diff --git a/crates/dbx-core/src/transfer.rs b/crates/dbx-core/src/transfer.rs index e250fbe367..600ce951d2 100644 --- a/crates/dbx-core/src/transfer.rs +++ b/crates/dbx-core/src/transfer.rs @@ -2194,6 +2194,24 @@ async fn execute_transfer_ddl_on_pool( Ok(()) } +fn transfer_table_already_exists_error(error: &str) -> bool { + let lower = error.to_ascii_lowercase(); + lower.contains("already exists") + || lower.contains("there is already") + || lower.contains("duplicate_table") + || lower.contains("42p07") + || error.contains("已经存在") + || error.contains("已存在") +} + +fn transfer_create_table_created(result: Result<(), String>, error_prefix: &str) -> Result { + match result { + Ok(_) => Ok(true), + Err(e) if transfer_table_already_exists_error(&e) => Ok(false), + Err(e) => Err(format!("{error_prefix}: {e}")), + } +} + fn transfer_ddl_statements(sql: &str, db_type: &DatabaseType) -> Vec { if matches!(db_type, DatabaseType::Postgres) { let statements = split_sql_statements(sql); @@ -3328,42 +3346,55 @@ where sql_target_columns.iter().map(|column| Some(column.data_type.clone())).collect(); if request.create_table { - let ddl = generate_create_table_ddl( - &sql_target_columns, - &target_table, - &request.source_schema, + let target_table_preexisting = crate::schema::list_tables_core( + state, + &request.target_connection_id, + &request.target_database, &request.target_schema, - target_db_type, - source_db_type, + Some(&target_table), + Some(1), None, - ); - let table_exists = match execute_on_pool(state, target_pool_key, &ddl).await { - Ok(_) => true, - Err(e) => { - let err_lower = e.to_lowercase(); - if err_lower.contains("already exists") || err_lower.contains("there is already") { - true - } else { - return Err(format!("Failed to create table from MongoDB collection '{table}': {e}")); - } - } - }; - if table_exists { - for stmt in generate_comment_ddl( + None, + ) + .await + .map(|tables| !tables.is_empty()) + .unwrap_or(false); + if !target_table_preexisting { + let ddl = generate_create_table_ddl( &sql_target_columns, &target_table, + &request.source_schema, &request.target_schema, target_db_type, + source_db_type, None, - ) { - if let Err(e) = execute_on_pool(state, target_pool_key, &stmt).await { - log::warn!( - "[transfer] failed to set MongoDB transfer column comment for {}: {}", - target_table, - e - ); + ); + let target_table_created = transfer_create_table_created( + execute_on_pool(state, target_pool_key, &ddl).await.map(|_| ()), + &format!("Failed to create table from MongoDB collection '{table}'"), + )?; + if target_table_created { + for stmt in generate_comment_ddl( + &sql_target_columns, + &target_table, + &request.target_schema, + target_db_type, + None, + ) { + if let Err(e) = execute_on_pool(state, target_pool_key, &stmt).await { + log::warn!( + "[transfer] failed to set MongoDB transfer column comment for {}: {}", + target_table, + e + ); + } } } + } else { + log::info!( + "[transfer] target table {} already exists, skipping create-table DDL", + target_table + ); } } @@ -3513,7 +3544,7 @@ where .next() .and_then(|t| t.comment); - let target_table_preexisting = crate::schema::list_tables_core( + let mut target_table_preexisting = crate::schema::list_tables_core( state, &request.target_connection_id, &request.target_database, @@ -3566,31 +3597,52 @@ where .await .map_err(|e| format!("Failed to ensure schema exists: {e}"))?; } - let owned_sequences = prepare_postgres_owned_sequences_for_transfer( - state, - request, - table, - &target_table, - source_pool_key, - target_pool_key, - pg_compat_transfer, - preserves_target_table_name, - target_table_preexisting, - ) - .await?; - let can_reuse_source_ddl = - can_reuse_source_table_ddl(source_db_type, target_db_type, preserves_target_table_name); - let ddl = if can_reuse_source_ddl { - let source_ddl = crate::schema::get_table_ddl_core( - &state, - &request.source_connection_id, - &request.source_database, - &request.source_schema, + if target_table_preexisting { + log::info!("[transfer] target table {} already exists, skipping create-table DDL", target_table); + } else { + let owned_sequences = prepare_postgres_owned_sequences_for_transfer( + state, + request, table, - None, + &target_table, + source_pool_key, + target_pool_key, + pg_compat_transfer, + preserves_target_table_name, + target_table_preexisting, ) - .await - .unwrap_or_else(|_| { + .await?; + let can_reuse_source_ddl = + can_reuse_source_table_ddl(source_db_type, target_db_type, preserves_target_table_name); + let ddl = if can_reuse_source_ddl { + let source_ddl = crate::schema::get_table_ddl_core( + &state, + &request.source_connection_id, + &request.source_database, + &request.source_schema, + table, + None, + ) + .await + .unwrap_or_else(|_| { + generate_create_table_ddl( + &columns, + &target_table, + &request.source_schema, + &request.target_schema, + target_db_type, + source_db_type, + table_comment.as_deref(), + ) + }); + rewrite_transfer_source_table_ddl( + &source_ddl, + &request.source_schema, + &request.target_schema, + source_db_type, + target_db_type, + ) + } else { generate_create_table_ddl( &columns, &target_table, @@ -3600,58 +3652,38 @@ where source_db_type, table_comment.as_deref(), ) - }); - rewrite_transfer_source_table_ddl( - &source_ddl, - &request.source_schema, - &request.target_schema, - source_db_type, - target_db_type, - ) - } else { - generate_create_table_ddl( - &columns, - &target_table, - &request.source_schema, - &request.target_schema, - target_db_type, - source_db_type, - table_comment.as_deref(), - ) - }; - log::info!("[transfer] creating target table: {}", ddl.chars().take(200).collect::()); - let table_exists = match execute_transfer_ddl_on_pool(state, target_pool_key, &ddl, target_db_type).await { - Ok(_) => true, - Err(e) => { - let err_lower = e.to_lowercase(); - if err_lower.contains("already exists") || err_lower.contains("there is already") { - true - } else { - return Err(format!("Failed to create table: {e}")); + }; + log::info!("[transfer] creating target table: {}", ddl.chars().take(200).collect::()); + let target_table_created = transfer_create_table_created( + execute_transfer_ddl_on_pool(state, target_pool_key, &ddl, target_db_type).await, + "Failed to create table", + )?; + if target_table_created { + let comment_stmts = generate_comment_ddl( + &columns, + &target_table, + &request.target_schema, + target_db_type, + table_comment.as_deref(), + ); + for stmt in &comment_stmts { + if let Err(e) = execute_on_pool(state, target_pool_key, stmt).await { + log::warn!("[transfer] failed to set column comment for {}: {}", target_table, e); + } } + bind_postgres_owned_sequences_for_transfer( + state, + request, + &target_table, + target_pool_key, + &owned_sequences, + ) + .await?; + } else { + // DDL may report the table already exists even when metadata + // lookup missed it (case/schema differences or localized errors). + target_table_preexisting = true; } - }; - if table_exists { - let comment_stmts = generate_comment_ddl( - &columns, - &target_table, - &request.target_schema, - target_db_type, - table_comment.as_deref(), - ); - for stmt in &comment_stmts { - if let Err(e) = execute_on_pool(state, target_pool_key, stmt).await { - log::warn!("[transfer] failed to set column comment for {}: {}", target_table, e); - } - } - bind_postgres_owned_sequences_for_transfer( - state, - request, - &target_table, - target_pool_key, - &owned_sequences, - ) - .await?; } } @@ -4537,6 +4569,28 @@ mod tests { ); } + #[test] + fn transfer_create_table_result_treats_existing_table_as_preexisting() { + assert_eq!( + transfer_create_table_created( + Err("ERROR: relation \"items\" already exists (SQLSTATE 42P07)".to_string()), + "create" + ) + .unwrap(), + false + ); + assert_eq!( + transfer_create_table_created(Err("错误: 关系 \"items\" 已经存在".to_string()), "create").unwrap(), + false + ); + assert_eq!(transfer_create_table_created(Ok(()), "create").unwrap(), true); + assert_eq!( + transfer_create_table_created(Err("permission denied for schema public".to_string()), "create") + .unwrap_err(), + "create: permission denied for schema public" + ); + } + #[test] fn non_postgres_transfer_ddl_keeps_statement_text_intact() { let ddl = "CREATE TABLE `items` (`id` int);\nALTER TABLE `items` COMMENT = 'items';"; diff --git a/crates/dbx-core/tests/live_postgres_transfer.rs b/crates/dbx-core/tests/live_postgres_transfer.rs index a751361785..5ad2d50429 100644 --- a/crates/dbx-core/tests/live_postgres_transfer.rs +++ b/crates/dbx-core/tests/live_postgres_transfer.rs @@ -402,3 +402,121 @@ async fn live_postgres_transfer_preserves_data_and_schema_objects() { let _ = postgres::execute_batch(&target_pool, &[cleanup_sql[1].clone()]).await; let _ = std::fs::remove_dir_all(dir); } + +#[tokio::test] +#[ignore = "requires PostgreSQL URLs via DBX_LIVE_PG_TRANSFER_SOURCE_URL and DBX_LIVE_PG_TRANSFER_TARGET_URL"] +async fn live_postgres_transfer_skips_create_ddl_for_existing_target_table() { + let source_url = std::env::var("DBX_LIVE_PG_TRANSFER_SOURCE_URL").expect("DBX_LIVE_PG_TRANSFER_SOURCE_URL"); + let target_url = std::env::var("DBX_LIVE_PG_TRANSFER_TARGET_URL").unwrap_or_else(|_| source_url.clone()); + + let source_pool = postgres::connect(&source_url, std::time::Duration::from_secs(5)).await.unwrap(); + let target_pool = postgres::connect(&target_url, std::time::Duration::from_secs(5)).await.unwrap(); + + let source_database = query_scalar(&source_pool, "SELECT current_database()").await.as_str().unwrap().to_string(); + let target_database = query_scalar(&target_pool, "SELECT current_database()").await.as_str().unwrap().to_string(); + + let suffix = uuid::Uuid::new_v4().simple().to_string(); + let source_schema = format!("dbx_src_existing_{}", &suffix[..8]); + let target_schema = format!("dbx_dst_existing_{}", &suffix[..8]); + + let cleanup_sql = [ + format!("DROP SCHEMA IF EXISTS \"{}\" CASCADE", source_schema), + format!("DROP SCHEMA IF EXISTS \"{}\" CASCADE", target_schema), + ]; + let _ = postgres::execute_batch(&source_pool, &[cleanup_sql[0].clone()]).await; + let _ = postgres::execute_batch(&target_pool, &[cleanup_sql[1].clone()]).await; + + postgres::execute_batch( + &source_pool, + &[ + format!("CREATE SCHEMA \"{}\"", source_schema), + format!( + "CREATE TABLE \"{}\".\"items\" (\"id\" integer PRIMARY KEY, \"name\" text NOT NULL)", + source_schema + ), + format!( + "INSERT INTO \"{}\".\"items\" (\"id\", \"name\") VALUES (1, 'existing-target-transfer')", + source_schema + ), + ], + ) + .await + .unwrap(); + postgres::execute_batch( + &target_pool, + &[ + format!("CREATE SCHEMA \"{}\"", target_schema), + format!( + "CREATE TABLE \"{}\".\"items\" (\"id\" integer PRIMARY KEY, \"name\" text NOT NULL)", + target_schema + ), + ], + ) + .await + .unwrap(); + + let dir = std::env::temp_dir().join(format!("dbx-live-existing-transfer-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&dir).unwrap(); + let storage = Storage::open(&dir.join("storage.db")).await.unwrap(); + let state = AppState::new(storage); + + let source_connection_id = "live-existing-source"; + let target_connection_id = "live-existing-target"; + let source_pool_key = format!("{source_connection_id}:{source_database}"); + let target_pool_key = format!("{target_connection_id}:{target_database}"); + + state.connections.write().await.insert(source_pool_key.clone(), PoolKind::Postgres(source_pool.clone())); + state.connections.write().await.insert(target_pool_key.clone(), PoolKind::Postgres(target_pool.clone())); + state + .configs + .write() + .await + .insert(source_connection_id.to_string(), postgres_test_config(source_connection_id, &source_database)); + state + .configs + .write() + .await + .insert(target_connection_id.to_string(), postgres_test_config(target_connection_id, &target_database)); + + let request = TransferRequest { + transfer_id: format!("live-existing-transfer-{suffix}"), + source_connection_id: source_connection_id.to_string(), + source_database: source_database.clone(), + source_schema: source_schema.clone(), + target_connection_id: target_connection_id.to_string(), + target_database: target_database.clone(), + target_schema: target_schema.clone(), + tables: vec!["items".to_string()], + create_table: true, + mode: TransferMode::Append, + target_table_name_case: TransferTableNameCase::Preserve, + batch_size: 100, + }; + + let source_db_type = get_db_type(&state, source_connection_id).await.unwrap(); + let target_db_type = get_db_type(&state, target_connection_id).await.unwrap(); + let transferred = transfer_table( + &state, + &request, + "items", + 0, + &source_db_type, + &target_db_type, + &source_pool_key, + &target_pool_key, + |_| {}, + ) + .await + .unwrap(); + + assert_eq!(transferred, 1); + assert_eq!( + query_scalar(&target_pool, &format!("SELECT \"name\" FROM \"{}\".\"items\" WHERE \"id\" = 1", target_schema)) + .await, + json!("existing-target-transfer") + ); + + let _ = postgres::execute_batch(&source_pool, &[cleanup_sql[0].clone()]).await; + let _ = postgres::execute_batch(&target_pool, &[cleanup_sql[1].clone()]).await; + let _ = std::fs::remove_dir_all(dir); +}