fix(transfer): skip create DDL for existing targets

This commit is contained in:
t8y2 committed 2026-07-02 00:49:32 +08:00
1 parent bd659c322c
commit a435c76250
2 files changed
+273 -101

No files matched your search

+155 -101
View File
@@ -2194,6 +2194,24 @@ async fn execute_transfer_ddl_on_pool(
Ok(())
}
fn transfer_table_already_exists_error(error: &str) -> bool {
let lower = error.to_ascii_lowercase();
lower.contains("already exists")
|| lower.contains("there is already")
|| lower.contains("duplicate_table")
|| lower.contains("42p07")
|| error.contains("已经存在")
|| error.contains("已存在")
}
fn transfer_create_table_created(result: Result<(), String>, error_prefix: &str) -> Result<bool, String> {
match result {
Ok(_) => Ok(true),
Err(e) if transfer_table_already_exists_error(&e) => Ok(false),
Err(e) => Err(format!("{error_prefix}: {e}")),
}
}
fn transfer_ddl_statements(sql: &str, db_type: &DatabaseType) -> Vec<String> {
if matches!(db_type, DatabaseType::Postgres) {
let statements = split_sql_statements(sql);
@@ -3328,42 +3346,55 @@ where
sql_target_columns.iter().map(|column| Some(column.data_type.clone())).collect();
if request.create_table {
let ddl = generate_create_table_ddl(
&sql_target_columns,
&target_table,
&request.source_schema,
let target_table_preexisting = crate::schema::list_tables_core(
state,
&request.target_connection_id,
&request.target_database,
&request.target_schema,
target_db_type,
source_db_type,
Some(&target_table),
Some(1),
None,
);
let table_exists = match execute_on_pool(state, target_pool_key, &ddl).await {
Ok(_) => true,
Err(e) => {
let err_lower = e.to_lowercase();
if err_lower.contains("already exists") || err_lower.contains("there is already") {
true
} else {
return Err(format!("Failed to create table from MongoDB collection '{table}': {e}"));
}
}
};
if table_exists {
for stmt in generate_comment_ddl(
None,
)
.await
.map(|tables| !tables.is_empty())
.unwrap_or(false);
if !target_table_preexisting {
let ddl = generate_create_table_ddl(
&sql_target_columns,
&target_table,
&request.source_schema,
&request.target_schema,
target_db_type,
source_db_type,
None,
) {
if let Err(e) = execute_on_pool(state, target_pool_key, &stmt).await {
log::warn!(
"[transfer] failed to set MongoDB transfer column comment for {}: {}",
target_table,
e
);
);
let target_table_created = transfer_create_table_created(
execute_on_pool(state, target_pool_key, &ddl).await.map(|_| ()),
&format!("Failed to create table from MongoDB collection '{table}'"),
)?;
if target_table_created {
for stmt in generate_comment_ddl(
&sql_target_columns,
&target_table,
&request.target_schema,
target_db_type,
None,
) {
if let Err(e) = execute_on_pool(state, target_pool_key, &stmt).await {
log::warn!(
"[transfer] failed to set MongoDB transfer column comment for {}: {}",
target_table,
e
);
}
}
}
} else {
log::info!(
"[transfer] target table {} already exists, skipping create-table DDL",
target_table
);
}
}
@@ -3513,7 +3544,7 @@ where
.next()
.and_then(|t| t.comment);
let target_table_preexisting = crate::schema::list_tables_core(
let mut target_table_preexisting = crate::schema::list_tables_core(
state,
&request.target_connection_id,
&request.target_database,
@@ -3566,31 +3597,52 @@ where
.await
.map_err(|e| format!("Failed to ensure schema exists: {e}"))?;
}
let owned_sequences = prepare_postgres_owned_sequences_for_transfer(
state,
request,
table,
&target_table,
source_pool_key,
target_pool_key,
pg_compat_transfer,
preserves_target_table_name,
target_table_preexisting,
)
.await?;
let can_reuse_source_ddl =
can_reuse_source_table_ddl(source_db_type, target_db_type, preserves_target_table_name);
let ddl = if can_reuse_source_ddl {
let source_ddl = crate::schema::get_table_ddl_core(
&state,
&request.source_connection_id,
&request.source_database,
&request.source_schema,
if target_table_preexisting {
log::info!("[transfer] target table {} already exists, skipping create-table DDL", target_table);
} else {
let owned_sequences = prepare_postgres_owned_sequences_for_transfer(
state,
request,
table,
None,
&target_table,
source_pool_key,
target_pool_key,
pg_compat_transfer,
preserves_target_table_name,
target_table_preexisting,
)
.await
.unwrap_or_else(|_| {
.await?;
let can_reuse_source_ddl =
can_reuse_source_table_ddl(source_db_type, target_db_type, preserves_target_table_name);
let ddl = if can_reuse_source_ddl {
let source_ddl = crate::schema::get_table_ddl_core(
&state,
&request.source_connection_id,
&request.source_database,
&request.source_schema,
table,
None,
)
.await
.unwrap_or_else(|_| {
generate_create_table_ddl(
&columns,
&target_table,
&request.source_schema,
&request.target_schema,
target_db_type,
source_db_type,
table_comment.as_deref(),
)
});
rewrite_transfer_source_table_ddl(
&source_ddl,
&request.source_schema,
&request.target_schema,
source_db_type,
target_db_type,
)
} else {
generate_create_table_ddl(
&columns,
&target_table,
@@ -3600,58 +3652,38 @@ where
source_db_type,
table_comment.as_deref(),
)
});
rewrite_transfer_source_table_ddl(
&source_ddl,
&request.source_schema,
&request.target_schema,
source_db_type,
target_db_type,
)
} else {
generate_create_table_ddl(
&columns,
&target_table,
&request.source_schema,
&request.target_schema,
target_db_type,
source_db_type,
table_comment.as_deref(),
)
};
log::info!("[transfer] creating target table: {}", ddl.chars().take(200).collect::<String>());
let table_exists = match execute_transfer_ddl_on_pool(state, target_pool_key, &ddl, target_db_type).await {
Ok(_) => true,
Err(e) => {
let err_lower = e.to_lowercase();
if err_lower.contains("already exists") || err_lower.contains("there is already") {
true
} else {
return Err(format!("Failed to create table: {e}"));
};
log::info!("[transfer] creating target table: {}", ddl.chars().take(200).collect::<String>());
let target_table_created = transfer_create_table_created(
execute_transfer_ddl_on_pool(state, target_pool_key, &ddl, target_db_type).await,
"Failed to create table",
)?;
if target_table_created {
let comment_stmts = generate_comment_ddl(
&columns,
&target_table,
&request.target_schema,
target_db_type,
table_comment.as_deref(),
);
for stmt in &comment_stmts {
if let Err(e) = execute_on_pool(state, target_pool_key, stmt).await {
log::warn!("[transfer] failed to set column comment for {}: {}", target_table, e);
}
}
bind_postgres_owned_sequences_for_transfer(
state,
request,
&target_table,
target_pool_key,
&owned_sequences,
)
.await?;
} else {
// DDL may report the table already exists even when metadata
// lookup missed it (case/schema differences or localized errors).
target_table_preexisting = true;
}
};
if table_exists {
let comment_stmts = generate_comment_ddl(
&columns,
&target_table,
&request.target_schema,
target_db_type,
table_comment.as_deref(),
);
for stmt in &comment_stmts {
if let Err(e) = execute_on_pool(state, target_pool_key, stmt).await {
log::warn!("[transfer] failed to set column comment for {}: {}", target_table, e);
}
}
bind_postgres_owned_sequences_for_transfer(
state,
request,
&target_table,
target_pool_key,
&owned_sequences,
)
.await?;
}
}
@@ -4537,6 +4569,28 @@ mod tests {
);
}
#[test]
fn transfer_create_table_result_treats_existing_table_as_preexisting() {
assert_eq!(
transfer_create_table_created(
Err("ERROR: relation \"items\" already exists (SQLSTATE 42P07)".to_string()),
"create"
)
.unwrap(),
false
);
assert_eq!(
transfer_create_table_created(Err("错误: 关系 \"items\" 已经存在".to_string()), "create").unwrap(),
false
);
assert_eq!(transfer_create_table_created(Ok(()), "create").unwrap(), true);
assert_eq!(
transfer_create_table_created(Err("permission denied for schema public".to_string()), "create")
.unwrap_err(),
"create: permission denied for schema public"
);
}
#[test]
fn non_postgres_transfer_ddl_keeps_statement_text_intact() {
let ddl = "CREATE TABLE `items` (`id` int);\nALTER TABLE `items` COMMENT = 'items';";
@@ -402,3 +402,121 @@ async fn live_postgres_transfer_preserves_data_and_schema_objects() {
let _ = postgres::execute_batch(&target_pool, &[cleanup_sql[1].clone()]).await;
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test]
#[ignore = "requires PostgreSQL URLs via DBX_LIVE_PG_TRANSFER_SOURCE_URL and DBX_LIVE_PG_TRANSFER_TARGET_URL"]
async fn live_postgres_transfer_skips_create_ddl_for_existing_target_table() {
let source_url = std::env::var("DBX_LIVE_PG_TRANSFER_SOURCE_URL").expect("DBX_LIVE_PG_TRANSFER_SOURCE_URL");
let target_url = std::env::var("DBX_LIVE_PG_TRANSFER_TARGET_URL").unwrap_or_else(|_| source_url.clone());
let source_pool = postgres::connect(&source_url, std::time::Duration::from_secs(5)).await.unwrap();
let target_pool = postgres::connect(&target_url, std::time::Duration::from_secs(5)).await.unwrap();
let source_database = query_scalar(&source_pool, "SELECT current_database()").await.as_str().unwrap().to_string();
let target_database = query_scalar(&target_pool, "SELECT current_database()").await.as_str().unwrap().to_string();
let suffix = uuid::Uuid::new_v4().simple().to_string();
let source_schema = format!("dbx_src_existing_{}", &suffix[..8]);
let target_schema = format!("dbx_dst_existing_{}", &suffix[..8]);
let cleanup_sql = [
format!("DROP SCHEMA IF EXISTS \"{}\" CASCADE", source_schema),
format!("DROP SCHEMA IF EXISTS \"{}\" CASCADE", target_schema),
];
let _ = postgres::execute_batch(&source_pool, &[cleanup_sql[0].clone()]).await;
let _ = postgres::execute_batch(&target_pool, &[cleanup_sql[1].clone()]).await;
postgres::execute_batch(
&source_pool,
&[
format!("CREATE SCHEMA \"{}\"", source_schema),
format!(
"CREATE TABLE \"{}\".\"items\" (\"id\" integer PRIMARY KEY, \"name\" text NOT NULL)",
source_schema
),
format!(
"INSERT INTO \"{}\".\"items\" (\"id\", \"name\") VALUES (1, 'existing-target-transfer')",
source_schema
),
],
)
.await
.unwrap();
postgres::execute_batch(
&target_pool,
&[
format!("CREATE SCHEMA \"{}\"", target_schema),
format!(
"CREATE TABLE \"{}\".\"items\" (\"id\" integer PRIMARY KEY, \"name\" text NOT NULL)",
target_schema
),
],
)
.await
.unwrap();
let dir = std::env::temp_dir().join(format!("dbx-live-existing-transfer-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
let state = AppState::new(storage);
let source_connection_id = "live-existing-source";
let target_connection_id = "live-existing-target";
let source_pool_key = format!("{source_connection_id}:{source_database}");
let target_pool_key = format!("{target_connection_id}:{target_database}");
state.connections.write().await.insert(source_pool_key.clone(), PoolKind::Postgres(source_pool.clone()));
state.connections.write().await.insert(target_pool_key.clone(), PoolKind::Postgres(target_pool.clone()));
state
.configs
.write()
.await
.insert(source_connection_id.to_string(), postgres_test_config(source_connection_id, &source_database));
state
.configs
.write()
.await
.insert(target_connection_id.to_string(), postgres_test_config(target_connection_id, &target_database));
let request = TransferRequest {
transfer_id: format!("live-existing-transfer-{suffix}"),
source_connection_id: source_connection_id.to_string(),
source_database: source_database.clone(),
source_schema: source_schema.clone(),
target_connection_id: target_connection_id.to_string(),
target_database: target_database.clone(),
target_schema: target_schema.clone(),
tables: vec!["items".to_string()],
create_table: true,
mode: TransferMode::Append,
target_table_name_case: TransferTableNameCase::Preserve,
batch_size: 100,
};
let source_db_type = get_db_type(&state, source_connection_id).await.unwrap();
let target_db_type = get_db_type(&state, target_connection_id).await.unwrap();
let transferred = transfer_table(
&state,
&request,
"items",
0,
&source_db_type,
&target_db_type,
&source_pool_key,
&target_pool_key,
|_| {},
)
.await
.unwrap();
assert_eq!(transferred, 1);
assert_eq!(
query_scalar(&target_pool, &format!("SELECT \"name\" FROM \"{}\".\"items\" WHERE \"id\" = 1", target_schema))
.await,
json!("existing-target-transfer")
);
let _ = postgres::execute_batch(&source_pool, &[cleanup_sql[0].clone()]).await;
let _ = postgres::execute_batch(&target_pool, &[cleanup_sql[1].clone()]).await;
let _ = std::fs::remove_dir_all(dir);
}