Files
dbx/crates/dbx-core/src/transfer/ddl_plan.rs
T

459 lines
18 KiB
Rust

//! Prepare target CREATE TABLE and deferred foreign keys without executing DDL.
//!
//! The transfer pass and the ownership preview share this planning helper so both always
//! agree on the DDL that will run and the names it will use. Nothing here executes DDL:
//! execution stays with `execute_transfer_create_table_ddl_on_pool` and the caller.
use super::*;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub(super) struct PreparedTableDdl {
pub ddl: String,
pub reused_source_ddl: bool,
pub deferred_fk_alters: Vec<String>,
}
#[allow(clippy::too_many_arguments)]
pub(super) async fn prepare_table_ddl(
state: &AppState,
request: &TransferRequest,
table: &str,
target_table: &str,
source_db_type: &DatabaseType,
target_db_type: &DatabaseType,
source_pool_key: &str,
columns: &[db::ColumnInfo],
table_comment: Option<&str>,
known_foreign_keys: &HashMap<String, Vec<db::ForeignKeyInfo>>,
) -> Result<PreparedTableDdl, String> {
let rebuild = request.drop_target_before_create;
let foreign_keys = if let Some(foreign_keys) = known_foreign_keys.get(table) {
foreign_keys.clone()
} else if rebuild {
// Strict for rebuilds: silently dropping foreign keys the inspection failed on
// would produce a rebuilt table that enforces fewer constraints than the source.
crate::schema::list_foreign_keys_core(
state,
&request.source_connection_id,
&request.source_database,
&request.source_schema,
table,
)
.await
.map_err(|e| format!("Failed to inspect source foreign keys for table '{table}' before rebuilding: {e}"))?
} else if supports_deferred_mysql_foreign_keys(target_db_type) {
// Ordinary MySQL-family targets keep the legacy best-effort behavior: fall back
// to the generated DDL without foreign keys when the inspection fails. Other
// targets never use the foreign keys, so no query is issued at all.
match crate::schema::list_foreign_keys_core(
state,
&request.source_connection_id,
&request.source_database,
&request.source_schema,
table,
)
.await
{
Ok(foreign_keys) => foreign_keys,
Err(e) => {
log::warn!("[transfer] failed to inspect source foreign keys for {table}: {e}");
Vec::new()
}
}
} else {
Vec::new()
};
let (source_driver_profile, target_driver_profile) = {
let configs = state.configs.read().await;
(
configs.get(&request.source_connection_id).and_then(|config| config.driver_profile.clone()),
configs.get(&request.target_connection_id).and_then(|config| config.driver_profile.clone()),
)
};
let can_reuse = can_reuse_source_table_ddl(
source_db_type,
target_db_type,
source_driver_profile.as_deref(),
target_driver_profile.as_deref(),
target_table == table,
) && (request.quote_target_column_names
|| !matches!(target_db_type, DatabaseType::Gaussdb | DatabaseType::OpenGauss));
// A rebuild recreates the source structure exactly. When the structure comes from
// the source DDL (the `can_reuse` path) the column list is only used for data
// mapping, so an empty list is fine — the DDL is read directly from the source. Only
// when the DDL must be *generated* from the column list does incomplete metadata
// become fatal, because there would be nothing to build the CREATE TABLE from.
if rebuild && !can_reuse {
let incomplete = columns.is_empty()
|| columns.iter().any(|column| column.name.trim().is_empty() || column.data_type.trim().is_empty());
if incomplete {
return Err(format!(
"Cannot rebuild table '{target_table}': the source returned incomplete column metadata, and the \
target cannot reuse the source DDL."
));
}
}
let mut reused_source_ddl = false;
let ddl = if can_reuse {
let (source_ddl, source_ddl_was_read) = if let Some(catalog) =
resolve_external_transfer_catalog(request.source_catalog.as_deref(), source_db_type)
{
// Doris/StarRocks external catalog: read DDL directly via
// SHOW CREATE TABLE catalog.database.table using the existing source
// pool (bare MySQL — addresses any catalog).
let pool = {
let pool =
state.pool_handle(source_pool_key).await.ok_or_else(|| "Source pool not found".to_string())?;
let PoolKind::Mysql(p, _) = &pool else {
return Err("Source pool must be MySQL-family for catalog DDL".to_string());
};
p.clone()
};
match db::doris::get_catalog_table_ddl(&pool, catalog, &request.source_database, table).await {
Ok(ddl) => (ddl, true),
Err(err) => {
log::warn!("[transfer] catalog DDL read failed for {table} in catalog '{catalog}': {err}; falling back to generated DDL");
(
generate_create_table_ddl_with_column_quoting(
columns,
target_table,
&request.source_schema,
&request.target_schema,
target_db_type,
source_db_type,
table_comment,
request.target_catalog.as_deref(),
request.quote_target_column_names,
),
false,
)
}
}
} else {
match crate::schema::get_table_ddl_core(
state,
&request.source_connection_id,
&request.source_database,
&request.source_schema,
table,
None,
)
.await
{
Ok(ddl) => (ddl, true),
Err(e)
if rebuild || matches!((source_db_type, target_db_type), (DatabaseType::H2, DatabaseType::H2)) =>
{
// Rebuilds and H2-to-H2 transfers must not silently discard source
// constraints or generated columns when native DDL cannot be read.
return Err(format!("Failed to read source DDL for table '{table}' before creating target: {e}"));
}
Err(_) => (
generate_create_table_ddl_with_column_quoting(
columns,
target_table,
&request.source_schema,
&request.target_schema,
target_db_type,
source_db_type,
table_comment,
request.target_catalog.as_deref(),
request.quote_target_column_names,
),
false,
),
}
};
if contains_oceanbase_mysql_table_options(&source_ddl)
&& !db::oceanbase_mysql::is_profile(target_db_type, target_driver_profile.as_deref())
{
generate_create_table_ddl_with_column_quoting(
columns,
target_table,
&request.source_schema,
&request.target_schema,
target_db_type,
source_db_type,
table_comment,
request.target_catalog.as_deref(),
request.quote_target_column_names,
)
} else if let Some(rewritten) = rewrite_transfer_source_table_ddl(
&source_ddl,
&request.source_schema,
&request.target_schema,
source_db_type,
target_db_type,
table,
target_table,
) {
reused_source_ddl = source_ddl_was_read;
rewritten
} else {
// The reused DDL's CREATE TABLE head could not be renamed safely; fall back
// to the generated DDL rather than creating the table under the wrong name.
generate_create_table_ddl_with_column_quoting(
columns,
target_table,
&request.source_schema,
&request.target_schema,
target_db_type,
source_db_type,
table_comment,
request.target_catalog.as_deref(),
request.quote_target_column_names,
)
}
} else {
generate_create_table_ddl_with_column_quoting(
columns,
target_table,
&request.source_schema,
&request.target_schema,
target_db_type,
source_db_type,
table_comment,
request.target_catalog.as_deref(),
request.quote_target_column_names,
)
};
// MySQL-family targets: defer the foreign keys to ALTER statements so the table
// creation order never has to satisfy foreign key dependencies (a foreign key cycle
// has no valid CREATE TABLE order at all). The rebuild path above guarantees the
// metadata behind `foreign_keys` is trustworthy before anything is stripped.
let mut ddl = ddl;
let mut deferred_fk_alters = Vec::new();
if supports_deferred_mysql_foreign_keys(target_db_type) && !foreign_keys.is_empty() {
ddl = strip_inline_foreign_key_constraint_lines(&ddl);
deferred_fk_alters =
generate_mysql_foreign_key_alter_statements(&foreign_keys, request, target_table, target_db_type);
}
Ok(PreparedTableDdl { ddl, reused_source_ddl, deferred_fk_alters })
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
async fn state_fixture() -> (Arc<AppState>, tempfile::TempDir) {
let directory = tempfile::tempdir().unwrap();
let storage = crate::storage::Storage::open(&directory.path().join("storage.db")).await.unwrap();
let state = Arc::new(AppState::new_with_plugin_dir(storage, directory.path().join("plugins")));
(state, directory)
}
fn request(rebuild: bool) -> TransferRequest {
serde_json::from_value(json!({
"transferId": "ddl-plan-test",
"sourceConnectionId": "source", "sourceDatabase": "main", "sourceSchema": "main",
"targetConnectionId": "target", "targetDatabase": "main", "targetSchema": "main",
"tables": ["child", "Parent"], "createTable": true, "content": "structureAndData",
"mode": "append", "batchSize": 10, "dropTargetBeforeCreate": rebuild,
"dropTargetConfirmed": true
}))
.unwrap()
}
fn columns() -> Vec<db::ColumnInfo> {
vec![
db::ColumnInfo {
name: "id".into(),
data_type: "INTEGER".into(),
is_primary_key: true,
..Default::default()
},
db::ColumnInfo { name: "parent_id".into(), data_type: "INTEGER".into(), ..Default::default() },
]
}
#[tokio::test]
async fn prepared_sqlite_ddl_preserves_source_foreign_keys_without_mutating_source() {
let (state, directory) = state_fixture().await;
let path = directory.path().join("source.db");
let pool = crate::db::sqlite::connect_path_create_if_missing(path.to_str().unwrap()).await.unwrap();
pool.with_connection(|connection| {
connection
.execute_batch(
"PRAGMA foreign_keys = ON;
CREATE TABLE Parent (id INTEGER PRIMARY KEY);
CREATE TABLE child (id INTEGER PRIMARY KEY, parent_id INTEGER REFERENCES Parent(id) ON DELETE CASCADE);
INSERT INTO Parent VALUES (7);
INSERT INTO child VALUES (11, 7);",
)
.map_err(|error| error.to_string())
})
.unwrap();
let config: ConnectionConfig = serde_json::from_value(json!({
"id": "source", "name": "source", "db_type": "sqlite", "host": path.to_str().unwrap(),
"port": 0, "username": "", "password": "", "database": null,
"one_time": false, "save_password": false, "read_only": false
}))
.unwrap();
state.configs.write().await.insert("source".into(), config);
let source_pool_key = ensure_transfer_pool(&state, "source", "main", None).await.unwrap();
let columns = crate::db::sqlite::get_columns(&pool, "main", "child").await.unwrap();
let prepared = prepare_table_ddl(
&state,
&request(true),
"child",
"child",
&DatabaseType::Sqlite,
&DatabaseType::Sqlite,
&source_pool_key,
&columns,
None,
&HashMap::new(),
)
.await
.unwrap();
let source_rows = crate::db::sqlite::execute_query(&pool, "SELECT id, parent_id FROM child").await.unwrap();
assert_eq!(source_rows.rows, vec![vec![json!(11), json!(7)]]);
let target =
crate::db::sqlite::connect_path_create_if_missing(directory.path().join("target.db").to_str().unwrap())
.await
.unwrap();
target
.with_connection(|connection| {
connection
.execute_batch("PRAGMA foreign_keys = ON; CREATE TABLE Parent (id INTEGER PRIMARY KEY);")
.map_err(|error| error.to_string())?;
connection.execute_batch(&prepared.ddl).map_err(|error| error.to_string())
})
.unwrap();
let foreign_keys = crate::db::sqlite::list_foreign_keys(&target, "main", "child").await.unwrap();
assert_eq!(foreign_keys.len(), 1, "source FK must survive the prepared CREATE TABLE: {}", prepared.ddl);
assert_eq!(foreign_keys[0].ref_table, "Parent");
assert!(crate::db::sqlite::execute_query(&target, "INSERT INTO child VALUES (1, 999)").await.is_err());
assert!(prepared.reused_source_ddl);
}
#[tokio::test]
async fn rebuild_rejects_failed_foreign_key_inspection() {
let (state, _directory) = state_fixture().await;
let error = prepare_table_ddl(
&state,
&request(true),
"child",
"child",
&DatabaseType::Sqlite,
&DatabaseType::Mysql,
"missing-source-pool",
&columns(),
None,
&HashMap::new(),
)
.await
.expect_err("a rebuild cannot discard foreign keys when source metadata is unavailable");
assert!(error.contains("child"), "{error}");
assert!(error.to_ascii_lowercase().contains("foreign key"), "{error}");
}
#[tokio::test]
async fn ordinary_foreign_key_metadata_failure_preserves_generated_fallback() {
let (state, _directory) = state_fixture().await;
let prepared = prepare_table_ddl(
&state,
&request(false),
"child",
"child",
&DatabaseType::Sqlite,
&DatabaseType::Mysql,
"missing-source-pool",
&columns(),
None,
&HashMap::new(),
)
.await
.unwrap();
assert!(!prepared.reused_source_ddl);
assert!(prepared.ddl.contains("CREATE TABLE"));
}
#[tokio::test]
async fn h2_native_ddl_read_failure_does_not_fall_back_to_lossy_metadata() {
let (state, _directory) = state_fixture().await;
let error = prepare_table_ddl(
&state,
&request(false),
"child",
"child",
&DatabaseType::H2,
&DatabaseType::H2,
"missing-source-pool",
&columns(),
None,
&HashMap::new(),
)
.await
.unwrap_err();
assert!(error.contains("Failed to read source DDL"), "{error}");
}
#[tokio::test]
async fn deferred_foreign_key_uses_actual_target_and_case_converted_parent() {
let (state, _directory) = state_fixture().await;
let mut request = request(true);
request.target_table_name_case = TransferTableNameCase::Lower;
let known = HashMap::from([(
"child".into(),
vec![db::ForeignKeyInfo {
name: "fk_child_parent".into(),
column: "parent_id".into(),
ref_schema: Some("main".into()),
ref_table: "Parent".into(),
ref_column: "id".into(),
on_delete: Some("CASCADE".into()),
on_update: Some("RESTRICT".into()),
}],
)]);
let prepared = prepare_table_ddl(
&state,
&request,
"child",
"resolved_child",
&DatabaseType::Sqlite,
&DatabaseType::Mysql,
"missing-source-pool",
&columns(),
None,
&known,
)
.await
.unwrap();
assert_eq!(prepared.deferred_fk_alters, vec![
"ALTER TABLE `resolved_child` ADD CONSTRAINT `fk_child_parent` FOREIGN KEY (`parent_id`) REFERENCES `parent` (`id`) ON DELETE CASCADE ON UPDATE RESTRICT"
]);
assert!(prepared.ddl.contains("`resolved_child`"), "{}", prepared.ddl);
}
#[tokio::test]
async fn rebuild_requires_complete_column_metadata() {
let (state, _directory) = state_fixture().await;
let incomplete = vec![db::ColumnInfo { name: "id".into(), ..Default::default() }];
for columns in [&[][..], incomplete.as_slice()] {
let result = prepare_table_ddl(
&state,
&request(true),
"child",
"child",
&DatabaseType::Sqlite,
&DatabaseType::Mysql,
"missing-source-pool",
columns,
None,
&HashMap::from([("child".into(), Vec::new())]),
)
.await;
assert!(result.is_err(), "incomplete source metadata must fail before any target DDL: {result:?}");
}
}
}