fix(transfer): isolate table transfer task

This commit is contained in:
Guoyu Su authored and GitHub committed 2026-09-01 23:45:33 +08:00
1 parent 51ca40e86e
commit 48f4fac89e
4 files changed
+356 -89

No files matched your search

+186 -78
View File
@@ -1,6 +1,7 @@
use regex::Regex;
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use tokio::sync::RwLock;
use crate::connection::{config_for_pool_key, AppState, PoolKind};
@@ -798,8 +799,66 @@ fn existing_transfer_target_table_name(
tables.iter().find(|table| table.name.eq_ignore_ascii_case(requested_name)).map(|table| table.name.clone())
}
const TRANSFER_PROGRESS_CHANNEL_CAPACITY: usize = 16;
fn try_send_transfer_progress(sender: &tokio::sync::mpsc::Sender<TransferProgress>, progress: TransferProgress) {
let _ = sender.try_send(progress);
}
struct AbortTransferTaskOnDrop(tokio::task::AbortHandle);
impl Drop for AbortTransferTaskOnDrop {
fn drop(&mut self) {
self.0.abort();
}
}
#[allow(clippy::too_many_arguments)]
async fn list_transfer_tables_isolated(
state: Arc<AppState>,
connection_id: String,
database: String,
schema: String,
catalog: Option<String>,
db_type: DatabaseType,
table: String,
limit: usize,
) -> Result<Vec<db::TableInfo>, String> {
let task = tokio::spawn(async move {
if let Some(catalog) = resolve_external_transfer_catalog(catalog.as_deref(), &db_type) {
crate::schema::list_doris_catalog_tables_core(
&state,
&connection_id,
catalog,
&database,
Some(&table),
Some(limit),
None,
None,
None,
)
.await
} else {
crate::schema::list_tables_core(
&state,
&connection_id,
&database,
&schema,
Some(&table),
Some(limit),
None,
None,
None,
)
.await
}
});
let _abort_on_drop = AbortTransferTaskOnDrop(task.abort_handle());
task.await.map_err(|error| format!("Transfer table metadata task failed: {error}"))?
}
async fn resolve_transfer_target_table_name(
state: &AppState,
state: &Arc<AppState>,
request: &TransferRequest,
source_table: &str,
target_pool_key: &str,
@@ -818,41 +877,21 @@ async fn resolve_transfer_target_table_name(
// Route through the catalog-aware path when targeting an external
// Doris/StarRocks catalog — otherwise the lookup runs against the
// default / internal catalog and can miss or misidentify the table.
let tables = if let Some(catalog) = resolve_external_transfer_catalog(target_catalog, target_db_type) {
crate::schema::list_doris_catalog_tables_core(
state,
&request.target_connection_id,
catalog,
&request.target_database,
Some(&requested_name),
Some(TRANSFER_TARGET_TABLE_LOOKUP_LIMIT),
None,
None,
None,
)
.await
.unwrap_or_else(|error| {
log::debug!("[transfer] failed to resolve target table metadata for {requested_name} in catalog '{catalog}': {error}");
Vec::new()
})
} else {
crate::schema::list_tables_core(
state,
&request.target_connection_id,
&request.target_database,
&request.target_schema,
Some(&requested_name),
Some(TRANSFER_TARGET_TABLE_LOOKUP_LIMIT),
None,
None,
None,
)
.await
.unwrap_or_else(|error| {
log::debug!("[transfer] failed to resolve target table metadata for {requested_name}: {error}");
Vec::new()
})
};
let tables = list_transfer_tables_isolated(
state.clone(),
request.target_connection_id.clone(),
request.target_database.clone(),
request.target_schema.clone(),
target_catalog.map(str::to_string),
*target_db_type,
requested_name.clone(),
TRANSFER_TARGET_TABLE_LOOKUP_LIMIT,
)
.await
.unwrap_or_else(|error| {
log::debug!("[transfer] failed to resolve target table metadata for {requested_name}: {error}");
Vec::new()
});
if let Some(existing_name) =
existing_transfer_target_table_name(&requested_name, &tables, allow_case_insensitive_match)
@@ -6827,7 +6866,7 @@ pub(crate) fn sort_table_names_by_dependencies(
#[allow(clippy::too_many_arguments)]
async fn transfer_mongodb_table<F>(
state: &AppState,
state: &Arc<AppState>,
request: &TransferRequest,
table: &str,
table_index: usize,
@@ -7187,8 +7226,8 @@ async fn close_hive_server_transfer_cursor(state: &AppState, pool_key: &str, cur
/// Transfer a single table. Returns rows transferred.
/// `progress_callback` is invoked for progress updates.
#[allow(clippy::too_many_arguments)]
pub async fn transfer_table<F>(
state: &AppState,
async fn transfer_table_inner<F>(
state: &Arc<AppState>,
request: &TransferRequest,
table: &str,
table_index: usize,
@@ -7265,45 +7304,23 @@ where
log::info!("[transfer] {} has {} columns, counting rows...", table, columns.len());
}
// Fetch source table comment
// Route through the catalog-aware path for Doris/StarRocks external catalogs
// so the comment comes from the selected catalog, not the default one.
let table_comment: Option<String> =
if let Some(catalog) = resolve_external_transfer_catalog(request.source_catalog.as_deref(), source_db_type) {
crate::schema::list_doris_catalog_tables_core(
state,
&request.source_connection_id,
catalog,
&request.source_database,
Some(table),
Some(1),
None,
None,
None,
)
.await
.unwrap_or_default()
.into_iter()
.next()
.and_then(|t| t.comment)
} else {
crate::schema::list_tables_core(
state,
&request.source_connection_id,
&request.source_database,
&request.source_schema,
Some(table),
Some(1),
None,
None,
None,
)
.await
.unwrap_or_default()
.into_iter()
.next()
.and_then(|t| t.comment)
};
// Fetch source table comment. Keep this list-tables metadata chain on its
// own task stack, just like the target-table lookup above.
let table_comment = list_transfer_tables_isolated(
state.clone(),
request.source_connection_id.clone(),
request.source_database.clone(),
request.source_schema.clone(),
request.source_catalog.clone(),
*source_db_type,
table.to_string(),
1,
)
.await
.unwrap_or_default()
.into_iter()
.next()
.and_then(|table| table.comment);
let source_indexes =
if request.create_table && pg_compat_transfer && preserves_target_table_name && !target_table_preexisting {
@@ -7862,6 +7879,74 @@ where
Ok(total_transferred)
}
/// Transfer one table on its own Tokio task so the large transfer future and
/// nested driver metadata futures do not share a single worker stack.
#[allow(clippy::too_many_arguments)]
pub async fn transfer_table<F>(
state: &Arc<AppState>,
request: &TransferRequest,
table: &str,
table_index: usize,
source_db_type: &DatabaseType,
target_db_type: &DatabaseType,
source_pool_key: &str,
target_pool_key: &str,
known_foreign_keys: &HashMap<String, Vec<db::ForeignKeyInfo>>,
pending_fk_alters: &mut Vec<(String, String)>,
mut progress_callback: F,
) -> Result<u64, String>
where
F: FnMut(TransferProgress),
{
let state = state.clone();
let request = request.clone();
let table = table.to_string();
let source_db_type = *source_db_type;
let target_db_type = *target_db_type;
let source_pool_key = source_pool_key.to_string();
let target_pool_key = target_pool_key.to_string();
let known_foreign_keys = known_foreign_keys
.get(&table)
.map(|foreign_keys| HashMap::from([(table.clone(), foreign_keys.clone())]))
.unwrap_or_default();
let (progress_tx, mut progress_rx) = tokio::sync::mpsc::channel(TRANSFER_PROGRESS_CHANNEL_CAPACITY);
let mut task = tokio::spawn(async move {
let mut task_pending_fk_alters = Vec::new();
let result = transfer_table_inner(
&state,
&request,
&table,
table_index,
&source_db_type,
&target_db_type,
&source_pool_key,
&target_pool_key,
&known_foreign_keys,
&mut task_pending_fk_alters,
move |progress| {
try_send_transfer_progress(&progress_tx, progress);
},
)
.await;
(result, task_pending_fk_alters)
});
let _abort_on_drop = AbortTransferTaskOnDrop(task.abort_handle());
loop {
tokio::select! {
biased;
Some(progress) = progress_rx.recv() => progress_callback(progress),
result = &mut task => {
let (result, task_pending_fk_alters) =
result.map_err(|error| format!("Transfer table task failed: {error}"))?;
pending_fk_alters.extend(task_pending_fk_alters);
return result;
}
}
}
}
pub async fn transfer_postgres_schema_dependencies<F>(
state: &AppState,
request: &TransferRequest,
@@ -12992,4 +13077,27 @@ CREATE INDEX items_name_idx ON public.items (id);"#;
assert_eq!(statements.len(), 1);
assert!(statements[0].starts_with("CREATE TABLE"));
}
#[test]
fn transfer_progress_queue_has_bounded_capacity() {
let (sender, receiver) = tokio::sync::mpsc::channel(TRANSFER_PROGRESS_CHANNEL_CAPACITY);
for rows_transferred in 0..=TRANSFER_PROGRESS_CHANNEL_CAPACITY {
try_send_transfer_progress(
&sender,
TransferProgress {
transfer_id: "transfer".to_string(),
table: "table".to_string(),
table_index: 0,
total_tables: 1,
rows_transferred: rows_transferred as u64,
total_rows: None,
status: TransferStatus::Running,
error: None,
terminal: false,
},
);
}
assert_eq!(receiver.len(), TRANSFER_PROGRESS_CHANNEL_CAPACITY);
}
}
+163 -5
View File
@@ -6,6 +6,7 @@ use dbx_core::transfer::{
transfer_table, TransferContent, TransferMode, TransferOwnershipPolicy, TransferRequest, TransferTableNameCase,
};
use serde_json::json;
use std::sync::Arc;
use std::time::Duration;
fn live_mysql_config(id: &str) -> ConnectionConfig {
@@ -98,6 +99,163 @@ async fn query_text(pool: &mysql::MySqlPool, sql: &str) -> String {
mysql::execute_query(pool, sql, false).await.unwrap().rows[0][0].as_str().unwrap().to_string()
}
#[test]
#[ignore = "requires disposable MySQL 8.0.33 source and 8.0.45 target endpoints via DBX_LIVE_MYSQL_TRANSFER_SOURCE_*/TARGET_* variables"]
fn live_mysql_cross_version_transfer_completes_on_small_stack() {
std::thread::Builder::new()
.name("live-mysql-transfer-small-stack".to_string())
.stack_size(2 * 1024 * 1024)
.spawn(|| {
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap()
.block_on(run_live_mysql_cross_version_transfer_completes_on_small_stack());
})
.unwrap()
.join()
.unwrap();
}
async fn run_live_mysql_cross_version_transfer_completes_on_small_stack() {
let suffix = uuid::Uuid::new_v4().simple().to_string();
let source_connection_id = format!("small-stack-source-{suffix}");
let target_connection_id = format!("small-stack-target-{suffix}");
let source_database = format!("dbx_stack_src_{}", &suffix[..12]);
let target_database = format!("dbx_stack_dst_{}", &suffix[..12]);
let source_config = live_cross_version_mysql_config(&source_connection_id, "SOURCE");
let target_config = live_cross_version_mysql_config(&target_connection_id, "TARGET");
let source_setup_pool = mysql::connect(&mysql_url(&source_config), Duration::from_secs(10)).await.unwrap();
let target_setup_pool = mysql::connect(&mysql_url(&target_config), Duration::from_secs(10)).await.unwrap();
mysql::execute_query(
&source_setup_pool,
&format!(
"CREATE DATABASE {source_database} CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;\
CREATE TABLE {source_database}.title_task (\
id VARCHAR(32) NOT NULL, deleted BIT(1) NOT NULL DEFAULT b'0', creator BIGINT NOT NULL,\
create_time DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, updater BIGINT DEFAULT NULL,\
update_time DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,\
creator_name VARCHAR(255), updater_name VARCHAR(255), task_name VARCHAR(100),\
title_level VARCHAR(100), task_year INT, review_result VARCHAR(100),\
review_mode TINYINT NOT NULL DEFAULT 0, score_snapshot_json TEXT,\
task_start_time DATETIME, task_end_time DATETIME, task_status INT DEFAULT 0,\
is_remove_highest_score INT DEFAULT 0, is_remove_minimum_score INT DEFAULT 0,\
task_files TEXT, task_level VARCHAR(30), org_id BIGINT, org_code VARCHAR(255),\
org_name VARCHAR(255), school_id BIGINT, school_name VARCHAR(255),\
province_code VARCHAR(50), province_name VARCHAR(255), city_code VARCHAR(50),\
city_name VARCHAR(255), area_code VARCHAR(50), area_name VARCHAR(255),\
PRIMARY KEY (id), KEY idx_task_name (task_name), KEY idx_task_year (task_year),\
KEY idx_task_status (task_status), KEY idx_org_id (org_id), KEY idx_org_code (org_code)\
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;\
INSERT INTO {source_database}.title_task \
(id, deleted, creator, review_result, review_mode, score_snapshot_json) VALUES\
('small-stack-default', b'0', 1, 'pending', 0, '{{\"score\":88}}'),\
('small-stack-non-default', b'0', 2, 'approved', 2, '{{\"score\":95}}')"
),
true,
)
.await
.unwrap();
mysql::execute_query(&target_setup_pool, &format!("CREATE DATABASE {target_database}"), false).await.unwrap();
let task_tmp = std::path::PathBuf::from(
std::env::var("DBX_LIVE_MYSQL_TRANSFER_TMP_DIR").expect("DBX_LIVE_MYSQL_TRANSFER_TMP_DIR"),
);
let dir = task_tmp.join(format!("small-stack-transfer-{suffix}"));
std::fs::create_dir_all(&dir).unwrap();
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
let state = Arc::new(AppState::new(storage));
state.configs.write().await.insert(source_connection_id.clone(), source_config);
state.configs.write().await.insert(target_connection_id.clone(), target_config);
let source_pool_key = state.get_or_create_pool(&source_connection_id, Some(&source_database)).await.unwrap();
let target_pool_key = state.get_or_create_pool(&target_connection_id, Some(&target_database)).await.unwrap();
let request = TransferRequest {
transfer_id: format!("small-stack-transfer-{suffix}"),
source_connection_id,
source_database: source_database.clone(),
source_schema: source_database.clone(),
source_catalog: None,
target_connection_id,
target_database: target_database.clone(),
target_schema: target_database.clone(),
target_catalog: None,
tables: vec!["title_task".to_string()],
create_table: true,
content: TransferContent::default(),
objects: Vec::new(),
mode: TransferMode::Append,
target_table_name_case: TransferTableNameCase::Preserve,
quote_target_column_names: true,
ownership_policy: TransferOwnershipPolicy::Preserve,
batch_size: 100,
};
let test_result = async {
let transferred = transfer_table(
&state,
&request,
"title_task",
0,
&DatabaseType::Mysql,
&DatabaseType::Mysql,
&source_pool_key,
&target_pool_key,
&std::collections::HashMap::new(),
&mut Vec::new(),
|_| {},
)
.await?;
assert_eq!(transferred, 2);
assert_eq!(
query_text(
&target_setup_pool,
&format!(
"SELECT CAST(COUNT(*) AS CHAR) FROM information_schema.columns \
WHERE table_schema = '{target_database}' AND table_name = 'title_task'"
),
)
.await,
"32"
);
assert_eq!(
query_text(
&target_setup_pool,
&format!(
"SELECT GROUP_CONCAT(CONCAT(id, ':', review_mode) ORDER BY id SEPARATOR ',') \
FROM {target_database}.title_task"
),
)
.await,
"small-stack-default:0,small-stack-non-default:2"
);
let review_mode = mysql::execute_query(
&target_setup_pool,
&format!(
"SELECT ordinal_position, column_type, is_nullable, column_default \
FROM information_schema.columns WHERE table_schema = '{target_database}' \
AND table_name = 'title_task' AND column_name = 'review_mode'"
),
false,
)
.await?;
assert_eq!(review_mode.rows.len(), 1);
assert_eq!(review_mode.rows[0][0], json!("13"));
assert_eq!(review_mode.rows[0][1], json!("tinyint"));
assert_eq!(review_mode.rows[0][2], json!("NO"));
assert_eq!(review_mode.rows[0][3], json!("0"));
Ok::<_, String>(())
}
.await;
mysql::execute_query(&source_setup_pool, &format!("DROP DATABASE {source_database}"), false).await.unwrap();
mysql::execute_query(&target_setup_pool, &format!("DROP DATABASE {target_database}"), false).await.unwrap();
source_setup_pool.disconnect().await.unwrap();
target_setup_pool.disconnect().await.unwrap();
let _ = std::fs::remove_dir_all(dir);
test_result.unwrap();
}
#[tokio::test]
#[ignore = "requires disposable MySQL 8 endpoints via DBX_LIVE_MYSQL_TRANSFER_* variables"]
async fn live_mysql_transfer_keeps_columns_whose_comment_mentions_foreign_key() {
@@ -159,7 +317,7 @@ async fn live_mysql_transfer_keeps_columns_whose_comment_mentions_foreign_key()
let dir = task_tmp.join(format!("live-mysql-fk-comment-transfer-{suffix}"));
std::fs::create_dir_all(&dir).unwrap();
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
let state = AppState::new(storage);
let state = Arc::new(AppState::new(storage));
state.configs.write().await.insert(source_connection_id.clone(), source_config);
state.configs.write().await.insert(target_connection_id.clone(), target_config);
let source_pool_key = state.get_or_create_pool(&source_connection_id, Some(&source_database)).await.unwrap();
@@ -312,7 +470,7 @@ async fn live_mysql_transfer_downgrades_unsupported_source_collations() {
let dir = task_tmp.join(format!("live-mysql-collation-transfer-{suffix}"));
std::fs::create_dir_all(&dir).unwrap();
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
let state = AppState::new(storage);
let state = Arc::new(AppState::new(storage));
state.configs.write().await.insert(source_connection_id.clone(), source_config);
state.configs.write().await.insert(target_connection_id.clone(), target_config);
let source_pool_key = state.get_or_create_pool(&source_connection_id, Some(&source_database)).await.unwrap();
@@ -493,7 +651,7 @@ async fn live_mysql_transfer_preserves_spatial_values_and_modes() {
let dir = std::env::temp_dir().join(format!("dbx-live-mysql-transfer-{suffix}"));
std::fs::create_dir_all(&dir).unwrap();
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
let state = AppState::new(storage);
let state = Arc::new(AppState::new(storage));
state.configs.write().await.insert(connection_id.clone(), config);
let source_pool_key = state.get_or_create_pool(&connection_id, Some(&source_database)).await.unwrap();
let target_pool_key = state.get_or_create_pool(&connection_id, Some(&target_database)).await.unwrap();
@@ -673,7 +831,7 @@ async fn live_mysql_transfer_structure_overwrite_rejects_incompatible_target_col
let dir = std::env::temp_dir().join(format!("dbx-live-mysql-transfer-struct-{suffix}"));
std::fs::create_dir_all(&dir).unwrap();
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
let state = AppState::new(storage);
let state = Arc::new(AppState::new(storage));
state.configs.write().await.insert(connection_id.clone(), config);
let source_pool_key = state.get_or_create_pool(&connection_id, Some(&source_database)).await.unwrap();
let target_pool_key = state.get_or_create_pool(&connection_id, Some(&target_database)).await.unwrap();
@@ -823,7 +981,7 @@ async fn live_mysql_transfer_structure_only_rejects_incompatible_target_columns(
let dir = std::env::temp_dir().join(format!("dbx-live-mysql-transfer-structonly-{suffix}"));
std::fs::create_dir_all(&dir).unwrap();
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
let state = AppState::new(storage);
let state = Arc::new(AppState::new(storage));
state.configs.write().await.insert(connection_id.clone(), config);
let source_pool_key = state.get_or_create_pool(&connection_id, Some(&source_database)).await.unwrap();
let target_pool_key = state.get_or_create_pool(&connection_id, Some(&target_database)).await.unwrap();
@@ -7,6 +7,7 @@ use dbx_core::transfer::{
TransferObjectKind, TransferObjectSelection, TransferOwnershipPolicy, TransferRequest, TransferTableNameCase,
};
use serde_json::json;
use std::sync::Arc;
fn postgres_test_config(id: &str, database: &str) -> ConnectionConfig {
ConnectionConfig {
@@ -193,7 +194,7 @@ async fn live_postgres_transfer_upserts_generated_always_identity_values() {
let dir = std::env::temp_dir().join(format!("dbx-live-always-transfer-{suffix}"));
std::fs::create_dir_all(&dir).unwrap();
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
let state = AppState::new(storage);
let state = Arc::new(AppState::new(storage));
let source_connection_id = "live-always-source";
let target_connection_id = "live-always-target";
let source_pool_key = format!("{source_connection_id}:{source_database}");
@@ -357,7 +358,7 @@ async fn live_postgres_structure_only_preserves_table_indexes() {
let dir = std::env::temp_dir().join(format!("dbx-live-structure-only-transfer-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
let state = AppState::new(storage);
let state = Arc::new(AppState::new(storage));
let source_connection_id = "live-structure-only-source";
let target_connection_id = "live-structure-only-target";
let source_pool_key = format!("{source_connection_id}:{source_database}");
@@ -600,7 +601,7 @@ async fn live_postgres_transfer_preserves_data_and_schema_objects() {
let dir = std::env::temp_dir().join(format!("dbx-live-transfer-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
let state = AppState::new(storage);
let state = Arc::new(AppState::new(storage));
let source_connection_id = "live-source";
let target_connection_id = "live-target";
@@ -888,7 +889,7 @@ async fn live_postgres_transfer_skips_create_ddl_for_existing_target_table() {
let dir = std::env::temp_dir().join(format!("dbx-live-existing-transfer-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
let state = AppState::new(storage);
let state = Arc::new(AppState::new(storage));
let source_connection_id = "live-existing-source";
let target_connection_id = "live-existing-target";
@@ -1007,7 +1008,7 @@ async fn live_postgres_transfer_creates_selected_sequence_before_referencing_tab
let dir = std::env::temp_dir().join(format!("dbx-live-sequence-transfer-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
let state = AppState::new(storage);
let state = Arc::new(AppState::new(storage));
let source_connection_id = "live-sequence-source";
let target_connection_id = "live-sequence-target";
let source_pool_key = format!("{source_connection_id}:{source_database}");
@@ -1351,7 +1351,7 @@ async fn live_sqlserver_transfer_table_skips_rowversion_insert_column() {
let dir = std::env::temp_dir().join(format!("dbx-live-sqlserver-rowversion-{suffix}"));
std::fs::create_dir_all(&dir).unwrap();
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
let state = AppState::new(storage);
let state = Arc::new(AppState::new(storage));
let config = live_sqlserver_config("live-sqlserver-rowversion", &database);
state.configs.write().await.insert(config.id.clone(), config);
let source_pool_key =