fix(transfer): isolate table transfer task
This commit is contained in:
1 parent
51ca40e86e
commit
48f4fac89e
4 files changed
+356
-89
No files matched your search
+186
-78
@@ -1,6 +1,7 @@
|
||||
use regex::Regex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use crate::connection::{config_for_pool_key, AppState, PoolKind};
|
||||
@@ -798,8 +799,66 @@ fn existing_transfer_target_table_name(
|
||||
tables.iter().find(|table| table.name.eq_ignore_ascii_case(requested_name)).map(|table| table.name.clone())
|
||||
}
|
||||
|
||||
const TRANSFER_PROGRESS_CHANNEL_CAPACITY: usize = 16;
|
||||
|
||||
fn try_send_transfer_progress(sender: &tokio::sync::mpsc::Sender<TransferProgress>, progress: TransferProgress) {
|
||||
let _ = sender.try_send(progress);
|
||||
}
|
||||
|
||||
struct AbortTransferTaskOnDrop(tokio::task::AbortHandle);
|
||||
|
||||
impl Drop for AbortTransferTaskOnDrop {
|
||||
fn drop(&mut self) {
|
||||
self.0.abort();
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn list_transfer_tables_isolated(
|
||||
state: Arc<AppState>,
|
||||
connection_id: String,
|
||||
database: String,
|
||||
schema: String,
|
||||
catalog: Option<String>,
|
||||
db_type: DatabaseType,
|
||||
table: String,
|
||||
limit: usize,
|
||||
) -> Result<Vec<db::TableInfo>, String> {
|
||||
let task = tokio::spawn(async move {
|
||||
if let Some(catalog) = resolve_external_transfer_catalog(catalog.as_deref(), &db_type) {
|
||||
crate::schema::list_doris_catalog_tables_core(
|
||||
&state,
|
||||
&connection_id,
|
||||
catalog,
|
||||
&database,
|
||||
Some(&table),
|
||||
Some(limit),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
crate::schema::list_tables_core(
|
||||
&state,
|
||||
&connection_id,
|
||||
&database,
|
||||
&schema,
|
||||
Some(&table),
|
||||
Some(limit),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
});
|
||||
let _abort_on_drop = AbortTransferTaskOnDrop(task.abort_handle());
|
||||
task.await.map_err(|error| format!("Transfer table metadata task failed: {error}"))?
|
||||
}
|
||||
|
||||
async fn resolve_transfer_target_table_name(
|
||||
state: &AppState,
|
||||
state: &Arc<AppState>,
|
||||
request: &TransferRequest,
|
||||
source_table: &str,
|
||||
target_pool_key: &str,
|
||||
@@ -818,41 +877,21 @@ async fn resolve_transfer_target_table_name(
|
||||
// Route through the catalog-aware path when targeting an external
|
||||
// Doris/StarRocks catalog — otherwise the lookup runs against the
|
||||
// default / internal catalog and can miss or misidentify the table.
|
||||
let tables = if let Some(catalog) = resolve_external_transfer_catalog(target_catalog, target_db_type) {
|
||||
crate::schema::list_doris_catalog_tables_core(
|
||||
state,
|
||||
&request.target_connection_id,
|
||||
catalog,
|
||||
&request.target_database,
|
||||
Some(&requested_name),
|
||||
Some(TRANSFER_TARGET_TABLE_LOOKUP_LIMIT),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|error| {
|
||||
log::debug!("[transfer] failed to resolve target table metadata for {requested_name} in catalog '{catalog}': {error}");
|
||||
Vec::new()
|
||||
})
|
||||
} else {
|
||||
crate::schema::list_tables_core(
|
||||
state,
|
||||
&request.target_connection_id,
|
||||
&request.target_database,
|
||||
&request.target_schema,
|
||||
Some(&requested_name),
|
||||
Some(TRANSFER_TARGET_TABLE_LOOKUP_LIMIT),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|error| {
|
||||
log::debug!("[transfer] failed to resolve target table metadata for {requested_name}: {error}");
|
||||
Vec::new()
|
||||
})
|
||||
};
|
||||
let tables = list_transfer_tables_isolated(
|
||||
state.clone(),
|
||||
request.target_connection_id.clone(),
|
||||
request.target_database.clone(),
|
||||
request.target_schema.clone(),
|
||||
target_catalog.map(str::to_string),
|
||||
*target_db_type,
|
||||
requested_name.clone(),
|
||||
TRANSFER_TARGET_TABLE_LOOKUP_LIMIT,
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|error| {
|
||||
log::debug!("[transfer] failed to resolve target table metadata for {requested_name}: {error}");
|
||||
Vec::new()
|
||||
});
|
||||
|
||||
if let Some(existing_name) =
|
||||
existing_transfer_target_table_name(&requested_name, &tables, allow_case_insensitive_match)
|
||||
@@ -6827,7 +6866,7 @@ pub(crate) fn sort_table_names_by_dependencies(
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn transfer_mongodb_table<F>(
|
||||
state: &AppState,
|
||||
state: &Arc<AppState>,
|
||||
request: &TransferRequest,
|
||||
table: &str,
|
||||
table_index: usize,
|
||||
@@ -7187,8 +7226,8 @@ async fn close_hive_server_transfer_cursor(state: &AppState, pool_key: &str, cur
|
||||
/// Transfer a single table. Returns rows transferred.
|
||||
/// `progress_callback` is invoked for progress updates.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn transfer_table<F>(
|
||||
state: &AppState,
|
||||
async fn transfer_table_inner<F>(
|
||||
state: &Arc<AppState>,
|
||||
request: &TransferRequest,
|
||||
table: &str,
|
||||
table_index: usize,
|
||||
@@ -7265,45 +7304,23 @@ where
|
||||
log::info!("[transfer] {} has {} columns, counting rows...", table, columns.len());
|
||||
}
|
||||
|
||||
// Fetch source table comment
|
||||
// Route through the catalog-aware path for Doris/StarRocks external catalogs
|
||||
// so the comment comes from the selected catalog, not the default one.
|
||||
let table_comment: Option<String> =
|
||||
if let Some(catalog) = resolve_external_transfer_catalog(request.source_catalog.as_deref(), source_db_type) {
|
||||
crate::schema::list_doris_catalog_tables_core(
|
||||
state,
|
||||
&request.source_connection_id,
|
||||
catalog,
|
||||
&request.source_database,
|
||||
Some(table),
|
||||
Some(1),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.next()
|
||||
.and_then(|t| t.comment)
|
||||
} else {
|
||||
crate::schema::list_tables_core(
|
||||
state,
|
||||
&request.source_connection_id,
|
||||
&request.source_database,
|
||||
&request.source_schema,
|
||||
Some(table),
|
||||
Some(1),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.next()
|
||||
.and_then(|t| t.comment)
|
||||
};
|
||||
// Fetch source table comment. Keep this list-tables metadata chain on its
|
||||
// own task stack, just like the target-table lookup above.
|
||||
let table_comment = list_transfer_tables_isolated(
|
||||
state.clone(),
|
||||
request.source_connection_id.clone(),
|
||||
request.source_database.clone(),
|
||||
request.source_schema.clone(),
|
||||
request.source_catalog.clone(),
|
||||
*source_db_type,
|
||||
table.to_string(),
|
||||
1,
|
||||
)
|
||||
.await
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.next()
|
||||
.and_then(|table| table.comment);
|
||||
|
||||
let source_indexes =
|
||||
if request.create_table && pg_compat_transfer && preserves_target_table_name && !target_table_preexisting {
|
||||
@@ -7862,6 +7879,74 @@ where
|
||||
Ok(total_transferred)
|
||||
}
|
||||
|
||||
/// Transfer one table on its own Tokio task so the large transfer future and
|
||||
/// nested driver metadata futures do not share a single worker stack.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn transfer_table<F>(
|
||||
state: &Arc<AppState>,
|
||||
request: &TransferRequest,
|
||||
table: &str,
|
||||
table_index: usize,
|
||||
source_db_type: &DatabaseType,
|
||||
target_db_type: &DatabaseType,
|
||||
source_pool_key: &str,
|
||||
target_pool_key: &str,
|
||||
known_foreign_keys: &HashMap<String, Vec<db::ForeignKeyInfo>>,
|
||||
pending_fk_alters: &mut Vec<(String, String)>,
|
||||
mut progress_callback: F,
|
||||
) -> Result<u64, String>
|
||||
where
|
||||
F: FnMut(TransferProgress),
|
||||
{
|
||||
let state = state.clone();
|
||||
let request = request.clone();
|
||||
let table = table.to_string();
|
||||
let source_db_type = *source_db_type;
|
||||
let target_db_type = *target_db_type;
|
||||
let source_pool_key = source_pool_key.to_string();
|
||||
let target_pool_key = target_pool_key.to_string();
|
||||
let known_foreign_keys = known_foreign_keys
|
||||
.get(&table)
|
||||
.map(|foreign_keys| HashMap::from([(table.clone(), foreign_keys.clone())]))
|
||||
.unwrap_or_default();
|
||||
let (progress_tx, mut progress_rx) = tokio::sync::mpsc::channel(TRANSFER_PROGRESS_CHANNEL_CAPACITY);
|
||||
|
||||
let mut task = tokio::spawn(async move {
|
||||
let mut task_pending_fk_alters = Vec::new();
|
||||
let result = transfer_table_inner(
|
||||
&state,
|
||||
&request,
|
||||
&table,
|
||||
table_index,
|
||||
&source_db_type,
|
||||
&target_db_type,
|
||||
&source_pool_key,
|
||||
&target_pool_key,
|
||||
&known_foreign_keys,
|
||||
&mut task_pending_fk_alters,
|
||||
move |progress| {
|
||||
try_send_transfer_progress(&progress_tx, progress);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
(result, task_pending_fk_alters)
|
||||
});
|
||||
let _abort_on_drop = AbortTransferTaskOnDrop(task.abort_handle());
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
biased;
|
||||
Some(progress) = progress_rx.recv() => progress_callback(progress),
|
||||
result = &mut task => {
|
||||
let (result, task_pending_fk_alters) =
|
||||
result.map_err(|error| format!("Transfer table task failed: {error}"))?;
|
||||
pending_fk_alters.extend(task_pending_fk_alters);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn transfer_postgres_schema_dependencies<F>(
|
||||
state: &AppState,
|
||||
request: &TransferRequest,
|
||||
@@ -12992,4 +13077,27 @@ CREATE INDEX items_name_idx ON public.items (id);"#;
|
||||
assert_eq!(statements.len(), 1);
|
||||
assert!(statements[0].starts_with("CREATE TABLE"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transfer_progress_queue_has_bounded_capacity() {
|
||||
let (sender, receiver) = tokio::sync::mpsc::channel(TRANSFER_PROGRESS_CHANNEL_CAPACITY);
|
||||
for rows_transferred in 0..=TRANSFER_PROGRESS_CHANNEL_CAPACITY {
|
||||
try_send_transfer_progress(
|
||||
&sender,
|
||||
TransferProgress {
|
||||
transfer_id: "transfer".to_string(),
|
||||
table: "table".to_string(),
|
||||
table_index: 0,
|
||||
total_tables: 1,
|
||||
rows_transferred: rows_transferred as u64,
|
||||
total_rows: None,
|
||||
status: TransferStatus::Running,
|
||||
error: None,
|
||||
terminal: false,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
assert_eq!(receiver.len(), TRANSFER_PROGRESS_CHANNEL_CAPACITY);
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ use dbx_core::transfer::{
|
||||
transfer_table, TransferContent, TransferMode, TransferOwnershipPolicy, TransferRequest, TransferTableNameCase,
|
||||
};
|
||||
use serde_json::json;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
fn live_mysql_config(id: &str) -> ConnectionConfig {
|
||||
@@ -98,6 +99,163 @@ async fn query_text(pool: &mysql::MySqlPool, sql: &str) -> String {
|
||||
mysql::execute_query(pool, sql, false).await.unwrap().rows[0][0].as_str().unwrap().to_string()
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires disposable MySQL 8.0.33 source and 8.0.45 target endpoints via DBX_LIVE_MYSQL_TRANSFER_SOURCE_*/TARGET_* variables"]
|
||||
fn live_mysql_cross_version_transfer_completes_on_small_stack() {
|
||||
std::thread::Builder::new()
|
||||
.name("live-mysql-transfer-small-stack".to_string())
|
||||
.stack_size(2 * 1024 * 1024)
|
||||
.spawn(|| {
|
||||
tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.unwrap()
|
||||
.block_on(run_live_mysql_cross_version_transfer_completes_on_small_stack());
|
||||
})
|
||||
.unwrap()
|
||||
.join()
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn run_live_mysql_cross_version_transfer_completes_on_small_stack() {
|
||||
let suffix = uuid::Uuid::new_v4().simple().to_string();
|
||||
let source_connection_id = format!("small-stack-source-{suffix}");
|
||||
let target_connection_id = format!("small-stack-target-{suffix}");
|
||||
let source_database = format!("dbx_stack_src_{}", &suffix[..12]);
|
||||
let target_database = format!("dbx_stack_dst_{}", &suffix[..12]);
|
||||
let source_config = live_cross_version_mysql_config(&source_connection_id, "SOURCE");
|
||||
let target_config = live_cross_version_mysql_config(&target_connection_id, "TARGET");
|
||||
let source_setup_pool = mysql::connect(&mysql_url(&source_config), Duration::from_secs(10)).await.unwrap();
|
||||
let target_setup_pool = mysql::connect(&mysql_url(&target_config), Duration::from_secs(10)).await.unwrap();
|
||||
|
||||
mysql::execute_query(
|
||||
&source_setup_pool,
|
||||
&format!(
|
||||
"CREATE DATABASE {source_database} CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci;\
|
||||
CREATE TABLE {source_database}.title_task (\
|
||||
id VARCHAR(32) NOT NULL, deleted BIT(1) NOT NULL DEFAULT b'0', creator BIGINT NOT NULL,\
|
||||
create_time DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, updater BIGINT DEFAULT NULL,\
|
||||
update_time DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,\
|
||||
creator_name VARCHAR(255), updater_name VARCHAR(255), task_name VARCHAR(100),\
|
||||
title_level VARCHAR(100), task_year INT, review_result VARCHAR(100),\
|
||||
review_mode TINYINT NOT NULL DEFAULT 0, score_snapshot_json TEXT,\
|
||||
task_start_time DATETIME, task_end_time DATETIME, task_status INT DEFAULT 0,\
|
||||
is_remove_highest_score INT DEFAULT 0, is_remove_minimum_score INT DEFAULT 0,\
|
||||
task_files TEXT, task_level VARCHAR(30), org_id BIGINT, org_code VARCHAR(255),\
|
||||
org_name VARCHAR(255), school_id BIGINT, school_name VARCHAR(255),\
|
||||
province_code VARCHAR(50), province_name VARCHAR(255), city_code VARCHAR(50),\
|
||||
city_name VARCHAR(255), area_code VARCHAR(50), area_name VARCHAR(255),\
|
||||
PRIMARY KEY (id), KEY idx_task_name (task_name), KEY idx_task_year (task_year),\
|
||||
KEY idx_task_status (task_status), KEY idx_org_id (org_id), KEY idx_org_code (org_code)\
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;\
|
||||
INSERT INTO {source_database}.title_task \
|
||||
(id, deleted, creator, review_result, review_mode, score_snapshot_json) VALUES\
|
||||
('small-stack-default', b'0', 1, 'pending', 0, '{{\"score\":88}}'),\
|
||||
('small-stack-non-default', b'0', 2, 'approved', 2, '{{\"score\":95}}')"
|
||||
),
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
mysql::execute_query(&target_setup_pool, &format!("CREATE DATABASE {target_database}"), false).await.unwrap();
|
||||
|
||||
let task_tmp = std::path::PathBuf::from(
|
||||
std::env::var("DBX_LIVE_MYSQL_TRANSFER_TMP_DIR").expect("DBX_LIVE_MYSQL_TRANSFER_TMP_DIR"),
|
||||
);
|
||||
let dir = task_tmp.join(format!("small-stack-transfer-{suffix}"));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
|
||||
let state = Arc::new(AppState::new(storage));
|
||||
state.configs.write().await.insert(source_connection_id.clone(), source_config);
|
||||
state.configs.write().await.insert(target_connection_id.clone(), target_config);
|
||||
let source_pool_key = state.get_or_create_pool(&source_connection_id, Some(&source_database)).await.unwrap();
|
||||
let target_pool_key = state.get_or_create_pool(&target_connection_id, Some(&target_database)).await.unwrap();
|
||||
let request = TransferRequest {
|
||||
transfer_id: format!("small-stack-transfer-{suffix}"),
|
||||
source_connection_id,
|
||||
source_database: source_database.clone(),
|
||||
source_schema: source_database.clone(),
|
||||
source_catalog: None,
|
||||
target_connection_id,
|
||||
target_database: target_database.clone(),
|
||||
target_schema: target_database.clone(),
|
||||
target_catalog: None,
|
||||
tables: vec!["title_task".to_string()],
|
||||
create_table: true,
|
||||
content: TransferContent::default(),
|
||||
objects: Vec::new(),
|
||||
mode: TransferMode::Append,
|
||||
target_table_name_case: TransferTableNameCase::Preserve,
|
||||
quote_target_column_names: true,
|
||||
ownership_policy: TransferOwnershipPolicy::Preserve,
|
||||
batch_size: 100,
|
||||
};
|
||||
|
||||
let test_result = async {
|
||||
let transferred = transfer_table(
|
||||
&state,
|
||||
&request,
|
||||
"title_task",
|
||||
0,
|
||||
&DatabaseType::Mysql,
|
||||
&DatabaseType::Mysql,
|
||||
&source_pool_key,
|
||||
&target_pool_key,
|
||||
&std::collections::HashMap::new(),
|
||||
&mut Vec::new(),
|
||||
|_| {},
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(transferred, 2);
|
||||
assert_eq!(
|
||||
query_text(
|
||||
&target_setup_pool,
|
||||
&format!(
|
||||
"SELECT CAST(COUNT(*) AS CHAR) FROM information_schema.columns \
|
||||
WHERE table_schema = '{target_database}' AND table_name = 'title_task'"
|
||||
),
|
||||
)
|
||||
.await,
|
||||
"32"
|
||||
);
|
||||
assert_eq!(
|
||||
query_text(
|
||||
&target_setup_pool,
|
||||
&format!(
|
||||
"SELECT GROUP_CONCAT(CONCAT(id, ':', review_mode) ORDER BY id SEPARATOR ',') \
|
||||
FROM {target_database}.title_task"
|
||||
),
|
||||
)
|
||||
.await,
|
||||
"small-stack-default:0,small-stack-non-default:2"
|
||||
);
|
||||
let review_mode = mysql::execute_query(
|
||||
&target_setup_pool,
|
||||
&format!(
|
||||
"SELECT ordinal_position, column_type, is_nullable, column_default \
|
||||
FROM information_schema.columns WHERE table_schema = '{target_database}' \
|
||||
AND table_name = 'title_task' AND column_name = 'review_mode'"
|
||||
),
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(review_mode.rows.len(), 1);
|
||||
assert_eq!(review_mode.rows[0][0], json!("13"));
|
||||
assert_eq!(review_mode.rows[0][1], json!("tinyint"));
|
||||
assert_eq!(review_mode.rows[0][2], json!("NO"));
|
||||
assert_eq!(review_mode.rows[0][3], json!("0"));
|
||||
Ok::<_, String>(())
|
||||
}
|
||||
.await;
|
||||
|
||||
mysql::execute_query(&source_setup_pool, &format!("DROP DATABASE {source_database}"), false).await.unwrap();
|
||||
mysql::execute_query(&target_setup_pool, &format!("DROP DATABASE {target_database}"), false).await.unwrap();
|
||||
source_setup_pool.disconnect().await.unwrap();
|
||||
target_setup_pool.disconnect().await.unwrap();
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
test_result.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires disposable MySQL 8 endpoints via DBX_LIVE_MYSQL_TRANSFER_* variables"]
|
||||
async fn live_mysql_transfer_keeps_columns_whose_comment_mentions_foreign_key() {
|
||||
@@ -159,7 +317,7 @@ async fn live_mysql_transfer_keeps_columns_whose_comment_mentions_foreign_key()
|
||||
let dir = task_tmp.join(format!("live-mysql-fk-comment-transfer-{suffix}"));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
|
||||
let state = AppState::new(storage);
|
||||
let state = Arc::new(AppState::new(storage));
|
||||
state.configs.write().await.insert(source_connection_id.clone(), source_config);
|
||||
state.configs.write().await.insert(target_connection_id.clone(), target_config);
|
||||
let source_pool_key = state.get_or_create_pool(&source_connection_id, Some(&source_database)).await.unwrap();
|
||||
@@ -312,7 +470,7 @@ async fn live_mysql_transfer_downgrades_unsupported_source_collations() {
|
||||
let dir = task_tmp.join(format!("live-mysql-collation-transfer-{suffix}"));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
|
||||
let state = AppState::new(storage);
|
||||
let state = Arc::new(AppState::new(storage));
|
||||
state.configs.write().await.insert(source_connection_id.clone(), source_config);
|
||||
state.configs.write().await.insert(target_connection_id.clone(), target_config);
|
||||
let source_pool_key = state.get_or_create_pool(&source_connection_id, Some(&source_database)).await.unwrap();
|
||||
@@ -493,7 +651,7 @@ async fn live_mysql_transfer_preserves_spatial_values_and_modes() {
|
||||
let dir = std::env::temp_dir().join(format!("dbx-live-mysql-transfer-{suffix}"));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
|
||||
let state = AppState::new(storage);
|
||||
let state = Arc::new(AppState::new(storage));
|
||||
state.configs.write().await.insert(connection_id.clone(), config);
|
||||
let source_pool_key = state.get_or_create_pool(&connection_id, Some(&source_database)).await.unwrap();
|
||||
let target_pool_key = state.get_or_create_pool(&connection_id, Some(&target_database)).await.unwrap();
|
||||
@@ -673,7 +831,7 @@ async fn live_mysql_transfer_structure_overwrite_rejects_incompatible_target_col
|
||||
let dir = std::env::temp_dir().join(format!("dbx-live-mysql-transfer-struct-{suffix}"));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
|
||||
let state = AppState::new(storage);
|
||||
let state = Arc::new(AppState::new(storage));
|
||||
state.configs.write().await.insert(connection_id.clone(), config);
|
||||
let source_pool_key = state.get_or_create_pool(&connection_id, Some(&source_database)).await.unwrap();
|
||||
let target_pool_key = state.get_or_create_pool(&connection_id, Some(&target_database)).await.unwrap();
|
||||
@@ -823,7 +981,7 @@ async fn live_mysql_transfer_structure_only_rejects_incompatible_target_columns(
|
||||
let dir = std::env::temp_dir().join(format!("dbx-live-mysql-transfer-structonly-{suffix}"));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
|
||||
let state = AppState::new(storage);
|
||||
let state = Arc::new(AppState::new(storage));
|
||||
state.configs.write().await.insert(connection_id.clone(), config);
|
||||
let source_pool_key = state.get_or_create_pool(&connection_id, Some(&source_database)).await.unwrap();
|
||||
let target_pool_key = state.get_or_create_pool(&connection_id, Some(&target_database)).await.unwrap();
|
||||
|
||||
@@ -7,6 +7,7 @@ use dbx_core::transfer::{
|
||||
TransferObjectKind, TransferObjectSelection, TransferOwnershipPolicy, TransferRequest, TransferTableNameCase,
|
||||
};
|
||||
use serde_json::json;
|
||||
use std::sync::Arc;
|
||||
|
||||
fn postgres_test_config(id: &str, database: &str) -> ConnectionConfig {
|
||||
ConnectionConfig {
|
||||
@@ -193,7 +194,7 @@ async fn live_postgres_transfer_upserts_generated_always_identity_values() {
|
||||
let dir = std::env::temp_dir().join(format!("dbx-live-always-transfer-{suffix}"));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
|
||||
let state = AppState::new(storage);
|
||||
let state = Arc::new(AppState::new(storage));
|
||||
let source_connection_id = "live-always-source";
|
||||
let target_connection_id = "live-always-target";
|
||||
let source_pool_key = format!("{source_connection_id}:{source_database}");
|
||||
@@ -357,7 +358,7 @@ async fn live_postgres_structure_only_preserves_table_indexes() {
|
||||
let dir = std::env::temp_dir().join(format!("dbx-live-structure-only-transfer-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
|
||||
let state = AppState::new(storage);
|
||||
let state = Arc::new(AppState::new(storage));
|
||||
let source_connection_id = "live-structure-only-source";
|
||||
let target_connection_id = "live-structure-only-target";
|
||||
let source_pool_key = format!("{source_connection_id}:{source_database}");
|
||||
@@ -600,7 +601,7 @@ async fn live_postgres_transfer_preserves_data_and_schema_objects() {
|
||||
let dir = std::env::temp_dir().join(format!("dbx-live-transfer-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
|
||||
let state = AppState::new(storage);
|
||||
let state = Arc::new(AppState::new(storage));
|
||||
|
||||
let source_connection_id = "live-source";
|
||||
let target_connection_id = "live-target";
|
||||
@@ -888,7 +889,7 @@ async fn live_postgres_transfer_skips_create_ddl_for_existing_target_table() {
|
||||
let dir = std::env::temp_dir().join(format!("dbx-live-existing-transfer-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
|
||||
let state = AppState::new(storage);
|
||||
let state = Arc::new(AppState::new(storage));
|
||||
|
||||
let source_connection_id = "live-existing-source";
|
||||
let target_connection_id = "live-existing-target";
|
||||
@@ -1007,7 +1008,7 @@ async fn live_postgres_transfer_creates_selected_sequence_before_referencing_tab
|
||||
let dir = std::env::temp_dir().join(format!("dbx-live-sequence-transfer-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
|
||||
let state = AppState::new(storage);
|
||||
let state = Arc::new(AppState::new(storage));
|
||||
let source_connection_id = "live-sequence-source";
|
||||
let target_connection_id = "live-sequence-target";
|
||||
let source_pool_key = format!("{source_connection_id}:{source_database}");
|
||||
|
||||
@@ -1351,7 +1351,7 @@ async fn live_sqlserver_transfer_table_skips_rowversion_insert_column() {
|
||||
let dir = std::env::temp_dir().join(format!("dbx-live-sqlserver-rowversion-{suffix}"));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
|
||||
let state = AppState::new(storage);
|
||||
let state = Arc::new(AppState::new(storage));
|
||||
let config = live_sqlserver_config("live-sqlserver-rowversion", &database);
|
||||
state.configs.write().await.insert(config.id.clone(), config);
|
||||
let source_pool_key =
|
||||
|
||||
Reference in new issue
Block a user