fix(transfer): rebuild dropped pools between transfer statements
This commit is contained in:
3 files changed
+221
-13
No files matched your search
@@ -2274,3 +2274,91 @@ async fn live_mysql_transfer_drop_target_inspects_dependent_views_without_the_us
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
cleanup.unwrap();
|
||||
}
|
||||
|
||||
/// A single timed-out statement must not take the rest of the transfer down with it.
|
||||
///
|
||||
/// `execute_on_pool_once` drops the pool of a timed-out driver so a late server response
|
||||
/// cannot be reused; because a bulk transfer shares one pool key across every table, the
|
||||
/// pool has to be replaced before the next statement runs. Without that replacement the
|
||||
/// next table reported the misleading `Connection not found` instead of executing.
|
||||
#[tokio::test]
|
||||
#[ignore = "requires a disposable MySQL 5.7+ endpoint via DBX_LIVE_MYSQL_TRANSFER_* variables"]
|
||||
async fn live_mysql_transfer_recovers_the_pool_after_a_query_timeout() {
|
||||
let suffix = uuid::Uuid::new_v4().simple().to_string();
|
||||
let connection_id = format!("live-mysql-transfer-timeout-{suffix}");
|
||||
let database = format!("dbx_transfer_timeout_{}", &suffix[..12]);
|
||||
let mut config = live_mysql_config(&connection_id);
|
||||
// A one-second budget lets `SELECT SLEEP(3)` time out while leaving normal
|
||||
// statements (and the replacement pool) usable.
|
||||
config.query_timeout_secs = 1;
|
||||
|
||||
let setup_pool = mysql::connect(&mysql_url(&config), Duration::from_secs(10)).await.unwrap();
|
||||
mysql::execute_query(&setup_pool, &format!("CREATE DATABASE `{database}`"), true).await.unwrap();
|
||||
|
||||
let dir = std::env::temp_dir().join(format!("dbx-live-mysql-transfer-timeout-{suffix}"));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let storage = dbx_core::persistence::test_storage::open(&dir.join("storage.db")).await.unwrap();
|
||||
let state = Arc::new(AppState::new(storage));
|
||||
state.configs.write().await.insert(connection_id.clone(), config);
|
||||
let pool_key = state.get_or_create_pool(&connection_id, Some(&database)).await.unwrap();
|
||||
|
||||
let timed_out = execute_on_pool(&state, &pool_key, "SELECT SLEEP(3)")
|
||||
.await
|
||||
.expect_err("the statement must hit the configured one-second budget");
|
||||
assert!(timed_out.to_lowercase().contains("timed out"), "expected a timeout error, got: {timed_out}");
|
||||
|
||||
let follow_up = execute_on_pool(&state, &pool_key, "SELECT 1").await;
|
||||
assert!(
|
||||
follow_up.is_ok(),
|
||||
"a later transfer statement must run on a replacement pool, not report a missing connection: {:?}",
|
||||
follow_up.err()
|
||||
);
|
||||
|
||||
let cleanup = mysql::execute_query(&setup_pool, &format!("DROP DATABASE `{database}`"), true).await;
|
||||
setup_pool.disconnect().await.unwrap();
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
cleanup.unwrap();
|
||||
}
|
||||
|
||||
/// A pool removed while it is idle between two statements must be rebuilt instead of
|
||||
/// failing the rest of the transfer with `Connection not found`.
|
||||
///
|
||||
/// The connection keepalive tears a pool down whenever its ping fails or times out, and a
|
||||
/// bulk transfer shares one pool key across every table. The next statement therefore has
|
||||
/// to reconnect on its own rather than report a missing pool -- the statement has not run
|
||||
/// yet, so nothing is replayed.
|
||||
#[tokio::test]
|
||||
#[ignore = "requires a disposable MySQL 5.7+ endpoint via DBX_LIVE_MYSQL_TRANSFER_* variables"]
|
||||
async fn live_mysql_transfer_recreates_a_pool_removed_between_statements() {
|
||||
let suffix = uuid::Uuid::new_v4().simple().to_string();
|
||||
let connection_id = format!("live-mysql-transfer-removed-pool-{suffix}");
|
||||
let database = format!("dbx_transfer_recreate_{}", &suffix[..12]);
|
||||
let config = live_mysql_config(&connection_id);
|
||||
|
||||
let setup_pool = mysql::connect(&mysql_url(&config), Duration::from_secs(10)).await.unwrap();
|
||||
mysql::execute_query(&setup_pool, &format!("CREATE DATABASE `{database}`"), true).await.unwrap();
|
||||
|
||||
let dir = std::env::temp_dir().join(format!("dbx-live-mysql-transfer-recreate-{suffix}"));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let storage = dbx_core::persistence::test_storage::open(&dir.join("storage.db")).await.unwrap();
|
||||
let state = Arc::new(AppState::new(storage));
|
||||
state.configs.write().await.insert(connection_id.clone(), config);
|
||||
let pool_key = state.get_or_create_pool(&connection_id, Some(&database)).await.unwrap();
|
||||
execute_on_pool(&state, &pool_key, "SELECT 1").await.unwrap();
|
||||
|
||||
// Same effect as a keepalive invalidation (or any other path that drops a transfer
|
||||
// pool) while the transfer waits between two tables.
|
||||
assert!(state.remove_pool_by_key(&pool_key).await, "the pool under test must exist first");
|
||||
|
||||
let follow_up = execute_on_pool(&state, &pool_key, "SELECT 1").await;
|
||||
assert!(
|
||||
follow_up.is_ok(),
|
||||
"a statement after the pool disappeared must reconnect, not report a missing connection: {:?}",
|
||||
follow_up.err()
|
||||
);
|
||||
|
||||
let cleanup = mysql::execute_query(&setup_pool, &format!("DROP DATABASE `{database}`"), true).await;
|
||||
setup_pool.disconnect().await.unwrap();
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
cleanup.unwrap();
|
||||
}
|
||||
Reference in new issue
Block a user