From a41aec09eb167f597ef19e8f98202d9a8fb8404a Mon Sep 17 00:00:00 2001 From: zipg Date: Wed, 30 Sep 2026 00:59:43 +0800 Subject: [PATCH] fix(transfer): rebuild dropped pools between transfer statements --- crates/dbx-core/src/data/transfer.rs | 144 +++++++++++++++++-- crates/dbx-core/src/query/mod.rs | 2 +- crates/dbx-core/tests/live_mysql_transfer.rs | 88 ++++++++++++ 3 files changed, 221 insertions(+), 13 deletions(-) diff --git a/crates/dbx-core/src/data/transfer.rs b/crates/dbx-core/src/data/transfer.rs index aff46b861a..27de55a237 100644 --- a/crates/dbx-core/src/data/transfer.rs +++ b/crates/dbx-core/src/data/transfer.rs @@ -27,7 +27,8 @@ use crate::models::connection::{ConnectionConfig, DatabaseType}; use crate::object_source_sql::{build_executable_object_source_statements, EditableObjectSourceSqlInput}; use crate::query::{ agent_execute_query_params, is_dbx_query_timeout_error, pool_error_action, query_timeout_duration, - wait_for_query_opt, PoolErrorAction, QueryExecutionOptions, StreamProgressClock, AGENT_PROTOCOL_MAX_ROWS, + should_discard_pool_after_query_timeout, wait_for_query_opt, PoolErrorAction, QueryExecutionOptions, + StreamProgressClock, AGENT_PROTOCOL_MAX_ROWS, }; use crate::sql::{split_sql_statements, split_sql_statements_for_database}; use crate::sql_dialect::{ @@ -1920,9 +1921,9 @@ async fn execute_transfer_write_statement( crate::query::check_read_only_for_connection(state, target_pool_key, sql).await?; let (_, _, _, query_timeout_secs) = transfer_pool_context(state, target_pool_key).await; let query_timeout = query_timeout_duration(query_timeout_secs); - let pool_handle = state.pool_handle(target_pool_key).await; - let client = match pool_handle.as_ref() { - Some(PoolKind::SqlServer(client)) => client.clone(), + let pool = ensure_transfer_statement_pool(state, target_pool_key).await?; + let client = match &pool { + PoolKind::SqlServer(client) => client.clone(), _ => return Err("SQL Server connection not found".to_string()), }; let mut client = client.lock().await; @@ -6378,6 +6379,52 @@ fn should_prepare_fresh_agent_transfer_session(db_type: Option, er ) } +/// Whether a discarded transfer pool has to be replaced before the next statement runs. +/// +/// Every error that reaches the `Discard` arm tears the pool down: the failed statement +/// is never replayed (its outcome on the server is unknown), and a driver whose query +/// timed out may still own a checked-out connection. The pool key, though, is shared by +/// every table of the transfer, so leaving it missing turns one transient failure into a +/// *misleading* "Connection not found" on the next table instead of that table running +/// (or failing) on its own. Prepare a replacement for native drivers; agent/JDBC +/// sessions keep their stricter rule, where recovery stays gated on the structured +/// quarantine decision. +fn should_reconnect_discarded_transfer_pool(db_type: Option) -> bool { + db_type.is_some_and(|db_type| !crate::database_capabilities::is_agent_type(&db_type)) +} + +/// Resolve the pool a transfer statement runs on, re-creating it when it is gone. +/// +/// A bulk transfer lives for tens of minutes and shares one pool key across every table, +/// so a pool that disappears between two statements used to abort the run with a +/// misleading `Connection not found` that says nothing about the database being +/// unreachable. The pool can legitimately be gone: `execute_on_pool_once` drops the pool +/// of a driver whose query timed out, the connection keepalive tears down a pool whose +/// ping failed or timed out, and every other transfer statement reuses the same key. +/// Rebuilding it here is safe -- the statement has not run yet, so nothing is replayed -- +/// and keeps the transfer working on a fresh connection instead of failing the table. +async fn ensure_transfer_statement_pool(state: &AppState, pool_key: &str) -> Result { + if let Some(pool) = state.pool_handle(pool_key).await { + return Ok(pool); + } + let (connection_id, database, _, _) = transfer_pool_context(state, pool_key).await; + let Some(connection_id) = connection_id else { + return Err("Connection not found".to_string()); + }; + let catalog = catalog_from_pool_key(pool_key).map(str::to_string); + let client_session_id = client_session_id_from_pool_key(pool_key).map(str::to_string); + state + .get_or_create_pool_for_session_with_catalog( + &connection_id, + database.as_deref(), + catalog.as_deref(), + client_session_id.as_deref(), + ) + .await + .map_err(|error| format!("Connection not found: {error}"))?; + state.pool_handle(pool_key).await.ok_or_else(|| "Connection not found".to_string()) +} + async fn transfer_pool_context( state: &AppState, pool_key: &str, @@ -6437,7 +6484,8 @@ async fn execute_on_pool_with_options( // the failed statement above, only prepares the pool for // whichever statement runs next. let prepare_agent_session = should_prepare_fresh_agent_transfer_session(db_type, error); - if pool_error_action(db_type, error) == PoolErrorAction::ReconnectAndRetry || prepare_agent_session { + let reconnect_pool = should_reconnect_discarded_transfer_pool(db_type); + if reconnect_pool || prepare_agent_session { if let Some(connection_id) = connection_id.as_deref() { let catalog = catalog_from_pool_key(¤t_pool_key).map(str::to_string); let recovery = if prepare_agent_session { @@ -6501,13 +6549,12 @@ async fn execute_on_pool_once( sql: &str, max_rows: Option, ) -> Result { - let (_connection_id, _database, _db_type, query_timeout_secs) = transfer_pool_context(state, pool_key).await; + let (_connection_id, _database, db_type, query_timeout_secs) = transfer_pool_context(state, pool_key).await; let query_timeout = query_timeout_duration(query_timeout_secs); // Read-only check: block transfer operations in readonly mode. crate::query::check_read_only_for_connection(state, pool_key, sql).await?; - let pool_handle = state.pool_handle(pool_key).await; - let pool = pool_handle.as_ref().ok_or("Connection not found")?; + let pool = ensure_transfer_statement_pool(state, pool_key).await?; // Transfer reads run under the per-connection operation budget. Drivers that // expose an incremental result stream (MySQL, PostgreSQL, SQLite, SQL Server) @@ -6517,7 +6564,7 @@ async fn execute_on_pool_once( // whose protocol returns the whole result in one shot — ClickHouse, InfluxDB, // the Agent/JDBC path, external drivers and the DuckDB sidecar worker — expose // no incremental progress, so they keep the plain wall-clock timeout. - let result = match pool { + let result = match &pool { PoolKind::Mysql(p, mode) => { let p = p.clone(); let bare = *mode == crate::connection::MysqlMode::Bare; @@ -6627,11 +6674,17 @@ async fn execute_on_pool_once( } _ => Err("Unsupported database type for transfer".to_string()), }; - drop(pool_handle); - if result.as_ref().is_err_and(|error| is_transfer_query_timeout(error)) { + drop(pool); + if result.as_ref().is_err_and(|error| is_transfer_query_timeout(error)) + && should_discard_pool_after_query_timeout(db_type) + { // A timed-out native driver future may still own a checked-out // connection. Discard the pool so a late server response cannot be - // reused by the next transfer statement. + // reused by the next transfer statement. Drivers that keep their pool on + // a timeout (`pool_error_action` -> `Keep`, e.g. the embedded SQLite + // worker) must not lose it here either: the pool key is shared by every + // table in the transfer, so dropping it would make all later tables fail + // with "Connection not found". state.remove_pool_by_key(pool_key).await; } result @@ -18385,4 +18438,71 @@ CREATE INDEX items_name_idx ON public.items (id);"#; assert_eq!(receiver.len(), TRANSFER_PROGRESS_CHANNEL_CAPACITY); } + + #[test] + fn discarded_transfer_pool_is_replaced_for_native_drivers() { + // Discarding a pool never replays the failed statement, but the rest of a + // multi-table transfer keeps reusing the same pool key, so the pool has to + // be replaced for later tables. Only agent/JDBC sessions are excluded: their + // recovery stays gated on the structured quarantine decision. + for db_type in [ + DatabaseType::Mysql, + DatabaseType::Postgres, + DatabaseType::SqlServer, + DatabaseType::ClickHouse, + DatabaseType::Gaussdb, + DatabaseType::Sqlite, + ] { + assert!( + should_reconnect_discarded_transfer_pool(Some(db_type)), + "{db_type:?} must prepare a replacement pool for later tables" + ); + } + for db_type in [DatabaseType::Oracle, DatabaseType::Dameng] { + assert!( + !should_reconnect_discarded_transfer_pool(Some(db_type)), + "{db_type:?} keeps the stricter agent session rule" + ); + } + assert!(!should_reconnect_discarded_transfer_pool(None)); + } + + #[test] + fn every_discard_decision_leaves_a_usable_pool_behind() { + // Connection drops on a write are downgraded to Discard (the write is never + // replayed) and query timeouts tear the pool down too. Whatever produced the + // Discard, a native driver must still get a pool for the next table instead + // of the misleading "Connection not found" seen in the wild. + for (db_type, error) in [ + (DatabaseType::Postgres, "connection reset by peer"), + (DatabaseType::Mysql, "Connection timed out"), + (DatabaseType::Postgres, "Query timed out after 30 seconds"), + (DatabaseType::Sqlite, "PostgreSQL schema.reset cleanup failed: schema.reset timed out after 3 seconds"), + ] { + assert_eq!( + transfer_pool_error_action(TransferExecutionSafety::WriteNoReplay, Some(db_type), error), + PoolErrorAction::Discard, + "{db_type:?} must discard its pool for: {error}" + ); + assert!( + should_reconnect_discarded_transfer_pool(Some(db_type)), + "{db_type:?} must be handed a replacement pool after discarding" + ); + } + // Errors that keep the pool never reach the reconnect branch. + assert_eq!( + transfer_pool_error_action( + TransferExecutionSafety::WriteNoReplay, + Some(DatabaseType::Postgres), + "duplicate key value violates unique constraint \"items_pkey\"" + ), + PoolErrorAction::Keep + ); + // Agent sessions only replace the session on a structured quarantine hint. + assert!(!should_prepare_fresh_agent_transfer_session( + Some(DatabaseType::Oracle), + "Query timed out after 30 seconds" + )); + assert!(!should_reconnect_discarded_transfer_pool(Some(DatabaseType::Oracle))); + } } diff --git a/crates/dbx-core/src/query/mod.rs b/crates/dbx-core/src/query/mod.rs index ae0c3f4201..76cc557895 100644 --- a/crates/dbx-core/src/query/mod.rs +++ b/crates/dbx-core/src/query/mod.rs @@ -1429,7 +1429,7 @@ fn options_for_sequential_statements( statement_options } -fn should_discard_pool_after_query_timeout(db_type: Option) -> bool { +pub(crate) fn should_discard_pool_after_query_timeout(db_type: Option) -> bool { let Some(db_type) = db_type else { return false; }; diff --git a/crates/dbx-core/tests/live_mysql_transfer.rs b/crates/dbx-core/tests/live_mysql_transfer.rs index 1707f73198..9e39798985 100644 --- a/crates/dbx-core/tests/live_mysql_transfer.rs +++ b/crates/dbx-core/tests/live_mysql_transfer.rs @@ -2274,3 +2274,91 @@ async fn live_mysql_transfer_drop_target_inspects_dependent_views_without_the_us let _ = std::fs::remove_dir_all(dir); cleanup.unwrap(); } + +/// A single timed-out statement must not take the rest of the transfer down with it. +/// +/// `execute_on_pool_once` drops the pool of a timed-out driver so a late server response +/// cannot be reused; because a bulk transfer shares one pool key across every table, the +/// pool has to be replaced before the next statement runs. Without that replacement the +/// next table reported the misleading `Connection not found` instead of executing. +#[tokio::test] +#[ignore = "requires a disposable MySQL 5.7+ endpoint via DBX_LIVE_MYSQL_TRANSFER_* variables"] +async fn live_mysql_transfer_recovers_the_pool_after_a_query_timeout() { + let suffix = uuid::Uuid::new_v4().simple().to_string(); + let connection_id = format!("live-mysql-transfer-timeout-{suffix}"); + let database = format!("dbx_transfer_timeout_{}", &suffix[..12]); + let mut config = live_mysql_config(&connection_id); + // A one-second budget lets `SELECT SLEEP(3)` time out while leaving normal + // statements (and the replacement pool) usable. + config.query_timeout_secs = 1; + + let setup_pool = mysql::connect(&mysql_url(&config), Duration::from_secs(10)).await.unwrap(); + mysql::execute_query(&setup_pool, &format!("CREATE DATABASE `{database}`"), true).await.unwrap(); + + let dir = std::env::temp_dir().join(format!("dbx-live-mysql-transfer-timeout-{suffix}")); + std::fs::create_dir_all(&dir).unwrap(); + let storage = dbx_core::persistence::test_storage::open(&dir.join("storage.db")).await.unwrap(); + let state = Arc::new(AppState::new(storage)); + state.configs.write().await.insert(connection_id.clone(), config); + let pool_key = state.get_or_create_pool(&connection_id, Some(&database)).await.unwrap(); + + let timed_out = execute_on_pool(&state, &pool_key, "SELECT SLEEP(3)") + .await + .expect_err("the statement must hit the configured one-second budget"); + assert!(timed_out.to_lowercase().contains("timed out"), "expected a timeout error, got: {timed_out}"); + + let follow_up = execute_on_pool(&state, &pool_key, "SELECT 1").await; + assert!( + follow_up.is_ok(), + "a later transfer statement must run on a replacement pool, not report a missing connection: {:?}", + follow_up.err() + ); + + let cleanup = mysql::execute_query(&setup_pool, &format!("DROP DATABASE `{database}`"), true).await; + setup_pool.disconnect().await.unwrap(); + let _ = std::fs::remove_dir_all(dir); + cleanup.unwrap(); +} + +/// A pool removed while it is idle between two statements must be rebuilt instead of +/// failing the rest of the transfer with `Connection not found`. +/// +/// The connection keepalive tears a pool down whenever its ping fails or times out, and a +/// bulk transfer shares one pool key across every table. The next statement therefore has +/// to reconnect on its own rather than report a missing pool -- the statement has not run +/// yet, so nothing is replayed. +#[tokio::test] +#[ignore = "requires a disposable MySQL 5.7+ endpoint via DBX_LIVE_MYSQL_TRANSFER_* variables"] +async fn live_mysql_transfer_recreates_a_pool_removed_between_statements() { + let suffix = uuid::Uuid::new_v4().simple().to_string(); + let connection_id = format!("live-mysql-transfer-removed-pool-{suffix}"); + let database = format!("dbx_transfer_recreate_{}", &suffix[..12]); + let config = live_mysql_config(&connection_id); + + let setup_pool = mysql::connect(&mysql_url(&config), Duration::from_secs(10)).await.unwrap(); + mysql::execute_query(&setup_pool, &format!("CREATE DATABASE `{database}`"), true).await.unwrap(); + + let dir = std::env::temp_dir().join(format!("dbx-live-mysql-transfer-recreate-{suffix}")); + std::fs::create_dir_all(&dir).unwrap(); + let storage = dbx_core::persistence::test_storage::open(&dir.join("storage.db")).await.unwrap(); + let state = Arc::new(AppState::new(storage)); + state.configs.write().await.insert(connection_id.clone(), config); + let pool_key = state.get_or_create_pool(&connection_id, Some(&database)).await.unwrap(); + execute_on_pool(&state, &pool_key, "SELECT 1").await.unwrap(); + + // Same effect as a keepalive invalidation (or any other path that drops a transfer + // pool) while the transfer waits between two tables. + assert!(state.remove_pool_by_key(&pool_key).await, "the pool under test must exist first"); + + let follow_up = execute_on_pool(&state, &pool_key, "SELECT 1").await; + assert!( + follow_up.is_ok(), + "a statement after the pool disappeared must reconnect, not report a missing connection: {:?}", + follow_up.err() + ); + + let cleanup = mysql::execute_query(&setup_pool, &format!("DROP DATABASE `{database}`"), true).await; + setup_pool.disconnect().await.unwrap(); + let _ = std::fs::remove_dir_all(dir); + cleanup.unwrap(); +}