fix(transfer): rebuild dropped pools between transfer statements
This commit is contained in:
3 files changed
+221
-13
No files matched your search
@@ -27,7 +27,8 @@ use crate::models::connection::{ConnectionConfig, DatabaseType};
|
|||||||
use crate::object_source_sql::{build_executable_object_source_statements, EditableObjectSourceSqlInput};
|
use crate::object_source_sql::{build_executable_object_source_statements, EditableObjectSourceSqlInput};
|
||||||
use crate::query::{
|
use crate::query::{
|
||||||
agent_execute_query_params, is_dbx_query_timeout_error, pool_error_action, query_timeout_duration,
|
agent_execute_query_params, is_dbx_query_timeout_error, pool_error_action, query_timeout_duration,
|
||||||
wait_for_query_opt, PoolErrorAction, QueryExecutionOptions, StreamProgressClock, AGENT_PROTOCOL_MAX_ROWS,
|
should_discard_pool_after_query_timeout, wait_for_query_opt, PoolErrorAction, QueryExecutionOptions,
|
||||||
|
StreamProgressClock, AGENT_PROTOCOL_MAX_ROWS,
|
||||||
};
|
};
|
||||||
use crate::sql::{split_sql_statements, split_sql_statements_for_database};
|
use crate::sql::{split_sql_statements, split_sql_statements_for_database};
|
||||||
use crate::sql_dialect::{
|
use crate::sql_dialect::{
|
||||||
@@ -1920,9 +1921,9 @@ async fn execute_transfer_write_statement(
|
|||||||
crate::query::check_read_only_for_connection(state, target_pool_key, sql).await?;
|
crate::query::check_read_only_for_connection(state, target_pool_key, sql).await?;
|
||||||
let (_, _, _, query_timeout_secs) = transfer_pool_context(state, target_pool_key).await;
|
let (_, _, _, query_timeout_secs) = transfer_pool_context(state, target_pool_key).await;
|
||||||
let query_timeout = query_timeout_duration(query_timeout_secs);
|
let query_timeout = query_timeout_duration(query_timeout_secs);
|
||||||
let pool_handle = state.pool_handle(target_pool_key).await;
|
let pool = ensure_transfer_statement_pool(state, target_pool_key).await?;
|
||||||
let client = match pool_handle.as_ref() {
|
let client = match &pool {
|
||||||
Some(PoolKind::SqlServer(client)) => client.clone(),
|
PoolKind::SqlServer(client) => client.clone(),
|
||||||
_ => return Err("SQL Server connection not found".to_string()),
|
_ => return Err("SQL Server connection not found".to_string()),
|
||||||
};
|
};
|
||||||
let mut client = client.lock().await;
|
let mut client = client.lock().await;
|
||||||
@@ -6378,6 +6379,52 @@ fn should_prepare_fresh_agent_transfer_session(db_type: Option<DatabaseType>, er
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Whether a discarded transfer pool has to be replaced before the next statement runs.
|
||||||
|
///
|
||||||
|
/// Every error that reaches the `Discard` arm tears the pool down: the failed statement
|
||||||
|
/// is never replayed (its outcome on the server is unknown), and a driver whose query
|
||||||
|
/// timed out may still own a checked-out connection. The pool key, though, is shared by
|
||||||
|
/// every table of the transfer, so leaving it missing turns one transient failure into a
|
||||||
|
/// *misleading* "Connection not found" on the next table instead of that table running
|
||||||
|
/// (or failing) on its own. Prepare a replacement for native drivers; agent/JDBC
|
||||||
|
/// sessions keep their stricter rule, where recovery stays gated on the structured
|
||||||
|
/// quarantine decision.
|
||||||
|
fn should_reconnect_discarded_transfer_pool(db_type: Option<DatabaseType>) -> bool {
|
||||||
|
db_type.is_some_and(|db_type| !crate::database_capabilities::is_agent_type(&db_type))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resolve the pool a transfer statement runs on, re-creating it when it is gone.
|
||||||
|
///
|
||||||
|
/// A bulk transfer lives for tens of minutes and shares one pool key across every table,
|
||||||
|
/// so a pool that disappears between two statements used to abort the run with a
|
||||||
|
/// misleading `Connection not found` that says nothing about the database being
|
||||||
|
/// unreachable. The pool can legitimately be gone: `execute_on_pool_once` drops the pool
|
||||||
|
/// of a driver whose query timed out, the connection keepalive tears down a pool whose
|
||||||
|
/// ping failed or timed out, and every other transfer statement reuses the same key.
|
||||||
|
/// Rebuilding it here is safe -- the statement has not run yet, so nothing is replayed --
|
||||||
|
/// and keeps the transfer working on a fresh connection instead of failing the table.
|
||||||
|
async fn ensure_transfer_statement_pool(state: &AppState, pool_key: &str) -> Result<PoolKind, String> {
|
||||||
|
if let Some(pool) = state.pool_handle(pool_key).await {
|
||||||
|
return Ok(pool);
|
||||||
|
}
|
||||||
|
let (connection_id, database, _, _) = transfer_pool_context(state, pool_key).await;
|
||||||
|
let Some(connection_id) = connection_id else {
|
||||||
|
return Err("Connection not found".to_string());
|
||||||
|
};
|
||||||
|
let catalog = catalog_from_pool_key(pool_key).map(str::to_string);
|
||||||
|
let client_session_id = client_session_id_from_pool_key(pool_key).map(str::to_string);
|
||||||
|
state
|
||||||
|
.get_or_create_pool_for_session_with_catalog(
|
||||||
|
&connection_id,
|
||||||
|
database.as_deref(),
|
||||||
|
catalog.as_deref(),
|
||||||
|
client_session_id.as_deref(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|error| format!("Connection not found: {error}"))?;
|
||||||
|
state.pool_handle(pool_key).await.ok_or_else(|| "Connection not found".to_string())
|
||||||
|
}
|
||||||
|
|
||||||
async fn transfer_pool_context(
|
async fn transfer_pool_context(
|
||||||
state: &AppState,
|
state: &AppState,
|
||||||
pool_key: &str,
|
pool_key: &str,
|
||||||
@@ -6437,7 +6484,8 @@ async fn execute_on_pool_with_options(
|
|||||||
// the failed statement above, only prepares the pool for
|
// the failed statement above, only prepares the pool for
|
||||||
// whichever statement runs next.
|
// whichever statement runs next.
|
||||||
let prepare_agent_session = should_prepare_fresh_agent_transfer_session(db_type, error);
|
let prepare_agent_session = should_prepare_fresh_agent_transfer_session(db_type, error);
|
||||||
if pool_error_action(db_type, error) == PoolErrorAction::ReconnectAndRetry || prepare_agent_session {
|
let reconnect_pool = should_reconnect_discarded_transfer_pool(db_type);
|
||||||
|
if reconnect_pool || prepare_agent_session {
|
||||||
if let Some(connection_id) = connection_id.as_deref() {
|
if let Some(connection_id) = connection_id.as_deref() {
|
||||||
let catalog = catalog_from_pool_key(¤t_pool_key).map(str::to_string);
|
let catalog = catalog_from_pool_key(¤t_pool_key).map(str::to_string);
|
||||||
let recovery = if prepare_agent_session {
|
let recovery = if prepare_agent_session {
|
||||||
@@ -6501,13 +6549,12 @@ async fn execute_on_pool_once(
|
|||||||
sql: &str,
|
sql: &str,
|
||||||
max_rows: Option<usize>,
|
max_rows: Option<usize>,
|
||||||
) -> Result<db::QueryResult, String> {
|
) -> Result<db::QueryResult, String> {
|
||||||
let (_connection_id, _database, _db_type, query_timeout_secs) = transfer_pool_context(state, pool_key).await;
|
let (_connection_id, _database, db_type, query_timeout_secs) = transfer_pool_context(state, pool_key).await;
|
||||||
let query_timeout = query_timeout_duration(query_timeout_secs);
|
let query_timeout = query_timeout_duration(query_timeout_secs);
|
||||||
|
|
||||||
// Read-only check: block transfer operations in readonly mode.
|
// Read-only check: block transfer operations in readonly mode.
|
||||||
crate::query::check_read_only_for_connection(state, pool_key, sql).await?;
|
crate::query::check_read_only_for_connection(state, pool_key, sql).await?;
|
||||||
let pool_handle = state.pool_handle(pool_key).await;
|
let pool = ensure_transfer_statement_pool(state, pool_key).await?;
|
||||||
let pool = pool_handle.as_ref().ok_or("Connection not found")?;
|
|
||||||
|
|
||||||
// Transfer reads run under the per-connection operation budget. Drivers that
|
// Transfer reads run under the per-connection operation budget. Drivers that
|
||||||
// expose an incremental result stream (MySQL, PostgreSQL, SQLite, SQL Server)
|
// expose an incremental result stream (MySQL, PostgreSQL, SQLite, SQL Server)
|
||||||
@@ -6517,7 +6564,7 @@ async fn execute_on_pool_once(
|
|||||||
// whose protocol returns the whole result in one shot — ClickHouse, InfluxDB,
|
// whose protocol returns the whole result in one shot — ClickHouse, InfluxDB,
|
||||||
// the Agent/JDBC path, external drivers and the DuckDB sidecar worker — expose
|
// the Agent/JDBC path, external drivers and the DuckDB sidecar worker — expose
|
||||||
// no incremental progress, so they keep the plain wall-clock timeout.
|
// no incremental progress, so they keep the plain wall-clock timeout.
|
||||||
let result = match pool {
|
let result = match &pool {
|
||||||
PoolKind::Mysql(p, mode) => {
|
PoolKind::Mysql(p, mode) => {
|
||||||
let p = p.clone();
|
let p = p.clone();
|
||||||
let bare = *mode == crate::connection::MysqlMode::Bare;
|
let bare = *mode == crate::connection::MysqlMode::Bare;
|
||||||
@@ -6627,11 +6674,17 @@ async fn execute_on_pool_once(
|
|||||||
}
|
}
|
||||||
_ => Err("Unsupported database type for transfer".to_string()),
|
_ => Err("Unsupported database type for transfer".to_string()),
|
||||||
};
|
};
|
||||||
drop(pool_handle);
|
drop(pool);
|
||||||
if result.as_ref().is_err_and(|error| is_transfer_query_timeout(error)) {
|
if result.as_ref().is_err_and(|error| is_transfer_query_timeout(error))
|
||||||
|
&& should_discard_pool_after_query_timeout(db_type)
|
||||||
|
{
|
||||||
// A timed-out native driver future may still own a checked-out
|
// A timed-out native driver future may still own a checked-out
|
||||||
// connection. Discard the pool so a late server response cannot be
|
// connection. Discard the pool so a late server response cannot be
|
||||||
// reused by the next transfer statement.
|
// reused by the next transfer statement. Drivers that keep their pool on
|
||||||
|
// a timeout (`pool_error_action` -> `Keep`, e.g. the embedded SQLite
|
||||||
|
// worker) must not lose it here either: the pool key is shared by every
|
||||||
|
// table in the transfer, so dropping it would make all later tables fail
|
||||||
|
// with "Connection not found".
|
||||||
state.remove_pool_by_key(pool_key).await;
|
state.remove_pool_by_key(pool_key).await;
|
||||||
}
|
}
|
||||||
result
|
result
|
||||||
@@ -18385,4 +18438,71 @@ CREATE INDEX items_name_idx ON public.items (id);"#;
|
|||||||
|
|
||||||
assert_eq!(receiver.len(), TRANSFER_PROGRESS_CHANNEL_CAPACITY);
|
assert_eq!(receiver.len(), TRANSFER_PROGRESS_CHANNEL_CAPACITY);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn discarded_transfer_pool_is_replaced_for_native_drivers() {
|
||||||
|
// Discarding a pool never replays the failed statement, but the rest of a
|
||||||
|
// multi-table transfer keeps reusing the same pool key, so the pool has to
|
||||||
|
// be replaced for later tables. Only agent/JDBC sessions are excluded: their
|
||||||
|
// recovery stays gated on the structured quarantine decision.
|
||||||
|
for db_type in [
|
||||||
|
DatabaseType::Mysql,
|
||||||
|
DatabaseType::Postgres,
|
||||||
|
DatabaseType::SqlServer,
|
||||||
|
DatabaseType::ClickHouse,
|
||||||
|
DatabaseType::Gaussdb,
|
||||||
|
DatabaseType::Sqlite,
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
should_reconnect_discarded_transfer_pool(Some(db_type)),
|
||||||
|
"{db_type:?} must prepare a replacement pool for later tables"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for db_type in [DatabaseType::Oracle, DatabaseType::Dameng] {
|
||||||
|
assert!(
|
||||||
|
!should_reconnect_discarded_transfer_pool(Some(db_type)),
|
||||||
|
"{db_type:?} keeps the stricter agent session rule"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(!should_reconnect_discarded_transfer_pool(None));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn every_discard_decision_leaves_a_usable_pool_behind() {
|
||||||
|
// Connection drops on a write are downgraded to Discard (the write is never
|
||||||
|
// replayed) and query timeouts tear the pool down too. Whatever produced the
|
||||||
|
// Discard, a native driver must still get a pool for the next table instead
|
||||||
|
// of the misleading "Connection not found" seen in the wild.
|
||||||
|
for (db_type, error) in [
|
||||||
|
(DatabaseType::Postgres, "connection reset by peer"),
|
||||||
|
(DatabaseType::Mysql, "Connection timed out"),
|
||||||
|
(DatabaseType::Postgres, "Query timed out after 30 seconds"),
|
||||||
|
(DatabaseType::Sqlite, "PostgreSQL schema.reset cleanup failed: schema.reset timed out after 3 seconds"),
|
||||||
|
] {
|
||||||
|
assert_eq!(
|
||||||
|
transfer_pool_error_action(TransferExecutionSafety::WriteNoReplay, Some(db_type), error),
|
||||||
|
PoolErrorAction::Discard,
|
||||||
|
"{db_type:?} must discard its pool for: {error}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
should_reconnect_discarded_transfer_pool(Some(db_type)),
|
||||||
|
"{db_type:?} must be handed a replacement pool after discarding"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// Errors that keep the pool never reach the reconnect branch.
|
||||||
|
assert_eq!(
|
||||||
|
transfer_pool_error_action(
|
||||||
|
TransferExecutionSafety::WriteNoReplay,
|
||||||
|
Some(DatabaseType::Postgres),
|
||||||
|
"duplicate key value violates unique constraint \"items_pkey\""
|
||||||
|
),
|
||||||
|
PoolErrorAction::Keep
|
||||||
|
);
|
||||||
|
// Agent sessions only replace the session on a structured quarantine hint.
|
||||||
|
assert!(!should_prepare_fresh_agent_transfer_session(
|
||||||
|
Some(DatabaseType::Oracle),
|
||||||
|
"Query timed out after 30 seconds"
|
||||||
|
));
|
||||||
|
assert!(!should_reconnect_discarded_transfer_pool(Some(DatabaseType::Oracle)));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -1429,7 +1429,7 @@ fn options_for_sequential_statements(
|
|||||||
statement_options
|
statement_options
|
||||||
}
|
}
|
||||||
|
|
||||||
fn should_discard_pool_after_query_timeout(db_type: Option<DatabaseType>) -> bool {
|
pub(crate) fn should_discard_pool_after_query_timeout(db_type: Option<DatabaseType>) -> bool {
|
||||||
let Some(db_type) = db_type else {
|
let Some(db_type) = db_type else {
|
||||||
return false;
|
return false;
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -2274,3 +2274,91 @@ async fn live_mysql_transfer_drop_target_inspects_dependent_views_without_the_us
|
|||||||
let _ = std::fs::remove_dir_all(dir);
|
let _ = std::fs::remove_dir_all(dir);
|
||||||
cleanup.unwrap();
|
cleanup.unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A single timed-out statement must not take the rest of the transfer down with it.
|
||||||
|
///
|
||||||
|
/// `execute_on_pool_once` drops the pool of a timed-out driver so a late server response
|
||||||
|
/// cannot be reused; because a bulk transfer shares one pool key across every table, the
|
||||||
|
/// pool has to be replaced before the next statement runs. Without that replacement the
|
||||||
|
/// next table reported the misleading `Connection not found` instead of executing.
|
||||||
|
#[tokio::test]
|
||||||
|
#[ignore = "requires a disposable MySQL 5.7+ endpoint via DBX_LIVE_MYSQL_TRANSFER_* variables"]
|
||||||
|
async fn live_mysql_transfer_recovers_the_pool_after_a_query_timeout() {
|
||||||
|
let suffix = uuid::Uuid::new_v4().simple().to_string();
|
||||||
|
let connection_id = format!("live-mysql-transfer-timeout-{suffix}");
|
||||||
|
let database = format!("dbx_transfer_timeout_{}", &suffix[..12]);
|
||||||
|
let mut config = live_mysql_config(&connection_id);
|
||||||
|
// A one-second budget lets `SELECT SLEEP(3)` time out while leaving normal
|
||||||
|
// statements (and the replacement pool) usable.
|
||||||
|
config.query_timeout_secs = 1;
|
||||||
|
|
||||||
|
let setup_pool = mysql::connect(&mysql_url(&config), Duration::from_secs(10)).await.unwrap();
|
||||||
|
mysql::execute_query(&setup_pool, &format!("CREATE DATABASE `{database}`"), true).await.unwrap();
|
||||||
|
|
||||||
|
let dir = std::env::temp_dir().join(format!("dbx-live-mysql-transfer-timeout-{suffix}"));
|
||||||
|
std::fs::create_dir_all(&dir).unwrap();
|
||||||
|
let storage = dbx_core::persistence::test_storage::open(&dir.join("storage.db")).await.unwrap();
|
||||||
|
let state = Arc::new(AppState::new(storage));
|
||||||
|
state.configs.write().await.insert(connection_id.clone(), config);
|
||||||
|
let pool_key = state.get_or_create_pool(&connection_id, Some(&database)).await.unwrap();
|
||||||
|
|
||||||
|
let timed_out = execute_on_pool(&state, &pool_key, "SELECT SLEEP(3)")
|
||||||
|
.await
|
||||||
|
.expect_err("the statement must hit the configured one-second budget");
|
||||||
|
assert!(timed_out.to_lowercase().contains("timed out"), "expected a timeout error, got: {timed_out}");
|
||||||
|
|
||||||
|
let follow_up = execute_on_pool(&state, &pool_key, "SELECT 1").await;
|
||||||
|
assert!(
|
||||||
|
follow_up.is_ok(),
|
||||||
|
"a later transfer statement must run on a replacement pool, not report a missing connection: {:?}",
|
||||||
|
follow_up.err()
|
||||||
|
);
|
||||||
|
|
||||||
|
let cleanup = mysql::execute_query(&setup_pool, &format!("DROP DATABASE `{database}`"), true).await;
|
||||||
|
setup_pool.disconnect().await.unwrap();
|
||||||
|
let _ = std::fs::remove_dir_all(dir);
|
||||||
|
cleanup.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A pool removed while it is idle between two statements must be rebuilt instead of
|
||||||
|
/// failing the rest of the transfer with `Connection not found`.
|
||||||
|
///
|
||||||
|
/// The connection keepalive tears a pool down whenever its ping fails or times out, and a
|
||||||
|
/// bulk transfer shares one pool key across every table. The next statement therefore has
|
||||||
|
/// to reconnect on its own rather than report a missing pool -- the statement has not run
|
||||||
|
/// yet, so nothing is replayed.
|
||||||
|
#[tokio::test]
|
||||||
|
#[ignore = "requires a disposable MySQL 5.7+ endpoint via DBX_LIVE_MYSQL_TRANSFER_* variables"]
|
||||||
|
async fn live_mysql_transfer_recreates_a_pool_removed_between_statements() {
|
||||||
|
let suffix = uuid::Uuid::new_v4().simple().to_string();
|
||||||
|
let connection_id = format!("live-mysql-transfer-removed-pool-{suffix}");
|
||||||
|
let database = format!("dbx_transfer_recreate_{}", &suffix[..12]);
|
||||||
|
let config = live_mysql_config(&connection_id);
|
||||||
|
|
||||||
|
let setup_pool = mysql::connect(&mysql_url(&config), Duration::from_secs(10)).await.unwrap();
|
||||||
|
mysql::execute_query(&setup_pool, &format!("CREATE DATABASE `{database}`"), true).await.unwrap();
|
||||||
|
|
||||||
|
let dir = std::env::temp_dir().join(format!("dbx-live-mysql-transfer-recreate-{suffix}"));
|
||||||
|
std::fs::create_dir_all(&dir).unwrap();
|
||||||
|
let storage = dbx_core::persistence::test_storage::open(&dir.join("storage.db")).await.unwrap();
|
||||||
|
let state = Arc::new(AppState::new(storage));
|
||||||
|
state.configs.write().await.insert(connection_id.clone(), config);
|
||||||
|
let pool_key = state.get_or_create_pool(&connection_id, Some(&database)).await.unwrap();
|
||||||
|
execute_on_pool(&state, &pool_key, "SELECT 1").await.unwrap();
|
||||||
|
|
||||||
|
// Same effect as a keepalive invalidation (or any other path that drops a transfer
|
||||||
|
// pool) while the transfer waits between two tables.
|
||||||
|
assert!(state.remove_pool_by_key(&pool_key).await, "the pool under test must exist first");
|
||||||
|
|
||||||
|
let follow_up = execute_on_pool(&state, &pool_key, "SELECT 1").await;
|
||||||
|
assert!(
|
||||||
|
follow_up.is_ok(),
|
||||||
|
"a statement after the pool disappeared must reconnect, not report a missing connection: {:?}",
|
||||||
|
follow_up.err()
|
||||||
|
);
|
||||||
|
|
||||||
|
let cleanup = mysql::execute_query(&setup_pool, &format!("DROP DATABASE `{database}`"), true).await;
|
||||||
|
setup_pool.disconnect().await.unwrap();
|
||||||
|
let _ = std::fs::remove_dir_all(dir);
|
||||||
|
cleanup.unwrap();
|
||||||
|
}
|
||||||
Reference in new issue
Block a user