fix(import): keep doris import transactions alive against pool probing
This commit is contained in:
2 files changed
+151
-2
No files matched your search
@@ -2178,6 +2178,12 @@ impl AppState {
|
|||||||
match result {
|
match result {
|
||||||
Ok(Ok(())) => {}
|
Ok(Ok(())) => {}
|
||||||
Ok(Err(err)) => {
|
Ok(Err(err)) => {
|
||||||
|
if !keepalive_failure_proves_pool_dead(&err.to_string()) {
|
||||||
|
log::debug!(
|
||||||
|
"Connection keepalive for '{key}' could not check out a connection; keeping the busy pool: {err}"
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
log::warn!("Connection keepalive failed for '{key}': {err}; invalidating pool");
|
log::warn!("Connection keepalive failed for '{key}': {err}; invalidating pool");
|
||||||
let replace_runtime =
|
let replace_runtime =
|
||||||
err.recovery_decision().is_some_and(RecoveryDecision::replaces_runtime);
|
err.recovery_decision().is_some_and(RecoveryDecision::replaces_runtime);
|
||||||
@@ -5936,6 +5942,18 @@ impl From<String> for KeepaliveError {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Whether a failed keepalive probe is evidence that the pool is dead.
|
||||||
|
///
|
||||||
|
/// A probe that ran out of its checkout budget because every connection is in use reports
|
||||||
|
/// pool saturation, which says nothing about the health of the pooled connections. That is
|
||||||
|
/// the normal state of a session-scoped pool (a single connection) while a batch import or
|
||||||
|
/// transfer holds a long transaction on it. Invalidating the pool there used to abort the
|
||||||
|
/// running operation with "Connection not found for transaction" instead of letting it
|
||||||
|
/// finish. Every other probe failure keeps the invalidate-and-reconnect behaviour.
|
||||||
|
fn keepalive_failure_proves_pool_dead(error: &str) -> bool {
|
||||||
|
!crate::query::is_pool_saturation_error(error)
|
||||||
|
}
|
||||||
|
|
||||||
impl KeepaliveTarget {
|
impl KeepaliveTarget {
|
||||||
fn matches_pool(&self, pool: &PoolKind) -> bool {
|
fn matches_pool(&self, pool: &PoolKind) -> bool {
|
||||||
match (self, pool) {
|
match (self, pool) {
|
||||||
@@ -6913,8 +6931,8 @@ mod tests {
|
|||||||
connection_probe_endpoints, connection_remote_endpoint, connection_url_for_endpoint,
|
connection_probe_endpoints, connection_remote_endpoint, connection_url_for_endpoint,
|
||||||
database_connection_config, database_connection_config_with_catalog,
|
database_connection_config, database_connection_config_with_catalog,
|
||||||
gaussdb_identifier_quote_from_query_result, gaussdb_m_jdbc_config_for_endpoint, gaussdb_uses_m_jdbc_driver,
|
gaussdb_identifier_quote_from_query_result, gaussdb_m_jdbc_config_for_endpoint, gaussdb_uses_m_jdbc_driver,
|
||||||
kafka_single_loopback_bootstrap_endpoint, metadata_connection_config, metadata_pool_database,
|
kafka_single_loopback_bootstrap_endpoint, keepalive_failure_proves_pool_dead, metadata_connection_config,
|
||||||
mysql_metadata_fallback_url, mysql_pool_setup_queries, oceanbase_mysql_setup_queries,
|
metadata_pool_database, mysql_metadata_fallback_url, mysql_pool_setup_queries, oceanbase_mysql_setup_queries,
|
||||||
prestosql_jdbc_config_for_endpoint, redacted_connection_url_for_endpoint, redis_sentinel_transport_id,
|
prestosql_jdbc_config_for_endpoint, redacted_connection_url_for_endpoint, redis_sentinel_transport_id,
|
||||||
redis_sentinel_transport_prefix, sqlserver_legacy_agent_config, sqlserver_legacy_driver_error,
|
redis_sentinel_transport_prefix, sqlserver_legacy_agent_config, sqlserver_legacy_driver_error,
|
||||||
sqlserver_uses_legacy_driver, task_client_session_id, transport_layers_through_last_ssh,
|
sqlserver_uses_legacy_driver, task_client_session_id, transport_layers_through_last_ssh,
|
||||||
@@ -8345,6 +8363,23 @@ mod tests {
|
|||||||
let _ = std::fs::remove_dir_all(dir);
|
let _ = std::fs::remove_dir_all(dir);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn keepalive_probe_keeps_a_busy_pool_but_invalidates_a_dead_one() {
|
||||||
|
// Every connection is checked out: the probe learned nothing about pool health, so the
|
||||||
|
// pool must survive. Invalidating it here aborts whatever holds the connection — a
|
||||||
|
// truncate import reports "Connection not found for transaction" on its next chunk.
|
||||||
|
assert!(!keepalive_failure_proves_pool_dead(
|
||||||
|
"MySQL connection pool checkout timed out [stage=wait, timeout_ms=10000]"
|
||||||
|
));
|
||||||
|
// A checkout failure while creating a connection is still evidence the pool is dead.
|
||||||
|
assert!(keepalive_failure_proves_pool_dead(
|
||||||
|
"MySQL connection pool checkout failed [stage=create]: connection refused"
|
||||||
|
));
|
||||||
|
assert!(keepalive_failure_proves_pool_dead(
|
||||||
|
"MySQL connection pool checkout timed out [stage=create, timeout_ms=10000]"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn jdbc_plugin_env_uses_managed_jre_when_installed() {
|
async fn jdbc_plugin_env_uses_managed_jre_when_installed() {
|
||||||
let dir = std::env::temp_dir().join(format!("dbx-core-jdbc-managed-jre-{}", uuid::Uuid::new_v4()));
|
let dir = std::env::temp_dir().join(format!("dbx-core-jdbc-managed-jre-{}", uuid::Uuid::new_v4()));
|
||||||
|
|||||||
@@ -78,6 +78,16 @@ pub fn table_import_client_session_id(import_id: &str) -> String {
|
|||||||
task_client_session_id("table-import", import_id)
|
task_client_session_id("table-import", import_id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Execution id registered with `RunningQueries` for the whole duration of an import.
|
||||||
|
///
|
||||||
|
/// Background pool maintenance (the keepalive probe, connection health refresh, idle
|
||||||
|
/// reclamation) consults `is_pool_active` before touching a pool. An import keeps a
|
||||||
|
/// connection checked out for the entire run — a session-scoped pool has exactly one
|
||||||
|
/// connection — so without this registration the pool looks idle while it is in use.
|
||||||
|
fn table_import_execution_id(import_id: &str) -> String {
|
||||||
|
format!("table-import-exec:{import_id}")
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct ParsedImportFile {
|
pub struct ParsedImportFile {
|
||||||
pub columns: Vec<String>,
|
pub columns: Vec<String>,
|
||||||
@@ -7063,6 +7073,16 @@ where
|
|||||||
let mut db_write_ms = 0u128;
|
let mut db_write_ms = 0u128;
|
||||||
let mut statement_count = 0usize;
|
let mut statement_count = 0usize;
|
||||||
let batch_size = if request.batch_size == 0 { DEFAULT_BATCH_SIZE } else { request.batch_size };
|
let batch_size = if request.batch_size == 0 { DEFAULT_BATCH_SIZE } else { request.batch_size };
|
||||||
|
// Mark the import's pool busy for the whole run. A session-scoped pool keeps a single
|
||||||
|
// connection, and the transactional batch path holds it for an entire chunk, so the
|
||||||
|
// keepalive probe cannot check out a connection on time. Without this registration the
|
||||||
|
// probe treats that healthy-but-busy pool as dead, invalidates it, and the next chunk
|
||||||
|
// fails with "Connection not found for transaction".
|
||||||
|
let import_execution_id = table_import_execution_id(&request.import_id);
|
||||||
|
let _import_activity = state.running_queries.register(import_execution_id.clone());
|
||||||
|
state.running_queries.set_pool_key(&import_execution_id, pool_key);
|
||||||
|
state.touch_pool_activity(pool_key).await;
|
||||||
|
let _activity_touch = state.pool_activity_touch(pool_key);
|
||||||
let kingbase_oracle_mode = kingbase_oracle_compatibility_mode(state, pool_key, db_type).await;
|
let kingbase_oracle_mode = kingbase_oracle_compatibility_mode(state, pool_key, db_type).await;
|
||||||
let conflict_policy = request.effective_conflict_policy();
|
let conflict_policy = request.effective_conflict_policy();
|
||||||
if let Err(error) = validate_update_existing_target(conflict_policy, db_type, request.create_table) {
|
if let Err(error) = validate_update_existing_target(conflict_policy, db_type, request.create_table) {
|
||||||
@@ -13255,6 +13275,100 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A session-scoped pool keeps a single connection, and the transactional batch path holds
|
||||||
|
// it for a whole chunk, so the keepalive probe cannot check out a connection while the
|
||||||
|
// import is running. The import must therefore advertise its pool as busy for its whole
|
||||||
|
// run; otherwise the probe invalidates the healthy pool and the next chunk fails with
|
||||||
|
// "Connection not found for transaction".
|
||||||
|
#[tokio::test]
|
||||||
|
async fn truncate_import_marks_its_pool_active_until_it_finishes() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let storage = crate::persistence::test_storage::open(&dir.path().join("storage.db")).await.unwrap();
|
||||||
|
let state = AppState::new(storage);
|
||||||
|
let connection_id = "sqlite-truncate-activity";
|
||||||
|
let pool_key = format!("{connection_id}:session:import");
|
||||||
|
let database_path = dir.path().join("target.db");
|
||||||
|
let sqlite = crate::db::sqlite::connect_path_create_if_missing(database_path.to_str().unwrap()).await.unwrap();
|
||||||
|
crate::db::sqlite::execute_query(&sqlite, "CREATE TABLE items (id INTEGER, name TEXT)").await.unwrap();
|
||||||
|
state
|
||||||
|
.update_connection_pools(|connections| {
|
||||||
|
connections.insert(pool_key.clone(), PoolKind::Sqlite(sqlite.clone()));
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
let config: ConnectionConfig = serde_json::from_value(serde_json::json!({
|
||||||
|
"id": connection_id,
|
||||||
|
"name": "SQLite truncate activity test",
|
||||||
|
"db_type": "sqlite",
|
||||||
|
"host": "",
|
||||||
|
"port": 0,
|
||||||
|
"username": "",
|
||||||
|
"password": "",
|
||||||
|
"database": database_path.to_string_lossy()
|
||||||
|
}))
|
||||||
|
.unwrap();
|
||||||
|
state.configs.write().await.insert(connection_id.to_string(), config);
|
||||||
|
let data_path = dir.path().join("rows.txt");
|
||||||
|
std::fs::write(&data_path, b"id%name\n1%Ada\n2%Grace\n3%Linus\n").unwrap();
|
||||||
|
let request = TableImportRequest {
|
||||||
|
import_id: "sqlite-truncate-activity".to_string(),
|
||||||
|
connection_id: connection_id.to_string(),
|
||||||
|
database: String::new(),
|
||||||
|
schema: String::new(),
|
||||||
|
table: "items".to_string(),
|
||||||
|
file_path: data_path.to_string_lossy().to_string(),
|
||||||
|
source_ref: None,
|
||||||
|
source_format: Some(TableImportSourceFormat::Delimited),
|
||||||
|
parse_options: TableImportParseOptions {
|
||||||
|
delimiter: Some("%".to_string()),
|
||||||
|
..TableImportParseOptions::default()
|
||||||
|
},
|
||||||
|
mappings: vec![
|
||||||
|
TableImportColumnMapping {
|
||||||
|
source_column: "id".to_string(),
|
||||||
|
target_column: "id".to_string(),
|
||||||
|
target_data_type: None,
|
||||||
|
},
|
||||||
|
TableImportColumnMapping {
|
||||||
|
source_column: "name".to_string(),
|
||||||
|
target_column: "name".to_string(),
|
||||||
|
target_data_type: None,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
mode: TableImportMode::Truncate,
|
||||||
|
create_table: false,
|
||||||
|
batch_size: 2,
|
||||||
|
date_time_format: None,
|
||||||
|
prepared_source: None,
|
||||||
|
skip_duplicate_rows: false,
|
||||||
|
conflict_policy: None,
|
||||||
|
retain_source: false,
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut active_during_import = Vec::new();
|
||||||
|
let summary = import_table_file_core(
|
||||||
|
&state,
|
||||||
|
&request,
|
||||||
|
&DatabaseType::Sqlite,
|
||||||
|
&pool_key,
|
||||||
|
|_| Box::pin(async { false }),
|
||||||
|
|progress| {
|
||||||
|
active_during_import.push((progress.status, state.running_queries.is_pool_active(&pool_key)));
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(summary.rows_imported, 3);
|
||||||
|
assert!(!active_during_import.is_empty(), "the import must report progress while it owns the pool");
|
||||||
|
for (status, active) in &active_during_import {
|
||||||
|
assert!(active, "progress event {status:?} happened while the import's pool was idle to sweeps");
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
!state.running_queries.is_pool_active(&pool_key),
|
||||||
|
"the import must release its pool registration when it finishes"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn sqlite_update_existing_import_counts_updated_rows_once() {
|
async fn sqlite_update_existing_import_counts_updated_rows_once() {
|
||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
|||||||
Reference in new issue
Block a user