fix(import): keep doris import transactions alive against pool probing

This commit is contained in:
zipg authored and GitHub committed 2026-09-29 16:41:05 +08:00
1 parent d55b4c966a
commit 497d7c9e1e
2 files changed
+151 -2

No files matched your search

+37 -2
View File
@@ -2178,6 +2178,12 @@ impl AppState {
match result { match result {
Ok(Ok(())) => {} Ok(Ok(())) => {}
Ok(Err(err)) => { Ok(Err(err)) => {
if !keepalive_failure_proves_pool_dead(&err.to_string()) {
log::debug!(
"Connection keepalive for '{key}' could not check out a connection; keeping the busy pool: {err}"
);
continue;
}
log::warn!("Connection keepalive failed for '{key}': {err}; invalidating pool"); log::warn!("Connection keepalive failed for '{key}': {err}; invalidating pool");
let replace_runtime = let replace_runtime =
err.recovery_decision().is_some_and(RecoveryDecision::replaces_runtime); err.recovery_decision().is_some_and(RecoveryDecision::replaces_runtime);
@@ -5936,6 +5942,18 @@ impl From<String> for KeepaliveError {
} }
} }
/// Whether a failed keepalive probe is evidence that the pool is dead.
///
/// A probe that ran out of its checkout budget because every connection is in use reports
/// pool saturation, which says nothing about the health of the pooled connections. That is
/// the normal state of a session-scoped pool (a single connection) while a batch import or
/// transfer holds a long transaction on it. Invalidating the pool there used to abort the
/// running operation with "Connection not found for transaction" instead of letting it
/// finish. Every other probe failure keeps the invalidate-and-reconnect behaviour.
fn keepalive_failure_proves_pool_dead(error: &str) -> bool {
!crate::query::is_pool_saturation_error(error)
}
impl KeepaliveTarget { impl KeepaliveTarget {
fn matches_pool(&self, pool: &PoolKind) -> bool { fn matches_pool(&self, pool: &PoolKind) -> bool {
match (self, pool) { match (self, pool) {
@@ -6913,8 +6931,8 @@ mod tests {
connection_probe_endpoints, connection_remote_endpoint, connection_url_for_endpoint, connection_probe_endpoints, connection_remote_endpoint, connection_url_for_endpoint,
database_connection_config, database_connection_config_with_catalog, database_connection_config, database_connection_config_with_catalog,
gaussdb_identifier_quote_from_query_result, gaussdb_m_jdbc_config_for_endpoint, gaussdb_uses_m_jdbc_driver, gaussdb_identifier_quote_from_query_result, gaussdb_m_jdbc_config_for_endpoint, gaussdb_uses_m_jdbc_driver,
kafka_single_loopback_bootstrap_endpoint, metadata_connection_config, metadata_pool_database, kafka_single_loopback_bootstrap_endpoint, keepalive_failure_proves_pool_dead, metadata_connection_config,
mysql_metadata_fallback_url, mysql_pool_setup_queries, oceanbase_mysql_setup_queries, metadata_pool_database, mysql_metadata_fallback_url, mysql_pool_setup_queries, oceanbase_mysql_setup_queries,
prestosql_jdbc_config_for_endpoint, redacted_connection_url_for_endpoint, redis_sentinel_transport_id, prestosql_jdbc_config_for_endpoint, redacted_connection_url_for_endpoint, redis_sentinel_transport_id,
redis_sentinel_transport_prefix, sqlserver_legacy_agent_config, sqlserver_legacy_driver_error, redis_sentinel_transport_prefix, sqlserver_legacy_agent_config, sqlserver_legacy_driver_error,
sqlserver_uses_legacy_driver, task_client_session_id, transport_layers_through_last_ssh, sqlserver_uses_legacy_driver, task_client_session_id, transport_layers_through_last_ssh,
@@ -8345,6 +8363,23 @@ mod tests {
let _ = std::fs::remove_dir_all(dir); let _ = std::fs::remove_dir_all(dir);
} }
#[test]
fn keepalive_probe_keeps_a_busy_pool_but_invalidates_a_dead_one() {
// Every connection is checked out: the probe learned nothing about pool health, so the
// pool must survive. Invalidating it here aborts whatever holds the connection — a
// truncate import reports "Connection not found for transaction" on its next chunk.
assert!(!keepalive_failure_proves_pool_dead(
"MySQL connection pool checkout timed out [stage=wait, timeout_ms=10000]"
));
// A checkout failure while creating a connection is still evidence the pool is dead.
assert!(keepalive_failure_proves_pool_dead(
"MySQL connection pool checkout failed [stage=create]: connection refused"
));
assert!(keepalive_failure_proves_pool_dead(
"MySQL connection pool checkout timed out [stage=create, timeout_ms=10000]"
));
}
#[tokio::test] #[tokio::test]
async fn jdbc_plugin_env_uses_managed_jre_when_installed() { async fn jdbc_plugin_env_uses_managed_jre_when_installed() {
let dir = std::env::temp_dir().join(format!("dbx-core-jdbc-managed-jre-{}", uuid::Uuid::new_v4())); let dir = std::env::temp_dir().join(format!("dbx-core-jdbc-managed-jre-{}", uuid::Uuid::new_v4()));
+114
View File
@@ -78,6 +78,16 @@ pub fn table_import_client_session_id(import_id: &str) -> String {
task_client_session_id("table-import", import_id) task_client_session_id("table-import", import_id)
} }
/// Execution id registered with `RunningQueries` for the whole duration of an import.
///
/// Background pool maintenance (the keepalive probe, connection health refresh, idle
/// reclamation) consults `is_pool_active` before touching a pool. An import keeps a
/// connection checked out for the entire run — a session-scoped pool has exactly one
/// connection — so without this registration the pool looks idle while it is in use.
fn table_import_execution_id(import_id: &str) -> String {
format!("table-import-exec:{import_id}")
}
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct ParsedImportFile { pub struct ParsedImportFile {
pub columns: Vec<String>, pub columns: Vec<String>,
@@ -7063,6 +7073,16 @@ where
let mut db_write_ms = 0u128; let mut db_write_ms = 0u128;
let mut statement_count = 0usize; let mut statement_count = 0usize;
let batch_size = if request.batch_size == 0 { DEFAULT_BATCH_SIZE } else { request.batch_size }; let batch_size = if request.batch_size == 0 { DEFAULT_BATCH_SIZE } else { request.batch_size };
// Mark the import's pool busy for the whole run. A session-scoped pool keeps a single
// connection, and the transactional batch path holds it for an entire chunk, so the
// keepalive probe cannot check out a connection on time. Without this registration the
// probe treats that healthy-but-busy pool as dead, invalidates it, and the next chunk
// fails with "Connection not found for transaction".
let import_execution_id = table_import_execution_id(&request.import_id);
let _import_activity = state.running_queries.register(import_execution_id.clone());
state.running_queries.set_pool_key(&import_execution_id, pool_key);
state.touch_pool_activity(pool_key).await;
let _activity_touch = state.pool_activity_touch(pool_key);
let kingbase_oracle_mode = kingbase_oracle_compatibility_mode(state, pool_key, db_type).await; let kingbase_oracle_mode = kingbase_oracle_compatibility_mode(state, pool_key, db_type).await;
let conflict_policy = request.effective_conflict_policy(); let conflict_policy = request.effective_conflict_policy();
if let Err(error) = validate_update_existing_target(conflict_policy, db_type, request.create_table) { if let Err(error) = validate_update_existing_target(conflict_policy, db_type, request.create_table) {
@@ -13255,6 +13275,100 @@ mod tests {
); );
} }
// A session-scoped pool keeps a single connection, and the transactional batch path holds
// it for a whole chunk, so the keepalive probe cannot check out a connection while the
// import is running. The import must therefore advertise its pool as busy for its whole
// run; otherwise the probe invalidates the healthy pool and the next chunk fails with
// "Connection not found for transaction".
#[tokio::test]
async fn truncate_import_marks_its_pool_active_until_it_finishes() {
let dir = tempfile::tempdir().unwrap();
let storage = crate::persistence::test_storage::open(&dir.path().join("storage.db")).await.unwrap();
let state = AppState::new(storage);
let connection_id = "sqlite-truncate-activity";
let pool_key = format!("{connection_id}:session:import");
let database_path = dir.path().join("target.db");
let sqlite = crate::db::sqlite::connect_path_create_if_missing(database_path.to_str().unwrap()).await.unwrap();
crate::db::sqlite::execute_query(&sqlite, "CREATE TABLE items (id INTEGER, name TEXT)").await.unwrap();
state
.update_connection_pools(|connections| {
connections.insert(pool_key.clone(), PoolKind::Sqlite(sqlite.clone()));
})
.await;
let config: ConnectionConfig = serde_json::from_value(serde_json::json!({
"id": connection_id,
"name": "SQLite truncate activity test",
"db_type": "sqlite",
"host": "",
"port": 0,
"username": "",
"password": "",
"database": database_path.to_string_lossy()
}))
.unwrap();
state.configs.write().await.insert(connection_id.to_string(), config);
let data_path = dir.path().join("rows.txt");
std::fs::write(&data_path, b"id%name\n1%Ada\n2%Grace\n3%Linus\n").unwrap();
let request = TableImportRequest {
import_id: "sqlite-truncate-activity".to_string(),
connection_id: connection_id.to_string(),
database: String::new(),
schema: String::new(),
table: "items".to_string(),
file_path: data_path.to_string_lossy().to_string(),
source_ref: None,
source_format: Some(TableImportSourceFormat::Delimited),
parse_options: TableImportParseOptions {
delimiter: Some("%".to_string()),
..TableImportParseOptions::default()
},
mappings: vec![
TableImportColumnMapping {
source_column: "id".to_string(),
target_column: "id".to_string(),
target_data_type: None,
},
TableImportColumnMapping {
source_column: "name".to_string(),
target_column: "name".to_string(),
target_data_type: None,
},
],
mode: TableImportMode::Truncate,
create_table: false,
batch_size: 2,
date_time_format: None,
prepared_source: None,
skip_duplicate_rows: false,
conflict_policy: None,
retain_source: false,
};
let mut active_during_import = Vec::new();
let summary = import_table_file_core(
&state,
&request,
&DatabaseType::Sqlite,
&pool_key,
|_| Box::pin(async { false }),
|progress| {
active_during_import.push((progress.status, state.running_queries.is_pool_active(&pool_key)));
},
)
.await
.unwrap();
assert_eq!(summary.rows_imported, 3);
assert!(!active_during_import.is_empty(), "the import must report progress while it owns the pool");
for (status, active) in &active_during_import {
assert!(active, "progress event {status:?} happened while the import's pool was idle to sweeps");
}
assert!(
!state.running_queries.is_pool_active(&pool_key),
"the import must release its pool registration when it finishes"
);
}
#[tokio::test] #[tokio::test]
async fn sqlite_update_existing_import_counts_updated_rows_once() { async fn sqlite_update_existing_import_counts_updated_rows_once() {
let dir = tempfile::tempdir().unwrap(); let dir = tempfile::tempdir().unwrap();