From 497d7c9e1ee58953d6cd6ca39bab0ee17848adcd Mon Sep 17 00:00:00 2001 From: zipg Date: Tue, 29 Sep 2026 16:41:05 +0800 Subject: [PATCH] fix(import): keep doris import transactions alive against pool probing --- crates/dbx-core/src/connection/mod.rs | 39 +++++++- crates/dbx-core/src/data/table_import.rs | 114 +++++++++++++++++++++++ 2 files changed, 151 insertions(+), 2 deletions(-) diff --git a/crates/dbx-core/src/connection/mod.rs b/crates/dbx-core/src/connection/mod.rs index 51ad92c411..b83afe393d 100644 --- a/crates/dbx-core/src/connection/mod.rs +++ b/crates/dbx-core/src/connection/mod.rs @@ -2178,6 +2178,12 @@ impl AppState { match result { Ok(Ok(())) => {} Ok(Err(err)) => { + if !keepalive_failure_proves_pool_dead(&err.to_string()) { + log::debug!( + "Connection keepalive for '{key}' could not check out a connection; keeping the busy pool: {err}" + ); + continue; + } log::warn!("Connection keepalive failed for '{key}': {err}; invalidating pool"); let replace_runtime = err.recovery_decision().is_some_and(RecoveryDecision::replaces_runtime); @@ -5936,6 +5942,18 @@ impl From for KeepaliveError { } } +/// Whether a failed keepalive probe is evidence that the pool is dead. +/// +/// A probe that ran out of its checkout budget because every connection is in use reports +/// pool saturation, which says nothing about the health of the pooled connections. That is +/// the normal state of a session-scoped pool (a single connection) while a batch import or +/// transfer holds a long transaction on it. Invalidating the pool there used to abort the +/// running operation with "Connection not found for transaction" instead of letting it +/// finish. Every other probe failure keeps the invalidate-and-reconnect behaviour. +fn keepalive_failure_proves_pool_dead(error: &str) -> bool { + !crate::query::is_pool_saturation_error(error) +} + impl KeepaliveTarget { fn matches_pool(&self, pool: &PoolKind) -> bool { match (self, pool) { @@ -6913,8 +6931,8 @@ mod tests { connection_probe_endpoints, connection_remote_endpoint, connection_url_for_endpoint, database_connection_config, database_connection_config_with_catalog, gaussdb_identifier_quote_from_query_result, gaussdb_m_jdbc_config_for_endpoint, gaussdb_uses_m_jdbc_driver, - kafka_single_loopback_bootstrap_endpoint, metadata_connection_config, metadata_pool_database, - mysql_metadata_fallback_url, mysql_pool_setup_queries, oceanbase_mysql_setup_queries, + kafka_single_loopback_bootstrap_endpoint, keepalive_failure_proves_pool_dead, metadata_connection_config, + metadata_pool_database, mysql_metadata_fallback_url, mysql_pool_setup_queries, oceanbase_mysql_setup_queries, prestosql_jdbc_config_for_endpoint, redacted_connection_url_for_endpoint, redis_sentinel_transport_id, redis_sentinel_transport_prefix, sqlserver_legacy_agent_config, sqlserver_legacy_driver_error, sqlserver_uses_legacy_driver, task_client_session_id, transport_layers_through_last_ssh, @@ -8345,6 +8363,23 @@ mod tests { let _ = std::fs::remove_dir_all(dir); } + #[test] + fn keepalive_probe_keeps_a_busy_pool_but_invalidates_a_dead_one() { + // Every connection is checked out: the probe learned nothing about pool health, so the + // pool must survive. Invalidating it here aborts whatever holds the connection — a + // truncate import reports "Connection not found for transaction" on its next chunk. + assert!(!keepalive_failure_proves_pool_dead( + "MySQL connection pool checkout timed out [stage=wait, timeout_ms=10000]" + )); + // A checkout failure while creating a connection is still evidence the pool is dead. + assert!(keepalive_failure_proves_pool_dead( + "MySQL connection pool checkout failed [stage=create]: connection refused" + )); + assert!(keepalive_failure_proves_pool_dead( + "MySQL connection pool checkout timed out [stage=create, timeout_ms=10000]" + )); + } + #[tokio::test] async fn jdbc_plugin_env_uses_managed_jre_when_installed() { let dir = std::env::temp_dir().join(format!("dbx-core-jdbc-managed-jre-{}", uuid::Uuid::new_v4())); diff --git a/crates/dbx-core/src/data/table_import.rs b/crates/dbx-core/src/data/table_import.rs index e07618764c..9ae6fdbcb7 100644 --- a/crates/dbx-core/src/data/table_import.rs +++ b/crates/dbx-core/src/data/table_import.rs @@ -78,6 +78,16 @@ pub fn table_import_client_session_id(import_id: &str) -> String { task_client_session_id("table-import", import_id) } +/// Execution id registered with `RunningQueries` for the whole duration of an import. +/// +/// Background pool maintenance (the keepalive probe, connection health refresh, idle +/// reclamation) consults `is_pool_active` before touching a pool. An import keeps a +/// connection checked out for the entire run — a session-scoped pool has exactly one +/// connection — so without this registration the pool looks idle while it is in use. +fn table_import_execution_id(import_id: &str) -> String { + format!("table-import-exec:{import_id}") +} + #[derive(Debug, Clone)] pub struct ParsedImportFile { pub columns: Vec, @@ -7063,6 +7073,16 @@ where let mut db_write_ms = 0u128; let mut statement_count = 0usize; let batch_size = if request.batch_size == 0 { DEFAULT_BATCH_SIZE } else { request.batch_size }; + // Mark the import's pool busy for the whole run. A session-scoped pool keeps a single + // connection, and the transactional batch path holds it for an entire chunk, so the + // keepalive probe cannot check out a connection on time. Without this registration the + // probe treats that healthy-but-busy pool as dead, invalidates it, and the next chunk + // fails with "Connection not found for transaction". + let import_execution_id = table_import_execution_id(&request.import_id); + let _import_activity = state.running_queries.register(import_execution_id.clone()); + state.running_queries.set_pool_key(&import_execution_id, pool_key); + state.touch_pool_activity(pool_key).await; + let _activity_touch = state.pool_activity_touch(pool_key); let kingbase_oracle_mode = kingbase_oracle_compatibility_mode(state, pool_key, db_type).await; let conflict_policy = request.effective_conflict_policy(); if let Err(error) = validate_update_existing_target(conflict_policy, db_type, request.create_table) { @@ -13255,6 +13275,100 @@ mod tests { ); } + // A session-scoped pool keeps a single connection, and the transactional batch path holds + // it for a whole chunk, so the keepalive probe cannot check out a connection while the + // import is running. The import must therefore advertise its pool as busy for its whole + // run; otherwise the probe invalidates the healthy pool and the next chunk fails with + // "Connection not found for transaction". + #[tokio::test] + async fn truncate_import_marks_its_pool_active_until_it_finishes() { + let dir = tempfile::tempdir().unwrap(); + let storage = crate::persistence::test_storage::open(&dir.path().join("storage.db")).await.unwrap(); + let state = AppState::new(storage); + let connection_id = "sqlite-truncate-activity"; + let pool_key = format!("{connection_id}:session:import"); + let database_path = dir.path().join("target.db"); + let sqlite = crate::db::sqlite::connect_path_create_if_missing(database_path.to_str().unwrap()).await.unwrap(); + crate::db::sqlite::execute_query(&sqlite, "CREATE TABLE items (id INTEGER, name TEXT)").await.unwrap(); + state + .update_connection_pools(|connections| { + connections.insert(pool_key.clone(), PoolKind::Sqlite(sqlite.clone())); + }) + .await; + let config: ConnectionConfig = serde_json::from_value(serde_json::json!({ + "id": connection_id, + "name": "SQLite truncate activity test", + "db_type": "sqlite", + "host": "", + "port": 0, + "username": "", + "password": "", + "database": database_path.to_string_lossy() + })) + .unwrap(); + state.configs.write().await.insert(connection_id.to_string(), config); + let data_path = dir.path().join("rows.txt"); + std::fs::write(&data_path, b"id%name\n1%Ada\n2%Grace\n3%Linus\n").unwrap(); + let request = TableImportRequest { + import_id: "sqlite-truncate-activity".to_string(), + connection_id: connection_id.to_string(), + database: String::new(), + schema: String::new(), + table: "items".to_string(), + file_path: data_path.to_string_lossy().to_string(), + source_ref: None, + source_format: Some(TableImportSourceFormat::Delimited), + parse_options: TableImportParseOptions { + delimiter: Some("%".to_string()), + ..TableImportParseOptions::default() + }, + mappings: vec![ + TableImportColumnMapping { + source_column: "id".to_string(), + target_column: "id".to_string(), + target_data_type: None, + }, + TableImportColumnMapping { + source_column: "name".to_string(), + target_column: "name".to_string(), + target_data_type: None, + }, + ], + mode: TableImportMode::Truncate, + create_table: false, + batch_size: 2, + date_time_format: None, + prepared_source: None, + skip_duplicate_rows: false, + conflict_policy: None, + retain_source: false, + }; + + let mut active_during_import = Vec::new(); + let summary = import_table_file_core( + &state, + &request, + &DatabaseType::Sqlite, + &pool_key, + |_| Box::pin(async { false }), + |progress| { + active_during_import.push((progress.status, state.running_queries.is_pool_active(&pool_key))); + }, + ) + .await + .unwrap(); + + assert_eq!(summary.rows_imported, 3); + assert!(!active_during_import.is_empty(), "the import must report progress while it owns the pool"); + for (status, active) in &active_during_import { + assert!(active, "progress event {status:?} happened while the import's pool was idle to sweeps"); + } + assert!( + !state.running_queries.is_pool_active(&pool_key), + "the import must release its pool registration when it finishes" + ); + } + #[tokio::test] async fn sqlite_update_existing_import_counts_updated_rows_once() { let dir = tempfile::tempdir().unwrap();