fix(sqlserver): rebuild pool after driver panic

This commit is contained in:
t8y2 committed 2026-06-18 13:43:03 +08:00
1 parent 991043e232
commit 17f186db26
4 files changed
+118 -18

No files matched your search

+9 -4
View File
@@ -13,6 +13,7 @@ use tokio::net::TcpStream;
use tokio_util::compat::{Compat, TokioAsyncWriteCompatExt};
pub type SqlServerClient = Client<Compat<TcpStream>>;
pub const SQLSERVER_DRIVER_PANIC_ERROR_PREFIX: &str = "SQL Server driver panic:";
const SIMPLE_QUERY_MODULE_KEYWORDS: &[&str] = &["FUNCTION", "PROC", "PROCEDURE", "TRIGGER", "VIEW"];
#[derive(Debug, PartialEq, Eq)]
@@ -175,13 +176,17 @@ where
{
match AssertUnwindSafe(future).catch_unwind().await {
Ok(result) => result.map_err(|e| e.to_string()),
Err(_) => {
Err("SQL Server driver could not decode this result set. Unsupported columns may need to be cast to text."
.to_string())
}
Err(_) => Err(format!(
"{SQLSERVER_DRIVER_PANIC_ERROR_PREFIX} the current client will be rebuilt. \
Unsupported columns may need to be cast to text."
)),
}
}
pub fn is_driver_panic_error(error: &str) -> bool {
error.starts_with(SQLSERVER_DRIVER_PANIC_ERROR_PREFIX)
}
async fn describe_sqlserver_result_set(
client: &mut SqlServerClient,
sql: &str,
+92 -8
View File
@@ -18,6 +18,7 @@ use tokio::time::timeout;
use tokio_util::sync::CancellationToken;
use crate::connection::{AppState, PoolKind};
use crate::database_capabilities;
use crate::db;
use crate::models::connection::DatabaseType;
#[cfg(feature = "duckdb-bundled")]
@@ -30,6 +31,13 @@ pub const QUERY_CANCELED: &str = "Query canceled";
#[cfg(feature = "duckdb-bundled")]
const DUCKDB_INTERRUPT_DRAIN_TIMEOUT: Duration = Duration::from_secs(2);
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PoolErrorAction {
Keep,
Discard,
ReconnectAndRetry,
}
/// Check read-only protection for a connection, blocking write SQL statements.
/// Only clones the connection name when read-only mode is active, avoiding
/// unnecessary allocations otherwise.
@@ -683,6 +691,28 @@ fn should_discard_agent_pool_after_error(err: &str) -> bool {
|| lower.contains("agent rpc task failed")
}
pub fn pool_error_action(db_type: Option<DatabaseType>, err: &str) -> PoolErrorAction {
let lower = err.to_lowercase();
if db::sqlserver::is_driver_panic_error(err)
|| (db_type == Some(DatabaseType::SqlServer) && is_dbx_query_timeout_error(&lower))
|| (db_type.is_some_and(|db_type| database_capabilities::is_agent_type(&db_type))
&& should_discard_agent_pool_after_error(err)
&& !is_connection_error(err))
{
return PoolErrorAction::Discard;
}
if is_connection_error(err) {
PoolErrorAction::ReconnectAndRetry
} else {
PoolErrorAction::Keep
}
}
pub fn should_discard_pool_after_error(db_type: Option<DatabaseType>, err: &str) -> bool {
matches!(pool_error_action(db_type, err), PoolErrorAction::Discard | PoolErrorAction::ReconnectAndRetry)
}
fn is_os_connection_error(lower: &str) -> bool {
let os_error_codes = ["10053", "10054", "10057", "10058", "10060", "10061"];
if let Some(pos) = lower.find("os error ") {
@@ -905,13 +935,18 @@ pub async fn do_execute(
},
None => client.lock().await,
};
wait_for_query_opt(
let result = wait_for_query_opt(
cancel_token,
query_timeout,
db::sqlserver::execute_query_with_max_rows(&mut client, sql, max_rows),
)
.await
.map(|result| truncate_result_with_max_rows(result, max_rows))
.map(|result| truncate_result_with_max_rows(result, max_rows));
drop(client);
if matches!(result.as_ref(), Err(err) if should_discard_pool_after_error(pool_db_type, err)) {
state.remove_pool_by_key(pool_key).await;
}
result
}
PoolKind::Elasticsearch(client) => {
let client = client.clone();
@@ -957,7 +992,7 @@ pub async fn do_execute(
})
.await
.map(|result| normalize_query_result_for_js(truncate_result_with_max_rows(result, max_rows)));
if matches!(result.as_ref(), Err(err) if should_discard_agent_pool_after_error(err)) {
if matches!(result.as_ref(), Err(err) if should_discard_pool_after_error(pool_db_type, err)) {
state.remove_pool_by_key(pool_key).await;
}
result
@@ -1103,7 +1138,9 @@ pub async fn execute_sql_statement_with_options(
.await;
match &result {
Err(e) if is_connection_error(e) && !is_canceled(&cancel_token) => {
Err(e)
if pool_error_action(db_type, e) == PoolErrorAction::ReconnectAndRetry && !is_canceled(&cancel_token) =>
{
let db_opt = if database.is_empty() { None } else { Some(database) };
let new_key =
state.reconnect_pool_for_session(connection_id, db_opt, options.client_session_id.as_deref()).await?;
@@ -1452,9 +1489,13 @@ async fn execute_multi_sqlserver(
None => client.lock().await,
};
match db::sqlserver::execute_batch_with_max_rows(&mut client, batch, max_rows).await {
let result = db::sqlserver::execute_batch_with_max_rows(&mut client, batch, max_rows).await;
drop(client);
match result {
Ok(results) => all_results.extend(results),
Err(e) => {
let action = pool_error_action(Some(DatabaseType::SqlServer), &e);
all_results.push(db::QueryResult {
columns: vec!["Error".to_string()],
column_types: Vec::new(),
@@ -1466,6 +1507,10 @@ async fn execute_multi_sqlserver(
session_id: None,
has_more: false,
});
if matches!(action, PoolErrorAction::Discard | PoolErrorAction::ReconnectAndRetry) {
state.remove_pool_by_key(pool_key).await;
break;
}
}
}
}
@@ -1522,9 +1567,15 @@ pub async fn execute_statements(
total_affected += result.affected_rows;
}
Err(e) => {
if is_connection_error(&e) {
let db_opt = if database.is_empty() { None } else { Some(database) };
let _ = state.reconnect_pool(connection_id, db_opt).await;
match pool_error_action(connection_database_type(state, connection_id).await, &e) {
PoolErrorAction::ReconnectAndRetry => {
let db_opt = if database.is_empty() { None } else { Some(database) };
let _ = state.reconnect_pool(connection_id, db_opt).await;
}
PoolErrorAction::Discard => {
let _ = state.remove_pool_by_key(&pool_key).await;
}
PoolErrorAction::Keep => {}
}
return Err(format!(
"Statement {} failed: {}. Previous {} statement(s) may have been committed.",
@@ -2013,6 +2064,31 @@ mod tests {
assert!(!is_connection_error("os error 13"));
}
#[test]
fn pool_error_action_discards_sqlserver_driver_panic_without_retry() {
let err = format!("{} the current client will be rebuilt.", db::sqlserver::SQLSERVER_DRIVER_PANIC_ERROR_PREFIX);
assert_eq!(pool_error_action(Some(DatabaseType::SqlServer), &err), PoolErrorAction::Discard);
assert!(should_discard_pool_after_error(Some(DatabaseType::SqlServer), &err));
assert!(!is_connection_error(&err));
}
#[test]
fn pool_error_action_discards_sqlserver_timeout_without_retry() {
let err = "Query timed out after 30 seconds";
assert_eq!(pool_error_action(Some(DatabaseType::SqlServer), err), PoolErrorAction::Discard);
assert_eq!(pool_error_action(Some(DatabaseType::Mysql), err), PoolErrorAction::Keep);
}
#[test]
fn pool_error_action_reconnects_connection_errors() {
let err = "connection reset by peer";
assert_eq!(pool_error_action(Some(DatabaseType::SqlServer), err), PoolErrorAction::ReconnectAndRetry);
assert_eq!(pool_error_action(Some(DatabaseType::Postgres), err), PoolErrorAction::ReconnectAndRetry);
}
#[cfg(feature = "duckdb-bundled")]
#[test]
fn duckdb_execute_preserves_double_precision() {
@@ -2397,6 +2473,10 @@ mod tests {
assert!(should_discard_agent_pool_after_error("Query timed out after 30 seconds"));
assert!(should_discard_agent_pool_after_error("Agent RPC call timed out (30s)"));
assert!(!is_connection_error("Agent RPC call timed out (30s)"));
assert_eq!(
pool_error_action(Some(DatabaseType::Oracle), "Agent RPC call timed out (30s)"),
PoolErrorAction::Discard
);
}
#[test]
@@ -2405,6 +2485,10 @@ mod tests {
assert!(should_discard_agent_pool_after_error("Agent stdout not available"));
assert!(is_connection_error("Agent stdin not available"));
assert!(is_connection_error("Agent stdout not available"));
assert_eq!(
pool_error_action(Some(DatabaseType::Oracle), "Agent stdin not available"),
PoolErrorAction::ReconnectAndRetry
);
}
#[test]
+9 -4
View File
@@ -1,7 +1,7 @@
use crate::connection::{connection_url_for_endpoint, database_connection_config, AppState, MysqlMode, PoolKind};
use crate::db;
use crate::models::connection::{ConnectionConfig, DatabaseType};
use crate::query::{agent_execute_query_params, QueryExecutionOptions};
use crate::query::{agent_execute_query_params, should_discard_pool_after_error, QueryExecutionOptions};
use std::future::Future;
use std::sync::Arc;
use std::time::Duration;
@@ -2019,10 +2019,15 @@ pub async fn get_object_source_core(
if let Some(client) = extract_pool!(&connections, &pool_key, SqlServer) {
drop(connections);
let mut client = client.lock().await;
first_string_cell(
let result =
db::sqlserver::execute_query(&mut client, &sqlserver_object_source_sql(schema, name, &object_type))
.await?,
)?
.await;
drop(client);
if matches!(result.as_ref(), Err(err) if should_discard_pool_after_error(Some(DatabaseType::SqlServer), err))
{
state.remove_pool_by_key(&pool_key).await;
}
first_string_cell(result?)?
} else if let Some(client) = extract_pool!(&connections, &pool_key, Agent) {
drop(connections);
if db_config.as_ref().is_some_and(|config| config.db_type == DatabaseType::Oracle)
+8 -2
View File
@@ -8,7 +8,7 @@ use crate::db;
use crate::db::mongo_driver::MongoDocumentResult;
use crate::models::connection::DatabaseType;
use crate::object_source_sql::{build_executable_object_source_statements, EditableObjectSourceSqlInput};
use crate::query::{agent_execute_query_params, QueryExecutionOptions};
use crate::query::{agent_execute_query_params, should_discard_pool_after_error, QueryExecutionOptions};
#[cfg(feature = "duckdb-bundled")]
use crate::sql::starts_with_executable_sql_keyword;
use crate::sql_dialect::{qualified_transfer_table, quote_transfer_identifier};
@@ -1851,7 +1851,13 @@ pub async fn execute_on_pool_with_max_rows(
let client = client.clone();
drop(connections);
let mut client = client.lock().await;
db::sqlserver::execute_query_with_max_rows(&mut client, sql, max_rows).await
let result = db::sqlserver::execute_query_with_max_rows(&mut client, sql, max_rows).await;
drop(client);
if matches!(result.as_ref(), Err(err) if should_discard_pool_after_error(Some(DatabaseType::SqlServer), err))
{
state.remove_pool_by_key(pool_key).await;
}
result
}
PoolKind::Agent(client) => {
let client = client.clone();