fix(connection): 修复数据库连接超时后无法可靠恢复的问题

* feat(core): implement PIP-0001 database connection timeout recovery

- Add DbOperationBudget struct for unified execution budget model
- Add deadpool create/recycle timeout for PostgreSQL pools
- Add checkout_postgres_client helper replacing 14 bare pool.get() calls
  with timeout + cancel token support
- Fix PostgreSQL TLS cancel: rebuild TLS connector from stored context,
  fallback to NoTls on failure
- Add timeout to MySQL kill_query and kill_query_with_opts (5s)
- Extend is_connection_error to match deadpool pool timeout messages
- Unify keepalive default to 30s (was 60s Rust / 0s frontend mismatch)
- Inject PostgreSQL TCP keepalive params (idle=30s, interval=10s, retries=3)
- Inject MySQL tcp_keepalive=30s in OptsBuilder
- Make MySQL inactive_connection_ttl configurable from idle_timeout_secs
- Add stage-based structured logging for pool.checkout/health.check/cancel
- Store PostgresCancelContext in AppState for TLS cancel reconstruction
- Clean up cancel contexts on pool removal/drain
- Add unit tests for DbOperationBudget and pool timeout error detection

* fix(core): align timeout recovery with PIP-0001

* fix(core): handle timeout recovery review issues

* fix(core): complete connection timeout recovery

* fix(core): fall back postgres cancel connector build

* refactor: revert comment translations to English in connection.rs and query.rs

- Revert DbOperationBudget doc comments to English (query.rs)
- Revert PostgresCancelContext and TLS cancel comments to English (connection.rs)
- Keep Chinese error-matching strings in is_connection_error unchanged

---------

Co-authored-by: onceMisery <onceMisery@users.noreply.github.com>
This commit is contained in:
miracleandonceMisery authored and GitHub committed 2026-06-26 21:41:56 +08:00
1 parent 796b0ce5b7
commit d128604743
11 files changed
+1134 -169

No files matched your search

+50 -15
View File
@@ -33,7 +33,7 @@ pub const JDBC_PLUGIN_NOT_INSTALLED: &str =
pub const PRESTOSQL_JDBC_DRIVER_CLASS: &str = "io.prestosql.jdbc.PrestoDriver";
const DEFAULT_AGENT_CONNECT_TIMEOUT_SECS: u64 = 30;
const ACCESS_AGENT_CONNECT_TIMEOUT_SECS: u64 = 30;
const POOL_CLOSE_TIMEOUT_SECS: u64 = 5;
const POOL_CLOSE_TIMEOUT_SECS: u64 = 3;
#[cfg(feature = "duckdb-bundled")]
mod duckdb_types {
@@ -136,6 +136,9 @@ pub struct AppState {
pub plugins: PluginRegistry,
pub agent_manager: crate::agent_manager::AgentManager,
pub nacos_registry: crate::nacos::NacosAdminRegistry,
/// PostgreSQL TLS cancel context, keyed by pool_key.
/// Used to reconstruct a TLS connector compatible with the original connection when cancelling.
postgres_cancel_contexts: Arc<RwLock<HashMap<String, db::postgres::PostgresCancelContext>>>,
#[cfg(feature = "mq-admin")]
pub mq_registry: crate::mq::MqAdminRegistry,
}
@@ -214,6 +217,7 @@ pub async fn connect_mysql_metadata_pool(
max_connections: usize,
) -> Result<(db::mysql::MySqlPool, MysqlMode), String> {
let url = connection_url_for_endpoint(db_config, host, port);
let idle_timeout_secs = Some(db_config.idle_timeout_secs);
if db_config.needs_bare_mysql() {
return match db::mysql::connect_bare_with_pool_limit(&url, connect_timeout, max_connections).await {
Ok(pool) => Ok((pool, MysqlMode::Bare)),
@@ -233,11 +237,12 @@ pub async fn connect_mysql_metadata_pool(
};
}
match db::mysql::connect_with_ca_cert_and_pool_limit(
match db::mysql::connect_with_ca_cert_pool_limit_and_idle(
&url,
Some(&db_config.ca_cert_path),
connect_timeout,
max_connections,
idle_timeout_secs,
)
.await
{
@@ -251,11 +256,12 @@ pub async fn connect_mysql_metadata_pool(
log::info!(
"MySQL metadata connection without a default database failed ({err}); retrying with configured default database."
);
let pool = db::mysql::connect_with_ca_cert_and_pool_limit(
let pool = db::mysql::connect_with_ca_cert_pool_limit_and_idle(
&fallback_url,
Some(&config.ca_cert_path),
connect_timeout,
max_connections,
idle_timeout_secs,
)
.await?;
let mode = detect_ob_oracle_mode(config, &pool).await;
@@ -365,6 +371,7 @@ impl AppState {
app_version,
),
nacos_registry: crate::nacos::NacosAdminRegistry::new(),
postgres_cancel_contexts: Arc::new(RwLock::new(HashMap::new())),
#[cfg(feature = "mq-admin")]
mq_registry: crate::mq::MqAdminRegistry::new(),
}
@@ -420,9 +427,10 @@ impl AppState {
self.stop_keepalive_task(&pool_key).await;
self.pool_activity.write().await.insert(pool_key.clone(), PoolActivity::now());
self.start_keepalive_task(&pool_key, &pool, config).await;
let previous_key = pool_key.clone();
let previous = self.connections.write().await.insert(pool_key, pool);
if let Some(pool) = previous {
close_pool_kind(pool).await;
close_pool_kind_with_timeout(previous_key, pool).await;
}
}
@@ -461,7 +469,7 @@ impl AppState {
config: &ConnectionConfig,
) -> Result<(), String> {
if let Err(err) = self.ensure_current_connection_attempt(connection_id, Some(attempt)).await {
close_pool_kind(pool).await;
close_pool_kind_with_timeout(pool_key, pool).await;
return Err(err);
}
self.insert_connection_pool(pool_key, pool, config).await;
@@ -495,6 +503,7 @@ impl AppState {
let connections = self.connections.clone();
let keepalive_tasks = self.keepalive_tasks.clone();
let pool_activity = self.pool_activity.clone();
let cancel_contexts = self.postgres_cancel_contexts.clone();
let running_queries = self.running_queries.clone();
let idle_timeout = Duration::from_secs(idle_timeout_secs.max(1));
let handle = tokio::spawn(async move {
@@ -517,6 +526,7 @@ impl AppState {
);
keepalive_tasks.write().await.remove(&key);
pool_activity.write().await.remove(&key);
cancel_contexts.write().await.remove(&key);
let removed = connections.write().await.remove(&key);
if let Some(pool) = removed {
close_pool_kind_with_timeout(key, pool).await;
@@ -533,6 +543,7 @@ impl AppState {
log::warn!("Connection keepalive failed for '{key}': {err}; invalidating pool");
keepalive_tasks.write().await.remove(&key);
pool_activity.write().await.remove(&key);
cancel_contexts.write().await.remove(&key);
let removed = connections.write().await.remove(&key);
if let Some(pool) = removed {
close_pool_kind_with_timeout(key, pool).await;
@@ -546,6 +557,7 @@ impl AppState {
);
keepalive_tasks.write().await.remove(&key);
pool_activity.write().await.remove(&key);
cancel_contexts.write().await.remove(&key);
let removed = connections.write().await.remove(&key);
if let Some(pool) = removed {
close_pool_kind_with_timeout(key, pool).await;
@@ -582,6 +594,11 @@ impl AppState {
self.pool_activity.write().await.insert(pool_key.to_string(), PoolActivity::now());
}
/// Get the PostgreSQL TLS cancel context (used to reconstruct the TLS connector when cancelling a query).
pub async fn get_postgres_cancel_context(&self, pool_key: &str) -> Option<db::postgres::PostgresCancelContext> {
self.postgres_cancel_contexts.read().await.get(pool_key).cloned()
}
pub fn pool_activity_touch(&self, pool_key: &str) -> PoolActivityTouch {
PoolActivityTouch {
pool_key: pool_key.to_string(),
@@ -678,7 +695,14 @@ impl AppState {
| DatabaseType::Gaussdb
| DatabaseType::Kwdb
| DatabaseType::Questdb
| DatabaseType::OpenGauss => PoolKind::Postgres(db::postgres::connect(&url, connect_timeout).await?),
| DatabaseType::OpenGauss => {
let pg_pool = db::postgres::connect(&url, connect_timeout).await?;
// Build TLS cancel context for reconstructing TLS connection during cancel
if let Some(ctx) = db::postgres::build_postgres_cancel_context(&url) {
self.postgres_cancel_contexts.write().await.insert(pool_key.clone(), ctx);
}
PoolKind::Postgres(pg_pool)
}
DatabaseType::Sqlite => {
let extensions = db::sqlite::sqlite_extension_specs_from_url_params(db_config.url_params.as_deref())
.into_iter()
@@ -1005,7 +1029,7 @@ impl AppState {
};
if let Err(err) = self.ensure_current_connection_attempt(connection_id, connection_attempt).await {
close_pool_kind(pool).await;
close_pool_kind_with_timeout(pool_key.clone(), pool).await;
return Err(err);
}
self.insert_connection_pool(pool_key.clone(), pool, &db_config).await;
@@ -1316,9 +1340,10 @@ impl AppState {
self.stop_keepalive_task(pool_key).await;
self.pool_activity.write().await.remove(pool_key);
self.postgres_cancel_contexts.write().await.remove(pool_key);
let removed = self.connections.write().await.remove(pool_key);
if let Some(pool) = removed {
close_pool_kind(pool).await;
close_pool_kind_with_timeout(pool_key.to_string(), pool).await;
true
} else {
false
@@ -1347,9 +1372,10 @@ impl AppState {
} else {
self.stop_keepalive_task(&pool_key).await;
self.pool_activity.write().await.remove(&pool_key);
self.postgres_cancel_contexts.write().await.remove(&pool_key);
let removed = self.connections.write().await.remove(&pool_key);
if let Some(pool) = removed {
close_pool_kind(pool).await;
close_pool_kind_with_timeout(pool_key.clone(), pool).await;
}
}
self.get_or_create_pool_for_session(connection_id, database, client_session_id).await
@@ -1376,9 +1402,10 @@ impl AppState {
}
self.stop_keepalive_task(&pool_key).await;
self.pool_activity.write().await.remove(&pool_key);
self.postgres_cancel_contexts.write().await.remove(&pool_key);
let removed = self.connections.write().await.remove(&pool_key);
if let Some(pool) = removed {
close_pool_kind(pool).await;
close_pool_kind_with_timeout(pool_key, pool).await;
Ok(true)
} else {
Ok(false)
@@ -1388,9 +1415,10 @@ impl AppState {
pub async fn remove_pool_by_key(&self, pool_key: &str) -> bool {
self.stop_keepalive_task(pool_key).await;
self.pool_activity.write().await.remove(pool_key);
self.postgres_cancel_contexts.write().await.remove(pool_key);
let removed = self.connections.write().await.remove(pool_key);
if let Some(pool) = removed {
close_pool_kind(pool).await;
close_pool_kind_with_timeout(pool_key.to_string(), pool).await;
true
} else {
false
@@ -1418,21 +1446,23 @@ impl AppState {
self.stop_keepalive_tasks(&keys_to_remove).await;
{
let mut activity = self.pool_activity.write().await;
let mut cancel_contexts = self.postgres_cancel_contexts.write().await;
for key in &keys_to_remove {
activity.remove(key);
cancel_contexts.remove(key);
}
}
let mut conns = self.connections.write().await;
let mut removed = Vec::with_capacity(keys_to_remove.len());
for key in keys_to_remove {
if let Some(pool) = conns.remove(&key) {
removed.push(pool);
removed.push((key, pool));
}
}
drop(conns);
let closed = !removed.is_empty();
for pool in removed {
close_pool_kind(pool).await;
for (key, pool) in removed {
close_pool_kind_with_timeout(key, pool).await;
}
Ok(closed)
}
@@ -1720,11 +1750,14 @@ impl AppState {
}
}
let mut conns = self.connections.write().await;
let mut removed = Vec::with_capacity(dead_keys.len());
for key in &dead_keys {
if let Some(pool) = conns.remove(key) {
close_pool_kind(pool).await;
removed.push((key.clone(), pool));
}
}
drop(conns);
close_removed_pools(removed).await;
}
// Re-establish SSH tunnels that have died
@@ -1766,8 +1799,10 @@ impl AppState {
self.stop_keepalive_tasks(&keys_to_remove).await;
{
let mut activity = self.pool_activity.write().await;
let mut cancel_contexts = self.postgres_cancel_contexts.write().await;
for key in &keys_to_remove {
activity.remove(key);
cancel_contexts.remove(key);
}
}
let mut conns = self.connections.write().await;
+192 -40
View File
@@ -10,6 +10,7 @@ use std::path::PathBuf;
use std::sync::atomic::AtomicBool;
use std::time::Duration;
use std::time::Instant;
use tokio_util::sync::CancellationToken;
use crate::models::connection::DatabaseType;
use crate::sql::starts_with_executable_sql_keyword;
@@ -386,16 +387,26 @@ pub async fn connect_with_ca_cert_and_pool_limit(
ca_cert_path: Option<&str>,
fallback_timeout: Duration,
max_connections: usize,
) -> Result<MySqlPool, String> {
connect_with_ca_cert_pool_limit_and_idle(url, ca_cert_path, fallback_timeout, max_connections, None).await
}
pub async fn connect_with_ca_cert_pool_limit_and_idle(
url: &str,
ca_cert_path: Option<&str>,
fallback_timeout: Duration,
max_connections: usize,
idle_timeout_secs: Option<u64>,
) -> Result<MySqlPool, String> {
let timeout = super::parse_connect_timeout_with_fallback(url, fallback_timeout);
let pool = create_pool(url, ca_cert_path, max_connections)?;
let pool = create_pool(url, ca_cert_path, max_connections, idle_timeout_secs)?;
let result = verify_pool_connection(&pool, timeout).await;
if let Err(ref e) = result {
if mysql_error_should_retry_without_ssl(e) {
if let Some(fallback_url) = ssl_fallback_url(url) {
log::info!("SSL handshake failed, retrying with ssl-mode=disabled");
let fallback_pool = create_pool(&fallback_url, None, max_connections)?;
let fallback_pool = create_pool(&fallback_url, None, max_connections, idle_timeout_secs)?;
return match verify_pool_connection(&fallback_pool, timeout).await {
Ok(()) => Ok(fallback_pool),
Err(e) => Err(e),
@@ -413,7 +424,12 @@ struct MySqlTlsFiles {
sslkey: Option<String>,
}
fn create_pool(url: &str, ca_cert_path: Option<&str>, max_connections: usize) -> Result<MySqlPool, String> {
fn create_pool(
url: &str,
ca_cert_path: Option<&str>,
max_connections: usize,
idle_timeout_secs: Option<u64>,
) -> Result<MySqlPool, String> {
let tls_url = mysql_tls_url(url)?;
let opts =
mysql_async::Opts::from_url(&mysql_async_url(&tls_url.url)).map_err(|e| format!("Invalid MySQL URL: {e}"))?;
@@ -422,14 +438,17 @@ fn create_pool(url: &str, ca_cert_path: Option<&str>, max_connections: usize) ->
// Single-connection pools (max_connections == 1) are client session pools that
// must preserve session state (e.g. TEMPORARY TABLEs) across queries.
// Disable COM_RESET_CONNECTION for these pools to avoid clearing that state.
let inactive_ttl =
idle_timeout_secs.filter(|&s| s >= 30).map(Duration::from_secs).unwrap_or(Duration::from_secs(300));
let pool_opts = mysql_async::PoolOpts::new()
.with_constraints(mysql_async::PoolConstraints::new(1, max_connections).unwrap())
.with_inactive_connection_ttl(Duration::from_secs(300))
.with_inactive_connection_ttl(inactive_ttl)
.with_reset_connection(max_connections > 1);
let mut builder = mysql_async::OptsBuilder::from_opts(opts)
.stmt_cache_size(0)
.prefer_socket(false)
.pool_opts(Some(pool_opts))
.tcp_keepalive(Some(30u32))
.setup(mysql_setup_queries(url));
if let Some(ssl_opts) = mysql_ssl_opts(base_ssl_opts, url, ca_cert_path, &tls_url.files)? {
builder = builder.ssl_opts(ssl_opts);
@@ -992,12 +1011,12 @@ pub async fn connect_bare_with_pool_limit(
max_connections: usize,
) -> Result<MySqlPool, String> {
let timeout = super::parse_connect_timeout_with_fallback(url, fallback_timeout);
let pool = create_pool(url, None, max_connections)?;
let pool = create_pool(url, None, max_connections, None)?;
verify_pool_connection(&pool, timeout).await.map(|_| pool)
}
pub async fn list_databases(pool: &MySqlPool) -> Result<Vec<DatabaseInfo>, String> {
let mut conn = pool.get_conn().await.map_err(|e| e.to_string())?;
let mut conn = get_conn_with_timeout(pool, super::connection_timeout()).await?;
let result = match conn.query_iter("SELECT SCHEMA_NAME FROM information_schema.SCHEMATA ORDER BY SCHEMA_NAME").await
{
Ok(result) => result,
@@ -1018,7 +1037,7 @@ pub async fn list_databases(pool: &MySqlPool) -> Result<Vec<DatabaseInfo>, Strin
}
pub async fn list_databases_show(pool: &MySqlPool) -> Result<Vec<DatabaseInfo>, String> {
let mut conn = pool.get_conn().await.map_err(|e| e.to_string())?;
let mut conn = get_conn_with_timeout(pool, super::connection_timeout()).await?;
let result = conn.query_iter("SHOW DATABASES").await.map_err(|e| e.to_string())?;
let rows: Vec<mysql_async::Row> = result.collect_and_drop().await.map_err(|e| e.to_string())?;
Ok(database_infos_from_names(rows.iter().map(|row| get_str(row, 0)), true))
@@ -1057,7 +1076,7 @@ pub async fn list_tables_filtered(
object_types: Option<&[String]>,
) -> Result<Vec<TableInfo>, String> {
let sql = list_tables_sql(database, filter, limit, offset, object_types);
let mut conn = pool.get_conn().await.map_err(|e| e.to_string())?;
let mut conn = get_conn_with_timeout(pool, super::connection_timeout()).await?;
let result = match conn.query_iter(&sql).await {
Ok(result) => result,
Err(err) => {
@@ -1198,7 +1217,7 @@ pub async fn completion_assistant_search(
request.object_kinds.clone()
};
let pattern = mysql_completion_like_pattern(&request.mask, request.match_mode.as_ref());
let mut conn = pool.get_conn().await.map_err(|e| e.to_string())?;
let mut conn = get_conn_with_timeout(pool, super::connection_timeout()).await?;
let mut candidates = Vec::new();
if kinds
@@ -1421,7 +1440,7 @@ fn table_comment_sql(database: &str, table: &str) -> String {
pub async fn get_table_comment(pool: &MySqlPool, database: &str, table: &str) -> Result<Option<String>, String> {
let sql = table_comment_sql(database, table);
let mut conn = pool.get_conn().await.map_err(|e| e.to_string())?;
let mut conn = get_conn_with_timeout(pool, super::connection_timeout()).await?;
let result = conn.query_iter(&sql).await.map_err(|e| e.to_string())?;
let rows: Vec<mysql_async::Row> = result.collect_and_drop().await.map_err(|e| e.to_string())?;
Ok(rows
@@ -1444,7 +1463,7 @@ async fn list_table_status_show(pool: &MySqlPool, database: &str) -> Result<Hash
} else {
format!("SHOW TABLE STATUS FROM {}", quote_identifier(database))
};
let mut conn = pool.get_conn().await.map_err(|e| e.to_string())?;
let mut conn = get_conn_with_timeout(pool, super::connection_timeout()).await?;
let result = conn.query_iter(&sql).await.map_err(|e| e.to_string())?;
let rows: Vec<mysql_async::Row> = result.collect_and_drop().await.map_err(|e| e.to_string())?;
Ok(rows
@@ -1467,7 +1486,7 @@ async fn list_table_status_show(pool: &MySqlPool, database: &str) -> Result<Hash
async fn list_table_names_show(pool: &MySqlPool, database: &str) -> Result<Vec<TableInfo>, String> {
let sql = show_tables_sql(database, true);
let mut conn = pool.get_conn().await.map_err(|e| e.to_string())?;
let mut conn = get_conn_with_timeout(pool, super::connection_timeout()).await?;
let rows: Vec<mysql_async::Row> = match conn.query_iter(&sql).await {
Ok(result) => result.collect_and_drop().await.map_err(|e| e.to_string())?,
Err(_) => {
@@ -1589,7 +1608,7 @@ fn row_to_object(row: &mysql_async::Row, database: &str) -> ObjectInfo {
}
pub async fn list_objects(pool: &MySqlPool, database: &str) -> Result<Vec<ObjectInfo>, String> {
let mut conn = pool.get_conn().await.map_err(|e| e.to_string())?;
let mut conn = get_conn_with_timeout(pool, super::connection_timeout()).await?;
let tables_sql = list_tables_objects_sql(database);
let result = conn.query_iter(&tables_sql).await.map_err(|e| e.to_string())?;
@@ -1625,7 +1644,7 @@ pub async fn list_object_statistics(pool: &MySqlPool, database: &str) -> Result<
ORDER BY TABLE_NAME",
quote_value(database),
);
let mut conn = pool.get_conn().await.map_err(|e| e.to_string())?;
let mut conn = get_conn_with_timeout(pool, super::connection_timeout()).await?;
let result = conn.query_iter(&sql).await.map_err(|e| e.to_string())?;
let rows: Vec<mysql_async::Row> = result.collect_and_drop().await.map_err(|e| e.to_string())?;
Ok(rows
@@ -1672,7 +1691,7 @@ pub async fn list_table_objects_show(pool: &MySqlPool, database: &str) -> Result
}
async fn list_routine_objects(pool: &MySqlPool, database: &str) -> Result<Vec<ObjectInfo>, String> {
let mut conn = pool.get_conn().await.map_err(|e| e.to_string())?;
let mut conn = get_conn_with_timeout(pool, super::connection_timeout()).await?;
let routines_sql = list_routines_sql(database);
let result = conn.query_iter(&routines_sql).await.map_err(|e| e.to_string())?;
let rows: Vec<mysql_async::Row> = result.collect_and_drop().await.map_err(|e| e.to_string())?;
@@ -1680,7 +1699,7 @@ async fn list_routine_objects(pool: &MySqlPool, database: &str) -> Result<Vec<Ob
}
pub async fn list_completion_objects(pool: &MySqlPool, database: &str) -> Result<Vec<ObjectInfo>, String> {
let mut conn = pool.get_conn().await.map_err(|e| e.to_string())?;
let mut conn = get_conn_with_timeout(pool, super::connection_timeout()).await?;
let mut objects = Vec::new();
let routines_sql = list_routines_sql(database);
@@ -1991,31 +2010,115 @@ fn skip_mysql_quoted(sql: &str, start: usize, quote: u8) -> usize {
/// Get a connection from the pool with a health check. If the connection is dead
/// (e.g. after app was backgrounded), it tries again with a fresh connection.
pub async fn get_conn_with_health_check(pool: &MySqlPool) -> Result<mysql_async::Conn, String> {
let timeout = crate::db::connection_timeout();
let mut conn = get_conn_with_timeout(pool, timeout).await?;
match ping_conn_with_timeout(&mut conn, timeout).await {
Ok(()) => Ok(conn),
_ => {
let _ = tokio::time::timeout(timeout, conn.disconnect()).await;
let mut conn = get_conn_with_timeout(pool, timeout).await?;
ping_conn_with_timeout(&mut conn, timeout).await?;
get_conn_with_health_check_with_timeout(pool, super::connection_timeout()).await
}
pub async fn get_conn_with_health_check_with_timeout(
pool: &MySqlPool,
timeout: Duration,
) -> Result<mysql_async::Conn, String> {
get_conn_with_health_check_with_cancel(pool, timeout, timeout, None).await
}
pub async fn get_conn_with_health_check_with_cancel(
pool: &MySqlPool,
timeout: Duration,
cleanup_timeout: Duration,
cancel_token: Option<&CancellationToken>,
) -> Result<mysql_async::Conn, String> {
let start = Instant::now();
let mut conn = get_conn_with_timeout_and_cancel(pool, timeout, cancel_token).await?;
match ping_conn_with_timeout_and_cancel(&mut conn, timeout, cancel_token).await {
Ok(()) => {
log::debug!(
"[db:health.check:done] elapsed_ms={} timeout_ms={}",
start.elapsed().as_millis(),
timeout.as_millis()
);
Ok(conn)
}
Err(err) if err == crate::query::QUERY_CANCELED => {
let _ = tokio::time::timeout(cleanup_timeout, conn.disconnect()).await;
Err(err)
}
Err(err) => {
log::warn!(
"[db:health.check:error] elapsed_ms={} timeout_ms={} error={}; retrying",
start.elapsed().as_millis(),
timeout.as_millis(),
err
);
let _ = tokio::time::timeout(cleanup_timeout, conn.disconnect()).await;
let mut conn = get_conn_with_timeout_and_cancel(pool, timeout, cancel_token).await?;
if let Err(err) = ping_conn_with_timeout_and_cancel(&mut conn, timeout, cancel_token).await {
if err == crate::query::QUERY_CANCELED {
let _ = tokio::time::timeout(cleanup_timeout, conn.disconnect()).await;
}
return Err(err);
}
log::info!(
"[db:health.check:recovered] elapsed_ms={} timeout_ms={}",
start.elapsed().as_millis(),
timeout.as_millis()
);
Ok(conn)
}
}
}
async fn get_conn_with_timeout(pool: &MySqlPool, timeout: Duration) -> Result<mysql_async::Conn, String> {
async fn get_conn_with_timeout_and_cancel(
pool: &MySqlPool,
timeout: Duration,
cancel_token: Option<&CancellationToken>,
) -> Result<mysql_async::Conn, String> {
let get_future = async {
tokio::time::timeout(timeout, pool.get_conn())
.await
.map_err(|_| "MySQL get connection timed out".to_string())?
.map_err(|e| e.to_string())
};
match cancel_token {
Some(token) => {
tokio::select! {
biased;
_ = token.cancelled() => Err(crate::query::canceled_error()),
result = get_future => result,
}
}
None => get_future.await,
}
}
pub async fn get_conn_with_timeout(pool: &MySqlPool, timeout: Duration) -> Result<mysql_async::Conn, String> {
tokio::time::timeout(timeout, pool.get_conn())
.await
.map_err(|_| "MySQL get connection timed out".to_string())?
.map_err(|e| e.to_string())
}
async fn ping_conn_with_timeout(conn: &mut mysql_async::Conn, timeout: Duration) -> Result<(), String> {
tokio::time::timeout(timeout, conn.ping())
.await
.map_err(|_| "MySQL ping timed out".to_string())?
.map_err(|e| e.to_string())
async fn ping_conn_with_timeout_and_cancel(
conn: &mut mysql_async::Conn,
timeout: Duration,
cancel_token: Option<&CancellationToken>,
) -> Result<(), String> {
let ping_future = async {
tokio::time::timeout(timeout, conn.ping())
.await
.map_err(|_| "MySQL ping timed out".to_string())?
.map_err(|e| e.to_string())
};
match cancel_token {
Some(token) => {
tokio::select! {
biased;
_ = token.cancelled() => Err(crate::query::canceled_error()),
result = ping_future => result,
}
}
None => ping_future.await,
}
}
async fn execute_result_set_with_text_protocol_on_conn(
@@ -2234,13 +2337,61 @@ async fn stream_query_rows_prepared(
}
pub async fn kill_query(pool: &MySqlPool, connection_id: u32) -> Result<(), String> {
let mut conn = pool.get_conn().await.map_err(|e| e.to_string())?;
conn.query_drop(format!("KILL QUERY {connection_id}")).await.map_err(|e| e.to_string())
let start = Instant::now();
let timeout = super::connection_timeout();
let mut conn = tokio::time::timeout(timeout, pool.get_conn())
.await
.map_err(|_| {
log::warn!(
"[db:cancel:error] elapsed_ms={} timeout_ms={} error=MySQL kill connection checkout timed out",
start.elapsed().as_millis(),
timeout.as_millis()
);
"MySQL kill connection checkout timed out".to_string()
})?
.map_err(|e| e.to_string())?;
tokio::time::timeout(timeout, conn.query_drop(format!("KILL QUERY {connection_id}")))
.await
.map_err(|_| {
log::warn!(
"[db:cancel:error] elapsed_ms={} timeout_ms={} error=MySQL KILL QUERY timed out",
start.elapsed().as_millis(),
timeout.as_millis()
);
"MySQL KILL QUERY timed out".to_string()
})?
.map_err(|e| e.to_string())?;
log::info!("[db:cancel:done] elapsed_ms={} timeout_ms={}", start.elapsed().as_millis(), timeout.as_millis());
Ok(())
}
pub async fn kill_query_with_opts(opts: mysql_async::Opts, connection_id: u32) -> Result<(), String> {
let mut conn = mysql_async::Conn::new(opts).await.map_err(|e| e.to_string())?;
conn.query_drop(format!("KILL QUERY {connection_id}")).await.map_err(|e| e.to_string())
let start = Instant::now();
let timeout = super::connection_timeout();
let mut conn = tokio::time::timeout(timeout, mysql_async::Conn::new(opts))
.await
.map_err(|_| {
log::warn!(
"[db:cancel:error] elapsed_ms={} timeout_ms={} error=MySQL kill connection timed out",
start.elapsed().as_millis(),
timeout.as_millis()
);
"MySQL kill connection timed out".to_string()
})?
.map_err(|e| e.to_string())?;
tokio::time::timeout(timeout, conn.query_drop(format!("KILL QUERY {connection_id}")))
.await
.map_err(|_| {
log::warn!(
"[db:cancel:error] elapsed_ms={} timeout_ms={} error=MySQL KILL QUERY execution timed out",
start.elapsed().as_millis(),
timeout.as_millis()
);
"MySQL KILL QUERY execution timed out".to_string()
})?
.map_err(|e| e.to_string())?;
log::info!("[db:cancel:done] elapsed_ms={} timeout_ms={}", start.elapsed().as_millis(), timeout.as_millis());
Ok(())
}
pub async fn execute_query_on_conn_with_max_rows(
@@ -2400,7 +2551,7 @@ pub async fn list_indexes(pool: &MySqlPool, database: &str, table: &str) -> Resu
quote_value(database),
quote_value(table),
);
let mut conn = pool.get_conn().await.map_err(|e| e.to_string())?;
let mut conn = get_conn_with_timeout(pool, super::connection_timeout()).await?;
let result = conn.query_iter(&sql).await.map_err(|e| e.to_string())?;
let rows: Vec<mysql_async::Row> = result.collect_and_drop().await.map_err(|e| e.to_string())?;
@@ -2438,7 +2589,7 @@ pub async fn list_foreign_keys(pool: &MySqlPool, database: &str, table: &str) ->
quote_value(database),
quote_value(table),
);
let mut conn = pool.get_conn().await.map_err(|e| e.to_string())?;
let mut conn = get_conn_with_timeout(pool, super::connection_timeout()).await?;
let result = conn.query_iter(&sql).await.map_err(|e| e.to_string())?;
let rows: Vec<mysql_async::Row> = result.collect_and_drop().await.map_err(|e| e.to_string())?;
@@ -2465,7 +2616,7 @@ pub async fn list_triggers(pool: &MySqlPool, database: &str, table: &str) -> Res
quote_value(database),
quote_value(table),
);
let mut conn = pool.get_conn().await.map_err(|e| e.to_string())?;
let mut conn = get_conn_with_timeout(pool, super::connection_timeout()).await?;
let result = conn.query_iter(&sql).await.map_err(|e| e.to_string())?;
let rows: Vec<mysql_async::Row> = result.collect_and_drop().await.map_err(|e| e.to_string())?;
@@ -2483,6 +2634,7 @@ pub async fn list_triggers(pool: &MySqlPool, database: &str, table: &str) -> Res
#[cfg(test)]
mod tests {
use super::*;
use crate::db::connection_timeout;
use mysql_async::consts::ColumnFlags;
#[test]
@@ -3044,7 +3196,7 @@ mod tests {
#[test]
fn parse_connect_timeout_ignores_out_of_range() {
let default = crate::db::connection_timeout();
let default = connection_timeout();
let url = "mysql://host:3306/db?connect_timeout=999";
assert_eq!(crate::db::parse_connect_timeout(url), default);
let url2 = "mysql://host:3306/db?connect_timeout=0";
@@ -3053,14 +3205,14 @@ mod tests {
#[test]
fn parse_connect_timeout_returns_default_when_missing() {
let default = crate::db::connection_timeout();
let default = connection_timeout();
let url = "mysql://host:3306/db?ssl-mode=preferred&charset=utf8mb4";
assert_eq!(crate::db::parse_connect_timeout(url), default);
}
#[test]
fn parse_connect_timeout_returns_default_when_no_query() {
let default = crate::db::connection_timeout();
let default = connection_timeout();
let url = "mysql://host:3306/db";
assert_eq!(crate::db::parse_connect_timeout(url), default);
}
+319 -69
View File
@@ -21,6 +21,7 @@ use tokio_postgres::{NoTls, Row, SimpleQueryMessage};
use tokio_util::sync::CancellationToken;
use super::file_validator::validate_file_path;
use crate::query::DbOperationBudget;
use crate::sql::starts_with_executable_sql_keyword;
use crate::types::{
ColumnInfo, CompletionAssistantCandidate, CompletionAssistantCandidateKind, CompletionAssistantMatchMode,
@@ -718,8 +719,7 @@ pub async fn stream_query_rows(
cancelled: &AtomicBool,
mut on_row: impl FnMut(&[serde_json::Value]) -> Result<(), String>,
) -> Result<u64, String> {
let client =
pool.get().await.map_err(|e| format!("PostgreSQL connection failed: {}", pg_pool_error_to_string(e)))?;
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
match stream_query_rows_on_client(&client, sql, max_rows, cancelled, &mut on_row).await {
Ok(rows) => Ok(rows),
Err(error) if should_retry_postgres_text_query_message(&error.to_ascii_lowercase()) => {
@@ -797,7 +797,8 @@ async fn stream_query_rows_text_on_client(
}
pub async fn connect(url: &str, fallback_timeout: Duration) -> Result<Pool, String> {
let postgres_url = postgres_connection_url(url)?;
let url_with_keepalive = inject_postgres_keepalive_params(url);
let postgres_url = postgres_connection_url(&url_with_keepalive)?;
let _ = rustls::crypto::aws_lc_rs::default_provider().install_default();
let timeout = super::parse_connect_timeout_with_fallback(url, fallback_timeout);
@@ -823,6 +824,8 @@ pub async fn connect(url: &str, fallback_timeout: Duration) -> Result<Pool, Stri
.max_size(10)
.runtime(Runtime::Tokio1)
.wait_timeout(Some(timeout))
.create_timeout(Some(timeout))
.recycle_timeout(Some(timeout))
.build()
.map_err(|e| format!("Failed to create PostgreSQL pool: {e}"))?;
@@ -843,10 +846,46 @@ pub async fn connect(url: &str, fallback_timeout: Duration) -> Result<Pool, Stri
}
#[derive(Debug, Default, Clone, PartialEq, Eq)]
struct PostgresSslFiles {
sslcert: Option<String>,
sslkey: Option<String>,
sslrootcert: Option<String>,
pub struct PostgresSslFiles {
pub sslcert: Option<String>,
pub sslkey: Option<String>,
pub sslrootcert: Option<String>,
}
/// TLS context info, used to reconstruct the TLS connector when cancelling a query.
#[derive(Debug, Clone)]
pub struct PostgresCancelContext {
pub ssl_files: PostgresSslFiles,
pub accepts_invalid_certs: bool,
pub verifies_hostname: bool,
pub ssl_mode: SslMode,
}
/// Build a TLS cancel context from the connection URL.
/// Returns None if URL parsing fails or sslmode=disable (no TLS cancel needed).
pub fn build_postgres_cancel_context(url: &str) -> Option<PostgresCancelContext> {
let postgres_url = postgres_connection_url(url).ok()?;
let pg_config = tokio_postgres::Config::from_str(&postgres_url.url).ok()?;
if pg_config.get_ssl_mode() == SslMode::Disable {
return None;
}
Some(PostgresCancelContext {
ssl_files: postgres_url.ssl_files,
accepts_invalid_certs: postgres_url.accepts_invalid_certs,
verifies_hostname: postgres_url.verifies_hostname,
ssl_mode: pg_config.get_ssl_mode(),
})
}
/// Reconstruct a TLS connector from the cancel context, used for TLS connection cancellation.
fn make_rustls_connect_from_context(
ctx: &PostgresCancelContext,
) -> Result<tokio_postgres_rustls::MakeRustlsConnect, String> {
// Build a minimal pg_config solely for ssl_mode determination
let mut pg_config = tokio_postgres::Config::new();
pg_config.ssl_mode(ctx.ssl_mode);
let tls_config = postgres_tls_config(&pg_config, &ctx.ssl_files, ctx.accepts_invalid_certs, ctx.verifies_hostname)?;
Ok(tokio_postgres_rustls::MakeRustlsConnect::new(tls_config))
}
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -857,6 +896,27 @@ struct PostgresConnectionUrl {
verifies_hostname: bool,
}
/// Inject TCP keepalive parameters into the PostgreSQL URL (only when the user has not explicitly specified them).
/// Default parameters shorten half-open connection detection time, suitable for desktop/VPN/NAT environments.
fn inject_postgres_keepalive_params(url: &str) -> String {
let (base, fragment) = url.split_once('#').map_or((url, ""), |(base, fragment)| (base, fragment));
let query = base.split('?').nth(1);
let has_keepalives = query
.map(|q| q.split('&').any(|p| p.split('=').next().is_some_and(|k| k.eq_ignore_ascii_case("keepalives"))))
.unwrap_or(false);
if has_keepalives {
return url.to_string(); // User has explicitly configured keepalive
}
let separator = if base.contains('?') { "&" } else { "?" };
let injected =
format!("{base}{separator}keepalives=1&keepalives_idle=30&keepalives_interval=10&keepalives_retries=3");
if fragment.is_empty() {
injected
} else {
format!("{injected}#{fragment}")
}
}
fn postgres_connection_url(url: &str) -> Result<PostgresConnectionUrl, String> {
let Some(query_start) = url.find('?') else {
let pg_config =
@@ -1144,7 +1204,7 @@ fn validate_postgres_ssl_paths(url: &str) -> Result<(), String> {
}
pub async fn list_databases(pool: &Pool) -> Result<Vec<DatabaseInfo>, String> {
let client = pool.get().await.map_err(|e| e.to_string())?;
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
let stmt = client
.prepare_cached(
"SELECT datname FROM pg_database \
@@ -1176,7 +1236,7 @@ pub async fn list_tables_filtered(
if crate::sql::fuzzy_filter_enabled(filter) { like_fuzzy_pattern(filter) } else { String::new() };
let limit_param = limit.and_then(|value| i64::try_from(value).ok());
let offset_param = offset.and_then(|value| i64::try_from(value).ok()).unwrap_or(0);
let client = pool.get().await.map_err(|e| e.to_string())?;
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
let stmt = client.prepare_cached(postgres_tables_sql()).await.map_err(|e| e.to_string())?;
let rows = client
.query(&stmt, &[&schema, &filter_pattern, &fuzzy_filter_pattern, &limit_param, &offset_param])
@@ -1207,7 +1267,7 @@ pub async fn completion_assistant_search(
request.object_kinds.clone()
};
let pattern = postgres_completion_like_pattern(&request.mask, request.match_mode.as_ref());
let client = pool.get().await.map_err(|e| e.to_string())?;
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
let mut candidates = Vec::new();
if kinds.iter().any(|kind| matches!(kind, CompletionAssistantObjectKind::Schema)) {
@@ -1390,7 +1450,7 @@ fn postgres_completion_like_pattern(value: &str, mode: Option<&CompletionAssista
pub async fn get_table_comment(pool: &Pool, schema: &str, table: &str) -> Result<Option<String>, String> {
let schema = if schema.is_empty() { "public" } else { schema };
let client = pool.get().await.map_err(|e| e.to_string())?;
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
let stmt = client.prepare_cached(postgres_table_comment_sql()).await.map_err(|e| e.to_string())?;
let rows = client.query(&stmt, &[&schema, &table]).await.map_err(|e| e.to_string())?;
Ok(rows.first().and_then(|row| row.try_get::<_, Option<String>>(0).ok().flatten()).filter(|s| !s.is_empty()))
@@ -1532,7 +1592,7 @@ fn list_objects_sql(include_timestamps: bool) -> &'static str {
}
pub async fn list_objects(pool: &Pool, schema: &str) -> Result<Vec<ObjectInfo>, String> {
let client = pool.get().await.map_err(|e| e.to_string())?;
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
let stmt = client.prepare_cached(list_objects_sql(true)).await.map_err(|e| e.to_string())?;
let rows = match client.query(&stmt, &[&schema]).await {
Ok(rows) => rows,
@@ -1559,7 +1619,7 @@ pub async fn list_objects(pool: &Pool, schema: &str) -> Result<Vec<ObjectInfo>,
pub async fn list_object_statistics(pool: &Pool, schema: &str) -> Result<Vec<ObjectStatistics>, String> {
let schema = if schema.is_empty() { "public" } else { schema };
let client = pool.get().await.map_err(|e| e.to_string())?;
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
let stmt = client
.prepare_cached(
"SELECT c.relname, \
@@ -1589,7 +1649,7 @@ pub async fn list_schemas(pool: &Pool) -> Result<Vec<String>, String> {
}
pub async fn list_schema_infos(pool: &Pool) -> Result<Vec<SchemaInfo>, String> {
let client = pool.get().await.map_err(|e| e.to_string())?;
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
let stmt = client
.prepare_cached(
"SELECT n.nspname AS schema_name, d.description AS schema_comment \
@@ -1736,7 +1796,7 @@ async fn get_columns_with_sql(
pub async fn get_columns(pool: &Pool, schema: &str, table: &str) -> Result<Vec<ColumnInfo>, String> {
let schema = if schema.is_empty() { "public" } else { schema };
let client = pool.get().await.map_err(|e| e.to_string())?;
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
match get_columns_with_sql(&client, POSTGRES_COLUMNS_SQL, schema, table).await {
Ok(columns) => Ok(columns),
Err(primary_error) => match get_columns_with_sql(&client, POSTGRES_COLUMNS_COMPAT_SQL, schema, table).await {
@@ -1783,10 +1843,10 @@ pub async fn execute_query_with_max_rows(
let row_limit = query_result_row_limit(max_rows);
if starts_with_executable_sql_keyword(sql, &["SELECT", "SHOW", "EXPLAIN", "WITH", "TABLE"]) {
let client = pool.get().await.map_err(|e| e.to_string())?;
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
execute_select_query(&client, sql, start, row_limit).await
} else {
let client = pool.get().await.map_err(|e| e.to_string())?;
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
let affected = client.execute(sql, &[]).await.map_err(pg_error_to_string)?;
clear_postgres_caches_after_ddl(pool, Some(&client), sql);
@@ -1809,14 +1869,17 @@ pub async fn execute_query_with_max_rows_and_cancel(
sql: &str,
max_rows: Option<usize>,
cancel_token: Option<CancellationToken>,
timeout_duration: Option<Duration>,
budget: DbOperationBudget,
cancel_context: Option<PostgresCancelContext>,
) -> Result<QueryResult, String> {
let client = pool.get().await.map_err(|e| e.to_string())?;
let client = checkout_postgres_client(pool, cancel_token.as_ref(), budget.checkout_timeout).await?;
let pg_cancel_token = client.cancel_token();
wait_postgres_query(
pg_cancel_token,
cancel_context,
cancel_token,
timeout_duration,
budget.query_timeout,
budget.cancel_timeout,
execute_query_with_max_rows_inner(&client, sql, max_rows),
)
.await
@@ -1834,7 +1897,7 @@ pub async fn execute_query_with_schema_and_max_rows(
) -> Result<QueryResult, String> {
let start = Instant::now();
let checkout_start = Instant::now();
let client = pool.get().await.map_err(|e| e.to_string())?;
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
log::info!(
"[postgres][execute_with_schema:pool:done] elapsed_ms={} total_ms={} schema={}",
checkout_start.elapsed().as_millis(),
@@ -1850,10 +1913,13 @@ pub async fn execute_query_with_schema_and_max_rows(
}
let set_schema_start = Instant::now();
client
.execute(&format!("SET search_path TO {}, public", pg_quote_ident(schema)), &[])
.await
.map_err(pg_error_to_string)?;
execute_postgres_infra_statement(
&client,
&format!("SET search_path TO {}, public", pg_quote_ident(schema)),
super::connection_timeout(),
"schema.set",
)
.await?;
log::info!(
"[postgres][execute_with_schema:set-search-path:done] elapsed_ms={} total_ms={}",
set_schema_start.elapsed().as_millis(),
@@ -1872,16 +1938,8 @@ pub async fn execute_query_with_schema_and_max_rows(
result.is_ok()
);
// Always reset search_path so the connection is clean when returned to the pool
let reset_start = Instant::now();
let _ = client.execute("RESET search_path", &[]).await;
log::info!(
"[postgres][execute_with_schema:reset-search-path:done] elapsed_ms={} total_ms={}",
reset_start.elapsed().as_millis(),
start.elapsed().as_millis()
);
result
let reset_result = reset_postgres_search_path(&client, super::connection_timeout(), start).await;
merge_postgres_query_and_reset_result(result, reset_result)
}
pub async fn execute_query_with_schema_and_max_rows_and_cancel(
@@ -1890,11 +1948,12 @@ pub async fn execute_query_with_schema_and_max_rows_and_cancel(
sql: &str,
max_rows: Option<usize>,
cancel_token: Option<CancellationToken>,
timeout_duration: Option<Duration>,
budget: DbOperationBudget,
cancel_context: Option<PostgresCancelContext>,
) -> Result<QueryResult, String> {
let start = Instant::now();
let checkout_start = Instant::now();
let client = pool.get().await.map_err(|e| e.to_string())?;
let client = checkout_postgres_client(pool, cancel_token.as_ref(), budget.checkout_timeout).await?;
log::info!(
"[postgres][execute_with_schema:pool:done] elapsed_ms={} total_ms={} schema={}",
checkout_start.elapsed().as_millis(),
@@ -1909,18 +1968,23 @@ pub async fn execute_query_with_schema_and_max_rows_and_cancel(
let pg_cancel_token = client.cancel_token();
return wait_postgres_query(
pg_cancel_token,
cancel_context,
cancel_token,
timeout_duration,
budget.query_timeout,
budget.cancel_timeout,
execute_query_with_max_rows_inner(&client, sql, max_rows),
)
.await;
}
let set_schema_start = Instant::now();
client
.execute(&format!("SET search_path TO {}, public", pg_quote_ident(schema)), &[])
.await
.map_err(pg_error_to_string)?;
execute_postgres_infra_statement(
&client,
&format!("SET search_path TO {}, public", pg_quote_ident(schema)),
budget.recycle_timeout,
"schema.set",
)
.await?;
log::info!(
"[postgres][execute_with_schema:set-search-path:done] elapsed_ms={} total_ms={}",
set_schema_start.elapsed().as_millis(),
@@ -1931,8 +1995,10 @@ pub async fn execute_query_with_schema_and_max_rows_and_cancel(
let pg_cancel_token = client.cancel_token();
let result = wait_postgres_query(
pg_cancel_token,
cancel_context,
cancel_token,
timeout_duration,
budget.query_timeout,
budget.cancel_timeout,
execute_query_with_max_rows_inner(&client, sql, max_rows),
)
.await;
@@ -1946,21 +2012,84 @@ pub async fn execute_query_with_schema_and_max_rows_and_cancel(
result.is_ok()
);
let reset_start = Instant::now();
let _ = client.execute("RESET search_path", &[]).await;
log::info!(
"[postgres][execute_with_schema:reset-search-path:done] elapsed_ms={} total_ms={}",
reset_start.elapsed().as_millis(),
start.elapsed().as_millis()
);
let reset_result = reset_postgres_search_path(&client, budget.cleanup_timeout, start).await;
merge_postgres_query_and_reset_result(result, reset_result)
}
result
async fn reset_postgres_search_path(
client: &deadpool_postgres::Client,
timeout_duration: Duration,
start: Instant,
) -> Result<(), String> {
let reset_start = Instant::now();
match execute_postgres_infra_statement(client, "RESET search_path", timeout_duration, "schema.reset").await {
Ok(_) => {
log::info!(
"[postgres][execute_with_schema:reset-search-path:done] elapsed_ms={} total_ms={}",
reset_start.elapsed().as_millis(),
start.elapsed().as_millis()
);
Ok(())
}
Err(err) => {
log::warn!(
"[postgres][execute_with_schema:reset-search-path:error] elapsed_ms={} total_ms={} error={}",
reset_start.elapsed().as_millis(),
start.elapsed().as_millis(),
err
);
Err(postgres_schema_reset_cleanup_error(err))
}
}
}
fn merge_postgres_query_and_reset_result(
query_result: Result<QueryResult, String>,
reset_result: Result<(), String>,
) -> Result<QueryResult, String> {
match (query_result, reset_result) {
(Ok(result), Ok(())) => Ok(result),
(Err(query_err), Ok(())) => Err(query_err),
(Ok(_), Err(reset_err)) => Err(reset_err),
(Err(query_err), Err(reset_err)) => Err(format!("{query_err}; {reset_err}")),
}
}
fn postgres_schema_reset_cleanup_error(err: String) -> String {
format!("PostgreSQL schema.reset cleanup failed: {err}")
}
pub(crate) async fn execute_postgres_infra_statement(
client: &deadpool_postgres::Client,
sql: &str,
timeout_duration: Duration,
stage: &str,
) -> Result<u64, String> {
tokio::time::timeout(timeout_duration, client.execute(sql, &[]))
.await
.map_err(|_| format!("PostgreSQL {stage} timed out after {} seconds", timeout_duration.as_secs()))?
.map_err(pg_error_to_string)
}
pub(crate) async fn wait_postgres_operation<T, F>(
pg_cancel_token: tokio_postgres::CancelToken,
cancel_context: Option<PostgresCancelContext>,
timeout_duration: Option<Duration>,
cancel_timeout: Duration,
future: F,
) -> Result<T, String>
where
F: Future<Output = Result<T, String>>,
{
wait_postgres_query(pg_cancel_token, cancel_context, None, timeout_duration, cancel_timeout, future).await
}
async fn wait_postgres_query<T, F>(
pg_cancel_token: tokio_postgres::CancelToken,
cancel_context: Option<PostgresCancelContext>,
cancel_token: Option<CancellationToken>,
timeout_duration: Option<Duration>,
cancel_timeout: Duration,
future: F,
) -> Result<T, String>
where
@@ -1971,13 +2100,13 @@ where
tokio::select! {
biased;
_ = token.cancelled() => {
cancel_postgres_query(pg_cancel_token).await;
cancel_postgres_query(pg_cancel_token, cancel_context.as_ref(), cancel_timeout).await;
Err(crate::query::canceled_error())
}
result = tokio::time::timeout(duration, future) => match result {
Ok(result) => result,
Err(_) => {
cancel_postgres_query(pg_cancel_token).await;
cancel_postgres_query(pg_cancel_token, cancel_context.as_ref(), cancel_timeout).await;
Err(format!("Query timed out after {} seconds", duration.as_secs()))
}
},
@@ -1986,7 +2115,7 @@ where
(None, Some(duration)) => match tokio::time::timeout(duration, future).await {
Ok(result) => result,
Err(_) => {
cancel_postgres_query(pg_cancel_token).await;
cancel_postgres_query(pg_cancel_token, cancel_context.as_ref(), cancel_timeout).await;
Err(format!("Query timed out after {} seconds", duration.as_secs()))
}
},
@@ -1994,7 +2123,7 @@ where
tokio::select! {
biased;
_ = token.cancelled() => {
cancel_postgres_query(pg_cancel_token).await;
cancel_postgres_query(pg_cancel_token, cancel_context.as_ref(), cancel_timeout).await;
Err(crate::query::canceled_error())
}
result = future => result,
@@ -2004,14 +2133,103 @@ where
}
}
async fn cancel_postgres_query(pg_cancel_token: tokio_postgres::CancelToken) {
match tokio::time::timeout(Duration::from_secs(2), pg_cancel_token.cancel_query(NoTls)).await {
/// PostgreSQL pool checkout with timeout and cancel token support.
/// When the checkout phase is stuck, the cancel token can terminate the wait early.
/// The timeout error message includes "checkout timed out" to ensure is_connection_error can classify it correctly.
pub async fn checkout_postgres_client(
pool: &Pool,
cancel_token: Option<&CancellationToken>,
checkout_timeout: Duration,
) -> Result<deadpool_postgres::Object, String> {
let start = Instant::now();
let get_future = async {
tokio::time::timeout(checkout_timeout, pool.get())
.await
.map_err(|_| {
let elapsed = start.elapsed().as_millis();
log::warn!(
"[db:pool.checkout:error] elapsed_ms={} timeout_ms={} error=checkout timed out",
elapsed,
checkout_timeout.as_millis()
);
format!("PostgreSQL connection pool checkout timed out ({}s)", checkout_timeout.as_secs())
})?
.map_err(|e| {
let elapsed = start.elapsed().as_millis();
let err = pg_pool_error_to_string(e);
log::warn!(
"[db:pool.checkout:error] elapsed_ms={} timeout_ms={} error={}",
elapsed,
checkout_timeout.as_millis(),
err
);
format!("PostgreSQL connection pool checkout failed: {err}")
})
};
let result = match cancel_token {
Some(token) => tokio::select! {
biased;
_ = token.cancelled() => {
log::info!(
"[db:pool.checkout:cancelled] elapsed_ms={} timeout_ms={}",
start.elapsed().as_millis(),
checkout_timeout.as_millis()
);
return Err(crate::query::canceled_error());
}
result = get_future => result,
},
None => get_future.await,
};
if result.is_ok() {
log::debug!(
"[db:pool.checkout:done] elapsed_ms={} timeout_ms={}",
start.elapsed().as_millis(),
checkout_timeout.as_millis()
);
}
result
}
async fn cancel_postgres_query(
pg_cancel_token: tokio_postgres::CancelToken,
cancel_context: Option<&PostgresCancelContext>,
cancel_timeout: Duration,
) {
let cancel_timeout = postgres_cancel_attempt_timeout(cancel_timeout, cancel_context);
if let Some(ctx) = cancel_context {
match make_rustls_connect_from_context(ctx) {
Ok(tls) => match tokio::time::timeout(cancel_timeout, pg_cancel_token.cancel_query(tls)).await {
Ok(Ok(())) => return,
Ok(Err(err)) => {
log::warn!("Failed to send PostgreSQL TLS cancel request: {err}");
return;
}
Err(_) => {
log::warn!("Timed out sending PostgreSQL TLS cancel request ({}s)", cancel_timeout.as_secs());
return;
}
},
Err(err) => {
log::warn!("Failed to build TLS connector for cancel: {err}; falling back to NoTls cancel");
}
}
}
match tokio::time::timeout(cancel_timeout, pg_cancel_token.cancel_query(NoTls)).await {
Ok(Ok(())) => {}
Ok(Err(err)) => log::warn!("Failed to send PostgreSQL cancel request: {err}"),
Err(_) => log::warn!("Timed out sending PostgreSQL cancel request"),
Err(_) => log::warn!("Timed out sending PostgreSQL cancel request ({}s)", cancel_timeout.as_secs()),
}
}
fn postgres_cancel_attempt_timeout(
cancel_timeout: Duration,
_cancel_context: Option<&PostgresCancelContext>,
) -> Duration {
cancel_timeout
}
fn is_transaction_recovery_statement(sql: &str) -> bool {
starts_with_executable_sql_keyword(sql, &["ROLLBACK", "ABORT", "COMMIT", "END"])
}
@@ -2140,7 +2358,7 @@ async fn list_indexes_with_sql(
}
pub async fn list_indexes(pool: &Pool, schema: &str, table: &str) -> Result<Vec<IndexInfo>, String> {
let client = pool.get().await.map_err(|e| e.to_string())?;
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
match list_indexes_with_sql(&client, POSTGRES_INDEXES_SQL, schema, table).await {
Ok(indexes) => Ok(indexes),
Err(primary_error) => match list_indexes_with_sql(&client, POSTGRES_INDEXES_COMPAT_SQL, schema, table).await {
@@ -2160,7 +2378,7 @@ pub async fn list_indexes(pool: &Pool, schema: &str, table: &str) -> Result<Vec<
}
pub async fn list_foreign_keys(pool: &Pool, schema: &str, table: &str) -> Result<Vec<ForeignKeyInfo>, String> {
let client = pool.get().await.map_err(|e| e.to_string())?;
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
let stmt = client
.prepare_cached(
"SELECT fk.constraint_name, fk.column_name, \
@@ -2201,7 +2419,7 @@ pub async fn list_foreign_keys(pool: &Pool, schema: &str, table: &str) -> Result
}
pub async fn list_triggers(pool: &Pool, schema: &str, table: &str) -> Result<Vec<TriggerInfo>, String> {
let client = pool.get().await.map_err(|e| e.to_string())?;
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
let stmt = client
.prepare_cached(
"SELECT trigger_name, event_manipulation, action_timing \
@@ -2225,7 +2443,7 @@ pub async fn list_triggers(pool: &Pool, schema: &str, table: &str) -> Result<Vec
}
pub async fn list_functions(pool: &Pool, schema: &str) -> Result<Vec<FunctionInfo>, String> {
let client = pool.get().await.map_err(|e| e.to_string())?;
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
// Use pg_proc + pg_get_functiondef() instead of information_schema.routines
// for reliable function definition retrieval (information_schema.routines.routine_definition
// is NULL for non-SQL functions like plpgsql)
@@ -2267,7 +2485,7 @@ pub async fn list_functions(pool: &Pool, schema: &str) -> Result<Vec<FunctionInf
}
pub async fn list_sequences(pool: &Pool, schema: &str, with_last_values: bool) -> Result<Vec<SequenceInfo>, String> {
let client = pool.get().await.map_err(|e| e.to_string())?;
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
// Use pg_class + pg_sequence + pg_namespace instead of pg_sequences view
// for better compatibility and permission handling
let stmt = client
@@ -2327,7 +2545,7 @@ pub async fn list_sequences(pool: &Pool, schema: &str, with_last_values: bool) -
}
pub async fn list_rules(pool: &Pool, schema: &str) -> Result<Vec<RuleInfo>, String> {
let client = pool.get().await.map_err(|e| e.to_string())?;
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
let stmt = client
.prepare_cached(
"SELECT schemaname, tablename, rulename, definition \
@@ -2350,7 +2568,7 @@ pub async fn list_rules(pool: &Pool, schema: &str) -> Result<Vec<RuleInfo>, Stri
}
pub async fn list_owners(pool: &Pool, schema: &str) -> Result<Vec<OwnerInfo>, String> {
let client = pool.get().await.map_err(|e| e.to_string())?;
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
let stmt = client.prepare_cached(POSTGRES_OWNERS_SQL).await.map_err(|e| e.to_string())?;
let rows = client.query(&stmt, &[&schema]).await.map_err(|e| e.to_string())?;
@@ -2374,14 +2592,14 @@ pub async fn execute_batch(pool: &Pool, statements: &[String]) -> Result<(), Str
if combined.is_empty() {
return Ok(());
}
let client = pool.get().await.map_err(|e| e.to_string())?;
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
client.batch_execute(&combined).await.map_err(pg_error_to_string)?;
clear_postgres_caches_after_ddl(pool, Some(&client), &combined);
Ok(())
}
pub async fn terminate_current_user_database_backends(pool: &Pool, database: &str) -> Result<u64, String> {
let client = pool.get().await.map_err(|e| e.to_string())?;
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
client
.execute(
"SELECT pg_terminate_backend(pid) \
@@ -2417,7 +2635,7 @@ fn invalidates_postgres_statement_cache(sql: &str) -> bool {
/// `COPY table (col1, col2) TO STDOUT (FORMAT CSV, HEADER)`.
/// Returns the raw COPY output bytes.
pub async fn copy_out(pool: &Pool, sql: &str) -> Result<Vec<u8>, String> {
let client = pool.get().await.map_err(|e| e.to_string())?;
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
let stream = client.copy_out(sql).await.map_err(pg_error_to_string)?;
tokio::pin!(stream);
let mut result = Vec::new();
@@ -2431,7 +2649,7 @@ pub async fn copy_out(pool: &Pool, sql: &str) -> Result<Vec<u8>, String> {
/// `COPY table (col1, col2) FROM STDIN (FORMAT CSV)`.
/// `data` is the raw input in the format specified by the COPY command.
pub async fn copy_in(pool: &Pool, sql: &str, data: &[u8]) -> Result<(), String> {
let client = pool.get().await.map_err(|e| e.to_string())?;
let client = checkout_postgres_client(pool, None, super::connection_timeout()).await?;
let sink = client.copy_in::<str, bytes::Bytes>(sql).await.map_err(pg_error_to_string)?;
let mut sink = Box::pin(sink);
sink.as_mut().send(bytes::Bytes::copy_from_slice(data)).await.map_err(pg_error_to_string)?;
@@ -2794,6 +3012,38 @@ mod tests {
assert!(error.contains("sslkey"));
}
#[test]
fn inject_postgres_keepalive_params_preserves_url_fragment() {
let url = "postgres://localhost/app?sslmode=require#read-only";
assert_eq!(
inject_postgres_keepalive_params(url),
"postgres://localhost/app?sslmode=require&keepalives=1&keepalives_idle=30&keepalives_interval=10&keepalives_retries=3#read-only"
);
}
#[test]
fn postgres_cancel_attempt_timeout_is_single_budget() {
assert_eq!(postgres_cancel_attempt_timeout(Duration::from_secs(5), None), Duration::from_secs(5));
assert_eq!(
postgres_cancel_attempt_timeout(
Duration::from_secs(5),
Some(&PostgresCancelContext {
ssl_files: PostgresSslFiles::default(),
accepts_invalid_certs: true,
verifies_hostname: false,
ssl_mode: SslMode::Require,
})
),
Duration::from_secs(5)
);
}
#[test]
fn postgres_cancel_context_omits_disabled_ssl_mode() {
assert!(build_postgres_cancel_context("postgres://localhost/app?sslmode=disable").is_none());
}
#[test]
fn postgres_tls_accepts_invalid_certs_for_require_sslmode() {
let pg_config = tokio_postgres::Config::from_str("postgres://localhost/db?sslmode=require").unwrap();
+1 -1
View File
@@ -211,7 +211,7 @@ pub fn default_idle_timeout_secs() -> u64 {
}
pub fn default_keepalive_interval_secs() -> u64 {
60
30
}
fn default_proxy_port() -> u16 {
+354 -37
View File
@@ -20,7 +20,7 @@ use tokio_util::sync::CancellationToken;
use crate::connection::{AppState, PoolKind};
use crate::database_capabilities;
use crate::db;
use crate::models::connection::DatabaseType;
use crate::models::connection::{ConnectionConfig, DatabaseType};
#[cfg(feature = "duckdb-bundled")]
use crate::sql::starts_with_duckdb_result_sql_keyword;
use crate::sql::{split_sql_batches, split_sql_statements};
@@ -38,6 +38,54 @@ pub enum PoolErrorAction {
ReconnectAndRetry,
}
/// Unified database operation execution budget.
/// query_timeout = None only means SQL execution has no upper limit;
/// checkout/connect/recycle/cancel/cleanup always have hard upper limits and cannot be disabled.
#[derive(Debug, Clone)]
pub struct DbOperationBudget {
pub checkout_timeout: Duration,
pub connect_timeout: Duration,
pub recycle_timeout: Duration,
pub query_timeout: Option<Duration>,
pub cancel_timeout: Duration,
pub cleanup_timeout: Duration,
}
impl DbOperationBudget {
/// Build an execution budget from connection config.
/// checkout/connect/recycle use connect_timeout_secs (clamped to 1s minimum, 300s maximum).
/// query_timeout follows resolve_query_timeout semantics (Some(0) -> None).
/// cancel/cleanup are fixed values and cannot be disabled.
pub fn from_config(connect_timeout_secs: u64, query_timeout_secs: Option<u64>) -> Self {
let infra_timeout = Duration::from_secs(connect_timeout_secs.clamp(1, 300));
Self {
checkout_timeout: infra_timeout,
connect_timeout: infra_timeout,
recycle_timeout: infra_timeout,
query_timeout: resolve_query_timeout(query_timeout_secs),
cancel_timeout: Duration::from_secs(5),
cleanup_timeout: Duration::from_secs(3),
}
}
pub fn from_connection_config(config: &ConnectionConfig) -> Self {
Self::from_config(config.effective_connect_timeout_secs(), Some(config.query_timeout_secs))
}
/// Use global default values (when no connection config is available).
pub fn with_defaults() -> Self {
let default_infra = db::connection_timeout();
Self {
checkout_timeout: default_infra,
connect_timeout: default_infra,
recycle_timeout: default_infra,
query_timeout: Some(QUERY_TIMEOUT),
cancel_timeout: Duration::from_secs(5),
cleanup_timeout: Duration::from_secs(3),
}
}
}
/// Check read-only protection for a connection, blocking write SQL statements.
/// Only clones the connection name when read-only mode is active, avoiding
/// unnecessary allocations otherwise.
@@ -656,6 +704,7 @@ pub fn is_connection_error(err: &str) -> bool {
|| lower.contains("broken pipe")
|| lower.contains("reset by peer")
|| lower.contains("timed out")
|| (lower.contains("pool") && lower.contains("timeout"))
|| lower.contains("closed")
|| lower.contains("关闭的连接")
|| lower.contains("连接已关闭")
@@ -680,6 +729,10 @@ fn is_agent_rpc_timeout_error(lower: &str) -> bool {
lower.starts_with("agent rpc call timed out ")
}
fn is_schema_reset_cleanup_error(lower: &str) -> bool {
lower.contains("schema.reset cleanup failed")
}
fn should_discard_agent_pool_after_error(err: &str) -> bool {
let lower = err.to_lowercase();
is_dbx_query_timeout_error(&lower)
@@ -695,6 +748,7 @@ pub fn pool_error_action(db_type: Option<DatabaseType>, err: &str) -> PoolErrorA
let lower = err.to_lowercase();
if db::sqlserver::is_driver_panic_error(err)
|| (is_dbx_query_timeout_error(&lower) && should_discard_pool_after_query_timeout(db_type))
|| is_schema_reset_cleanup_error(&lower)
|| (db_type.is_some_and(|db_type| database_capabilities::is_agent_type(&db_type))
&& should_discard_agent_pool_after_error(err)
&& !is_connection_error(err))
@@ -824,6 +878,23 @@ fn resolve_query_timeout(timeout_secs: Option<u64>) -> Option<Duration> {
}
}
async fn operation_budget_for_pool_key(
state: &AppState,
pool_key: &str,
query_timeout: Option<Duration>,
) -> DbOperationBudget {
let mut budget = configured_operation_budget_for_pool_key(state, pool_key).await;
budget.query_timeout = query_timeout;
budget
}
async fn configured_operation_budget_for_pool_key(state: &AppState, pool_key: &str) -> DbOperationBudget {
let configs = state.configs.read().await;
crate::connection::config_for_pool_key(pool_key, &configs)
.map(DbOperationBudget::from_connection_config)
.unwrap_or_else(DbOperationBudget::with_defaults)
}
#[allow(clippy::too_many_arguments)]
pub async fn do_execute(
state: &AppState,
@@ -851,6 +922,7 @@ pub async fn do_execute(
let conn_name = config.filter(|c| c.read_only).map(|c| c.name.clone());
(attached, conn_name)
};
let operation_budget = operation_budget_for_pool_key(state, pool_key, query_timeout).await;
if let Some(name) = conn_name_if_readonly {
crate::query_execution_sql::check_read_only(sql, &name)?;
}
@@ -889,7 +961,21 @@ pub async fn do_execute(
let bare = *mode == crate::connection::MysqlMode::Bare;
let max_rows = options.max_rows;
drop(connections);
let mut conn = db::mysql::get_conn_with_health_check(&p).await?;
let mut conn = match db::mysql::get_conn_with_health_check_with_cancel(
&p,
operation_budget.checkout_timeout,
operation_budget.cleanup_timeout,
cancel_token.as_ref(),
)
.await
{
Ok(conn) => conn,
Err(err) if err == QUERY_CANCELED => {
state.remove_pool_by_key(pool_key).await;
return Err(err);
}
Err(err) => return Err(err),
};
let connection_id = conn.id();
if let Some(ref execution_id) = options.execution_id {
let kill_opts = conn.opts().clone();
@@ -913,7 +999,7 @@ pub async fn do_execute(
let p = p.clone();
let schema = schema.map(|s| s.to_string());
let max_rows = options.max_rows;
let query_timeout = query_timeout;
let cancel_context = state.get_postgres_cancel_context(pool_key).await;
drop(connections);
if let Some(schema) = schema {
db::postgres::execute_query_with_schema_and_max_rows_and_cancel(
@@ -922,12 +1008,20 @@ pub async fn do_execute(
sql,
max_rows,
cancel_token,
query_timeout,
operation_budget.clone(),
cancel_context,
)
.await
} else {
db::postgres::execute_query_with_max_rows_and_cancel(&p, sql, max_rows, cancel_token, query_timeout)
.await
db::postgres::execute_query_with_max_rows_and_cancel(
&p,
sql,
max_rows,
cancel_token,
operation_budget.clone(),
cancel_context,
)
.await
}
}
PoolKind::Sqlite(p) => {
@@ -1553,12 +1647,21 @@ async fn execute_multi_mysql(
options: QueryExecutionOptions,
) -> Result<Vec<db::QueryResult>, String> {
let query_timeout = resolve_query_timeout(options.timeout_secs);
let operation_budget = operation_budget_for_pool_key(state, pool_key, query_timeout).await;
let bare = mode == crate::connection::MysqlMode::Bare;
let max_rows = options.max_rows;
let mut conn = match db::mysql::get_conn_with_health_check(pool).await {
let mut conn = match db::mysql::get_conn_with_health_check_with_cancel(
pool,
operation_budget.checkout_timeout,
operation_budget.cleanup_timeout,
cancel_token.as_ref(),
)
.await
{
Ok(conn) => conn,
Err(err) => {
if matches!(pool_error_action(db_type, &err), PoolErrorAction::Discard | PoolErrorAction::ReconnectAndRetry)
|| err == QUERY_CANCELED
{
state.remove_pool_by_key(pool_key).await;
}
@@ -1850,6 +1953,7 @@ pub async fn execute_statements_in_transaction(
let start = std::time::Instant::now();
let db_type = connection_database_type(state, connection_id).await;
let operation_budget = configured_operation_budget_for_pool_key(state, &pool_key).await;
// Clone the pool handle within the lock, then drop it before any async work.
let path = {
@@ -1886,8 +1990,13 @@ pub async fn execute_statements_in_transaction(
};
let result = match path {
Some(TxPath::Pg(pool)) => exec_tx_pg_inner(pool, statements, schema, start).await,
Some(TxPath::Mysql(pool, _bare)) => exec_tx_mysql_inner(pool, statements, start).await,
Some(TxPath::Pg(pool)) => {
let cancel_context = state.get_postgres_cancel_context(&pool_key).await;
exec_tx_pg_inner(pool, statements, schema, start, operation_budget.clone(), cancel_context).await
}
Some(TxPath::Mysql(pool, _bare)) => {
exec_tx_mysql_inner(pool, statements, start, operation_budget.clone()).await
}
Some(TxPath::Sqlite(pool)) => exec_tx_sqlite_inner(pool, statements, start).await,
Some(TxPath::Explicit) => {
let mysql_dialect = connection_mysql_query_dialect(state, connection_id).await;
@@ -1926,24 +2035,41 @@ async fn exec_tx_pg_inner(
statements: &[String],
schema: Option<&str>,
start: std::time::Instant,
budget: DbOperationBudget,
cancel_context: Option<db::postgres::PostgresCancelContext>,
) -> Result<db::QueryResult, String> {
let mut client = pool.get().await.map_err(|e| format!("Failed to acquire connection: {}", e))?;
let mut client = db::postgres::checkout_postgres_client(&pool, None, budget.checkout_timeout)
.await
.map_err(|e| format!("Failed to acquire connection: {}", e))?;
let had_schema = schema.is_some();
if let Some(s) = schema {
client
.execute(&format!("SET search_path TO {}", db::postgres::pg_quote_ident(s)), &[])
.await
.map_err(|e| format!("SET search_path failed: {}", e))?;
db::postgres::execute_postgres_infra_statement(
&client,
&format!("SET search_path TO {}", db::postgres::pg_quote_ident(s)),
budget.recycle_timeout,
"schema.set",
)
.await
.map_err(|e| format!("SET search_path failed: {}", e))?;
}
let tx_result = exec_tx_pg_statements(&mut client, statements).await;
let tx_result = exec_tx_pg_statements(&mut client, statements, &budget, cancel_context).await;
// Always reset search_path so the connection is clean when returned to the pool
if had_schema {
let _ = client.execute("RESET search_path", &[]).await;
}
let reset_result = if had_schema {
db::postgres::execute_postgres_infra_statement(
&client,
"RESET search_path",
budget.cleanup_timeout,
"schema.reset",
)
.await
.map_err(|err| format!("PostgreSQL schema.reset cleanup failed: {err}"))
} else {
Ok(0)
};
match tx_result {
Ok(total_affected) => Ok(db::QueryResult {
match (tx_result, reset_result) {
(Ok(total_affected), Ok(_)) => Ok(db::QueryResult {
columns: vec![],
column_types: Vec::new(),
column_sortables: vec![],
@@ -1954,23 +2080,42 @@ async fn exec_tx_pg_inner(
session_id: None,
has_more: false,
}),
Err(e) => Err(e),
(Err(e), Ok(_)) => Err(e),
(Ok(_), Err(reset_err)) => Err(reset_err),
(Err(e), Err(reset_err)) => Err(format!("{e}; {reset_err}")),
}
}
async fn exec_tx_pg_statements(client: &mut deadpool_postgres::Client, statements: &[String]) -> Result<u64, String> {
let tx = client.transaction().await.map_err(|e| format!("Failed to begin transaction: {}", e))?;
async fn exec_tx_pg_statements(
client: &mut deadpool_postgres::Client,
statements: &[String],
budget: &DbOperationBudget,
cancel_context: Option<db::postgres::PostgresCancelContext>,
) -> Result<u64, String> {
let tx = tokio::time::timeout(budget.recycle_timeout, client.transaction())
.await
.map_err(|_| {
format!("Failed to begin transaction: timed out after {} seconds", budget.recycle_timeout.as_secs())
})?
.map_err(|e| format!("Failed to begin transaction: {}", e))?;
let mut total_affected: u64 = 0;
for (i, sql) in statements.iter().enumerate() {
match tx.execute(sql, &[]).await {
Ok(affected) => total_affected += affected,
Err(e) => {
// Transaction auto-rollbacks on drop
return Err(format!("Statement {} failed: {}", i + 1, e));
}
}
let pg_cancel_token = tx.client().cancel_token();
let affected = db::postgres::wait_postgres_operation(
pg_cancel_token,
cancel_context.clone(),
budget.query_timeout,
budget.cancel_timeout,
async { tx.execute(sql, &[]).await.map_err(|e| e.to_string()) },
)
.await
.map_err(|e| format!("Statement {} failed: {}", i + 1, e))?;
total_affected += affected;
}
tx.commit().await.map_err(|e| format!("COMMIT failed: {}", e))?;
tokio::time::timeout(budget.cleanup_timeout, tx.commit())
.await
.map_err(|_| format!("COMMIT timed out after {} seconds", budget.cleanup_timeout.as_secs()))?
.map_err(|e| format!("COMMIT failed: {}", e))?;
Ok(total_affected)
}
@@ -1978,20 +2123,28 @@ async fn exec_tx_mysql_inner(
pool: mysql_async::Pool,
statements: &[String],
start: std::time::Instant,
budget: DbOperationBudget,
) -> Result<db::QueryResult, String> {
let mut conn = db::mysql::get_conn_with_health_check(&pool).await?;
conn.query_drop("START TRANSACTION").await.map_err(|e| format!("Failed to begin transaction: {}", e))?;
let mut conn = db::mysql::get_conn_with_health_check_with_timeout(&pool, budget.checkout_timeout).await?;
mysql_query_drop_with_timeout(
&mut conn,
"START TRANSACTION",
budget.recycle_timeout,
"Failed to begin transaction",
)
.await?;
let mut total_affected: u64 = 0;
for (i, sql) in statements.iter().enumerate() {
match conn.query_iter(sql).await {
Ok(result) => total_affected += result.affected_rows(),
match mysql_query_iter_with_timeout(&mut conn, sql, budget.query_timeout).await {
Ok(affected) => total_affected += affected,
Err(e) => {
let _ = conn.query_drop("ROLLBACK").await;
let _ = mysql_query_drop_with_timeout(&mut conn, "ROLLBACK", budget.cleanup_timeout, "ROLLBACK failed")
.await;
return Err(format!("Statement {} failed: {}", i + 1, e));
}
}
}
conn.query_drop("COMMIT").await.map_err(|e| format!("COMMIT failed: {}", e))?;
mysql_query_drop_with_timeout(&mut conn, "COMMIT", budget.cleanup_timeout, "COMMIT failed").await?;
Ok(db::QueryResult {
columns: vec![],
column_types: Vec::new(),
@@ -2005,6 +2158,33 @@ async fn exec_tx_mysql_inner(
})
}
async fn mysql_query_drop_with_timeout(
conn: &mut mysql_async::Conn,
sql: &str,
timeout_duration: Duration,
context: &str,
) -> Result<(), String> {
tokio::time::timeout(timeout_duration, conn.query_drop(sql))
.await
.map_err(|_| format!("{context}: timed out after {} seconds", timeout_duration.as_secs()))?
.map_err(|e| format!("{context}: {e}"))
}
async fn mysql_query_iter_with_timeout(
conn: &mut mysql_async::Conn,
sql: &str,
timeout_duration: Option<Duration>,
) -> Result<u64, String> {
match timeout_duration {
Some(timeout_duration) => tokio::time::timeout(timeout_duration, conn.query_iter(sql))
.await
.map_err(|_| format!("Query timed out after {} seconds", timeout_duration.as_secs()))?
.map(|result| result.affected_rows())
.map_err(|e| e.to_string()),
None => conn.query_iter(sql).await.map(|result| result.affected_rows()).map_err(|e| e.to_string()),
}
}
async fn exec_tx_sqlite_inner(
pool: db::sqlite::SqliteHandle,
statements: &[String],
@@ -2175,6 +2355,55 @@ mod tests {
use super::*;
use crate::models::connection::{default_redis_key_separator, ConnectionConfig, DatabaseType};
fn test_connection_config(db_type: DatabaseType) -> ConnectionConfig {
ConnectionConfig {
id: "conn-1".to_string(),
name: "Connection".to_string(),
db_type,
driver_profile: None,
driver_label: None,
url_params: None,
host: "localhost".to_string(),
port: 0,
username: String::new(),
password: String::new(),
database: None,
visible_databases: None,
visible_schemas: None,
attached_databases: Vec::new(),
color: None,
transport_layers: Vec::new(),
connect_timeout_secs: 10,
query_timeout_secs: 30,
idle_timeout_secs: 60,
keepalive_interval_secs: 30,
ssl: false,
ca_cert_path: String::new(),
client_cert_path: String::new(),
client_key_path: String::new(),
sysdba: false,
oracle_connection_type: None,
connection_string: None,
redis_connection_mode: None,
redis_sentinel_master: String::new(),
redis_sentinel_nodes: String::new(),
redis_sentinel_username: String::new(),
redis_sentinel_password: String::new(),
redis_sentinel_tls: false,
redis_cluster_nodes: String::new(),
redis_key_separator: default_redis_key_separator(),
redis_scan_page_size: None,
etcd_endpoints: String::new(),
gbase_server: String::new(),
informix_server: String::new(),
external_config: None,
jdbc_driver_class: None,
jdbc_driver_paths: Vec::new(),
one_time: false,
read_only: false,
}
}
#[tokio::test]
async fn wait_for_query_returns_cancelled_when_token_is_cancelled() {
let token = CancellationToken::new();
@@ -2252,6 +2481,71 @@ mod tests {
assert_eq!(follow_up.rows, vec![vec![serde_json::json!(1)]]);
}
#[test]
fn db_operation_budget_from_config() {
let budget = DbOperationBudget::from_config(10, Some(30));
assert_eq!(budget.checkout_timeout, Duration::from_secs(10));
assert_eq!(budget.connect_timeout, Duration::from_secs(10));
assert_eq!(budget.recycle_timeout, Duration::from_secs(10));
assert_eq!(budget.query_timeout, Some(Duration::from_secs(30)));
assert_eq!(budget.cancel_timeout, Duration::from_secs(5));
assert_eq!(budget.cleanup_timeout, Duration::from_secs(3));
}
#[test]
fn db_operation_budget_from_connection_config_uses_connection_settings() {
let mut config = test_connection_config(DatabaseType::Postgres);
config.connect_timeout_secs = 12;
config.query_timeout_secs = 0;
let budget = DbOperationBudget::from_connection_config(&config);
assert_eq!(budget.checkout_timeout, Duration::from_secs(12));
assert_eq!(budget.connect_timeout, Duration::from_secs(12));
assert_eq!(budget.recycle_timeout, Duration::from_secs(12));
assert_eq!(budget.query_timeout, None);
assert_eq!(budget.cancel_timeout, Duration::from_secs(5));
assert_eq!(budget.cleanup_timeout, Duration::from_secs(3));
}
#[test]
fn db_operation_budget_query_timeout_zero_means_no_limit() {
let budget = DbOperationBudget::from_config(10, Some(0));
assert_eq!(budget.query_timeout, None);
// Infrastructure timeouts still have hard limits
assert_eq!(budget.checkout_timeout, Duration::from_secs(10));
assert_eq!(budget.cancel_timeout, Duration::from_secs(5));
}
#[test]
fn db_operation_budget_query_timeout_zero_keeps_transaction_infra_limits() {
let mut config = test_connection_config(DatabaseType::Mysql);
config.connect_timeout_secs = 7;
config.query_timeout_secs = 0;
let budget = DbOperationBudget::from_connection_config(&config);
assert_eq!(budget.query_timeout, None);
assert_eq!(budget.checkout_timeout, Duration::from_secs(7));
assert_eq!(budget.recycle_timeout, Duration::from_secs(7));
assert_eq!(budget.cleanup_timeout, Duration::from_secs(3));
}
#[test]
fn db_operation_budget_clamps_infra_timeout() {
let budget = DbOperationBudget::from_config(0, Some(30));
assert_eq!(budget.checkout_timeout, Duration::from_secs(1)); // clamped to min 1s
let budget = DbOperationBudget::from_config(600, Some(30));
assert_eq!(budget.checkout_timeout, Duration::from_secs(300)); // clamped to max 300s
}
#[test]
fn db_operation_budget_with_defaults() {
let budget = DbOperationBudget::with_defaults();
assert_eq!(budget.checkout_timeout, db::connection_timeout());
assert_eq!(budget.query_timeout, Some(QUERY_TIMEOUT));
}
#[test]
fn is_connection_error_detects_english_messages() {
assert!(is_connection_error("connection reset"));
@@ -2300,6 +2594,20 @@ mod tests {
assert!(!is_connection_error("os error 13"));
}
#[test]
fn is_connection_error_detects_deadpool_pool_timeouts() {
// deadpool-postgres PoolError::Timeout messages (contain "pool" + "timeout" but not "timed out")
assert!(is_connection_error("pool wait timeout"));
assert!(is_connection_error("pool create timeout"));
assert!(is_connection_error("pool recycle timeout"));
// checkout helper timeout messages
assert!(is_connection_error("PostgreSQL connection pool checkout timed out (5s)"));
assert!(is_connection_error("MySQL get connection timed out"));
assert!(is_connection_error("MySQL ping timed out"));
assert!(is_connection_error("MySQL kill connection checkout timed out"));
assert!(is_connection_error("MySQL KILL QUERY timed out"));
}
#[test]
fn pool_error_action_discards_sqlserver_driver_panic_without_retry() {
let err = format!("{} the current client will be rebuilt.", db::sqlserver::SQLSERVER_DRIVER_PANIC_ERROR_PREFIX);
@@ -2322,6 +2630,15 @@ mod tests {
assert_eq!(pool_error_action(Some(DatabaseType::DuckDb), err), PoolErrorAction::Keep);
}
#[test]
fn pool_error_action_discards_schema_reset_cleanup_without_retry() {
let err = "PostgreSQL schema.reset cleanup failed: PostgreSQL schema.reset timed out after 3 seconds";
assert_eq!(pool_error_action(Some(DatabaseType::Postgres), err), PoolErrorAction::Discard);
assert_eq!(pool_error_action(Some(DatabaseType::OpenGauss), err), PoolErrorAction::Discard);
assert!(should_discard_pool_after_error(Some(DatabaseType::Postgres), err));
}
#[test]
fn pool_error_action_reconnects_connection_errors() {
let err = "connection reset by peer";
+1 -1
View File
@@ -3186,7 +3186,7 @@ async fn mysql_object_source(
) -> Result<String, String> {
use mysql_async::prelude::*;
let sql = mysql_object_source_sql(name, kind);
let mut conn = pool.get_conn().await.map_err(|e| e.to_string())?;
let mut conn = db::mysql::get_conn_with_timeout(pool, db::connection_timeout()).await?;
let result = conn.query_iter(&sql).await.map_err(|e| e.to_string())?;
let rows: Vec<mysql_async::Row> = result.collect_and_drop().await.map_err(|e| e.to_string())?;
let row = rows.first().ok_or("Object source not found")?;