fix(metadata): bound cache maintenance and pool recovery

This commit is contained in:
miracle authored and GitHub committed 2026-08-19 11:10:12 +08:00
1 parent 21af5cf600
commit 8c2b232243
41 files changed
+2747 -350

No files matched your search

+57
View File
@@ -52,6 +52,7 @@ pub mod victoriametrics_driver;
pub mod wkb;
use reqwest::ClientBuilder;
use std::fmt;
use std::future::Future;
use std::time::Duration;
@@ -62,6 +63,62 @@ pub use file_validator::validate_file_path;
pub const CONNECTION_TIMEOUT_SECS: u64 = 5;
pub const TCP_PROBE_TIMEOUT_SECS: u64 = 3;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PoolCheckoutStage {
Wait,
Create,
Recycle,
Unknown,
}
impl PoolCheckoutStage {
pub fn as_str(self) -> &'static str {
match self {
Self::Wait => "wait",
Self::Create => "create",
Self::Recycle => "recycle",
Self::Unknown => "unknown",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum PoolCheckoutError {
Timeout { database: &'static str, stage: PoolCheckoutStage, timeout: Duration },
Failed { database: &'static str, stage: PoolCheckoutStage, detail: String },
Canceled,
}
impl PoolCheckoutError {
pub fn stage(&self) -> Option<PoolCheckoutStage> {
match self {
Self::Timeout { stage, .. } | Self::Failed { stage, .. } => Some(*stage),
Self::Canceled => None,
}
}
pub fn is_pool_saturation(&self) -> bool {
matches!(self, Self::Timeout { stage: PoolCheckoutStage::Wait, .. })
}
}
impl fmt::Display for PoolCheckoutError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Timeout { database, stage, timeout } => write!(
formatter,
"{database} connection pool checkout timed out [stage={}, timeout_ms={}]",
stage.as_str(),
timeout.as_millis()
),
Self::Failed { database, stage, detail } => {
write!(formatter, "{database} connection pool checkout failed [stage={}]: {detail}", stage.as_str())
}
Self::Canceled => formatter.write_str(crate::query::QUERY_CANCELED),
}
}
}
pub fn connection_timeout() -> Duration {
Duration::from_secs(CONNECTION_TIMEOUT_SECS)
}
+270 -25
View File
@@ -9,7 +9,9 @@ use sqlparser::dialect::MySqlDialect;
use sqlparser::parser::Parser;
use std::borrow::Cow;
use std::collections::{HashMap, HashSet};
#[cfg(test)]
use std::future::Future;
use std::ops::Deref;
use std::path::PathBuf;
use std::sync::atomic::AtomicBool;
use std::time::Duration;
@@ -28,7 +30,62 @@ use crate::types::{
use super::file_validator::validate_file_path;
pub type MySqlPool = mysql_async::Pool;
/// DBX-owned MySQL pool handle. The driver does not expose its configured
/// maximum, so retain that value beside the driver pool for checkout phase
/// classification instead of guessing it from aggregate metrics.
#[derive(Debug, Clone)]
pub struct MySqlPool {
inner: mysql_async::Pool,
max_connections: usize,
}
impl MySqlPool {
pub(crate) fn new<O>(opts: O, max_connections: usize) -> Self
where
mysql_async::Opts: TryFrom<O>,
<mysql_async::Opts as TryFrom<O>>::Error: std::error::Error,
{
Self { inner: mysql_async::Pool::new(opts), max_connections: max_connections.max(1) }
}
pub async fn disconnect(self) -> Result<(), mysql_async::Error> {
self.inner.disconnect().await
}
}
impl Deref for MySqlPool {
type Target = mysql_async::Pool;
fn deref(&self) -> &Self::Target {
&self.inner
}
}
#[doc(hidden)]
pub trait MySqlPoolAccess {
fn driver_pool(&self) -> &mysql_async::Pool;
fn checkout_max_connections(&self) -> Option<usize>;
}
impl MySqlPoolAccess for MySqlPool {
fn driver_pool(&self) -> &mysql_async::Pool {
&self.inner
}
fn checkout_max_connections(&self) -> Option<usize> {
Some(self.max_connections)
}
}
impl MySqlPoolAccess for mysql_async::Pool {
fn driver_pool(&self) -> &mysql_async::Pool {
self
}
fn checkout_max_connections(&self) -> Option<usize> {
None
}
}
const MYSQL_TCP_KEEPALIVE_MS: u32 = 30_000;
const MYSQL_SQL_PACKET_MARGIN_MAX_BYTES: usize = 64 * 1024;
@@ -1242,7 +1299,7 @@ fn create_pool(
// to paths explicitly supplied by the user instead of enabling arbitrary reads.
builder = builder.local_infile_handler(Some(mysql_async::WhiteListFsHandler::new(local_infile_paths)));
}
Ok(MySqlPool::new(builder))
Ok(MySqlPool::new(builder, max_connections))
}
fn mysql_async_tcp_host(host: &str) -> &str {
@@ -3603,7 +3660,10 @@ fn parse_mysql_enum_values(column_type: &str) -> Option<Vec<String>> {
}
}
pub async fn get_columns(pool: &MySqlPool, database: &str, table: &str) -> Result<Vec<ColumnInfo>, String> {
pub async fn get_columns<P>(pool: &P, database: &str, table: &str) -> Result<Vec<ColumnInfo>, String>
where
P: MySqlPoolAccess + ?Sized,
{
let sql = columns_sql(database, table);
let mut conn = get_conn_with_health_check(pool).await?;
let result = match conn.query_iter(&sql).await {
@@ -3675,7 +3735,10 @@ pub async fn get_columns(pool: &MySqlPool, database: &str, table: &str) -> Resul
Ok(columns)
}
pub async fn get_columns_show(pool: &MySqlPool, database: &str, table: &str) -> Result<Vec<ColumnInfo>, String> {
pub async fn get_columns_show<P>(pool: &P, database: &str, table: &str) -> Result<Vec<ColumnInfo>, String>
where
P: MySqlPoolAccess + ?Sized,
{
let sql = show_columns_sql(database, table, true);
let mut conn = get_conn_with_health_check(pool).await?;
let rows: Vec<mysql_async::Row> = match conn.query_iter(&sql).await {
@@ -3852,23 +3915,32 @@ pub(super) fn skip_mysql_quoted(sql: &str, start: usize, quote: u8) -> usize {
/// Get a connection from the pool with a health check. If the connection is dead
/// (e.g. after app was backgrounded), it tries again with a fresh connection.
pub async fn get_conn_with_health_check(pool: &MySqlPool) -> Result<mysql_async::Conn, String> {
pub async fn get_conn_with_health_check<P>(pool: &P) -> Result<mysql_async::Conn, String>
where
P: MySqlPoolAccess + ?Sized,
{
get_conn_with_health_check_with_timeout(pool, super::connection_timeout()).await
}
pub async fn get_conn_with_health_check_with_timeout(
pool: &MySqlPool,
pub async fn get_conn_with_health_check_with_timeout<P>(
pool: &P,
timeout: Duration,
) -> Result<mysql_async::Conn, String> {
) -> Result<mysql_async::Conn, String>
where
P: MySqlPoolAccess + ?Sized,
{
get_conn_with_health_check_with_cancel(pool, timeout, timeout, None).await
}
pub async fn get_conn_with_health_check_with_cancel(
pool: &MySqlPool,
pub async fn get_conn_with_health_check_with_cancel<P>(
pool: &P,
timeout: Duration,
cleanup_timeout: Duration,
cancel_token: Option<&CancellationToken>,
) -> Result<mysql_async::Conn, String> {
) -> Result<mysql_async::Conn, String>
where
P: MySqlPoolAccess + ?Sized,
{
let start = Instant::now();
let mut conn = get_conn_with_timeout_and_cancel(pool, timeout, cancel_token).await?;
match ping_conn_with_timeout_and_cancel(&mut conn, timeout, cancel_token).await {
@@ -3909,13 +3981,67 @@ pub async fn get_conn_with_health_check_with_cancel(
}
}
async fn get_conn_with_timeout_and_cancel(
pool: &MySqlPool,
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct MysqlCheckoutSnapshot {
pub(crate) connection_count: usize,
pub(crate) connections_in_pool: usize,
pub(crate) max_connections: usize,
}
pub(crate) type MysqlCheckoutStage = super::PoolCheckoutStage;
pub(crate) fn classify_mysql_checkout_stage(snapshot: MysqlCheckoutSnapshot) -> MysqlCheckoutStage {
if snapshot.connections_in_pool > 0 {
MysqlCheckoutStage::Recycle
} else if snapshot.connection_count < snapshot.max_connections {
MysqlCheckoutStage::Create
} else {
MysqlCheckoutStage::Wait
}
}
fn mysql_checkout_snapshot<P>(pool: &P, max_connections: usize) -> MysqlCheckoutSnapshot
where
P: MySqlPoolAccess + ?Sized,
{
let metrics = pool.driver_pool().metrics();
use std::sync::atomic::Ordering;
MysqlCheckoutSnapshot {
connection_count: metrics.connection_count.load(Ordering::Relaxed),
connections_in_pool: metrics.connections_in_pool.load(Ordering::Relaxed),
max_connections,
}
}
pub(crate) async fn checkout_mysql_conn<P>(
pool: &P,
timeout: Duration,
) -> Result<mysql_async::Conn, super::PoolCheckoutError>
where
P: MySqlPoolAccess + ?Sized,
{
// Capture the phase before entering the driver future. The phase is tied
// to this checkout's capacity decision, not to a later aggregate metric
// snapshot that may already include another request.
let stage = pool.checkout_max_connections().map_or(MysqlCheckoutStage::Unknown, |max_connections| {
classify_mysql_checkout_stage(mysql_checkout_snapshot(pool, max_connections))
});
match tokio::time::timeout(timeout, pool.driver_pool().get_conn()).await {
Ok(Ok(conn)) => Ok(conn),
Ok(Err(error)) => Err(super::PoolCheckoutError::Failed { database: "MySQL", stage, detail: error.to_string() }),
Err(_) => Err(super::PoolCheckoutError::Timeout { database: "MySQL", stage, timeout }),
}
}
async fn get_conn_with_timeout_and_cancel<P>(
pool: &P,
timeout: Duration,
cancel_token: Option<&CancellationToken>,
) -> Result<mysql_async::Conn, String> {
let get_future =
connection_result_with_timeout(timeout, async { pool.get_conn().await.map_err(|error| error.to_string()) });
) -> Result<mysql_async::Conn, String>
where
P: MySqlPoolAccess + ?Sized,
{
let get_future = async { checkout_mysql_conn(pool, timeout).await.map_err(|error| error.to_string()) };
match cancel_token {
Some(token) => {
@@ -3929,10 +4055,14 @@ async fn get_conn_with_timeout_and_cancel(
}
}
pub async fn get_conn_with_timeout(pool: &MySqlPool, timeout: Duration) -> Result<mysql_async::Conn, String> {
connection_result_with_timeout(timeout, async { pool.get_conn().await.map_err(|error| error.to_string()) }).await
pub async fn get_conn_with_timeout<P>(pool: &P, timeout: Duration) -> Result<mysql_async::Conn, String>
where
P: MySqlPoolAccess + ?Sized,
{
checkout_mysql_conn(pool, timeout).await.map_err(|error| error.to_string())
}
#[cfg(test)]
async fn connection_result_with_timeout<T, F>(timeout: Duration, future: F) -> Result<T, String>
where
F: Future<Output = Result<T, String>>,
@@ -4564,13 +4694,16 @@ pub(crate) fn mysql_sql_statement_hard_limit(max_allowed_packet: u64) -> Option<
packet_bytes.checked_sub(margin).filter(|limit| *limit > 0)
}
pub async fn execute_query_with_max_rows(
pool: &MySqlPool,
pub async fn execute_query_with_max_rows<P>(
pool: &P,
sql: &str,
bare: bool,
max_rows: Option<usize>,
dialect: MySqlQueryDialect,
) -> Result<QueryResult, String> {
) -> Result<QueryResult, String>
where
P: MySqlPoolAccess + ?Sized,
{
let mut conn = get_conn_with_health_check(pool).await?;
execute_query_on_conn_with_max_rows(&mut conn, sql, bare, max_rows, dialect).await
}
@@ -5357,10 +5490,10 @@ mod tests {
tokio::time::sleep(Duration::from_millis(10)).await;
Ok::<_, String>("connection")
});
let configured_five = connection_result_with_timeout(Duration::from_millis(50), async {
tokio::time::sleep(Duration::from_millis(60)).await;
Ok::<_, String>("connection")
});
let configured_five = connection_result_with_timeout(
Duration::from_millis(10),
std::future::pending::<Result<&'static str, String>>(),
);
let (exact, adjacent, configured_five) = tokio::join!(exact, adjacent, configured_five);
@@ -5369,6 +5502,118 @@ mod tests {
assert_eq!(configured_five, Err("MySQL get connection timed out".to_string()));
}
#[tokio::test]
async fn mysql_checkout_fault_injection_classifies_full_pool_as_wait() {
assert!(tokio::time::timeout(Duration::from_millis(1), std::future::pending::<()>()).await.is_err());
assert_eq!(
classify_mysql_checkout_stage(MysqlCheckoutSnapshot {
connection_count: 1,
connections_in_pool: 0,
max_connections: 1,
}),
MysqlCheckoutStage::Wait
);
}
#[tokio::test]
async fn mysql_checkout_fault_injection_classifies_hung_connection_create() {
assert!(tokio::time::timeout(Duration::from_millis(1), std::future::pending::<()>()).await.is_err());
assert_eq!(
classify_mysql_checkout_stage(MysqlCheckoutSnapshot {
connection_count: 0,
connections_in_pool: 0,
max_connections: 2,
}),
MysqlCheckoutStage::Create
);
}
#[tokio::test]
async fn mysql_checkout_fault_injection_preserves_create_stage_when_handshake_hangs() {
let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)).await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (_socket, _) = listener.accept().await.unwrap();
std::future::pending::<()>().await;
});
let pool_options =
mysql_async::PoolOpts::new().with_constraints(mysql_async::PoolConstraints::new(1, 2).unwrap());
let options = mysql_async::OptsBuilder::default()
.ip_or_hostname(address.ip().to_string())
.tcp_port(address.port())
.user(Some("fault-injection"))
.pass(Some("fault-injection"))
.pool_opts(Some(pool_options));
let pool = MySqlPool::new(options, 2);
let error = checkout_mysql_conn(&pool, Duration::from_millis(50)).await.unwrap_err();
assert!(matches!(error, super::super::PoolCheckoutError::Timeout { stage: MysqlCheckoutStage::Create, .. }));
server.abort();
let _ = tokio::time::timeout(Duration::from_secs(1), pool.disconnect()).await;
}
#[tokio::test]
async fn mysql_checkout_raw_driver_pool_timeout_keeps_stage_unknown() {
let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)).await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (_socket, _) = listener.accept().await.unwrap();
std::future::pending::<()>().await;
});
let options = mysql_async::OptsBuilder::default()
.ip_or_hostname(address.ip().to_string())
.tcp_port(address.port())
.user(Some("fault-injection"))
.pass(Some("fault-injection"));
let pool = mysql_async::Pool::new(options);
let error = checkout_mysql_conn(&pool, Duration::from_millis(50)).await.unwrap_err();
assert!(matches!(error, super::super::PoolCheckoutError::Timeout { stage: MysqlCheckoutStage::Unknown, .. }));
server.abort();
let _ = tokio::time::timeout(Duration::from_secs(1), pool.disconnect()).await;
}
#[tokio::test]
#[ignore = "requires DBX_REVIEW_MYSQL_* environment variables"]
async fn mysql_checkout_fault_injection_preserves_wait_stage_when_real_pool_is_full() {
let host = std::env::var("DBX_REVIEW_MYSQL_HOST").expect("DBX_REVIEW_MYSQL_HOST is required");
let port = std::env::var("DBX_REVIEW_MYSQL_PORT").expect("DBX_REVIEW_MYSQL_PORT is required");
let user = std::env::var("DBX_REVIEW_MYSQL_USER").expect("DBX_REVIEW_MYSQL_USER is required");
let password = std::env::var("DBX_REVIEW_MYSQL_PASSWORD").expect("DBX_REVIEW_MYSQL_PASSWORD is required");
let database = std::env::var("DBX_REVIEW_MYSQL_DATABASE").expect("DBX_REVIEW_MYSQL_DATABASE is required");
let pool_options =
mysql_async::PoolOpts::new().with_constraints(mysql_async::PoolConstraints::new(1, 1).unwrap());
let options = mysql_async::OptsBuilder::default()
.ip_or_hostname(host)
.tcp_port(port.parse().expect("DBX_REVIEW_MYSQL_PORT must be a valid port"))
.user(Some(user))
.pass(Some(password))
.db_name(Some(database))
.pool_opts(Some(pool_options));
let pool = MySqlPool::new(options, 1);
let held_connection = tokio::time::timeout(Duration::from_secs(10), pool.get_conn())
.await
.expect("real MySQL checkout timed out")
.expect("real MySQL checkout failed");
let error = checkout_mysql_conn(&pool, Duration::from_millis(50)).await.unwrap_err();
assert!(matches!(error, super::super::PoolCheckoutError::Timeout { stage: MysqlCheckoutStage::Wait, .. }));
drop(held_connection);
let _ = tokio::time::timeout(Duration::from_secs(2), pool.disconnect()).await;
}
#[test]
fn mysql_public_metadata_and_query_helpers_accept_driver_pool_for_compatibility() {
let pool = mysql_async::Pool::new(mysql_async::OptsBuilder::default());
let columns = get_columns(&pool, "database", "table");
let query = execute_query_with_max_rows(&pool, "SELECT 1", false, Some(1), MySqlQueryDialect::default());
drop((columns, query));
}
#[tokio::test]
async fn mysql_database_list_timeout_preserves_immediate_errors_and_fallback_plan() {
let error =
+105 -26
View File
@@ -5900,40 +5900,70 @@ async fn execute_postgres_user_query(
/// PostgreSQL pool checkout with timeout and cancel token support.
/// When the checkout phase is stuck, the cancel token can terminate the wait early.
/// The timeout error message includes "checkout timed out" to ensure is_connection_error can classify it correctly.
pub async fn checkout_postgres_client(
pub(crate) type PostgresCheckoutStage = super::PoolCheckoutStage;
pub(crate) fn classify_postgres_checkout_stage(status: deadpool_postgres::Status) -> PostgresCheckoutStage {
if status.waiting > 0 || (status.available == 0 && status.size >= status.max_size) {
PostgresCheckoutStage::Wait
} else if status.available > 0 {
PostgresCheckoutStage::Recycle
} else if status.size < status.max_size {
PostgresCheckoutStage::Create
} else {
PostgresCheckoutStage::Unknown
}
}
fn postgres_pool_error_stage(error: &PoolError) -> PostgresCheckoutStage {
match error {
PoolError::Timeout(deadpool_postgres::TimeoutType::Wait) => PostgresCheckoutStage::Wait,
PoolError::Timeout(deadpool_postgres::TimeoutType::Create) => PostgresCheckoutStage::Create,
PoolError::Timeout(deadpool_postgres::TimeoutType::Recycle) => PostgresCheckoutStage::Recycle,
PoolError::Backend(_) | PoolError::PostCreateHook(_) => PostgresCheckoutStage::Create,
PoolError::Closed | PoolError::NoRuntimeSpecified => PostgresCheckoutStage::Unknown,
}
}
pub(crate) async fn checkout_postgres_client_classified(
pool: &Pool,
cancel_token: Option<&CancellationToken>,
checkout_timeout: Duration,
) -> Result<deadpool_postgres::Object, String> {
let checkout_timeout = effective_postgres_checkout_timeout(pool, checkout_timeout);
) -> Result<deadpool_postgres::Object, super::PoolCheckoutError> {
let start = Instant::now();
let get_future = async {
tokio::time::timeout(checkout_timeout, pool.get())
.await
.map_err(|_| {
let elapsed = start.elapsed().as_millis();
match tokio::time::timeout(checkout_timeout, pool.get()).await {
Ok(Ok(client)) => Ok(client),
Ok(Err(error)) => {
let stage = postgres_pool_error_stage(&error);
let timed_out = matches!(&error, PoolError::Timeout(_));
let detail = pg_pool_error_to_string(error);
log::warn!(
"[db:pool.checkout:error] elapsed_ms={} timeout_ms={} error=checkout timed out",
elapsed,
checkout_timeout.as_millis()
);
format!("PostgreSQL connection pool checkout timed out ({}s)", checkout_timeout.as_secs())
})?
.map_err(|e| {
let elapsed = start.elapsed().as_millis();
let err = pg_pool_error_to_string(e);
log::warn!(
"[db:pool.checkout:error] elapsed_ms={} timeout_ms={} error={}",
elapsed,
"[db:pool.checkout:error] elapsed_ms={} timeout_ms={} stage={} error={}",
start.elapsed().as_millis(),
checkout_timeout.as_millis(),
err
stage.as_str(),
detail
);
format!("PostgreSQL connection pool checkout failed: {err}")
})
if timed_out {
Err(super::PoolCheckoutError::Timeout { database: "PostgreSQL", stage, timeout: checkout_timeout })
} else {
Err(super::PoolCheckoutError::Failed { database: "PostgreSQL", stage, detail })
}
}
Err(_) => {
let stage = classify_postgres_checkout_stage(pool.status());
log::warn!(
"[db:pool.checkout:error] elapsed_ms={} timeout_ms={} stage={} error=checkout timed out",
start.elapsed().as_millis(),
checkout_timeout.as_millis(),
stage.as_str()
);
Err(super::PoolCheckoutError::Timeout { database: "PostgreSQL", stage, timeout: checkout_timeout })
}
}
};
let result = match cancel_token {
match cancel_token {
Some(token) => tokio::select! {
biased;
_ = token.cancelled() => {
@@ -5942,12 +5972,24 @@ pub async fn checkout_postgres_client(
start.elapsed().as_millis(),
checkout_timeout.as_millis()
);
return Err(crate::query::canceled_error());
Err(super::PoolCheckoutError::Canceled)
}
result = get_future => result,
},
None => get_future.await,
};
}
}
pub async fn checkout_postgres_client(
pool: &Pool,
cancel_token: Option<&CancellationToken>,
checkout_timeout: Duration,
) -> Result<deadpool_postgres::Object, String> {
let checkout_timeout = effective_postgres_checkout_timeout(pool, checkout_timeout);
let start = Instant::now();
let result = checkout_postgres_client_classified(pool, cancel_token, checkout_timeout)
.await
.map_err(|error| error.to_string());
if let Ok(client) = &result {
log::debug!(
"[db:pool.checkout:done] elapsed_ms={} timeout_ms={}",
@@ -8763,6 +8805,43 @@ mod tests {
assert_eq!(effective_postgres_checkout_timeout(&pool, Duration::from_secs(15)), Duration::from_secs(15));
}
#[tokio::test]
async fn postgres_checkout_fault_injection_classifies_full_pool_as_wait() {
assert!(tokio::time::timeout(Duration::from_millis(1), std::future::pending::<()>()).await.is_err());
assert_eq!(
classify_postgres_checkout_stage(deadpool_postgres::Status {
max_size: 1,
size: 1,
available: 0,
waiting: 1
}),
PostgresCheckoutStage::Wait
);
}
#[tokio::test]
async fn postgres_checkout_fault_injection_preserves_create_and_recycle_stages() {
assert!(tokio::time::timeout(Duration::from_millis(1), std::future::pending::<()>()).await.is_err());
assert_eq!(
classify_postgres_checkout_stage(deadpool_postgres::Status {
max_size: 2,
size: 1,
available: 0,
waiting: 0
}),
PostgresCheckoutStage::Create
);
assert_eq!(
classify_postgres_checkout_stage(deadpool_postgres::Status {
max_size: 1,
size: 1,
available: 1,
waiting: 0
}),
PostgresCheckoutStage::Recycle
);
}
#[tokio::test]
async fn postgres_session_setup_reports_its_own_timeout_stage() {
let error =