fix(metadata): bound cache maintenance and pool recovery
This commit is contained in:
1 parent
21af5cf600
commit
8c2b232243
41 files changed
+2747
-350
No files matched your search
@@ -52,6 +52,7 @@ pub mod victoriametrics_driver;
|
||||
pub mod wkb;
|
||||
|
||||
use reqwest::ClientBuilder;
|
||||
use std::fmt;
|
||||
use std::future::Future;
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -62,6 +63,62 @@ pub use file_validator::validate_file_path;
|
||||
pub const CONNECTION_TIMEOUT_SECS: u64 = 5;
|
||||
pub const TCP_PROBE_TIMEOUT_SECS: u64 = 3;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum PoolCheckoutStage {
|
||||
Wait,
|
||||
Create,
|
||||
Recycle,
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl PoolCheckoutStage {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Wait => "wait",
|
||||
Self::Create => "create",
|
||||
Self::Recycle => "recycle",
|
||||
Self::Unknown => "unknown",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum PoolCheckoutError {
|
||||
Timeout { database: &'static str, stage: PoolCheckoutStage, timeout: Duration },
|
||||
Failed { database: &'static str, stage: PoolCheckoutStage, detail: String },
|
||||
Canceled,
|
||||
}
|
||||
|
||||
impl PoolCheckoutError {
|
||||
pub fn stage(&self) -> Option<PoolCheckoutStage> {
|
||||
match self {
|
||||
Self::Timeout { stage, .. } | Self::Failed { stage, .. } => Some(*stage),
|
||||
Self::Canceled => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_pool_saturation(&self) -> bool {
|
||||
matches!(self, Self::Timeout { stage: PoolCheckoutStage::Wait, .. })
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for PoolCheckoutError {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Timeout { database, stage, timeout } => write!(
|
||||
formatter,
|
||||
"{database} connection pool checkout timed out [stage={}, timeout_ms={}]",
|
||||
stage.as_str(),
|
||||
timeout.as_millis()
|
||||
),
|
||||
Self::Failed { database, stage, detail } => {
|
||||
write!(formatter, "{database} connection pool checkout failed [stage={}]: {detail}", stage.as_str())
|
||||
}
|
||||
Self::Canceled => formatter.write_str(crate::query::QUERY_CANCELED),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn connection_timeout() -> Duration {
|
||||
Duration::from_secs(CONNECTION_TIMEOUT_SECS)
|
||||
}
|
||||
|
||||
+270
-25
@@ -9,7 +9,9 @@ use sqlparser::dialect::MySqlDialect;
|
||||
use sqlparser::parser::Parser;
|
||||
use std::borrow::Cow;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
#[cfg(test)]
|
||||
use std::future::Future;
|
||||
use std::ops::Deref;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::time::Duration;
|
||||
@@ -28,7 +30,62 @@ use crate::types::{
|
||||
|
||||
use super::file_validator::validate_file_path;
|
||||
|
||||
pub type MySqlPool = mysql_async::Pool;
|
||||
/// DBX-owned MySQL pool handle. The driver does not expose its configured
|
||||
/// maximum, so retain that value beside the driver pool for checkout phase
|
||||
/// classification instead of guessing it from aggregate metrics.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MySqlPool {
|
||||
inner: mysql_async::Pool,
|
||||
max_connections: usize,
|
||||
}
|
||||
|
||||
impl MySqlPool {
|
||||
pub(crate) fn new<O>(opts: O, max_connections: usize) -> Self
|
||||
where
|
||||
mysql_async::Opts: TryFrom<O>,
|
||||
<mysql_async::Opts as TryFrom<O>>::Error: std::error::Error,
|
||||
{
|
||||
Self { inner: mysql_async::Pool::new(opts), max_connections: max_connections.max(1) }
|
||||
}
|
||||
|
||||
pub async fn disconnect(self) -> Result<(), mysql_async::Error> {
|
||||
self.inner.disconnect().await
|
||||
}
|
||||
}
|
||||
|
||||
impl Deref for MySqlPool {
|
||||
type Target = mysql_async::Pool;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.inner
|
||||
}
|
||||
}
|
||||
|
||||
#[doc(hidden)]
|
||||
pub trait MySqlPoolAccess {
|
||||
fn driver_pool(&self) -> &mysql_async::Pool;
|
||||
fn checkout_max_connections(&self) -> Option<usize>;
|
||||
}
|
||||
|
||||
impl MySqlPoolAccess for MySqlPool {
|
||||
fn driver_pool(&self) -> &mysql_async::Pool {
|
||||
&self.inner
|
||||
}
|
||||
|
||||
fn checkout_max_connections(&self) -> Option<usize> {
|
||||
Some(self.max_connections)
|
||||
}
|
||||
}
|
||||
|
||||
impl MySqlPoolAccess for mysql_async::Pool {
|
||||
fn driver_pool(&self) -> &mysql_async::Pool {
|
||||
self
|
||||
}
|
||||
|
||||
fn checkout_max_connections(&self) -> Option<usize> {
|
||||
None
|
||||
}
|
||||
}
|
||||
const MYSQL_TCP_KEEPALIVE_MS: u32 = 30_000;
|
||||
const MYSQL_SQL_PACKET_MARGIN_MAX_BYTES: usize = 64 * 1024;
|
||||
|
||||
@@ -1242,7 +1299,7 @@ fn create_pool(
|
||||
// to paths explicitly supplied by the user instead of enabling arbitrary reads.
|
||||
builder = builder.local_infile_handler(Some(mysql_async::WhiteListFsHandler::new(local_infile_paths)));
|
||||
}
|
||||
Ok(MySqlPool::new(builder))
|
||||
Ok(MySqlPool::new(builder, max_connections))
|
||||
}
|
||||
|
||||
fn mysql_async_tcp_host(host: &str) -> &str {
|
||||
@@ -3603,7 +3660,10 @@ fn parse_mysql_enum_values(column_type: &str) -> Option<Vec<String>> {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_columns(pool: &MySqlPool, database: &str, table: &str) -> Result<Vec<ColumnInfo>, String> {
|
||||
pub async fn get_columns<P>(pool: &P, database: &str, table: &str) -> Result<Vec<ColumnInfo>, String>
|
||||
where
|
||||
P: MySqlPoolAccess + ?Sized,
|
||||
{
|
||||
let sql = columns_sql(database, table);
|
||||
let mut conn = get_conn_with_health_check(pool).await?;
|
||||
let result = match conn.query_iter(&sql).await {
|
||||
@@ -3675,7 +3735,10 @@ pub async fn get_columns(pool: &MySqlPool, database: &str, table: &str) -> Resul
|
||||
Ok(columns)
|
||||
}
|
||||
|
||||
pub async fn get_columns_show(pool: &MySqlPool, database: &str, table: &str) -> Result<Vec<ColumnInfo>, String> {
|
||||
pub async fn get_columns_show<P>(pool: &P, database: &str, table: &str) -> Result<Vec<ColumnInfo>, String>
|
||||
where
|
||||
P: MySqlPoolAccess + ?Sized,
|
||||
{
|
||||
let sql = show_columns_sql(database, table, true);
|
||||
let mut conn = get_conn_with_health_check(pool).await?;
|
||||
let rows: Vec<mysql_async::Row> = match conn.query_iter(&sql).await {
|
||||
@@ -3852,23 +3915,32 @@ pub(super) fn skip_mysql_quoted(sql: &str, start: usize, quote: u8) -> usize {
|
||||
|
||||
/// Get a connection from the pool with a health check. If the connection is dead
|
||||
/// (e.g. after app was backgrounded), it tries again with a fresh connection.
|
||||
pub async fn get_conn_with_health_check(pool: &MySqlPool) -> Result<mysql_async::Conn, String> {
|
||||
pub async fn get_conn_with_health_check<P>(pool: &P) -> Result<mysql_async::Conn, String>
|
||||
where
|
||||
P: MySqlPoolAccess + ?Sized,
|
||||
{
|
||||
get_conn_with_health_check_with_timeout(pool, super::connection_timeout()).await
|
||||
}
|
||||
|
||||
pub async fn get_conn_with_health_check_with_timeout(
|
||||
pool: &MySqlPool,
|
||||
pub async fn get_conn_with_health_check_with_timeout<P>(
|
||||
pool: &P,
|
||||
timeout: Duration,
|
||||
) -> Result<mysql_async::Conn, String> {
|
||||
) -> Result<mysql_async::Conn, String>
|
||||
where
|
||||
P: MySqlPoolAccess + ?Sized,
|
||||
{
|
||||
get_conn_with_health_check_with_cancel(pool, timeout, timeout, None).await
|
||||
}
|
||||
|
||||
pub async fn get_conn_with_health_check_with_cancel(
|
||||
pool: &MySqlPool,
|
||||
pub async fn get_conn_with_health_check_with_cancel<P>(
|
||||
pool: &P,
|
||||
timeout: Duration,
|
||||
cleanup_timeout: Duration,
|
||||
cancel_token: Option<&CancellationToken>,
|
||||
) -> Result<mysql_async::Conn, String> {
|
||||
) -> Result<mysql_async::Conn, String>
|
||||
where
|
||||
P: MySqlPoolAccess + ?Sized,
|
||||
{
|
||||
let start = Instant::now();
|
||||
let mut conn = get_conn_with_timeout_and_cancel(pool, timeout, cancel_token).await?;
|
||||
match ping_conn_with_timeout_and_cancel(&mut conn, timeout, cancel_token).await {
|
||||
@@ -3909,13 +3981,67 @@ pub async fn get_conn_with_health_check_with_cancel(
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_conn_with_timeout_and_cancel(
|
||||
pool: &MySqlPool,
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) struct MysqlCheckoutSnapshot {
|
||||
pub(crate) connection_count: usize,
|
||||
pub(crate) connections_in_pool: usize,
|
||||
pub(crate) max_connections: usize,
|
||||
}
|
||||
|
||||
pub(crate) type MysqlCheckoutStage = super::PoolCheckoutStage;
|
||||
|
||||
pub(crate) fn classify_mysql_checkout_stage(snapshot: MysqlCheckoutSnapshot) -> MysqlCheckoutStage {
|
||||
if snapshot.connections_in_pool > 0 {
|
||||
MysqlCheckoutStage::Recycle
|
||||
} else if snapshot.connection_count < snapshot.max_connections {
|
||||
MysqlCheckoutStage::Create
|
||||
} else {
|
||||
MysqlCheckoutStage::Wait
|
||||
}
|
||||
}
|
||||
|
||||
fn mysql_checkout_snapshot<P>(pool: &P, max_connections: usize) -> MysqlCheckoutSnapshot
|
||||
where
|
||||
P: MySqlPoolAccess + ?Sized,
|
||||
{
|
||||
let metrics = pool.driver_pool().metrics();
|
||||
use std::sync::atomic::Ordering;
|
||||
MysqlCheckoutSnapshot {
|
||||
connection_count: metrics.connection_count.load(Ordering::Relaxed),
|
||||
connections_in_pool: metrics.connections_in_pool.load(Ordering::Relaxed),
|
||||
max_connections,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn checkout_mysql_conn<P>(
|
||||
pool: &P,
|
||||
timeout: Duration,
|
||||
) -> Result<mysql_async::Conn, super::PoolCheckoutError>
|
||||
where
|
||||
P: MySqlPoolAccess + ?Sized,
|
||||
{
|
||||
// Capture the phase before entering the driver future. The phase is tied
|
||||
// to this checkout's capacity decision, not to a later aggregate metric
|
||||
// snapshot that may already include another request.
|
||||
let stage = pool.checkout_max_connections().map_or(MysqlCheckoutStage::Unknown, |max_connections| {
|
||||
classify_mysql_checkout_stage(mysql_checkout_snapshot(pool, max_connections))
|
||||
});
|
||||
match tokio::time::timeout(timeout, pool.driver_pool().get_conn()).await {
|
||||
Ok(Ok(conn)) => Ok(conn),
|
||||
Ok(Err(error)) => Err(super::PoolCheckoutError::Failed { database: "MySQL", stage, detail: error.to_string() }),
|
||||
Err(_) => Err(super::PoolCheckoutError::Timeout { database: "MySQL", stage, timeout }),
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_conn_with_timeout_and_cancel<P>(
|
||||
pool: &P,
|
||||
timeout: Duration,
|
||||
cancel_token: Option<&CancellationToken>,
|
||||
) -> Result<mysql_async::Conn, String> {
|
||||
let get_future =
|
||||
connection_result_with_timeout(timeout, async { pool.get_conn().await.map_err(|error| error.to_string()) });
|
||||
) -> Result<mysql_async::Conn, String>
|
||||
where
|
||||
P: MySqlPoolAccess + ?Sized,
|
||||
{
|
||||
let get_future = async { checkout_mysql_conn(pool, timeout).await.map_err(|error| error.to_string()) };
|
||||
|
||||
match cancel_token {
|
||||
Some(token) => {
|
||||
@@ -3929,10 +4055,14 @@ async fn get_conn_with_timeout_and_cancel(
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_conn_with_timeout(pool: &MySqlPool, timeout: Duration) -> Result<mysql_async::Conn, String> {
|
||||
connection_result_with_timeout(timeout, async { pool.get_conn().await.map_err(|error| error.to_string()) }).await
|
||||
pub async fn get_conn_with_timeout<P>(pool: &P, timeout: Duration) -> Result<mysql_async::Conn, String>
|
||||
where
|
||||
P: MySqlPoolAccess + ?Sized,
|
||||
{
|
||||
checkout_mysql_conn(pool, timeout).await.map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn connection_result_with_timeout<T, F>(timeout: Duration, future: F) -> Result<T, String>
|
||||
where
|
||||
F: Future<Output = Result<T, String>>,
|
||||
@@ -4564,13 +4694,16 @@ pub(crate) fn mysql_sql_statement_hard_limit(max_allowed_packet: u64) -> Option<
|
||||
packet_bytes.checked_sub(margin).filter(|limit| *limit > 0)
|
||||
}
|
||||
|
||||
pub async fn execute_query_with_max_rows(
|
||||
pool: &MySqlPool,
|
||||
pub async fn execute_query_with_max_rows<P>(
|
||||
pool: &P,
|
||||
sql: &str,
|
||||
bare: bool,
|
||||
max_rows: Option<usize>,
|
||||
dialect: MySqlQueryDialect,
|
||||
) -> Result<QueryResult, String> {
|
||||
) -> Result<QueryResult, String>
|
||||
where
|
||||
P: MySqlPoolAccess + ?Sized,
|
||||
{
|
||||
let mut conn = get_conn_with_health_check(pool).await?;
|
||||
execute_query_on_conn_with_max_rows(&mut conn, sql, bare, max_rows, dialect).await
|
||||
}
|
||||
@@ -5357,10 +5490,10 @@ mod tests {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
Ok::<_, String>("connection")
|
||||
});
|
||||
let configured_five = connection_result_with_timeout(Duration::from_millis(50), async {
|
||||
tokio::time::sleep(Duration::from_millis(60)).await;
|
||||
Ok::<_, String>("connection")
|
||||
});
|
||||
let configured_five = connection_result_with_timeout(
|
||||
Duration::from_millis(10),
|
||||
std::future::pending::<Result<&'static str, String>>(),
|
||||
);
|
||||
|
||||
let (exact, adjacent, configured_five) = tokio::join!(exact, adjacent, configured_five);
|
||||
|
||||
@@ -5369,6 +5502,118 @@ mod tests {
|
||||
assert_eq!(configured_five, Err("MySQL get connection timed out".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mysql_checkout_fault_injection_classifies_full_pool_as_wait() {
|
||||
assert!(tokio::time::timeout(Duration::from_millis(1), std::future::pending::<()>()).await.is_err());
|
||||
assert_eq!(
|
||||
classify_mysql_checkout_stage(MysqlCheckoutSnapshot {
|
||||
connection_count: 1,
|
||||
connections_in_pool: 0,
|
||||
max_connections: 1,
|
||||
}),
|
||||
MysqlCheckoutStage::Wait
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mysql_checkout_fault_injection_classifies_hung_connection_create() {
|
||||
assert!(tokio::time::timeout(Duration::from_millis(1), std::future::pending::<()>()).await.is_err());
|
||||
assert_eq!(
|
||||
classify_mysql_checkout_stage(MysqlCheckoutSnapshot {
|
||||
connection_count: 0,
|
||||
connections_in_pool: 0,
|
||||
max_connections: 2,
|
||||
}),
|
||||
MysqlCheckoutStage::Create
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mysql_checkout_fault_injection_preserves_create_stage_when_handshake_hangs() {
|
||||
let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)).await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
let (_socket, _) = listener.accept().await.unwrap();
|
||||
std::future::pending::<()>().await;
|
||||
});
|
||||
let pool_options =
|
||||
mysql_async::PoolOpts::new().with_constraints(mysql_async::PoolConstraints::new(1, 2).unwrap());
|
||||
let options = mysql_async::OptsBuilder::default()
|
||||
.ip_or_hostname(address.ip().to_string())
|
||||
.tcp_port(address.port())
|
||||
.user(Some("fault-injection"))
|
||||
.pass(Some("fault-injection"))
|
||||
.pool_opts(Some(pool_options));
|
||||
let pool = MySqlPool::new(options, 2);
|
||||
|
||||
let error = checkout_mysql_conn(&pool, Duration::from_millis(50)).await.unwrap_err();
|
||||
|
||||
assert!(matches!(error, super::super::PoolCheckoutError::Timeout { stage: MysqlCheckoutStage::Create, .. }));
|
||||
server.abort();
|
||||
let _ = tokio::time::timeout(Duration::from_secs(1), pool.disconnect()).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mysql_checkout_raw_driver_pool_timeout_keeps_stage_unknown() {
|
||||
let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)).await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
let (_socket, _) = listener.accept().await.unwrap();
|
||||
std::future::pending::<()>().await;
|
||||
});
|
||||
let options = mysql_async::OptsBuilder::default()
|
||||
.ip_or_hostname(address.ip().to_string())
|
||||
.tcp_port(address.port())
|
||||
.user(Some("fault-injection"))
|
||||
.pass(Some("fault-injection"));
|
||||
let pool = mysql_async::Pool::new(options);
|
||||
|
||||
let error = checkout_mysql_conn(&pool, Duration::from_millis(50)).await.unwrap_err();
|
||||
|
||||
assert!(matches!(error, super::super::PoolCheckoutError::Timeout { stage: MysqlCheckoutStage::Unknown, .. }));
|
||||
server.abort();
|
||||
let _ = tokio::time::timeout(Duration::from_secs(1), pool.disconnect()).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DBX_REVIEW_MYSQL_* environment variables"]
|
||||
async fn mysql_checkout_fault_injection_preserves_wait_stage_when_real_pool_is_full() {
|
||||
let host = std::env::var("DBX_REVIEW_MYSQL_HOST").expect("DBX_REVIEW_MYSQL_HOST is required");
|
||||
let port = std::env::var("DBX_REVIEW_MYSQL_PORT").expect("DBX_REVIEW_MYSQL_PORT is required");
|
||||
let user = std::env::var("DBX_REVIEW_MYSQL_USER").expect("DBX_REVIEW_MYSQL_USER is required");
|
||||
let password = std::env::var("DBX_REVIEW_MYSQL_PASSWORD").expect("DBX_REVIEW_MYSQL_PASSWORD is required");
|
||||
let database = std::env::var("DBX_REVIEW_MYSQL_DATABASE").expect("DBX_REVIEW_MYSQL_DATABASE is required");
|
||||
let pool_options =
|
||||
mysql_async::PoolOpts::new().with_constraints(mysql_async::PoolConstraints::new(1, 1).unwrap());
|
||||
let options = mysql_async::OptsBuilder::default()
|
||||
.ip_or_hostname(host)
|
||||
.tcp_port(port.parse().expect("DBX_REVIEW_MYSQL_PORT must be a valid port"))
|
||||
.user(Some(user))
|
||||
.pass(Some(password))
|
||||
.db_name(Some(database))
|
||||
.pool_opts(Some(pool_options));
|
||||
let pool = MySqlPool::new(options, 1);
|
||||
let held_connection = tokio::time::timeout(Duration::from_secs(10), pool.get_conn())
|
||||
.await
|
||||
.expect("real MySQL checkout timed out")
|
||||
.expect("real MySQL checkout failed");
|
||||
|
||||
let error = checkout_mysql_conn(&pool, Duration::from_millis(50)).await.unwrap_err();
|
||||
|
||||
assert!(matches!(error, super::super::PoolCheckoutError::Timeout { stage: MysqlCheckoutStage::Wait, .. }));
|
||||
drop(held_connection);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(2), pool.disconnect()).await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mysql_public_metadata_and_query_helpers_accept_driver_pool_for_compatibility() {
|
||||
let pool = mysql_async::Pool::new(mysql_async::OptsBuilder::default());
|
||||
let columns = get_columns(&pool, "database", "table");
|
||||
let query = execute_query_with_max_rows(&pool, "SELECT 1", false, Some(1), MySqlQueryDialect::default());
|
||||
|
||||
drop((columns, query));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mysql_database_list_timeout_preserves_immediate_errors_and_fallback_plan() {
|
||||
let error =
|
||||
|
||||
@@ -5900,40 +5900,70 @@ async fn execute_postgres_user_query(
|
||||
|
||||
/// PostgreSQL pool checkout with timeout and cancel token support.
|
||||
/// When the checkout phase is stuck, the cancel token can terminate the wait early.
|
||||
/// The timeout error message includes "checkout timed out" to ensure is_connection_error can classify it correctly.
|
||||
pub async fn checkout_postgres_client(
|
||||
pub(crate) type PostgresCheckoutStage = super::PoolCheckoutStage;
|
||||
|
||||
pub(crate) fn classify_postgres_checkout_stage(status: deadpool_postgres::Status) -> PostgresCheckoutStage {
|
||||
if status.waiting > 0 || (status.available == 0 && status.size >= status.max_size) {
|
||||
PostgresCheckoutStage::Wait
|
||||
} else if status.available > 0 {
|
||||
PostgresCheckoutStage::Recycle
|
||||
} else if status.size < status.max_size {
|
||||
PostgresCheckoutStage::Create
|
||||
} else {
|
||||
PostgresCheckoutStage::Unknown
|
||||
}
|
||||
}
|
||||
|
||||
fn postgres_pool_error_stage(error: &PoolError) -> PostgresCheckoutStage {
|
||||
match error {
|
||||
PoolError::Timeout(deadpool_postgres::TimeoutType::Wait) => PostgresCheckoutStage::Wait,
|
||||
PoolError::Timeout(deadpool_postgres::TimeoutType::Create) => PostgresCheckoutStage::Create,
|
||||
PoolError::Timeout(deadpool_postgres::TimeoutType::Recycle) => PostgresCheckoutStage::Recycle,
|
||||
PoolError::Backend(_) | PoolError::PostCreateHook(_) => PostgresCheckoutStage::Create,
|
||||
PoolError::Closed | PoolError::NoRuntimeSpecified => PostgresCheckoutStage::Unknown,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn checkout_postgres_client_classified(
|
||||
pool: &Pool,
|
||||
cancel_token: Option<&CancellationToken>,
|
||||
checkout_timeout: Duration,
|
||||
) -> Result<deadpool_postgres::Object, String> {
|
||||
let checkout_timeout = effective_postgres_checkout_timeout(pool, checkout_timeout);
|
||||
) -> Result<deadpool_postgres::Object, super::PoolCheckoutError> {
|
||||
let start = Instant::now();
|
||||
let get_future = async {
|
||||
tokio::time::timeout(checkout_timeout, pool.get())
|
||||
.await
|
||||
.map_err(|_| {
|
||||
let elapsed = start.elapsed().as_millis();
|
||||
match tokio::time::timeout(checkout_timeout, pool.get()).await {
|
||||
Ok(Ok(client)) => Ok(client),
|
||||
Ok(Err(error)) => {
|
||||
let stage = postgres_pool_error_stage(&error);
|
||||
let timed_out = matches!(&error, PoolError::Timeout(_));
|
||||
let detail = pg_pool_error_to_string(error);
|
||||
log::warn!(
|
||||
"[db:pool.checkout:error] elapsed_ms={} timeout_ms={} error=checkout timed out",
|
||||
elapsed,
|
||||
checkout_timeout.as_millis()
|
||||
);
|
||||
format!("PostgreSQL connection pool checkout timed out ({}s)", checkout_timeout.as_secs())
|
||||
})?
|
||||
.map_err(|e| {
|
||||
let elapsed = start.elapsed().as_millis();
|
||||
let err = pg_pool_error_to_string(e);
|
||||
log::warn!(
|
||||
"[db:pool.checkout:error] elapsed_ms={} timeout_ms={} error={}",
|
||||
elapsed,
|
||||
"[db:pool.checkout:error] elapsed_ms={} timeout_ms={} stage={} error={}",
|
||||
start.elapsed().as_millis(),
|
||||
checkout_timeout.as_millis(),
|
||||
err
|
||||
stage.as_str(),
|
||||
detail
|
||||
);
|
||||
format!("PostgreSQL connection pool checkout failed: {err}")
|
||||
})
|
||||
if timed_out {
|
||||
Err(super::PoolCheckoutError::Timeout { database: "PostgreSQL", stage, timeout: checkout_timeout })
|
||||
} else {
|
||||
Err(super::PoolCheckoutError::Failed { database: "PostgreSQL", stage, detail })
|
||||
}
|
||||
}
|
||||
Err(_) => {
|
||||
let stage = classify_postgres_checkout_stage(pool.status());
|
||||
log::warn!(
|
||||
"[db:pool.checkout:error] elapsed_ms={} timeout_ms={} stage={} error=checkout timed out",
|
||||
start.elapsed().as_millis(),
|
||||
checkout_timeout.as_millis(),
|
||||
stage.as_str()
|
||||
);
|
||||
Err(super::PoolCheckoutError::Timeout { database: "PostgreSQL", stage, timeout: checkout_timeout })
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let result = match cancel_token {
|
||||
match cancel_token {
|
||||
Some(token) => tokio::select! {
|
||||
biased;
|
||||
_ = token.cancelled() => {
|
||||
@@ -5942,12 +5972,24 @@ pub async fn checkout_postgres_client(
|
||||
start.elapsed().as_millis(),
|
||||
checkout_timeout.as_millis()
|
||||
);
|
||||
return Err(crate::query::canceled_error());
|
||||
Err(super::PoolCheckoutError::Canceled)
|
||||
}
|
||||
result = get_future => result,
|
||||
},
|
||||
None => get_future.await,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn checkout_postgres_client(
|
||||
pool: &Pool,
|
||||
cancel_token: Option<&CancellationToken>,
|
||||
checkout_timeout: Duration,
|
||||
) -> Result<deadpool_postgres::Object, String> {
|
||||
let checkout_timeout = effective_postgres_checkout_timeout(pool, checkout_timeout);
|
||||
let start = Instant::now();
|
||||
let result = checkout_postgres_client_classified(pool, cancel_token, checkout_timeout)
|
||||
.await
|
||||
.map_err(|error| error.to_string());
|
||||
if let Ok(client) = &result {
|
||||
log::debug!(
|
||||
"[db:pool.checkout:done] elapsed_ms={} timeout_ms={}",
|
||||
@@ -8763,6 +8805,43 @@ mod tests {
|
||||
assert_eq!(effective_postgres_checkout_timeout(&pool, Duration::from_secs(15)), Duration::from_secs(15));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn postgres_checkout_fault_injection_classifies_full_pool_as_wait() {
|
||||
assert!(tokio::time::timeout(Duration::from_millis(1), std::future::pending::<()>()).await.is_err());
|
||||
assert_eq!(
|
||||
classify_postgres_checkout_stage(deadpool_postgres::Status {
|
||||
max_size: 1,
|
||||
size: 1,
|
||||
available: 0,
|
||||
waiting: 1
|
||||
}),
|
||||
PostgresCheckoutStage::Wait
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn postgres_checkout_fault_injection_preserves_create_and_recycle_stages() {
|
||||
assert!(tokio::time::timeout(Duration::from_millis(1), std::future::pending::<()>()).await.is_err());
|
||||
assert_eq!(
|
||||
classify_postgres_checkout_stage(deadpool_postgres::Status {
|
||||
max_size: 2,
|
||||
size: 1,
|
||||
available: 0,
|
||||
waiting: 0
|
||||
}),
|
||||
PostgresCheckoutStage::Create
|
||||
);
|
||||
assert_eq!(
|
||||
classify_postgres_checkout_stage(deadpool_postgres::Status {
|
||||
max_size: 1,
|
||||
size: 1,
|
||||
available: 1,
|
||||
waiting: 0
|
||||
}),
|
||||
PostgresCheckoutStage::Recycle
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn postgres_session_setup_reports_its_own_timeout_stage() {
|
||||
let error =
|
||||
|
||||
Reference in new issue
Block a user