fix(metadata): bound cache maintenance and pool recovery
This commit is contained in:
1 parent
21af5cf600
commit
8c2b232243
41 files changed
+2747
-350
No files matched your search
@@ -3,7 +3,7 @@ use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::sync::{watch, Mutex, RwLock};
|
||||
use tokio::sync::{watch, Mutex, OwnedSemaphorePermit, RwLock, Semaphore};
|
||||
|
||||
use mysql_async::prelude::Queryable;
|
||||
use mysql_async::Row as MysqlRow;
|
||||
@@ -44,6 +44,9 @@ const DEFAULT_AGENT_CONNECT_TIMEOUT_SECS: u64 = 30;
|
||||
const ACCESS_AGENT_CONNECT_TIMEOUT_SECS: u64 = 30;
|
||||
const POOL_CLOSE_TIMEOUT_SECS: u64 = 3;
|
||||
const HEALTH_CHECK_POOL_ACQUIRE_TIMEOUT: Duration = Duration::from_millis(500);
|
||||
pub(crate) const METADATA_POOL_ACQUIRE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
pub(crate) const METADATA_POOL_DEFAULT_LIMIT: usize = 6;
|
||||
pub(crate) const METADATA_POOL_SQLSERVER_LIMIT: usize = 1;
|
||||
|
||||
mod duckdb_types {
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
@@ -115,6 +118,42 @@ pub enum PoolKind {
|
||||
}
|
||||
|
||||
impl PoolKind {
|
||||
/// Clone only the handles used by metadata operations so the global pool
|
||||
/// map lock can be released before any database I/O begins.
|
||||
pub(crate) fn clone_for_metadata(&self) -> Option<Self> {
|
||||
match self {
|
||||
Self::Mysql(pool, mode) => Some(Self::Mysql(pool.clone(), *mode)),
|
||||
Self::Postgres(pool) => Some(Self::Postgres(pool.clone())),
|
||||
Self::Sqlite(pool) => Some(Self::Sqlite(pool.clone())),
|
||||
Self::Rqlite(client) => Some(Self::Rqlite(client.clone())),
|
||||
Self::Turso(client) => Some(Self::Turso(client.clone())),
|
||||
Self::CloudflareD1(client) => Some(Self::CloudflareD1(client.clone())),
|
||||
Self::DuckDbWorker(client) => Some(Self::DuckDbWorker(client.clone())),
|
||||
Self::MongoDb(client) => Some(Self::MongoDb(client.clone())),
|
||||
Self::ClickHouse(client) => Some(Self::ClickHouse(client.clone())),
|
||||
Self::SqlServer(client) => Some(Self::SqlServer(client.clone())),
|
||||
Self::Elasticsearch(client) => Some(Self::Elasticsearch(client.clone())),
|
||||
Self::Easysearch(client) => Some(Self::Easysearch(client.clone())),
|
||||
Self::Meilisearch(client) => Some(Self::Meilisearch(client.clone())),
|
||||
Self::HBase(client) => Some(Self::HBase(client.clone())),
|
||||
Self::VectorDb(client) => Some(Self::VectorDb(client.clone())),
|
||||
Self::InfluxDb(client) => Some(Self::InfluxDb(client.clone())),
|
||||
Self::VictoriaMetrics(client) => Some(Self::VictoriaMetrics(client.clone())),
|
||||
Self::Agent(client) => Some(Self::Agent(client.clone())),
|
||||
Self::ExternalDriver { driver_id, config, session } => Some(Self::ExternalDriver {
|
||||
driver_id: driver_id.clone(),
|
||||
config: config.clone(),
|
||||
session: session.clone(),
|
||||
}),
|
||||
Self::MessageQueue => Some(Self::MessageQueue),
|
||||
Self::Nacos => Some(Self::Nacos),
|
||||
Self::Consul(client) => Some(Self::Consul(client.clone())),
|
||||
#[cfg(feature = "mq-admin")]
|
||||
Self::Mqtt(client) => Some(Self::Mqtt(client.clone())),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn agent(client: db::agent_driver::AgentDriverClient) -> Self {
|
||||
Self::Agent(Arc::new(db::agent_driver::PooledAgentClient::new(client)))
|
||||
}
|
||||
@@ -221,6 +260,7 @@ pub struct AppState {
|
||||
/// Web 端 owner 为已认证会话 token,不同登录会话互不可见。建池/池重建/
|
||||
/// AI/元数据从它读取,前端通过状态接口查询。
|
||||
pub session_credentials: SessionCredentialStore,
|
||||
metadata_gates: Arc<Mutex<HashMap<String, Arc<Semaphore>>>>,
|
||||
#[cfg(feature = "mq-admin")]
|
||||
pub mq_registry: crate::mq::MqAdminRegistry,
|
||||
}
|
||||
@@ -250,6 +290,32 @@ struct ConnectionAttemptState {
|
||||
client_attempt: Option<u64>,
|
||||
}
|
||||
|
||||
pub(crate) fn metadata_concurrency_limit(db_type: DatabaseType, max_connections: usize) -> usize {
|
||||
if db_type == DatabaseType::SqlServer {
|
||||
METADATA_POOL_SQLSERVER_LIMIT
|
||||
} else {
|
||||
max_connections.saturating_sub(2).clamp(1, METADATA_POOL_DEFAULT_LIMIT)
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn uses_metadata_gate(db_type: DatabaseType) -> bool {
|
||||
matches!(db_type, DatabaseType::Mysql | DatabaseType::Postgres | DatabaseType::SqlServer)
|
||||
}
|
||||
|
||||
fn metadata_gate_key(
|
||||
connection_id: &str,
|
||||
database: Option<&str>,
|
||||
db_type: DatabaseType,
|
||||
client_session_id: Option<&str>,
|
||||
) -> String {
|
||||
let session = if db_type == DatabaseType::SqlServer {
|
||||
""
|
||||
} else {
|
||||
client_session_id.map(str::trim).filter(|session| !session.is_empty()).unwrap_or_default()
|
||||
};
|
||||
format!("{connection_id}\0{}\0{}", database.unwrap_or_default(), session)
|
||||
}
|
||||
|
||||
struct PoolDrainGuard {
|
||||
pool_key: String,
|
||||
draining_pools: Arc<std::sync::Mutex<HashMap<String, watch::Sender<bool>>>>,
|
||||
@@ -1091,11 +1157,65 @@ impl AppState {
|
||||
postgres_cancel_contexts: Arc::new(RwLock::new(HashMap::new())),
|
||||
transaction_sessions: Arc::new(RwLock::new(HashMap::new())),
|
||||
session_credentials: SessionCredentialStore::new(),
|
||||
metadata_gates: Arc::new(Mutex::new(HashMap::new())),
|
||||
#[cfg(feature = "mq-admin")]
|
||||
mq_registry: crate::mq::MqAdminRegistry::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn acquire_metadata_permit(
|
||||
&self,
|
||||
connection_id: &str,
|
||||
database: Option<&str>,
|
||||
db_type: DatabaseType,
|
||||
client_session_id: Option<&str>,
|
||||
) -> Result<OwnedSemaphorePermit, String> {
|
||||
let key = metadata_gate_key(connection_id, database, db_type, client_session_id);
|
||||
let max_connections =
|
||||
if client_session_id.map(str::trim).is_some_and(|session| !session.is_empty()) { 1 } else { 10 };
|
||||
let limit = metadata_concurrency_limit(db_type, max_connections);
|
||||
let gate = {
|
||||
let mut gates = self.metadata_gates.lock().await;
|
||||
gates.entry(key.clone()).or_insert_with(|| Arc::new(Semaphore::new(limit))).clone()
|
||||
};
|
||||
let started = Instant::now();
|
||||
let queued = gate.available_permits() == 0;
|
||||
let permit = match tokio::time::timeout(METADATA_POOL_ACQUIRE_TIMEOUT, gate.acquire_owned()).await {
|
||||
Ok(Ok(permit)) => permit,
|
||||
Ok(Err(_)) => return Err(crate::query::METADATA_POOL_BUSY_ERROR.to_string()),
|
||||
Err(_) => {
|
||||
log::warn!(
|
||||
"[metadata:pool:busy] connection_id={} database={} wait_ms={} limit={}",
|
||||
connection_id,
|
||||
database.unwrap_or_default(),
|
||||
started.elapsed().as_millis(),
|
||||
limit
|
||||
);
|
||||
return Err(crate::query::METADATA_POOL_BUSY_ERROR.to_string());
|
||||
}
|
||||
};
|
||||
if queued {
|
||||
log::debug!(
|
||||
"[metadata:pool:acquired] connection_id={} database={} wait_ms={} limit={}",
|
||||
connection_id,
|
||||
database.unwrap_or_default(),
|
||||
started.elapsed().as_millis(),
|
||||
limit
|
||||
);
|
||||
}
|
||||
Ok(permit)
|
||||
}
|
||||
|
||||
async fn clear_metadata_gates_for_connection(&self, connection_id: &str) {
|
||||
let prefix = format!("{connection_id}\0");
|
||||
self.metadata_gates.lock().await.retain(|key, _| !key.starts_with(&prefix));
|
||||
}
|
||||
|
||||
async fn clear_metadata_gate_for_database(&self, connection_id: &str, database: Option<&str>) {
|
||||
let prefix = format!("{connection_id}\0{}\0", database.unwrap_or_default());
|
||||
self.metadata_gates.lock().await.retain(|key, _| !key.starts_with(&prefix));
|
||||
}
|
||||
|
||||
pub fn jdbc_unavailable_error(&self) -> String {
|
||||
match self.plugins.find_driver("jdbc") {
|
||||
Ok(Some(_)) => "JDBC plugin is installed, but the connection could not be opened.".to_string(),
|
||||
@@ -3021,18 +3141,18 @@ impl AppState {
|
||||
PoolKind::Mysql(pool, _) => {
|
||||
let pool = pool.clone();
|
||||
drop(connections);
|
||||
match tokio::time::timeout(HEALTH_CHECK_POOL_ACQUIRE_TIMEOUT, pool.get_conn()).await {
|
||||
match db::mysql::checkout_mysql_conn(&pool, HEALTH_CHECK_POOL_ACQUIRE_TIMEOUT).await {
|
||||
// Pool saturation means active work, not a dead connection. Removing this pool would
|
||||
// start a competing reconnect while foreground queries and metadata are still running.
|
||||
Err(_) => {
|
||||
log::debug!("MySQL connection pool '{pool_key}' is busy; skipping health probe");
|
||||
Err(err) if err.is_pool_saturation() => {
|
||||
log::debug!("MySQL connection pool '{pool_key}' is busy; skipping health probe: {err}");
|
||||
false
|
||||
}
|
||||
Ok(Err(err)) => {
|
||||
Err(err) => {
|
||||
log::warn!("MySQL connection pool '{pool_key}' is stale: {err}");
|
||||
true
|
||||
}
|
||||
Ok(Ok(mut conn)) => {
|
||||
Ok(mut conn) => {
|
||||
let timeout = crate::db::connection_timeout();
|
||||
match tokio::time::timeout(timeout, conn.ping()).await {
|
||||
Ok(Ok(())) => false,
|
||||
@@ -3052,8 +3172,14 @@ impl AppState {
|
||||
let pool = pool.clone();
|
||||
drop(connections);
|
||||
let timeout = crate::db::connection_timeout();
|
||||
match tokio::time::timeout(HEALTH_CHECK_POOL_ACQUIRE_TIMEOUT, pool.get()).await {
|
||||
Ok(Ok(client)) => match tokio::time::timeout(timeout, client.simple_query("SELECT 1")).await {
|
||||
match db::postgres::checkout_postgres_client_classified(
|
||||
&pool,
|
||||
None,
|
||||
HEALTH_CHECK_POOL_ACQUIRE_TIMEOUT,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(client) => match tokio::time::timeout(timeout, client.simple_query("SELECT 1")).await {
|
||||
Ok(Ok(_)) => false,
|
||||
Ok(Err(err)) => {
|
||||
log::warn!("PostgreSQL connection pool '{pool_key}' is stale: {err}");
|
||||
@@ -3064,14 +3190,16 @@ impl AppState {
|
||||
true
|
||||
}
|
||||
},
|
||||
Ok(Err(err)) => {
|
||||
Err(err) if err.is_pool_saturation() => {
|
||||
log::debug!(
|
||||
"PostgreSQL connection pool '{pool_key}' is busy; skipping health probe: {err}"
|
||||
);
|
||||
false
|
||||
}
|
||||
Err(err) => {
|
||||
log::warn!("PostgreSQL connection pool '{pool_key}' is stale: {err}");
|
||||
true
|
||||
}
|
||||
Err(_) => {
|
||||
log::debug!("PostgreSQL connection pool '{pool_key}' is busy; skipping health probe");
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
PoolKind::SqlServer(client) => {
|
||||
@@ -3742,6 +3870,7 @@ impl AppState {
|
||||
}
|
||||
drop(conns);
|
||||
let closed = !removed.is_empty();
|
||||
self.clear_metadata_gate_for_database(connection_id, database).await;
|
||||
for (key, pool) in removed {
|
||||
self.pool_routing_control().close_pool_with_timeout(key, pool).await;
|
||||
}
|
||||
@@ -4034,32 +4163,38 @@ impl AppState {
|
||||
let healthy = match pool {
|
||||
PoolKind::Mysql(p, _) => match db::mysql::get_conn_with_health_check(p).await {
|
||||
Ok(_) => true,
|
||||
Err(e) if crate::query::is_pool_saturation_error(&e) => {
|
||||
log::debug!("MySQL connection pool '{key}' is busy; skipping health probe: {e}");
|
||||
true
|
||||
}
|
||||
Err(e) => {
|
||||
log::warn!("MySQL connection pool '{key}' is unhealthy: {e}");
|
||||
false
|
||||
}
|
||||
},
|
||||
PoolKind::Postgres(p) => match tokio::time::timeout(timeout, p.get()).await {
|
||||
Ok(Ok(client)) => match tokio::time::timeout(timeout, client.simple_query("SELECT 1")).await {
|
||||
Ok(Ok(_)) => true,
|
||||
Ok(Err(e)) => {
|
||||
log::warn!("PostgreSQL connection pool '{key}' is unhealthy: {e}");
|
||||
PoolKind::Postgres(p) => {
|
||||
match db::postgres::checkout_postgres_client_classified(p, None, timeout).await {
|
||||
Ok(client) => match tokio::time::timeout(timeout, client.simple_query("SELECT 1")).await {
|
||||
Ok(Ok(_)) => true,
|
||||
Ok(Err(e)) => {
|
||||
log::warn!("PostgreSQL connection pool '{key}' is unhealthy: {e}");
|
||||
false
|
||||
}
|
||||
Err(_) => {
|
||||
log::warn!("PostgreSQL connection pool '{key}' is unhealthy: health check timed out");
|
||||
false
|
||||
}
|
||||
},
|
||||
Err(error) if error.is_pool_saturation() => {
|
||||
log::debug!("PostgreSQL connection pool '{key}' is busy; skipping health probe: {error}");
|
||||
true
|
||||
}
|
||||
Err(error) => {
|
||||
log::warn!("PostgreSQL connection pool '{key}' is unhealthy: {error}");
|
||||
false
|
||||
}
|
||||
Err(_) => {
|
||||
log::warn!("PostgreSQL connection pool '{key}' is unhealthy: health check timed out");
|
||||
false
|
||||
}
|
||||
},
|
||||
Ok(Err(e)) => {
|
||||
log::warn!("PostgreSQL connection pool '{key}' is unhealthy: {e}");
|
||||
false
|
||||
}
|
||||
Err(_) => {
|
||||
log::warn!("PostgreSQL connection pool '{key}' is unhealthy: get connection timed out");
|
||||
false
|
||||
}
|
||||
},
|
||||
}
|
||||
PoolKind::SqlServer(client) => {
|
||||
let mut client = client.lock().await;
|
||||
match db::sqlserver::test_connection(&mut client).await {
|
||||
@@ -4280,11 +4415,13 @@ impl AppState {
|
||||
|
||||
pub async fn remove_connection_pools(&self, connection_id: &str) {
|
||||
let removed = self.drain_connection_pools(connection_id).await;
|
||||
self.clear_metadata_gates_for_connection(connection_id).await;
|
||||
self.pool_routing_control().close_removed(removed).await;
|
||||
}
|
||||
|
||||
pub async fn remove_connection_pools_detached(&self, connection_id: &str) {
|
||||
let removed = self.drain_connection_pools(connection_id).await;
|
||||
self.clear_metadata_gates_for_connection(connection_id).await;
|
||||
self.pool_routing_control().close_removed_in_background(removed);
|
||||
}
|
||||
|
||||
@@ -4303,6 +4440,7 @@ impl AppState {
|
||||
self.session_credentials.clear_pool_owners();
|
||||
self.postgres_cancel_contexts.write().await.clear();
|
||||
self.draining_pools.lock().unwrap_or_else(|error| error.into_inner()).clear();
|
||||
self.metadata_gates.lock().await.clear();
|
||||
self.connections.write().await.drain().collect()
|
||||
}
|
||||
|
||||
@@ -6859,6 +6997,37 @@ mod tests {
|
||||
assert_eq!(super::mysql_pool_max_connections_for_session(Some("tab-1")), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_concurrency_reserves_query_capacity() {
|
||||
assert_eq!(super::metadata_concurrency_limit(DatabaseType::Mysql, 10), 6);
|
||||
assert_eq!(super::metadata_concurrency_limit(DatabaseType::Postgres, 10), 6);
|
||||
assert_eq!(super::metadata_concurrency_limit(DatabaseType::Mysql, 3), 1);
|
||||
assert_eq!(super::metadata_concurrency_limit(DatabaseType::Postgres, 1), 1);
|
||||
assert_eq!(super::metadata_concurrency_limit(DatabaseType::SqlServer, 10), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_gate_is_limited_to_supported_pool_drivers() {
|
||||
assert!(super::uses_metadata_gate(DatabaseType::Mysql));
|
||||
assert!(super::uses_metadata_gate(DatabaseType::Postgres));
|
||||
assert!(super::uses_metadata_gate(DatabaseType::SqlServer));
|
||||
assert!(!super::uses_metadata_gate(DatabaseType::Oracle));
|
||||
assert!(!super::uses_metadata_gate(DatabaseType::Sqlite));
|
||||
assert!(!super::uses_metadata_gate(DatabaseType::MongoDb));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sqlserver_metadata_gate_is_shared_across_client_sessions() {
|
||||
assert_eq!(
|
||||
super::metadata_gate_key("conn", Some("app"), DatabaseType::SqlServer, Some("metadata-1")),
|
||||
super::metadata_gate_key("conn", Some("app"), DatabaseType::SqlServer, Some("metadata-2"))
|
||||
);
|
||||
assert_ne!(
|
||||
super::metadata_gate_key("conn", Some("app"), DatabaseType::Mysql, Some("tab-1")),
|
||||
super::metadata_gate_key("conn", Some("app"), DatabaseType::Mysql, Some("tab-2"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prestosql_uses_external_driver_pool_not_agent_pool() {
|
||||
assert!(!super::uses_agent_connection_pool(&DatabaseType::PrestoSql));
|
||||
|
||||
@@ -1848,6 +1848,17 @@ pub async fn export_database_sql_core(
|
||||
state: &crate::connection::AppState,
|
||||
request: &DatabaseExportRequest,
|
||||
on_progress: impl Fn(ExportProgress) + Sync,
|
||||
) -> Result<(), String> {
|
||||
// Keep the large export state machine on the heap. Besides making the
|
||||
// caller future small, this prevents the metadata-prefetch locals from
|
||||
// exhausting the bounded stack used by test and runtime worker threads.
|
||||
Box::pin(export_database_sql_core_inner(state, request, on_progress)).await
|
||||
}
|
||||
|
||||
async fn export_database_sql_core_inner(
|
||||
state: &crate::connection::AppState,
|
||||
request: &DatabaseExportRequest,
|
||||
on_progress: impl Fn(ExportProgress) + Sync,
|
||||
) -> Result<(), String> {
|
||||
let _snapshot_keep_alive = if let Some(snapshot_session_id) = request.snapshot_session_id.as_deref() {
|
||||
Some(crate::query::keep_manual_transaction_alive(state, snapshot_session_id).await?)
|
||||
@@ -1869,12 +1880,15 @@ pub async fn export_database_sql_core(
|
||||
|
||||
// 2. Get pool
|
||||
let client_session_id = database_export_client_session_id(&request.export_id);
|
||||
let pool_key = state
|
||||
.get_or_create_pool_for_session(&request.connection_id, Some(&request.database), Some(&client_session_id))
|
||||
.await?;
|
||||
let pool_key = Box::pin(state.get_or_create_pool_for_session(
|
||||
&request.connection_id,
|
||||
Some(&request.database),
|
||||
Some(&client_session_id),
|
||||
))
|
||||
.await?;
|
||||
|
||||
// 3. List tables
|
||||
let all_tables = crate::schema::list_tables_core(
|
||||
let all_tables = Box::pin(crate::schema::list_tables_core(
|
||||
state,
|
||||
&request.connection_id,
|
||||
&request.database,
|
||||
@@ -1884,7 +1898,7 @@ pub async fn export_database_sql_core(
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
))
|
||||
.await?;
|
||||
// 4. Create file
|
||||
let mut expected_destination_dev = None;
|
||||
|
||||
@@ -52,6 +52,7 @@ pub mod victoriametrics_driver;
|
||||
pub mod wkb;
|
||||
|
||||
use reqwest::ClientBuilder;
|
||||
use std::fmt;
|
||||
use std::future::Future;
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -62,6 +63,62 @@ pub use file_validator::validate_file_path;
|
||||
pub const CONNECTION_TIMEOUT_SECS: u64 = 5;
|
||||
pub const TCP_PROBE_TIMEOUT_SECS: u64 = 3;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum PoolCheckoutStage {
|
||||
Wait,
|
||||
Create,
|
||||
Recycle,
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl PoolCheckoutStage {
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Wait => "wait",
|
||||
Self::Create => "create",
|
||||
Self::Recycle => "recycle",
|
||||
Self::Unknown => "unknown",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum PoolCheckoutError {
|
||||
Timeout { database: &'static str, stage: PoolCheckoutStage, timeout: Duration },
|
||||
Failed { database: &'static str, stage: PoolCheckoutStage, detail: String },
|
||||
Canceled,
|
||||
}
|
||||
|
||||
impl PoolCheckoutError {
|
||||
pub fn stage(&self) -> Option<PoolCheckoutStage> {
|
||||
match self {
|
||||
Self::Timeout { stage, .. } | Self::Failed { stage, .. } => Some(*stage),
|
||||
Self::Canceled => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_pool_saturation(&self) -> bool {
|
||||
matches!(self, Self::Timeout { stage: PoolCheckoutStage::Wait, .. })
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for PoolCheckoutError {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Timeout { database, stage, timeout } => write!(
|
||||
formatter,
|
||||
"{database} connection pool checkout timed out [stage={}, timeout_ms={}]",
|
||||
stage.as_str(),
|
||||
timeout.as_millis()
|
||||
),
|
||||
Self::Failed { database, stage, detail } => {
|
||||
write!(formatter, "{database} connection pool checkout failed [stage={}]: {detail}", stage.as_str())
|
||||
}
|
||||
Self::Canceled => formatter.write_str(crate::query::QUERY_CANCELED),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn connection_timeout() -> Duration {
|
||||
Duration::from_secs(CONNECTION_TIMEOUT_SECS)
|
||||
}
|
||||
|
||||
+270
-25
@@ -9,7 +9,9 @@ use sqlparser::dialect::MySqlDialect;
|
||||
use sqlparser::parser::Parser;
|
||||
use std::borrow::Cow;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
#[cfg(test)]
|
||||
use std::future::Future;
|
||||
use std::ops::Deref;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::time::Duration;
|
||||
@@ -28,7 +30,62 @@ use crate::types::{
|
||||
|
||||
use super::file_validator::validate_file_path;
|
||||
|
||||
pub type MySqlPool = mysql_async::Pool;
|
||||
/// DBX-owned MySQL pool handle. The driver does not expose its configured
|
||||
/// maximum, so retain that value beside the driver pool for checkout phase
|
||||
/// classification instead of guessing it from aggregate metrics.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MySqlPool {
|
||||
inner: mysql_async::Pool,
|
||||
max_connections: usize,
|
||||
}
|
||||
|
||||
impl MySqlPool {
|
||||
pub(crate) fn new<O>(opts: O, max_connections: usize) -> Self
|
||||
where
|
||||
mysql_async::Opts: TryFrom<O>,
|
||||
<mysql_async::Opts as TryFrom<O>>::Error: std::error::Error,
|
||||
{
|
||||
Self { inner: mysql_async::Pool::new(opts), max_connections: max_connections.max(1) }
|
||||
}
|
||||
|
||||
pub async fn disconnect(self) -> Result<(), mysql_async::Error> {
|
||||
self.inner.disconnect().await
|
||||
}
|
||||
}
|
||||
|
||||
impl Deref for MySqlPool {
|
||||
type Target = mysql_async::Pool;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.inner
|
||||
}
|
||||
}
|
||||
|
||||
#[doc(hidden)]
|
||||
pub trait MySqlPoolAccess {
|
||||
fn driver_pool(&self) -> &mysql_async::Pool;
|
||||
fn checkout_max_connections(&self) -> Option<usize>;
|
||||
}
|
||||
|
||||
impl MySqlPoolAccess for MySqlPool {
|
||||
fn driver_pool(&self) -> &mysql_async::Pool {
|
||||
&self.inner
|
||||
}
|
||||
|
||||
fn checkout_max_connections(&self) -> Option<usize> {
|
||||
Some(self.max_connections)
|
||||
}
|
||||
}
|
||||
|
||||
impl MySqlPoolAccess for mysql_async::Pool {
|
||||
fn driver_pool(&self) -> &mysql_async::Pool {
|
||||
self
|
||||
}
|
||||
|
||||
fn checkout_max_connections(&self) -> Option<usize> {
|
||||
None
|
||||
}
|
||||
}
|
||||
const MYSQL_TCP_KEEPALIVE_MS: u32 = 30_000;
|
||||
const MYSQL_SQL_PACKET_MARGIN_MAX_BYTES: usize = 64 * 1024;
|
||||
|
||||
@@ -1242,7 +1299,7 @@ fn create_pool(
|
||||
// to paths explicitly supplied by the user instead of enabling arbitrary reads.
|
||||
builder = builder.local_infile_handler(Some(mysql_async::WhiteListFsHandler::new(local_infile_paths)));
|
||||
}
|
||||
Ok(MySqlPool::new(builder))
|
||||
Ok(MySqlPool::new(builder, max_connections))
|
||||
}
|
||||
|
||||
fn mysql_async_tcp_host(host: &str) -> &str {
|
||||
@@ -3603,7 +3660,10 @@ fn parse_mysql_enum_values(column_type: &str) -> Option<Vec<String>> {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_columns(pool: &MySqlPool, database: &str, table: &str) -> Result<Vec<ColumnInfo>, String> {
|
||||
pub async fn get_columns<P>(pool: &P, database: &str, table: &str) -> Result<Vec<ColumnInfo>, String>
|
||||
where
|
||||
P: MySqlPoolAccess + ?Sized,
|
||||
{
|
||||
let sql = columns_sql(database, table);
|
||||
let mut conn = get_conn_with_health_check(pool).await?;
|
||||
let result = match conn.query_iter(&sql).await {
|
||||
@@ -3675,7 +3735,10 @@ pub async fn get_columns(pool: &MySqlPool, database: &str, table: &str) -> Resul
|
||||
Ok(columns)
|
||||
}
|
||||
|
||||
pub async fn get_columns_show(pool: &MySqlPool, database: &str, table: &str) -> Result<Vec<ColumnInfo>, String> {
|
||||
pub async fn get_columns_show<P>(pool: &P, database: &str, table: &str) -> Result<Vec<ColumnInfo>, String>
|
||||
where
|
||||
P: MySqlPoolAccess + ?Sized,
|
||||
{
|
||||
let sql = show_columns_sql(database, table, true);
|
||||
let mut conn = get_conn_with_health_check(pool).await?;
|
||||
let rows: Vec<mysql_async::Row> = match conn.query_iter(&sql).await {
|
||||
@@ -3852,23 +3915,32 @@ pub(super) fn skip_mysql_quoted(sql: &str, start: usize, quote: u8) -> usize {
|
||||
|
||||
/// Get a connection from the pool with a health check. If the connection is dead
|
||||
/// (e.g. after app was backgrounded), it tries again with a fresh connection.
|
||||
pub async fn get_conn_with_health_check(pool: &MySqlPool) -> Result<mysql_async::Conn, String> {
|
||||
pub async fn get_conn_with_health_check<P>(pool: &P) -> Result<mysql_async::Conn, String>
|
||||
where
|
||||
P: MySqlPoolAccess + ?Sized,
|
||||
{
|
||||
get_conn_with_health_check_with_timeout(pool, super::connection_timeout()).await
|
||||
}
|
||||
|
||||
pub async fn get_conn_with_health_check_with_timeout(
|
||||
pool: &MySqlPool,
|
||||
pub async fn get_conn_with_health_check_with_timeout<P>(
|
||||
pool: &P,
|
||||
timeout: Duration,
|
||||
) -> Result<mysql_async::Conn, String> {
|
||||
) -> Result<mysql_async::Conn, String>
|
||||
where
|
||||
P: MySqlPoolAccess + ?Sized,
|
||||
{
|
||||
get_conn_with_health_check_with_cancel(pool, timeout, timeout, None).await
|
||||
}
|
||||
|
||||
pub async fn get_conn_with_health_check_with_cancel(
|
||||
pool: &MySqlPool,
|
||||
pub async fn get_conn_with_health_check_with_cancel<P>(
|
||||
pool: &P,
|
||||
timeout: Duration,
|
||||
cleanup_timeout: Duration,
|
||||
cancel_token: Option<&CancellationToken>,
|
||||
) -> Result<mysql_async::Conn, String> {
|
||||
) -> Result<mysql_async::Conn, String>
|
||||
where
|
||||
P: MySqlPoolAccess + ?Sized,
|
||||
{
|
||||
let start = Instant::now();
|
||||
let mut conn = get_conn_with_timeout_and_cancel(pool, timeout, cancel_token).await?;
|
||||
match ping_conn_with_timeout_and_cancel(&mut conn, timeout, cancel_token).await {
|
||||
@@ -3909,13 +3981,67 @@ pub async fn get_conn_with_health_check_with_cancel(
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_conn_with_timeout_and_cancel(
|
||||
pool: &MySqlPool,
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) struct MysqlCheckoutSnapshot {
|
||||
pub(crate) connection_count: usize,
|
||||
pub(crate) connections_in_pool: usize,
|
||||
pub(crate) max_connections: usize,
|
||||
}
|
||||
|
||||
pub(crate) type MysqlCheckoutStage = super::PoolCheckoutStage;
|
||||
|
||||
pub(crate) fn classify_mysql_checkout_stage(snapshot: MysqlCheckoutSnapshot) -> MysqlCheckoutStage {
|
||||
if snapshot.connections_in_pool > 0 {
|
||||
MysqlCheckoutStage::Recycle
|
||||
} else if snapshot.connection_count < snapshot.max_connections {
|
||||
MysqlCheckoutStage::Create
|
||||
} else {
|
||||
MysqlCheckoutStage::Wait
|
||||
}
|
||||
}
|
||||
|
||||
fn mysql_checkout_snapshot<P>(pool: &P, max_connections: usize) -> MysqlCheckoutSnapshot
|
||||
where
|
||||
P: MySqlPoolAccess + ?Sized,
|
||||
{
|
||||
let metrics = pool.driver_pool().metrics();
|
||||
use std::sync::atomic::Ordering;
|
||||
MysqlCheckoutSnapshot {
|
||||
connection_count: metrics.connection_count.load(Ordering::Relaxed),
|
||||
connections_in_pool: metrics.connections_in_pool.load(Ordering::Relaxed),
|
||||
max_connections,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn checkout_mysql_conn<P>(
|
||||
pool: &P,
|
||||
timeout: Duration,
|
||||
) -> Result<mysql_async::Conn, super::PoolCheckoutError>
|
||||
where
|
||||
P: MySqlPoolAccess + ?Sized,
|
||||
{
|
||||
// Capture the phase before entering the driver future. The phase is tied
|
||||
// to this checkout's capacity decision, not to a later aggregate metric
|
||||
// snapshot that may already include another request.
|
||||
let stage = pool.checkout_max_connections().map_or(MysqlCheckoutStage::Unknown, |max_connections| {
|
||||
classify_mysql_checkout_stage(mysql_checkout_snapshot(pool, max_connections))
|
||||
});
|
||||
match tokio::time::timeout(timeout, pool.driver_pool().get_conn()).await {
|
||||
Ok(Ok(conn)) => Ok(conn),
|
||||
Ok(Err(error)) => Err(super::PoolCheckoutError::Failed { database: "MySQL", stage, detail: error.to_string() }),
|
||||
Err(_) => Err(super::PoolCheckoutError::Timeout { database: "MySQL", stage, timeout }),
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_conn_with_timeout_and_cancel<P>(
|
||||
pool: &P,
|
||||
timeout: Duration,
|
||||
cancel_token: Option<&CancellationToken>,
|
||||
) -> Result<mysql_async::Conn, String> {
|
||||
let get_future =
|
||||
connection_result_with_timeout(timeout, async { pool.get_conn().await.map_err(|error| error.to_string()) });
|
||||
) -> Result<mysql_async::Conn, String>
|
||||
where
|
||||
P: MySqlPoolAccess + ?Sized,
|
||||
{
|
||||
let get_future = async { checkout_mysql_conn(pool, timeout).await.map_err(|error| error.to_string()) };
|
||||
|
||||
match cancel_token {
|
||||
Some(token) => {
|
||||
@@ -3929,10 +4055,14 @@ async fn get_conn_with_timeout_and_cancel(
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_conn_with_timeout(pool: &MySqlPool, timeout: Duration) -> Result<mysql_async::Conn, String> {
|
||||
connection_result_with_timeout(timeout, async { pool.get_conn().await.map_err(|error| error.to_string()) }).await
|
||||
pub async fn get_conn_with_timeout<P>(pool: &P, timeout: Duration) -> Result<mysql_async::Conn, String>
|
||||
where
|
||||
P: MySqlPoolAccess + ?Sized,
|
||||
{
|
||||
checkout_mysql_conn(pool, timeout).await.map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn connection_result_with_timeout<T, F>(timeout: Duration, future: F) -> Result<T, String>
|
||||
where
|
||||
F: Future<Output = Result<T, String>>,
|
||||
@@ -4564,13 +4694,16 @@ pub(crate) fn mysql_sql_statement_hard_limit(max_allowed_packet: u64) -> Option<
|
||||
packet_bytes.checked_sub(margin).filter(|limit| *limit > 0)
|
||||
}
|
||||
|
||||
pub async fn execute_query_with_max_rows(
|
||||
pool: &MySqlPool,
|
||||
pub async fn execute_query_with_max_rows<P>(
|
||||
pool: &P,
|
||||
sql: &str,
|
||||
bare: bool,
|
||||
max_rows: Option<usize>,
|
||||
dialect: MySqlQueryDialect,
|
||||
) -> Result<QueryResult, String> {
|
||||
) -> Result<QueryResult, String>
|
||||
where
|
||||
P: MySqlPoolAccess + ?Sized,
|
||||
{
|
||||
let mut conn = get_conn_with_health_check(pool).await?;
|
||||
execute_query_on_conn_with_max_rows(&mut conn, sql, bare, max_rows, dialect).await
|
||||
}
|
||||
@@ -5357,10 +5490,10 @@ mod tests {
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
Ok::<_, String>("connection")
|
||||
});
|
||||
let configured_five = connection_result_with_timeout(Duration::from_millis(50), async {
|
||||
tokio::time::sleep(Duration::from_millis(60)).await;
|
||||
Ok::<_, String>("connection")
|
||||
});
|
||||
let configured_five = connection_result_with_timeout(
|
||||
Duration::from_millis(10),
|
||||
std::future::pending::<Result<&'static str, String>>(),
|
||||
);
|
||||
|
||||
let (exact, adjacent, configured_five) = tokio::join!(exact, adjacent, configured_five);
|
||||
|
||||
@@ -5369,6 +5502,118 @@ mod tests {
|
||||
assert_eq!(configured_five, Err("MySQL get connection timed out".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mysql_checkout_fault_injection_classifies_full_pool_as_wait() {
|
||||
assert!(tokio::time::timeout(Duration::from_millis(1), std::future::pending::<()>()).await.is_err());
|
||||
assert_eq!(
|
||||
classify_mysql_checkout_stage(MysqlCheckoutSnapshot {
|
||||
connection_count: 1,
|
||||
connections_in_pool: 0,
|
||||
max_connections: 1,
|
||||
}),
|
||||
MysqlCheckoutStage::Wait
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mysql_checkout_fault_injection_classifies_hung_connection_create() {
|
||||
assert!(tokio::time::timeout(Duration::from_millis(1), std::future::pending::<()>()).await.is_err());
|
||||
assert_eq!(
|
||||
classify_mysql_checkout_stage(MysqlCheckoutSnapshot {
|
||||
connection_count: 0,
|
||||
connections_in_pool: 0,
|
||||
max_connections: 2,
|
||||
}),
|
||||
MysqlCheckoutStage::Create
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mysql_checkout_fault_injection_preserves_create_stage_when_handshake_hangs() {
|
||||
let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)).await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
let (_socket, _) = listener.accept().await.unwrap();
|
||||
std::future::pending::<()>().await;
|
||||
});
|
||||
let pool_options =
|
||||
mysql_async::PoolOpts::new().with_constraints(mysql_async::PoolConstraints::new(1, 2).unwrap());
|
||||
let options = mysql_async::OptsBuilder::default()
|
||||
.ip_or_hostname(address.ip().to_string())
|
||||
.tcp_port(address.port())
|
||||
.user(Some("fault-injection"))
|
||||
.pass(Some("fault-injection"))
|
||||
.pool_opts(Some(pool_options));
|
||||
let pool = MySqlPool::new(options, 2);
|
||||
|
||||
let error = checkout_mysql_conn(&pool, Duration::from_millis(50)).await.unwrap_err();
|
||||
|
||||
assert!(matches!(error, super::super::PoolCheckoutError::Timeout { stage: MysqlCheckoutStage::Create, .. }));
|
||||
server.abort();
|
||||
let _ = tokio::time::timeout(Duration::from_secs(1), pool.disconnect()).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mysql_checkout_raw_driver_pool_timeout_keeps_stage_unknown() {
|
||||
let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)).await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
let (_socket, _) = listener.accept().await.unwrap();
|
||||
std::future::pending::<()>().await;
|
||||
});
|
||||
let options = mysql_async::OptsBuilder::default()
|
||||
.ip_or_hostname(address.ip().to_string())
|
||||
.tcp_port(address.port())
|
||||
.user(Some("fault-injection"))
|
||||
.pass(Some("fault-injection"));
|
||||
let pool = mysql_async::Pool::new(options);
|
||||
|
||||
let error = checkout_mysql_conn(&pool, Duration::from_millis(50)).await.unwrap_err();
|
||||
|
||||
assert!(matches!(error, super::super::PoolCheckoutError::Timeout { stage: MysqlCheckoutStage::Unknown, .. }));
|
||||
server.abort();
|
||||
let _ = tokio::time::timeout(Duration::from_secs(1), pool.disconnect()).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DBX_REVIEW_MYSQL_* environment variables"]
|
||||
async fn mysql_checkout_fault_injection_preserves_wait_stage_when_real_pool_is_full() {
|
||||
let host = std::env::var("DBX_REVIEW_MYSQL_HOST").expect("DBX_REVIEW_MYSQL_HOST is required");
|
||||
let port = std::env::var("DBX_REVIEW_MYSQL_PORT").expect("DBX_REVIEW_MYSQL_PORT is required");
|
||||
let user = std::env::var("DBX_REVIEW_MYSQL_USER").expect("DBX_REVIEW_MYSQL_USER is required");
|
||||
let password = std::env::var("DBX_REVIEW_MYSQL_PASSWORD").expect("DBX_REVIEW_MYSQL_PASSWORD is required");
|
||||
let database = std::env::var("DBX_REVIEW_MYSQL_DATABASE").expect("DBX_REVIEW_MYSQL_DATABASE is required");
|
||||
let pool_options =
|
||||
mysql_async::PoolOpts::new().with_constraints(mysql_async::PoolConstraints::new(1, 1).unwrap());
|
||||
let options = mysql_async::OptsBuilder::default()
|
||||
.ip_or_hostname(host)
|
||||
.tcp_port(port.parse().expect("DBX_REVIEW_MYSQL_PORT must be a valid port"))
|
||||
.user(Some(user))
|
||||
.pass(Some(password))
|
||||
.db_name(Some(database))
|
||||
.pool_opts(Some(pool_options));
|
||||
let pool = MySqlPool::new(options, 1);
|
||||
let held_connection = tokio::time::timeout(Duration::from_secs(10), pool.get_conn())
|
||||
.await
|
||||
.expect("real MySQL checkout timed out")
|
||||
.expect("real MySQL checkout failed");
|
||||
|
||||
let error = checkout_mysql_conn(&pool, Duration::from_millis(50)).await.unwrap_err();
|
||||
|
||||
assert!(matches!(error, super::super::PoolCheckoutError::Timeout { stage: MysqlCheckoutStage::Wait, .. }));
|
||||
drop(held_connection);
|
||||
let _ = tokio::time::timeout(Duration::from_secs(2), pool.disconnect()).await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mysql_public_metadata_and_query_helpers_accept_driver_pool_for_compatibility() {
|
||||
let pool = mysql_async::Pool::new(mysql_async::OptsBuilder::default());
|
||||
let columns = get_columns(&pool, "database", "table");
|
||||
let query = execute_query_with_max_rows(&pool, "SELECT 1", false, Some(1), MySqlQueryDialect::default());
|
||||
|
||||
drop((columns, query));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mysql_database_list_timeout_preserves_immediate_errors_and_fallback_plan() {
|
||||
let error =
|
||||
|
||||
@@ -5900,40 +5900,70 @@ async fn execute_postgres_user_query(
|
||||
|
||||
/// PostgreSQL pool checkout with timeout and cancel token support.
|
||||
/// When the checkout phase is stuck, the cancel token can terminate the wait early.
|
||||
/// The timeout error message includes "checkout timed out" to ensure is_connection_error can classify it correctly.
|
||||
pub async fn checkout_postgres_client(
|
||||
pub(crate) type PostgresCheckoutStage = super::PoolCheckoutStage;
|
||||
|
||||
pub(crate) fn classify_postgres_checkout_stage(status: deadpool_postgres::Status) -> PostgresCheckoutStage {
|
||||
if status.waiting > 0 || (status.available == 0 && status.size >= status.max_size) {
|
||||
PostgresCheckoutStage::Wait
|
||||
} else if status.available > 0 {
|
||||
PostgresCheckoutStage::Recycle
|
||||
} else if status.size < status.max_size {
|
||||
PostgresCheckoutStage::Create
|
||||
} else {
|
||||
PostgresCheckoutStage::Unknown
|
||||
}
|
||||
}
|
||||
|
||||
fn postgres_pool_error_stage(error: &PoolError) -> PostgresCheckoutStage {
|
||||
match error {
|
||||
PoolError::Timeout(deadpool_postgres::TimeoutType::Wait) => PostgresCheckoutStage::Wait,
|
||||
PoolError::Timeout(deadpool_postgres::TimeoutType::Create) => PostgresCheckoutStage::Create,
|
||||
PoolError::Timeout(deadpool_postgres::TimeoutType::Recycle) => PostgresCheckoutStage::Recycle,
|
||||
PoolError::Backend(_) | PoolError::PostCreateHook(_) => PostgresCheckoutStage::Create,
|
||||
PoolError::Closed | PoolError::NoRuntimeSpecified => PostgresCheckoutStage::Unknown,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn checkout_postgres_client_classified(
|
||||
pool: &Pool,
|
||||
cancel_token: Option<&CancellationToken>,
|
||||
checkout_timeout: Duration,
|
||||
) -> Result<deadpool_postgres::Object, String> {
|
||||
let checkout_timeout = effective_postgres_checkout_timeout(pool, checkout_timeout);
|
||||
) -> Result<deadpool_postgres::Object, super::PoolCheckoutError> {
|
||||
let start = Instant::now();
|
||||
let get_future = async {
|
||||
tokio::time::timeout(checkout_timeout, pool.get())
|
||||
.await
|
||||
.map_err(|_| {
|
||||
let elapsed = start.elapsed().as_millis();
|
||||
match tokio::time::timeout(checkout_timeout, pool.get()).await {
|
||||
Ok(Ok(client)) => Ok(client),
|
||||
Ok(Err(error)) => {
|
||||
let stage = postgres_pool_error_stage(&error);
|
||||
let timed_out = matches!(&error, PoolError::Timeout(_));
|
||||
let detail = pg_pool_error_to_string(error);
|
||||
log::warn!(
|
||||
"[db:pool.checkout:error] elapsed_ms={} timeout_ms={} error=checkout timed out",
|
||||
elapsed,
|
||||
checkout_timeout.as_millis()
|
||||
);
|
||||
format!("PostgreSQL connection pool checkout timed out ({}s)", checkout_timeout.as_secs())
|
||||
})?
|
||||
.map_err(|e| {
|
||||
let elapsed = start.elapsed().as_millis();
|
||||
let err = pg_pool_error_to_string(e);
|
||||
log::warn!(
|
||||
"[db:pool.checkout:error] elapsed_ms={} timeout_ms={} error={}",
|
||||
elapsed,
|
||||
"[db:pool.checkout:error] elapsed_ms={} timeout_ms={} stage={} error={}",
|
||||
start.elapsed().as_millis(),
|
||||
checkout_timeout.as_millis(),
|
||||
err
|
||||
stage.as_str(),
|
||||
detail
|
||||
);
|
||||
format!("PostgreSQL connection pool checkout failed: {err}")
|
||||
})
|
||||
if timed_out {
|
||||
Err(super::PoolCheckoutError::Timeout { database: "PostgreSQL", stage, timeout: checkout_timeout })
|
||||
} else {
|
||||
Err(super::PoolCheckoutError::Failed { database: "PostgreSQL", stage, detail })
|
||||
}
|
||||
}
|
||||
Err(_) => {
|
||||
let stage = classify_postgres_checkout_stage(pool.status());
|
||||
log::warn!(
|
||||
"[db:pool.checkout:error] elapsed_ms={} timeout_ms={} stage={} error=checkout timed out",
|
||||
start.elapsed().as_millis(),
|
||||
checkout_timeout.as_millis(),
|
||||
stage.as_str()
|
||||
);
|
||||
Err(super::PoolCheckoutError::Timeout { database: "PostgreSQL", stage, timeout: checkout_timeout })
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let result = match cancel_token {
|
||||
match cancel_token {
|
||||
Some(token) => tokio::select! {
|
||||
biased;
|
||||
_ = token.cancelled() => {
|
||||
@@ -5942,12 +5972,24 @@ pub async fn checkout_postgres_client(
|
||||
start.elapsed().as_millis(),
|
||||
checkout_timeout.as_millis()
|
||||
);
|
||||
return Err(crate::query::canceled_error());
|
||||
Err(super::PoolCheckoutError::Canceled)
|
||||
}
|
||||
result = get_future => result,
|
||||
},
|
||||
None => get_future.await,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn checkout_postgres_client(
|
||||
pool: &Pool,
|
||||
cancel_token: Option<&CancellationToken>,
|
||||
checkout_timeout: Duration,
|
||||
) -> Result<deadpool_postgres::Object, String> {
|
||||
let checkout_timeout = effective_postgres_checkout_timeout(pool, checkout_timeout);
|
||||
let start = Instant::now();
|
||||
let result = checkout_postgres_client_classified(pool, cancel_token, checkout_timeout)
|
||||
.await
|
||||
.map_err(|error| error.to_string());
|
||||
if let Ok(client) = &result {
|
||||
log::debug!(
|
||||
"[db:pool.checkout:done] elapsed_ms={} timeout_ms={}",
|
||||
@@ -8763,6 +8805,43 @@ mod tests {
|
||||
assert_eq!(effective_postgres_checkout_timeout(&pool, Duration::from_secs(15)), Duration::from_secs(15));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn postgres_checkout_fault_injection_classifies_full_pool_as_wait() {
|
||||
assert!(tokio::time::timeout(Duration::from_millis(1), std::future::pending::<()>()).await.is_err());
|
||||
assert_eq!(
|
||||
classify_postgres_checkout_stage(deadpool_postgres::Status {
|
||||
max_size: 1,
|
||||
size: 1,
|
||||
available: 0,
|
||||
waiting: 1
|
||||
}),
|
||||
PostgresCheckoutStage::Wait
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn postgres_checkout_fault_injection_preserves_create_and_recycle_stages() {
|
||||
assert!(tokio::time::timeout(Duration::from_millis(1), std::future::pending::<()>()).await.is_err());
|
||||
assert_eq!(
|
||||
classify_postgres_checkout_stage(deadpool_postgres::Status {
|
||||
max_size: 2,
|
||||
size: 1,
|
||||
available: 0,
|
||||
waiting: 0
|
||||
}),
|
||||
PostgresCheckoutStage::Create
|
||||
);
|
||||
assert_eq!(
|
||||
classify_postgres_checkout_stage(deadpool_postgres::Status {
|
||||
max_size: 1,
|
||||
size: 1,
|
||||
available: 1,
|
||||
waiting: 0
|
||||
}),
|
||||
PostgresCheckoutStage::Recycle
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn postgres_session_setup_reports_its_own_timeout_stage() {
|
||||
let error =
|
||||
|
||||
@@ -32,6 +32,16 @@ pub const QUERY_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
pub const MAX_ROWS: usize = 10000;
|
||||
pub const AGENT_PROTOCOL_MAX_ROWS: usize = i32::MAX as usize;
|
||||
pub const QUERY_CANCELED: &str = "Query canceled";
|
||||
pub const METADATA_POOL_BUSY_ERROR: &str = "DBX metadata pool is busy; please retry";
|
||||
|
||||
/// Returns true when a metadata request failed because all pool/client slots
|
||||
/// were temporarily occupied. This is deliberately separate from
|
||||
/// `is_connection_error`: a saturated pool is healthy and must not trigger a
|
||||
/// shared-pool reconnect.
|
||||
pub fn is_pool_saturation_error(err: &str) -> bool {
|
||||
let lower = err.to_lowercase();
|
||||
lower.contains("connection pool checkout timed out [stage=wait") || lower.contains("dbx metadata pool is busy")
|
||||
}
|
||||
/// Fallback when a Mongo connection hits the generic SQL executor instead of the shell path.
|
||||
/// Wording must match packages/mongo-shell `MONGO_SHELL_COMMAND_HINT`
|
||||
/// (desktop/CLI diagnose first; this is only the Rust SQL-executor backstop).
|
||||
@@ -1070,7 +1080,7 @@ pub fn agent_close_query_session_params(session_id: &str) -> serde_json::Value {
|
||||
|
||||
pub fn is_connection_error(err: &str) -> bool {
|
||||
let lower = err.to_lowercase();
|
||||
if is_dbx_query_timeout_error(&lower) || is_agent_rpc_timeout_error(&lower) {
|
||||
if is_dbx_query_timeout_error(&lower) || is_agent_rpc_timeout_error(&lower) || is_pool_saturation_error(err) {
|
||||
return false;
|
||||
}
|
||||
lower.contains("connection")
|
||||
@@ -3849,7 +3859,7 @@ pub async fn execute_statements_in_transaction_on_pool_typed(
|
||||
/// Owned pool variants for safe dispatch across async boundaries.
|
||||
enum TxPath {
|
||||
Pg(deadpool_postgres::Pool),
|
||||
Mysql(mysql_async::Pool, bool),
|
||||
Mysql(db::mysql::MySqlPool, bool),
|
||||
Sqlite(db::sqlite::SqliteHandle),
|
||||
CloudflareD1(db::cloudflare_d1_driver::CloudflareD1Client),
|
||||
Agent(Arc<crate::db::agent_driver::PooledAgentClient>),
|
||||
@@ -3962,7 +3972,7 @@ async fn exec_tx_pg_statements(
|
||||
async fn exec_tx_mysql_inner(
|
||||
state: &AppState,
|
||||
pool_key: &str,
|
||||
pool: mysql_async::Pool,
|
||||
pool: db::mysql::MySqlPool,
|
||||
statements: &[String],
|
||||
start: std::time::Instant,
|
||||
budget: DbOperationBudget,
|
||||
@@ -6905,17 +6915,43 @@ for line in sys.stdin:
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_connection_error_detects_deadpool_pool_timeouts() {
|
||||
fn is_connection_error_distinguishes_checkout_saturation_from_connection_timeouts() {
|
||||
// deadpool-postgres PoolError::Timeout messages (contain "pool" + "timeout" but not "timed out")
|
||||
assert!(is_connection_error("pool wait timeout"));
|
||||
assert!(is_connection_error("pool create timeout"));
|
||||
assert!(is_connection_error("pool recycle timeout"));
|
||||
// checkout helper timeout messages
|
||||
assert!(is_connection_error("PostgreSQL connection pool checkout timed out (5s)"));
|
||||
assert!(is_connection_error("MySQL get connection timed out"));
|
||||
assert!(!is_connection_error("PostgreSQL connection pool checkout timed out [stage=wait, timeout_ms=5000]"));
|
||||
assert!(is_connection_error("PostgreSQL connection pool checkout timed out [stage=create, timeout_ms=5000]"));
|
||||
assert!(is_connection_error("PostgreSQL connection pool checkout timed out [stage=recycle, timeout_ms=5000]"));
|
||||
assert!(!is_connection_error("MySQL connection pool checkout timed out [stage=wait, timeout_ms=5000]"));
|
||||
assert!(!is_connection_error(METADATA_POOL_BUSY_ERROR));
|
||||
assert!(is_connection_error("MySQL ping timed out"));
|
||||
assert!(is_connection_error("MySQL kill connection checkout timed out"));
|
||||
assert!(is_connection_error("MySQL KILL QUERY timed out"));
|
||||
|
||||
assert!(is_pool_saturation_error(
|
||||
"PostgreSQL connection pool checkout timed out [stage=wait, timeout_ms=5000]"
|
||||
));
|
||||
assert!(is_pool_saturation_error("MySQL connection pool checkout timed out [stage=wait, timeout_ms=5000]"));
|
||||
assert!(!is_pool_saturation_error(
|
||||
"PostgreSQL connection pool checkout timed out [stage=create, timeout_ms=5000]"
|
||||
));
|
||||
assert!(is_pool_saturation_error(METADATA_POOL_BUSY_ERROR));
|
||||
assert_eq!(
|
||||
pool_error_action(
|
||||
Some(DatabaseType::Mysql),
|
||||
"MySQL connection pool checkout timed out [stage=wait, timeout_ms=5000]"
|
||||
),
|
||||
PoolErrorAction::Keep
|
||||
);
|
||||
assert_eq!(
|
||||
pool_error_action(
|
||||
Some(DatabaseType::Postgres),
|
||||
"PostgreSQL connection pool checkout timed out [stage=wait, timeout_ms=5000]"
|
||||
),
|
||||
PoolErrorAction::Keep
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
+315
-131
@@ -1,7 +1,7 @@
|
||||
use crate::agent_recovery::{RecoveryDecision, RecoveryPolicy, RecoveryScope};
|
||||
use crate::connection::{
|
||||
connection_url_for_endpoint, database_connection_config, gaussdb_uses_m_jdbc_driver, task_client_session_id,
|
||||
AppState, MysqlMode, PoolKind,
|
||||
uses_metadata_gate, AppState, MysqlMode, PoolKind, METADATA_POOL_ACQUIRE_TIMEOUT,
|
||||
};
|
||||
use crate::db;
|
||||
use crate::models::connection::{ConnectionConfig, DatabaseType};
|
||||
@@ -33,6 +33,10 @@ macro_rules! dispatch_mysql {
|
||||
};
|
||||
}
|
||||
|
||||
async fn clone_metadata_pool(state: &AppState, pool_key: &str) -> Option<PoolKind> {
|
||||
state.connections.read().await.get(pool_key).and_then(PoolKind::clone_for_metadata)
|
||||
}
|
||||
|
||||
struct EphemeralAgentMetadataSession {
|
||||
client_session_id: Option<String>,
|
||||
cleanup_guard: Option<crate::connection::ClientSessionPoolCleanupGuard>,
|
||||
@@ -68,12 +72,25 @@ macro_rules! try_sqlserver {
|
||||
($connections:expr, $pool_key:expr, $method:ident $(, $arg:expr)*) => {
|
||||
if let Some(client) = extract_pool!(&$connections, $pool_key, SqlServer) {
|
||||
drop($connections);
|
||||
let mut client = client.lock().await;
|
||||
let mut client = lock_sqlserver_metadata_client(&client).await?;
|
||||
return db::sqlserver::$method(&mut client $(, $arg)*).await;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async fn lock_sqlserver_metadata_client<'a>(
|
||||
client: &'a Arc<tokio::sync::Mutex<db::sqlserver::SqlServerClient>>,
|
||||
) -> Result<tokio::sync::MutexGuard<'a, db::sqlserver::SqlServerClient>, String> {
|
||||
lock_metadata_mutex_with_timeout(client, METADATA_POOL_ACQUIRE_TIMEOUT).await
|
||||
}
|
||||
|
||||
async fn lock_metadata_mutex_with_timeout<T>(
|
||||
client: &Arc<tokio::sync::Mutex<T>>,
|
||||
timeout: Duration,
|
||||
) -> Result<tokio::sync::MutexGuard<'_, T>, String> {
|
||||
tokio::time::timeout(timeout, client.lock()).await.map_err(|_| crate::query::METADATA_POOL_BUSY_ERROR.to_string())
|
||||
}
|
||||
|
||||
const ORACLE_TABLE_COMMENT_BATCH_SIZE: usize = 500;
|
||||
const TDENGINE_COMMENT_SEARCH_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const TDENGINE_COMMENT_SEARCH_CACHE_TTL: Duration = Duration::from_secs(10);
|
||||
@@ -194,6 +211,17 @@ pub async fn list_database_storage_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
database_names: &[String],
|
||||
) -> Result<Vec<db::DatabaseStorageInfo>, String> {
|
||||
retry_metadata_connection(state, connection_id, None, || {
|
||||
list_database_storage_once(state, connection_id, database_names)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn list_database_storage_once(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
database_names: &[String],
|
||||
) -> Result<Vec<db::DatabaseStorageInfo>, String> {
|
||||
const DATABASE_STORAGE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const MAX_DATABASE_STORAGE_NAMES: usize = 2048;
|
||||
@@ -244,10 +272,11 @@ pub async fn list_sqlserver_linked_servers_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
) -> Result<Vec<db::LinkedServerInfo>, String> {
|
||||
let _metadata_permit = state.acquire_metadata_permit(connection_id, None, DatabaseType::SqlServer, None).await?;
|
||||
let connections = state.connections.read().await;
|
||||
if let Some(client) = extract_pool!(&connections, connection_id, SqlServer) {
|
||||
drop(connections);
|
||||
let mut client = client.lock().await;
|
||||
let mut client = lock_sqlserver_metadata_client(&client).await?;
|
||||
return db::sqlserver::list_linked_servers(&mut client).await;
|
||||
}
|
||||
Ok(vec![])
|
||||
@@ -307,10 +336,11 @@ pub async fn list_sqlserver_linked_server_catalogs_core(
|
||||
connection_id: &str,
|
||||
server: &str,
|
||||
) -> Result<Vec<db::DatabaseInfo>, String> {
|
||||
let _metadata_permit = state.acquire_metadata_permit(connection_id, None, DatabaseType::SqlServer, None).await?;
|
||||
let connections = state.connections.read().await;
|
||||
if let Some(client) = extract_pool!(&connections, connection_id, SqlServer) {
|
||||
drop(connections);
|
||||
let mut client = client.lock().await;
|
||||
let mut client = lock_sqlserver_metadata_client(&client).await?;
|
||||
return db::sqlserver::list_linked_server_catalogs(&mut client, server).await;
|
||||
}
|
||||
Ok(vec![])
|
||||
@@ -322,10 +352,11 @@ pub async fn list_sqlserver_linked_server_schemas_core(
|
||||
server: &str,
|
||||
catalog: &str,
|
||||
) -> Result<Vec<String>, String> {
|
||||
let _metadata_permit = state.acquire_metadata_permit(connection_id, None, DatabaseType::SqlServer, None).await?;
|
||||
let connections = state.connections.read().await;
|
||||
if let Some(client) = extract_pool!(&connections, connection_id, SqlServer) {
|
||||
drop(connections);
|
||||
let mut client = client.lock().await;
|
||||
let mut client = lock_sqlserver_metadata_client(&client).await?;
|
||||
return db::sqlserver::list_linked_server_schemas(&mut client, server, catalog).await;
|
||||
}
|
||||
Ok(vec![])
|
||||
@@ -341,10 +372,11 @@ pub async fn list_sqlserver_linked_server_tables_core(
|
||||
limit: Option<usize>,
|
||||
offset: Option<usize>,
|
||||
) -> Result<Vec<db::TableInfo>, String> {
|
||||
let _metadata_permit = state.acquire_metadata_permit(connection_id, None, DatabaseType::SqlServer, None).await?;
|
||||
let connections = state.connections.read().await;
|
||||
if let Some(client) = extract_pool!(&connections, connection_id, SqlServer) {
|
||||
drop(connections);
|
||||
let mut client = client.lock().await;
|
||||
let mut client = lock_sqlserver_metadata_client(&client).await?;
|
||||
return db::sqlserver::list_linked_server_tables(&mut client, server, catalog, schema, filter, limit, offset)
|
||||
.await;
|
||||
}
|
||||
@@ -367,14 +399,17 @@ pub async fn list_sqlserver_linked_server_tables_core(
|
||||
/// use the MySQL protocol, so this is a defensive no-op); the caller's
|
||||
/// flat-sidebar fallback then renders the standard database list.
|
||||
pub async fn list_doris_catalogs_core(state: &AppState, connection_id: &str) -> Result<Vec<db::CatalogInfo>, String> {
|
||||
retry_metadata_connection(state, connection_id, None, || list_doris_catalogs_once(state, connection_id)).await
|
||||
}
|
||||
|
||||
async fn list_doris_catalogs_once(state: &AppState, connection_id: &str) -> Result<Vec<db::CatalogInfo>, String> {
|
||||
let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, None, None).await?;
|
||||
let db_config = connection_config(state, connection_id).await;
|
||||
let connections = state.connections.read().await;
|
||||
if let Some(PoolKind::Mysql(p, _)) = connections.get(&pool_key) {
|
||||
if let Some(PoolKind::Mysql(p, _)) = clone_metadata_pool(state, &pool_key).await {
|
||||
return if db_config.as_ref().is_some_and(db::starrocks::is_config) {
|
||||
db::starrocks::list_catalogs(p).await
|
||||
db::starrocks::list_catalogs(&p).await
|
||||
} else {
|
||||
db::doris::list_catalogs(p).await
|
||||
db::doris::list_catalogs(&p).await
|
||||
};
|
||||
}
|
||||
Ok(vec![])
|
||||
@@ -389,12 +424,22 @@ pub async fn list_doris_catalog_databases_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
catalog: &str,
|
||||
) -> Result<Vec<db::DatabaseInfo>, String> {
|
||||
retry_metadata_connection(state, connection_id, None, || {
|
||||
list_doris_catalog_databases_once(state, connection_id, catalog)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn list_doris_catalog_databases_once(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
catalog: &str,
|
||||
) -> Result<Vec<db::DatabaseInfo>, String> {
|
||||
let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, None, None).await?;
|
||||
let db_config = connection_config(state, connection_id).await;
|
||||
let connections = state.connections.read().await;
|
||||
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
|
||||
let PoolKind::Mysql(p, _) = pool else {
|
||||
let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
|
||||
let PoolKind::Mysql(p, _) = &pool else {
|
||||
return Ok(vec![]);
|
||||
};
|
||||
let databases = if db_config.as_ref().is_some_and(db::starrocks::is_config) {
|
||||
@@ -433,12 +478,38 @@ pub async fn list_doris_catalog_tables_core(
|
||||
offset: Option<usize>,
|
||||
object_types: Option<&[String]>,
|
||||
table_name_filter: Option<&TableNameFilter>,
|
||||
) -> Result<Vec<db::TableInfo>, String> {
|
||||
retry_metadata_connection(state, connection_id, Some(database), || {
|
||||
list_doris_catalog_tables_once(
|
||||
state,
|
||||
connection_id,
|
||||
catalog,
|
||||
database,
|
||||
filter,
|
||||
limit,
|
||||
offset,
|
||||
object_types,
|
||||
table_name_filter,
|
||||
)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn list_doris_catalog_tables_once(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
catalog: &str,
|
||||
database: &str,
|
||||
filter: Option<&str>,
|
||||
limit: Option<usize>,
|
||||
offset: Option<usize>,
|
||||
object_types: Option<&[String]>,
|
||||
table_name_filter: Option<&TableNameFilter>,
|
||||
) -> Result<Vec<db::TableInfo>, String> {
|
||||
let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, None, None).await?;
|
||||
let db_config = connection_config(state, connection_id).await;
|
||||
let connections = state.connections.read().await;
|
||||
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
|
||||
let PoolKind::Mysql(p, _) = pool else {
|
||||
let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
|
||||
let PoolKind::Mysql(p, _) = &pool else {
|
||||
return Ok(vec![]);
|
||||
};
|
||||
let tables = if db_config.as_ref().is_some_and(db::starrocks::is_config) {
|
||||
@@ -456,12 +527,24 @@ pub async fn get_doris_catalog_columns_core(
|
||||
catalog: &str,
|
||||
database: &str,
|
||||
table: &str,
|
||||
) -> Result<Vec<db::ColumnInfo>, String> {
|
||||
retry_metadata_connection(state, connection_id, Some(database), || {
|
||||
get_doris_catalog_columns_once(state, connection_id, catalog, database, table)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_doris_catalog_columns_once(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
catalog: &str,
|
||||
database: &str,
|
||||
table: &str,
|
||||
) -> Result<Vec<db::ColumnInfo>, String> {
|
||||
let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, None, None).await?;
|
||||
let db_config = connection_config(state, connection_id).await;
|
||||
let connections = state.connections.read().await;
|
||||
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
|
||||
let PoolKind::Mysql(p, _) = pool else {
|
||||
let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
|
||||
let PoolKind::Mysql(p, _) = &pool else {
|
||||
return Ok(vec![]);
|
||||
};
|
||||
let columns = if db_config.as_ref().is_some_and(db::starrocks::is_config) {
|
||||
@@ -479,12 +562,24 @@ pub async fn get_doris_catalog_table_ddl_core(
|
||||
catalog: &str,
|
||||
database: &str,
|
||||
table: &str,
|
||||
) -> Result<String, String> {
|
||||
retry_metadata_connection(state, connection_id, Some(database), || {
|
||||
get_doris_catalog_table_ddl_once(state, connection_id, catalog, database, table)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_doris_catalog_table_ddl_once(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
catalog: &str,
|
||||
database: &str,
|
||||
table: &str,
|
||||
) -> Result<String, String> {
|
||||
let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, None, None).await?;
|
||||
let db_config = connection_config(state, connection_id).await;
|
||||
let connections = state.connections.read().await;
|
||||
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
|
||||
let PoolKind::Mysql(p, _) = pool else {
|
||||
let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
|
||||
let PoolKind::Mysql(p, _) = &pool else {
|
||||
return Err("DDL not supported for this connection".to_string());
|
||||
};
|
||||
if db_config.as_ref().is_some_and(db::starrocks::is_config) {
|
||||
@@ -501,12 +596,24 @@ pub async fn list_doris_catalog_indexes_core(
|
||||
catalog: &str,
|
||||
database: &str,
|
||||
table: &str,
|
||||
) -> Result<Vec<db::IndexInfo>, String> {
|
||||
retry_metadata_connection(state, connection_id, Some(database), || {
|
||||
list_doris_catalog_indexes_once(state, connection_id, catalog, database, table)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn list_doris_catalog_indexes_once(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
catalog: &str,
|
||||
database: &str,
|
||||
table: &str,
|
||||
) -> Result<Vec<db::IndexInfo>, String> {
|
||||
let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, None, None).await?;
|
||||
let db_config = connection_config(state, connection_id).await;
|
||||
let connections = state.connections.read().await;
|
||||
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
|
||||
let PoolKind::Mysql(p, _) = pool else {
|
||||
let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
|
||||
let PoolKind::Mysql(p, _) = &pool else {
|
||||
return Ok(vec![]);
|
||||
};
|
||||
if db_config.as_ref().is_some_and(db::starrocks::is_config) {
|
||||
@@ -603,8 +710,7 @@ async fn list_databases_once(state: &AppState, connection_id: &str) -> Result<Ve
|
||||
}
|
||||
try_sqlserver!(connections, connection_id, list_databases);
|
||||
if let Some(client) = extract_pool!(&connections, connection_id, Agent) {
|
||||
let is_mongo =
|
||||
state.configs.read().await.get(connection_id).is_some_and(|c| c.db_type == DatabaseType::MongoDb);
|
||||
let is_mongo = db_config.as_ref().is_some_and(|config| config.db_type == DatabaseType::MongoDb);
|
||||
if is_mongo {
|
||||
drop(connections);
|
||||
let dbs = crate::mongo_ops::mongo_list_databases_core(state, connection_id).await?;
|
||||
@@ -618,10 +724,9 @@ async fn list_databases_once(state: &AppState, connection_id: &str) -> Result<Ve
|
||||
|
||||
let db_config = connection_config(state, connection_id).await;
|
||||
let mysql_database_list_timeout = mysql_database_list_timeout(db_config.as_ref());
|
||||
let connections = state.connections.read().await;
|
||||
let pool = connections.get(connection_id).ok_or("Connection not found")?;
|
||||
let pool = clone_metadata_pool(state, connection_id).await.ok_or("Connection not found")?;
|
||||
|
||||
match pool {
|
||||
match &pool {
|
||||
PoolKind::Mysql(p, mode)
|
||||
if *mode != MysqlMode::OceanBaseOracle && db_config.as_ref().is_some_and(db::dolt::is_config) =>
|
||||
{
|
||||
@@ -640,11 +745,7 @@ async fn list_databases_once(state: &AppState, connection_id: &str) -> Result<Ve
|
||||
PoolKind::Turso(client) => db::turso_driver::list_databases(client).await,
|
||||
PoolKind::HBase(client) => db::hbase_driver::list_namespaces(client).await,
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
PoolKind::DuckDbWorker(client) => {
|
||||
let client = client.clone();
|
||||
drop(connections);
|
||||
client.list_databases().await
|
||||
}
|
||||
PoolKind::DuckDbWorker(client) => client.list_databases().await,
|
||||
PoolKind::CloudflareD1(client) => db::cloudflare_d1_driver::list_databases(client).await,
|
||||
_ => Ok(vec![]),
|
||||
}
|
||||
@@ -661,7 +762,7 @@ async fn list_database_metadata_once(state: &AppState, connection_id: &str) -> R
|
||||
let connections = state.connections.read().await;
|
||||
if let Some(client) = extract_pool!(&connections, connection_id, SqlServer) {
|
||||
drop(connections);
|
||||
let mut client = client.lock().await;
|
||||
let mut client = lock_sqlserver_metadata_client(&client).await?;
|
||||
return db::sqlserver::list_database_metadata(&mut client).await;
|
||||
}
|
||||
if let Some(PoolKind::Mysql(pool, mode)) = connections.get(connection_id) {
|
||||
@@ -717,11 +818,8 @@ async fn list_schema_infos_once(
|
||||
let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, Some(database), None).await?;
|
||||
let db_config = connection_config(state, connection_id).await;
|
||||
let show_system_schemas = db_config.as_ref().is_some_and(|config| config.show_system_schemas);
|
||||
{
|
||||
let connections = state.connections.read().await;
|
||||
if let Some(PoolKind::Postgres(pool)) = connections.get(&pool_key) {
|
||||
return db::postgres::list_schema_infos_with_system(pool, show_system_schemas).await;
|
||||
}
|
||||
if let Some(PoolKind::Postgres(pool)) = clone_metadata_pool(state, &pool_key).await {
|
||||
return db::postgres::list_schema_infos_with_system(&pool, show_system_schemas).await;
|
||||
}
|
||||
|
||||
let schemas = list_schemas_once(state, connection_id, database, false).await?;
|
||||
@@ -866,10 +964,9 @@ async fn list_schemas_once(
|
||||
}
|
||||
}
|
||||
|
||||
let connections = state.connections.read().await;
|
||||
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
|
||||
let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
|
||||
|
||||
match pool {
|
||||
match &pool {
|
||||
PoolKind::Mysql(p, mode) if *mode == MysqlMode::OceanBaseOracle => db::ob_oracle::list_schemas(p)
|
||||
.await
|
||||
.map(|schemas| filter_visible_schema_names(schemas, visible_schema_filter.as_deref())),
|
||||
@@ -878,9 +975,7 @@ async fn list_schemas_once(
|
||||
.map(|schemas| filter_visible_schema_names(schemas, visible_schema_filter.as_deref())),
|
||||
#[cfg(feature = "duckdb-sidecar")]
|
||||
PoolKind::DuckDbWorker(client) => {
|
||||
let client = client.clone();
|
||||
let database = database.to_string();
|
||||
drop(connections);
|
||||
client
|
||||
.list_schemas(database)
|
||||
.await
|
||||
@@ -1085,10 +1180,9 @@ async fn get_table_comment_core_for_session(
|
||||
}
|
||||
}
|
||||
|
||||
let connections = state.connections.read().await;
|
||||
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
|
||||
let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
|
||||
|
||||
match pool {
|
||||
match &pool {
|
||||
PoolKind::Mysql(p, mode)
|
||||
if *mode != MysqlMode::OceanBaseOracle
|
||||
&& !db_config.as_ref().is_some_and(db::mysql_compatible::uses_show_metadata)
|
||||
@@ -1955,7 +2049,7 @@ async fn list_tables_once(
|
||||
if let Some(linked) = crate::sql_dialect::parse_sqlserver_linked_schema_ref(schema) {
|
||||
if let Some(client) = extract_pool!(&connections, &pool_key, SqlServer) {
|
||||
drop(connections);
|
||||
let mut client = client.lock().await;
|
||||
let mut client = lock_sqlserver_metadata_client(&client).await?;
|
||||
return db::sqlserver::list_linked_server_tables(
|
||||
&mut client,
|
||||
&linked.server,
|
||||
@@ -1972,7 +2066,7 @@ async fn list_tables_once(
|
||||
if object_types.is_some() || table_name_filter.is_some_and(|filter| !filter.is_empty()) {
|
||||
if let Some(client) = extract_pool!(&connections, &pool_key, SqlServer) {
|
||||
drop(connections);
|
||||
let mut client = client.lock().await;
|
||||
let mut client = lock_sqlserver_metadata_client(&client).await?;
|
||||
return db::sqlserver::list_tables(&mut client, schema, filter, None, None)
|
||||
.await
|
||||
.map(|tables| filter_table_infos(tables, filter, limit, offset, object_types, table_name_filter));
|
||||
@@ -2156,10 +2250,9 @@ async fn list_tables_once(
|
||||
}
|
||||
}
|
||||
|
||||
let connections = state.connections.read().await;
|
||||
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
|
||||
let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
|
||||
|
||||
match pool {
|
||||
match &pool {
|
||||
PoolKind::Mysql(p, _) if db_config.as_ref().is_some_and(db::starrocks::is_config) => {
|
||||
db::starrocks::list_tables(p, database)
|
||||
.await
|
||||
@@ -3282,8 +3375,9 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_retry_recovers_missing_pool_only_as_transient_state() {
|
||||
assert!(is_retryable_metadata_error("Pool not found"));
|
||||
fn metadata_retry_excludes_pool_saturation_from_reconnects() {
|
||||
assert!(!is_retryable_metadata_error("Pool not found"));
|
||||
assert!(!is_retryable_metadata_error(crate::query::METADATA_POOL_BUSY_ERROR));
|
||||
assert!(is_retryable_metadata_error("connection reset by peer"));
|
||||
assert!(is_retryable_metadata_error("Agent RPC error (-1): dm.jdbc.driver.DMException: 网络通信异常"));
|
||||
assert!(is_retryable_metadata_error(
|
||||
@@ -3302,6 +3396,78 @@ mod tests {
|
||||
assert!(!is_retryable_metadata_error("Access denied for user"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_pool_races_retry_once_and_saturation_returns_busy() {
|
||||
let dir = std::env::temp_dir().join(format!("dbx-schema-metadata-pool-race-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let storage = crate::storage::Storage::open(&dir.join("storage.db")).await.unwrap();
|
||||
let state = crate::connection::AppState::new(storage);
|
||||
state.configs.write().await.insert("conn".to_string(), test_connection_config(DatabaseType::Mysql));
|
||||
|
||||
let mut recovered_attempts = 0;
|
||||
let recovered = super::retry_metadata_connection_for_session(&state, "conn", Some("app"), None, || {
|
||||
recovered_attempts += 1;
|
||||
let attempt = recovered_attempts;
|
||||
async move {
|
||||
if attempt == 1 {
|
||||
Err("Pool not found".to_string())
|
||||
} else {
|
||||
Ok("loaded")
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
assert_eq!(recovered, Ok("loaded"));
|
||||
assert_eq!(recovered_attempts, 2);
|
||||
|
||||
let mut missing_attempts = 0;
|
||||
let missing = super::retry_metadata_connection_for_session(&state, "conn", Some("app"), None, || {
|
||||
missing_attempts += 1;
|
||||
async { Err::<(), _>("Pool not found".to_string()) }
|
||||
})
|
||||
.await;
|
||||
assert_eq!(missing.err().as_deref(), Some(crate::query::METADATA_POOL_BUSY_ERROR));
|
||||
assert_eq!(missing_attempts, 2);
|
||||
|
||||
let mut saturation_attempts = 0;
|
||||
let saturated = super::retry_metadata_connection_for_session(&state, "conn", Some("app"), None, || {
|
||||
saturation_attempts += 1;
|
||||
async { Err::<(), _>("MySQL connection pool checkout timed out [stage=wait, timeout_ms=500]".to_string()) }
|
||||
})
|
||||
.await;
|
||||
assert_eq!(saturated.err().as_deref(), Some(crate::query::METADATA_POOL_BUSY_ERROR));
|
||||
assert_eq!(saturation_attempts, 1);
|
||||
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sqlserver_metadata_mutex_contention_returns_busy_after_timeout() {
|
||||
let client = std::sync::Arc::new(tokio::sync::Mutex::new(()));
|
||||
let _held = client.lock().await;
|
||||
let result = super::lock_metadata_mutex_with_timeout(&client, Duration::from_millis(1)).await;
|
||||
assert_eq!(result.err().as_deref(), Some(crate::query::METADATA_POOL_BUSY_ERROR));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_pool_snapshot_releases_global_connections_lock() {
|
||||
let dir = std::env::temp_dir().join(format!("dbx-schema-metadata-snapshot-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let storage = crate::storage::Storage::open(&dir.join("storage.db")).await.unwrap();
|
||||
let state = crate::connection::AppState::new(storage);
|
||||
let pool = crate::db::sqlite::connect_path(":memory:").await.unwrap();
|
||||
state.connections.write().await.insert("conn".to_string(), super::PoolKind::Sqlite(pool));
|
||||
|
||||
let snapshot = super::clone_metadata_pool(&state, "conn").await.expect("metadata pool snapshot");
|
||||
let write_guard = state.connections.try_write().expect("snapshot must not retain the global pool-map lock");
|
||||
|
||||
assert!(matches!(snapshot, super::PoolKind::Sqlite(_)));
|
||||
drop(write_guard);
|
||||
drop(snapshot);
|
||||
drop(state);
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_error_action_applies_fail_stop_to_every_attempt() {
|
||||
let quarantine = "Agent RPC error (-1): connection lost\nDBX_AGENT_ERROR_DATA:{\"category\":\"connection\",\"sessionDisposition\":\"quarantine\"}";
|
||||
@@ -4793,8 +4959,8 @@ pub async fn completion_assistant_search_core(
|
||||
{
|
||||
let connections = state.connections.read().await;
|
||||
if let Some(client) = extract_pool!(&connections, &pool_key, Agent) {
|
||||
let db_config = connection_config(state, &request.connection_id).await;
|
||||
drop(connections);
|
||||
let db_config = connection_config(state, &request.connection_id).await;
|
||||
let mut client = client.lock().await;
|
||||
match client
|
||||
.completion_assistant_search::<db::CompletionAssistantResponse>(
|
||||
@@ -5030,8 +5196,9 @@ async fn list_object_statistics_once(
|
||||
drop(connections);
|
||||
return db::victoriametrics_driver::list_object_statistics(&client).await;
|
||||
}
|
||||
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
|
||||
match pool {
|
||||
drop(connections);
|
||||
let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
|
||||
match &pool {
|
||||
PoolKind::Mysql(p, mode) => {
|
||||
if *mode == MysqlMode::OceanBaseOracle || db_config.as_ref().is_some_and(db::manticoresearch::is_config) {
|
||||
Ok(vec![])
|
||||
@@ -5111,7 +5278,7 @@ async fn list_objects_once(
|
||||
}
|
||||
if let Some(client) = extract_pool!(&connections, &pool_key, SqlServer) {
|
||||
drop(connections);
|
||||
let mut client = client.lock().await;
|
||||
let mut client = lock_sqlserver_metadata_client(&client).await?;
|
||||
return db::sqlserver::list_objects(&mut client, schema).await.map(unpaged_object_list);
|
||||
}
|
||||
if let Some(client) = extract_pool!(&connections, &pool_key, Agent) {
|
||||
@@ -5225,10 +5392,9 @@ async fn list_objects_once(
|
||||
}
|
||||
}
|
||||
|
||||
let connections = state.connections.read().await;
|
||||
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
|
||||
let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
|
||||
|
||||
match pool {
|
||||
match &pool {
|
||||
PoolKind::Mysql(p, mode) => {
|
||||
// Note: mysql and ob_oracle take different second args (database vs schema)
|
||||
if *mode == MysqlMode::OceanBaseOracle {
|
||||
@@ -5260,31 +5426,28 @@ async fn list_objects_once(
|
||||
.await
|
||||
.map(unpaged_object_list)
|
||||
}
|
||||
_ => {
|
||||
drop(connections);
|
||||
Ok(unpaged_object_list(
|
||||
list_tables_core(state, connection_id, database, schema, None, None, None, None, None)
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|table| db::ObjectInfo {
|
||||
name: table.name,
|
||||
object_type: table.table_type,
|
||||
schema: if schema.is_empty() { None } else { Some(schema.to_string()) },
|
||||
valid: None,
|
||||
signature: None,
|
||||
custom_type_kind: None,
|
||||
has_members: None,
|
||||
comment: table.comment,
|
||||
created_at: None,
|
||||
updated_at: None,
|
||||
parent_schema: table.parent_schema,
|
||||
parent_name: table.parent_name,
|
||||
trigger: None,
|
||||
xugu_type_members_expandable: None,
|
||||
})
|
||||
.collect(),
|
||||
))
|
||||
}
|
||||
_ => Ok(unpaged_object_list(
|
||||
list_tables_core(state, connection_id, database, schema, None, None, None, None, None)
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|table| db::ObjectInfo {
|
||||
name: table.name,
|
||||
object_type: table.table_type,
|
||||
schema: if schema.is_empty() { None } else { Some(schema.to_string()) },
|
||||
valid: None,
|
||||
signature: None,
|
||||
custom_type_kind: None,
|
||||
has_members: None,
|
||||
comment: table.comment,
|
||||
created_at: None,
|
||||
updated_at: None,
|
||||
parent_schema: table.parent_schema,
|
||||
parent_name: table.parent_name,
|
||||
trigger: None,
|
||||
xugu_type_members_expandable: None,
|
||||
})
|
||||
.collect(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5373,8 +5536,9 @@ async fn list_completion_objects_once(
|
||||
return Ok(filter_completion_objects(objects));
|
||||
}
|
||||
|
||||
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
|
||||
match pool {
|
||||
drop(connections);
|
||||
let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
|
||||
match &pool {
|
||||
PoolKind::Mysql(p, mode) if *mode != MysqlMode::OceanBaseOracle => {
|
||||
db::mysql::list_completion_objects(p, database).await
|
||||
}
|
||||
@@ -5391,7 +5555,6 @@ async fn list_completion_objects_once(
|
||||
db::postgres::list_objects(p, schema, true, true, false).await.map(filter_completion_objects)
|
||||
}
|
||||
PoolKind::SqlServer(_) => {
|
||||
drop(connections);
|
||||
let outcome =
|
||||
list_objects_once(state, connection_id, database, schema, None, None, None, None, None).await?;
|
||||
Ok(filter_completion_objects(outcome.objects))
|
||||
@@ -5464,9 +5627,42 @@ where
|
||||
let configs = state.configs.read().await;
|
||||
configs.get(connection_id).map(|config| config.db_type)
|
||||
};
|
||||
let _metadata_permit = match db_type.filter(|db_type| uses_metadata_gate(*db_type)) {
|
||||
Some(db_type) => {
|
||||
Some(state.acquire_metadata_permit(connection_id, database, db_type, client_session_id).await?)
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
let mut retried = false;
|
||||
let mut missing_pool_retry = false;
|
||||
loop {
|
||||
let result = operation().await;
|
||||
if result.as_ref().err().is_some_and(|error| error == "Pool not found") {
|
||||
if !missing_pool_retry {
|
||||
missing_pool_retry = true;
|
||||
log::debug!(
|
||||
"[metadata:pool:missing-retry] connection_id={} database={}",
|
||||
connection_id,
|
||||
database.unwrap_or_default()
|
||||
);
|
||||
continue;
|
||||
}
|
||||
log::warn!(
|
||||
"[metadata:pool:missing] connection_id={} database={}",
|
||||
connection_id,
|
||||
database.unwrap_or_default()
|
||||
);
|
||||
return Err(crate::query::METADATA_POOL_BUSY_ERROR.to_string());
|
||||
}
|
||||
if result.as_ref().err().is_some_and(|error| crate::query::is_pool_saturation_error(error)) {
|
||||
log::warn!(
|
||||
"[metadata:pool:saturation] connection_id={} database={} error={}",
|
||||
connection_id,
|
||||
database.unwrap_or_default(),
|
||||
result.as_ref().err().map(String::as_str).unwrap_or_default()
|
||||
);
|
||||
return Err(crate::query::METADATA_POOL_BUSY_ERROR.to_string());
|
||||
}
|
||||
let recovery =
|
||||
result.as_ref().err().map(|error| metadata_recovery(db_type, error, retried)).unwrap_or_default();
|
||||
match recovery.action {
|
||||
@@ -5592,7 +5788,7 @@ fn is_retryable_metadata_error(error: &str) -> bool {
|
||||
return RecoveryPolicy::decide(&error, RecoveryScope::ReadOnlyMetadata { retried: false })
|
||||
== RecoveryDecision::RetryReadOnlyMetadata;
|
||||
}
|
||||
error == "Pool not found" || crate::query::is_connection_error(error)
|
||||
crate::query::is_connection_error(error)
|
||||
}
|
||||
|
||||
pub async fn get_columns_core(
|
||||
@@ -5748,7 +5944,7 @@ async fn get_columns_core_for_session_inner(
|
||||
if let Some(linked) = crate::sql_dialect::parse_sqlserver_linked_schema_ref(schema) {
|
||||
if let Some(client) = extract_pool!(&connections, &pool_key, SqlServer) {
|
||||
drop(connections);
|
||||
let mut client = client.lock().await;
|
||||
let mut client = lock_sqlserver_metadata_client(&client).await?;
|
||||
return db::sqlserver::get_linked_server_columns(
|
||||
&mut client,
|
||||
&linked.server,
|
||||
@@ -5872,10 +6068,9 @@ async fn get_columns_core_for_session_inner(
|
||||
}
|
||||
}
|
||||
|
||||
let connections = state.connections.read().await;
|
||||
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
|
||||
let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
|
||||
|
||||
match pool {
|
||||
match &pool {
|
||||
PoolKind::Mysql(p, _) if db_config.as_ref().is_some_and(db::manticoresearch::is_config) => {
|
||||
let metadata_database = mysql_show_metadata_database_for_config(db_config.as_ref(), database);
|
||||
db::manticoresearch::get_columns(p, metadata_database, table).await.map(deduplicate_column_infos)
|
||||
@@ -6072,10 +6267,9 @@ async fn list_indexes_core_for_session(
|
||||
}
|
||||
}
|
||||
|
||||
let connections = state.connections.read().await;
|
||||
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
|
||||
let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
|
||||
|
||||
match pool {
|
||||
match &pool {
|
||||
PoolKind::Mysql(p, mode) => {
|
||||
if db_config.as_ref().is_some_and(db::manticoresearch::is_config) {
|
||||
return db::manticoresearch::list_indexes(p, table).await;
|
||||
@@ -6160,10 +6354,9 @@ async fn list_foreign_keys_core_for_session(
|
||||
}
|
||||
}
|
||||
|
||||
let connections = state.connections.read().await;
|
||||
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
|
||||
let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
|
||||
|
||||
match pool {
|
||||
match &pool {
|
||||
PoolKind::Mysql(p, mode) => {
|
||||
if *mode == MysqlMode::OceanBaseOracle {
|
||||
db::ob_oracle::list_foreign_keys(p, schema, table).await
|
||||
@@ -6206,10 +6399,9 @@ pub async fn list_triggers_core(
|
||||
}
|
||||
}
|
||||
|
||||
let connections = state.connections.read().await;
|
||||
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
|
||||
let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
|
||||
|
||||
match pool {
|
||||
match &pool {
|
||||
PoolKind::Mysql(p, mode) => {
|
||||
if *mode == MysqlMode::OceanBaseOracle {
|
||||
db::ob_oracle::list_triggers(p, schema, table).await
|
||||
@@ -6361,10 +6553,9 @@ pub async fn list_functions_core(
|
||||
) -> Result<Vec<db::FunctionInfo>, String> {
|
||||
retry_metadata_connection(state, connection_id, Some(database), || async {
|
||||
let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, Some(database), None).await?;
|
||||
let connections = state.connections.read().await;
|
||||
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
|
||||
let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
|
||||
|
||||
match pool {
|
||||
match &pool {
|
||||
PoolKind::Postgres(p) => db::postgres::list_functions(p, schema).await,
|
||||
_ => Ok(vec![]),
|
||||
}
|
||||
@@ -6382,10 +6573,9 @@ pub async fn list_sequences_core(
|
||||
retry_metadata_connection(state, connection_id, Some(database), || async {
|
||||
let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, Some(database), None).await?;
|
||||
let db_config = connection_config(state, connection_id).await;
|
||||
let connections = state.connections.read().await;
|
||||
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
|
||||
let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
|
||||
|
||||
match pool {
|
||||
match &pool {
|
||||
PoolKind::Postgres(p) if db_config.as_ref().is_some_and(is_opengauss_family_config) => {
|
||||
db::postgres::list_opengauss_sequences(p, schema, with_last_values).await
|
||||
}
|
||||
@@ -6404,10 +6594,9 @@ pub async fn list_rules_core(
|
||||
) -> Result<Vec<db::RuleInfo>, String> {
|
||||
retry_metadata_connection(state, connection_id, Some(database), || async {
|
||||
let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, Some(database), None).await?;
|
||||
let connections = state.connections.read().await;
|
||||
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
|
||||
let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
|
||||
|
||||
match pool {
|
||||
match &pool {
|
||||
PoolKind::Postgres(p) => db::postgres::list_rules(p, schema).await,
|
||||
_ => Ok(vec![]),
|
||||
}
|
||||
@@ -6433,10 +6622,9 @@ pub async fn list_extensions_core(
|
||||
}
|
||||
}
|
||||
|
||||
let connections = state.connections.read().await;
|
||||
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
|
||||
let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
|
||||
|
||||
match pool {
|
||||
match &pool {
|
||||
PoolKind::Postgres(p) => db::postgres::list_extensions(p, schema).await,
|
||||
_ => Ok(vec![]),
|
||||
}
|
||||
@@ -6465,10 +6653,9 @@ pub async fn list_available_extensions_core(
|
||||
}
|
||||
}
|
||||
|
||||
let connections = state.connections.read().await;
|
||||
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
|
||||
let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
|
||||
|
||||
match pool {
|
||||
match &pool {
|
||||
PoolKind::Postgres(p) => db::postgres::list_available_extensions(p).await,
|
||||
_ => Ok(vec![]),
|
||||
}
|
||||
@@ -6484,10 +6671,9 @@ pub async fn list_owners_core(
|
||||
) -> Result<Vec<db::OwnerInfo>, String> {
|
||||
retry_metadata_connection(state, connection_id, Some(database), || async {
|
||||
let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, Some(database), None).await?;
|
||||
let connections = state.connections.read().await;
|
||||
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
|
||||
let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
|
||||
|
||||
match pool {
|
||||
match &pool {
|
||||
PoolKind::Postgres(p) => db::postgres::list_owners(p, schema).await,
|
||||
_ => Ok(vec![]),
|
||||
}
|
||||
@@ -6709,7 +6895,7 @@ async fn get_table_ddl_once(
|
||||
}
|
||||
if let Some(client) = extract_pool!(&connections, &pool_key, SqlServer) {
|
||||
drop(connections);
|
||||
let mut client = client.lock().await;
|
||||
let mut client = lock_sqlserver_metadata_client(&client).await?;
|
||||
return build_sqlserver_ddl(&mut client, schema, table).await;
|
||||
}
|
||||
if let Some(client) = extract_pool!(&connections, &pool_key, Agent) {
|
||||
@@ -6771,10 +6957,9 @@ async fn get_table_ddl_once(
|
||||
}
|
||||
}
|
||||
|
||||
let connections = state.connections.read().await;
|
||||
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
|
||||
let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
|
||||
|
||||
match pool {
|
||||
match &pool {
|
||||
PoolKind::Mysql(p, _) => mysql_ddl(p, mysql_table_metadata_catalog(database, schema), table).await,
|
||||
PoolKind::Postgres(p) if db_config.as_ref().is_some_and(is_opengauss_family_config) => {
|
||||
match opengauss_table_ddl(p, schema, table).await {
|
||||
@@ -7567,9 +7752,8 @@ async fn get_custom_type_details_once(
|
||||
.await;
|
||||
}
|
||||
}
|
||||
let connections = state.connections.read().await;
|
||||
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
|
||||
match pool {
|
||||
let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
|
||||
match &pool {
|
||||
PoolKind::Postgres(p) => db::postgres::get_custom_type_details(p, schema, name).await,
|
||||
_ => Err("custom type details are not supported for this connection type".to_string()),
|
||||
}
|
||||
@@ -7662,7 +7846,7 @@ async fn get_object_source_once(
|
||||
}
|
||||
if let Some(client) = extract_pool!(&connections, &pool_key, SqlServer) {
|
||||
drop(connections);
|
||||
let mut client = client.lock().await;
|
||||
let mut client = lock_sqlserver_metadata_client(&client).await?;
|
||||
let result =
|
||||
db::sqlserver::execute_query(&mut client, &sqlserver_object_source_sql(schema, name, &object_type))
|
||||
.await;
|
||||
|
||||
+439
-14
@@ -192,6 +192,8 @@ pub struct DesktopSettings {
|
||||
pub close_action_prompted: bool,
|
||||
#[serde(default)]
|
||||
pub debug_logging_enabled: bool,
|
||||
#[serde(default = "default_metadata_cache_max_memory_mb")]
|
||||
pub metadata_cache_max_memory_mb: usize,
|
||||
#[serde(default)]
|
||||
pub duckdb_worker_process_isolation: bool,
|
||||
#[serde(default = "default_duckdb_worker_max_processes")]
|
||||
@@ -244,6 +246,31 @@ pub const DUCKDB_WORKER_MAX_PROCESSES_MIN: usize = 1;
|
||||
pub const DUCKDB_WORKER_MAX_PROCESSES_MAX: usize = 16;
|
||||
pub const DUCKDB_WORKER_MAX_PROCESSES_DEFAULT: usize = 4;
|
||||
|
||||
pub const METADATA_CACHE_MEMORY_MIN_MB: usize = 16;
|
||||
pub const METADATA_CACHE_MEMORY_RECOMMENDED_MAX_MB: usize = 256;
|
||||
pub const METADATA_CACHE_MEMORY_HARD_MAX_MB: usize = 512;
|
||||
pub const METADATA_CACHE_MEMORY_DEFAULT_MB: usize = 64;
|
||||
|
||||
pub fn default_metadata_cache_max_memory_mb() -> usize {
|
||||
METADATA_CACHE_MEMORY_DEFAULT_MB
|
||||
}
|
||||
|
||||
pub fn normalize_metadata_cache_max_memory_mb(value: usize) -> usize {
|
||||
if value > METADATA_CACHE_MEMORY_HARD_MAX_MB {
|
||||
log::warn!(
|
||||
"Metadata cache memory limit {value} MB exceeds the hard limit; falling back to {METADATA_CACHE_MEMORY_DEFAULT_MB} MB"
|
||||
);
|
||||
METADATA_CACHE_MEMORY_DEFAULT_MB
|
||||
} else {
|
||||
if value > METADATA_CACHE_MEMORY_RECOMMENDED_MAX_MB {
|
||||
log::warn!(
|
||||
"Metadata cache memory limit {value} MB exceeds the recommended {METADATA_CACHE_MEMORY_RECOMMENDED_MAX_MB} MB"
|
||||
);
|
||||
}
|
||||
value.clamp(METADATA_CACHE_MEMORY_MIN_MB, METADATA_CACHE_MEMORY_HARD_MAX_MB)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn default_duckdb_worker_max_processes() -> usize {
|
||||
DUCKDB_WORKER_MAX_PROCESSES_DEFAULT
|
||||
}
|
||||
@@ -260,6 +287,7 @@ impl Default for DesktopSettings {
|
||||
quit_on_close: false,
|
||||
close_action_prompted: false,
|
||||
debug_logging_enabled: false,
|
||||
metadata_cache_max_memory_mb: default_metadata_cache_max_memory_mb(),
|
||||
duckdb_worker_process_isolation: false,
|
||||
duckdb_worker_max_processes: default_duckdb_worker_max_processes(),
|
||||
saved_sql_sync_dir: None,
|
||||
@@ -351,7 +379,11 @@ const SCHEMA_STATEMENTS: &[&str] = &[
|
||||
"CREATE TABLE IF NOT EXISTS schema_cache (
|
||||
cache_key TEXT PRIMARY KEY,
|
||||
payload_json TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
updated_at TEXT NOT NULL,
|
||||
updated_at_ms INTEGER NOT NULL DEFAULT 0,
|
||||
last_accessed_at_ms INTEGER NOT NULL DEFAULT 0,
|
||||
byte_size INTEGER NOT NULL DEFAULT 0,
|
||||
owner_id TEXT NOT NULL DEFAULT ''
|
||||
)",
|
||||
"CREATE TABLE IF NOT EXISTS tab_runtime_cache (
|
||||
cache_key TEXT PRIMARY KEY,
|
||||
@@ -452,6 +484,7 @@ impl Storage {
|
||||
ensure_history_columns_sync(conn)?;
|
||||
ensure_saved_sql_columns_sync(conn)?;
|
||||
ensure_tab_runtime_cache_columns_sync(conn)?;
|
||||
ensure_schema_cache_columns_sync(conn)?;
|
||||
ensure_ai_configs_columns_sync(conn)?;
|
||||
ensure_state_store_columns_sync(conn)?;
|
||||
Ok(())
|
||||
@@ -485,6 +518,52 @@ fn inspect_sqlite_db_file(path: &Path) -> Result<SqliteDbFileState, String> {
|
||||
}
|
||||
}
|
||||
|
||||
fn ensure_schema_cache_columns_sync(conn: &Connection) -> Result<(), String> {
|
||||
let mut columns = HashSet::new();
|
||||
let mut statement = conn.prepare("PRAGMA table_info(schema_cache)").map_err(|error| error.to_string())?;
|
||||
let rows = statement.query_map([], |row| row.get::<_, String>(1)).map_err(|error| error.to_string())?;
|
||||
for row in rows {
|
||||
columns.insert(row.map_err(|error| error.to_string())?);
|
||||
}
|
||||
|
||||
for (name, definition) in [
|
||||
("updated_at_ms", "INTEGER NOT NULL DEFAULT 0"),
|
||||
("last_accessed_at_ms", "INTEGER NOT NULL DEFAULT 0"),
|
||||
("byte_size", "INTEGER NOT NULL DEFAULT 0"),
|
||||
("owner_id", "TEXT NOT NULL DEFAULT ''"),
|
||||
] {
|
||||
if !columns.contains(name) {
|
||||
conn.execute(&format!("ALTER TABLE schema_cache ADD COLUMN {name} {definition}"), [])
|
||||
.map_err(|error| error.to_string())?;
|
||||
}
|
||||
}
|
||||
|
||||
conn.execute(
|
||||
"UPDATE schema_cache
|
||||
SET byte_size = length(payload_json)
|
||||
WHERE byte_size = 0 AND payload_json IS NOT NULL",
|
||||
[],
|
||||
)
|
||||
.map_err(|error| error.to_string())?;
|
||||
conn.execute(
|
||||
"UPDATE schema_cache
|
||||
SET updated_at_ms = COALESCE(CAST(strftime('%s', updated_at) AS INTEGER) * 1000, 0),
|
||||
last_accessed_at_ms = COALESCE(CAST(strftime('%s', updated_at) AS INTEGER) * 1000, 0)
|
||||
WHERE updated_at_ms = 0",
|
||||
[],
|
||||
)
|
||||
.map_err(|error| error.to_string())?;
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_schema_cache_updated_at_ms ON schema_cache (updated_at_ms)", [])
|
||||
.map_err(|error| error.to_string())?;
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_schema_cache_owner_lru
|
||||
ON schema_cache (owner_id, last_accessed_at_ms, updated_at_ms, cache_key)",
|
||||
[],
|
||||
)
|
||||
.map_err(|error| error.to_string())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn open_read_only_sqlite(path: &Path) -> Result<Connection, String> {
|
||||
Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY)
|
||||
.map_err(|e| format!("Failed to open db read-only: {e}"))
|
||||
@@ -1538,6 +1617,12 @@ impl Storage {
|
||||
"debug_logging_enabled".to_string(),
|
||||
serde_json::Value::Bool(desktop_settings.debug_logging_enabled),
|
||||
);
|
||||
settings.insert(
|
||||
"metadata_cache_max_memory_mb".to_string(),
|
||||
serde_json::Value::Number(serde_json::Number::from(normalize_metadata_cache_max_memory_mb(
|
||||
desktop_settings.metadata_cache_max_memory_mb,
|
||||
))),
|
||||
);
|
||||
settings.insert(
|
||||
"duckdb_worker_process_isolation".to_string(),
|
||||
serde_json::Value::Bool(desktop_settings.duckdb_worker_process_isolation),
|
||||
@@ -1608,6 +1693,12 @@ impl Storage {
|
||||
.get("debug_logging_enabled")
|
||||
.and_then(|value| value.as_bool())
|
||||
.unwrap_or_else(|| DesktopSettings::default().debug_logging_enabled),
|
||||
metadata_cache_max_memory_mb: settings
|
||||
.get("metadata_cache_max_memory_mb")
|
||||
.and_then(|value| value.as_u64())
|
||||
.and_then(|value| usize::try_from(value).ok())
|
||||
.map(normalize_metadata_cache_max_memory_mb)
|
||||
.unwrap_or_else(|| DesktopSettings::default().metadata_cache_max_memory_mb),
|
||||
duckdb_worker_process_isolation: settings
|
||||
.get("duckdb_worker_process_isolation")
|
||||
.and_then(|value| value.as_bool())
|
||||
@@ -3342,31 +3433,167 @@ impl Storage {
|
||||
|
||||
// Schema cache
|
||||
|
||||
const SCHEMA_CACHE_MAX_TOTAL_BYTES: i64 = 256 * 1024 * 1024;
|
||||
const SCHEMA_CACHE_MAX_CONNECTION_BYTES: i64 = 64 * 1024 * 1024;
|
||||
const SCHEMA_CACHE_MAX_ENTRIES: usize = 50_000;
|
||||
const SCHEMA_CACHE_MAX_AGE_MILLIS: i64 = 24 * 60 * 60 * 1000;
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct SchemaCachePolicy {
|
||||
max_total_bytes: i64,
|
||||
max_connection_bytes: i64,
|
||||
max_entries: usize,
|
||||
max_age_millis: i64,
|
||||
}
|
||||
|
||||
impl Default for SchemaCachePolicy {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
max_total_bytes: SCHEMA_CACHE_MAX_TOTAL_BYTES,
|
||||
max_connection_bytes: SCHEMA_CACHE_MAX_CONNECTION_BYTES,
|
||||
max_entries: SCHEMA_CACHE_MAX_ENTRIES,
|
||||
max_age_millis: SCHEMA_CACHE_MAX_AGE_MILLIS,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn schema_cache_owner(cache_key: &str) -> &str {
|
||||
let mut parts = cache_key.split(':');
|
||||
match (parts.next(), parts.next(), parts.next()) {
|
||||
(Some("object-ddl" | "object-meta"), Some("v1"), Some(owner)) => owner,
|
||||
_ => "",
|
||||
}
|
||||
}
|
||||
|
||||
fn delete_oldest_schema_cache_entry(conn: &Connection, owner_id: Option<&str>) -> Result<bool, String> {
|
||||
let deleted = match owner_id {
|
||||
Some(owner_id) => conn.execute(
|
||||
"DELETE FROM schema_cache
|
||||
WHERE cache_key = (
|
||||
SELECT cache_key FROM schema_cache WHERE owner_id = ?1
|
||||
ORDER BY last_accessed_at_ms ASC, updated_at_ms ASC, cache_key ASC LIMIT 1
|
||||
)",
|
||||
[owner_id],
|
||||
),
|
||||
None => conn.execute(
|
||||
"DELETE FROM schema_cache
|
||||
WHERE cache_key = (
|
||||
SELECT cache_key FROM schema_cache
|
||||
ORDER BY last_accessed_at_ms ASC, updated_at_ms ASC, cache_key ASC LIMIT 1
|
||||
)",
|
||||
[],
|
||||
),
|
||||
}
|
||||
.map_err(|error| error.to_string())?;
|
||||
Ok(deleted > 0)
|
||||
}
|
||||
|
||||
fn prune_schema_cache(conn: &Connection, policy: SchemaCachePolicy, now_ms: i64) -> Result<(), String> {
|
||||
let expires_before = now_ms.saturating_sub(policy.max_age_millis.max(0));
|
||||
conn.execute("DELETE FROM schema_cache WHERE updated_at_ms = 0 OR updated_at_ms <= ?1", [expires_before])
|
||||
.map_err(|error| error.to_string())?;
|
||||
|
||||
loop {
|
||||
let over_budget_owner: Option<String> = conn
|
||||
.query_row(
|
||||
"SELECT owner_id FROM schema_cache
|
||||
GROUP BY owner_id
|
||||
HAVING SUM(byte_size) > ?1
|
||||
ORDER BY SUM(byte_size) DESC, owner_id ASC
|
||||
LIMIT 1",
|
||||
[policy.max_connection_bytes.max(0)],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.optional()
|
||||
.map_err(|error| error.to_string())?;
|
||||
let Some(owner_id) = over_budget_owner else {
|
||||
break;
|
||||
};
|
||||
if !delete_oldest_schema_cache_entry(conn, Some(&owner_id))? {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
loop {
|
||||
let (entry_count, total_bytes): (i64, i64) = conn
|
||||
.query_row("SELECT COUNT(*), COALESCE(SUM(byte_size), 0) FROM schema_cache", [], |row| {
|
||||
Ok((row.get(0)?, row.get(1)?))
|
||||
})
|
||||
.map_err(|error| error.to_string())?;
|
||||
if entry_count <= policy.max_entries as i64 && total_bytes <= policy.max_total_bytes.max(0) {
|
||||
break;
|
||||
}
|
||||
if !delete_oldest_schema_cache_entry(conn, None)? {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl Storage {
|
||||
pub async fn save_schema_cache(&self, cache_key: &str, payload: &serde_json::Value) -> Result<(), String> {
|
||||
self.save_schema_cache_with_policy(cache_key, payload, SchemaCachePolicy::default()).await
|
||||
}
|
||||
|
||||
async fn save_schema_cache_with_policy(
|
||||
&self,
|
||||
cache_key: &str,
|
||||
payload: &serde_json::Value,
|
||||
policy: SchemaCachePolicy,
|
||||
) -> Result<(), String> {
|
||||
let cache_key = cache_key.to_string();
|
||||
let json = serde_json::to_string(payload).map_err(|e| e.to_string())?;
|
||||
let byte_size = json.len().min(i64::MAX as usize) as i64;
|
||||
let owner_id = schema_cache_owner(&cache_key).to_string();
|
||||
let now_ms = unix_timestamp_millis();
|
||||
self.with_conn(move |conn| {
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO schema_cache (cache_key, payload_json, updated_at) \
|
||||
VALUES (?1, ?2, datetime('now'))",
|
||||
params![cache_key, json],
|
||||
)
|
||||
.map(|_| ())
|
||||
.map_err(|e| e.to_string())
|
||||
let transaction = conn.transaction().map_err(|error| error.to_string())?;
|
||||
transaction
|
||||
.execute(
|
||||
"INSERT INTO schema_cache (
|
||||
cache_key, payload_json, updated_at, updated_at_ms, last_accessed_at_ms, byte_size, owner_id
|
||||
) VALUES (?1, ?2, datetime('now'), ?3, ?3, ?4, ?5)
|
||||
ON CONFLICT(cache_key) DO UPDATE SET
|
||||
payload_json = excluded.payload_json,
|
||||
updated_at = excluded.updated_at,
|
||||
updated_at_ms = excluded.updated_at_ms,
|
||||
last_accessed_at_ms = excluded.last_accessed_at_ms,
|
||||
byte_size = excluded.byte_size,
|
||||
owner_id = excluded.owner_id",
|
||||
params![cache_key, json, now_ms, byte_size, owner_id],
|
||||
)
|
||||
.map_err(|error| error.to_string())?;
|
||||
prune_schema_cache(&transaction, policy, now_ms)?;
|
||||
transaction.commit().map_err(|error| error.to_string())
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn load_schema_cache(&self, cache_key: &str) -> Result<Option<serde_json::Value>, String> {
|
||||
let cache_key = cache_key.to_string();
|
||||
let now_ms = unix_timestamp_millis();
|
||||
let json: Option<String> = self
|
||||
.with_conn(move |conn| {
|
||||
conn.query_row("SELECT payload_json FROM schema_cache WHERE cache_key = ?1", [cache_key], |row| {
|
||||
row.get(0)
|
||||
})
|
||||
.optional()
|
||||
.map_err(|e| e.to_string())
|
||||
let transaction = conn.transaction().map_err(|error| error.to_string())?;
|
||||
let json = transaction
|
||||
.query_row(
|
||||
"SELECT payload_json FROM schema_cache
|
||||
WHERE cache_key = ?1 AND updated_at_ms > ?2",
|
||||
params![cache_key, now_ms.saturating_sub(SCHEMA_CACHE_MAX_AGE_MILLIS.max(0))],
|
||||
|row| row.get(0),
|
||||
)
|
||||
.optional()
|
||||
.map_err(|error| error.to_string())?;
|
||||
if json.is_some() {
|
||||
transaction
|
||||
.execute(
|
||||
"UPDATE schema_cache SET last_accessed_at_ms = ?2 WHERE cache_key = ?1",
|
||||
params![cache_key, now_ms],
|
||||
)
|
||||
.map_err(|error| error.to_string())?;
|
||||
}
|
||||
transaction.commit().map_err(|error| error.to_string())?;
|
||||
Ok(json)
|
||||
})
|
||||
.await?;
|
||||
json.map(|value| serde_json::from_str(&value).map_err(|e| e.to_string())).transpose()
|
||||
@@ -4097,7 +4324,7 @@ mod tests {
|
||||
};
|
||||
use crate::saved_sql::SavedSqlFile;
|
||||
use rusqlite::Connection;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
fn temp_db_path(name: &str) -> std::path::PathBuf {
|
||||
let stamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos();
|
||||
@@ -5240,6 +5467,202 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_cache_memory_budget_is_bounded() {
|
||||
assert_eq!(super::normalize_metadata_cache_max_memory_mb(1), 16);
|
||||
assert_eq!(super::normalize_metadata_cache_max_memory_mb(256), 256);
|
||||
assert_eq!(super::normalize_metadata_cache_max_memory_mb(512), 512);
|
||||
assert_eq!(super::normalize_metadata_cache_max_memory_mb(513), 64);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn schema_cache_prunes_expired_rows_on_write_without_maintaining_during_reads() {
|
||||
let path = temp_db_path("schema-cache-ttl-prune");
|
||||
let storage = Storage::open(&path).await.unwrap();
|
||||
storage
|
||||
.save_schema_cache(
|
||||
"object-ddl:v1:conn-a:db:public:old::TABLE:",
|
||||
&serde_json::json!({ "version": 1, "ddl": "old" }),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
storage
|
||||
.with_conn(|conn| {
|
||||
conn.execute(
|
||||
"UPDATE schema_cache SET updated_at = datetime('now', '-25 hours'), updated_at_ms = CAST(strftime('%s', 'now', '-25 hours') AS INTEGER) * 1000",
|
||||
[],
|
||||
)
|
||||
.map(|_| ())
|
||||
.map_err(|error| error.to_string())
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(storage.load_schema_cache("object-ddl:v1:conn-a:db:public:old::TABLE:").await.unwrap(), None);
|
||||
let remaining = storage
|
||||
.with_conn(|conn| {
|
||||
conn.query_row("SELECT COUNT(*) FROM schema_cache", [], |row| row.get::<_, i64>(0))
|
||||
.map_err(|error| error.to_string())
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(remaining, 1, "L2 reads must remain indexed point lookups without global maintenance");
|
||||
|
||||
storage
|
||||
.save_schema_cache(
|
||||
"object-ddl:v1:conn-a:db:public:new::TABLE:",
|
||||
&serde_json::json!({ "version": 1, "ddl": "new" }),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let remaining = storage
|
||||
.with_conn(|conn| {
|
||||
conn.query_row("SELECT COUNT(*) FROM schema_cache", [], |row| row.get::<_, i64>(0))
|
||||
.map_err(|error| error.to_string())
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(remaining, 1, "the next write must prune the expired row");
|
||||
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn schema_cache_budget_covers_multiple_connections_objects_and_facets() {
|
||||
let path = temp_db_path("schema-cache-capacity");
|
||||
let storage = Storage::open(&path).await.unwrap();
|
||||
let policy = super::SchemaCachePolicy {
|
||||
max_total_bytes: 1_100,
|
||||
max_connection_bytes: 700,
|
||||
max_entries: 5,
|
||||
max_age_millis: 86_400_000,
|
||||
};
|
||||
let payload = serde_json::json!({ "value": "x".repeat(180) });
|
||||
let keys = [
|
||||
"object-ddl:v1:conn-a:db:public:accounts::TABLE:",
|
||||
"object-meta:v1:conn-a:db:public:accounts::TABLE:columns:",
|
||||
"object-meta:v1:conn-a:db:public:billing::TABLE:indexes:",
|
||||
"object-ddl:v1:conn-b:db:public:events::TABLE:",
|
||||
"object-meta:v1:conn-b:db:public:events::TABLE:triggers:",
|
||||
"object-meta:v1:conn-c:db:public:audit::TABLE:comment:",
|
||||
];
|
||||
for key in keys {
|
||||
storage.save_schema_cache_with_policy(key, &payload, policy).await.unwrap();
|
||||
}
|
||||
|
||||
let (entries, bytes, conn_a_bytes) = storage
|
||||
.with_conn(|conn| {
|
||||
conn.query_row(
|
||||
"SELECT COUNT(*), COALESCE(SUM(byte_size), 0), COALESCE(SUM(CASE WHEN owner_id = 'conn-a' THEN byte_size ELSE 0 END), 0) FROM schema_cache",
|
||||
[],
|
||||
|row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?, row.get::<_, i64>(2)?)),
|
||||
)
|
||||
.map_err(|error| error.to_string())
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(entries <= policy.max_entries as i64);
|
||||
assert!(bytes <= policy.max_total_bytes);
|
||||
assert!(conn_a_bytes <= policy.max_connection_bytes);
|
||||
assert!(storage.load_schema_cache(keys.last().unwrap()).await.unwrap().is_some());
|
||||
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn schema_cache_lru_keeps_recently_accessed_entries() {
|
||||
let path = temp_db_path("schema-cache-lru");
|
||||
let storage = Storage::open(&path).await.unwrap();
|
||||
let policy = super::SchemaCachePolicy {
|
||||
max_total_bytes: i64::MAX,
|
||||
max_connection_bytes: i64::MAX,
|
||||
max_entries: 2,
|
||||
max_age_millis: 86_400_000,
|
||||
};
|
||||
let first = "object-ddl:v1:conn-a:db:public:first::TABLE:";
|
||||
let second = "object-meta:v1:conn-b:db:public:second::TABLE:columns:";
|
||||
let newest = "object-meta:v1:conn-c:db:public:newest::TABLE:indexes:";
|
||||
storage.save_schema_cache_with_policy(first, &serde_json::json!({ "value": 1 }), policy).await.unwrap();
|
||||
storage.save_schema_cache_with_policy(second, &serde_json::json!({ "value": 2 }), policy).await.unwrap();
|
||||
storage
|
||||
.with_conn(move |conn| {
|
||||
conn.execute(
|
||||
"UPDATE schema_cache SET last_accessed_at_ms = CASE cache_key WHEN ?1 THEN 1 ELSE 2 END",
|
||||
rusqlite::params![first],
|
||||
)
|
||||
.map(|_| ())
|
||||
.map_err(|error| error.to_string())
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(storage.load_schema_cache(first).await.unwrap().is_some());
|
||||
storage.save_schema_cache_with_policy(newest, &serde_json::json!({ "value": 3 }), policy).await.unwrap();
|
||||
|
||||
assert!(storage.load_schema_cache(first).await.unwrap().is_some());
|
||||
assert_eq!(storage.load_schema_cache(second).await.unwrap(), None);
|
||||
assert!(storage.load_schema_cache(newest).await.unwrap().is_some());
|
||||
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn schema_cache_50k_point_reads_stay_below_performance_gate() {
|
||||
const ENTRY_COUNT: usize = 50_000;
|
||||
const SAMPLE_COUNT: usize = 40;
|
||||
let path = temp_db_path("schema-cache-50k-read-performance");
|
||||
let storage = Storage::open(&path).await.unwrap();
|
||||
storage
|
||||
.with_conn(|conn| {
|
||||
let transaction = conn.transaction().map_err(|error| error.to_string())?;
|
||||
{
|
||||
let mut statement = transaction
|
||||
.prepare(
|
||||
"INSERT INTO schema_cache (
|
||||
cache_key, payload_json, updated_at, updated_at_ms,
|
||||
last_accessed_at_ms, byte_size, owner_id
|
||||
) VALUES (?1, ?2, datetime('now'), ?3, ?3, ?4, ?5)",
|
||||
)
|
||||
.map_err(|error| error.to_string())?;
|
||||
for index in 0..ENTRY_COUNT {
|
||||
let cache_key =
|
||||
format!("object-meta:v1:conn-{}:db:public:table-{index}::TABLE:columns:", index % 32);
|
||||
let payload = format!(r#"{{"version":1,"value":{index}}}"#);
|
||||
statement
|
||||
.execute(rusqlite::params![
|
||||
cache_key,
|
||||
payload,
|
||||
super::unix_timestamp_millis(),
|
||||
32_i64,
|
||||
format!("conn-{}", index % 32),
|
||||
])
|
||||
.map_err(|error| error.to_string())?;
|
||||
}
|
||||
}
|
||||
transaction.commit().map_err(|error| error.to_string())
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut samples = Vec::with_capacity(SAMPLE_COUNT);
|
||||
for sample in 0..SAMPLE_COUNT {
|
||||
let index = sample * (ENTRY_COUNT / SAMPLE_COUNT);
|
||||
let cache_key = format!("object-meta:v1:conn-{}:db:public:table-{index}::TABLE:columns:", index % 32);
|
||||
let started = Instant::now();
|
||||
assert!(storage.load_schema_cache(&cache_key).await.unwrap().is_some());
|
||||
samples.push(started.elapsed());
|
||||
}
|
||||
samples.sort_unstable();
|
||||
let total = samples.iter().copied().sum::<Duration>();
|
||||
let average = total / SAMPLE_COUNT as u32;
|
||||
let p95 = samples[(SAMPLE_COUNT * 95 / 100).saturating_sub(1)];
|
||||
eprintln!("schema_cache_50k_point_reads average={average:?} p95={p95:?}");
|
||||
|
||||
assert!(average < Duration::from_millis(20), "50k L2 point-read average {average:?} exceeded 20 ms");
|
||||
assert!(p95 < Duration::from_millis(50), "50k L2 point-read P95 {p95:?} exceeded 50 ms");
|
||||
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn desktop_settings_preserve_existing_password_hash() {
|
||||
let path = temp_db_path("desktop-settings-preserve-password");
|
||||
@@ -5253,6 +5676,7 @@ mod tests {
|
||||
quit_on_close: true,
|
||||
close_action_prompted: false,
|
||||
debug_logging_enabled: true,
|
||||
metadata_cache_max_memory_mb: 128,
|
||||
duckdb_worker_process_isolation: false,
|
||||
duckdb_worker_max_processes: DesktopSettings::default().duckdb_worker_max_processes,
|
||||
saved_sql_sync_dir: None,
|
||||
@@ -5273,6 +5697,7 @@ mod tests {
|
||||
quit_on_close: true,
|
||||
close_action_prompted: false,
|
||||
debug_logging_enabled: true,
|
||||
metadata_cache_max_memory_mb: 128,
|
||||
duckdb_worker_process_isolation: false,
|
||||
duckdb_worker_max_processes: DesktopSettings::default().duckdb_worker_max_processes,
|
||||
saved_sql_sync_dir: None,
|
||||
|
||||
Reference in new issue
Block a user