feat(duckdb): add worker concurrency setting

This commit is contained in:
miracle authored and GitHub committed 2026-07-06 00:13:11 +08:00
1 parent 4666f8a5f2
commit 4afc3297e1
16 files changed
+192 -31

No files matched your search

+10 -3
View File
@@ -1,6 +1,6 @@
use std::collections::{HashMap, HashSet};
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::{Mutex, RwLock};
@@ -27,7 +27,7 @@ use crate::models::connection::{
use crate::path_utils::expand_tilde;
use crate::plugins::{PluginDriverSession, PluginRegistry, PluginRuntimeEnv};
use crate::query_cancel::RunningQueries;
use crate::storage::Storage;
use crate::storage::{normalize_duckdb_worker_max_processes, Storage, DUCKDB_WORKER_MAX_PROCESSES_DEFAULT};
pub const JDBC_PLUGIN_NOT_INSTALLED: &str =
"JDBC plugin is not installed. Install the optional JDBC plugin to use this connection.";
@@ -172,6 +172,7 @@ pub struct AppState {
pub agent_manager: crate::agent_manager::AgentManager,
pub nacos_registry: crate::nacos::NacosAdminRegistry,
duckdb_worker_process_isolation: AtomicBool,
duckdb_worker_max_processes: AtomicUsize,
/// PostgreSQL TLS cancel context, keyed by pool_key.
/// Used to reconstruct a TLS connector compatible with the original connection when cancelling.
postgres_cancel_contexts: Arc<RwLock<HashMap<String, db::postgres::PostgresCancelContext>>>,
@@ -479,6 +480,7 @@ impl AppState {
),
nacos_registry: crate::nacos::NacosAdminRegistry::new(),
duckdb_worker_process_isolation: AtomicBool::new(false),
duckdb_worker_max_processes: AtomicUsize::new(DUCKDB_WORKER_MAX_PROCESSES_DEFAULT),
postgres_cancel_contexts: Arc::new(RwLock::new(HashMap::new())),
transaction_sessions: Arc::new(RwLock::new(HashMap::new())),
#[cfg(feature = "mq-admin")]
@@ -498,6 +500,10 @@ impl AppState {
self.duckdb_worker_process_isolation.store(enabled, Ordering::Relaxed);
}
pub fn set_duckdb_worker_max_processes(&self, max_processes: usize) {
self.duckdb_worker_max_processes.store(normalize_duckdb_worker_max_processes(max_processes), Ordering::Relaxed);
}
pub async fn apply_duckdb_worker_process_isolation(&self, enabled: bool) {
let previous = self.duckdb_worker_process_isolation.swap(enabled, Ordering::Relaxed);
if previous != enabled {
@@ -949,9 +955,10 @@ impl AppState {
path: expand_tilde(&attached.path),
})
.collect();
let client = db::duckdb_worker_process::DuckDbWorkerClient::open(
let client = db::duckdb_worker_process::DuckDbWorkerClient::open_with_process_limit(
expand_tilde(&db_config.host),
attached_databases,
self.duckdb_worker_max_processes.load(Ordering::Relaxed),
)
.await?;
PoolKind::DuckDbWorker(Arc::new(client))
+52 -10
View File
@@ -4,7 +4,7 @@ use std::collections::HashMap;
use std::path::PathBuf;
use std::process::Stdio;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, OnceLock};
use std::sync::{Arc, Mutex as StdMutex, OnceLock};
use std::time::Duration;
use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
@@ -18,12 +18,11 @@ use crate::db::duckdb_worker_protocol::{
DuckDbWorkerResponse,
};
use crate::models::connection::AttachedDatabaseConfig;
use crate::storage::{normalize_duckdb_worker_max_processes, DUCKDB_WORKER_MAX_PROCESSES_DEFAULT};
const DEFAULT_WORKER_REQUEST_TIMEOUT: Duration = Duration::from_secs(60);
const DEFAULT_WORKER_KILL_WAIT: Duration = Duration::from_secs(3);
const DEFAULT_WORKER_START_WAIT: Duration = Duration::from_secs(5);
const DEFAULT_DUCKDB_WORKER_PROCESS_LIMIT: usize = 4;
type PendingRequests = Arc<Mutex<HashMap<String, PendingRequest>>>;
struct PendingRequest {
@@ -42,6 +41,7 @@ struct DuckDbWorkerClientInner {
connect_lock: Mutex<()>,
query_lock: Mutex<()>,
process_limiter: Arc<Semaphore>,
process_limit: usize,
executable: PathBuf,
connect_params: DuckDbWorkerConnectParams,
request_timeout: Duration,
@@ -63,18 +63,44 @@ impl DuckDbWorkerClient {
Self::open_with_executable(executable, path, attached_databases).await
}
pub async fn open_with_process_limit(
path: String,
attached_databases: Vec<AttachedDatabaseConfig>,
process_limit: usize,
) -> Result<Self, String> {
let executable = std::env::current_exe().map_err(|e| e.to_string())?;
Self::open_with_executable_and_process_limit(executable, path, attached_databases, process_limit).await
}
pub async fn open_with_executable(
executable: PathBuf,
path: String,
attached_databases: Vec<AttachedDatabaseConfig>,
) -> Result<Self, String> {
Self::open_with_executable_and_process_limit(
executable,
path,
attached_databases,
DUCKDB_WORKER_MAX_PROCESSES_DEFAULT,
)
.await
}
pub async fn open_with_executable_and_process_limit(
executable: PathBuf,
path: String,
attached_databases: Vec<AttachedDatabaseConfig>,
process_limit: usize,
) -> Result<Self, String> {
let (process_limiter, process_limit) = duckdb_worker_process_limiter(process_limit);
let client = Self {
inner: Arc::new(DuckDbWorkerClientInner {
state: Mutex::new(WorkerProcessState::default()),
pending: Arc::new(Mutex::new(HashMap::new())),
connect_lock: Mutex::new(()),
query_lock: Mutex::new(()),
process_limiter: duckdb_worker_process_limiter(),
process_limiter,
process_limit,
executable,
connect_params: DuckDbWorkerConnectParams { path, attached_databases },
request_timeout: DEFAULT_WORKER_REQUEST_TIMEOUT,
@@ -285,7 +311,7 @@ impl DuckDbWorkerClient {
let permit =
tokio::time::timeout(self.inner.worker_start_timeout, self.inner.process_limiter.clone().acquire_owned())
.await
.map_err(|_| duckdb_worker_process_limit_error())?
.map_err(|_| duckdb_worker_process_limit_error(self.inner.process_limit))?
.map_err(|_| "DuckDB worker process limiter is closed".to_string())?;
log::info!("[duckdb-worker:start] executable={}", self.inner.executable.display());
@@ -380,15 +406,31 @@ impl DuckDbWorkerClient {
}
}
fn duckdb_worker_process_limiter() -> Arc<Semaphore> {
static LIMITER: OnceLock<Arc<Semaphore>> = OnceLock::new();
LIMITER.get_or_init(|| Arc::new(Semaphore::new(DEFAULT_DUCKDB_WORKER_PROCESS_LIMIT))).clone()
struct WorkerProcessLimiterState {
limit: usize,
limiter: Arc<Semaphore>,
}
fn duckdb_worker_process_limit_error() -> String {
fn duckdb_worker_process_limiter(process_limit: usize) -> (Arc<Semaphore>, usize) {
static LIMITER: OnceLock<StdMutex<WorkerProcessLimiterState>> = OnceLock::new();
let process_limit = normalize_duckdb_worker_max_processes(process_limit);
let state = LIMITER.get_or_init(|| {
StdMutex::new(WorkerProcessLimiterState {
limit: process_limit,
limiter: Arc::new(Semaphore::new(process_limit)),
})
});
let mut state = state.lock().expect("DuckDB worker process limiter lock poisoned");
if state.limit != process_limit && state.limiter.available_permits() == state.limit {
*state = WorkerProcessLimiterState { limit: process_limit, limiter: Arc::new(Semaphore::new(process_limit)) };
}
(state.limiter.clone(), state.limit)
}
fn duckdb_worker_process_limit_error(process_limit: usize) -> String {
format!(
"DuckDB worker process limit reached (max {}). Close another DuckDB worker connection or retry later.",
DEFAULT_DUCKDB_WORKER_PROCESS_LIMIT
process_limit
)
}
+42
View File
@@ -123,6 +123,8 @@ pub struct DesktopSettings {
pub debug_logging_enabled: bool,
#[serde(default)]
pub duckdb_worker_process_isolation: bool,
#[serde(default = "default_duckdb_worker_max_processes")]
pub duckdb_worker_max_processes: usize,
#[serde(default)]
pub saved_sql_sync_dir: Option<String>,
#[serde(default)]
@@ -139,6 +141,18 @@ fn default_sidebar_table_page_size() -> usize {
1000
}
pub const DUCKDB_WORKER_MAX_PROCESSES_MIN: usize = 1;
pub const DUCKDB_WORKER_MAX_PROCESSES_MAX: usize = 16;
pub const DUCKDB_WORKER_MAX_PROCESSES_DEFAULT: usize = 4;
pub fn default_duckdb_worker_max_processes() -> usize {
DUCKDB_WORKER_MAX_PROCESSES_DEFAULT
}
pub fn normalize_duckdb_worker_max_processes(value: usize) -> usize {
value.clamp(DUCKDB_WORKER_MAX_PROCESSES_MIN, DUCKDB_WORKER_MAX_PROCESSES_MAX)
}
impl Default for DesktopSettings {
fn default() -> Self {
Self {
@@ -148,6 +162,7 @@ impl Default for DesktopSettings {
close_action_prompted: false,
debug_logging_enabled: false,
duckdb_worker_process_isolation: false,
duckdb_worker_max_processes: default_duckdb_worker_max_processes(),
saved_sql_sync_dir: None,
driver_store_dir: None,
plugin_store_dir: None,
@@ -811,6 +826,12 @@ impl Storage {
"duckdb_worker_process_isolation".to_string(),
serde_json::Value::Bool(desktop_settings.duckdb_worker_process_isolation),
);
settings.insert(
"duckdb_worker_max_processes".to_string(),
serde_json::Value::Number(serde_json::Number::from(normalize_duckdb_worker_max_processes(
desktop_settings.duckdb_worker_max_processes,
))),
);
match desktop_settings.saved_sql_sync_dir.as_ref().filter(|path| !path.trim().is_empty()) {
Some(path) => {
settings.insert("saved_sql_sync_dir".to_string(), serde_json::Value::String(path.clone()));
@@ -875,6 +896,12 @@ impl Storage {
.get("duckdb_worker_process_isolation")
.and_then(|value| value.as_bool())
.unwrap_or_else(|| DesktopSettings::default().duckdb_worker_process_isolation),
duckdb_worker_max_processes: settings
.get("duckdb_worker_max_processes")
.and_then(|value| value.as_u64())
.and_then(|value| usize::try_from(value).ok())
.map(normalize_duckdb_worker_max_processes)
.unwrap_or_else(|| DesktopSettings::default().duckdb_worker_max_processes),
saved_sql_sync_dir: settings
.get("saved_sql_sync_dir")
.and_then(|value| value.as_str())
@@ -2804,6 +2831,7 @@ mod tests {
close_action_prompted: false,
debug_logging_enabled: true,
duckdb_worker_process_isolation: false,
duckdb_worker_max_processes: DesktopSettings::default().duckdb_worker_max_processes,
saved_sql_sync_dir: None,
driver_store_dir: Some("/tmp/dbx-drivers".to_string()),
plugin_store_dir: Some("/tmp/dbx-plugins".to_string()),
@@ -2823,6 +2851,7 @@ mod tests {
close_action_prompted: false,
debug_logging_enabled: true,
duckdb_worker_process_isolation: false,
duckdb_worker_max_processes: DesktopSettings::default().duckdb_worker_max_processes,
saved_sql_sync_dir: None,
driver_store_dir: Some("/tmp/dbx-drivers".to_string()),
plugin_store_dir: Some("/tmp/dbx-plugins".to_string()),
@@ -2872,6 +2901,19 @@ mod tests {
assert_eq!(storage.load_desktop_settings().await.unwrap().sidebar_table_page_size, 1234);
}
#[tokio::test]
async fn desktop_settings_persist_duckdb_worker_max_processes() {
let path = temp_db_path("desktop-settings-duckdb-worker-max-processes");
let storage = Storage::open(&path).await.unwrap();
storage
.save_desktop_settings(&DesktopSettings { duckdb_worker_max_processes: 8, ..DesktopSettings::default() })
.await
.unwrap();
assert_eq!(storage.load_desktop_settings().await.unwrap().duckdb_worker_max_processes, 8);
}
#[tokio::test]
async fn password_hash_preserves_existing_desktop_settings() {
let path = temp_db_path("password-preserve-desktop-settings");