fix(duckdb): close worker process reliably
This commit is contained in:
1 parent
da88d03ad4
commit
c32d0b7d6e
6 files changed
+600
-47
No files matched your search
@@ -20,6 +20,11 @@ use crate::db::duckdb_worker_protocol::{
|
||||
use crate::models::connection::AttachedDatabaseConfig;
|
||||
use crate::storage::{normalize_duckdb_worker_max_processes, DUCKDB_WORKER_MAX_PROCESSES_DEFAULT};
|
||||
|
||||
/// Error code the worker reports when a query error left the DuckDB connection poisoned
|
||||
/// (see the duckdb-rs Parser Error bug). The client kills the worker on this code so the
|
||||
/// next request starts a fresh one. Must match the code emitted in `duckdb_worker_runtime`.
|
||||
const DUCKDB_WORKER_POISONED_CODE: &str = "duckdb_worker_poisoned";
|
||||
const DUCKDB_WORKER_REQUEST_TIMEOUT_CODE: &str = "duckdb_worker_request_timeout";
|
||||
const DEFAULT_WORKER_REQUEST_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
const DEFAULT_WORKER_KILL_WAIT: Duration = Duration::from_secs(3);
|
||||
const DEFAULT_WORKER_START_WAIT: Duration = Duration::from_secs(5);
|
||||
@@ -92,8 +97,29 @@ impl DuckDbWorkerClient {
|
||||
attached_databases: Vec<AttachedDatabaseConfig>,
|
||||
process_limit: usize,
|
||||
) -> Result<Self, String> {
|
||||
let client = Self::new_unconnected_with_timeouts(
|
||||
executable,
|
||||
path,
|
||||
attached_databases,
|
||||
process_limit,
|
||||
DEFAULT_WORKER_REQUEST_TIMEOUT,
|
||||
DEFAULT_WORKER_START_WAIT,
|
||||
);
|
||||
client.ensure_connected().await?;
|
||||
Ok(client)
|
||||
}
|
||||
|
||||
#[doc(hidden)]
|
||||
pub fn new_unconnected_with_timeouts(
|
||||
executable: PathBuf,
|
||||
path: String,
|
||||
attached_databases: Vec<AttachedDatabaseConfig>,
|
||||
process_limit: usize,
|
||||
request_timeout: Duration,
|
||||
worker_start_timeout: Duration,
|
||||
) -> Self {
|
||||
let (process_limiter, process_limit) = duckdb_worker_process_limiter(process_limit);
|
||||
let client = Self {
|
||||
Self {
|
||||
inner: Arc::new(DuckDbWorkerClientInner {
|
||||
state: Mutex::new(WorkerProcessState::default()),
|
||||
pending: Arc::new(Mutex::new(HashMap::new())),
|
||||
@@ -103,13 +129,11 @@ impl DuckDbWorkerClient {
|
||||
process_limit,
|
||||
executable,
|
||||
connect_params: DuckDbWorkerConnectParams { path, attached_databases },
|
||||
request_timeout: DEFAULT_WORKER_REQUEST_TIMEOUT,
|
||||
worker_start_timeout: DEFAULT_WORKER_START_WAIT,
|
||||
request_timeout,
|
||||
worker_start_timeout,
|
||||
next_id: AtomicU64::new(1),
|
||||
}),
|
||||
};
|
||||
client.ensure_connected().await?;
|
||||
Ok(client)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn execute(
|
||||
@@ -122,14 +146,32 @@ impl DuckDbWorkerClient {
|
||||
) -> Result<db::QueryResult, String> {
|
||||
let _query_guard = self.inner.query_lock.lock().await;
|
||||
let client = self.clone();
|
||||
// Cancellation and timeout restart the worker via cancel_or_kill below. An ordinary
|
||||
// query error (parser/binder/catalog/runtime) leaves the worker healthy, so we keep it
|
||||
// alive to preserve session-local state (:memory: contents, temp tables, SET, raw ATTACH).
|
||||
// The one exception is a poisoned connection: duckdb-rs can leave the connection unusable
|
||||
// after a syntax Parser Error, and the worker reports that with `duckdb_worker_poisoned`.
|
||||
// We kill on that code so the next request restarts a fresh worker. Killing here (rather
|
||||
// than letting the worker self-exit) avoids racing our own next request against a dying
|
||||
// worker, and OS-level kill never runs the destructor that would abort the process.
|
||||
let future = async move {
|
||||
client
|
||||
.request::<db::QueryResult>(
|
||||
client.ensure_connected().await?;
|
||||
match client
|
||||
.send_request_structured::<db::QueryResult>(
|
||||
DuckDbWorkerMethod::Execute,
|
||||
DuckDbWorkerExecuteParams { sql, database, max_rows },
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => Ok(result),
|
||||
Err(error) => {
|
||||
if error.code == DUCKDB_WORKER_POISONED_CODE {
|
||||
client.kill().await;
|
||||
}
|
||||
Err(error.message)
|
||||
}
|
||||
}
|
||||
};
|
||||
tokio::pin!(future);
|
||||
|
||||
@@ -165,23 +207,17 @@ impl DuckDbWorkerClient {
|
||||
}
|
||||
|
||||
pub async fn list_databases(&self) -> Result<Vec<db::DatabaseInfo>, String> {
|
||||
self.request(DuckDbWorkerMethod::ListDatabases, serde_json::json!({}), Some(self.inner.request_timeout)).await
|
||||
self.metadata_request(DuckDbWorkerMethod::ListDatabases, serde_json::json!({})).await
|
||||
}
|
||||
|
||||
pub async fn list_schemas(&self, database: String) -> Result<Vec<String>, String> {
|
||||
self.request(
|
||||
DuckDbWorkerMethod::ListSchemas,
|
||||
serde_json::json!({ "database": database }),
|
||||
Some(self.inner.request_timeout),
|
||||
)
|
||||
.await
|
||||
self.metadata_request(DuckDbWorkerMethod::ListSchemas, serde_json::json!({ "database": database })).await
|
||||
}
|
||||
|
||||
pub async fn list_tables(&self, database: String, schema: String) -> Result<Vec<db::TableInfo>, String> {
|
||||
self.request(
|
||||
self.metadata_request(
|
||||
DuckDbWorkerMethod::ListTables,
|
||||
serde_json::json!({ "database": database, "schema": schema }),
|
||||
Some(self.inner.request_timeout),
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -192,24 +228,36 @@ impl DuckDbWorkerClient {
|
||||
schema: String,
|
||||
table: String,
|
||||
) -> Result<Vec<db::ColumnInfo>, String> {
|
||||
self.request(
|
||||
self.metadata_request(
|
||||
DuckDbWorkerMethod::ListColumns,
|
||||
serde_json::json!({ "database": database, "schema": schema, "table": table }),
|
||||
Some(self.inner.request_timeout),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn attach_database(&self, attached: AttachedDatabaseConfig) -> Result<(), String> {
|
||||
self.request::<serde_json::Value>(
|
||||
DuckDbWorkerMethod::AttachDatabase,
|
||||
attached,
|
||||
Some(self.inner.request_timeout),
|
||||
)
|
||||
.await?;
|
||||
self.metadata_request::<serde_json::Value>(DuckDbWorkerMethod::AttachDatabase, attached).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Runs a metadata request that executes synchronously inside the worker's stdin loop.
|
||||
/// If it times out the worker is likely stuck inside DuckDB and can no longer read further
|
||||
/// requests, so we kill it; the next call restarts a fresh worker via `ensure_connected`.
|
||||
/// Ordinary errors (e.g. a missing table) leave the worker healthy and do not kill it.
|
||||
async fn metadata_request<T>(&self, method: DuckDbWorkerMethod, params: impl serde::Serialize) -> Result<T, String>
|
||||
where
|
||||
T: serde::de::DeserializeOwned,
|
||||
{
|
||||
let timeout = self.inner.request_timeout;
|
||||
match tokio::time::timeout(timeout, self.request::<T>(method, params, None)).await {
|
||||
Ok(result) => result,
|
||||
Err(_) => {
|
||||
self.kill().await;
|
||||
Err(format!("DuckDB worker request timed out after {}s", timeout.as_secs()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn cancel(&self) -> Result<(), String> {
|
||||
self.kill().await;
|
||||
Ok(())
|
||||
@@ -272,12 +320,38 @@ impl DuckDbWorkerClient {
|
||||
}
|
||||
}
|
||||
|
||||
self.send_request::<serde_json::Value>(
|
||||
DuckDbWorkerMethod::Connect,
|
||||
self.inner.connect_params.clone(),
|
||||
Some(self.inner.request_timeout),
|
||||
)
|
||||
.await?;
|
||||
let mut attempts = 0;
|
||||
loop {
|
||||
match self
|
||||
.send_request_structured::<serde_json::Value>(
|
||||
DuckDbWorkerMethod::Connect,
|
||||
self.inner.connect_params.clone(),
|
||||
Some(self.inner.request_timeout),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => break,
|
||||
Err(error) => {
|
||||
let retry_delay = duckdb_connect_file_lock_retry_delay(attempts, &error.message);
|
||||
// A failed Connect leaves this worker without a valid session. Keep no stale
|
||||
// child around, especially after OS file-lock errors where the user may retry
|
||||
// once the competing process exits.
|
||||
self.kill().await;
|
||||
if let Some(delay) = retry_delay {
|
||||
attempts += 1;
|
||||
log::warn!(
|
||||
"[duckdb-worker:connect-retry] attempt={} delay_ms={} error={}",
|
||||
attempts,
|
||||
delay.as_millis(),
|
||||
error.message
|
||||
);
|
||||
tokio::time::sleep(delay).await;
|
||||
continue;
|
||||
}
|
||||
return Err(error.message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut state = self.inner.state.lock().await;
|
||||
state.connected = true;
|
||||
@@ -340,6 +414,21 @@ impl DuckDbWorkerClient {
|
||||
params: impl serde::Serialize,
|
||||
timeout: Option<Duration>,
|
||||
) -> Result<T, String>
|
||||
where
|
||||
T: serde::de::DeserializeOwned,
|
||||
{
|
||||
self.send_request_structured(method, params, timeout).await.map_err(|error| error.message)
|
||||
}
|
||||
|
||||
/// Like `send_request` but preserves the worker error `code` so callers can react to
|
||||
/// specific conditions (e.g. `duckdb_worker_poisoned`). A protocol/transport failure is
|
||||
/// surfaced as an error with code `duckdb_worker_error`.
|
||||
async fn send_request_structured<T>(
|
||||
&self,
|
||||
method: DuckDbWorkerMethod,
|
||||
params: impl serde::Serialize,
|
||||
timeout: Option<Duration>,
|
||||
) -> Result<T, DuckDbWorkerError>
|
||||
where
|
||||
T: serde::de::DeserializeOwned,
|
||||
{
|
||||
@@ -365,30 +454,33 @@ impl DuckDbWorkerClient {
|
||||
|
||||
if let Err(err) = write_result {
|
||||
self.inner.pending.lock().await.remove(&id);
|
||||
return Err(err);
|
||||
return Err(err.into());
|
||||
}
|
||||
|
||||
let response = match timeout {
|
||||
Some(timeout) => match tokio::time::timeout(timeout, rx).await {
|
||||
Ok(Ok(response)) => response,
|
||||
Ok(Err(_)) => return Err("DuckDB worker response channel closed".to_string()),
|
||||
Ok(Err(_)) => return Err("DuckDB worker response channel closed".into()),
|
||||
Err(_) => {
|
||||
self.inner.pending.lock().await.remove(&id);
|
||||
return Err(format!("DuckDB worker request timed out after {}s", timeout.as_secs()));
|
||||
return Err(DuckDbWorkerError::new(
|
||||
DUCKDB_WORKER_REQUEST_TIMEOUT_CODE,
|
||||
format!("DuckDB worker request timed out after {}s", timeout.as_secs()),
|
||||
));
|
||||
}
|
||||
},
|
||||
None => rx.await.map_err(|_| "DuckDB worker response channel closed".to_string())?,
|
||||
None => rx.await.map_err(|_| DuckDbWorkerError::from("DuckDB worker response channel closed"))?,
|
||||
};
|
||||
|
||||
if !response.ok {
|
||||
let error = response
|
||||
.error
|
||||
.unwrap_or_else(|| DuckDbWorkerError::new("duckdb_worker_error", "DuckDB worker request failed"));
|
||||
return Err(error.message);
|
||||
return Err(error);
|
||||
}
|
||||
|
||||
let result = response.result.unwrap_or(serde_json::Value::Null);
|
||||
serde_json::from_value(result).map_err(|e| e.to_string())
|
||||
serde_json::from_value(result).map_err(|e| DuckDbWorkerError::from(e.to_string()))
|
||||
}
|
||||
|
||||
async fn fail_pending_for_generation(&self, generation: u64, code: &'static str, message: &'static str) {
|
||||
@@ -434,6 +526,35 @@ fn duckdb_worker_process_limit_error(process_limit: usize) -> String {
|
||||
)
|
||||
}
|
||||
|
||||
fn duckdb_connect_file_lock_retry_delay(attempts: usize, message: &str) -> Option<Duration> {
|
||||
if !is_transient_duckdb_file_lock_error(message) {
|
||||
return None;
|
||||
}
|
||||
match attempts {
|
||||
0 => Some(Duration::from_millis(50)),
|
||||
1 => Some(Duration::from_millis(100)),
|
||||
2 => Some(Duration::from_millis(200)),
|
||||
3 => Some(Duration::from_millis(400)),
|
||||
4 => Some(Duration::from_millis(800)),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_transient_duckdb_file_lock_error(message: &str) -> bool {
|
||||
let lower = message.to_ascii_lowercase();
|
||||
let mentions_file_open = lower.contains("cannot open file")
|
||||
|| lower.contains("could not set lock")
|
||||
|| lower.contains("file is already open");
|
||||
let mentions_lock = lower.contains("file is already open")
|
||||
|| lower.contains("being used by another process")
|
||||
|| lower.contains("process cannot access the file")
|
||||
|| lower.contains("sharing violation")
|
||||
|| lower.contains("resource temporarily unavailable")
|
||||
|| message.contains("另一个程序正在使用")
|
||||
|| message.contains("进程无法访问");
|
||||
mentions_file_open && mentions_lock
|
||||
}
|
||||
|
||||
fn spawn_stdout_reader(
|
||||
stdout: tokio::process::ChildStdout,
|
||||
pending: PendingRequests,
|
||||
|
||||
@@ -102,6 +102,27 @@ impl DuckDbWorkerSession {
|
||||
let connection = self.connection.as_ref().ok_or("DuckDB worker is not connected")?.clone();
|
||||
Ok(connection.interrupt_handle())
|
||||
}
|
||||
|
||||
/// Probes whether the connection is still usable after an execute error.
|
||||
///
|
||||
/// duckdb-rs 1.10503.1 has a bug where `Connection::prepare()` failing at the
|
||||
/// `duckdb_extract_statements` stage (a syntax `Parser Error`) permanently poisons
|
||||
/// the connection: every later operation returns `resource deadlock would occur`,
|
||||
/// and dropping the connection aborts the whole process. Benign errors (binder,
|
||||
/// catalog, runtime) do not poison the connection.
|
||||
///
|
||||
/// This probe runs a trivial query through `execute_batch`, which uses the
|
||||
/// non-poisoning `duckdb_query_arrow` path and fails fast on a poisoned connection.
|
||||
/// It returns `true` when the connection is healthy and safe to reuse.
|
||||
fn is_connection_healthy(&self) -> bool {
|
||||
match self.connection.as_ref() {
|
||||
Some(connection) => match connection.lock() {
|
||||
Ok(locked) => locked.execute_batch("SELECT 1").is_ok(),
|
||||
Err(_) => false,
|
||||
},
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
@@ -175,19 +196,39 @@ impl DuckDbWorkerRuntime {
|
||||
tokio::spawn(async move {
|
||||
let id = request.id.clone();
|
||||
let params = request.parse_params::<DuckDbWorkerExecuteParams>();
|
||||
let result = match params {
|
||||
Ok(params) => tokio::task::spawn_blocking(move || {
|
||||
let mut session = session.lock().unwrap_or_else(|e| e.into_inner());
|
||||
session.execute(params)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())
|
||||
.and_then(|result| result),
|
||||
Err(err) => Err(err.message),
|
||||
let (result, poisoned) = match params {
|
||||
Ok(params) => {
|
||||
let probe_session = session.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let mut session = probe_session.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let result = session.execute(params);
|
||||
// Only probe connection health when execute failed: a syntax
|
||||
// Parser Error can poison the connection (duckdb-rs bug), and any
|
||||
// later use — including drop — would abort the process.
|
||||
let poisoned = result.is_err() && !session.is_connection_healthy();
|
||||
(result, poisoned)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())
|
||||
.unwrap_or_else(|err| (Err(err), true))
|
||||
}
|
||||
Err(err) => (Err(err.message), false),
|
||||
};
|
||||
*active_interrupt.lock().unwrap_or_else(|e| e.into_inner()) = None;
|
||||
let response = match result {
|
||||
Ok(result) => DuckDbWorkerResponse::ok(id, result),
|
||||
// A poisoned connection cannot be reused or even safely dropped in-process.
|
||||
// Report a distinct code so the parent kills this worker (OS-level, skipping
|
||||
// destructors) and restarts a fresh one on the next request. We must not exit
|
||||
// here ourselves: process::exit races the parent's next request, which could
|
||||
// be written to a dying worker. Killing on the parent side has no such window.
|
||||
Err(err) if poisoned => {
|
||||
log::warn!("[duckdb-worker:poisoned-connection] reporting to parent for restart");
|
||||
DuckDbWorkerResponse::err(
|
||||
id,
|
||||
DuckDbWorkerError::from_message("duckdb_worker_poisoned", err),
|
||||
)
|
||||
}
|
||||
Err(err) => {
|
||||
DuckDbWorkerResponse::err(id, DuckDbWorkerError::from_message("duckdb_execute_failed", err))
|
||||
}
|
||||
@@ -256,7 +297,7 @@ pub async fn run_stdio_worker() -> Result<(), String> {
|
||||
loop {
|
||||
let line = lines.next_line().await.map_err(|e| e.to_string())?;
|
||||
let Some(line) = line else {
|
||||
break;
|
||||
std::process::exit(0);
|
||||
};
|
||||
if line.trim().is_empty() {
|
||||
continue;
|
||||
|
||||
Reference in new issue
Block a user