fix(duckdb): checkpoint wal on worker shutdown

This commit is contained in:
t8y2 committed 2026-08-26 15:23:29 +08:00
1 parent 6686d1628e
commit 8475c84d48
5 files changed
+168 -9

No files matched your search

@@ -28,6 +28,10 @@ const DUCKDB_WORKER_POISONED_CODE: &str = "duckdb_worker_poisoned";
const DUCKDB_WORKER_REQUEST_TIMEOUT_CODE: &str = "duckdb_worker_request_timeout";
const DEFAULT_WORKER_REQUEST_TIMEOUT: Duration = Duration::from_secs(60);
const DEFAULT_WORKER_KILL_WAIT: Duration = Duration::from_secs(3);
/// Grace period for a shutdown-acked worker to exit by itself (session drop
/// plus DuckDB shutdown checkpoint). Kept below the 3s pool close timeout in
/// `connection.rs` so pool close cancellation cannot cut the checkpoint short.
const WORKER_SHUTDOWN_EXIT_WAIT: Duration = Duration::from_millis(2500);
const DEFAULT_WORKER_START_WAIT: Duration = Duration::from_secs(5);
pub const DUCKDB_DRIVER_PATH_ENV: &str = "DBX_DUCKDB_DRIVER_PATH";
type PendingRequests = Arc<Mutex<HashMap<String, PendingRequest>>>;
@@ -362,7 +366,61 @@ impl DuckDbWorkerClient {
Some(self.inner.request_timeout),
)
.await;
self.kill().await;
self.wait_for_exit_or_kill().await;
}
/// Lets a shutdown-acked worker exit on its own so its session drop runs
/// the DuckDB shutdown checkpoint that removes the database WAL; SIGKILL
/// stays the fallback for stuck workers. Killing immediately after the
/// ack would truncate that checkpoint and leave the WAL file behind.
/// The wait must fit inside the caller's pool close timeout so the
/// detached close task is not cancelled mid-checkpoint.
async fn wait_for_exit_or_kill(&self) {
let (child, generation) = {
let mut state = self.inner.state.lock().await;
let child = state.child.take();
let generation = state.generation;
state.stdin = None;
state.connected = false;
(child, generation)
};
let Some(mut child) = child else { return };
match tokio::time::timeout(WORKER_SHUTDOWN_EXIT_WAIT, child.wait()).await {
Ok(Ok(status)) => {
log::info!("[duckdb-worker:shutdown:exit] status={status}");
self.fail_pending_for_generation(
generation,
"duckdb_worker_killed",
"DuckDB worker process was killed",
)
.await;
return;
}
Ok(Err(err)) => {
log::warn!("[duckdb-worker:shutdown:wait-failed] error={err}");
self.fail_pending_for_generation(
generation,
"duckdb_worker_killed",
"DuckDB worker process was killed",
)
.await;
return;
}
Err(_) => {
log::warn!("[duckdb-worker:shutdown:wait-timeout] wait_ms={}", WORKER_SHUTDOWN_EXIT_WAIT.as_millis());
}
}
let _ = child.start_kill();
match tokio::time::timeout(DEFAULT_WORKER_KILL_WAIT, child.wait()).await {
Ok(Ok(status)) => log::info!("[duckdb-worker:kill:exit] status={status}"),
Ok(Err(err)) => log::warn!("[duckdb-worker:kill:wait-failed] error={err}"),
Err(_) => {
log::warn!("[duckdb-worker:kill:wait-timeout] wait_ms={}", DEFAULT_WORKER_KILL_WAIT.as_millis())
}
}
self.fail_pending_for_generation(generation, "duckdb_worker_killed", "DuckDB worker process was killed").await;
}
pub async fn kill(&self) {