fix(duckdb): checkpoint wal on worker shutdown
This commit is contained in:
5 files changed
+168
-9
No files matched your search
@@ -28,6 +28,10 @@ const DUCKDB_WORKER_POISONED_CODE: &str = "duckdb_worker_poisoned";
|
||||
const DUCKDB_WORKER_REQUEST_TIMEOUT_CODE: &str = "duckdb_worker_request_timeout";
|
||||
const DEFAULT_WORKER_REQUEST_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
const DEFAULT_WORKER_KILL_WAIT: Duration = Duration::from_secs(3);
|
||||
/// Grace period for a shutdown-acked worker to exit by itself (session drop
|
||||
/// plus DuckDB shutdown checkpoint). Kept below the 3s pool close timeout in
|
||||
/// `connection.rs` so pool close cancellation cannot cut the checkpoint short.
|
||||
const WORKER_SHUTDOWN_EXIT_WAIT: Duration = Duration::from_millis(2500);
|
||||
const DEFAULT_WORKER_START_WAIT: Duration = Duration::from_secs(5);
|
||||
pub const DUCKDB_DRIVER_PATH_ENV: &str = "DBX_DUCKDB_DRIVER_PATH";
|
||||
type PendingRequests = Arc<Mutex<HashMap<String, PendingRequest>>>;
|
||||
@@ -362,7 +366,61 @@ impl DuckDbWorkerClient {
|
||||
Some(self.inner.request_timeout),
|
||||
)
|
||||
.await;
|
||||
self.kill().await;
|
||||
self.wait_for_exit_or_kill().await;
|
||||
}
|
||||
|
||||
/// Lets a shutdown-acked worker exit on its own so its session drop runs
|
||||
/// the DuckDB shutdown checkpoint that removes the database WAL; SIGKILL
|
||||
/// stays the fallback for stuck workers. Killing immediately after the
|
||||
/// ack would truncate that checkpoint and leave the WAL file behind.
|
||||
/// The wait must fit inside the caller's pool close timeout so the
|
||||
/// detached close task is not cancelled mid-checkpoint.
|
||||
async fn wait_for_exit_or_kill(&self) {
|
||||
let (child, generation) = {
|
||||
let mut state = self.inner.state.lock().await;
|
||||
let child = state.child.take();
|
||||
let generation = state.generation;
|
||||
state.stdin = None;
|
||||
state.connected = false;
|
||||
(child, generation)
|
||||
};
|
||||
|
||||
let Some(mut child) = child else { return };
|
||||
match tokio::time::timeout(WORKER_SHUTDOWN_EXIT_WAIT, child.wait()).await {
|
||||
Ok(Ok(status)) => {
|
||||
log::info!("[duckdb-worker:shutdown:exit] status={status}");
|
||||
self.fail_pending_for_generation(
|
||||
generation,
|
||||
"duckdb_worker_killed",
|
||||
"DuckDB worker process was killed",
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
Ok(Err(err)) => {
|
||||
log::warn!("[duckdb-worker:shutdown:wait-failed] error={err}");
|
||||
self.fail_pending_for_generation(
|
||||
generation,
|
||||
"duckdb_worker_killed",
|
||||
"DuckDB worker process was killed",
|
||||
)
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
Err(_) => {
|
||||
log::warn!("[duckdb-worker:shutdown:wait-timeout] wait_ms={}", WORKER_SHUTDOWN_EXIT_WAIT.as_millis());
|
||||
}
|
||||
}
|
||||
|
||||
let _ = child.start_kill();
|
||||
match tokio::time::timeout(DEFAULT_WORKER_KILL_WAIT, child.wait()).await {
|
||||
Ok(Ok(status)) => log::info!("[duckdb-worker:kill:exit] status={status}"),
|
||||
Ok(Err(err)) => log::warn!("[duckdb-worker:kill:wait-failed] error={err}"),
|
||||
Err(_) => {
|
||||
log::warn!("[duckdb-worker:kill:wait-timeout] wait_ms={}", DEFAULT_WORKER_KILL_WAIT.as_millis())
|
||||
}
|
||||
}
|
||||
self.fail_pending_for_generation(generation, "duckdb_worker_killed", "DuckDB worker process was killed").await;
|
||||
}
|
||||
|
||||
pub async fn kill(&self) {
|
||||
|
||||
Reference in new issue
Block a user