feat(export): support safe background query exports
This commit is contained in:
1 parent
2f44d36018
commit
a07def6042
17 files changed
+577
-195
No files matched your search
@@ -2,6 +2,7 @@ use serde::{Deserialize, Serialize};
|
||||
use std::fs::File;
|
||||
use std::io::{BufWriter, Seek, Write};
|
||||
use std::mem;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{
|
||||
atomic::{AtomicBool, Ordering},
|
||||
Arc,
|
||||
@@ -102,6 +103,55 @@ pub struct QueryResultExportRequest {
|
||||
pub numeric_column_right_align: bool,
|
||||
}
|
||||
|
||||
pub struct StagedExportTarget {
|
||||
destination: PathBuf,
|
||||
temporary: tempfile::TempPath,
|
||||
}
|
||||
|
||||
impl StagedExportTarget {
|
||||
pub fn new(destination: &str) -> Result<Self, String> {
|
||||
let destination = PathBuf::from(destination);
|
||||
let parent = destination.parent().filter(|path| !path.as_os_str().is_empty()).unwrap_or(Path::new("."));
|
||||
let file_name = destination.file_name().and_then(|name| name.to_str()).unwrap_or("query-result");
|
||||
let temporary = tempfile::Builder::new()
|
||||
.prefix(&format!(".{file_name}.dbx-export-"))
|
||||
.tempfile_in(parent)
|
||||
.map_err(|error| format!("Failed to create temporary export file: {error}"))?
|
||||
.into_temp_path();
|
||||
Ok(Self { destination, temporary })
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &Path {
|
||||
self.temporary.as_ref()
|
||||
}
|
||||
|
||||
pub fn path_string(&self) -> Result<String, String> {
|
||||
self.path()
|
||||
.to_str()
|
||||
.map(ToOwned::to_owned)
|
||||
.ok_or_else(|| "Temporary export path is not valid UTF-8".to_string())
|
||||
}
|
||||
|
||||
pub fn commit(self) -> Result<(), String> {
|
||||
let staged_file = std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.open(self.path())
|
||||
.map_err(|error| format!("Failed to open staged export file: {error}"))?;
|
||||
if let Ok(metadata) = std::fs::metadata(&self.destination) {
|
||||
staged_file
|
||||
.set_permissions(metadata.permissions())
|
||||
.map_err(|error| format!("Failed to preserve export destination permissions: {error}"))?;
|
||||
}
|
||||
staged_file.sync_all().map_err(|error| format!("Failed to synchronize export file: {error}"))?;
|
||||
drop(staged_file);
|
||||
self.temporary
|
||||
.persist(&self.destination)
|
||||
.map(|_| ())
|
||||
.map_err(|error| format!("Failed to replace export destination: {}", error.error))
|
||||
}
|
||||
}
|
||||
|
||||
fn safe_postgres_temp_setup_sql(setup_sql: &[String]) -> Option<Vec<String>> {
|
||||
if setup_sql.is_empty() {
|
||||
return None;
|
||||
@@ -231,41 +281,30 @@ fn sql_insert_column_types(request: &QueryResultExportRequest, column_types: &[S
|
||||
}
|
||||
}
|
||||
|
||||
/// Bounded SQL INSERT writer with temp-file + atomic-rename safety.
|
||||
/// Bounded SQL INSERT writer with staged-file replacement safety.
|
||||
///
|
||||
/// Rows are buffered and flushed to a temp file every [`SQL_INSERT_BATCH_SIZE`]
|
||||
/// rows, so memory stays bounded regardless of the query page size. The temp file
|
||||
/// lives alongside the user-chosen target (`<target>.dbx-export-tmp`) and is
|
||||
/// renamed onto the target only when [`SqlInsertWriter::finish`] succeeds. If the
|
||||
/// writer is dropped without finishing (export error or cancellation), the temp
|
||||
/// file is removed and the user's target file is never truncated or deleted.
|
||||
/// rows, so memory stays bounded regardless of the query page size. The unique
|
||||
/// temp file lives alongside the target and replaces it only after
|
||||
/// [`SqlInsertWriter::finish`] flushes and synchronizes the complete output.
|
||||
struct SqlInsertWriter {
|
||||
file: Option<BufWriter<File>>,
|
||||
temp_path: std::path::PathBuf,
|
||||
target_path: std::path::PathBuf,
|
||||
target: Option<StagedExportTarget>,
|
||||
pending_rows: Vec<Vec<Value>>,
|
||||
columns: Vec<String>,
|
||||
column_types: Vec<Option<String>>,
|
||||
database_type: DatabaseType,
|
||||
schema: Option<String>,
|
||||
table_name: String,
|
||||
finished: bool,
|
||||
}
|
||||
|
||||
impl SqlInsertWriter {
|
||||
/// Create the writer and open the temp file. Column metadata is supplied later
|
||||
/// via [`SqlInsertWriter::set_columns`] once the executed result is known.
|
||||
fn create(request: &QueryResultExportRequest) -> Result<Self, String> {
|
||||
let target_path = std::path::PathBuf::from(&request.file_path);
|
||||
let mut temp_path = target_path.clone();
|
||||
let mut temp_name = target_path
|
||||
.file_name()
|
||||
.map(|name| name.to_os_string())
|
||||
.unwrap_or_else(|| std::ffi::OsString::from("export.sql"));
|
||||
temp_name.push(".dbx-export-tmp");
|
||||
temp_path.set_file_name(temp_name);
|
||||
let target = StagedExportTarget::new(&request.file_path)?;
|
||||
let file = BufWriter::new(
|
||||
File::create(&temp_path).map_err(|e| format!("Failed to create SQL export temp file: {e}"))?,
|
||||
File::create(target.path()).map_err(|e| format!("Failed to create SQL export temp file: {e}"))?,
|
||||
);
|
||||
let table_name = request
|
||||
.export_table_name
|
||||
@@ -275,15 +314,13 @@ impl SqlInsertWriter {
|
||||
.to_string();
|
||||
Ok(Self {
|
||||
file: Some(file),
|
||||
temp_path,
|
||||
target_path,
|
||||
target: Some(target),
|
||||
pending_rows: Vec::new(),
|
||||
columns: Vec::new(),
|
||||
column_types: Vec::new(),
|
||||
database_type: request.database_type,
|
||||
schema: request.schema.clone(),
|
||||
table_name,
|
||||
finished: false,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -330,30 +367,18 @@ impl SqlInsertWriter {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Flush remaining rows, close the temp file, and atomically rename it onto the
|
||||
/// target path. After this succeeds the writer no longer removes the file on drop.
|
||||
/// Flush remaining rows, close the temp file, and atomically replace the target.
|
||||
fn finish(mut self) -> Result<(), String> {
|
||||
self.flush_batch()?;
|
||||
if let Some(file) = self.file.as_mut() {
|
||||
file.flush().map_err(|e| format!("Failed to flush SQL file: {e}"))?;
|
||||
}
|
||||
// Close the file handle before rename (required on Windows).
|
||||
self.file.take();
|
||||
self.finished = true;
|
||||
std::fs::rename(&self.temp_path, &self.target_path)
|
||||
.map_err(|e| format!("Failed to finalize SQL export file: {e}"))?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for SqlInsertWriter {
|
||||
fn drop(&mut self) {
|
||||
if !self.finished {
|
||||
// Best-effort: close the handle then remove the temp file so the user's
|
||||
// chosen target is never left truncated or deleted by a failed export.
|
||||
self.file.take();
|
||||
let _ = std::fs::remove_file(&self.temp_path);
|
||||
}
|
||||
self.target
|
||||
.take()
|
||||
.ok_or_else(|| "SQL export target already finalized".to_string())?
|
||||
.commit()
|
||||
.map_err(|error| format!("Failed to finalize SQL export file: {error}"))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1730,6 +1755,42 @@ async fn try_export_sqlserver_query_result_stream(
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn staged_export_target_preserves_existing_destination_on_discard_and_replace_failure() {
|
||||
let dir = tempfile::tempdir().expect("temp dir");
|
||||
let destination = dir.path().join("result.csv");
|
||||
std::fs::write(&destination, "original").expect("write destination");
|
||||
|
||||
let discarded = StagedExportTarget::new(destination.to_str().expect("destination path")).expect("target");
|
||||
std::fs::write(discarded.path(), "partial").expect("write partial export");
|
||||
drop(discarded);
|
||||
assert_eq!(std::fs::read_to_string(&destination).expect("read destination"), "original");
|
||||
|
||||
let failed = StagedExportTarget::new(destination.to_str().expect("destination path")).expect("target");
|
||||
std::fs::write(failed.path(), "replacement").expect("write replacement");
|
||||
std::fs::remove_file(failed.path()).expect("remove staged path");
|
||||
assert!(failed.commit().expect_err("replace should fail").contains("open staged export file"));
|
||||
assert_eq!(std::fs::read_to_string(destination).expect("read destination"), "original");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn staged_export_targets_are_unique_same_directory_and_replace_existing_destination() {
|
||||
let dir = tempfile::tempdir().expect("temp dir");
|
||||
let destination = dir.path().join("result.csv");
|
||||
std::fs::write(&destination, "original").expect("write destination");
|
||||
let first = StagedExportTarget::new(destination.to_str().expect("destination path")).expect("first target");
|
||||
let second = StagedExportTarget::new(destination.to_str().expect("destination path")).expect("second target");
|
||||
|
||||
assert_eq!(first.path().parent(), destination.parent());
|
||||
assert_eq!(second.path().parent(), destination.parent());
|
||||
assert_ne!(first.path(), second.path());
|
||||
std::fs::write(first.path(), "replacement").expect("write replacement");
|
||||
first.commit().expect("commit export");
|
||||
drop(second);
|
||||
|
||||
assert_eq!(std::fs::read_to_string(destination).expect("read destination"), "replacement");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stream_cancel_detection_covers_driver_token_and_export_flags() {
|
||||
assert!(stream_export_was_cancelled(QUERY_CANCELED, false, false));
|
||||
|
||||
Reference in new issue
Block a user