feat(export): support safe background query exports

This commit is contained in:
zhangyaxi authored and GitHub committed 2026-07-29 03:18:06 +08:00
1 parent 2f44d36018
commit a07def6042
17 files changed
+577 -195

No files matched your search

+100 -39
View File
@@ -2,6 +2,7 @@ use serde::{Deserialize, Serialize};
use std::fs::File;
use std::io::{BufWriter, Seek, Write};
use std::mem;
use std::path::{Path, PathBuf};
use std::sync::{
atomic::{AtomicBool, Ordering},
Arc,
@@ -102,6 +103,55 @@ pub struct QueryResultExportRequest {
pub numeric_column_right_align: bool,
}
pub struct StagedExportTarget {
destination: PathBuf,
temporary: tempfile::TempPath,
}
impl StagedExportTarget {
pub fn new(destination: &str) -> Result<Self, String> {
let destination = PathBuf::from(destination);
let parent = destination.parent().filter(|path| !path.as_os_str().is_empty()).unwrap_or(Path::new("."));
let file_name = destination.file_name().and_then(|name| name.to_str()).unwrap_or("query-result");
let temporary = tempfile::Builder::new()
.prefix(&format!(".{file_name}.dbx-export-"))
.tempfile_in(parent)
.map_err(|error| format!("Failed to create temporary export file: {error}"))?
.into_temp_path();
Ok(Self { destination, temporary })
}
pub fn path(&self) -> &Path {
self.temporary.as_ref()
}
pub fn path_string(&self) -> Result<String, String> {
self.path()
.to_str()
.map(ToOwned::to_owned)
.ok_or_else(|| "Temporary export path is not valid UTF-8".to_string())
}
pub fn commit(self) -> Result<(), String> {
let staged_file = std::fs::OpenOptions::new()
.read(true)
.write(true)
.open(self.path())
.map_err(|error| format!("Failed to open staged export file: {error}"))?;
if let Ok(metadata) = std::fs::metadata(&self.destination) {
staged_file
.set_permissions(metadata.permissions())
.map_err(|error| format!("Failed to preserve export destination permissions: {error}"))?;
}
staged_file.sync_all().map_err(|error| format!("Failed to synchronize export file: {error}"))?;
drop(staged_file);
self.temporary
.persist(&self.destination)
.map(|_| ())
.map_err(|error| format!("Failed to replace export destination: {}", error.error))
}
}
fn safe_postgres_temp_setup_sql(setup_sql: &[String]) -> Option<Vec<String>> {
if setup_sql.is_empty() {
return None;
@@ -231,41 +281,30 @@ fn sql_insert_column_types(request: &QueryResultExportRequest, column_types: &[S
}
}
/// Bounded SQL INSERT writer with temp-file + atomic-rename safety.
/// Bounded SQL INSERT writer with staged-file replacement safety.
///
/// Rows are buffered and flushed to a temp file every [`SQL_INSERT_BATCH_SIZE`]
/// rows, so memory stays bounded regardless of the query page size. The temp file
/// lives alongside the user-chosen target (`<target>.dbx-export-tmp`) and is
/// renamed onto the target only when [`SqlInsertWriter::finish`] succeeds. If the
/// writer is dropped without finishing (export error or cancellation), the temp
/// file is removed and the user's target file is never truncated or deleted.
/// rows, so memory stays bounded regardless of the query page size. The unique
/// temp file lives alongside the target and replaces it only after
/// [`SqlInsertWriter::finish`] flushes and synchronizes the complete output.
struct SqlInsertWriter {
file: Option<BufWriter<File>>,
temp_path: std::path::PathBuf,
target_path: std::path::PathBuf,
target: Option<StagedExportTarget>,
pending_rows: Vec<Vec<Value>>,
columns: Vec<String>,
column_types: Vec<Option<String>>,
database_type: DatabaseType,
schema: Option<String>,
table_name: String,
finished: bool,
}
impl SqlInsertWriter {
/// Create the writer and open the temp file. Column metadata is supplied later
/// via [`SqlInsertWriter::set_columns`] once the executed result is known.
fn create(request: &QueryResultExportRequest) -> Result<Self, String> {
let target_path = std::path::PathBuf::from(&request.file_path);
let mut temp_path = target_path.clone();
let mut temp_name = target_path
.file_name()
.map(|name| name.to_os_string())
.unwrap_or_else(|| std::ffi::OsString::from("export.sql"));
temp_name.push(".dbx-export-tmp");
temp_path.set_file_name(temp_name);
let target = StagedExportTarget::new(&request.file_path)?;
let file = BufWriter::new(
File::create(&temp_path).map_err(|e| format!("Failed to create SQL export temp file: {e}"))?,
File::create(target.path()).map_err(|e| format!("Failed to create SQL export temp file: {e}"))?,
);
let table_name = request
.export_table_name
@@ -275,15 +314,13 @@ impl SqlInsertWriter {
.to_string();
Ok(Self {
file: Some(file),
temp_path,
target_path,
target: Some(target),
pending_rows: Vec::new(),
columns: Vec::new(),
column_types: Vec::new(),
database_type: request.database_type,
schema: request.schema.clone(),
table_name,
finished: false,
})
}
@@ -330,30 +367,18 @@ impl SqlInsertWriter {
Ok(())
}
/// Flush remaining rows, close the temp file, and atomically rename it onto the
/// target path. After this succeeds the writer no longer removes the file on drop.
/// Flush remaining rows, close the temp file, and atomically replace the target.
fn finish(mut self) -> Result<(), String> {
self.flush_batch()?;
if let Some(file) = self.file.as_mut() {
file.flush().map_err(|e| format!("Failed to flush SQL file: {e}"))?;
}
// Close the file handle before rename (required on Windows).
self.file.take();
self.finished = true;
std::fs::rename(&self.temp_path, &self.target_path)
.map_err(|e| format!("Failed to finalize SQL export file: {e}"))?;
Ok(())
}
}
impl Drop for SqlInsertWriter {
fn drop(&mut self) {
if !self.finished {
// Best-effort: close the handle then remove the temp file so the user's
// chosen target is never left truncated or deleted by a failed export.
self.file.take();
let _ = std::fs::remove_file(&self.temp_path);
}
self.target
.take()
.ok_or_else(|| "SQL export target already finalized".to_string())?
.commit()
.map_err(|error| format!("Failed to finalize SQL export file: {error}"))
}
}
@@ -1730,6 +1755,42 @@ async fn try_export_sqlserver_query_result_stream(
mod tests {
use super::*;
#[test]
fn staged_export_target_preserves_existing_destination_on_discard_and_replace_failure() {
let dir = tempfile::tempdir().expect("temp dir");
let destination = dir.path().join("result.csv");
std::fs::write(&destination, "original").expect("write destination");
let discarded = StagedExportTarget::new(destination.to_str().expect("destination path")).expect("target");
std::fs::write(discarded.path(), "partial").expect("write partial export");
drop(discarded);
assert_eq!(std::fs::read_to_string(&destination).expect("read destination"), "original");
let failed = StagedExportTarget::new(destination.to_str().expect("destination path")).expect("target");
std::fs::write(failed.path(), "replacement").expect("write replacement");
std::fs::remove_file(failed.path()).expect("remove staged path");
assert!(failed.commit().expect_err("replace should fail").contains("open staged export file"));
assert_eq!(std::fs::read_to_string(destination).expect("read destination"), "original");
}
#[test]
fn staged_export_targets_are_unique_same_directory_and_replace_existing_destination() {
let dir = tempfile::tempdir().expect("temp dir");
let destination = dir.path().join("result.csv");
std::fs::write(&destination, "original").expect("write destination");
let first = StagedExportTarget::new(destination.to_str().expect("destination path")).expect("first target");
let second = StagedExportTarget::new(destination.to_str().expect("destination path")).expect("second target");
assert_eq!(first.path().parent(), destination.parent());
assert_eq!(second.path().parent(), destination.parent());
assert_ne!(first.path(), second.path());
std::fs::write(first.path(), "replacement").expect("write replacement");
first.commit().expect("commit export");
drop(second);
assert_eq!(std::fs::read_to_string(destination).expect("read destination"), "replacement");
}
#[test]
fn stream_cancel_detection_covers_driver_token_and_export_flags() {
assert!(stream_export_was_cancelled(QUERY_CANCELED, false, false));