fix(sql-file): stream sqlserver zip imports incrementally

This commit is contained in:
КГБ Lieu authored and GitHub committed 2026-10-01 02:40:02 +08:00
1 parent e0f802c031
commit 64e5847f74
7 files changed
+659 -66

No files matched your search

+70 -12
View File
@@ -8,8 +8,8 @@ use axum::response::sse::{Event, Sse};
use axum::Json;
use dbx_core::sql::{self, SqlFileProgress, SqlFileRequest, SqlFileStatus};
use dbx_core::sql_file_import::{
execute_sql_file_paths, sql_file_error_progress, sql_file_progress as build_sql_file_progress,
SqlFileProgressEmitter,
execute_sql_file_paths, execute_sql_file_zip_package_paths, sql_file_error_progress,
sql_file_progress as build_sql_file_progress, SqlFileProgressEmitter,
};
use futures::stream::Stream;
use serde::Deserialize;
@@ -223,7 +223,11 @@ async fn preview_uploaded_sql_file(
let tmp_dir = file_path.parent().ok_or_else(|| AppError::from("Invalid SQL file path".to_string()))?;
if file_name.to_ascii_lowercase().ends_with(".zip") {
let extraction_dir = tmp_dir.join(format!("package-{}", Uuid::new_v4()));
let extraction_dir = tmp_dir.join(format!(
"{}{}",
dbx_core::sql_file_zip_package::SQL_FILE_ZIP_EXTRACTION_DIR_PREFIX_WEB,
Uuid::new_v4()
));
let package = dbx_core::sql_file_zip_package::extract_sql_file_zip_package(file_path, &extraction_dir)
.map_err(AppError::from)?;
let paths = dbx_core::sql_file_zip_package::extracted_sql_zip_paths(&extraction_dir, &package)
@@ -272,6 +276,7 @@ pub async fn execute_sql_file(
.iter()
.map(|file_path| validated_uploaded_sql_path(&state.data_dir, file_path))
.collect::<Result<Vec<_>, _>>()?;
let is_zip_package = execution_targets_zip_package(&file_paths);
let managed_previews = state.managed_sql_previews.claim(&file_paths)?;
// Fast-fail: reject early if the connection is read-only (individual statements are also checked in do_execute)
@@ -344,10 +349,17 @@ pub async fn execute_sql_file(
let file_path_refs: Vec<&Path> = file_paths.iter().map(PathBuf::as_path).collect();
// The core executor emits exactly one terminal Error with the latest
// cumulative counters before returning Err.
let _ = execute_sql_file_paths(&app, &req, &file_path_refs, token, started_at, |progress| {
progress_emitter.emit(progress);
})
.await;
let _ = if is_zip_package {
execute_sql_file_zip_package_paths(&app, &req, &file_path_refs, token, started_at, |progress| {
progress_emitter.emit(progress);
})
.await
} else {
execute_sql_file_paths(&app, &req, &file_path_refs, token, started_at, |progress| {
progress_emitter.emit(progress);
})
.await
};
cleanup_sql_file_package_paths(&file_paths);
cleanup_sql_file_execution(&state_clone, &req.execution_id).await;
@@ -365,11 +377,11 @@ fn send_sql_file_progress(tx: &broadcast::Sender<String>, progress: SqlFileProgr
fn cleanup_sql_file_package_paths(file_paths: &[PathBuf]) {
let mut directories = std::collections::HashSet::new();
for path in file_paths {
let Some(parent) = path.parent() else {
continue;
};
if parent.file_name().and_then(|name| name.to_str()).is_some_and(|name| name.starts_with("package-")) {
directories.insert(parent.to_path_buf());
if dbx_core::sql_file_zip_package::is_extracted_sql_zip_package_path(
path,
dbx_core::sql_file_zip_package::SQL_FILE_ZIP_EXTRACTION_DIR_PREFIX_WEB,
) {
directories.insert(path.parent().expect("checked by is_extracted_sql_zip_package_path").to_path_buf());
}
}
for directory in directories {
@@ -436,6 +448,23 @@ fn cleanup_sql_file_uploads_except(tmp_dir: &Path, max_age: Duration, active: &H
}
}
/// The frontend always sends the extracted `.sql` part paths as the execution
/// paths for a ZIP upload, never the original `.zip` path (that stays only in
/// `SqlFileRequest::file_path`), so routing to the ZIP-package incremental
/// -flush importer must be derived from those actual execution paths, not
/// from `req.file_path`'s extension. Web only ever extracts under the
/// `package-` prefix, and every path reaching here has already passed
/// `validated_uploaded_sql_path`, which confines it to the managed upload
/// tmp dir.
fn execution_targets_zip_package(file_paths: &[PathBuf]) -> bool {
file_paths.iter().any(|path| {
dbx_core::sql_file_zip_package::is_extracted_sql_zip_package_path(
path,
dbx_core::sql_file_zip_package::SQL_FILE_ZIP_EXTRACTION_DIR_PREFIX_WEB,
)
})
}
fn validated_uploaded_sql_path(data_dir: &Path, file_path: &str) -> Result<PathBuf, AppError> {
let path = PathBuf::from(file_path);
if !path.is_absolute() {
@@ -633,6 +662,35 @@ mod tests {
assert!(state.sql_file_executions.read().await.contains_key("existing-job"));
}
#[test]
fn zip_package_routing_is_selected_from_extracted_part_paths_not_the_original_zip_name() {
// Regression for PR #10632 review: the frontend sends the extracted
// `.sql` part paths as execution paths, never the original `.zip`
// upload path, so routing must key off the actual paths, not
// `req.file_path`'s extension.
let zip_upload_part_paths = vec![
PathBuf::from("/data/tmp/sql_file/package-11111111-1111-1111-1111-111111111111/00001-dump.sql"),
PathBuf::from("/data/tmp/sql_file/package-11111111-1111-1111-1111-111111111111/00002-dump.sql"),
];
assert!(execution_targets_zip_package(&zip_upload_part_paths));
let ordinary_upload_paths = vec![PathBuf::from("/data/tmp/sql_file/upload-22222222.sql")];
assert!(!execution_targets_zip_package(&ordinary_upload_paths));
// A single-file (non-multi-file) request always executes exactly the
// `req.file_path` value as its only path, so this also exercises the
// no-`file_paths`-supplied call shape from `execute_sql_file`.
let single_zip_part_as_sole_path = vec![PathBuf::from("/data/tmp/sql_file/package-33333333/00001-dump.sql")];
assert!(execution_targets_zip_package(&single_zip_part_as_sole_path));
// Desktop extraction dirs (`dbx-sql-package-`, under the OS temp dir)
// are not this site's to match: web only ever extracts `package-*`
// dirs, and the generic name must not pull user directories into the
// ZIP importer or its cleanup.
let desktop_style_paths = vec![PathBuf::from("/tmp/dbx-sql-package-44444444/00001-dump.sql")];
assert!(!execution_targets_zip_package(&desktop_style_paths));
}
#[test]
fn sql_file_upload_limit_defaults_and_clamps() {
assert_eq!(sql_file_upload_max_bytes_from_value(None), SQL_FILE_UPLOAD_MAX_BYTES);