feat(sql-file): import split SQL ZIP packages with bounded extraction

This commit is contained in:
КГБ Lieu authored and GitHub committed 2026-09-16 14:15:09 +08:00
1 parent f43bf80015
commit b0d30e6f73
22 files changed
+550 -118

No files matched your search

+55 -9
View File
@@ -5,7 +5,7 @@ use std::time::Duration;
use axum::extract::{Multipart, Path as AxumPath, State};
use axum::response::sse::{Event, Sse};
use axum::Json;
use dbx_core::sql::{SqlFileProgress, SqlFileRequest, SqlFileStatus};
use dbx_core::sql::{self, SqlFileProgress, SqlFileRequest, SqlFileStatus};
use dbx_core::sql_file_import::{
execute_sql_file_paths, sql_file_error_progress, sql_file_progress as build_sql_file_progress,
SqlFileProgressEmitter,
@@ -95,6 +95,31 @@ pub async fn preview_sql_file(
let file_path = safe_uploaded_sql_path(&tmp_dir, &file_name)?;
std::fs::write(&file_path, &data).map_err(|e| AppError::from(e.to_string()))?;
if file_name.to_ascii_lowercase().ends_with(".zip") {
let extraction_dir = tmp_dir.join(format!("package-{}", Uuid::new_v4()));
let package = dbx_core::sql_file_zip_package::extract_sql_file_zip_package(&file_path, &extraction_dir)
.map_err(AppError::from)?;
let paths = dbx_core::sql_file_zip_package::extracted_sql_zip_paths(&extraction_dir, &package)
.into_iter()
.map(|path| path.to_string_lossy().to_string())
.collect::<Vec<_>>();
let content =
sql::decode_sql_file_bytes(&std::fs::read(&paths[0]).map_err(|e| AppError::from(e.to_string()))?)
.map_err(AppError::from)?;
let preview: String = content.chars().take(20_000).collect();
let bootstrap_analysis = dbx_core::sql_file_import::mysql_like_sql_file_bootstrap_analysis(&content);
return Ok(Json(serde_json::json!({
"fileName": file_name,
"filePath": file_path.to_string_lossy(),
"sizeBytes": data.len(),
"preview": preview,
"canExecuteWithoutSelectedDatabase": bootstrap_analysis.can_execute_without_selected_database,
"establishesDatabaseContext": bootstrap_analysis.establishes_database_context,
"packageFilePaths": paths,
"packagePartCount": package.part_names.len(),
})));
}
let size_bytes = data.len() as u64;
let content =
dbx_core::sql_file_import::read_sql_file_preview(&file_path, 1_000_000).await.map_err(AppError::from)?;
@@ -199,6 +224,7 @@ pub async fn execute_sql_file(
})
.await;
cleanup_sql_file_package_paths(&file_paths);
cleanup_sql_file_execution(&state_clone, &req.execution_id).await;
});
@@ -211,6 +237,21 @@ fn send_sql_file_progress(tx: &broadcast::Sender<String>, progress: SqlFileProgr
}
}
fn cleanup_sql_file_package_paths(file_paths: &[PathBuf]) {
let mut directories = std::collections::HashSet::new();
for path in file_paths {
let Some(parent) = path.parent() else {
continue;
};
if parent.file_name().and_then(|name| name.to_str()).is_some_and(|name| name.starts_with("package-")) {
directories.insert(parent.to_path_buf());
}
}
for directory in directories {
let _ = std::fs::remove_dir_all(directory);
}
}
async fn cleanup_sql_file_execution(state: &WebState, execution_id: &str) {
state.remove_sse_channel(execution_id).await;
state.sql_file_executions.write().await.remove(execution_id);
@@ -245,14 +286,19 @@ fn cleanup_sql_file_uploads_older_than(tmp_dir: &Path, max_age: Duration) {
};
for entry in entries.flatten() {
let path = entry.path();
let expired = entry
.metadata()
.ok()
.filter(|metadata| metadata.is_file())
.and_then(|metadata| metadata.modified().ok())
.and_then(|modified| modified.elapsed().ok())
.is_some_and(|age| age >= max_age);
if expired {
let Ok(metadata) = entry.metadata() else {
continue;
};
// Extracted `package-*` directories count too: a preview that never reaches execution
// leaves them behind, and they must expire with the uploaded files.
let expired =
metadata.modified().ok().and_then(|modified| modified.elapsed().ok()).is_some_and(|age| age >= max_age);
if !expired {
continue;
}
if metadata.is_dir() {
let _ = std::fs::remove_dir_all(path);
} else {
let _ = std::fs::remove_file(path);
}
}