feat(sql-file): import split SQL ZIP packages with bounded extraction
This commit is contained in:
1 parent
f43bf80015
commit
b0d30e6f73
22 files changed
+550
-118
No files matched your search
@@ -5,7 +5,7 @@ use std::time::Duration;
|
||||
use axum::extract::{Multipart, Path as AxumPath, State};
|
||||
use axum::response::sse::{Event, Sse};
|
||||
use axum::Json;
|
||||
use dbx_core::sql::{SqlFileProgress, SqlFileRequest, SqlFileStatus};
|
||||
use dbx_core::sql::{self, SqlFileProgress, SqlFileRequest, SqlFileStatus};
|
||||
use dbx_core::sql_file_import::{
|
||||
execute_sql_file_paths, sql_file_error_progress, sql_file_progress as build_sql_file_progress,
|
||||
SqlFileProgressEmitter,
|
||||
@@ -95,6 +95,31 @@ pub async fn preview_sql_file(
|
||||
let file_path = safe_uploaded_sql_path(&tmp_dir, &file_name)?;
|
||||
std::fs::write(&file_path, &data).map_err(|e| AppError::from(e.to_string()))?;
|
||||
|
||||
if file_name.to_ascii_lowercase().ends_with(".zip") {
|
||||
let extraction_dir = tmp_dir.join(format!("package-{}", Uuid::new_v4()));
|
||||
let package = dbx_core::sql_file_zip_package::extract_sql_file_zip_package(&file_path, &extraction_dir)
|
||||
.map_err(AppError::from)?;
|
||||
let paths = dbx_core::sql_file_zip_package::extracted_sql_zip_paths(&extraction_dir, &package)
|
||||
.into_iter()
|
||||
.map(|path| path.to_string_lossy().to_string())
|
||||
.collect::<Vec<_>>();
|
||||
let content =
|
||||
sql::decode_sql_file_bytes(&std::fs::read(&paths[0]).map_err(|e| AppError::from(e.to_string()))?)
|
||||
.map_err(AppError::from)?;
|
||||
let preview: String = content.chars().take(20_000).collect();
|
||||
let bootstrap_analysis = dbx_core::sql_file_import::mysql_like_sql_file_bootstrap_analysis(&content);
|
||||
return Ok(Json(serde_json::json!({
|
||||
"fileName": file_name,
|
||||
"filePath": file_path.to_string_lossy(),
|
||||
"sizeBytes": data.len(),
|
||||
"preview": preview,
|
||||
"canExecuteWithoutSelectedDatabase": bootstrap_analysis.can_execute_without_selected_database,
|
||||
"establishesDatabaseContext": bootstrap_analysis.establishes_database_context,
|
||||
"packageFilePaths": paths,
|
||||
"packagePartCount": package.part_names.len(),
|
||||
})));
|
||||
}
|
||||
|
||||
let size_bytes = data.len() as u64;
|
||||
let content =
|
||||
dbx_core::sql_file_import::read_sql_file_preview(&file_path, 1_000_000).await.map_err(AppError::from)?;
|
||||
@@ -199,6 +224,7 @@ pub async fn execute_sql_file(
|
||||
})
|
||||
.await;
|
||||
|
||||
cleanup_sql_file_package_paths(&file_paths);
|
||||
cleanup_sql_file_execution(&state_clone, &req.execution_id).await;
|
||||
});
|
||||
|
||||
@@ -211,6 +237,21 @@ fn send_sql_file_progress(tx: &broadcast::Sender<String>, progress: SqlFileProgr
|
||||
}
|
||||
}
|
||||
|
||||
fn cleanup_sql_file_package_paths(file_paths: &[PathBuf]) {
|
||||
let mut directories = std::collections::HashSet::new();
|
||||
for path in file_paths {
|
||||
let Some(parent) = path.parent() else {
|
||||
continue;
|
||||
};
|
||||
if parent.file_name().and_then(|name| name.to_str()).is_some_and(|name| name.starts_with("package-")) {
|
||||
directories.insert(parent.to_path_buf());
|
||||
}
|
||||
}
|
||||
for directory in directories {
|
||||
let _ = std::fs::remove_dir_all(directory);
|
||||
}
|
||||
}
|
||||
|
||||
async fn cleanup_sql_file_execution(state: &WebState, execution_id: &str) {
|
||||
state.remove_sse_channel(execution_id).await;
|
||||
state.sql_file_executions.write().await.remove(execution_id);
|
||||
@@ -245,14 +286,19 @@ fn cleanup_sql_file_uploads_older_than(tmp_dir: &Path, max_age: Duration) {
|
||||
};
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
let expired = entry
|
||||
.metadata()
|
||||
.ok()
|
||||
.filter(|metadata| metadata.is_file())
|
||||
.and_then(|metadata| metadata.modified().ok())
|
||||
.and_then(|modified| modified.elapsed().ok())
|
||||
.is_some_and(|age| age >= max_age);
|
||||
if expired {
|
||||
let Ok(metadata) = entry.metadata() else {
|
||||
continue;
|
||||
};
|
||||
// Extracted `package-*` directories count too: a preview that never reaches execution
|
||||
// leaves them behind, and they must expire with the uploaded files.
|
||||
let expired =
|
||||
metadata.modified().ok().and_then(|modified| modified.elapsed().ok()).is_some_and(|age| age >= max_age);
|
||||
if !expired {
|
||||
continue;
|
||||
}
|
||||
if metadata.is_dir() {
|
||||
let _ = std::fs::remove_dir_all(path);
|
||||
} else {
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user