Files
dbx/crates/dbx-web/src/routes/sql_file.rs
T

462 lines
17 KiB
Rust

use std::path::{Path, PathBuf};
use std::sync::Arc;
use axum::extract::{Multipart, Path as AxumPath, State};
use axum::response::sse::{Event, Sse};
use axum::Json;
use dbx_core::query;
use dbx_core::sql;
use dbx_core::sql::{SqlFileImportStatement, SqlFileImportStatementKind};
use futures::stream::Stream;
use serde::Deserialize;
use crate::error::AppError;
use crate::state::WebState;
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct SqlFileExecuteRequest {
pub execution_id: String,
pub connection_id: String,
pub database: String,
pub file_path: String,
pub continue_on_error: bool,
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct SqlFileExecuteWrapper {
pub request: SqlFileExecuteRequest,
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct CancelSqlFileRequest {
pub execution_id: String,
}
pub async fn preview_sql_file(
State(state): State<Arc<WebState>>,
mut multipart: Multipart,
) -> Result<Json<serde_json::Value>, AppError> {
let tmp_dir = state.data_dir.join("tmp");
std::fs::create_dir_all(&tmp_dir).map_err(|e| AppError(e.to_string()))?;
if let Some(field) = multipart.next_field().await.map_err(|e| AppError(e.to_string()))? {
let file_name = field.file_name().unwrap_or("upload.sql").to_string();
let data = field.bytes().await.map_err(|e| AppError(e.to_string()))?;
let file_path = safe_uploaded_sql_path(&tmp_dir, &file_name)?;
std::fs::write(&file_path, &data).map_err(|e| AppError(e.to_string()))?;
let size_bytes = data.len() as u64;
let content = sql::decode_sql_file_bytes(&data).map_err(AppError)?;
let preview: String = content.chars().take(20_000).collect();
return Ok(Json(serde_json::json!({
"fileName": file_name,
"filePath": file_path.to_string_lossy(),
"sizeBytes": size_bytes,
"preview": preview,
})));
}
Err(AppError("No file uploaded".to_string()))
}
pub async fn execute_sql_file(
State(state): State<Arc<WebState>>,
Json(body): Json<SqlFileExecuteWrapper>,
) -> Result<Json<serde_json::Value>, AppError> {
let req = body.request;
let execution_id = req.execution_id.clone();
let (tx, _) = tokio::sync::broadcast::channel::<String>(256);
state.sse_channels.write().await.insert(execution_id.clone(), tx.clone());
let app = state.app.clone();
let state_clone = state.clone();
let file_path = validated_uploaded_sql_path(&state.data_dir, &req.file_path)?;
tokio::spawn(async move {
match std::fs::metadata(&file_path) {
Ok(meta) if meta.len() > 200 * 1024 * 1024 => {
let progress = dbx_core::sql::SqlFileProgress {
execution_id: req.execution_id.clone(),
status: dbx_core::sql::SqlFileStatus::Error,
statement_index: 0,
success_count: 0,
failure_count: 0,
affected_rows: 0,
elapsed_ms: 0,
statement_summary: String::new(),
error: Some(format!("File too large: {} bytes (max {} bytes)", meta.len(), 200 * 1024 * 1024)),
};
if let Ok(json) = serde_json::to_string(&progress) {
let _ = tx.send(json);
}
return;
}
Err(e) => {
let progress = dbx_core::sql::SqlFileProgress {
execution_id: req.execution_id.clone(),
status: dbx_core::sql::SqlFileStatus::Error,
statement_index: 0,
success_count: 0,
failure_count: 0,
affected_rows: 0,
elapsed_ms: 0,
statement_summary: String::new(),
error: Some(e.to_string()),
};
if let Ok(json) = serde_json::to_string(&progress) {
let _ = tx.send(json);
}
return;
}
_ => {}
}
let file_content = match std::fs::read(&file_path).and_then(|bytes| {
sql::decode_sql_file_bytes(&bytes)
.map_err(|message| std::io::Error::new(std::io::ErrorKind::InvalidData, message))
}) {
Ok(content) => content,
Err(e) => {
let progress = dbx_core::sql::SqlFileProgress {
execution_id: req.execution_id.clone(),
status: dbx_core::sql::SqlFileStatus::Error,
statement_index: 0,
success_count: 0,
failure_count: 0,
affected_rows: 0,
elapsed_ms: 0,
statement_summary: String::new(),
error: Some(e.to_string()),
};
if let Ok(json) = serde_json::to_string(&progress) {
let _ = tx.send(json);
}
return;
}
};
// Send started
let started = dbx_core::sql::SqlFileProgress {
execution_id: req.execution_id.clone(),
status: dbx_core::sql::SqlFileStatus::Started,
statement_index: 0,
success_count: 0,
failure_count: 0,
affected_rows: 0,
elapsed_ms: 0,
statement_summary: String::new(),
error: None,
};
if let Ok(json) = serde_json::to_string(&started) {
let _ = tx.send(json);
}
let import_target = {
let configs = app.configs.read().await;
configs.get(&req.connection_id).map(|config| (config.db_type, config.driver_profile.clone()))
};
let statements = import_target
.as_ref()
.map(|(db_type, _)| sql::split_sql_statements_for_database(&file_content, *db_type))
.unwrap_or_else(|| sql::split_sql_statements(&file_content));
let planned_statements = sql::optimize_sql_file_import_statements(
&statements,
import_target.as_ref().map(|(db_type, _)| *db_type),
import_target.as_ref().and_then(|(_, driver_profile)| driver_profile.as_deref()),
);
let start = std::time::Instant::now();
let mut success_count = 0usize;
let mut failure_count = 0usize;
let mut total_affected: u64 = 0;
let mut statement_index = 0usize;
for planned_statement in planned_statements {
let next_statement_index = statement_index + planned_statement.source_statement_count;
let summary = sql::statement_summary(&planned_statement.sql);
// Send running
let running = dbx_core::sql::SqlFileProgress {
execution_id: req.execution_id.clone(),
status: dbx_core::sql::SqlFileStatus::Running,
statement_index: next_statement_index,
success_count,
failure_count,
affected_rows: total_affected,
elapsed_ms: start.elapsed().as_millis(),
statement_summary: summary.clone(),
error: None,
};
if let Ok(json) = serde_json::to_string(&running) {
let _ = tx.send(json);
}
if planned_statement.kind == SqlFileImportStatementKind::Skip {
success_count += planned_statement.source_statement_count;
let done = dbx_core::sql::SqlFileProgress {
execution_id: req.execution_id.clone(),
status: dbx_core::sql::SqlFileStatus::StatementDone,
statement_index: next_statement_index,
success_count,
failure_count,
affected_rows: total_affected,
elapsed_ms: start.elapsed().as_millis(),
statement_summary: summary,
error: None,
};
if let Ok(json) = serde_json::to_string(&done) {
let _ = tx.send(json);
}
statement_index = next_statement_index;
continue;
}
match query::execute_sql_statement(
&app,
&req.connection_id,
&req.database,
&planned_statement.sql,
None,
None,
)
.await
{
Ok(result) => {
success_count += planned_statement.source_statement_count;
total_affected += result.affected_rows;
let done = dbx_core::sql::SqlFileProgress {
execution_id: req.execution_id.clone(),
status: dbx_core::sql::SqlFileStatus::StatementDone,
statement_index: next_statement_index,
success_count,
failure_count,
affected_rows: total_affected,
elapsed_ms: start.elapsed().as_millis(),
statement_summary: summary,
error: None,
};
if let Ok(json) = serde_json::to_string(&done) {
let _ = tx.send(json);
}
}
Err(e) => {
let stopped = execute_web_merged_statement_fallback(
&app,
&req,
&tx,
start,
statement_index + 1,
&planned_statement,
&mut success_count,
&mut failure_count,
&mut total_affected,
e,
)
.await;
if stopped || !req.continue_on_error {
break;
}
}
}
statement_index = next_statement_index;
}
// Send final done
let final_done = dbx_core::sql::SqlFileProgress {
execution_id: req.execution_id.clone(),
status: dbx_core::sql::SqlFileStatus::Done,
statement_index,
success_count,
failure_count,
affected_rows: total_affected,
elapsed_ms: start.elapsed().as_millis(),
statement_summary: String::new(),
error: None,
};
if let Ok(json) = serde_json::to_string(&final_done) {
let _ = tx.send(json);
}
state_clone.remove_sse_channel(&req.execution_id).await;
});
Ok(Json(serde_json::json!({ "executionId": execution_id })))
}
async fn execute_web_merged_statement_fallback(
app: &Arc<dbx_core::connection::AppState>,
req: &SqlFileExecuteRequest,
tx: &tokio::sync::broadcast::Sender<String>,
start: std::time::Instant,
first_statement_index: usize,
statement: &SqlFileImportStatement,
success_count: &mut usize,
failure_count: &mut usize,
total_affected: &mut u64,
batch_error: String,
) -> bool {
if statement.source_statement_count <= 1 {
let summary = sql::statement_summary(&statement.sql);
*failure_count += 1;
let failed = dbx_core::sql::SqlFileProgress {
execution_id: req.execution_id.clone(),
status: dbx_core::sql::SqlFileStatus::StatementFailed,
statement_index: first_statement_index,
success_count: *success_count,
failure_count: *failure_count,
affected_rows: *total_affected,
elapsed_ms: start.elapsed().as_millis(),
statement_summary: summary,
error: Some(batch_error),
};
if let Ok(json) = serde_json::to_string(&failed) {
let _ = tx.send(json);
}
return false;
}
for (offset, source_sql) in statement.source_sqls.iter().enumerate() {
let statement_index = first_statement_index + offset;
let summary = sql::statement_summary(source_sql);
let running = dbx_core::sql::SqlFileProgress {
execution_id: req.execution_id.clone(),
status: dbx_core::sql::SqlFileStatus::Running,
statement_index,
success_count: *success_count,
failure_count: *failure_count,
affected_rows: *total_affected,
elapsed_ms: start.elapsed().as_millis(),
statement_summary: summary.clone(),
error: None,
};
if let Ok(json) = serde_json::to_string(&running) {
let _ = tx.send(json);
}
match query::execute_sql_statement(app, &req.connection_id, &req.database, source_sql, None, None).await {
Ok(result) => {
*success_count += 1;
*total_affected += result.affected_rows;
let done = dbx_core::sql::SqlFileProgress {
execution_id: req.execution_id.clone(),
status: dbx_core::sql::SqlFileStatus::StatementDone,
statement_index,
success_count: *success_count,
failure_count: *failure_count,
affected_rows: *total_affected,
elapsed_ms: start.elapsed().as_millis(),
statement_summary: summary,
error: None,
};
if let Ok(json) = serde_json::to_string(&done) {
let _ = tx.send(json);
}
}
Err(error) => {
*failure_count += 1;
let failed = dbx_core::sql::SqlFileProgress {
execution_id: req.execution_id.clone(),
status: dbx_core::sql::SqlFileStatus::StatementFailed,
statement_index,
success_count: *success_count,
failure_count: *failure_count,
affected_rows: *total_affected,
elapsed_ms: start.elapsed().as_millis(),
statement_summary: summary,
error: Some(error),
};
if let Ok(json) = serde_json::to_string(&failed) {
let _ = tx.send(json);
}
if !req.continue_on_error {
return true;
}
}
}
}
false
}
fn safe_uploaded_sql_path(tmp_dir: &Path, file_name: &str) -> Result<PathBuf, AppError> {
let base_name =
file_name.rsplit(|ch| ch == '/' || ch == '\\').find(|part| !part.is_empty()).unwrap_or("upload.sql").trim();
if base_name.is_empty() || base_name == "." || base_name == ".." {
return Err(AppError("Invalid SQL file name".to_string()));
}
Ok(tmp_dir.join(base_name))
}
fn validated_uploaded_sql_path(data_dir: &Path, file_path: &str) -> Result<PathBuf, AppError> {
let path = PathBuf::from(file_path);
if !path.is_absolute() {
return Err(AppError("File path must be absolute".to_string()));
}
let tmp_dir = data_dir.join("tmp").canonicalize().map_err(|e| AppError(e.to_string()))?;
let canonical_path = path.canonicalize().map_err(|e| AppError(e.to_string()))?;
if !canonical_path.starts_with(&tmp_dir) {
return Err(AppError("File path must be inside the uploaded SQL directory".to_string()));
}
Ok(canonical_path)
}
pub async fn sql_file_progress(
State(state): State<Arc<WebState>>,
AxumPath(execution_id): AxumPath<String>,
) -> Result<Sse<impl Stream<Item = Result<Event, std::convert::Infallible>>>, AppError> {
let channels = state.sse_channels.read().await;
let tx = channels.get(&execution_id).ok_or_else(|| AppError("Execution not found".to_string()))?;
let rx = tx.subscribe();
drop(channels);
Ok(crate::sse::sse_from_channel(rx))
}
pub async fn cancel_sql_file(
State(state): State<Arc<WebState>>,
Json(req): Json<CancelSqlFileRequest>,
) -> Json<serde_json::Value> {
// Remove the channel to stop the execution loop
state.sse_channels.write().await.remove(&req.execution_id);
Json(serde_json::json!({ "cancelled": true }))
}
#[cfg(test)]
mod tests {
use super::{safe_uploaded_sql_path, validated_uploaded_sql_path};
#[test]
fn uploaded_sql_path_uses_only_the_file_name() {
let data_dir = std::env::temp_dir().join(format!("dbx-web-sql-file-test-{}", uuid::Uuid::new_v4()));
let tmp_dir = data_dir.join("tmp");
let path = match safe_uploaded_sql_path(&tmp_dir, "../outside.sql") {
Ok(path) => path,
Err(error) => panic!("{}", error.0),
};
assert_eq!(path, tmp_dir.join("outside.sql"));
let _ = std::fs::remove_dir_all(data_dir);
}
#[test]
fn execution_path_must_stay_inside_uploaded_tmp_dir() {
let data_dir = std::env::temp_dir().join(format!("dbx-web-sql-file-test-{}", uuid::Uuid::new_v4()));
let tmp_dir = data_dir.join("tmp");
std::fs::create_dir_all(&tmp_dir).unwrap();
let outside = data_dir.join("outside.sql");
std::fs::write(&outside, "select 1;").unwrap();
let result = validated_uploaded_sql_path(&data_dir, &outside.to_string_lossy());
assert!(result.is_err());
let _ = std::fs::remove_dir_all(data_dir);
}
}