use std::path::{Path as StdPath, PathBuf}; use std::sync::Arc; use std::time::{Duration, Instant, SystemTime}; use axum::body::Bytes; use axum::extract::{Multipart, Path, State}; use axum::response::sse::{Event, Sse}; use axum::Json; use dbx_core::table_import::{ self, TableImportParseOptions, TableImportPhase, TableImportPreviewRequest, TableImportProgress, TableImportRequest, TableImportSourceFormat, TableImportStatus, }; use dbx_core::transfer; use futures::stream::Stream; use futures::StreamExt; use serde::Deserialize; use tokio::io::AsyncWriteExt; use crate::error::AppError; use crate::state::WebState; const MAX_IMPORT_UPLOAD_BYTES: usize = 100 * 1024 * 1024; const TABLE_IMPORT_PROGRESS_TTL: Duration = Duration::from_secs(30); fn initial_import_progress(import_id: &str, started_at: Instant) -> TableImportProgress { TableImportProgress { import_id: import_id.to_string(), status: TableImportStatus::Running, phase: TableImportPhase::Preparing, rows_imported: 0, total_rows: 0, total_rows_exact: false, bytes_read: 0, total_bytes: 0, elapsed_ms: started_at.elapsed().as_millis(), error: None, } } fn failed_import_progress(import_id: &str, started_at: Instant, error: String) -> TableImportProgress { TableImportProgress { import_id: import_id.to_string(), status: TableImportStatus::Error, phase: TableImportPhase::Done, rows_imported: 0, total_rows: 0, total_rows_exact: true, bytes_read: 0, total_bytes: 0, elapsed_ms: started_at.elapsed().as_millis(), error: Some(error), } } fn send_import_progress(tx: &tokio::sync::watch::Sender, progress: &TableImportProgress) { if let Ok(json) = serde_json::to_string(progress) { tx.send_replace(json); } } fn schedule_import_progress_cleanup(state: Arc, import_id: String) { // Keep the terminal snapshot briefly so an SSE client that connects after completion can // still observe the result, then release the channel and its captured state. tokio::spawn(async move { tokio::time::sleep(TABLE_IMPORT_PROGRESS_TTL).await; state.table_import_channels.write().await.remove(&import_id); }); } #[derive(Deserialize)] #[serde(rename_all = "camelCase")] pub struct ExecuteImportWrapper { pub request: TableImportRequest, } #[derive(Deserialize)] #[serde(rename_all = "camelCase")] pub struct CancelImportRequest { pub import_id: String, } #[derive(Deserialize)] #[serde(rename_all = "camelCase")] pub struct PreviewUploadedImportRequest { pub source_ref: String, #[serde(default)] pub source_format: Option, #[serde(default)] pub parse_options: TableImportParseOptions, #[serde(default)] pub preview_limit: Option, } #[derive(Deserialize)] #[serde(rename_all = "camelCase")] pub struct ReleaseImportSourceRequest { pub source_ref: String, } pub async fn preview_import( State(state): State>, mut multipart: Multipart, ) -> Result, AppError> { let tmp_dir = import_upload_dir(&state.data_dir); std::fs::create_dir_all(&tmp_dir).map_err(|e| AppError::from(e.to_string()))?; cleanup_expired_import_uploads(&tmp_dir, Duration::from_secs(24 * 60 * 60)); let mut uploaded_file: Option<(String, PathBuf)> = None; let mut source_format: Option = None; let mut parse_options = TableImportParseOptions::default(); let mut preview_limit: Option = None; loop { let field = match multipart.next_field().await { Ok(Some(field)) => field, Ok(None) => break, Err(error) => { cleanup_pending_upload(&uploaded_file).await; return Err(AppError::from(error.to_string())); } }; let name = field.name().unwrap_or_default().to_string(); if name == "file" { if uploaded_file.is_some() { cleanup_pending_upload(&uploaded_file).await; return Err(AppError::from("Only one import file may be uploaded".to_string())); } let file_name = field.file_name().unwrap_or("upload.csv").to_string(); let source_ref = uuid::Uuid::new_v4().to_string(); let file_path = safe_uploaded_import_path(&tmp_dir, &file_name, &source_ref)?; if let Err(error) = write_import_upload(field, &file_path).await { cleanup_uploaded_import_path(&file_path).await; return Err(error); } uploaded_file = Some((source_ref, file_path)); } else { let value = match field.text().await { Ok(value) => value, Err(error) => { cleanup_pending_upload(&uploaded_file).await; return Err(AppError::from(error.to_string())); } }; match name.as_str() { "sourceFormat" => { source_format = match serde_json::from_value(serde_json::Value::String(value)) { Ok(source_format) => Some(source_format), Err(error) => { cleanup_pending_upload(&uploaded_file).await; return Err(AppError::from(error.to_string())); } }; } "parseOptions" => { parse_options = match serde_json::from_str(&value) { Ok(parse_options) => parse_options, Err(error) => { cleanup_pending_upload(&uploaded_file).await; return Err(AppError::from(error.to_string())); } }; } "previewLimit" => { preview_limit = value.parse::().ok(); } _ => {} } } } if let Some((source_ref, file_path)) = uploaded_file { let file_path_str = file_path.to_string_lossy().to_string(); let preview = table_import::preview_table_import_file_with_request(TableImportPreviewRequest { file_path: file_path_str, source_ref: Some(source_ref), source_format, parse_options, preview_limit, }) .await; let preview = match preview { Ok(preview) => preview, Err(error) => { cleanup_uploaded_import_path(&file_path).await; return Err(AppError::from(error)); } }; let preview = match serde_json::to_value(preview) { Ok(preview) => preview, Err(error) => { cleanup_uploaded_import_path(&file_path).await; return Err(AppError::from(error.to_string())); } }; return Ok(Json(preview)); } Err(AppError::from("No file uploaded".to_string())) } pub async fn preview_uploaded_import( State(state): State>, Json(request): Json, ) -> Result, AppError> { let file_path = uploaded_import_path_for_source_ref(&state.data_dir, &request.source_ref)?; let preview = table_import::preview_table_import_file_with_request(TableImportPreviewRequest { file_path: file_path.to_string_lossy().to_string(), source_ref: Some(request.source_ref), source_format: request.source_format, parse_options: request.parse_options, preview_limit: request.preview_limit, }) .await .map_err(AppError::from)?; serde_json::to_value(preview).map(Json).map_err(|error| AppError::from(error.to_string())) } pub async fn release_import_source( State(state): State>, Json(request): Json, ) -> Json { let released = match uploaded_import_path_for_source_ref(&state.data_dir, &request.source_ref) { Ok(file_path) => tokio::fs::remove_file(file_path).await.is_ok(), Err(_) => false, }; Json(serde_json::json!({ "released": released })) } async fn write_import_upload(field: axum::extract::multipart::Field<'_>, file_path: &StdPath) -> Result<(), AppError> { write_import_upload_stream(field, file_path, MAX_IMPORT_UPLOAD_BYTES).await } async fn write_import_upload_stream( mut chunks: S, file_path: &StdPath, max_upload_bytes: usize, ) -> Result<(), AppError> where S: Stream> + Unpin, E: std::fmt::Display, { let mut upload = tokio::fs::File::create(file_path).await.map_err(|error| AppError::from(error.to_string()))?; let mut uploaded_bytes = 0usize; // Stream uploads to disk so valid large CSV files don't require a second full-size in-memory copy. let result = async { while let Some(chunk) = chunks.next().await { let chunk = chunk.map_err(|error| AppError::from(error.to_string()))?; uploaded_bytes = uploaded_bytes.saturating_add(chunk.len()); if uploaded_bytes > max_upload_bytes { return Err(AppError::from(format!( "File too large: {uploaded_bytes} bytes received (max {max_upload_bytes} bytes)" ))); } upload.write_all(&chunk).await.map_err(|error| AppError::from(error.to_string()))?; } upload.flush().await.map_err(|error| AppError::from(error.to_string())) } .await; drop(upload); if result.is_err() { cleanup_uploaded_import_path(file_path).await; } result } pub async fn execute_import( State(state): State>, Json(body): Json, ) -> Result, AppError> { let started_at = Instant::now(); let mut req = body.request; let file_path = validated_uploaded_import_path(&state.data_dir, &req.file_path)?; req.file_path = file_path.to_string_lossy().to_string(); // Reject import early if the connection is read-only if let Some(name) = dbx_core::query::connection_readonly_name(&state.app, &req.connection_id).await { if !req.retain_source { cleanup_uploaded_import_source(&req.file_path).await; } return Err(AppError::from(format!( "Read-only mode: connection '{}' has read-only protection enabled. Import blocked.", name ))); } let import_id = req.import_id.clone(); let initial_progress = serde_json::to_string(&initial_import_progress(&import_id, started_at)) .map_err(|error| AppError::from(error.to_string()))?; // Unlike broadcast, watch retains the latest progress and cannot drop the terminal update // when the browser subscribes slightly after the import starts. let (tx, _) = tokio::sync::watch::channel(initial_progress); state.table_import_channels.write().await.insert(import_id.clone(), tx.clone()); let app = state.app.clone(); let state_clone = state.clone(); tokio::spawn(async move { let db_type = match transfer::get_db_type(&app, &req.connection_id).await { Ok(t) => t, Err(e) => { send_import_progress(&tx, &failed_import_progress(&req.import_id, started_at, e)); if !req.retain_source { cleanup_uploaded_import_source(&req.file_path).await; } schedule_import_progress_cleanup(state_clone, req.import_id.clone()); return; } }; let pool_key = match app.get_or_create_pool(&req.connection_id, Some(&req.database)).await { Ok(k) => k, Err(e) => { send_import_progress(&tx, &failed_import_progress(&req.import_id, started_at, e)); if !req.retain_source { cleanup_uploaded_import_source(&req.file_path).await; } schedule_import_progress_cleanup(state_clone, req.import_id.clone()); return; } }; let tx_clone = tx.clone(); let result = table_import::import_table_file_core( &app, &req, &db_type, &pool_key, |id: &str| { let id = id.to_string(); Box::pin(async move { transfer::is_cancelled(&id).await }) }, |progress| send_import_progress(&tx_clone, &progress), ) .await; let _ = result; if !req.retain_source { cleanup_uploaded_import_source(&req.file_path).await; } schedule_import_progress_cleanup(state_clone, req.import_id.clone()); }); Ok(Json(serde_json::json!({ "importId": import_id }))) } pub async fn import_progress( State(state): State>, Path(import_id): Path, ) -> Result>>, AppError> { let channels = state.table_import_channels.read().await; let tx = channels.get(&import_id).ok_or_else(|| AppError::from("Import not found".to_string()))?; let rx = tx.subscribe(); drop(channels); Ok(crate::sse::sse_from_watch(rx)) } pub async fn cancel_import( State(_state): State>, Json(req): Json, ) -> Json { transfer::set_cancelled(&req.import_id).await; Json(serde_json::json!({ "cancelled": true })) } fn import_upload_dir(data_dir: &StdPath) -> PathBuf { data_dir.join("tmp").join("table_import") } fn safe_uploaded_import_path(tmp_dir: &StdPath, file_name: &str, source_ref: &str) -> Result { let base_name = file_name.rsplit(['/', '\\']).find(|part| !part.is_empty()).unwrap_or("upload.csv").trim(); if base_name.is_empty() || base_name == "." || base_name == ".." { return Err(AppError::from("Invalid import file name".to_string())); } Ok(tmp_dir.join(format!("{source_ref}-{base_name}"))) } fn validated_uploaded_import_path(data_dir: &StdPath, file_path: &str) -> Result { let path = PathBuf::from(file_path); if !path.is_absolute() { return Err(AppError::from("Import source path must be absolute".to_string())); } let tmp_dir = import_upload_dir(data_dir).canonicalize().map_err(|e| AppError::from(e.to_string()))?; let canonical_path = path.canonicalize().map_err(|e| AppError::from(format!("Import source is no longer available: {e}")))?; if !canonical_path.starts_with(&tmp_dir) { return Err(AppError::from("Import source must be inside the uploaded import directory".to_string())); } Ok(canonical_path) } fn uploaded_import_path_for_source_ref(data_dir: &StdPath, source_ref: &str) -> Result { uuid::Uuid::parse_str(source_ref).map_err(|_| AppError::from("Invalid import source reference".to_string()))?; let tmp_dir = import_upload_dir(data_dir); let prefix = format!("{source_ref}-"); let mut matches = std::fs::read_dir(&tmp_dir) .map_err(|_| AppError::from("Import source is no longer available".to_string()))? .filter_map(Result::ok) .filter(|entry| entry.file_name().to_string_lossy().starts_with(&prefix)) .map(|entry| entry.path()); let file_path = matches.next().ok_or_else(|| AppError::from("Import source is no longer available".to_string()))?; if matches.next().is_some() { return Err(AppError::from("Import source reference is ambiguous".to_string())); } validated_uploaded_import_path(data_dir, &file_path.to_string_lossy()) } fn cleanup_expired_import_uploads(tmp_dir: &StdPath, max_age: Duration) { let Ok(entries) = std::fs::read_dir(tmp_dir) else { return; }; let now = SystemTime::now(); for entry in entries.flatten() { let Ok(metadata) = entry.metadata() else { continue; }; let Ok(modified) = metadata.modified() else { continue; }; if now.duration_since(modified).map(|age| age > max_age).unwrap_or(false) { let _ = std::fs::remove_file(entry.path()); } } } async fn cleanup_uploaded_import_source(file_path: &str) { let _ = tokio::fs::remove_file(file_path).await; } async fn cleanup_uploaded_import_path(file_path: &StdPath) { let _ = tokio::fs::remove_file(file_path).await; } async fn cleanup_pending_upload(uploaded_file: &Option<(String, PathBuf)>) { if let Some((_, file_path)) = uploaded_file { cleanup_uploaded_import_path(file_path).await; } } #[cfg(test)] mod tests { use super::*; use futures::stream; fn test_upload_path() -> PathBuf { std::env::temp_dir().join(format!("dbx-table-import-test-{}", uuid::Uuid::new_v4())) } #[tokio::test] async fn streams_import_upload_to_disk() { let file_path = test_upload_path(); let chunks = stream::iter([Ok::<_, String>(Bytes::from_static(b"a,b\n")), Ok(Bytes::from_static(b"1,2\n"))]); assert!(write_import_upload_stream(chunks, &file_path, 8).await.is_ok()); assert_eq!(tokio::fs::read(&file_path).await.unwrap(), b"a,b\n1,2\n"); cleanup_uploaded_import_path(&file_path).await; } #[tokio::test] async fn removes_partial_upload_when_size_limit_is_exceeded() { let file_path = test_upload_path(); let chunks = stream::iter([Ok::<_, String>(Bytes::from_static(b"1234")), Ok(Bytes::from_static(b"5"))]); let error = write_import_upload_stream(chunks, &file_path, 4).await.unwrap_err(); assert!(error.message.contains("File too large")); assert!(!file_path.exists()); } #[tokio::test] async fn removes_partial_upload_when_stream_read_fails() { let file_path = test_upload_path(); let chunks = stream::iter([Ok(Bytes::from_static(b"1234")), Err("multipart stream failed")]); let error = write_import_upload_stream(chunks, &file_path, 8).await.unwrap_err(); assert_eq!(error.message, "multipart stream failed"); assert!(!file_path.exists()); } #[test] fn resolves_uploaded_import_source_by_uuid_reference() { let data_dir = std::env::temp_dir().join(format!("dbx-table-import-data-{}", uuid::Uuid::new_v4())); let upload_dir = import_upload_dir(&data_dir); std::fs::create_dir_all(&upload_dir).unwrap(); let source_ref = uuid::Uuid::new_v4().to_string(); let file_path = upload_dir.join(format!("{source_ref}-users.csv")); std::fs::write(&file_path, b"id,name\n1,Ada\n").unwrap(); let resolved = uploaded_import_path_for_source_ref(&data_dir, &source_ref) .unwrap_or_else(|error| panic!("failed to resolve uploaded source: {}", error.message)); assert_eq!(resolved, file_path.canonicalize().unwrap()); assert!(uploaded_import_path_for_source_ref(&data_dir, "../users.csv").is_err()); assert!(uploaded_import_path_for_source_ref(&data_dir, &uuid::Uuid::new_v4().to_string()).is_err()); let _ = std::fs::remove_dir_all(data_dir); } #[test] fn import_progress_watch_replays_terminal_state_to_late_subscribers() { let started_at = Instant::now(); let initial = serde_json::to_string(&initial_import_progress("import-1", started_at)).unwrap(); let (tx, _) = tokio::sync::watch::channel(initial); let terminal = TableImportProgress { import_id: "import-1".to_string(), status: TableImportStatus::Done, phase: TableImportPhase::Done, rows_imported: 2, total_rows: 2, total_rows_exact: true, bytes_read: 20, total_bytes: 20, elapsed_ms: 10, error: None, }; send_import_progress(&tx, &terminal); let late_subscriber = tx.subscribe(); let replayed: serde_json::Value = serde_json::from_str(&late_subscriber.borrow()).unwrap(); assert_eq!(replayed["status"], "done"); assert_eq!(replayed["rowsImported"], 2); } }