Files
dbx/crates/dbx-web/src/routes/table_import.rs
T

529 lines
20 KiB
Rust

use std::path::{Path as StdPath, PathBuf};
use std::sync::Arc;
use std::time::{Duration, Instant, SystemTime};
use axum::body::Bytes;
use axum::extract::{Multipart, Path, State};
use axum::response::sse::{Event, Sse};
use axum::Json;
use dbx_core::table_import::{
self, TableImportParseOptions, TableImportPhase, TableImportPreviewRequest, TableImportProgress,
TableImportRequest, TableImportSourceFormat, TableImportStatus,
};
use dbx_core::transfer;
use futures::stream::Stream;
use futures::StreamExt;
use serde::Deserialize;
use tokio::io::AsyncWriteExt;
use crate::error::AppError;
use crate::state::WebState;
const MAX_IMPORT_UPLOAD_BYTES: usize = 100 * 1024 * 1024;
const TABLE_IMPORT_PROGRESS_TTL: Duration = Duration::from_secs(30);
fn initial_import_progress(import_id: &str, started_at: Instant) -> TableImportProgress {
TableImportProgress {
import_id: import_id.to_string(),
status: TableImportStatus::Running,
phase: TableImportPhase::Preparing,
rows_imported: 0,
total_rows: 0,
total_rows_exact: false,
bytes_read: 0,
total_bytes: 0,
elapsed_ms: started_at.elapsed().as_millis(),
error: None,
}
}
fn failed_import_progress(import_id: &str, started_at: Instant, error: String) -> TableImportProgress {
TableImportProgress {
import_id: import_id.to_string(),
status: TableImportStatus::Error,
phase: TableImportPhase::Done,
rows_imported: 0,
total_rows: 0,
total_rows_exact: true,
bytes_read: 0,
total_bytes: 0,
elapsed_ms: started_at.elapsed().as_millis(),
error: Some(error),
}
}
fn send_import_progress(tx: &tokio::sync::watch::Sender<String>, progress: &TableImportProgress) {
if let Ok(json) = serde_json::to_string(progress) {
tx.send_replace(json);
}
}
fn schedule_import_progress_cleanup(state: Arc<WebState>, import_id: String) {
// Keep the terminal snapshot briefly so an SSE client that connects after completion can
// still observe the result, then release the channel and its captured state.
tokio::spawn(async move {
tokio::time::sleep(TABLE_IMPORT_PROGRESS_TTL).await;
state.table_import_channels.write().await.remove(&import_id);
});
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ExecuteImportWrapper {
pub request: TableImportRequest,
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct CancelImportRequest {
pub import_id: String,
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct PreviewUploadedImportRequest {
pub source_ref: String,
#[serde(default)]
pub source_format: Option<TableImportSourceFormat>,
#[serde(default)]
pub parse_options: TableImportParseOptions,
#[serde(default)]
pub preview_limit: Option<usize>,
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ReleaseImportSourceRequest {
pub source_ref: String,
}
pub async fn preview_import(
State(state): State<Arc<WebState>>,
mut multipart: Multipart,
) -> Result<Json<serde_json::Value>, AppError> {
let tmp_dir = import_upload_dir(&state.data_dir);
std::fs::create_dir_all(&tmp_dir).map_err(|e| AppError::from(e.to_string()))?;
cleanup_expired_import_uploads(&tmp_dir, Duration::from_secs(24 * 60 * 60));
let mut uploaded_file: Option<(String, PathBuf)> = None;
let mut source_format: Option<TableImportSourceFormat> = None;
let mut parse_options = TableImportParseOptions::default();
let mut preview_limit: Option<usize> = None;
loop {
let field = match multipart.next_field().await {
Ok(Some(field)) => field,
Ok(None) => break,
Err(error) => {
cleanup_pending_upload(&uploaded_file).await;
return Err(AppError::from(error.to_string()));
}
};
let name = field.name().unwrap_or_default().to_string();
if name == "file" {
if uploaded_file.is_some() {
cleanup_pending_upload(&uploaded_file).await;
return Err(AppError::from("Only one import file may be uploaded".to_string()));
}
let file_name = field.file_name().unwrap_or("upload.csv").to_string();
let source_ref = uuid::Uuid::new_v4().to_string();
let file_path = safe_uploaded_import_path(&tmp_dir, &file_name, &source_ref)?;
if let Err(error) = write_import_upload(field, &file_path).await {
cleanup_uploaded_import_path(&file_path).await;
return Err(error);
}
uploaded_file = Some((source_ref, file_path));
} else {
let value = match field.text().await {
Ok(value) => value,
Err(error) => {
cleanup_pending_upload(&uploaded_file).await;
return Err(AppError::from(error.to_string()));
}
};
match name.as_str() {
"sourceFormat" => {
source_format = match serde_json::from_value(serde_json::Value::String(value)) {
Ok(source_format) => Some(source_format),
Err(error) => {
cleanup_pending_upload(&uploaded_file).await;
return Err(AppError::from(error.to_string()));
}
};
}
"parseOptions" => {
parse_options = match serde_json::from_str(&value) {
Ok(parse_options) => parse_options,
Err(error) => {
cleanup_pending_upload(&uploaded_file).await;
return Err(AppError::from(error.to_string()));
}
};
}
"previewLimit" => {
preview_limit = value.parse::<usize>().ok();
}
_ => {}
}
}
}
if let Some((source_ref, file_path)) = uploaded_file {
let file_path_str = file_path.to_string_lossy().to_string();
let preview = table_import::preview_table_import_file_with_request(TableImportPreviewRequest {
file_path: file_path_str,
source_ref: Some(source_ref),
source_format,
parse_options,
preview_limit,
})
.await;
let preview = match preview {
Ok(preview) => preview,
Err(error) => {
cleanup_uploaded_import_path(&file_path).await;
return Err(AppError::from(error));
}
};
let preview = match serde_json::to_value(preview) {
Ok(preview) => preview,
Err(error) => {
cleanup_uploaded_import_path(&file_path).await;
return Err(AppError::from(error.to_string()));
}
};
return Ok(Json(preview));
}
Err(AppError::from("No file uploaded".to_string()))
}
pub async fn preview_uploaded_import(
State(state): State<Arc<WebState>>,
Json(request): Json<PreviewUploadedImportRequest>,
) -> Result<Json<serde_json::Value>, AppError> {
let file_path = uploaded_import_path_for_source_ref(&state.data_dir, &request.source_ref)?;
let preview = table_import::preview_table_import_file_with_request(TableImportPreviewRequest {
file_path: file_path.to_string_lossy().to_string(),
source_ref: Some(request.source_ref),
source_format: request.source_format,
parse_options: request.parse_options,
preview_limit: request.preview_limit,
})
.await
.map_err(AppError::from)?;
serde_json::to_value(preview).map(Json).map_err(|error| AppError::from(error.to_string()))
}
pub async fn release_import_source(
State(state): State<Arc<WebState>>,
Json(request): Json<ReleaseImportSourceRequest>,
) -> Json<serde_json::Value> {
let released = match uploaded_import_path_for_source_ref(&state.data_dir, &request.source_ref) {
Ok(file_path) => tokio::fs::remove_file(file_path).await.is_ok(),
Err(_) => false,
};
Json(serde_json::json!({ "released": released }))
}
async fn write_import_upload(field: axum::extract::multipart::Field<'_>, file_path: &StdPath) -> Result<(), AppError> {
write_import_upload_stream(field, file_path, MAX_IMPORT_UPLOAD_BYTES).await
}
async fn write_import_upload_stream<S, E>(
mut chunks: S,
file_path: &StdPath,
max_upload_bytes: usize,
) -> Result<(), AppError>
where
S: Stream<Item = Result<Bytes, E>> + Unpin,
E: std::fmt::Display,
{
let mut upload = tokio::fs::File::create(file_path).await.map_err(|error| AppError::from(error.to_string()))?;
let mut uploaded_bytes = 0usize;
// Stream uploads to disk so valid large CSV files don't require a second full-size in-memory copy.
let result = async {
while let Some(chunk) = chunks.next().await {
let chunk = chunk.map_err(|error| AppError::from(error.to_string()))?;
uploaded_bytes = uploaded_bytes.saturating_add(chunk.len());
if uploaded_bytes > max_upload_bytes {
return Err(AppError::from(format!(
"File too large: {uploaded_bytes} bytes received (max {max_upload_bytes} bytes)"
)));
}
upload.write_all(&chunk).await.map_err(|error| AppError::from(error.to_string()))?;
}
upload.flush().await.map_err(|error| AppError::from(error.to_string()))
}
.await;
drop(upload);
if result.is_err() {
cleanup_uploaded_import_path(file_path).await;
}
result
}
pub async fn execute_import(
State(state): State<Arc<WebState>>,
Json(body): Json<ExecuteImportWrapper>,
) -> Result<Json<serde_json::Value>, AppError> {
let started_at = Instant::now();
let mut req = body.request;
let file_path = validated_uploaded_import_path(&state.data_dir, &req.file_path)?;
req.file_path = file_path.to_string_lossy().to_string();
// Reject import early if the connection is read-only
if let Some(name) = dbx_core::query::connection_readonly_name(&state.app, &req.connection_id).await {
if !req.retain_source {
cleanup_uploaded_import_source(&req.file_path).await;
}
return Err(AppError::from(format!(
"Read-only mode: connection '{}' has read-only protection enabled. Import blocked.",
name
)));
}
let import_id = req.import_id.clone();
let initial_progress = serde_json::to_string(&initial_import_progress(&import_id, started_at))
.map_err(|error| AppError::from(error.to_string()))?;
// Unlike broadcast, watch retains the latest progress and cannot drop the terminal update
// when the browser subscribes slightly after the import starts.
let (tx, _) = tokio::sync::watch::channel(initial_progress);
state.table_import_channels.write().await.insert(import_id.clone(), tx.clone());
let app = state.app.clone();
let state_clone = state.clone();
tokio::spawn(async move {
let db_type = match transfer::get_db_type(&app, &req.connection_id).await {
Ok(t) => t,
Err(e) => {
send_import_progress(&tx, &failed_import_progress(&req.import_id, started_at, e));
if !req.retain_source {
cleanup_uploaded_import_source(&req.file_path).await;
}
schedule_import_progress_cleanup(state_clone, req.import_id.clone());
return;
}
};
let pool_key = match app.get_or_create_pool(&req.connection_id, Some(&req.database)).await {
Ok(k) => k,
Err(e) => {
send_import_progress(&tx, &failed_import_progress(&req.import_id, started_at, e));
if !req.retain_source {
cleanup_uploaded_import_source(&req.file_path).await;
}
schedule_import_progress_cleanup(state_clone, req.import_id.clone());
return;
}
};
let tx_clone = tx.clone();
let result = table_import::import_table_file_core(
&app,
&req,
&db_type,
&pool_key,
|id: &str| {
let id = id.to_string();
Box::pin(async move { transfer::is_cancelled(&id).await })
},
|progress| send_import_progress(&tx_clone, &progress),
)
.await;
let _ = result;
if !req.retain_source {
cleanup_uploaded_import_source(&req.file_path).await;
}
schedule_import_progress_cleanup(state_clone, req.import_id.clone());
});
Ok(Json(serde_json::json!({ "importId": import_id })))
}
pub async fn import_progress(
State(state): State<Arc<WebState>>,
Path(import_id): Path<String>,
) -> Result<Sse<impl Stream<Item = Result<Event, std::convert::Infallible>>>, AppError> {
let channels = state.table_import_channels.read().await;
let tx = channels.get(&import_id).ok_or_else(|| AppError::from("Import not found".to_string()))?;
let rx = tx.subscribe();
drop(channels);
Ok(crate::sse::sse_from_watch(rx))
}
pub async fn cancel_import(
State(_state): State<Arc<WebState>>,
Json(req): Json<CancelImportRequest>,
) -> Json<serde_json::Value> {
transfer::set_cancelled(&req.import_id).await;
Json(serde_json::json!({ "cancelled": true }))
}
fn import_upload_dir(data_dir: &StdPath) -> PathBuf {
data_dir.join("tmp").join("table_import")
}
fn safe_uploaded_import_path(tmp_dir: &StdPath, file_name: &str, source_ref: &str) -> Result<PathBuf, AppError> {
let base_name = file_name.rsplit(['/', '\\']).find(|part| !part.is_empty()).unwrap_or("upload.csv").trim();
if base_name.is_empty() || base_name == "." || base_name == ".." {
return Err(AppError::from("Invalid import file name".to_string()));
}
Ok(tmp_dir.join(format!("{source_ref}-{base_name}")))
}
fn validated_uploaded_import_path(data_dir: &StdPath, file_path: &str) -> Result<PathBuf, AppError> {
let path = PathBuf::from(file_path);
if !path.is_absolute() {
return Err(AppError::from("Import source path must be absolute".to_string()));
}
let tmp_dir = import_upload_dir(data_dir).canonicalize().map_err(|e| AppError::from(e.to_string()))?;
let canonical_path =
path.canonicalize().map_err(|e| AppError::from(format!("Import source is no longer available: {e}")))?;
if !canonical_path.starts_with(&tmp_dir) {
return Err(AppError::from("Import source must be inside the uploaded import directory".to_string()));
}
Ok(canonical_path)
}
fn uploaded_import_path_for_source_ref(data_dir: &StdPath, source_ref: &str) -> Result<PathBuf, AppError> {
uuid::Uuid::parse_str(source_ref).map_err(|_| AppError::from("Invalid import source reference".to_string()))?;
let tmp_dir = import_upload_dir(data_dir);
let prefix = format!("{source_ref}-");
let mut matches = std::fs::read_dir(&tmp_dir)
.map_err(|_| AppError::from("Import source is no longer available".to_string()))?
.filter_map(Result::ok)
.filter(|entry| entry.file_name().to_string_lossy().starts_with(&prefix))
.map(|entry| entry.path());
let file_path = matches.next().ok_or_else(|| AppError::from("Import source is no longer available".to_string()))?;
if matches.next().is_some() {
return Err(AppError::from("Import source reference is ambiguous".to_string()));
}
validated_uploaded_import_path(data_dir, &file_path.to_string_lossy())
}
fn cleanup_expired_import_uploads(tmp_dir: &StdPath, max_age: Duration) {
let Ok(entries) = std::fs::read_dir(tmp_dir) else {
return;
};
let now = SystemTime::now();
for entry in entries.flatten() {
let Ok(metadata) = entry.metadata() else {
continue;
};
let Ok(modified) = metadata.modified() else {
continue;
};
if now.duration_since(modified).map(|age| age > max_age).unwrap_or(false) {
let _ = std::fs::remove_file(entry.path());
}
}
}
async fn cleanup_uploaded_import_source(file_path: &str) {
let _ = tokio::fs::remove_file(file_path).await;
}
async fn cleanup_uploaded_import_path(file_path: &StdPath) {
let _ = tokio::fs::remove_file(file_path).await;
}
async fn cleanup_pending_upload(uploaded_file: &Option<(String, PathBuf)>) {
if let Some((_, file_path)) = uploaded_file {
cleanup_uploaded_import_path(file_path).await;
}
}
#[cfg(test)]
mod tests {
use super::*;
use futures::stream;
fn test_upload_path() -> PathBuf {
std::env::temp_dir().join(format!("dbx-table-import-test-{}", uuid::Uuid::new_v4()))
}
#[tokio::test]
async fn streams_import_upload_to_disk() {
let file_path = test_upload_path();
let chunks = stream::iter([Ok::<_, String>(Bytes::from_static(b"a,b\n")), Ok(Bytes::from_static(b"1,2\n"))]);
assert!(write_import_upload_stream(chunks, &file_path, 8).await.is_ok());
assert_eq!(tokio::fs::read(&file_path).await.unwrap(), b"a,b\n1,2\n");
cleanup_uploaded_import_path(&file_path).await;
}
#[tokio::test]
async fn removes_partial_upload_when_size_limit_is_exceeded() {
let file_path = test_upload_path();
let chunks = stream::iter([Ok::<_, String>(Bytes::from_static(b"1234")), Ok(Bytes::from_static(b"5"))]);
let error = write_import_upload_stream(chunks, &file_path, 4).await.unwrap_err();
assert!(error.message.contains("File too large"));
assert!(!file_path.exists());
}
#[tokio::test]
async fn removes_partial_upload_when_stream_read_fails() {
let file_path = test_upload_path();
let chunks = stream::iter([Ok(Bytes::from_static(b"1234")), Err("multipart stream failed")]);
let error = write_import_upload_stream(chunks, &file_path, 8).await.unwrap_err();
assert_eq!(error.message, "multipart stream failed");
assert!(!file_path.exists());
}
#[test]
fn resolves_uploaded_import_source_by_uuid_reference() {
let data_dir = std::env::temp_dir().join(format!("dbx-table-import-data-{}", uuid::Uuid::new_v4()));
let upload_dir = import_upload_dir(&data_dir);
std::fs::create_dir_all(&upload_dir).unwrap();
let source_ref = uuid::Uuid::new_v4().to_string();
let file_path = upload_dir.join(format!("{source_ref}-users.csv"));
std::fs::write(&file_path, b"id,name\n1,Ada\n").unwrap();
let resolved = uploaded_import_path_for_source_ref(&data_dir, &source_ref)
.unwrap_or_else(|error| panic!("failed to resolve uploaded source: {}", error.message));
assert_eq!(resolved, file_path.canonicalize().unwrap());
assert!(uploaded_import_path_for_source_ref(&data_dir, "../users.csv").is_err());
assert!(uploaded_import_path_for_source_ref(&data_dir, &uuid::Uuid::new_v4().to_string()).is_err());
let _ = std::fs::remove_dir_all(data_dir);
}
#[test]
fn import_progress_watch_replays_terminal_state_to_late_subscribers() {
let started_at = Instant::now();
let initial = serde_json::to_string(&initial_import_progress("import-1", started_at)).unwrap();
let (tx, _) = tokio::sync::watch::channel(initial);
let terminal = TableImportProgress {
import_id: "import-1".to_string(),
status: TableImportStatus::Done,
phase: TableImportPhase::Done,
rows_imported: 2,
total_rows: 2,
total_rows_exact: true,
bytes_read: 20,
total_bytes: 20,
elapsed_ms: 10,
error: None,
};
send_import_progress(&tx, &terminal);
let late_subscriber = tx.subscribe();
let replayed: serde_json::Value = serde_json::from_str(&late_subscriber.borrow()).unwrap();
assert_eq!(replayed["status"], "done");
assert_eq!(replayed["rowsImported"], 2);
}
}