perf(import): optimize large file imports

This commit is contained in:
miracle authored and GitHub committed 2026-07-22 16:56:49 +08:00
1 parent 9602cc3329
commit 0e33c73035
29 files changed
+5032 -625

No files matched your search

+3
View File
@@ -181,6 +181,7 @@ async fn main() {
password_hash: RwLock::new(password_hash),
sessions: RwLock::new(HashSet::new()),
sse_channels: RwLock::new(HashMap::new()),
table_import_channels: RwLock::new(HashMap::new()),
sql_file_executions: RwLock::new(HashMap::new()),
login_rate_limit: tokio::sync::Mutex::new(state::LoginRateLimit { fail_count: 0, locked_until: None }),
export_files: RwLock::new(HashMap::new()),
@@ -561,6 +562,8 @@ async fn main() {
.route("/sql-file/cancel", post(routes::sql_file::cancel_sql_file))
// Table import
.route("/import/preview", post(routes::table_import::preview_import))
.route("/import/preview-source", post(routes::table_import::preview_uploaded_import))
.route("/import/source/release", post(routes::table_import::release_import_source))
.route("/import/execute", post(routes::table_import::execute_import))
.route("/import/progress/{importId}", get(routes::table_import::import_progress))
.route("/import/cancel", post(routes::table_import::cancel_import))
+1
View File
@@ -493,6 +493,7 @@ mod tests {
password_hash: RwLock::new(None),
sessions: RwLock::new(HashSet::new()),
sse_channels: RwLock::new(HashMap::new()),
table_import_channels: RwLock::new(HashMap::new()),
sql_file_executions: RwLock::new(HashMap::new()),
login_rate_limit: Mutex::new(LoginRateLimit { fail_count: 0, locked_until: None }),
export_files: RwLock::new(HashMap::new()),
+1
View File
@@ -834,6 +834,7 @@ mod tests {
password_hash: RwLock::new(None),
sessions: RwLock::new(HashSet::new()),
sse_channels: RwLock::new(HashMap::new()),
table_import_channels: RwLock::new(HashMap::new()),
sql_file_executions: RwLock::new(HashMap::new()),
login_rate_limit: Mutex::new(LoginRateLimit { fail_count: 0, locked_until: None }),
export_files: RwLock::new(HashMap::new()),
+182 -58
View File
@@ -1,13 +1,14 @@
use std::path::{Path as StdPath, PathBuf};
use std::sync::Arc;
use std::time::{Duration, SystemTime};
use std::time::{Duration, Instant, SystemTime};
use axum::body::Bytes;
use axum::extract::{Multipart, Path, State};
use axum::response::sse::{Event, Sse};
use axum::Json;
use dbx_core::table_import::{
self, TableImportParseOptions, TableImportPreviewRequest, TableImportRequest, TableImportSourceFormat,
self, TableImportParseOptions, TableImportPhase, TableImportPreviewRequest, TableImportProgress,
TableImportRequest, TableImportSourceFormat, TableImportStatus,
};
use dbx_core::transfer;
use futures::stream::Stream;
@@ -19,6 +20,52 @@ use crate::error::AppError;
use crate::state::WebState;
const MAX_IMPORT_UPLOAD_BYTES: usize = 100 * 1024 * 1024;
const TABLE_IMPORT_PROGRESS_TTL: Duration = Duration::from_secs(30);
fn initial_import_progress(import_id: &str, started_at: Instant) -> TableImportProgress {
TableImportProgress {
import_id: import_id.to_string(),
status: TableImportStatus::Running,
phase: TableImportPhase::Preparing,
rows_imported: 0,
total_rows: 0,
total_rows_exact: false,
bytes_read: 0,
total_bytes: 0,
elapsed_ms: started_at.elapsed().as_millis(),
error: None,
}
}
fn failed_import_progress(import_id: &str, started_at: Instant, error: String) -> TableImportProgress {
TableImportProgress {
import_id: import_id.to_string(),
status: TableImportStatus::Error,
phase: TableImportPhase::Done,
rows_imported: 0,
total_rows: 0,
total_rows_exact: true,
bytes_read: 0,
total_bytes: 0,
elapsed_ms: started_at.elapsed().as_millis(),
error: Some(error),
}
}
fn send_import_progress(tx: &tokio::sync::watch::Sender<String>, progress: &TableImportProgress) {
if let Ok(json) = serde_json::to_string(progress) {
tx.send_replace(json);
}
}
fn schedule_import_progress_cleanup(state: Arc<WebState>, import_id: String) {
// Keep the terminal snapshot briefly so an SSE client that connects after completion can
// still observe the result, then release the channel and its captured state.
tokio::spawn(async move {
tokio::time::sleep(TABLE_IMPORT_PROGRESS_TTL).await;
state.table_import_channels.write().await.remove(&import_id);
});
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
@@ -32,6 +79,24 @@ pub struct CancelImportRequest {
pub import_id: String,
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct PreviewUploadedImportRequest {
pub source_ref: String,
#[serde(default)]
pub source_format: Option<TableImportSourceFormat>,
#[serde(default)]
pub parse_options: TableImportParseOptions,
#[serde(default)]
pub preview_limit: Option<usize>,
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ReleaseImportSourceRequest {
pub source_ref: String,
}
pub async fn preview_import(
State(state): State<Arc<WebState>>,
mut multipart: Multipart,
@@ -133,6 +198,34 @@ pub async fn preview_import(
Err(AppError::from("No file uploaded".to_string()))
}
pub async fn preview_uploaded_import(
State(state): State<Arc<WebState>>,
Json(request): Json<PreviewUploadedImportRequest>,
) -> Result<Json<serde_json::Value>, AppError> {
let file_path = uploaded_import_path_for_source_ref(&state.data_dir, &request.source_ref)?;
let preview = table_import::preview_table_import_file_with_request(TableImportPreviewRequest {
file_path: file_path.to_string_lossy().to_string(),
source_ref: Some(request.source_ref),
source_format: request.source_format,
parse_options: request.parse_options,
preview_limit: request.preview_limit,
})
.await
.map_err(AppError::from)?;
serde_json::to_value(preview).map(Json).map_err(|error| AppError::from(error.to_string()))
}
pub async fn release_import_source(
State(state): State<Arc<WebState>>,
Json(request): Json<ReleaseImportSourceRequest>,
) -> Json<serde_json::Value> {
let released = match uploaded_import_path_for_source_ref(&state.data_dir, &request.source_ref) {
Ok(file_path) => tokio::fs::remove_file(file_path).await.is_ok(),
Err(_) => false,
};
Json(serde_json::json!({ "released": released }))
}
async fn write_import_upload(field: axum::extract::multipart::Field<'_>, file_path: &StdPath) -> Result<(), AppError> {
write_import_upload_stream(field, file_path, MAX_IMPORT_UPLOAD_BYTES).await
}
@@ -176,13 +269,16 @@ pub async fn execute_import(
State(state): State<Arc<WebState>>,
Json(body): Json<ExecuteImportWrapper>,
) -> Result<Json<serde_json::Value>, AppError> {
let started_at = Instant::now();
let mut req = body.request;
let file_path = validated_uploaded_import_path(&state.data_dir, &req.file_path)?;
req.file_path = file_path.to_string_lossy().to_string();
// Reject import early if the connection is read-only
if let Some(name) = dbx_core::query::connection_readonly_name(&state.app, &req.connection_id).await {
cleanup_uploaded_import_source(&req.file_path).await;
if !req.retain_source {
cleanup_uploaded_import_source(&req.file_path).await;
}
return Err(AppError::from(format!(
"Read-only mode: connection '{}' has read-only protection enabled. Import blocked.",
name
@@ -191,8 +287,12 @@ pub async fn execute_import(
let import_id = req.import_id.clone();
let (tx, _) = tokio::sync::broadcast::channel::<String>(256);
state.sse_channels.write().await.insert(import_id.clone(), tx.clone());
let initial_progress = serde_json::to_string(&initial_import_progress(&import_id, started_at))
.map_err(|error| AppError::from(error.to_string()))?;
// Unlike broadcast, watch retains the latest progress and cannot drop the terminal update
// when the browser subscribes slightly after the import starts.
let (tx, _) = tokio::sync::watch::channel(initial_progress);
state.table_import_channels.write().await.insert(import_id.clone(), tx.clone());
let app = state.app.clone();
let state_clone = state.clone();
@@ -201,18 +301,11 @@ pub async fn execute_import(
let db_type = match transfer::get_db_type(&app, &req.connection_id).await {
Ok(t) => t,
Err(e) => {
let _ = tx.send(
serde_json::json!({
"importId": req.import_id.clone(),
"status": "error",
"rowsImported": 0,
"totalRows": 0,
"error": e
})
.to_string(),
);
cleanup_uploaded_import_source(&req.file_path).await;
state_clone.sse_channels.write().await.remove(&req.import_id);
send_import_progress(&tx, &failed_import_progress(&req.import_id, started_at, e));
if !req.retain_source {
cleanup_uploaded_import_source(&req.file_path).await;
}
schedule_import_progress_cleanup(state_clone, req.import_id.clone());
return;
}
};
@@ -220,24 +313,16 @@ pub async fn execute_import(
let pool_key = match app.get_or_create_pool(&req.connection_id, Some(&req.database)).await {
Ok(k) => k,
Err(e) => {
let _ = tx.send(
serde_json::json!({
"importId": req.import_id.clone(),
"status": "error",
"rowsImported": 0,
"totalRows": 0,
"error": e
})
.to_string(),
);
cleanup_uploaded_import_source(&req.file_path).await;
state_clone.sse_channels.write().await.remove(&req.import_id);
send_import_progress(&tx, &failed_import_progress(&req.import_id, started_at, e));
if !req.retain_source {
cleanup_uploaded_import_source(&req.file_path).await;
}
schedule_import_progress_cleanup(state_clone, req.import_id.clone());
return;
}
};
let tx_clone = tx.clone();
let import_id_for_cancel = req.import_id.clone();
let result = table_import::import_table_file_core(
&app,
&req,
@@ -247,36 +332,16 @@ pub async fn execute_import(
let id = id.to_string();
Box::pin(async move { transfer::is_cancelled(&id).await })
},
|progress| {
if let Ok(json) = serde_json::to_string(&progress) {
let _ = tx_clone.send(json);
}
},
|progress| send_import_progress(&tx_clone, &progress),
)
.await;
match result {
Ok(summary) => {
if let Ok(json) = serde_json::to_string(&summary) {
let _ = tx.send(json);
}
}
Err(e) => {
let _ = tx.send(
serde_json::json!({
"importId": import_id_for_cancel,
"status": "error",
"rowsImported": 0,
"totalRows": 0,
"error": e
})
.to_string(),
);
}
}
let _ = result;
cleanup_uploaded_import_source(&req.file_path).await;
state_clone.sse_channels.write().await.remove(&req.import_id);
if !req.retain_source {
cleanup_uploaded_import_source(&req.file_path).await;
}
schedule_import_progress_cleanup(state_clone, req.import_id.clone());
});
Ok(Json(serde_json::json!({ "importId": import_id })))
@@ -286,11 +351,11 @@ pub async fn import_progress(
State(state): State<Arc<WebState>>,
Path(import_id): Path<String>,
) -> Result<Sse<impl Stream<Item = Result<Event, std::convert::Infallible>>>, AppError> {
let channels = state.sse_channels.read().await;
let channels = state.table_import_channels.read().await;
let tx = channels.get(&import_id).ok_or_else(|| AppError::from("Import not found".to_string()))?;
let rx = tx.subscribe();
drop(channels);
Ok(crate::sse::sse_from_channel(rx))
Ok(crate::sse::sse_from_watch(rx))
}
pub async fn cancel_import(
@@ -328,6 +393,22 @@ fn validated_uploaded_import_path(data_dir: &StdPath, file_path: &str) -> Result
Ok(canonical_path)
}
fn uploaded_import_path_for_source_ref(data_dir: &StdPath, source_ref: &str) -> Result<PathBuf, AppError> {
uuid::Uuid::parse_str(source_ref).map_err(|_| AppError::from("Invalid import source reference".to_string()))?;
let tmp_dir = import_upload_dir(data_dir);
let prefix = format!("{source_ref}-");
let mut matches = std::fs::read_dir(&tmp_dir)
.map_err(|_| AppError::from("Import source is no longer available".to_string()))?
.filter_map(Result::ok)
.filter(|entry| entry.file_name().to_string_lossy().starts_with(&prefix))
.map(|entry| entry.path());
let file_path = matches.next().ok_or_else(|| AppError::from("Import source is no longer available".to_string()))?;
if matches.next().is_some() {
return Err(AppError::from("Import source reference is ambiguous".to_string()));
}
validated_uploaded_import_path(data_dir, &file_path.to_string_lossy())
}
fn cleanup_expired_import_uploads(tmp_dir: &StdPath, max_age: Duration) {
let Ok(entries) = std::fs::read_dir(tmp_dir) else {
return;
@@ -401,4 +482,47 @@ mod tests {
assert_eq!(error.message, "multipart stream failed");
assert!(!file_path.exists());
}
#[test]
fn resolves_uploaded_import_source_by_uuid_reference() {
let data_dir = std::env::temp_dir().join(format!("dbx-table-import-data-{}", uuid::Uuid::new_v4()));
let upload_dir = import_upload_dir(&data_dir);
std::fs::create_dir_all(&upload_dir).unwrap();
let source_ref = uuid::Uuid::new_v4().to_string();
let file_path = upload_dir.join(format!("{source_ref}-users.csv"));
std::fs::write(&file_path, b"id,name\n1,Ada\n").unwrap();
let resolved = uploaded_import_path_for_source_ref(&data_dir, &source_ref)
.unwrap_or_else(|error| panic!("failed to resolve uploaded source: {}", error.message));
assert_eq!(resolved, file_path.canonicalize().unwrap());
assert!(uploaded_import_path_for_source_ref(&data_dir, "../users.csv").is_err());
assert!(uploaded_import_path_for_source_ref(&data_dir, &uuid::Uuid::new_v4().to_string()).is_err());
let _ = std::fs::remove_dir_all(data_dir);
}
#[test]
fn import_progress_watch_replays_terminal_state_to_late_subscribers() {
let started_at = Instant::now();
let initial = serde_json::to_string(&initial_import_progress("import-1", started_at)).unwrap();
let (tx, _) = tokio::sync::watch::channel(initial);
let terminal = TableImportProgress {
import_id: "import-1".to_string(),
status: TableImportStatus::Done,
phase: TableImportPhase::Done,
rows_imported: 2,
total_rows: 2,
total_rows_exact: true,
bytes_read: 20,
total_bytes: 20,
elapsed_ms: 10,
error: None,
};
send_import_progress(&tx, &terminal);
let late_subscriber = tx.subscribe();
let replayed: serde_json::Value = serde_json::from_str(&late_subscriber.borrow()).unwrap();
assert_eq!(replayed["status"], "done");
assert_eq!(replayed["rowsImported"], 2);
}
}
+19
View File
@@ -1,6 +1,7 @@
use axum::response::sse::{Event, KeepAlive, Sse};
use futures::stream::Stream;
use tokio::sync::broadcast::{self, error::RecvError};
use tokio::sync::watch;
pub fn sse_from_channel(
rx: broadcast::Receiver<String>,
@@ -32,3 +33,21 @@ fn sse_from_channel_with_lag_policy(
};
Sse::new(stream).keep_alive(KeepAlive::default())
}
pub fn sse_from_watch(
mut rx: watch::Receiver<String>,
) -> Sse<impl Stream<Item = Result<Event, std::convert::Infallible>>> {
// A watch channel stores the latest state, so late subscribers immediately receive the
// current progress, including a terminal result, instead of waiting for a new event.
let stream = async_stream::stream! {
let initial = rx.borrow().clone();
if !initial.is_empty() {
yield Ok(Event::default().data(initial));
}
while rx.changed().await.is_ok() {
let update = rx.borrow().clone();
yield Ok(Event::default().data(update));
}
};
Sse::new(stream).keep_alive(KeepAlive::default())
}
+2 -1
View File
@@ -2,7 +2,7 @@ use dbx_core::connection::AppState;
use std::collections::{HashMap, HashSet};
use std::path::PathBuf;
use std::sync::Arc;
use tokio::sync::{broadcast, Mutex, RwLock};
use tokio::sync::{broadcast, watch, Mutex, RwLock};
use tokio_util::sync::CancellationToken;
pub struct LoginRateLimit {
@@ -18,6 +18,7 @@ pub struct WebState {
pub password_hash: RwLock<Option<String>>,
pub sessions: RwLock<HashSet<String>>,
pub sse_channels: RwLock<HashMap<String, broadcast::Sender<String>>>,
pub table_import_channels: RwLock<HashMap<String, watch::Sender<String>>>,
pub sql_file_executions: RwLock<HashMap<String, CancellationToken>>,
pub login_rate_limit: Mutex<LoginRateLimit>,
/// Table export temp files: export_id -> (file_path, format)