perf(import): optimize large file imports
This commit is contained in:
1 parent
9602cc3329
commit
0e33c73035
29 files changed
+5032
-625
No files matched your search
@@ -181,6 +181,7 @@ async fn main() {
|
||||
password_hash: RwLock::new(password_hash),
|
||||
sessions: RwLock::new(HashSet::new()),
|
||||
sse_channels: RwLock::new(HashMap::new()),
|
||||
table_import_channels: RwLock::new(HashMap::new()),
|
||||
sql_file_executions: RwLock::new(HashMap::new()),
|
||||
login_rate_limit: tokio::sync::Mutex::new(state::LoginRateLimit { fail_count: 0, locked_until: None }),
|
||||
export_files: RwLock::new(HashMap::new()),
|
||||
@@ -561,6 +562,8 @@ async fn main() {
|
||||
.route("/sql-file/cancel", post(routes::sql_file::cancel_sql_file))
|
||||
// Table import
|
||||
.route("/import/preview", post(routes::table_import::preview_import))
|
||||
.route("/import/preview-source", post(routes::table_import::preview_uploaded_import))
|
||||
.route("/import/source/release", post(routes::table_import::release_import_source))
|
||||
.route("/import/execute", post(routes::table_import::execute_import))
|
||||
.route("/import/progress/{importId}", get(routes::table_import::import_progress))
|
||||
.route("/import/cancel", post(routes::table_import::cancel_import))
|
||||
|
||||
@@ -493,6 +493,7 @@ mod tests {
|
||||
password_hash: RwLock::new(None),
|
||||
sessions: RwLock::new(HashSet::new()),
|
||||
sse_channels: RwLock::new(HashMap::new()),
|
||||
table_import_channels: RwLock::new(HashMap::new()),
|
||||
sql_file_executions: RwLock::new(HashMap::new()),
|
||||
login_rate_limit: Mutex::new(LoginRateLimit { fail_count: 0, locked_until: None }),
|
||||
export_files: RwLock::new(HashMap::new()),
|
||||
|
||||
@@ -834,6 +834,7 @@ mod tests {
|
||||
password_hash: RwLock::new(None),
|
||||
sessions: RwLock::new(HashSet::new()),
|
||||
sse_channels: RwLock::new(HashMap::new()),
|
||||
table_import_channels: RwLock::new(HashMap::new()),
|
||||
sql_file_executions: RwLock::new(HashMap::new()),
|
||||
login_rate_limit: Mutex::new(LoginRateLimit { fail_count: 0, locked_until: None }),
|
||||
export_files: RwLock::new(HashMap::new()),
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
use std::path::{Path as StdPath, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime};
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
|
||||
use axum::body::Bytes;
|
||||
use axum::extract::{Multipart, Path, State};
|
||||
use axum::response::sse::{Event, Sse};
|
||||
use axum::Json;
|
||||
use dbx_core::table_import::{
|
||||
self, TableImportParseOptions, TableImportPreviewRequest, TableImportRequest, TableImportSourceFormat,
|
||||
self, TableImportParseOptions, TableImportPhase, TableImportPreviewRequest, TableImportProgress,
|
||||
TableImportRequest, TableImportSourceFormat, TableImportStatus,
|
||||
};
|
||||
use dbx_core::transfer;
|
||||
use futures::stream::Stream;
|
||||
@@ -19,6 +20,52 @@ use crate::error::AppError;
|
||||
use crate::state::WebState;
|
||||
|
||||
const MAX_IMPORT_UPLOAD_BYTES: usize = 100 * 1024 * 1024;
|
||||
const TABLE_IMPORT_PROGRESS_TTL: Duration = Duration::from_secs(30);
|
||||
|
||||
fn initial_import_progress(import_id: &str, started_at: Instant) -> TableImportProgress {
|
||||
TableImportProgress {
|
||||
import_id: import_id.to_string(),
|
||||
status: TableImportStatus::Running,
|
||||
phase: TableImportPhase::Preparing,
|
||||
rows_imported: 0,
|
||||
total_rows: 0,
|
||||
total_rows_exact: false,
|
||||
bytes_read: 0,
|
||||
total_bytes: 0,
|
||||
elapsed_ms: started_at.elapsed().as_millis(),
|
||||
error: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn failed_import_progress(import_id: &str, started_at: Instant, error: String) -> TableImportProgress {
|
||||
TableImportProgress {
|
||||
import_id: import_id.to_string(),
|
||||
status: TableImportStatus::Error,
|
||||
phase: TableImportPhase::Done,
|
||||
rows_imported: 0,
|
||||
total_rows: 0,
|
||||
total_rows_exact: true,
|
||||
bytes_read: 0,
|
||||
total_bytes: 0,
|
||||
elapsed_ms: started_at.elapsed().as_millis(),
|
||||
error: Some(error),
|
||||
}
|
||||
}
|
||||
|
||||
fn send_import_progress(tx: &tokio::sync::watch::Sender<String>, progress: &TableImportProgress) {
|
||||
if let Ok(json) = serde_json::to_string(progress) {
|
||||
tx.send_replace(json);
|
||||
}
|
||||
}
|
||||
|
||||
fn schedule_import_progress_cleanup(state: Arc<WebState>, import_id: String) {
|
||||
// Keep the terminal snapshot briefly so an SSE client that connects after completion can
|
||||
// still observe the result, then release the channel and its captured state.
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(TABLE_IMPORT_PROGRESS_TTL).await;
|
||||
state.table_import_channels.write().await.remove(&import_id);
|
||||
});
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
@@ -32,6 +79,24 @@ pub struct CancelImportRequest {
|
||||
pub import_id: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct PreviewUploadedImportRequest {
|
||||
pub source_ref: String,
|
||||
#[serde(default)]
|
||||
pub source_format: Option<TableImportSourceFormat>,
|
||||
#[serde(default)]
|
||||
pub parse_options: TableImportParseOptions,
|
||||
#[serde(default)]
|
||||
pub preview_limit: Option<usize>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ReleaseImportSourceRequest {
|
||||
pub source_ref: String,
|
||||
}
|
||||
|
||||
pub async fn preview_import(
|
||||
State(state): State<Arc<WebState>>,
|
||||
mut multipart: Multipart,
|
||||
@@ -133,6 +198,34 @@ pub async fn preview_import(
|
||||
Err(AppError::from("No file uploaded".to_string()))
|
||||
}
|
||||
|
||||
pub async fn preview_uploaded_import(
|
||||
State(state): State<Arc<WebState>>,
|
||||
Json(request): Json<PreviewUploadedImportRequest>,
|
||||
) -> Result<Json<serde_json::Value>, AppError> {
|
||||
let file_path = uploaded_import_path_for_source_ref(&state.data_dir, &request.source_ref)?;
|
||||
let preview = table_import::preview_table_import_file_with_request(TableImportPreviewRequest {
|
||||
file_path: file_path.to_string_lossy().to_string(),
|
||||
source_ref: Some(request.source_ref),
|
||||
source_format: request.source_format,
|
||||
parse_options: request.parse_options,
|
||||
preview_limit: request.preview_limit,
|
||||
})
|
||||
.await
|
||||
.map_err(AppError::from)?;
|
||||
serde_json::to_value(preview).map(Json).map_err(|error| AppError::from(error.to_string()))
|
||||
}
|
||||
|
||||
pub async fn release_import_source(
|
||||
State(state): State<Arc<WebState>>,
|
||||
Json(request): Json<ReleaseImportSourceRequest>,
|
||||
) -> Json<serde_json::Value> {
|
||||
let released = match uploaded_import_path_for_source_ref(&state.data_dir, &request.source_ref) {
|
||||
Ok(file_path) => tokio::fs::remove_file(file_path).await.is_ok(),
|
||||
Err(_) => false,
|
||||
};
|
||||
Json(serde_json::json!({ "released": released }))
|
||||
}
|
||||
|
||||
async fn write_import_upload(field: axum::extract::multipart::Field<'_>, file_path: &StdPath) -> Result<(), AppError> {
|
||||
write_import_upload_stream(field, file_path, MAX_IMPORT_UPLOAD_BYTES).await
|
||||
}
|
||||
@@ -176,13 +269,16 @@ pub async fn execute_import(
|
||||
State(state): State<Arc<WebState>>,
|
||||
Json(body): Json<ExecuteImportWrapper>,
|
||||
) -> Result<Json<serde_json::Value>, AppError> {
|
||||
let started_at = Instant::now();
|
||||
let mut req = body.request;
|
||||
let file_path = validated_uploaded_import_path(&state.data_dir, &req.file_path)?;
|
||||
req.file_path = file_path.to_string_lossy().to_string();
|
||||
|
||||
// Reject import early if the connection is read-only
|
||||
if let Some(name) = dbx_core::query::connection_readonly_name(&state.app, &req.connection_id).await {
|
||||
cleanup_uploaded_import_source(&req.file_path).await;
|
||||
if !req.retain_source {
|
||||
cleanup_uploaded_import_source(&req.file_path).await;
|
||||
}
|
||||
return Err(AppError::from(format!(
|
||||
"Read-only mode: connection '{}' has read-only protection enabled. Import blocked.",
|
||||
name
|
||||
@@ -191,8 +287,12 @@ pub async fn execute_import(
|
||||
|
||||
let import_id = req.import_id.clone();
|
||||
|
||||
let (tx, _) = tokio::sync::broadcast::channel::<String>(256);
|
||||
state.sse_channels.write().await.insert(import_id.clone(), tx.clone());
|
||||
let initial_progress = serde_json::to_string(&initial_import_progress(&import_id, started_at))
|
||||
.map_err(|error| AppError::from(error.to_string()))?;
|
||||
// Unlike broadcast, watch retains the latest progress and cannot drop the terminal update
|
||||
// when the browser subscribes slightly after the import starts.
|
||||
let (tx, _) = tokio::sync::watch::channel(initial_progress);
|
||||
state.table_import_channels.write().await.insert(import_id.clone(), tx.clone());
|
||||
|
||||
let app = state.app.clone();
|
||||
let state_clone = state.clone();
|
||||
@@ -201,18 +301,11 @@ pub async fn execute_import(
|
||||
let db_type = match transfer::get_db_type(&app, &req.connection_id).await {
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
let _ = tx.send(
|
||||
serde_json::json!({
|
||||
"importId": req.import_id.clone(),
|
||||
"status": "error",
|
||||
"rowsImported": 0,
|
||||
"totalRows": 0,
|
||||
"error": e
|
||||
})
|
||||
.to_string(),
|
||||
);
|
||||
cleanup_uploaded_import_source(&req.file_path).await;
|
||||
state_clone.sse_channels.write().await.remove(&req.import_id);
|
||||
send_import_progress(&tx, &failed_import_progress(&req.import_id, started_at, e));
|
||||
if !req.retain_source {
|
||||
cleanup_uploaded_import_source(&req.file_path).await;
|
||||
}
|
||||
schedule_import_progress_cleanup(state_clone, req.import_id.clone());
|
||||
return;
|
||||
}
|
||||
};
|
||||
@@ -220,24 +313,16 @@ pub async fn execute_import(
|
||||
let pool_key = match app.get_or_create_pool(&req.connection_id, Some(&req.database)).await {
|
||||
Ok(k) => k,
|
||||
Err(e) => {
|
||||
let _ = tx.send(
|
||||
serde_json::json!({
|
||||
"importId": req.import_id.clone(),
|
||||
"status": "error",
|
||||
"rowsImported": 0,
|
||||
"totalRows": 0,
|
||||
"error": e
|
||||
})
|
||||
.to_string(),
|
||||
);
|
||||
cleanup_uploaded_import_source(&req.file_path).await;
|
||||
state_clone.sse_channels.write().await.remove(&req.import_id);
|
||||
send_import_progress(&tx, &failed_import_progress(&req.import_id, started_at, e));
|
||||
if !req.retain_source {
|
||||
cleanup_uploaded_import_source(&req.file_path).await;
|
||||
}
|
||||
schedule_import_progress_cleanup(state_clone, req.import_id.clone());
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let tx_clone = tx.clone();
|
||||
let import_id_for_cancel = req.import_id.clone();
|
||||
let result = table_import::import_table_file_core(
|
||||
&app,
|
||||
&req,
|
||||
@@ -247,36 +332,16 @@ pub async fn execute_import(
|
||||
let id = id.to_string();
|
||||
Box::pin(async move { transfer::is_cancelled(&id).await })
|
||||
},
|
||||
|progress| {
|
||||
if let Ok(json) = serde_json::to_string(&progress) {
|
||||
let _ = tx_clone.send(json);
|
||||
}
|
||||
},
|
||||
|progress| send_import_progress(&tx_clone, &progress),
|
||||
)
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(summary) => {
|
||||
if let Ok(json) = serde_json::to_string(&summary) {
|
||||
let _ = tx.send(json);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
let _ = tx.send(
|
||||
serde_json::json!({
|
||||
"importId": import_id_for_cancel,
|
||||
"status": "error",
|
||||
"rowsImported": 0,
|
||||
"totalRows": 0,
|
||||
"error": e
|
||||
})
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
}
|
||||
let _ = result;
|
||||
|
||||
cleanup_uploaded_import_source(&req.file_path).await;
|
||||
state_clone.sse_channels.write().await.remove(&req.import_id);
|
||||
if !req.retain_source {
|
||||
cleanup_uploaded_import_source(&req.file_path).await;
|
||||
}
|
||||
schedule_import_progress_cleanup(state_clone, req.import_id.clone());
|
||||
});
|
||||
|
||||
Ok(Json(serde_json::json!({ "importId": import_id })))
|
||||
@@ -286,11 +351,11 @@ pub async fn import_progress(
|
||||
State(state): State<Arc<WebState>>,
|
||||
Path(import_id): Path<String>,
|
||||
) -> Result<Sse<impl Stream<Item = Result<Event, std::convert::Infallible>>>, AppError> {
|
||||
let channels = state.sse_channels.read().await;
|
||||
let channels = state.table_import_channels.read().await;
|
||||
let tx = channels.get(&import_id).ok_or_else(|| AppError::from("Import not found".to_string()))?;
|
||||
let rx = tx.subscribe();
|
||||
drop(channels);
|
||||
Ok(crate::sse::sse_from_channel(rx))
|
||||
Ok(crate::sse::sse_from_watch(rx))
|
||||
}
|
||||
|
||||
pub async fn cancel_import(
|
||||
@@ -328,6 +393,22 @@ fn validated_uploaded_import_path(data_dir: &StdPath, file_path: &str) -> Result
|
||||
Ok(canonical_path)
|
||||
}
|
||||
|
||||
fn uploaded_import_path_for_source_ref(data_dir: &StdPath, source_ref: &str) -> Result<PathBuf, AppError> {
|
||||
uuid::Uuid::parse_str(source_ref).map_err(|_| AppError::from("Invalid import source reference".to_string()))?;
|
||||
let tmp_dir = import_upload_dir(data_dir);
|
||||
let prefix = format!("{source_ref}-");
|
||||
let mut matches = std::fs::read_dir(&tmp_dir)
|
||||
.map_err(|_| AppError::from("Import source is no longer available".to_string()))?
|
||||
.filter_map(Result::ok)
|
||||
.filter(|entry| entry.file_name().to_string_lossy().starts_with(&prefix))
|
||||
.map(|entry| entry.path());
|
||||
let file_path = matches.next().ok_or_else(|| AppError::from("Import source is no longer available".to_string()))?;
|
||||
if matches.next().is_some() {
|
||||
return Err(AppError::from("Import source reference is ambiguous".to_string()));
|
||||
}
|
||||
validated_uploaded_import_path(data_dir, &file_path.to_string_lossy())
|
||||
}
|
||||
|
||||
fn cleanup_expired_import_uploads(tmp_dir: &StdPath, max_age: Duration) {
|
||||
let Ok(entries) = std::fs::read_dir(tmp_dir) else {
|
||||
return;
|
||||
@@ -401,4 +482,47 @@ mod tests {
|
||||
assert_eq!(error.message, "multipart stream failed");
|
||||
assert!(!file_path.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolves_uploaded_import_source_by_uuid_reference() {
|
||||
let data_dir = std::env::temp_dir().join(format!("dbx-table-import-data-{}", uuid::Uuid::new_v4()));
|
||||
let upload_dir = import_upload_dir(&data_dir);
|
||||
std::fs::create_dir_all(&upload_dir).unwrap();
|
||||
let source_ref = uuid::Uuid::new_v4().to_string();
|
||||
let file_path = upload_dir.join(format!("{source_ref}-users.csv"));
|
||||
std::fs::write(&file_path, b"id,name\n1,Ada\n").unwrap();
|
||||
|
||||
let resolved = uploaded_import_path_for_source_ref(&data_dir, &source_ref)
|
||||
.unwrap_or_else(|error| panic!("failed to resolve uploaded source: {}", error.message));
|
||||
|
||||
assert_eq!(resolved, file_path.canonicalize().unwrap());
|
||||
assert!(uploaded_import_path_for_source_ref(&data_dir, "../users.csv").is_err());
|
||||
assert!(uploaded_import_path_for_source_ref(&data_dir, &uuid::Uuid::new_v4().to_string()).is_err());
|
||||
let _ = std::fs::remove_dir_all(data_dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn import_progress_watch_replays_terminal_state_to_late_subscribers() {
|
||||
let started_at = Instant::now();
|
||||
let initial = serde_json::to_string(&initial_import_progress("import-1", started_at)).unwrap();
|
||||
let (tx, _) = tokio::sync::watch::channel(initial);
|
||||
let terminal = TableImportProgress {
|
||||
import_id: "import-1".to_string(),
|
||||
status: TableImportStatus::Done,
|
||||
phase: TableImportPhase::Done,
|
||||
rows_imported: 2,
|
||||
total_rows: 2,
|
||||
total_rows_exact: true,
|
||||
bytes_read: 20,
|
||||
total_bytes: 20,
|
||||
elapsed_ms: 10,
|
||||
error: None,
|
||||
};
|
||||
send_import_progress(&tx, &terminal);
|
||||
|
||||
let late_subscriber = tx.subscribe();
|
||||
let replayed: serde_json::Value = serde_json::from_str(&late_subscriber.borrow()).unwrap();
|
||||
assert_eq!(replayed["status"], "done");
|
||||
assert_eq!(replayed["rowsImported"], 2);
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
use axum::response::sse::{Event, KeepAlive, Sse};
|
||||
use futures::stream::Stream;
|
||||
use tokio::sync::broadcast::{self, error::RecvError};
|
||||
use tokio::sync::watch;
|
||||
|
||||
pub fn sse_from_channel(
|
||||
rx: broadcast::Receiver<String>,
|
||||
@@ -32,3 +33,21 @@ fn sse_from_channel_with_lag_policy(
|
||||
};
|
||||
Sse::new(stream).keep_alive(KeepAlive::default())
|
||||
}
|
||||
|
||||
pub fn sse_from_watch(
|
||||
mut rx: watch::Receiver<String>,
|
||||
) -> Sse<impl Stream<Item = Result<Event, std::convert::Infallible>>> {
|
||||
// A watch channel stores the latest state, so late subscribers immediately receive the
|
||||
// current progress, including a terminal result, instead of waiting for a new event.
|
||||
let stream = async_stream::stream! {
|
||||
let initial = rx.borrow().clone();
|
||||
if !initial.is_empty() {
|
||||
yield Ok(Event::default().data(initial));
|
||||
}
|
||||
while rx.changed().await.is_ok() {
|
||||
let update = rx.borrow().clone();
|
||||
yield Ok(Event::default().data(update));
|
||||
}
|
||||
};
|
||||
Sse::new(stream).keep_alive(KeepAlive::default())
|
||||
}
|
||||
@@ -2,7 +2,7 @@ use dbx_core::connection::AppState;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::{broadcast, Mutex, RwLock};
|
||||
use tokio::sync::{broadcast, watch, Mutex, RwLock};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
pub struct LoginRateLimit {
|
||||
@@ -18,6 +18,7 @@ pub struct WebState {
|
||||
pub password_hash: RwLock<Option<String>>,
|
||||
pub sessions: RwLock<HashSet<String>>,
|
||||
pub sse_channels: RwLock<HashMap<String, broadcast::Sender<String>>>,
|
||||
pub table_import_channels: RwLock<HashMap<String, watch::Sender<String>>>,
|
||||
pub sql_file_executions: RwLock<HashMap<String, CancellationToken>>,
|
||||
pub login_rate_limit: Mutex<LoginRateLimit>,
|
||||
/// Table export temp files: export_id -> (file_path, format)
|
||||
|
||||
Reference in new issue
Block a user