perf(import): optimize large file imports

This commit is contained in:
miracle authored and GitHub committed 2026-07-22 16:56:49 +08:00
1 parent 9602cc3329
commit 0e33c73035
29 files changed
+5032 -625

No files matched your search

+1
View File
@@ -88,3 +88,4 @@ bytes = "1"
font-kit = "0.14.3"
zip = { version = "4", default-features = false, features = ["deflate"] }
sysinfo = { version = "0.32", features = ["system"] }
tempfile = "3"
@@ -0,0 +1,211 @@
use std::fs::{self, File};
use std::io::{BufWriter, Write};
use std::path::Path;
use std::time::Instant;
use dbx_core::models::connection::DatabaseType;
use dbx_core::table_import::{
build_import_insert_batches, parse_delimited_file_with_options, parse_xlsx_file_with_options,
preview_table_import_file_with_request, ParsedImportFile, TableImportColumnMapping, TableImportParseOptions,
TableImportPreviewRequest, TableImportSourceFormat,
};
use dbx_core::xlsx_export::{build_xlsx_workbook, XlsxWorksheetData};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum BenchFormat {
Csv,
Xlsx,
All,
}
struct Options {
rows: usize,
columns: usize,
batch_size: usize,
format: BenchFormat,
}
fn print_help() {
println!(
"Table import benchmark\n\n\
Usage:\n cargo run -p dbx-core --example table_import_bench --release -- [options]\n\n\
Options:\n --rows=10000 Number of data rows\n --columns=10 Number of columns\n --batch-size=500 SQL rows per batch\n --format=all csv, xlsx, or all\n --help Show this help\n"
);
}
fn parse_options() -> Result<Options, String> {
let mut options = Options { rows: 10_000, columns: 10, batch_size: 500, format: BenchFormat::All };
for argument in std::env::args().skip(1) {
if argument == "--help" || argument == "-h" {
print_help();
std::process::exit(0);
}
let (key, value) = argument.split_once('=').ok_or_else(|| format!("Invalid option: {argument}"))?;
match key {
"--rows" => options.rows = value.parse().map_err(|_| format!("Invalid row count: {value}"))?,
"--columns" => options.columns = value.parse().map_err(|_| format!("Invalid column count: {value}"))?,
"--batch-size" => options.batch_size = value.parse().map_err(|_| format!("Invalid batch size: {value}"))?,
"--format" => {
options.format = match value {
"csv" => BenchFormat::Csv,
"xlsx" => BenchFormat::Xlsx,
"all" => BenchFormat::All,
_ => return Err(format!("Invalid format: {value}")),
}
}
_ => return Err(format!("Unknown option: {key}")),
}
}
if options.rows == 0 || options.columns == 0 || options.batch_size == 0 {
return Err("rows, columns, and batch-size must be greater than zero".to_string());
}
Ok(options)
}
fn columns(count: usize) -> Vec<String> {
(0..count).map(|index| format!("column_{}", index + 1)).collect()
}
fn row(row_index: usize, column_count: usize) -> Vec<serde_json::Value> {
(0..column_count)
.map(|column_index| {
if column_index == 0 {
serde_json::json!(row_index + 1)
} else if column_index % 3 == 0 {
serde_json::json!(format!("2026-07-{:02} 12:34:56", row_index % 28 + 1))
} else {
serde_json::json!(format!("value-{row_index}-{column_index}"))
}
})
.collect()
}
fn write_csv(path: &Path, row_count: usize, column_count: usize) -> Result<(), String> {
let file = File::create(path).map_err(|error| error.to_string())?;
let mut writer = BufWriter::new(file);
writeln!(writer, "{}", columns(column_count).join(",")).map_err(|error| error.to_string())?;
for row_index in 0..row_count {
let values = row(row_index, column_count)
.into_iter()
.map(|value| value.as_str().map(str::to_string).unwrap_or_else(|| value.to_string()))
.collect::<Vec<_>>();
writeln!(writer, "{}", values.join(",")).map_err(|error| error.to_string())?;
}
writer.flush().map_err(|error| error.to_string())
}
fn write_xlsx(path: &Path, row_count: usize, column_count: usize) -> Result<(), String> {
let workbook = build_xlsx_workbook(&XlsxWorksheetData {
sheet_name: Some("Benchmark".to_string()),
columns: columns(column_count),
column_types: vec![],
rows: (0..row_count).map(|row_index| row(row_index, column_count)).collect(),
})?;
fs::write(path, workbook).map_err(|error| error.to_string())
}
fn mappings(data: &ParsedImportFile) -> Vec<TableImportColumnMapping> {
data.columns
.iter()
.map(|column| TableImportColumnMapping {
source_column: column.clone(),
target_column: column.clone(),
target_data_type: None,
})
.collect()
}
async fn benchmark_file(
path: &Path,
format: TableImportSourceFormat,
options: &Options,
) -> Result<serde_json::Value, String> {
let path_text = path.to_string_lossy();
let parse_options = TableImportParseOptions::default();
let preview_started = Instant::now();
let preview = preview_table_import_file_with_request(TableImportPreviewRequest {
file_path: path_text.to_string(),
source_ref: None,
source_format: Some(format),
parse_options: parse_options.clone(),
preview_limit: Some(50),
})
.await?;
let preview_ms = preview_started.elapsed().as_secs_f64() * 1000.0;
let full_parse_started = Instant::now();
let parsed = match format {
TableImportSourceFormat::Csv => {
parse_delimited_file_with_options(&path_text, format, &parse_options, usize::MAX)?
}
TableImportSourceFormat::Excel => parse_xlsx_file_with_options(&path_text, &parse_options, usize::MAX)?,
_ => unreachable!(),
};
let full_parse_ms = full_parse_started.elapsed().as_secs_f64() * 1000.0;
let batch_started = Instant::now();
let batches = build_import_insert_batches(
&parsed,
&mappings(&parsed),
&[],
"benchmark_import",
"main",
&DatabaseType::Sqlite,
options.batch_size,
)?;
let batch_build_ms = batch_started.elapsed().as_secs_f64() * 1000.0;
let sql_bytes = batches.iter().map(|batch| batch.sql.len()).sum::<usize>();
Ok(serde_json::json!({
"format": format.label(),
"fileBytes": fs::metadata(path).map_err(|error| error.to_string())?.len(),
"rows": parsed.total_rows,
"columns": preview.columns.len(),
"previewRows": preview.rows.len(),
"previewTotalRowsExact": preview.total_rows_exact,
"previewMs": preview_ms,
"fullParseMs": full_parse_ms,
"batchBuildMs": batch_build_ms,
"batchCount": batches.len(),
"sqlBytes": sql_bytes,
}))
}
async fn run() -> Result<(), String> {
let options = parse_options()?;
let temp_dir = std::env::temp_dir().join(format!("dbx-table-import-bench-{}", uuid::Uuid::new_v4()));
fs::create_dir_all(&temp_dir).map_err(|error| error.to_string())?;
let mut results = Vec::new();
if matches!(options.format, BenchFormat::Csv | BenchFormat::All) {
let path = temp_dir.join("benchmark.csv");
write_csv(&path, options.rows, options.columns)?;
results.push(benchmark_file(&path, TableImportSourceFormat::Csv, &options).await?);
}
if matches!(options.format, BenchFormat::Xlsx | BenchFormat::All) {
let path = temp_dir.join("benchmark.xlsx");
write_xlsx(&path, options.rows, options.columns)?;
results.push(benchmark_file(&path, TableImportSourceFormat::Excel, &options).await?);
}
println!(
"{}",
serde_json::to_string_pretty(&serde_json::json!({
"rows": options.rows,
"columns": options.columns,
"batchSize": options.batch_size,
"results": results,
}))
.map_err(|error| error.to_string())?
);
let _ = fs::remove_dir_all(temp_dir);
Ok(())
}
#[tokio::main]
async fn main() {
if let Err(error) = run().await {
eprintln!("{error}");
std::process::exit(1);
}
}
+64 -14
View File
@@ -2308,9 +2308,38 @@ impl AppState {
database: Option<&str>,
client_session_id: &str,
) -> Result<bool, String> {
let Some((pool_key, pool)) = self.take_client_session_pool(connection_id, database, client_session_id).await?
else {
return Ok(false);
};
close_pool_kind_with_timeout(pool_key, pool).await;
Ok(true)
}
/// Removes a session-scoped pool immediately and schedules the potentially slow driver
/// shutdown on the supervised background task set.
pub async fn detach_client_session_pool(
&self,
connection_id: &str,
database: Option<&str>,
client_session_id: &str,
) -> Result<bool, String> {
let Some(removed) = self.take_client_session_pool(connection_id, database, client_session_id).await? else {
return Ok(false);
};
close_removed_pools_in_background(&self.task_supervisor, vec![removed]);
Ok(true)
}
async fn take_client_session_pool(
&self,
connection_id: &str,
database: Option<&str>,
client_session_id: &str,
) -> Result<Option<(String, PoolKind)>, String> {
let session = normalize_client_session_id(Some(client_session_id));
let Some(session) = session else {
return Ok(false);
return Ok(None);
};
let config = {
let configs = self.configs.read().await;
@@ -2320,18 +2349,13 @@ impl AppState {
let base_pool_key = base_pool_key_for(db_type, connection_id, database, false);
let pool_key = session_scoped_pool_key_for(config.as_ref(), base_pool_key.clone(), Some(&session));
if pool_key == base_pool_key {
return Ok(false);
return Ok(None);
}
self.stop_keepalive_task(&pool_key).await;
self.pool_activity.write().await.remove(&pool_key);
self.postgres_cancel_contexts.write().await.remove(&pool_key);
let removed = self.connections.write().await.remove(&pool_key);
if let Some(pool) = removed {
close_pool_kind_with_timeout(pool_key, pool).await;
Ok(true)
} else {
Ok(false)
}
Ok(removed.map(|pool| (pool_key, pool)))
}
pub async fn remove_pool_by_key(&self, pool_key: &str) -> bool {
@@ -2453,7 +2477,7 @@ impl AppState {
self.postgres_cancel_contexts.write().await.remove(pool_key);
let removed = self.connections.write().await.remove(pool_key);
if let Some(pool) = removed {
close_removed_pools_in_background(vec![(pool_key.to_string(), pool)]);
close_removed_pools_in_background(&self.task_supervisor, vec![(pool_key.to_string(), pool)]);
true
} else {
false
@@ -3008,13 +3032,13 @@ impl AppState {
pub async fn remove_connection_pools_detached(&self, connection_id: &str) {
let removed = self.drain_connection_pools(connection_id).await;
close_removed_pools_in_background(removed);
close_removed_pools_in_background(&self.task_supervisor, removed);
}
#[cfg(feature = "duckdb-bundled")]
async fn remove_duckdb_pools_detached(&self) {
let removed = self.drain_duckdb_pools().await;
close_removed_pools_in_background(removed);
close_removed_pools_in_background(&self.task_supervisor, removed);
}
#[cfg(not(feature = "duckdb-bundled"))]
@@ -3463,13 +3487,19 @@ async fn close_removed_pools(removed: Vec<(String, PoolKind)>) {
}
}
fn close_removed_pools_in_background(removed: Vec<(String, PoolKind)>) {
fn close_removed_pools_in_background(supervisor: &TaskSupervisor, removed: Vec<(String, PoolKind)>) {
if removed.is_empty() {
return;
}
tokio::spawn(async move {
// Supervision keeps detached cleanup visible to application shutdown instead of leaving an
// untracked Tokio task that may be abandoned silently.
let pool_count = removed.len();
let task_key = format!("pool-close:{}", uuid::Uuid::new_v4());
if !supervisor.spawn_once(task_key, move |_| async move {
close_removed_pools(removed).await;
});
}) {
log::debug!("Dropped {pool_count} detached pool handle(s) during application shutdown");
}
}
async fn close_pool_kind_with_timeout(pool_key: String, pool: PoolKind) {
@@ -5244,6 +5274,26 @@ for line in sys.stdin:
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test]
async fn detach_client_session_pool_removes_pool_before_background_close() {
let (state, dir) = test_app_state().await;
let mut config = mysql_config(None);
config.id = "conn".to_string();
config.db_type = DatabaseType::Sqlite;
state.configs.write().await.insert(config.id.clone(), config);
let pool_key = "conn:session:import-1";
let pool = crate::db::sqlite::connect_path(":memory:").await.unwrap();
state.connections.write().await.insert(pool_key.to_string(), PoolKind::Sqlite(pool));
state.pool_activity.write().await.insert(pool_key.to_string(), super::PoolActivity::now());
assert!(state.detach_client_session_pool("conn", None, "import-1").await.unwrap());
assert!(!state.connections.read().await.contains_key(pool_key));
assert!(!state.pool_activity.read().await.contains_key(pool_key));
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test]
async fn closing_oracle_table_tab_keeps_connection_scoped_pool() {
let (state, dir) = test_app_state().await;
+21 -8
View File
@@ -2551,17 +2551,30 @@ pub async fn execute_statements_in_transaction(
.map_err(|e| query_error_with_omitted_sql_context(&e, sql_ctx))?
};
execute_statements_in_transaction_on_pool(state, &pool_key, connection_id, database, statements, schema).await
}
/// Execute multiple SQL statements transactionally on an already-resolved pool.
/// This preserves session-scoped pools used by long-running imports.
pub async fn execute_statements_in_transaction_on_pool(
state: &AppState,
pool_key: &str,
connection_id: &str,
database: &str,
statements: &[String],
schema: Option<&str>,
) -> Result<db::QueryResult, String> {
// Read-only check: intercept all transaction paths before dispatching
check_read_only_for_connection_multi(state, &pool_key, statements).await?;
check_read_only_for_connection_multi(state, pool_key, statements).await?;
let start = std::time::Instant::now();
let db_type = connection_database_type(state, connection_id).await;
let operation_budget = configured_operation_budget_for_pool_key(state, &pool_key).await;
let operation_budget = configured_operation_budget_for_pool_key(state, pool_key).await;
// Clone the pool handle within the lock, then drop it before any async work.
let path = {
let conns = state.connections.read().await;
conns.get(&pool_key).map(|p| match p {
conns.get(pool_key).map(|p| match p {
PoolKind::Postgres(pg) => TxPath::Pg(pg.clone()),
PoolKind::Mysql(mp, _mode) => TxPath::Mysql(mp.clone(), false),
PoolKind::Sqlite(sq) => TxPath::Sqlite(sq.clone()),
@@ -2597,11 +2610,11 @@ pub async fn execute_statements_in_transaction(
let result = match path {
Some(TxPath::Pg(pool)) => {
let cancel_context = state.get_postgres_cancel_context(&pool_key).await;
let cancel_context = state.get_postgres_cancel_context(pool_key).await;
exec_tx_pg_inner(pool, statements, schema, start, operation_budget.clone(), cancel_context).await
}
Some(TxPath::Mysql(pool, _bare)) => {
exec_tx_mysql_inner(state, &pool_key, pool, statements, start, operation_budget.clone()).await
exec_tx_mysql_inner(state, pool_key, pool, statements, start, operation_budget.clone()).await
}
Some(TxPath::Sqlite(pool)) => exec_tx_sqlite_inner(pool, statements, start).await,
Some(TxPath::CloudflareD1(client)) => {
@@ -2615,18 +2628,18 @@ pub async fn execute_statements_in_transaction(
}
Some(TxPath::Explicit) => {
let mysql_dialect = connection_mysql_query_dialect(state, connection_id).await;
exec_tx_explicit_inner(state, &pool_key, mysql_dialect, Some(database), statements, schema, start).await
exec_tx_explicit_inner(state, pool_key, mysql_dialect, Some(database), statements, schema, start).await
}
Some(TxPath::None) => {
let mysql_dialect = connection_mysql_query_dialect(state, connection_id).await;
exec_tx_none_inner(state, &pool_key, mysql_dialect, Some(database), statements, schema, start).await
exec_tx_none_inner(state, pool_key, mysql_dialect, Some(database), statements, schema, start).await
}
None => Err("Connection not found for transaction".to_string()),
};
if let Err(err) = result.as_ref() {
if matches!(pool_error_action(db_type, err), PoolErrorAction::Discard | PoolErrorAction::ReconnectAndRetry) {
state.remove_pool_by_key(&pool_key).await;
state.remove_pool_by_key(pool_key).await;
}
}
File diff suppressed because it is too large. Load diff
+55 -2
View File
@@ -27,17 +27,47 @@ impl TaskSupervisor {
F: FnOnce(CancellationToken) -> Fut,
Fut: Future<Output = ()> + Send + 'static,
{
let key = key.into();
let mut tasks = self.tasks.lock().unwrap_or_else(|error| error.into_inner());
if !self.accepting.load(Ordering::Acquire) {
return false;
}
let handle = tokio::spawn(task(self.cancellation.child_token()));
if let Some(previous) = self.tasks.lock().unwrap_or_else(|error| error.into_inner()).insert(key.into(), handle)
{
if let Some(previous) = tasks.insert(key, handle) {
previous.abort();
}
true
}
pub fn spawn_once<F, Fut>(&self, key: impl Into<String>, task: F) -> bool
where
F: FnOnce(CancellationToken) -> Fut + Send + 'static,
Fut: Future<Output = ()> + Send + 'static,
{
// One-shot cleanup tasks are keyed so duplicate requests do not accumulate, and
// completed handles remove themselves from the supervisor.
let key = key.into();
let mut tasks = self.tasks.lock().unwrap_or_else(|error| error.into_inner());
if !self.accepting.load(Ordering::Acquire) || tasks.contains_key(&key) {
return false;
}
let cleanup = self.clone();
let task_key = key.clone();
// Do not let a very short task finish before its handle is registered in the map.
let (start, started) = tokio::sync::oneshot::channel();
let handle = tokio::spawn(async move {
if started.await.is_err() {
return;
}
task(cleanup.cancellation.child_token()).await;
cleanup.remove_completed(&task_key);
});
tasks.insert(key, handle);
drop(tasks);
let _ = start.send(());
true
}
pub fn stop(&self, key: &str) {
if let Some(task) = self.tasks.lock().unwrap_or_else(|error| error.into_inner()).remove(key) {
task.abort();
@@ -62,6 +92,10 @@ impl TaskSupervisor {
self.tasks.lock().unwrap_or_else(|error| error.into_inner()).len()
}
fn remove_completed(&self, key: &str) {
self.tasks.lock().unwrap_or_else(|error| error.into_inner()).remove(key);
}
pub async fn shutdown(&self, deadline: Duration) {
self.accepting.store(false, Ordering::Release);
self.cancellation.cancel();
@@ -116,4 +150,23 @@ mod tests {
assert_eq!(supervisor.active_count(), 0);
assert!(!supervisor.spawn_replace("late", |_| async {}));
}
#[tokio::test]
async fn spawn_once_removes_completed_task() {
let supervisor = TaskSupervisor::new();
assert!(supervisor.spawn_once("one-shot", |_| async {}));
tokio::time::sleep(Duration::from_millis(1)).await;
assert_eq!(supervisor.active_count(), 0);
}
#[tokio::test]
async fn spawn_once_rejects_duplicate_active_key() {
let supervisor = TaskSupervisor::new();
assert!(supervisor.spawn_once("one-shot", |token| async move {
token.cancelled().await;
}));
assert!(!supervisor.spawn_once("one-shot", |_| async {}));
assert_eq!(supervisor.active_count(), 1);
supervisor.shutdown(Duration::from_secs(1)).await;
}
}
+72
View File
@@ -2082,6 +2082,63 @@ fn generate_transfer_write_sql(
}
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn generate_insert_typed_sql_batches(
columns: &[String],
column_types: &[Option<String>],
rows: &[Vec<serde_json::Value>],
table: &str,
schema: &str,
db_type: &DatabaseType,
requested_max_rows: usize,
) -> Vec<(String, usize)> {
if rows.is_empty() {
return Vec::new();
}
let max_rows = requested_max_rows.max(1);
let max_sql_bytes = match db_type {
DatabaseType::CloudflareD1 => crate::db::cloudflare_d1::MAX_SQL_STATEMENT_BYTES,
_ => MAX_TRANSFER_WRITE_SQL_BYTES,
};
let mut statements = Vec::new();
let mut start = 0;
// First honor the row limit, then use binary search to find the largest statement that
// also fits the backend byte limit. This avoids generating every intermediate size.
while start < rows.len() {
let max_end = start.saturating_add(max_rows).min(rows.len());
let mut end = max_end;
let mut accepted = generate_insert_typed(columns, column_types, &rows[start..max_end], table, schema, db_type);
if accepted.len() > max_sql_bytes && max_end > start + 1 {
end = start + 1;
accepted = generate_insert_typed(columns, column_types, &rows[start..end], table, schema, db_type);
let mut low = start + 2;
let mut high = max_end;
while low <= high {
let candidate_end = low + (high - low) / 2;
let candidate =
generate_insert_typed(columns, column_types, &rows[start..candidate_end], table, schema, db_type);
if candidate.len() <= max_sql_bytes {
accepted = candidate;
end = candidate_end;
low = candidate_end + 1;
} else {
high = candidate_end - 1;
}
}
}
if !accepted.is_empty() {
statements.push((accepted, end - start));
}
start = end;
}
statements
}
#[allow(clippy::too_many_arguments)]
fn generate_transfer_write_sql_batches(
mode: &TransferMode,
@@ -2097,6 +2154,21 @@ fn generate_transfer_write_sql_batches(
return Vec::new();
}
if matches!(mode, TransferMode::Append | TransferMode::Overwrite) {
return generate_insert_typed_sql_batches(
columns,
column_types,
rows,
table,
schema,
db_type,
max_transfer_write_rows(db_type, mode),
)
.into_iter()
.map(|(sql, _)| sql)
.collect();
}
let max_rows = max_transfer_write_rows(db_type, mode);
let max_sql_bytes = match db_type {
DatabaseType::CloudflareD1 => crate::db::cloudflare_d1::MAX_SQL_STATEMENT_BYTES,