perf(import): optimize large file imports
This commit is contained in:
1 parent
9602cc3329
commit
0e33c73035
29 files changed
+5032
-625
No files matched your search
@@ -88,3 +88,4 @@ bytes = "1"
|
||||
font-kit = "0.14.3"
|
||||
zip = { version = "4", default-features = false, features = ["deflate"] }
|
||||
sysinfo = { version = "0.32", features = ["system"] }
|
||||
tempfile = "3"
|
||||
@@ -0,0 +1,211 @@
|
||||
use std::fs::{self, File};
|
||||
use std::io::{BufWriter, Write};
|
||||
use std::path::Path;
|
||||
use std::time::Instant;
|
||||
|
||||
use dbx_core::models::connection::DatabaseType;
|
||||
use dbx_core::table_import::{
|
||||
build_import_insert_batches, parse_delimited_file_with_options, parse_xlsx_file_with_options,
|
||||
preview_table_import_file_with_request, ParsedImportFile, TableImportColumnMapping, TableImportParseOptions,
|
||||
TableImportPreviewRequest, TableImportSourceFormat,
|
||||
};
|
||||
use dbx_core::xlsx_export::{build_xlsx_workbook, XlsxWorksheetData};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum BenchFormat {
|
||||
Csv,
|
||||
Xlsx,
|
||||
All,
|
||||
}
|
||||
|
||||
struct Options {
|
||||
rows: usize,
|
||||
columns: usize,
|
||||
batch_size: usize,
|
||||
format: BenchFormat,
|
||||
}
|
||||
|
||||
fn print_help() {
|
||||
println!(
|
||||
"Table import benchmark\n\n\
|
||||
Usage:\n cargo run -p dbx-core --example table_import_bench --release -- [options]\n\n\
|
||||
Options:\n --rows=10000 Number of data rows\n --columns=10 Number of columns\n --batch-size=500 SQL rows per batch\n --format=all csv, xlsx, or all\n --help Show this help\n"
|
||||
);
|
||||
}
|
||||
|
||||
fn parse_options() -> Result<Options, String> {
|
||||
let mut options = Options { rows: 10_000, columns: 10, batch_size: 500, format: BenchFormat::All };
|
||||
for argument in std::env::args().skip(1) {
|
||||
if argument == "--help" || argument == "-h" {
|
||||
print_help();
|
||||
std::process::exit(0);
|
||||
}
|
||||
let (key, value) = argument.split_once('=').ok_or_else(|| format!("Invalid option: {argument}"))?;
|
||||
match key {
|
||||
"--rows" => options.rows = value.parse().map_err(|_| format!("Invalid row count: {value}"))?,
|
||||
"--columns" => options.columns = value.parse().map_err(|_| format!("Invalid column count: {value}"))?,
|
||||
"--batch-size" => options.batch_size = value.parse().map_err(|_| format!("Invalid batch size: {value}"))?,
|
||||
"--format" => {
|
||||
options.format = match value {
|
||||
"csv" => BenchFormat::Csv,
|
||||
"xlsx" => BenchFormat::Xlsx,
|
||||
"all" => BenchFormat::All,
|
||||
_ => return Err(format!("Invalid format: {value}")),
|
||||
}
|
||||
}
|
||||
_ => return Err(format!("Unknown option: {key}")),
|
||||
}
|
||||
}
|
||||
if options.rows == 0 || options.columns == 0 || options.batch_size == 0 {
|
||||
return Err("rows, columns, and batch-size must be greater than zero".to_string());
|
||||
}
|
||||
Ok(options)
|
||||
}
|
||||
|
||||
fn columns(count: usize) -> Vec<String> {
|
||||
(0..count).map(|index| format!("column_{}", index + 1)).collect()
|
||||
}
|
||||
|
||||
fn row(row_index: usize, column_count: usize) -> Vec<serde_json::Value> {
|
||||
(0..column_count)
|
||||
.map(|column_index| {
|
||||
if column_index == 0 {
|
||||
serde_json::json!(row_index + 1)
|
||||
} else if column_index % 3 == 0 {
|
||||
serde_json::json!(format!("2026-07-{:02} 12:34:56", row_index % 28 + 1))
|
||||
} else {
|
||||
serde_json::json!(format!("value-{row_index}-{column_index}"))
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn write_csv(path: &Path, row_count: usize, column_count: usize) -> Result<(), String> {
|
||||
let file = File::create(path).map_err(|error| error.to_string())?;
|
||||
let mut writer = BufWriter::new(file);
|
||||
writeln!(writer, "{}", columns(column_count).join(",")).map_err(|error| error.to_string())?;
|
||||
for row_index in 0..row_count {
|
||||
let values = row(row_index, column_count)
|
||||
.into_iter()
|
||||
.map(|value| value.as_str().map(str::to_string).unwrap_or_else(|| value.to_string()))
|
||||
.collect::<Vec<_>>();
|
||||
writeln!(writer, "{}", values.join(",")).map_err(|error| error.to_string())?;
|
||||
}
|
||||
writer.flush().map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
fn write_xlsx(path: &Path, row_count: usize, column_count: usize) -> Result<(), String> {
|
||||
let workbook = build_xlsx_workbook(&XlsxWorksheetData {
|
||||
sheet_name: Some("Benchmark".to_string()),
|
||||
columns: columns(column_count),
|
||||
column_types: vec![],
|
||||
rows: (0..row_count).map(|row_index| row(row_index, column_count)).collect(),
|
||||
})?;
|
||||
fs::write(path, workbook).map_err(|error| error.to_string())
|
||||
}
|
||||
|
||||
fn mappings(data: &ParsedImportFile) -> Vec<TableImportColumnMapping> {
|
||||
data.columns
|
||||
.iter()
|
||||
.map(|column| TableImportColumnMapping {
|
||||
source_column: column.clone(),
|
||||
target_column: column.clone(),
|
||||
target_data_type: None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn benchmark_file(
|
||||
path: &Path,
|
||||
format: TableImportSourceFormat,
|
||||
options: &Options,
|
||||
) -> Result<serde_json::Value, String> {
|
||||
let path_text = path.to_string_lossy();
|
||||
let parse_options = TableImportParseOptions::default();
|
||||
let preview_started = Instant::now();
|
||||
let preview = preview_table_import_file_with_request(TableImportPreviewRequest {
|
||||
file_path: path_text.to_string(),
|
||||
source_ref: None,
|
||||
source_format: Some(format),
|
||||
parse_options: parse_options.clone(),
|
||||
preview_limit: Some(50),
|
||||
})
|
||||
.await?;
|
||||
let preview_ms = preview_started.elapsed().as_secs_f64() * 1000.0;
|
||||
|
||||
let full_parse_started = Instant::now();
|
||||
let parsed = match format {
|
||||
TableImportSourceFormat::Csv => {
|
||||
parse_delimited_file_with_options(&path_text, format, &parse_options, usize::MAX)?
|
||||
}
|
||||
TableImportSourceFormat::Excel => parse_xlsx_file_with_options(&path_text, &parse_options, usize::MAX)?,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
let full_parse_ms = full_parse_started.elapsed().as_secs_f64() * 1000.0;
|
||||
|
||||
let batch_started = Instant::now();
|
||||
let batches = build_import_insert_batches(
|
||||
&parsed,
|
||||
&mappings(&parsed),
|
||||
&[],
|
||||
"benchmark_import",
|
||||
"main",
|
||||
&DatabaseType::Sqlite,
|
||||
options.batch_size,
|
||||
)?;
|
||||
let batch_build_ms = batch_started.elapsed().as_secs_f64() * 1000.0;
|
||||
let sql_bytes = batches.iter().map(|batch| batch.sql.len()).sum::<usize>();
|
||||
|
||||
Ok(serde_json::json!({
|
||||
"format": format.label(),
|
||||
"fileBytes": fs::metadata(path).map_err(|error| error.to_string())?.len(),
|
||||
"rows": parsed.total_rows,
|
||||
"columns": preview.columns.len(),
|
||||
"previewRows": preview.rows.len(),
|
||||
"previewTotalRowsExact": preview.total_rows_exact,
|
||||
"previewMs": preview_ms,
|
||||
"fullParseMs": full_parse_ms,
|
||||
"batchBuildMs": batch_build_ms,
|
||||
"batchCount": batches.len(),
|
||||
"sqlBytes": sql_bytes,
|
||||
}))
|
||||
}
|
||||
|
||||
async fn run() -> Result<(), String> {
|
||||
let options = parse_options()?;
|
||||
let temp_dir = std::env::temp_dir().join(format!("dbx-table-import-bench-{}", uuid::Uuid::new_v4()));
|
||||
fs::create_dir_all(&temp_dir).map_err(|error| error.to_string())?;
|
||||
let mut results = Vec::new();
|
||||
|
||||
if matches!(options.format, BenchFormat::Csv | BenchFormat::All) {
|
||||
let path = temp_dir.join("benchmark.csv");
|
||||
write_csv(&path, options.rows, options.columns)?;
|
||||
results.push(benchmark_file(&path, TableImportSourceFormat::Csv, &options).await?);
|
||||
}
|
||||
if matches!(options.format, BenchFormat::Xlsx | BenchFormat::All) {
|
||||
let path = temp_dir.join("benchmark.xlsx");
|
||||
write_xlsx(&path, options.rows, options.columns)?;
|
||||
results.push(benchmark_file(&path, TableImportSourceFormat::Excel, &options).await?);
|
||||
}
|
||||
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&serde_json::json!({
|
||||
"rows": options.rows,
|
||||
"columns": options.columns,
|
||||
"batchSize": options.batch_size,
|
||||
"results": results,
|
||||
}))
|
||||
.map_err(|error| error.to_string())?
|
||||
);
|
||||
let _ = fs::remove_dir_all(temp_dir);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
if let Err(error) = run().await {
|
||||
eprintln!("{error}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
@@ -2308,9 +2308,38 @@ impl AppState {
|
||||
database: Option<&str>,
|
||||
client_session_id: &str,
|
||||
) -> Result<bool, String> {
|
||||
let Some((pool_key, pool)) = self.take_client_session_pool(connection_id, database, client_session_id).await?
|
||||
else {
|
||||
return Ok(false);
|
||||
};
|
||||
close_pool_kind_with_timeout(pool_key, pool).await;
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Removes a session-scoped pool immediately and schedules the potentially slow driver
|
||||
/// shutdown on the supervised background task set.
|
||||
pub async fn detach_client_session_pool(
|
||||
&self,
|
||||
connection_id: &str,
|
||||
database: Option<&str>,
|
||||
client_session_id: &str,
|
||||
) -> Result<bool, String> {
|
||||
let Some(removed) = self.take_client_session_pool(connection_id, database, client_session_id).await? else {
|
||||
return Ok(false);
|
||||
};
|
||||
close_removed_pools_in_background(&self.task_supervisor, vec![removed]);
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn take_client_session_pool(
|
||||
&self,
|
||||
connection_id: &str,
|
||||
database: Option<&str>,
|
||||
client_session_id: &str,
|
||||
) -> Result<Option<(String, PoolKind)>, String> {
|
||||
let session = normalize_client_session_id(Some(client_session_id));
|
||||
let Some(session) = session else {
|
||||
return Ok(false);
|
||||
return Ok(None);
|
||||
};
|
||||
let config = {
|
||||
let configs = self.configs.read().await;
|
||||
@@ -2320,18 +2349,13 @@ impl AppState {
|
||||
let base_pool_key = base_pool_key_for(db_type, connection_id, database, false);
|
||||
let pool_key = session_scoped_pool_key_for(config.as_ref(), base_pool_key.clone(), Some(&session));
|
||||
if pool_key == base_pool_key {
|
||||
return Ok(false);
|
||||
return Ok(None);
|
||||
}
|
||||
self.stop_keepalive_task(&pool_key).await;
|
||||
self.pool_activity.write().await.remove(&pool_key);
|
||||
self.postgres_cancel_contexts.write().await.remove(&pool_key);
|
||||
let removed = self.connections.write().await.remove(&pool_key);
|
||||
if let Some(pool) = removed {
|
||||
close_pool_kind_with_timeout(pool_key, pool).await;
|
||||
Ok(true)
|
||||
} else {
|
||||
Ok(false)
|
||||
}
|
||||
Ok(removed.map(|pool| (pool_key, pool)))
|
||||
}
|
||||
|
||||
pub async fn remove_pool_by_key(&self, pool_key: &str) -> bool {
|
||||
@@ -2453,7 +2477,7 @@ impl AppState {
|
||||
self.postgres_cancel_contexts.write().await.remove(pool_key);
|
||||
let removed = self.connections.write().await.remove(pool_key);
|
||||
if let Some(pool) = removed {
|
||||
close_removed_pools_in_background(vec![(pool_key.to_string(), pool)]);
|
||||
close_removed_pools_in_background(&self.task_supervisor, vec![(pool_key.to_string(), pool)]);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
@@ -3008,13 +3032,13 @@ impl AppState {
|
||||
|
||||
pub async fn remove_connection_pools_detached(&self, connection_id: &str) {
|
||||
let removed = self.drain_connection_pools(connection_id).await;
|
||||
close_removed_pools_in_background(removed);
|
||||
close_removed_pools_in_background(&self.task_supervisor, removed);
|
||||
}
|
||||
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
async fn remove_duckdb_pools_detached(&self) {
|
||||
let removed = self.drain_duckdb_pools().await;
|
||||
close_removed_pools_in_background(removed);
|
||||
close_removed_pools_in_background(&self.task_supervisor, removed);
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "duckdb-bundled"))]
|
||||
@@ -3463,13 +3487,19 @@ async fn close_removed_pools(removed: Vec<(String, PoolKind)>) {
|
||||
}
|
||||
}
|
||||
|
||||
fn close_removed_pools_in_background(removed: Vec<(String, PoolKind)>) {
|
||||
fn close_removed_pools_in_background(supervisor: &TaskSupervisor, removed: Vec<(String, PoolKind)>) {
|
||||
if removed.is_empty() {
|
||||
return;
|
||||
}
|
||||
tokio::spawn(async move {
|
||||
// Supervision keeps detached cleanup visible to application shutdown instead of leaving an
|
||||
// untracked Tokio task that may be abandoned silently.
|
||||
let pool_count = removed.len();
|
||||
let task_key = format!("pool-close:{}", uuid::Uuid::new_v4());
|
||||
if !supervisor.spawn_once(task_key, move |_| async move {
|
||||
close_removed_pools(removed).await;
|
||||
});
|
||||
}) {
|
||||
log::debug!("Dropped {pool_count} detached pool handle(s) during application shutdown");
|
||||
}
|
||||
}
|
||||
|
||||
async fn close_pool_kind_with_timeout(pool_key: String, pool: PoolKind) {
|
||||
@@ -5244,6 +5274,26 @@ for line in sys.stdin:
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn detach_client_session_pool_removes_pool_before_background_close() {
|
||||
let (state, dir) = test_app_state().await;
|
||||
let mut config = mysql_config(None);
|
||||
config.id = "conn".to_string();
|
||||
config.db_type = DatabaseType::Sqlite;
|
||||
state.configs.write().await.insert(config.id.clone(), config);
|
||||
|
||||
let pool_key = "conn:session:import-1";
|
||||
let pool = crate::db::sqlite::connect_path(":memory:").await.unwrap();
|
||||
state.connections.write().await.insert(pool_key.to_string(), PoolKind::Sqlite(pool));
|
||||
state.pool_activity.write().await.insert(pool_key.to_string(), super::PoolActivity::now());
|
||||
|
||||
assert!(state.detach_client_session_pool("conn", None, "import-1").await.unwrap());
|
||||
assert!(!state.connections.read().await.contains_key(pool_key));
|
||||
assert!(!state.pool_activity.read().await.contains_key(pool_key));
|
||||
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn closing_oracle_table_tab_keeps_connection_scoped_pool() {
|
||||
let (state, dir) = test_app_state().await;
|
||||
|
||||
@@ -2551,17 +2551,30 @@ pub async fn execute_statements_in_transaction(
|
||||
.map_err(|e| query_error_with_omitted_sql_context(&e, sql_ctx))?
|
||||
};
|
||||
|
||||
execute_statements_in_transaction_on_pool(state, &pool_key, connection_id, database, statements, schema).await
|
||||
}
|
||||
|
||||
/// Execute multiple SQL statements transactionally on an already-resolved pool.
|
||||
/// This preserves session-scoped pools used by long-running imports.
|
||||
pub async fn execute_statements_in_transaction_on_pool(
|
||||
state: &AppState,
|
||||
pool_key: &str,
|
||||
connection_id: &str,
|
||||
database: &str,
|
||||
statements: &[String],
|
||||
schema: Option<&str>,
|
||||
) -> Result<db::QueryResult, String> {
|
||||
// Read-only check: intercept all transaction paths before dispatching
|
||||
check_read_only_for_connection_multi(state, &pool_key, statements).await?;
|
||||
check_read_only_for_connection_multi(state, pool_key, statements).await?;
|
||||
|
||||
let start = std::time::Instant::now();
|
||||
let db_type = connection_database_type(state, connection_id).await;
|
||||
let operation_budget = configured_operation_budget_for_pool_key(state, &pool_key).await;
|
||||
let operation_budget = configured_operation_budget_for_pool_key(state, pool_key).await;
|
||||
|
||||
// Clone the pool handle within the lock, then drop it before any async work.
|
||||
let path = {
|
||||
let conns = state.connections.read().await;
|
||||
conns.get(&pool_key).map(|p| match p {
|
||||
conns.get(pool_key).map(|p| match p {
|
||||
PoolKind::Postgres(pg) => TxPath::Pg(pg.clone()),
|
||||
PoolKind::Mysql(mp, _mode) => TxPath::Mysql(mp.clone(), false),
|
||||
PoolKind::Sqlite(sq) => TxPath::Sqlite(sq.clone()),
|
||||
@@ -2597,11 +2610,11 @@ pub async fn execute_statements_in_transaction(
|
||||
|
||||
let result = match path {
|
||||
Some(TxPath::Pg(pool)) => {
|
||||
let cancel_context = state.get_postgres_cancel_context(&pool_key).await;
|
||||
let cancel_context = state.get_postgres_cancel_context(pool_key).await;
|
||||
exec_tx_pg_inner(pool, statements, schema, start, operation_budget.clone(), cancel_context).await
|
||||
}
|
||||
Some(TxPath::Mysql(pool, _bare)) => {
|
||||
exec_tx_mysql_inner(state, &pool_key, pool, statements, start, operation_budget.clone()).await
|
||||
exec_tx_mysql_inner(state, pool_key, pool, statements, start, operation_budget.clone()).await
|
||||
}
|
||||
Some(TxPath::Sqlite(pool)) => exec_tx_sqlite_inner(pool, statements, start).await,
|
||||
Some(TxPath::CloudflareD1(client)) => {
|
||||
@@ -2615,18 +2628,18 @@ pub async fn execute_statements_in_transaction(
|
||||
}
|
||||
Some(TxPath::Explicit) => {
|
||||
let mysql_dialect = connection_mysql_query_dialect(state, connection_id).await;
|
||||
exec_tx_explicit_inner(state, &pool_key, mysql_dialect, Some(database), statements, schema, start).await
|
||||
exec_tx_explicit_inner(state, pool_key, mysql_dialect, Some(database), statements, schema, start).await
|
||||
}
|
||||
Some(TxPath::None) => {
|
||||
let mysql_dialect = connection_mysql_query_dialect(state, connection_id).await;
|
||||
exec_tx_none_inner(state, &pool_key, mysql_dialect, Some(database), statements, schema, start).await
|
||||
exec_tx_none_inner(state, pool_key, mysql_dialect, Some(database), statements, schema, start).await
|
||||
}
|
||||
None => Err("Connection not found for transaction".to_string()),
|
||||
};
|
||||
|
||||
if let Err(err) = result.as_ref() {
|
||||
if matches!(pool_error_action(db_type, err), PoolErrorAction::Discard | PoolErrorAction::ReconnectAndRetry) {
|
||||
state.remove_pool_by_key(&pool_key).await;
|
||||
state.remove_pool_by_key(pool_key).await;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+3951
-503
File diff suppressed because it is too large.
Load diff
@@ -27,17 +27,47 @@ impl TaskSupervisor {
|
||||
F: FnOnce(CancellationToken) -> Fut,
|
||||
Fut: Future<Output = ()> + Send + 'static,
|
||||
{
|
||||
let key = key.into();
|
||||
let mut tasks = self.tasks.lock().unwrap_or_else(|error| error.into_inner());
|
||||
if !self.accepting.load(Ordering::Acquire) {
|
||||
return false;
|
||||
}
|
||||
let handle = tokio::spawn(task(self.cancellation.child_token()));
|
||||
if let Some(previous) = self.tasks.lock().unwrap_or_else(|error| error.into_inner()).insert(key.into(), handle)
|
||||
{
|
||||
if let Some(previous) = tasks.insert(key, handle) {
|
||||
previous.abort();
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
pub fn spawn_once<F, Fut>(&self, key: impl Into<String>, task: F) -> bool
|
||||
where
|
||||
F: FnOnce(CancellationToken) -> Fut + Send + 'static,
|
||||
Fut: Future<Output = ()> + Send + 'static,
|
||||
{
|
||||
// One-shot cleanup tasks are keyed so duplicate requests do not accumulate, and
|
||||
// completed handles remove themselves from the supervisor.
|
||||
let key = key.into();
|
||||
let mut tasks = self.tasks.lock().unwrap_or_else(|error| error.into_inner());
|
||||
if !self.accepting.load(Ordering::Acquire) || tasks.contains_key(&key) {
|
||||
return false;
|
||||
}
|
||||
let cleanup = self.clone();
|
||||
let task_key = key.clone();
|
||||
// Do not let a very short task finish before its handle is registered in the map.
|
||||
let (start, started) = tokio::sync::oneshot::channel();
|
||||
let handle = tokio::spawn(async move {
|
||||
if started.await.is_err() {
|
||||
return;
|
||||
}
|
||||
task(cleanup.cancellation.child_token()).await;
|
||||
cleanup.remove_completed(&task_key);
|
||||
});
|
||||
tasks.insert(key, handle);
|
||||
drop(tasks);
|
||||
let _ = start.send(());
|
||||
true
|
||||
}
|
||||
|
||||
pub fn stop(&self, key: &str) {
|
||||
if let Some(task) = self.tasks.lock().unwrap_or_else(|error| error.into_inner()).remove(key) {
|
||||
task.abort();
|
||||
@@ -62,6 +92,10 @@ impl TaskSupervisor {
|
||||
self.tasks.lock().unwrap_or_else(|error| error.into_inner()).len()
|
||||
}
|
||||
|
||||
fn remove_completed(&self, key: &str) {
|
||||
self.tasks.lock().unwrap_or_else(|error| error.into_inner()).remove(key);
|
||||
}
|
||||
|
||||
pub async fn shutdown(&self, deadline: Duration) {
|
||||
self.accepting.store(false, Ordering::Release);
|
||||
self.cancellation.cancel();
|
||||
@@ -116,4 +150,23 @@ mod tests {
|
||||
assert_eq!(supervisor.active_count(), 0);
|
||||
assert!(!supervisor.spawn_replace("late", |_| async {}));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn spawn_once_removes_completed_task() {
|
||||
let supervisor = TaskSupervisor::new();
|
||||
assert!(supervisor.spawn_once("one-shot", |_| async {}));
|
||||
tokio::time::sleep(Duration::from_millis(1)).await;
|
||||
assert_eq!(supervisor.active_count(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn spawn_once_rejects_duplicate_active_key() {
|
||||
let supervisor = TaskSupervisor::new();
|
||||
assert!(supervisor.spawn_once("one-shot", |token| async move {
|
||||
token.cancelled().await;
|
||||
}));
|
||||
assert!(!supervisor.spawn_once("one-shot", |_| async {}));
|
||||
assert_eq!(supervisor.active_count(), 1);
|
||||
supervisor.shutdown(Duration::from_secs(1)).await;
|
||||
}
|
||||
}
|
||||
@@ -2082,6 +2082,63 @@ fn generate_transfer_write_sql(
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn generate_insert_typed_sql_batches(
|
||||
columns: &[String],
|
||||
column_types: &[Option<String>],
|
||||
rows: &[Vec<serde_json::Value>],
|
||||
table: &str,
|
||||
schema: &str,
|
||||
db_type: &DatabaseType,
|
||||
requested_max_rows: usize,
|
||||
) -> Vec<(String, usize)> {
|
||||
if rows.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let max_rows = requested_max_rows.max(1);
|
||||
let max_sql_bytes = match db_type {
|
||||
DatabaseType::CloudflareD1 => crate::db::cloudflare_d1::MAX_SQL_STATEMENT_BYTES,
|
||||
_ => MAX_TRANSFER_WRITE_SQL_BYTES,
|
||||
};
|
||||
let mut statements = Vec::new();
|
||||
let mut start = 0;
|
||||
|
||||
// First honor the row limit, then use binary search to find the largest statement that
|
||||
// also fits the backend byte limit. This avoids generating every intermediate size.
|
||||
while start < rows.len() {
|
||||
let max_end = start.saturating_add(max_rows).min(rows.len());
|
||||
let mut end = max_end;
|
||||
let mut accepted = generate_insert_typed(columns, column_types, &rows[start..max_end], table, schema, db_type);
|
||||
|
||||
if accepted.len() > max_sql_bytes && max_end > start + 1 {
|
||||
end = start + 1;
|
||||
accepted = generate_insert_typed(columns, column_types, &rows[start..end], table, schema, db_type);
|
||||
let mut low = start + 2;
|
||||
let mut high = max_end;
|
||||
while low <= high {
|
||||
let candidate_end = low + (high - low) / 2;
|
||||
let candidate =
|
||||
generate_insert_typed(columns, column_types, &rows[start..candidate_end], table, schema, db_type);
|
||||
if candidate.len() <= max_sql_bytes {
|
||||
accepted = candidate;
|
||||
end = candidate_end;
|
||||
low = candidate_end + 1;
|
||||
} else {
|
||||
high = candidate_end - 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !accepted.is_empty() {
|
||||
statements.push((accepted, end - start));
|
||||
}
|
||||
start = end;
|
||||
}
|
||||
|
||||
statements
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn generate_transfer_write_sql_batches(
|
||||
mode: &TransferMode,
|
||||
@@ -2097,6 +2154,21 @@ fn generate_transfer_write_sql_batches(
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
if matches!(mode, TransferMode::Append | TransferMode::Overwrite) {
|
||||
return generate_insert_typed_sql_batches(
|
||||
columns,
|
||||
column_types,
|
||||
rows,
|
||||
table,
|
||||
schema,
|
||||
db_type,
|
||||
max_transfer_write_rows(db_type, mode),
|
||||
)
|
||||
.into_iter()
|
||||
.map(|(sql, _)| sql)
|
||||
.collect();
|
||||
}
|
||||
|
||||
let max_rows = max_transfer_write_rows(db_type, mode);
|
||||
let max_sql_bytes = match db_type {
|
||||
DatabaseType::CloudflareD1 => crate::db::cloudflare_d1::MAX_SQL_STATEMENT_BYTES,
|
||||
|
||||
Reference in new issue
Block a user