feat(export): add SQL INSERT batch or single-row mode

This commit is contained in:
jearhe authored and GitHub committed 2026-09-06 10:27:34 +08:00
1 parent 5d65be03a3
commit ab35a5b1da
29 files changed
+563 -23

No files matched your search

+34 -3
View File
@@ -20,6 +20,7 @@ use crate::csv_export::{
pub use crate::database_export::ExportStatus;
use crate::database_export::{
build_export_insert_statements, is_export_cancelled, is_internal_export_column, BuildExportInsertStatementsOptions,
SqlInsertMode,
};
use crate::db::agent_driver::AgentTableReadStartParams;
use crate::models::connection::DatabaseType;
@@ -54,6 +55,8 @@ pub struct TableExportRequest {
/// "csv", "xlsx", "json", "markdown", "sql", or "txt"
pub format: String,
#[serde(default)]
pub insert_mode: SqlInsertMode,
#[serde(default)]
pub csv_quote_mode: CsvQuoteMode,
#[serde(default)]
pub columns: Option<Vec<String>>,
@@ -1268,7 +1271,7 @@ async fn try_export_native_table_stream(
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: std::mem::take(pending_rows),
batch_size: Some(SQL_INSERT_BATCH_SIZE),
batch_size: Some(request.insert_mode.batch_size(SQL_INSERT_BATCH_SIZE)),
})?;
if !statements.is_empty() {
if wrote_statements {
@@ -1290,7 +1293,7 @@ async fn try_export_native_table_stream(
cancel_token.clone(),
|row| {
pending_rows.push(row.to_vec());
if pending_rows.len() >= SQL_INSERT_BATCH_SIZE {
if request.insert_mode.flush_each_row() || pending_rows.len() >= SQL_INSERT_BATCH_SIZE {
flush_pending(&mut file, &mut pending_rows)?;
}
rows_exported += 1;
@@ -2102,7 +2105,7 @@ async fn export_table_data_core_inner(
spatial_columns: result.spatial_columns.clone(),
spatial_values: result.spatial_values.clone(),
rows: result.rows.clone(),
batch_size: Some(100),
batch_size: Some(request.insert_mode.batch_size(SQL_INSERT_BATCH_SIZE)),
})?;
if !statements.is_empty() {
if wrote_statements {
@@ -2177,6 +2180,22 @@ mod tests {
#[cfg(unix)]
use std::os::unix::fs::PermissionsExt;
#[test]
fn table_export_request_defaults_to_batch_insert_mode() {
let request: TableExportRequest = serde_json::from_value(json!({
"exportId": "export-1",
"connectionId": "conn-1",
"database": "db",
"schema": null,
"tableName": "users",
"filePath": "users.sql",
"format": "sql"
}))
.expect("deserialize table export request");
assert_eq!(request.insert_mode, SqlInsertMode::Batch);
}
#[cfg(unix)]
struct ExternalDriverExportFixture {
state: AppState,
@@ -2266,6 +2285,7 @@ mod tests {
table_name: "EXPORT_SAMPLE".to_string(),
file_path: output.to_string_lossy().into_owned(),
format: "csv".to_string(),
insert_mode: Default::default(),
columns: Some(vec!["id".to_string(), "name".to_string()]),
column_types: Some(vec![Some("INTEGER".to_string()), Some("VARCHAR".to_string())]),
primary_keys: Some(vec!["id".to_string()]),
@@ -2427,6 +2447,7 @@ mod tests {
table_name: "weather".to_string(),
file_path: output.to_string_lossy().into_owned(),
format: "csv".to_string(),
insert_mode: Default::default(),
csv_quote_mode: CsvQuoteMode::All,
columns: None,
column_types: None,
@@ -2591,6 +2612,7 @@ mod tests {
table_name: "device2".to_string(),
file_path: "device2.csv".to_string(),
format: "csv".to_string(),
insert_mode: Default::default(),
columns: None,
column_types: None,
primary_keys: None,
@@ -2648,6 +2670,7 @@ mod tests {
table_name: "device2".to_string(),
file_path: "device2.csv".to_string(),
format: "csv".to_string(),
insert_mode: Default::default(),
columns: None,
column_types: None,
primary_keys: None,
@@ -2683,6 +2706,7 @@ mod tests {
table_name: "device2".to_string(),
file_path: "device2.csv".to_string(),
format: "csv".to_string(),
insert_mode: Default::default(),
columns: None,
column_types: None,
primary_keys: None,
@@ -2713,6 +2737,7 @@ mod tests {
table_name: "device2".to_string(),
file_path: "device2.csv".to_string(),
format: "csv".to_string(),
insert_mode: Default::default(),
columns: None,
column_types: None,
primary_keys: None,
@@ -2751,6 +2776,7 @@ mod tests {
table_name: "samples".to_string(),
file_path: "samples.csv".to_string(),
format: "csv".to_string(),
insert_mode: Default::default(),
columns: None,
column_types: None,
primary_keys: None,
@@ -2801,6 +2827,7 @@ mod tests {
table_name: "events".to_string(),
file_path: "events.csv".to_string(),
format: "csv".to_string(),
insert_mode: Default::default(),
columns: None,
column_types: None,
primary_keys: None,
@@ -2845,6 +2872,7 @@ mod tests {
table_name: "orders".to_string(),
file_path: "orders.txt".to_string(),
format: "txt".to_string(),
insert_mode: Default::default(),
columns: None,
column_types: None,
primary_keys: None,
@@ -2917,6 +2945,7 @@ mod tests {
table_name: "order".to_string(),
file_path: "order.csv".to_string(),
format: "csv".to_string(),
insert_mode: Default::default(),
columns: None,
column_types: None,
primary_keys: None,
@@ -2971,6 +3000,7 @@ mod tests {
table_name: "spatial_data".to_string(),
file_path: "spatial_data.sql".to_string(),
format: "sql".to_string(),
insert_mode: Default::default(),
columns: None,
column_types: None,
primary_keys: None,
@@ -3027,6 +3057,7 @@ mod tests {
table_name: "USERS".to_string(),
file_path: "users.sql".to_string(),
format: "sql".to_string(),
insert_mode: Default::default(),
columns: None,
column_types: None,
primary_keys: None,