feat(export): add SQL INSERT batch or single-row mode

This commit is contained in:
jearhe authored and GitHub committed 2026-09-06 10:27:34 +08:00
1 parent 5d65be03a3
commit ab35a5b1da
29 files changed
+563 -23

No files matched your search

+22
View File
@@ -42,6 +42,28 @@ pub enum DatabaseExportOutputCompression {
Gzip,
}
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case")]
pub enum SqlInsertMode {
#[default]
Batch,
Single,
}
impl SqlInsertMode {
pub(crate) const fn flush_each_row(self) -> bool {
matches!(self, Self::Single)
}
pub(crate) const fn batch_size(self, default: usize) -> usize {
if self.flush_each_row() || default == 0 {
1
} else {
default
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct DatabaseExportRequest {
+62 -5
View File
@@ -15,7 +15,9 @@ use crate::csv_export::{
CsvQuoteMode,
};
pub use crate::database_export::ExportStatus;
use crate::database_export::{build_export_insert_statements, is_export_cancelled, BuildExportInsertStatementsOptions};
use crate::database_export::{
build_export_insert_statements, is_export_cancelled, BuildExportInsertStatementsOptions, SqlInsertMode,
};
use crate::models::connection::DatabaseType;
use crate::query::{
await_stream_with_progress_timeout, canceled_error, close_query_session, execute_sql_statement_with_options,
@@ -82,6 +84,8 @@ pub struct QueryResultExportRequest {
pub file_path: String,
pub format: String,
#[serde(default)]
pub insert_mode: SqlInsertMode,
#[serde(default)]
pub csv_quote_mode: CsvQuoteMode,
#[serde(default)]
pub include_sql_sheet: bool,
@@ -300,8 +304,9 @@ fn sql_insert_column_types(request: &QueryResultExportRequest, column_types: &[S
/// Bounded SQL INSERT writer with staged-file replacement safety.
///
/// Rows are buffered and flushed to a temp file every [`SQL_INSERT_BATCH_SIZE`]
/// rows, so memory stays bounded regardless of the query page size. The unique
/// Batch mode buffers rows up to [`SQL_INSERT_BATCH_SIZE`], while single mode
/// flushes each row immediately. Both modes keep memory bounded regardless of
/// the query page size. The unique
/// temp file lives alongside the target and replaces it only after
/// [`SqlInsertWriter::finish`] flushes and synchronizes the complete output.
struct SqlInsertWriter {
@@ -309,6 +314,7 @@ struct SqlInsertWriter {
target: Option<StagedExportTarget>,
pending_rows: Vec<Vec<Value>>,
pending_spatial_values: Vec<Vec<Option<u32>>>,
insert_mode: SqlInsertMode,
columns: Vec<String>,
column_types: Vec<Option<String>>,
spatial_columns: Vec<SpatialColumn>,
@@ -337,6 +343,7 @@ impl SqlInsertWriter {
target: Some(target),
pending_rows: Vec::new(),
pending_spatial_values: Vec::new(),
insert_mode: request.insert_mode,
columns: Vec::new(),
column_types: Vec::new(),
spatial_columns: Vec::new(),
@@ -365,7 +372,7 @@ impl SqlInsertWriter {
fn write_row(&mut self, row: Vec<Value>, spatial_values: Option<Vec<Option<u32>>>) -> Result<(), String> {
self.pending_rows.push(row);
self.pending_spatial_values.push(spatial_values.unwrap_or_default());
if self.pending_rows.len() >= SQL_INSERT_BATCH_SIZE {
if self.insert_mode.flush_each_row() || self.pending_rows.len() >= SQL_INSERT_BATCH_SIZE {
self.flush_batch()?;
}
Ok(())
@@ -387,7 +394,7 @@ impl SqlInsertWriter {
spatial_columns: self.spatial_columns.clone(),
spatial_values: mem::take(&mut self.pending_spatial_values),
rows: mem::take(&mut self.pending_rows),
batch_size: Some(SQL_INSERT_BATCH_SIZE),
batch_size: Some(self.insert_mode.batch_size(SQL_INSERT_BATCH_SIZE)),
})?;
let file = self.file.as_mut().ok_or_else(|| "SQL export file already closed".to_string())?;
for stmt in &stmts {
@@ -1965,6 +1972,7 @@ mod tests {
use_agent_cursor: false,
file_path: "out.csv".to_string(),
format: format.to_string(),
insert_mode: SqlInsertMode::default(),
include_sql_sheet: false,
page_size: 1000,
row_limit,
@@ -1984,6 +1992,55 @@ mod tests {
}
}
fn rendered_sql_insert_output(mode: SqlInsertMode) -> String {
let dir = tempfile::tempdir().expect("temp dir");
let destination = dir.path().join("users.sql");
let mut req = request("sql", None, None);
req.database_type = DatabaseType::Mysql;
req.file_path = destination.to_string_lossy().into_owned();
req.export_table_name = Some("users".to_string());
req.insert_mode = mode;
let mut writer = SqlInsertWriter::create(&req).expect("create SQL writer");
writer.set_columns(
vec!["id".to_string(), "name".to_string()],
&["int".to_string(), "text".to_string()],
&[],
&req,
);
writer.write_row(vec![serde_json::json!(1), serde_json::json!("Ada")], None).expect("write first row");
writer.write_row(vec![serde_json::json!(2), serde_json::json!("Lin")], None).expect("write second row");
writer.finish().expect("finish SQL writer");
std::fs::read_to_string(destination).expect("read SQL output")
}
#[test]
fn sql_insert_mode_defaults_to_batch_when_request_field_is_missing() {
let mut serialized = serde_json::to_value(request("sql", None, None)).expect("serialize request");
serialized.as_object_mut().expect("request object").remove("insertMode");
let decoded: QueryResultExportRequest = serde_json::from_value(serialized).expect("deserialize request");
assert_eq!(decoded.insert_mode, SqlInsertMode::Batch);
}
#[test]
fn sql_insert_writer_uses_one_batched_statement_by_default() {
let output = rendered_sql_insert_output(SqlInsertMode::Batch);
assert_eq!(output.matches("INSERT INTO").count(), 1);
assert!(output.contains("VALUES (1, 'Ada'), (2, 'Lin');"));
}
#[test]
fn sql_insert_writer_emits_one_complete_statement_per_row_in_single_mode() {
let output = rendered_sql_insert_output(SqlInsertMode::Single);
assert_eq!(output.matches("INSERT INTO").count(), 2);
assert!(output.contains("VALUES (1, 'Ada');\nINSERT INTO `users` (`id`, `name`) VALUES (2, 'Lin');"));
assert!(!output.contains("), ("));
}
#[test]
fn highgo_unrewritable_query_export_reports_actionable_pagination_error() {
let mut req = request("csv", None, None);
+34 -3
View File
@@ -20,6 +20,7 @@ use crate::csv_export::{
pub use crate::database_export::ExportStatus;
use crate::database_export::{
build_export_insert_statements, is_export_cancelled, is_internal_export_column, BuildExportInsertStatementsOptions,
SqlInsertMode,
};
use crate::db::agent_driver::AgentTableReadStartParams;
use crate::models::connection::DatabaseType;
@@ -54,6 +55,8 @@ pub struct TableExportRequest {
/// "csv", "xlsx", "json", "markdown", "sql", or "txt"
pub format: String,
#[serde(default)]
pub insert_mode: SqlInsertMode,
#[serde(default)]
pub csv_quote_mode: CsvQuoteMode,
#[serde(default)]
pub columns: Option<Vec<String>>,
@@ -1268,7 +1271,7 @@ async fn try_export_native_table_stream(
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: std::mem::take(pending_rows),
batch_size: Some(SQL_INSERT_BATCH_SIZE),
batch_size: Some(request.insert_mode.batch_size(SQL_INSERT_BATCH_SIZE)),
})?;
if !statements.is_empty() {
if wrote_statements {
@@ -1290,7 +1293,7 @@ async fn try_export_native_table_stream(
cancel_token.clone(),
|row| {
pending_rows.push(row.to_vec());
if pending_rows.len() >= SQL_INSERT_BATCH_SIZE {
if request.insert_mode.flush_each_row() || pending_rows.len() >= SQL_INSERT_BATCH_SIZE {
flush_pending(&mut file, &mut pending_rows)?;
}
rows_exported += 1;
@@ -2102,7 +2105,7 @@ async fn export_table_data_core_inner(
spatial_columns: result.spatial_columns.clone(),
spatial_values: result.spatial_values.clone(),
rows: result.rows.clone(),
batch_size: Some(100),
batch_size: Some(request.insert_mode.batch_size(SQL_INSERT_BATCH_SIZE)),
})?;
if !statements.is_empty() {
if wrote_statements {
@@ -2177,6 +2180,22 @@ mod tests {
#[cfg(unix)]
use std::os::unix::fs::PermissionsExt;
#[test]
fn table_export_request_defaults_to_batch_insert_mode() {
let request: TableExportRequest = serde_json::from_value(json!({
"exportId": "export-1",
"connectionId": "conn-1",
"database": "db",
"schema": null,
"tableName": "users",
"filePath": "users.sql",
"format": "sql"
}))
.expect("deserialize table export request");
assert_eq!(request.insert_mode, SqlInsertMode::Batch);
}
#[cfg(unix)]
struct ExternalDriverExportFixture {
state: AppState,
@@ -2266,6 +2285,7 @@ mod tests {
table_name: "EXPORT_SAMPLE".to_string(),
file_path: output.to_string_lossy().into_owned(),
format: "csv".to_string(),
insert_mode: Default::default(),
columns: Some(vec!["id".to_string(), "name".to_string()]),
column_types: Some(vec![Some("INTEGER".to_string()), Some("VARCHAR".to_string())]),
primary_keys: Some(vec!["id".to_string()]),
@@ -2427,6 +2447,7 @@ mod tests {
table_name: "weather".to_string(),
file_path: output.to_string_lossy().into_owned(),
format: "csv".to_string(),
insert_mode: Default::default(),
csv_quote_mode: CsvQuoteMode::All,
columns: None,
column_types: None,
@@ -2591,6 +2612,7 @@ mod tests {
table_name: "device2".to_string(),
file_path: "device2.csv".to_string(),
format: "csv".to_string(),
insert_mode: Default::default(),
columns: None,
column_types: None,
primary_keys: None,
@@ -2648,6 +2670,7 @@ mod tests {
table_name: "device2".to_string(),
file_path: "device2.csv".to_string(),
format: "csv".to_string(),
insert_mode: Default::default(),
columns: None,
column_types: None,
primary_keys: None,
@@ -2683,6 +2706,7 @@ mod tests {
table_name: "device2".to_string(),
file_path: "device2.csv".to_string(),
format: "csv".to_string(),
insert_mode: Default::default(),
columns: None,
column_types: None,
primary_keys: None,
@@ -2713,6 +2737,7 @@ mod tests {
table_name: "device2".to_string(),
file_path: "device2.csv".to_string(),
format: "csv".to_string(),
insert_mode: Default::default(),
columns: None,
column_types: None,
primary_keys: None,
@@ -2751,6 +2776,7 @@ mod tests {
table_name: "samples".to_string(),
file_path: "samples.csv".to_string(),
format: "csv".to_string(),
insert_mode: Default::default(),
columns: None,
column_types: None,
primary_keys: None,
@@ -2801,6 +2827,7 @@ mod tests {
table_name: "events".to_string(),
file_path: "events.csv".to_string(),
format: "csv".to_string(),
insert_mode: Default::default(),
columns: None,
column_types: None,
primary_keys: None,
@@ -2845,6 +2872,7 @@ mod tests {
table_name: "orders".to_string(),
file_path: "orders.txt".to_string(),
format: "txt".to_string(),
insert_mode: Default::default(),
columns: None,
column_types: None,
primary_keys: None,
@@ -2917,6 +2945,7 @@ mod tests {
table_name: "order".to_string(),
file_path: "order.csv".to_string(),
format: "csv".to_string(),
insert_mode: Default::default(),
columns: None,
column_types: None,
primary_keys: None,
@@ -2971,6 +3000,7 @@ mod tests {
table_name: "spatial_data".to_string(),
file_path: "spatial_data.sql".to_string(),
format: "sql".to_string(),
insert_mode: Default::default(),
columns: None,
column_types: None,
primary_keys: None,
@@ -3027,6 +3057,7 @@ mod tests {
table_name: "USERS".to_string(),
file_path: "users.sql".to_string(),
format: "sql".to_string(),
insert_mode: Default::default(),
columns: None,
column_types: None,
primary_keys: None,
@@ -92,6 +92,7 @@ async fn live_clickhouse_query_result_export_xlsx_streams_random_order_query_onc
use_agent_cursor: false,
file_path: file_path.to_string_lossy().to_string(),
format: "xlsx".to_string(),
insert_mode: Default::default(),
include_sql_sheet: false,
page_size: 100,
row_limit: None,
+2
View File
@@ -465,6 +465,7 @@ async fn live_mysql_query_result_export_xlsx_streams_single_query_without_duplic
use_agent_cursor: false,
file_path: file_path.to_string_lossy().to_string(),
format: "xlsx".to_string(),
insert_mode: Default::default(),
include_sql_sheet: false,
page_size: 50,
row_limit: None,
@@ -558,6 +559,7 @@ async fn live_mysql_xlsx_export_can_outlive_query_timeout_while_rows_keep_arrivi
use_agent_cursor: false,
file_path: file_path.to_string_lossy().to_string(),
format: "xlsx".to_string(),
insert_mode: Default::default(),
include_sql_sheet: false,
page_size: 10_000,
row_limit: None,
@@ -134,6 +134,7 @@ async fn live_postgres_query_result_export_uses_single_streamed_query() {
use_agent_cursor: false,
file_path: file_path.to_string_lossy().to_string(),
format: "csv".to_string(),
insert_mode: Default::default(),
include_sql_sheet: false,
page_size: 100,
row_limit: None,
@@ -217,6 +218,7 @@ async fn live_postgres_query_result_xlsx_preserves_temporal_cell_types() {
use_agent_cursor: false,
file_path: file_path.to_string_lossy().to_string(),
format: "xlsx".to_string(),
insert_mode: Default::default(),
include_sql_sheet: false,
page_size: 100,
row_limit: None,
@@ -289,6 +291,7 @@ async fn live_postgres_numeric_xlsx_ignores_fractional_trailing_zeros() {
use_agent_cursor: false,
file_path: file_path.to_string_lossy().to_string(),
format: "xlsx".to_string(),
insert_mode: Default::default(),
include_sql_sheet: false,
page_size: 100,
row_limit: None,
@@ -366,6 +369,7 @@ async fn live_postgres_truncated_batch_result_export_replays_safe_temp_setup() {
use_agent_cursor: false,
file_path: dir.join("result.csv").to_string_lossy().to_string(),
format: "csv".to_string(),
insert_mode: Default::default(),
include_sql_sheet: false,
page_size: 2000,
row_limit: None,
@@ -445,6 +449,7 @@ async fn live_postgres_xlsx_export_can_outlive_query_timeout_while_rows_keep_arr
use_agent_cursor: false,
file_path: file_path.to_string_lossy().to_string(),
format: "xlsx".to_string(),
insert_mode: Default::default(),
include_sql_sheet: false,
page_size: 10_000,
row_limit: None,
@@ -515,6 +520,7 @@ async fn live_postgres_stream_still_times_out_without_progress_and_recovers() {
use_agent_cursor: false,
file_path: file_path.to_string_lossy().to_string(),
format: "csv".to_string(),
insert_mode: Default::default(),
include_sql_sheet: false,
page_size: 100,
row_limit: None,
@@ -1196,6 +1196,7 @@ async fn live_sqlserver_query_result_export_streams_cte_query_to_csv() {
use_agent_cursor: false,
file_path: file_path.to_string_lossy().to_string(),
format: "csv".to_string(),
insert_mode: Default::default(),
include_sql_sheet: false,
page_size: 1,
row_limit: None,
@@ -113,6 +113,7 @@ async fn live_sqlserver_xlsx_export_can_outlive_query_timeout_while_rows_keep_ar
use_agent_cursor: false,
file_path: file_path.to_string_lossy().to_string(),
format: "xlsx".to_string(),
insert_mode: Default::default(),
include_sql_sheet: false,
page_size: 5000,
row_limit: Some(200_000),