feat(export): add SQL INSERT batch or single-row mode
This commit is contained in:
1 parent
5d65be03a3
commit
ab35a5b1da
29 files changed
+563
-23
No files matched your search
@@ -42,6 +42,28 @@ pub enum DatabaseExportOutputCompression {
|
||||
Gzip,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum SqlInsertMode {
|
||||
#[default]
|
||||
Batch,
|
||||
Single,
|
||||
}
|
||||
|
||||
impl SqlInsertMode {
|
||||
pub(crate) const fn flush_each_row(self) -> bool {
|
||||
matches!(self, Self::Single)
|
||||
}
|
||||
|
||||
pub(crate) const fn batch_size(self, default: usize) -> usize {
|
||||
if self.flush_each_row() || default == 0 {
|
||||
1
|
||||
} else {
|
||||
default
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DatabaseExportRequest {
|
||||
|
||||
@@ -15,7 +15,9 @@ use crate::csv_export::{
|
||||
CsvQuoteMode,
|
||||
};
|
||||
pub use crate::database_export::ExportStatus;
|
||||
use crate::database_export::{build_export_insert_statements, is_export_cancelled, BuildExportInsertStatementsOptions};
|
||||
use crate::database_export::{
|
||||
build_export_insert_statements, is_export_cancelled, BuildExportInsertStatementsOptions, SqlInsertMode,
|
||||
};
|
||||
use crate::models::connection::DatabaseType;
|
||||
use crate::query::{
|
||||
await_stream_with_progress_timeout, canceled_error, close_query_session, execute_sql_statement_with_options,
|
||||
@@ -82,6 +84,8 @@ pub struct QueryResultExportRequest {
|
||||
pub file_path: String,
|
||||
pub format: String,
|
||||
#[serde(default)]
|
||||
pub insert_mode: SqlInsertMode,
|
||||
#[serde(default)]
|
||||
pub csv_quote_mode: CsvQuoteMode,
|
||||
#[serde(default)]
|
||||
pub include_sql_sheet: bool,
|
||||
@@ -300,8 +304,9 @@ fn sql_insert_column_types(request: &QueryResultExportRequest, column_types: &[S
|
||||
|
||||
/// Bounded SQL INSERT writer with staged-file replacement safety.
|
||||
///
|
||||
/// Rows are buffered and flushed to a temp file every [`SQL_INSERT_BATCH_SIZE`]
|
||||
/// rows, so memory stays bounded regardless of the query page size. The unique
|
||||
/// Batch mode buffers rows up to [`SQL_INSERT_BATCH_SIZE`], while single mode
|
||||
/// flushes each row immediately. Both modes keep memory bounded regardless of
|
||||
/// the query page size. The unique
|
||||
/// temp file lives alongside the target and replaces it only after
|
||||
/// [`SqlInsertWriter::finish`] flushes and synchronizes the complete output.
|
||||
struct SqlInsertWriter {
|
||||
@@ -309,6 +314,7 @@ struct SqlInsertWriter {
|
||||
target: Option<StagedExportTarget>,
|
||||
pending_rows: Vec<Vec<Value>>,
|
||||
pending_spatial_values: Vec<Vec<Option<u32>>>,
|
||||
insert_mode: SqlInsertMode,
|
||||
columns: Vec<String>,
|
||||
column_types: Vec<Option<String>>,
|
||||
spatial_columns: Vec<SpatialColumn>,
|
||||
@@ -337,6 +343,7 @@ impl SqlInsertWriter {
|
||||
target: Some(target),
|
||||
pending_rows: Vec::new(),
|
||||
pending_spatial_values: Vec::new(),
|
||||
insert_mode: request.insert_mode,
|
||||
columns: Vec::new(),
|
||||
column_types: Vec::new(),
|
||||
spatial_columns: Vec::new(),
|
||||
@@ -365,7 +372,7 @@ impl SqlInsertWriter {
|
||||
fn write_row(&mut self, row: Vec<Value>, spatial_values: Option<Vec<Option<u32>>>) -> Result<(), String> {
|
||||
self.pending_rows.push(row);
|
||||
self.pending_spatial_values.push(spatial_values.unwrap_or_default());
|
||||
if self.pending_rows.len() >= SQL_INSERT_BATCH_SIZE {
|
||||
if self.insert_mode.flush_each_row() || self.pending_rows.len() >= SQL_INSERT_BATCH_SIZE {
|
||||
self.flush_batch()?;
|
||||
}
|
||||
Ok(())
|
||||
@@ -387,7 +394,7 @@ impl SqlInsertWriter {
|
||||
spatial_columns: self.spatial_columns.clone(),
|
||||
spatial_values: mem::take(&mut self.pending_spatial_values),
|
||||
rows: mem::take(&mut self.pending_rows),
|
||||
batch_size: Some(SQL_INSERT_BATCH_SIZE),
|
||||
batch_size: Some(self.insert_mode.batch_size(SQL_INSERT_BATCH_SIZE)),
|
||||
})?;
|
||||
let file = self.file.as_mut().ok_or_else(|| "SQL export file already closed".to_string())?;
|
||||
for stmt in &stmts {
|
||||
@@ -1965,6 +1972,7 @@ mod tests {
|
||||
use_agent_cursor: false,
|
||||
file_path: "out.csv".to_string(),
|
||||
format: format.to_string(),
|
||||
insert_mode: SqlInsertMode::default(),
|
||||
include_sql_sheet: false,
|
||||
page_size: 1000,
|
||||
row_limit,
|
||||
@@ -1984,6 +1992,55 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
fn rendered_sql_insert_output(mode: SqlInsertMode) -> String {
|
||||
let dir = tempfile::tempdir().expect("temp dir");
|
||||
let destination = dir.path().join("users.sql");
|
||||
let mut req = request("sql", None, None);
|
||||
req.database_type = DatabaseType::Mysql;
|
||||
req.file_path = destination.to_string_lossy().into_owned();
|
||||
req.export_table_name = Some("users".to_string());
|
||||
req.insert_mode = mode;
|
||||
|
||||
let mut writer = SqlInsertWriter::create(&req).expect("create SQL writer");
|
||||
writer.set_columns(
|
||||
vec!["id".to_string(), "name".to_string()],
|
||||
&["int".to_string(), "text".to_string()],
|
||||
&[],
|
||||
&req,
|
||||
);
|
||||
writer.write_row(vec![serde_json::json!(1), serde_json::json!("Ada")], None).expect("write first row");
|
||||
writer.write_row(vec![serde_json::json!(2), serde_json::json!("Lin")], None).expect("write second row");
|
||||
writer.finish().expect("finish SQL writer");
|
||||
|
||||
std::fs::read_to_string(destination).expect("read SQL output")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sql_insert_mode_defaults_to_batch_when_request_field_is_missing() {
|
||||
let mut serialized = serde_json::to_value(request("sql", None, None)).expect("serialize request");
|
||||
serialized.as_object_mut().expect("request object").remove("insertMode");
|
||||
let decoded: QueryResultExportRequest = serde_json::from_value(serialized).expect("deserialize request");
|
||||
|
||||
assert_eq!(decoded.insert_mode, SqlInsertMode::Batch);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sql_insert_writer_uses_one_batched_statement_by_default() {
|
||||
let output = rendered_sql_insert_output(SqlInsertMode::Batch);
|
||||
|
||||
assert_eq!(output.matches("INSERT INTO").count(), 1);
|
||||
assert!(output.contains("VALUES (1, 'Ada'), (2, 'Lin');"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sql_insert_writer_emits_one_complete_statement_per_row_in_single_mode() {
|
||||
let output = rendered_sql_insert_output(SqlInsertMode::Single);
|
||||
|
||||
assert_eq!(output.matches("INSERT INTO").count(), 2);
|
||||
assert!(output.contains("VALUES (1, 'Ada');\nINSERT INTO `users` (`id`, `name`) VALUES (2, 'Lin');"));
|
||||
assert!(!output.contains("), ("));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn highgo_unrewritable_query_export_reports_actionable_pagination_error() {
|
||||
let mut req = request("csv", None, None);
|
||||
|
||||
@@ -20,6 +20,7 @@ use crate::csv_export::{
|
||||
pub use crate::database_export::ExportStatus;
|
||||
use crate::database_export::{
|
||||
build_export_insert_statements, is_export_cancelled, is_internal_export_column, BuildExportInsertStatementsOptions,
|
||||
SqlInsertMode,
|
||||
};
|
||||
use crate::db::agent_driver::AgentTableReadStartParams;
|
||||
use crate::models::connection::DatabaseType;
|
||||
@@ -54,6 +55,8 @@ pub struct TableExportRequest {
|
||||
/// "csv", "xlsx", "json", "markdown", "sql", or "txt"
|
||||
pub format: String,
|
||||
#[serde(default)]
|
||||
pub insert_mode: SqlInsertMode,
|
||||
#[serde(default)]
|
||||
pub csv_quote_mode: CsvQuoteMode,
|
||||
#[serde(default)]
|
||||
pub columns: Option<Vec<String>>,
|
||||
@@ -1268,7 +1271,7 @@ async fn try_export_native_table_stream(
|
||||
spatial_columns: Vec::new(),
|
||||
spatial_values: Vec::new(),
|
||||
rows: std::mem::take(pending_rows),
|
||||
batch_size: Some(SQL_INSERT_BATCH_SIZE),
|
||||
batch_size: Some(request.insert_mode.batch_size(SQL_INSERT_BATCH_SIZE)),
|
||||
})?;
|
||||
if !statements.is_empty() {
|
||||
if wrote_statements {
|
||||
@@ -1290,7 +1293,7 @@ async fn try_export_native_table_stream(
|
||||
cancel_token.clone(),
|
||||
|row| {
|
||||
pending_rows.push(row.to_vec());
|
||||
if pending_rows.len() >= SQL_INSERT_BATCH_SIZE {
|
||||
if request.insert_mode.flush_each_row() || pending_rows.len() >= SQL_INSERT_BATCH_SIZE {
|
||||
flush_pending(&mut file, &mut pending_rows)?;
|
||||
}
|
||||
rows_exported += 1;
|
||||
@@ -2102,7 +2105,7 @@ async fn export_table_data_core_inner(
|
||||
spatial_columns: result.spatial_columns.clone(),
|
||||
spatial_values: result.spatial_values.clone(),
|
||||
rows: result.rows.clone(),
|
||||
batch_size: Some(100),
|
||||
batch_size: Some(request.insert_mode.batch_size(SQL_INSERT_BATCH_SIZE)),
|
||||
})?;
|
||||
if !statements.is_empty() {
|
||||
if wrote_statements {
|
||||
@@ -2177,6 +2180,22 @@ mod tests {
|
||||
#[cfg(unix)]
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
#[test]
|
||||
fn table_export_request_defaults_to_batch_insert_mode() {
|
||||
let request: TableExportRequest = serde_json::from_value(json!({
|
||||
"exportId": "export-1",
|
||||
"connectionId": "conn-1",
|
||||
"database": "db",
|
||||
"schema": null,
|
||||
"tableName": "users",
|
||||
"filePath": "users.sql",
|
||||
"format": "sql"
|
||||
}))
|
||||
.expect("deserialize table export request");
|
||||
|
||||
assert_eq!(request.insert_mode, SqlInsertMode::Batch);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
struct ExternalDriverExportFixture {
|
||||
state: AppState,
|
||||
@@ -2266,6 +2285,7 @@ mod tests {
|
||||
table_name: "EXPORT_SAMPLE".to_string(),
|
||||
file_path: output.to_string_lossy().into_owned(),
|
||||
format: "csv".to_string(),
|
||||
insert_mode: Default::default(),
|
||||
columns: Some(vec!["id".to_string(), "name".to_string()]),
|
||||
column_types: Some(vec![Some("INTEGER".to_string()), Some("VARCHAR".to_string())]),
|
||||
primary_keys: Some(vec!["id".to_string()]),
|
||||
@@ -2427,6 +2447,7 @@ mod tests {
|
||||
table_name: "weather".to_string(),
|
||||
file_path: output.to_string_lossy().into_owned(),
|
||||
format: "csv".to_string(),
|
||||
insert_mode: Default::default(),
|
||||
csv_quote_mode: CsvQuoteMode::All,
|
||||
columns: None,
|
||||
column_types: None,
|
||||
@@ -2591,6 +2612,7 @@ mod tests {
|
||||
table_name: "device2".to_string(),
|
||||
file_path: "device2.csv".to_string(),
|
||||
format: "csv".to_string(),
|
||||
insert_mode: Default::default(),
|
||||
columns: None,
|
||||
column_types: None,
|
||||
primary_keys: None,
|
||||
@@ -2648,6 +2670,7 @@ mod tests {
|
||||
table_name: "device2".to_string(),
|
||||
file_path: "device2.csv".to_string(),
|
||||
format: "csv".to_string(),
|
||||
insert_mode: Default::default(),
|
||||
columns: None,
|
||||
column_types: None,
|
||||
primary_keys: None,
|
||||
@@ -2683,6 +2706,7 @@ mod tests {
|
||||
table_name: "device2".to_string(),
|
||||
file_path: "device2.csv".to_string(),
|
||||
format: "csv".to_string(),
|
||||
insert_mode: Default::default(),
|
||||
columns: None,
|
||||
column_types: None,
|
||||
primary_keys: None,
|
||||
@@ -2713,6 +2737,7 @@ mod tests {
|
||||
table_name: "device2".to_string(),
|
||||
file_path: "device2.csv".to_string(),
|
||||
format: "csv".to_string(),
|
||||
insert_mode: Default::default(),
|
||||
columns: None,
|
||||
column_types: None,
|
||||
primary_keys: None,
|
||||
@@ -2751,6 +2776,7 @@ mod tests {
|
||||
table_name: "samples".to_string(),
|
||||
file_path: "samples.csv".to_string(),
|
||||
format: "csv".to_string(),
|
||||
insert_mode: Default::default(),
|
||||
columns: None,
|
||||
column_types: None,
|
||||
primary_keys: None,
|
||||
@@ -2801,6 +2827,7 @@ mod tests {
|
||||
table_name: "events".to_string(),
|
||||
file_path: "events.csv".to_string(),
|
||||
format: "csv".to_string(),
|
||||
insert_mode: Default::default(),
|
||||
columns: None,
|
||||
column_types: None,
|
||||
primary_keys: None,
|
||||
@@ -2845,6 +2872,7 @@ mod tests {
|
||||
table_name: "orders".to_string(),
|
||||
file_path: "orders.txt".to_string(),
|
||||
format: "txt".to_string(),
|
||||
insert_mode: Default::default(),
|
||||
columns: None,
|
||||
column_types: None,
|
||||
primary_keys: None,
|
||||
@@ -2917,6 +2945,7 @@ mod tests {
|
||||
table_name: "order".to_string(),
|
||||
file_path: "order.csv".to_string(),
|
||||
format: "csv".to_string(),
|
||||
insert_mode: Default::default(),
|
||||
columns: None,
|
||||
column_types: None,
|
||||
primary_keys: None,
|
||||
@@ -2971,6 +3000,7 @@ mod tests {
|
||||
table_name: "spatial_data".to_string(),
|
||||
file_path: "spatial_data.sql".to_string(),
|
||||
format: "sql".to_string(),
|
||||
insert_mode: Default::default(),
|
||||
columns: None,
|
||||
column_types: None,
|
||||
primary_keys: None,
|
||||
@@ -3027,6 +3057,7 @@ mod tests {
|
||||
table_name: "USERS".to_string(),
|
||||
file_path: "users.sql".to_string(),
|
||||
format: "sql".to_string(),
|
||||
insert_mode: Default::default(),
|
||||
columns: None,
|
||||
column_types: None,
|
||||
primary_keys: None,
|
||||
|
||||
Reference in new issue
Block a user