perf(export): optimize SQL export performance

This commit is contained in:
miracle authored and GitHub committed 2026-08-05 14:57:06 +08:00
1 parent 7fd4aef060
commit 605d20dd4f
14 files changed
+665 -293

No files matched your search

+173 -79
View File
@@ -1,7 +1,7 @@
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::collections::{HashMap, HashSet};
use std::io::Write;
use std::io::{BufWriter, Write};
use std::sync::RwLock;
use crate::connection::task_client_session_id;
@@ -11,9 +11,8 @@ use crate::object_source_sql::build_export_object_source_sql;
use crate::sql_dialect::{qualified_table_name, quote_table_identifier, uses_single_row_insert_statements};
use crate::transfer::{
format_ch_array_sql_literal, format_pg_array_sql_literal, is_identity_column_extra,
is_mysql_generated_column_extra, quote_identifier, quote_postgres_string_literal,
selected_columns_include_identity_extras, wrap_dameng_identity_insert_sql,
wrap_dameng_identity_insert_sql_for_table,
is_mysql_generated_column_extra, keyset_pagination_sql_with_identifier_quote, quote_identifier,
quote_postgres_string_literal, wrap_dameng_identity_insert_sql_for_table,
};
use crate::types::ObjectSourceKind;
@@ -231,6 +230,7 @@ pub enum ExportStatus {
pub const DATABASE_EXPORT_ROW_LIMIT: usize = 10_000;
pub const DATABASE_EXPORT_PAGE_SIZE: usize = 500;
pub const DATABASE_EXPORT_INSERT_BATCH_SIZE: usize = 100;
pub const DATABASE_EXPORT_TARGET_STATEMENT_BYTES: usize = 512 * 1024;
#[derive(Debug, Clone, PartialEq, Eq)]
struct PostgresExportSequence {
@@ -859,7 +859,12 @@ pub fn build_export_insert_statements(options: BuildExportInsertStatementsOption
let batch_size = if options.database_type.is_some_and(uses_single_row_insert_statements) {
1
} else {
options.batch_size.unwrap_or(DATABASE_EXPORT_INSERT_BATCH_SIZE).max(1)
let requested = options.batch_size.unwrap_or(DATABASE_EXPORT_INSERT_BATCH_SIZE).max(1);
if options.database_type == Some(DatabaseType::SqlServer) {
requested.min(1000)
} else {
requested
}
};
let columns = insert_columns
.iter()
@@ -872,37 +877,69 @@ pub fn build_export_insert_statements(options: BuildExportInsertStatementsOption
is_identity_column_extra(options.column_extras.get(*index).and_then(|value| value.as_deref()))
});
for rows in options.rows.chunks(batch_size) {
let values = rows
.iter()
.map(|row| {
let values = insert_columns
.iter()
.map(|(index, _)| {
let value = row.get(*index).unwrap_or(&Value::Null);
format_export_sql_literal_typed(
value,
options.database_type,
options.column_types.get(*index).and_then(|value| value.as_deref()),
)
})
.collect::<Vec<_>>()
.join(", ");
format!("({values})")
})
.collect::<Vec<_>>()
.join(", ");
let insert_sql = format!("INSERT INTO {table} ({columns}) VALUES {values};");
let statement_prefix = format!("INSERT INTO {table} ({columns}) VALUES ");
let statement_overhead_bytes = export_sql_statement_bytes(options.database_type, &statement_prefix) + 1;
let target_statement_bytes = DATABASE_EXPORT_TARGET_STATEMENT_BYTES;
let separator_bytes = export_sql_statement_bytes(options.database_type, ", ");
let mut current_values = Vec::with_capacity(batch_size);
let mut current_values_bytes = 0usize;
let flush_values = |statements: &mut Vec<String>, values: &mut Vec<String>, values_bytes: &mut usize| {
if values.is_empty() {
return;
}
let insert_sql = format!("{statement_prefix}{};", values.join(", "));
if needs_dameng_identity_insert {
statements.push(wrap_dameng_identity_insert_sql_for_table(&insert_sql, &table));
} else {
statements.push(insert_sql);
}
values.clear();
*values_bytes = 0;
};
for row in options.rows {
let row_values = insert_columns
.iter()
.map(|(index, _)| {
let value = row.get(*index).unwrap_or(&Value::Null);
format_export_sql_literal_typed(
value,
options.database_type,
options.column_types.get(*index).and_then(|value| value.as_deref()),
)
})
.collect::<Vec<_>>()
.join(", ");
let rendered_row = format!("({row_values})");
let rendered_row_bytes = export_sql_statement_bytes(options.database_type, &rendered_row);
let candidate_bytes = statement_overhead_bytes
+ current_values_bytes
+ if current_values.is_empty() { 0 } else { separator_bytes }
+ rendered_row_bytes;
if !current_values.is_empty()
&& (current_values.len() >= batch_size || candidate_bytes > target_statement_bytes)
{
flush_values(&mut statements, &mut current_values, &mut current_values_bytes);
}
current_values_bytes += if current_values.is_empty() { 0 } else { separator_bytes };
current_values_bytes += rendered_row_bytes;
current_values.push(rendered_row);
}
flush_values(&mut statements, &mut current_values, &mut current_values_bytes);
Ok(statements)
}
fn export_sql_statement_bytes(database_type: Option<DatabaseType>, text: &str) -> usize {
if database_type == Some(DatabaseType::SqlServer) {
text.encode_utf16().count() * 2
} else {
text.len()
}
}
pub(crate) fn is_internal_export_column(database_type: Option<DatabaseType>, column: &str) -> bool {
// Oracle-compatible ROWID is injected only to identify editable rows. It
// is not a physical table column and must never propagate into exports.
@@ -1331,7 +1368,7 @@ fn database_export_metadata_prefetch_concurrency(db_type: DatabaseType) -> usize
}
}
fn record_export_error(file: &mut std::fs::File, fail_on_error: bool, message: String) -> Result<(), String> {
fn record_export_error<W: Write>(file: &mut W, fail_on_error: bool, message: String) -> Result<(), String> {
if fail_on_error {
Err(message)
} else {
@@ -1345,8 +1382,8 @@ fn database_export_select_sql(columns: &[String], table: &str, schema: &str, db_
format!("SELECT {columns} FROM {table}")
}
fn write_database_export_rows(
file: &mut std::fs::File,
fn write_database_export_rows<W: Write>(
file: &mut W,
rows: &[Vec<Value>],
columns: &[String],
column_types: &[Option<String>],
@@ -1396,28 +1433,25 @@ fn write_database_export_rows(
filtered_rows.as_slice(),
)
};
let mut insert_sql = crate::transfer::generate_insert_typed(
insert_columns,
insert_column_types,
insert_rows,
table,
schema,
db_type,
None,
);
if *db_type == DatabaseType::Dameng
&& selected_columns_include_identity_extras(insert_columns, insert_column_extras)
{
insert_sql = wrap_dameng_identity_insert_sql(&insert_sql, table, schema);
}
if insert_sql.is_empty() {
return Ok(());
}
if insert_sql.trim_end().ends_with(';') {
writeln!(file, "{}\n", insert_sql).map_err(|error| format!("Failed to write file: {error}"))
} else {
writeln!(file, "{};\n", insert_sql).map_err(|error| format!("Failed to write file: {error}"))
// Database exports use the same table qualification as the SELECT/DDL
// path and the legacy typed INSERT writer. In particular, MySQL uses the
// selected database rather than a schema-qualified table name.
let qualified_table_name = crate::transfer::qualified_table(table, schema, db_type, None);
let statements = build_export_insert_statements(BuildExportInsertStatementsOptions {
database_type: Some(*db_type),
schema: (!schema.is_empty()).then(|| schema.to_string()),
table_name: Some(table.to_string()),
qualified_table_name: Some(qualified_table_name),
columns: insert_columns.to_vec(),
column_types: insert_column_types.to_vec(),
column_extras: insert_column_extras.to_vec(),
rows: insert_rows.to_vec(),
batch_size: Some(DATABASE_EXPORT_INSERT_BATCH_SIZE),
})?;
for statement in statements {
writeln!(file, "{statement}\n").map_err(|error| format!("Failed to write file: {error}"))?;
}
Ok(())
}
fn emit_database_export_running(
@@ -1479,7 +1513,8 @@ pub async fn export_database_sql_core(
)
.await?;
// 4. Create file
let mut file = std::fs::File::create(&request.file_path).map_err(|e| format!("Failed to write file: {e}"))?;
let file = std::fs::File::create(&request.file_path).map_err(|e| format!("Failed to write file: {e}"))?;
let mut file = BufWriter::new(file);
let create_database_preamble = if request.include_create_database && matches!(db_type, DatabaseType::Mysql) {
Some(mysql_database_export_preamble_for_request(state, request).await)
@@ -1962,24 +1997,21 @@ pub async fn export_database_sql_core(
)
.await?;
} else {
let count_query = crate::transfer::count_sql(table_name, &request.schema, &db_type, None);
let total_rows = match crate::transfer::execute_read_on_pool(state, &pool_key, &count_query).await {
Ok(result) => {
let count = result.rows.first().and_then(|row| row.first()).and_then(|value| match value {
serde_json::Value::Number(number) => number.as_u64(),
serde_json::Value::String(text) => text.parse::<u64>().ok(),
_ => None,
});
if request.fail_on_error && count.is_none() {
return Err(format!("Failed to read row count for table {table_name}"));
}
count
}
Err(error) if request.fail_on_error => {
return Err(format!("Failed to read row count for table {table_name}: {error}"));
}
Err(_) => None,
};
// Exact COUNT(*) is deliberately skipped for manual database exports. It adds a full
// table scan before the real read and does not improve correctness. Progress reports
// exported rows while total_rows remains indeterminate.
let total_rows = None;
let primary_keys = columns
.iter()
.filter(|column| column.is_primary_key)
.map(|column| column.name.clone())
.collect::<Vec<_>>();
let primary_key_indices = primary_keys
.iter()
.filter_map(|primary_key| col_names.iter().position(|column| column == primary_key))
.collect::<Vec<_>>();
let use_keyset = !primary_keys.is_empty() && primary_key_indices.len() == primary_keys.len();
let mut last_primary_key_values = Vec::new();
let mut offset = 0_u64;
loop {
@@ -1998,14 +2030,27 @@ pub async fn export_database_sql_core(
return Ok(());
}
let sql = crate::transfer::pagination_sql(
&col_names,
table_name,
&request.schema,
&db_type,
offset,
batch_size,
);
let sql = if use_keyset {
keyset_pagination_sql_with_identifier_quote(
&col_names,
table_name,
&request.schema,
&db_type,
&primary_keys,
&last_primary_key_values,
batch_size,
None,
)
} else {
crate::transfer::pagination_sql(
&col_names,
table_name,
&request.schema,
&db_type,
offset,
batch_size,
)
};
let result = match crate::transfer::execute_read_on_pool(state, &pool_key, &sql).await {
Ok(result) => result,
Err(error) => {
@@ -2032,7 +2077,14 @@ pub async fn export_database_sql_core(
&db_type,
)?;
total_rows_exported += row_count as u64;
offset += row_count as u64;
if use_keyset {
if let Some(last_row) = result.rows.last() {
last_primary_key_values =
primary_key_indices.iter().map(|&index| last_row[index].clone()).collect();
}
} else {
offset += row_count as u64;
}
on_progress(ExportProgress {
export_id: request.export_id.clone(),
current_object: table_name.clone(),
@@ -2239,6 +2291,8 @@ pub async fn export_database_sql_core(
writeln!(file, "SET FOREIGN_KEY_CHECKS = 1;").map_err(|e| format!("Failed to write file: {e}"))?;
}
file.flush().map_err(|e| format!("Failed to finalize export file: {e}"))?;
// Emit Done progress
on_progress(ExportProgress {
export_id: request.export_id.clone(),
@@ -2670,6 +2724,46 @@ mod tests {
);
}
#[test]
fn export_insert_batches_split_on_statement_bytes() {
let long_value = "x".repeat(300_000);
let statements = build_export_insert_statements(BuildExportInsertStatementsOptions {
database_type: Some(DatabaseType::Mysql),
schema: None,
table_name: Some("payloads".to_string()),
qualified_table_name: None,
columns: vec!["payload".to_string()],
column_types: vec![Some("longtext".to_string())],
column_extras: Vec::new(),
rows: vec![vec![json!(long_value.clone())], vec![json!(long_value)]],
batch_size: Some(100),
})
.unwrap();
assert_eq!(statements.len(), 2);
assert!(statements.iter().all(|statement| statement.matches("INSERT INTO").count() == 1));
}
#[test]
fn sqlserver_export_caps_multi_row_insert_at_1000_rows() {
let statements = build_export_insert_statements(BuildExportInsertStatementsOptions {
database_type: Some(DatabaseType::SqlServer),
schema: Some("dbo".to_string()),
table_name: Some("items".to_string()),
qualified_table_name: None,
columns: vec!["id".to_string()],
column_types: vec![Some("int".to_string())],
column_extras: Vec::new(),
rows: (0..1001).map(|id| vec![json!(id)]).collect(),
batch_size: Some(2000),
})
.unwrap();
assert_eq!(statements.len(), 2);
assert_eq!(statements[0].matches("), (").count(), 999);
assert_eq!(statements[1].matches("), (").count(), 0);
}
#[test]
fn doris_export_inserts_escape_control_characters() {
let statements = build_export_insert_statements(BuildExportInsertStatementsOptions {
@@ -3215,7 +3309,7 @@ mod tests {
assert_eq!(
std::fs::read_to_string(path).unwrap(),
"INSERT INTO `orders` (`id`, `quantity`, `created_at`) VALUES\n(7, 2, '2026-07-30 08:00:00');\n\n"
"INSERT INTO `orders` (`id`, `quantity`, `created_at`) VALUES (7, 2, '2026-07-30 08:00:00');\n\n"
);
}
+4 -2
View File
@@ -1292,7 +1292,8 @@ pub(crate) fn is_mysql_generated_column_extra(extra: Option<&str>) -> bool {
})
}
pub(crate) fn selected_columns_include_identity_extras(columns: &[String], column_extras: &[Option<String>]) -> bool {
#[cfg(test)]
fn selected_columns_include_identity_extras(columns: &[String], column_extras: &[Option<String>]) -> bool {
columns
.iter()
.enumerate()
@@ -1344,7 +1345,8 @@ fn dameng_identity_insert_statement(table: &str, schema: &str, enabled: bool) ->
format!("SET IDENTITY_INSERT {full_table} {}", if enabled { "ON" } else { "OFF" })
}
pub(crate) fn wrap_dameng_identity_insert_sql(insert_sql: &str, table: &str, schema: &str) -> String {
#[cfg(test)]
fn wrap_dameng_identity_insert_sql(insert_sql: &str, table: &str, schema: &str) -> String {
let full_table = qualified_table(table, schema, &DatabaseType::Dameng, None);
wrap_dameng_identity_insert_sql_for_table(insert_sql, &full_table)
}