diff --git a/crates/dbx-core/examples/data_transfer_bench.rs b/crates/dbx-core/examples/data_transfer_bench.rs new file mode 100644 index 0000000000..4a96f0a9f1 --- /dev/null +++ b/crates/dbx-core/examples/data_transfer_bench.rs @@ -0,0 +1,154 @@ +use std::time::Instant; + +use dbx_core::csv_export::format_query_result_csv_rows; +use dbx_core::database_export::{build_export_insert_statements, BuildExportInsertStatementsOptions}; +use dbx_core::models::connection::DatabaseType; +use dbx_core::table_import::{build_import_insert_batches, ParsedImportFile, TableImportColumnMapping}; +use dbx_core::xlsx_export::{build_xlsx_workbook, XlsxWorksheetData}; + +struct Options { + rows: usize, + columns: usize, + batch_size: usize, +} + +fn parse_options() -> Result { + let mut options = Options { rows: 20_000, columns: 10, batch_size: 500 }; + for argument in std::env::args().skip(1) { + let (key, value) = argument.split_once('=').ok_or_else(|| format!("Invalid option: {argument}"))?; + match key { + "--rows" => options.rows = value.parse().map_err(|_| format!("Invalid row count: {value}"))?, + "--columns" => options.columns = value.parse().map_err(|_| format!("Invalid column count: {value}"))?, + "--batch-size" => options.batch_size = value.parse().map_err(|_| format!("Invalid batch size: {value}"))?, + _ => return Err(format!("Unknown option: {key}")), + } + } + if options.rows == 0 || options.columns == 0 || options.batch_size == 0 { + return Err("rows, columns, and batch-size must be greater than zero".to_string()); + } + Ok(options) +} + +fn columns(count: usize) -> Vec { + (0..count).map(|index| format!("column_{}", index + 1)).collect() +} + +fn rows(row_count: usize, column_count: usize) -> Vec> { + (0..row_count) + .map(|row_index| { + (0..column_count) + .map(|column_index| { + if column_index == 0 { + serde_json::json!(row_index + 1) + } else if column_index % 3 == 0 { + serde_json::json!(format!("2026-07-{:02} 12:34:56", row_index % 28 + 1)) + } else { + serde_json::json!(format!("value-{row_index}-{column_index}")) + } + }) + .collect() + }) + .collect() +} + +fn elapsed_ms(started_at: Instant) -> f64 { + started_at.elapsed().as_secs_f64() * 1000.0 +} + +fn run() -> Result<(), String> { + let options = parse_options()?; + let columns = columns(options.columns); + let rows = rows(options.rows, options.columns); + let mappings = columns + .iter() + .map(|column| TableImportColumnMapping { + source_column: column.clone(), + target_column: column.clone(), + target_data_type: None, + }) + .collect::>(); + let parsed = ParsedImportFile { + columns: columns.clone(), + rows: rows.clone(), + total_rows: rows.len(), + effective_encoding: None, + }; + + let import_started = Instant::now(); + let import_batches = build_import_insert_batches( + &parsed, + &mappings, + &[], + "benchmark_import", + "main", + &DatabaseType::Sqlite, + options.batch_size, + )?; + let import_ms = elapsed_ms(import_started); + + let csv_started = Instant::now(); + let csv = format_query_result_csv_rows(&rows); + let csv_ms = elapsed_ms(csv_started); + + let sql_started = Instant::now(); + let sql = build_export_insert_statements(BuildExportInsertStatementsOptions { + database_type: Some(DatabaseType::Sqlite), + identifier_quote: None, + schema: Some("main".to_string()), + table_name: Some("benchmark_export".to_string()), + qualified_table_name: None, + columns: columns.clone(), + column_types: vec![None; columns.len()], + column_extras: Vec::new(), + rows: rows.clone(), + batch_size: Some(options.batch_size), + })?; + let sql_ms = elapsed_ms(sql_started); + + let xlsx_started = Instant::now(); + let xlsx = build_xlsx_workbook(&XlsxWorksheetData { + sheet_name: Some("Benchmark".to_string()), + columns, + column_types: Vec::new(), + column_comments: Vec::new(), + rows, + numeric_column_right_align: false, + })?; + let xlsx_ms = elapsed_ms(xlsx_started); + + println!( + "{}", + serde_json::to_string_pretty(&serde_json::json!({ + "rows": options.rows, + "columns": options.columns, + "batchSize": options.batch_size, + "importSql": { + "milliseconds": import_ms, + "batches": import_batches.len(), + "bytes": import_batches.iter().map(|batch| batch.sql.len()).sum::(), + }, + "csv": { + "milliseconds": csv_ms, + "bytes": csv.len(), + }, + "sqlExport": { + "milliseconds": sql_ms, + "statements": sql.len(), + "bytes": sql.iter().map(String::len).sum::(), + }, + "xlsx": { + "milliseconds": xlsx_ms, + "bytes": xlsx.len(), + }, + })) + .map_err(|error| error.to_string())? + ); + Ok(()) +} + +fn main() { + if let Err(error) = run() { + eprintln!("{error}"); + std::process::exit(1); + } +} diff --git a/crates/dbx-core/src/csv_export.rs b/crates/dbx-core/src/csv_export.rs index 7568142065..ad484d4bd6 100644 --- a/crates/dbx-core/src/csv_export.rs +++ b/crates/dbx-core/src/csv_export.rs @@ -100,6 +100,15 @@ fn push_tsv_value(out: &mut String, value: &Value) { } } +pub(crate) fn push_tsv_row(out: &mut String, row: &[Value]) { + for (cell_index, cell) in row.iter().enumerate() { + if cell_index > 0 { + out.push('\t'); + } + push_tsv_value(out, cell); + } +} + /// 预分配粗估:按全部行的实际单元格数求和(不假设等宽),饱和运算防溢出, /// 并设上限——估算只是性能提示,绝不能因病态输入放大成巨额分配 const ROWS_CAPACITY_ESTIMATE_MAX: usize = 16 * 1024 * 1024; @@ -133,15 +142,11 @@ fn push_tsv_rows(out: &mut String, rows: &[Vec]) { if row_index > 0 { out.push('\n'); } - for (cell_index, cell) in row.iter().enumerate() { - if cell_index > 0 { - out.push('\t'); - } - push_tsv_value(out, cell); - } + push_tsv_row(out, row); } } +#[cfg_attr(not(test), allow(dead_code))] pub(crate) fn format_tsv_rows(rows: &[Vec]) -> String { let mut out = String::with_capacity(estimated_rows_capacity(rows)); push_tsv_rows(&mut out, rows); @@ -166,17 +171,30 @@ pub(crate) fn format_tsv(columns: &[String], rows: &[Vec]) -> String { /// Format query-result rows as CSV text without a header row. Database NULLs /// use the same empty-cell representation as table-data exports. Used by the /// streaming query-result export for batches after the first. +pub(crate) fn push_query_result_csv_row(out: &mut String, row: &[Value]) { + for (cell_index, cell) in row.iter().enumerate() { + if cell_index > 0 { + out.push(','); + } + push_csv_value(out, cell); + } +} + +pub(crate) fn push_table_csv_row(out: &mut String, row: &[Value]) { + for (cell_index, cell) in row.iter().enumerate() { + if cell_index > 0 { + out.push(','); + } + push_csv_text_value(out, cell); + } +} + fn push_query_result_csv_rows(out: &mut String, rows: &[Vec]) { for (row_index, row) in rows.iter().enumerate() { if row_index > 0 { out.push('\n'); } - for (cell_index, cell) in row.iter().enumerate() { - if cell_index > 0 { - out.push(','); - } - push_csv_value(out, cell); - } + push_query_result_csv_row(out, row); } } @@ -399,5 +417,22 @@ mod tests { } } + #[test] + fn reusable_row_buffers_match_batch_formatters() { + let row = vec![Value::Null, json!("NULL"), json!("line\n\"two\""), json!(42)]; + + let mut csv = String::new(); + super::push_query_result_csv_row(&mut csv, &row); + assert_eq!(csv, super::format_query_result_csv_rows(&[row.clone()])); + + let mut table_csv = String::new(); + super::push_table_csv_row(&mut table_csv, &row); + assert_eq!(table_csv, "\"\",\"NULL\",\"line\n\"\"two\"\"\",\"42\""); + + let mut tsv = String::new(); + super::push_tsv_row(&mut tsv, &row); + assert_eq!(tsv, super::format_tsv_rows(&[row])); + } + use serde_json::Value; } diff --git a/crates/dbx-core/src/database_export.rs b/crates/dbx-core/src/database_export.rs index 9f33f13839..e63852f145 100644 --- a/crates/dbx-core/src/database_export.rs +++ b/crates/dbx-core/src/database_export.rs @@ -994,54 +994,63 @@ pub fn build_export_insert_statements(options: BuildExportInsertStatementsOption let statement_overhead_bytes = export_sql_statement_bytes(options.database_type, &statement_prefix) + 1; let target_statement_bytes = DATABASE_EXPORT_TARGET_STATEMENT_BYTES; let separator_bytes = export_sql_statement_bytes(options.database_type, ", "); - let mut current_values = Vec::with_capacity(batch_size); + let mut current_values = String::new(); let mut current_values_bytes = 0usize; + let mut current_row_count = 0usize; - let flush_values = |statements: &mut Vec, values: &mut Vec, values_bytes: &mut usize| { - if values.is_empty() { - return; - } - let insert_sql = format!("{statement_prefix}{};", values.join(", ")); - if needs_dameng_identity_insert { - statements.push(wrap_dameng_identity_insert_sql_for_table(&insert_sql, &table)); - } else { - statements.push(insert_sql); - } - values.clear(); - *values_bytes = 0; - }; + let flush_values = + |statements: &mut Vec, values: &mut String, values_bytes: &mut usize, row_count: &mut usize| { + if *row_count == 0 { + return; + } + let mut insert_sql = String::with_capacity(statement_prefix.len() + values.len() + 1); + insert_sql.push_str(&statement_prefix); + insert_sql.push_str(values); + insert_sql.push(';'); + if needs_dameng_identity_insert { + statements.push(wrap_dameng_identity_insert_sql_for_table(&insert_sql, &table)); + } else { + statements.push(insert_sql); + } + values.clear(); + *values_bytes = 0; + *row_count = 0; + }; for row in options.rows { - let row_values = insert_columns - .iter() - .map(|(index, _, sqlserver_unicode_string)| { - let value = row.get(*index).unwrap_or(&Value::Null); - format_export_sql_literal_typed( - value, - options.database_type, - options.column_types.get(*index).and_then(|value| value.as_deref()), - *sqlserver_unicode_string, - ) - }) - .collect::>() - .join(", "); - let rendered_row = format!("({row_values})"); + let mut rendered_row = String::with_capacity(insert_columns.len().saturating_mul(16).saturating_add(2)); + rendered_row.push('('); + for (column_index, (index, _, sqlserver_unicode_string)) in insert_columns.iter().enumerate() { + if column_index > 0 { + rendered_row.push_str(", "); + } + let value = row.get(*index).unwrap_or(&Value::Null); + rendered_row.push_str(&format_export_sql_literal_typed( + value, + options.database_type, + options.column_types.get(*index).and_then(|value| value.as_deref()), + *sqlserver_unicode_string, + )); + } + rendered_row.push(')'); let rendered_row_bytes = export_sql_statement_bytes(options.database_type, &rendered_row); let candidate_bytes = statement_overhead_bytes + current_values_bytes - + if current_values.is_empty() { 0 } else { separator_bytes } + + if current_row_count == 0 { 0 } else { separator_bytes } + rendered_row_bytes; - if !current_values.is_empty() - && (current_values.len() >= batch_size || candidate_bytes > target_statement_bytes) - { - flush_values(&mut statements, &mut current_values, &mut current_values_bytes); + if current_row_count > 0 && (current_row_count >= batch_size || candidate_bytes > target_statement_bytes) { + flush_values(&mut statements, &mut current_values, &mut current_values_bytes, &mut current_row_count); } - current_values_bytes += if current_values.is_empty() { 0 } else { separator_bytes }; + if current_row_count > 0 { + current_values.push_str(", "); + current_values_bytes += separator_bytes; + } + current_values.push_str(&rendered_row); current_values_bytes += rendered_row_bytes; - current_values.push(rendered_row); + current_row_count += 1; } - flush_values(&mut statements, &mut current_values, &mut current_values_bytes); + flush_values(&mut statements, &mut current_values, &mut current_values_bytes, &mut current_row_count); Ok(statements) } diff --git a/crates/dbx-core/src/query_result_export.rs b/crates/dbx-core/src/query_result_export.rs index 9f6449daad..3971dc77c8 100644 --- a/crates/dbx-core/src/query_result_export.rs +++ b/crates/dbx-core/src/query_result_export.rs @@ -10,7 +10,7 @@ use std::sync::{ use std::time::{Duration, Instant}; use crate::connection::{AppState, PoolKind}; -use crate::csv_export::{format_query_result_csv, format_query_result_csv_rows, format_tsv, format_tsv_rows}; +use crate::csv_export::{format_query_result_csv, format_tsv, push_query_result_csv_row, push_tsv_row}; pub use crate::database_export::ExportStatus; use crate::database_export::{build_export_insert_statements, is_export_cancelled, BuildExportInsertStatementsOptions}; use crate::models::connection::DatabaseType; @@ -400,12 +400,38 @@ fn format_text_export_header(format: &str, columns: &[String]) -> String { content.strip_suffix('\n').unwrap_or(&content).to_string() } -fn format_text_export_rows(format: &str, rows: &[Vec]) -> String { +fn write_text_export_row( + file: &mut W, + format: &str, + row: &[Value], + buffer: &mut String, +) -> Result<(), String> { + buffer.clear(); + buffer.push('\n'); if format == "csv" { - format_query_result_csv_rows(rows) + push_query_result_csv_row(buffer, row); } else { - format_tsv_rows(rows) + push_tsv_row(buffer, row); } + file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}")) +} + +fn write_text_export_rows( + file: &mut W, + format: &str, + rows: &[Vec], + buffer: &mut String, +) -> Result<(), String> { + buffer.clear(); + for row in rows { + buffer.push('\n'); + if format == "csv" { + push_query_result_csv_row(buffer, row); + } else { + push_tsv_row(buffer, row); + } + } + file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}")) } fn should_emit_stream_progress( @@ -683,6 +709,7 @@ async fn export_query_result_core_inner( } let mut xlsx = None; + let mut text_buffer = String::new(); let mut columns: Vec = Vec::new(); let mut column_types: Vec = Vec::new(); let mut rows_exported: u64 = 0; @@ -838,7 +865,7 @@ async fn export_query_result_core_inner( result.rows.truncate(this_page); } let row_count = result.rows.len(); - let formatted_rows = crate::temporal_format::format_temporal_export_rows_with_string_types( + let formatted_rows = crate::temporal_format::format_temporal_export_rows_with_string_types_cow( &result.rows, &column_types, request.date_time_format.as_deref(), @@ -850,18 +877,16 @@ async fn export_query_result_core_inner( let header = format_text_export_header(&format, &columns); file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?; if row_count > 0 { - let rows = format_text_export_rows(&format, &formatted_rows); - write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?; + write_text_export_rows(file, &format, formatted_rows.as_ref(), &mut text_buffer)?; } wrote_text_header = true; } else if row_count > 0 { - let rows = format_text_export_rows(&format, &formatted_rows); - write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?; + write_text_export_rows(file, &format, formatted_rows.as_ref(), &mut text_buffer)?; } } } else if format == "sql" { let writer = sql_writer.as_mut().ok_or_else(|| "SQL export writer missing".to_string())?; - for row in formatted_rows { + for row in formatted_rows.into_owned() { writer.write_row(row)?; } } else { @@ -876,7 +901,7 @@ async fn export_query_result_core_inner( )?); } if let Some(writer) = xlsx.as_mut() { - for row in &formatted_rows { + for row in formatted_rows.as_ref() { writer.write_row(row).map_err(|e| format!("Failed to write XLSX row: {e}"))?; } } @@ -1013,6 +1038,7 @@ async fn try_export_postgres_query_result_stream( None }; let mut xlsx = None; + let mut text_buffer = String::new(); let mut sql_writer: Option = if format == "sql" { Some(SqlInsertWriter::create(request)?) } else { None }; let budget = operation_budget_for_pool_key(state, &pool_key, query_export_timeout(request.timeout_secs)).await; @@ -1050,25 +1076,26 @@ async fn try_export_postgres_query_result_stream( } } crate::db::postgres::PostgresQueryStreamItem::Row(row) => { - let formatted = crate::temporal_format::format_temporal_export_row_with_string_types( + let formatted = crate::temporal_format::format_temporal_export_row_with_string_types_cow( &row, &temporal_column_types, request.date_time_format.as_deref(), ); if let Some(writer) = sql_writer.as_mut() { - writer.write_row(formatted)?; + writer.write_row(formatted.into_owned())?; } else if let Some(file) = text_file.as_mut() { - let rows = format_text_export_rows(format, std::slice::from_ref(&formatted)); - write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?; + write_text_export_row(file, format, formatted.as_ref(), &mut text_buffer)?; } else if let Some(writer) = xlsx.as_mut() { - writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?; + writer.write_row(formatted.as_ref()).map_err(|e| format!("Failed to write XLSX row: {e}"))?; } else { let xlsx_file = File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?; xlsx = Some(start_query_result_xlsx_workbook(BufWriter::new(xlsx_file), request, &columns, &[])?); if let Some(writer) = xlsx.as_mut() { - writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?; + writer + .write_row(formatted.as_ref()) + .map_err(|e| format!("Failed to write XLSX row: {e}"))?; } } rows_exported += 1; @@ -1205,6 +1232,7 @@ async fn try_export_mysql_query_result_stream( None }; let mut xlsx = None; + let mut text_buffer = String::new(); let mut sql_writer: Option = if format == "sql" { Some(SqlInsertWriter::create(request)?) } else { None }; let query_timeout = query_export_timeout(request.timeout_secs); @@ -1292,25 +1320,26 @@ async fn try_export_mysql_query_result_stream( } } crate::db::mysql::MySqlQueryStreamItem::Row(row) => { - let formatted = crate::temporal_format::format_temporal_export_row_with_string_types( + let formatted = crate::temporal_format::format_temporal_export_row_with_string_types_cow( &row, &temporal_column_types, request.date_time_format.as_deref(), ); if let Some(writer) = sql_writer.as_mut() { - writer.write_row(formatted)?; + writer.write_row(formatted.into_owned())?; } else if let Some(file) = text_file.as_mut() { - let rows = format_text_export_rows(format, std::slice::from_ref(&formatted)); - write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?; + write_text_export_row(file, format, formatted.as_ref(), &mut text_buffer)?; } else if let Some(writer) = xlsx.as_mut() { - writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?; + writer.write_row(formatted.as_ref()).map_err(|e| format!("Failed to write XLSX row: {e}"))?; } else { let xlsx_file = File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?; xlsx = Some(start_query_result_xlsx_workbook(BufWriter::new(xlsx_file), request, &columns, &[])?); if let Some(writer) = xlsx.as_mut() { - writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?; + writer + .write_row(formatted.as_ref()) + .map_err(|e| format!("Failed to write XLSX row: {e}"))?; } } rows_exported += 1; @@ -1467,6 +1496,7 @@ async fn try_export_clickhouse_query_result_stream( None }; let mut xlsx = None; + let mut text_buffer = String::new(); let mut sql_writer: Option = if format == "sql" { Some(SqlInsertWriter::create(request)?) } else { None }; let query_timeout = query_export_timeout(request.timeout_secs); @@ -1505,25 +1535,26 @@ async fn try_export_clickhouse_query_result_stream( } } crate::db::clickhouse_driver::ClickHouseQueryStreamItem::Row(row) => { - let formatted = crate::temporal_format::format_temporal_export_row_with_string_types( + let formatted = crate::temporal_format::format_temporal_export_row_with_string_types_cow( &row, &temporal_column_types, request.date_time_format.as_deref(), ); if let Some(writer) = sql_writer.as_mut() { - writer.write_row(formatted)?; + writer.write_row(formatted.into_owned())?; } else if let Some(file) = text_file.as_mut() { - let rows = format_text_export_rows(format, std::slice::from_ref(&formatted)); - write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?; + write_text_export_row(file, format, formatted.as_ref(), &mut text_buffer)?; } else if let Some(writer) = xlsx.as_mut() { - writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?; + writer.write_row(formatted.as_ref()).map_err(|e| format!("Failed to write XLSX row: {e}"))?; } else { let xlsx_file = File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?; xlsx = Some(start_query_result_xlsx_workbook(BufWriter::new(xlsx_file), request, &columns, &[])?); if let Some(writer) = xlsx.as_mut() { - writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?; + writer + .write_row(formatted.as_ref()) + .map_err(|e| format!("Failed to write XLSX row: {e}"))?; } } rows_exported += 1; @@ -1635,6 +1666,7 @@ async fn try_export_sqlserver_query_result_stream( None }; let mut xlsx = None; + let mut text_buffer = String::new(); let mut sql_writer: Option = if format == "sql" { Some(SqlInsertWriter::create(request)?) } else { None }; let query_timeout = query_export_timeout(request.timeout_secs); @@ -1683,25 +1715,26 @@ async fn try_export_sqlserver_query_result_stream( } } crate::db::sqlserver::SqlServerStreamItem::Row(row) => { - let formatted = crate::temporal_format::format_temporal_export_row_with_string_types( + let formatted = crate::temporal_format::format_temporal_export_row_with_string_types_cow( row, &temporal_column_types, request.date_time_format.as_deref(), ); if let Some(writer) = sql_writer.as_mut() { - writer.write_row(formatted)?; + writer.write_row(formatted.into_owned())?; } else if let Some(file) = text_file.as_mut() { - let rows = format_text_export_rows(format, std::slice::from_ref(&formatted)); - write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?; + write_text_export_row(file, format, formatted.as_ref(), &mut text_buffer)?; } else if let Some(writer) = xlsx.as_mut() { - writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?; + writer.write_row(formatted.as_ref()).map_err(|e| format!("Failed to write XLSX row: {e}"))?; } else { let xlsx_file = File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?; xlsx = Some(start_query_result_xlsx_workbook(BufWriter::new(xlsx_file), request, &columns, &[])?); if let Some(writer) = xlsx.as_mut() { - writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?; + writer + .write_row(formatted.as_ref()) + .map_err(|e| format!("Failed to write XLSX row: {e}"))?; } } rows_exported += 1; @@ -1893,6 +1926,16 @@ mod tests { assert_eq!(format_text_export_header("txt", &["id".to_string(), "note".to_string()]), "id\tnote"); } + #[test] + fn reusable_text_row_buffer_preserves_query_null_semantics() { + let row = vec![Value::Null, serde_json::json!(""), serde_json::json!("line\n\"two\"")]; + let mut output = Vec::new(); + let mut buffer = String::new(); + + write_text_export_row(&mut output, "csv", &row, &mut buffer).expect("write csv row"); + assert_eq!(String::from_utf8(output).expect("utf8 csv"), "\n,\"\",\"line\n\"\"two\"\"\""); + } + #[test] fn xlsx_no_row_limit_has_no_query_layer_cap() { // Without the old hard limit, XLSX uses the writer's internal splitting. diff --git a/crates/dbx-core/src/table_export.rs b/crates/dbx-core/src/table_export.rs index a566fc8002..18508f4cdb 100644 --- a/crates/dbx-core/src/table_export.rs +++ b/crates/dbx-core/src/table_export.rs @@ -9,7 +9,9 @@ use tokio_util::sync::CancellationToken; use crate::connection::MysqlMode; use crate::connection::{config_for_pool_key, task_client_session_id, AppState, PoolKind}; -use crate::csv_export::{format_csv, format_tsv, format_tsv_rows, push_csv_text_value}; +#[cfg(test)] +use crate::csv_export::format_tsv_rows; +use crate::csv_export::{format_csv, format_tsv, push_table_csv_row, push_tsv_row}; pub use crate::database_export::ExportStatus; use crate::database_export::{ build_export_insert_statements, is_export_cancelled, is_internal_export_column, BuildExportInsertStatementsOptions, @@ -82,6 +84,7 @@ pub struct TableExportProgress { /// Format rows as CSV text without a header row. /// Used for streaming subsequent pagination batches. +#[cfg_attr(not(test), allow(dead_code))] fn format_csv_rows(rows: &[Vec]) -> String { // 注意:该无表头批次路径的 Null 输出为 ""(带引号空串),与查询结果导出 // 及首批 format_csv 的裸空单元格语义不同,直写化必须保留该差异 @@ -90,16 +93,40 @@ fn format_csv_rows(rows: &[Vec]) -> String { if row_index > 0 { out.push('\n'); } - for (cell_index, cell) in row.iter().enumerate() { - if cell_index > 0 { - out.push(','); - } - push_csv_text_value(&mut out, cell); - } + push_table_csv_row(&mut out, row); } out } +fn write_table_text_row(file: &mut W, csv: bool, row: &[Value], buffer: &mut String) -> Result<(), String> { + buffer.clear(); + buffer.push('\n'); + if csv { + push_table_csv_row(buffer, row); + } else { + push_tsv_row(buffer, row); + } + file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}")) +} + +fn write_table_text_rows( + file: &mut W, + csv: bool, + rows: &[Vec], + buffer: &mut String, +) -> Result<(), String> { + buffer.clear(); + for row in rows { + buffer.push('\n'); + if csv { + push_table_csv_row(buffer, row); + } else { + push_tsv_row(buffer, row); + } + } + file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}")) +} + fn export_column_types(request: &TableExportRequest) -> Vec { request .column_types @@ -928,6 +955,7 @@ async fn try_export_native_table_stream( let header = format_csv(col_names, &[]); let header = header.strip_suffix('\n').unwrap_or(&header); file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?; + let mut row_buffer = String::new(); let result = stream_native_table_rows( state, @@ -938,13 +966,12 @@ async fn try_export_native_table_stream( &cancelled, cancel_token.clone(), |row| { - let formatted = crate::temporal_format::format_temporal_export_row( + let formatted = crate::temporal_format::format_temporal_export_row_cow( row, column_types, request.date_time_format.as_deref(), ); - let row_csv = format_csv_rows(&[formatted]); - write!(file, "\n{row_csv}").map_err(|e| format!("Failed to write CSV rows: {e}"))?; + write_table_text_row(&mut file, true, formatted.as_ref(), &mut row_buffer)?; rows_exported += 1; if rows_exported.is_multiple_of(progress_interval) { on_progress(TableExportProgress { @@ -972,6 +999,7 @@ async fn try_export_native_table_stream( let header = format_tsv(col_names, &[]); let header = header.strip_suffix('\n').unwrap_or(&header); file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write TXT: {e}"))?; + let mut row_buffer = String::new(); let result = stream_native_table_rows( state, @@ -982,13 +1010,12 @@ async fn try_export_native_table_stream( &cancelled, cancel_token.clone(), |row| { - let formatted = crate::temporal_format::format_temporal_export_row( + let formatted = crate::temporal_format::format_temporal_export_row_cow( row, column_types, request.date_time_format.as_deref(), ); - let row_tsv = format_tsv_rows(&[formatted]); - write!(file, "\n{row_tsv}").map_err(|e| format!("Failed to write TXT rows: {e}"))?; + write_table_text_row(&mut file, false, formatted.as_ref(), &mut row_buffer)?; rows_exported += 1; if rows_exported.is_multiple_of(progress_interval) { on_progress(TableExportProgress { @@ -1033,12 +1060,12 @@ async fn try_export_native_table_stream( &cancelled, cancel_token.clone(), |row| { - let formatted = crate::temporal_format::format_temporal_export_row( + let formatted = crate::temporal_format::format_temporal_export_row_cow( row, column_types, request.date_time_format.as_deref(), ); - writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?; + writer.write_row(formatted.as_ref()).map_err(|e| format!("Failed to write XLSX row: {e}"))?; rows_exported += 1; if rows_exported.is_multiple_of(progress_interval) { on_progress(TableExportProgress { @@ -1087,12 +1114,12 @@ async fn try_export_native_table_stream( if !is_first_row { file.write_all(b",\n").map_err(|e| format!("Failed to write JSON: {e}"))?; } - let formatted = crate::temporal_format::format_temporal_export_row( + let formatted = crate::temporal_format::format_temporal_export_row_cow( row, column_types, request.date_time_format.as_deref(), ); - write_json_row_object(&mut file, col_names, &formatted)?; + write_json_row_object(&mut file, col_names, formatted.as_ref())?; is_first_row = false; rows_exported += 1; if rows_exported.is_multiple_of(progress_interval) { @@ -1473,6 +1500,7 @@ async fn export_table_data_core_inner( // 8. Create output file let file = std::fs::File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?; let mut file = BufWriter::new(file); + let mut text_buffer = String::new(); let mut rows_exported: u64 = 0; let batch_size = request.batch_size.unwrap_or(DEFAULT_BATCH_SIZE).max(1); @@ -1533,7 +1561,7 @@ async fn export_table_data_core_inner( if row_count == 0 { break; } - let formatted_rows = crate::temporal_format::format_temporal_export_rows( + let formatted_rows = crate::temporal_format::format_temporal_export_rows_cow( &result.rows, &column_types, request.date_time_format.as_deref(), @@ -1541,15 +1569,12 @@ async fn export_table_data_core_inner( if is_first_batch { // First batch: write header + rows via format_csv - let csv_content = format_csv(&col_names, &formatted_rows); + let csv_content = format_csv(&col_names, formatted_rows.as_ref()); file.write_all(csv_content.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?; is_first_batch = false; } else { // Subsequent batches: write rows only (prepend newline for separation) - let rows_csv = format_csv_rows(&formatted_rows); - if !rows_csv.is_empty() { - write!(file, "\n{rows_csv}").map_err(|e| format!("Failed to write CSV rows: {e}"))?; - } + write_table_text_rows(&mut file, true, formatted_rows.as_ref(), &mut text_buffer)?; } rows_exported += row_count as u64; @@ -1624,21 +1649,17 @@ async fn export_table_data_core_inner( if row_count == 0 { break; } - let formatted_rows = crate::temporal_format::format_temporal_export_rows( + let formatted_rows = crate::temporal_format::format_temporal_export_rows_cow( &result.rows, &column_types, request.date_time_format.as_deref(), ); if is_first_batch { - let rows_tsv = format_tsv_rows(&formatted_rows); - write!(file, "\n{rows_tsv}").map_err(|e| format!("Failed to write TXT rows: {e}"))?; + write_table_text_rows(&mut file, false, formatted_rows.as_ref(), &mut text_buffer)?; is_first_batch = false; } else { - let rows_tsv = format_tsv_rows(&formatted_rows); - if !rows_tsv.is_empty() { - write!(file, "\n{rows_tsv}").map_err(|e| format!("Failed to write TXT rows: {e}"))?; - } + write_table_text_rows(&mut file, false, formatted_rows.as_ref(), &mut text_buffer)?; } rows_exported += row_count as u64; @@ -1728,12 +1749,12 @@ async fn export_table_data_core_inner( } for row in &result.rows { - let formatted = crate::temporal_format::format_temporal_export_row( + let formatted = crate::temporal_format::format_temporal_export_row_cow( row, &column_types, request.date_time_format.as_deref(), ); - writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?; + writer.write_row(formatted.as_ref()).map_err(|e| format!("Failed to write XLSX row: {e}"))?; } rows_exported += row_count as u64; @@ -1827,12 +1848,12 @@ async fn export_table_data_core_inner( if !is_first_row { file.write_all(b",\n").map_err(|e| format!("Failed to write JSON: {e}"))?; } - let formatted = crate::temporal_format::format_temporal_export_row( + let formatted = crate::temporal_format::format_temporal_export_row_cow( row, &column_types, request.date_time_format.as_deref(), ); - write_json_row_object(&mut file, &col_names, &formatted)?; + write_json_row_object(&mut file, &col_names, formatted.as_ref())?; is_first_row = false; } @@ -1908,12 +1929,12 @@ async fn export_table_data_core_inner( break; } - let formatted_rows = crate::temporal_format::format_temporal_export_rows( + let formatted_rows = crate::temporal_format::format_temporal_export_rows_cow( &result.rows, &column_types, request.date_time_format.as_deref(), ); - let rows_markdown = format_markdown_rows(&formatted_rows); + let rows_markdown = format_markdown_rows(formatted_rows.as_ref()); if !rows_markdown.is_empty() { if wrote_rows { file.write_all(b"\n").map_err(|e| format!("Failed to write Markdown: {e}"))?; @@ -2270,6 +2291,16 @@ mod tests { assert_eq!(out, "\"just\",\"one\""); } + #[test] + fn reusable_text_row_buffer_preserves_table_null_semantics() { + let row = vec![Value::Null, json!(""), json!("line\n\"two\"")]; + let mut output = Vec::new(); + let mut buffer = String::new(); + + write_table_text_row(&mut output, true, &row, &mut buffer).expect("write csv row"); + assert_eq!(String::from_utf8(output).expect("utf8 csv"), "\n\"\",\"\",\"line\n\"\"two\"\"\""); + } + // ----------------------------------------------------------------------- // format_tsv (Navicat-style TXT export) // ----------------------------------------------------------------------- diff --git a/crates/dbx-core/src/table_import.rs b/crates/dbx-core/src/table_import.rs index d3f99b9657..1667669b58 100644 --- a/crates/dbx-core/src/table_import.rs +++ b/crates/dbx-core/src/table_import.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use std::fs::File; use std::io::{BufRead, BufReader, Read as IoRead, Seek, SeekFrom, Write as IoWrite}; @@ -19,8 +20,9 @@ use sha2::{Digest, Sha256}; use crate::connection::{task_client_session_id, AppState, PoolKind}; use crate::models::connection::DatabaseType; use crate::transfer::{ - execute_on_pool, generate_insert_typed, generate_insert_typed_sql_batches, get_columns_for_transfer, - normalize_integer_literal, normalize_thousands_numeric_literal, qualified_table, quote_identifier, SqlBatchLimits, + escape_value_typed, execute_on_pool, generate_insert_typed_from_value_rows, + generate_insert_typed_sql_batches_from_value_rows, get_columns_for_transfer, normalize_integer_literal, + normalize_thousands_numeric_literal, qualified_table, quote_identifier, SqlBatchLimits, }; pub const DEFAULT_PREVIEW_LIMIT: usize = 50; @@ -3142,9 +3144,8 @@ fn build_import_insert_batch_with_plan( if rows.is_empty() { return Ok(None); } - let mapped_rows = map_import_rows_with_plan(rows, plan, db_type, kingbase_oracle_mode, date_time_format); - let sql = - generate_insert_typed(&plan.target_columns, &plan.column_types, &mapped_rows, table, schema, db_type, None); + let value_rows = import_value_rows_sql(rows, plan, db_type, kingbase_oracle_mode, date_time_format); + let sql = generate_insert_typed_from_value_rows(&plan.target_columns, &value_rows, table, schema, db_type, None); Ok((!sql.trim().is_empty()).then_some(ImportSqlBatch { sql, row_count: rows.len() })) } @@ -3161,11 +3162,10 @@ fn build_import_insert_batches_with_plan( if rows.is_empty() { return Ok(Vec::new()); } - let mapped_rows = map_import_rows_with_plan(rows, plan, db_type, kingbase_oracle_mode, date_time_format); - let batches = generate_insert_typed_sql_batches( + let value_rows = import_value_rows_sql(rows, plan, db_type, kingbase_oracle_mode, date_time_format); + let batches = generate_insert_typed_sql_batches_from_value_rows( &plan.target_columns, - &plan.column_types, - &mapped_rows, + &value_rows, table, schema, db_type, @@ -3175,16 +3175,31 @@ fn build_import_insert_batches_with_plan( Ok(batches.into_iter().map(|(sql, row_count)| ImportSqlBatch { sql, row_count }).collect()) } -fn map_import_rows_with_plan( +fn import_value_rows_sql( rows: &[Vec], plan: &CompiledImportPlan, db_type: &DatabaseType, kingbase_oracle_mode: bool, date_time_format: Option<&str>, -) -> Vec> { - rows.iter() - .map(|row| map_import_row_with_plan(row, plan, db_type, kingbase_oracle_mode, date_time_format)) - .collect() +) -> Vec { + let mut value_rows = Vec::with_capacity(rows.len()); + for row in rows { + let mut values = String::with_capacity(plan.mapped_source_indexes.len().saturating_mul(16).saturating_add(2)); + values.push('('); + for (target_index, source_index) in plan.mapped_source_indexes.iter().enumerate() { + if target_index > 0 { + values.push_str(", "); + } + let source_value = row.get(*source_index).unwrap_or(&serde_json::Value::Null); + let data_type = plan.column_types.get(target_index).and_then(|data_type| data_type.as_deref()); + let normalized = + normalize_import_value_cow(source_value, data_type, db_type, kingbase_oracle_mode, date_time_format); + values.push_str(&escape_value_typed(normalized.as_ref(), db_type, data_type)); + } + values.push(')'); + value_rows.push(values); + } + value_rows } fn map_import_row_with_plan( @@ -3274,17 +3289,17 @@ fn effective_import_batch_size(db_type: &DatabaseType, requested: usize) -> usiz requested.max(1).min(max_rows) } -fn normalize_import_temporal_value( - value: &serde_json::Value, +fn normalize_import_temporal_value_cow<'a>( + value: &'a serde_json::Value, data_type: Option<&str>, db_type: &DatabaseType, kingbase_oracle_mode: bool, date_time_format: Option<&str>, -) -> serde_json::Value { +) -> Cow<'a, serde_json::Value> { let date_type_preserves_time = (matches!(db_type, DatabaseType::Oracle | DatabaseType::OceanbaseOracle) || (*db_type == DatabaseType::Kingbase && kingbase_oracle_mode)) && data_type.is_some_and(|data_type| data_type.trim().eq_ignore_ascii_case("date")); - crate::temporal_format::normalize_temporal_import_value( + crate::temporal_format::normalize_temporal_import_value_cow( value, if date_type_preserves_time { Some("datetime") } else { data_type }, date_time_format, @@ -3352,6 +3367,41 @@ fn textual_source_columns_for_import( .collect() } +fn normalize_import_value_cow<'a>( + value: &'a serde_json::Value, + data_type: Option<&str>, + db_type: &DatabaseType, + kingbase_oracle_mode: bool, + date_time_format: Option<&str>, +) -> Cow<'a, serde_json::Value> { + let normalized = + normalize_import_temporal_value_cow(value, data_type, db_type, kingbase_oracle_mode, date_time_format); + // Strip validated thousands separators before integer canonicalization so "1,234.00" + // still collapses to a plain integer literal for integer targets. + let thousands_canonical = + normalized.as_str().and_then(|value| normalize_thousands_numeric_literal(value, db_type, data_type)); + if let Some(integer_text) = thousands_canonical + .as_deref() + .or_else(|| normalized.as_str()) + .and_then(|value| normalize_integer_literal(value, db_type, data_type)) + .and_then(|value| value.parse::().ok()) + { + // Normalize before both INSERT and COPY paths; COPY does not pass through SQL literal escaping. + return Cow::Owned(serde_json::Value::Number(integer_text.into())); + } + if let Some(number) = normalized.as_number() { + if let Some(integer_text) = normalize_integer_literal(&number.to_string(), db_type, data_type) + .and_then(|value| value.parse::().ok()) + { + return Cow::Owned(serde_json::Value::Number(integer_text.into())); + } + } + if let Some(canonical) = thousands_canonical { + return Cow::Owned(serde_json::Value::String(canonical)); + } + normalized +} + fn normalize_import_value( value: &serde_json::Value, data_type: Option<&str>, @@ -3359,28 +3409,7 @@ fn normalize_import_value( kingbase_oracle_mode: bool, date_time_format: Option<&str>, ) -> serde_json::Value { - let normalized = normalize_import_temporal_value(value, data_type, db_type, kingbase_oracle_mode, date_time_format); - // Strip validated thousands separators before integer canonicalization so "1,234.00" - // still collapses to a plain integer literal for integer targets. - let thousands_canonical = - normalized.as_str().and_then(|value| normalize_thousands_numeric_literal(value, db_type, data_type)); - let integer_text = thousands_canonical - .as_deref() - .or_else(|| normalized.as_str()) - .map(str::to_owned) - .or_else(|| normalized.as_number().map(ToString::to_string)); - if let Some(integer_text) = integer_text - .as_deref() - .and_then(|value| normalize_integer_literal(value, db_type, data_type)) - .and_then(|value| value.parse::().ok()) - { - // Normalize before both INSERT and COPY paths; COPY does not pass through SQL literal escaping. - return serde_json::Value::Number(integer_text.into()); - } - if let Some(canonical) = thousands_canonical { - return serde_json::Value::String(canonical); - } - normalized + normalize_import_value_cow(value, data_type, db_type, kingbase_oracle_mode, date_time_format).into_owned() } pub fn build_import_insert_batches( diff --git a/crates/dbx-core/src/temporal_format.rs b/crates/dbx-core/src/temporal_format.rs index 64ac6b3c61..898f04d233 100644 --- a/crates/dbx-core/src/temporal_format.rs +++ b/crates/dbx-core/src/temporal_format.rs @@ -1,5 +1,6 @@ use chrono::{DateTime, FixedOffset, NaiveDate, NaiveDateTime, NaiveTime, Timelike}; use serde_json::Value; +use std::borrow::Cow; use std::fmt::Write as _; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -192,17 +193,51 @@ pub fn format_temporal_export_value(value: &Value, data_type: Option<&str>, patt .unwrap_or_else(|| value.clone()) } +fn active_temporal_pattern(pattern: Option<&str>) -> Option<&str> { + pattern.filter(|pattern| !pattern.trim().is_empty()) +} + +pub fn format_temporal_export_row_cow<'a>( + row: &'a [Value], + column_types: &[Option], + pattern: Option<&str>, +) -> Cow<'a, [Value]> { + let Some(pattern) = active_temporal_pattern(pattern) else { + return Cow::Borrowed(row); + }; + if !column_types.iter().any(|data_type| temporal_kind(data_type.as_deref()).is_some()) { + return Cow::Borrowed(row); + } + Cow::Owned( + row.iter() + .enumerate() + .map(|(index, value)| { + format_temporal_export_value( + value, + column_types.get(index).and_then(|data_type| data_type.as_deref()), + Some(pattern), + ) + }) + .collect(), + ) +} + pub fn format_temporal_export_row(row: &[Value], column_types: &[Option], pattern: Option<&str>) -> Vec { - row.iter() - .enumerate() - .map(|(index, value)| { - format_temporal_export_value( - value, - column_types.get(index).and_then(|data_type| data_type.as_deref()), - pattern, - ) - }) - .collect() + format_temporal_export_row_cow(row, column_types, pattern).into_owned() +} + +pub fn format_temporal_export_rows_cow<'a>( + rows: &'a [Vec], + column_types: &[Option], + pattern: Option<&str>, +) -> Cow<'a, [Vec]> { + let Some(pattern) = active_temporal_pattern(pattern) else { + return Cow::Borrowed(rows); + }; + if !column_types.iter().any(|data_type| temporal_kind(data_type.as_deref()).is_some()) { + return Cow::Borrowed(rows); + } + Cow::Owned(rows.iter().map(|row| format_temporal_export_row(row, column_types, Some(pattern))).collect()) } pub fn format_temporal_export_rows( @@ -210,7 +245,28 @@ pub fn format_temporal_export_rows( column_types: &[Option], pattern: Option<&str>, ) -> Vec> { - rows.iter().map(|row| format_temporal_export_row(row, column_types, pattern)).collect() + format_temporal_export_rows_cow(rows, column_types, pattern).into_owned() +} + +pub fn format_temporal_export_row_with_string_types_cow<'a>( + row: &'a [Value], + column_types: &[String], + pattern: Option<&str>, +) -> Cow<'a, [Value]> { + let Some(pattern) = active_temporal_pattern(pattern) else { + return Cow::Borrowed(row); + }; + if !column_types.iter().any(|data_type| temporal_kind(Some(data_type)).is_some()) { + return Cow::Borrowed(row); + } + Cow::Owned( + row.iter() + .enumerate() + .map(|(index, value)| { + format_temporal_export_value(value, column_types.get(index).map(String::as_str), Some(pattern)) + }) + .collect(), + ) } pub fn format_temporal_export_row_with_string_types( @@ -218,10 +274,23 @@ pub fn format_temporal_export_row_with_string_types( column_types: &[String], pattern: Option<&str>, ) -> Vec { - row.iter() - .enumerate() - .map(|(index, value)| format_temporal_export_value(value, column_types.get(index).map(String::as_str), pattern)) - .collect() + format_temporal_export_row_with_string_types_cow(row, column_types, pattern).into_owned() +} + +pub fn format_temporal_export_rows_with_string_types_cow<'a>( + rows: &'a [Vec], + column_types: &[String], + pattern: Option<&str>, +) -> Cow<'a, [Vec]> { + let Some(pattern) = active_temporal_pattern(pattern) else { + return Cow::Borrowed(rows); + }; + if !column_types.iter().any(|data_type| temporal_kind(Some(data_type)).is_some()) { + return Cow::Borrowed(rows); + } + Cow::Owned( + rows.iter().map(|row| format_temporal_export_row_with_string_types(row, column_types, Some(pattern))).collect(), + ) } pub fn format_temporal_export_rows_with_string_types( @@ -229,40 +298,44 @@ pub fn format_temporal_export_rows_with_string_types( column_types: &[String], pattern: Option<&str>, ) -> Vec> { - rows.iter().map(|row| format_temporal_export_row_with_string_types(row, column_types, pattern)).collect() + format_temporal_export_rows_with_string_types_cow(rows, column_types, pattern).into_owned() } -pub fn normalize_temporal_import_value(value: &Value, data_type: Option<&str>, pattern: Option<&str>) -> Value { +pub(crate) fn normalize_temporal_import_value_cow<'a>( + value: &'a Value, + data_type: Option<&str>, + pattern: Option<&str>, +) -> Cow<'a, Value> { let Some(kind) = temporal_kind(data_type) else { - return value.clone(); + return Cow::Borrowed(value); }; let Some(raw) = value.as_str() else { - return value.clone(); + return Cow::Borrowed(value); }; let Some(parsed) = parse_temporal(raw.trim(), pattern) else { - return value.clone(); + return Cow::Borrowed(value); }; let normalized = match (kind, parsed) { (TemporalKind::Date, ParsedTemporal::Zoned(value)) => value.date_naive().format("%Y-%m-%d").to_string(), (TemporalKind::Date, ParsedTemporal::DateTime(value)) => value.date().format("%Y-%m-%d").to_string(), (TemporalKind::Date, ParsedTemporal::Date(value)) => value.format("%Y-%m-%d").to_string(), - (TemporalKind::Date, ParsedTemporal::Time(_)) => return value.clone(), + (TemporalKind::Date, ParsedTemporal::Time(_)) => return Cow::Borrowed(value), (TemporalKind::Time, ParsedTemporal::Zoned(value)) => value.time().format("%H:%M:%S%.f").to_string(), (TemporalKind::Time, ParsedTemporal::DateTime(value)) => value.time().format("%H:%M:%S%.f").to_string(), (TemporalKind::Time, ParsedTemporal::Time(value)) => value.format("%H:%M:%S%.f").to_string(), - (TemporalKind::Time, ParsedTemporal::Date(_)) => return value.clone(), + (TemporalKind::Time, ParsedTemporal::Date(_)) => return Cow::Borrowed(value), (TemporalKind::DateTime, ParsedTemporal::Zoned(value)) => { value.naive_local().format("%Y-%m-%d %H:%M:%S%.f").to_string() } (TemporalKind::DateTime, ParsedTemporal::DateTime(value)) => value.format("%Y-%m-%d %H:%M:%S%.f").to_string(), (TemporalKind::DateTime, ParsedTemporal::Date(date)) => { let Some(value) = date.and_hms_opt(0, 0, 0) else { - return value.clone(); + return Cow::Borrowed(value); }; value.format("%Y-%m-%d %H:%M:%S").to_string() } - (TemporalKind::DateTime, ParsedTemporal::Time(_)) => return value.clone(), + (TemporalKind::DateTime, ParsedTemporal::Time(_)) => return Cow::Borrowed(value), (TemporalKind::DateTimeWithTimeZone, ParsedTemporal::Zoned(value)) => { value.format("%Y-%m-%dT%H:%M:%S%.f%:z").to_string() } @@ -271,13 +344,17 @@ pub fn normalize_temporal_import_value(value: &Value, data_type: Option<&str>, p } (TemporalKind::DateTimeWithTimeZone, ParsedTemporal::Date(date)) => { let Some(value) = date.and_hms_opt(0, 0, 0) else { - return value.clone(); + return Cow::Borrowed(value); }; value.format("%Y-%m-%d %H:%M:%S").to_string() } - (TemporalKind::DateTimeWithTimeZone, ParsedTemporal::Time(_)) => return value.clone(), + (TemporalKind::DateTimeWithTimeZone, ParsedTemporal::Time(_)) => return Cow::Borrowed(value), }; - Value::String(normalized) + Cow::Owned(Value::String(normalized)) +} + +pub fn normalize_temporal_import_value(value: &Value, data_type: Option<&str>, pattern: Option<&str>) -> Value { + normalize_temporal_import_value_cow(value, data_type, pattern).into_owned() } #[cfg(test)] @@ -358,4 +435,31 @@ mod tests { json!("2024-02-25T13:02:15+08:00") ); } + + #[test] + fn export_cow_borrows_when_no_formatting_is_needed() { + let row = vec![json!(1), json!("2024-02-25 13:02:15")]; + let rows = vec![row.clone()]; + + assert!(matches!( + format_temporal_export_row_cow(&row, &[None, Some("TIMESTAMP".to_string())], None), + Cow::Borrowed(_) + )); + assert!(matches!( + format_temporal_export_rows_cow(&rows, &[None, Some("VARCHAR".to_string())], Some("YYYY/MM/DD")), + Cow::Borrowed(_) + )); + } + + #[test] + fn import_cow_borrows_unchanged_values_and_owns_normalized_values() { + let text = json!("plain"); + assert!(matches!(normalize_temporal_import_value_cow(&text, Some("TEXT"), None), Cow::Borrowed(_))); + + let timestamp = json!("2024/2/25 13:02:15"); + assert_eq!( + normalize_temporal_import_value_cow(×tamp, Some("TIMESTAMP"), None).into_owned(), + json!("2024-02-25 13:02:15") + ); + } } diff --git a/crates/dbx-core/src/transfer.rs b/crates/dbx-core/src/transfer.rs index c60baae7f2..c822d53b68 100644 --- a/crates/dbx-core/src/transfer.rs +++ b/crates/dbx-core/src/transfer.rs @@ -3263,9 +3263,19 @@ pub fn generate_insert_typed( return String::new(); } - let template = InsertSqlTemplate::new(columns, table, schema, db_type, catalog, false); let value_rows = value_rows_sql(rows, column_types, db_type, false); - template.build(&value_rows) + generate_insert_typed_from_value_rows(columns, &value_rows, table, schema, db_type, catalog) +} + +pub(crate) fn generate_insert_typed_from_value_rows( + columns: &[String], + value_rows: &[String], + table: &str, + schema: &str, + db_type: &DatabaseType, + catalog: Option<&str>, +) -> String { + InsertSqlTemplate::new(columns, table, schema, db_type, catalog, false).build(value_rows) } #[derive(Debug)] @@ -3302,7 +3312,13 @@ impl InsertSqlTemplate { return String::new(); } if let Some(into_prefix) = self.oracle_into_prefix.as_deref().filter(|_| value_rows.len() > 1) { - let mut sql = String::from("INSERT ALL\n"); + let capacity = "INSERT ALL\n".len() + + into_prefix.len().saturating_mul(value_rows.len()) + + value_rows.iter().map(String::len).sum::() + + value_rows.len().saturating_sub(1) + + "\nSELECT 1 FROM dual".len(); + let mut sql = String::with_capacity(capacity); + sql.push_str("INSERT ALL\n"); for (index, values) in value_rows.iter().enumerate() { if index > 0 { sql.push('\n'); @@ -3314,8 +3330,17 @@ impl InsertSqlTemplate { return sql; } - let mut sql = self.standard_prefix.clone(); - sql.push_str(&value_rows.join(",\n")); + let capacity = self.standard_prefix.len() + + value_rows.iter().map(String::len).sum::() + + ",\n".len().saturating_mul(value_rows.len().saturating_sub(1)); + let mut sql = String::with_capacity(capacity); + sql.push_str(&self.standard_prefix); + for (index, values) in value_rows.iter().enumerate() { + if index > 0 { + sql.push_str(",\n"); + } + sql.push_str(values); + } sql } @@ -3350,8 +3375,12 @@ fn value_rows_sql( ) -> Vec { let mut out = Vec::with_capacity(rows.len()); for row in rows { - let mut vals = Vec::with_capacity(row.len()); + let mut values = String::with_capacity(row.len().saturating_mul(16).saturating_add(2)); + values.push('('); for (index, v) in row.iter().enumerate() { + if index > 0 { + values.push_str(", "); + } let column_type = column_types.get(index).and_then(|value| value.as_deref()); let value = if mysql_spatial_markers { crate::database_export::format_mysql_spatial_export_literal(v, Some(*db_type), column_type) @@ -3359,9 +3388,10 @@ fn value_rows_sql( None } .unwrap_or_else(|| escape_value_typed(v, db_type, column_type)); - vals.push(value); + values.push_str(&value); } - out.push(format!("({})", vals.join(", "))); + values.push(')'); + out.push(values); } out } @@ -3714,6 +3744,7 @@ fn generate_transfer_write_sql( } #[allow(clippy::too_many_arguments)] +#[cfg_attr(not(test), allow(dead_code))] pub(crate) fn generate_insert_typed_sql_batches( columns: &[String], column_types: &[Option], @@ -3724,18 +3755,21 @@ pub(crate) fn generate_insert_typed_sql_batches( catalog: Option<&str>, limits: SqlBatchLimits, ) -> Result, String> { - generate_insert_typed_sql_batches_for_transfer( - columns, - column_types, - rows, - table, - schema, - db_type, - catalog, - limits, - false, - false, - ) + let value_rows = value_rows_sql(rows, column_types, db_type, false); + generate_insert_typed_sql_batches_from_value_rows(columns, &value_rows, table, schema, db_type, catalog, limits) +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn generate_insert_typed_sql_batches_from_value_rows( + columns: &[String], + value_rows: &[String], + table: &str, + schema: &str, + db_type: &DatabaseType, + catalog: Option<&str>, + limits: SqlBatchLimits, +) -> Result, String> { + generate_insert_sql_batches_from_value_rows(columns, value_rows, table, schema, db_type, catalog, limits, false) } #[allow(clippy::too_many_arguments)] @@ -3755,6 +3789,34 @@ fn generate_insert_typed_sql_batches_for_transfer( return Ok(Vec::new()); } + let value_rows = value_rows_sql(rows, column_types, db_type, mysql_spatial_markers); + generate_insert_sql_batches_from_value_rows( + columns, + &value_rows, + table, + schema, + db_type, + catalog, + limits, + overrides_postgres_system_values, + ) +} + +#[allow(clippy::too_many_arguments)] +fn generate_insert_sql_batches_from_value_rows( + columns: &[String], + value_rows: &[String], + table: &str, + schema: &str, + db_type: &DatabaseType, + catalog: Option<&str>, + limits: SqlBatchLimits, + overrides_postgres_system_values: bool, +) -> Result, String> { + if value_rows.is_empty() { + return Ok(Vec::new()); + } + let max_rows = limits.max_rows.max(1).min(match db_type { DatabaseType::SqlServer => MAX_SQLSERVER_INSERT_ROWS, DatabaseType::Oracle => MAX_ORACLE_INSERT_ALL_ROWS, @@ -3763,7 +3825,6 @@ fn generate_insert_typed_sql_batches_for_transfer( let target_sql_bytes = limits.target_sql_bytes.max(1); let batch_sql_bytes = limits.hard_sql_bytes.map_or(target_sql_bytes, |hard| target_sql_bytes.min(hard)); let template = InsertSqlTemplate::new(columns, table, schema, db_type, catalog, overrides_postgres_system_values); - let value_rows = value_rows_sql(rows, column_types, db_type, mysql_spatial_markers); let value_row_bytes = value_rows.iter().map(|row| sql_text_bytes(row, db_type)).collect::>(); let mut statements = Vec::new(); let mut start = 0usize; diff --git a/crates/dbx-core/src/xlsx_export.rs b/crates/dbx-core/src/xlsx_export.rs index 6b524fedcb..ed0229c1d5 100644 --- a/crates/dbx-core/src/xlsx_export.rs +++ b/crates/dbx-core/src/xlsx_export.rs @@ -1,5 +1,6 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; +use std::fmt::Write as FmtWrite; use std::io::{Cursor, Seek, Write}; use crate::temporal_format::{excel_temporal_serial, ExcelTemporalKind}; @@ -130,6 +131,7 @@ pub struct StreamingXlsxWriter { column_comments: Vec>, date_time_format: Option, numeric_right_align: bool, + row_buffer: String, } /// Estimate column widths from header names only (used by the streaming path @@ -159,8 +161,8 @@ fn cols_xml(widths: &[usize]) -> String { /// Resolve the effective header text: prefer a non-empty column comment, fall /// back to the original column name. -fn effective_header(column: &str, comment: Option<&str>) -> String { - comment.filter(|c| !c.is_empty()).unwrap_or(column).to_string() +fn effective_header<'a>(column: &'a str, comment: Option<&'a str>) -> &'a str { + comment.filter(|c| !c.is_empty()).unwrap_or(column) } /// Build a single `` XML fragment for the header row (row 1). @@ -172,30 +174,36 @@ pub(crate) fn header_row_xml(columns: &[String], column_comments: &[Option() ) } -fn data_row_xml_with_date_time_format( +fn push_data_row_xml( + output: &mut String, row_number: usize, columns: &[String], column_types: &[String], row: &[Value], date_time_format: Option<&str>, numeric_right_align: bool, -) -> String { - let cells = columns - .iter() - .enumerate() - .map(|(col_index, _)| { - let col_type = column_types.get(col_index); - let align_style = numeric_column_style(col_type, numeric_right_align); - typed_cell_xml(row.get(col_index), col_type, row_number - 1, col_index, align_style, date_time_format) - }) - .collect::(); - format!("{cells}") +) { + write!(output, "").expect("writing XLSX XML into a String cannot fail"); + for (col_index, _) in columns.iter().enumerate() { + let col_type = column_types.get(col_index); + let align_style = numeric_column_style(col_type, numeric_right_align); + push_typed_cell_xml( + output, + row.get(col_index), + col_type, + row_number - 1, + col_index, + align_style, + date_time_format, + ); + } + output.push_str(""); } /// Shared ZIP entry options for all XLSX parts. XLSX is a ZIP of XML, and the @@ -298,6 +306,7 @@ fn start_xlsx_writer_inner( column_comments: column_comments.to_vec(), date_time_format: date_time_format.map(str::to_string), numeric_right_align, + row_buffer: String::with_capacity(columns.len().saturating_mul(48)), }) } @@ -355,19 +364,17 @@ impl StreamingXlsxWriter { self.finish_current_sheet()?; self.start_next_data_sheet()?; } - self.zip - .write_all( - data_row_xml_with_date_time_format( - self.next_row_number, - &self.columns, - &self.column_types, - row, - self.date_time_format.as_deref(), - self.numeric_right_align, - ) - .as_bytes(), - ) - .map_err(|err| err.to_string())?; + self.row_buffer.clear(); + push_data_row_xml( + &mut self.row_buffer, + self.next_row_number, + &self.columns, + &self.column_types, + row, + self.date_time_format.as_deref(), + self.numeric_right_align, + ); + self.zip.write_all(self.row_buffer.as_bytes()).map_err(|err| err.to_string())?; self.next_row_number += 1; self.current_data_rows += 1; Ok(()) @@ -469,6 +476,11 @@ pub(crate) fn finish_streaming_xlsx_workbook(writer: StreamingX fn escape_xml(value: &str) -> String { let mut result = String::with_capacity(value.len()); + push_xml_escaped(&mut result, value); + result +} + +fn push_xml_escaped(result: &mut String, value: &str) { for ch in value.chars() { let code = ch as u32; if code != 9 && code != 10 && code != 13 && code < 32 { @@ -482,22 +494,36 @@ fn escape_xml(value: &str) -> String { _ => result.push(ch), } } - result } fn column_name(index: usize) -> String { - let mut out = String::new(); + let mut out = String::with_capacity(4); + push_column_name(&mut out, index); + out +} + +fn push_column_name(out: &mut String, index: usize) { + let mut buffer = [0u8; 16]; + let mut cursor = buffer.len(); let mut n = index + 1; while n > 0 { let rem = (n - 1) % 26; - out.push((b'A' + rem as u8) as char); + cursor -= 1; + buffer[cursor] = b'A' + rem as u8; n = (n - 1) / 26; } - out.chars().rev().collect() + out.push_str(std::str::from_utf8(&buffer[cursor..]).expect("XLSX column names are ASCII")); } fn cell_ref(row_index: usize, col_index: usize) -> String { - format!("{}{}", column_name(col_index), row_index + 1) + let mut reference = String::with_capacity(8); + push_cell_ref(&mut reference, row_index, col_index); + reference +} + +fn push_cell_ref(out: &mut String, row_index: usize, col_index: usize) { + push_column_name(out, col_index); + write!(out, "{}", row_index + 1).expect("writing XLSX cell references into a String cannot fail"); } fn sheet_range(column_count: usize, row_count: usize) -> String { @@ -560,6 +586,56 @@ fn cell_xml(value: Option<&Value>, row_index: usize, col_index: usize, style: Op } } +fn push_cell_style(output: &mut String, style: Option) { + if let Some(style) = style { + write!(output, " s=\"{style}\"").expect("writing XLSX cell styles into a String cannot fail"); + } +} + +fn push_cell_xml(output: &mut String, value: Option<&Value>, row_index: usize, col_index: usize, style: Option) { + output.push_str(" { + push_cell_style(output, style); + output.push_str("/>"); + } + Some(Value::Bool(value)) => { + let bool_value = if *value { 1 } else { 0 }; + output.push_str(" t=\"b\""); + push_cell_style(output, style); + write!(output, ">{bool_value}").expect("writing XLSX booleans into a String cannot fail"); + } + Some(Value::Number(value)) => { + if value.as_f64().is_some_and(|number| number.is_finite()) { + push_cell_style(output, style); + write!(output, ">{value}").expect("writing XLSX numbers into a String cannot fail"); + } else { + output.push_str(" t=\"inlineStr\""); + push_cell_style(output, style); + output.push_str(">"); + push_xml_escaped(output, &value.to_string()); + output.push_str(""); + } + } + Some(Value::String(value)) => { + output.push_str(" t=\"inlineStr\""); + push_cell_style(output, style); + output.push_str(">"); + push_xml_escaped(output, value); + output.push_str(""); + } + Some(value) => { + output.push_str(" t=\"inlineStr\""); + push_cell_style(output, style); + output.push_str(">"); + push_xml_escaped(output, &value.to_string()); + output.push_str(""); + } + } +} + fn is_numeric_column_type(column_type: Option<&String>) -> bool { let mut normalized = column_type.map(|value| value.trim().to_ascii_lowercase()).unwrap_or_default(); while normalized.ends_with(')') { @@ -659,36 +735,43 @@ fn safe_excel_number(value: &str) -> Option<&str> { (significant_digits <= 15).then_some(trimmed) } -fn typed_cell_xml( +fn push_typed_cell_xml( + output: &mut String, value: Option<&Value>, column_type: Option<&String>, row_index: usize, col_index: usize, style: Option, date_time_format: Option<&str>, -) -> String { +) { if let Some(Value::String(value)) = value { if let Some((serial, temporal_kind)) = excel_temporal_serial(value, column_type.map(String::as_str), date_time_format) { - let reference = cell_ref(row_index, col_index); let style = match temporal_kind { ExcelTemporalKind::Date => XLSX_DATE_STYLE, ExcelTemporalKind::DateTime => XLSX_DATETIME_STYLE, }; - return format!("{serial}"); + output.push_str("{serial}") + .expect("writing XLSX temporal cells into a String cannot fail"); + return; } } if is_numeric_column_type(column_type) { if let Some(Value::String(value)) = value { if let Some(number) = safe_excel_number(value) { - let reference = cell_ref(row_index, col_index); - let style_attr = style.map_or(String::new(), |style| format!(" s=\"{style}\"")); - return format!("{number}"); + output.push_str("{number}").expect("writing XLSX numeric cells into a String cannot fail"); + return; } } } - cell_xml(value, row_index, col_index, style) + push_cell_xml(output, value, row_index, col_index, style); } fn write_worksheet_xml(writer: &mut W, segment: &WorksheetSegment) -> Result<(), String> { @@ -718,21 +801,20 @@ fn write_worksheet_xml(writer: &mut W, segment: &WorksheetSegment) -> .write_all(header_row_xml(segment.columns, segment.column_comments).as_bytes()) .map_err(|err| err.to_string())?; + let mut row_buffer = String::with_capacity(segment.columns.len().saturating_mul(48)); for (row_index, row) in segment.rows.iter().enumerate() { let excel_row = row_index + 2; - writer - .write_all( - data_row_xml_with_date_time_format( - excel_row, - segment.columns, - segment.column_types, - row, - None, - segment.numeric_column_right_align, - ) - .as_bytes(), - ) - .map_err(|err| err.to_string())?; + row_buffer.clear(); + push_data_row_xml( + &mut row_buffer, + excel_row, + segment.columns, + segment.column_types, + row, + None, + segment.numeric_column_right_align, + ); + writer.write_all(row_buffer.as_bytes()).map_err(|err| err.to_string())?; } writer @@ -1983,4 +2065,17 @@ mod tests { let sheet2 = read_zip_entry(&data, "xl/worksheets/sheet2.xml"); assert!(sheet2.contains("row_3"), "row 3 should be on sheet 2: {sheet2}"); } + + #[test] + fn reusable_cell_buffer_matches_reference_xml() { + let values = [Value::Null, json!(true), json!(42.5), json!("a<&\"b"), json!({"key": "value"})]; + for (index, value) in values.iter().enumerate() { + for style in [None, Some(1), Some(4)] { + let expected = super::cell_xml(Some(value), index, index + 1, style); + let mut actual = String::new(); + super::push_cell_xml(&mut actual, Some(value), index, index + 1, style); + assert_eq!(actual, expected, "value={value:?}, style={style:?}"); + } + } + } }