fix(transfer): reduce import and export allocations

This commit is contained in:
t8y2 committed 2026-08-21 01:10:17 +08:00
1 parent b5d3343f9a
commit 30f75d8388
9 files changed
+822 -261

No files matched your search

@@ -0,0 +1,154 @@
use std::time::Instant;
use dbx_core::csv_export::format_query_result_csv_rows;
use dbx_core::database_export::{build_export_insert_statements, BuildExportInsertStatementsOptions};
use dbx_core::models::connection::DatabaseType;
use dbx_core::table_import::{build_import_insert_batches, ParsedImportFile, TableImportColumnMapping};
use dbx_core::xlsx_export::{build_xlsx_workbook, XlsxWorksheetData};
struct Options {
rows: usize,
columns: usize,
batch_size: usize,
}
fn parse_options() -> Result<Options, String> {
let mut options = Options { rows: 20_000, columns: 10, batch_size: 500 };
for argument in std::env::args().skip(1) {
let (key, value) = argument.split_once('=').ok_or_else(|| format!("Invalid option: {argument}"))?;
match key {
"--rows" => options.rows = value.parse().map_err(|_| format!("Invalid row count: {value}"))?,
"--columns" => options.columns = value.parse().map_err(|_| format!("Invalid column count: {value}"))?,
"--batch-size" => options.batch_size = value.parse().map_err(|_| format!("Invalid batch size: {value}"))?,
_ => return Err(format!("Unknown option: {key}")),
}
}
if options.rows == 0 || options.columns == 0 || options.batch_size == 0 {
return Err("rows, columns, and batch-size must be greater than zero".to_string());
}
Ok(options)
}
fn columns(count: usize) -> Vec<String> {
(0..count).map(|index| format!("column_{}", index + 1)).collect()
}
fn rows(row_count: usize, column_count: usize) -> Vec<Vec<serde_json::Value>> {
(0..row_count)
.map(|row_index| {
(0..column_count)
.map(|column_index| {
if column_index == 0 {
serde_json::json!(row_index + 1)
} else if column_index % 3 == 0 {
serde_json::json!(format!("2026-07-{:02} 12:34:56", row_index % 28 + 1))
} else {
serde_json::json!(format!("value-{row_index}-{column_index}"))
}
})
.collect()
})
.collect()
}
fn elapsed_ms(started_at: Instant) -> f64 {
started_at.elapsed().as_secs_f64() * 1000.0
}
fn run() -> Result<(), String> {
let options = parse_options()?;
let columns = columns(options.columns);
let rows = rows(options.rows, options.columns);
let mappings = columns
.iter()
.map(|column| TableImportColumnMapping {
source_column: column.clone(),
target_column: column.clone(),
target_data_type: None,
})
.collect::<Vec<_>>();
let parsed = ParsedImportFile {
columns: columns.clone(),
rows: rows.clone(),
total_rows: rows.len(),
effective_encoding: None,
};
let import_started = Instant::now();
let import_batches = build_import_insert_batches(
&parsed,
&mappings,
&[],
"benchmark_import",
"main",
&DatabaseType::Sqlite,
options.batch_size,
)?;
let import_ms = elapsed_ms(import_started);
let csv_started = Instant::now();
let csv = format_query_result_csv_rows(&rows);
let csv_ms = elapsed_ms(csv_started);
let sql_started = Instant::now();
let sql = build_export_insert_statements(BuildExportInsertStatementsOptions {
database_type: Some(DatabaseType::Sqlite),
identifier_quote: None,
schema: Some("main".to_string()),
table_name: Some("benchmark_export".to_string()),
qualified_table_name: None,
columns: columns.clone(),
column_types: vec![None; columns.len()],
column_extras: Vec::new(),
rows: rows.clone(),
batch_size: Some(options.batch_size),
})?;
let sql_ms = elapsed_ms(sql_started);
let xlsx_started = Instant::now();
let xlsx = build_xlsx_workbook(&XlsxWorksheetData {
sheet_name: Some("Benchmark".to_string()),
columns,
column_types: Vec::new(),
column_comments: Vec::new(),
rows,
numeric_column_right_align: false,
})?;
let xlsx_ms = elapsed_ms(xlsx_started);
println!(
"{}",
serde_json::to_string_pretty(&serde_json::json!({
"rows": options.rows,
"columns": options.columns,
"batchSize": options.batch_size,
"importSql": {
"milliseconds": import_ms,
"batches": import_batches.len(),
"bytes": import_batches.iter().map(|batch| batch.sql.len()).sum::<usize>(),
},
"csv": {
"milliseconds": csv_ms,
"bytes": csv.len(),
},
"sqlExport": {
"milliseconds": sql_ms,
"statements": sql.len(),
"bytes": sql.iter().map(String::len).sum::<usize>(),
},
"xlsx": {
"milliseconds": xlsx_ms,
"bytes": xlsx.len(),
},
}))
.map_err(|error| error.to_string())?
);
Ok(())
}
fn main() {
if let Err(error) = run() {
eprintln!("{error}");
std::process::exit(1);
}
}
+47 -12
View File
@@ -100,6 +100,15 @@ fn push_tsv_value(out: &mut String, value: &Value) {
} }
} }
pub(crate) fn push_tsv_row(out: &mut String, row: &[Value]) {
for (cell_index, cell) in row.iter().enumerate() {
if cell_index > 0 {
out.push('\t');
}
push_tsv_value(out, cell);
}
}
/// 预分配粗估:按全部行的实际单元格数求和(不假设等宽),饱和运算防溢出, /// 预分配粗估:按全部行的实际单元格数求和(不假设等宽),饱和运算防溢出,
/// 并设上限——估算只是性能提示,绝不能因病态输入放大成巨额分配 /// 并设上限——估算只是性能提示,绝不能因病态输入放大成巨额分配
const ROWS_CAPACITY_ESTIMATE_MAX: usize = 16 * 1024 * 1024; const ROWS_CAPACITY_ESTIMATE_MAX: usize = 16 * 1024 * 1024;
@@ -133,15 +142,11 @@ fn push_tsv_rows(out: &mut String, rows: &[Vec<Value>]) {
if row_index > 0 { if row_index > 0 {
out.push('\n'); out.push('\n');
} }
for (cell_index, cell) in row.iter().enumerate() { push_tsv_row(out, row);
if cell_index > 0 {
out.push('\t');
}
push_tsv_value(out, cell);
}
} }
} }
#[cfg_attr(not(test), allow(dead_code))]
pub(crate) fn format_tsv_rows(rows: &[Vec<Value>]) -> String { pub(crate) fn format_tsv_rows(rows: &[Vec<Value>]) -> String {
let mut out = String::with_capacity(estimated_rows_capacity(rows)); let mut out = String::with_capacity(estimated_rows_capacity(rows));
push_tsv_rows(&mut out, rows); push_tsv_rows(&mut out, rows);
@@ -166,17 +171,30 @@ pub(crate) fn format_tsv(columns: &[String], rows: &[Vec<Value>]) -> String {
/// Format query-result rows as CSV text without a header row. Database NULLs /// Format query-result rows as CSV text without a header row. Database NULLs
/// use the same empty-cell representation as table-data exports. Used by the /// use the same empty-cell representation as table-data exports. Used by the
/// streaming query-result export for batches after the first. /// streaming query-result export for batches after the first.
pub(crate) fn push_query_result_csv_row(out: &mut String, row: &[Value]) {
for (cell_index, cell) in row.iter().enumerate() {
if cell_index > 0 {
out.push(',');
}
push_csv_value(out, cell);
}
}
pub(crate) fn push_table_csv_row(out: &mut String, row: &[Value]) {
for (cell_index, cell) in row.iter().enumerate() {
if cell_index > 0 {
out.push(',');
}
push_csv_text_value(out, cell);
}
}
fn push_query_result_csv_rows(out: &mut String, rows: &[Vec<Value>]) { fn push_query_result_csv_rows(out: &mut String, rows: &[Vec<Value>]) {
for (row_index, row) in rows.iter().enumerate() { for (row_index, row) in rows.iter().enumerate() {
if row_index > 0 { if row_index > 0 {
out.push('\n'); out.push('\n');
} }
for (cell_index, cell) in row.iter().enumerate() { push_query_result_csv_row(out, row);
if cell_index > 0 {
out.push(',');
}
push_csv_value(out, cell);
}
} }
} }
@@ -399,5 +417,22 @@ mod tests {
} }
} }
#[test]
fn reusable_row_buffers_match_batch_formatters() {
let row = vec![Value::Null, json!("NULL"), json!("line\n\"two\""), json!(42)];
let mut csv = String::new();
super::push_query_result_csv_row(&mut csv, &row);
assert_eq!(csv, super::format_query_result_csv_rows(&[row.clone()]));
let mut table_csv = String::new();
super::push_table_csv_row(&mut table_csv, &row);
assert_eq!(table_csv, "\"\",\"NULL\",\"line\n\"\"two\"\"\",\"42\"");
let mut tsv = String::new();
super::push_tsv_row(&mut tsv, &row);
assert_eq!(tsv, super::format_tsv_rows(&[row]));
}
use serde_json::Value; use serde_json::Value;
} }
+45 -36
View File
@@ -994,54 +994,63 @@ pub fn build_export_insert_statements(options: BuildExportInsertStatementsOption
let statement_overhead_bytes = export_sql_statement_bytes(options.database_type, &statement_prefix) + 1; let statement_overhead_bytes = export_sql_statement_bytes(options.database_type, &statement_prefix) + 1;
let target_statement_bytes = DATABASE_EXPORT_TARGET_STATEMENT_BYTES; let target_statement_bytes = DATABASE_EXPORT_TARGET_STATEMENT_BYTES;
let separator_bytes = export_sql_statement_bytes(options.database_type, ", "); let separator_bytes = export_sql_statement_bytes(options.database_type, ", ");
let mut current_values = Vec::with_capacity(batch_size); let mut current_values = String::new();
let mut current_values_bytes = 0usize; let mut current_values_bytes = 0usize;
let mut current_row_count = 0usize;
let flush_values = |statements: &mut Vec<String>, values: &mut Vec<String>, values_bytes: &mut usize| { let flush_values =
if values.is_empty() { |statements: &mut Vec<String>, values: &mut String, values_bytes: &mut usize, row_count: &mut usize| {
return; if *row_count == 0 {
} return;
let insert_sql = format!("{statement_prefix}{};", values.join(", ")); }
if needs_dameng_identity_insert { let mut insert_sql = String::with_capacity(statement_prefix.len() + values.len() + 1);
statements.push(wrap_dameng_identity_insert_sql_for_table(&insert_sql, &table)); insert_sql.push_str(&statement_prefix);
} else { insert_sql.push_str(values);
statements.push(insert_sql); insert_sql.push(';');
} if needs_dameng_identity_insert {
values.clear(); statements.push(wrap_dameng_identity_insert_sql_for_table(&insert_sql, &table));
*values_bytes = 0; } else {
}; statements.push(insert_sql);
}
values.clear();
*values_bytes = 0;
*row_count = 0;
};
for row in options.rows { for row in options.rows {
let row_values = insert_columns let mut rendered_row = String::with_capacity(insert_columns.len().saturating_mul(16).saturating_add(2));
.iter() rendered_row.push('(');
.map(|(index, _, sqlserver_unicode_string)| { for (column_index, (index, _, sqlserver_unicode_string)) in insert_columns.iter().enumerate() {
let value = row.get(*index).unwrap_or(&Value::Null); if column_index > 0 {
format_export_sql_literal_typed( rendered_row.push_str(", ");
value, }
options.database_type, let value = row.get(*index).unwrap_or(&Value::Null);
options.column_types.get(*index).and_then(|value| value.as_deref()), rendered_row.push_str(&format_export_sql_literal_typed(
*sqlserver_unicode_string, value,
) options.database_type,
}) options.column_types.get(*index).and_then(|value| value.as_deref()),
.collect::<Vec<_>>() *sqlserver_unicode_string,
.join(", "); ));
let rendered_row = format!("({row_values})"); }
rendered_row.push(')');
let rendered_row_bytes = export_sql_statement_bytes(options.database_type, &rendered_row); let rendered_row_bytes = export_sql_statement_bytes(options.database_type, &rendered_row);
let candidate_bytes = statement_overhead_bytes let candidate_bytes = statement_overhead_bytes
+ current_values_bytes + current_values_bytes
+ if current_values.is_empty() { 0 } else { separator_bytes } + if current_row_count == 0 { 0 } else { separator_bytes }
+ rendered_row_bytes; + rendered_row_bytes;
if !current_values.is_empty() if current_row_count > 0 && (current_row_count >= batch_size || candidate_bytes > target_statement_bytes) {
&& (current_values.len() >= batch_size || candidate_bytes > target_statement_bytes) flush_values(&mut statements, &mut current_values, &mut current_values_bytes, &mut current_row_count);
{
flush_values(&mut statements, &mut current_values, &mut current_values_bytes);
} }
current_values_bytes += if current_values.is_empty() { 0 } else { separator_bytes }; if current_row_count > 0 {
current_values.push_str(", ");
current_values_bytes += separator_bytes;
}
current_values.push_str(&rendered_row);
current_values_bytes += rendered_row_bytes; current_values_bytes += rendered_row_bytes;
current_values.push(rendered_row); current_row_count += 1;
} }
flush_values(&mut statements, &mut current_values, &mut current_values_bytes); flush_values(&mut statements, &mut current_values, &mut current_values_bytes, &mut current_row_count);
Ok(statements) Ok(statements)
} }
+78 -35
View File
@@ -10,7 +10,7 @@ use std::sync::{
use std::time::{Duration, Instant}; use std::time::{Duration, Instant};
use crate::connection::{AppState, PoolKind}; use crate::connection::{AppState, PoolKind};
use crate::csv_export::{format_query_result_csv, format_query_result_csv_rows, format_tsv, format_tsv_rows}; use crate::csv_export::{format_query_result_csv, format_tsv, push_query_result_csv_row, push_tsv_row};
pub use crate::database_export::ExportStatus; pub use crate::database_export::ExportStatus;
use crate::database_export::{build_export_insert_statements, is_export_cancelled, BuildExportInsertStatementsOptions}; use crate::database_export::{build_export_insert_statements, is_export_cancelled, BuildExportInsertStatementsOptions};
use crate::models::connection::DatabaseType; use crate::models::connection::DatabaseType;
@@ -400,12 +400,38 @@ fn format_text_export_header(format: &str, columns: &[String]) -> String {
content.strip_suffix('\n').unwrap_or(&content).to_string() content.strip_suffix('\n').unwrap_or(&content).to_string()
} }
fn format_text_export_rows(format: &str, rows: &[Vec<Value>]) -> String { fn write_text_export_row<W: Write>(
file: &mut W,
format: &str,
row: &[Value],
buffer: &mut String,
) -> Result<(), String> {
buffer.clear();
buffer.push('\n');
if format == "csv" { if format == "csv" {
format_query_result_csv_rows(rows) push_query_result_csv_row(buffer, row);
} else { } else {
format_tsv_rows(rows) push_tsv_row(buffer, row);
} }
file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}"))
}
fn write_text_export_rows<W: Write>(
file: &mut W,
format: &str,
rows: &[Vec<Value>],
buffer: &mut String,
) -> Result<(), String> {
buffer.clear();
for row in rows {
buffer.push('\n');
if format == "csv" {
push_query_result_csv_row(buffer, row);
} else {
push_tsv_row(buffer, row);
}
}
file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}"))
} }
fn should_emit_stream_progress( fn should_emit_stream_progress(
@@ -683,6 +709,7 @@ async fn export_query_result_core_inner(
} }
let mut xlsx = None; let mut xlsx = None;
let mut text_buffer = String::new();
let mut columns: Vec<String> = Vec::new(); let mut columns: Vec<String> = Vec::new();
let mut column_types: Vec<String> = Vec::new(); let mut column_types: Vec<String> = Vec::new();
let mut rows_exported: u64 = 0; let mut rows_exported: u64 = 0;
@@ -838,7 +865,7 @@ async fn export_query_result_core_inner(
result.rows.truncate(this_page); result.rows.truncate(this_page);
} }
let row_count = result.rows.len(); let row_count = result.rows.len();
let formatted_rows = crate::temporal_format::format_temporal_export_rows_with_string_types( let formatted_rows = crate::temporal_format::format_temporal_export_rows_with_string_types_cow(
&result.rows, &result.rows,
&column_types, &column_types,
request.date_time_format.as_deref(), request.date_time_format.as_deref(),
@@ -850,18 +877,16 @@ async fn export_query_result_core_inner(
let header = format_text_export_header(&format, &columns); let header = format_text_export_header(&format, &columns);
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?; file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
if row_count > 0 { if row_count > 0 {
let rows = format_text_export_rows(&format, &formatted_rows); write_text_export_rows(file, &format, formatted_rows.as_ref(), &mut text_buffer)?;
write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?;
} }
wrote_text_header = true; wrote_text_header = true;
} else if row_count > 0 { } else if row_count > 0 {
let rows = format_text_export_rows(&format, &formatted_rows); write_text_export_rows(file, &format, formatted_rows.as_ref(), &mut text_buffer)?;
write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?;
} }
} }
} else if format == "sql" { } else if format == "sql" {
let writer = sql_writer.as_mut().ok_or_else(|| "SQL export writer missing".to_string())?; let writer = sql_writer.as_mut().ok_or_else(|| "SQL export writer missing".to_string())?;
for row in formatted_rows { for row in formatted_rows.into_owned() {
writer.write_row(row)?; writer.write_row(row)?;
} }
} else { } else {
@@ -876,7 +901,7 @@ async fn export_query_result_core_inner(
)?); )?);
} }
if let Some(writer) = xlsx.as_mut() { if let Some(writer) = xlsx.as_mut() {
for row in &formatted_rows { for row in formatted_rows.as_ref() {
writer.write_row(row).map_err(|e| format!("Failed to write XLSX row: {e}"))?; writer.write_row(row).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
} }
} }
@@ -1013,6 +1038,7 @@ async fn try_export_postgres_query_result_stream(
None None
}; };
let mut xlsx = None; let mut xlsx = None;
let mut text_buffer = String::new();
let mut sql_writer: Option<SqlInsertWriter> = let mut sql_writer: Option<SqlInsertWriter> =
if format == "sql" { Some(SqlInsertWriter::create(request)?) } else { None }; if format == "sql" { Some(SqlInsertWriter::create(request)?) } else { None };
let budget = operation_budget_for_pool_key(state, &pool_key, query_export_timeout(request.timeout_secs)).await; let budget = operation_budget_for_pool_key(state, &pool_key, query_export_timeout(request.timeout_secs)).await;
@@ -1050,25 +1076,26 @@ async fn try_export_postgres_query_result_stream(
} }
} }
crate::db::postgres::PostgresQueryStreamItem::Row(row) => { crate::db::postgres::PostgresQueryStreamItem::Row(row) => {
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types( let formatted = crate::temporal_format::format_temporal_export_row_with_string_types_cow(
&row, &row,
&temporal_column_types, &temporal_column_types,
request.date_time_format.as_deref(), request.date_time_format.as_deref(),
); );
if let Some(writer) = sql_writer.as_mut() { if let Some(writer) = sql_writer.as_mut() {
writer.write_row(formatted)?; writer.write_row(formatted.into_owned())?;
} else if let Some(file) = text_file.as_mut() { } else if let Some(file) = text_file.as_mut() {
let rows = format_text_export_rows(format, std::slice::from_ref(&formatted)); write_text_export_row(file, format, formatted.as_ref(), &mut text_buffer)?;
write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?;
} else if let Some(writer) = xlsx.as_mut() { } else if let Some(writer) = xlsx.as_mut() {
writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?; writer.write_row(formatted.as_ref()).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
} else { } else {
let xlsx_file = let xlsx_file =
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?; File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
xlsx = xlsx =
Some(start_query_result_xlsx_workbook(BufWriter::new(xlsx_file), request, &columns, &[])?); Some(start_query_result_xlsx_workbook(BufWriter::new(xlsx_file), request, &columns, &[])?);
if let Some(writer) = xlsx.as_mut() { if let Some(writer) = xlsx.as_mut() {
writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?; writer
.write_row(formatted.as_ref())
.map_err(|e| format!("Failed to write XLSX row: {e}"))?;
} }
} }
rows_exported += 1; rows_exported += 1;
@@ -1205,6 +1232,7 @@ async fn try_export_mysql_query_result_stream(
None None
}; };
let mut xlsx = None; let mut xlsx = None;
let mut text_buffer = String::new();
let mut sql_writer: Option<SqlInsertWriter> = let mut sql_writer: Option<SqlInsertWriter> =
if format == "sql" { Some(SqlInsertWriter::create(request)?) } else { None }; if format == "sql" { Some(SqlInsertWriter::create(request)?) } else { None };
let query_timeout = query_export_timeout(request.timeout_secs); let query_timeout = query_export_timeout(request.timeout_secs);
@@ -1292,25 +1320,26 @@ async fn try_export_mysql_query_result_stream(
} }
} }
crate::db::mysql::MySqlQueryStreamItem::Row(row) => { crate::db::mysql::MySqlQueryStreamItem::Row(row) => {
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types( let formatted = crate::temporal_format::format_temporal_export_row_with_string_types_cow(
&row, &row,
&temporal_column_types, &temporal_column_types,
request.date_time_format.as_deref(), request.date_time_format.as_deref(),
); );
if let Some(writer) = sql_writer.as_mut() { if let Some(writer) = sql_writer.as_mut() {
writer.write_row(formatted)?; writer.write_row(formatted.into_owned())?;
} else if let Some(file) = text_file.as_mut() { } else if let Some(file) = text_file.as_mut() {
let rows = format_text_export_rows(format, std::slice::from_ref(&formatted)); write_text_export_row(file, format, formatted.as_ref(), &mut text_buffer)?;
write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?;
} else if let Some(writer) = xlsx.as_mut() { } else if let Some(writer) = xlsx.as_mut() {
writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?; writer.write_row(formatted.as_ref()).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
} else { } else {
let xlsx_file = let xlsx_file =
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?; File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
xlsx = xlsx =
Some(start_query_result_xlsx_workbook(BufWriter::new(xlsx_file), request, &columns, &[])?); Some(start_query_result_xlsx_workbook(BufWriter::new(xlsx_file), request, &columns, &[])?);
if let Some(writer) = xlsx.as_mut() { if let Some(writer) = xlsx.as_mut() {
writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?; writer
.write_row(formatted.as_ref())
.map_err(|e| format!("Failed to write XLSX row: {e}"))?;
} }
} }
rows_exported += 1; rows_exported += 1;
@@ -1467,6 +1496,7 @@ async fn try_export_clickhouse_query_result_stream(
None None
}; };
let mut xlsx = None; let mut xlsx = None;
let mut text_buffer = String::new();
let mut sql_writer: Option<SqlInsertWriter> = let mut sql_writer: Option<SqlInsertWriter> =
if format == "sql" { Some(SqlInsertWriter::create(request)?) } else { None }; if format == "sql" { Some(SqlInsertWriter::create(request)?) } else { None };
let query_timeout = query_export_timeout(request.timeout_secs); let query_timeout = query_export_timeout(request.timeout_secs);
@@ -1505,25 +1535,26 @@ async fn try_export_clickhouse_query_result_stream(
} }
} }
crate::db::clickhouse_driver::ClickHouseQueryStreamItem::Row(row) => { crate::db::clickhouse_driver::ClickHouseQueryStreamItem::Row(row) => {
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types( let formatted = crate::temporal_format::format_temporal_export_row_with_string_types_cow(
&row, &row,
&temporal_column_types, &temporal_column_types,
request.date_time_format.as_deref(), request.date_time_format.as_deref(),
); );
if let Some(writer) = sql_writer.as_mut() { if let Some(writer) = sql_writer.as_mut() {
writer.write_row(formatted)?; writer.write_row(formatted.into_owned())?;
} else if let Some(file) = text_file.as_mut() { } else if let Some(file) = text_file.as_mut() {
let rows = format_text_export_rows(format, std::slice::from_ref(&formatted)); write_text_export_row(file, format, formatted.as_ref(), &mut text_buffer)?;
write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?;
} else if let Some(writer) = xlsx.as_mut() { } else if let Some(writer) = xlsx.as_mut() {
writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?; writer.write_row(formatted.as_ref()).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
} else { } else {
let xlsx_file = let xlsx_file =
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?; File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
xlsx = xlsx =
Some(start_query_result_xlsx_workbook(BufWriter::new(xlsx_file), request, &columns, &[])?); Some(start_query_result_xlsx_workbook(BufWriter::new(xlsx_file), request, &columns, &[])?);
if let Some(writer) = xlsx.as_mut() { if let Some(writer) = xlsx.as_mut() {
writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?; writer
.write_row(formatted.as_ref())
.map_err(|e| format!("Failed to write XLSX row: {e}"))?;
} }
} }
rows_exported += 1; rows_exported += 1;
@@ -1635,6 +1666,7 @@ async fn try_export_sqlserver_query_result_stream(
None None
}; };
let mut xlsx = None; let mut xlsx = None;
let mut text_buffer = String::new();
let mut sql_writer: Option<SqlInsertWriter> = let mut sql_writer: Option<SqlInsertWriter> =
if format == "sql" { Some(SqlInsertWriter::create(request)?) } else { None }; if format == "sql" { Some(SqlInsertWriter::create(request)?) } else { None };
let query_timeout = query_export_timeout(request.timeout_secs); let query_timeout = query_export_timeout(request.timeout_secs);
@@ -1683,25 +1715,26 @@ async fn try_export_sqlserver_query_result_stream(
} }
} }
crate::db::sqlserver::SqlServerStreamItem::Row(row) => { crate::db::sqlserver::SqlServerStreamItem::Row(row) => {
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types( let formatted = crate::temporal_format::format_temporal_export_row_with_string_types_cow(
row, row,
&temporal_column_types, &temporal_column_types,
request.date_time_format.as_deref(), request.date_time_format.as_deref(),
); );
if let Some(writer) = sql_writer.as_mut() { if let Some(writer) = sql_writer.as_mut() {
writer.write_row(formatted)?; writer.write_row(formatted.into_owned())?;
} else if let Some(file) = text_file.as_mut() { } else if let Some(file) = text_file.as_mut() {
let rows = format_text_export_rows(format, std::slice::from_ref(&formatted)); write_text_export_row(file, format, formatted.as_ref(), &mut text_buffer)?;
write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?;
} else if let Some(writer) = xlsx.as_mut() { } else if let Some(writer) = xlsx.as_mut() {
writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?; writer.write_row(formatted.as_ref()).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
} else { } else {
let xlsx_file = let xlsx_file =
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?; File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
xlsx = xlsx =
Some(start_query_result_xlsx_workbook(BufWriter::new(xlsx_file), request, &columns, &[])?); Some(start_query_result_xlsx_workbook(BufWriter::new(xlsx_file), request, &columns, &[])?);
if let Some(writer) = xlsx.as_mut() { if let Some(writer) = xlsx.as_mut() {
writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?; writer
.write_row(formatted.as_ref())
.map_err(|e| format!("Failed to write XLSX row: {e}"))?;
} }
} }
rows_exported += 1; rows_exported += 1;
@@ -1893,6 +1926,16 @@ mod tests {
assert_eq!(format_text_export_header("txt", &["id".to_string(), "note".to_string()]), "id\tnote"); assert_eq!(format_text_export_header("txt", &["id".to_string(), "note".to_string()]), "id\tnote");
} }
#[test]
fn reusable_text_row_buffer_preserves_query_null_semantics() {
let row = vec![Value::Null, serde_json::json!(""), serde_json::json!("line\n\"two\"")];
let mut output = Vec::new();
let mut buffer = String::new();
write_text_export_row(&mut output, "csv", &row, &mut buffer).expect("write csv row");
assert_eq!(String::from_utf8(output).expect("utf8 csv"), "\n,\"\",\"line\n\"\"two\"\"\"");
}
#[test] #[test]
fn xlsx_no_row_limit_has_no_query_layer_cap() { fn xlsx_no_row_limit_has_no_query_layer_cap() {
// Without the old hard limit, XLSX uses the writer's internal splitting. // Without the old hard limit, XLSX uses the writer's internal splitting.
+67 -36
View File
@@ -9,7 +9,9 @@ use tokio_util::sync::CancellationToken;
use crate::connection::MysqlMode; use crate::connection::MysqlMode;
use crate::connection::{config_for_pool_key, task_client_session_id, AppState, PoolKind}; use crate::connection::{config_for_pool_key, task_client_session_id, AppState, PoolKind};
use crate::csv_export::{format_csv, format_tsv, format_tsv_rows, push_csv_text_value}; #[cfg(test)]
use crate::csv_export::format_tsv_rows;
use crate::csv_export::{format_csv, format_tsv, push_table_csv_row, push_tsv_row};
pub use crate::database_export::ExportStatus; pub use crate::database_export::ExportStatus;
use crate::database_export::{ use crate::database_export::{
build_export_insert_statements, is_export_cancelled, is_internal_export_column, BuildExportInsertStatementsOptions, build_export_insert_statements, is_export_cancelled, is_internal_export_column, BuildExportInsertStatementsOptions,
@@ -82,6 +84,7 @@ pub struct TableExportProgress {
/// Format rows as CSV text without a header row. /// Format rows as CSV text without a header row.
/// Used for streaming subsequent pagination batches. /// Used for streaming subsequent pagination batches.
#[cfg_attr(not(test), allow(dead_code))]
fn format_csv_rows(rows: &[Vec<Value>]) -> String { fn format_csv_rows(rows: &[Vec<Value>]) -> String {
// 注意:该无表头批次路径的 Null 输出为 ""(带引号空串),与查询结果导出 // 注意:该无表头批次路径的 Null 输出为 ""(带引号空串),与查询结果导出
// 及首批 format_csv 的裸空单元格语义不同,直写化必须保留该差异 // 及首批 format_csv 的裸空单元格语义不同,直写化必须保留该差异
@@ -90,16 +93,40 @@ fn format_csv_rows(rows: &[Vec<Value>]) -> String {
if row_index > 0 { if row_index > 0 {
out.push('\n'); out.push('\n');
} }
for (cell_index, cell) in row.iter().enumerate() { push_table_csv_row(&mut out, row);
if cell_index > 0 {
out.push(',');
}
push_csv_text_value(&mut out, cell);
}
} }
out out
} }
fn write_table_text_row<W: Write>(file: &mut W, csv: bool, row: &[Value], buffer: &mut String) -> Result<(), String> {
buffer.clear();
buffer.push('\n');
if csv {
push_table_csv_row(buffer, row);
} else {
push_tsv_row(buffer, row);
}
file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}"))
}
fn write_table_text_rows<W: Write>(
file: &mut W,
csv: bool,
rows: &[Vec<Value>],
buffer: &mut String,
) -> Result<(), String> {
buffer.clear();
for row in rows {
buffer.push('\n');
if csv {
push_table_csv_row(buffer, row);
} else {
push_tsv_row(buffer, row);
}
}
file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}"))
}
fn export_column_types(request: &TableExportRequest) -> Vec<String> { fn export_column_types(request: &TableExportRequest) -> Vec<String> {
request request
.column_types .column_types
@@ -928,6 +955,7 @@ async fn try_export_native_table_stream(
let header = format_csv(col_names, &[]); let header = format_csv(col_names, &[]);
let header = header.strip_suffix('\n').unwrap_or(&header); let header = header.strip_suffix('\n').unwrap_or(&header);
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?; file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?;
let mut row_buffer = String::new();
let result = stream_native_table_rows( let result = stream_native_table_rows(
state, state,
@@ -938,13 +966,12 @@ async fn try_export_native_table_stream(
&cancelled, &cancelled,
cancel_token.clone(), cancel_token.clone(),
|row| { |row| {
let formatted = crate::temporal_format::format_temporal_export_row( let formatted = crate::temporal_format::format_temporal_export_row_cow(
row, row,
column_types, column_types,
request.date_time_format.as_deref(), request.date_time_format.as_deref(),
); );
let row_csv = format_csv_rows(&[formatted]); write_table_text_row(&mut file, true, formatted.as_ref(), &mut row_buffer)?;
write!(file, "\n{row_csv}").map_err(|e| format!("Failed to write CSV rows: {e}"))?;
rows_exported += 1; rows_exported += 1;
if rows_exported.is_multiple_of(progress_interval) { if rows_exported.is_multiple_of(progress_interval) {
on_progress(TableExportProgress { on_progress(TableExportProgress {
@@ -972,6 +999,7 @@ async fn try_export_native_table_stream(
let header = format_tsv(col_names, &[]); let header = format_tsv(col_names, &[]);
let header = header.strip_suffix('\n').unwrap_or(&header); let header = header.strip_suffix('\n').unwrap_or(&header);
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write TXT: {e}"))?; file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write TXT: {e}"))?;
let mut row_buffer = String::new();
let result = stream_native_table_rows( let result = stream_native_table_rows(
state, state,
@@ -982,13 +1010,12 @@ async fn try_export_native_table_stream(
&cancelled, &cancelled,
cancel_token.clone(), cancel_token.clone(),
|row| { |row| {
let formatted = crate::temporal_format::format_temporal_export_row( let formatted = crate::temporal_format::format_temporal_export_row_cow(
row, row,
column_types, column_types,
request.date_time_format.as_deref(), request.date_time_format.as_deref(),
); );
let row_tsv = format_tsv_rows(&[formatted]); write_table_text_row(&mut file, false, formatted.as_ref(), &mut row_buffer)?;
write!(file, "\n{row_tsv}").map_err(|e| format!("Failed to write TXT rows: {e}"))?;
rows_exported += 1; rows_exported += 1;
if rows_exported.is_multiple_of(progress_interval) { if rows_exported.is_multiple_of(progress_interval) {
on_progress(TableExportProgress { on_progress(TableExportProgress {
@@ -1033,12 +1060,12 @@ async fn try_export_native_table_stream(
&cancelled, &cancelled,
cancel_token.clone(), cancel_token.clone(),
|row| { |row| {
let formatted = crate::temporal_format::format_temporal_export_row( let formatted = crate::temporal_format::format_temporal_export_row_cow(
row, row,
column_types, column_types,
request.date_time_format.as_deref(), request.date_time_format.as_deref(),
); );
writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?; writer.write_row(formatted.as_ref()).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
rows_exported += 1; rows_exported += 1;
if rows_exported.is_multiple_of(progress_interval) { if rows_exported.is_multiple_of(progress_interval) {
on_progress(TableExportProgress { on_progress(TableExportProgress {
@@ -1087,12 +1114,12 @@ async fn try_export_native_table_stream(
if !is_first_row { if !is_first_row {
file.write_all(b",\n").map_err(|e| format!("Failed to write JSON: {e}"))?; file.write_all(b",\n").map_err(|e| format!("Failed to write JSON: {e}"))?;
} }
let formatted = crate::temporal_format::format_temporal_export_row( let formatted = crate::temporal_format::format_temporal_export_row_cow(
row, row,
column_types, column_types,
request.date_time_format.as_deref(), request.date_time_format.as_deref(),
); );
write_json_row_object(&mut file, col_names, &formatted)?; write_json_row_object(&mut file, col_names, formatted.as_ref())?;
is_first_row = false; is_first_row = false;
rows_exported += 1; rows_exported += 1;
if rows_exported.is_multiple_of(progress_interval) { if rows_exported.is_multiple_of(progress_interval) {
@@ -1473,6 +1500,7 @@ async fn export_table_data_core_inner(
// 8. Create output file // 8. Create output file
let file = std::fs::File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?; let file = std::fs::File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?;
let mut file = BufWriter::new(file); let mut file = BufWriter::new(file);
let mut text_buffer = String::new();
let mut rows_exported: u64 = 0; let mut rows_exported: u64 = 0;
let batch_size = request.batch_size.unwrap_or(DEFAULT_BATCH_SIZE).max(1); let batch_size = request.batch_size.unwrap_or(DEFAULT_BATCH_SIZE).max(1);
@@ -1533,7 +1561,7 @@ async fn export_table_data_core_inner(
if row_count == 0 { if row_count == 0 {
break; break;
} }
let formatted_rows = crate::temporal_format::format_temporal_export_rows( let formatted_rows = crate::temporal_format::format_temporal_export_rows_cow(
&result.rows, &result.rows,
&column_types, &column_types,
request.date_time_format.as_deref(), request.date_time_format.as_deref(),
@@ -1541,15 +1569,12 @@ async fn export_table_data_core_inner(
if is_first_batch { if is_first_batch {
// First batch: write header + rows via format_csv // First batch: write header + rows via format_csv
let csv_content = format_csv(&col_names, &formatted_rows); let csv_content = format_csv(&col_names, formatted_rows.as_ref());
file.write_all(csv_content.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?; file.write_all(csv_content.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?;
is_first_batch = false; is_first_batch = false;
} else { } else {
// Subsequent batches: write rows only (prepend newline for separation) // Subsequent batches: write rows only (prepend newline for separation)
let rows_csv = format_csv_rows(&formatted_rows); write_table_text_rows(&mut file, true, formatted_rows.as_ref(), &mut text_buffer)?;
if !rows_csv.is_empty() {
write!(file, "\n{rows_csv}").map_err(|e| format!("Failed to write CSV rows: {e}"))?;
}
} }
rows_exported += row_count as u64; rows_exported += row_count as u64;
@@ -1624,21 +1649,17 @@ async fn export_table_data_core_inner(
if row_count == 0 { if row_count == 0 {
break; break;
} }
let formatted_rows = crate::temporal_format::format_temporal_export_rows( let formatted_rows = crate::temporal_format::format_temporal_export_rows_cow(
&result.rows, &result.rows,
&column_types, &column_types,
request.date_time_format.as_deref(), request.date_time_format.as_deref(),
); );
if is_first_batch { if is_first_batch {
let rows_tsv = format_tsv_rows(&formatted_rows); write_table_text_rows(&mut file, false, formatted_rows.as_ref(), &mut text_buffer)?;
write!(file, "\n{rows_tsv}").map_err(|e| format!("Failed to write TXT rows: {e}"))?;
is_first_batch = false; is_first_batch = false;
} else { } else {
let rows_tsv = format_tsv_rows(&formatted_rows); write_table_text_rows(&mut file, false, formatted_rows.as_ref(), &mut text_buffer)?;
if !rows_tsv.is_empty() {
write!(file, "\n{rows_tsv}").map_err(|e| format!("Failed to write TXT rows: {e}"))?;
}
} }
rows_exported += row_count as u64; rows_exported += row_count as u64;
@@ -1728,12 +1749,12 @@ async fn export_table_data_core_inner(
} }
for row in &result.rows { for row in &result.rows {
let formatted = crate::temporal_format::format_temporal_export_row( let formatted = crate::temporal_format::format_temporal_export_row_cow(
row, row,
&column_types, &column_types,
request.date_time_format.as_deref(), request.date_time_format.as_deref(),
); );
writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?; writer.write_row(formatted.as_ref()).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
} }
rows_exported += row_count as u64; rows_exported += row_count as u64;
@@ -1827,12 +1848,12 @@ async fn export_table_data_core_inner(
if !is_first_row { if !is_first_row {
file.write_all(b",\n").map_err(|e| format!("Failed to write JSON: {e}"))?; file.write_all(b",\n").map_err(|e| format!("Failed to write JSON: {e}"))?;
} }
let formatted = crate::temporal_format::format_temporal_export_row( let formatted = crate::temporal_format::format_temporal_export_row_cow(
row, row,
&column_types, &column_types,
request.date_time_format.as_deref(), request.date_time_format.as_deref(),
); );
write_json_row_object(&mut file, &col_names, &formatted)?; write_json_row_object(&mut file, &col_names, formatted.as_ref())?;
is_first_row = false; is_first_row = false;
} }
@@ -1908,12 +1929,12 @@ async fn export_table_data_core_inner(
break; break;
} }
let formatted_rows = crate::temporal_format::format_temporal_export_rows( let formatted_rows = crate::temporal_format::format_temporal_export_rows_cow(
&result.rows, &result.rows,
&column_types, &column_types,
request.date_time_format.as_deref(), request.date_time_format.as_deref(),
); );
let rows_markdown = format_markdown_rows(&formatted_rows); let rows_markdown = format_markdown_rows(formatted_rows.as_ref());
if !rows_markdown.is_empty() { if !rows_markdown.is_empty() {
if wrote_rows { if wrote_rows {
file.write_all(b"\n").map_err(|e| format!("Failed to write Markdown: {e}"))?; file.write_all(b"\n").map_err(|e| format!("Failed to write Markdown: {e}"))?;
@@ -2270,6 +2291,16 @@ mod tests {
assert_eq!(out, "\"just\",\"one\""); assert_eq!(out, "\"just\",\"one\"");
} }
#[test]
fn reusable_text_row_buffer_preserves_table_null_semantics() {
let row = vec![Value::Null, json!(""), json!("line\n\"two\"")];
let mut output = Vec::new();
let mut buffer = String::new();
write_table_text_row(&mut output, true, &row, &mut buffer).expect("write csv row");
assert_eq!(String::from_utf8(output).expect("utf8 csv"), "\n\"\",\"\",\"line\n\"\"two\"\"\"");
}
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
// format_tsv (Navicat-style TXT export) // format_tsv (Navicat-style TXT export)
// ----------------------------------------------------------------------- // -----------------------------------------------------------------------
+69 -40
View File
@@ -1,3 +1,4 @@
use std::borrow::Cow;
use std::collections::{HashMap, HashSet}; use std::collections::{HashMap, HashSet};
use std::fs::File; use std::fs::File;
use std::io::{BufRead, BufReader, Read as IoRead, Seek, SeekFrom, Write as IoWrite}; use std::io::{BufRead, BufReader, Read as IoRead, Seek, SeekFrom, Write as IoWrite};
@@ -19,8 +20,9 @@ use sha2::{Digest, Sha256};
use crate::connection::{task_client_session_id, AppState, PoolKind}; use crate::connection::{task_client_session_id, AppState, PoolKind};
use crate::models::connection::DatabaseType; use crate::models::connection::DatabaseType;
use crate::transfer::{ use crate::transfer::{
execute_on_pool, generate_insert_typed, generate_insert_typed_sql_batches, get_columns_for_transfer, escape_value_typed, execute_on_pool, generate_insert_typed_from_value_rows,
normalize_integer_literal, normalize_thousands_numeric_literal, qualified_table, quote_identifier, SqlBatchLimits, generate_insert_typed_sql_batches_from_value_rows, get_columns_for_transfer, normalize_integer_literal,
normalize_thousands_numeric_literal, qualified_table, quote_identifier, SqlBatchLimits,
}; };
pub const DEFAULT_PREVIEW_LIMIT: usize = 50; pub const DEFAULT_PREVIEW_LIMIT: usize = 50;
@@ -3142,9 +3144,8 @@ fn build_import_insert_batch_with_plan(
if rows.is_empty() { if rows.is_empty() {
return Ok(None); return Ok(None);
} }
let mapped_rows = map_import_rows_with_plan(rows, plan, db_type, kingbase_oracle_mode, date_time_format); let value_rows = import_value_rows_sql(rows, plan, db_type, kingbase_oracle_mode, date_time_format);
let sql = let sql = generate_insert_typed_from_value_rows(&plan.target_columns, &value_rows, table, schema, db_type, None);
generate_insert_typed(&plan.target_columns, &plan.column_types, &mapped_rows, table, schema, db_type, None);
Ok((!sql.trim().is_empty()).then_some(ImportSqlBatch { sql, row_count: rows.len() })) Ok((!sql.trim().is_empty()).then_some(ImportSqlBatch { sql, row_count: rows.len() }))
} }
@@ -3161,11 +3162,10 @@ fn build_import_insert_batches_with_plan(
if rows.is_empty() { if rows.is_empty() {
return Ok(Vec::new()); return Ok(Vec::new());
} }
let mapped_rows = map_import_rows_with_plan(rows, plan, db_type, kingbase_oracle_mode, date_time_format); let value_rows = import_value_rows_sql(rows, plan, db_type, kingbase_oracle_mode, date_time_format);
let batches = generate_insert_typed_sql_batches( let batches = generate_insert_typed_sql_batches_from_value_rows(
&plan.target_columns, &plan.target_columns,
&plan.column_types, &value_rows,
&mapped_rows,
table, table,
schema, schema,
db_type, db_type,
@@ -3175,16 +3175,31 @@ fn build_import_insert_batches_with_plan(
Ok(batches.into_iter().map(|(sql, row_count)| ImportSqlBatch { sql, row_count }).collect()) Ok(batches.into_iter().map(|(sql, row_count)| ImportSqlBatch { sql, row_count }).collect())
} }
fn map_import_rows_with_plan( fn import_value_rows_sql(
rows: &[Vec<serde_json::Value>], rows: &[Vec<serde_json::Value>],
plan: &CompiledImportPlan, plan: &CompiledImportPlan,
db_type: &DatabaseType, db_type: &DatabaseType,
kingbase_oracle_mode: bool, kingbase_oracle_mode: bool,
date_time_format: Option<&str>, date_time_format: Option<&str>,
) -> Vec<Vec<serde_json::Value>> { ) -> Vec<String> {
rows.iter() let mut value_rows = Vec::with_capacity(rows.len());
.map(|row| map_import_row_with_plan(row, plan, db_type, kingbase_oracle_mode, date_time_format)) for row in rows {
.collect() let mut values = String::with_capacity(plan.mapped_source_indexes.len().saturating_mul(16).saturating_add(2));
values.push('(');
for (target_index, source_index) in plan.mapped_source_indexes.iter().enumerate() {
if target_index > 0 {
values.push_str(", ");
}
let source_value = row.get(*source_index).unwrap_or(&serde_json::Value::Null);
let data_type = plan.column_types.get(target_index).and_then(|data_type| data_type.as_deref());
let normalized =
normalize_import_value_cow(source_value, data_type, db_type, kingbase_oracle_mode, date_time_format);
values.push_str(&escape_value_typed(normalized.as_ref(), db_type, data_type));
}
values.push(')');
value_rows.push(values);
}
value_rows
} }
fn map_import_row_with_plan( fn map_import_row_with_plan(
@@ -3274,17 +3289,17 @@ fn effective_import_batch_size(db_type: &DatabaseType, requested: usize) -> usiz
requested.max(1).min(max_rows) requested.max(1).min(max_rows)
} }
fn normalize_import_temporal_value( fn normalize_import_temporal_value_cow<'a>(
value: &serde_json::Value, value: &'a serde_json::Value,
data_type: Option<&str>, data_type: Option<&str>,
db_type: &DatabaseType, db_type: &DatabaseType,
kingbase_oracle_mode: bool, kingbase_oracle_mode: bool,
date_time_format: Option<&str>, date_time_format: Option<&str>,
) -> serde_json::Value { ) -> Cow<'a, serde_json::Value> {
let date_type_preserves_time = (matches!(db_type, DatabaseType::Oracle | DatabaseType::OceanbaseOracle) let date_type_preserves_time = (matches!(db_type, DatabaseType::Oracle | DatabaseType::OceanbaseOracle)
|| (*db_type == DatabaseType::Kingbase && kingbase_oracle_mode)) || (*db_type == DatabaseType::Kingbase && kingbase_oracle_mode))
&& data_type.is_some_and(|data_type| data_type.trim().eq_ignore_ascii_case("date")); && data_type.is_some_and(|data_type| data_type.trim().eq_ignore_ascii_case("date"));
crate::temporal_format::normalize_temporal_import_value( crate::temporal_format::normalize_temporal_import_value_cow(
value, value,
if date_type_preserves_time { Some("datetime") } else { data_type }, if date_type_preserves_time { Some("datetime") } else { data_type },
date_time_format, date_time_format,
@@ -3352,6 +3367,41 @@ fn textual_source_columns_for_import(
.collect() .collect()
} }
fn normalize_import_value_cow<'a>(
value: &'a serde_json::Value,
data_type: Option<&str>,
db_type: &DatabaseType,
kingbase_oracle_mode: bool,
date_time_format: Option<&str>,
) -> Cow<'a, serde_json::Value> {
let normalized =
normalize_import_temporal_value_cow(value, data_type, db_type, kingbase_oracle_mode, date_time_format);
// Strip validated thousands separators before integer canonicalization so "1,234.00"
// still collapses to a plain integer literal for integer targets.
let thousands_canonical =
normalized.as_str().and_then(|value| normalize_thousands_numeric_literal(value, db_type, data_type));
if let Some(integer_text) = thousands_canonical
.as_deref()
.or_else(|| normalized.as_str())
.and_then(|value| normalize_integer_literal(value, db_type, data_type))
.and_then(|value| value.parse::<i64>().ok())
{
// Normalize before both INSERT and COPY paths; COPY does not pass through SQL literal escaping.
return Cow::Owned(serde_json::Value::Number(integer_text.into()));
}
if let Some(number) = normalized.as_number() {
if let Some(integer_text) = normalize_integer_literal(&number.to_string(), db_type, data_type)
.and_then(|value| value.parse::<i64>().ok())
{
return Cow::Owned(serde_json::Value::Number(integer_text.into()));
}
}
if let Some(canonical) = thousands_canonical {
return Cow::Owned(serde_json::Value::String(canonical));
}
normalized
}
fn normalize_import_value( fn normalize_import_value(
value: &serde_json::Value, value: &serde_json::Value,
data_type: Option<&str>, data_type: Option<&str>,
@@ -3359,28 +3409,7 @@ fn normalize_import_value(
kingbase_oracle_mode: bool, kingbase_oracle_mode: bool,
date_time_format: Option<&str>, date_time_format: Option<&str>,
) -> serde_json::Value { ) -> serde_json::Value {
let normalized = normalize_import_temporal_value(value, data_type, db_type, kingbase_oracle_mode, date_time_format); normalize_import_value_cow(value, data_type, db_type, kingbase_oracle_mode, date_time_format).into_owned()
// Strip validated thousands separators before integer canonicalization so "1,234.00"
// still collapses to a plain integer literal for integer targets.
let thousands_canonical =
normalized.as_str().and_then(|value| normalize_thousands_numeric_literal(value, db_type, data_type));
let integer_text = thousands_canonical
.as_deref()
.or_else(|| normalized.as_str())
.map(str::to_owned)
.or_else(|| normalized.as_number().map(ToString::to_string));
if let Some(integer_text) = integer_text
.as_deref()
.and_then(|value| normalize_integer_literal(value, db_type, data_type))
.and_then(|value| value.parse::<i64>().ok())
{
// Normalize before both INSERT and COPY paths; COPY does not pass through SQL literal escaping.
return serde_json::Value::Number(integer_text.into());
}
if let Some(canonical) = thousands_canonical {
return serde_json::Value::String(canonical);
}
normalized
} }
pub fn build_import_insert_batches( pub fn build_import_insert_batches(
+131 -27
View File
@@ -1,5 +1,6 @@
use chrono::{DateTime, FixedOffset, NaiveDate, NaiveDateTime, NaiveTime, Timelike}; use chrono::{DateTime, FixedOffset, NaiveDate, NaiveDateTime, NaiveTime, Timelike};
use serde_json::Value; use serde_json::Value;
use std::borrow::Cow;
use std::fmt::Write as _; use std::fmt::Write as _;
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -192,17 +193,51 @@ pub fn format_temporal_export_value(value: &Value, data_type: Option<&str>, patt
.unwrap_or_else(|| value.clone()) .unwrap_or_else(|| value.clone())
} }
fn active_temporal_pattern(pattern: Option<&str>) -> Option<&str> {
pattern.filter(|pattern| !pattern.trim().is_empty())
}
pub fn format_temporal_export_row_cow<'a>(
row: &'a [Value],
column_types: &[Option<String>],
pattern: Option<&str>,
) -> Cow<'a, [Value]> {
let Some(pattern) = active_temporal_pattern(pattern) else {
return Cow::Borrowed(row);
};
if !column_types.iter().any(|data_type| temporal_kind(data_type.as_deref()).is_some()) {
return Cow::Borrowed(row);
}
Cow::Owned(
row.iter()
.enumerate()
.map(|(index, value)| {
format_temporal_export_value(
value,
column_types.get(index).and_then(|data_type| data_type.as_deref()),
Some(pattern),
)
})
.collect(),
)
}
pub fn format_temporal_export_row(row: &[Value], column_types: &[Option<String>], pattern: Option<&str>) -> Vec<Value> { pub fn format_temporal_export_row(row: &[Value], column_types: &[Option<String>], pattern: Option<&str>) -> Vec<Value> {
row.iter() format_temporal_export_row_cow(row, column_types, pattern).into_owned()
.enumerate() }
.map(|(index, value)| {
format_temporal_export_value( pub fn format_temporal_export_rows_cow<'a>(
value, rows: &'a [Vec<Value>],
column_types.get(index).and_then(|data_type| data_type.as_deref()), column_types: &[Option<String>],
pattern, pattern: Option<&str>,
) ) -> Cow<'a, [Vec<Value>]> {
}) let Some(pattern) = active_temporal_pattern(pattern) else {
.collect() return Cow::Borrowed(rows);
};
if !column_types.iter().any(|data_type| temporal_kind(data_type.as_deref()).is_some()) {
return Cow::Borrowed(rows);
}
Cow::Owned(rows.iter().map(|row| format_temporal_export_row(row, column_types, Some(pattern))).collect())
} }
pub fn format_temporal_export_rows( pub fn format_temporal_export_rows(
@@ -210,7 +245,28 @@ pub fn format_temporal_export_rows(
column_types: &[Option<String>], column_types: &[Option<String>],
pattern: Option<&str>, pattern: Option<&str>,
) -> Vec<Vec<Value>> { ) -> Vec<Vec<Value>> {
rows.iter().map(|row| format_temporal_export_row(row, column_types, pattern)).collect() format_temporal_export_rows_cow(rows, column_types, pattern).into_owned()
}
pub fn format_temporal_export_row_with_string_types_cow<'a>(
row: &'a [Value],
column_types: &[String],
pattern: Option<&str>,
) -> Cow<'a, [Value]> {
let Some(pattern) = active_temporal_pattern(pattern) else {
return Cow::Borrowed(row);
};
if !column_types.iter().any(|data_type| temporal_kind(Some(data_type)).is_some()) {
return Cow::Borrowed(row);
}
Cow::Owned(
row.iter()
.enumerate()
.map(|(index, value)| {
format_temporal_export_value(value, column_types.get(index).map(String::as_str), Some(pattern))
})
.collect(),
)
} }
pub fn format_temporal_export_row_with_string_types( pub fn format_temporal_export_row_with_string_types(
@@ -218,10 +274,23 @@ pub fn format_temporal_export_row_with_string_types(
column_types: &[String], column_types: &[String],
pattern: Option<&str>, pattern: Option<&str>,
) -> Vec<Value> { ) -> Vec<Value> {
row.iter() format_temporal_export_row_with_string_types_cow(row, column_types, pattern).into_owned()
.enumerate() }
.map(|(index, value)| format_temporal_export_value(value, column_types.get(index).map(String::as_str), pattern))
.collect() pub fn format_temporal_export_rows_with_string_types_cow<'a>(
rows: &'a [Vec<Value>],
column_types: &[String],
pattern: Option<&str>,
) -> Cow<'a, [Vec<Value>]> {
let Some(pattern) = active_temporal_pattern(pattern) else {
return Cow::Borrowed(rows);
};
if !column_types.iter().any(|data_type| temporal_kind(Some(data_type)).is_some()) {
return Cow::Borrowed(rows);
}
Cow::Owned(
rows.iter().map(|row| format_temporal_export_row_with_string_types(row, column_types, Some(pattern))).collect(),
)
} }
pub fn format_temporal_export_rows_with_string_types( pub fn format_temporal_export_rows_with_string_types(
@@ -229,40 +298,44 @@ pub fn format_temporal_export_rows_with_string_types(
column_types: &[String], column_types: &[String],
pattern: Option<&str>, pattern: Option<&str>,
) -> Vec<Vec<Value>> { ) -> Vec<Vec<Value>> {
rows.iter().map(|row| format_temporal_export_row_with_string_types(row, column_types, pattern)).collect() format_temporal_export_rows_with_string_types_cow(rows, column_types, pattern).into_owned()
} }
pub fn normalize_temporal_import_value(value: &Value, data_type: Option<&str>, pattern: Option<&str>) -> Value { pub(crate) fn normalize_temporal_import_value_cow<'a>(
value: &'a Value,
data_type: Option<&str>,
pattern: Option<&str>,
) -> Cow<'a, Value> {
let Some(kind) = temporal_kind(data_type) else { let Some(kind) = temporal_kind(data_type) else {
return value.clone(); return Cow::Borrowed(value);
}; };
let Some(raw) = value.as_str() else { let Some(raw) = value.as_str() else {
return value.clone(); return Cow::Borrowed(value);
}; };
let Some(parsed) = parse_temporal(raw.trim(), pattern) else { let Some(parsed) = parse_temporal(raw.trim(), pattern) else {
return value.clone(); return Cow::Borrowed(value);
}; };
let normalized = match (kind, parsed) { let normalized = match (kind, parsed) {
(TemporalKind::Date, ParsedTemporal::Zoned(value)) => value.date_naive().format("%Y-%m-%d").to_string(), (TemporalKind::Date, ParsedTemporal::Zoned(value)) => value.date_naive().format("%Y-%m-%d").to_string(),
(TemporalKind::Date, ParsedTemporal::DateTime(value)) => value.date().format("%Y-%m-%d").to_string(), (TemporalKind::Date, ParsedTemporal::DateTime(value)) => value.date().format("%Y-%m-%d").to_string(),
(TemporalKind::Date, ParsedTemporal::Date(value)) => value.format("%Y-%m-%d").to_string(), (TemporalKind::Date, ParsedTemporal::Date(value)) => value.format("%Y-%m-%d").to_string(),
(TemporalKind::Date, ParsedTemporal::Time(_)) => return value.clone(), (TemporalKind::Date, ParsedTemporal::Time(_)) => return Cow::Borrowed(value),
(TemporalKind::Time, ParsedTemporal::Zoned(value)) => value.time().format("%H:%M:%S%.f").to_string(), (TemporalKind::Time, ParsedTemporal::Zoned(value)) => value.time().format("%H:%M:%S%.f").to_string(),
(TemporalKind::Time, ParsedTemporal::DateTime(value)) => value.time().format("%H:%M:%S%.f").to_string(), (TemporalKind::Time, ParsedTemporal::DateTime(value)) => value.time().format("%H:%M:%S%.f").to_string(),
(TemporalKind::Time, ParsedTemporal::Time(value)) => value.format("%H:%M:%S%.f").to_string(), (TemporalKind::Time, ParsedTemporal::Time(value)) => value.format("%H:%M:%S%.f").to_string(),
(TemporalKind::Time, ParsedTemporal::Date(_)) => return value.clone(), (TemporalKind::Time, ParsedTemporal::Date(_)) => return Cow::Borrowed(value),
(TemporalKind::DateTime, ParsedTemporal::Zoned(value)) => { (TemporalKind::DateTime, ParsedTemporal::Zoned(value)) => {
value.naive_local().format("%Y-%m-%d %H:%M:%S%.f").to_string() value.naive_local().format("%Y-%m-%d %H:%M:%S%.f").to_string()
} }
(TemporalKind::DateTime, ParsedTemporal::DateTime(value)) => value.format("%Y-%m-%d %H:%M:%S%.f").to_string(), (TemporalKind::DateTime, ParsedTemporal::DateTime(value)) => value.format("%Y-%m-%d %H:%M:%S%.f").to_string(),
(TemporalKind::DateTime, ParsedTemporal::Date(date)) => { (TemporalKind::DateTime, ParsedTemporal::Date(date)) => {
let Some(value) = date.and_hms_opt(0, 0, 0) else { let Some(value) = date.and_hms_opt(0, 0, 0) else {
return value.clone(); return Cow::Borrowed(value);
}; };
value.format("%Y-%m-%d %H:%M:%S").to_string() value.format("%Y-%m-%d %H:%M:%S").to_string()
} }
(TemporalKind::DateTime, ParsedTemporal::Time(_)) => return value.clone(), (TemporalKind::DateTime, ParsedTemporal::Time(_)) => return Cow::Borrowed(value),
(TemporalKind::DateTimeWithTimeZone, ParsedTemporal::Zoned(value)) => { (TemporalKind::DateTimeWithTimeZone, ParsedTemporal::Zoned(value)) => {
value.format("%Y-%m-%dT%H:%M:%S%.f%:z").to_string() value.format("%Y-%m-%dT%H:%M:%S%.f%:z").to_string()
} }
@@ -271,13 +344,17 @@ pub fn normalize_temporal_import_value(value: &Value, data_type: Option<&str>, p
} }
(TemporalKind::DateTimeWithTimeZone, ParsedTemporal::Date(date)) => { (TemporalKind::DateTimeWithTimeZone, ParsedTemporal::Date(date)) => {
let Some(value) = date.and_hms_opt(0, 0, 0) else { let Some(value) = date.and_hms_opt(0, 0, 0) else {
return value.clone(); return Cow::Borrowed(value);
}; };
value.format("%Y-%m-%d %H:%M:%S").to_string() value.format("%Y-%m-%d %H:%M:%S").to_string()
} }
(TemporalKind::DateTimeWithTimeZone, ParsedTemporal::Time(_)) => return value.clone(), (TemporalKind::DateTimeWithTimeZone, ParsedTemporal::Time(_)) => return Cow::Borrowed(value),
}; };
Value::String(normalized) Cow::Owned(Value::String(normalized))
}
pub fn normalize_temporal_import_value(value: &Value, data_type: Option<&str>, pattern: Option<&str>) -> Value {
normalize_temporal_import_value_cow(value, data_type, pattern).into_owned()
} }
#[cfg(test)] #[cfg(test)]
@@ -358,4 +435,31 @@ mod tests {
json!("2024-02-25T13:02:15+08:00") json!("2024-02-25T13:02:15+08:00")
); );
} }
#[test]
fn export_cow_borrows_when_no_formatting_is_needed() {
let row = vec![json!(1), json!("2024-02-25 13:02:15")];
let rows = vec![row.clone()];
assert!(matches!(
format_temporal_export_row_cow(&row, &[None, Some("TIMESTAMP".to_string())], None),
Cow::Borrowed(_)
));
assert!(matches!(
format_temporal_export_rows_cow(&rows, &[None, Some("VARCHAR".to_string())], Some("YYYY/MM/DD")),
Cow::Borrowed(_)
));
}
#[test]
fn import_cow_borrows_unchanged_values_and_owns_normalized_values() {
let text = json!("plain");
assert!(matches!(normalize_temporal_import_value_cow(&text, Some("TEXT"), None), Cow::Borrowed(_)));
let timestamp = json!("2024/2/25 13:02:15");
assert_eq!(
normalize_temporal_import_value_cow(&timestamp, Some("TIMESTAMP"), None).into_owned(),
json!("2024-02-25 13:02:15")
);
}
} }
+82 -21
View File
@@ -3263,9 +3263,19 @@ pub fn generate_insert_typed(
return String::new(); return String::new();
} }
let template = InsertSqlTemplate::new(columns, table, schema, db_type, catalog, false);
let value_rows = value_rows_sql(rows, column_types, db_type, false); let value_rows = value_rows_sql(rows, column_types, db_type, false);
template.build(&value_rows) generate_insert_typed_from_value_rows(columns, &value_rows, table, schema, db_type, catalog)
}
pub(crate) fn generate_insert_typed_from_value_rows(
columns: &[String],
value_rows: &[String],
table: &str,
schema: &str,
db_type: &DatabaseType,
catalog: Option<&str>,
) -> String {
InsertSqlTemplate::new(columns, table, schema, db_type, catalog, false).build(value_rows)
} }
#[derive(Debug)] #[derive(Debug)]
@@ -3302,7 +3312,13 @@ impl InsertSqlTemplate {
return String::new(); return String::new();
} }
if let Some(into_prefix) = self.oracle_into_prefix.as_deref().filter(|_| value_rows.len() > 1) { if let Some(into_prefix) = self.oracle_into_prefix.as_deref().filter(|_| value_rows.len() > 1) {
let mut sql = String::from("INSERT ALL\n"); let capacity = "INSERT ALL\n".len()
+ into_prefix.len().saturating_mul(value_rows.len())
+ value_rows.iter().map(String::len).sum::<usize>()
+ value_rows.len().saturating_sub(1)
+ "\nSELECT 1 FROM dual".len();
let mut sql = String::with_capacity(capacity);
sql.push_str("INSERT ALL\n");
for (index, values) in value_rows.iter().enumerate() { for (index, values) in value_rows.iter().enumerate() {
if index > 0 { if index > 0 {
sql.push('\n'); sql.push('\n');
@@ -3314,8 +3330,17 @@ impl InsertSqlTemplate {
return sql; return sql;
} }
let mut sql = self.standard_prefix.clone(); let capacity = self.standard_prefix.len()
sql.push_str(&value_rows.join(",\n")); + value_rows.iter().map(String::len).sum::<usize>()
+ ",\n".len().saturating_mul(value_rows.len().saturating_sub(1));
let mut sql = String::with_capacity(capacity);
sql.push_str(&self.standard_prefix);
for (index, values) in value_rows.iter().enumerate() {
if index > 0 {
sql.push_str(",\n");
}
sql.push_str(values);
}
sql sql
} }
@@ -3350,8 +3375,12 @@ fn value_rows_sql(
) -> Vec<String> { ) -> Vec<String> {
let mut out = Vec::with_capacity(rows.len()); let mut out = Vec::with_capacity(rows.len());
for row in rows { for row in rows {
let mut vals = Vec::with_capacity(row.len()); let mut values = String::with_capacity(row.len().saturating_mul(16).saturating_add(2));
values.push('(');
for (index, v) in row.iter().enumerate() { for (index, v) in row.iter().enumerate() {
if index > 0 {
values.push_str(", ");
}
let column_type = column_types.get(index).and_then(|value| value.as_deref()); let column_type = column_types.get(index).and_then(|value| value.as_deref());
let value = if mysql_spatial_markers { let value = if mysql_spatial_markers {
crate::database_export::format_mysql_spatial_export_literal(v, Some(*db_type), column_type) crate::database_export::format_mysql_spatial_export_literal(v, Some(*db_type), column_type)
@@ -3359,9 +3388,10 @@ fn value_rows_sql(
None None
} }
.unwrap_or_else(|| escape_value_typed(v, db_type, column_type)); .unwrap_or_else(|| escape_value_typed(v, db_type, column_type));
vals.push(value); values.push_str(&value);
} }
out.push(format!("({})", vals.join(", "))); values.push(')');
out.push(values);
} }
out out
} }
@@ -3714,6 +3744,7 @@ fn generate_transfer_write_sql(
} }
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
#[cfg_attr(not(test), allow(dead_code))]
pub(crate) fn generate_insert_typed_sql_batches( pub(crate) fn generate_insert_typed_sql_batches(
columns: &[String], columns: &[String],
column_types: &[Option<String>], column_types: &[Option<String>],
@@ -3724,18 +3755,21 @@ pub(crate) fn generate_insert_typed_sql_batches(
catalog: Option<&str>, catalog: Option<&str>,
limits: SqlBatchLimits, limits: SqlBatchLimits,
) -> Result<Vec<(String, usize)>, String> { ) -> Result<Vec<(String, usize)>, String> {
generate_insert_typed_sql_batches_for_transfer( let value_rows = value_rows_sql(rows, column_types, db_type, false);
columns, generate_insert_typed_sql_batches_from_value_rows(columns, &value_rows, table, schema, db_type, catalog, limits)
column_types, }
rows,
table, #[allow(clippy::too_many_arguments)]
schema, pub(crate) fn generate_insert_typed_sql_batches_from_value_rows(
db_type, columns: &[String],
catalog, value_rows: &[String],
limits, table: &str,
false, schema: &str,
false, db_type: &DatabaseType,
) catalog: Option<&str>,
limits: SqlBatchLimits,
) -> Result<Vec<(String, usize)>, String> {
generate_insert_sql_batches_from_value_rows(columns, value_rows, table, schema, db_type, catalog, limits, false)
} }
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
@@ -3755,6 +3789,34 @@ fn generate_insert_typed_sql_batches_for_transfer(
return Ok(Vec::new()); return Ok(Vec::new());
} }
let value_rows = value_rows_sql(rows, column_types, db_type, mysql_spatial_markers);
generate_insert_sql_batches_from_value_rows(
columns,
&value_rows,
table,
schema,
db_type,
catalog,
limits,
overrides_postgres_system_values,
)
}
#[allow(clippy::too_many_arguments)]
fn generate_insert_sql_batches_from_value_rows(
columns: &[String],
value_rows: &[String],
table: &str,
schema: &str,
db_type: &DatabaseType,
catalog: Option<&str>,
limits: SqlBatchLimits,
overrides_postgres_system_values: bool,
) -> Result<Vec<(String, usize)>, String> {
if value_rows.is_empty() {
return Ok(Vec::new());
}
let max_rows = limits.max_rows.max(1).min(match db_type { let max_rows = limits.max_rows.max(1).min(match db_type {
DatabaseType::SqlServer => MAX_SQLSERVER_INSERT_ROWS, DatabaseType::SqlServer => MAX_SQLSERVER_INSERT_ROWS,
DatabaseType::Oracle => MAX_ORACLE_INSERT_ALL_ROWS, DatabaseType::Oracle => MAX_ORACLE_INSERT_ALL_ROWS,
@@ -3763,7 +3825,6 @@ fn generate_insert_typed_sql_batches_for_transfer(
let target_sql_bytes = limits.target_sql_bytes.max(1); let target_sql_bytes = limits.target_sql_bytes.max(1);
let batch_sql_bytes = limits.hard_sql_bytes.map_or(target_sql_bytes, |hard| target_sql_bytes.min(hard)); let batch_sql_bytes = limits.hard_sql_bytes.map_or(target_sql_bytes, |hard| target_sql_bytes.min(hard));
let template = InsertSqlTemplate::new(columns, table, schema, db_type, catalog, overrides_postgres_system_values); let template = InsertSqlTemplate::new(columns, table, schema, db_type, catalog, overrides_postgres_system_values);
let value_rows = value_rows_sql(rows, column_types, db_type, mysql_spatial_markers);
let value_row_bytes = value_rows.iter().map(|row| sql_text_bytes(row, db_type)).collect::<Vec<_>>(); let value_row_bytes = value_rows.iter().map(|row| sql_text_bytes(row, db_type)).collect::<Vec<_>>();
let mut statements = Vec::new(); let mut statements = Vec::new();
let mut start = 0usize; let mut start = 0usize;
+149 -54
View File
@@ -1,5 +1,6 @@
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_json::Value; use serde_json::Value;
use std::fmt::Write as FmtWrite;
use std::io::{Cursor, Seek, Write}; use std::io::{Cursor, Seek, Write};
use crate::temporal_format::{excel_temporal_serial, ExcelTemporalKind}; use crate::temporal_format::{excel_temporal_serial, ExcelTemporalKind};
@@ -130,6 +131,7 @@ pub struct StreamingXlsxWriter<W: Write + Seek> {
column_comments: Vec<Option<String>>, column_comments: Vec<Option<String>>,
date_time_format: Option<String>, date_time_format: Option<String>,
numeric_right_align: bool, numeric_right_align: bool,
row_buffer: String,
} }
/// Estimate column widths from header names only (used by the streaming path /// Estimate column widths from header names only (used by the streaming path
@@ -159,8 +161,8 @@ fn cols_xml(widths: &[usize]) -> String {
/// Resolve the effective header text: prefer a non-empty column comment, fall /// Resolve the effective header text: prefer a non-empty column comment, fall
/// back to the original column name. /// back to the original column name.
fn effective_header(column: &str, comment: Option<&str>) -> String { fn effective_header<'a>(column: &'a str, comment: Option<&'a str>) -> &'a str {
comment.filter(|c| !c.is_empty()).unwrap_or(column).to_string() comment.filter(|c| !c.is_empty()).unwrap_or(column)
} }
/// Build a single `<row>` XML fragment for the header row (row 1). /// Build a single `<row>` XML fragment for the header row (row 1).
@@ -172,30 +174,36 @@ pub(crate) fn header_row_xml(columns: &[String], column_comments: &[Option<Strin
.enumerate() .enumerate()
.map(|(index, col)| { .map(|(index, col)| {
let header = effective_header(col, column_comments.get(index).and_then(|c| c.as_deref())); let header = effective_header(col, column_comments.get(index).and_then(|c| c.as_deref()));
cell_xml(Some(&Value::String(header)), 0, index, Some(1)) cell_xml(Some(&Value::String(header.to_string())), 0, index, Some(1))
}) })
.collect::<String>() .collect::<String>()
) )
} }
fn data_row_xml_with_date_time_format( fn push_data_row_xml(
output: &mut String,
row_number: usize, row_number: usize,
columns: &[String], columns: &[String],
column_types: &[String], column_types: &[String],
row: &[Value], row: &[Value],
date_time_format: Option<&str>, date_time_format: Option<&str>,
numeric_right_align: bool, numeric_right_align: bool,
) -> String { ) {
let cells = columns write!(output, "<row r=\"{row_number}\">").expect("writing XLSX XML into a String cannot fail");
.iter() for (col_index, _) in columns.iter().enumerate() {
.enumerate() let col_type = column_types.get(col_index);
.map(|(col_index, _)| { let align_style = numeric_column_style(col_type, numeric_right_align);
let col_type = column_types.get(col_index); push_typed_cell_xml(
let align_style = numeric_column_style(col_type, numeric_right_align); output,
typed_cell_xml(row.get(col_index), col_type, row_number - 1, col_index, align_style, date_time_format) row.get(col_index),
}) col_type,
.collect::<String>(); row_number - 1,
format!("<row r=\"{row_number}\">{cells}</row>") col_index,
align_style,
date_time_format,
);
}
output.push_str("</row>");
} }
/// Shared ZIP entry options for all XLSX parts. XLSX is a ZIP of XML, and the /// Shared ZIP entry options for all XLSX parts. XLSX is a ZIP of XML, and the
@@ -298,6 +306,7 @@ fn start_xlsx_writer_inner<W: Write + Seek>(
column_comments: column_comments.to_vec(), column_comments: column_comments.to_vec(),
date_time_format: date_time_format.map(str::to_string), date_time_format: date_time_format.map(str::to_string),
numeric_right_align, numeric_right_align,
row_buffer: String::with_capacity(columns.len().saturating_mul(48)),
}) })
} }
@@ -355,19 +364,17 @@ impl<W: Write + Seek> StreamingXlsxWriter<W> {
self.finish_current_sheet()?; self.finish_current_sheet()?;
self.start_next_data_sheet()?; self.start_next_data_sheet()?;
} }
self.zip self.row_buffer.clear();
.write_all( push_data_row_xml(
data_row_xml_with_date_time_format( &mut self.row_buffer,
self.next_row_number, self.next_row_number,
&self.columns, &self.columns,
&self.column_types, &self.column_types,
row, row,
self.date_time_format.as_deref(), self.date_time_format.as_deref(),
self.numeric_right_align, self.numeric_right_align,
) );
.as_bytes(), self.zip.write_all(self.row_buffer.as_bytes()).map_err(|err| err.to_string())?;
)
.map_err(|err| err.to_string())?;
self.next_row_number += 1; self.next_row_number += 1;
self.current_data_rows += 1; self.current_data_rows += 1;
Ok(()) Ok(())
@@ -469,6 +476,11 @@ pub(crate) fn finish_streaming_xlsx_workbook<W: Write + Seek>(writer: StreamingX
fn escape_xml(value: &str) -> String { fn escape_xml(value: &str) -> String {
let mut result = String::with_capacity(value.len()); let mut result = String::with_capacity(value.len());
push_xml_escaped(&mut result, value);
result
}
fn push_xml_escaped(result: &mut String, value: &str) {
for ch in value.chars() { for ch in value.chars() {
let code = ch as u32; let code = ch as u32;
if code != 9 && code != 10 && code != 13 && code < 32 { if code != 9 && code != 10 && code != 13 && code < 32 {
@@ -482,22 +494,36 @@ fn escape_xml(value: &str) -> String {
_ => result.push(ch), _ => result.push(ch),
} }
} }
result
} }
fn column_name(index: usize) -> String { fn column_name(index: usize) -> String {
let mut out = String::new(); let mut out = String::with_capacity(4);
push_column_name(&mut out, index);
out
}
fn push_column_name(out: &mut String, index: usize) {
let mut buffer = [0u8; 16];
let mut cursor = buffer.len();
let mut n = index + 1; let mut n = index + 1;
while n > 0 { while n > 0 {
let rem = (n - 1) % 26; let rem = (n - 1) % 26;
out.push((b'A' + rem as u8) as char); cursor -= 1;
buffer[cursor] = b'A' + rem as u8;
n = (n - 1) / 26; n = (n - 1) / 26;
} }
out.chars().rev().collect() out.push_str(std::str::from_utf8(&buffer[cursor..]).expect("XLSX column names are ASCII"));
} }
fn cell_ref(row_index: usize, col_index: usize) -> String { fn cell_ref(row_index: usize, col_index: usize) -> String {
format!("{}{}", column_name(col_index), row_index + 1) let mut reference = String::with_capacity(8);
push_cell_ref(&mut reference, row_index, col_index);
reference
}
fn push_cell_ref(out: &mut String, row_index: usize, col_index: usize) {
push_column_name(out, col_index);
write!(out, "{}", row_index + 1).expect("writing XLSX cell references into a String cannot fail");
} }
fn sheet_range(column_count: usize, row_count: usize) -> String { fn sheet_range(column_count: usize, row_count: usize) -> String {
@@ -560,6 +586,56 @@ fn cell_xml(value: Option<&Value>, row_index: usize, col_index: usize, style: Op
} }
} }
fn push_cell_style(output: &mut String, style: Option<usize>) {
if let Some(style) = style {
write!(output, " s=\"{style}\"").expect("writing XLSX cell styles into a String cannot fail");
}
}
fn push_cell_xml(output: &mut String, value: Option<&Value>, row_index: usize, col_index: usize, style: Option<usize>) {
output.push_str("<c r=\"");
push_cell_ref(output, row_index, col_index);
output.push('"');
match value {
Some(Value::Null) | None => {
push_cell_style(output, style);
output.push_str("/>");
}
Some(Value::Bool(value)) => {
let bool_value = if *value { 1 } else { 0 };
output.push_str(" t=\"b\"");
push_cell_style(output, style);
write!(output, "><v>{bool_value}</v></c>").expect("writing XLSX booleans into a String cannot fail");
}
Some(Value::Number(value)) => {
if value.as_f64().is_some_and(|number| number.is_finite()) {
push_cell_style(output, style);
write!(output, "><v>{value}</v></c>").expect("writing XLSX numbers into a String cannot fail");
} else {
output.push_str(" t=\"inlineStr\"");
push_cell_style(output, style);
output.push_str("><is><t>");
push_xml_escaped(output, &value.to_string());
output.push_str("</t></is></c>");
}
}
Some(Value::String(value)) => {
output.push_str(" t=\"inlineStr\"");
push_cell_style(output, style);
output.push_str("><is><t>");
push_xml_escaped(output, value);
output.push_str("</t></is></c>");
}
Some(value) => {
output.push_str(" t=\"inlineStr\"");
push_cell_style(output, style);
output.push_str("><is><t>");
push_xml_escaped(output, &value.to_string());
output.push_str("</t></is></c>");
}
}
}
fn is_numeric_column_type(column_type: Option<&String>) -> bool { fn is_numeric_column_type(column_type: Option<&String>) -> bool {
let mut normalized = column_type.map(|value| value.trim().to_ascii_lowercase()).unwrap_or_default(); let mut normalized = column_type.map(|value| value.trim().to_ascii_lowercase()).unwrap_or_default();
while normalized.ends_with(')') { while normalized.ends_with(')') {
@@ -659,36 +735,43 @@ fn safe_excel_number(value: &str) -> Option<&str> {
(significant_digits <= 15).then_some(trimmed) (significant_digits <= 15).then_some(trimmed)
} }
fn typed_cell_xml( fn push_typed_cell_xml(
output: &mut String,
value: Option<&Value>, value: Option<&Value>,
column_type: Option<&String>, column_type: Option<&String>,
row_index: usize, row_index: usize,
col_index: usize, col_index: usize,
style: Option<usize>, style: Option<usize>,
date_time_format: Option<&str>, date_time_format: Option<&str>,
) -> String { ) {
if let Some(Value::String(value)) = value { if let Some(Value::String(value)) = value {
if let Some((serial, temporal_kind)) = if let Some((serial, temporal_kind)) =
excel_temporal_serial(value, column_type.map(String::as_str), date_time_format) excel_temporal_serial(value, column_type.map(String::as_str), date_time_format)
{ {
let reference = cell_ref(row_index, col_index);
let style = match temporal_kind { let style = match temporal_kind {
ExcelTemporalKind::Date => XLSX_DATE_STYLE, ExcelTemporalKind::Date => XLSX_DATE_STYLE,
ExcelTemporalKind::DateTime => XLSX_DATETIME_STYLE, ExcelTemporalKind::DateTime => XLSX_DATETIME_STYLE,
}; };
return format!("<c r=\"{reference}\" s=\"{style}\"><v>{serial}</v></c>"); output.push_str("<c r=\"");
push_cell_ref(output, row_index, col_index);
write!(output, "\" s=\"{style}\"><v>{serial}</v></c>")
.expect("writing XLSX temporal cells into a String cannot fail");
return;
} }
} }
if is_numeric_column_type(column_type) { if is_numeric_column_type(column_type) {
if let Some(Value::String(value)) = value { if let Some(Value::String(value)) = value {
if let Some(number) = safe_excel_number(value) { if let Some(number) = safe_excel_number(value) {
let reference = cell_ref(row_index, col_index); output.push_str("<c r=\"");
let style_attr = style.map_or(String::new(), |style| format!(" s=\"{style}\"")); push_cell_ref(output, row_index, col_index);
return format!("<c r=\"{reference}\"{style_attr}><v>{number}</v></c>"); output.push('"');
push_cell_style(output, style);
write!(output, "><v>{number}</v></c>").expect("writing XLSX numeric cells into a String cannot fail");
return;
} }
} }
} }
cell_xml(value, row_index, col_index, style) push_cell_xml(output, value, row_index, col_index, style);
} }
fn write_worksheet_xml<W: Write>(writer: &mut W, segment: &WorksheetSegment) -> Result<(), String> { fn write_worksheet_xml<W: Write>(writer: &mut W, segment: &WorksheetSegment) -> Result<(), String> {
@@ -718,21 +801,20 @@ fn write_worksheet_xml<W: Write>(writer: &mut W, segment: &WorksheetSegment) ->
.write_all(header_row_xml(segment.columns, segment.column_comments).as_bytes()) .write_all(header_row_xml(segment.columns, segment.column_comments).as_bytes())
.map_err(|err| err.to_string())?; .map_err(|err| err.to_string())?;
let mut row_buffer = String::with_capacity(segment.columns.len().saturating_mul(48));
for (row_index, row) in segment.rows.iter().enumerate() { for (row_index, row) in segment.rows.iter().enumerate() {
let excel_row = row_index + 2; let excel_row = row_index + 2;
writer row_buffer.clear();
.write_all( push_data_row_xml(
data_row_xml_with_date_time_format( &mut row_buffer,
excel_row, excel_row,
segment.columns, segment.columns,
segment.column_types, segment.column_types,
row, row,
None, None,
segment.numeric_column_right_align, segment.numeric_column_right_align,
) );
.as_bytes(), writer.write_all(row_buffer.as_bytes()).map_err(|err| err.to_string())?;
)
.map_err(|err| err.to_string())?;
} }
writer writer
@@ -1983,4 +2065,17 @@ mod tests {
let sheet2 = read_zip_entry(&data, "xl/worksheets/sheet2.xml"); let sheet2 = read_zip_entry(&data, "xl/worksheets/sheet2.xml");
assert!(sheet2.contains("row_3"), "row 3 should be on sheet 2: {sheet2}"); assert!(sheet2.contains("row_3"), "row 3 should be on sheet 2: {sheet2}");
} }
#[test]
fn reusable_cell_buffer_matches_reference_xml() {
let values = [Value::Null, json!(true), json!(42.5), json!("a<&\"b"), json!({"key": "value"})];
for (index, value) in values.iter().enumerate() {
for style in [None, Some(1), Some(4)] {
let expected = super::cell_xml(Some(value), index, index + 1, style);
let mut actual = String::new();
super::push_cell_xml(&mut actual, Some(value), index, index + 1, style);
assert_eq!(actual, expected, "value={value:?}, style={style:?}");
}
}
}
} }