fix(transfer): reduce import and export allocations
This commit is contained in:
9 files changed
+822
-261
No files matched your search
@@ -100,6 +100,15 @@ fn push_tsv_value(out: &mut String, value: &Value) {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn push_tsv_row(out: &mut String, row: &[Value]) {
|
||||
for (cell_index, cell) in row.iter().enumerate() {
|
||||
if cell_index > 0 {
|
||||
out.push('\t');
|
||||
}
|
||||
push_tsv_value(out, cell);
|
||||
}
|
||||
}
|
||||
|
||||
/// 预分配粗估:按全部行的实际单元格数求和(不假设等宽),饱和运算防溢出,
|
||||
/// 并设上限——估算只是性能提示,绝不能因病态输入放大成巨额分配
|
||||
const ROWS_CAPACITY_ESTIMATE_MAX: usize = 16 * 1024 * 1024;
|
||||
@@ -133,15 +142,11 @@ fn push_tsv_rows(out: &mut String, rows: &[Vec<Value>]) {
|
||||
if row_index > 0 {
|
||||
out.push('\n');
|
||||
}
|
||||
for (cell_index, cell) in row.iter().enumerate() {
|
||||
if cell_index > 0 {
|
||||
out.push('\t');
|
||||
}
|
||||
push_tsv_value(out, cell);
|
||||
}
|
||||
push_tsv_row(out, row);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg_attr(not(test), allow(dead_code))]
|
||||
pub(crate) fn format_tsv_rows(rows: &[Vec<Value>]) -> String {
|
||||
let mut out = String::with_capacity(estimated_rows_capacity(rows));
|
||||
push_tsv_rows(&mut out, rows);
|
||||
@@ -166,17 +171,30 @@ pub(crate) fn format_tsv(columns: &[String], rows: &[Vec<Value>]) -> String {
|
||||
/// Format query-result rows as CSV text without a header row. Database NULLs
|
||||
/// use the same empty-cell representation as table-data exports. Used by the
|
||||
/// streaming query-result export for batches after the first.
|
||||
pub(crate) fn push_query_result_csv_row(out: &mut String, row: &[Value]) {
|
||||
for (cell_index, cell) in row.iter().enumerate() {
|
||||
if cell_index > 0 {
|
||||
out.push(',');
|
||||
}
|
||||
push_csv_value(out, cell);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn push_table_csv_row(out: &mut String, row: &[Value]) {
|
||||
for (cell_index, cell) in row.iter().enumerate() {
|
||||
if cell_index > 0 {
|
||||
out.push(',');
|
||||
}
|
||||
push_csv_text_value(out, cell);
|
||||
}
|
||||
}
|
||||
|
||||
fn push_query_result_csv_rows(out: &mut String, rows: &[Vec<Value>]) {
|
||||
for (row_index, row) in rows.iter().enumerate() {
|
||||
if row_index > 0 {
|
||||
out.push('\n');
|
||||
}
|
||||
for (cell_index, cell) in row.iter().enumerate() {
|
||||
if cell_index > 0 {
|
||||
out.push(',');
|
||||
}
|
||||
push_csv_value(out, cell);
|
||||
}
|
||||
push_query_result_csv_row(out, row);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -399,5 +417,22 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reusable_row_buffers_match_batch_formatters() {
|
||||
let row = vec![Value::Null, json!("NULL"), json!("line\n\"two\""), json!(42)];
|
||||
|
||||
let mut csv = String::new();
|
||||
super::push_query_result_csv_row(&mut csv, &row);
|
||||
assert_eq!(csv, super::format_query_result_csv_rows(&[row.clone()]));
|
||||
|
||||
let mut table_csv = String::new();
|
||||
super::push_table_csv_row(&mut table_csv, &row);
|
||||
assert_eq!(table_csv, "\"\",\"NULL\",\"line\n\"\"two\"\"\",\"42\"");
|
||||
|
||||
let mut tsv = String::new();
|
||||
super::push_tsv_row(&mut tsv, &row);
|
||||
assert_eq!(tsv, super::format_tsv_rows(&[row]));
|
||||
}
|
||||
|
||||
use serde_json::Value;
|
||||
}
|
||||
@@ -994,54 +994,63 @@ pub fn build_export_insert_statements(options: BuildExportInsertStatementsOption
|
||||
let statement_overhead_bytes = export_sql_statement_bytes(options.database_type, &statement_prefix) + 1;
|
||||
let target_statement_bytes = DATABASE_EXPORT_TARGET_STATEMENT_BYTES;
|
||||
let separator_bytes = export_sql_statement_bytes(options.database_type, ", ");
|
||||
let mut current_values = Vec::with_capacity(batch_size);
|
||||
let mut current_values = String::new();
|
||||
let mut current_values_bytes = 0usize;
|
||||
let mut current_row_count = 0usize;
|
||||
|
||||
let flush_values = |statements: &mut Vec<String>, values: &mut Vec<String>, values_bytes: &mut usize| {
|
||||
if values.is_empty() {
|
||||
return;
|
||||
}
|
||||
let insert_sql = format!("{statement_prefix}{};", values.join(", "));
|
||||
if needs_dameng_identity_insert {
|
||||
statements.push(wrap_dameng_identity_insert_sql_for_table(&insert_sql, &table));
|
||||
} else {
|
||||
statements.push(insert_sql);
|
||||
}
|
||||
values.clear();
|
||||
*values_bytes = 0;
|
||||
};
|
||||
let flush_values =
|
||||
|statements: &mut Vec<String>, values: &mut String, values_bytes: &mut usize, row_count: &mut usize| {
|
||||
if *row_count == 0 {
|
||||
return;
|
||||
}
|
||||
let mut insert_sql = String::with_capacity(statement_prefix.len() + values.len() + 1);
|
||||
insert_sql.push_str(&statement_prefix);
|
||||
insert_sql.push_str(values);
|
||||
insert_sql.push(';');
|
||||
if needs_dameng_identity_insert {
|
||||
statements.push(wrap_dameng_identity_insert_sql_for_table(&insert_sql, &table));
|
||||
} else {
|
||||
statements.push(insert_sql);
|
||||
}
|
||||
values.clear();
|
||||
*values_bytes = 0;
|
||||
*row_count = 0;
|
||||
};
|
||||
|
||||
for row in options.rows {
|
||||
let row_values = insert_columns
|
||||
.iter()
|
||||
.map(|(index, _, sqlserver_unicode_string)| {
|
||||
let value = row.get(*index).unwrap_or(&Value::Null);
|
||||
format_export_sql_literal_typed(
|
||||
value,
|
||||
options.database_type,
|
||||
options.column_types.get(*index).and_then(|value| value.as_deref()),
|
||||
*sqlserver_unicode_string,
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
let rendered_row = format!("({row_values})");
|
||||
let mut rendered_row = String::with_capacity(insert_columns.len().saturating_mul(16).saturating_add(2));
|
||||
rendered_row.push('(');
|
||||
for (column_index, (index, _, sqlserver_unicode_string)) in insert_columns.iter().enumerate() {
|
||||
if column_index > 0 {
|
||||
rendered_row.push_str(", ");
|
||||
}
|
||||
let value = row.get(*index).unwrap_or(&Value::Null);
|
||||
rendered_row.push_str(&format_export_sql_literal_typed(
|
||||
value,
|
||||
options.database_type,
|
||||
options.column_types.get(*index).and_then(|value| value.as_deref()),
|
||||
*sqlserver_unicode_string,
|
||||
));
|
||||
}
|
||||
rendered_row.push(')');
|
||||
let rendered_row_bytes = export_sql_statement_bytes(options.database_type, &rendered_row);
|
||||
let candidate_bytes = statement_overhead_bytes
|
||||
+ current_values_bytes
|
||||
+ if current_values.is_empty() { 0 } else { separator_bytes }
|
||||
+ if current_row_count == 0 { 0 } else { separator_bytes }
|
||||
+ rendered_row_bytes;
|
||||
|
||||
if !current_values.is_empty()
|
||||
&& (current_values.len() >= batch_size || candidate_bytes > target_statement_bytes)
|
||||
{
|
||||
flush_values(&mut statements, &mut current_values, &mut current_values_bytes);
|
||||
if current_row_count > 0 && (current_row_count >= batch_size || candidate_bytes > target_statement_bytes) {
|
||||
flush_values(&mut statements, &mut current_values, &mut current_values_bytes, &mut current_row_count);
|
||||
}
|
||||
current_values_bytes += if current_values.is_empty() { 0 } else { separator_bytes };
|
||||
if current_row_count > 0 {
|
||||
current_values.push_str(", ");
|
||||
current_values_bytes += separator_bytes;
|
||||
}
|
||||
current_values.push_str(&rendered_row);
|
||||
current_values_bytes += rendered_row_bytes;
|
||||
current_values.push(rendered_row);
|
||||
current_row_count += 1;
|
||||
}
|
||||
flush_values(&mut statements, &mut current_values, &mut current_values_bytes);
|
||||
flush_values(&mut statements, &mut current_values, &mut current_values_bytes, &mut current_row_count);
|
||||
|
||||
Ok(statements)
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@ use std::sync::{
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crate::connection::{AppState, PoolKind};
|
||||
use crate::csv_export::{format_query_result_csv, format_query_result_csv_rows, format_tsv, format_tsv_rows};
|
||||
use crate::csv_export::{format_query_result_csv, format_tsv, push_query_result_csv_row, push_tsv_row};
|
||||
pub use crate::database_export::ExportStatus;
|
||||
use crate::database_export::{build_export_insert_statements, is_export_cancelled, BuildExportInsertStatementsOptions};
|
||||
use crate::models::connection::DatabaseType;
|
||||
@@ -400,12 +400,38 @@ fn format_text_export_header(format: &str, columns: &[String]) -> String {
|
||||
content.strip_suffix('\n').unwrap_or(&content).to_string()
|
||||
}
|
||||
|
||||
fn format_text_export_rows(format: &str, rows: &[Vec<Value>]) -> String {
|
||||
fn write_text_export_row<W: Write>(
|
||||
file: &mut W,
|
||||
format: &str,
|
||||
row: &[Value],
|
||||
buffer: &mut String,
|
||||
) -> Result<(), String> {
|
||||
buffer.clear();
|
||||
buffer.push('\n');
|
||||
if format == "csv" {
|
||||
format_query_result_csv_rows(rows)
|
||||
push_query_result_csv_row(buffer, row);
|
||||
} else {
|
||||
format_tsv_rows(rows)
|
||||
push_tsv_row(buffer, row);
|
||||
}
|
||||
file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}"))
|
||||
}
|
||||
|
||||
fn write_text_export_rows<W: Write>(
|
||||
file: &mut W,
|
||||
format: &str,
|
||||
rows: &[Vec<Value>],
|
||||
buffer: &mut String,
|
||||
) -> Result<(), String> {
|
||||
buffer.clear();
|
||||
for row in rows {
|
||||
buffer.push('\n');
|
||||
if format == "csv" {
|
||||
push_query_result_csv_row(buffer, row);
|
||||
} else {
|
||||
push_tsv_row(buffer, row);
|
||||
}
|
||||
}
|
||||
file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}"))
|
||||
}
|
||||
|
||||
fn should_emit_stream_progress(
|
||||
@@ -683,6 +709,7 @@ async fn export_query_result_core_inner(
|
||||
}
|
||||
|
||||
let mut xlsx = None;
|
||||
let mut text_buffer = String::new();
|
||||
let mut columns: Vec<String> = Vec::new();
|
||||
let mut column_types: Vec<String> = Vec::new();
|
||||
let mut rows_exported: u64 = 0;
|
||||
@@ -838,7 +865,7 @@ async fn export_query_result_core_inner(
|
||||
result.rows.truncate(this_page);
|
||||
}
|
||||
let row_count = result.rows.len();
|
||||
let formatted_rows = crate::temporal_format::format_temporal_export_rows_with_string_types(
|
||||
let formatted_rows = crate::temporal_format::format_temporal_export_rows_with_string_types_cow(
|
||||
&result.rows,
|
||||
&column_types,
|
||||
request.date_time_format.as_deref(),
|
||||
@@ -850,18 +877,16 @@ async fn export_query_result_core_inner(
|
||||
let header = format_text_export_header(&format, &columns);
|
||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
||||
if row_count > 0 {
|
||||
let rows = format_text_export_rows(&format, &formatted_rows);
|
||||
write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?;
|
||||
write_text_export_rows(file, &format, formatted_rows.as_ref(), &mut text_buffer)?;
|
||||
}
|
||||
wrote_text_header = true;
|
||||
} else if row_count > 0 {
|
||||
let rows = format_text_export_rows(&format, &formatted_rows);
|
||||
write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?;
|
||||
write_text_export_rows(file, &format, formatted_rows.as_ref(), &mut text_buffer)?;
|
||||
}
|
||||
}
|
||||
} else if format == "sql" {
|
||||
let writer = sql_writer.as_mut().ok_or_else(|| "SQL export writer missing".to_string())?;
|
||||
for row in formatted_rows {
|
||||
for row in formatted_rows.into_owned() {
|
||||
writer.write_row(row)?;
|
||||
}
|
||||
} else {
|
||||
@@ -876,7 +901,7 @@ async fn export_query_result_core_inner(
|
||||
)?);
|
||||
}
|
||||
if let Some(writer) = xlsx.as_mut() {
|
||||
for row in &formatted_rows {
|
||||
for row in formatted_rows.as_ref() {
|
||||
writer.write_row(row).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||
}
|
||||
}
|
||||
@@ -1013,6 +1038,7 @@ async fn try_export_postgres_query_result_stream(
|
||||
None
|
||||
};
|
||||
let mut xlsx = None;
|
||||
let mut text_buffer = String::new();
|
||||
let mut sql_writer: Option<SqlInsertWriter> =
|
||||
if format == "sql" { Some(SqlInsertWriter::create(request)?) } else { None };
|
||||
let budget = operation_budget_for_pool_key(state, &pool_key, query_export_timeout(request.timeout_secs)).await;
|
||||
@@ -1050,25 +1076,26 @@ async fn try_export_postgres_query_result_stream(
|
||||
}
|
||||
}
|
||||
crate::db::postgres::PostgresQueryStreamItem::Row(row) => {
|
||||
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types(
|
||||
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types_cow(
|
||||
&row,
|
||||
&temporal_column_types,
|
||||
request.date_time_format.as_deref(),
|
||||
);
|
||||
if let Some(writer) = sql_writer.as_mut() {
|
||||
writer.write_row(formatted)?;
|
||||
writer.write_row(formatted.into_owned())?;
|
||||
} else if let Some(file) = text_file.as_mut() {
|
||||
let rows = format_text_export_rows(format, std::slice::from_ref(&formatted));
|
||||
write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?;
|
||||
write_text_export_row(file, format, formatted.as_ref(), &mut text_buffer)?;
|
||||
} else if let Some(writer) = xlsx.as_mut() {
|
||||
writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||
writer.write_row(formatted.as_ref()).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||
} else {
|
||||
let xlsx_file =
|
||||
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
|
||||
xlsx =
|
||||
Some(start_query_result_xlsx_workbook(BufWriter::new(xlsx_file), request, &columns, &[])?);
|
||||
if let Some(writer) = xlsx.as_mut() {
|
||||
writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||
writer
|
||||
.write_row(formatted.as_ref())
|
||||
.map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||
}
|
||||
}
|
||||
rows_exported += 1;
|
||||
@@ -1205,6 +1232,7 @@ async fn try_export_mysql_query_result_stream(
|
||||
None
|
||||
};
|
||||
let mut xlsx = None;
|
||||
let mut text_buffer = String::new();
|
||||
let mut sql_writer: Option<SqlInsertWriter> =
|
||||
if format == "sql" { Some(SqlInsertWriter::create(request)?) } else { None };
|
||||
let query_timeout = query_export_timeout(request.timeout_secs);
|
||||
@@ -1292,25 +1320,26 @@ async fn try_export_mysql_query_result_stream(
|
||||
}
|
||||
}
|
||||
crate::db::mysql::MySqlQueryStreamItem::Row(row) => {
|
||||
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types(
|
||||
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types_cow(
|
||||
&row,
|
||||
&temporal_column_types,
|
||||
request.date_time_format.as_deref(),
|
||||
);
|
||||
if let Some(writer) = sql_writer.as_mut() {
|
||||
writer.write_row(formatted)?;
|
||||
writer.write_row(formatted.into_owned())?;
|
||||
} else if let Some(file) = text_file.as_mut() {
|
||||
let rows = format_text_export_rows(format, std::slice::from_ref(&formatted));
|
||||
write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?;
|
||||
write_text_export_row(file, format, formatted.as_ref(), &mut text_buffer)?;
|
||||
} else if let Some(writer) = xlsx.as_mut() {
|
||||
writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||
writer.write_row(formatted.as_ref()).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||
} else {
|
||||
let xlsx_file =
|
||||
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
|
||||
xlsx =
|
||||
Some(start_query_result_xlsx_workbook(BufWriter::new(xlsx_file), request, &columns, &[])?);
|
||||
if let Some(writer) = xlsx.as_mut() {
|
||||
writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||
writer
|
||||
.write_row(formatted.as_ref())
|
||||
.map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||
}
|
||||
}
|
||||
rows_exported += 1;
|
||||
@@ -1467,6 +1496,7 @@ async fn try_export_clickhouse_query_result_stream(
|
||||
None
|
||||
};
|
||||
let mut xlsx = None;
|
||||
let mut text_buffer = String::new();
|
||||
let mut sql_writer: Option<SqlInsertWriter> =
|
||||
if format == "sql" { Some(SqlInsertWriter::create(request)?) } else { None };
|
||||
let query_timeout = query_export_timeout(request.timeout_secs);
|
||||
@@ -1505,25 +1535,26 @@ async fn try_export_clickhouse_query_result_stream(
|
||||
}
|
||||
}
|
||||
crate::db::clickhouse_driver::ClickHouseQueryStreamItem::Row(row) => {
|
||||
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types(
|
||||
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types_cow(
|
||||
&row,
|
||||
&temporal_column_types,
|
||||
request.date_time_format.as_deref(),
|
||||
);
|
||||
if let Some(writer) = sql_writer.as_mut() {
|
||||
writer.write_row(formatted)?;
|
||||
writer.write_row(formatted.into_owned())?;
|
||||
} else if let Some(file) = text_file.as_mut() {
|
||||
let rows = format_text_export_rows(format, std::slice::from_ref(&formatted));
|
||||
write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?;
|
||||
write_text_export_row(file, format, formatted.as_ref(), &mut text_buffer)?;
|
||||
} else if let Some(writer) = xlsx.as_mut() {
|
||||
writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||
writer.write_row(formatted.as_ref()).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||
} else {
|
||||
let xlsx_file =
|
||||
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
|
||||
xlsx =
|
||||
Some(start_query_result_xlsx_workbook(BufWriter::new(xlsx_file), request, &columns, &[])?);
|
||||
if let Some(writer) = xlsx.as_mut() {
|
||||
writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||
writer
|
||||
.write_row(formatted.as_ref())
|
||||
.map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||
}
|
||||
}
|
||||
rows_exported += 1;
|
||||
@@ -1635,6 +1666,7 @@ async fn try_export_sqlserver_query_result_stream(
|
||||
None
|
||||
};
|
||||
let mut xlsx = None;
|
||||
let mut text_buffer = String::new();
|
||||
let mut sql_writer: Option<SqlInsertWriter> =
|
||||
if format == "sql" { Some(SqlInsertWriter::create(request)?) } else { None };
|
||||
let query_timeout = query_export_timeout(request.timeout_secs);
|
||||
@@ -1683,25 +1715,26 @@ async fn try_export_sqlserver_query_result_stream(
|
||||
}
|
||||
}
|
||||
crate::db::sqlserver::SqlServerStreamItem::Row(row) => {
|
||||
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types(
|
||||
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types_cow(
|
||||
row,
|
||||
&temporal_column_types,
|
||||
request.date_time_format.as_deref(),
|
||||
);
|
||||
if let Some(writer) = sql_writer.as_mut() {
|
||||
writer.write_row(formatted)?;
|
||||
writer.write_row(formatted.into_owned())?;
|
||||
} else if let Some(file) = text_file.as_mut() {
|
||||
let rows = format_text_export_rows(format, std::slice::from_ref(&formatted));
|
||||
write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?;
|
||||
write_text_export_row(file, format, formatted.as_ref(), &mut text_buffer)?;
|
||||
} else if let Some(writer) = xlsx.as_mut() {
|
||||
writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||
writer.write_row(formatted.as_ref()).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||
} else {
|
||||
let xlsx_file =
|
||||
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
|
||||
xlsx =
|
||||
Some(start_query_result_xlsx_workbook(BufWriter::new(xlsx_file), request, &columns, &[])?);
|
||||
if let Some(writer) = xlsx.as_mut() {
|
||||
writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||
writer
|
||||
.write_row(formatted.as_ref())
|
||||
.map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||
}
|
||||
}
|
||||
rows_exported += 1;
|
||||
@@ -1893,6 +1926,16 @@ mod tests {
|
||||
assert_eq!(format_text_export_header("txt", &["id".to_string(), "note".to_string()]), "id\tnote");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reusable_text_row_buffer_preserves_query_null_semantics() {
|
||||
let row = vec![Value::Null, serde_json::json!(""), serde_json::json!("line\n\"two\"")];
|
||||
let mut output = Vec::new();
|
||||
let mut buffer = String::new();
|
||||
|
||||
write_text_export_row(&mut output, "csv", &row, &mut buffer).expect("write csv row");
|
||||
assert_eq!(String::from_utf8(output).expect("utf8 csv"), "\n,\"\",\"line\n\"\"two\"\"\"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn xlsx_no_row_limit_has_no_query_layer_cap() {
|
||||
// Without the old hard limit, XLSX uses the writer's internal splitting.
|
||||
|
||||
@@ -9,7 +9,9 @@ use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::connection::MysqlMode;
|
||||
use crate::connection::{config_for_pool_key, task_client_session_id, AppState, PoolKind};
|
||||
use crate::csv_export::{format_csv, format_tsv, format_tsv_rows, push_csv_text_value};
|
||||
#[cfg(test)]
|
||||
use crate::csv_export::format_tsv_rows;
|
||||
use crate::csv_export::{format_csv, format_tsv, push_table_csv_row, push_tsv_row};
|
||||
pub use crate::database_export::ExportStatus;
|
||||
use crate::database_export::{
|
||||
build_export_insert_statements, is_export_cancelled, is_internal_export_column, BuildExportInsertStatementsOptions,
|
||||
@@ -82,6 +84,7 @@ pub struct TableExportProgress {
|
||||
|
||||
/// Format rows as CSV text without a header row.
|
||||
/// Used for streaming subsequent pagination batches.
|
||||
#[cfg_attr(not(test), allow(dead_code))]
|
||||
fn format_csv_rows(rows: &[Vec<Value>]) -> String {
|
||||
// 注意:该无表头批次路径的 Null 输出为 ""(带引号空串),与查询结果导出
|
||||
// 及首批 format_csv 的裸空单元格语义不同,直写化必须保留该差异
|
||||
@@ -90,16 +93,40 @@ fn format_csv_rows(rows: &[Vec<Value>]) -> String {
|
||||
if row_index > 0 {
|
||||
out.push('\n');
|
||||
}
|
||||
for (cell_index, cell) in row.iter().enumerate() {
|
||||
if cell_index > 0 {
|
||||
out.push(',');
|
||||
}
|
||||
push_csv_text_value(&mut out, cell);
|
||||
}
|
||||
push_table_csv_row(&mut out, row);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn write_table_text_row<W: Write>(file: &mut W, csv: bool, row: &[Value], buffer: &mut String) -> Result<(), String> {
|
||||
buffer.clear();
|
||||
buffer.push('\n');
|
||||
if csv {
|
||||
push_table_csv_row(buffer, row);
|
||||
} else {
|
||||
push_tsv_row(buffer, row);
|
||||
}
|
||||
file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}"))
|
||||
}
|
||||
|
||||
fn write_table_text_rows<W: Write>(
|
||||
file: &mut W,
|
||||
csv: bool,
|
||||
rows: &[Vec<Value>],
|
||||
buffer: &mut String,
|
||||
) -> Result<(), String> {
|
||||
buffer.clear();
|
||||
for row in rows {
|
||||
buffer.push('\n');
|
||||
if csv {
|
||||
push_table_csv_row(buffer, row);
|
||||
} else {
|
||||
push_tsv_row(buffer, row);
|
||||
}
|
||||
}
|
||||
file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}"))
|
||||
}
|
||||
|
||||
fn export_column_types(request: &TableExportRequest) -> Vec<String> {
|
||||
request
|
||||
.column_types
|
||||
@@ -928,6 +955,7 @@ async fn try_export_native_table_stream(
|
||||
let header = format_csv(col_names, &[]);
|
||||
let header = header.strip_suffix('\n').unwrap_or(&header);
|
||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?;
|
||||
let mut row_buffer = String::new();
|
||||
|
||||
let result = stream_native_table_rows(
|
||||
state,
|
||||
@@ -938,13 +966,12 @@ async fn try_export_native_table_stream(
|
||||
&cancelled,
|
||||
cancel_token.clone(),
|
||||
|row| {
|
||||
let formatted = crate::temporal_format::format_temporal_export_row(
|
||||
let formatted = crate::temporal_format::format_temporal_export_row_cow(
|
||||
row,
|
||||
column_types,
|
||||
request.date_time_format.as_deref(),
|
||||
);
|
||||
let row_csv = format_csv_rows(&[formatted]);
|
||||
write!(file, "\n{row_csv}").map_err(|e| format!("Failed to write CSV rows: {e}"))?;
|
||||
write_table_text_row(&mut file, true, formatted.as_ref(), &mut row_buffer)?;
|
||||
rows_exported += 1;
|
||||
if rows_exported.is_multiple_of(progress_interval) {
|
||||
on_progress(TableExportProgress {
|
||||
@@ -972,6 +999,7 @@ async fn try_export_native_table_stream(
|
||||
let header = format_tsv(col_names, &[]);
|
||||
let header = header.strip_suffix('\n').unwrap_or(&header);
|
||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write TXT: {e}"))?;
|
||||
let mut row_buffer = String::new();
|
||||
|
||||
let result = stream_native_table_rows(
|
||||
state,
|
||||
@@ -982,13 +1010,12 @@ async fn try_export_native_table_stream(
|
||||
&cancelled,
|
||||
cancel_token.clone(),
|
||||
|row| {
|
||||
let formatted = crate::temporal_format::format_temporal_export_row(
|
||||
let formatted = crate::temporal_format::format_temporal_export_row_cow(
|
||||
row,
|
||||
column_types,
|
||||
request.date_time_format.as_deref(),
|
||||
);
|
||||
let row_tsv = format_tsv_rows(&[formatted]);
|
||||
write!(file, "\n{row_tsv}").map_err(|e| format!("Failed to write TXT rows: {e}"))?;
|
||||
write_table_text_row(&mut file, false, formatted.as_ref(), &mut row_buffer)?;
|
||||
rows_exported += 1;
|
||||
if rows_exported.is_multiple_of(progress_interval) {
|
||||
on_progress(TableExportProgress {
|
||||
@@ -1033,12 +1060,12 @@ async fn try_export_native_table_stream(
|
||||
&cancelled,
|
||||
cancel_token.clone(),
|
||||
|row| {
|
||||
let formatted = crate::temporal_format::format_temporal_export_row(
|
||||
let formatted = crate::temporal_format::format_temporal_export_row_cow(
|
||||
row,
|
||||
column_types,
|
||||
request.date_time_format.as_deref(),
|
||||
);
|
||||
writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||
writer.write_row(formatted.as_ref()).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||
rows_exported += 1;
|
||||
if rows_exported.is_multiple_of(progress_interval) {
|
||||
on_progress(TableExportProgress {
|
||||
@@ -1087,12 +1114,12 @@ async fn try_export_native_table_stream(
|
||||
if !is_first_row {
|
||||
file.write_all(b",\n").map_err(|e| format!("Failed to write JSON: {e}"))?;
|
||||
}
|
||||
let formatted = crate::temporal_format::format_temporal_export_row(
|
||||
let formatted = crate::temporal_format::format_temporal_export_row_cow(
|
||||
row,
|
||||
column_types,
|
||||
request.date_time_format.as_deref(),
|
||||
);
|
||||
write_json_row_object(&mut file, col_names, &formatted)?;
|
||||
write_json_row_object(&mut file, col_names, formatted.as_ref())?;
|
||||
is_first_row = false;
|
||||
rows_exported += 1;
|
||||
if rows_exported.is_multiple_of(progress_interval) {
|
||||
@@ -1473,6 +1500,7 @@ async fn export_table_data_core_inner(
|
||||
// 8. Create output file
|
||||
let file = std::fs::File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?;
|
||||
let mut file = BufWriter::new(file);
|
||||
let mut text_buffer = String::new();
|
||||
|
||||
let mut rows_exported: u64 = 0;
|
||||
let batch_size = request.batch_size.unwrap_or(DEFAULT_BATCH_SIZE).max(1);
|
||||
@@ -1533,7 +1561,7 @@ async fn export_table_data_core_inner(
|
||||
if row_count == 0 {
|
||||
break;
|
||||
}
|
||||
let formatted_rows = crate::temporal_format::format_temporal_export_rows(
|
||||
let formatted_rows = crate::temporal_format::format_temporal_export_rows_cow(
|
||||
&result.rows,
|
||||
&column_types,
|
||||
request.date_time_format.as_deref(),
|
||||
@@ -1541,15 +1569,12 @@ async fn export_table_data_core_inner(
|
||||
|
||||
if is_first_batch {
|
||||
// First batch: write header + rows via format_csv
|
||||
let csv_content = format_csv(&col_names, &formatted_rows);
|
||||
let csv_content = format_csv(&col_names, formatted_rows.as_ref());
|
||||
file.write_all(csv_content.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?;
|
||||
is_first_batch = false;
|
||||
} else {
|
||||
// Subsequent batches: write rows only (prepend newline for separation)
|
||||
let rows_csv = format_csv_rows(&formatted_rows);
|
||||
if !rows_csv.is_empty() {
|
||||
write!(file, "\n{rows_csv}").map_err(|e| format!("Failed to write CSV rows: {e}"))?;
|
||||
}
|
||||
write_table_text_rows(&mut file, true, formatted_rows.as_ref(), &mut text_buffer)?;
|
||||
}
|
||||
|
||||
rows_exported += row_count as u64;
|
||||
@@ -1624,21 +1649,17 @@ async fn export_table_data_core_inner(
|
||||
if row_count == 0 {
|
||||
break;
|
||||
}
|
||||
let formatted_rows = crate::temporal_format::format_temporal_export_rows(
|
||||
let formatted_rows = crate::temporal_format::format_temporal_export_rows_cow(
|
||||
&result.rows,
|
||||
&column_types,
|
||||
request.date_time_format.as_deref(),
|
||||
);
|
||||
|
||||
if is_first_batch {
|
||||
let rows_tsv = format_tsv_rows(&formatted_rows);
|
||||
write!(file, "\n{rows_tsv}").map_err(|e| format!("Failed to write TXT rows: {e}"))?;
|
||||
write_table_text_rows(&mut file, false, formatted_rows.as_ref(), &mut text_buffer)?;
|
||||
is_first_batch = false;
|
||||
} else {
|
||||
let rows_tsv = format_tsv_rows(&formatted_rows);
|
||||
if !rows_tsv.is_empty() {
|
||||
write!(file, "\n{rows_tsv}").map_err(|e| format!("Failed to write TXT rows: {e}"))?;
|
||||
}
|
||||
write_table_text_rows(&mut file, false, formatted_rows.as_ref(), &mut text_buffer)?;
|
||||
}
|
||||
|
||||
rows_exported += row_count as u64;
|
||||
@@ -1728,12 +1749,12 @@ async fn export_table_data_core_inner(
|
||||
}
|
||||
|
||||
for row in &result.rows {
|
||||
let formatted = crate::temporal_format::format_temporal_export_row(
|
||||
let formatted = crate::temporal_format::format_temporal_export_row_cow(
|
||||
row,
|
||||
&column_types,
|
||||
request.date_time_format.as_deref(),
|
||||
);
|
||||
writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||
writer.write_row(formatted.as_ref()).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||
}
|
||||
rows_exported += row_count as u64;
|
||||
|
||||
@@ -1827,12 +1848,12 @@ async fn export_table_data_core_inner(
|
||||
if !is_first_row {
|
||||
file.write_all(b",\n").map_err(|e| format!("Failed to write JSON: {e}"))?;
|
||||
}
|
||||
let formatted = crate::temporal_format::format_temporal_export_row(
|
||||
let formatted = crate::temporal_format::format_temporal_export_row_cow(
|
||||
row,
|
||||
&column_types,
|
||||
request.date_time_format.as_deref(),
|
||||
);
|
||||
write_json_row_object(&mut file, &col_names, &formatted)?;
|
||||
write_json_row_object(&mut file, &col_names, formatted.as_ref())?;
|
||||
is_first_row = false;
|
||||
}
|
||||
|
||||
@@ -1908,12 +1929,12 @@ async fn export_table_data_core_inner(
|
||||
break;
|
||||
}
|
||||
|
||||
let formatted_rows = crate::temporal_format::format_temporal_export_rows(
|
||||
let formatted_rows = crate::temporal_format::format_temporal_export_rows_cow(
|
||||
&result.rows,
|
||||
&column_types,
|
||||
request.date_time_format.as_deref(),
|
||||
);
|
||||
let rows_markdown = format_markdown_rows(&formatted_rows);
|
||||
let rows_markdown = format_markdown_rows(formatted_rows.as_ref());
|
||||
if !rows_markdown.is_empty() {
|
||||
if wrote_rows {
|
||||
file.write_all(b"\n").map_err(|e| format!("Failed to write Markdown: {e}"))?;
|
||||
@@ -2270,6 +2291,16 @@ mod tests {
|
||||
assert_eq!(out, "\"just\",\"one\"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reusable_text_row_buffer_preserves_table_null_semantics() {
|
||||
let row = vec![Value::Null, json!(""), json!("line\n\"two\"")];
|
||||
let mut output = Vec::new();
|
||||
let mut buffer = String::new();
|
||||
|
||||
write_table_text_row(&mut output, true, &row, &mut buffer).expect("write csv row");
|
||||
assert_eq!(String::from_utf8(output).expect("utf8 csv"), "\n\"\",\"\",\"line\n\"\"two\"\"\"");
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// format_tsv (Navicat-style TXT export)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use std::borrow::Cow;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fs::File;
|
||||
use std::io::{BufRead, BufReader, Read as IoRead, Seek, SeekFrom, Write as IoWrite};
|
||||
@@ -19,8 +20,9 @@ use sha2::{Digest, Sha256};
|
||||
use crate::connection::{task_client_session_id, AppState, PoolKind};
|
||||
use crate::models::connection::DatabaseType;
|
||||
use crate::transfer::{
|
||||
execute_on_pool, generate_insert_typed, generate_insert_typed_sql_batches, get_columns_for_transfer,
|
||||
normalize_integer_literal, normalize_thousands_numeric_literal, qualified_table, quote_identifier, SqlBatchLimits,
|
||||
escape_value_typed, execute_on_pool, generate_insert_typed_from_value_rows,
|
||||
generate_insert_typed_sql_batches_from_value_rows, get_columns_for_transfer, normalize_integer_literal,
|
||||
normalize_thousands_numeric_literal, qualified_table, quote_identifier, SqlBatchLimits,
|
||||
};
|
||||
|
||||
pub const DEFAULT_PREVIEW_LIMIT: usize = 50;
|
||||
@@ -3142,9 +3144,8 @@ fn build_import_insert_batch_with_plan(
|
||||
if rows.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let mapped_rows = map_import_rows_with_plan(rows, plan, db_type, kingbase_oracle_mode, date_time_format);
|
||||
let sql =
|
||||
generate_insert_typed(&plan.target_columns, &plan.column_types, &mapped_rows, table, schema, db_type, None);
|
||||
let value_rows = import_value_rows_sql(rows, plan, db_type, kingbase_oracle_mode, date_time_format);
|
||||
let sql = generate_insert_typed_from_value_rows(&plan.target_columns, &value_rows, table, schema, db_type, None);
|
||||
Ok((!sql.trim().is_empty()).then_some(ImportSqlBatch { sql, row_count: rows.len() }))
|
||||
}
|
||||
|
||||
@@ -3161,11 +3162,10 @@ fn build_import_insert_batches_with_plan(
|
||||
if rows.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let mapped_rows = map_import_rows_with_plan(rows, plan, db_type, kingbase_oracle_mode, date_time_format);
|
||||
let batches = generate_insert_typed_sql_batches(
|
||||
let value_rows = import_value_rows_sql(rows, plan, db_type, kingbase_oracle_mode, date_time_format);
|
||||
let batches = generate_insert_typed_sql_batches_from_value_rows(
|
||||
&plan.target_columns,
|
||||
&plan.column_types,
|
||||
&mapped_rows,
|
||||
&value_rows,
|
||||
table,
|
||||
schema,
|
||||
db_type,
|
||||
@@ -3175,16 +3175,31 @@ fn build_import_insert_batches_with_plan(
|
||||
Ok(batches.into_iter().map(|(sql, row_count)| ImportSqlBatch { sql, row_count }).collect())
|
||||
}
|
||||
|
||||
fn map_import_rows_with_plan(
|
||||
fn import_value_rows_sql(
|
||||
rows: &[Vec<serde_json::Value>],
|
||||
plan: &CompiledImportPlan,
|
||||
db_type: &DatabaseType,
|
||||
kingbase_oracle_mode: bool,
|
||||
date_time_format: Option<&str>,
|
||||
) -> Vec<Vec<serde_json::Value>> {
|
||||
rows.iter()
|
||||
.map(|row| map_import_row_with_plan(row, plan, db_type, kingbase_oracle_mode, date_time_format))
|
||||
.collect()
|
||||
) -> Vec<String> {
|
||||
let mut value_rows = Vec::with_capacity(rows.len());
|
||||
for row in rows {
|
||||
let mut values = String::with_capacity(plan.mapped_source_indexes.len().saturating_mul(16).saturating_add(2));
|
||||
values.push('(');
|
||||
for (target_index, source_index) in plan.mapped_source_indexes.iter().enumerate() {
|
||||
if target_index > 0 {
|
||||
values.push_str(", ");
|
||||
}
|
||||
let source_value = row.get(*source_index).unwrap_or(&serde_json::Value::Null);
|
||||
let data_type = plan.column_types.get(target_index).and_then(|data_type| data_type.as_deref());
|
||||
let normalized =
|
||||
normalize_import_value_cow(source_value, data_type, db_type, kingbase_oracle_mode, date_time_format);
|
||||
values.push_str(&escape_value_typed(normalized.as_ref(), db_type, data_type));
|
||||
}
|
||||
values.push(')');
|
||||
value_rows.push(values);
|
||||
}
|
||||
value_rows
|
||||
}
|
||||
|
||||
fn map_import_row_with_plan(
|
||||
@@ -3274,17 +3289,17 @@ fn effective_import_batch_size(db_type: &DatabaseType, requested: usize) -> usiz
|
||||
requested.max(1).min(max_rows)
|
||||
}
|
||||
|
||||
fn normalize_import_temporal_value(
|
||||
value: &serde_json::Value,
|
||||
fn normalize_import_temporal_value_cow<'a>(
|
||||
value: &'a serde_json::Value,
|
||||
data_type: Option<&str>,
|
||||
db_type: &DatabaseType,
|
||||
kingbase_oracle_mode: bool,
|
||||
date_time_format: Option<&str>,
|
||||
) -> serde_json::Value {
|
||||
) -> Cow<'a, serde_json::Value> {
|
||||
let date_type_preserves_time = (matches!(db_type, DatabaseType::Oracle | DatabaseType::OceanbaseOracle)
|
||||
|| (*db_type == DatabaseType::Kingbase && kingbase_oracle_mode))
|
||||
&& data_type.is_some_and(|data_type| data_type.trim().eq_ignore_ascii_case("date"));
|
||||
crate::temporal_format::normalize_temporal_import_value(
|
||||
crate::temporal_format::normalize_temporal_import_value_cow(
|
||||
value,
|
||||
if date_type_preserves_time { Some("datetime") } else { data_type },
|
||||
date_time_format,
|
||||
@@ -3352,6 +3367,41 @@ fn textual_source_columns_for_import(
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn normalize_import_value_cow<'a>(
|
||||
value: &'a serde_json::Value,
|
||||
data_type: Option<&str>,
|
||||
db_type: &DatabaseType,
|
||||
kingbase_oracle_mode: bool,
|
||||
date_time_format: Option<&str>,
|
||||
) -> Cow<'a, serde_json::Value> {
|
||||
let normalized =
|
||||
normalize_import_temporal_value_cow(value, data_type, db_type, kingbase_oracle_mode, date_time_format);
|
||||
// Strip validated thousands separators before integer canonicalization so "1,234.00"
|
||||
// still collapses to a plain integer literal for integer targets.
|
||||
let thousands_canonical =
|
||||
normalized.as_str().and_then(|value| normalize_thousands_numeric_literal(value, db_type, data_type));
|
||||
if let Some(integer_text) = thousands_canonical
|
||||
.as_deref()
|
||||
.or_else(|| normalized.as_str())
|
||||
.and_then(|value| normalize_integer_literal(value, db_type, data_type))
|
||||
.and_then(|value| value.parse::<i64>().ok())
|
||||
{
|
||||
// Normalize before both INSERT and COPY paths; COPY does not pass through SQL literal escaping.
|
||||
return Cow::Owned(serde_json::Value::Number(integer_text.into()));
|
||||
}
|
||||
if let Some(number) = normalized.as_number() {
|
||||
if let Some(integer_text) = normalize_integer_literal(&number.to_string(), db_type, data_type)
|
||||
.and_then(|value| value.parse::<i64>().ok())
|
||||
{
|
||||
return Cow::Owned(serde_json::Value::Number(integer_text.into()));
|
||||
}
|
||||
}
|
||||
if let Some(canonical) = thousands_canonical {
|
||||
return Cow::Owned(serde_json::Value::String(canonical));
|
||||
}
|
||||
normalized
|
||||
}
|
||||
|
||||
fn normalize_import_value(
|
||||
value: &serde_json::Value,
|
||||
data_type: Option<&str>,
|
||||
@@ -3359,28 +3409,7 @@ fn normalize_import_value(
|
||||
kingbase_oracle_mode: bool,
|
||||
date_time_format: Option<&str>,
|
||||
) -> serde_json::Value {
|
||||
let normalized = normalize_import_temporal_value(value, data_type, db_type, kingbase_oracle_mode, date_time_format);
|
||||
// Strip validated thousands separators before integer canonicalization so "1,234.00"
|
||||
// still collapses to a plain integer literal for integer targets.
|
||||
let thousands_canonical =
|
||||
normalized.as_str().and_then(|value| normalize_thousands_numeric_literal(value, db_type, data_type));
|
||||
let integer_text = thousands_canonical
|
||||
.as_deref()
|
||||
.or_else(|| normalized.as_str())
|
||||
.map(str::to_owned)
|
||||
.or_else(|| normalized.as_number().map(ToString::to_string));
|
||||
if let Some(integer_text) = integer_text
|
||||
.as_deref()
|
||||
.and_then(|value| normalize_integer_literal(value, db_type, data_type))
|
||||
.and_then(|value| value.parse::<i64>().ok())
|
||||
{
|
||||
// Normalize before both INSERT and COPY paths; COPY does not pass through SQL literal escaping.
|
||||
return serde_json::Value::Number(integer_text.into());
|
||||
}
|
||||
if let Some(canonical) = thousands_canonical {
|
||||
return serde_json::Value::String(canonical);
|
||||
}
|
||||
normalized
|
||||
normalize_import_value_cow(value, data_type, db_type, kingbase_oracle_mode, date_time_format).into_owned()
|
||||
}
|
||||
|
||||
pub fn build_import_insert_batches(
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use chrono::{DateTime, FixedOffset, NaiveDate, NaiveDateTime, NaiveTime, Timelike};
|
||||
use serde_json::Value;
|
||||
use std::borrow::Cow;
|
||||
use std::fmt::Write as _;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -192,17 +193,51 @@ pub fn format_temporal_export_value(value: &Value, data_type: Option<&str>, patt
|
||||
.unwrap_or_else(|| value.clone())
|
||||
}
|
||||
|
||||
fn active_temporal_pattern(pattern: Option<&str>) -> Option<&str> {
|
||||
pattern.filter(|pattern| !pattern.trim().is_empty())
|
||||
}
|
||||
|
||||
pub fn format_temporal_export_row_cow<'a>(
|
||||
row: &'a [Value],
|
||||
column_types: &[Option<String>],
|
||||
pattern: Option<&str>,
|
||||
) -> Cow<'a, [Value]> {
|
||||
let Some(pattern) = active_temporal_pattern(pattern) else {
|
||||
return Cow::Borrowed(row);
|
||||
};
|
||||
if !column_types.iter().any(|data_type| temporal_kind(data_type.as_deref()).is_some()) {
|
||||
return Cow::Borrowed(row);
|
||||
}
|
||||
Cow::Owned(
|
||||
row.iter()
|
||||
.enumerate()
|
||||
.map(|(index, value)| {
|
||||
format_temporal_export_value(
|
||||
value,
|
||||
column_types.get(index).and_then(|data_type| data_type.as_deref()),
|
||||
Some(pattern),
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn format_temporal_export_row(row: &[Value], column_types: &[Option<String>], pattern: Option<&str>) -> Vec<Value> {
|
||||
row.iter()
|
||||
.enumerate()
|
||||
.map(|(index, value)| {
|
||||
format_temporal_export_value(
|
||||
value,
|
||||
column_types.get(index).and_then(|data_type| data_type.as_deref()),
|
||||
pattern,
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
format_temporal_export_row_cow(row, column_types, pattern).into_owned()
|
||||
}
|
||||
|
||||
pub fn format_temporal_export_rows_cow<'a>(
|
||||
rows: &'a [Vec<Value>],
|
||||
column_types: &[Option<String>],
|
||||
pattern: Option<&str>,
|
||||
) -> Cow<'a, [Vec<Value>]> {
|
||||
let Some(pattern) = active_temporal_pattern(pattern) else {
|
||||
return Cow::Borrowed(rows);
|
||||
};
|
||||
if !column_types.iter().any(|data_type| temporal_kind(data_type.as_deref()).is_some()) {
|
||||
return Cow::Borrowed(rows);
|
||||
}
|
||||
Cow::Owned(rows.iter().map(|row| format_temporal_export_row(row, column_types, Some(pattern))).collect())
|
||||
}
|
||||
|
||||
pub fn format_temporal_export_rows(
|
||||
@@ -210,7 +245,28 @@ pub fn format_temporal_export_rows(
|
||||
column_types: &[Option<String>],
|
||||
pattern: Option<&str>,
|
||||
) -> Vec<Vec<Value>> {
|
||||
rows.iter().map(|row| format_temporal_export_row(row, column_types, pattern)).collect()
|
||||
format_temporal_export_rows_cow(rows, column_types, pattern).into_owned()
|
||||
}
|
||||
|
||||
pub fn format_temporal_export_row_with_string_types_cow<'a>(
|
||||
row: &'a [Value],
|
||||
column_types: &[String],
|
||||
pattern: Option<&str>,
|
||||
) -> Cow<'a, [Value]> {
|
||||
let Some(pattern) = active_temporal_pattern(pattern) else {
|
||||
return Cow::Borrowed(row);
|
||||
};
|
||||
if !column_types.iter().any(|data_type| temporal_kind(Some(data_type)).is_some()) {
|
||||
return Cow::Borrowed(row);
|
||||
}
|
||||
Cow::Owned(
|
||||
row.iter()
|
||||
.enumerate()
|
||||
.map(|(index, value)| {
|
||||
format_temporal_export_value(value, column_types.get(index).map(String::as_str), Some(pattern))
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn format_temporal_export_row_with_string_types(
|
||||
@@ -218,10 +274,23 @@ pub fn format_temporal_export_row_with_string_types(
|
||||
column_types: &[String],
|
||||
pattern: Option<&str>,
|
||||
) -> Vec<Value> {
|
||||
row.iter()
|
||||
.enumerate()
|
||||
.map(|(index, value)| format_temporal_export_value(value, column_types.get(index).map(String::as_str), pattern))
|
||||
.collect()
|
||||
format_temporal_export_row_with_string_types_cow(row, column_types, pattern).into_owned()
|
||||
}
|
||||
|
||||
pub fn format_temporal_export_rows_with_string_types_cow<'a>(
|
||||
rows: &'a [Vec<Value>],
|
||||
column_types: &[String],
|
||||
pattern: Option<&str>,
|
||||
) -> Cow<'a, [Vec<Value>]> {
|
||||
let Some(pattern) = active_temporal_pattern(pattern) else {
|
||||
return Cow::Borrowed(rows);
|
||||
};
|
||||
if !column_types.iter().any(|data_type| temporal_kind(Some(data_type)).is_some()) {
|
||||
return Cow::Borrowed(rows);
|
||||
}
|
||||
Cow::Owned(
|
||||
rows.iter().map(|row| format_temporal_export_row_with_string_types(row, column_types, Some(pattern))).collect(),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn format_temporal_export_rows_with_string_types(
|
||||
@@ -229,40 +298,44 @@ pub fn format_temporal_export_rows_with_string_types(
|
||||
column_types: &[String],
|
||||
pattern: Option<&str>,
|
||||
) -> Vec<Vec<Value>> {
|
||||
rows.iter().map(|row| format_temporal_export_row_with_string_types(row, column_types, pattern)).collect()
|
||||
format_temporal_export_rows_with_string_types_cow(rows, column_types, pattern).into_owned()
|
||||
}
|
||||
|
||||
pub fn normalize_temporal_import_value(value: &Value, data_type: Option<&str>, pattern: Option<&str>) -> Value {
|
||||
pub(crate) fn normalize_temporal_import_value_cow<'a>(
|
||||
value: &'a Value,
|
||||
data_type: Option<&str>,
|
||||
pattern: Option<&str>,
|
||||
) -> Cow<'a, Value> {
|
||||
let Some(kind) = temporal_kind(data_type) else {
|
||||
return value.clone();
|
||||
return Cow::Borrowed(value);
|
||||
};
|
||||
let Some(raw) = value.as_str() else {
|
||||
return value.clone();
|
||||
return Cow::Borrowed(value);
|
||||
};
|
||||
let Some(parsed) = parse_temporal(raw.trim(), pattern) else {
|
||||
return value.clone();
|
||||
return Cow::Borrowed(value);
|
||||
};
|
||||
|
||||
let normalized = match (kind, parsed) {
|
||||
(TemporalKind::Date, ParsedTemporal::Zoned(value)) => value.date_naive().format("%Y-%m-%d").to_string(),
|
||||
(TemporalKind::Date, ParsedTemporal::DateTime(value)) => value.date().format("%Y-%m-%d").to_string(),
|
||||
(TemporalKind::Date, ParsedTemporal::Date(value)) => value.format("%Y-%m-%d").to_string(),
|
||||
(TemporalKind::Date, ParsedTemporal::Time(_)) => return value.clone(),
|
||||
(TemporalKind::Date, ParsedTemporal::Time(_)) => return Cow::Borrowed(value),
|
||||
(TemporalKind::Time, ParsedTemporal::Zoned(value)) => value.time().format("%H:%M:%S%.f").to_string(),
|
||||
(TemporalKind::Time, ParsedTemporal::DateTime(value)) => value.time().format("%H:%M:%S%.f").to_string(),
|
||||
(TemporalKind::Time, ParsedTemporal::Time(value)) => value.format("%H:%M:%S%.f").to_string(),
|
||||
(TemporalKind::Time, ParsedTemporal::Date(_)) => return value.clone(),
|
||||
(TemporalKind::Time, ParsedTemporal::Date(_)) => return Cow::Borrowed(value),
|
||||
(TemporalKind::DateTime, ParsedTemporal::Zoned(value)) => {
|
||||
value.naive_local().format("%Y-%m-%d %H:%M:%S%.f").to_string()
|
||||
}
|
||||
(TemporalKind::DateTime, ParsedTemporal::DateTime(value)) => value.format("%Y-%m-%d %H:%M:%S%.f").to_string(),
|
||||
(TemporalKind::DateTime, ParsedTemporal::Date(date)) => {
|
||||
let Some(value) = date.and_hms_opt(0, 0, 0) else {
|
||||
return value.clone();
|
||||
return Cow::Borrowed(value);
|
||||
};
|
||||
value.format("%Y-%m-%d %H:%M:%S").to_string()
|
||||
}
|
||||
(TemporalKind::DateTime, ParsedTemporal::Time(_)) => return value.clone(),
|
||||
(TemporalKind::DateTime, ParsedTemporal::Time(_)) => return Cow::Borrowed(value),
|
||||
(TemporalKind::DateTimeWithTimeZone, ParsedTemporal::Zoned(value)) => {
|
||||
value.format("%Y-%m-%dT%H:%M:%S%.f%:z").to_string()
|
||||
}
|
||||
@@ -271,13 +344,17 @@ pub fn normalize_temporal_import_value(value: &Value, data_type: Option<&str>, p
|
||||
}
|
||||
(TemporalKind::DateTimeWithTimeZone, ParsedTemporal::Date(date)) => {
|
||||
let Some(value) = date.and_hms_opt(0, 0, 0) else {
|
||||
return value.clone();
|
||||
return Cow::Borrowed(value);
|
||||
};
|
||||
value.format("%Y-%m-%d %H:%M:%S").to_string()
|
||||
}
|
||||
(TemporalKind::DateTimeWithTimeZone, ParsedTemporal::Time(_)) => return value.clone(),
|
||||
(TemporalKind::DateTimeWithTimeZone, ParsedTemporal::Time(_)) => return Cow::Borrowed(value),
|
||||
};
|
||||
Value::String(normalized)
|
||||
Cow::Owned(Value::String(normalized))
|
||||
}
|
||||
|
||||
pub fn normalize_temporal_import_value(value: &Value, data_type: Option<&str>, pattern: Option<&str>) -> Value {
|
||||
normalize_temporal_import_value_cow(value, data_type, pattern).into_owned()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -358,4 +435,31 @@ mod tests {
|
||||
json!("2024-02-25T13:02:15+08:00")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_cow_borrows_when_no_formatting_is_needed() {
|
||||
let row = vec![json!(1), json!("2024-02-25 13:02:15")];
|
||||
let rows = vec![row.clone()];
|
||||
|
||||
assert!(matches!(
|
||||
format_temporal_export_row_cow(&row, &[None, Some("TIMESTAMP".to_string())], None),
|
||||
Cow::Borrowed(_)
|
||||
));
|
||||
assert!(matches!(
|
||||
format_temporal_export_rows_cow(&rows, &[None, Some("VARCHAR".to_string())], Some("YYYY/MM/DD")),
|
||||
Cow::Borrowed(_)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn import_cow_borrows_unchanged_values_and_owns_normalized_values() {
|
||||
let text = json!("plain");
|
||||
assert!(matches!(normalize_temporal_import_value_cow(&text, Some("TEXT"), None), Cow::Borrowed(_)));
|
||||
|
||||
let timestamp = json!("2024/2/25 13:02:15");
|
||||
assert_eq!(
|
||||
normalize_temporal_import_value_cow(×tamp, Some("TIMESTAMP"), None).into_owned(),
|
||||
json!("2024-02-25 13:02:15")
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -3263,9 +3263,19 @@ pub fn generate_insert_typed(
|
||||
return String::new();
|
||||
}
|
||||
|
||||
let template = InsertSqlTemplate::new(columns, table, schema, db_type, catalog, false);
|
||||
let value_rows = value_rows_sql(rows, column_types, db_type, false);
|
||||
template.build(&value_rows)
|
||||
generate_insert_typed_from_value_rows(columns, &value_rows, table, schema, db_type, catalog)
|
||||
}
|
||||
|
||||
pub(crate) fn generate_insert_typed_from_value_rows(
|
||||
columns: &[String],
|
||||
value_rows: &[String],
|
||||
table: &str,
|
||||
schema: &str,
|
||||
db_type: &DatabaseType,
|
||||
catalog: Option<&str>,
|
||||
) -> String {
|
||||
InsertSqlTemplate::new(columns, table, schema, db_type, catalog, false).build(value_rows)
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -3302,7 +3312,13 @@ impl InsertSqlTemplate {
|
||||
return String::new();
|
||||
}
|
||||
if let Some(into_prefix) = self.oracle_into_prefix.as_deref().filter(|_| value_rows.len() > 1) {
|
||||
let mut sql = String::from("INSERT ALL\n");
|
||||
let capacity = "INSERT ALL\n".len()
|
||||
+ into_prefix.len().saturating_mul(value_rows.len())
|
||||
+ value_rows.iter().map(String::len).sum::<usize>()
|
||||
+ value_rows.len().saturating_sub(1)
|
||||
+ "\nSELECT 1 FROM dual".len();
|
||||
let mut sql = String::with_capacity(capacity);
|
||||
sql.push_str("INSERT ALL\n");
|
||||
for (index, values) in value_rows.iter().enumerate() {
|
||||
if index > 0 {
|
||||
sql.push('\n');
|
||||
@@ -3314,8 +3330,17 @@ impl InsertSqlTemplate {
|
||||
return sql;
|
||||
}
|
||||
|
||||
let mut sql = self.standard_prefix.clone();
|
||||
sql.push_str(&value_rows.join(",\n"));
|
||||
let capacity = self.standard_prefix.len()
|
||||
+ value_rows.iter().map(String::len).sum::<usize>()
|
||||
+ ",\n".len().saturating_mul(value_rows.len().saturating_sub(1));
|
||||
let mut sql = String::with_capacity(capacity);
|
||||
sql.push_str(&self.standard_prefix);
|
||||
for (index, values) in value_rows.iter().enumerate() {
|
||||
if index > 0 {
|
||||
sql.push_str(",\n");
|
||||
}
|
||||
sql.push_str(values);
|
||||
}
|
||||
sql
|
||||
}
|
||||
|
||||
@@ -3350,8 +3375,12 @@ fn value_rows_sql(
|
||||
) -> Vec<String> {
|
||||
let mut out = Vec::with_capacity(rows.len());
|
||||
for row in rows {
|
||||
let mut vals = Vec::with_capacity(row.len());
|
||||
let mut values = String::with_capacity(row.len().saturating_mul(16).saturating_add(2));
|
||||
values.push('(');
|
||||
for (index, v) in row.iter().enumerate() {
|
||||
if index > 0 {
|
||||
values.push_str(", ");
|
||||
}
|
||||
let column_type = column_types.get(index).and_then(|value| value.as_deref());
|
||||
let value = if mysql_spatial_markers {
|
||||
crate::database_export::format_mysql_spatial_export_literal(v, Some(*db_type), column_type)
|
||||
@@ -3359,9 +3388,10 @@ fn value_rows_sql(
|
||||
None
|
||||
}
|
||||
.unwrap_or_else(|| escape_value_typed(v, db_type, column_type));
|
||||
vals.push(value);
|
||||
values.push_str(&value);
|
||||
}
|
||||
out.push(format!("({})", vals.join(", ")));
|
||||
values.push(')');
|
||||
out.push(values);
|
||||
}
|
||||
out
|
||||
}
|
||||
@@ -3714,6 +3744,7 @@ fn generate_transfer_write_sql(
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[cfg_attr(not(test), allow(dead_code))]
|
||||
pub(crate) fn generate_insert_typed_sql_batches(
|
||||
columns: &[String],
|
||||
column_types: &[Option<String>],
|
||||
@@ -3724,18 +3755,21 @@ pub(crate) fn generate_insert_typed_sql_batches(
|
||||
catalog: Option<&str>,
|
||||
limits: SqlBatchLimits,
|
||||
) -> Result<Vec<(String, usize)>, String> {
|
||||
generate_insert_typed_sql_batches_for_transfer(
|
||||
columns,
|
||||
column_types,
|
||||
rows,
|
||||
table,
|
||||
schema,
|
||||
db_type,
|
||||
catalog,
|
||||
limits,
|
||||
false,
|
||||
false,
|
||||
)
|
||||
let value_rows = value_rows_sql(rows, column_types, db_type, false);
|
||||
generate_insert_typed_sql_batches_from_value_rows(columns, &value_rows, table, schema, db_type, catalog, limits)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn generate_insert_typed_sql_batches_from_value_rows(
|
||||
columns: &[String],
|
||||
value_rows: &[String],
|
||||
table: &str,
|
||||
schema: &str,
|
||||
db_type: &DatabaseType,
|
||||
catalog: Option<&str>,
|
||||
limits: SqlBatchLimits,
|
||||
) -> Result<Vec<(String, usize)>, String> {
|
||||
generate_insert_sql_batches_from_value_rows(columns, value_rows, table, schema, db_type, catalog, limits, false)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
@@ -3755,6 +3789,34 @@ fn generate_insert_typed_sql_batches_for_transfer(
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let value_rows = value_rows_sql(rows, column_types, db_type, mysql_spatial_markers);
|
||||
generate_insert_sql_batches_from_value_rows(
|
||||
columns,
|
||||
&value_rows,
|
||||
table,
|
||||
schema,
|
||||
db_type,
|
||||
catalog,
|
||||
limits,
|
||||
overrides_postgres_system_values,
|
||||
)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn generate_insert_sql_batches_from_value_rows(
|
||||
columns: &[String],
|
||||
value_rows: &[String],
|
||||
table: &str,
|
||||
schema: &str,
|
||||
db_type: &DatabaseType,
|
||||
catalog: Option<&str>,
|
||||
limits: SqlBatchLimits,
|
||||
overrides_postgres_system_values: bool,
|
||||
) -> Result<Vec<(String, usize)>, String> {
|
||||
if value_rows.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let max_rows = limits.max_rows.max(1).min(match db_type {
|
||||
DatabaseType::SqlServer => MAX_SQLSERVER_INSERT_ROWS,
|
||||
DatabaseType::Oracle => MAX_ORACLE_INSERT_ALL_ROWS,
|
||||
@@ -3763,7 +3825,6 @@ fn generate_insert_typed_sql_batches_for_transfer(
|
||||
let target_sql_bytes = limits.target_sql_bytes.max(1);
|
||||
let batch_sql_bytes = limits.hard_sql_bytes.map_or(target_sql_bytes, |hard| target_sql_bytes.min(hard));
|
||||
let template = InsertSqlTemplate::new(columns, table, schema, db_type, catalog, overrides_postgres_system_values);
|
||||
let value_rows = value_rows_sql(rows, column_types, db_type, mysql_spatial_markers);
|
||||
let value_row_bytes = value_rows.iter().map(|row| sql_text_bytes(row, db_type)).collect::<Vec<_>>();
|
||||
let mut statements = Vec::new();
|
||||
let mut start = 0usize;
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use std::fmt::Write as FmtWrite;
|
||||
use std::io::{Cursor, Seek, Write};
|
||||
|
||||
use crate::temporal_format::{excel_temporal_serial, ExcelTemporalKind};
|
||||
@@ -130,6 +131,7 @@ pub struct StreamingXlsxWriter<W: Write + Seek> {
|
||||
column_comments: Vec<Option<String>>,
|
||||
date_time_format: Option<String>,
|
||||
numeric_right_align: bool,
|
||||
row_buffer: String,
|
||||
}
|
||||
|
||||
/// Estimate column widths from header names only (used by the streaming path
|
||||
@@ -159,8 +161,8 @@ fn cols_xml(widths: &[usize]) -> String {
|
||||
|
||||
/// Resolve the effective header text: prefer a non-empty column comment, fall
|
||||
/// back to the original column name.
|
||||
fn effective_header(column: &str, comment: Option<&str>) -> String {
|
||||
comment.filter(|c| !c.is_empty()).unwrap_or(column).to_string()
|
||||
fn effective_header<'a>(column: &'a str, comment: Option<&'a str>) -> &'a str {
|
||||
comment.filter(|c| !c.is_empty()).unwrap_or(column)
|
||||
}
|
||||
|
||||
/// Build a single `<row>` XML fragment for the header row (row 1).
|
||||
@@ -172,30 +174,36 @@ pub(crate) fn header_row_xml(columns: &[String], column_comments: &[Option<Strin
|
||||
.enumerate()
|
||||
.map(|(index, col)| {
|
||||
let header = effective_header(col, column_comments.get(index).and_then(|c| c.as_deref()));
|
||||
cell_xml(Some(&Value::String(header)), 0, index, Some(1))
|
||||
cell_xml(Some(&Value::String(header.to_string())), 0, index, Some(1))
|
||||
})
|
||||
.collect::<String>()
|
||||
)
|
||||
}
|
||||
|
||||
fn data_row_xml_with_date_time_format(
|
||||
fn push_data_row_xml(
|
||||
output: &mut String,
|
||||
row_number: usize,
|
||||
columns: &[String],
|
||||
column_types: &[String],
|
||||
row: &[Value],
|
||||
date_time_format: Option<&str>,
|
||||
numeric_right_align: bool,
|
||||
) -> String {
|
||||
let cells = columns
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(col_index, _)| {
|
||||
let col_type = column_types.get(col_index);
|
||||
let align_style = numeric_column_style(col_type, numeric_right_align);
|
||||
typed_cell_xml(row.get(col_index), col_type, row_number - 1, col_index, align_style, date_time_format)
|
||||
})
|
||||
.collect::<String>();
|
||||
format!("<row r=\"{row_number}\">{cells}</row>")
|
||||
) {
|
||||
write!(output, "<row r=\"{row_number}\">").expect("writing XLSX XML into a String cannot fail");
|
||||
for (col_index, _) in columns.iter().enumerate() {
|
||||
let col_type = column_types.get(col_index);
|
||||
let align_style = numeric_column_style(col_type, numeric_right_align);
|
||||
push_typed_cell_xml(
|
||||
output,
|
||||
row.get(col_index),
|
||||
col_type,
|
||||
row_number - 1,
|
||||
col_index,
|
||||
align_style,
|
||||
date_time_format,
|
||||
);
|
||||
}
|
||||
output.push_str("</row>");
|
||||
}
|
||||
|
||||
/// Shared ZIP entry options for all XLSX parts. XLSX is a ZIP of XML, and the
|
||||
@@ -298,6 +306,7 @@ fn start_xlsx_writer_inner<W: Write + Seek>(
|
||||
column_comments: column_comments.to_vec(),
|
||||
date_time_format: date_time_format.map(str::to_string),
|
||||
numeric_right_align,
|
||||
row_buffer: String::with_capacity(columns.len().saturating_mul(48)),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -355,19 +364,17 @@ impl<W: Write + Seek> StreamingXlsxWriter<W> {
|
||||
self.finish_current_sheet()?;
|
||||
self.start_next_data_sheet()?;
|
||||
}
|
||||
self.zip
|
||||
.write_all(
|
||||
data_row_xml_with_date_time_format(
|
||||
self.next_row_number,
|
||||
&self.columns,
|
||||
&self.column_types,
|
||||
row,
|
||||
self.date_time_format.as_deref(),
|
||||
self.numeric_right_align,
|
||||
)
|
||||
.as_bytes(),
|
||||
)
|
||||
.map_err(|err| err.to_string())?;
|
||||
self.row_buffer.clear();
|
||||
push_data_row_xml(
|
||||
&mut self.row_buffer,
|
||||
self.next_row_number,
|
||||
&self.columns,
|
||||
&self.column_types,
|
||||
row,
|
||||
self.date_time_format.as_deref(),
|
||||
self.numeric_right_align,
|
||||
);
|
||||
self.zip.write_all(self.row_buffer.as_bytes()).map_err(|err| err.to_string())?;
|
||||
self.next_row_number += 1;
|
||||
self.current_data_rows += 1;
|
||||
Ok(())
|
||||
@@ -469,6 +476,11 @@ pub(crate) fn finish_streaming_xlsx_workbook<W: Write + Seek>(writer: StreamingX
|
||||
|
||||
fn escape_xml(value: &str) -> String {
|
||||
let mut result = String::with_capacity(value.len());
|
||||
push_xml_escaped(&mut result, value);
|
||||
result
|
||||
}
|
||||
|
||||
fn push_xml_escaped(result: &mut String, value: &str) {
|
||||
for ch in value.chars() {
|
||||
let code = ch as u32;
|
||||
if code != 9 && code != 10 && code != 13 && code < 32 {
|
||||
@@ -482,22 +494,36 @@ fn escape_xml(value: &str) -> String {
|
||||
_ => result.push(ch),
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn column_name(index: usize) -> String {
|
||||
let mut out = String::new();
|
||||
let mut out = String::with_capacity(4);
|
||||
push_column_name(&mut out, index);
|
||||
out
|
||||
}
|
||||
|
||||
fn push_column_name(out: &mut String, index: usize) {
|
||||
let mut buffer = [0u8; 16];
|
||||
let mut cursor = buffer.len();
|
||||
let mut n = index + 1;
|
||||
while n > 0 {
|
||||
let rem = (n - 1) % 26;
|
||||
out.push((b'A' + rem as u8) as char);
|
||||
cursor -= 1;
|
||||
buffer[cursor] = b'A' + rem as u8;
|
||||
n = (n - 1) / 26;
|
||||
}
|
||||
out.chars().rev().collect()
|
||||
out.push_str(std::str::from_utf8(&buffer[cursor..]).expect("XLSX column names are ASCII"));
|
||||
}
|
||||
|
||||
fn cell_ref(row_index: usize, col_index: usize) -> String {
|
||||
format!("{}{}", column_name(col_index), row_index + 1)
|
||||
let mut reference = String::with_capacity(8);
|
||||
push_cell_ref(&mut reference, row_index, col_index);
|
||||
reference
|
||||
}
|
||||
|
||||
fn push_cell_ref(out: &mut String, row_index: usize, col_index: usize) {
|
||||
push_column_name(out, col_index);
|
||||
write!(out, "{}", row_index + 1).expect("writing XLSX cell references into a String cannot fail");
|
||||
}
|
||||
|
||||
fn sheet_range(column_count: usize, row_count: usize) -> String {
|
||||
@@ -560,6 +586,56 @@ fn cell_xml(value: Option<&Value>, row_index: usize, col_index: usize, style: Op
|
||||
}
|
||||
}
|
||||
|
||||
fn push_cell_style(output: &mut String, style: Option<usize>) {
|
||||
if let Some(style) = style {
|
||||
write!(output, " s=\"{style}\"").expect("writing XLSX cell styles into a String cannot fail");
|
||||
}
|
||||
}
|
||||
|
||||
fn push_cell_xml(output: &mut String, value: Option<&Value>, row_index: usize, col_index: usize, style: Option<usize>) {
|
||||
output.push_str("<c r=\"");
|
||||
push_cell_ref(output, row_index, col_index);
|
||||
output.push('"');
|
||||
match value {
|
||||
Some(Value::Null) | None => {
|
||||
push_cell_style(output, style);
|
||||
output.push_str("/>");
|
||||
}
|
||||
Some(Value::Bool(value)) => {
|
||||
let bool_value = if *value { 1 } else { 0 };
|
||||
output.push_str(" t=\"b\"");
|
||||
push_cell_style(output, style);
|
||||
write!(output, "><v>{bool_value}</v></c>").expect("writing XLSX booleans into a String cannot fail");
|
||||
}
|
||||
Some(Value::Number(value)) => {
|
||||
if value.as_f64().is_some_and(|number| number.is_finite()) {
|
||||
push_cell_style(output, style);
|
||||
write!(output, "><v>{value}</v></c>").expect("writing XLSX numbers into a String cannot fail");
|
||||
} else {
|
||||
output.push_str(" t=\"inlineStr\"");
|
||||
push_cell_style(output, style);
|
||||
output.push_str("><is><t>");
|
||||
push_xml_escaped(output, &value.to_string());
|
||||
output.push_str("</t></is></c>");
|
||||
}
|
||||
}
|
||||
Some(Value::String(value)) => {
|
||||
output.push_str(" t=\"inlineStr\"");
|
||||
push_cell_style(output, style);
|
||||
output.push_str("><is><t>");
|
||||
push_xml_escaped(output, value);
|
||||
output.push_str("</t></is></c>");
|
||||
}
|
||||
Some(value) => {
|
||||
output.push_str(" t=\"inlineStr\"");
|
||||
push_cell_style(output, style);
|
||||
output.push_str("><is><t>");
|
||||
push_xml_escaped(output, &value.to_string());
|
||||
output.push_str("</t></is></c>");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn is_numeric_column_type(column_type: Option<&String>) -> bool {
|
||||
let mut normalized = column_type.map(|value| value.trim().to_ascii_lowercase()).unwrap_or_default();
|
||||
while normalized.ends_with(')') {
|
||||
@@ -659,36 +735,43 @@ fn safe_excel_number(value: &str) -> Option<&str> {
|
||||
(significant_digits <= 15).then_some(trimmed)
|
||||
}
|
||||
|
||||
fn typed_cell_xml(
|
||||
fn push_typed_cell_xml(
|
||||
output: &mut String,
|
||||
value: Option<&Value>,
|
||||
column_type: Option<&String>,
|
||||
row_index: usize,
|
||||
col_index: usize,
|
||||
style: Option<usize>,
|
||||
date_time_format: Option<&str>,
|
||||
) -> String {
|
||||
) {
|
||||
if let Some(Value::String(value)) = value {
|
||||
if let Some((serial, temporal_kind)) =
|
||||
excel_temporal_serial(value, column_type.map(String::as_str), date_time_format)
|
||||
{
|
||||
let reference = cell_ref(row_index, col_index);
|
||||
let style = match temporal_kind {
|
||||
ExcelTemporalKind::Date => XLSX_DATE_STYLE,
|
||||
ExcelTemporalKind::DateTime => XLSX_DATETIME_STYLE,
|
||||
};
|
||||
return format!("<c r=\"{reference}\" s=\"{style}\"><v>{serial}</v></c>");
|
||||
output.push_str("<c r=\"");
|
||||
push_cell_ref(output, row_index, col_index);
|
||||
write!(output, "\" s=\"{style}\"><v>{serial}</v></c>")
|
||||
.expect("writing XLSX temporal cells into a String cannot fail");
|
||||
return;
|
||||
}
|
||||
}
|
||||
if is_numeric_column_type(column_type) {
|
||||
if let Some(Value::String(value)) = value {
|
||||
if let Some(number) = safe_excel_number(value) {
|
||||
let reference = cell_ref(row_index, col_index);
|
||||
let style_attr = style.map_or(String::new(), |style| format!(" s=\"{style}\""));
|
||||
return format!("<c r=\"{reference}\"{style_attr}><v>{number}</v></c>");
|
||||
output.push_str("<c r=\"");
|
||||
push_cell_ref(output, row_index, col_index);
|
||||
output.push('"');
|
||||
push_cell_style(output, style);
|
||||
write!(output, "><v>{number}</v></c>").expect("writing XLSX numeric cells into a String cannot fail");
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
cell_xml(value, row_index, col_index, style)
|
||||
push_cell_xml(output, value, row_index, col_index, style);
|
||||
}
|
||||
|
||||
fn write_worksheet_xml<W: Write>(writer: &mut W, segment: &WorksheetSegment) -> Result<(), String> {
|
||||
@@ -718,21 +801,20 @@ fn write_worksheet_xml<W: Write>(writer: &mut W, segment: &WorksheetSegment) ->
|
||||
.write_all(header_row_xml(segment.columns, segment.column_comments).as_bytes())
|
||||
.map_err(|err| err.to_string())?;
|
||||
|
||||
let mut row_buffer = String::with_capacity(segment.columns.len().saturating_mul(48));
|
||||
for (row_index, row) in segment.rows.iter().enumerate() {
|
||||
let excel_row = row_index + 2;
|
||||
writer
|
||||
.write_all(
|
||||
data_row_xml_with_date_time_format(
|
||||
excel_row,
|
||||
segment.columns,
|
||||
segment.column_types,
|
||||
row,
|
||||
None,
|
||||
segment.numeric_column_right_align,
|
||||
)
|
||||
.as_bytes(),
|
||||
)
|
||||
.map_err(|err| err.to_string())?;
|
||||
row_buffer.clear();
|
||||
push_data_row_xml(
|
||||
&mut row_buffer,
|
||||
excel_row,
|
||||
segment.columns,
|
||||
segment.column_types,
|
||||
row,
|
||||
None,
|
||||
segment.numeric_column_right_align,
|
||||
);
|
||||
writer.write_all(row_buffer.as_bytes()).map_err(|err| err.to_string())?;
|
||||
}
|
||||
|
||||
writer
|
||||
@@ -1983,4 +2065,17 @@ mod tests {
|
||||
let sheet2 = read_zip_entry(&data, "xl/worksheets/sheet2.xml");
|
||||
assert!(sheet2.contains("row_3"), "row 3 should be on sheet 2: {sheet2}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reusable_cell_buffer_matches_reference_xml() {
|
||||
let values = [Value::Null, json!(true), json!(42.5), json!("a<&\"b"), json!({"key": "value"})];
|
||||
for (index, value) in values.iter().enumerate() {
|
||||
for style in [None, Some(1), Some(4)] {
|
||||
let expected = super::cell_xml(Some(value), index, index + 1, style);
|
||||
let mut actual = String::new();
|
||||
super::push_cell_xml(&mut actual, Some(value), index, index + 1, style);
|
||||
assert_eq!(actual, expected, "value={value:?}, style={style:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user