fix(transfer): reduce import and export allocations
This commit is contained in:
9 files changed
+822
-261
No files matched your search
@@ -0,0 +1,154 @@
|
|||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
use dbx_core::csv_export::format_query_result_csv_rows;
|
||||||
|
use dbx_core::database_export::{build_export_insert_statements, BuildExportInsertStatementsOptions};
|
||||||
|
use dbx_core::models::connection::DatabaseType;
|
||||||
|
use dbx_core::table_import::{build_import_insert_batches, ParsedImportFile, TableImportColumnMapping};
|
||||||
|
use dbx_core::xlsx_export::{build_xlsx_workbook, XlsxWorksheetData};
|
||||||
|
|
||||||
|
struct Options {
|
||||||
|
rows: usize,
|
||||||
|
columns: usize,
|
||||||
|
batch_size: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_options() -> Result<Options, String> {
|
||||||
|
let mut options = Options { rows: 20_000, columns: 10, batch_size: 500 };
|
||||||
|
for argument in std::env::args().skip(1) {
|
||||||
|
let (key, value) = argument.split_once('=').ok_or_else(|| format!("Invalid option: {argument}"))?;
|
||||||
|
match key {
|
||||||
|
"--rows" => options.rows = value.parse().map_err(|_| format!("Invalid row count: {value}"))?,
|
||||||
|
"--columns" => options.columns = value.parse().map_err(|_| format!("Invalid column count: {value}"))?,
|
||||||
|
"--batch-size" => options.batch_size = value.parse().map_err(|_| format!("Invalid batch size: {value}"))?,
|
||||||
|
_ => return Err(format!("Unknown option: {key}")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if options.rows == 0 || options.columns == 0 || options.batch_size == 0 {
|
||||||
|
return Err("rows, columns, and batch-size must be greater than zero".to_string());
|
||||||
|
}
|
||||||
|
Ok(options)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn columns(count: usize) -> Vec<String> {
|
||||||
|
(0..count).map(|index| format!("column_{}", index + 1)).collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn rows(row_count: usize, column_count: usize) -> Vec<Vec<serde_json::Value>> {
|
||||||
|
(0..row_count)
|
||||||
|
.map(|row_index| {
|
||||||
|
(0..column_count)
|
||||||
|
.map(|column_index| {
|
||||||
|
if column_index == 0 {
|
||||||
|
serde_json::json!(row_index + 1)
|
||||||
|
} else if column_index % 3 == 0 {
|
||||||
|
serde_json::json!(format!("2026-07-{:02} 12:34:56", row_index % 28 + 1))
|
||||||
|
} else {
|
||||||
|
serde_json::json!(format!("value-{row_index}-{column_index}"))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn elapsed_ms(started_at: Instant) -> f64 {
|
||||||
|
started_at.elapsed().as_secs_f64() * 1000.0
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run() -> Result<(), String> {
|
||||||
|
let options = parse_options()?;
|
||||||
|
let columns = columns(options.columns);
|
||||||
|
let rows = rows(options.rows, options.columns);
|
||||||
|
let mappings = columns
|
||||||
|
.iter()
|
||||||
|
.map(|column| TableImportColumnMapping {
|
||||||
|
source_column: column.clone(),
|
||||||
|
target_column: column.clone(),
|
||||||
|
target_data_type: None,
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let parsed = ParsedImportFile {
|
||||||
|
columns: columns.clone(),
|
||||||
|
rows: rows.clone(),
|
||||||
|
total_rows: rows.len(),
|
||||||
|
effective_encoding: None,
|
||||||
|
};
|
||||||
|
|
||||||
|
let import_started = Instant::now();
|
||||||
|
let import_batches = build_import_insert_batches(
|
||||||
|
&parsed,
|
||||||
|
&mappings,
|
||||||
|
&[],
|
||||||
|
"benchmark_import",
|
||||||
|
"main",
|
||||||
|
&DatabaseType::Sqlite,
|
||||||
|
options.batch_size,
|
||||||
|
)?;
|
||||||
|
let import_ms = elapsed_ms(import_started);
|
||||||
|
|
||||||
|
let csv_started = Instant::now();
|
||||||
|
let csv = format_query_result_csv_rows(&rows);
|
||||||
|
let csv_ms = elapsed_ms(csv_started);
|
||||||
|
|
||||||
|
let sql_started = Instant::now();
|
||||||
|
let sql = build_export_insert_statements(BuildExportInsertStatementsOptions {
|
||||||
|
database_type: Some(DatabaseType::Sqlite),
|
||||||
|
identifier_quote: None,
|
||||||
|
schema: Some("main".to_string()),
|
||||||
|
table_name: Some("benchmark_export".to_string()),
|
||||||
|
qualified_table_name: None,
|
||||||
|
columns: columns.clone(),
|
||||||
|
column_types: vec![None; columns.len()],
|
||||||
|
column_extras: Vec::new(),
|
||||||
|
rows: rows.clone(),
|
||||||
|
batch_size: Some(options.batch_size),
|
||||||
|
})?;
|
||||||
|
let sql_ms = elapsed_ms(sql_started);
|
||||||
|
|
||||||
|
let xlsx_started = Instant::now();
|
||||||
|
let xlsx = build_xlsx_workbook(&XlsxWorksheetData {
|
||||||
|
sheet_name: Some("Benchmark".to_string()),
|
||||||
|
columns,
|
||||||
|
column_types: Vec::new(),
|
||||||
|
column_comments: Vec::new(),
|
||||||
|
rows,
|
||||||
|
numeric_column_right_align: false,
|
||||||
|
})?;
|
||||||
|
let xlsx_ms = elapsed_ms(xlsx_started);
|
||||||
|
|
||||||
|
println!(
|
||||||
|
"{}",
|
||||||
|
serde_json::to_string_pretty(&serde_json::json!({
|
||||||
|
"rows": options.rows,
|
||||||
|
"columns": options.columns,
|
||||||
|
"batchSize": options.batch_size,
|
||||||
|
"importSql": {
|
||||||
|
"milliseconds": import_ms,
|
||||||
|
"batches": import_batches.len(),
|
||||||
|
"bytes": import_batches.iter().map(|batch| batch.sql.len()).sum::<usize>(),
|
||||||
|
},
|
||||||
|
"csv": {
|
||||||
|
"milliseconds": csv_ms,
|
||||||
|
"bytes": csv.len(),
|
||||||
|
},
|
||||||
|
"sqlExport": {
|
||||||
|
"milliseconds": sql_ms,
|
||||||
|
"statements": sql.len(),
|
||||||
|
"bytes": sql.iter().map(String::len).sum::<usize>(),
|
||||||
|
},
|
||||||
|
"xlsx": {
|
||||||
|
"milliseconds": xlsx_ms,
|
||||||
|
"bytes": xlsx.len(),
|
||||||
|
},
|
||||||
|
}))
|
||||||
|
.map_err(|error| error.to_string())?
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn main() {
|
||||||
|
if let Err(error) = run() {
|
||||||
|
eprintln!("{error}");
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -100,6 +100,15 @@ fn push_tsv_value(out: &mut String, value: &Value) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn push_tsv_row(out: &mut String, row: &[Value]) {
|
||||||
|
for (cell_index, cell) in row.iter().enumerate() {
|
||||||
|
if cell_index > 0 {
|
||||||
|
out.push('\t');
|
||||||
|
}
|
||||||
|
push_tsv_value(out, cell);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// 预分配粗估:按全部行的实际单元格数求和(不假设等宽),饱和运算防溢出,
|
/// 预分配粗估:按全部行的实际单元格数求和(不假设等宽),饱和运算防溢出,
|
||||||
/// 并设上限——估算只是性能提示,绝不能因病态输入放大成巨额分配
|
/// 并设上限——估算只是性能提示,绝不能因病态输入放大成巨额分配
|
||||||
const ROWS_CAPACITY_ESTIMATE_MAX: usize = 16 * 1024 * 1024;
|
const ROWS_CAPACITY_ESTIMATE_MAX: usize = 16 * 1024 * 1024;
|
||||||
@@ -133,15 +142,11 @@ fn push_tsv_rows(out: &mut String, rows: &[Vec<Value>]) {
|
|||||||
if row_index > 0 {
|
if row_index > 0 {
|
||||||
out.push('\n');
|
out.push('\n');
|
||||||
}
|
}
|
||||||
for (cell_index, cell) in row.iter().enumerate() {
|
push_tsv_row(out, row);
|
||||||
if cell_index > 0 {
|
|
||||||
out.push('\t');
|
|
||||||
}
|
|
||||||
push_tsv_value(out, cell);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg_attr(not(test), allow(dead_code))]
|
||||||
pub(crate) fn format_tsv_rows(rows: &[Vec<Value>]) -> String {
|
pub(crate) fn format_tsv_rows(rows: &[Vec<Value>]) -> String {
|
||||||
let mut out = String::with_capacity(estimated_rows_capacity(rows));
|
let mut out = String::with_capacity(estimated_rows_capacity(rows));
|
||||||
push_tsv_rows(&mut out, rows);
|
push_tsv_rows(&mut out, rows);
|
||||||
@@ -166,17 +171,30 @@ pub(crate) fn format_tsv(columns: &[String], rows: &[Vec<Value>]) -> String {
|
|||||||
/// Format query-result rows as CSV text without a header row. Database NULLs
|
/// Format query-result rows as CSV text without a header row. Database NULLs
|
||||||
/// use the same empty-cell representation as table-data exports. Used by the
|
/// use the same empty-cell representation as table-data exports. Used by the
|
||||||
/// streaming query-result export for batches after the first.
|
/// streaming query-result export for batches after the first.
|
||||||
|
pub(crate) fn push_query_result_csv_row(out: &mut String, row: &[Value]) {
|
||||||
|
for (cell_index, cell) in row.iter().enumerate() {
|
||||||
|
if cell_index > 0 {
|
||||||
|
out.push(',');
|
||||||
|
}
|
||||||
|
push_csv_value(out, cell);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn push_table_csv_row(out: &mut String, row: &[Value]) {
|
||||||
|
for (cell_index, cell) in row.iter().enumerate() {
|
||||||
|
if cell_index > 0 {
|
||||||
|
out.push(',');
|
||||||
|
}
|
||||||
|
push_csv_text_value(out, cell);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn push_query_result_csv_rows(out: &mut String, rows: &[Vec<Value>]) {
|
fn push_query_result_csv_rows(out: &mut String, rows: &[Vec<Value>]) {
|
||||||
for (row_index, row) in rows.iter().enumerate() {
|
for (row_index, row) in rows.iter().enumerate() {
|
||||||
if row_index > 0 {
|
if row_index > 0 {
|
||||||
out.push('\n');
|
out.push('\n');
|
||||||
}
|
}
|
||||||
for (cell_index, cell) in row.iter().enumerate() {
|
push_query_result_csv_row(out, row);
|
||||||
if cell_index > 0 {
|
|
||||||
out.push(',');
|
|
||||||
}
|
|
||||||
push_csv_value(out, cell);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -399,5 +417,22 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reusable_row_buffers_match_batch_formatters() {
|
||||||
|
let row = vec![Value::Null, json!("NULL"), json!("line\n\"two\""), json!(42)];
|
||||||
|
|
||||||
|
let mut csv = String::new();
|
||||||
|
super::push_query_result_csv_row(&mut csv, &row);
|
||||||
|
assert_eq!(csv, super::format_query_result_csv_rows(&[row.clone()]));
|
||||||
|
|
||||||
|
let mut table_csv = String::new();
|
||||||
|
super::push_table_csv_row(&mut table_csv, &row);
|
||||||
|
assert_eq!(table_csv, "\"\",\"NULL\",\"line\n\"\"two\"\"\",\"42\"");
|
||||||
|
|
||||||
|
let mut tsv = String::new();
|
||||||
|
super::push_tsv_row(&mut tsv, &row);
|
||||||
|
assert_eq!(tsv, super::format_tsv_rows(&[row]));
|
||||||
|
}
|
||||||
|
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
}
|
}
|
||||||
@@ -994,54 +994,63 @@ pub fn build_export_insert_statements(options: BuildExportInsertStatementsOption
|
|||||||
let statement_overhead_bytes = export_sql_statement_bytes(options.database_type, &statement_prefix) + 1;
|
let statement_overhead_bytes = export_sql_statement_bytes(options.database_type, &statement_prefix) + 1;
|
||||||
let target_statement_bytes = DATABASE_EXPORT_TARGET_STATEMENT_BYTES;
|
let target_statement_bytes = DATABASE_EXPORT_TARGET_STATEMENT_BYTES;
|
||||||
let separator_bytes = export_sql_statement_bytes(options.database_type, ", ");
|
let separator_bytes = export_sql_statement_bytes(options.database_type, ", ");
|
||||||
let mut current_values = Vec::with_capacity(batch_size);
|
let mut current_values = String::new();
|
||||||
let mut current_values_bytes = 0usize;
|
let mut current_values_bytes = 0usize;
|
||||||
|
let mut current_row_count = 0usize;
|
||||||
|
|
||||||
let flush_values = |statements: &mut Vec<String>, values: &mut Vec<String>, values_bytes: &mut usize| {
|
let flush_values =
|
||||||
if values.is_empty() {
|
|statements: &mut Vec<String>, values: &mut String, values_bytes: &mut usize, row_count: &mut usize| {
|
||||||
return;
|
if *row_count == 0 {
|
||||||
}
|
return;
|
||||||
let insert_sql = format!("{statement_prefix}{};", values.join(", "));
|
}
|
||||||
if needs_dameng_identity_insert {
|
let mut insert_sql = String::with_capacity(statement_prefix.len() + values.len() + 1);
|
||||||
statements.push(wrap_dameng_identity_insert_sql_for_table(&insert_sql, &table));
|
insert_sql.push_str(&statement_prefix);
|
||||||
} else {
|
insert_sql.push_str(values);
|
||||||
statements.push(insert_sql);
|
insert_sql.push(';');
|
||||||
}
|
if needs_dameng_identity_insert {
|
||||||
values.clear();
|
statements.push(wrap_dameng_identity_insert_sql_for_table(&insert_sql, &table));
|
||||||
*values_bytes = 0;
|
} else {
|
||||||
};
|
statements.push(insert_sql);
|
||||||
|
}
|
||||||
|
values.clear();
|
||||||
|
*values_bytes = 0;
|
||||||
|
*row_count = 0;
|
||||||
|
};
|
||||||
|
|
||||||
for row in options.rows {
|
for row in options.rows {
|
||||||
let row_values = insert_columns
|
let mut rendered_row = String::with_capacity(insert_columns.len().saturating_mul(16).saturating_add(2));
|
||||||
.iter()
|
rendered_row.push('(');
|
||||||
.map(|(index, _, sqlserver_unicode_string)| {
|
for (column_index, (index, _, sqlserver_unicode_string)) in insert_columns.iter().enumerate() {
|
||||||
let value = row.get(*index).unwrap_or(&Value::Null);
|
if column_index > 0 {
|
||||||
format_export_sql_literal_typed(
|
rendered_row.push_str(", ");
|
||||||
value,
|
}
|
||||||
options.database_type,
|
let value = row.get(*index).unwrap_or(&Value::Null);
|
||||||
options.column_types.get(*index).and_then(|value| value.as_deref()),
|
rendered_row.push_str(&format_export_sql_literal_typed(
|
||||||
*sqlserver_unicode_string,
|
value,
|
||||||
)
|
options.database_type,
|
||||||
})
|
options.column_types.get(*index).and_then(|value| value.as_deref()),
|
||||||
.collect::<Vec<_>>()
|
*sqlserver_unicode_string,
|
||||||
.join(", ");
|
));
|
||||||
let rendered_row = format!("({row_values})");
|
}
|
||||||
|
rendered_row.push(')');
|
||||||
let rendered_row_bytes = export_sql_statement_bytes(options.database_type, &rendered_row);
|
let rendered_row_bytes = export_sql_statement_bytes(options.database_type, &rendered_row);
|
||||||
let candidate_bytes = statement_overhead_bytes
|
let candidate_bytes = statement_overhead_bytes
|
||||||
+ current_values_bytes
|
+ current_values_bytes
|
||||||
+ if current_values.is_empty() { 0 } else { separator_bytes }
|
+ if current_row_count == 0 { 0 } else { separator_bytes }
|
||||||
+ rendered_row_bytes;
|
+ rendered_row_bytes;
|
||||||
|
|
||||||
if !current_values.is_empty()
|
if current_row_count > 0 && (current_row_count >= batch_size || candidate_bytes > target_statement_bytes) {
|
||||||
&& (current_values.len() >= batch_size || candidate_bytes > target_statement_bytes)
|
flush_values(&mut statements, &mut current_values, &mut current_values_bytes, &mut current_row_count);
|
||||||
{
|
|
||||||
flush_values(&mut statements, &mut current_values, &mut current_values_bytes);
|
|
||||||
}
|
}
|
||||||
current_values_bytes += if current_values.is_empty() { 0 } else { separator_bytes };
|
if current_row_count > 0 {
|
||||||
|
current_values.push_str(", ");
|
||||||
|
current_values_bytes += separator_bytes;
|
||||||
|
}
|
||||||
|
current_values.push_str(&rendered_row);
|
||||||
current_values_bytes += rendered_row_bytes;
|
current_values_bytes += rendered_row_bytes;
|
||||||
current_values.push(rendered_row);
|
current_row_count += 1;
|
||||||
}
|
}
|
||||||
flush_values(&mut statements, &mut current_values, &mut current_values_bytes);
|
flush_values(&mut statements, &mut current_values, &mut current_values_bytes, &mut current_row_count);
|
||||||
|
|
||||||
Ok(statements)
|
Ok(statements)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ use std::sync::{
|
|||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
use crate::connection::{AppState, PoolKind};
|
use crate::connection::{AppState, PoolKind};
|
||||||
use crate::csv_export::{format_query_result_csv, format_query_result_csv_rows, format_tsv, format_tsv_rows};
|
use crate::csv_export::{format_query_result_csv, format_tsv, push_query_result_csv_row, push_tsv_row};
|
||||||
pub use crate::database_export::ExportStatus;
|
pub use crate::database_export::ExportStatus;
|
||||||
use crate::database_export::{build_export_insert_statements, is_export_cancelled, BuildExportInsertStatementsOptions};
|
use crate::database_export::{build_export_insert_statements, is_export_cancelled, BuildExportInsertStatementsOptions};
|
||||||
use crate::models::connection::DatabaseType;
|
use crate::models::connection::DatabaseType;
|
||||||
@@ -400,12 +400,38 @@ fn format_text_export_header(format: &str, columns: &[String]) -> String {
|
|||||||
content.strip_suffix('\n').unwrap_or(&content).to_string()
|
content.strip_suffix('\n').unwrap_or(&content).to_string()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn format_text_export_rows(format: &str, rows: &[Vec<Value>]) -> String {
|
fn write_text_export_row<W: Write>(
|
||||||
|
file: &mut W,
|
||||||
|
format: &str,
|
||||||
|
row: &[Value],
|
||||||
|
buffer: &mut String,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
buffer.clear();
|
||||||
|
buffer.push('\n');
|
||||||
if format == "csv" {
|
if format == "csv" {
|
||||||
format_query_result_csv_rows(rows)
|
push_query_result_csv_row(buffer, row);
|
||||||
} else {
|
} else {
|
||||||
format_tsv_rows(rows)
|
push_tsv_row(buffer, row);
|
||||||
}
|
}
|
||||||
|
file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_text_export_rows<W: Write>(
|
||||||
|
file: &mut W,
|
||||||
|
format: &str,
|
||||||
|
rows: &[Vec<Value>],
|
||||||
|
buffer: &mut String,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
buffer.clear();
|
||||||
|
for row in rows {
|
||||||
|
buffer.push('\n');
|
||||||
|
if format == "csv" {
|
||||||
|
push_query_result_csv_row(buffer, row);
|
||||||
|
} else {
|
||||||
|
push_tsv_row(buffer, row);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}"))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn should_emit_stream_progress(
|
fn should_emit_stream_progress(
|
||||||
@@ -683,6 +709,7 @@ async fn export_query_result_core_inner(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let mut xlsx = None;
|
let mut xlsx = None;
|
||||||
|
let mut text_buffer = String::new();
|
||||||
let mut columns: Vec<String> = Vec::new();
|
let mut columns: Vec<String> = Vec::new();
|
||||||
let mut column_types: Vec<String> = Vec::new();
|
let mut column_types: Vec<String> = Vec::new();
|
||||||
let mut rows_exported: u64 = 0;
|
let mut rows_exported: u64 = 0;
|
||||||
@@ -838,7 +865,7 @@ async fn export_query_result_core_inner(
|
|||||||
result.rows.truncate(this_page);
|
result.rows.truncate(this_page);
|
||||||
}
|
}
|
||||||
let row_count = result.rows.len();
|
let row_count = result.rows.len();
|
||||||
let formatted_rows = crate::temporal_format::format_temporal_export_rows_with_string_types(
|
let formatted_rows = crate::temporal_format::format_temporal_export_rows_with_string_types_cow(
|
||||||
&result.rows,
|
&result.rows,
|
||||||
&column_types,
|
&column_types,
|
||||||
request.date_time_format.as_deref(),
|
request.date_time_format.as_deref(),
|
||||||
@@ -850,18 +877,16 @@ async fn export_query_result_core_inner(
|
|||||||
let header = format_text_export_header(&format, &columns);
|
let header = format_text_export_header(&format, &columns);
|
||||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
||||||
if row_count > 0 {
|
if row_count > 0 {
|
||||||
let rows = format_text_export_rows(&format, &formatted_rows);
|
write_text_export_rows(file, &format, formatted_rows.as_ref(), &mut text_buffer)?;
|
||||||
write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?;
|
|
||||||
}
|
}
|
||||||
wrote_text_header = true;
|
wrote_text_header = true;
|
||||||
} else if row_count > 0 {
|
} else if row_count > 0 {
|
||||||
let rows = format_text_export_rows(&format, &formatted_rows);
|
write_text_export_rows(file, &format, formatted_rows.as_ref(), &mut text_buffer)?;
|
||||||
write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else if format == "sql" {
|
} else if format == "sql" {
|
||||||
let writer = sql_writer.as_mut().ok_or_else(|| "SQL export writer missing".to_string())?;
|
let writer = sql_writer.as_mut().ok_or_else(|| "SQL export writer missing".to_string())?;
|
||||||
for row in formatted_rows {
|
for row in formatted_rows.into_owned() {
|
||||||
writer.write_row(row)?;
|
writer.write_row(row)?;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
@@ -876,7 +901,7 @@ async fn export_query_result_core_inner(
|
|||||||
)?);
|
)?);
|
||||||
}
|
}
|
||||||
if let Some(writer) = xlsx.as_mut() {
|
if let Some(writer) = xlsx.as_mut() {
|
||||||
for row in &formatted_rows {
|
for row in formatted_rows.as_ref() {
|
||||||
writer.write_row(row).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
writer.write_row(row).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1013,6 +1038,7 @@ async fn try_export_postgres_query_result_stream(
|
|||||||
None
|
None
|
||||||
};
|
};
|
||||||
let mut xlsx = None;
|
let mut xlsx = None;
|
||||||
|
let mut text_buffer = String::new();
|
||||||
let mut sql_writer: Option<SqlInsertWriter> =
|
let mut sql_writer: Option<SqlInsertWriter> =
|
||||||
if format == "sql" { Some(SqlInsertWriter::create(request)?) } else { None };
|
if format == "sql" { Some(SqlInsertWriter::create(request)?) } else { None };
|
||||||
let budget = operation_budget_for_pool_key(state, &pool_key, query_export_timeout(request.timeout_secs)).await;
|
let budget = operation_budget_for_pool_key(state, &pool_key, query_export_timeout(request.timeout_secs)).await;
|
||||||
@@ -1050,25 +1076,26 @@ async fn try_export_postgres_query_result_stream(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
crate::db::postgres::PostgresQueryStreamItem::Row(row) => {
|
crate::db::postgres::PostgresQueryStreamItem::Row(row) => {
|
||||||
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types(
|
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types_cow(
|
||||||
&row,
|
&row,
|
||||||
&temporal_column_types,
|
&temporal_column_types,
|
||||||
request.date_time_format.as_deref(),
|
request.date_time_format.as_deref(),
|
||||||
);
|
);
|
||||||
if let Some(writer) = sql_writer.as_mut() {
|
if let Some(writer) = sql_writer.as_mut() {
|
||||||
writer.write_row(formatted)?;
|
writer.write_row(formatted.into_owned())?;
|
||||||
} else if let Some(file) = text_file.as_mut() {
|
} else if let Some(file) = text_file.as_mut() {
|
||||||
let rows = format_text_export_rows(format, std::slice::from_ref(&formatted));
|
write_text_export_row(file, format, formatted.as_ref(), &mut text_buffer)?;
|
||||||
write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?;
|
|
||||||
} else if let Some(writer) = xlsx.as_mut() {
|
} else if let Some(writer) = xlsx.as_mut() {
|
||||||
writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
writer.write_row(formatted.as_ref()).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||||
} else {
|
} else {
|
||||||
let xlsx_file =
|
let xlsx_file =
|
||||||
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
|
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
|
||||||
xlsx =
|
xlsx =
|
||||||
Some(start_query_result_xlsx_workbook(BufWriter::new(xlsx_file), request, &columns, &[])?);
|
Some(start_query_result_xlsx_workbook(BufWriter::new(xlsx_file), request, &columns, &[])?);
|
||||||
if let Some(writer) = xlsx.as_mut() {
|
if let Some(writer) = xlsx.as_mut() {
|
||||||
writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
writer
|
||||||
|
.write_row(formatted.as_ref())
|
||||||
|
.map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
rows_exported += 1;
|
rows_exported += 1;
|
||||||
@@ -1205,6 +1232,7 @@ async fn try_export_mysql_query_result_stream(
|
|||||||
None
|
None
|
||||||
};
|
};
|
||||||
let mut xlsx = None;
|
let mut xlsx = None;
|
||||||
|
let mut text_buffer = String::new();
|
||||||
let mut sql_writer: Option<SqlInsertWriter> =
|
let mut sql_writer: Option<SqlInsertWriter> =
|
||||||
if format == "sql" { Some(SqlInsertWriter::create(request)?) } else { None };
|
if format == "sql" { Some(SqlInsertWriter::create(request)?) } else { None };
|
||||||
let query_timeout = query_export_timeout(request.timeout_secs);
|
let query_timeout = query_export_timeout(request.timeout_secs);
|
||||||
@@ -1292,25 +1320,26 @@ async fn try_export_mysql_query_result_stream(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
crate::db::mysql::MySqlQueryStreamItem::Row(row) => {
|
crate::db::mysql::MySqlQueryStreamItem::Row(row) => {
|
||||||
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types(
|
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types_cow(
|
||||||
&row,
|
&row,
|
||||||
&temporal_column_types,
|
&temporal_column_types,
|
||||||
request.date_time_format.as_deref(),
|
request.date_time_format.as_deref(),
|
||||||
);
|
);
|
||||||
if let Some(writer) = sql_writer.as_mut() {
|
if let Some(writer) = sql_writer.as_mut() {
|
||||||
writer.write_row(formatted)?;
|
writer.write_row(formatted.into_owned())?;
|
||||||
} else if let Some(file) = text_file.as_mut() {
|
} else if let Some(file) = text_file.as_mut() {
|
||||||
let rows = format_text_export_rows(format, std::slice::from_ref(&formatted));
|
write_text_export_row(file, format, formatted.as_ref(), &mut text_buffer)?;
|
||||||
write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?;
|
|
||||||
} else if let Some(writer) = xlsx.as_mut() {
|
} else if let Some(writer) = xlsx.as_mut() {
|
||||||
writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
writer.write_row(formatted.as_ref()).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||||
} else {
|
} else {
|
||||||
let xlsx_file =
|
let xlsx_file =
|
||||||
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
|
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
|
||||||
xlsx =
|
xlsx =
|
||||||
Some(start_query_result_xlsx_workbook(BufWriter::new(xlsx_file), request, &columns, &[])?);
|
Some(start_query_result_xlsx_workbook(BufWriter::new(xlsx_file), request, &columns, &[])?);
|
||||||
if let Some(writer) = xlsx.as_mut() {
|
if let Some(writer) = xlsx.as_mut() {
|
||||||
writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
writer
|
||||||
|
.write_row(formatted.as_ref())
|
||||||
|
.map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
rows_exported += 1;
|
rows_exported += 1;
|
||||||
@@ -1467,6 +1496,7 @@ async fn try_export_clickhouse_query_result_stream(
|
|||||||
None
|
None
|
||||||
};
|
};
|
||||||
let mut xlsx = None;
|
let mut xlsx = None;
|
||||||
|
let mut text_buffer = String::new();
|
||||||
let mut sql_writer: Option<SqlInsertWriter> =
|
let mut sql_writer: Option<SqlInsertWriter> =
|
||||||
if format == "sql" { Some(SqlInsertWriter::create(request)?) } else { None };
|
if format == "sql" { Some(SqlInsertWriter::create(request)?) } else { None };
|
||||||
let query_timeout = query_export_timeout(request.timeout_secs);
|
let query_timeout = query_export_timeout(request.timeout_secs);
|
||||||
@@ -1505,25 +1535,26 @@ async fn try_export_clickhouse_query_result_stream(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
crate::db::clickhouse_driver::ClickHouseQueryStreamItem::Row(row) => {
|
crate::db::clickhouse_driver::ClickHouseQueryStreamItem::Row(row) => {
|
||||||
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types(
|
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types_cow(
|
||||||
&row,
|
&row,
|
||||||
&temporal_column_types,
|
&temporal_column_types,
|
||||||
request.date_time_format.as_deref(),
|
request.date_time_format.as_deref(),
|
||||||
);
|
);
|
||||||
if let Some(writer) = sql_writer.as_mut() {
|
if let Some(writer) = sql_writer.as_mut() {
|
||||||
writer.write_row(formatted)?;
|
writer.write_row(formatted.into_owned())?;
|
||||||
} else if let Some(file) = text_file.as_mut() {
|
} else if let Some(file) = text_file.as_mut() {
|
||||||
let rows = format_text_export_rows(format, std::slice::from_ref(&formatted));
|
write_text_export_row(file, format, formatted.as_ref(), &mut text_buffer)?;
|
||||||
write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?;
|
|
||||||
} else if let Some(writer) = xlsx.as_mut() {
|
} else if let Some(writer) = xlsx.as_mut() {
|
||||||
writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
writer.write_row(formatted.as_ref()).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||||
} else {
|
} else {
|
||||||
let xlsx_file =
|
let xlsx_file =
|
||||||
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
|
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
|
||||||
xlsx =
|
xlsx =
|
||||||
Some(start_query_result_xlsx_workbook(BufWriter::new(xlsx_file), request, &columns, &[])?);
|
Some(start_query_result_xlsx_workbook(BufWriter::new(xlsx_file), request, &columns, &[])?);
|
||||||
if let Some(writer) = xlsx.as_mut() {
|
if let Some(writer) = xlsx.as_mut() {
|
||||||
writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
writer
|
||||||
|
.write_row(formatted.as_ref())
|
||||||
|
.map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
rows_exported += 1;
|
rows_exported += 1;
|
||||||
@@ -1635,6 +1666,7 @@ async fn try_export_sqlserver_query_result_stream(
|
|||||||
None
|
None
|
||||||
};
|
};
|
||||||
let mut xlsx = None;
|
let mut xlsx = None;
|
||||||
|
let mut text_buffer = String::new();
|
||||||
let mut sql_writer: Option<SqlInsertWriter> =
|
let mut sql_writer: Option<SqlInsertWriter> =
|
||||||
if format == "sql" { Some(SqlInsertWriter::create(request)?) } else { None };
|
if format == "sql" { Some(SqlInsertWriter::create(request)?) } else { None };
|
||||||
let query_timeout = query_export_timeout(request.timeout_secs);
|
let query_timeout = query_export_timeout(request.timeout_secs);
|
||||||
@@ -1683,25 +1715,26 @@ async fn try_export_sqlserver_query_result_stream(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
crate::db::sqlserver::SqlServerStreamItem::Row(row) => {
|
crate::db::sqlserver::SqlServerStreamItem::Row(row) => {
|
||||||
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types(
|
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types_cow(
|
||||||
row,
|
row,
|
||||||
&temporal_column_types,
|
&temporal_column_types,
|
||||||
request.date_time_format.as_deref(),
|
request.date_time_format.as_deref(),
|
||||||
);
|
);
|
||||||
if let Some(writer) = sql_writer.as_mut() {
|
if let Some(writer) = sql_writer.as_mut() {
|
||||||
writer.write_row(formatted)?;
|
writer.write_row(formatted.into_owned())?;
|
||||||
} else if let Some(file) = text_file.as_mut() {
|
} else if let Some(file) = text_file.as_mut() {
|
||||||
let rows = format_text_export_rows(format, std::slice::from_ref(&formatted));
|
write_text_export_row(file, format, formatted.as_ref(), &mut text_buffer)?;
|
||||||
write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?;
|
|
||||||
} else if let Some(writer) = xlsx.as_mut() {
|
} else if let Some(writer) = xlsx.as_mut() {
|
||||||
writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
writer.write_row(formatted.as_ref()).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||||
} else {
|
} else {
|
||||||
let xlsx_file =
|
let xlsx_file =
|
||||||
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
|
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
|
||||||
xlsx =
|
xlsx =
|
||||||
Some(start_query_result_xlsx_workbook(BufWriter::new(xlsx_file), request, &columns, &[])?);
|
Some(start_query_result_xlsx_workbook(BufWriter::new(xlsx_file), request, &columns, &[])?);
|
||||||
if let Some(writer) = xlsx.as_mut() {
|
if let Some(writer) = xlsx.as_mut() {
|
||||||
writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
writer
|
||||||
|
.write_row(formatted.as_ref())
|
||||||
|
.map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
rows_exported += 1;
|
rows_exported += 1;
|
||||||
@@ -1893,6 +1926,16 @@ mod tests {
|
|||||||
assert_eq!(format_text_export_header("txt", &["id".to_string(), "note".to_string()]), "id\tnote");
|
assert_eq!(format_text_export_header("txt", &["id".to_string(), "note".to_string()]), "id\tnote");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reusable_text_row_buffer_preserves_query_null_semantics() {
|
||||||
|
let row = vec![Value::Null, serde_json::json!(""), serde_json::json!("line\n\"two\"")];
|
||||||
|
let mut output = Vec::new();
|
||||||
|
let mut buffer = String::new();
|
||||||
|
|
||||||
|
write_text_export_row(&mut output, "csv", &row, &mut buffer).expect("write csv row");
|
||||||
|
assert_eq!(String::from_utf8(output).expect("utf8 csv"), "\n,\"\",\"line\n\"\"two\"\"\"");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn xlsx_no_row_limit_has_no_query_layer_cap() {
|
fn xlsx_no_row_limit_has_no_query_layer_cap() {
|
||||||
// Without the old hard limit, XLSX uses the writer's internal splitting.
|
// Without the old hard limit, XLSX uses the writer's internal splitting.
|
||||||
|
|||||||
@@ -9,7 +9,9 @@ use tokio_util::sync::CancellationToken;
|
|||||||
|
|
||||||
use crate::connection::MysqlMode;
|
use crate::connection::MysqlMode;
|
||||||
use crate::connection::{config_for_pool_key, task_client_session_id, AppState, PoolKind};
|
use crate::connection::{config_for_pool_key, task_client_session_id, AppState, PoolKind};
|
||||||
use crate::csv_export::{format_csv, format_tsv, format_tsv_rows, push_csv_text_value};
|
#[cfg(test)]
|
||||||
|
use crate::csv_export::format_tsv_rows;
|
||||||
|
use crate::csv_export::{format_csv, format_tsv, push_table_csv_row, push_tsv_row};
|
||||||
pub use crate::database_export::ExportStatus;
|
pub use crate::database_export::ExportStatus;
|
||||||
use crate::database_export::{
|
use crate::database_export::{
|
||||||
build_export_insert_statements, is_export_cancelled, is_internal_export_column, BuildExportInsertStatementsOptions,
|
build_export_insert_statements, is_export_cancelled, is_internal_export_column, BuildExportInsertStatementsOptions,
|
||||||
@@ -82,6 +84,7 @@ pub struct TableExportProgress {
|
|||||||
|
|
||||||
/// Format rows as CSV text without a header row.
|
/// Format rows as CSV text without a header row.
|
||||||
/// Used for streaming subsequent pagination batches.
|
/// Used for streaming subsequent pagination batches.
|
||||||
|
#[cfg_attr(not(test), allow(dead_code))]
|
||||||
fn format_csv_rows(rows: &[Vec<Value>]) -> String {
|
fn format_csv_rows(rows: &[Vec<Value>]) -> String {
|
||||||
// 注意:该无表头批次路径的 Null 输出为 ""(带引号空串),与查询结果导出
|
// 注意:该无表头批次路径的 Null 输出为 ""(带引号空串),与查询结果导出
|
||||||
// 及首批 format_csv 的裸空单元格语义不同,直写化必须保留该差异
|
// 及首批 format_csv 的裸空单元格语义不同,直写化必须保留该差异
|
||||||
@@ -90,16 +93,40 @@ fn format_csv_rows(rows: &[Vec<Value>]) -> String {
|
|||||||
if row_index > 0 {
|
if row_index > 0 {
|
||||||
out.push('\n');
|
out.push('\n');
|
||||||
}
|
}
|
||||||
for (cell_index, cell) in row.iter().enumerate() {
|
push_table_csv_row(&mut out, row);
|
||||||
if cell_index > 0 {
|
|
||||||
out.push(',');
|
|
||||||
}
|
|
||||||
push_csv_text_value(&mut out, cell);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
out
|
out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn write_table_text_row<W: Write>(file: &mut W, csv: bool, row: &[Value], buffer: &mut String) -> Result<(), String> {
|
||||||
|
buffer.clear();
|
||||||
|
buffer.push('\n');
|
||||||
|
if csv {
|
||||||
|
push_table_csv_row(buffer, row);
|
||||||
|
} else {
|
||||||
|
push_tsv_row(buffer, row);
|
||||||
|
}
|
||||||
|
file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_table_text_rows<W: Write>(
|
||||||
|
file: &mut W,
|
||||||
|
csv: bool,
|
||||||
|
rows: &[Vec<Value>],
|
||||||
|
buffer: &mut String,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
buffer.clear();
|
||||||
|
for row in rows {
|
||||||
|
buffer.push('\n');
|
||||||
|
if csv {
|
||||||
|
push_table_csv_row(buffer, row);
|
||||||
|
} else {
|
||||||
|
push_tsv_row(buffer, row);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}"))
|
||||||
|
}
|
||||||
|
|
||||||
fn export_column_types(request: &TableExportRequest) -> Vec<String> {
|
fn export_column_types(request: &TableExportRequest) -> Vec<String> {
|
||||||
request
|
request
|
||||||
.column_types
|
.column_types
|
||||||
@@ -928,6 +955,7 @@ async fn try_export_native_table_stream(
|
|||||||
let header = format_csv(col_names, &[]);
|
let header = format_csv(col_names, &[]);
|
||||||
let header = header.strip_suffix('\n').unwrap_or(&header);
|
let header = header.strip_suffix('\n').unwrap_or(&header);
|
||||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?;
|
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?;
|
||||||
|
let mut row_buffer = String::new();
|
||||||
|
|
||||||
let result = stream_native_table_rows(
|
let result = stream_native_table_rows(
|
||||||
state,
|
state,
|
||||||
@@ -938,13 +966,12 @@ async fn try_export_native_table_stream(
|
|||||||
&cancelled,
|
&cancelled,
|
||||||
cancel_token.clone(),
|
cancel_token.clone(),
|
||||||
|row| {
|
|row| {
|
||||||
let formatted = crate::temporal_format::format_temporal_export_row(
|
let formatted = crate::temporal_format::format_temporal_export_row_cow(
|
||||||
row,
|
row,
|
||||||
column_types,
|
column_types,
|
||||||
request.date_time_format.as_deref(),
|
request.date_time_format.as_deref(),
|
||||||
);
|
);
|
||||||
let row_csv = format_csv_rows(&[formatted]);
|
write_table_text_row(&mut file, true, formatted.as_ref(), &mut row_buffer)?;
|
||||||
write!(file, "\n{row_csv}").map_err(|e| format!("Failed to write CSV rows: {e}"))?;
|
|
||||||
rows_exported += 1;
|
rows_exported += 1;
|
||||||
if rows_exported.is_multiple_of(progress_interval) {
|
if rows_exported.is_multiple_of(progress_interval) {
|
||||||
on_progress(TableExportProgress {
|
on_progress(TableExportProgress {
|
||||||
@@ -972,6 +999,7 @@ async fn try_export_native_table_stream(
|
|||||||
let header = format_tsv(col_names, &[]);
|
let header = format_tsv(col_names, &[]);
|
||||||
let header = header.strip_suffix('\n').unwrap_or(&header);
|
let header = header.strip_suffix('\n').unwrap_or(&header);
|
||||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write TXT: {e}"))?;
|
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write TXT: {e}"))?;
|
||||||
|
let mut row_buffer = String::new();
|
||||||
|
|
||||||
let result = stream_native_table_rows(
|
let result = stream_native_table_rows(
|
||||||
state,
|
state,
|
||||||
@@ -982,13 +1010,12 @@ async fn try_export_native_table_stream(
|
|||||||
&cancelled,
|
&cancelled,
|
||||||
cancel_token.clone(),
|
cancel_token.clone(),
|
||||||
|row| {
|
|row| {
|
||||||
let formatted = crate::temporal_format::format_temporal_export_row(
|
let formatted = crate::temporal_format::format_temporal_export_row_cow(
|
||||||
row,
|
row,
|
||||||
column_types,
|
column_types,
|
||||||
request.date_time_format.as_deref(),
|
request.date_time_format.as_deref(),
|
||||||
);
|
);
|
||||||
let row_tsv = format_tsv_rows(&[formatted]);
|
write_table_text_row(&mut file, false, formatted.as_ref(), &mut row_buffer)?;
|
||||||
write!(file, "\n{row_tsv}").map_err(|e| format!("Failed to write TXT rows: {e}"))?;
|
|
||||||
rows_exported += 1;
|
rows_exported += 1;
|
||||||
if rows_exported.is_multiple_of(progress_interval) {
|
if rows_exported.is_multiple_of(progress_interval) {
|
||||||
on_progress(TableExportProgress {
|
on_progress(TableExportProgress {
|
||||||
@@ -1033,12 +1060,12 @@ async fn try_export_native_table_stream(
|
|||||||
&cancelled,
|
&cancelled,
|
||||||
cancel_token.clone(),
|
cancel_token.clone(),
|
||||||
|row| {
|
|row| {
|
||||||
let formatted = crate::temporal_format::format_temporal_export_row(
|
let formatted = crate::temporal_format::format_temporal_export_row_cow(
|
||||||
row,
|
row,
|
||||||
column_types,
|
column_types,
|
||||||
request.date_time_format.as_deref(),
|
request.date_time_format.as_deref(),
|
||||||
);
|
);
|
||||||
writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
writer.write_row(formatted.as_ref()).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||||
rows_exported += 1;
|
rows_exported += 1;
|
||||||
if rows_exported.is_multiple_of(progress_interval) {
|
if rows_exported.is_multiple_of(progress_interval) {
|
||||||
on_progress(TableExportProgress {
|
on_progress(TableExportProgress {
|
||||||
@@ -1087,12 +1114,12 @@ async fn try_export_native_table_stream(
|
|||||||
if !is_first_row {
|
if !is_first_row {
|
||||||
file.write_all(b",\n").map_err(|e| format!("Failed to write JSON: {e}"))?;
|
file.write_all(b",\n").map_err(|e| format!("Failed to write JSON: {e}"))?;
|
||||||
}
|
}
|
||||||
let formatted = crate::temporal_format::format_temporal_export_row(
|
let formatted = crate::temporal_format::format_temporal_export_row_cow(
|
||||||
row,
|
row,
|
||||||
column_types,
|
column_types,
|
||||||
request.date_time_format.as_deref(),
|
request.date_time_format.as_deref(),
|
||||||
);
|
);
|
||||||
write_json_row_object(&mut file, col_names, &formatted)?;
|
write_json_row_object(&mut file, col_names, formatted.as_ref())?;
|
||||||
is_first_row = false;
|
is_first_row = false;
|
||||||
rows_exported += 1;
|
rows_exported += 1;
|
||||||
if rows_exported.is_multiple_of(progress_interval) {
|
if rows_exported.is_multiple_of(progress_interval) {
|
||||||
@@ -1473,6 +1500,7 @@ async fn export_table_data_core_inner(
|
|||||||
// 8. Create output file
|
// 8. Create output file
|
||||||
let file = std::fs::File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?;
|
let file = std::fs::File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?;
|
||||||
let mut file = BufWriter::new(file);
|
let mut file = BufWriter::new(file);
|
||||||
|
let mut text_buffer = String::new();
|
||||||
|
|
||||||
let mut rows_exported: u64 = 0;
|
let mut rows_exported: u64 = 0;
|
||||||
let batch_size = request.batch_size.unwrap_or(DEFAULT_BATCH_SIZE).max(1);
|
let batch_size = request.batch_size.unwrap_or(DEFAULT_BATCH_SIZE).max(1);
|
||||||
@@ -1533,7 +1561,7 @@ async fn export_table_data_core_inner(
|
|||||||
if row_count == 0 {
|
if row_count == 0 {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
let formatted_rows = crate::temporal_format::format_temporal_export_rows(
|
let formatted_rows = crate::temporal_format::format_temporal_export_rows_cow(
|
||||||
&result.rows,
|
&result.rows,
|
||||||
&column_types,
|
&column_types,
|
||||||
request.date_time_format.as_deref(),
|
request.date_time_format.as_deref(),
|
||||||
@@ -1541,15 +1569,12 @@ async fn export_table_data_core_inner(
|
|||||||
|
|
||||||
if is_first_batch {
|
if is_first_batch {
|
||||||
// First batch: write header + rows via format_csv
|
// First batch: write header + rows via format_csv
|
||||||
let csv_content = format_csv(&col_names, &formatted_rows);
|
let csv_content = format_csv(&col_names, formatted_rows.as_ref());
|
||||||
file.write_all(csv_content.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?;
|
file.write_all(csv_content.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?;
|
||||||
is_first_batch = false;
|
is_first_batch = false;
|
||||||
} else {
|
} else {
|
||||||
// Subsequent batches: write rows only (prepend newline for separation)
|
// Subsequent batches: write rows only (prepend newline for separation)
|
||||||
let rows_csv = format_csv_rows(&formatted_rows);
|
write_table_text_rows(&mut file, true, formatted_rows.as_ref(), &mut text_buffer)?;
|
||||||
if !rows_csv.is_empty() {
|
|
||||||
write!(file, "\n{rows_csv}").map_err(|e| format!("Failed to write CSV rows: {e}"))?;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
rows_exported += row_count as u64;
|
rows_exported += row_count as u64;
|
||||||
@@ -1624,21 +1649,17 @@ async fn export_table_data_core_inner(
|
|||||||
if row_count == 0 {
|
if row_count == 0 {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
let formatted_rows = crate::temporal_format::format_temporal_export_rows(
|
let formatted_rows = crate::temporal_format::format_temporal_export_rows_cow(
|
||||||
&result.rows,
|
&result.rows,
|
||||||
&column_types,
|
&column_types,
|
||||||
request.date_time_format.as_deref(),
|
request.date_time_format.as_deref(),
|
||||||
);
|
);
|
||||||
|
|
||||||
if is_first_batch {
|
if is_first_batch {
|
||||||
let rows_tsv = format_tsv_rows(&formatted_rows);
|
write_table_text_rows(&mut file, false, formatted_rows.as_ref(), &mut text_buffer)?;
|
||||||
write!(file, "\n{rows_tsv}").map_err(|e| format!("Failed to write TXT rows: {e}"))?;
|
|
||||||
is_first_batch = false;
|
is_first_batch = false;
|
||||||
} else {
|
} else {
|
||||||
let rows_tsv = format_tsv_rows(&formatted_rows);
|
write_table_text_rows(&mut file, false, formatted_rows.as_ref(), &mut text_buffer)?;
|
||||||
if !rows_tsv.is_empty() {
|
|
||||||
write!(file, "\n{rows_tsv}").map_err(|e| format!("Failed to write TXT rows: {e}"))?;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
rows_exported += row_count as u64;
|
rows_exported += row_count as u64;
|
||||||
@@ -1728,12 +1749,12 @@ async fn export_table_data_core_inner(
|
|||||||
}
|
}
|
||||||
|
|
||||||
for row in &result.rows {
|
for row in &result.rows {
|
||||||
let formatted = crate::temporal_format::format_temporal_export_row(
|
let formatted = crate::temporal_format::format_temporal_export_row_cow(
|
||||||
row,
|
row,
|
||||||
&column_types,
|
&column_types,
|
||||||
request.date_time_format.as_deref(),
|
request.date_time_format.as_deref(),
|
||||||
);
|
);
|
||||||
writer.write_row(&formatted).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
writer.write_row(formatted.as_ref()).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||||
}
|
}
|
||||||
rows_exported += row_count as u64;
|
rows_exported += row_count as u64;
|
||||||
|
|
||||||
@@ -1827,12 +1848,12 @@ async fn export_table_data_core_inner(
|
|||||||
if !is_first_row {
|
if !is_first_row {
|
||||||
file.write_all(b",\n").map_err(|e| format!("Failed to write JSON: {e}"))?;
|
file.write_all(b",\n").map_err(|e| format!("Failed to write JSON: {e}"))?;
|
||||||
}
|
}
|
||||||
let formatted = crate::temporal_format::format_temporal_export_row(
|
let formatted = crate::temporal_format::format_temporal_export_row_cow(
|
||||||
row,
|
row,
|
||||||
&column_types,
|
&column_types,
|
||||||
request.date_time_format.as_deref(),
|
request.date_time_format.as_deref(),
|
||||||
);
|
);
|
||||||
write_json_row_object(&mut file, &col_names, &formatted)?;
|
write_json_row_object(&mut file, &col_names, formatted.as_ref())?;
|
||||||
is_first_row = false;
|
is_first_row = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1908,12 +1929,12 @@ async fn export_table_data_core_inner(
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
let formatted_rows = crate::temporal_format::format_temporal_export_rows(
|
let formatted_rows = crate::temporal_format::format_temporal_export_rows_cow(
|
||||||
&result.rows,
|
&result.rows,
|
||||||
&column_types,
|
&column_types,
|
||||||
request.date_time_format.as_deref(),
|
request.date_time_format.as_deref(),
|
||||||
);
|
);
|
||||||
let rows_markdown = format_markdown_rows(&formatted_rows);
|
let rows_markdown = format_markdown_rows(formatted_rows.as_ref());
|
||||||
if !rows_markdown.is_empty() {
|
if !rows_markdown.is_empty() {
|
||||||
if wrote_rows {
|
if wrote_rows {
|
||||||
file.write_all(b"\n").map_err(|e| format!("Failed to write Markdown: {e}"))?;
|
file.write_all(b"\n").map_err(|e| format!("Failed to write Markdown: {e}"))?;
|
||||||
@@ -2270,6 +2291,16 @@ mod tests {
|
|||||||
assert_eq!(out, "\"just\",\"one\"");
|
assert_eq!(out, "\"just\",\"one\"");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reusable_text_row_buffer_preserves_table_null_semantics() {
|
||||||
|
let row = vec![Value::Null, json!(""), json!("line\n\"two\"")];
|
||||||
|
let mut output = Vec::new();
|
||||||
|
let mut buffer = String::new();
|
||||||
|
|
||||||
|
write_table_text_row(&mut output, true, &row, &mut buffer).expect("write csv row");
|
||||||
|
assert_eq!(String::from_utf8(output).expect("utf8 csv"), "\n\"\",\"\",\"line\n\"\"two\"\"\"");
|
||||||
|
}
|
||||||
|
|
||||||
// -----------------------------------------------------------------------
|
// -----------------------------------------------------------------------
|
||||||
// format_tsv (Navicat-style TXT export)
|
// format_tsv (Navicat-style TXT export)
|
||||||
// -----------------------------------------------------------------------
|
// -----------------------------------------------------------------------
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
use std::borrow::Cow;
|
||||||
use std::collections::{HashMap, HashSet};
|
use std::collections::{HashMap, HashSet};
|
||||||
use std::fs::File;
|
use std::fs::File;
|
||||||
use std::io::{BufRead, BufReader, Read as IoRead, Seek, SeekFrom, Write as IoWrite};
|
use std::io::{BufRead, BufReader, Read as IoRead, Seek, SeekFrom, Write as IoWrite};
|
||||||
@@ -19,8 +20,9 @@ use sha2::{Digest, Sha256};
|
|||||||
use crate::connection::{task_client_session_id, AppState, PoolKind};
|
use crate::connection::{task_client_session_id, AppState, PoolKind};
|
||||||
use crate::models::connection::DatabaseType;
|
use crate::models::connection::DatabaseType;
|
||||||
use crate::transfer::{
|
use crate::transfer::{
|
||||||
execute_on_pool, generate_insert_typed, generate_insert_typed_sql_batches, get_columns_for_transfer,
|
escape_value_typed, execute_on_pool, generate_insert_typed_from_value_rows,
|
||||||
normalize_integer_literal, normalize_thousands_numeric_literal, qualified_table, quote_identifier, SqlBatchLimits,
|
generate_insert_typed_sql_batches_from_value_rows, get_columns_for_transfer, normalize_integer_literal,
|
||||||
|
normalize_thousands_numeric_literal, qualified_table, quote_identifier, SqlBatchLimits,
|
||||||
};
|
};
|
||||||
|
|
||||||
pub const DEFAULT_PREVIEW_LIMIT: usize = 50;
|
pub const DEFAULT_PREVIEW_LIMIT: usize = 50;
|
||||||
@@ -3142,9 +3144,8 @@ fn build_import_insert_batch_with_plan(
|
|||||||
if rows.is_empty() {
|
if rows.is_empty() {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
let mapped_rows = map_import_rows_with_plan(rows, plan, db_type, kingbase_oracle_mode, date_time_format);
|
let value_rows = import_value_rows_sql(rows, plan, db_type, kingbase_oracle_mode, date_time_format);
|
||||||
let sql =
|
let sql = generate_insert_typed_from_value_rows(&plan.target_columns, &value_rows, table, schema, db_type, None);
|
||||||
generate_insert_typed(&plan.target_columns, &plan.column_types, &mapped_rows, table, schema, db_type, None);
|
|
||||||
Ok((!sql.trim().is_empty()).then_some(ImportSqlBatch { sql, row_count: rows.len() }))
|
Ok((!sql.trim().is_empty()).then_some(ImportSqlBatch { sql, row_count: rows.len() }))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3161,11 +3162,10 @@ fn build_import_insert_batches_with_plan(
|
|||||||
if rows.is_empty() {
|
if rows.is_empty() {
|
||||||
return Ok(Vec::new());
|
return Ok(Vec::new());
|
||||||
}
|
}
|
||||||
let mapped_rows = map_import_rows_with_plan(rows, plan, db_type, kingbase_oracle_mode, date_time_format);
|
let value_rows = import_value_rows_sql(rows, plan, db_type, kingbase_oracle_mode, date_time_format);
|
||||||
let batches = generate_insert_typed_sql_batches(
|
let batches = generate_insert_typed_sql_batches_from_value_rows(
|
||||||
&plan.target_columns,
|
&plan.target_columns,
|
||||||
&plan.column_types,
|
&value_rows,
|
||||||
&mapped_rows,
|
|
||||||
table,
|
table,
|
||||||
schema,
|
schema,
|
||||||
db_type,
|
db_type,
|
||||||
@@ -3175,16 +3175,31 @@ fn build_import_insert_batches_with_plan(
|
|||||||
Ok(batches.into_iter().map(|(sql, row_count)| ImportSqlBatch { sql, row_count }).collect())
|
Ok(batches.into_iter().map(|(sql, row_count)| ImportSqlBatch { sql, row_count }).collect())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn map_import_rows_with_plan(
|
fn import_value_rows_sql(
|
||||||
rows: &[Vec<serde_json::Value>],
|
rows: &[Vec<serde_json::Value>],
|
||||||
plan: &CompiledImportPlan,
|
plan: &CompiledImportPlan,
|
||||||
db_type: &DatabaseType,
|
db_type: &DatabaseType,
|
||||||
kingbase_oracle_mode: bool,
|
kingbase_oracle_mode: bool,
|
||||||
date_time_format: Option<&str>,
|
date_time_format: Option<&str>,
|
||||||
) -> Vec<Vec<serde_json::Value>> {
|
) -> Vec<String> {
|
||||||
rows.iter()
|
let mut value_rows = Vec::with_capacity(rows.len());
|
||||||
.map(|row| map_import_row_with_plan(row, plan, db_type, kingbase_oracle_mode, date_time_format))
|
for row in rows {
|
||||||
.collect()
|
let mut values = String::with_capacity(plan.mapped_source_indexes.len().saturating_mul(16).saturating_add(2));
|
||||||
|
values.push('(');
|
||||||
|
for (target_index, source_index) in plan.mapped_source_indexes.iter().enumerate() {
|
||||||
|
if target_index > 0 {
|
||||||
|
values.push_str(", ");
|
||||||
|
}
|
||||||
|
let source_value = row.get(*source_index).unwrap_or(&serde_json::Value::Null);
|
||||||
|
let data_type = plan.column_types.get(target_index).and_then(|data_type| data_type.as_deref());
|
||||||
|
let normalized =
|
||||||
|
normalize_import_value_cow(source_value, data_type, db_type, kingbase_oracle_mode, date_time_format);
|
||||||
|
values.push_str(&escape_value_typed(normalized.as_ref(), db_type, data_type));
|
||||||
|
}
|
||||||
|
values.push(')');
|
||||||
|
value_rows.push(values);
|
||||||
|
}
|
||||||
|
value_rows
|
||||||
}
|
}
|
||||||
|
|
||||||
fn map_import_row_with_plan(
|
fn map_import_row_with_plan(
|
||||||
@@ -3274,17 +3289,17 @@ fn effective_import_batch_size(db_type: &DatabaseType, requested: usize) -> usiz
|
|||||||
requested.max(1).min(max_rows)
|
requested.max(1).min(max_rows)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn normalize_import_temporal_value(
|
fn normalize_import_temporal_value_cow<'a>(
|
||||||
value: &serde_json::Value,
|
value: &'a serde_json::Value,
|
||||||
data_type: Option<&str>,
|
data_type: Option<&str>,
|
||||||
db_type: &DatabaseType,
|
db_type: &DatabaseType,
|
||||||
kingbase_oracle_mode: bool,
|
kingbase_oracle_mode: bool,
|
||||||
date_time_format: Option<&str>,
|
date_time_format: Option<&str>,
|
||||||
) -> serde_json::Value {
|
) -> Cow<'a, serde_json::Value> {
|
||||||
let date_type_preserves_time = (matches!(db_type, DatabaseType::Oracle | DatabaseType::OceanbaseOracle)
|
let date_type_preserves_time = (matches!(db_type, DatabaseType::Oracle | DatabaseType::OceanbaseOracle)
|
||||||
|| (*db_type == DatabaseType::Kingbase && kingbase_oracle_mode))
|
|| (*db_type == DatabaseType::Kingbase && kingbase_oracle_mode))
|
||||||
&& data_type.is_some_and(|data_type| data_type.trim().eq_ignore_ascii_case("date"));
|
&& data_type.is_some_and(|data_type| data_type.trim().eq_ignore_ascii_case("date"));
|
||||||
crate::temporal_format::normalize_temporal_import_value(
|
crate::temporal_format::normalize_temporal_import_value_cow(
|
||||||
value,
|
value,
|
||||||
if date_type_preserves_time { Some("datetime") } else { data_type },
|
if date_type_preserves_time { Some("datetime") } else { data_type },
|
||||||
date_time_format,
|
date_time_format,
|
||||||
@@ -3352,6 +3367,41 @@ fn textual_source_columns_for_import(
|
|||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn normalize_import_value_cow<'a>(
|
||||||
|
value: &'a serde_json::Value,
|
||||||
|
data_type: Option<&str>,
|
||||||
|
db_type: &DatabaseType,
|
||||||
|
kingbase_oracle_mode: bool,
|
||||||
|
date_time_format: Option<&str>,
|
||||||
|
) -> Cow<'a, serde_json::Value> {
|
||||||
|
let normalized =
|
||||||
|
normalize_import_temporal_value_cow(value, data_type, db_type, kingbase_oracle_mode, date_time_format);
|
||||||
|
// Strip validated thousands separators before integer canonicalization so "1,234.00"
|
||||||
|
// still collapses to a plain integer literal for integer targets.
|
||||||
|
let thousands_canonical =
|
||||||
|
normalized.as_str().and_then(|value| normalize_thousands_numeric_literal(value, db_type, data_type));
|
||||||
|
if let Some(integer_text) = thousands_canonical
|
||||||
|
.as_deref()
|
||||||
|
.or_else(|| normalized.as_str())
|
||||||
|
.and_then(|value| normalize_integer_literal(value, db_type, data_type))
|
||||||
|
.and_then(|value| value.parse::<i64>().ok())
|
||||||
|
{
|
||||||
|
// Normalize before both INSERT and COPY paths; COPY does not pass through SQL literal escaping.
|
||||||
|
return Cow::Owned(serde_json::Value::Number(integer_text.into()));
|
||||||
|
}
|
||||||
|
if let Some(number) = normalized.as_number() {
|
||||||
|
if let Some(integer_text) = normalize_integer_literal(&number.to_string(), db_type, data_type)
|
||||||
|
.and_then(|value| value.parse::<i64>().ok())
|
||||||
|
{
|
||||||
|
return Cow::Owned(serde_json::Value::Number(integer_text.into()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(canonical) = thousands_canonical {
|
||||||
|
return Cow::Owned(serde_json::Value::String(canonical));
|
||||||
|
}
|
||||||
|
normalized
|
||||||
|
}
|
||||||
|
|
||||||
fn normalize_import_value(
|
fn normalize_import_value(
|
||||||
value: &serde_json::Value,
|
value: &serde_json::Value,
|
||||||
data_type: Option<&str>,
|
data_type: Option<&str>,
|
||||||
@@ -3359,28 +3409,7 @@ fn normalize_import_value(
|
|||||||
kingbase_oracle_mode: bool,
|
kingbase_oracle_mode: bool,
|
||||||
date_time_format: Option<&str>,
|
date_time_format: Option<&str>,
|
||||||
) -> serde_json::Value {
|
) -> serde_json::Value {
|
||||||
let normalized = normalize_import_temporal_value(value, data_type, db_type, kingbase_oracle_mode, date_time_format);
|
normalize_import_value_cow(value, data_type, db_type, kingbase_oracle_mode, date_time_format).into_owned()
|
||||||
// Strip validated thousands separators before integer canonicalization so "1,234.00"
|
|
||||||
// still collapses to a plain integer literal for integer targets.
|
|
||||||
let thousands_canonical =
|
|
||||||
normalized.as_str().and_then(|value| normalize_thousands_numeric_literal(value, db_type, data_type));
|
|
||||||
let integer_text = thousands_canonical
|
|
||||||
.as_deref()
|
|
||||||
.or_else(|| normalized.as_str())
|
|
||||||
.map(str::to_owned)
|
|
||||||
.or_else(|| normalized.as_number().map(ToString::to_string));
|
|
||||||
if let Some(integer_text) = integer_text
|
|
||||||
.as_deref()
|
|
||||||
.and_then(|value| normalize_integer_literal(value, db_type, data_type))
|
|
||||||
.and_then(|value| value.parse::<i64>().ok())
|
|
||||||
{
|
|
||||||
// Normalize before both INSERT and COPY paths; COPY does not pass through SQL literal escaping.
|
|
||||||
return serde_json::Value::Number(integer_text.into());
|
|
||||||
}
|
|
||||||
if let Some(canonical) = thousands_canonical {
|
|
||||||
return serde_json::Value::String(canonical);
|
|
||||||
}
|
|
||||||
normalized
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn build_import_insert_batches(
|
pub fn build_import_insert_batches(
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
use chrono::{DateTime, FixedOffset, NaiveDate, NaiveDateTime, NaiveTime, Timelike};
|
use chrono::{DateTime, FixedOffset, NaiveDate, NaiveDateTime, NaiveTime, Timelike};
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
|
use std::borrow::Cow;
|
||||||
use std::fmt::Write as _;
|
use std::fmt::Write as _;
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
@@ -192,17 +193,51 @@ pub fn format_temporal_export_value(value: &Value, data_type: Option<&str>, patt
|
|||||||
.unwrap_or_else(|| value.clone())
|
.unwrap_or_else(|| value.clone())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn active_temporal_pattern(pattern: Option<&str>) -> Option<&str> {
|
||||||
|
pattern.filter(|pattern| !pattern.trim().is_empty())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn format_temporal_export_row_cow<'a>(
|
||||||
|
row: &'a [Value],
|
||||||
|
column_types: &[Option<String>],
|
||||||
|
pattern: Option<&str>,
|
||||||
|
) -> Cow<'a, [Value]> {
|
||||||
|
let Some(pattern) = active_temporal_pattern(pattern) else {
|
||||||
|
return Cow::Borrowed(row);
|
||||||
|
};
|
||||||
|
if !column_types.iter().any(|data_type| temporal_kind(data_type.as_deref()).is_some()) {
|
||||||
|
return Cow::Borrowed(row);
|
||||||
|
}
|
||||||
|
Cow::Owned(
|
||||||
|
row.iter()
|
||||||
|
.enumerate()
|
||||||
|
.map(|(index, value)| {
|
||||||
|
format_temporal_export_value(
|
||||||
|
value,
|
||||||
|
column_types.get(index).and_then(|data_type| data_type.as_deref()),
|
||||||
|
Some(pattern),
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn format_temporal_export_row(row: &[Value], column_types: &[Option<String>], pattern: Option<&str>) -> Vec<Value> {
|
pub fn format_temporal_export_row(row: &[Value], column_types: &[Option<String>], pattern: Option<&str>) -> Vec<Value> {
|
||||||
row.iter()
|
format_temporal_export_row_cow(row, column_types, pattern).into_owned()
|
||||||
.enumerate()
|
}
|
||||||
.map(|(index, value)| {
|
|
||||||
format_temporal_export_value(
|
pub fn format_temporal_export_rows_cow<'a>(
|
||||||
value,
|
rows: &'a [Vec<Value>],
|
||||||
column_types.get(index).and_then(|data_type| data_type.as_deref()),
|
column_types: &[Option<String>],
|
||||||
pattern,
|
pattern: Option<&str>,
|
||||||
)
|
) -> Cow<'a, [Vec<Value>]> {
|
||||||
})
|
let Some(pattern) = active_temporal_pattern(pattern) else {
|
||||||
.collect()
|
return Cow::Borrowed(rows);
|
||||||
|
};
|
||||||
|
if !column_types.iter().any(|data_type| temporal_kind(data_type.as_deref()).is_some()) {
|
||||||
|
return Cow::Borrowed(rows);
|
||||||
|
}
|
||||||
|
Cow::Owned(rows.iter().map(|row| format_temporal_export_row(row, column_types, Some(pattern))).collect())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn format_temporal_export_rows(
|
pub fn format_temporal_export_rows(
|
||||||
@@ -210,7 +245,28 @@ pub fn format_temporal_export_rows(
|
|||||||
column_types: &[Option<String>],
|
column_types: &[Option<String>],
|
||||||
pattern: Option<&str>,
|
pattern: Option<&str>,
|
||||||
) -> Vec<Vec<Value>> {
|
) -> Vec<Vec<Value>> {
|
||||||
rows.iter().map(|row| format_temporal_export_row(row, column_types, pattern)).collect()
|
format_temporal_export_rows_cow(rows, column_types, pattern).into_owned()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn format_temporal_export_row_with_string_types_cow<'a>(
|
||||||
|
row: &'a [Value],
|
||||||
|
column_types: &[String],
|
||||||
|
pattern: Option<&str>,
|
||||||
|
) -> Cow<'a, [Value]> {
|
||||||
|
let Some(pattern) = active_temporal_pattern(pattern) else {
|
||||||
|
return Cow::Borrowed(row);
|
||||||
|
};
|
||||||
|
if !column_types.iter().any(|data_type| temporal_kind(Some(data_type)).is_some()) {
|
||||||
|
return Cow::Borrowed(row);
|
||||||
|
}
|
||||||
|
Cow::Owned(
|
||||||
|
row.iter()
|
||||||
|
.enumerate()
|
||||||
|
.map(|(index, value)| {
|
||||||
|
format_temporal_export_value(value, column_types.get(index).map(String::as_str), Some(pattern))
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn format_temporal_export_row_with_string_types(
|
pub fn format_temporal_export_row_with_string_types(
|
||||||
@@ -218,10 +274,23 @@ pub fn format_temporal_export_row_with_string_types(
|
|||||||
column_types: &[String],
|
column_types: &[String],
|
||||||
pattern: Option<&str>,
|
pattern: Option<&str>,
|
||||||
) -> Vec<Value> {
|
) -> Vec<Value> {
|
||||||
row.iter()
|
format_temporal_export_row_with_string_types_cow(row, column_types, pattern).into_owned()
|
||||||
.enumerate()
|
}
|
||||||
.map(|(index, value)| format_temporal_export_value(value, column_types.get(index).map(String::as_str), pattern))
|
|
||||||
.collect()
|
pub fn format_temporal_export_rows_with_string_types_cow<'a>(
|
||||||
|
rows: &'a [Vec<Value>],
|
||||||
|
column_types: &[String],
|
||||||
|
pattern: Option<&str>,
|
||||||
|
) -> Cow<'a, [Vec<Value>]> {
|
||||||
|
let Some(pattern) = active_temporal_pattern(pattern) else {
|
||||||
|
return Cow::Borrowed(rows);
|
||||||
|
};
|
||||||
|
if !column_types.iter().any(|data_type| temporal_kind(Some(data_type)).is_some()) {
|
||||||
|
return Cow::Borrowed(rows);
|
||||||
|
}
|
||||||
|
Cow::Owned(
|
||||||
|
rows.iter().map(|row| format_temporal_export_row_with_string_types(row, column_types, Some(pattern))).collect(),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn format_temporal_export_rows_with_string_types(
|
pub fn format_temporal_export_rows_with_string_types(
|
||||||
@@ -229,40 +298,44 @@ pub fn format_temporal_export_rows_with_string_types(
|
|||||||
column_types: &[String],
|
column_types: &[String],
|
||||||
pattern: Option<&str>,
|
pattern: Option<&str>,
|
||||||
) -> Vec<Vec<Value>> {
|
) -> Vec<Vec<Value>> {
|
||||||
rows.iter().map(|row| format_temporal_export_row_with_string_types(row, column_types, pattern)).collect()
|
format_temporal_export_rows_with_string_types_cow(rows, column_types, pattern).into_owned()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn normalize_temporal_import_value(value: &Value, data_type: Option<&str>, pattern: Option<&str>) -> Value {
|
pub(crate) fn normalize_temporal_import_value_cow<'a>(
|
||||||
|
value: &'a Value,
|
||||||
|
data_type: Option<&str>,
|
||||||
|
pattern: Option<&str>,
|
||||||
|
) -> Cow<'a, Value> {
|
||||||
let Some(kind) = temporal_kind(data_type) else {
|
let Some(kind) = temporal_kind(data_type) else {
|
||||||
return value.clone();
|
return Cow::Borrowed(value);
|
||||||
};
|
};
|
||||||
let Some(raw) = value.as_str() else {
|
let Some(raw) = value.as_str() else {
|
||||||
return value.clone();
|
return Cow::Borrowed(value);
|
||||||
};
|
};
|
||||||
let Some(parsed) = parse_temporal(raw.trim(), pattern) else {
|
let Some(parsed) = parse_temporal(raw.trim(), pattern) else {
|
||||||
return value.clone();
|
return Cow::Borrowed(value);
|
||||||
};
|
};
|
||||||
|
|
||||||
let normalized = match (kind, parsed) {
|
let normalized = match (kind, parsed) {
|
||||||
(TemporalKind::Date, ParsedTemporal::Zoned(value)) => value.date_naive().format("%Y-%m-%d").to_string(),
|
(TemporalKind::Date, ParsedTemporal::Zoned(value)) => value.date_naive().format("%Y-%m-%d").to_string(),
|
||||||
(TemporalKind::Date, ParsedTemporal::DateTime(value)) => value.date().format("%Y-%m-%d").to_string(),
|
(TemporalKind::Date, ParsedTemporal::DateTime(value)) => value.date().format("%Y-%m-%d").to_string(),
|
||||||
(TemporalKind::Date, ParsedTemporal::Date(value)) => value.format("%Y-%m-%d").to_string(),
|
(TemporalKind::Date, ParsedTemporal::Date(value)) => value.format("%Y-%m-%d").to_string(),
|
||||||
(TemporalKind::Date, ParsedTemporal::Time(_)) => return value.clone(),
|
(TemporalKind::Date, ParsedTemporal::Time(_)) => return Cow::Borrowed(value),
|
||||||
(TemporalKind::Time, ParsedTemporal::Zoned(value)) => value.time().format("%H:%M:%S%.f").to_string(),
|
(TemporalKind::Time, ParsedTemporal::Zoned(value)) => value.time().format("%H:%M:%S%.f").to_string(),
|
||||||
(TemporalKind::Time, ParsedTemporal::DateTime(value)) => value.time().format("%H:%M:%S%.f").to_string(),
|
(TemporalKind::Time, ParsedTemporal::DateTime(value)) => value.time().format("%H:%M:%S%.f").to_string(),
|
||||||
(TemporalKind::Time, ParsedTemporal::Time(value)) => value.format("%H:%M:%S%.f").to_string(),
|
(TemporalKind::Time, ParsedTemporal::Time(value)) => value.format("%H:%M:%S%.f").to_string(),
|
||||||
(TemporalKind::Time, ParsedTemporal::Date(_)) => return value.clone(),
|
(TemporalKind::Time, ParsedTemporal::Date(_)) => return Cow::Borrowed(value),
|
||||||
(TemporalKind::DateTime, ParsedTemporal::Zoned(value)) => {
|
(TemporalKind::DateTime, ParsedTemporal::Zoned(value)) => {
|
||||||
value.naive_local().format("%Y-%m-%d %H:%M:%S%.f").to_string()
|
value.naive_local().format("%Y-%m-%d %H:%M:%S%.f").to_string()
|
||||||
}
|
}
|
||||||
(TemporalKind::DateTime, ParsedTemporal::DateTime(value)) => value.format("%Y-%m-%d %H:%M:%S%.f").to_string(),
|
(TemporalKind::DateTime, ParsedTemporal::DateTime(value)) => value.format("%Y-%m-%d %H:%M:%S%.f").to_string(),
|
||||||
(TemporalKind::DateTime, ParsedTemporal::Date(date)) => {
|
(TemporalKind::DateTime, ParsedTemporal::Date(date)) => {
|
||||||
let Some(value) = date.and_hms_opt(0, 0, 0) else {
|
let Some(value) = date.and_hms_opt(0, 0, 0) else {
|
||||||
return value.clone();
|
return Cow::Borrowed(value);
|
||||||
};
|
};
|
||||||
value.format("%Y-%m-%d %H:%M:%S").to_string()
|
value.format("%Y-%m-%d %H:%M:%S").to_string()
|
||||||
}
|
}
|
||||||
(TemporalKind::DateTime, ParsedTemporal::Time(_)) => return value.clone(),
|
(TemporalKind::DateTime, ParsedTemporal::Time(_)) => return Cow::Borrowed(value),
|
||||||
(TemporalKind::DateTimeWithTimeZone, ParsedTemporal::Zoned(value)) => {
|
(TemporalKind::DateTimeWithTimeZone, ParsedTemporal::Zoned(value)) => {
|
||||||
value.format("%Y-%m-%dT%H:%M:%S%.f%:z").to_string()
|
value.format("%Y-%m-%dT%H:%M:%S%.f%:z").to_string()
|
||||||
}
|
}
|
||||||
@@ -271,13 +344,17 @@ pub fn normalize_temporal_import_value(value: &Value, data_type: Option<&str>, p
|
|||||||
}
|
}
|
||||||
(TemporalKind::DateTimeWithTimeZone, ParsedTemporal::Date(date)) => {
|
(TemporalKind::DateTimeWithTimeZone, ParsedTemporal::Date(date)) => {
|
||||||
let Some(value) = date.and_hms_opt(0, 0, 0) else {
|
let Some(value) = date.and_hms_opt(0, 0, 0) else {
|
||||||
return value.clone();
|
return Cow::Borrowed(value);
|
||||||
};
|
};
|
||||||
value.format("%Y-%m-%d %H:%M:%S").to_string()
|
value.format("%Y-%m-%d %H:%M:%S").to_string()
|
||||||
}
|
}
|
||||||
(TemporalKind::DateTimeWithTimeZone, ParsedTemporal::Time(_)) => return value.clone(),
|
(TemporalKind::DateTimeWithTimeZone, ParsedTemporal::Time(_)) => return Cow::Borrowed(value),
|
||||||
};
|
};
|
||||||
Value::String(normalized)
|
Cow::Owned(Value::String(normalized))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn normalize_temporal_import_value(value: &Value, data_type: Option<&str>, pattern: Option<&str>) -> Value {
|
||||||
|
normalize_temporal_import_value_cow(value, data_type, pattern).into_owned()
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -358,4 +435,31 @@ mod tests {
|
|||||||
json!("2024-02-25T13:02:15+08:00")
|
json!("2024-02-25T13:02:15+08:00")
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn export_cow_borrows_when_no_formatting_is_needed() {
|
||||||
|
let row = vec![json!(1), json!("2024-02-25 13:02:15")];
|
||||||
|
let rows = vec![row.clone()];
|
||||||
|
|
||||||
|
assert!(matches!(
|
||||||
|
format_temporal_export_row_cow(&row, &[None, Some("TIMESTAMP".to_string())], None),
|
||||||
|
Cow::Borrowed(_)
|
||||||
|
));
|
||||||
|
assert!(matches!(
|
||||||
|
format_temporal_export_rows_cow(&rows, &[None, Some("VARCHAR".to_string())], Some("YYYY/MM/DD")),
|
||||||
|
Cow::Borrowed(_)
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn import_cow_borrows_unchanged_values_and_owns_normalized_values() {
|
||||||
|
let text = json!("plain");
|
||||||
|
assert!(matches!(normalize_temporal_import_value_cow(&text, Some("TEXT"), None), Cow::Borrowed(_)));
|
||||||
|
|
||||||
|
let timestamp = json!("2024/2/25 13:02:15");
|
||||||
|
assert_eq!(
|
||||||
|
normalize_temporal_import_value_cow(×tamp, Some("TIMESTAMP"), None).into_owned(),
|
||||||
|
json!("2024-02-25 13:02:15")
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -3263,9 +3263,19 @@ pub fn generate_insert_typed(
|
|||||||
return String::new();
|
return String::new();
|
||||||
}
|
}
|
||||||
|
|
||||||
let template = InsertSqlTemplate::new(columns, table, schema, db_type, catalog, false);
|
|
||||||
let value_rows = value_rows_sql(rows, column_types, db_type, false);
|
let value_rows = value_rows_sql(rows, column_types, db_type, false);
|
||||||
template.build(&value_rows)
|
generate_insert_typed_from_value_rows(columns, &value_rows, table, schema, db_type, catalog)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn generate_insert_typed_from_value_rows(
|
||||||
|
columns: &[String],
|
||||||
|
value_rows: &[String],
|
||||||
|
table: &str,
|
||||||
|
schema: &str,
|
||||||
|
db_type: &DatabaseType,
|
||||||
|
catalog: Option<&str>,
|
||||||
|
) -> String {
|
||||||
|
InsertSqlTemplate::new(columns, table, schema, db_type, catalog, false).build(value_rows)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
@@ -3302,7 +3312,13 @@ impl InsertSqlTemplate {
|
|||||||
return String::new();
|
return String::new();
|
||||||
}
|
}
|
||||||
if let Some(into_prefix) = self.oracle_into_prefix.as_deref().filter(|_| value_rows.len() > 1) {
|
if let Some(into_prefix) = self.oracle_into_prefix.as_deref().filter(|_| value_rows.len() > 1) {
|
||||||
let mut sql = String::from("INSERT ALL\n");
|
let capacity = "INSERT ALL\n".len()
|
||||||
|
+ into_prefix.len().saturating_mul(value_rows.len())
|
||||||
|
+ value_rows.iter().map(String::len).sum::<usize>()
|
||||||
|
+ value_rows.len().saturating_sub(1)
|
||||||
|
+ "\nSELECT 1 FROM dual".len();
|
||||||
|
let mut sql = String::with_capacity(capacity);
|
||||||
|
sql.push_str("INSERT ALL\n");
|
||||||
for (index, values) in value_rows.iter().enumerate() {
|
for (index, values) in value_rows.iter().enumerate() {
|
||||||
if index > 0 {
|
if index > 0 {
|
||||||
sql.push('\n');
|
sql.push('\n');
|
||||||
@@ -3314,8 +3330,17 @@ impl InsertSqlTemplate {
|
|||||||
return sql;
|
return sql;
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut sql = self.standard_prefix.clone();
|
let capacity = self.standard_prefix.len()
|
||||||
sql.push_str(&value_rows.join(",\n"));
|
+ value_rows.iter().map(String::len).sum::<usize>()
|
||||||
|
+ ",\n".len().saturating_mul(value_rows.len().saturating_sub(1));
|
||||||
|
let mut sql = String::with_capacity(capacity);
|
||||||
|
sql.push_str(&self.standard_prefix);
|
||||||
|
for (index, values) in value_rows.iter().enumerate() {
|
||||||
|
if index > 0 {
|
||||||
|
sql.push_str(",\n");
|
||||||
|
}
|
||||||
|
sql.push_str(values);
|
||||||
|
}
|
||||||
sql
|
sql
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -3350,8 +3375,12 @@ fn value_rows_sql(
|
|||||||
) -> Vec<String> {
|
) -> Vec<String> {
|
||||||
let mut out = Vec::with_capacity(rows.len());
|
let mut out = Vec::with_capacity(rows.len());
|
||||||
for row in rows {
|
for row in rows {
|
||||||
let mut vals = Vec::with_capacity(row.len());
|
let mut values = String::with_capacity(row.len().saturating_mul(16).saturating_add(2));
|
||||||
|
values.push('(');
|
||||||
for (index, v) in row.iter().enumerate() {
|
for (index, v) in row.iter().enumerate() {
|
||||||
|
if index > 0 {
|
||||||
|
values.push_str(", ");
|
||||||
|
}
|
||||||
let column_type = column_types.get(index).and_then(|value| value.as_deref());
|
let column_type = column_types.get(index).and_then(|value| value.as_deref());
|
||||||
let value = if mysql_spatial_markers {
|
let value = if mysql_spatial_markers {
|
||||||
crate::database_export::format_mysql_spatial_export_literal(v, Some(*db_type), column_type)
|
crate::database_export::format_mysql_spatial_export_literal(v, Some(*db_type), column_type)
|
||||||
@@ -3359,9 +3388,10 @@ fn value_rows_sql(
|
|||||||
None
|
None
|
||||||
}
|
}
|
||||||
.unwrap_or_else(|| escape_value_typed(v, db_type, column_type));
|
.unwrap_or_else(|| escape_value_typed(v, db_type, column_type));
|
||||||
vals.push(value);
|
values.push_str(&value);
|
||||||
}
|
}
|
||||||
out.push(format!("({})", vals.join(", ")));
|
values.push(')');
|
||||||
|
out.push(values);
|
||||||
}
|
}
|
||||||
out
|
out
|
||||||
}
|
}
|
||||||
@@ -3714,6 +3744,7 @@ fn generate_transfer_write_sql(
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
#[cfg_attr(not(test), allow(dead_code))]
|
||||||
pub(crate) fn generate_insert_typed_sql_batches(
|
pub(crate) fn generate_insert_typed_sql_batches(
|
||||||
columns: &[String],
|
columns: &[String],
|
||||||
column_types: &[Option<String>],
|
column_types: &[Option<String>],
|
||||||
@@ -3724,18 +3755,21 @@ pub(crate) fn generate_insert_typed_sql_batches(
|
|||||||
catalog: Option<&str>,
|
catalog: Option<&str>,
|
||||||
limits: SqlBatchLimits,
|
limits: SqlBatchLimits,
|
||||||
) -> Result<Vec<(String, usize)>, String> {
|
) -> Result<Vec<(String, usize)>, String> {
|
||||||
generate_insert_typed_sql_batches_for_transfer(
|
let value_rows = value_rows_sql(rows, column_types, db_type, false);
|
||||||
columns,
|
generate_insert_typed_sql_batches_from_value_rows(columns, &value_rows, table, schema, db_type, catalog, limits)
|
||||||
column_types,
|
}
|
||||||
rows,
|
|
||||||
table,
|
#[allow(clippy::too_many_arguments)]
|
||||||
schema,
|
pub(crate) fn generate_insert_typed_sql_batches_from_value_rows(
|
||||||
db_type,
|
columns: &[String],
|
||||||
catalog,
|
value_rows: &[String],
|
||||||
limits,
|
table: &str,
|
||||||
false,
|
schema: &str,
|
||||||
false,
|
db_type: &DatabaseType,
|
||||||
)
|
catalog: Option<&str>,
|
||||||
|
limits: SqlBatchLimits,
|
||||||
|
) -> Result<Vec<(String, usize)>, String> {
|
||||||
|
generate_insert_sql_batches_from_value_rows(columns, value_rows, table, schema, db_type, catalog, limits, false)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
@@ -3755,6 +3789,34 @@ fn generate_insert_typed_sql_batches_for_transfer(
|
|||||||
return Ok(Vec::new());
|
return Ok(Vec::new());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let value_rows = value_rows_sql(rows, column_types, db_type, mysql_spatial_markers);
|
||||||
|
generate_insert_sql_batches_from_value_rows(
|
||||||
|
columns,
|
||||||
|
&value_rows,
|
||||||
|
table,
|
||||||
|
schema,
|
||||||
|
db_type,
|
||||||
|
catalog,
|
||||||
|
limits,
|
||||||
|
overrides_postgres_system_values,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
fn generate_insert_sql_batches_from_value_rows(
|
||||||
|
columns: &[String],
|
||||||
|
value_rows: &[String],
|
||||||
|
table: &str,
|
||||||
|
schema: &str,
|
||||||
|
db_type: &DatabaseType,
|
||||||
|
catalog: Option<&str>,
|
||||||
|
limits: SqlBatchLimits,
|
||||||
|
overrides_postgres_system_values: bool,
|
||||||
|
) -> Result<Vec<(String, usize)>, String> {
|
||||||
|
if value_rows.is_empty() {
|
||||||
|
return Ok(Vec::new());
|
||||||
|
}
|
||||||
|
|
||||||
let max_rows = limits.max_rows.max(1).min(match db_type {
|
let max_rows = limits.max_rows.max(1).min(match db_type {
|
||||||
DatabaseType::SqlServer => MAX_SQLSERVER_INSERT_ROWS,
|
DatabaseType::SqlServer => MAX_SQLSERVER_INSERT_ROWS,
|
||||||
DatabaseType::Oracle => MAX_ORACLE_INSERT_ALL_ROWS,
|
DatabaseType::Oracle => MAX_ORACLE_INSERT_ALL_ROWS,
|
||||||
@@ -3763,7 +3825,6 @@ fn generate_insert_typed_sql_batches_for_transfer(
|
|||||||
let target_sql_bytes = limits.target_sql_bytes.max(1);
|
let target_sql_bytes = limits.target_sql_bytes.max(1);
|
||||||
let batch_sql_bytes = limits.hard_sql_bytes.map_or(target_sql_bytes, |hard| target_sql_bytes.min(hard));
|
let batch_sql_bytes = limits.hard_sql_bytes.map_or(target_sql_bytes, |hard| target_sql_bytes.min(hard));
|
||||||
let template = InsertSqlTemplate::new(columns, table, schema, db_type, catalog, overrides_postgres_system_values);
|
let template = InsertSqlTemplate::new(columns, table, schema, db_type, catalog, overrides_postgres_system_values);
|
||||||
let value_rows = value_rows_sql(rows, column_types, db_type, mysql_spatial_markers);
|
|
||||||
let value_row_bytes = value_rows.iter().map(|row| sql_text_bytes(row, db_type)).collect::<Vec<_>>();
|
let value_row_bytes = value_rows.iter().map(|row| sql_text_bytes(row, db_type)).collect::<Vec<_>>();
|
||||||
let mut statements = Vec::new();
|
let mut statements = Vec::new();
|
||||||
let mut start = 0usize;
|
let mut start = 0usize;
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
|
use std::fmt::Write as FmtWrite;
|
||||||
use std::io::{Cursor, Seek, Write};
|
use std::io::{Cursor, Seek, Write};
|
||||||
|
|
||||||
use crate::temporal_format::{excel_temporal_serial, ExcelTemporalKind};
|
use crate::temporal_format::{excel_temporal_serial, ExcelTemporalKind};
|
||||||
@@ -130,6 +131,7 @@ pub struct StreamingXlsxWriter<W: Write + Seek> {
|
|||||||
column_comments: Vec<Option<String>>,
|
column_comments: Vec<Option<String>>,
|
||||||
date_time_format: Option<String>,
|
date_time_format: Option<String>,
|
||||||
numeric_right_align: bool,
|
numeric_right_align: bool,
|
||||||
|
row_buffer: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Estimate column widths from header names only (used by the streaming path
|
/// Estimate column widths from header names only (used by the streaming path
|
||||||
@@ -159,8 +161,8 @@ fn cols_xml(widths: &[usize]) -> String {
|
|||||||
|
|
||||||
/// Resolve the effective header text: prefer a non-empty column comment, fall
|
/// Resolve the effective header text: prefer a non-empty column comment, fall
|
||||||
/// back to the original column name.
|
/// back to the original column name.
|
||||||
fn effective_header(column: &str, comment: Option<&str>) -> String {
|
fn effective_header<'a>(column: &'a str, comment: Option<&'a str>) -> &'a str {
|
||||||
comment.filter(|c| !c.is_empty()).unwrap_or(column).to_string()
|
comment.filter(|c| !c.is_empty()).unwrap_or(column)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Build a single `<row>` XML fragment for the header row (row 1).
|
/// Build a single `<row>` XML fragment for the header row (row 1).
|
||||||
@@ -172,30 +174,36 @@ pub(crate) fn header_row_xml(columns: &[String], column_comments: &[Option<Strin
|
|||||||
.enumerate()
|
.enumerate()
|
||||||
.map(|(index, col)| {
|
.map(|(index, col)| {
|
||||||
let header = effective_header(col, column_comments.get(index).and_then(|c| c.as_deref()));
|
let header = effective_header(col, column_comments.get(index).and_then(|c| c.as_deref()));
|
||||||
cell_xml(Some(&Value::String(header)), 0, index, Some(1))
|
cell_xml(Some(&Value::String(header.to_string())), 0, index, Some(1))
|
||||||
})
|
})
|
||||||
.collect::<String>()
|
.collect::<String>()
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn data_row_xml_with_date_time_format(
|
fn push_data_row_xml(
|
||||||
|
output: &mut String,
|
||||||
row_number: usize,
|
row_number: usize,
|
||||||
columns: &[String],
|
columns: &[String],
|
||||||
column_types: &[String],
|
column_types: &[String],
|
||||||
row: &[Value],
|
row: &[Value],
|
||||||
date_time_format: Option<&str>,
|
date_time_format: Option<&str>,
|
||||||
numeric_right_align: bool,
|
numeric_right_align: bool,
|
||||||
) -> String {
|
) {
|
||||||
let cells = columns
|
write!(output, "<row r=\"{row_number}\">").expect("writing XLSX XML into a String cannot fail");
|
||||||
.iter()
|
for (col_index, _) in columns.iter().enumerate() {
|
||||||
.enumerate()
|
let col_type = column_types.get(col_index);
|
||||||
.map(|(col_index, _)| {
|
let align_style = numeric_column_style(col_type, numeric_right_align);
|
||||||
let col_type = column_types.get(col_index);
|
push_typed_cell_xml(
|
||||||
let align_style = numeric_column_style(col_type, numeric_right_align);
|
output,
|
||||||
typed_cell_xml(row.get(col_index), col_type, row_number - 1, col_index, align_style, date_time_format)
|
row.get(col_index),
|
||||||
})
|
col_type,
|
||||||
.collect::<String>();
|
row_number - 1,
|
||||||
format!("<row r=\"{row_number}\">{cells}</row>")
|
col_index,
|
||||||
|
align_style,
|
||||||
|
date_time_format,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
output.push_str("</row>");
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Shared ZIP entry options for all XLSX parts. XLSX is a ZIP of XML, and the
|
/// Shared ZIP entry options for all XLSX parts. XLSX is a ZIP of XML, and the
|
||||||
@@ -298,6 +306,7 @@ fn start_xlsx_writer_inner<W: Write + Seek>(
|
|||||||
column_comments: column_comments.to_vec(),
|
column_comments: column_comments.to_vec(),
|
||||||
date_time_format: date_time_format.map(str::to_string),
|
date_time_format: date_time_format.map(str::to_string),
|
||||||
numeric_right_align,
|
numeric_right_align,
|
||||||
|
row_buffer: String::with_capacity(columns.len().saturating_mul(48)),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -355,19 +364,17 @@ impl<W: Write + Seek> StreamingXlsxWriter<W> {
|
|||||||
self.finish_current_sheet()?;
|
self.finish_current_sheet()?;
|
||||||
self.start_next_data_sheet()?;
|
self.start_next_data_sheet()?;
|
||||||
}
|
}
|
||||||
self.zip
|
self.row_buffer.clear();
|
||||||
.write_all(
|
push_data_row_xml(
|
||||||
data_row_xml_with_date_time_format(
|
&mut self.row_buffer,
|
||||||
self.next_row_number,
|
self.next_row_number,
|
||||||
&self.columns,
|
&self.columns,
|
||||||
&self.column_types,
|
&self.column_types,
|
||||||
row,
|
row,
|
||||||
self.date_time_format.as_deref(),
|
self.date_time_format.as_deref(),
|
||||||
self.numeric_right_align,
|
self.numeric_right_align,
|
||||||
)
|
);
|
||||||
.as_bytes(),
|
self.zip.write_all(self.row_buffer.as_bytes()).map_err(|err| err.to_string())?;
|
||||||
)
|
|
||||||
.map_err(|err| err.to_string())?;
|
|
||||||
self.next_row_number += 1;
|
self.next_row_number += 1;
|
||||||
self.current_data_rows += 1;
|
self.current_data_rows += 1;
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -469,6 +476,11 @@ pub(crate) fn finish_streaming_xlsx_workbook<W: Write + Seek>(writer: StreamingX
|
|||||||
|
|
||||||
fn escape_xml(value: &str) -> String {
|
fn escape_xml(value: &str) -> String {
|
||||||
let mut result = String::with_capacity(value.len());
|
let mut result = String::with_capacity(value.len());
|
||||||
|
push_xml_escaped(&mut result, value);
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_xml_escaped(result: &mut String, value: &str) {
|
||||||
for ch in value.chars() {
|
for ch in value.chars() {
|
||||||
let code = ch as u32;
|
let code = ch as u32;
|
||||||
if code != 9 && code != 10 && code != 13 && code < 32 {
|
if code != 9 && code != 10 && code != 13 && code < 32 {
|
||||||
@@ -482,22 +494,36 @@ fn escape_xml(value: &str) -> String {
|
|||||||
_ => result.push(ch),
|
_ => result.push(ch),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
result
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn column_name(index: usize) -> String {
|
fn column_name(index: usize) -> String {
|
||||||
let mut out = String::new();
|
let mut out = String::with_capacity(4);
|
||||||
|
push_column_name(&mut out, index);
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_column_name(out: &mut String, index: usize) {
|
||||||
|
let mut buffer = [0u8; 16];
|
||||||
|
let mut cursor = buffer.len();
|
||||||
let mut n = index + 1;
|
let mut n = index + 1;
|
||||||
while n > 0 {
|
while n > 0 {
|
||||||
let rem = (n - 1) % 26;
|
let rem = (n - 1) % 26;
|
||||||
out.push((b'A' + rem as u8) as char);
|
cursor -= 1;
|
||||||
|
buffer[cursor] = b'A' + rem as u8;
|
||||||
n = (n - 1) / 26;
|
n = (n - 1) / 26;
|
||||||
}
|
}
|
||||||
out.chars().rev().collect()
|
out.push_str(std::str::from_utf8(&buffer[cursor..]).expect("XLSX column names are ASCII"));
|
||||||
}
|
}
|
||||||
|
|
||||||
fn cell_ref(row_index: usize, col_index: usize) -> String {
|
fn cell_ref(row_index: usize, col_index: usize) -> String {
|
||||||
format!("{}{}", column_name(col_index), row_index + 1)
|
let mut reference = String::with_capacity(8);
|
||||||
|
push_cell_ref(&mut reference, row_index, col_index);
|
||||||
|
reference
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_cell_ref(out: &mut String, row_index: usize, col_index: usize) {
|
||||||
|
push_column_name(out, col_index);
|
||||||
|
write!(out, "{}", row_index + 1).expect("writing XLSX cell references into a String cannot fail");
|
||||||
}
|
}
|
||||||
|
|
||||||
fn sheet_range(column_count: usize, row_count: usize) -> String {
|
fn sheet_range(column_count: usize, row_count: usize) -> String {
|
||||||
@@ -560,6 +586,56 @@ fn cell_xml(value: Option<&Value>, row_index: usize, col_index: usize, style: Op
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn push_cell_style(output: &mut String, style: Option<usize>) {
|
||||||
|
if let Some(style) = style {
|
||||||
|
write!(output, " s=\"{style}\"").expect("writing XLSX cell styles into a String cannot fail");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_cell_xml(output: &mut String, value: Option<&Value>, row_index: usize, col_index: usize, style: Option<usize>) {
|
||||||
|
output.push_str("<c r=\"");
|
||||||
|
push_cell_ref(output, row_index, col_index);
|
||||||
|
output.push('"');
|
||||||
|
match value {
|
||||||
|
Some(Value::Null) | None => {
|
||||||
|
push_cell_style(output, style);
|
||||||
|
output.push_str("/>");
|
||||||
|
}
|
||||||
|
Some(Value::Bool(value)) => {
|
||||||
|
let bool_value = if *value { 1 } else { 0 };
|
||||||
|
output.push_str(" t=\"b\"");
|
||||||
|
push_cell_style(output, style);
|
||||||
|
write!(output, "><v>{bool_value}</v></c>").expect("writing XLSX booleans into a String cannot fail");
|
||||||
|
}
|
||||||
|
Some(Value::Number(value)) => {
|
||||||
|
if value.as_f64().is_some_and(|number| number.is_finite()) {
|
||||||
|
push_cell_style(output, style);
|
||||||
|
write!(output, "><v>{value}</v></c>").expect("writing XLSX numbers into a String cannot fail");
|
||||||
|
} else {
|
||||||
|
output.push_str(" t=\"inlineStr\"");
|
||||||
|
push_cell_style(output, style);
|
||||||
|
output.push_str("><is><t>");
|
||||||
|
push_xml_escaped(output, &value.to_string());
|
||||||
|
output.push_str("</t></is></c>");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Some(Value::String(value)) => {
|
||||||
|
output.push_str(" t=\"inlineStr\"");
|
||||||
|
push_cell_style(output, style);
|
||||||
|
output.push_str("><is><t>");
|
||||||
|
push_xml_escaped(output, value);
|
||||||
|
output.push_str("</t></is></c>");
|
||||||
|
}
|
||||||
|
Some(value) => {
|
||||||
|
output.push_str(" t=\"inlineStr\"");
|
||||||
|
push_cell_style(output, style);
|
||||||
|
output.push_str("><is><t>");
|
||||||
|
push_xml_escaped(output, &value.to_string());
|
||||||
|
output.push_str("</t></is></c>");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn is_numeric_column_type(column_type: Option<&String>) -> bool {
|
fn is_numeric_column_type(column_type: Option<&String>) -> bool {
|
||||||
let mut normalized = column_type.map(|value| value.trim().to_ascii_lowercase()).unwrap_or_default();
|
let mut normalized = column_type.map(|value| value.trim().to_ascii_lowercase()).unwrap_or_default();
|
||||||
while normalized.ends_with(')') {
|
while normalized.ends_with(')') {
|
||||||
@@ -659,36 +735,43 @@ fn safe_excel_number(value: &str) -> Option<&str> {
|
|||||||
(significant_digits <= 15).then_some(trimmed)
|
(significant_digits <= 15).then_some(trimmed)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn typed_cell_xml(
|
fn push_typed_cell_xml(
|
||||||
|
output: &mut String,
|
||||||
value: Option<&Value>,
|
value: Option<&Value>,
|
||||||
column_type: Option<&String>,
|
column_type: Option<&String>,
|
||||||
row_index: usize,
|
row_index: usize,
|
||||||
col_index: usize,
|
col_index: usize,
|
||||||
style: Option<usize>,
|
style: Option<usize>,
|
||||||
date_time_format: Option<&str>,
|
date_time_format: Option<&str>,
|
||||||
) -> String {
|
) {
|
||||||
if let Some(Value::String(value)) = value {
|
if let Some(Value::String(value)) = value {
|
||||||
if let Some((serial, temporal_kind)) =
|
if let Some((serial, temporal_kind)) =
|
||||||
excel_temporal_serial(value, column_type.map(String::as_str), date_time_format)
|
excel_temporal_serial(value, column_type.map(String::as_str), date_time_format)
|
||||||
{
|
{
|
||||||
let reference = cell_ref(row_index, col_index);
|
|
||||||
let style = match temporal_kind {
|
let style = match temporal_kind {
|
||||||
ExcelTemporalKind::Date => XLSX_DATE_STYLE,
|
ExcelTemporalKind::Date => XLSX_DATE_STYLE,
|
||||||
ExcelTemporalKind::DateTime => XLSX_DATETIME_STYLE,
|
ExcelTemporalKind::DateTime => XLSX_DATETIME_STYLE,
|
||||||
};
|
};
|
||||||
return format!("<c r=\"{reference}\" s=\"{style}\"><v>{serial}</v></c>");
|
output.push_str("<c r=\"");
|
||||||
|
push_cell_ref(output, row_index, col_index);
|
||||||
|
write!(output, "\" s=\"{style}\"><v>{serial}</v></c>")
|
||||||
|
.expect("writing XLSX temporal cells into a String cannot fail");
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if is_numeric_column_type(column_type) {
|
if is_numeric_column_type(column_type) {
|
||||||
if let Some(Value::String(value)) = value {
|
if let Some(Value::String(value)) = value {
|
||||||
if let Some(number) = safe_excel_number(value) {
|
if let Some(number) = safe_excel_number(value) {
|
||||||
let reference = cell_ref(row_index, col_index);
|
output.push_str("<c r=\"");
|
||||||
let style_attr = style.map_or(String::new(), |style| format!(" s=\"{style}\""));
|
push_cell_ref(output, row_index, col_index);
|
||||||
return format!("<c r=\"{reference}\"{style_attr}><v>{number}</v></c>");
|
output.push('"');
|
||||||
|
push_cell_style(output, style);
|
||||||
|
write!(output, "><v>{number}</v></c>").expect("writing XLSX numeric cells into a String cannot fail");
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
cell_xml(value, row_index, col_index, style)
|
push_cell_xml(output, value, row_index, col_index, style);
|
||||||
}
|
}
|
||||||
|
|
||||||
fn write_worksheet_xml<W: Write>(writer: &mut W, segment: &WorksheetSegment) -> Result<(), String> {
|
fn write_worksheet_xml<W: Write>(writer: &mut W, segment: &WorksheetSegment) -> Result<(), String> {
|
||||||
@@ -718,21 +801,20 @@ fn write_worksheet_xml<W: Write>(writer: &mut W, segment: &WorksheetSegment) ->
|
|||||||
.write_all(header_row_xml(segment.columns, segment.column_comments).as_bytes())
|
.write_all(header_row_xml(segment.columns, segment.column_comments).as_bytes())
|
||||||
.map_err(|err| err.to_string())?;
|
.map_err(|err| err.to_string())?;
|
||||||
|
|
||||||
|
let mut row_buffer = String::with_capacity(segment.columns.len().saturating_mul(48));
|
||||||
for (row_index, row) in segment.rows.iter().enumerate() {
|
for (row_index, row) in segment.rows.iter().enumerate() {
|
||||||
let excel_row = row_index + 2;
|
let excel_row = row_index + 2;
|
||||||
writer
|
row_buffer.clear();
|
||||||
.write_all(
|
push_data_row_xml(
|
||||||
data_row_xml_with_date_time_format(
|
&mut row_buffer,
|
||||||
excel_row,
|
excel_row,
|
||||||
segment.columns,
|
segment.columns,
|
||||||
segment.column_types,
|
segment.column_types,
|
||||||
row,
|
row,
|
||||||
None,
|
None,
|
||||||
segment.numeric_column_right_align,
|
segment.numeric_column_right_align,
|
||||||
)
|
);
|
||||||
.as_bytes(),
|
writer.write_all(row_buffer.as_bytes()).map_err(|err| err.to_string())?;
|
||||||
)
|
|
||||||
.map_err(|err| err.to_string())?;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
writer
|
writer
|
||||||
@@ -1983,4 +2065,17 @@ mod tests {
|
|||||||
let sheet2 = read_zip_entry(&data, "xl/worksheets/sheet2.xml");
|
let sheet2 = read_zip_entry(&data, "xl/worksheets/sheet2.xml");
|
||||||
assert!(sheet2.contains("row_3"), "row 3 should be on sheet 2: {sheet2}");
|
assert!(sheet2.contains("row_3"), "row 3 should be on sheet 2: {sheet2}");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reusable_cell_buffer_matches_reference_xml() {
|
||||||
|
let values = [Value::Null, json!(true), json!(42.5), json!("a<&\"b"), json!({"key": "value"})];
|
||||||
|
for (index, value) in values.iter().enumerate() {
|
||||||
|
for style in [None, Some(1), Some(4)] {
|
||||||
|
let expected = super::cell_xml(Some(value), index, index + 1, style);
|
||||||
|
let mut actual = String::new();
|
||||||
|
super::push_cell_xml(&mut actual, Some(value), index, index + 1, style);
|
||||||
|
assert_eq!(actual, expected, "value={value:?}, style={style:?}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Reference in new issue
Block a user