fix(export): preserve numeric types in XLSX files

This commit is contained in:
t8y2 committed 2026-07-12 09:45:27 +08:00
1 parent c2b6aef315
commit be66d6f46b
9 files changed
+282 -45

No files matched your search

+64 -24
View File
@@ -350,6 +350,7 @@ async fn export_query_result_core_inner(
let mut xlsx = None;
let mut columns: Vec<String> = Vec::new();
let mut column_types: Vec<String> = Vec::new();
let mut rows_exported: u64 = 0;
let mut offset: usize = 0;
let mut wrote_csv_header = false;
@@ -476,6 +477,7 @@ async fn export_query_result_core_inner(
if columns.is_empty() {
columns = result.columns.clone();
column_types = result.column_types.clone();
}
let fetched_row_count = result.rows.len();
if xlsx_hard_limit_active {
@@ -504,7 +506,12 @@ async fn export_query_result_core_inner(
if xlsx.is_none() {
let xlsx_file =
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
xlsx = Some(start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?);
xlsx = Some(start_streaming_xlsx_workbook(
BufWriter::new(xlsx_file),
Some("Result"),
&columns,
&column_types,
)?);
}
if let Some(writer) = xlsx.as_mut() {
for row in &result.rows {
@@ -565,7 +572,7 @@ async fn export_query_result_core_inner(
buf.flush().map_err(|e| format!("Failed to flush XLSX file: {e}"))?;
} else {
let xlsx_file = File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
let writer = start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?;
let writer = start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns, &column_types)?;
let mut buf =
finish_streaming_xlsx_workbook(writer).map_err(|e| format!("Failed to finalize XLSX file: {e}"))?;
buf.flush().map_err(|e| format!("Failed to flush XLSX file: {e}"))?;
@@ -649,7 +656,7 @@ async fn try_export_postgres_query_result_stream(
cancel_context,
|item| {
match item {
crate::db::postgres::PostgresQueryStreamItem::Columns { columns: stream_columns, .. } => {
crate::db::postgres::PostgresQueryStreamItem::Columns { columns: stream_columns, column_types } => {
columns = stream_columns;
if let Some(file) = csv_file.as_mut() {
let csv = format_query_result_csv(&columns, &[]);
@@ -658,8 +665,12 @@ async fn try_export_postgres_query_result_stream(
} else {
let xlsx_file =
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
xlsx =
Some(start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?);
xlsx = Some(start_streaming_xlsx_workbook(
BufWriter::new(xlsx_file),
Some("Result"),
&columns,
&column_types,
)?);
}
}
crate::db::postgres::PostgresQueryStreamItem::Row(row) => {
@@ -674,8 +685,12 @@ async fn try_export_postgres_query_result_stream(
} else {
let xlsx_file =
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
xlsx =
Some(start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?);
xlsx = Some(start_streaming_xlsx_workbook(
BufWriter::new(xlsx_file),
Some("Result"),
&columns,
&[],
)?);
if let Some(writer) = xlsx.as_mut() {
writer.write_row(&row).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
}
@@ -712,7 +727,7 @@ async fn try_export_postgres_query_result_stream(
buf.flush().map_err(|e| format!("Failed to flush XLSX file: {e}"))?;
} else if format == "xlsx" {
let xlsx_file = File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
let writer = start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?;
let writer = start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns, &[])?;
let mut buf =
finish_streaming_xlsx_workbook(writer).map_err(|e| format!("Failed to finalize XLSX file: {e}"))?;
buf.flush().map_err(|e| format!("Failed to flush XLSX file: {e}"))?;
@@ -857,7 +872,7 @@ async fn try_export_mysql_query_result_stream(
return Err(canceled_error());
}
match item {
crate::db::mysql::MySqlQueryStreamItem::Columns { columns: stream_columns, .. } => {
crate::db::mysql::MySqlQueryStreamItem::Columns { columns: stream_columns, column_types } => {
columns = stream_columns;
if let Some(file) = csv_file.as_mut() {
let csv = format_query_result_csv(&columns, &[]);
@@ -866,8 +881,12 @@ async fn try_export_mysql_query_result_stream(
} else {
let xlsx_file =
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
xlsx =
Some(start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?);
xlsx = Some(start_streaming_xlsx_workbook(
BufWriter::new(xlsx_file),
Some("Result"),
&columns,
&column_types,
)?);
}
}
crate::db::mysql::MySqlQueryStreamItem::Row(row) => {
@@ -882,8 +901,12 @@ async fn try_export_mysql_query_result_stream(
} else {
let xlsx_file =
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
xlsx =
Some(start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?);
xlsx = Some(start_streaming_xlsx_workbook(
BufWriter::new(xlsx_file),
Some("Result"),
&columns,
&[],
)?);
if let Some(writer) = xlsx.as_mut() {
writer.write_row(&row).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
}
@@ -947,7 +970,7 @@ async fn try_export_mysql_query_result_stream(
buf.flush().map_err(|e| format!("Failed to flush XLSX file: {e}"))?;
} else if format == "xlsx" {
let xlsx_file = File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
let writer = start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?;
let writer = start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns, &[])?;
let mut buf =
finish_streaming_xlsx_workbook(writer).map_err(|e| format!("Failed to finalize XLSX file: {e}"))?;
buf.flush().map_err(|e| format!("Failed to flush XLSX file: {e}"))?;
@@ -1029,7 +1052,8 @@ async fn try_export_clickhouse_query_result_stream(
|item| {
match item {
crate::db::clickhouse_driver::ClickHouseQueryStreamItem::Columns {
columns: stream_columns, ..
columns: stream_columns,
column_types,
} => {
columns = stream_columns;
if let Some(file) = csv_file.as_mut() {
@@ -1039,8 +1063,12 @@ async fn try_export_clickhouse_query_result_stream(
} else {
let xlsx_file =
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
xlsx =
Some(start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?);
xlsx = Some(start_streaming_xlsx_workbook(
BufWriter::new(xlsx_file),
Some("Result"),
&columns,
&column_types,
)?);
}
}
crate::db::clickhouse_driver::ClickHouseQueryStreamItem::Row(row) => {
@@ -1055,8 +1083,12 @@ async fn try_export_clickhouse_query_result_stream(
} else {
let xlsx_file =
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
xlsx =
Some(start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?);
xlsx = Some(start_streaming_xlsx_workbook(
BufWriter::new(xlsx_file),
Some("Result"),
&columns,
&[],
)?);
if let Some(writer) = xlsx.as_mut() {
writer.write_row(&row).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
}
@@ -1115,7 +1147,7 @@ async fn try_export_clickhouse_query_result_stream(
buf.flush().map_err(|e| format!("Failed to flush XLSX file: {e}"))?;
} else if format == "xlsx" {
let xlsx_file = File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
let writer = start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?;
let writer = start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns, &[])?;
let mut buf =
finish_streaming_xlsx_workbook(writer).map_err(|e| format!("Failed to finalize XLSX file: {e}"))?;
buf.flush().map_err(|e| format!("Failed to flush XLSX file: {e}"))?;
@@ -1195,8 +1227,12 @@ async fn try_export_sqlserver_query_result_stream(
} else {
let xlsx_file =
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
xlsx =
Some(start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?);
xlsx = Some(start_streaming_xlsx_workbook(
BufWriter::new(xlsx_file),
Some("Result"),
&columns,
&[],
)?);
}
}
crate::db::sqlserver::SqlServerStreamItem::Row(row) => {
@@ -1211,8 +1247,12 @@ async fn try_export_sqlserver_query_result_stream(
} else {
let xlsx_file =
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
xlsx =
Some(start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?);
xlsx = Some(start_streaming_xlsx_workbook(
BufWriter::new(xlsx_file),
Some("Result"),
&columns,
&[],
)?);
if let Some(writer) = xlsx.as_mut() {
writer.write_row(row).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
}
+25 -4
View File
@@ -73,6 +73,16 @@ fn format_csv_rows(rows: &[Vec<Value>]) -> String {
.join("\n")
}
fn export_column_types(request: &TableExportRequest) -> Vec<String> {
request
.column_types
.as_deref()
.unwrap_or_default()
.iter()
.map(|column_type| column_type.clone().unwrap_or_default())
.collect()
}
fn write_json_row_object<W: Write>(writer: &mut W, columns: &[String], row: &[Value]) -> Result<(), String> {
writer.write_all(b"{\n").map_err(|e| format!("Failed to write JSON: {e}"))?;
let mut first = true;
@@ -492,10 +502,15 @@ async fn try_export_native_table_stream(
result
}
"xlsx" => {
let column_types = export_column_types(request);
let xlsx_file =
std::fs::File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
let mut writer =
start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some(&request.table_name), col_names)?;
let mut writer = start_streaming_xlsx_workbook(
BufWriter::new(xlsx_file),
Some(&request.table_name),
col_names,
&column_types,
)?;
let result = stream_native_table_rows(
state,
pool_key,
@@ -958,13 +973,18 @@ pub async fn export_table_data_core(
}
}
"xlsx" => {
let column_types = export_column_types(request);
// Create a dedicated file handle for the streaming XLSX writer
// instead of cloning the outer BufWriter's handle. This avoids
// sharing a file descriptor between two independent buffers.
let xlsx_file =
std::fs::File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
let mut writer =
start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some(&request.table_name), &col_names)?;
let mut writer = start_streaming_xlsx_workbook(
BufWriter::new(xlsx_file),
Some(&request.table_name),
&col_names,
&column_types,
)?;
loop {
// Check cancellation between batches
@@ -1490,6 +1510,7 @@ mod tests {
let data = XlsxWorksheetData {
sheet_name: Some("employees".to_string()),
columns: vec!["id".to_string(), "name".to_string(), "salary".to_string()],
column_types: vec![],
rows: vec![
vec![json!(1), json!("Alice"), json!(75000.50)],
vec![json!(2), json!("Bob"), json!(82000)],
+2
View File
@@ -1702,11 +1702,13 @@ mod tests {
XlsxWorksheetData {
sheet_name: Some("First".to_string()),
columns: vec!["id".to_string()],
column_types: vec![],
rows: vec![vec![serde_json::json!(1)]],
},
XlsxWorksheetData {
sheet_name: Some("Second".to_string()),
columns: vec!["name".to_string()],
column_types: vec![],
rows: vec![vec![serde_json::json!("Ada")]],
},
])
+122 -6
View File
@@ -7,6 +7,8 @@ use std::io::{Cursor, Seek, Write};
pub struct XlsxWorksheetData {
pub sheet_name: Option<String>,
pub columns: Vec<String>,
#[serde(default)]
pub column_types: Vec<String>,
pub rows: Vec<Vec<Value>>,
}
@@ -16,6 +18,7 @@ pub struct XlsxWorksheetData {
pub struct StreamingXlsxWriter<W: Write + Seek> {
zip: zip::ZipWriter<W>,
columns: Vec<String>,
column_types: Vec<String>,
next_row_number: usize,
}
@@ -50,11 +53,13 @@ pub(crate) fn header_row_xml(columns: &[String]) -> String {
}
/// Build a single `<row>` XML fragment for a data row.
pub(crate) fn data_row_xml(row_number: usize, columns: &[String], row: &[Value]) -> String {
pub(crate) fn data_row_xml(row_number: usize, columns: &[String], column_types: &[String], row: &[Value]) -> String {
let cells = columns
.iter()
.enumerate()
.map(|(col_index, _)| cell_xml(row.get(col_index), row_number - 1, col_index, None))
.map(|(col_index, _)| {
typed_cell_xml(row.get(col_index), column_types.get(col_index), row_number - 1, col_index, None)
})
.collect::<String>();
format!("<row r=\"{row_number}\">{cells}</row>")
}
@@ -79,6 +84,7 @@ pub(crate) fn start_streaming_xlsx_workbook<W: Write + Seek>(
writer: W,
sheet_name: Option<&str>,
columns: &[String],
column_types: &[String],
) -> Result<StreamingXlsxWriter<W>, String> {
let sheet_name = normalize_sheet_name(sheet_name);
let widths = estimate_header_widths(columns);
@@ -111,14 +117,14 @@ pub(crate) fn start_streaming_xlsx_workbook<W: Write + Seek>(
zip.write_all(sheet_header.as_bytes()).map_err(|err| err.to_string())?;
zip.write_all(header_row_xml(columns).as_bytes()).map_err(|err| err.to_string())?;
Ok(StreamingXlsxWriter { zip, columns: columns.to_vec(), next_row_number: 2 })
Ok(StreamingXlsxWriter { zip, columns: columns.to_vec(), column_types: column_types.to_vec(), next_row_number: 2 })
}
impl<W: Write + Seek> StreamingXlsxWriter<W> {
/// Append a single data row to the worksheet.
pub fn write_row(&mut self, row: &[Value]) -> Result<(), String> {
self.zip
.write_all(data_row_xml(self.next_row_number, &self.columns, row).as_bytes())
.write_all(data_row_xml(self.next_row_number, &self.columns, &self.column_types, row).as_bytes())
.map_err(|err| err.to_string())?;
self.next_row_number += 1;
Ok(())
@@ -248,6 +254,79 @@ fn cell_xml(value: Option<&Value>, row_index: usize, col_index: usize, style: Op
}
}
fn is_numeric_column_type(column_type: Option<&String>) -> bool {
let normalized = column_type.map(|value| value.trim().to_ascii_lowercase()).unwrap_or_default();
let base = normalized.split(['(', ' ', '[']).next().unwrap_or_default();
matches!(
base,
"bit"
| "tinyint"
| "smallint"
| "mediumint"
| "int"
| "integer"
| "bigint"
| "int2"
| "int4"
| "int8"
| "uint8"
| "uint16"
| "uint32"
| "uint64"
| "uint128"
| "uint256"
| "float"
| "float4"
| "float8"
| "float32"
| "float64"
| "real"
| "double"
| "decimal"
| "numeric"
| "number"
| "money"
| "smallmoney"
)
}
fn safe_excel_number(value: &str) -> Option<&str> {
let trimmed = value.trim();
if trimmed.is_empty() || trimmed.parse::<f64>().ok().is_none_or(|number| !number.is_finite()) {
return None;
}
// Excel stores numbers with at most 15 significant decimal digits. Keep
// higher-precision database values as text rather than silently rounding.
let significant_digits = trimmed
.split(['e', 'E'])
.next()
.unwrap_or(trimmed)
.chars()
.filter(|ch| ch.is_ascii_digit())
.skip_while(|ch| *ch == '0')
.count();
(significant_digits <= 15).then_some(trimmed)
}
fn typed_cell_xml(
value: Option<&Value>,
column_type: Option<&String>,
row_index: usize,
col_index: usize,
style: Option<usize>,
) -> String {
if is_numeric_column_type(column_type) {
if let Some(Value::String(value)) = value {
if let Some(number) = safe_excel_number(value) {
let reference = cell_ref(row_index, col_index);
let style_attr = style.map_or(String::new(), |style| format!(" s=\"{style}\""));
return format!("<c r=\"{reference}\"{style_attr}><v>{number}</v></c>");
}
}
}
cell_xml(value, row_index, col_index, style)
}
fn worksheet_xml(data: &XlsxWorksheetData) -> String {
let total_rows = data.rows.len() + 1;
let range = sheet_range(data.columns.len(), total_rows);
@@ -280,7 +359,9 @@ fn worksheet_xml(data: &XlsxWorksheetData) -> String {
.columns
.iter()
.enumerate()
.map(|(col_index, _)| cell_xml(row.get(col_index), excel_row - 1, col_index, None))
.map(|(col_index, _)| {
typed_cell_xml(row.get(col_index), data.column_types.get(col_index), excel_row - 1, col_index, None)
})
.collect::<String>();
format!("<row r=\"{excel_row}\">{cells}</row>")
})
@@ -492,6 +573,7 @@ mod tests {
let workbook = build_xlsx_workbook(&XlsxWorksheetData {
sheet_name: Some("Users".to_string()),
columns: vec!["id".to_string(), "name".to_string(), "active".to_string()],
column_types: vec![],
rows: vec![vec![json!(1), json!("Ada & Bob"), json!(true)], vec![json!(2), json!(null), json!(false)]],
})
.expect("build workbook");
@@ -510,11 +592,43 @@ mod tests {
assert_all_entries_deflated(&workbook);
}
#[test]
fn writes_safe_numeric_strings_as_numbers_for_numeric_columns() {
let workbook = build_xlsx_workbook(&XlsxWorksheetData {
sheet_name: Some("Amounts".to_string()),
columns: vec!["quantity".to_string(), "amount".to_string(), "code".to_string()],
column_types: vec!["decimal(10,5)".to_string(), "numeric".to_string(), "varchar".to_string()],
rows: vec![vec![json!("1.00000"), json!("2800.000000"), json!("00123")]],
})
.expect("build workbook");
let sheet = read_zip_entry(&workbook, "xl/worksheets/sheet1.xml");
assert!(sheet.contains("<c r=\"A2\"><v>1.00000</v></c>"));
assert!(sheet.contains("<c r=\"B2\"><v>2800.000000</v></c>"));
assert!(sheet.contains("<c r=\"C2\" t=\"inlineStr\"><is><t>00123</t></is></c>"));
}
#[test]
fn preserves_high_precision_numeric_strings_as_text() {
let workbook = build_xlsx_workbook(&XlsxWorksheetData {
sheet_name: Some("Precision".to_string()),
columns: vec!["large_id".to_string(), "precise_amount".to_string()],
column_types: vec!["bigint".to_string(), "decimal(30,10)".to_string()],
rows: vec![vec![json!("9223372036854775807"), json!("123456789012345.6789000000")]],
})
.expect("build workbook");
let sheet = read_zip_entry(&workbook, "xl/worksheets/sheet1.xml");
assert!(sheet.contains("t=\"inlineStr\"") && sheet.contains("9223372036854775807"));
assert!(sheet.contains("123456789012345.6789000000"));
}
#[test]
fn sanitizes_invalid_sheet_name() {
let workbook = build_xlsx_workbook(&XlsxWorksheetData {
sheet_name: Some("bad/name:with*chars?and-a-very-long-tail".to_string()),
columns: vec!["value".to_string()],
column_types: vec![],
rows: vec![vec![json!("ok")]],
})
.expect("build workbook");
@@ -529,11 +643,13 @@ mod tests {
XlsxWorksheetData {
sheet_name: Some("Result 1".to_string()),
columns: vec!["id".to_string()],
column_types: vec![],
rows: vec![vec![json!(1)]],
},
XlsxWorksheetData {
sheet_name: Some("Result 2".to_string()),
columns: vec!["name".to_string()],
column_types: vec![],
rows: vec![vec![json!("Ada")]],
},
])
@@ -556,7 +672,7 @@ mod tests {
{
let file = fs::File::create(&path).expect("create temp xlsx");
let mut writer =
start_streaming_xlsx_workbook(file, Some("Streamed"), &["id".to_string(), "name".to_string()])
start_streaming_xlsx_workbook(file, Some("Streamed"), &["id".to_string(), "name".to_string()], &[])
.expect("start workbook");
writer.write_row(&[json!(1), json!("Ada")]).expect("write row");
writer.write_row(&[json!(2), json!("Bob")]).expect("write row");