fix(export): preserve CSV null semantics

This commit is contained in:
二丫讲梵 authored and GitHub committed 2026-07-14 01:24:10 +08:00
1 parent 65e1100db8
commit 8e87ab2e39
6 files changed
+142 -34

No files matched your search

+36 -27
View File
@@ -41,10 +41,10 @@ pub(crate) fn value_to_csv_text(value: &Value) -> String {
}
}
pub(crate) fn value_to_query_result_csv_text(value: &Value) -> String {
fn format_csv_value(value: &Value) -> String {
match value {
Value::Null => "NULL".to_string(),
other => value_to_csv_text(other),
Value::Null => String::new(),
other => escape_csv(&value_to_csv_text(other)),
}
}
@@ -69,41 +69,32 @@ pub(crate) fn format_tsv(columns: &[String], rows: &[Vec<Value>]) -> String {
format!("{header}\n{body}")
}
/// Format query-result rows as CSV text without a header row, using the
/// query-result NULL semantics (NULL → "NULL"). Used by the streaming
/// query-result export for batches after the first.
/// Format query-result rows as CSV text without a header row. Database NULLs
/// use the same empty-cell representation as table-data exports. Used by the
/// streaming query-result export for batches after the first.
pub fn format_query_result_csv_rows(rows: &[Vec<Value>]) -> String {
rows.iter()
.map(|row| {
row.iter().map(|cell| escape_csv(&value_to_query_result_csv_text(cell))).collect::<Vec<_>>().join(",")
})
.collect::<Vec<_>>()
.join("\n")
rows.iter().map(|row| row.iter().map(format_csv_value).collect::<Vec<_>>().join(",")).collect::<Vec<_>>().join("\n")
}
fn format_csv_with_value_formatter(
columns: &[String],
rows: &[Vec<Value>],
value_formatter: fn(&Value) -> String,
) -> String {
fn format_csv_with_value_formatter(columns: &[String], rows: &[Vec<Value>]) -> String {
let header = columns.iter().map(|col| escape_csv(col)).collect::<Vec<_>>().join(",");
let body = rows
.iter()
.map(|row| row.iter().map(|cell| escape_csv(&value_formatter(cell))).collect::<Vec<_>>().join(","))
.map(|row| row.iter().map(format_csv_value).collect::<Vec<_>>().join(","))
.collect::<Vec<_>>()
.join("\n");
format!("{header}\n{body}")
}
pub fn format_csv(columns: &[String], rows: &[Vec<Value>]) -> String {
format_csv_with_value_formatter(columns, rows, value_to_csv_text)
format_csv_with_value_formatter(columns, rows)
}
pub fn format_query_result_csv(columns: &[String], rows: &[Vec<Value>]) -> String {
format_csv_with_value_formatter(columns, rows, value_to_query_result_csv_text)
format_csv(columns, rows)
}
fn write_csv_row(writer: &mut impl Write, values: impl IntoIterator<Item = String>) -> Result<(), String> {
fn write_csv_text_row(writer: &mut impl Write, values: impl IntoIterator<Item = String>) -> Result<(), String> {
let mut first = true;
for value in values {
if !first {
@@ -115,6 +106,18 @@ fn write_csv_row(writer: &mut impl Write, values: impl IntoIterator<Item = Strin
Ok(())
}
fn write_csv_value_row(writer: &mut impl Write, values: impl IntoIterator<Item = Value>) -> Result<(), String> {
let mut first = true;
for value in values {
if !first {
writer.write_all(b",").map_err(|err| err.to_string())?;
}
first = false;
writer.write_all(format_csv_value(&value).as_bytes()).map_err(|err| err.to_string())?;
}
Ok(())
}
async fn connection_database_type(state: &AppState, connection_id: &str) -> Result<DatabaseType, String> {
state
.configs
@@ -168,7 +171,7 @@ pub async fn export_table_data_csv_core(state: &AppState, options: TableCsvExpor
.await?;
if !wrote_header {
write_csv_row(&mut writer, result.columns)?;
write_csv_text_row(&mut writer, result.columns)?;
wrote_header = true;
}
@@ -178,7 +181,7 @@ pub async fn export_table_data_csv_core(state: &AppState, options: TableCsvExpor
}
for row in result.rows {
writer.write_all(b"\n").map_err(|err| err.to_string())?;
write_csv_row(&mut writer, row.iter().map(value_to_csv_text))?;
write_csv_value_row(&mut writer, row)?;
}
rows_exported += fetched as u64;
@@ -197,7 +200,7 @@ pub async fn export_table_data_csv_core(state: &AppState, options: TableCsvExpor
#[cfg(test)]
mod tests {
use super::{format_csv, format_query_result_csv, format_tsv};
use super::{format_csv, format_query_result_csv, format_query_result_csv_rows, format_tsv};
use serde_json::json;
#[test]
@@ -209,13 +212,19 @@ mod tests {
#[test]
fn formats_null_as_empty_cell() {
let out = format_csv(&["id".to_string(), "note".to_string()], &[vec![json!(1), Value::Null]]);
assert_eq!(out, "\"id\",\"note\"\n\"1\",\"\"");
assert_eq!(out, "\"id\",\"note\"\n\"1\",");
}
#[test]
fn formats_query_result_null_as_null_text() {
fn formats_query_result_null_as_empty_cell() {
let out = format_query_result_csv(&["id".to_string(), "note".to_string()], &[vec![json!(1), Value::Null]]);
assert_eq!(out, "\"id\",\"note\"\n\"1\",\"NULL\"");
assert_eq!(out, "\"id\",\"note\"\n\"1\",");
}
#[test]
fn formats_streamed_query_result_null_as_empty_cell_and_preserves_literal_null() {
let out = format_query_result_csv_rows(&[vec![Value::Null, json!("NULL"), json!("")]]);
assert_eq!(out, ",\"NULL\",\"\"");
}
#[test]
+73
View File
@@ -608,6 +608,58 @@ mod tests {
assert!(sheet.contains("<c r=\"C2\" t=\"inlineStr\"><is><t>00123</t></is></c>"));
}
#[test]
fn writes_mysql_57_numeric_strings_as_numeric_cells() {
let workbook = build_xlsx_workbook(&XlsxWorksheetData {
sheet_name: Some("MySQL 5.7".to_string()),
columns: vec![
"id".to_string(),
"nullable_int".to_string(),
"tinyint_value".to_string(),
"unsigned_int_value".to_string(),
"bigint_safe".to_string(),
"float_value".to_string(),
"double_value".to_string(),
"decimal_value".to_string(),
],
column_types: vec![
"bigint".to_string(),
"int(11)".to_string(),
"tinyint(4)".to_string(),
"int(10) unsigned".to_string(),
"bigint(20)".to_string(),
"float".to_string(),
"double".to_string(),
"decimal(18,6)".to_string(),
],
rows: vec![vec![
json!("2"),
json!("42"),
json!("-7"),
json!("4000000000"),
json!("123456789012345"),
json!("123.5"),
json!("987654.321"),
json!("2800.000000"),
]],
})
.expect("build workbook");
let sheet = read_zip_entry(&workbook, "xl/worksheets/sheet1.xml");
for (reference, value) in [
("A2", "2"),
("B2", "42"),
("C2", "-7"),
("D2", "4000000000"),
("E2", "123456789012345"),
("F2", "123.5"),
("G2", "987654.321"),
("H2", "2800.000000"),
] {
assert!(sheet.contains(&format!("<c r=\"{reference}\"><v>{value}</v></c>")), "sheet={sheet}");
}
}
#[test]
fn preserves_high_precision_numeric_strings_as_text() {
let workbook = build_xlsx_workbook(&XlsxWorksheetData {
@@ -686,4 +738,25 @@ mod tests {
assert_eq!(range.get_value((2, 1)).expect("row2"), &calamine::Data::String("Bob".to_string()));
let _ = fs::remove_file(&path);
}
#[test]
fn streams_mysql_numeric_strings_as_numeric_cells() {
let path = std::env::temp_dir().join(format!("dbx-mysql-stream-test-{}.xlsx", uuid::Uuid::new_v4()));
{
let file = fs::File::create(&path).expect("create temp xlsx");
let columns = ["nullable_int".to_string(), "float_value".to_string(), "decimal_value".to_string()];
let column_types = ["int(11)".to_string(), "float".to_string(), "decimal(18,6)".to_string()];
let mut writer = start_streaming_xlsx_workbook(file, Some("MySQL Stream"), &columns, &column_types)
.expect("start workbook");
writer.write_row(&[json!("42"), json!("123.5"), json!("2800.000000")]).expect("write row");
drop(writer.finish().expect("finish workbook"));
}
let bytes = fs::read(&path).expect("read workbook");
let sheet = read_zip_entry(&bytes, "xl/worksheets/sheet1.xml");
assert!(sheet.contains("<c r=\"A2\"><v>42</v></c>"));
assert!(sheet.contains("<c r=\"B2\"><v>123.5</v></c>"));
assert!(sheet.contains("<c r=\"C2\"><v>2800.000000</v></c>"));
let _ = fs::remove_file(&path);
}
}