fix(export): preserve CSV null semantics
This commit is contained in:
6 files changed
+142
-34
No files matched your search
@@ -6,7 +6,7 @@ export type ExportCellValue = string | number | boolean | null;
|
|||||||
export function formatCsv(columns: string[], rows: ExportCellValue[][]): string {
|
export function formatCsv(columns: string[], rows: ExportCellValue[][]): string {
|
||||||
const esc = (v: string) => `"${v.replace(/"/g, '""')}"`;
|
const esc = (v: string) => `"${v.replace(/"/g, '""')}"`;
|
||||||
const header = columns.map(esc).join(",");
|
const header = columns.map(esc).join(",");
|
||||||
const body = rows.map((row) => row.map((c) => esc(c === null ? "NULL" : String(c))).join(",")).join("\n");
|
const body = rows.map((row) => row.map((c) => (c === null ? "" : esc(String(c)))).join(",")).join("\n");
|
||||||
return `${header}\n${body}`;
|
return `${header}\n${body}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -41,10 +41,10 @@ pub(crate) fn value_to_csv_text(value: &Value) -> String {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn value_to_query_result_csv_text(value: &Value) -> String {
|
fn format_csv_value(value: &Value) -> String {
|
||||||
match value {
|
match value {
|
||||||
Value::Null => "NULL".to_string(),
|
Value::Null => String::new(),
|
||||||
other => value_to_csv_text(other),
|
other => escape_csv(&value_to_csv_text(other)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -69,41 +69,32 @@ pub(crate) fn format_tsv(columns: &[String], rows: &[Vec<Value>]) -> String {
|
|||||||
format!("{header}\n{body}")
|
format!("{header}\n{body}")
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Format query-result rows as CSV text without a header row, using the
|
/// Format query-result rows as CSV text without a header row. Database NULLs
|
||||||
/// query-result NULL semantics (NULL → "NULL"). Used by the streaming
|
/// use the same empty-cell representation as table-data exports. Used by the
|
||||||
/// query-result export for batches after the first.
|
/// streaming query-result export for batches after the first.
|
||||||
pub fn format_query_result_csv_rows(rows: &[Vec<Value>]) -> String {
|
pub fn format_query_result_csv_rows(rows: &[Vec<Value>]) -> String {
|
||||||
rows.iter()
|
rows.iter().map(|row| row.iter().map(format_csv_value).collect::<Vec<_>>().join(",")).collect::<Vec<_>>().join("\n")
|
||||||
.map(|row| {
|
|
||||||
row.iter().map(|cell| escape_csv(&value_to_query_result_csv_text(cell))).collect::<Vec<_>>().join(",")
|
|
||||||
})
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
.join("\n")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn format_csv_with_value_formatter(
|
fn format_csv_with_value_formatter(columns: &[String], rows: &[Vec<Value>]) -> String {
|
||||||
columns: &[String],
|
|
||||||
rows: &[Vec<Value>],
|
|
||||||
value_formatter: fn(&Value) -> String,
|
|
||||||
) -> String {
|
|
||||||
let header = columns.iter().map(|col| escape_csv(col)).collect::<Vec<_>>().join(",");
|
let header = columns.iter().map(|col| escape_csv(col)).collect::<Vec<_>>().join(",");
|
||||||
let body = rows
|
let body = rows
|
||||||
.iter()
|
.iter()
|
||||||
.map(|row| row.iter().map(|cell| escape_csv(&value_formatter(cell))).collect::<Vec<_>>().join(","))
|
.map(|row| row.iter().map(format_csv_value).collect::<Vec<_>>().join(","))
|
||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
.join("\n");
|
.join("\n");
|
||||||
format!("{header}\n{body}")
|
format!("{header}\n{body}")
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn format_csv(columns: &[String], rows: &[Vec<Value>]) -> String {
|
pub fn format_csv(columns: &[String], rows: &[Vec<Value>]) -> String {
|
||||||
format_csv_with_value_formatter(columns, rows, value_to_csv_text)
|
format_csv_with_value_formatter(columns, rows)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn format_query_result_csv(columns: &[String], rows: &[Vec<Value>]) -> String {
|
pub fn format_query_result_csv(columns: &[String], rows: &[Vec<Value>]) -> String {
|
||||||
format_csv_with_value_formatter(columns, rows, value_to_query_result_csv_text)
|
format_csv(columns, rows)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn write_csv_row(writer: &mut impl Write, values: impl IntoIterator<Item = String>) -> Result<(), String> {
|
fn write_csv_text_row(writer: &mut impl Write, values: impl IntoIterator<Item = String>) -> Result<(), String> {
|
||||||
let mut first = true;
|
let mut first = true;
|
||||||
for value in values {
|
for value in values {
|
||||||
if !first {
|
if !first {
|
||||||
@@ -115,6 +106,18 @@ fn write_csv_row(writer: &mut impl Write, values: impl IntoIterator<Item = Strin
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn write_csv_value_row(writer: &mut impl Write, values: impl IntoIterator<Item = Value>) -> Result<(), String> {
|
||||||
|
let mut first = true;
|
||||||
|
for value in values {
|
||||||
|
if !first {
|
||||||
|
writer.write_all(b",").map_err(|err| err.to_string())?;
|
||||||
|
}
|
||||||
|
first = false;
|
||||||
|
writer.write_all(format_csv_value(&value).as_bytes()).map_err(|err| err.to_string())?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
async fn connection_database_type(state: &AppState, connection_id: &str) -> Result<DatabaseType, String> {
|
async fn connection_database_type(state: &AppState, connection_id: &str) -> Result<DatabaseType, String> {
|
||||||
state
|
state
|
||||||
.configs
|
.configs
|
||||||
@@ -168,7 +171,7 @@ pub async fn export_table_data_csv_core(state: &AppState, options: TableCsvExpor
|
|||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
if !wrote_header {
|
if !wrote_header {
|
||||||
write_csv_row(&mut writer, result.columns)?;
|
write_csv_text_row(&mut writer, result.columns)?;
|
||||||
wrote_header = true;
|
wrote_header = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -178,7 +181,7 @@ pub async fn export_table_data_csv_core(state: &AppState, options: TableCsvExpor
|
|||||||
}
|
}
|
||||||
for row in result.rows {
|
for row in result.rows {
|
||||||
writer.write_all(b"\n").map_err(|err| err.to_string())?;
|
writer.write_all(b"\n").map_err(|err| err.to_string())?;
|
||||||
write_csv_row(&mut writer, row.iter().map(value_to_csv_text))?;
|
write_csv_value_row(&mut writer, row)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
rows_exported += fetched as u64;
|
rows_exported += fetched as u64;
|
||||||
@@ -197,7 +200,7 @@ pub async fn export_table_data_csv_core(state: &AppState, options: TableCsvExpor
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{format_csv, format_query_result_csv, format_tsv};
|
use super::{format_csv, format_query_result_csv, format_query_result_csv_rows, format_tsv};
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -209,13 +212,19 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn formats_null_as_empty_cell() {
|
fn formats_null_as_empty_cell() {
|
||||||
let out = format_csv(&["id".to_string(), "note".to_string()], &[vec![json!(1), Value::Null]]);
|
let out = format_csv(&["id".to_string(), "note".to_string()], &[vec![json!(1), Value::Null]]);
|
||||||
assert_eq!(out, "\"id\",\"note\"\n\"1\",\"\"");
|
assert_eq!(out, "\"id\",\"note\"\n\"1\",");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn formats_query_result_null_as_null_text() {
|
fn formats_query_result_null_as_empty_cell() {
|
||||||
let out = format_query_result_csv(&["id".to_string(), "note".to_string()], &[vec![json!(1), Value::Null]]);
|
let out = format_query_result_csv(&["id".to_string(), "note".to_string()], &[vec![json!(1), Value::Null]]);
|
||||||
assert_eq!(out, "\"id\",\"note\"\n\"1\",\"NULL\"");
|
assert_eq!(out, "\"id\",\"note\"\n\"1\",");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn formats_streamed_query_result_null_as_empty_cell_and_preserves_literal_null() {
|
||||||
|
let out = format_query_result_csv_rows(&[vec![Value::Null, json!("NULL"), json!("")]]);
|
||||||
|
assert_eq!(out, ",\"NULL\",\"\"");
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -608,6 +608,58 @@ mod tests {
|
|||||||
assert!(sheet.contains("<c r=\"C2\" t=\"inlineStr\"><is><t>00123</t></is></c>"));
|
assert!(sheet.contains("<c r=\"C2\" t=\"inlineStr\"><is><t>00123</t></is></c>"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn writes_mysql_57_numeric_strings_as_numeric_cells() {
|
||||||
|
let workbook = build_xlsx_workbook(&XlsxWorksheetData {
|
||||||
|
sheet_name: Some("MySQL 5.7".to_string()),
|
||||||
|
columns: vec![
|
||||||
|
"id".to_string(),
|
||||||
|
"nullable_int".to_string(),
|
||||||
|
"tinyint_value".to_string(),
|
||||||
|
"unsigned_int_value".to_string(),
|
||||||
|
"bigint_safe".to_string(),
|
||||||
|
"float_value".to_string(),
|
||||||
|
"double_value".to_string(),
|
||||||
|
"decimal_value".to_string(),
|
||||||
|
],
|
||||||
|
column_types: vec![
|
||||||
|
"bigint".to_string(),
|
||||||
|
"int(11)".to_string(),
|
||||||
|
"tinyint(4)".to_string(),
|
||||||
|
"int(10) unsigned".to_string(),
|
||||||
|
"bigint(20)".to_string(),
|
||||||
|
"float".to_string(),
|
||||||
|
"double".to_string(),
|
||||||
|
"decimal(18,6)".to_string(),
|
||||||
|
],
|
||||||
|
rows: vec![vec![
|
||||||
|
json!("2"),
|
||||||
|
json!("42"),
|
||||||
|
json!("-7"),
|
||||||
|
json!("4000000000"),
|
||||||
|
json!("123456789012345"),
|
||||||
|
json!("123.5"),
|
||||||
|
json!("987654.321"),
|
||||||
|
json!("2800.000000"),
|
||||||
|
]],
|
||||||
|
})
|
||||||
|
.expect("build workbook");
|
||||||
|
|
||||||
|
let sheet = read_zip_entry(&workbook, "xl/worksheets/sheet1.xml");
|
||||||
|
for (reference, value) in [
|
||||||
|
("A2", "2"),
|
||||||
|
("B2", "42"),
|
||||||
|
("C2", "-7"),
|
||||||
|
("D2", "4000000000"),
|
||||||
|
("E2", "123456789012345"),
|
||||||
|
("F2", "123.5"),
|
||||||
|
("G2", "987654.321"),
|
||||||
|
("H2", "2800.000000"),
|
||||||
|
] {
|
||||||
|
assert!(sheet.contains(&format!("<c r=\"{reference}\"><v>{value}</v></c>")), "sheet={sheet}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn preserves_high_precision_numeric_strings_as_text() {
|
fn preserves_high_precision_numeric_strings_as_text() {
|
||||||
let workbook = build_xlsx_workbook(&XlsxWorksheetData {
|
let workbook = build_xlsx_workbook(&XlsxWorksheetData {
|
||||||
@@ -686,4 +738,25 @@ mod tests {
|
|||||||
assert_eq!(range.get_value((2, 1)).expect("row2"), &calamine::Data::String("Bob".to_string()));
|
assert_eq!(range.get_value((2, 1)).expect("row2"), &calamine::Data::String("Bob".to_string()));
|
||||||
let _ = fs::remove_file(&path);
|
let _ = fs::remove_file(&path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn streams_mysql_numeric_strings_as_numeric_cells() {
|
||||||
|
let path = std::env::temp_dir().join(format!("dbx-mysql-stream-test-{}.xlsx", uuid::Uuid::new_v4()));
|
||||||
|
{
|
||||||
|
let file = fs::File::create(&path).expect("create temp xlsx");
|
||||||
|
let columns = ["nullable_int".to_string(), "float_value".to_string(), "decimal_value".to_string()];
|
||||||
|
let column_types = ["int(11)".to_string(), "float".to_string(), "decimal(18,6)".to_string()];
|
||||||
|
let mut writer = start_streaming_xlsx_workbook(file, Some("MySQL Stream"), &columns, &column_types)
|
||||||
|
.expect("start workbook");
|
||||||
|
writer.write_row(&[json!("42"), json!("123.5"), json!("2800.000000")]).expect("write row");
|
||||||
|
drop(writer.finish().expect("finish workbook"));
|
||||||
|
}
|
||||||
|
|
||||||
|
let bytes = fs::read(&path).expect("read workbook");
|
||||||
|
let sheet = read_zip_entry(&bytes, "xl/worksheets/sheet1.xml");
|
||||||
|
assert!(sheet.contains("<c r=\"A2\"><v>42</v></c>"));
|
||||||
|
assert!(sheet.contains("<c r=\"B2\"><v>123.5</v></c>"));
|
||||||
|
assert!(sheet.contains("<c r=\"C2\"><v>2800.000000</v></c>"));
|
||||||
|
let _ = fs::remove_file(&path);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
+12
-2
@@ -2,6 +2,16 @@ import { strict as assert } from "node:assert";
|
|||||||
import { test } from "vitest";
|
import { test } from "vitest";
|
||||||
import { formatCsv } from "../../apps/desktop/src/lib/export/exportFormats.ts";
|
import { formatCsv } from "../../apps/desktop/src/lib/export/exportFormats.ts";
|
||||||
|
|
||||||
test("formatCsv preserves query result null display text", () => {
|
test("formatCsv writes database null as an empty cell and preserves literal NULL", () => {
|
||||||
assert.equal(formatCsv(["id", "note"], [[1, null]]), "\"id\",\"note\"\n\"1\",\"NULL\"");
|
assert.equal(
|
||||||
|
formatCsv(
|
||||||
|
["id", "note"],
|
||||||
|
[
|
||||||
|
[1, null],
|
||||||
|
[2, ""],
|
||||||
|
[3, "NULL"],
|
||||||
|
],
|
||||||
|
),
|
||||||
|
'"id","note"\n"1",\n"2",""\n"3","NULL"',
|
||||||
|
);
|
||||||
});
|
});
|
||||||
+4
-4
@@ -47,7 +47,7 @@ function installMemoryStorage() {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function buildExportHarness(options: { currentResultLabel?: string; exportFileBaseName?: string } = {}) {
|
function buildExportHarness(options: { currentResultLabel?: string; exportFileBaseName?: string; columnTypes?: Array<string | undefined> } = {}) {
|
||||||
const exportProgressDialog = ref(false);
|
const exportProgressDialog = ref(false);
|
||||||
const exportProgressState = ref({
|
const exportProgressState = ref({
|
||||||
title: "",
|
title: "",
|
||||||
@@ -95,7 +95,7 @@ function buildExportHarness(options: { currentResultLabel?: string; exportFileBa
|
|||||||
database: computed(() => "db"),
|
database: computed(() => "db"),
|
||||||
context: computed(() => "results"),
|
context: computed(() => "results"),
|
||||||
sourceColumns: computed(() => undefined),
|
sourceColumns: computed(() => undefined),
|
||||||
columnTypes: computed(() => undefined),
|
columnTypes: computed(() => options.columnTypes),
|
||||||
whereInput: computed(() => undefined),
|
whereInput: computed(() => undefined),
|
||||||
orderBy: computed(() => undefined),
|
orderBy: computed(() => undefined),
|
||||||
exportBatchSize: computed(() => 1000),
|
exportBatchSize: computed(() => 1000),
|
||||||
@@ -725,7 +725,7 @@ test("selected query result CSV export keeps the existing in-memory path", async
|
|||||||
});
|
});
|
||||||
|
|
||||||
test("selected query result XLSX export uses the current source label as the sheet name", async () => {
|
test("selected query result XLSX export uses the current source label as the sheet name", async () => {
|
||||||
const { composable, queryResultExportRequest } = buildExportHarness({ currentResultLabel: "aaa.apis" });
|
const { composable, queryResultExportRequest } = buildExportHarness({ currentResultLabel: "aaa.apis", columnTypes: ["bigint(20)", "varchar(64)"] });
|
||||||
|
|
||||||
await composable.exportXlsx([1]);
|
await composable.exportXlsx([1]);
|
||||||
|
|
||||||
@@ -734,7 +734,7 @@ test("selected query result XLSX export uses the current source label as the she
|
|||||||
assert.equal(apiMock.exportQueryResultXlsx.mock.calls.length, 1);
|
assert.equal(apiMock.exportQueryResultXlsx.mock.calls.length, 1);
|
||||||
assert.equal(apiMock.exportQueryResultXlsx.mock.calls[0][1], "aaa.apis");
|
assert.equal(apiMock.exportQueryResultXlsx.mock.calls[0][1], "aaa.apis");
|
||||||
assert.deepEqual(apiMock.exportQueryResultXlsx.mock.calls[0][2], ["id", "name"]);
|
assert.deepEqual(apiMock.exportQueryResultXlsx.mock.calls[0][2], ["id", "name"]);
|
||||||
assert.deepEqual(apiMock.exportQueryResultXlsx.mock.calls[0][3], []);
|
assert.deepEqual(apiMock.exportQueryResultXlsx.mock.calls[0][3], ["bigint(20)", "varchar(64)"]);
|
||||||
assert.deepEqual(apiMock.exportQueryResultXlsx.mock.calls[0][4], [[1, "Ada"]]);
|
assert.deepEqual(apiMock.exportQueryResultXlsx.mock.calls[0][4], [[1, "Ada"]]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
+16
@@ -33,3 +33,19 @@ test("sanitizes invalid sheet names", () => {
|
|||||||
|
|
||||||
assert.match(text, /name="bad name with chars and-a-very-"/);
|
assert.match(text, /name="bad name with chars and-a-very-"/);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("writes MySQL 5.7 numeric strings as numeric cells", () => {
|
||||||
|
const workbook = buildXlsxWorkbook({
|
||||||
|
sheetName: "MySQL 5.7",
|
||||||
|
columns: ["nullable_int", "float_value", "double_value", "decimal_value", "bigint_high_precision"],
|
||||||
|
columnTypes: ["int(11)", "float", "double", "decimal(18,6)", "bigint(20)"],
|
||||||
|
rows: [["42", "123.5", "987654.321", "2800.000000", "9007199254740992"]],
|
||||||
|
});
|
||||||
|
const text = new TextDecoder().decode(workbook);
|
||||||
|
|
||||||
|
assert.match(text, /<c r="A2"><v>42<\/v><\/c>/);
|
||||||
|
assert.match(text, /<c r="B2"><v>123\.5<\/v><\/c>/);
|
||||||
|
assert.match(text, /<c r="C2"><v>987654\.321<\/v><\/c>/);
|
||||||
|
assert.match(text, /<c r="D2"><v>2800\.000000<\/v><\/c>/);
|
||||||
|
assert.match(text, /<c r="E2" t="inlineStr"><is><t>9007199254740992<\/t><\/is><\/c>/);
|
||||||
|
});
|
||||||
Reference in new issue
Block a user