fix(export): emit CSV temporal values without formula wrapper
This commit is contained in:
4 files changed
+50
-160
No files matched your search
@@ -914,11 +914,10 @@ async fn export_query_result_core_inner(
|
||||
result.rows.truncate(this_page);
|
||||
}
|
||||
let row_count = result.rows.len();
|
||||
let formatted_rows = crate::temporal_format::format_temporal_export_rows_with_string_types_for_csv_cow(
|
||||
let formatted_rows = crate::temporal_format::format_temporal_export_rows_with_string_types_cow(
|
||||
&result.rows,
|
||||
&column_types,
|
||||
request.date_time_format.as_deref(),
|
||||
format == "csv",
|
||||
);
|
||||
|
||||
if format == "csv" || format == "txt" {
|
||||
@@ -1137,11 +1136,10 @@ async fn try_export_postgres_query_result_stream(
|
||||
}
|
||||
}
|
||||
crate::db::postgres::PostgresQueryStreamItem::Row(row) => {
|
||||
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types_for_csv_cow(
|
||||
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types_cow(
|
||||
&row,
|
||||
&temporal_column_types,
|
||||
request.date_time_format.as_deref(),
|
||||
format == "csv",
|
||||
);
|
||||
if let Some(writer) = sql_writer.as_mut() {
|
||||
writer.write_row(formatted.into_owned(), None)?;
|
||||
@@ -1379,11 +1377,10 @@ async fn try_export_mysql_query_result_stream(
|
||||
}
|
||||
}
|
||||
crate::db::mysql::MySqlQueryStreamItem::Row(row) => {
|
||||
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types_for_csv_cow(
|
||||
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types_cow(
|
||||
&row,
|
||||
&temporal_column_types,
|
||||
request.date_time_format.as_deref(),
|
||||
format == "csv",
|
||||
);
|
||||
if let Some(writer) = sql_writer.as_mut() {
|
||||
writer.write_row(formatted.into_owned(), None)?;
|
||||
@@ -1600,11 +1597,10 @@ async fn try_export_clickhouse_query_result_stream(
|
||||
}
|
||||
}
|
||||
crate::db::clickhouse_driver::ClickHouseQueryStreamItem::Row(row) => {
|
||||
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types_for_csv_cow(
|
||||
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types_cow(
|
||||
&row,
|
||||
&temporal_column_types,
|
||||
request.date_time_format.as_deref(),
|
||||
format == "csv",
|
||||
);
|
||||
if let Some(writer) = sql_writer.as_mut() {
|
||||
writer.write_row(formatted.into_owned(), None)?;
|
||||
@@ -1786,11 +1782,10 @@ async fn try_export_sqlserver_query_result_stream(
|
||||
}
|
||||
}
|
||||
crate::db::sqlserver::SqlServerStreamItem::Row(row) => {
|
||||
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types_for_csv_cow(
|
||||
let formatted = crate::temporal_format::format_temporal_export_row_with_string_types_cow(
|
||||
row,
|
||||
&temporal_column_types,
|
||||
request.date_time_format.as_deref(),
|
||||
format == "csv",
|
||||
);
|
||||
if let Some(writer) = sql_writer.as_mut() {
|
||||
writer.write_row(formatted.into_owned(), None)?;
|
||||
|
||||
@@ -1071,11 +1071,10 @@ async fn try_export_native_table_stream(
|
||||
&cancelled,
|
||||
cancel_token.clone(),
|
||||
|row| {
|
||||
let formatted = crate::temporal_format::format_temporal_export_row_for_csv_cow(
|
||||
let formatted = crate::temporal_format::format_temporal_export_row_cow(
|
||||
row,
|
||||
column_types,
|
||||
request.date_time_format.as_deref(),
|
||||
request.format.eq_ignore_ascii_case("csv"),
|
||||
);
|
||||
write_table_text_row(&mut file, true, formatted.as_ref(), &mut row_buffer, request.csv_quote_mode)?;
|
||||
rows_exported += 1;
|
||||
@@ -1681,11 +1680,10 @@ async fn export_table_data_core_inner(
|
||||
if row_count == 0 {
|
||||
break;
|
||||
}
|
||||
let formatted_rows = crate::temporal_format::format_temporal_export_rows_for_csv_cow(
|
||||
let formatted_rows = crate::temporal_format::format_temporal_export_rows_cow(
|
||||
&result.rows,
|
||||
&column_types,
|
||||
request.date_time_format.as_deref(),
|
||||
request.format.eq_ignore_ascii_case("csv"),
|
||||
);
|
||||
|
||||
if is_first_batch {
|
||||
|
||||
@@ -492,9 +492,8 @@ pub fn effective_delimited_config(
|
||||
})
|
||||
}
|
||||
|
||||
/// Undo the `="..."` force-text wrapper that CSV export writes around temporal
|
||||
/// cells (see `temporal_format::wrap_csv_force_text`). Without this, exporting a
|
||||
/// table with a datetime column and importing the file straight back stores the
|
||||
/// Undo the `="..."` force-text wrapper that older DBX versions wrote around
|
||||
/// temporal CSV cells. Without this, importing an older DBX export stores the
|
||||
/// literal `="2026-06-24 02:00:07"` instead of the timestamp, which every
|
||||
/// temporal column type then rejects.
|
||||
///
|
||||
|
||||
@@ -333,125 +333,6 @@ pub fn format_temporal_export_rows_with_string_types(
|
||||
format_temporal_export_rows_with_string_types_cow(rows, column_types, pattern).into_owned()
|
||||
}
|
||||
|
||||
/// CSV is untyped text: spreadsheet apps (WPS/Excel/OnlyOffice) re-guess a
|
||||
/// quoted date-looking cell's type on open regardless of RFC4180 quoting,
|
||||
/// silently truncating/reformatting it (dropping the date part or the
|
||||
/// milliseconds). Wrapping the value as an `="..."` formula is the standard
|
||||
/// cross-app way to force text interpretation; the existing CSV
|
||||
/// quoting/escaping (which doubles embedded `"`) turns it into valid CSV for
|
||||
/// free. Applied whenever the column is a temporal type, independent of
|
||||
/// whether `pattern` re-formats the value or the raw driver string passes
|
||||
/// through unchanged (`保持数据库原始值`) — both shapes are equally prone to
|
||||
/// spreadsheet mis-parsing.
|
||||
fn wrap_csv_force_text(value: Value) -> Value {
|
||||
match value {
|
||||
Value::String(text) => Value::String(format!("=\"{text}\"")),
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn format_temporal_export_row_for_csv_cow<'a>(
|
||||
row: &'a [Value],
|
||||
column_types: &[Option<String>],
|
||||
pattern: Option<&str>,
|
||||
force_csv_text: bool,
|
||||
) -> Cow<'a, [Value]> {
|
||||
if !force_csv_text {
|
||||
return format_temporal_export_row_cow(row, column_types, pattern);
|
||||
}
|
||||
if !column_types.iter().any(|data_type| temporal_kind(data_type.as_deref()).is_some()) {
|
||||
return Cow::Borrowed(row);
|
||||
}
|
||||
let active_pattern = active_temporal_pattern(pattern);
|
||||
Cow::Owned(
|
||||
row.iter()
|
||||
.enumerate()
|
||||
.map(|(index, value)| {
|
||||
let data_type = column_types.get(index).and_then(|data_type| data_type.as_deref());
|
||||
if temporal_kind(data_type).is_none() {
|
||||
return value.clone();
|
||||
}
|
||||
let value = match active_pattern {
|
||||
Some(pattern) => format_temporal_export_value(value, data_type, Some(pattern)),
|
||||
None => value.clone(),
|
||||
};
|
||||
wrap_csv_force_text(value)
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn format_temporal_export_rows_for_csv_cow<'a>(
|
||||
rows: &'a [Vec<Value>],
|
||||
column_types: &[Option<String>],
|
||||
pattern: Option<&str>,
|
||||
force_csv_text: bool,
|
||||
) -> Cow<'a, [Vec<Value>]> {
|
||||
if !force_csv_text {
|
||||
return format_temporal_export_rows_cow(rows, column_types, pattern);
|
||||
}
|
||||
if !column_types.iter().any(|data_type| temporal_kind(data_type.as_deref()).is_some()) {
|
||||
return Cow::Borrowed(rows);
|
||||
}
|
||||
Cow::Owned(
|
||||
rows.iter()
|
||||
.map(|row| format_temporal_export_row_for_csv_cow(row, column_types, pattern, true).into_owned())
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn format_temporal_export_row_with_string_types_for_csv_cow<'a>(
|
||||
row: &'a [Value],
|
||||
column_types: &[String],
|
||||
pattern: Option<&str>,
|
||||
force_csv_text: bool,
|
||||
) -> Cow<'a, [Value]> {
|
||||
if !force_csv_text {
|
||||
return format_temporal_export_row_with_string_types_cow(row, column_types, pattern);
|
||||
}
|
||||
if !column_types.iter().any(|data_type| temporal_kind(Some(data_type)).is_some()) {
|
||||
return Cow::Borrowed(row);
|
||||
}
|
||||
let active_pattern = active_temporal_pattern(pattern);
|
||||
Cow::Owned(
|
||||
row.iter()
|
||||
.enumerate()
|
||||
.map(|(index, value)| {
|
||||
let data_type = column_types.get(index).map(String::as_str);
|
||||
if temporal_kind(data_type).is_none() {
|
||||
return value.clone();
|
||||
}
|
||||
let value = match active_pattern {
|
||||
Some(pattern) => format_temporal_export_value(value, data_type, Some(pattern)),
|
||||
None => value.clone(),
|
||||
};
|
||||
wrap_csv_force_text(value)
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn format_temporal_export_rows_with_string_types_for_csv_cow<'a>(
|
||||
rows: &'a [Vec<Value>],
|
||||
column_types: &[String],
|
||||
pattern: Option<&str>,
|
||||
force_csv_text: bool,
|
||||
) -> Cow<'a, [Vec<Value>]> {
|
||||
if !force_csv_text {
|
||||
return format_temporal_export_rows_with_string_types_cow(rows, column_types, pattern);
|
||||
}
|
||||
if !column_types.iter().any(|data_type| temporal_kind(Some(data_type)).is_some()) {
|
||||
return Cow::Borrowed(rows);
|
||||
}
|
||||
Cow::Owned(
|
||||
rows.iter()
|
||||
.map(|row| {
|
||||
format_temporal_export_row_with_string_types_for_csv_cow(row, column_types, pattern, true).into_owned()
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn normalize_temporal_import_value_cow<'a>(
|
||||
value: &'a Value,
|
||||
data_type: Option<&str>,
|
||||
@@ -528,12 +409,13 @@ mod tests {
|
||||
let seconds = format_temporal_export_value(&json!("1785042135"), Some("TIMESTAMP(ms)"), pattern);
|
||||
assert_eq!(seconds.as_str().expect("string").get(..4), rendered.get(..4));
|
||||
|
||||
// The CSV force-text wrapper now wraps a real date instead of an epoch.
|
||||
// CSV keeps the formatted date as plain text instead of changing it
|
||||
// into a formula.
|
||||
let rows = vec![vec![json!("1785042135456"), json!("device-a")]];
|
||||
let types = vec!["TIMESTAMP(ms)".to_string(), "TEXT".to_string()];
|
||||
let wrapped = format_temporal_export_rows_with_string_types_for_csv_cow(&rows, &types, pattern, true);
|
||||
let cell = wrapped[0][0].as_str().expect("string");
|
||||
assert!(cell.starts_with("=\"") && cell.ends_with(":15.456\""), "cell={cell}");
|
||||
let exported = format_temporal_export_rows_with_string_types_cow(&rows, &types, pattern);
|
||||
let cell = exported[0][0].as_str().expect("string");
|
||||
assert_eq!(cell, rendered);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -586,55 +468,71 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn csv_export_wraps_formatted_temporal_values_as_force_text_formulas() {
|
||||
fn csv_export_preserves_formatted_temporal_values_without_formula_wrapper() {
|
||||
let row = vec![json!(1), json!("2024-02-25 13:02:15"), json!("plain text")];
|
||||
let column_types = [Some("NUMBER".into()), Some("TIMESTAMP".into()), Some("VARCHAR2".into())];
|
||||
assert_eq!(
|
||||
format_temporal_export_row_for_csv_cow(&row, &column_types, Some("YYYY/M/D HH:mm:ss"), true).into_owned(),
|
||||
vec![json!(1), json!("=\"2024/2/25 13:02:15\""), json!("plain text")]
|
||||
format_temporal_export_row_cow(&row, &column_types, Some("YYYY/M/D HH:mm:ss")).into_owned(),
|
||||
vec![json!(1), json!("2024/2/25 13:02:15"), json!("plain text")]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn csv_export_wraps_raw_temporal_values_even_without_a_configured_pattern() {
|
||||
// "保持数据库原始值": no export pattern configured, but the raw driver
|
||||
// string is still date-shaped and equally prone to spreadsheet
|
||||
// mis-parsing, so the force-text wrap must still apply.
|
||||
fn csv_export_serializes_temporal_values_without_formula_wrapper() {
|
||||
let row = vec![json!("2024-02-25 13:02:15")];
|
||||
let column_types = [Some("TIMESTAMP".into())];
|
||||
let formatted = format_temporal_export_row_cow(&row, &column_types, None);
|
||||
let csv = crate::csv_export::format_csv_with_quote_mode(
|
||||
&["created_at".to_string()],
|
||||
&[formatted.into_owned()],
|
||||
crate::csv_export::CsvQuoteMode::All,
|
||||
);
|
||||
|
||||
assert_eq!(csv, "\"created_at\"\n\"2024-02-25 13:02:15\"");
|
||||
assert!(!csv.contains("=\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn csv_export_preserves_raw_temporal_values_without_formula_wrapper() {
|
||||
// "保持数据库原始值": no export pattern configured, so the raw driver
|
||||
// string is preserved as-is.
|
||||
let row = vec![json!("2024-02-25 13:02:15.000")];
|
||||
let column_types = [Some("DATETIME".into())];
|
||||
assert_eq!(
|
||||
format_temporal_export_row_for_csv_cow(&row, &column_types, None, true).into_owned(),
|
||||
vec![json!("=\"2024-02-25 13:02:15.000\"")]
|
||||
format_temporal_export_row_cow(&row, &column_types, None).into_owned(),
|
||||
vec![json!("2024-02-25 13:02:15.000")]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn csv_export_does_not_wrap_null_temporal_values() {
|
||||
fn csv_export_preserves_null_temporal_values() {
|
||||
let row = vec![json!(Value::Null)];
|
||||
let column_types = [Some("DATE".into())];
|
||||
assert_eq!(
|
||||
format_temporal_export_row_for_csv_cow(&row, &column_types, None, true).into_owned(),
|
||||
vec![json!(Value::Null)]
|
||||
);
|
||||
assert_eq!(format_temporal_export_row_cow(&row, &column_types, None).into_owned(), vec![json!(Value::Null)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn csv_export_force_text_disabled_matches_plain_temporal_formatting() {
|
||||
fn csv_export_force_text_flag_is_ignored() {
|
||||
let row = vec![json!("2024-02-25 13:02:15")];
|
||||
let column_types = [Some("TIMESTAMP".into())];
|
||||
let expected = format_temporal_export_row(&row, &column_types, Some("YYYY/M/D HH:mm:ss"));
|
||||
assert_eq!(
|
||||
format_temporal_export_row_for_csv_cow(&row, &column_types, Some("YYYY/M/D HH:mm:ss"), false).into_owned(),
|
||||
format_temporal_export_row(&row, &column_types, Some("YYYY/M/D HH:mm:ss"))
|
||||
format_temporal_export_row_cow(&row, &column_types, Some("YYYY/M/D HH:mm:ss")).into_owned(),
|
||||
expected
|
||||
);
|
||||
assert_eq!(
|
||||
format_temporal_export_row_cow(&row, &column_types, Some("YYYY/M/D HH:mm:ss")).into_owned(),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn csv_export_with_string_types_wraps_temporal_values() {
|
||||
fn csv_export_with_string_types_preserves_temporal_values() {
|
||||
let row = vec![json!("2024-02-25"), json!(42)];
|
||||
let column_types = ["DATE".to_string(), "INT".to_string()];
|
||||
assert_eq!(
|
||||
format_temporal_export_row_with_string_types_for_csv_cow(&row, &column_types, None, true).into_owned(),
|
||||
vec![json!("=\"2024-02-25\""), json!(42)]
|
||||
format_temporal_export_row_with_string_types_cow(&row, &column_types, None).into_owned(),
|
||||
vec![json!("2024-02-25"), json!(42)]
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user