feat(import): support comma decimal separator for CSV/TSV
This commit is contained in:
15 files changed
+184
-6
No files matched your search
@@ -5,6 +5,7 @@ use std::io::{BufRead, BufReader, Read as IoRead, Seek, SeekFrom, Write as IoWri
|
||||
use std::path::Path;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::sync::OnceLock;
|
||||
use std::time::{Duration, Instant, UNIX_EPOCH};
|
||||
|
||||
use calamine::{
|
||||
@@ -211,6 +212,7 @@ impl TableImportTextEncoding {
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct TableImportParseOptions {
|
||||
pub delimiter: Option<String>,
|
||||
pub decimal_separator: Option<String>,
|
||||
pub encoding: Option<TableImportTextEncoding>,
|
||||
pub has_header: Option<bool>,
|
||||
pub title_row: Option<usize>,
|
||||
@@ -234,6 +236,7 @@ impl Default for TableImportParseOptions {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
delimiter: None,
|
||||
decimal_separator: None,
|
||||
encoding: Some(TableImportTextEncoding::Auto),
|
||||
has_header: None,
|
||||
title_row: None,
|
||||
@@ -669,6 +672,7 @@ fn unique_import_headers(headers: impl IntoIterator<Item = String>) -> Vec<Strin
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DelimitedParseConfig {
|
||||
pub delimiter: u8,
|
||||
pub decimal_separator: char,
|
||||
pub trim_values: bool,
|
||||
pub empty_string_as_null: bool,
|
||||
/// 命中的字段按 NULL 处理;`None` 表示不按字面量识别 NULL。
|
||||
@@ -735,9 +739,15 @@ pub fn effective_delimited_config(
|
||||
};
|
||||
|
||||
let null_literal = effective_delimited_null_literal(options.null_literal.as_deref());
|
||||
let decimal_separator = match options.decimal_separator.as_deref() {
|
||||
None | Some(".") => '.',
|
||||
Some(",") => ',',
|
||||
_ => return Err("Decimal separator must be '.' or ','".to_string()),
|
||||
};
|
||||
|
||||
Ok(DelimitedParseConfig {
|
||||
delimiter,
|
||||
decimal_separator,
|
||||
trim_values: options.trim_values.unwrap_or(false),
|
||||
// 配了 NULL 字面量时空字段一律是空串:否则字面量刚把 NULL 和空串分开,
|
||||
// 这里又会把空串重新当成 NULL。
|
||||
@@ -776,7 +786,16 @@ pub fn csv_value_with_config(value: &str, config: &DelimitedParseConfig) -> serd
|
||||
if config.empty_string_as_null && value.is_empty() {
|
||||
serde_json::Value::Null
|
||||
} else {
|
||||
serde_json::Value::String(value.to_string())
|
||||
static COMMA_DECIMAL: OnceLock<regex::Regex> = OnceLock::new();
|
||||
let is_comma_decimal = config.decimal_separator == ','
|
||||
&& value.contains(',')
|
||||
&& COMMA_DECIMAL
|
||||
.get_or_init(|| {
|
||||
regex::Regex::new(r"^[+-]?(?:[0-9]+(?:,[0-9]*)?|,[0-9]+)(?:[eE][+-]?[0-9]+)?$").unwrap()
|
||||
})
|
||||
.is_match(value);
|
||||
let normalized = if is_comma_decimal { value.replace(',', ".") } else { value.to_string() };
|
||||
serde_json::Value::String(normalized)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -785,6 +804,7 @@ pub fn csv_value(value: &str) -> serde_json::Value {
|
||||
value,
|
||||
&DelimitedParseConfig {
|
||||
delimiter: b',',
|
||||
decimal_separator: '.',
|
||||
trim_values: false,
|
||||
empty_string_as_null: true,
|
||||
null_literal: None,
|
||||
@@ -9868,6 +9888,82 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn comma_decimal_separator_matches_preview_and_streamed_rows() {
|
||||
let huge = format!("{},25", "9".repeat(400));
|
||||
let bytes = format!("amount;note\n12,50;part,number\n-0,25;\\N\n1,2e3;\n1.234,50;plain\n{huge};large\n");
|
||||
let path = std::env::temp_dir().join(format!("dbx-comma-decimal-{}.csv", uuid::Uuid::new_v4()));
|
||||
std::fs::write(&path, &bytes).unwrap();
|
||||
let options = TableImportParseOptions {
|
||||
delimiter: Some(";".into()),
|
||||
decimal_separator: Some(",".into()),
|
||||
encoding: Some(TableImportTextEncoding::Utf8),
|
||||
..TableImportParseOptions::default()
|
||||
};
|
||||
let preview =
|
||||
parse_delimited_bytes_with_options(bytes.as_bytes(), TableImportSourceFormat::Delimited, &options, 10)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
preview.rows,
|
||||
vec![
|
||||
vec![serde_json::json!("12.50"), serde_json::json!("part,number")],
|
||||
vec![serde_json::json!("-0.25"), serde_json::Value::Null],
|
||||
vec![serde_json::json!("1.2e3"), serde_json::json!("")],
|
||||
vec![serde_json::json!("1.234,50"), serde_json::json!("plain")],
|
||||
vec![serde_json::json!(huge.replace(',', ".")), serde_json::json!("large")],
|
||||
]
|
||||
);
|
||||
|
||||
let (sender, mut receiver) = tokio::sync::mpsc::channel(16);
|
||||
stream_delimited_rows_to_channel(
|
||||
&path.to_string_lossy(),
|
||||
TableImportSourceFormat::Delimited,
|
||||
&options,
|
||||
2,
|
||||
sender,
|
||||
)
|
||||
.unwrap();
|
||||
let rows = std::iter::from_fn(|| receiver.blocking_recv())
|
||||
.map(|message| message.unwrap())
|
||||
.filter_map(|message| match message {
|
||||
DelimitedStreamMessage::Rows { rows, .. } => Some(rows),
|
||||
_ => None,
|
||||
})
|
||||
.flatten()
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(rows, preview.rows);
|
||||
|
||||
let default = parse_delimited_bytes_with_options(
|
||||
bytes.as_bytes(),
|
||||
TableImportSourceFormat::Delimited,
|
||||
&TableImportParseOptions { delimiter: Some(";".into()), ..TableImportParseOptions::default() },
|
||||
10,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(default.rows[0][0], serde_json::json!("12,50"));
|
||||
|
||||
for (format, bytes) in [
|
||||
(TableImportSourceFormat::Csv, &b"amount,note\n\"12,50\",plain\n"[..]),
|
||||
(TableImportSourceFormat::Tsv, &b"amount\tnote\n12,50\tplain\n"[..]),
|
||||
] {
|
||||
let parsed = parse_delimited_bytes_with_options(
|
||||
bytes,
|
||||
format,
|
||||
&TableImportParseOptions { decimal_separator: Some(",".into()), ..TableImportParseOptions::default() },
|
||||
10,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(parsed.rows[0], vec![serde_json::json!("12.50"), serde_json::json!("plain")]);
|
||||
}
|
||||
let invalid = effective_delimited_config(
|
||||
TableImportSourceFormat::Csv,
|
||||
&TableImportParseOptions { decimal_separator: Some(";".into()), ..TableImportParseOptions::default() },
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(invalid.contains("Decimal separator"));
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_delimited_text_without_header_and_trims_values() {
|
||||
let options = TableImportParseOptions {
|
||||
|
||||
Reference in new issue
Block a user