github/dbx
4
0
Fork 0
mirror of https://github.com/t8y2/dbx.git synced 2026-10-20 00:17:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(import): support comma decimal separator for CSV/TSV

Browse Source
This commit is contained in:
ccq
2026-09-29 22:15:41 +08:00
committed by GitHub
parent 9a1c64b9f0
commit 4269a61e2c
15 changed files with 184 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/desktop/src/components/import/TableImportDialog.vue
+17 -2
View File
@@ -135,6 +135,7 @@ const errorMessage = ref("");
const wizardStep = ref<TableImportWizardStep>("source");
const fileInput = ref<HTMLInputElement | null>(null);
const delimiter = ref(",");
const decimalSeparator = ref<"." | ",">(".");
const textEncoding = ref<api.TableImportTextEncoding>("auto");
const titleRow = ref(1);
const dataStartRow = ref(2);
@@ -397,6 +398,7 @@ function resetState() {
activeTaskIndex.value = 0;
sourceFormat.value = "csv";
delimiter.value = ",";
decimalSeparator.value = ".";
textEncoding.value = "auto";
titleRow.value = 1;
dataStartRow.value = 2;
@@ -465,6 +467,7 @@ function taskParseOptions(format: api.TableImportSourceFormat, sheetName = ""):
return buildTableImportParseOptions({
format,
delimiter: delimiter.value,
decimalSeparator: decimalSeparator.value,
textEncoding: textEncoding.value,
titleRow: titleRow.value,
dataStartRow: dataStartRow.value,
@@ -1223,7 +1226,7 @@ watch(
onBeforeUnmount(stopDialogDrag);
watch([sourceFormat, delimiter, titleRow, dataStartRow, lastDataRow, trimValues, emptyStringAsNull, selectedSheet, jsonShape, previewLimit], schedulePreviewReload);
watch(textEncoding, schedulePreviewReloadAfterEncodingChange);
watch([textEncoding, decimalSeparator], schedulePreviewReloadAfterEncodingChange);
watch([newTableName, columnMapping, columnDataTypes], saveActiveBatchTask, { deep: true });
watch(wizardStep, (step) => {
if (step !== "mapping") closeDataTypePicker();
@@ -1434,7 +1437,7 @@ watch(rawProgressPercent, (percent) => {
</div>
</div>
<div v-if="sourceFormat === 'csv' || sourceFormat === 'tsv' || sourceFormat === 'delimited'" class="grid grid-cols-5 gap-3 rounded-md border p-3">
<div v-if="sourceFormat === 'csv' || sourceFormat === 'tsv' || sourceFormat === 'delimited'" class="grid grid-cols-2 gap-3 rounded-md border p-3 md:grid-cols-3 xl:grid-cols-6">
<div class="space-y-1.5">
<Label class="text-xs">{{ t("tableImport.encoding") }}</Label>
<Select :model-value="textEncoding" @update:model-value="(value: any) => (textEncoding = value)">
@@ -1455,6 +1458,18 @@ watch(rawProgressPercent, (percent) => {
<Label class="text-xs">{{ t("tableImport.delimiter") }}</Label>
<Input v-model="delimiter" :disabled="sourceFormat !== 'delimited'" class="h-8 text-xs font-mono" />
</div>
<div class="space-y-1.5">
<Label class="text-xs">{{ t("tableImport.decimalSeparator") }}</Label>
<Select v-model="decimalSeparator">
<SelectTrigger class="h-8 text-xs font-mono" :aria-label="t('tableImport.decimalSeparator')">
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value=".">.</SelectItem>
<SelectItem value=",">,</SelectItem>
</SelectContent>
</Select>
</div>
<div class="space-y-1.5">
<Label class="text-xs">{{ t("tableImport.titleRow") }}</Label>
<Input v-model.number="titleRow" type="number" min="0" class="h-8 text-xs" />
apps/desktop/src/components/import/__tests__/TableImportDialog.batchSelection.spec.ts
Vendored
+40 -3
View File
@@ -113,16 +113,29 @@ vi.mock("@/components/ui/label", async () => {
});
vi.mock("@/components/ui/select", async () => {
const { defineComponent, h } = await import("vue");
const { defineComponent, h, inject, provide } = await import("vue");
const passthrough = defineComponent({
setup(_props, { slots }) {
return () => h("div", slots.default?.());
},
});
return {
Select: passthrough,
Select: defineComponent({
props: { modelValue: String },
emits: ["update:modelValue"],
setup(_props, { emit, slots }) {
provide("selectOption", (value: string) => emit("update:modelValue", value));
return () => h("div", slots.default?.());
},
}),
SelectContent: passthrough,
SelectItem: passthrough,
SelectItem: defineComponent({
props: { value: String },
setup(props, { slots }) {
const selectOption = inject<(value: string) => void>("selectOption");
return () => h("span", { "data-select-option": props.value, onClick: () => selectOption?.(props.value || "") }, slots.default?.());
},
}),
SelectTrigger: passthrough,
SelectValue: passthrough,
};
@@ -279,6 +292,30 @@ afterEach(() => {
});
describe("TableImportDialog batch selection", () => {
it("reloads a single CSV preview and passes the selected decimal separator to import", async () => {
mocks.previewTableImportFile.mockImplementation(() => Promise.resolve(delimitedPreview()));
await mountDialog([new File(['amount,note\n"12,50",plain'], "rows.csv")]);
expect(mocks.previewTableImportFile.mock.calls.at(-1)?.[1].parseOptions.decimalSeparator).toBe(".");
document.body.querySelector<HTMLElement>('[data-select-option=","]')!.click();
await new Promise((resolve) => setTimeout(resolve, 300));
await flushAsyncUpdates();
expect(mocks.previewTableImportFile.mock.calls.at(-1)?.[1].parseOptions.decimalSeparator).toBe(",");
await startImport();
expect(mocks.importTableFile.mock.calls[0]![0].parseOptions?.decimalSeparator).toBe(",");
});
it("reloads selected batch CSV previews when the decimal separator changes", async () => {
mocks.previewTableImportFile.mockImplementation(() => Promise.resolve(delimitedPreview()));
await mountDialog([new File(["id,name"], "first.csv"), new File(["id,name"], "second.csv")]);
mocks.previewTableImportFile.mockClear();
document.body.querySelector<HTMLElement>('[data-select-option=","]')!.click();
await flushAsyncUpdates();
expect(mocks.previewTableImportFile).toHaveBeenCalledTimes(2);
expect(mocks.previewTableImportFile.mock.calls.map(([, options]) => options.parseOptions.decimalSeparator)).toEqual([",", ","]);
});
it("uses all desktop grid columns when Parquet is unavailable", async () => {
await mountSourceDialog();
apps/desktop/src/i18n/locales/en.ts
+1
View File
@@ -6992,6 +6992,7 @@ export default {
parquetOnlyDuckdb: "Parquet import is only supported for DuckDB connections",
previewRows: "Preview rows",
delimiter: "Delimiter",
decimalSeparator: "Decimal separator",
encoding: "Encoding",
encodingAuto: "Auto-detect",
encodingUtf8: "UTF-8",
apps/desktop/src/i18n/locales/es.ts
+1
View File
@@ -6380,6 +6380,7 @@ export default withEnglishFallback({
status_cancelled: "Import cancelled",
targetColumnsTimeout: "Se agotó el tiempo de espera al cargar los campos de la tabla de destino. Compruebe la conexión e inténtelo de nuevo.",
retry: "Reintentar",
decimalSeparator: "Separador decimal",
},
dataGenerate: {
title: "Generación de datos",
apps/desktop/src/i18n/locales/it.ts
+1
View File
@@ -6302,6 +6302,7 @@ export default withEnglishFallback({
status_cancelled: "Import cancelled",
targetColumnsTimeout: "Timeout nel caricamento dei campi della tabella di destinazione. Controlla la connessione e riprova.",
retry: "Riprova",
decimalSeparator: "Separatore decimale",
},
dataGenerate: {
title: "Generazione dati",
apps/desktop/src/i18n/locales/ja.ts
+1
View File
@@ -6333,6 +6333,7 @@ export default withEnglishFallback({
lastDataRow: "データ終了行（0 は末尾まで）",
targetColumnsTimeout: "ターゲットテーブルのフィールド読み込みがタイムアウトしました。接続を確認して再試行してください。",
retry: "再試行",
decimalSeparator: "小数点記号",
},
dataGenerate: {
title: "データ生成",
apps/desktop/src/i18n/locales/ko.ts
+1
View File
@@ -6035,6 +6035,7 @@ export default withEnglishFallback({
status_cancelled: "가져오기 취소됨",
targetColumnsTimeout: "대상 테이블 필드를 로드하는 시간이 초과되었습니다. 연결을 확인한 후 다시 시도하십시오.",
retry: "재시도",
decimalSeparator: "소수점 구분 기호",
},
dataGenerate: {
title: "데이터 생성",
apps/desktop/src/i18n/locales/pt-BR.ts
+1
View File
@@ -6304,6 +6304,7 @@ export default withEnglishFallback({
status_cancelled: "Import cancelled",
targetColumnsTimeout: "Tempo esgotado ao carregar os campos da tabela de destino. Verifique a conexão e tente novamente.",
retry: "Tentar novamente",
decimalSeparator: "Separador decimal",
},
dataGenerate: {
title: "Geração de dados",
apps/desktop/src/i18n/locales/ru.ts
+1
View File
@@ -7413,6 +7413,7 @@ export default withEnglishFallback({
parquetOnlyDuckdb: "Импорт Parquet поддерживается только для подключений DuckDB",
previewRows: "Строк предпросмотра",
delimiter: "Разделитель",
decimalSeparator: "Десятичный разделитель",
encoding: "Кодировка",
encodingAuto: "Автоопределение",
encodingUtf8: "UTF-8",
apps/desktop/src/i18n/locales/zh-CN.ts
+1
View File
@@ -6959,6 +6959,7 @@ export default withEnglishFallback({
parquetOnlyDuckdb: "Parquet 导入仅支持 DuckDB 连接",
previewRows: "预览行数",
delimiter: "分隔符",
decimalSeparator: "小数分隔符",
encoding: "文件编码",
encodingAuto: "自动检测",
encodingUtf8: "UTF-8",
apps/desktop/src/i18n/locales/zh-TW.ts
+1
View File
@@ -5620,6 +5620,7 @@ export default withEnglishFallback({
status_cancelled: "匯入已取消",
targetColumnsTimeout: "載入目標資料表欄位逾時，請檢查連線後重試。",
retry: "重試",
decimalSeparator: "小數分隔符",
},
dataGenerate: {
title: "資料生成",
apps/desktop/src/lib/__tests__/table/tableImport.spec.ts
Vendored
+18
View File
@@ -45,6 +45,24 @@ describe("tableImport", () => {
expect(legacyEmptyField.emptyStringAsNull).toBe(true);
});
it("passes the decimal separator only for delimited sources", () => {
const settings = {
delimiter: ";",
decimalSeparator: ",",
textEncoding: "auto" as const,
titleRow: 1,
dataStartRow: 2,
lastDataRow: 0,
trimValues: false,
emptyStringAsNull: false,
jsonShape: "auto" as const,
};
expect(buildTableImportParseOptions({ ...settings, format: "delimited" }).decimalSeparator).toBe(",");
expect(buildTableImportParseOptions({ ...settings, format: "csv" }).decimalSeparator).toBe(",");
expect(buildTableImportParseOptions({ ...settings, format: "tsv" }).decimalSeparator).toBe(",");
expect(buildTableImportParseOptions({ ...settings, format: "json" }).decimalSeparator).toBeNull();
});
it("formats import elapsed time for progress and terminal summaries", () => {
expect(formatTableImportElapsed(0)).toBe("0 ms");
expect(formatTableImportElapsed(999)).toBe("999 ms");
apps/desktop/src/lib/backend/tauri.ts
+1
View File
@@ -5494,6 +5494,7 @@ export interface TableImportColumnMapping {
export interface TableImportParseOptions {
delimiter?: string | null;
decimalSeparator?: string | null;
encoding?: TableImportTextEncoding | null;
hasHeader?: boolean | null;
titleRow?: number | null;
apps/desktop/src/lib/table/tableImport.ts
+2
View File
@@ -86,6 +86,7 @@ export function tableImportProgressPercent(progress: TableImportProgressLike | n
export interface TableImportParseSettings {
format: TableImportSourceFormat;
delimiter: string;
decimalSeparator?: "." | ",";
textEncoding: TableImportTextEncoding;
titleRow: number;
dataStartRow: number;
@@ -114,6 +115,7 @@ export function buildTableImportParseOptions(settings: TableImportParseSettings)
const isTextSource = isDelimited || settings.format === "sql";
return {
delimiter: settings.format === "tsv" ? "\\t" : settings.format === "csv" ? "," : settings.delimiter,
decimalSeparator: isDelimited ? (settings.decimalSeparator ?? ".") : null,
encoding: isTextSource ? settings.textEncoding : null,
titleRow: settings.titleRow,
dataStartRow: settings.dataStartRow,
crates/dbx-core/src/data/table_import.rs
+97 -1
View File
@@ -5,6 +5,7 @@ use std::io::{BufRead, BufReader, Read as IoRead, Seek, SeekFrom, Write as IoWri
use std::path::Path;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::Arc;
use std::sync::OnceLock;
use std::time::{Duration, Instant, UNIX_EPOCH};
use calamine::{
@@ -211,6 +212,7 @@ impl TableImportTextEncoding {
#[serde(rename_all = "camelCase")]
pub struct TableImportParseOptions {
pub delimiter: Option<String>,
pub decimal_separator: Option<String>,
pub encoding: Option<TableImportTextEncoding>,
pub has_header: Option<bool>,
pub title_row: Option<usize>,
@@ -234,6 +236,7 @@ impl Default for TableImportParseOptions {
fn default() -> Self {
Self {
delimiter: None,
decimal_separator: None,
encoding: Some(TableImportTextEncoding::Auto),
has_header: None,
title_row: None,
@@ -669,6 +672,7 @@ fn unique_import_headers(headers: impl IntoIterator<Item = String>) -> Vec<Strin
#[derive(Debug, Clone)]
pub struct DelimitedParseConfig {
pub delimiter: u8,
pub decimal_separator: char,
pub trim_values: bool,
pub empty_string_as_null: bool,
/// 命中的字段按 NULL 处理；`None` 表示不按字面量识别 NULL。
@@ -735,9 +739,15 @@ pub fn effective_delimited_config(
};
let null_literal = effective_delimited_null_literal(options.null_literal.as_deref());
let decimal_separator = match options.decimal_separator.as_deref() {
None | Some(".") => '.',
Some(",") => ',',
_ => return Err("Decimal separator must be '.' or ','".to_string()),
};
Ok(DelimitedParseConfig {
delimiter,
decimal_separator,
trim_values: options.trim_values.unwrap_or(false),
// 配了 NULL 字面量时空字段一律是空串：否则字面量刚把 NULL 和空串分开，
// 这里又会把空串重新当成 NULL。
@@ -776,7 +786,16 @@ pub fn csv_value_with_config(value: &str, config: &DelimitedParseConfig) -> serd
if config.empty_string_as_null && value.is_empty() {
serde_json::Value::Null
} else {
serde_json::Value::String(value.to_string())
static COMMA_DECIMAL: OnceLock<regex::Regex> = OnceLock::new();
let is_comma_decimal = config.decimal_separator == ','
&& value.contains(',')
&& COMMA_DECIMAL
.get_or_init(|| {
regex::Regex::new(r"^[+-]?(?:[0-9]+(?:,[0-9]*)?|,[0-9]+)(?:[eE][+-]?[0-9]+)?$").unwrap()
})
.is_match(value);
let normalized = if is_comma_decimal { value.replace(',', ".") } else { value.to_string() };
serde_json::Value::String(normalized)
}
}
@@ -785,6 +804,7 @@ pub fn csv_value(value: &str) -> serde_json::Value {
value,
&DelimitedParseConfig {
delimiter: b',',
decimal_separator: '.',
trim_values: false,
empty_string_as_null: true,
null_literal: None,
@@ -9868,6 +9888,82 @@ mod tests {
);
}
#[test]
fn comma_decimal_separator_matches_preview_and_streamed_rows() {
let huge = format!("{},25", "9".repeat(400));
let bytes = format!("amount;note\n12,50;part,number\n-0,25;\\N\n1,2e3;\n1.234,50;plain\n{huge};large\n");
let path = std::env::temp_dir().join(format!("dbx-comma-decimal-{}.csv", uuid::Uuid::new_v4()));
std::fs::write(&path, &bytes).unwrap();
let options = TableImportParseOptions {
delimiter: Some(";".into()),
decimal_separator: Some(",".into()),
encoding: Some(TableImportTextEncoding::Utf8),
..TableImportParseOptions::default()
};
let preview =
parse_delimited_bytes_with_options(bytes.as_bytes(), TableImportSourceFormat::Delimited, &options, 10)
.unwrap();
assert_eq!(
preview.rows,
vec![
vec![serde_json::json!("12.50"), serde_json::json!("part,number")],
vec![serde_json::json!("-0.25"), serde_json::Value::Null],
vec![serde_json::json!("1.2e3"), serde_json::json!("")],
vec![serde_json::json!("1.234,50"), serde_json::json!("plain")],
vec![serde_json::json!(huge.replace(',', ".")), serde_json::json!("large")],
]
);
let (sender, mut receiver) = tokio::sync::mpsc::channel(16);
stream_delimited_rows_to_channel(
&path.to_string_lossy(),
TableImportSourceFormat::Delimited,
&options,
2,
sender,
)
.unwrap();
let rows = std::iter::from_fn(|| receiver.blocking_recv())
.map(|message| message.unwrap())
.filter_map(|message| match message {
DelimitedStreamMessage::Rows { rows, .. } => Some(rows),
_ => None,
})
.flatten()
.collect::<Vec<_>>();
assert_eq!(rows, preview.rows);
let default = parse_delimited_bytes_with_options(
bytes.as_bytes(),
TableImportSourceFormat::Delimited,
&TableImportParseOptions { delimiter: Some(";".into()), ..TableImportParseOptions::default() },
10,
)
.unwrap();
assert_eq!(default.rows[0][0], serde_json::json!("12,50"));
for (format, bytes) in [
(TableImportSourceFormat::Csv, &b"amount,note\n\"12,50\",plain\n"[..]),
(TableImportSourceFormat::Tsv, &b"amount\tnote\n12,50\tplain\n"[..]),
] {
let parsed = parse_delimited_bytes_with_options(
bytes,
format,
&TableImportParseOptions { decimal_separator: Some(",".into()), ..TableImportParseOptions::default() },
10,
)
.unwrap();
assert_eq!(parsed.rows[0], vec![serde_json::json!("12.50"), serde_json::json!("plain")]);
}
let invalid = effective_delimited_config(
TableImportSourceFormat::Csv,
&TableImportParseOptions { decimal_separator: Some(";".into()), ..TableImportParseOptions::default() },
)
.unwrap_err();
assert!(invalid.contains("Decimal separator"));
let _ = std::fs::remove_file(path);
}
#[test]
fn parses_delimited_text_without_header_and_trims_values() {
let options = TableImportParseOptions {