diff --git a/crates/dbx-core/src/table_import.rs b/crates/dbx-core/src/table_import.rs index ae8a0b1904..cbef4b3794 100644 --- a/crates/dbx-core/src/table_import.rs +++ b/crates/dbx-core/src/table_import.rs @@ -1134,10 +1134,22 @@ fn xlsx_datetime_label(value: &ExcelDateTime, temporal_kind: Option) -> serde_json::Value { +fn xlsx_string_value(value: &str, empty_string_as_null: bool) -> serde_json::Value { + if empty_string_as_null && value.is_empty() { + serde_json::Value::Null + } else { + serde_json::Value::String(value.to_string()) + } +} + +fn xlsx_cell_value_with_temporal_kind( + cell: &Data, + temporal_kind: Option, + empty_string_as_null: bool, +) -> serde_json::Value { match cell { Data::Empty => serde_json::Value::Null, - Data::String(s) => csv_value(s), + Data::String(s) => xlsx_string_value(s, empty_string_as_null), Data::Float(n) => { serde_json::Number::from_f64(*n).map(serde_json::Value::Number).unwrap_or(serde_json::Value::Null) } @@ -1182,7 +1194,7 @@ fn xlsx_cell_text_value(cell: &Data, style: Option<&XlsxCellStyle>) -> Option serde_json::Value { - xlsx_cell_value_with_temporal_kind(cell, None) + xlsx_cell_value_with_temporal_kind(cell, None, true) } fn xlsx_cell_label_with_temporal_kind(cell: &Data, temporal_kind: Option) -> String { @@ -1206,11 +1218,12 @@ pub fn xlsx_cell_label(cell: &Data) -> String { fn xlsx_cell_ref_value_with_temporal_kind( cell: &DataRef<'_>, temporal_kind: Option, + empty_string_as_null: bool, ) -> serde_json::Value { match cell { DataRef::Empty => serde_json::Value::Null, - DataRef::String(s) => csv_value(s), - DataRef::SharedString(s) => csv_value(s), + DataRef::String(s) => xlsx_string_value(s, empty_string_as_null), + DataRef::SharedString(s) => xlsx_string_value(s, empty_string_as_null), DataRef::Float(n) => { serde_json::Number::from_f64(*n).map(serde_json::Value::Number).unwrap_or(serde_json::Value::Null) } @@ -1543,6 +1556,8 @@ struct XlsxPreviewRawCell { style_id: Option, value: String, inline_value: String, + has_value: bool, + has_inline_value: bool, } fn xlsx_dimension_bounds(reference: &str) -> Option<((usize, usize), (usize, usize))> { @@ -1809,6 +1824,7 @@ fn xlsx_preview_cell_value( shared_strings: &HashMap, styles: &[XlsxCellStyle], date_1904: bool, + empty_string_as_null: bool, ) -> serde_json::Value { let cell_type = cell.cell_type.as_deref().unwrap_or_default(); match cell_type { @@ -1817,9 +1833,12 @@ fn xlsx_preview_cell_value( .parse::() .ok() .and_then(|index| shared_strings.get(&index)) - .map_or(serde_json::Value::Null, |value| csv_value(value)), - "inlineStr" => csv_value(&cell.inline_value), - "str" | "d" | "e" => csv_value(&cell.value), + .map_or(serde_json::Value::Null, |value| xlsx_string_value(value, empty_string_as_null)), + "inlineStr" if cell.has_inline_value => xlsx_string_value(&cell.inline_value, empty_string_as_null), + "inlineStr" => serde_json::Value::Null, + "str" if cell.has_value => xlsx_string_value(&cell.value, empty_string_as_null), + "str" => serde_json::Value::Null, + "d" | "e" => csv_value(&cell.value), "b" => serde_json::Value::Bool(matches!(cell.value.trim(), "1" | "true" | "TRUE")), _ => { let Some(number) = cell.value.trim().parse::().ok() else { @@ -1878,6 +1897,7 @@ fn parse_xlsx_preview_file_with_options( let styles_xml = read_xlsx_zip_text(&mut zip, "xl/styles.xml").unwrap_or_default(); let styles = parse_xlsx_styles(&styles_xml); let date_1904 = xlsx_workbook_uses_1904_date_system(&workbook_xml); + let empty_string_as_null = options.empty_string_as_null.unwrap_or(true); let row_range = effective_import_row_range(options)?; let preview_limit = preview_limit.max(1); let preview_last_row = row_range.data_start_row.saturating_add(preview_limit.saturating_sub(1)); @@ -1951,11 +1971,19 @@ fn parse_xlsx_preview_file_with_options( }; } Ok(Event::Start(element)) if xml_local_name_eq(element.name().as_ref(), b"v") => { + current_cell.has_value = true; in_value = true; } + Ok(Event::Empty(element)) if xml_local_name_eq(element.name().as_ref(), b"v") => { + current_cell.has_value = true; + } Ok(Event::Start(element)) if xml_local_name_eq(element.name().as_ref(), b"t") => { + current_cell.has_inline_value = true; in_inline_text = inline_phonetic_depth == 0; } + Ok(Event::Empty(element)) if xml_local_name_eq(element.name().as_ref(), b"t") => { + current_cell.has_inline_value = true; + } Ok(Event::Start(element)) if xml_local_name_eq(element.name().as_ref(), b"rPh") => { inline_phonetic_depth = inline_phonetic_depth.saturating_add(1); } @@ -2039,7 +2067,9 @@ fn parse_xlsx_preview_file_with_options( let column = start_column + index; let value = raw_cells .get(&(absolute_title_row, column)) - .map(|cell| xlsx_preview_cell_value(cell, &shared_strings, &styles, date_1904)) + .map(|cell| { + xlsx_preview_cell_value(cell, &shared_strings, &styles, date_1904, empty_string_as_null) + }) .unwrap_or(serde_json::Value::Null); normalize_header(&xlsx_preview_cell_label(&value), index) }) @@ -2066,7 +2096,9 @@ fn parse_xlsx_preview_file_with_options( .map(|index| { raw_cells .get(&(absolute_row, start_column + index)) - .map(|cell| xlsx_preview_cell_value(cell, &shared_strings, &styles, date_1904)) + .map(|cell| { + xlsx_preview_cell_value(cell, &shared_strings, &styles, date_1904, empty_string_as_null) + }) .unwrap_or(serde_json::Value::Null) }) .collect::>() @@ -2232,6 +2264,7 @@ fn xlsx_stream_cell_value( styles: &[XlsxCellStyle], date_1904: bool, format_as_text: bool, + empty_string_as_null: bool, ) -> Result { if format_as_text && cell.cell_type.as_deref().unwrap_or_default().is_empty() { if let Ok(number) = cell.value.trim().parse::() { @@ -2242,12 +2275,14 @@ fn xlsx_stream_cell_value( } } if cell.cell_type.as_deref() != Some("s") { - return Ok(xlsx_preview_cell_value(cell, &HashMap::new(), styles, date_1904)); + return Ok(xlsx_preview_cell_value(cell, &HashMap::new(), styles, date_1904, empty_string_as_null)); } let Some(index) = cell.value.parse::().ok() else { return Ok(serde_json::Value::Null); }; - Ok(shared_strings.get(index)?.map_or(serde_json::Value::Null, |value| csv_value(&value))) + Ok(shared_strings + .get(index)? + .map_or(serde_json::Value::Null, |value| xlsx_string_value(&value, empty_string_as_null))) } fn xlsx_cell_ref_text_value(cell: &DataRef<'_>, style: Option<&XlsxCellStyle>) -> Option { @@ -2498,6 +2533,7 @@ fn stream_xlsx_rows_to_channel_with_control( // This producer runs on a blocking thread and communicates in bounded batches. The small // channel capacity applies backpressure when database writes are slower than XML parsing. let total_bytes = std::fs::metadata(path).map(|metadata| metadata.len()).unwrap_or_default(); + let empty_string_as_null = options.empty_string_as_null.unwrap_or(true); let mut zip = zip::ZipArchive::new(File::open(path).map_err(|error| error.to_string())?) .map_err(|error| error.to_string())?; let workbook_xml = read_xlsx_zip_text(&mut zip, "xl/workbook.xml")?; @@ -2615,10 +2651,20 @@ fn stream_xlsx_rows_to_channel_with_control( ..XlsxPreviewRawCell::default() }; } - Ok(Event::Start(element)) if xml_local_name_eq(element.name().as_ref(), b"v") => in_value = true, + Ok(Event::Start(element)) if xml_local_name_eq(element.name().as_ref(), b"v") => { + current_cell.has_value = true; + in_value = true; + } + Ok(Event::Empty(element)) if xml_local_name_eq(element.name().as_ref(), b"v") => { + current_cell.has_value = true; + } Ok(Event::Start(element)) if xml_local_name_eq(element.name().as_ref(), b"t") => { + current_cell.has_inline_value = true; in_inline_text = inline_phonetic_depth == 0; } + Ok(Event::Empty(element)) if xml_local_name_eq(element.name().as_ref(), b"t") => { + current_cell.has_inline_value = true; + } Ok(Event::Start(element)) if xml_local_name_eq(element.name().as_ref(), b"rPh") => { inline_phonetic_depth = inline_phonetic_depth.saturating_add(1); } @@ -2636,8 +2682,14 @@ fn stream_xlsx_rows_to_channel_with_control( Ok(Event::End(element)) if xml_local_name_eq(element.name().as_ref(), b"c") => { if let Some((row, column)) = current_position.take() { let format_as_text = rows.is_text_source_column(row, column, &text_source_columns); - let value = - xlsx_stream_cell_value(¤t_cell, &mut shared_strings, &styles, date_1904, format_as_text)?; + let value = xlsx_stream_cell_value( + ¤t_cell, + &mut shared_strings, + &styles, + date_1904, + format_as_text, + empty_string_as_null, + )?; rows.push_cell(row, column, value, progress)?; current_cell = XlsxPreviewRawCell::default(); } @@ -2755,13 +2807,14 @@ fn parse_xlsx_range( where T: CellType, Label: Fn(&T, Option) -> String, - Value: Fn(&T, Option) -> serde_json::Value, + Value: Fn(&T, Option, bool) -> serde_json::Value, TextValue: Fn(&T, Option<&XlsxCellStyle>) -> Option, IsNumeric: Fn(&T) -> bool, { let (range_start_row, range_start_column) = range.start().map(|(row, column)| (row as usize, column as usize)).unwrap_or_default(); let row_range = effective_import_row_range(options)?; + let empty_string_as_null = options.empty_string_as_null.unwrap_or(true); let mut columns = Vec::new(); let mut rows = Vec::new(); let mut total_rows = 0; @@ -2812,7 +2865,7 @@ where return Ok(serde_json::Value::String(text)); } } - Ok(cell_value(cell, style.and_then(|style| style.temporal_kind))) + Ok(cell_value(cell, style.and_then(|style| style.temporal_kind), empty_string_as_null)) }) .transpose()? .unwrap_or(serde_json::Value::Null); @@ -6514,6 +6567,81 @@ mod tests { zip.finish().unwrap().into_inner() } + fn assert_xlsx_empty_string_option(options: TableImportParseOptions, expected_row: Vec) { + let path = + std::env::temp_dir().join(format!("dbx-table-import-empty-string-option-{}.xlsx", uuid::Uuid::new_v4())); + let sheet_xml = r#" + + + + + inline + shared + formula + absent + empty_cell + + + + 0 + "" + + + +"#; + let shared_strings_xml = r#" + + +"#; + std::fs::write(&path, build_preview_test_xlsx(sheet_xml, Some(shared_strings_xml))).unwrap(); + + let parsed = parse_xlsx_file_with_options(&path.to_string_lossy(), &options, 10).unwrap(); + let (preview, _) = parse_xlsx_preview_file_with_options(&path.to_string_lossy(), &options, 10).unwrap(); + let (sender, mut receiver) = tokio::sync::mpsc::channel(16); + stream_xlsx_rows_to_channel(&path.to_string_lossy(), &options, 500, None, HashSet::new(), false, sender) + .unwrap(); + + let mut streamed_columns = Vec::new(); + let mut streamed_rows = Vec::new(); + while let Some(message) = receiver.blocking_recv() { + match message.unwrap() { + XlsxStreamMessage::Header(columns) => streamed_columns = columns, + XlsxStreamMessage::Rows(rows) => streamed_rows.extend(rows), + _ => {} + } + } + + assert_eq!(parsed.columns, vec!["inline", "shared", "formula", "absent", "empty_cell"]); + assert_eq!(preview.columns, parsed.columns); + assert_eq!(streamed_columns, parsed.columns); + assert_eq!(parsed.rows, vec![expected_row.clone()]); + assert_eq!(preview.rows, parsed.rows); + assert_eq!(streamed_rows, parsed.rows); + let _ = std::fs::remove_file(path); + } + + #[test] + fn xlsx_preserves_explicit_empty_strings_when_configured() { + let options = + TableImportParseOptions { empty_string_as_null: Some(false), ..TableImportParseOptions::default() }; + + assert_xlsx_empty_string_option( + options, + vec![ + serde_json::json!(""), + serde_json::json!(""), + serde_json::json!(""), + serde_json::Value::Null, + serde_json::Value::Null, + ], + ); + } + + #[test] + fn xlsx_defaults_explicit_empty_strings_to_null() { + assert_xlsx_empty_string_option(TableImportParseOptions::default(), vec![serde_json::Value::Null; 5]); + } + #[test] fn retains_only_temporal_and_text_target_xlsx_styles() { let styles = vec![ @@ -8039,10 +8167,10 @@ mod tests { let date = DataRef::DateTime(ExcelDateTime::new(45996.0, calamine::ExcelDateTimeType::DateTime, false)); assert_eq!(xlsx_cell_ref_label_with_temporal_kind(&shared_string, None), "Ada"); - assert_eq!(xlsx_cell_ref_value_with_temporal_kind(&shared_string, None), serde_json::json!("Ada")); - assert_eq!(xlsx_cell_ref_value_with_temporal_kind(&number, None), serde_json::json!(42.5)); + assert_eq!(xlsx_cell_ref_value_with_temporal_kind(&shared_string, None, true), serde_json::json!("Ada")); + assert_eq!(xlsx_cell_ref_value_with_temporal_kind(&number, None, true), serde_json::json!(42.5)); assert_eq!( - xlsx_cell_ref_value_with_temporal_kind(&date, Some(XlsxTemporalKind::Date)), + xlsx_cell_ref_value_with_temporal_kind(&date, Some(XlsxTemporalKind::Date), true), serde_json::json!("2025-12-05") ); } @@ -9223,7 +9351,8 @@ mod tests { }]; let excel_date_time = Data::DateTime(ExcelDateTime::new(45959.686111111, calamine::ExcelDateTimeType::DateTime, false)); - let imported_value = xlsx_cell_value_with_temporal_kind(&excel_date_time, Some(XlsxTemporalKind::DateTime)); + let imported_value = + xlsx_cell_value_with_temporal_kind(&excel_date_time, Some(XlsxTemporalKind::DateTime), true); assert_eq!(imported_value, serde_json::json!("2025-10-29 16:28:00")); let data = ParsedImportFile { columns: vec!["created_at".to_string()],