diff --git a/crates/dbx-core/src/data/table_import.rs b/crates/dbx-core/src/data/table_import.rs index 745003c9eb..9c55f01f44 100644 --- a/crates/dbx-core/src/data/table_import.rs +++ b/crates/dbx-core/src/data/table_import.rs @@ -2382,6 +2382,14 @@ pub fn xlsx_cell_label(cell: &Data) -> String { xlsx_cell_label_with_temporal_kind(cell, None) } +fn xlsx_cell_is_empty(cell: &Data) -> bool { + matches!(cell, Data::Empty) +} + +fn xlsx_cell_ref_is_empty(cell: &DataRef<'_>) -> bool { + matches!(cell, DataRef::Empty) +} + fn xlsx_cell_ref_value_with_temporal_kind( cell: &DataRef<'_>, temporal_kind: Option, @@ -2729,6 +2737,20 @@ struct XlsxPreviewRawCell { has_inline_value: bool, } +impl XlsxPreviewRawCell { + /// Whether the worksheet recorded a value for this cell. + /// + /// Excel and LibreOffice write style-only cells (``, no + /// ``/`` child) for rows a user inserted and left blank. Such a row + /// carries no data and must not be imported as an all-NULL row (#8604), + /// while any cell with a value element keeps the row - including explicit + /// empty strings (``), matching the `.xls` parser and dbx's + /// own empty-string options. + fn has_content(&self) -> bool { + self.has_inline_value || self.has_value + } +} + fn xlsx_dimension_bounds(reference: &str) -> Option<((usize, usize), (usize, usize))> { let mut parts = reference.split(':'); let start = xlsx_cell_ref_position(parts.next()?)?; @@ -3256,7 +3278,12 @@ fn parse_xlsx_preview_file_with_options( if last_preview_row < first_preview_row { return Err("Import file has no data rows in the selected row range".to_string()); } + // Rows made of style-only cells (blank rows inserted by a spreadsheet app) carry no + // data and are skipped here as well so the preview matches what the import writes (#8604). + let rows_with_content = + raw_cells.iter().filter(|(_, cell)| cell.has_content()).map(|((row, _), _)| *row).collect::>(); let rows = (first_preview_row..=last_preview_row) + .filter(|absolute_row| rows_with_content.contains(absolute_row)) .map(|absolute_row| { (0..columns.len()) .map(|index| { @@ -3384,6 +3411,7 @@ fn parse_xlsx_file_with_options_and_text_columns( xlsx_cell_ref_value_with_temporal_kind, xlsx_cell_ref_text_value, xlsx_cell_ref_is_numeric, + xlsx_cell_ref_is_empty, ); } @@ -3412,6 +3440,7 @@ fn parse_xlsx_file_with_options_and_text_columns( xlsx_cell_value_with_temporal_kind, xlsx_cell_text_value, xlsx_cell_is_numeric, + xlsx_cell_is_empty, ) } @@ -3482,6 +3511,7 @@ struct XlsxStreamRowsState { rows_seen: usize, current_row: Option, current_values: Vec, + current_row_has_content: bool, batch_size: usize, } @@ -3507,6 +3537,7 @@ impl XlsxStreamRowsState { rows_seen: 0, current_row: None, current_values: Vec::new(), + current_row_has_content: false, batch_size, } } @@ -3555,6 +3586,7 @@ impl XlsxStreamRowsState { absolute_row: usize, absolute_column: usize, value: serde_json::Value, + has_content: bool, progress: u64, ) -> Result<(), String> { self.initialize_range(absolute_row, absolute_column); @@ -3562,6 +3594,7 @@ impl XlsxStreamRowsState { self.flush_current_row(progress)?; self.current_row = Some(absolute_row); } + self.current_row_has_content |= has_content; let column_offset = absolute_column.checked_sub(self.start_column).ok_or_else(|| { format!("Excel row {absolute_row} contains a cell before the detected import range start column") })?; @@ -3580,13 +3613,15 @@ impl XlsxStreamRowsState { return Ok(()); }; let values = std::mem::take(&mut self.current_values); - self.flush_row(absolute_row, values, progress) + let has_content = std::mem::take(&mut self.current_row_has_content); + self.flush_row(absolute_row, values, has_content, progress) } fn flush_row( &mut self, absolute_row: usize, mut values: Vec, + has_content: bool, progress: u64, ) -> Result<(), String> { if self.row_range.title_row == Some(absolute_row) { @@ -3607,6 +3642,12 @@ impl XlsxStreamRowsState { { return Ok(()); } + // A row whose cells are all blank (Excel keeps style-only cells behind + // when a blank row is inserted) is not a data row: importing it would + // insert an all-NULL row and fail on NOT NULL columns (#8604). + if !has_content { + return Ok(()); + } if self.columns.is_empty() { let column_count = self.declared_column_count.unwrap_or(values.len()).max(values.len()); self.columns = (0..column_count).map(|index| format!("column_{}", index + 1)).collect(); @@ -3797,7 +3838,7 @@ fn stream_xlsx_rows_to_channel_with_control( .unwrap_or_else(|| (current_row.max(1), current_column.saturating_add(1).max(1))); current_row = position.0; current_column = position.1; - rows.push_cell(position.0, position.1, serde_json::Value::Null, progress)?; + rows.push_cell(position.0, position.1, serde_json::Value::Null, false, progress)?; } Ok(Event::Start(element)) if xml_local_name_eq(element.name().as_ref(), b"c") => { let position = xml_attr_value(&reader, &element, b"r") @@ -3852,7 +3893,7 @@ fn stream_xlsx_rows_to_channel_with_control( format_as_text, empty_string_as_null, )?; - rows.push_cell(row, column, value, progress)?; + rows.push_cell(row, column, value, current_cell.has_content(), progress)?; current_cell = XlsxPreviewRawCell::default(); } inline_phonetic_depth = 0; @@ -3954,7 +3995,8 @@ async fn validate_xlsx_worksheet_for_import( columns.ok_or_else(|| "Excel stream ended before providing a header".to_string()) } -fn parse_xlsx_range( +#[allow(clippy::too_many_arguments)] +fn parse_xlsx_range( range: &Range, options: &TableImportParseOptions, preview_limit: usize, @@ -3965,6 +4007,7 @@ fn parse_xlsx_range( cell_value: Value, cell_text_value: TextValue, is_numeric: IsNumeric, + is_empty: IsEmpty, ) -> Result where T: CellType, @@ -3972,6 +4015,7 @@ where Value: Fn(&T, Option, bool) -> serde_json::Value, TextValue: Fn(&T, Option<&XlsxCellStyle>) -> Option, IsNumeric: Fn(&T) -> bool, + IsEmpty: Fn(&T) -> bool, { let (range_start_row, range_start_column) = range.start().map(|(row, column)| (row as usize, column as usize)).unwrap_or_default(); @@ -3999,6 +4043,11 @@ where if row_range.last_data_row.is_some_and(|last| row_number > last) { break; } + // Style-only cells (blank rows inserted by a spreadsheet app) are not data, so the + // row must not be materialized as an all-NULL row (#8604). + if source_row.iter().all(&is_empty) { + continue; + } if columns.is_empty() { columns = (0..source_row.len()).map(|index| format!("column_{}", index + 1)).collect(); } @@ -8502,6 +8551,73 @@ mod tests { assert_xlsx_empty_string_option(TableImportParseOptions::default(), vec![serde_json::Value::Null; 5]); } + /// Excel/LibreOffice keep style-only cells behind when a user inserts a row and leaves it + /// blank. Such a row has no data and must not be imported as an all-NULL row - it used to + /// fail on NOT NULL target columns (#8604) - and the preview must agree with the import. + #[test] + fn xlsx_style_only_blank_rows_are_skipped_by_preview_parse_and_streaming() { + let blank_rows = [ + r#""#, + r#""#, + r#""#, + ]; + for blank_row in blank_rows { + let path = std::env::temp_dir().join(format!("dbx-table-import-blank-rows-{}.xlsx", uuid::Uuid::new_v4())); + let sheet_xml = format!( + r#" + + + + + id + other + + + 1 + a + + {blank_row} + + 2 + b + + +"# + ); + std::fs::write(&path, build_preview_test_xlsx(&sheet_xml, None)).unwrap(); + let options = TableImportParseOptions::default(); + let expected_rows = vec![ + vec![serde_json::json!(1), serde_json::json!("a")], + vec![serde_json::json!(2), serde_json::json!("b")], + ]; + + let parsed = parse_xlsx_file_with_options(&path.to_string_lossy(), &options, 10).unwrap(); + let (preview, _) = parse_xlsx_preview_file_with_options(&path.to_string_lossy(), &options, 10).unwrap(); + let (sender, mut receiver) = tokio::sync::mpsc::channel(16); + stream_xlsx_rows_to_channel(&path.to_string_lossy(), &options, 500, None, HashSet::new(), false, sender) + .unwrap(); + let mut streamed_columns = Vec::new(); + let mut streamed_rows = Vec::new(); + while let Some(message) = receiver.blocking_recv() { + match message.unwrap() { + XlsxStreamMessage::Header(columns) => streamed_columns = columns, + XlsxStreamMessage::Rows(rows) => streamed_rows.extend(rows), + _ => {} + } + } + + assert_eq!(parsed.columns, vec!["id", "other"], "{blank_row}"); + assert_eq!(parsed.rows, expected_rows, "{blank_row}"); + assert_eq!(parsed.total_rows, 2, "{blank_row}"); + assert_eq!(preview.columns, parsed.columns, "{blank_row}"); + assert_eq!(preview.rows, parsed.rows, "{blank_row}"); + assert_eq!(preview.total_rows, 2, "{blank_row}"); + assert_eq!(streamed_columns, parsed.columns, "{blank_row}"); + assert_eq!(streamed_rows, parsed.rows, "{blank_row}"); + let _ = std::fs::remove_file(path); + } + } + #[test] fn dbx_exported_xlsx_round_trip_preserves_empty_strings_when_configured() { let path =