fix(import): skip styled blank rows in Excel import

This commit is contained in:
zipg authored and GitHub committed 2026-09-27 10:26:02 +08:00
1 parent 9962796a04
commit 26138a6236
1 file changed
+120 -4
+120 -4
View File
@@ -2382,6 +2382,14 @@ pub fn xlsx_cell_label(cell: &Data) -> String {
xlsx_cell_label_with_temporal_kind(cell, None) xlsx_cell_label_with_temporal_kind(cell, None)
} }
fn xlsx_cell_is_empty(cell: &Data) -> bool {
matches!(cell, Data::Empty)
}
fn xlsx_cell_ref_is_empty(cell: &DataRef<'_>) -> bool {
matches!(cell, DataRef::Empty)
}
fn xlsx_cell_ref_value_with_temporal_kind( fn xlsx_cell_ref_value_with_temporal_kind(
cell: &DataRef<'_>, cell: &DataRef<'_>,
temporal_kind: Option<XlsxTemporalKind>, temporal_kind: Option<XlsxTemporalKind>,
@@ -2729,6 +2737,20 @@ struct XlsxPreviewRawCell {
has_inline_value: bool, has_inline_value: bool,
} }
impl XlsxPreviewRawCell {
/// Whether the worksheet recorded a value for this cell.
///
/// Excel and LibreOffice write style-only cells (`<c r="A3" s="4"/>`, no
/// `<v>`/`<is>` child) for rows a user inserted and left blank. Such a row
/// carries no data and must not be imported as an all-NULL row (#8604),
/// while any cell with a value element keeps the row - including explicit
/// empty strings (`<is><t></t></is>`), matching the `.xls` parser and dbx's
/// own empty-string options.
fn has_content(&self) -> bool {
self.has_inline_value || self.has_value
}
}
fn xlsx_dimension_bounds(reference: &str) -> Option<((usize, usize), (usize, usize))> { fn xlsx_dimension_bounds(reference: &str) -> Option<((usize, usize), (usize, usize))> {
let mut parts = reference.split(':'); let mut parts = reference.split(':');
let start = xlsx_cell_ref_position(parts.next()?)?; let start = xlsx_cell_ref_position(parts.next()?)?;
@@ -3256,7 +3278,12 @@ fn parse_xlsx_preview_file_with_options(
if last_preview_row < first_preview_row { if last_preview_row < first_preview_row {
return Err("Import file has no data rows in the selected row range".to_string()); return Err("Import file has no data rows in the selected row range".to_string());
} }
// Rows made of style-only cells (blank rows inserted by a spreadsheet app) carry no
// data and are skipped here as well so the preview matches what the import writes (#8604).
let rows_with_content =
raw_cells.iter().filter(|(_, cell)| cell.has_content()).map(|((row, _), _)| *row).collect::<HashSet<_>>();
let rows = (first_preview_row..=last_preview_row) let rows = (first_preview_row..=last_preview_row)
.filter(|absolute_row| rows_with_content.contains(absolute_row))
.map(|absolute_row| { .map(|absolute_row| {
(0..columns.len()) (0..columns.len())
.map(|index| { .map(|index| {
@@ -3384,6 +3411,7 @@ fn parse_xlsx_file_with_options_and_text_columns(
xlsx_cell_ref_value_with_temporal_kind, xlsx_cell_ref_value_with_temporal_kind,
xlsx_cell_ref_text_value, xlsx_cell_ref_text_value,
xlsx_cell_ref_is_numeric, xlsx_cell_ref_is_numeric,
xlsx_cell_ref_is_empty,
); );
} }
@@ -3412,6 +3440,7 @@ fn parse_xlsx_file_with_options_and_text_columns(
xlsx_cell_value_with_temporal_kind, xlsx_cell_value_with_temporal_kind,
xlsx_cell_text_value, xlsx_cell_text_value,
xlsx_cell_is_numeric, xlsx_cell_is_numeric,
xlsx_cell_is_empty,
) )
} }
@@ -3482,6 +3511,7 @@ struct XlsxStreamRowsState {
rows_seen: usize, rows_seen: usize,
current_row: Option<usize>, current_row: Option<usize>,
current_values: Vec<serde_json::Value>, current_values: Vec<serde_json::Value>,
current_row_has_content: bool,
batch_size: usize, batch_size: usize,
} }
@@ -3507,6 +3537,7 @@ impl XlsxStreamRowsState {
rows_seen: 0, rows_seen: 0,
current_row: None, current_row: None,
current_values: Vec::new(), current_values: Vec::new(),
current_row_has_content: false,
batch_size, batch_size,
} }
} }
@@ -3555,6 +3586,7 @@ impl XlsxStreamRowsState {
absolute_row: usize, absolute_row: usize,
absolute_column: usize, absolute_column: usize,
value: serde_json::Value, value: serde_json::Value,
has_content: bool,
progress: u64, progress: u64,
) -> Result<(), String> { ) -> Result<(), String> {
self.initialize_range(absolute_row, absolute_column); self.initialize_range(absolute_row, absolute_column);
@@ -3562,6 +3594,7 @@ impl XlsxStreamRowsState {
self.flush_current_row(progress)?; self.flush_current_row(progress)?;
self.current_row = Some(absolute_row); self.current_row = Some(absolute_row);
} }
self.current_row_has_content |= has_content;
let column_offset = absolute_column.checked_sub(self.start_column).ok_or_else(|| { let column_offset = absolute_column.checked_sub(self.start_column).ok_or_else(|| {
format!("Excel row {absolute_row} contains a cell before the detected import range start column") format!("Excel row {absolute_row} contains a cell before the detected import range start column")
})?; })?;
@@ -3580,13 +3613,15 @@ impl XlsxStreamRowsState {
return Ok(()); return Ok(());
}; };
let values = std::mem::take(&mut self.current_values); let values = std::mem::take(&mut self.current_values);
self.flush_row(absolute_row, values, progress) let has_content = std::mem::take(&mut self.current_row_has_content);
self.flush_row(absolute_row, values, has_content, progress)
} }
fn flush_row( fn flush_row(
&mut self, &mut self,
absolute_row: usize, absolute_row: usize,
mut values: Vec<serde_json::Value>, mut values: Vec<serde_json::Value>,
has_content: bool,
progress: u64, progress: u64,
) -> Result<(), String> { ) -> Result<(), String> {
if self.row_range.title_row == Some(absolute_row) { if self.row_range.title_row == Some(absolute_row) {
@@ -3607,6 +3642,12 @@ impl XlsxStreamRowsState {
{ {
return Ok(()); return Ok(());
} }
// A row whose cells are all blank (Excel keeps style-only cells behind
// when a blank row is inserted) is not a data row: importing it would
// insert an all-NULL row and fail on NOT NULL columns (#8604).
if !has_content {
return Ok(());
}
if self.columns.is_empty() { if self.columns.is_empty() {
let column_count = self.declared_column_count.unwrap_or(values.len()).max(values.len()); let column_count = self.declared_column_count.unwrap_or(values.len()).max(values.len());
self.columns = (0..column_count).map(|index| format!("column_{}", index + 1)).collect(); self.columns = (0..column_count).map(|index| format!("column_{}", index + 1)).collect();
@@ -3797,7 +3838,7 @@ fn stream_xlsx_rows_to_channel_with_control(
.unwrap_or_else(|| (current_row.max(1), current_column.saturating_add(1).max(1))); .unwrap_or_else(|| (current_row.max(1), current_column.saturating_add(1).max(1)));
current_row = position.0; current_row = position.0;
current_column = position.1; current_column = position.1;
rows.push_cell(position.0, position.1, serde_json::Value::Null, progress)?; rows.push_cell(position.0, position.1, serde_json::Value::Null, false, progress)?;
} }
Ok(Event::Start(element)) if xml_local_name_eq(element.name().as_ref(), b"c") => { Ok(Event::Start(element)) if xml_local_name_eq(element.name().as_ref(), b"c") => {
let position = xml_attr_value(&reader, &element, b"r") let position = xml_attr_value(&reader, &element, b"r")
@@ -3852,7 +3893,7 @@ fn stream_xlsx_rows_to_channel_with_control(
format_as_text, format_as_text,
empty_string_as_null, empty_string_as_null,
)?; )?;
rows.push_cell(row, column, value, progress)?; rows.push_cell(row, column, value, current_cell.has_content(), progress)?;
current_cell = XlsxPreviewRawCell::default(); current_cell = XlsxPreviewRawCell::default();
} }
inline_phonetic_depth = 0; inline_phonetic_depth = 0;
@@ -3954,7 +3995,8 @@ async fn validate_xlsx_worksheet_for_import(
columns.ok_or_else(|| "Excel stream ended before providing a header".to_string()) columns.ok_or_else(|| "Excel stream ended before providing a header".to_string())
} }
fn parse_xlsx_range<T, Label, Value, TextValue, IsNumeric>( #[allow(clippy::too_many_arguments)]
fn parse_xlsx_range<T, Label, Value, TextValue, IsNumeric, IsEmpty>(
range: &Range<T>, range: &Range<T>,
options: &TableImportParseOptions, options: &TableImportParseOptions,
preview_limit: usize, preview_limit: usize,
@@ -3965,6 +4007,7 @@ fn parse_xlsx_range<T, Label, Value, TextValue, IsNumeric>(
cell_value: Value, cell_value: Value,
cell_text_value: TextValue, cell_text_value: TextValue,
is_numeric: IsNumeric, is_numeric: IsNumeric,
is_empty: IsEmpty,
) -> Result<ParsedImportFile, String> ) -> Result<ParsedImportFile, String>
where where
T: CellType, T: CellType,
@@ -3972,6 +4015,7 @@ where
Value: Fn(&T, Option<XlsxTemporalKind>, bool) -> serde_json::Value, Value: Fn(&T, Option<XlsxTemporalKind>, bool) -> serde_json::Value,
TextValue: Fn(&T, Option<&XlsxCellStyle>) -> Option<String>, TextValue: Fn(&T, Option<&XlsxCellStyle>) -> Option<String>,
IsNumeric: Fn(&T) -> bool, IsNumeric: Fn(&T) -> bool,
IsEmpty: Fn(&T) -> bool,
{ {
let (range_start_row, range_start_column) = let (range_start_row, range_start_column) =
range.start().map(|(row, column)| (row as usize, column as usize)).unwrap_or_default(); range.start().map(|(row, column)| (row as usize, column as usize)).unwrap_or_default();
@@ -3999,6 +4043,11 @@ where
if row_range.last_data_row.is_some_and(|last| row_number > last) { if row_range.last_data_row.is_some_and(|last| row_number > last) {
break; break;
} }
// Style-only cells (blank rows inserted by a spreadsheet app) are not data, so the
// row must not be materialized as an all-NULL row (#8604).
if source_row.iter().all(&is_empty) {
continue;
}
if columns.is_empty() { if columns.is_empty() {
columns = (0..source_row.len()).map(|index| format!("column_{}", index + 1)).collect(); columns = (0..source_row.len()).map(|index| format!("column_{}", index + 1)).collect();
} }
@@ -8502,6 +8551,73 @@ mod tests {
assert_xlsx_empty_string_option(TableImportParseOptions::default(), vec![serde_json::Value::Null; 5]); assert_xlsx_empty_string_option(TableImportParseOptions::default(), vec![serde_json::Value::Null; 5]);
} }
/// Excel/LibreOffice keep style-only cells behind when a user inserts a row and leaves it
/// blank. Such a row has no data and must not be imported as an all-NULL row - it used to
/// fail on NOT NULL target columns (#8604) - and the preview must agree with the import.
#[test]
fn xlsx_style_only_blank_rows_are_skipped_by_preview_parse_and_streaming() {
let blank_rows = [
r#"<c r="A3" s="0"/><c r="B3" s="0"/>"#,
r#"<c r="A3" s="0"></c><c r="B3" s="0"></c>"#,
r#"<c r="A3"/><c r="B3"/>"#,
];
for blank_row in blank_rows {
let path = std::env::temp_dir().join(format!("dbx-table-import-blank-rows-{}.xlsx", uuid::Uuid::new_v4()));
let sheet_xml = format!(
r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">
<dimension ref="A1:B4"/>
<sheetData>
<row r="1">
<c r="A1" t="inlineStr"><is><t>id</t></is></c>
<c r="B1" t="inlineStr"><is><t>other</t></is></c>
</row>
<row r="2">
<c r="A2"><v>1</v></c>
<c r="B2" t="inlineStr"><is><t>a</t></is></c>
</row>
<row r="3">{blank_row}</row>
<row r="4">
<c r="A4"><v>2</v></c>
<c r="B4" t="inlineStr"><is><t>b</t></is></c>
</row>
</sheetData>
</worksheet>"#
);
std::fs::write(&path, build_preview_test_xlsx(&sheet_xml, None)).unwrap();
let options = TableImportParseOptions::default();
let expected_rows = vec![
vec![serde_json::json!(1), serde_json::json!("a")],
vec![serde_json::json!(2), serde_json::json!("b")],
];
let parsed = parse_xlsx_file_with_options(&path.to_string_lossy(), &options, 10).unwrap();
let (preview, _) = parse_xlsx_preview_file_with_options(&path.to_string_lossy(), &options, 10).unwrap();
let (sender, mut receiver) = tokio::sync::mpsc::channel(16);
stream_xlsx_rows_to_channel(&path.to_string_lossy(), &options, 500, None, HashSet::new(), false, sender)
.unwrap();
let mut streamed_columns = Vec::new();
let mut streamed_rows = Vec::new();
while let Some(message) = receiver.blocking_recv() {
match message.unwrap() {
XlsxStreamMessage::Header(columns) => streamed_columns = columns,
XlsxStreamMessage::Rows(rows) => streamed_rows.extend(rows),
_ => {}
}
}
assert_eq!(parsed.columns, vec!["id", "other"], "{blank_row}");
assert_eq!(parsed.rows, expected_rows, "{blank_row}");
assert_eq!(parsed.total_rows, 2, "{blank_row}");
assert_eq!(preview.columns, parsed.columns, "{blank_row}");
assert_eq!(preview.rows, parsed.rows, "{blank_row}");
assert_eq!(preview.total_rows, 2, "{blank_row}");
assert_eq!(streamed_columns, parsed.columns, "{blank_row}");
assert_eq!(streamed_rows, parsed.rows, "{blank_row}");
let _ = std::fs::remove_file(path);
}
}
#[test] #[test]
fn dbx_exported_xlsx_round_trip_preserves_empty_strings_when_configured() { fn dbx_exported_xlsx_round_trip_preserves_empty_strings_when_configured() {
let path = let path =