use serde::{Deserialize, Serialize}; use serde_json::Value; use std::io::{Cursor, Seek, Write}; #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct XlsxWorksheetData { pub sheet_name: Option, pub columns: Vec, pub rows: Vec>, } /// Streaming XLSX writer that incrementally writes rows to a ZIP-backed /// workbook. This avoids accumulating all rows in memory before building the /// final file, drastically reducing peak memory for large exports. pub struct StreamingXlsxWriter { zip: zip::ZipWriter, columns: Vec, next_row_number: usize, } /// Estimate column widths from header names only (used by the streaming path /// where full row data is not available up-front). Each width is clamped to /// [10, 60] to stay within reasonable bounds. fn estimate_header_widths(columns: &[String]) -> Vec { columns.iter().map(|col| (col.chars().count() + 2).clamp(10, 60)).collect() } /// Build the `` XML fragment from a width slice. fn cols_xml(widths: &[usize]) -> String { widths .iter() .enumerate() .map(|(index, width)| { format!("", index + 1, index + 1, width) }) .collect() } /// Build a single `` XML fragment for the header row (row 1). pub(crate) fn header_row_xml(columns: &[String]) -> String { format!( "{}", columns .iter() .enumerate() .map(|(index, col)| cell_xml(Some(&Value::String(col.clone())), 0, index, Some(1))) .collect::() ) } /// Build a single `` XML fragment for a data row. pub(crate) fn data_row_xml(row_number: usize, columns: &[String], row: &[Value]) -> String { let cells = columns .iter() .enumerate() .map(|(col_index, _)| cell_xml(row.get(col_index), row_number - 1, col_index, None)) .collect::(); format!("{cells}") } /// Shared ZIP entry options for all XLSX parts. XLSX is a ZIP of XML, and the /// `inlineStr` cell encoding is highly repetitive, so Deflate typically shrinks /// the file several-fold over `Stored` (matching what Excel/Navicat produce). fn xlsx_zip_options() -> zip::write::SimpleFileOptions { zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated) } fn write_zip_entry(zip: &mut zip::ZipWriter, path: &str, content: &str) -> Result<(), String> { zip.start_file(path, xlsx_zip_options()).map_err(|err| err.to_string())?; zip.write_all(content.as_bytes()).map_err(|err| err.to_string()) } /// Start a new streaming XLSX workbook. The ZIP skeleton, worksheet header, /// column widths (estimated from header names) and the header row are written /// immediately. Callers then feed data rows via [`StreamingXlsxWriter::write_row`] /// and finalize with [`StreamingXlsxWriter::finish`]. pub(crate) fn start_streaming_xlsx_workbook( writer: W, sheet_name: Option<&str>, columns: &[String], ) -> Result, String> { let sheet_name = normalize_sheet_name(sheet_name); let widths = estimate_header_widths(columns); let mut zip = zip::ZipWriter::new(writer); write_zip_entry(&mut zip, "[Content_Types].xml", &content_types_xml())?; write_zip_entry(&mut zip, "_rels/.rels", root_rels_xml())?; write_zip_entry(&mut zip, "xl/workbook.xml", &workbook_xml(&sheet_name))?; write_zip_entry(&mut zip, "xl/_rels/workbook.xml.rels", &workbook_rels_xml())?; write_zip_entry(&mut zip, "xl/styles.xml", styles_xml())?; // Begin the sheet1.xml entry with header, frozen pane, column widths and // the header row. let options = xlsx_zip_options(); zip.start_file("xl/worksheets/sheet1.xml", options).map_err(|err| err.to_string())?; let sheet_header = format!( concat!( "", "", "", "", "", "", "{cols}", "" ), cols = cols_xml(&widths), ); zip.write_all(sheet_header.as_bytes()).map_err(|err| err.to_string())?; zip.write_all(header_row_xml(columns).as_bytes()).map_err(|err| err.to_string())?; Ok(StreamingXlsxWriter { zip, columns: columns.to_vec(), next_row_number: 2 }) } impl StreamingXlsxWriter { /// Append a single data row to the worksheet. pub fn write_row(&mut self, row: &[Value]) -> Result<(), String> { self.zip .write_all(data_row_xml(self.next_row_number, &self.columns, row).as_bytes()) .map_err(|err| err.to_string())?; self.next_row_number += 1; Ok(()) } /// Finalize the worksheet and close the ZIP archive. Returns the /// underlying writer so callers can flush / close it as needed. pub fn finish(mut self) -> Result { let row_count = self.next_row_number.saturating_sub(1); let range = sheet_range(self.columns.len(), row_count); self.zip .write_all(format!("").as_bytes()) .map_err(|err| err.to_string())?; self.zip.finish().map_err(|err| err.to_string()) } } /// Convenience wrapper that finalizes a streaming workbook. pub(crate) fn finish_streaming_xlsx_workbook(writer: StreamingXlsxWriter) -> Result { writer.finish() } fn escape_xml(value: &str) -> String { let mut result = String::with_capacity(value.len()); for ch in value.chars() { let code = ch as u32; if code != 9 && code != 10 && code != 13 && code < 32 { continue; } match ch { '&' => result.push_str("&"), '<' => result.push_str("<"), '>' => result.push_str(">"), '"' => result.push_str("""), _ => result.push(ch), } } result } fn column_name(index: usize) -> String { let mut out = String::new(); let mut n = index + 1; while n > 0 { let rem = (n - 1) % 26; out.push((b'A' + rem as u8) as char); n = (n - 1) / 26; } out.chars().rev().collect() } fn cell_ref(row_index: usize, col_index: usize) -> String { format!("{}{}", column_name(col_index), row_index + 1) } fn sheet_range(column_count: usize, row_count: usize) -> String { if column_count == 0 || row_count == 0 { return "A1".to_string(); } format!("A1:{}{}", column_name(column_count - 1), row_count) } fn normalize_sheet_name(input: Option<&str>) -> String { let base = input.unwrap_or("Sheet1"); let name: String = base .chars() .map(|ch| match ch { '[' | ']' | ':' | '*' | '?' | '/' | '\\' => ' ', _ => ch, }) .collect::() .trim() .to_string(); let fallback = if name.is_empty() { "Sheet1" } else { &name }; fallback.chars().take(31).collect() } fn value_text(value: Option<&Value>) -> String { match value { Some(Value::Null) | None => String::new(), Some(Value::Bool(v)) => v.to_string(), Some(Value::Number(n)) => n.to_string(), Some(Value::String(s)) => s.clone(), Some(other) => other.to_string(), } } fn estimate_column_widths(columns: &[String], rows: &[Vec]) -> Vec { columns .iter() .enumerate() .map(|(col_index, column)| { let max_len = std::iter::once(column.chars().count().min(60)) .chain(rows.iter().take(100).map(|row| value_text(row.get(col_index)).chars().count().min(60))) .fold(8usize, usize::max); (max_len + 2).clamp(10, 60) }) .collect() } fn cell_xml(value: Option<&Value>, row_index: usize, col_index: usize, style: Option) -> String { let reference = cell_ref(row_index, col_index); let style_attr = style.map_or(String::new(), |s| format!(" s=\"{s}\"")); match value { Some(Value::Null) | None => format!(""), Some(Value::Bool(v)) => { let bool_v = if *v { 1 } else { 0 }; format!("{bool_v}") } Some(Value::Number(n)) => { if n.as_f64().is_some_and(|f| f.is_finite()) { format!("{}", n) } else { format!( "{}", escape_xml(&n.to_string()) ) } } Some(Value::String(s)) => { format!("{}", escape_xml(s)) } Some(other) => format!( "{}", escape_xml(&other.to_string()) ), } } fn worksheet_xml(data: &XlsxWorksheetData) -> String { let total_rows = data.rows.len() + 1; let range = sheet_range(data.columns.len(), total_rows); let widths = estimate_column_widths(&data.columns, &data.rows); let cols_xml = widths .iter() .enumerate() .map(|(index, width)| { format!("", index + 1, index + 1, width) }) .collect::(); let header_xml = format!( "{}", data.columns .iter() .enumerate() .map(|(index, col)| cell_xml(Some(&Value::String(col.clone())), 0, index, Some(1))) .collect::() ); let body_xml = data .rows .iter() .enumerate() .map(|(row_index, row)| { let excel_row = row_index + 2; let cells = data .columns .iter() .enumerate() .map(|(col_index, _)| cell_xml(row.get(col_index), excel_row - 1, col_index, None)) .collect::(); format!("{cells}") }) .collect::(); format!( concat!( "", "", "", "", "", "{cols_xml}", "{header_xml}{body_xml}", "", "" ), range = range, cols_xml = cols_xml, header_xml = header_xml, body_xml = body_xml, ) } fn content_types_xml() -> String { content_types_xml_for_sheet_count(1) } fn content_types_xml_for_sheet_count(sheet_count: usize) -> String { let worksheet_overrides = (1..=sheet_count) .map(|index| { format!( "" ) }) .collect::(); format!( concat!( "", "", "", "", "", "{}", "", "" ), worksheet_overrides ) } fn root_rels_xml() -> &'static str { concat!( "", "", "", "" ) } fn workbook_xml(sheet_name: &str) -> String { workbook_xml_for_sheets(&[sheet_name.to_string()]) } fn workbook_xml_for_sheets(sheet_names: &[String]) -> String { let sheets = sheet_names .iter() .enumerate() .map(|(index, sheet_name)| { let sheet_id = index + 1; format!("", escape_xml(sheet_name)) }) .collect::(); format!( concat!( "", "", "{}", "" ), sheets ) } fn workbook_rels_xml() -> String { workbook_rels_xml_for_sheet_count(1) } fn workbook_rels_xml_for_sheet_count(sheet_count: usize) -> String { let worksheet_rels = (1..=sheet_count) .map(|index| { format!( "" ) }) .collect::(); let styles_id = sheet_count + 1; format!( concat!( "", "", "{}", "", "" ), worksheet_rels, styles_id ) } fn styles_xml() -> &'static str { concat!( "", "", "", "", "", "", "", "", "" ) } fn normalize_unique_sheet_names(sheets: &[XlsxWorksheetData]) -> Vec { let mut names = Vec::with_capacity(sheets.len()); for (index, sheet) in sheets.iter().enumerate() { let base = normalize_sheet_name(sheet.sheet_name.as_deref().or(Some(&format!("Sheet{}", index + 1)))); let mut candidate = base.clone(); let mut suffix = 2; while names.iter().any(|name| name == &candidate) { let suffix_text = format!(" ({suffix})"); let max_base_len = 31usize.saturating_sub(suffix_text.chars().count()); candidate = format!("{}{}", base.chars().take(max_base_len).collect::(), suffix_text); suffix += 1; } names.push(candidate); } names } pub fn build_xlsx_workbook(data: &XlsxWorksheetData) -> Result, String> { build_xlsx_workbook_multi(std::slice::from_ref(data)) } pub fn build_xlsx_workbook_multi(sheets: &[XlsxWorksheetData]) -> Result, String> { if sheets.is_empty() { return Err("At least one worksheet is required".to_string()); } let sheet_names = normalize_unique_sheet_names(sheets); let files = vec![ ("[Content_Types].xml", content_types_xml_for_sheet_count(sheets.len())), ("_rels/.rels", root_rels_xml().to_string()), ("xl/workbook.xml", workbook_xml_for_sheets(&sheet_names)), ("xl/_rels/workbook.xml.rels", workbook_rels_xml_for_sheet_count(sheets.len())), ("xl/styles.xml", styles_xml().to_string()), ]; let cursor = Cursor::new(Vec::::new()); let mut zip = zip::ZipWriter::new(cursor); let options = xlsx_zip_options(); for (path, content) in files { zip.start_file(path, options).map_err(|err| err.to_string())?; zip.write_all(content.as_bytes()).map_err(|err| err.to_string())?; } for (index, sheet) in sheets.iter().enumerate() { zip.start_file(format!("xl/worksheets/sheet{}.xml", index + 1), options).map_err(|err| err.to_string())?; zip.write_all(worksheet_xml(sheet).as_bytes()).map_err(|err| err.to_string())?; } let output = zip.finish().map_err(|err| err.to_string())?; Ok(output.into_inner()) } #[cfg(test)] mod tests { use super::{build_xlsx_workbook, build_xlsx_workbook_multi, start_streaming_xlsx_workbook, XlsxWorksheetData}; use calamine::{open_workbook_auto, Reader}; use serde_json::json; use std::fs; use std::io::Read; /// Read and decompress a single entry from an in-memory XLSX (ZIP) buffer. fn read_zip_entry(bytes: &[u8], path: &str) -> String { let mut archive = zip::ZipArchive::new(std::io::Cursor::new(bytes.to_vec())).expect("open xlsx as zip archive"); let mut entry = archive.by_name(path).unwrap_or_else(|_| panic!("missing zip entry: {path}")); let mut content = String::new(); entry.read_to_string(&mut content).expect("read zip entry"); content } /// Assert every entry in the XLSX (ZIP) buffer is Deflate-compressed, which /// is what keeps exported workbooks small (see `xlsx_zip_options`). fn assert_all_entries_deflated(bytes: &[u8]) { let mut archive = zip::ZipArchive::new(std::io::Cursor::new(bytes.to_vec())).expect("open xlsx as zip archive"); for index in 0..archive.len() { let entry = archive.by_index(index).expect("zip entry"); assert_eq!( entry.compression(), zip::CompressionMethod::Deflated, "entry {} should be Deflate-compressed", entry.name() ); } } #[test] fn builds_xlsx_zip_with_sheet_data() { let workbook = build_xlsx_workbook(&XlsxWorksheetData { sheet_name: Some("Users".to_string()), columns: vec!["id".to_string(), "name".to_string(), "active".to_string()], rows: vec![vec![json!(1), json!("Ada & Bob"), json!(true)], vec![json!(2), json!(null), json!(false)]], }) .expect("build workbook"); // ZIP magic bytes. assert_eq!(workbook[0], 0x50); assert_eq!(workbook[1], 0x4b); // Entries are stored compressed; assert on their decompressed contents. let sheet = read_zip_entry(&workbook, "xl/worksheets/sheet1.xml"); let workbook_xml = read_zip_entry(&workbook, "xl/workbook.xml"); assert!(workbook_xml.contains("name=\"Users\"")); assert!(sheet.contains("1")); assert!(sheet.contains("Ada & Bob")); assert!(sheet.contains("1")); assert_all_entries_deflated(&workbook); } #[test] fn sanitizes_invalid_sheet_name() { let workbook = build_xlsx_workbook(&XlsxWorksheetData { sheet_name: Some("bad/name:with*chars?and-a-very-long-tail".to_string()), columns: vec!["value".to_string()], rows: vec![vec![json!("ok")]], }) .expect("build workbook"); let workbook_xml = read_zip_entry(&workbook, "xl/workbook.xml"); assert!(workbook_xml.contains("name=\"bad name with chars and-a-very-\"")); } #[test] fn builds_multi_sheet_xlsx_workbook() { let path = std::env::temp_dir().join(format!("dbx-multi-sheet-test-{}.xlsx", uuid::Uuid::new_v4())); let workbook = build_xlsx_workbook_multi(&[ XlsxWorksheetData { sheet_name: Some("Result 1".to_string()), columns: vec!["id".to_string()], rows: vec![vec![json!(1)]], }, XlsxWorksheetData { sheet_name: Some("Result 2".to_string()), columns: vec!["name".to_string()], rows: vec![vec![json!("Ada")]], }, ]) .expect("build multi-sheet workbook"); fs::write(&path, workbook).expect("write temp workbook"); let mut workbook = open_workbook_auto(&path).expect("open workbook"); let names = workbook.sheet_names().to_vec(); assert_eq!(names, vec!["Result 1".to_string(), "Result 2".to_string()]); let first = workbook.worksheet_range("Result 1").expect("read first worksheet"); let second = workbook.worksheet_range("Result 2").expect("read second worksheet"); assert_eq!(first.get_value((1, 0)).expect("first row"), &calamine::Data::Float(1.0)); assert_eq!(second.get_value((1, 0)).expect("second row"), &calamine::Data::String("Ada".to_string())); let _ = fs::remove_file(&path); } #[test] fn streams_xlsx_rows_to_a_readable_workbook() { let path = std::env::temp_dir().join(format!("dbx-stream-test-{}.xlsx", uuid::Uuid::new_v4())); { let file = fs::File::create(&path).expect("create temp xlsx"); let mut writer = start_streaming_xlsx_workbook(file, Some("Streamed"), &["id".to_string(), "name".to_string()]) .expect("start workbook"); writer.write_row(&[json!(1), json!("Ada")]).expect("write row"); writer.write_row(&[json!(2), json!("Bob")]).expect("write row"); drop(writer.finish().expect("finish workbook")); } let mut workbook = open_workbook_auto(&path).expect("open workbook"); let range = workbook.worksheet_range("Streamed").expect("read worksheet"); assert_eq!(range.get_value((0, 0)).expect("header"), &calamine::Data::String("id".to_string())); assert_eq!(range.get_value((1, 0)).expect("row1"), &calamine::Data::Float(1.0)); assert_eq!(range.get_value((2, 1)).expect("row2"), &calamine::Data::String("Bob".to_string())); let _ = fs::remove_file(&path); } }