fix(transfer): reduce import and export allocations

This commit is contained in:
t8y2 committed 2026-08-21 01:10:17 +08:00
1 parent b5d3343f9a
commit 30f75d8388
9 files changed
+822 -261

No files matched your search

+47 -12
View File
@@ -100,6 +100,15 @@ fn push_tsv_value(out: &mut String, value: &Value) {
}
}
pub(crate) fn push_tsv_row(out: &mut String, row: &[Value]) {
for (cell_index, cell) in row.iter().enumerate() {
if cell_index > 0 {
out.push('\t');
}
push_tsv_value(out, cell);
}
}
/// 预分配粗估:按全部行的实际单元格数求和(不假设等宽),饱和运算防溢出,
/// 并设上限——估算只是性能提示,绝不能因病态输入放大成巨额分配
const ROWS_CAPACITY_ESTIMATE_MAX: usize = 16 * 1024 * 1024;
@@ -133,15 +142,11 @@ fn push_tsv_rows(out: &mut String, rows: &[Vec<Value>]) {
if row_index > 0 {
out.push('\n');
}
for (cell_index, cell) in row.iter().enumerate() {
if cell_index > 0 {
out.push('\t');
}
push_tsv_value(out, cell);
}
push_tsv_row(out, row);
}
}
#[cfg_attr(not(test), allow(dead_code))]
pub(crate) fn format_tsv_rows(rows: &[Vec<Value>]) -> String {
let mut out = String::with_capacity(estimated_rows_capacity(rows));
push_tsv_rows(&mut out, rows);
@@ -166,17 +171,30 @@ pub(crate) fn format_tsv(columns: &[String], rows: &[Vec<Value>]) -> String {
/// Format query-result rows as CSV text without a header row. Database NULLs
/// use the same empty-cell representation as table-data exports. Used by the
/// streaming query-result export for batches after the first.
pub(crate) fn push_query_result_csv_row(out: &mut String, row: &[Value]) {
for (cell_index, cell) in row.iter().enumerate() {
if cell_index > 0 {
out.push(',');
}
push_csv_value(out, cell);
}
}
pub(crate) fn push_table_csv_row(out: &mut String, row: &[Value]) {
for (cell_index, cell) in row.iter().enumerate() {
if cell_index > 0 {
out.push(',');
}
push_csv_text_value(out, cell);
}
}
fn push_query_result_csv_rows(out: &mut String, rows: &[Vec<Value>]) {
for (row_index, row) in rows.iter().enumerate() {
if row_index > 0 {
out.push('\n');
}
for (cell_index, cell) in row.iter().enumerate() {
if cell_index > 0 {
out.push(',');
}
push_csv_value(out, cell);
}
push_query_result_csv_row(out, row);
}
}
@@ -399,5 +417,22 @@ mod tests {
}
}
#[test]
fn reusable_row_buffers_match_batch_formatters() {
let row = vec![Value::Null, json!("NULL"), json!("line\n\"two\""), json!(42)];
let mut csv = String::new();
super::push_query_result_csv_row(&mut csv, &row);
assert_eq!(csv, super::format_query_result_csv_rows(&[row.clone()]));
let mut table_csv = String::new();
super::push_table_csv_row(&mut table_csv, &row);
assert_eq!(table_csv, "\"\",\"NULL\",\"line\n\"\"two\"\"\",\"42\"");
let mut tsv = String::new();
super::push_tsv_row(&mut tsv, &row);
assert_eq!(tsv, super::format_tsv_rows(&[row]));
}
use serde_json::Value;
}