fix(export): mark NULL as \N in CSV and TSV exports
This commit is contained in:
49 files changed
+1056
-130
No files matched your search
@@ -30,6 +30,10 @@ pub struct TableCsvExportOptions {
|
||||
pub timeout_secs: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub csv_quote_mode: CsvQuoteMode,
|
||||
/// CSV 里 NULL 写成什么。默认 `\N`;空字符串表示关闭该字面量,
|
||||
/// 退回「NULL 写成空字段」的旧行为(此时 NULL 与空字符串无法区分)。
|
||||
#[serde(default = "default_csv_null_literal")]
|
||||
pub null_literal: String,
|
||||
}
|
||||
|
||||
async fn connection_database_type(state: &AppState, connection_id: &str) -> Result<DatabaseType, String> {
|
||||
@@ -156,7 +160,12 @@ where
|
||||
|
||||
for row in result.rows {
|
||||
writer.write_all(b"\n").map_err(|err| err.to_string())?;
|
||||
write_csv_value_row(writer, row, options.csv_quote_mode)?;
|
||||
write_csv_value_row_with_options(
|
||||
writer,
|
||||
row,
|
||||
options.csv_quote_mode,
|
||||
csv_null_literal(&options.null_literal),
|
||||
)?;
|
||||
}
|
||||
|
||||
rows_exported += fetched as u64;
|
||||
@@ -191,6 +200,7 @@ mod tests {
|
||||
page_size: Some(page_size),
|
||||
timeout_secs: Some(30),
|
||||
csv_quote_mode: CsvQuoteMode::default(),
|
||||
null_literal: String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -354,6 +364,194 @@ mod tests {
|
||||
config
|
||||
}
|
||||
|
||||
/// 表数据导出必须把 NULL 写成字面量、空字符串写成 `""`,否则导入端拿不到引号信息,
|
||||
/// 两者都会变成 NULL(正是「NOT NULL DEFAULT '' 的列导入报错」的成因)。
|
||||
#[tokio::test]
|
||||
async fn streamed_table_export_writes_null_literal_and_keeps_empty_strings() {
|
||||
fn result_with_a_null_and_an_empty_string() -> QueryResult {
|
||||
serde_json::from_value(serde_json::json!({
|
||||
"columns": ["name", "note"],
|
||||
"rows": [["a", null], ["b", ""]],
|
||||
"affected_rows": 0,
|
||||
"execution_time_ms": 1,
|
||||
"session_id": null,
|
||||
"has_more": false
|
||||
}))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
let mut options = export_options(10);
|
||||
options.null_literal = dbx_formats::csv_export::DEFAULT_CSV_NULL_LITERAL.to_string();
|
||||
let mut pages = vec![result_with_a_null_and_an_empty_string()].into_iter();
|
||||
let mut output = Vec::new();
|
||||
write_table_csv_pages(&mut output, DatabaseType::Postgres, &options, None, |_, _| {
|
||||
std::future::ready(Ok(pages.next().expect("single page")))
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(String::from_utf8(output).unwrap(), "\"name\",\"note\"\n\"a\",\\N\n\"b\",\"\"");
|
||||
|
||||
// 关闭字面量(旧行为):NULL 与空串都写空字段,二者在文件里已无法区分
|
||||
options.null_literal = String::new();
|
||||
let mut pages = vec![result_with_a_null_and_an_empty_string()].into_iter();
|
||||
let mut output = Vec::new();
|
||||
write_table_csv_pages(&mut output, DatabaseType::Postgres, &options, None, |_, _| {
|
||||
std::future::ready(Ok(pages.next().expect("single page")))
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(String::from_utf8(output).unwrap(), "\"name\",\"note\"\n\"a\",\n\"b\",\"\"");
|
||||
}
|
||||
|
||||
/// 用户反馈:`NOT NULL DEFAULT ''` 的列导出成 CSV、再导入到另一个环境的同名表时报
|
||||
/// 「不允许为 null」。用真实 SQLite 库走一遍「导出 → 导入」,确认空字符串仍是空字符串、
|
||||
/// NULL 仍是 NULL(而不是双双塌成 NULL)。
|
||||
#[tokio::test]
|
||||
async fn exported_csv_round_trips_empty_strings_and_nulls_through_a_real_import() {
|
||||
use crate::data::table_import::{
|
||||
import_table_file_core, TableImportColumnMapping, TableImportMode, TableImportParseOptions,
|
||||
TableImportRequest, TableImportSourceFormat,
|
||||
};
|
||||
use std::io::Write;
|
||||
|
||||
const DDL: &str = "CREATE TABLE contacts (id INTEGER, name TEXT NOT NULL DEFAULT '', note TEXT)";
|
||||
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let source_path = directory.path().join("source.db");
|
||||
let sqlite = crate::db::sqlite::connect_path_create_if_missing(source_path.to_str().unwrap()).await.unwrap();
|
||||
crate::db::sqlite::execute_query(&sqlite, DDL).await.unwrap();
|
||||
crate::db::sqlite::execute_query(
|
||||
&sqlite,
|
||||
"INSERT INTO contacts (id, name, note) VALUES (1, '', NULL), (2, 'Ada', 'x')",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut options = export_options(100);
|
||||
options.table_name = "contacts".to_string();
|
||||
options.columns = vec!["id".to_string(), "name".to_string(), "note".to_string()];
|
||||
options.null_literal = dbx_formats::csv_export::DEFAULT_CSV_NULL_LITERAL.to_string();
|
||||
|
||||
let mut exported_bytes = "\u{FEFF}".as_bytes().to_vec();
|
||||
let exported = write_table_csv_pages(&mut exported_bytes, DatabaseType::Sqlite, &options, None, |sql, _| {
|
||||
let sqlite = sqlite.clone();
|
||||
async move { crate::db::sqlite::execute_query(&sqlite, &sql).await.map_err(|error| error.to_string()) }
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(exported, 2);
|
||||
// 导出的 NULL 与空串必须已经可区分:空串 `""`、NULL 裸写 `\N`
|
||||
assert_eq!(
|
||||
String::from_utf8(exported_bytes.clone()).unwrap(),
|
||||
"\u{FEFF}\"id\",\"name\",\"note\"\n\"1\",\"\",\\N\n\"2\",\"Ada\",\"x\""
|
||||
);
|
||||
|
||||
let csv_path = directory.path().join("contacts.csv");
|
||||
let mut csv_file = std::fs::File::create(&csv_path).unwrap();
|
||||
csv_file.write_all(&exported_bytes).unwrap();
|
||||
csv_file.flush().unwrap();
|
||||
drop(csv_file);
|
||||
|
||||
// 目标环境:同名 DDL,`name` 同样是 NOT NULL DEFAULT ''
|
||||
let target_path = directory.path().join("target.db");
|
||||
let target = crate::db::sqlite::connect_path_create_if_missing(target_path.to_str().unwrap()).await.unwrap();
|
||||
crate::db::sqlite::execute_query(&target, DDL).await.unwrap();
|
||||
|
||||
let storage = crate::persistence::test_storage::open(&directory.path().join("storage.db")).await.unwrap();
|
||||
let state = AppState::new(storage);
|
||||
let connection_id = "csv-round-trip";
|
||||
let pool_key = format!("{connection_id}:session:import");
|
||||
state
|
||||
.update_connection_pools(|connections| {
|
||||
connections.insert(pool_key.clone(), crate::connection::PoolKind::Sqlite(target.clone()));
|
||||
})
|
||||
.await;
|
||||
let config: ConnectionConfig = serde_json::from_value(serde_json::json!({
|
||||
"id": connection_id,
|
||||
"name": "CSV round-trip",
|
||||
"db_type": "sqlite",
|
||||
"host": "",
|
||||
"port": 0,
|
||||
"username": "",
|
||||
"password": "",
|
||||
"database": target_path.to_string_lossy()
|
||||
}))
|
||||
.unwrap();
|
||||
state.configs.write().await.insert(connection_id.to_string(), config);
|
||||
|
||||
let request = TableImportRequest {
|
||||
import_id: "csv-round-trip".to_string(),
|
||||
connection_id: connection_id.to_string(),
|
||||
database: String::new(),
|
||||
schema: String::new(),
|
||||
table: "contacts".to_string(),
|
||||
file_path: csv_path.to_string_lossy().into_owned(),
|
||||
source_ref: None,
|
||||
source_format: Some(TableImportSourceFormat::Csv),
|
||||
parse_options: TableImportParseOptions::default(),
|
||||
mappings: ["id", "name", "note"]
|
||||
.into_iter()
|
||||
.map(|column| TableImportColumnMapping {
|
||||
source_column: column.to_string(),
|
||||
target_column: column.to_string(),
|
||||
target_data_type: None,
|
||||
})
|
||||
.collect(),
|
||||
mode: TableImportMode::Append,
|
||||
create_table: false,
|
||||
batch_size: 2,
|
||||
date_time_format: None,
|
||||
prepared_source: None,
|
||||
retain_source: false,
|
||||
conflict_policy: None,
|
||||
skip_duplicate_rows: false,
|
||||
};
|
||||
let summary = import_table_file_core(
|
||||
&state,
|
||||
&request,
|
||||
&DatabaseType::Sqlite,
|
||||
&pool_key,
|
||||
|_| Box::pin(async { false }),
|
||||
|_| {},
|
||||
)
|
||||
.await
|
||||
.expect("NOT NULL DEFAULT '' 的列必须能原样导入空字符串");
|
||||
|
||||
assert_eq!(summary.rows_imported, 2);
|
||||
assert_eq!(
|
||||
crate::db::sqlite::execute_query(&target, "SELECT id, name, note FROM contacts ORDER BY id")
|
||||
.await
|
||||
.unwrap()
|
||||
.rows,
|
||||
vec![
|
||||
vec![serde_json::json!(1), serde_json::json!(""), serde_json::Value::Null],
|
||||
vec![serde_json::json!(2), serde_json::json!("Ada"), serde_json::json!("x")],
|
||||
]
|
||||
);
|
||||
|
||||
// 反向验证:把 NULL 字面量关掉、恢复「空字段即 NULL」的旧配置再导一次同一个文件,
|
||||
// 必须重现用户报的「不允许为 null」,否则这条用例并没有真正覆盖该缺陷。
|
||||
let legacy_request = TableImportRequest {
|
||||
import_id: "csv-round-trip-legacy".to_string(),
|
||||
parse_options: TableImportParseOptions {
|
||||
null_literal: Some(String::new()),
|
||||
..TableImportParseOptions::default()
|
||||
},
|
||||
..request
|
||||
};
|
||||
let error = import_table_file_core(
|
||||
&state,
|
||||
&legacy_request,
|
||||
&DatabaseType::Sqlite,
|
||||
&pool_key,
|
||||
|_| Box::pin(async { false }),
|
||||
|_| {},
|
||||
)
|
||||
.await
|
||||
.expect_err("旧配置会把空字符串写成 NULL,必须触发 NOT NULL 约束错误");
|
||||
assert!(error.to_lowercase().contains("not null"), "unexpected error: {error}");
|
||||
}
|
||||
|
||||
/// The refusal has to land before `File::create`, otherwise the user is left
|
||||
/// holding a header-only CSV that looks like an object with no records.
|
||||
#[tokio::test]
|
||||
@@ -377,6 +575,7 @@ mod tests {
|
||||
page_size: None,
|
||||
timeout_secs: None,
|
||||
csv_quote_mode: CsvQuoteMode::default(),
|
||||
null_literal: String::new(),
|
||||
},
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -11,8 +11,8 @@ use std::time::{Duration, Instant};
|
||||
|
||||
use crate::connection::{AppState, PoolKind};
|
||||
use crate::csv_export::{
|
||||
format_query_result_csv_with_quote_mode, format_tsv, push_query_result_csv_row_with_quote_mode, push_tsv_row,
|
||||
CsvQuoteMode,
|
||||
csv_null_literal, format_query_result_csv_with_options, format_tsv, push_query_result_csv_row_with_options,
|
||||
push_tsv_row, CsvQuoteMode,
|
||||
};
|
||||
pub use crate::database_export::ExportStatus;
|
||||
use crate::database_export::{
|
||||
@@ -88,6 +88,10 @@ pub struct QueryResultExportRequest {
|
||||
pub insert_mode: SqlInsertMode,
|
||||
#[serde(default)]
|
||||
pub csv_quote_mode: CsvQuoteMode,
|
||||
/// CSV 里 NULL 写成什么。默认 `\N`;空字符串表示关闭该字面量,
|
||||
/// 退回「NULL 写成空字段」的旧行为(此时 NULL 与空字符串在文件里无法区分)。
|
||||
#[serde(default = "dbx_formats::csv_export::default_csv_null_literal")]
|
||||
pub null_literal: String,
|
||||
#[serde(default)]
|
||||
pub include_sql_sheet: bool,
|
||||
pub page_size: usize,
|
||||
@@ -543,9 +547,14 @@ fn effective_row_limit(request: &QueryResultExportRequest) -> Option<usize> {
|
||||
request.row_limit
|
||||
}
|
||||
|
||||
fn format_text_export_header(format: &str, columns: &[String], csv_quote_mode: CsvQuoteMode) -> String {
|
||||
fn format_text_export_header(
|
||||
format: &str,
|
||||
columns: &[String],
|
||||
csv_quote_mode: CsvQuoteMode,
|
||||
null_literal: Option<&str>,
|
||||
) -> String {
|
||||
let content = if format == "csv" {
|
||||
format_query_result_csv_with_quote_mode(columns, &[], csv_quote_mode)
|
||||
format_query_result_csv_with_options(columns, &[], csv_quote_mode, null_literal)
|
||||
} else {
|
||||
format_tsv(columns, &[])
|
||||
};
|
||||
@@ -558,13 +567,14 @@ fn write_text_export_row<W: Write>(
|
||||
row: &[Value],
|
||||
buffer: &mut String,
|
||||
csv_quote_mode: CsvQuoteMode,
|
||||
null_literal: Option<&str>,
|
||||
) -> Result<(), String> {
|
||||
buffer.clear();
|
||||
buffer.push('\n');
|
||||
if format == "csv" {
|
||||
push_query_result_csv_row_with_quote_mode(buffer, row, csv_quote_mode);
|
||||
push_query_result_csv_row_with_options(buffer, row, csv_quote_mode, null_literal);
|
||||
} else {
|
||||
push_tsv_row(buffer, row);
|
||||
push_tsv_row(buffer, row, null_literal);
|
||||
}
|
||||
file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}"))
|
||||
}
|
||||
@@ -575,14 +585,15 @@ fn write_text_export_rows<W: Write>(
|
||||
rows: &[Vec<Value>],
|
||||
buffer: &mut String,
|
||||
csv_quote_mode: CsvQuoteMode,
|
||||
null_literal: Option<&str>,
|
||||
) -> Result<(), String> {
|
||||
buffer.clear();
|
||||
for row in rows {
|
||||
buffer.push('\n');
|
||||
if format == "csv" {
|
||||
push_query_result_csv_row_with_quote_mode(buffer, row, csv_quote_mode);
|
||||
push_query_result_csv_row_with_options(buffer, row, csv_quote_mode, null_literal);
|
||||
} else {
|
||||
push_tsv_row(buffer, row);
|
||||
push_tsv_row(buffer, row, null_literal);
|
||||
}
|
||||
}
|
||||
file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}"))
|
||||
@@ -1093,7 +1104,12 @@ async fn export_query_result_core_inner(
|
||||
if format == "csv" || format == "txt" {
|
||||
if let Some(file) = text_file.as_mut() {
|
||||
if !wrote_text_header {
|
||||
let header = format_text_export_header(&format, &columns, request.csv_quote_mode);
|
||||
let header = format_text_export_header(
|
||||
&format,
|
||||
&columns,
|
||||
request.csv_quote_mode,
|
||||
csv_null_literal(&request.null_literal),
|
||||
);
|
||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
||||
if row_count > 0 {
|
||||
write_text_export_rows(
|
||||
@@ -1102,6 +1118,7 @@ async fn export_query_result_core_inner(
|
||||
formatted_rows.as_ref(),
|
||||
&mut text_buffer,
|
||||
request.csv_quote_mode,
|
||||
csv_null_literal(&request.null_literal),
|
||||
)?;
|
||||
}
|
||||
wrote_text_header = true;
|
||||
@@ -1112,6 +1129,7 @@ async fn export_query_result_core_inner(
|
||||
formatted_rows.as_ref(),
|
||||
&mut text_buffer,
|
||||
request.csv_quote_mode,
|
||||
csv_null_literal(&request.null_literal),
|
||||
)?;
|
||||
}
|
||||
}
|
||||
@@ -1184,7 +1202,12 @@ async fn export_query_result_core_inner(
|
||||
|
||||
if format == "csv" || format == "txt" {
|
||||
if !wrote_text_header {
|
||||
let header = format_text_export_header(&format, &columns, request.csv_quote_mode);
|
||||
let header = format_text_export_header(
|
||||
&format,
|
||||
&columns,
|
||||
request.csv_quote_mode,
|
||||
csv_null_literal(&request.null_literal),
|
||||
);
|
||||
if let Some(file) = text_file.as_mut() {
|
||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
||||
}
|
||||
@@ -1301,7 +1324,12 @@ async fn try_export_postgres_query_result_stream(
|
||||
if let Some(writer) = sql_writer.as_mut() {
|
||||
writer.set_columns(columns.clone(), &column_types, &[], request)?;
|
||||
} else if let Some(file) = text_file.as_mut() {
|
||||
let header = format_text_export_header(format, &columns, request.csv_quote_mode);
|
||||
let header = format_text_export_header(
|
||||
format,
|
||||
&columns,
|
||||
request.csv_quote_mode,
|
||||
csv_null_literal(&request.null_literal),
|
||||
);
|
||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
||||
} else if json_writer.is_none() {
|
||||
let xlsx_file =
|
||||
@@ -1329,6 +1357,7 @@ async fn try_export_postgres_query_result_stream(
|
||||
formatted.as_ref(),
|
||||
&mut text_buffer,
|
||||
request.csv_quote_mode,
|
||||
csv_null_literal(&request.null_literal),
|
||||
)?;
|
||||
} else if let Some(writer) = json_writer.as_mut() {
|
||||
writer.write_row(&columns, formatted.as_ref())?;
|
||||
@@ -1547,7 +1576,12 @@ async fn try_export_mysql_query_result_stream(
|
||||
if let Some(writer) = sql_writer.as_mut() {
|
||||
writer.set_columns(columns.clone(), &column_types, &[], request)?;
|
||||
} else if let Some(file) = text_file.as_mut() {
|
||||
let header = format_text_export_header(format, &columns, request.csv_quote_mode);
|
||||
let header = format_text_export_header(
|
||||
format,
|
||||
&columns,
|
||||
request.csv_quote_mode,
|
||||
csv_null_literal(&request.null_literal),
|
||||
);
|
||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
||||
} else if json_writer.is_none() {
|
||||
let xlsx_file =
|
||||
@@ -1575,6 +1609,7 @@ async fn try_export_mysql_query_result_stream(
|
||||
formatted.as_ref(),
|
||||
&mut text_buffer,
|
||||
request.csv_quote_mode,
|
||||
csv_null_literal(&request.null_literal),
|
||||
)?;
|
||||
} else if let Some(writer) = json_writer.as_mut() {
|
||||
writer.write_row(&columns, formatted.as_ref())?;
|
||||
@@ -1772,7 +1807,12 @@ async fn try_export_clickhouse_query_result_stream(
|
||||
if let Some(writer) = sql_writer.as_mut() {
|
||||
writer.set_columns(columns.clone(), &column_types, &[], request)?;
|
||||
} else if let Some(file) = text_file.as_mut() {
|
||||
let header = format_text_export_header(format, &columns, request.csv_quote_mode);
|
||||
let header = format_text_export_header(
|
||||
format,
|
||||
&columns,
|
||||
request.csv_quote_mode,
|
||||
csv_null_literal(&request.null_literal),
|
||||
);
|
||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
||||
} else if json_writer.is_none() {
|
||||
let xlsx_file =
|
||||
@@ -1800,6 +1840,7 @@ async fn try_export_clickhouse_query_result_stream(
|
||||
formatted.as_ref(),
|
||||
&mut text_buffer,
|
||||
request.csv_quote_mode,
|
||||
csv_null_literal(&request.null_literal),
|
||||
)?;
|
||||
} else if let Some(writer) = json_writer.as_mut() {
|
||||
writer.write_row(&columns, formatted.as_ref())?;
|
||||
@@ -1966,7 +2007,12 @@ async fn try_export_sqlserver_query_result_stream(
|
||||
if let Some(writer) = sql_writer.as_mut() {
|
||||
writer.set_columns(columns.clone(), &temporal_column_types, &[], request)?;
|
||||
} else if let Some(file) = text_file.as_mut() {
|
||||
let header = format_text_export_header(format, &columns, request.csv_quote_mode);
|
||||
let header = format_text_export_header(
|
||||
format,
|
||||
&columns,
|
||||
request.csv_quote_mode,
|
||||
csv_null_literal(&request.null_literal),
|
||||
);
|
||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
||||
} else if json_writer.is_none() {
|
||||
let xlsx_file =
|
||||
@@ -1990,6 +2036,7 @@ async fn try_export_sqlserver_query_result_stream(
|
||||
formatted.as_ref(),
|
||||
&mut text_buffer,
|
||||
request.csv_quote_mode,
|
||||
csv_null_literal(&request.null_literal),
|
||||
)?;
|
||||
} else if let Some(writer) = json_writer.as_mut() {
|
||||
writer.write_row(&columns, formatted.as_ref())?;
|
||||
@@ -2191,6 +2238,7 @@ mod tests {
|
||||
identifier_quote: None,
|
||||
insert_mode: SqlInsertMode::Batch,
|
||||
csv_quote_mode: Default::default(),
|
||||
null_literal: String::new(),
|
||||
exclude_primary_keys: false,
|
||||
primary_keys: Vec::new(),
|
||||
}
|
||||
@@ -2281,6 +2329,7 @@ mod tests {
|
||||
identifier_quote: None,
|
||||
insert_mode: Default::default(),
|
||||
csv_quote_mode: Default::default(),
|
||||
null_literal: String::new(),
|
||||
exclude_primary_keys: true,
|
||||
primary_keys: vec!["id".to_string()],
|
||||
};
|
||||
@@ -2357,6 +2406,7 @@ mod tests {
|
||||
execution_id: None,
|
||||
date_time_format: None,
|
||||
csv_quote_mode: CsvQuoteMode::All,
|
||||
null_literal: String::new(),
|
||||
export_table_name: None,
|
||||
export_column_types: None,
|
||||
selected_columns: None,
|
||||
@@ -2560,7 +2610,7 @@ mod tests {
|
||||
#[test]
|
||||
fn txt_export_header_keeps_columns_for_empty_results() {
|
||||
assert_eq!(
|
||||
format_text_export_header("txt", &["id".to_string(), "note".to_string()], CsvQuoteMode::All),
|
||||
format_text_export_header("txt", &["id".to_string(), "note".to_string()], CsvQuoteMode::All, None),
|
||||
"id\tnote"
|
||||
);
|
||||
}
|
||||
@@ -2602,7 +2652,7 @@ mod tests {
|
||||
let mut output = Vec::new();
|
||||
let mut buffer = String::new();
|
||||
|
||||
write_text_export_row(&mut output, "csv", &row, &mut buffer, CsvQuoteMode::All).expect("write csv row");
|
||||
write_text_export_row(&mut output, "csv", &row, &mut buffer, CsvQuoteMode::All, None).expect("write csv row");
|
||||
assert_eq!(String::from_utf8(output).expect("utf8 csv"), "\n,\"\",\"line\n\"\"two\"\"\"");
|
||||
}
|
||||
|
||||
|
||||
@@ -15,8 +15,8 @@ use crate::csv_export::format_csv;
|
||||
#[cfg(test)]
|
||||
use crate::csv_export::format_tsv_rows;
|
||||
use crate::csv_export::{
|
||||
format_csv_with_quote_mode, format_tsv, push_table_csv_row, push_table_csv_row_with_quote_mode, push_tsv_row,
|
||||
CsvQuoteMode,
|
||||
csv_null_literal, format_csv_with_options, format_tsv, push_table_csv_row, push_table_csv_row_with_options,
|
||||
push_tsv_row, CsvQuoteMode,
|
||||
};
|
||||
use crate::data_grid_sql::extra_is_auto_generated;
|
||||
pub use crate::database_export::ExportStatus;
|
||||
@@ -87,6 +87,10 @@ pub struct TableExportRequest {
|
||||
pub insert_dialect: SqlInsertDialect,
|
||||
#[serde(default)]
|
||||
pub csv_quote_mode: CsvQuoteMode,
|
||||
/// CSV 里 NULL 写成什么。默认 `\N`;空字符串表示关闭该字面量,
|
||||
/// 退回「NULL 写成空字段」的旧行为(此时 NULL 与空字符串在文件里无法区分)。
|
||||
#[serde(default = "dbx_formats::csv_export::default_csv_null_literal")]
|
||||
pub null_literal: String,
|
||||
#[serde(default)]
|
||||
pub columns: Option<Vec<String>>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
@@ -164,13 +168,14 @@ fn write_table_text_row<W: Write>(
|
||||
row: &[Value],
|
||||
buffer: &mut String,
|
||||
csv_quote_mode: CsvQuoteMode,
|
||||
null_literal: Option<&str>,
|
||||
) -> Result<(), String> {
|
||||
buffer.clear();
|
||||
buffer.push('\n');
|
||||
if csv {
|
||||
push_table_csv_row_with_quote_mode(buffer, row, csv_quote_mode);
|
||||
push_table_csv_row_with_options(buffer, row, csv_quote_mode, null_literal);
|
||||
} else {
|
||||
push_tsv_row(buffer, row);
|
||||
push_tsv_row(buffer, row, null_literal);
|
||||
}
|
||||
file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}"))
|
||||
}
|
||||
@@ -181,14 +186,15 @@ fn write_table_text_rows<W: Write>(
|
||||
rows: &[Vec<Value>],
|
||||
buffer: &mut String,
|
||||
csv_quote_mode: CsvQuoteMode,
|
||||
null_literal: Option<&str>,
|
||||
) -> Result<(), String> {
|
||||
buffer.clear();
|
||||
for row in rows {
|
||||
buffer.push('\n');
|
||||
if csv {
|
||||
push_table_csv_row_with_quote_mode(buffer, row, csv_quote_mode);
|
||||
push_table_csv_row_with_options(buffer, row, csv_quote_mode, null_literal);
|
||||
} else {
|
||||
push_tsv_row(buffer, row);
|
||||
push_tsv_row(buffer, row, null_literal);
|
||||
}
|
||||
}
|
||||
file.write_all(buffer.as_bytes()).map_err(|error| format!("Failed to write export rows: {error}"))
|
||||
@@ -1128,7 +1134,12 @@ async fn try_export_native_table_stream(
|
||||
std::fs::File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?,
|
||||
);
|
||||
file.write_all(b"\xEF\xBB\xBF").map_err(|e| format!("Failed to write BOM: {e}"))?;
|
||||
let header = format_csv_with_quote_mode(col_names, &[], request.csv_quote_mode);
|
||||
let header = format_csv_with_options(
|
||||
col_names,
|
||||
&[],
|
||||
request.csv_quote_mode,
|
||||
csv_null_literal(&request.null_literal),
|
||||
);
|
||||
let header = header.strip_suffix('\n').unwrap_or(&header);
|
||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?;
|
||||
let mut row_buffer = String::new();
|
||||
@@ -1147,7 +1158,14 @@ async fn try_export_native_table_stream(
|
||||
column_types,
|
||||
request.date_time_format.as_deref(),
|
||||
);
|
||||
write_table_text_row(&mut file, true, formatted.as_ref(), &mut row_buffer, request.csv_quote_mode)?;
|
||||
write_table_text_row(
|
||||
&mut file,
|
||||
true,
|
||||
formatted.as_ref(),
|
||||
&mut row_buffer,
|
||||
request.csv_quote_mode,
|
||||
csv_null_literal(&request.null_literal),
|
||||
)?;
|
||||
rows_exported += 1;
|
||||
if rows_exported.is_multiple_of(progress_interval) {
|
||||
on_progress(TableExportProgress {
|
||||
@@ -1197,6 +1215,7 @@ async fn try_export_native_table_stream(
|
||||
formatted.as_ref(),
|
||||
&mut row_buffer,
|
||||
request.csv_quote_mode,
|
||||
csv_null_literal(&request.null_literal),
|
||||
)?;
|
||||
rows_exported += 1;
|
||||
if rows_exported.is_multiple_of(progress_interval) {
|
||||
@@ -1780,8 +1799,12 @@ async fn export_table_data_core_inner(
|
||||
|
||||
if is_first_batch {
|
||||
// First batch: write header + rows via format_csv
|
||||
let csv_content =
|
||||
format_csv_with_quote_mode(&col_names, formatted_rows.as_ref(), request.csv_quote_mode);
|
||||
let csv_content = format_csv_with_options(
|
||||
&col_names,
|
||||
formatted_rows.as_ref(),
|
||||
request.csv_quote_mode,
|
||||
csv_null_literal(&request.null_literal),
|
||||
);
|
||||
file.write_all(csv_content.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?;
|
||||
is_first_batch = false;
|
||||
} else {
|
||||
@@ -1792,6 +1815,7 @@ async fn export_table_data_core_inner(
|
||||
formatted_rows.as_ref(),
|
||||
&mut text_buffer,
|
||||
request.csv_quote_mode,
|
||||
csv_null_literal(&request.null_literal),
|
||||
)?;
|
||||
}
|
||||
|
||||
@@ -1880,6 +1904,7 @@ async fn export_table_data_core_inner(
|
||||
formatted_rows.as_ref(),
|
||||
&mut text_buffer,
|
||||
request.csv_quote_mode,
|
||||
csv_null_literal(&request.null_literal),
|
||||
)?;
|
||||
is_first_batch = false;
|
||||
} else {
|
||||
@@ -1889,6 +1914,7 @@ async fn export_table_data_core_inner(
|
||||
formatted_rows.as_ref(),
|
||||
&mut text_buffer,
|
||||
request.csv_quote_mode,
|
||||
csv_null_literal(&request.null_literal),
|
||||
)?;
|
||||
}
|
||||
|
||||
@@ -2476,6 +2502,7 @@ mod tests {
|
||||
row_limit,
|
||||
date_time_format: None,
|
||||
csv_quote_mode: CsvQuoteMode::All,
|
||||
null_literal: String::new(),
|
||||
numeric_column_right_align: false,
|
||||
column_comments: None,
|
||||
auto_filter: None,
|
||||
@@ -2631,6 +2658,7 @@ mod tests {
|
||||
insert_mode: Default::default(),
|
||||
insert_dialect: Default::default(),
|
||||
csv_quote_mode: CsvQuoteMode::All,
|
||||
null_literal: String::new(),
|
||||
exclude_primary_keys: false,
|
||||
columns: None,
|
||||
selected_columns: None,
|
||||
@@ -2730,7 +2758,7 @@ mod tests {
|
||||
let mut output = Vec::new();
|
||||
let mut buffer = String::new();
|
||||
|
||||
write_table_text_row(&mut output, true, &row, &mut buffer, CsvQuoteMode::All).expect("write csv row");
|
||||
write_table_text_row(&mut output, true, &row, &mut buffer, CsvQuoteMode::All, None).expect("write csv row");
|
||||
assert_eq!(String::from_utf8(output).expect("utf8 csv"), "\n\"\",\"\",\"line\n\"\"two\"\"\"");
|
||||
}
|
||||
|
||||
@@ -2813,6 +2841,7 @@ mod tests {
|
||||
row_limit: None,
|
||||
date_time_format: None,
|
||||
csv_quote_mode: CsvQuoteMode::All,
|
||||
null_literal: String::new(),
|
||||
numeric_column_right_align: false,
|
||||
column_comments: None,
|
||||
auto_filter: None,
|
||||
@@ -2876,6 +2905,7 @@ mod tests {
|
||||
row_limit: None,
|
||||
date_time_format: None,
|
||||
csv_quote_mode: CsvQuoteMode::All,
|
||||
null_literal: String::new(),
|
||||
numeric_column_right_align: false,
|
||||
column_comments: None,
|
||||
auto_filter: None,
|
||||
@@ -2917,6 +2947,7 @@ mod tests {
|
||||
row_limit: None,
|
||||
date_time_format: None,
|
||||
csv_quote_mode: CsvQuoteMode::All,
|
||||
null_literal: String::new(),
|
||||
numeric_column_right_align: false,
|
||||
column_comments: None,
|
||||
auto_filter: None,
|
||||
@@ -2953,6 +2984,7 @@ mod tests {
|
||||
row_limit: None,
|
||||
date_time_format: None,
|
||||
csv_quote_mode: CsvQuoteMode::All,
|
||||
null_literal: String::new(),
|
||||
numeric_column_right_align: false,
|
||||
column_comments: None,
|
||||
auto_filter: None,
|
||||
@@ -2997,6 +3029,7 @@ mod tests {
|
||||
row_limit: None,
|
||||
date_time_format: None,
|
||||
csv_quote_mode: CsvQuoteMode::All,
|
||||
null_literal: String::new(),
|
||||
numeric_column_right_align: false,
|
||||
column_comments: None,
|
||||
auto_filter: None,
|
||||
@@ -3053,6 +3086,7 @@ mod tests {
|
||||
row_limit: Some(1000),
|
||||
date_time_format: None,
|
||||
csv_quote_mode: CsvQuoteMode::All,
|
||||
null_literal: String::new(),
|
||||
numeric_column_right_align: false,
|
||||
column_comments: None,
|
||||
auto_filter: None,
|
||||
@@ -3103,6 +3137,7 @@ mod tests {
|
||||
row_limit: None,
|
||||
date_time_format: None,
|
||||
csv_quote_mode: CsvQuoteMode::All,
|
||||
null_literal: String::new(),
|
||||
numeric_column_right_align: false,
|
||||
column_comments: None,
|
||||
auto_filter: None,
|
||||
@@ -3181,6 +3216,7 @@ mod tests {
|
||||
row_limit: None,
|
||||
date_time_format: None,
|
||||
csv_quote_mode: CsvQuoteMode::All,
|
||||
null_literal: String::new(),
|
||||
numeric_column_right_align: false,
|
||||
column_comments: None,
|
||||
auto_filter: None,
|
||||
@@ -3241,6 +3277,7 @@ mod tests {
|
||||
row_limit: None,
|
||||
date_time_format: None,
|
||||
csv_quote_mode: CsvQuoteMode::All,
|
||||
null_literal: String::new(),
|
||||
numeric_column_right_align: false,
|
||||
column_comments: None,
|
||||
auto_filter: None,
|
||||
@@ -3303,6 +3340,7 @@ mod tests {
|
||||
row_limit: None,
|
||||
date_time_format: None,
|
||||
csv_quote_mode: CsvQuoteMode::All,
|
||||
null_literal: String::new(),
|
||||
numeric_column_right_align: false,
|
||||
column_comments: None,
|
||||
auto_filter: None,
|
||||
|
||||
@@ -208,6 +208,10 @@ pub struct TableImportParseOptions {
|
||||
pub last_data_row: Option<usize>,
|
||||
pub trim_values: Option<bool>,
|
||||
pub empty_string_as_null: Option<bool>,
|
||||
/// 分隔文本里代表 NULL 的字面量。缺省表示使用与导出端一致的默认字面量(`\N`);
|
||||
/// 显式配置空串表示关闭字面量,退回「空字段即 NULL」的旧行为。
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub null_literal: Option<String>,
|
||||
pub sheet_name: Option<String>,
|
||||
pub sheet_index: Option<usize>,
|
||||
pub json_shape: Option<TableImportJsonShape>,
|
||||
@@ -227,6 +231,7 @@ impl Default for TableImportParseOptions {
|
||||
last_data_row: None,
|
||||
trim_values: Some(false),
|
||||
empty_string_as_null: Some(true),
|
||||
null_literal: None,
|
||||
sheet_name: None,
|
||||
sheet_index: None,
|
||||
json_shape: Some(TableImportJsonShape::Auto),
|
||||
@@ -651,14 +656,26 @@ fn unique_import_headers(headers: impl IntoIterator<Item = String>) -> Vec<Strin
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DelimitedParseConfig {
|
||||
pub delimiter: u8,
|
||||
pub trim_values: bool,
|
||||
pub empty_string_as_null: bool,
|
||||
/// 命中的字段按 NULL 处理;`None` 表示不按字面量识别 NULL。
|
||||
pub null_literal: Option<String>,
|
||||
pub row_range: ImportRowRange,
|
||||
}
|
||||
|
||||
/// 分隔文本里 NULL 字面量的默认值:与导出端共用 [`dbx_formats::csv_export::DEFAULT_CSV_NULL_LITERAL`],
|
||||
/// 因此 DBX 自己导出的 CSV 默认即可无损导入。
|
||||
fn effective_delimited_null_literal(raw: Option<&str>) -> Option<String> {
|
||||
match raw {
|
||||
None => Some(dbx_formats::csv_export::DEFAULT_CSV_NULL_LITERAL.to_string()),
|
||||
// 显式空串 = 关闭字面量
|
||||
Some(raw) => dbx_formats::csv_export::csv_null_literal(raw).map(str::to_string),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct ImportRowRange {
|
||||
pub title_row: Option<usize>,
|
||||
@@ -707,10 +724,15 @@ pub fn effective_delimited_config(
|
||||
}
|
||||
};
|
||||
|
||||
let null_literal = effective_delimited_null_literal(options.null_literal.as_deref());
|
||||
|
||||
Ok(DelimitedParseConfig {
|
||||
delimiter,
|
||||
trim_values: options.trim_values.unwrap_or(false),
|
||||
empty_string_as_null: options.empty_string_as_null.unwrap_or(true),
|
||||
// 配了 NULL 字面量时空字段一律是空串:否则字面量刚把 NULL 和空串分开,
|
||||
// 这里又会把空串重新当成 NULL。
|
||||
empty_string_as_null: null_literal.is_none() && options.empty_string_as_null.unwrap_or(true),
|
||||
null_literal,
|
||||
row_range: effective_import_row_range(options)?,
|
||||
})
|
||||
}
|
||||
@@ -734,9 +756,13 @@ fn unwrap_csv_force_text(value: &str) -> &str {
|
||||
inner
|
||||
}
|
||||
|
||||
pub fn csv_value_with_config(value: &str, config: DelimitedParseConfig) -> serde_json::Value {
|
||||
pub fn csv_value_with_config(value: &str, config: &DelimitedParseConfig) -> serde_json::Value {
|
||||
let value = if config.trim_values { value.trim() } else { value };
|
||||
let value = unwrap_csv_force_text(value);
|
||||
// 字面量优先:命中即为 NULL,空字段则原样保留为空字符串。
|
||||
if config.null_literal.as_deref() == Some(value) {
|
||||
return serde_json::Value::Null;
|
||||
}
|
||||
if config.empty_string_as_null && value.is_empty() {
|
||||
serde_json::Value::Null
|
||||
} else {
|
||||
@@ -747,10 +773,11 @@ pub fn csv_value_with_config(value: &str, config: DelimitedParseConfig) -> serde
|
||||
pub fn csv_value(value: &str) -> serde_json::Value {
|
||||
csv_value_with_config(
|
||||
value,
|
||||
DelimitedParseConfig {
|
||||
&DelimitedParseConfig {
|
||||
delimiter: b',',
|
||||
trim_values: false,
|
||||
empty_string_as_null: true,
|
||||
null_literal: None,
|
||||
row_range: ImportRowRange { title_row: Some(1), data_start_row: 2, last_data_row: None },
|
||||
},
|
||||
)
|
||||
@@ -1089,7 +1116,7 @@ fn open_delimited_csv_reader_with_progress(
|
||||
|
||||
pub fn parse_delimited_reader<R: std::io::Read>(
|
||||
reader: R,
|
||||
config: DelimitedParseConfig,
|
||||
config: &DelimitedParseConfig,
|
||||
preview_limit: usize,
|
||||
) -> Result<ParsedImportFile, String> {
|
||||
parse_decoded_delimited_reader(reader, config, preview_limit, TableImportTextEncoding::Utf8)
|
||||
@@ -1097,7 +1124,7 @@ pub fn parse_delimited_reader<R: std::io::Read>(
|
||||
|
||||
fn parse_decoded_delimited_reader<R: IoRead>(
|
||||
reader: R,
|
||||
config: DelimitedParseConfig,
|
||||
config: &DelimitedParseConfig,
|
||||
preview_limit: usize,
|
||||
effective_encoding: TableImportTextEncoding,
|
||||
) -> Result<ParsedImportFile, String> {
|
||||
@@ -1114,7 +1141,12 @@ pub fn parse_delimited_bytes_with_options(
|
||||
) -> Result<ParsedImportFile, String> {
|
||||
let (encoding, bom_len) = resolve_text_encoding_from_bytes(bytes, options.encoding)?;
|
||||
let reader = StrictTranscodingReader::new(std::io::Cursor::new(&bytes[bom_len..]), encoding)?;
|
||||
parse_decoded_delimited_reader(reader, effective_delimited_config(source_format, options)?, preview_limit, encoding)
|
||||
parse_decoded_delimited_reader(
|
||||
reader,
|
||||
&effective_delimited_config(source_format, options)?,
|
||||
preview_limit,
|
||||
encoding,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn parse_delimited_file_with_options(
|
||||
@@ -1132,7 +1164,7 @@ pub fn parse_delimited_file_with_options(
|
||||
explicit_options.encoding = Some(encoding);
|
||||
let (reader, config, encoding) =
|
||||
open_delimited_csv_reader_with_progress(path, source_format, &explicit_options, |_| {})?;
|
||||
return parse_csv_reader(reader, config, preview_limit, encoding);
|
||||
return parse_csv_reader(reader, &config, preview_limit, encoding);
|
||||
}
|
||||
|
||||
for encoding in [TableImportTextEncoding::Utf8, TableImportTextEncoding::Gbk] {
|
||||
@@ -1140,7 +1172,7 @@ pub fn parse_delimited_file_with_options(
|
||||
explicit_options.encoding = Some(encoding);
|
||||
let (reader, config, encoding) =
|
||||
open_delimited_csv_reader_with_progress(path, source_format, &explicit_options, |_| {})?;
|
||||
match parse_csv_reader(reader, config, preview_limit, encoding) {
|
||||
match parse_csv_reader(reader, &config, preview_limit, encoding) {
|
||||
Ok(parsed) => return Ok(parsed),
|
||||
Err(error) if error.starts_with("Invalid byte sequence for ") => continue,
|
||||
Err(error) => return Err(error),
|
||||
@@ -1150,12 +1182,12 @@ pub fn parse_delimited_file_with_options(
|
||||
}
|
||||
|
||||
let (reader, config, encoding) = open_delimited_csv_reader_with_progress(path, source_format, options, |_| {})?;
|
||||
parse_csv_reader(reader, config, preview_limit, encoding)
|
||||
parse_csv_reader(reader, &config, preview_limit, encoding)
|
||||
}
|
||||
|
||||
fn parse_csv_reader<R: IoRead>(
|
||||
mut reader: csv::Reader<R>,
|
||||
config: DelimitedParseConfig,
|
||||
config: &DelimitedParseConfig,
|
||||
preview_limit: usize,
|
||||
effective_encoding: TableImportTextEncoding,
|
||||
) -> Result<ParsedImportFile, String> {
|
||||
@@ -1164,7 +1196,7 @@ fn parse_csv_reader<R: IoRead>(
|
||||
|
||||
fn parse_csv_reader_bounded<R: IoRead>(
|
||||
mut reader: csv::Reader<R>,
|
||||
config: DelimitedParseConfig,
|
||||
config: &DelimitedParseConfig,
|
||||
preview_limit: usize,
|
||||
effective_encoding: TableImportTextEncoding,
|
||||
) -> Result<ParsedImportFile, String> {
|
||||
@@ -1173,7 +1205,7 @@ fn parse_csv_reader_bounded<R: IoRead>(
|
||||
|
||||
fn parse_csv_reader_inner<R: IoRead>(
|
||||
reader: &mut csv::Reader<R>,
|
||||
config: DelimitedParseConfig,
|
||||
config: &DelimitedParseConfig,
|
||||
preview_limit: usize,
|
||||
effective_encoding: TableImportTextEncoding,
|
||||
count_all_rows: bool,
|
||||
@@ -1236,7 +1268,7 @@ fn parse_delimited_preview_file_with_options(
|
||||
explicit_options.encoding = Some(encoding);
|
||||
let (reader, config, encoding) =
|
||||
open_delimited_csv_reader_with_progress(path, source_format, &explicit_options, |_| {})?;
|
||||
return parse_csv_reader_bounded(reader, config, preview_limit, encoding);
|
||||
return parse_csv_reader_bounded(reader, &config, preview_limit, encoding);
|
||||
}
|
||||
|
||||
for encoding in [TableImportTextEncoding::Utf8, TableImportTextEncoding::Gbk] {
|
||||
@@ -1244,7 +1276,7 @@ fn parse_delimited_preview_file_with_options(
|
||||
explicit_options.encoding = Some(encoding);
|
||||
let (reader, config, encoding) =
|
||||
open_delimited_csv_reader_with_progress(path, source_format, &explicit_options, |_| {})?;
|
||||
match parse_csv_reader_bounded(reader, config, preview_limit, encoding) {
|
||||
match parse_csv_reader_bounded(reader, &config, preview_limit, encoding) {
|
||||
Ok(parsed) => return Ok(parsed),
|
||||
Err(error) if error.starts_with("Invalid byte sequence for ") => continue,
|
||||
Err(error) => return Err(error),
|
||||
@@ -1254,7 +1286,7 @@ fn parse_delimited_preview_file_with_options(
|
||||
}
|
||||
|
||||
let (reader, config, encoding) = open_delimited_csv_reader_with_progress(path, source_format, options, |_| {})?;
|
||||
parse_csv_reader_bounded(reader, config, preview_limit, encoding)
|
||||
parse_csv_reader_bounded(reader, &config, preview_limit, encoding)
|
||||
}
|
||||
|
||||
pub fn parse_csv_bytes(bytes: &[u8], preview_limit: usize) -> Result<ParsedImportFile, String> {
|
||||
@@ -6291,7 +6323,7 @@ fn import_cancelled_progress(
|
||||
fn delimited_record_to_row(
|
||||
record: &csv::StringRecord,
|
||||
columns_len: usize,
|
||||
config: DelimitedParseConfig,
|
||||
config: &DelimitedParseConfig,
|
||||
) -> Vec<serde_json::Value> {
|
||||
(0..columns_len)
|
||||
.map(|index| {
|
||||
@@ -6302,7 +6334,7 @@ fn delimited_record_to_row(
|
||||
|
||||
fn delimited_columns_and_first_record<R: std::io::Read>(
|
||||
reader: &mut csv::Reader<R>,
|
||||
config: DelimitedParseConfig,
|
||||
config: &DelimitedParseConfig,
|
||||
) -> Result<(Vec<String>, Option<csv::StringRecord>), String> {
|
||||
let mut columns = Vec::new();
|
||||
for (index, record) in reader.records().enumerate() {
|
||||
@@ -6351,7 +6383,7 @@ fn stream_delimited_rows_to_channel(
|
||||
// Keep CSV parsing off the async executor while the bounded channel prevents unbounded
|
||||
// accumulation when the database consumer is under load.
|
||||
let (mut reader, config, _) = open_delimited_csv_reader_with_progress(path, source_format, options, |_| {})?;
|
||||
let (columns, first_record) = delimited_columns_and_first_record(&mut reader, config)?;
|
||||
let (columns, first_record) = delimited_columns_and_first_record(&mut reader, &config)?;
|
||||
sender
|
||||
.blocking_send(Ok(DelimitedStreamMessage::Header(columns.clone())))
|
||||
.map_err(|_| "Delimited import consumer closed before the stream started".to_string())?;
|
||||
@@ -6374,7 +6406,7 @@ fn stream_delimited_rows_to_channel(
|
||||
if config.row_range.last_data_row.is_some_and(|last| source_row_number > last) {
|
||||
break;
|
||||
}
|
||||
pending_rows.push(delimited_record_to_row(&record, columns.len(), config));
|
||||
pending_rows.push(delimited_record_to_row(&record, columns.len(), &config));
|
||||
if pending_rows.len() >= batch_size {
|
||||
sender
|
||||
.blocking_send(Ok(DelimitedStreamMessage::Rows {
|
||||
@@ -9372,12 +9404,93 @@ mod tests {
|
||||
parsed.rows[1],
|
||||
vec![
|
||||
serde_json::Value::String("2".to_string()),
|
||||
serde_json::Value::Null,
|
||||
// 默认 NULL 字面量生效后,空字段保留为空字符串,只有字面量才是 NULL
|
||||
serde_json::Value::String(String::new()),
|
||||
serde_json::Value::String("false".to_string()),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn effective_delimited_config_shares_the_export_null_literal_by_default() {
|
||||
let config =
|
||||
effective_delimited_config(TableImportSourceFormat::Csv, &TableImportParseOptions::default()).unwrap();
|
||||
assert_eq!(config.null_literal.as_deref(), Some(dbx_formats::csv_export::DEFAULT_CSV_NULL_LITERAL));
|
||||
assert!(!config.empty_string_as_null, "NULL 字面量生效时不再把空字段当作 NULL");
|
||||
|
||||
// 显式空串 = 关闭字面量,退回「空字段即 NULL」的旧行为
|
||||
let legacy =
|
||||
TableImportParseOptions { null_literal: Some(String::new()), ..TableImportParseOptions::default() };
|
||||
let config = effective_delimited_config(TableImportSourceFormat::Csv, &legacy).unwrap();
|
||||
assert_eq!(config.null_literal, None);
|
||||
assert!(config.empty_string_as_null);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn csv_null_literal_separates_null_from_an_empty_string() {
|
||||
// `\N` 是 NULL;`""` 与裸空字段都是空字符串
|
||||
let parsed = parse_csv_bytes(b"id,name\n1,\\N\n2,\"\"\n3,\n", 10).unwrap();
|
||||
|
||||
assert_eq!(parsed.total_rows, 3);
|
||||
assert_eq!(parsed.rows[0][1], serde_json::Value::Null);
|
||||
assert_eq!(parsed.rows[1][1], serde_json::json!(""));
|
||||
assert_eq!(parsed.rows[2][1], serde_json::json!(""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tsv_export_with_null_literal_round_trips_through_the_importer() {
|
||||
// 导出端(push_tsv_row + 默认字面量)写出的 TSV,按 TSV 规则导入:
|
||||
// NULL 仍是 NULL、空字符串仍是空字符串
|
||||
let mut text = String::from("id\tnote\n");
|
||||
dbx_formats::csv_export::push_tsv_row(
|
||||
&mut text,
|
||||
&[serde_json::json!(1), serde_json::Value::Null],
|
||||
Some(dbx_formats::csv_export::DEFAULT_CSV_NULL_LITERAL),
|
||||
);
|
||||
text.push('\n');
|
||||
dbx_formats::csv_export::push_tsv_row(
|
||||
&mut text,
|
||||
&[serde_json::json!(2), serde_json::json!("")],
|
||||
Some(dbx_formats::csv_export::DEFAULT_CSV_NULL_LITERAL),
|
||||
);
|
||||
|
||||
let parsed = parse_delimited_bytes_with_options(
|
||||
text.as_bytes(),
|
||||
TableImportSourceFormat::Tsv,
|
||||
&TableImportParseOptions::default(),
|
||||
10,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(parsed.rows[0][1], serde_json::Value::Null);
|
||||
assert_eq!(parsed.rows[1][1], serde_json::json!(""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn csv_null_literal_can_be_replaced_or_disabled() {
|
||||
let custom =
|
||||
TableImportParseOptions { null_literal: Some("(null)".to_string()), ..TableImportParseOptions::default() };
|
||||
let parsed =
|
||||
parse_delimited_bytes_with_options(b"id,name\n1,(null)\n2,\n", TableImportSourceFormat::Csv, &custom, 10)
|
||||
.unwrap();
|
||||
assert_eq!(parsed.rows[0][1], serde_json::Value::Null);
|
||||
assert_eq!(parsed.rows[1][1], serde_json::json!(""));
|
||||
|
||||
let legacy =
|
||||
TableImportParseOptions { null_literal: Some(String::new()), ..TableImportParseOptions::default() };
|
||||
let parsed =
|
||||
parse_delimited_bytes_with_options(b"id,name\n1,\n2,Ada\n", TableImportSourceFormat::Csv, &legacy, 10)
|
||||
.unwrap();
|
||||
assert_eq!(parsed.rows[0][1], serde_json::Value::Null);
|
||||
assert_eq!(parsed.rows[1][1], serde_json::json!("Ada"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn csv_null_literal_is_not_reported_as_a_missing_value() {
|
||||
// 字面量只按整字段匹配:夹带空格的 `\N ` 不是 NULL,避免误伤真实数据
|
||||
let parsed = parse_csv_bytes(b"id,name\n1, \\N \n", 10).unwrap();
|
||||
assert_eq!(parsed.rows[0][1], serde_json::json!(" \\N "));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn duplicate_csv_headers_map_to_distinct_source_indexes() {
|
||||
let parsed = parse_csv_bytes(b"name,name\nAda,Lovelace\n", 10).unwrap();
|
||||
@@ -9627,7 +9740,7 @@ mod tests {
|
||||
let config =
|
||||
effective_delimited_config(TableImportSourceFormat::Csv, &TableImportParseOptions::default()).unwrap();
|
||||
|
||||
let preview = parse_csv_reader_bounded(reader, config, 1, TableImportTextEncoding::Utf8).unwrap();
|
||||
let preview = parse_csv_reader_bounded(reader, &config, 1, TableImportTextEncoding::Utf8).unwrap();
|
||||
|
||||
assert_eq!(preview.columns, vec!["id", "name"]);
|
||||
assert_eq!(preview.rows, vec![vec![serde_json::json!("1"), serde_json::json!("Ada")]]);
|
||||
@@ -9754,7 +9867,9 @@ mod tests {
|
||||
assert_eq!(parsed.columns, vec!["column_1", "column_2"]);
|
||||
assert_eq!(parsed.total_rows, 2);
|
||||
assert_eq!(parsed.rows[0], vec![serde_json::json!("1"), serde_json::json!("Ada")]);
|
||||
assert_eq!(parsed.rows[1], vec![serde_json::json!("2"), serde_json::Value::Null]);
|
||||
// 默认 NULL 字面量生效时,显式打开的「空字符串作为 NULL」不再接管空字段:
|
||||
// 否则字面量刚把 NULL 和空串分开,这里又会把空串重新写成 NULL。
|
||||
assert_eq!(parsed.rows[1], vec![serde_json::json!("2"), serde_json::json!("")]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -102,6 +102,7 @@ async fn live_clickhouse_query_result_export_xlsx_streams_random_order_query_onc
|
||||
execution_id: Some(format!("live-clickhouse-query-export-{suffix}")),
|
||||
date_time_format: None,
|
||||
csv_quote_mode: Default::default(),
|
||||
null_literal: String::new(),
|
||||
export_table_name: None,
|
||||
export_column_types: None,
|
||||
selected_columns: None,
|
||||
|
||||
@@ -477,6 +477,7 @@ async fn live_mysql_query_result_export_xlsx_streams_single_query_without_duplic
|
||||
execution_id: Some(format!("live-mysql-query-export-{suffix}")),
|
||||
date_time_format: None,
|
||||
csv_quote_mode: Default::default(),
|
||||
null_literal: String::new(),
|
||||
export_table_name: None,
|
||||
export_column_types: None,
|
||||
selected_columns: None,
|
||||
@@ -572,6 +573,7 @@ async fn live_mysql_csv_temporal_export_round_trip_preserves_dbx_force_text_valu
|
||||
execution_id: Some(format!("live-mysql-issue-8803-{suffix}")),
|
||||
date_time_format: None,
|
||||
csv_quote_mode: Default::default(),
|
||||
null_literal: String::new(),
|
||||
export_table_name: None,
|
||||
export_column_types: None,
|
||||
selected_columns: None,
|
||||
@@ -691,6 +693,7 @@ async fn live_mysql_xlsx_export_can_outlive_query_timeout_while_rows_keep_arrivi
|
||||
execution_id: Some(format!("live-mysql-query-export-timeout-{suffix}")),
|
||||
date_time_format: None,
|
||||
csv_quote_mode: Default::default(),
|
||||
null_literal: String::new(),
|
||||
export_table_name: None,
|
||||
export_column_types: None,
|
||||
selected_columns: None,
|
||||
|
||||
@@ -150,6 +150,7 @@ async fn live_postgres_query_result_export_uses_single_streamed_query() {
|
||||
execution_id: Some(format!("live-postgres-query-export-{suffix}")),
|
||||
date_time_format: None,
|
||||
csv_quote_mode: Default::default(),
|
||||
null_literal: String::new(),
|
||||
export_table_name: None,
|
||||
export_column_types: None,
|
||||
selected_columns: None,
|
||||
@@ -238,6 +239,7 @@ async fn live_postgres_query_result_xlsx_preserves_temporal_cell_types() {
|
||||
execution_id: Some(format!("live-postgres-xlsx-temporal-{suffix}")),
|
||||
date_time_format: None,
|
||||
csv_quote_mode: Default::default(),
|
||||
null_literal: String::new(),
|
||||
export_table_name: None,
|
||||
export_column_types: None,
|
||||
selected_columns: None,
|
||||
@@ -315,6 +317,7 @@ async fn live_postgres_numeric_xlsx_ignores_fractional_trailing_zeros() {
|
||||
execution_id: Some(format!("live-postgres-xlsx-numeric-{suffix}")),
|
||||
date_time_format: None,
|
||||
csv_quote_mode: Default::default(),
|
||||
null_literal: String::new(),
|
||||
export_table_name: None,
|
||||
export_column_types: None,
|
||||
selected_columns: None,
|
||||
@@ -397,6 +400,7 @@ async fn live_postgres_truncated_batch_result_export_replays_safe_temp_setup() {
|
||||
execution_id: Some(format!("temp-export-csv-{short_suffix}")),
|
||||
date_time_format: None,
|
||||
csv_quote_mode: Default::default(),
|
||||
null_literal: String::new(),
|
||||
export_table_name: None,
|
||||
export_column_types: None,
|
||||
selected_columns: None,
|
||||
@@ -481,6 +485,7 @@ async fn live_postgres_xlsx_export_can_outlive_query_timeout_while_rows_keep_arr
|
||||
execution_id: Some(format!("live-postgres-query-export-timeout-{suffix}")),
|
||||
date_time_format: None,
|
||||
csv_quote_mode: Default::default(),
|
||||
null_literal: String::new(),
|
||||
export_table_name: None,
|
||||
export_column_types: None,
|
||||
selected_columns: None,
|
||||
@@ -556,6 +561,7 @@ async fn live_postgres_stream_still_times_out_without_progress_and_recovers() {
|
||||
execution_id: Some(format!("live-postgres-query-export-stall-{suffix}")),
|
||||
date_time_format: None,
|
||||
csv_quote_mode: Default::default(),
|
||||
null_literal: String::new(),
|
||||
export_table_name: None,
|
||||
export_column_types: None,
|
||||
selected_columns: None,
|
||||
|
||||
@@ -1215,6 +1215,7 @@ async fn live_sqlserver_query_result_export_streams_cte_query_to_csv() {
|
||||
execution_id: Some(format!("live-sqlserver-export-{suffix}")),
|
||||
date_time_format: None,
|
||||
csv_quote_mode: Default::default(),
|
||||
null_literal: String::new(),
|
||||
export_table_name: None,
|
||||
export_column_types: None,
|
||||
selected_columns: None,
|
||||
|
||||
@@ -129,6 +129,7 @@ async fn live_sqlserver_xlsx_export_can_outlive_query_timeout_while_rows_keep_ar
|
||||
execution_id: Some(format!("live-sqlserver-xlsx-{suffix}")),
|
||||
date_time_format: None,
|
||||
csv_quote_mode: Default::default(),
|
||||
null_literal: String::new(),
|
||||
export_table_name: None,
|
||||
export_column_types: None,
|
||||
selected_columns: None,
|
||||
|
||||
@@ -11,6 +11,27 @@ pub enum CsvQuoteMode {
|
||||
Necessary,
|
||||
}
|
||||
|
||||
/// CSV 里 NULL 的默认字面量(`\N`,MySQL `SELECT ... INTO OUTFILE` 惯例)。
|
||||
///
|
||||
/// 默认值必须同时被导出与导入两侧引用:导出写出该字面量、导入按该字面量还原 NULL,
|
||||
/// 空字符串则始终写成 `""`,这样「空字符串」与「NULL」在 CSV 中不再互相冒充。
|
||||
pub const DEFAULT_CSV_NULL_LITERAL: &str = "\\N";
|
||||
|
||||
/// 把导出/导入配置里的原始字面量解析成实际生效的值:空串表示「不写 NULL 标记」,
|
||||
/// 即沿用历史行为(NULL 写成空字段,无法与空字符串区分)。
|
||||
pub fn csv_null_literal(raw: &str) -> Option<&str> {
|
||||
if raw.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(raw)
|
||||
}
|
||||
}
|
||||
|
||||
/// 导出/导入配置里 NULL 字面量的 serde 默认值:未显式配置时用 [`DEFAULT_CSV_NULL_LITERAL`]。
|
||||
pub fn default_csv_null_literal() -> String {
|
||||
DEFAULT_CSV_NULL_LITERAL.to_string()
|
||||
}
|
||||
|
||||
/// CSV 转义直写目标 buffer:包引号 + 内部 `"` 翻倍。值不含 `"` 时整段拷贝,
|
||||
/// 不做 replace 分配(逐批流式导出对每个单元格调用,是导出热路径)。
|
||||
fn push_csv_escaped_content(out: &mut String, value: &str) {
|
||||
@@ -67,7 +88,26 @@ pub fn push_csv_text_value(out: &mut String, value: &Value) {
|
||||
out.push('"');
|
||||
}
|
||||
|
||||
fn push_csv_value_with_quote_mode(out: &mut String, value: &Value, quote_mode: CsvQuoteMode, quote_null: bool) {
|
||||
fn push_csv_value_with_quote_mode(
|
||||
out: &mut String,
|
||||
value: &Value,
|
||||
quote_mode: CsvQuoteMode,
|
||||
quote_null: bool,
|
||||
null_literal: Option<&str>,
|
||||
) {
|
||||
if value.is_null() {
|
||||
// 配置了 NULL 字面量时,NULL 写成该字面量;空字符串仍然写成 `""`。
|
||||
// 两者不再互相冒充,导出→导入才能无损往返。
|
||||
//
|
||||
// 字面量必须尽量裸写:PostgreSQL 的 COPY ... FORMAT csv 从不把带引号的值当成 NULL
|
||||
// (`"\N"` 是普通字符串,整数列直接报错),ClickHouse 的 CSV 读取更是在引号内遇到
|
||||
// `\N` 就解析失败;只有裸 `\N` 才是这两者与 MySQL 共同识别的 NULL。字面量自身含
|
||||
// 分隔符/引号/换行时才加引号(此时它本来也不再是标准 NULL 标记)。
|
||||
if let Some(literal) = null_literal {
|
||||
push_csv_field(out, literal, CsvQuoteMode::Necessary);
|
||||
return;
|
||||
}
|
||||
}
|
||||
if quote_mode == CsvQuoteMode::All {
|
||||
if value.is_null() && !quote_null {
|
||||
return;
|
||||
@@ -87,13 +127,6 @@ fn push_csv_value_with_quote_mode(out: &mut String, value: &Value, quote_mode: C
|
||||
}
|
||||
}
|
||||
|
||||
fn push_csv_value(out: &mut String, value: &Value) {
|
||||
if value.is_null() {
|
||||
return;
|
||||
}
|
||||
push_csv_text_value(out, value);
|
||||
}
|
||||
|
||||
/// TSV 转义直写:仅含特殊字符时包引号(语义与原 escape_tsv 一致)。
|
||||
fn push_tsv_escaped(out: &mut String, value: &str) {
|
||||
if value.contains('\t') || value.contains('\n') || value.contains('\r') || value.contains('"') {
|
||||
@@ -103,7 +136,14 @@ fn push_tsv_escaped(out: &mut String, value: &str) {
|
||||
}
|
||||
}
|
||||
|
||||
fn push_tsv_value(out: &mut String, value: &Value) {
|
||||
fn push_tsv_value(out: &mut String, value: &Value, null_literal: Option<&str>) {
|
||||
if value.is_null() {
|
||||
// 与 CSV 一致:NULL 写字面量、空字符串写空字段,导出→导入才能无损往返。
|
||||
if let Some(literal) = null_literal {
|
||||
push_tsv_escaped(out, literal);
|
||||
return;
|
||||
}
|
||||
}
|
||||
match value {
|
||||
Value::Null => {}
|
||||
Value::String(v) => push_tsv_escaped(out, v),
|
||||
@@ -115,12 +155,12 @@ fn push_tsv_value(out: &mut String, value: &Value) {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn push_tsv_row(out: &mut String, row: &[Value]) {
|
||||
pub fn push_tsv_row(out: &mut String, row: &[Value], null_literal: Option<&str>) {
|
||||
for (cell_index, cell) in row.iter().enumerate() {
|
||||
if cell_index > 0 {
|
||||
out.push('\t');
|
||||
}
|
||||
push_tsv_value(out, cell);
|
||||
push_tsv_value(out, cell, null_literal);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -146,12 +186,6 @@ fn escape_csv_with_quote_mode(value: &str, quote_mode: CsvQuoteMode) -> String {
|
||||
out
|
||||
}
|
||||
|
||||
fn format_csv_value(value: &Value) -> String {
|
||||
let mut out = String::new();
|
||||
push_csv_value(&mut out, value);
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg_attr(not(test), allow(dead_code))]
|
||||
pub fn escape_tsv(value: &str) -> String {
|
||||
let mut out = String::with_capacity(value.len());
|
||||
@@ -159,19 +193,19 @@ pub fn escape_tsv(value: &str) -> String {
|
||||
out
|
||||
}
|
||||
|
||||
fn push_tsv_rows(out: &mut String, rows: &[Vec<Value>]) {
|
||||
fn push_tsv_rows(out: &mut String, rows: &[Vec<Value>], null_literal: Option<&str>) {
|
||||
for (row_index, row) in rows.iter().enumerate() {
|
||||
if row_index > 0 {
|
||||
out.push('\n');
|
||||
}
|
||||
push_tsv_row(out, row);
|
||||
push_tsv_row(out, row, null_literal);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg_attr(not(test), allow(dead_code))]
|
||||
pub fn format_tsv_rows(rows: &[Vec<Value>]) -> String {
|
||||
let mut out = String::with_capacity(estimated_rows_capacity(rows));
|
||||
push_tsv_rows(&mut out, rows);
|
||||
push_tsv_rows(&mut out, rows, None);
|
||||
out
|
||||
}
|
||||
|
||||
@@ -186,7 +220,7 @@ pub fn format_tsv(columns: &[String], rows: &[Vec<Value>]) -> String {
|
||||
push_tsv_escaped(&mut out, column);
|
||||
}
|
||||
out.push('\n');
|
||||
push_tsv_rows(&mut out, rows);
|
||||
push_tsv_rows(&mut out, rows, None);
|
||||
out
|
||||
}
|
||||
|
||||
@@ -198,11 +232,21 @@ pub fn push_query_result_csv_row(out: &mut String, row: &[Value]) {
|
||||
}
|
||||
|
||||
pub fn push_query_result_csv_row_with_quote_mode(out: &mut String, row: &[Value], quote_mode: CsvQuoteMode) {
|
||||
push_query_result_csv_row_with_options(out, row, quote_mode, None);
|
||||
}
|
||||
|
||||
/// 与 [`push_query_result_csv_row_with_quote_mode`] 相同,但允许把 NULL 写成指定字面量。
|
||||
pub fn push_query_result_csv_row_with_options(
|
||||
out: &mut String,
|
||||
row: &[Value],
|
||||
quote_mode: CsvQuoteMode,
|
||||
null_literal: Option<&str>,
|
||||
) {
|
||||
for (cell_index, cell) in row.iter().enumerate() {
|
||||
if cell_index > 0 {
|
||||
out.push(',');
|
||||
}
|
||||
push_csv_value_with_quote_mode(out, cell, quote_mode, false);
|
||||
push_csv_value_with_quote_mode(out, cell, quote_mode, false, null_literal);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -211,11 +255,21 @@ pub fn push_table_csv_row(out: &mut String, row: &[Value]) {
|
||||
}
|
||||
|
||||
pub fn push_table_csv_row_with_quote_mode(out: &mut String, row: &[Value], quote_mode: CsvQuoteMode) {
|
||||
push_table_csv_row_with_options(out, row, quote_mode, None);
|
||||
}
|
||||
|
||||
/// 与 [`push_table_csv_row_with_quote_mode`] 相同,但允许把 NULL 写成指定字面量。
|
||||
pub fn push_table_csv_row_with_options(
|
||||
out: &mut String,
|
||||
row: &[Value],
|
||||
quote_mode: CsvQuoteMode,
|
||||
null_literal: Option<&str>,
|
||||
) {
|
||||
for (cell_index, cell) in row.iter().enumerate() {
|
||||
if cell_index > 0 {
|
||||
out.push(',');
|
||||
}
|
||||
push_csv_value_with_quote_mode(out, cell, quote_mode, true);
|
||||
push_csv_value_with_quote_mode(out, cell, quote_mode, true, null_literal);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -254,6 +308,19 @@ pub fn format_csv(columns: &[String], rows: &[Vec<Value>]) -> String {
|
||||
}
|
||||
|
||||
pub fn format_csv_with_quote_mode(columns: &[String], rows: &[Vec<Value>], quote_mode: CsvQuoteMode) -> String {
|
||||
format_csv_with_options(columns, rows, quote_mode, None)
|
||||
}
|
||||
|
||||
/// 与 [`format_csv_with_quote_mode`] 相同,但允许把 NULL 写成指定字面量。
|
||||
///
|
||||
/// 表数据导出的首批行走这里,后续分批走 [`push_table_csv_row_with_options`]:
|
||||
/// 两处必须收到同一个字面量,否则同一个文件里 NULL 会前后不一致。
|
||||
pub fn format_csv_with_options(
|
||||
columns: &[String],
|
||||
rows: &[Vec<Value>],
|
||||
quote_mode: CsvQuoteMode,
|
||||
null_literal: Option<&str>,
|
||||
) -> String {
|
||||
let mut out = String::with_capacity(
|
||||
estimated_rows_capacity(rows).saturating_add(columns.len().saturating_mul(12)).min(ROWS_CAPACITY_ESTIMATE_MAX),
|
||||
);
|
||||
@@ -268,7 +335,7 @@ pub fn format_csv_with_quote_mode(columns: &[String], rows: &[Vec<Value>], quote
|
||||
if row_index > 0 {
|
||||
out.push('\n');
|
||||
}
|
||||
push_query_result_csv_row_with_quote_mode(&mut out, row, quote_mode);
|
||||
push_query_result_csv_row_with_options(&mut out, row, quote_mode, null_literal);
|
||||
}
|
||||
out
|
||||
}
|
||||
@@ -282,7 +349,17 @@ pub fn format_query_result_csv_with_quote_mode(
|
||||
rows: &[Vec<Value>],
|
||||
quote_mode: CsvQuoteMode,
|
||||
) -> String {
|
||||
format_csv_with_quote_mode(columns, rows, quote_mode)
|
||||
format_csv_with_options(columns, rows, quote_mode, None)
|
||||
}
|
||||
|
||||
/// 与 [`format_query_result_csv_with_quote_mode`] 相同,但允许把 NULL 写成指定字面量。
|
||||
pub fn format_query_result_csv_with_options(
|
||||
columns: &[String],
|
||||
rows: &[Vec<Value>],
|
||||
quote_mode: CsvQuoteMode,
|
||||
null_literal: Option<&str>,
|
||||
) -> String {
|
||||
format_csv_with_options(columns, rows, quote_mode, null_literal)
|
||||
}
|
||||
|
||||
pub fn write_csv_text_row(
|
||||
@@ -305,20 +382,26 @@ pub fn write_csv_value_row(
|
||||
writer: &mut impl Write,
|
||||
values: impl IntoIterator<Item = Value>,
|
||||
quote_mode: CsvQuoteMode,
|
||||
) -> Result<(), String> {
|
||||
write_csv_value_row_with_options(writer, values, quote_mode, None)
|
||||
}
|
||||
|
||||
/// 与 [`write_csv_value_row`] 相同,但允许把 NULL 写成指定字面量。
|
||||
pub fn write_csv_value_row_with_options(
|
||||
writer: &mut impl Write,
|
||||
values: impl IntoIterator<Item = Value>,
|
||||
quote_mode: CsvQuoteMode,
|
||||
null_literal: Option<&str>,
|
||||
) -> Result<(), String> {
|
||||
let mut first = true;
|
||||
let mut formatted = String::new();
|
||||
for value in values {
|
||||
if !first {
|
||||
writer.write_all(b",").map_err(|err| err.to_string())?;
|
||||
}
|
||||
first = false;
|
||||
let formatted = if quote_mode == CsvQuoteMode::All {
|
||||
format_csv_value(&value)
|
||||
} else {
|
||||
let mut formatted = String::new();
|
||||
push_csv_value_with_quote_mode(&mut formatted, &value, quote_mode, false);
|
||||
formatted
|
||||
};
|
||||
formatted.clear();
|
||||
push_csv_value_with_quote_mode(&mut formatted, &value, quote_mode, false, null_literal);
|
||||
writer.write_all(formatted.as_bytes()).map_err(|err| err.to_string())?;
|
||||
}
|
||||
Ok(())
|
||||
@@ -327,8 +410,8 @@ pub fn write_csv_value_row(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
format_csv, format_csv_with_quote_mode, format_query_result_csv, format_query_result_csv_rows, format_tsv,
|
||||
CsvQuoteMode,
|
||||
format_csv, format_csv_with_options, format_csv_with_quote_mode, format_query_result_csv,
|
||||
format_query_result_csv_rows, format_tsv, CsvQuoteMode, DEFAULT_CSV_NULL_LITERAL,
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
@@ -434,6 +517,104 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_null_literal_is_the_mysql_outfile_convention() {
|
||||
assert_eq!(DEFAULT_CSV_NULL_LITERAL, "\\N");
|
||||
// 空串表示「不写 NULL 标记」,即沿用历史行为
|
||||
assert_eq!(super::csv_null_literal(""), None);
|
||||
assert_eq!(super::csv_null_literal(DEFAULT_CSV_NULL_LITERAL), Some("\\N"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tsv_null_literal_separates_null_from_an_empty_string() {
|
||||
let row = vec![Value::Null, json!("")];
|
||||
|
||||
// 未配置字面量(旧行为):NULL 与空串都写空字段,二者在文件里无法区分
|
||||
let mut out = String::new();
|
||||
super::push_tsv_row(&mut out, &row, None);
|
||||
assert_eq!(out, "\t");
|
||||
|
||||
// 配置字面量后:NULL 写成字面量,空字符串仍是空字段
|
||||
let mut out = String::new();
|
||||
super::push_tsv_row(&mut out, &row, Some(DEFAULT_CSV_NULL_LITERAL));
|
||||
assert_eq!(out, "\\N\t");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn null_literal_stops_null_from_impersonating_an_empty_string() {
|
||||
let columns = vec!["a".to_string(), "b".to_string()];
|
||||
let row = vec![Value::Null, json!("")];
|
||||
|
||||
// 未配置字面量(旧行为):NULL 写成裸空字段,空串写成 `""`,
|
||||
// 文件里看似可区分,但导入端拿不到引号信息,两者最终都会变成 NULL
|
||||
assert_eq!(
|
||||
format_csv_with_options(&columns, std::slice::from_ref(&row), CsvQuoteMode::All, None),
|
||||
"\"a\",\"b\"\n,\"\""
|
||||
);
|
||||
|
||||
// 配置字面量后:NULL 写成裸字面量,空串写成 `""`;即使 quote mode = All 也不给
|
||||
// 字面量加引号,否则 PostgreSQL/ClickHouse 的 CSV 读取不认它是 NULL
|
||||
assert_eq!(
|
||||
format_csv_with_options(
|
||||
&columns,
|
||||
std::slice::from_ref(&row),
|
||||
CsvQuoteMode::All,
|
||||
Some(DEFAULT_CSV_NULL_LITERAL)
|
||||
),
|
||||
"\"a\",\"b\"\n\\N,\"\""
|
||||
);
|
||||
// Necessary 模式:空串裸写为空字段,但 NULL 有独立字面量,仍然可区分
|
||||
assert_eq!(
|
||||
format_csv_with_options(
|
||||
&columns,
|
||||
std::slice::from_ref(&row),
|
||||
CsvQuoteMode::Necessary,
|
||||
Some(DEFAULT_CSV_NULL_LITERAL)
|
||||
),
|
||||
"a,b\n\\N,"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn null_literal_is_quoted_only_when_it_needs_quoting() {
|
||||
let columns = vec!["a".to_string()];
|
||||
let row = vec![Value::Null];
|
||||
|
||||
// 标准字面量裸写(PostgreSQL/ClickHouse/MySQL 共同识别的 NULL 写法)
|
||||
assert_eq!(
|
||||
format_csv_with_options(&columns, std::slice::from_ref(&row), CsvQuoteMode::All, Some("\\N")),
|
||||
"\"a\"\n\\N"
|
||||
);
|
||||
// 自定义字面量含分隔符时无法裸写,只能退化成带引号的普通字段
|
||||
assert_eq!(
|
||||
format_csv_with_options(&columns, std::slice::from_ref(&row), CsvQuoteMode::All, Some("a,b")),
|
||||
"\"a\"\n\"a,b\""
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn null_literal_applies_to_streaming_row_writers_too() {
|
||||
let row = vec![Value::Null, json!("NULL"), json!("")];
|
||||
|
||||
let mut table = String::new();
|
||||
super::push_table_csv_row_with_options(&mut table, &row, CsvQuoteMode::All, Some(DEFAULT_CSV_NULL_LITERAL));
|
||||
assert_eq!(table, "\\N,\"NULL\",\"\"");
|
||||
|
||||
let mut query = String::new();
|
||||
super::push_query_result_csv_row_with_options(
|
||||
&mut query,
|
||||
&row,
|
||||
CsvQuoteMode::All,
|
||||
Some(DEFAULT_CSV_NULL_LITERAL),
|
||||
);
|
||||
assert_eq!(query, "\\N,\"NULL\",\"\"");
|
||||
|
||||
let mut written = Vec::new();
|
||||
super::write_csv_value_row_with_options(&mut written, row, CsvQuoteMode::All, Some(DEFAULT_CSV_NULL_LITERAL))
|
||||
.unwrap();
|
||||
assert_eq!(String::from_utf8(written).unwrap(), "\\N,\"NULL\",\"\"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reusable_row_buffers_match_batch_formatters() {
|
||||
let row = vec![Value::Null, json!("NULL"), json!("line\n\"two\""), json!(42)];
|
||||
@@ -447,7 +628,7 @@ mod tests {
|
||||
assert_eq!(table_csv, "\"\",\"NULL\",\"line\n\"\"two\"\"\",\"42\"");
|
||||
|
||||
let mut tsv = String::new();
|
||||
super::push_tsv_row(&mut tsv, &row);
|
||||
super::push_tsv_row(&mut tsv, &row, None);
|
||||
assert_eq!(tsv, super::format_tsv_rows(&[row]));
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user