feat(export): add streaming TXT data export
This commit is contained in:
1 parent
99b84cbc92
commit
78b3bb1ba1
17 files changed
+471
-81
No files matched your search
@@ -48,6 +48,27 @@ pub(crate) fn value_to_query_result_csv_text(value: &Value) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn escape_tsv(value: &str) -> String {
|
||||
if value.contains('\t') || value.contains('\n') || value.contains('\r') || value.contains('"') {
|
||||
format!("\"{}\"", value.replace('"', "\"\""))
|
||||
} else {
|
||||
value.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn format_tsv_rows(rows: &[Vec<Value>]) -> String {
|
||||
rows.iter()
|
||||
.map(|row| row.iter().map(|cell| escape_tsv(&value_to_csv_text(cell))).collect::<Vec<_>>().join("\t"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
pub(crate) fn format_tsv(columns: &[String], rows: &[Vec<Value>]) -> String {
|
||||
let header = columns.iter().map(|col| escape_tsv(col)).collect::<Vec<_>>().join("\t");
|
||||
let body = format_tsv_rows(rows);
|
||||
format!("{header}\n{body}")
|
||||
}
|
||||
|
||||
/// Format query-result rows as CSV text without a header row, using the
|
||||
/// query-result NULL semantics (NULL → "NULL"). Used by the streaming
|
||||
/// query-result export for batches after the first.
|
||||
@@ -176,7 +197,7 @@ pub async fn export_table_data_csv_core(state: &AppState, options: TableCsvExpor
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{format_csv, format_query_result_csv};
|
||||
use super::{format_csv, format_query_result_csv, format_tsv};
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
@@ -197,5 +218,14 @@ mod tests {
|
||||
assert_eq!(out, "\"id\",\"note\"\n\"1\",\"NULL\"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn formats_tsv_with_empty_null_and_escaped_special_values() {
|
||||
let out = format_tsv(
|
||||
&["id".to_string(), "note".to_string()],
|
||||
&[vec![json!(1), Value::Null], vec![json!(2), json!("line1\n\"line2\"")]],
|
||||
);
|
||||
assert_eq!(out, "id\tnote\n1\t\n2\t\"line1\n\"\"line2\"\"\"");
|
||||
}
|
||||
|
||||
use serde_json::Value;
|
||||
}
|
||||
@@ -8,7 +8,7 @@ use std::sync::{
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crate::connection::{AppState, PoolKind};
|
||||
use crate::csv_export::{format_query_result_csv, format_query_result_csv_rows};
|
||||
use crate::csv_export::{format_query_result_csv, format_query_result_csv_rows, format_tsv, format_tsv_rows};
|
||||
use crate::database_export::is_export_cancelled;
|
||||
pub use crate::database_export::ExportStatus;
|
||||
use crate::models::connection::DatabaseType;
|
||||
@@ -110,6 +110,19 @@ fn xlsx_hard_limit_active(format: &str, request: &QueryResultExportRequest) -> b
|
||||
format == "xlsx" && request.row_limit.map_or(true, |limit| limit > XLSX_MAX_DATA_ROWS)
|
||||
}
|
||||
|
||||
fn format_text_export_header(format: &str, columns: &[String]) -> String {
|
||||
let content = if format == "csv" { format_query_result_csv(columns, &[]) } else { format_tsv(columns, &[]) };
|
||||
content.strip_suffix('\n').unwrap_or(&content).to_string()
|
||||
}
|
||||
|
||||
fn format_text_export_rows(format: &str, rows: &[Vec<Value>]) -> String {
|
||||
if format == "csv" {
|
||||
format_query_result_csv_rows(rows)
|
||||
} else {
|
||||
format_tsv_rows(rows)
|
||||
}
|
||||
}
|
||||
|
||||
fn should_emit_stream_progress(
|
||||
rows_exported: u64,
|
||||
last_progress_rows: u64,
|
||||
@@ -311,7 +324,7 @@ async fn export_query_result_core_inner(
|
||||
session_id: &mut Option<String>,
|
||||
) -> Result<(), String> {
|
||||
let format = request.format.to_lowercase();
|
||||
if format != "csv" && format != "xlsx" {
|
||||
if format != "csv" && format != "xlsx" && format != "txt" {
|
||||
return Err(format!("Unsupported streaming query-result export format: {format}"));
|
||||
}
|
||||
|
||||
@@ -351,12 +364,12 @@ async fn export_query_result_core_inner(
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut csv_file = if format == "csv" {
|
||||
let mut text_file = if format == "csv" || format == "txt" {
|
||||
Some(BufWriter::new(File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(file) = csv_file.as_mut() {
|
||||
if let Some(file) = text_file.as_mut() {
|
||||
file.write_all(b"\xEF\xBB\xBF").map_err(|e| format!("Failed to write BOM: {e}"))?;
|
||||
}
|
||||
|
||||
@@ -365,7 +378,7 @@ async fn export_query_result_core_inner(
|
||||
let mut column_types: Vec<String> = Vec::new();
|
||||
let mut rows_exported: u64 = 0;
|
||||
let mut offset: usize = 0;
|
||||
let mut wrote_csv_header = false;
|
||||
let mut wrote_text_header = false;
|
||||
let mut keyset_plan = build_keyset_plan(state, request).await;
|
||||
if keyset_plan.is_none() && !request.use_agent_cursor && !supports_streaming_offset_pagination(request, page_size) {
|
||||
return Err(STREAMING_PAGINATION_UNSUPPORTED_ERROR.to_string());
|
||||
@@ -503,15 +516,19 @@ async fn export_query_result_core_inner(
|
||||
}
|
||||
let row_count = result.rows.len();
|
||||
|
||||
if format == "csv" {
|
||||
if let Some(file) = csv_file.as_mut() {
|
||||
if !wrote_csv_header {
|
||||
let csv = format_query_result_csv(&columns, &result.rows);
|
||||
file.write_all(csv.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?;
|
||||
wrote_csv_header = true;
|
||||
if format == "csv" || format == "txt" {
|
||||
if let Some(file) = text_file.as_mut() {
|
||||
if !wrote_text_header {
|
||||
let header = format_text_export_header(&format, &columns);
|
||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
||||
if row_count > 0 {
|
||||
let rows = format_text_export_rows(&format, &result.rows);
|
||||
write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?;
|
||||
}
|
||||
wrote_text_header = true;
|
||||
} else if row_count > 0 {
|
||||
let rows_csv = format_query_result_csv_rows(&result.rows);
|
||||
write!(file, "\n{rows_csv}").map_err(|e| format!("Failed to write CSV rows: {e}"))?;
|
||||
let rows = format_text_export_rows(&format, &result.rows);
|
||||
write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -568,15 +585,15 @@ async fn export_query_result_core_inner(
|
||||
|
||||
on_progress(progress(request, rows_exported, ExportStatus::Writing, None));
|
||||
|
||||
if format == "csv" {
|
||||
if !wrote_csv_header {
|
||||
let csv = format_query_result_csv(&columns, &[]);
|
||||
if let Some(file) = csv_file.as_mut() {
|
||||
file.write_all(csv.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?;
|
||||
if format == "csv" || format == "txt" {
|
||||
if !wrote_text_header {
|
||||
let header = format_text_export_header(&format, &columns);
|
||||
if let Some(file) = text_file.as_mut() {
|
||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
||||
}
|
||||
}
|
||||
if let Some(file) = csv_file.as_mut() {
|
||||
file.flush().map_err(|e| format!("Failed to flush CSV file: {e}"))?;
|
||||
if let Some(file) = text_file.as_mut() {
|
||||
file.flush().map_err(|e| format!("Failed to flush text export file: {e}"))?;
|
||||
}
|
||||
} else if let Some(writer) = xlsx {
|
||||
let mut buf =
|
||||
@@ -646,7 +663,7 @@ async fn try_export_postgres_query_result_stream(
|
||||
let mut rows_exported = 0_u64;
|
||||
let mut last_progress_rows = 0_u64;
|
||||
let mut last_progress_at = Instant::now();
|
||||
let mut csv_file = if format == "csv" {
|
||||
let mut text_file = if format == "csv" || format == "txt" {
|
||||
let mut file =
|
||||
BufWriter::new(File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?);
|
||||
file.write_all(b"\xEF\xBB\xBF").map_err(|e| format!("Failed to write BOM: {e}"))?;
|
||||
@@ -670,10 +687,9 @@ async fn try_export_postgres_query_result_stream(
|
||||
match item {
|
||||
crate::db::postgres::PostgresQueryStreamItem::Columns { columns: stream_columns, column_types } => {
|
||||
columns = stream_columns;
|
||||
if let Some(file) = csv_file.as_mut() {
|
||||
let csv = format_query_result_csv(&columns, &[]);
|
||||
let header = csv.strip_suffix('\n').unwrap_or(&csv);
|
||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?;
|
||||
if let Some(file) = text_file.as_mut() {
|
||||
let header = format_text_export_header(format, &columns);
|
||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
||||
} else {
|
||||
let xlsx_file =
|
||||
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
|
||||
@@ -689,9 +705,9 @@ async fn try_export_postgres_query_result_stream(
|
||||
if xlsx_hard_limit_active && rows_exported as usize >= XLSX_MAX_DATA_ROWS {
|
||||
return Err(XLSX_ROW_LIMIT_ERROR.to_string());
|
||||
}
|
||||
if let Some(file) = csv_file.as_mut() {
|
||||
let rows_csv = format_query_result_csv_rows(std::slice::from_ref(&row));
|
||||
write!(file, "\n{rows_csv}").map_err(|e| format!("Failed to write CSV rows: {e}"))?;
|
||||
if let Some(file) = text_file.as_mut() {
|
||||
let rows = format_text_export_rows(format, std::slice::from_ref(&row));
|
||||
write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?;
|
||||
} else if let Some(writer) = xlsx.as_mut() {
|
||||
writer.write_row(&row).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||
} else {
|
||||
@@ -730,8 +746,8 @@ async fn try_export_postgres_query_result_stream(
|
||||
on_progress(progress(request, rows_exported, ExportStatus::Running, None));
|
||||
}
|
||||
on_progress(progress(request, rows_exported, ExportStatus::Writing, None));
|
||||
if let Some(file) = csv_file.as_mut() {
|
||||
file.flush().map_err(|e| format!("Failed to flush CSV file: {e}"))?;
|
||||
if let Some(file) = text_file.as_mut() {
|
||||
file.flush().map_err(|e| format!("Failed to flush text export file: {e}"))?;
|
||||
}
|
||||
if let Some(writer) = xlsx {
|
||||
let mut buf =
|
||||
@@ -813,7 +829,7 @@ async fn try_export_mysql_query_result_stream(
|
||||
let mut rows_exported = 0_u64;
|
||||
let mut last_progress_rows = 0_u64;
|
||||
let mut last_progress_at = Instant::now();
|
||||
let mut csv_file = if format == "csv" {
|
||||
let mut text_file = if format == "csv" || format == "txt" {
|
||||
let mut file =
|
||||
BufWriter::new(File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?);
|
||||
file.write_all(b"\xEF\xBB\xBF").map_err(|e| format!("Failed to write BOM: {e}"))?;
|
||||
@@ -886,10 +902,9 @@ async fn try_export_mysql_query_result_stream(
|
||||
match item {
|
||||
crate::db::mysql::MySqlQueryStreamItem::Columns { columns: stream_columns, column_types } => {
|
||||
columns = stream_columns;
|
||||
if let Some(file) = csv_file.as_mut() {
|
||||
let csv = format_query_result_csv(&columns, &[]);
|
||||
let header = csv.strip_suffix('\n').unwrap_or(&csv);
|
||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?;
|
||||
if let Some(file) = text_file.as_mut() {
|
||||
let header = format_text_export_header(format, &columns);
|
||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
||||
} else {
|
||||
let xlsx_file =
|
||||
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
|
||||
@@ -905,9 +920,9 @@ async fn try_export_mysql_query_result_stream(
|
||||
if xlsx_hard_limit_active && rows_exported as usize >= XLSX_MAX_DATA_ROWS {
|
||||
return Err(XLSX_ROW_LIMIT_ERROR.to_string());
|
||||
}
|
||||
if let Some(file) = csv_file.as_mut() {
|
||||
let rows_csv = format_query_result_csv_rows(std::slice::from_ref(&row));
|
||||
write!(file, "\n{rows_csv}").map_err(|e| format!("Failed to write CSV rows: {e}"))?;
|
||||
if let Some(file) = text_file.as_mut() {
|
||||
let rows = format_text_export_rows(format, std::slice::from_ref(&row));
|
||||
write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?;
|
||||
} else if let Some(writer) = xlsx.as_mut() {
|
||||
writer.write_row(&row).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||
} else {
|
||||
@@ -992,8 +1007,8 @@ async fn try_export_mysql_query_result_stream(
|
||||
on_progress(progress(request, rows_exported, ExportStatus::Running, None));
|
||||
}
|
||||
on_progress(progress(request, rows_exported, ExportStatus::Writing, None));
|
||||
if let Some(file) = csv_file.as_mut() {
|
||||
file.flush().map_err(|e| format!("Failed to flush CSV file: {e}"))?;
|
||||
if let Some(file) = text_file.as_mut() {
|
||||
file.flush().map_err(|e| format!("Failed to flush text export file: {e}"))?;
|
||||
}
|
||||
if let Some(writer) = xlsx {
|
||||
let mut buf =
|
||||
@@ -1063,7 +1078,7 @@ async fn try_export_clickhouse_query_result_stream(
|
||||
let mut rows_exported = 0_u64;
|
||||
let mut last_progress_rows = 0_u64;
|
||||
let mut last_progress_at = Instant::now();
|
||||
let mut csv_file = if format == "csv" {
|
||||
let mut text_file = if format == "csv" || format == "txt" {
|
||||
let mut file =
|
||||
BufWriter::new(File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?);
|
||||
file.write_all(b"\xEF\xBB\xBF").map_err(|e| format!("Failed to write BOM: {e}"))?;
|
||||
@@ -1087,10 +1102,9 @@ async fn try_export_clickhouse_query_result_stream(
|
||||
column_types,
|
||||
} => {
|
||||
columns = stream_columns;
|
||||
if let Some(file) = csv_file.as_mut() {
|
||||
let csv = format_query_result_csv(&columns, &[]);
|
||||
let header = csv.strip_suffix('\n').unwrap_or(&csv);
|
||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?;
|
||||
if let Some(file) = text_file.as_mut() {
|
||||
let header = format_text_export_header(format, &columns);
|
||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
||||
} else {
|
||||
let xlsx_file =
|
||||
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
|
||||
@@ -1106,9 +1120,9 @@ async fn try_export_clickhouse_query_result_stream(
|
||||
if xlsx_hard_limit_active && rows_exported as usize >= XLSX_MAX_DATA_ROWS {
|
||||
return Err(XLSX_ROW_LIMIT_ERROR.to_string());
|
||||
}
|
||||
if let Some(file) = csv_file.as_mut() {
|
||||
let rows_csv = format_query_result_csv_rows(std::slice::from_ref(&row));
|
||||
write!(file, "\n{rows_csv}").map_err(|e| format!("Failed to write CSV rows: {e}"))?;
|
||||
if let Some(file) = text_file.as_mut() {
|
||||
let rows = format_text_export_rows(format, std::slice::from_ref(&row));
|
||||
write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?;
|
||||
} else if let Some(writer) = xlsx.as_mut() {
|
||||
writer.write_row(&row).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||
} else {
|
||||
@@ -1169,8 +1183,8 @@ async fn try_export_clickhouse_query_result_stream(
|
||||
on_progress(progress(request, rows_exported, ExportStatus::Running, None));
|
||||
}
|
||||
on_progress(progress(request, rows_exported, ExportStatus::Writing, None));
|
||||
if let Some(file) = csv_file.as_mut() {
|
||||
file.flush().map_err(|e| format!("Failed to flush CSV file: {e}"))?;
|
||||
if let Some(file) = text_file.as_mut() {
|
||||
file.flush().map_err(|e| format!("Failed to flush text export file: {e}"))?;
|
||||
}
|
||||
if let Some(writer) = xlsx {
|
||||
let mut buf =
|
||||
@@ -1221,7 +1235,7 @@ async fn try_export_sqlserver_query_result_stream(
|
||||
let mut last_progress_rows = 0_u64;
|
||||
let mut last_progress_at = Instant::now();
|
||||
let progress_row_interval = request.page_size.max(1) as u64;
|
||||
let mut csv_file = if format == "csv" {
|
||||
let mut text_file = if format == "csv" || format == "txt" {
|
||||
let mut file =
|
||||
BufWriter::new(File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?);
|
||||
file.write_all(b"\xEF\xBB\xBF").map_err(|e| format!("Failed to write BOM: {e}"))?;
|
||||
@@ -1251,10 +1265,9 @@ async fn try_export_sqlserver_query_result_stream(
|
||||
match item {
|
||||
crate::db::sqlserver::SqlServerStreamItem::Columns(stream_columns) => {
|
||||
columns = stream_columns.to_vec();
|
||||
if let Some(file) = csv_file.as_mut() {
|
||||
let csv = format_query_result_csv(&columns, &[]);
|
||||
let header = csv.strip_suffix('\n').unwrap_or(&csv);
|
||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?;
|
||||
if let Some(file) = text_file.as_mut() {
|
||||
let header = format_text_export_header(format, &columns);
|
||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?;
|
||||
} else {
|
||||
let xlsx_file =
|
||||
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
|
||||
@@ -1270,9 +1283,9 @@ async fn try_export_sqlserver_query_result_stream(
|
||||
if xlsx_hard_limit_active && rows_exported as usize >= XLSX_MAX_DATA_ROWS {
|
||||
return Err(XLSX_ROW_LIMIT_ERROR.to_string());
|
||||
}
|
||||
if let Some(file) = csv_file.as_mut() {
|
||||
let rows_csv = format_query_result_csv_rows(&[row.to_vec()]);
|
||||
write!(file, "\n{rows_csv}").map_err(|e| format!("Failed to write CSV rows: {e}"))?;
|
||||
if let Some(file) = text_file.as_mut() {
|
||||
let rows = format_text_export_rows(format, &[row.to_vec()]);
|
||||
write!(file, "\n{rows}").map_err(|e| format!("Failed to write export rows: {e}"))?;
|
||||
} else if let Some(writer) = xlsx.as_mut() {
|
||||
writer.write_row(row).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||
} else {
|
||||
@@ -1317,8 +1330,8 @@ async fn try_export_sqlserver_query_result_stream(
|
||||
on_progress(progress(request, rows_exported, ExportStatus::Running, None));
|
||||
}
|
||||
on_progress(progress(request, rows_exported, ExportStatus::Writing, None));
|
||||
if let Some(file) = csv_file.as_mut() {
|
||||
file.flush().map_err(|e| format!("Failed to flush CSV file: {e}"))?;
|
||||
if let Some(file) = text_file.as_mut() {
|
||||
file.flush().map_err(|e| format!("Failed to flush text export file: {e}"))?;
|
||||
}
|
||||
if let Some(writer) = xlsx {
|
||||
let mut buf =
|
||||
@@ -1360,6 +1373,16 @@ mod tests {
|
||||
assert_eq!(effective_row_limit("csv", &request("csv", None, None)), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn txt_unlimited_export_has_no_effective_row_limit() {
|
||||
assert_eq!(effective_row_limit("txt", &request("txt", None, None)), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn txt_export_header_keeps_columns_for_empty_results() {
|
||||
assert_eq!(format_text_export_header("txt", &["id".to_string(), "note".to_string()]), "id\tnote");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn xlsx_unlimited_export_uses_excel_hard_limit() {
|
||||
assert_eq!(effective_row_limit("xlsx", &request("xlsx", None, None)), Some(XLSX_MAX_DATA_ROWS));
|
||||
|
||||
@@ -8,7 +8,7 @@ use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::connection::MysqlMode;
|
||||
use crate::connection::{task_client_session_id, AppState, PoolKind};
|
||||
use crate::csv_export::{escape_csv, format_csv, value_to_csv_text};
|
||||
use crate::csv_export::{escape_csv, format_csv, format_tsv, format_tsv_rows, value_to_csv_text};
|
||||
pub use crate::database_export::ExportStatus;
|
||||
use crate::database_export::{build_export_insert_statements, is_export_cancelled, BuildExportInsertStatementsOptions};
|
||||
use crate::db::agent_driver::AgentTableReadStartParams;
|
||||
@@ -36,7 +36,7 @@ pub struct TableExportRequest {
|
||||
pub schema: Option<String>,
|
||||
pub table_name: String,
|
||||
pub file_path: String,
|
||||
/// "csv", "xlsx", "json", "markdown", or "sql"
|
||||
/// "csv", "xlsx", "json", "markdown", "sql", or "txt"
|
||||
pub format: String,
|
||||
#[serde(default)]
|
||||
pub columns: Option<Vec<String>>,
|
||||
@@ -505,6 +505,45 @@ async fn try_export_native_table_stream(
|
||||
}
|
||||
result
|
||||
}
|
||||
"txt" => {
|
||||
let mut file = BufWriter::new(
|
||||
std::fs::File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?,
|
||||
);
|
||||
let header = format_tsv(col_names, &[]);
|
||||
let header = header.strip_suffix('\n').unwrap_or(&header);
|
||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write TXT: {e}"))?;
|
||||
|
||||
let result = stream_native_table_rows(
|
||||
state,
|
||||
pool_key,
|
||||
db_type,
|
||||
&sql,
|
||||
row_limit,
|
||||
&cancelled,
|
||||
cancel_token.clone(),
|
||||
|row| {
|
||||
let row_tsv = format_tsv_rows(&[row.to_vec()]);
|
||||
write!(file, "\n{row_tsv}").map_err(|e| format!("Failed to write TXT rows: {e}"))?;
|
||||
rows_exported += 1;
|
||||
if rows_exported % progress_interval == 0 {
|
||||
on_progress(TableExportProgress {
|
||||
export_id: request.export_id.clone(),
|
||||
table_name: request.table_name.clone(),
|
||||
rows_exported,
|
||||
total_rows,
|
||||
status: ExportStatus::Running,
|
||||
error_message: None,
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.await;
|
||||
if result.is_ok() {
|
||||
file.flush().map_err(|e| format!("Failed to flush export file: {e}"))?;
|
||||
}
|
||||
result
|
||||
}
|
||||
"xlsx" => {
|
||||
let column_types = export_column_types(request);
|
||||
let xlsx_file =
|
||||
@@ -979,6 +1018,85 @@ pub async fn export_table_data_core(
|
||||
}
|
||||
}
|
||||
}
|
||||
"txt" => {
|
||||
let mut is_first_batch = true;
|
||||
let header = format_tsv(&col_names, &[]);
|
||||
let header = header.strip_suffix('\n').unwrap_or(&header);
|
||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write TXT: {e}"))?;
|
||||
|
||||
loop {
|
||||
if is_export_cancelled(&request.export_id).await {
|
||||
on_progress(TableExportProgress {
|
||||
export_id: request.export_id.clone(),
|
||||
table_name: request.table_name.clone(),
|
||||
rows_exported,
|
||||
total_rows,
|
||||
status: ExportStatus::Cancelled,
|
||||
error_message: Some("Export cancelled".to_string()),
|
||||
});
|
||||
close_table_read_session_if_open(state, &pool_key, &mut table_read_session_id).await;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let Some(active_batch_size) = next_export_batch_size(row_limit, rows_exported, batch_size) else {
|
||||
break;
|
||||
};
|
||||
let result = fetch_table_export_batch(
|
||||
state,
|
||||
&pool_key,
|
||||
request,
|
||||
&db_type,
|
||||
&col_names,
|
||||
&primary_keys,
|
||||
use_keyset,
|
||||
&last_pk_values,
|
||||
offset,
|
||||
active_batch_size,
|
||||
&mut table_read_session_id,
|
||||
&mut table_read_attempted,
|
||||
&mut table_read_completed,
|
||||
)
|
||||
.await?;
|
||||
let row_count = result.rows.len();
|
||||
if row_count == 0 {
|
||||
break;
|
||||
}
|
||||
|
||||
if is_first_batch {
|
||||
let rows_tsv = format_tsv_rows(&result.rows);
|
||||
write!(file, "\n{rows_tsv}").map_err(|e| format!("Failed to write TXT rows: {e}"))?;
|
||||
is_first_batch = false;
|
||||
} else {
|
||||
let rows_tsv = format_tsv_rows(&result.rows);
|
||||
if !rows_tsv.is_empty() {
|
||||
write!(file, "\n{rows_tsv}").map_err(|e| format!("Failed to write TXT rows: {e}"))?;
|
||||
}
|
||||
}
|
||||
|
||||
rows_exported += row_count as u64;
|
||||
|
||||
if use_keyset {
|
||||
if let Some(last_row) = result.rows.last() {
|
||||
last_pk_values = pk_indices.iter().map(|&i| last_row[i].clone()).collect();
|
||||
}
|
||||
} else {
|
||||
offset += row_count as u64;
|
||||
}
|
||||
|
||||
on_progress(TableExportProgress {
|
||||
export_id: request.export_id.clone(),
|
||||
table_name: request.table_name.clone(),
|
||||
rows_exported,
|
||||
total_rows,
|
||||
status: ExportStatus::Running,
|
||||
error_message: None,
|
||||
});
|
||||
|
||||
if row_count < active_batch_size {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
"xlsx" => {
|
||||
let column_types = export_column_types(request);
|
||||
// Create a dedicated file handle for the streaming XLSX writer
|
||||
@@ -1403,6 +1521,44 @@ mod tests {
|
||||
assert_eq!(out, "\"just\",\"one\"");
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// format_tsv (Navicat-style TXT export)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn formats_tsv_with_header_and_tab_separated_values() {
|
||||
let columns = vec!["id".to_string(), "name".to_string()];
|
||||
let rows = vec![vec![json!(1), json!("Alice")], vec![json!(2), json!("Bob")]];
|
||||
assert_eq!(format_tsv(&columns, &rows), "id\tname\n1\tAlice\n2\tBob");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn formats_tsv_renders_null_as_empty() {
|
||||
let columns = vec!["id".to_string(), "note".to_string()];
|
||||
let rows = vec![vec![json!(1), Value::Null]];
|
||||
assert_eq!(format_tsv(&columns, &rows), "id\tnote\n1\t");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn formats_tsv_quotes_fields_containing_tab_or_newline() {
|
||||
let columns = vec!["a".to_string(), "b".to_string()];
|
||||
let rows = vec![vec![json!("x\ty"), json!("line1\nline2")]];
|
||||
assert_eq!(format_tsv(&columns, &rows), "a\tb\n\"x\ty\"\t\"line1\nline2\"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn formats_tsv_escapes_embedded_quotes() {
|
||||
let columns = vec!["name".to_string()];
|
||||
let rows = vec![vec![json!(r#"Bob "Builder""#)]];
|
||||
assert_eq!(format_tsv(&columns, &rows), "name\n\"Bob \"\"Builder\"\"\"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn formats_tsv_rows_returns_empty_for_empty_rows() {
|
||||
let rows: Vec<Vec<Value>> = vec![];
|
||||
assert_eq!(format_tsv_rows(&rows), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_batch_size_respects_row_limit_remaining_rows() {
|
||||
assert_eq!(next_export_batch_size(None, 12_000, 10_000), Some(10_000));
|
||||
|
||||
Reference in new issue
Block a user