feat: stream query result CSV/XLSX exports
* docs: add query result streaming export design * feat(core): stream query result exports * feat: expose query result streaming export APIs * feat(desktop): stream query result exports from grid * fix: route query result export cancellation through its command * fix: reduce query export batch size and cancel latency * opt: del local test files
This commit is contained in:
1 parent
849caf4b39
commit
588ba3352d
25 files changed
+1787
-47
No files matched your search
@@ -41,13 +41,25 @@ pub(crate) fn value_to_csv_text(value: &Value) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
fn value_to_query_result_csv_text(value: &Value) -> String {
|
||||
pub(crate) fn value_to_query_result_csv_text(value: &Value) -> String {
|
||||
match value {
|
||||
Value::Null => "NULL".to_string(),
|
||||
other => value_to_csv_text(other),
|
||||
}
|
||||
}
|
||||
|
||||
/// Format query-result rows as CSV text without a header row, using the
|
||||
/// query-result NULL semantics (NULL → "NULL"). Used by the streaming
|
||||
/// query-result export for batches after the first.
|
||||
pub fn format_query_result_csv_rows(rows: &[Vec<Value>]) -> String {
|
||||
rows.iter()
|
||||
.map(|row| {
|
||||
row.iter().map(|cell| escape_csv(&value_to_query_result_csv_text(cell))).collect::<Vec<_>>().join(",")
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
}
|
||||
|
||||
fn format_csv_with_value_formatter(
|
||||
columns: &[String],
|
||||
rows: &[Vec<Value>],
|
||||
|
||||
@@ -36,6 +36,7 @@ pub mod plugins;
|
||||
pub mod query;
|
||||
pub mod query_cancel;
|
||||
pub mod query_execution_sql;
|
||||
pub mod query_result_export;
|
||||
pub mod query_result_sql;
|
||||
pub mod redis_ops;
|
||||
pub mod saved_sql;
|
||||
|
||||
@@ -0,0 +1,621 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::fs::File;
|
||||
use std::io::{BufWriter, Write};
|
||||
|
||||
use crate::connection::AppState;
|
||||
use crate::csv_export::{format_query_result_csv, format_query_result_csv_rows};
|
||||
use crate::database_export::is_export_cancelled;
|
||||
pub use crate::database_export::ExportStatus;
|
||||
use crate::models::connection::DatabaseType;
|
||||
use crate::query::{close_query_session, execute_sql_statement_with_options, QueryExecutionOptions, QUERY_CANCELED};
|
||||
use crate::query_result_sql::{
|
||||
build_query_pagination_execution_plan, QueryPagination, QueryPaginationExecutionPlanOptions,
|
||||
};
|
||||
use crate::table_export::TableExportProgress;
|
||||
use crate::transfer::keyset_pagination_sql;
|
||||
use crate::xlsx_export::{finish_streaming_xlsx_workbook, start_streaming_xlsx_workbook};
|
||||
use serde_json::Value;
|
||||
use sqlparser::ast::{GroupByExpr, ObjectNamePart, OrderByKind, SelectItem, SetExpr, Statement, TableFactor};
|
||||
use sqlparser::dialect::GenericDialect;
|
||||
use sqlparser::parser::Parser;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
const AGENT_UNBOUNDED_ROW_LIMIT: usize = i32::MAX as usize;
|
||||
pub const XLSX_MAX_DATA_ROWS: usize = 1_048_575;
|
||||
const XLSX_ROW_LIMIT_ERROR: &str = "XLSX 最多支持 1,048,575 行数据,请改用 CSV 导出完整结果。";
|
||||
const STREAMING_PAGINATION_UNSUPPORTED_ERROR: &str = "当前查询暂不支持流式导出,请简化查询或使用受支持的驱动。";
|
||||
const AGENT_SESSION_MISSING_ERROR: &str = "查询结果流式导出需要驱动返回结果集会话,但当前驱动未返回 session_id。";
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct QueryResultExportRequest {
|
||||
pub export_id: String,
|
||||
pub connection_id: String,
|
||||
pub database: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub schema: Option<String>,
|
||||
pub sql: String,
|
||||
pub query_base_sql: String,
|
||||
pub database_type: DatabaseType,
|
||||
#[serde(default)]
|
||||
pub use_agent_cursor: bool,
|
||||
pub file_path: String,
|
||||
pub format: String,
|
||||
pub page_size: usize,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub row_limit: Option<usize>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub total_rows: Option<u64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub timeout_secs: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub keyset_optimization_enabled: bool,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub client_session_id: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub execution_id: Option<String>,
|
||||
}
|
||||
|
||||
fn progress(
|
||||
request: &QueryResultExportRequest,
|
||||
rows_exported: u64,
|
||||
status: ExportStatus,
|
||||
error_message: Option<String>,
|
||||
) -> TableExportProgress {
|
||||
let total_rows = request.total_rows.map(|total| {
|
||||
let format = request.format.to_lowercase();
|
||||
let limit = effective_row_limit(&format, request);
|
||||
limit.map_or(total, |limit| total.min(limit as u64))
|
||||
});
|
||||
TableExportProgress {
|
||||
export_id: request.export_id.clone(),
|
||||
table_name: String::new(),
|
||||
rows_exported,
|
||||
total_rows,
|
||||
status,
|
||||
error_message,
|
||||
}
|
||||
}
|
||||
|
||||
fn effective_row_limit(format: &str, request: &QueryResultExportRequest) -> Option<usize> {
|
||||
if format == "xlsx" {
|
||||
Some(request.row_limit.map_or(XLSX_MAX_DATA_ROWS, |limit| limit.min(XLSX_MAX_DATA_ROWS)))
|
||||
} else {
|
||||
request.row_limit
|
||||
}
|
||||
}
|
||||
|
||||
fn xlsx_hard_limit_active(format: &str, request: &QueryResultExportRequest) -> bool {
|
||||
format == "xlsx" && request.row_limit.map_or(true, |limit| limit > XLSX_MAX_DATA_ROWS)
|
||||
}
|
||||
|
||||
fn should_fetch_next_page(
|
||||
use_agent_result_session: bool,
|
||||
has_more: bool,
|
||||
fetched_row_count: usize,
|
||||
written_row_count: usize,
|
||||
requested_page_size: usize,
|
||||
) -> bool {
|
||||
if use_agent_result_session {
|
||||
has_more
|
||||
} else {
|
||||
fetched_row_count > written_row_count || written_row_count >= requested_page_size
|
||||
}
|
||||
}
|
||||
|
||||
fn supports_streaming_offset_pagination(request: &QueryResultExportRequest, page_size: usize) -> bool {
|
||||
let first_page = build_query_pagination_execution_plan(QueryPaginationExecutionPlanOptions {
|
||||
sql: request.sql.clone(),
|
||||
query_base_sql: request.query_base_sql.clone(),
|
||||
database_type: Some(request.database_type),
|
||||
pagination: QueryPagination { limit: page_size, offset: 0, session_id: None },
|
||||
use_agent_cursor: false,
|
||||
first_page_uses_actual_sql: true,
|
||||
});
|
||||
let second_page = build_query_pagination_execution_plan(QueryPaginationExecutionPlanOptions {
|
||||
sql: request.sql.clone(),
|
||||
query_base_sql: request.query_base_sql.clone(),
|
||||
database_type: Some(request.database_type),
|
||||
pagination: QueryPagination { limit: page_size, offset: page_size, session_id: None },
|
||||
use_agent_cursor: false,
|
||||
first_page_uses_actual_sql: true,
|
||||
});
|
||||
|
||||
let (Some(first_sql), Some(second_sql)) = (first_page.page_sql.as_deref(), second_page.page_sql.as_deref()) else {
|
||||
return false;
|
||||
};
|
||||
first_page.page_limit.is_some()
|
||||
&& second_page.page_limit.is_some()
|
||||
&& !first_sql.trim().eq_ignore_ascii_case(second_sql.trim())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
struct SafeKeysetCandidate {
|
||||
schema: Option<String>,
|
||||
table: String,
|
||||
}
|
||||
|
||||
struct KeysetPlan {
|
||||
columns: Vec<String>,
|
||||
primary_keys: Vec<String>,
|
||||
pk_indices: Vec<usize>,
|
||||
schema: String,
|
||||
table: String,
|
||||
last_pk_values: Vec<Value>,
|
||||
}
|
||||
|
||||
fn object_name_parts(name: &sqlparser::ast::ObjectName) -> Option<Vec<String>> {
|
||||
name.0
|
||||
.iter()
|
||||
.map(|part| match part {
|
||||
ObjectNamePart::Identifier(ident) => Some(ident.value.clone()),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn safe_keyset_candidate(sql: &str) -> Option<SafeKeysetCandidate> {
|
||||
let dialect = GenericDialect {};
|
||||
let statements = Parser::parse_sql(&dialect, sql).ok()?;
|
||||
let [Statement::Query(query)] = statements.as_slice() else {
|
||||
return None;
|
||||
};
|
||||
if query.with.is_some()
|
||||
|| query.limit_clause.is_some()
|
||||
|| query.fetch.is_some()
|
||||
|| !query.locks.is_empty()
|
||||
|| query.for_clause.is_some()
|
||||
|| query.settings.is_some()
|
||||
|| query.format_clause.is_some()
|
||||
|| !query.pipe_operators.is_empty()
|
||||
|| query
|
||||
.order_by
|
||||
.as_ref()
|
||||
.is_some_and(|order_by| !matches!(&order_by.kind, OrderByKind::Expressions(exprs) if exprs.is_empty()))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let SetExpr::Select(select) = query.body.as_ref() else {
|
||||
return None;
|
||||
};
|
||||
if select.distinct.is_some()
|
||||
|| !matches!(&select.group_by, GroupByExpr::Expressions(exprs, _) if exprs.is_empty())
|
||||
|| select.having.is_some()
|
||||
|| select.selection.is_some()
|
||||
|| select.from.len() != 1
|
||||
|| !(select.projection.len() == 1 && matches!(select.projection.first(), Some(SelectItem::Wildcard(_))))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let source = &select.from[0];
|
||||
if !source.joins.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let TableFactor::Table { name, .. } = &source.relation else {
|
||||
return None;
|
||||
};
|
||||
let parts = object_name_parts(name)?;
|
||||
match parts.as_slice() {
|
||||
[table] => Some(SafeKeysetCandidate { schema: None, table: table.clone() }),
|
||||
[schema, table] => Some(SafeKeysetCandidate { schema: Some(schema.clone()), table: table.clone() }),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
async fn build_keyset_plan(state: &AppState, request: &QueryResultExportRequest) -> Option<KeysetPlan> {
|
||||
if !request.keyset_optimization_enabled || request.use_agent_cursor {
|
||||
return None;
|
||||
}
|
||||
let candidate = safe_keyset_candidate(&request.sql)?;
|
||||
let schema = candidate.schema.or_else(|| request.schema.clone()).unwrap_or_default();
|
||||
let columns =
|
||||
crate::schema::get_columns_core(state, &request.connection_id, &request.database, &schema, &candidate.table)
|
||||
.await
|
||||
.ok()?;
|
||||
let col_names: Vec<String> = columns.iter().map(|column| column.name.clone()).collect();
|
||||
let primary_keys: Vec<String> =
|
||||
columns.iter().filter(|column| column.is_primary_key).map(|column| column.name.clone()).collect();
|
||||
if col_names.is_empty() || primary_keys.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let pk_indices: Vec<usize> = primary_keys
|
||||
.iter()
|
||||
.filter_map(|pk| col_names.iter().position(|column| column.eq_ignore_ascii_case(pk)))
|
||||
.collect();
|
||||
if pk_indices.len() != primary_keys.len() {
|
||||
return None;
|
||||
}
|
||||
Some(KeysetPlan {
|
||||
columns: col_names,
|
||||
primary_keys,
|
||||
pk_indices,
|
||||
schema,
|
||||
table: candidate.table,
|
||||
last_pk_values: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn export_query_result_core(
|
||||
state: &AppState,
|
||||
request: &QueryResultExportRequest,
|
||||
cancel_token: Option<CancellationToken>,
|
||||
on_progress: impl Fn(TableExportProgress),
|
||||
) -> Result<(), String> {
|
||||
let mut session_id: Option<String> = None;
|
||||
let result = export_query_result_core_inner(state, request, cancel_token, &on_progress, &mut session_id).await;
|
||||
|
||||
if let Some(session_id) = session_id {
|
||||
let _ = close_query_session(
|
||||
state,
|
||||
&request.connection_id,
|
||||
&request.database,
|
||||
&session_id,
|
||||
request.client_session_id.as_deref(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
if let Some(client_session_id) = request.client_session_id.as_deref() {
|
||||
let database = request.database.trim();
|
||||
let database = if database.is_empty() { None } else { Some(database) };
|
||||
let _ = state.close_client_session_pool(&request.connection_id, database, client_session_id).await;
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
async fn export_query_result_core_inner(
|
||||
state: &AppState,
|
||||
request: &QueryResultExportRequest,
|
||||
cancel_token: Option<CancellationToken>,
|
||||
on_progress: &impl Fn(TableExportProgress),
|
||||
session_id: &mut Option<String>,
|
||||
) -> Result<(), String> {
|
||||
let format = request.format.to_lowercase();
|
||||
if format != "csv" && format != "xlsx" {
|
||||
return Err(format!("Unsupported streaming query-result export format: {format}"));
|
||||
}
|
||||
|
||||
let page_size = request.page_size.max(1);
|
||||
let effective_row_limit = effective_row_limit(&format, request);
|
||||
let xlsx_hard_limit_active = xlsx_hard_limit_active(&format, request);
|
||||
if xlsx_hard_limit_active && request.total_rows.is_some_and(|total| total > XLSX_MAX_DATA_ROWS as u64) {
|
||||
return Err(XLSX_ROW_LIMIT_ERROR.to_string());
|
||||
}
|
||||
|
||||
let agent_max_rows = if xlsx_hard_limit_active {
|
||||
XLSX_MAX_DATA_ROWS + 1
|
||||
} else {
|
||||
effective_row_limit.unwrap_or(AGENT_UNBOUNDED_ROW_LIMIT)
|
||||
}
|
||||
.max(1);
|
||||
|
||||
on_progress(progress(request, 0, ExportStatus::Running, None));
|
||||
|
||||
let mut csv_file = if format == "csv" {
|
||||
Some(BufWriter::new(File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(file) = csv_file.as_mut() {
|
||||
file.write_all(b"\xEF\xBB\xBF").map_err(|e| format!("Failed to write BOM: {e}"))?;
|
||||
}
|
||||
|
||||
let mut xlsx = None;
|
||||
let mut columns: Vec<String> = Vec::new();
|
||||
let mut rows_exported: u64 = 0;
|
||||
let mut offset: usize = 0;
|
||||
let mut wrote_csv_header = false;
|
||||
let mut keyset_plan = build_keyset_plan(state, request).await;
|
||||
if keyset_plan.is_none() && !request.use_agent_cursor && !supports_streaming_offset_pagination(request, page_size) {
|
||||
return Err(STREAMING_PAGINATION_UNSUPPORTED_ERROR.to_string());
|
||||
}
|
||||
|
||||
loop {
|
||||
if cancel_token.as_ref().is_some_and(|token| token.is_cancelled())
|
||||
|| is_export_cancelled(&request.export_id).await
|
||||
{
|
||||
on_progress(progress(
|
||||
request,
|
||||
rows_exported,
|
||||
ExportStatus::Cancelled,
|
||||
Some("Export cancelled".to_string()),
|
||||
));
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let remaining = effective_row_limit.map(|limit| limit.saturating_sub(rows_exported as usize));
|
||||
if matches!(remaining, Some(0)) {
|
||||
break;
|
||||
}
|
||||
let this_page = remaining.map_or(page_size, |rem| rem.min(page_size)).max(1);
|
||||
let fetch_limit = if xlsx_hard_limit_active && remaining.is_some_and(|rem| rem <= page_size) {
|
||||
this_page.saturating_add(1)
|
||||
} else {
|
||||
this_page
|
||||
};
|
||||
|
||||
let (sql_to_execute, plan_limit, use_agent_result_session) = if let Some(plan) = keyset_plan.as_ref() {
|
||||
(
|
||||
keyset_pagination_sql(
|
||||
&plan.columns,
|
||||
&plan.table,
|
||||
&plan.schema,
|
||||
&request.database_type,
|
||||
&plan.primary_keys,
|
||||
&plan.last_pk_values,
|
||||
fetch_limit,
|
||||
),
|
||||
fetch_limit,
|
||||
false,
|
||||
)
|
||||
} else {
|
||||
let plan = build_query_pagination_execution_plan(QueryPaginationExecutionPlanOptions {
|
||||
sql: request.sql.clone(),
|
||||
query_base_sql: request.query_base_sql.clone(),
|
||||
database_type: Some(request.database_type),
|
||||
pagination: QueryPagination { limit: fetch_limit, offset, session_id: session_id.clone() },
|
||||
use_agent_cursor: request.use_agent_cursor,
|
||||
first_page_uses_actual_sql: true,
|
||||
});
|
||||
let Some(plan_limit) = plan.page_limit else {
|
||||
return Err("Failed to build query pagination plan for export".to_string());
|
||||
};
|
||||
(plan.sql_to_execute, plan_limit, plan.use_agent_result_session)
|
||||
};
|
||||
|
||||
let options = if use_agent_result_session {
|
||||
QueryExecutionOptions {
|
||||
page_size: Some(plan_limit),
|
||||
fetch_size: Some(plan_limit),
|
||||
result_session_id: session_id.clone(),
|
||||
max_rows: Some(agent_max_rows),
|
||||
timeout_secs: request.timeout_secs,
|
||||
client_session_id: request.client_session_id.clone(),
|
||||
execution_id: request.execution_id.clone(),
|
||||
}
|
||||
} else {
|
||||
QueryExecutionOptions {
|
||||
max_rows: Some(plan_limit),
|
||||
fetch_size: Some(plan_limit),
|
||||
timeout_secs: request.timeout_secs,
|
||||
client_session_id: request.client_session_id.clone(),
|
||||
execution_id: request.execution_id.clone(),
|
||||
..Default::default()
|
||||
}
|
||||
};
|
||||
|
||||
let mut result = match execute_sql_statement_with_options(
|
||||
state,
|
||||
&request.connection_id,
|
||||
&request.database,
|
||||
&sql_to_execute,
|
||||
request.schema.as_deref(),
|
||||
cancel_token.clone(),
|
||||
options,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(error) => {
|
||||
if error == QUERY_CANCELED
|
||||
|| cancel_token.as_ref().is_some_and(|token| token.is_cancelled())
|
||||
|| is_export_cancelled(&request.export_id).await
|
||||
{
|
||||
on_progress(progress(
|
||||
request,
|
||||
rows_exported,
|
||||
ExportStatus::Cancelled,
|
||||
Some("Export cancelled".to_string()),
|
||||
));
|
||||
return Ok(());
|
||||
}
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
|
||||
if cancel_token.as_ref().is_some_and(|token| token.is_cancelled())
|
||||
|| is_export_cancelled(&request.export_id).await
|
||||
{
|
||||
on_progress(progress(
|
||||
request,
|
||||
rows_exported,
|
||||
ExportStatus::Cancelled,
|
||||
Some("Export cancelled".to_string()),
|
||||
));
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if columns.is_empty() {
|
||||
columns = result.columns.clone();
|
||||
}
|
||||
let fetched_row_count = result.rows.len();
|
||||
if xlsx_hard_limit_active {
|
||||
let remaining_rows = XLSX_MAX_DATA_ROWS.saturating_sub(rows_exported as usize);
|
||||
if fetched_row_count > remaining_rows {
|
||||
return Err(XLSX_ROW_LIMIT_ERROR.to_string());
|
||||
}
|
||||
}
|
||||
if result.rows.len() > this_page {
|
||||
result.rows.truncate(this_page);
|
||||
}
|
||||
let row_count = result.rows.len();
|
||||
|
||||
if format == "csv" {
|
||||
if let Some(file) = csv_file.as_mut() {
|
||||
if !wrote_csv_header {
|
||||
let csv = format_query_result_csv(&columns, &result.rows);
|
||||
file.write_all(csv.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?;
|
||||
wrote_csv_header = true;
|
||||
} else if row_count > 0 {
|
||||
let rows_csv = format_query_result_csv_rows(&result.rows);
|
||||
write!(file, "\n{rows_csv}").map_err(|e| format!("Failed to write CSV rows: {e}"))?;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if xlsx.is_none() {
|
||||
let xlsx_file =
|
||||
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
|
||||
xlsx = Some(start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?);
|
||||
}
|
||||
if let Some(writer) = xlsx.as_mut() {
|
||||
for row in &result.rows {
|
||||
writer.write_row(row).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
rows_exported += row_count as u64;
|
||||
on_progress(progress(request, rows_exported, ExportStatus::Running, None));
|
||||
|
||||
if result.session_id.is_some() {
|
||||
*session_id = result.session_id.clone();
|
||||
}
|
||||
if use_agent_result_session && result.has_more && session_id.is_none() {
|
||||
return Err(AGENT_SESSION_MISSING_ERROR.to_string());
|
||||
}
|
||||
if let Some(plan) = keyset_plan.as_mut() {
|
||||
if let Some(last_row) = result.rows.last() {
|
||||
plan.last_pk_values =
|
||||
plan.pk_indices.iter().map(|&index| last_row.get(index).cloned().unwrap_or(Value::Null)).collect();
|
||||
}
|
||||
}
|
||||
let should_continue =
|
||||
should_fetch_next_page(use_agent_result_session, result.has_more, fetched_row_count, row_count, plan_limit);
|
||||
if cancel_token.as_ref().is_some_and(|token| token.is_cancelled())
|
||||
|| is_export_cancelled(&request.export_id).await
|
||||
{
|
||||
on_progress(progress(
|
||||
request,
|
||||
rows_exported,
|
||||
ExportStatus::Cancelled,
|
||||
Some("Export cancelled".to_string()),
|
||||
));
|
||||
return Ok(());
|
||||
}
|
||||
if !should_continue || row_count == 0 {
|
||||
break;
|
||||
}
|
||||
offset += row_count;
|
||||
}
|
||||
|
||||
on_progress(progress(request, rows_exported, ExportStatus::Writing, None));
|
||||
|
||||
if format == "csv" {
|
||||
if !wrote_csv_header {
|
||||
let csv = format_query_result_csv(&columns, &[]);
|
||||
if let Some(file) = csv_file.as_mut() {
|
||||
file.write_all(csv.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?;
|
||||
}
|
||||
}
|
||||
if let Some(file) = csv_file.as_mut() {
|
||||
file.flush().map_err(|e| format!("Failed to flush CSV file: {e}"))?;
|
||||
}
|
||||
} else if let Some(writer) = xlsx {
|
||||
let mut buf =
|
||||
finish_streaming_xlsx_workbook(writer).map_err(|e| format!("Failed to finalize XLSX file: {e}"))?;
|
||||
buf.flush().map_err(|e| format!("Failed to flush XLSX file: {e}"))?;
|
||||
} else {
|
||||
let xlsx_file = File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
|
||||
let writer = start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?;
|
||||
let mut buf =
|
||||
finish_streaming_xlsx_workbook(writer).map_err(|e| format!("Failed to finalize XLSX file: {e}"))?;
|
||||
buf.flush().map_err(|e| format!("Failed to flush XLSX file: {e}"))?;
|
||||
}
|
||||
|
||||
on_progress(progress(request, rows_exported, ExportStatus::Done, None));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn request(format: &str, row_limit: Option<usize>, total_rows: Option<u64>) -> QueryResultExportRequest {
|
||||
QueryResultExportRequest {
|
||||
export_id: "export-1".to_string(),
|
||||
connection_id: "conn-1".to_string(),
|
||||
database: "db".to_string(),
|
||||
schema: None,
|
||||
sql: "SELECT * FROM users".to_string(),
|
||||
query_base_sql: "SELECT * FROM users".to_string(),
|
||||
database_type: DatabaseType::Postgres,
|
||||
use_agent_cursor: false,
|
||||
file_path: "out.csv".to_string(),
|
||||
format: format.to_string(),
|
||||
page_size: 1000,
|
||||
row_limit,
|
||||
total_rows,
|
||||
timeout_secs: None,
|
||||
keyset_optimization_enabled: true,
|
||||
client_session_id: None,
|
||||
execution_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn csv_unlimited_export_has_no_effective_row_limit() {
|
||||
assert_eq!(effective_row_limit("csv", &request("csv", None, None)), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn xlsx_unlimited_export_uses_excel_hard_limit() {
|
||||
assert_eq!(effective_row_limit("xlsx", &request("xlsx", None, None)), Some(XLSX_MAX_DATA_ROWS));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn xlsx_row_limit_caps_to_excel_hard_limit() {
|
||||
assert_eq!(
|
||||
effective_row_limit("xlsx", &request("xlsx", Some(XLSX_MAX_DATA_ROWS + 10), None)),
|
||||
Some(XLSX_MAX_DATA_ROWS)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn xlsx_known_total_above_hard_limit_errors_before_export() {
|
||||
let req = request("xlsx", None, Some(XLSX_MAX_DATA_ROWS as u64 + 1));
|
||||
assert!(xlsx_hard_limit_active("xlsx", &req));
|
||||
assert!(req.total_rows.is_some_and(|total| total > XLSX_MAX_DATA_ROWS as u64));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_agent_pages_continue_after_trimming_probe_row() {
|
||||
assert!(should_fetch_next_page(false, false, 101, 100, 100));
|
||||
assert!(should_fetch_next_page(false, false, 100, 100, 100));
|
||||
assert!(!should_fetch_next_page(false, false, 42, 42, 100));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn agent_pages_follow_has_more_flag() {
|
||||
assert!(should_fetch_next_page(true, true, 42, 42, 100));
|
||||
assert!(!should_fetch_next_page(true, false, 100, 100, 100));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn streaming_offset_pagination_requires_distinct_followup_page_sql() {
|
||||
let req = request("csv", Some(1000), None);
|
||||
assert!(supports_streaming_offset_pagination(&req, 100));
|
||||
|
||||
let oracle_req =
|
||||
QueryResultExportRequest { database_type: DatabaseType::Oracle, ..request("csv", Some(1000), None) };
|
||||
assert!(!supports_streaming_offset_pagination(&oracle_req, 100));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keyset_candidate_accepts_simple_single_table_wildcard_query() {
|
||||
let candidate = safe_keyset_candidate("SELECT * FROM public.users").expect("safe keyset candidate");
|
||||
assert_eq!(candidate.schema.as_deref(), Some("public"));
|
||||
assert_eq!(candidate.table, "users");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keyset_candidate_rejects_join_and_sorted_queries() {
|
||||
assert!(safe_keyset_candidate("SELECT * FROM users u JOIN orders o ON o.user_id = u.id").is_none());
|
||||
assert!(safe_keyset_candidate("SELECT * FROM users ORDER BY name").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keyset_candidate_rejects_filters_and_projection_changes() {
|
||||
assert!(safe_keyset_candidate("SELECT * FROM users WHERE active = true").is_none());
|
||||
assert!(safe_keyset_candidate("SELECT id, name FROM users").is_none());
|
||||
}
|
||||
}
|
||||
@@ -32,6 +32,8 @@ pub struct QueryPaginationExecutionPlanOptions {
|
||||
pub database_type: Option<DatabaseType>,
|
||||
pub pagination: QueryPagination,
|
||||
pub use_agent_cursor: bool,
|
||||
#[serde(default)]
|
||||
pub first_page_uses_actual_sql: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
|
||||
@@ -128,7 +130,9 @@ pub fn build_query_pagination_execution_plan(
|
||||
}
|
||||
|
||||
if options.use_agent_cursor && options.pagination.offset == 0 {
|
||||
plan.sql_to_execute = options.query_base_sql;
|
||||
if !options.first_page_uses_actual_sql {
|
||||
plan.sql_to_execute = options.query_base_sql;
|
||||
}
|
||||
plan.page_limit = Some(options.pagination.limit);
|
||||
plan.page_offset = Some(options.pagination.offset);
|
||||
plan.use_agent_result_session = true;
|
||||
@@ -922,6 +926,7 @@ mod tests {
|
||||
database_type: Some(DatabaseType::SqlServer),
|
||||
pagination: QueryPagination { limit: 100, offset: 0, session_id: None },
|
||||
use_agent_cursor: false,
|
||||
first_page_uses_actual_sql: false,
|
||||
});
|
||||
|
||||
assert_eq!(plan.sql_to_execute, sql);
|
||||
@@ -1282,6 +1287,7 @@ WHERE u.id = picked.id;
|
||||
database_type: Some(DatabaseType::Oracle),
|
||||
pagination: QueryPagination { limit: 500, offset: 0, session_id: None },
|
||||
use_agent_cursor: true,
|
||||
first_page_uses_actual_sql: false,
|
||||
});
|
||||
|
||||
assert_eq!(plan.sql_to_execute, "SELECT * FROM events");
|
||||
@@ -1291,6 +1297,24 @@ WHERE u.id = picked.id;
|
||||
assert!(plan.use_agent_result_session);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_agent_cursor_first_page_keeps_actual_sorted_sql() {
|
||||
let plan = build_query_pagination_execution_plan(QueryPaginationExecutionPlanOptions {
|
||||
sql: "SELECT * FROM (SELECT * FROM events) t ORDER BY created_at DESC".to_string(),
|
||||
query_base_sql: "SELECT * FROM events".to_string(),
|
||||
database_type: Some(DatabaseType::Oracle),
|
||||
pagination: QueryPagination { limit: 500, offset: 0, session_id: None },
|
||||
use_agent_cursor: true,
|
||||
first_page_uses_actual_sql: true,
|
||||
});
|
||||
|
||||
assert_eq!(plan.sql_to_execute, "SELECT * FROM (SELECT * FROM events) t ORDER BY created_at DESC");
|
||||
assert_eq!(plan.page_limit, Some(500));
|
||||
assert_eq!(plan.page_offset, Some(0));
|
||||
assert!(plan.page_sql.is_none());
|
||||
assert!(plan.use_agent_result_session);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_sorted_query_sql() {
|
||||
let result = build_sorted_query_sql(SortedQuerySqlOptions {
|
||||
|
||||
@@ -59,9 +59,15 @@ pub(crate) fn data_row_xml(row_number: usize, columns: &[String], row: &[Value])
|
||||
format!("<row r=\"{row_number}\">{cells}</row>")
|
||||
}
|
||||
|
||||
/// Shared ZIP entry options for all XLSX parts. XLSX is a ZIP of XML, and the
|
||||
/// `inlineStr` cell encoding is highly repetitive, so Deflate typically shrinks
|
||||
/// the file several-fold over `Stored` (matching what Excel/Navicat produce).
|
||||
fn xlsx_zip_options() -> zip::write::SimpleFileOptions {
|
||||
zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated)
|
||||
}
|
||||
|
||||
fn write_zip_entry<W: Write + Seek>(zip: &mut zip::ZipWriter<W>, path: &str, content: &str) -> Result<(), String> {
|
||||
let options = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored);
|
||||
zip.start_file(path, options).map_err(|err| err.to_string())?;
|
||||
zip.start_file(path, xlsx_zip_options()).map_err(|err| err.to_string())?;
|
||||
zip.write_all(content.as_bytes()).map_err(|err| err.to_string())
|
||||
}
|
||||
|
||||
@@ -86,7 +92,7 @@ pub(crate) fn start_streaming_xlsx_workbook<W: Write + Seek>(
|
||||
|
||||
// Begin the sheet1.xml entry with header, frozen pane, column widths and
|
||||
// the header row.
|
||||
let options = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored);
|
||||
let options = xlsx_zip_options();
|
||||
zip.start_file("xl/worksheets/sheet1.xml", options).map_err(|err| err.to_string())?;
|
||||
|
||||
let sheet_header = format!(
|
||||
@@ -434,7 +440,7 @@ pub fn build_xlsx_workbook_multi(sheets: &[XlsxWorksheetData]) -> Result<Vec<u8>
|
||||
|
||||
let cursor = Cursor::new(Vec::<u8>::new());
|
||||
let mut zip = zip::ZipWriter::new(cursor);
|
||||
let options = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored);
|
||||
let options = xlsx_zip_options();
|
||||
|
||||
for (path, content) in files {
|
||||
zip.start_file(path, options).map_err(|err| err.to_string())?;
|
||||
@@ -455,6 +461,31 @@ mod tests {
|
||||
use calamine::{open_workbook_auto, Reader};
|
||||
use serde_json::json;
|
||||
use std::fs;
|
||||
use std::io::Read;
|
||||
|
||||
/// Read and decompress a single entry from an in-memory XLSX (ZIP) buffer.
|
||||
fn read_zip_entry(bytes: &[u8], path: &str) -> String {
|
||||
let mut archive = zip::ZipArchive::new(std::io::Cursor::new(bytes.to_vec())).expect("open xlsx as zip archive");
|
||||
let mut entry = archive.by_name(path).unwrap_or_else(|_| panic!("missing zip entry: {path}"));
|
||||
let mut content = String::new();
|
||||
entry.read_to_string(&mut content).expect("read zip entry");
|
||||
content
|
||||
}
|
||||
|
||||
/// Assert every entry in the XLSX (ZIP) buffer is Deflate-compressed, which
|
||||
/// is what keeps exported workbooks small (see `xlsx_zip_options`).
|
||||
fn assert_all_entries_deflated(bytes: &[u8]) {
|
||||
let mut archive = zip::ZipArchive::new(std::io::Cursor::new(bytes.to_vec())).expect("open xlsx as zip archive");
|
||||
for index in 0..archive.len() {
|
||||
let entry = archive.by_index(index).expect("zip entry");
|
||||
assert_eq!(
|
||||
entry.compression(),
|
||||
zip::CompressionMethod::Deflated,
|
||||
"entry {} should be Deflate-compressed",
|
||||
entry.name()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_xlsx_zip_with_sheet_data() {
|
||||
@@ -464,16 +495,19 @@ mod tests {
|
||||
rows: vec![vec![json!(1), json!("Ada & Bob"), json!(true)], vec![json!(2), json!(null), json!(false)]],
|
||||
})
|
||||
.expect("build workbook");
|
||||
let text = String::from_utf8_lossy(&workbook);
|
||||
|
||||
// ZIP magic bytes.
|
||||
assert_eq!(workbook[0], 0x50);
|
||||
assert_eq!(workbook[1], 0x4b);
|
||||
assert!(text.contains("[Content_Types].xml"));
|
||||
assert!(text.contains("xl/worksheets/sheet1.xml"));
|
||||
assert!(text.contains("name=\"Users\""));
|
||||
assert!(text.contains("<c r=\"A2\"><v>1</v></c>"));
|
||||
assert!(text.contains("Ada & Bob"));
|
||||
assert!(text.contains("<c r=\"C2\" t=\"b\"><v>1</v></c>"));
|
||||
|
||||
// Entries are stored compressed; assert on their decompressed contents.
|
||||
let sheet = read_zip_entry(&workbook, "xl/worksheets/sheet1.xml");
|
||||
let workbook_xml = read_zip_entry(&workbook, "xl/workbook.xml");
|
||||
assert!(workbook_xml.contains("name=\"Users\""));
|
||||
assert!(sheet.contains("<c r=\"A2\"><v>1</v></c>"));
|
||||
assert!(sheet.contains("Ada & Bob"));
|
||||
assert!(sheet.contains("<c r=\"C2\" t=\"b\"><v>1</v></c>"));
|
||||
assert_all_entries_deflated(&workbook);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -484,8 +518,8 @@ mod tests {
|
||||
rows: vec![vec![json!("ok")]],
|
||||
})
|
||||
.expect("build workbook");
|
||||
let text = String::from_utf8_lossy(&workbook);
|
||||
assert!(text.contains("name=\"bad name with chars and-a-very-\""));
|
||||
let workbook_xml = read_zip_entry(&workbook, "xl/workbook.xml");
|
||||
assert!(workbook_xml.contains("name=\"bad name with chars and-a-very-\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
Reference in new issue
Block a user