feat: stream query result CSV/XLSX exports

* docs: add query result streaming export design

* feat(core): stream query result exports

* feat: expose query result streaming export APIs

* feat(desktop): stream query result exports from grid

* fix: route query result export cancellation through its command

* fix: reduce query export batch size and cancel latency

* opt: del local test files
This commit is contained in:
miracle authored and GitHub committed 2026-06-24 21:18:11 +08:00
1 parent 849caf4b39
commit 588ba3352d
25 files changed
+1787 -47

No files matched your search

+13 -1
View File
@@ -41,13 +41,25 @@ pub(crate) fn value_to_csv_text(value: &Value) -> String {
}
}
fn value_to_query_result_csv_text(value: &Value) -> String {
pub(crate) fn value_to_query_result_csv_text(value: &Value) -> String {
match value {
Value::Null => "NULL".to_string(),
other => value_to_csv_text(other),
}
}
/// Format query-result rows as CSV text without a header row, using the
/// query-result NULL semantics (NULL → "NULL"). Used by the streaming
/// query-result export for batches after the first.
pub fn format_query_result_csv_rows(rows: &[Vec<Value>]) -> String {
rows.iter()
.map(|row| {
row.iter().map(|cell| escape_csv(&value_to_query_result_csv_text(cell))).collect::<Vec<_>>().join(",")
})
.collect::<Vec<_>>()
.join("\n")
}
fn format_csv_with_value_formatter(
columns: &[String],
rows: &[Vec<Value>],
+1
View File
@@ -36,6 +36,7 @@ pub mod plugins;
pub mod query;
pub mod query_cancel;
pub mod query_execution_sql;
pub mod query_result_export;
pub mod query_result_sql;
pub mod redis_ops;
pub mod saved_sql;
+621
View File
@@ -0,0 +1,621 @@
use serde::{Deserialize, Serialize};
use std::fs::File;
use std::io::{BufWriter, Write};
use crate::connection::AppState;
use crate::csv_export::{format_query_result_csv, format_query_result_csv_rows};
use crate::database_export::is_export_cancelled;
pub use crate::database_export::ExportStatus;
use crate::models::connection::DatabaseType;
use crate::query::{close_query_session, execute_sql_statement_with_options, QueryExecutionOptions, QUERY_CANCELED};
use crate::query_result_sql::{
build_query_pagination_execution_plan, QueryPagination, QueryPaginationExecutionPlanOptions,
};
use crate::table_export::TableExportProgress;
use crate::transfer::keyset_pagination_sql;
use crate::xlsx_export::{finish_streaming_xlsx_workbook, start_streaming_xlsx_workbook};
use serde_json::Value;
use sqlparser::ast::{GroupByExpr, ObjectNamePart, OrderByKind, SelectItem, SetExpr, Statement, TableFactor};
use sqlparser::dialect::GenericDialect;
use sqlparser::parser::Parser;
use tokio_util::sync::CancellationToken;
const AGENT_UNBOUNDED_ROW_LIMIT: usize = i32::MAX as usize;
pub const XLSX_MAX_DATA_ROWS: usize = 1_048_575;
const XLSX_ROW_LIMIT_ERROR: &str = "XLSX 最多支持 1,048,575 行数据,请改用 CSV 导出完整结果。";
const STREAMING_PAGINATION_UNSUPPORTED_ERROR: &str = "当前查询暂不支持流式导出,请简化查询或使用受支持的驱动。";
const AGENT_SESSION_MISSING_ERROR: &str = "查询结果流式导出需要驱动返回结果集会话,但当前驱动未返回 session_id。";
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct QueryResultExportRequest {
pub export_id: String,
pub connection_id: String,
pub database: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub schema: Option<String>,
pub sql: String,
pub query_base_sql: String,
pub database_type: DatabaseType,
#[serde(default)]
pub use_agent_cursor: bool,
pub file_path: String,
pub format: String,
pub page_size: usize,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub row_limit: Option<usize>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub total_rows: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub timeout_secs: Option<u64>,
#[serde(default)]
pub keyset_optimization_enabled: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub client_session_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub execution_id: Option<String>,
}
fn progress(
request: &QueryResultExportRequest,
rows_exported: u64,
status: ExportStatus,
error_message: Option<String>,
) -> TableExportProgress {
let total_rows = request.total_rows.map(|total| {
let format = request.format.to_lowercase();
let limit = effective_row_limit(&format, request);
limit.map_or(total, |limit| total.min(limit as u64))
});
TableExportProgress {
export_id: request.export_id.clone(),
table_name: String::new(),
rows_exported,
total_rows,
status,
error_message,
}
}
fn effective_row_limit(format: &str, request: &QueryResultExportRequest) -> Option<usize> {
if format == "xlsx" {
Some(request.row_limit.map_or(XLSX_MAX_DATA_ROWS, |limit| limit.min(XLSX_MAX_DATA_ROWS)))
} else {
request.row_limit
}
}
fn xlsx_hard_limit_active(format: &str, request: &QueryResultExportRequest) -> bool {
format == "xlsx" && request.row_limit.map_or(true, |limit| limit > XLSX_MAX_DATA_ROWS)
}
fn should_fetch_next_page(
use_agent_result_session: bool,
has_more: bool,
fetched_row_count: usize,
written_row_count: usize,
requested_page_size: usize,
) -> bool {
if use_agent_result_session {
has_more
} else {
fetched_row_count > written_row_count || written_row_count >= requested_page_size
}
}
fn supports_streaming_offset_pagination(request: &QueryResultExportRequest, page_size: usize) -> bool {
let first_page = build_query_pagination_execution_plan(QueryPaginationExecutionPlanOptions {
sql: request.sql.clone(),
query_base_sql: request.query_base_sql.clone(),
database_type: Some(request.database_type),
pagination: QueryPagination { limit: page_size, offset: 0, session_id: None },
use_agent_cursor: false,
first_page_uses_actual_sql: true,
});
let second_page = build_query_pagination_execution_plan(QueryPaginationExecutionPlanOptions {
sql: request.sql.clone(),
query_base_sql: request.query_base_sql.clone(),
database_type: Some(request.database_type),
pagination: QueryPagination { limit: page_size, offset: page_size, session_id: None },
use_agent_cursor: false,
first_page_uses_actual_sql: true,
});
let (Some(first_sql), Some(second_sql)) = (first_page.page_sql.as_deref(), second_page.page_sql.as_deref()) else {
return false;
};
first_page.page_limit.is_some()
&& second_page.page_limit.is_some()
&& !first_sql.trim().eq_ignore_ascii_case(second_sql.trim())
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct SafeKeysetCandidate {
schema: Option<String>,
table: String,
}
struct KeysetPlan {
columns: Vec<String>,
primary_keys: Vec<String>,
pk_indices: Vec<usize>,
schema: String,
table: String,
last_pk_values: Vec<Value>,
}
fn object_name_parts(name: &sqlparser::ast::ObjectName) -> Option<Vec<String>> {
name.0
.iter()
.map(|part| match part {
ObjectNamePart::Identifier(ident) => Some(ident.value.clone()),
_ => None,
})
.collect()
}
fn safe_keyset_candidate(sql: &str) -> Option<SafeKeysetCandidate> {
let dialect = GenericDialect {};
let statements = Parser::parse_sql(&dialect, sql).ok()?;
let [Statement::Query(query)] = statements.as_slice() else {
return None;
};
if query.with.is_some()
|| query.limit_clause.is_some()
|| query.fetch.is_some()
|| !query.locks.is_empty()
|| query.for_clause.is_some()
|| query.settings.is_some()
|| query.format_clause.is_some()
|| !query.pipe_operators.is_empty()
|| query
.order_by
.as_ref()
.is_some_and(|order_by| !matches!(&order_by.kind, OrderByKind::Expressions(exprs) if exprs.is_empty()))
{
return None;
}
let SetExpr::Select(select) = query.body.as_ref() else {
return None;
};
if select.distinct.is_some()
|| !matches!(&select.group_by, GroupByExpr::Expressions(exprs, _) if exprs.is_empty())
|| select.having.is_some()
|| select.selection.is_some()
|| select.from.len() != 1
|| !(select.projection.len() == 1 && matches!(select.projection.first(), Some(SelectItem::Wildcard(_))))
{
return None;
}
let source = &select.from[0];
if !source.joins.is_empty() {
return None;
}
let TableFactor::Table { name, .. } = &source.relation else {
return None;
};
let parts = object_name_parts(name)?;
match parts.as_slice() {
[table] => Some(SafeKeysetCandidate { schema: None, table: table.clone() }),
[schema, table] => Some(SafeKeysetCandidate { schema: Some(schema.clone()), table: table.clone() }),
_ => None,
}
}
async fn build_keyset_plan(state: &AppState, request: &QueryResultExportRequest) -> Option<KeysetPlan> {
if !request.keyset_optimization_enabled || request.use_agent_cursor {
return None;
}
let candidate = safe_keyset_candidate(&request.sql)?;
let schema = candidate.schema.or_else(|| request.schema.clone()).unwrap_or_default();
let columns =
crate::schema::get_columns_core(state, &request.connection_id, &request.database, &schema, &candidate.table)
.await
.ok()?;
let col_names: Vec<String> = columns.iter().map(|column| column.name.clone()).collect();
let primary_keys: Vec<String> =
columns.iter().filter(|column| column.is_primary_key).map(|column| column.name.clone()).collect();
if col_names.is_empty() || primary_keys.is_empty() {
return None;
}
let pk_indices: Vec<usize> = primary_keys
.iter()
.filter_map(|pk| col_names.iter().position(|column| column.eq_ignore_ascii_case(pk)))
.collect();
if pk_indices.len() != primary_keys.len() {
return None;
}
Some(KeysetPlan {
columns: col_names,
primary_keys,
pk_indices,
schema,
table: candidate.table,
last_pk_values: Vec::new(),
})
}
pub async fn export_query_result_core(
state: &AppState,
request: &QueryResultExportRequest,
cancel_token: Option<CancellationToken>,
on_progress: impl Fn(TableExportProgress),
) -> Result<(), String> {
let mut session_id: Option<String> = None;
let result = export_query_result_core_inner(state, request, cancel_token, &on_progress, &mut session_id).await;
if let Some(session_id) = session_id {
let _ = close_query_session(
state,
&request.connection_id,
&request.database,
&session_id,
request.client_session_id.as_deref(),
)
.await;
}
if let Some(client_session_id) = request.client_session_id.as_deref() {
let database = request.database.trim();
let database = if database.is_empty() { None } else { Some(database) };
let _ = state.close_client_session_pool(&request.connection_id, database, client_session_id).await;
}
result
}
async fn export_query_result_core_inner(
state: &AppState,
request: &QueryResultExportRequest,
cancel_token: Option<CancellationToken>,
on_progress: &impl Fn(TableExportProgress),
session_id: &mut Option<String>,
) -> Result<(), String> {
let format = request.format.to_lowercase();
if format != "csv" && format != "xlsx" {
return Err(format!("Unsupported streaming query-result export format: {format}"));
}
let page_size = request.page_size.max(1);
let effective_row_limit = effective_row_limit(&format, request);
let xlsx_hard_limit_active = xlsx_hard_limit_active(&format, request);
if xlsx_hard_limit_active && request.total_rows.is_some_and(|total| total > XLSX_MAX_DATA_ROWS as u64) {
return Err(XLSX_ROW_LIMIT_ERROR.to_string());
}
let agent_max_rows = if xlsx_hard_limit_active {
XLSX_MAX_DATA_ROWS + 1
} else {
effective_row_limit.unwrap_or(AGENT_UNBOUNDED_ROW_LIMIT)
}
.max(1);
on_progress(progress(request, 0, ExportStatus::Running, None));
let mut csv_file = if format == "csv" {
Some(BufWriter::new(File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?))
} else {
None
};
if let Some(file) = csv_file.as_mut() {
file.write_all(b"\xEF\xBB\xBF").map_err(|e| format!("Failed to write BOM: {e}"))?;
}
let mut xlsx = None;
let mut columns: Vec<String> = Vec::new();
let mut rows_exported: u64 = 0;
let mut offset: usize = 0;
let mut wrote_csv_header = false;
let mut keyset_plan = build_keyset_plan(state, request).await;
if keyset_plan.is_none() && !request.use_agent_cursor && !supports_streaming_offset_pagination(request, page_size) {
return Err(STREAMING_PAGINATION_UNSUPPORTED_ERROR.to_string());
}
loop {
if cancel_token.as_ref().is_some_and(|token| token.is_cancelled())
|| is_export_cancelled(&request.export_id).await
{
on_progress(progress(
request,
rows_exported,
ExportStatus::Cancelled,
Some("Export cancelled".to_string()),
));
return Ok(());
}
let remaining = effective_row_limit.map(|limit| limit.saturating_sub(rows_exported as usize));
if matches!(remaining, Some(0)) {
break;
}
let this_page = remaining.map_or(page_size, |rem| rem.min(page_size)).max(1);
let fetch_limit = if xlsx_hard_limit_active && remaining.is_some_and(|rem| rem <= page_size) {
this_page.saturating_add(1)
} else {
this_page
};
let (sql_to_execute, plan_limit, use_agent_result_session) = if let Some(plan) = keyset_plan.as_ref() {
(
keyset_pagination_sql(
&plan.columns,
&plan.table,
&plan.schema,
&request.database_type,
&plan.primary_keys,
&plan.last_pk_values,
fetch_limit,
),
fetch_limit,
false,
)
} else {
let plan = build_query_pagination_execution_plan(QueryPaginationExecutionPlanOptions {
sql: request.sql.clone(),
query_base_sql: request.query_base_sql.clone(),
database_type: Some(request.database_type),
pagination: QueryPagination { limit: fetch_limit, offset, session_id: session_id.clone() },
use_agent_cursor: request.use_agent_cursor,
first_page_uses_actual_sql: true,
});
let Some(plan_limit) = plan.page_limit else {
return Err("Failed to build query pagination plan for export".to_string());
};
(plan.sql_to_execute, plan_limit, plan.use_agent_result_session)
};
let options = if use_agent_result_session {
QueryExecutionOptions {
page_size: Some(plan_limit),
fetch_size: Some(plan_limit),
result_session_id: session_id.clone(),
max_rows: Some(agent_max_rows),
timeout_secs: request.timeout_secs,
client_session_id: request.client_session_id.clone(),
execution_id: request.execution_id.clone(),
}
} else {
QueryExecutionOptions {
max_rows: Some(plan_limit),
fetch_size: Some(plan_limit),
timeout_secs: request.timeout_secs,
client_session_id: request.client_session_id.clone(),
execution_id: request.execution_id.clone(),
..Default::default()
}
};
let mut result = match execute_sql_statement_with_options(
state,
&request.connection_id,
&request.database,
&sql_to_execute,
request.schema.as_deref(),
cancel_token.clone(),
options,
)
.await
{
Ok(result) => result,
Err(error) => {
if error == QUERY_CANCELED
|| cancel_token.as_ref().is_some_and(|token| token.is_cancelled())
|| is_export_cancelled(&request.export_id).await
{
on_progress(progress(
request,
rows_exported,
ExportStatus::Cancelled,
Some("Export cancelled".to_string()),
));
return Ok(());
}
return Err(error);
}
};
if cancel_token.as_ref().is_some_and(|token| token.is_cancelled())
|| is_export_cancelled(&request.export_id).await
{
on_progress(progress(
request,
rows_exported,
ExportStatus::Cancelled,
Some("Export cancelled".to_string()),
));
return Ok(());
}
if columns.is_empty() {
columns = result.columns.clone();
}
let fetched_row_count = result.rows.len();
if xlsx_hard_limit_active {
let remaining_rows = XLSX_MAX_DATA_ROWS.saturating_sub(rows_exported as usize);
if fetched_row_count > remaining_rows {
return Err(XLSX_ROW_LIMIT_ERROR.to_string());
}
}
if result.rows.len() > this_page {
result.rows.truncate(this_page);
}
let row_count = result.rows.len();
if format == "csv" {
if let Some(file) = csv_file.as_mut() {
if !wrote_csv_header {
let csv = format_query_result_csv(&columns, &result.rows);
file.write_all(csv.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?;
wrote_csv_header = true;
} else if row_count > 0 {
let rows_csv = format_query_result_csv_rows(&result.rows);
write!(file, "\n{rows_csv}").map_err(|e| format!("Failed to write CSV rows: {e}"))?;
}
}
} else {
if xlsx.is_none() {
let xlsx_file =
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
xlsx = Some(start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?);
}
if let Some(writer) = xlsx.as_mut() {
for row in &result.rows {
writer.write_row(row).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
}
}
}
rows_exported += row_count as u64;
on_progress(progress(request, rows_exported, ExportStatus::Running, None));
if result.session_id.is_some() {
*session_id = result.session_id.clone();
}
if use_agent_result_session && result.has_more && session_id.is_none() {
return Err(AGENT_SESSION_MISSING_ERROR.to_string());
}
if let Some(plan) = keyset_plan.as_mut() {
if let Some(last_row) = result.rows.last() {
plan.last_pk_values =
plan.pk_indices.iter().map(|&index| last_row.get(index).cloned().unwrap_or(Value::Null)).collect();
}
}
let should_continue =
should_fetch_next_page(use_agent_result_session, result.has_more, fetched_row_count, row_count, plan_limit);
if cancel_token.as_ref().is_some_and(|token| token.is_cancelled())
|| is_export_cancelled(&request.export_id).await
{
on_progress(progress(
request,
rows_exported,
ExportStatus::Cancelled,
Some("Export cancelled".to_string()),
));
return Ok(());
}
if !should_continue || row_count == 0 {
break;
}
offset += row_count;
}
on_progress(progress(request, rows_exported, ExportStatus::Writing, None));
if format == "csv" {
if !wrote_csv_header {
let csv = format_query_result_csv(&columns, &[]);
if let Some(file) = csv_file.as_mut() {
file.write_all(csv.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?;
}
}
if let Some(file) = csv_file.as_mut() {
file.flush().map_err(|e| format!("Failed to flush CSV file: {e}"))?;
}
} else if let Some(writer) = xlsx {
let mut buf =
finish_streaming_xlsx_workbook(writer).map_err(|e| format!("Failed to finalize XLSX file: {e}"))?;
buf.flush().map_err(|e| format!("Failed to flush XLSX file: {e}"))?;
} else {
let xlsx_file = File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
let writer = start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?;
let mut buf =
finish_streaming_xlsx_workbook(writer).map_err(|e| format!("Failed to finalize XLSX file: {e}"))?;
buf.flush().map_err(|e| format!("Failed to flush XLSX file: {e}"))?;
}
on_progress(progress(request, rows_exported, ExportStatus::Done, None));
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
fn request(format: &str, row_limit: Option<usize>, total_rows: Option<u64>) -> QueryResultExportRequest {
QueryResultExportRequest {
export_id: "export-1".to_string(),
connection_id: "conn-1".to_string(),
database: "db".to_string(),
schema: None,
sql: "SELECT * FROM users".to_string(),
query_base_sql: "SELECT * FROM users".to_string(),
database_type: DatabaseType::Postgres,
use_agent_cursor: false,
file_path: "out.csv".to_string(),
format: format.to_string(),
page_size: 1000,
row_limit,
total_rows,
timeout_secs: None,
keyset_optimization_enabled: true,
client_session_id: None,
execution_id: None,
}
}
#[test]
fn csv_unlimited_export_has_no_effective_row_limit() {
assert_eq!(effective_row_limit("csv", &request("csv", None, None)), None);
}
#[test]
fn xlsx_unlimited_export_uses_excel_hard_limit() {
assert_eq!(effective_row_limit("xlsx", &request("xlsx", None, None)), Some(XLSX_MAX_DATA_ROWS));
}
#[test]
fn xlsx_row_limit_caps_to_excel_hard_limit() {
assert_eq!(
effective_row_limit("xlsx", &request("xlsx", Some(XLSX_MAX_DATA_ROWS + 10), None)),
Some(XLSX_MAX_DATA_ROWS)
);
}
#[test]
fn xlsx_known_total_above_hard_limit_errors_before_export() {
let req = request("xlsx", None, Some(XLSX_MAX_DATA_ROWS as u64 + 1));
assert!(xlsx_hard_limit_active("xlsx", &req));
assert!(req.total_rows.is_some_and(|total| total > XLSX_MAX_DATA_ROWS as u64));
}
#[test]
fn non_agent_pages_continue_after_trimming_probe_row() {
assert!(should_fetch_next_page(false, false, 101, 100, 100));
assert!(should_fetch_next_page(false, false, 100, 100, 100));
assert!(!should_fetch_next_page(false, false, 42, 42, 100));
}
#[test]
fn agent_pages_follow_has_more_flag() {
assert!(should_fetch_next_page(true, true, 42, 42, 100));
assert!(!should_fetch_next_page(true, false, 100, 100, 100));
}
#[test]
fn streaming_offset_pagination_requires_distinct_followup_page_sql() {
let req = request("csv", Some(1000), None);
assert!(supports_streaming_offset_pagination(&req, 100));
let oracle_req =
QueryResultExportRequest { database_type: DatabaseType::Oracle, ..request("csv", Some(1000), None) };
assert!(!supports_streaming_offset_pagination(&oracle_req, 100));
}
#[test]
fn keyset_candidate_accepts_simple_single_table_wildcard_query() {
let candidate = safe_keyset_candidate("SELECT * FROM public.users").expect("safe keyset candidate");
assert_eq!(candidate.schema.as_deref(), Some("public"));
assert_eq!(candidate.table, "users");
}
#[test]
fn keyset_candidate_rejects_join_and_sorted_queries() {
assert!(safe_keyset_candidate("SELECT * FROM users u JOIN orders o ON o.user_id = u.id").is_none());
assert!(safe_keyset_candidate("SELECT * FROM users ORDER BY name").is_none());
}
#[test]
fn keyset_candidate_rejects_filters_and_projection_changes() {
assert!(safe_keyset_candidate("SELECT * FROM users WHERE active = true").is_none());
assert!(safe_keyset_candidate("SELECT id, name FROM users").is_none());
}
}
+25 -1
View File
@@ -32,6 +32,8 @@ pub struct QueryPaginationExecutionPlanOptions {
pub database_type: Option<DatabaseType>,
pub pagination: QueryPagination,
pub use_agent_cursor: bool,
#[serde(default)]
pub first_page_uses_actual_sql: bool,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
@@ -128,7 +130,9 @@ pub fn build_query_pagination_execution_plan(
}
if options.use_agent_cursor && options.pagination.offset == 0 {
plan.sql_to_execute = options.query_base_sql;
if !options.first_page_uses_actual_sql {
plan.sql_to_execute = options.query_base_sql;
}
plan.page_limit = Some(options.pagination.limit);
plan.page_offset = Some(options.pagination.offset);
plan.use_agent_result_session = true;
@@ -922,6 +926,7 @@ mod tests {
database_type: Some(DatabaseType::SqlServer),
pagination: QueryPagination { limit: 100, offset: 0, session_id: None },
use_agent_cursor: false,
first_page_uses_actual_sql: false,
});
assert_eq!(plan.sql_to_execute, sql);
@@ -1282,6 +1287,7 @@ WHERE u.id = picked.id;
database_type: Some(DatabaseType::Oracle),
pagination: QueryPagination { limit: 500, offset: 0, session_id: None },
use_agent_cursor: true,
first_page_uses_actual_sql: false,
});
assert_eq!(plan.sql_to_execute, "SELECT * FROM events");
@@ -1291,6 +1297,24 @@ WHERE u.id = picked.id;
assert!(plan.use_agent_result_session);
}
#[test]
fn export_agent_cursor_first_page_keeps_actual_sorted_sql() {
let plan = build_query_pagination_execution_plan(QueryPaginationExecutionPlanOptions {
sql: "SELECT * FROM (SELECT * FROM events) t ORDER BY created_at DESC".to_string(),
query_base_sql: "SELECT * FROM events".to_string(),
database_type: Some(DatabaseType::Oracle),
pagination: QueryPagination { limit: 500, offset: 0, session_id: None },
use_agent_cursor: true,
first_page_uses_actual_sql: true,
});
assert_eq!(plan.sql_to_execute, "SELECT * FROM (SELECT * FROM events) t ORDER BY created_at DESC");
assert_eq!(plan.page_limit, Some(500));
assert_eq!(plan.page_offset, Some(0));
assert!(plan.page_sql.is_none());
assert!(plan.use_agent_result_session);
}
#[test]
fn builds_sorted_query_sql() {
let result = build_sorted_query_sql(SortedQuerySqlOptions {
+47 -13
View File
@@ -59,9 +59,15 @@ pub(crate) fn data_row_xml(row_number: usize, columns: &[String], row: &[Value])
format!("<row r=\"{row_number}\">{cells}</row>")
}
/// Shared ZIP entry options for all XLSX parts. XLSX is a ZIP of XML, and the
/// `inlineStr` cell encoding is highly repetitive, so Deflate typically shrinks
/// the file several-fold over `Stored` (matching what Excel/Navicat produce).
fn xlsx_zip_options() -> zip::write::SimpleFileOptions {
zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated)
}
fn write_zip_entry<W: Write + Seek>(zip: &mut zip::ZipWriter<W>, path: &str, content: &str) -> Result<(), String> {
let options = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored);
zip.start_file(path, options).map_err(|err| err.to_string())?;
zip.start_file(path, xlsx_zip_options()).map_err(|err| err.to_string())?;
zip.write_all(content.as_bytes()).map_err(|err| err.to_string())
}
@@ -86,7 +92,7 @@ pub(crate) fn start_streaming_xlsx_workbook<W: Write + Seek>(
// Begin the sheet1.xml entry with header, frozen pane, column widths and
// the header row.
let options = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored);
let options = xlsx_zip_options();
zip.start_file("xl/worksheets/sheet1.xml", options).map_err(|err| err.to_string())?;
let sheet_header = format!(
@@ -434,7 +440,7 @@ pub fn build_xlsx_workbook_multi(sheets: &[XlsxWorksheetData]) -> Result<Vec<u8>
let cursor = Cursor::new(Vec::<u8>::new());
let mut zip = zip::ZipWriter::new(cursor);
let options = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored);
let options = xlsx_zip_options();
for (path, content) in files {
zip.start_file(path, options).map_err(|err| err.to_string())?;
@@ -455,6 +461,31 @@ mod tests {
use calamine::{open_workbook_auto, Reader};
use serde_json::json;
use std::fs;
use std::io::Read;
/// Read and decompress a single entry from an in-memory XLSX (ZIP) buffer.
fn read_zip_entry(bytes: &[u8], path: &str) -> String {
let mut archive = zip::ZipArchive::new(std::io::Cursor::new(bytes.to_vec())).expect("open xlsx as zip archive");
let mut entry = archive.by_name(path).unwrap_or_else(|_| panic!("missing zip entry: {path}"));
let mut content = String::new();
entry.read_to_string(&mut content).expect("read zip entry");
content
}
/// Assert every entry in the XLSX (ZIP) buffer is Deflate-compressed, which
/// is what keeps exported workbooks small (see `xlsx_zip_options`).
fn assert_all_entries_deflated(bytes: &[u8]) {
let mut archive = zip::ZipArchive::new(std::io::Cursor::new(bytes.to_vec())).expect("open xlsx as zip archive");
for index in 0..archive.len() {
let entry = archive.by_index(index).expect("zip entry");
assert_eq!(
entry.compression(),
zip::CompressionMethod::Deflated,
"entry {} should be Deflate-compressed",
entry.name()
);
}
}
#[test]
fn builds_xlsx_zip_with_sheet_data() {
@@ -464,16 +495,19 @@ mod tests {
rows: vec![vec![json!(1), json!("Ada & Bob"), json!(true)], vec![json!(2), json!(null), json!(false)]],
})
.expect("build workbook");
let text = String::from_utf8_lossy(&workbook);
// ZIP magic bytes.
assert_eq!(workbook[0], 0x50);
assert_eq!(workbook[1], 0x4b);
assert!(text.contains("[Content_Types].xml"));
assert!(text.contains("xl/worksheets/sheet1.xml"));
assert!(text.contains("name=\"Users\""));
assert!(text.contains("<c r=\"A2\"><v>1</v></c>"));
assert!(text.contains("Ada &amp; Bob"));
assert!(text.contains("<c r=\"C2\" t=\"b\"><v>1</v></c>"));
// Entries are stored compressed; assert on their decompressed contents.
let sheet = read_zip_entry(&workbook, "xl/worksheets/sheet1.xml");
let workbook_xml = read_zip_entry(&workbook, "xl/workbook.xml");
assert!(workbook_xml.contains("name=\"Users\""));
assert!(sheet.contains("<c r=\"A2\"><v>1</v></c>"));
assert!(sheet.contains("Ada &amp; Bob"));
assert!(sheet.contains("<c r=\"C2\" t=\"b\"><v>1</v></c>"));
assert_all_entries_deflated(&workbook);
}
#[test]
@@ -484,8 +518,8 @@ mod tests {
rows: vec![vec![json!("ok")]],
})
.expect("build workbook");
let text = String::from_utf8_lossy(&workbook);
assert!(text.contains("name=\"bad name with chars and-a-very-\""));
let workbook_xml = read_zip_entry(&workbook, "xl/workbook.xml");
assert!(workbook_xml.contains("name=\"bad name with chars and-a-very-\""));
}
#[test]