fix(export): stream postgres query result exports

This commit is contained in:
t8y2 committed 2026-07-03 20:02:19 +08:00
1 parent c8efe06ddc
commit 06e653b0bb
4 files changed
+487 -2

No files matched your search

+186
View File
@@ -712,6 +712,139 @@ async fn execute_select_query(
}
}
pub enum PostgresQueryStreamItem {
Columns { columns: Vec<String>, column_types: Vec<String> },
Row(Vec<serde_json::Value>),
}
enum PostgresQueryStreamError {
Postgres { err: tokio_postgres::Error, emitted: bool },
Export(String),
}
impl PostgresQueryStreamError {
fn into_string(self) -> String {
match self {
Self::Postgres { err, .. } => pg_error_to_string(err),
Self::Export(err) => err,
}
}
}
async fn stream_select_query_prepared(
client: &deadpool_postgres::Client,
sql: &str,
row_limit: Option<usize>,
on_item: &mut impl FnMut(PostgresQueryStreamItem) -> Result<(), String>,
) -> Result<u64, PostgresQueryStreamError> {
let stmt =
client.prepare_cached(sql).await.map_err(|err| PostgresQueryStreamError::Postgres { err, emitted: false })?;
let columns: Vec<String> = stmt.columns().iter().map(|c| c.name().to_string()).collect();
let column_types: Vec<String> = stmt.columns().iter().map(|c| c.type_().name().to_string()).collect();
let params: Vec<&(dyn tokio_postgres::types::ToSql + Sync)> = Vec::new();
let stream = client
.query_raw(&stmt, params)
.await
.map_err(|err| PostgresQueryStreamError::Postgres { err, emitted: false })?;
tokio::pin!(stream);
let mut rows_streamed = 0_u64;
let mut columns_emitted = false;
while let Some(row_result) = stream.next().await {
if row_limit.is_some_and(|limit| rows_streamed as usize >= limit) {
break;
}
let row = row_result
.map_err(|err| PostgresQueryStreamError::Postgres { err, emitted: columns_emitted || rows_streamed > 0 })?;
if !columns_emitted {
on_item(PostgresQueryStreamItem::Columns { columns: columns.clone(), column_types: column_types.clone() })
.map_err(PostgresQueryStreamError::Export)?;
columns_emitted = true;
}
let values = (0..row.columns().len())
.map(|i| pg_value_to_json(&row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
.collect();
on_item(PostgresQueryStreamItem::Row(values)).map_err(PostgresQueryStreamError::Export)?;
rows_streamed += 1;
}
if !columns_emitted {
on_item(PostgresQueryStreamItem::Columns { columns, column_types })
.map_err(PostgresQueryStreamError::Export)?;
}
Ok(rows_streamed)
}
async fn stream_select_query_text(
client: &deadpool_postgres::Client,
sql: &str,
row_limit: Option<usize>,
on_item: &mut impl FnMut(PostgresQueryStreamItem) -> Result<(), String>,
) -> Result<u64, String> {
let stream = client.simple_query_raw(sql).await.map_err(pg_error_to_string)?;
tokio::pin!(stream);
let mut columns: Vec<String> = Vec::new();
let mut rows_streamed = 0_u64;
while let Some(message) = stream.next().await {
match message.map_err(pg_error_to_string)? {
SimpleQueryMessage::RowDescription(cols) => {
columns = cols.iter().map(|c| c.name().to_string()).collect();
on_item(PostgresQueryStreamItem::Columns { columns: columns.clone(), column_types: Vec::new() })?;
}
SimpleQueryMessage::Row(row) => {
if row_limit.is_some_and(|limit| rows_streamed as usize >= limit) {
break;
}
if columns.is_empty() {
columns = row.columns().iter().map(|c| c.name().to_string()).collect();
on_item(PostgresQueryStreamItem::Columns { columns: columns.clone(), column_types: Vec::new() })?;
}
let mut values = Vec::with_capacity(row.len());
for i in 0..row.len() {
values.push(match row.try_get(i).map_err(pg_error_to_string)? {
Some(value) => serde_json::Value::String(value.to_string()),
None => serde_json::Value::Null,
});
}
on_item(PostgresQueryStreamItem::Row(values))?;
rows_streamed += 1;
}
SimpleQueryMessage::CommandComplete(_) => {}
_ => {}
}
}
Ok(rows_streamed)
}
async fn stream_select_query_inner(
client: &deadpool_postgres::Client,
sql: &str,
row_limit: Option<usize>,
on_item: &mut impl FnMut(PostgresQueryStreamItem) -> Result<(), String>,
) -> Result<u64, String> {
match stream_select_query_prepared(client, sql, row_limit, on_item).await {
Ok(rows) => Ok(rows),
Err(PostgresQueryStreamError::Postgres { err, emitted: false }) if should_retry_postgres_stale_cache(&err) => {
// The cached prepared statement can become stale after schema changes.
// Evict and retry once, matching the normal query execution path.
log::warn!("[postgres][stream:stale_cache] evicting cached statement: {}", pg_error_to_string(err));
client.statement_cache.remove(sql, &[]);
match stream_select_query_prepared(client, sql, row_limit, on_item).await {
Ok(rows) => Ok(rows),
Err(PostgresQueryStreamError::Postgres { err, emitted: false })
if should_retry_postgres_text_query(&err) =>
{
stream_select_query_text(client, sql, row_limit, on_item).await
}
Err(err) => Err(err.into_string()),
}
}
Err(PostgresQueryStreamError::Postgres { err, emitted: false }) if should_retry_postgres_text_query(&err) => {
stream_select_query_text(client, sql, row_limit, on_item).await
}
Err(err) => Err(err.into_string()),
}
}
pub async fn stream_query_rows(
pool: &Pool,
sql: &str,
@@ -1963,6 +2096,59 @@ pub async fn execute_query_with_max_rows_and_cancel(
.await
}
pub async fn stream_select_query_with_cancel(
pool: &Pool,
schema: Option<&str>,
sql: &str,
max_rows: Option<usize>,
cancel_token: Option<CancellationToken>,
budget: DbOperationBudget,
cancel_context: Option<PostgresCancelContext>,
on_item: impl FnMut(PostgresQueryStreamItem) -> Result<(), String>,
) -> Result<u64, String> {
let start = Instant::now();
let client = checkout_postgres_client(pool, cancel_token.as_ref(), budget.checkout_timeout).await?;
let mut on_item = on_item;
let row_limit = max_rows.map(|limit| limit.max(1));
let schema = schema.map(str::trim).filter(|schema| !schema.is_empty());
let schema_was_set = schema.is_some_and(|_| !is_transaction_recovery_statement(sql));
if let Some(schema) = schema.filter(|_| schema_was_set) {
// Match normal query execution: export may reference unqualified names
// in the active schema, so the streaming path must use the same search_path.
execute_postgres_infra_statement(
&client,
&format!("SET search_path TO {}, public", pg_quote_ident(schema)),
budget.recycle_timeout,
"schema.set",
)
.await?;
}
let pg_cancel_token = client.cancel_token();
let result = wait_postgres_query(
pg_cancel_token,
cancel_context,
cancel_token,
budget.query_timeout,
budget.cancel_timeout,
stream_select_query_inner(&client, sql, row_limit, &mut on_item),
)
.await;
if schema_was_set {
let reset_result = reset_postgres_search_path(&client, budget.cleanup_timeout, start).await;
match (result, reset_result) {
(Ok(rows), Ok(())) => Ok(rows),
(Err(query_err), Ok(())) => Err(query_err),
(Ok(_), Err(reset_err)) => Err(reset_err),
(Err(query_err), Err(reset_err)) => Err(format!("{query_err}; {reset_err}")),
}
} else {
result
}
}
pub async fn execute_query_with_schema(pool: &Pool, schema: &str, sql: &str) -> Result<QueryResult, String> {
execute_query_with_schema_and_max_rows(pool, schema, sql, None).await
}
+1 -1
View File
@@ -884,7 +884,7 @@ fn resolve_query_timeout(timeout_secs: Option<u64>) -> Option<Duration> {
}
}
async fn operation_budget_for_pool_key(
pub async fn operation_budget_for_pool_key(
state: &AppState,
pool_key: &str,
query_timeout: Option<Duration>,
+152 -1
View File
@@ -9,7 +9,8 @@ use crate::database_export::is_export_cancelled;
pub use crate::database_export::ExportStatus;
use crate::models::connection::DatabaseType;
use crate::query::{
canceled_error, close_query_session, execute_sql_statement_with_options, QueryExecutionOptions, QUERY_CANCELED,
canceled_error, close_query_session, execute_sql_statement_with_options, operation_budget_for_pool_key,
QueryExecutionOptions, QUERY_CANCELED,
};
use crate::query_result_sql::{
build_query_pagination_execution_plan, QueryPagination, QueryPaginationExecutionPlanOptions,
@@ -314,6 +315,10 @@ async fn export_query_result_core_inner(
on_progress(progress(request, 0, ExportStatus::Running, None));
if try_export_postgres_query_result_stream(state, request, &format, cancel_token.clone(), on_progress).await? {
return Ok(());
}
if try_export_sqlserver_query_result_stream(state, request, &format, cancel_token.clone(), on_progress).await? {
return Ok(());
}
@@ -554,6 +559,152 @@ async fn export_query_result_core_inner(
Ok(())
}
async fn try_export_postgres_query_result_stream(
state: &AppState,
request: &QueryResultExportRequest,
format: &str,
cancel_token: Option<CancellationToken>,
on_progress: &impl Fn(TableExportProgress),
) -> Result<bool, String> {
if request.use_agent_cursor
|| !crate::sql::starts_with_executable_sql_keyword(
&request.sql,
&["SELECT", "SHOW", "EXPLAIN", "WITH", "TABLE"],
)
{
return Ok(false);
}
let database = request.database.trim();
let pool_key = if database.is_empty() {
state.get_or_create_pool_for_session(&request.connection_id, None, request.client_session_id.as_deref()).await?
} else {
state
.get_or_create_pool_for_session(
&request.connection_id,
Some(database),
request.client_session_id.as_deref(),
)
.await?
};
let connections = state.connections.read().await;
let Some(pool) = connections.get(&pool_key).and_then(|pool| match pool {
PoolKind::Postgres(pool) => Some(pool.clone()),
_ => None,
}) else {
return Ok(false);
};
drop(connections);
if let Some(execution_id) = request.execution_id.as_deref() {
state.running_queries.set_pool_key(execution_id, pool_key.clone());
}
state.touch_pool_activity(&pool_key).await;
let _activity_touch = state.pool_activity_touch(&pool_key);
let xlsx_hard_limit_active = xlsx_hard_limit_active(format, request);
let row_limit = effective_row_limit(format, request);
let stream_row_limit =
if xlsx_hard_limit_active { row_limit.map(|limit| limit.saturating_add(1)) } else { row_limit };
let progress_row_interval = request.page_size.max(1) as u64;
let mut columns: Vec<String> = Vec::new();
let mut rows_exported = 0_u64;
let mut last_progress_rows = 0_u64;
let mut last_progress_at = Instant::now();
let mut csv_file = if format == "csv" {
let mut file =
BufWriter::new(File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?);
file.write_all(b"\xEF\xBB\xBF").map_err(|e| format!("Failed to write BOM: {e}"))?;
Some(file)
} else {
None
};
let mut xlsx = None;
let budget = operation_budget_for_pool_key(state, &pool_key, query_export_timeout(request.timeout_secs)).await;
let cancel_context = state.get_postgres_cancel_context(&pool_key).await;
crate::db::postgres::stream_select_query_with_cancel(
&pool,
request.schema.as_deref(),
&request.sql,
stream_row_limit,
cancel_token,
budget,
cancel_context,
|item| {
match item {
crate::db::postgres::PostgresQueryStreamItem::Columns { columns: stream_columns, .. } => {
columns = stream_columns;
if let Some(file) = csv_file.as_mut() {
let csv = format_query_result_csv(&columns, &[]);
let header = csv.strip_suffix('\n').unwrap_or(&csv);
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?;
} else {
let xlsx_file =
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
xlsx =
Some(start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?);
}
}
crate::db::postgres::PostgresQueryStreamItem::Row(row) => {
if xlsx_hard_limit_active && rows_exported as usize >= XLSX_MAX_DATA_ROWS {
return Err(XLSX_ROW_LIMIT_ERROR.to_string());
}
if let Some(file) = csv_file.as_mut() {
let rows_csv = format_query_result_csv_rows(std::slice::from_ref(&row));
write!(file, "\n{rows_csv}").map_err(|e| format!("Failed to write CSV rows: {e}"))?;
} else if let Some(writer) = xlsx.as_mut() {
writer.write_row(&row).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
} else {
let xlsx_file =
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
xlsx =
Some(start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?);
if let Some(writer) = xlsx.as_mut() {
writer.write_row(&row).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
}
}
rows_exported += 1;
let now = Instant::now();
if should_emit_stream_progress(
rows_exported,
last_progress_rows,
progress_row_interval,
now.duration_since(last_progress_at),
) {
on_progress(progress(request, rows_exported, ExportStatus::Running, None));
last_progress_rows = rows_exported;
last_progress_at = now;
}
}
}
Ok(())
},
)
.await?;
if rows_exported != last_progress_rows {
on_progress(progress(request, rows_exported, ExportStatus::Running, None));
}
on_progress(progress(request, rows_exported, ExportStatus::Writing, None));
if let Some(file) = csv_file.as_mut() {
file.flush().map_err(|e| format!("Failed to flush CSV file: {e}"))?;
}
if let Some(writer) = xlsx {
let mut buf =
finish_streaming_xlsx_workbook(writer).map_err(|e| format!("Failed to finalize XLSX file: {e}"))?;
buf.flush().map_err(|e| format!("Failed to flush XLSX file: {e}"))?;
} else if format == "xlsx" {
let xlsx_file = File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
let writer = start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?;
let mut buf =
finish_streaming_xlsx_workbook(writer).map_err(|e| format!("Failed to finalize XLSX file: {e}"))?;
buf.flush().map_err(|e| format!("Failed to flush XLSX file: {e}"))?;
}
on_progress(progress(request, rows_exported, ExportStatus::Done, None));
Ok(true)
}
async fn try_export_sqlserver_query_result_stream(
state: &AppState,
request: &QueryResultExportRequest,