fix(export): stream postgres query result exports
This commit is contained in:
1 parent
c8efe06ddc
commit
06e653b0bb
4 files changed
+487
-2
No files matched your search
@@ -712,6 +712,139 @@ async fn execute_select_query(
|
||||
}
|
||||
}
|
||||
|
||||
pub enum PostgresQueryStreamItem {
|
||||
Columns { columns: Vec<String>, column_types: Vec<String> },
|
||||
Row(Vec<serde_json::Value>),
|
||||
}
|
||||
|
||||
enum PostgresQueryStreamError {
|
||||
Postgres { err: tokio_postgres::Error, emitted: bool },
|
||||
Export(String),
|
||||
}
|
||||
|
||||
impl PostgresQueryStreamError {
|
||||
fn into_string(self) -> String {
|
||||
match self {
|
||||
Self::Postgres { err, .. } => pg_error_to_string(err),
|
||||
Self::Export(err) => err,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn stream_select_query_prepared(
|
||||
client: &deadpool_postgres::Client,
|
||||
sql: &str,
|
||||
row_limit: Option<usize>,
|
||||
on_item: &mut impl FnMut(PostgresQueryStreamItem) -> Result<(), String>,
|
||||
) -> Result<u64, PostgresQueryStreamError> {
|
||||
let stmt =
|
||||
client.prepare_cached(sql).await.map_err(|err| PostgresQueryStreamError::Postgres { err, emitted: false })?;
|
||||
let columns: Vec<String> = stmt.columns().iter().map(|c| c.name().to_string()).collect();
|
||||
let column_types: Vec<String> = stmt.columns().iter().map(|c| c.type_().name().to_string()).collect();
|
||||
|
||||
let params: Vec<&(dyn tokio_postgres::types::ToSql + Sync)> = Vec::new();
|
||||
let stream = client
|
||||
.query_raw(&stmt, params)
|
||||
.await
|
||||
.map_err(|err| PostgresQueryStreamError::Postgres { err, emitted: false })?;
|
||||
tokio::pin!(stream);
|
||||
let mut rows_streamed = 0_u64;
|
||||
let mut columns_emitted = false;
|
||||
while let Some(row_result) = stream.next().await {
|
||||
if row_limit.is_some_and(|limit| rows_streamed as usize >= limit) {
|
||||
break;
|
||||
}
|
||||
let row = row_result
|
||||
.map_err(|err| PostgresQueryStreamError::Postgres { err, emitted: columns_emitted || rows_streamed > 0 })?;
|
||||
if !columns_emitted {
|
||||
on_item(PostgresQueryStreamItem::Columns { columns: columns.clone(), column_types: column_types.clone() })
|
||||
.map_err(PostgresQueryStreamError::Export)?;
|
||||
columns_emitted = true;
|
||||
}
|
||||
let values = (0..row.columns().len())
|
||||
.map(|i| pg_value_to_json(&row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
|
||||
.collect();
|
||||
on_item(PostgresQueryStreamItem::Row(values)).map_err(PostgresQueryStreamError::Export)?;
|
||||
rows_streamed += 1;
|
||||
}
|
||||
if !columns_emitted {
|
||||
on_item(PostgresQueryStreamItem::Columns { columns, column_types })
|
||||
.map_err(PostgresQueryStreamError::Export)?;
|
||||
}
|
||||
Ok(rows_streamed)
|
||||
}
|
||||
|
||||
async fn stream_select_query_text(
|
||||
client: &deadpool_postgres::Client,
|
||||
sql: &str,
|
||||
row_limit: Option<usize>,
|
||||
on_item: &mut impl FnMut(PostgresQueryStreamItem) -> Result<(), String>,
|
||||
) -> Result<u64, String> {
|
||||
let stream = client.simple_query_raw(sql).await.map_err(pg_error_to_string)?;
|
||||
tokio::pin!(stream);
|
||||
let mut columns: Vec<String> = Vec::new();
|
||||
let mut rows_streamed = 0_u64;
|
||||
while let Some(message) = stream.next().await {
|
||||
match message.map_err(pg_error_to_string)? {
|
||||
SimpleQueryMessage::RowDescription(cols) => {
|
||||
columns = cols.iter().map(|c| c.name().to_string()).collect();
|
||||
on_item(PostgresQueryStreamItem::Columns { columns: columns.clone(), column_types: Vec::new() })?;
|
||||
}
|
||||
SimpleQueryMessage::Row(row) => {
|
||||
if row_limit.is_some_and(|limit| rows_streamed as usize >= limit) {
|
||||
break;
|
||||
}
|
||||
if columns.is_empty() {
|
||||
columns = row.columns().iter().map(|c| c.name().to_string()).collect();
|
||||
on_item(PostgresQueryStreamItem::Columns { columns: columns.clone(), column_types: Vec::new() })?;
|
||||
}
|
||||
let mut values = Vec::with_capacity(row.len());
|
||||
for i in 0..row.len() {
|
||||
values.push(match row.try_get(i).map_err(pg_error_to_string)? {
|
||||
Some(value) => serde_json::Value::String(value.to_string()),
|
||||
None => serde_json::Value::Null,
|
||||
});
|
||||
}
|
||||
on_item(PostgresQueryStreamItem::Row(values))?;
|
||||
rows_streamed += 1;
|
||||
}
|
||||
SimpleQueryMessage::CommandComplete(_) => {}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Ok(rows_streamed)
|
||||
}
|
||||
|
||||
async fn stream_select_query_inner(
|
||||
client: &deadpool_postgres::Client,
|
||||
sql: &str,
|
||||
row_limit: Option<usize>,
|
||||
on_item: &mut impl FnMut(PostgresQueryStreamItem) -> Result<(), String>,
|
||||
) -> Result<u64, String> {
|
||||
match stream_select_query_prepared(client, sql, row_limit, on_item).await {
|
||||
Ok(rows) => Ok(rows),
|
||||
Err(PostgresQueryStreamError::Postgres { err, emitted: false }) if should_retry_postgres_stale_cache(&err) => {
|
||||
// The cached prepared statement can become stale after schema changes.
|
||||
// Evict and retry once, matching the normal query execution path.
|
||||
log::warn!("[postgres][stream:stale_cache] evicting cached statement: {}", pg_error_to_string(err));
|
||||
client.statement_cache.remove(sql, &[]);
|
||||
match stream_select_query_prepared(client, sql, row_limit, on_item).await {
|
||||
Ok(rows) => Ok(rows),
|
||||
Err(PostgresQueryStreamError::Postgres { err, emitted: false })
|
||||
if should_retry_postgres_text_query(&err) =>
|
||||
{
|
||||
stream_select_query_text(client, sql, row_limit, on_item).await
|
||||
}
|
||||
Err(err) => Err(err.into_string()),
|
||||
}
|
||||
}
|
||||
Err(PostgresQueryStreamError::Postgres { err, emitted: false }) if should_retry_postgres_text_query(&err) => {
|
||||
stream_select_query_text(client, sql, row_limit, on_item).await
|
||||
}
|
||||
Err(err) => Err(err.into_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn stream_query_rows(
|
||||
pool: &Pool,
|
||||
sql: &str,
|
||||
@@ -1963,6 +2096,59 @@ pub async fn execute_query_with_max_rows_and_cancel(
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn stream_select_query_with_cancel(
|
||||
pool: &Pool,
|
||||
schema: Option<&str>,
|
||||
sql: &str,
|
||||
max_rows: Option<usize>,
|
||||
cancel_token: Option<CancellationToken>,
|
||||
budget: DbOperationBudget,
|
||||
cancel_context: Option<PostgresCancelContext>,
|
||||
on_item: impl FnMut(PostgresQueryStreamItem) -> Result<(), String>,
|
||||
) -> Result<u64, String> {
|
||||
let start = Instant::now();
|
||||
let client = checkout_postgres_client(pool, cancel_token.as_ref(), budget.checkout_timeout).await?;
|
||||
let mut on_item = on_item;
|
||||
let row_limit = max_rows.map(|limit| limit.max(1));
|
||||
let schema = schema.map(str::trim).filter(|schema| !schema.is_empty());
|
||||
let schema_was_set = schema.is_some_and(|_| !is_transaction_recovery_statement(sql));
|
||||
|
||||
if let Some(schema) = schema.filter(|_| schema_was_set) {
|
||||
// Match normal query execution: export may reference unqualified names
|
||||
// in the active schema, so the streaming path must use the same search_path.
|
||||
execute_postgres_infra_statement(
|
||||
&client,
|
||||
&format!("SET search_path TO {}, public", pg_quote_ident(schema)),
|
||||
budget.recycle_timeout,
|
||||
"schema.set",
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
let pg_cancel_token = client.cancel_token();
|
||||
let result = wait_postgres_query(
|
||||
pg_cancel_token,
|
||||
cancel_context,
|
||||
cancel_token,
|
||||
budget.query_timeout,
|
||||
budget.cancel_timeout,
|
||||
stream_select_query_inner(&client, sql, row_limit, &mut on_item),
|
||||
)
|
||||
.await;
|
||||
|
||||
if schema_was_set {
|
||||
let reset_result = reset_postgres_search_path(&client, budget.cleanup_timeout, start).await;
|
||||
match (result, reset_result) {
|
||||
(Ok(rows), Ok(())) => Ok(rows),
|
||||
(Err(query_err), Ok(())) => Err(query_err),
|
||||
(Ok(_), Err(reset_err)) => Err(reset_err),
|
||||
(Err(query_err), Err(reset_err)) => Err(format!("{query_err}; {reset_err}")),
|
||||
}
|
||||
} else {
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn execute_query_with_schema(pool: &Pool, schema: &str, sql: &str) -> Result<QueryResult, String> {
|
||||
execute_query_with_schema_and_max_rows(pool, schema, sql, None).await
|
||||
}
|
||||
|
||||
@@ -884,7 +884,7 @@ fn resolve_query_timeout(timeout_secs: Option<u64>) -> Option<Duration> {
|
||||
}
|
||||
}
|
||||
|
||||
async fn operation_budget_for_pool_key(
|
||||
pub async fn operation_budget_for_pool_key(
|
||||
state: &AppState,
|
||||
pool_key: &str,
|
||||
query_timeout: Option<Duration>,
|
||||
|
||||
@@ -9,7 +9,8 @@ use crate::database_export::is_export_cancelled;
|
||||
pub use crate::database_export::ExportStatus;
|
||||
use crate::models::connection::DatabaseType;
|
||||
use crate::query::{
|
||||
canceled_error, close_query_session, execute_sql_statement_with_options, QueryExecutionOptions, QUERY_CANCELED,
|
||||
canceled_error, close_query_session, execute_sql_statement_with_options, operation_budget_for_pool_key,
|
||||
QueryExecutionOptions, QUERY_CANCELED,
|
||||
};
|
||||
use crate::query_result_sql::{
|
||||
build_query_pagination_execution_plan, QueryPagination, QueryPaginationExecutionPlanOptions,
|
||||
@@ -314,6 +315,10 @@ async fn export_query_result_core_inner(
|
||||
|
||||
on_progress(progress(request, 0, ExportStatus::Running, None));
|
||||
|
||||
if try_export_postgres_query_result_stream(state, request, &format, cancel_token.clone(), on_progress).await? {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if try_export_sqlserver_query_result_stream(state, request, &format, cancel_token.clone(), on_progress).await? {
|
||||
return Ok(());
|
||||
}
|
||||
@@ -554,6 +559,152 @@ async fn export_query_result_core_inner(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn try_export_postgres_query_result_stream(
|
||||
state: &AppState,
|
||||
request: &QueryResultExportRequest,
|
||||
format: &str,
|
||||
cancel_token: Option<CancellationToken>,
|
||||
on_progress: &impl Fn(TableExportProgress),
|
||||
) -> Result<bool, String> {
|
||||
if request.use_agent_cursor
|
||||
|| !crate::sql::starts_with_executable_sql_keyword(
|
||||
&request.sql,
|
||||
&["SELECT", "SHOW", "EXPLAIN", "WITH", "TABLE"],
|
||||
)
|
||||
{
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let database = request.database.trim();
|
||||
let pool_key = if database.is_empty() {
|
||||
state.get_or_create_pool_for_session(&request.connection_id, None, request.client_session_id.as_deref()).await?
|
||||
} else {
|
||||
state
|
||||
.get_or_create_pool_for_session(
|
||||
&request.connection_id,
|
||||
Some(database),
|
||||
request.client_session_id.as_deref(),
|
||||
)
|
||||
.await?
|
||||
};
|
||||
let connections = state.connections.read().await;
|
||||
let Some(pool) = connections.get(&pool_key).and_then(|pool| match pool {
|
||||
PoolKind::Postgres(pool) => Some(pool.clone()),
|
||||
_ => None,
|
||||
}) else {
|
||||
return Ok(false);
|
||||
};
|
||||
drop(connections);
|
||||
|
||||
if let Some(execution_id) = request.execution_id.as_deref() {
|
||||
state.running_queries.set_pool_key(execution_id, pool_key.clone());
|
||||
}
|
||||
state.touch_pool_activity(&pool_key).await;
|
||||
let _activity_touch = state.pool_activity_touch(&pool_key);
|
||||
|
||||
let xlsx_hard_limit_active = xlsx_hard_limit_active(format, request);
|
||||
let row_limit = effective_row_limit(format, request);
|
||||
let stream_row_limit =
|
||||
if xlsx_hard_limit_active { row_limit.map(|limit| limit.saturating_add(1)) } else { row_limit };
|
||||
let progress_row_interval = request.page_size.max(1) as u64;
|
||||
let mut columns: Vec<String> = Vec::new();
|
||||
let mut rows_exported = 0_u64;
|
||||
let mut last_progress_rows = 0_u64;
|
||||
let mut last_progress_at = Instant::now();
|
||||
let mut csv_file = if format == "csv" {
|
||||
let mut file =
|
||||
BufWriter::new(File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?);
|
||||
file.write_all(b"\xEF\xBB\xBF").map_err(|e| format!("Failed to write BOM: {e}"))?;
|
||||
Some(file)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut xlsx = None;
|
||||
let budget = operation_budget_for_pool_key(state, &pool_key, query_export_timeout(request.timeout_secs)).await;
|
||||
let cancel_context = state.get_postgres_cancel_context(&pool_key).await;
|
||||
|
||||
crate::db::postgres::stream_select_query_with_cancel(
|
||||
&pool,
|
||||
request.schema.as_deref(),
|
||||
&request.sql,
|
||||
stream_row_limit,
|
||||
cancel_token,
|
||||
budget,
|
||||
cancel_context,
|
||||
|item| {
|
||||
match item {
|
||||
crate::db::postgres::PostgresQueryStreamItem::Columns { columns: stream_columns, .. } => {
|
||||
columns = stream_columns;
|
||||
if let Some(file) = csv_file.as_mut() {
|
||||
let csv = format_query_result_csv(&columns, &[]);
|
||||
let header = csv.strip_suffix('\n').unwrap_or(&csv);
|
||||
file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write CSV: {e}"))?;
|
||||
} else {
|
||||
let xlsx_file =
|
||||
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
|
||||
xlsx =
|
||||
Some(start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?);
|
||||
}
|
||||
}
|
||||
crate::db::postgres::PostgresQueryStreamItem::Row(row) => {
|
||||
if xlsx_hard_limit_active && rows_exported as usize >= XLSX_MAX_DATA_ROWS {
|
||||
return Err(XLSX_ROW_LIMIT_ERROR.to_string());
|
||||
}
|
||||
if let Some(file) = csv_file.as_mut() {
|
||||
let rows_csv = format_query_result_csv_rows(std::slice::from_ref(&row));
|
||||
write!(file, "\n{rows_csv}").map_err(|e| format!("Failed to write CSV rows: {e}"))?;
|
||||
} else if let Some(writer) = xlsx.as_mut() {
|
||||
writer.write_row(&row).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||
} else {
|
||||
let xlsx_file =
|
||||
File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
|
||||
xlsx =
|
||||
Some(start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?);
|
||||
if let Some(writer) = xlsx.as_mut() {
|
||||
writer.write_row(&row).map_err(|e| format!("Failed to write XLSX row: {e}"))?;
|
||||
}
|
||||
}
|
||||
rows_exported += 1;
|
||||
let now = Instant::now();
|
||||
if should_emit_stream_progress(
|
||||
rows_exported,
|
||||
last_progress_rows,
|
||||
progress_row_interval,
|
||||
now.duration_since(last_progress_at),
|
||||
) {
|
||||
on_progress(progress(request, rows_exported, ExportStatus::Running, None));
|
||||
last_progress_rows = rows_exported;
|
||||
last_progress_at = now;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
|
||||
if rows_exported != last_progress_rows {
|
||||
on_progress(progress(request, rows_exported, ExportStatus::Running, None));
|
||||
}
|
||||
on_progress(progress(request, rows_exported, ExportStatus::Writing, None));
|
||||
if let Some(file) = csv_file.as_mut() {
|
||||
file.flush().map_err(|e| format!("Failed to flush CSV file: {e}"))?;
|
||||
}
|
||||
if let Some(writer) = xlsx {
|
||||
let mut buf =
|
||||
finish_streaming_xlsx_workbook(writer).map_err(|e| format!("Failed to finalize XLSX file: {e}"))?;
|
||||
buf.flush().map_err(|e| format!("Failed to flush XLSX file: {e}"))?;
|
||||
} else if format == "xlsx" {
|
||||
let xlsx_file = File::create(&request.file_path).map_err(|e| format!("Failed to create XLSX file: {e}"))?;
|
||||
let writer = start_streaming_xlsx_workbook(BufWriter::new(xlsx_file), Some("Result"), &columns)?;
|
||||
let mut buf =
|
||||
finish_streaming_xlsx_workbook(writer).map_err(|e| format!("Failed to finalize XLSX file: {e}"))?;
|
||||
buf.flush().map_err(|e| format!("Failed to flush XLSX file: {e}"))?;
|
||||
}
|
||||
on_progress(progress(request, rows_exported, ExportStatus::Done, None));
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn try_export_sqlserver_query_result_stream(
|
||||
state: &AppState,
|
||||
request: &QueryResultExportRequest,
|
||||
|
||||
Reference in new issue
Block a user