fix(export): support PostgreSQL temporary-table result export
This commit is contained in:
18 files changed
+281
-28
No files matched your search
@@ -2563,6 +2563,7 @@ pub async fn execute_query_with_max_rows_and_cancel(
|
||||
pub async fn stream_select_query_with_cancel(
|
||||
pool: &Pool,
|
||||
schema: Option<&str>,
|
||||
setup_sql: &[String],
|
||||
sql: &str,
|
||||
max_rows: Option<usize>,
|
||||
cancel_token: Option<CancellationToken>,
|
||||
@@ -2589,28 +2590,72 @@ pub async fn stream_select_query_with_cancel(
|
||||
.await?;
|
||||
}
|
||||
|
||||
let pg_cancel_token = client.cancel_token();
|
||||
let setup_transaction_started = !setup_sql.is_empty();
|
||||
if setup_transaction_started {
|
||||
execute_postgres_infra_statement(&client, "BEGIN", budget.recycle_timeout, "export_setup.begin").await?;
|
||||
}
|
||||
|
||||
let query_timeout = budget.query_timeout;
|
||||
let timeout_error =
|
||||
format!("Query timed out after {} seconds", query_timeout.map_or(0, |timeout| timeout.as_secs()));
|
||||
let progress_clock = Arc::new(StreamProgressClock::new());
|
||||
let progress_clock_for_stream = progress_clock.clone();
|
||||
let mut on_stream_item = |item| {
|
||||
on_item(item)?;
|
||||
progress_clock_for_stream.mark();
|
||||
let setup_result = async {
|
||||
for setup_statement in setup_sql {
|
||||
wait_postgres_query(
|
||||
client.cancel_token(),
|
||||
cancel_context.clone(),
|
||||
cancel_token.clone(),
|
||||
query_timeout,
|
||||
budget.cancel_timeout,
|
||||
async {
|
||||
client.batch_execute(setup_statement).await.map_err(pg_error_to_string)?;
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
let result = await_stream_with_progress_timeout(
|
||||
stream_select_query_inner(&client, sql, row_limit, &mut on_stream_item),
|
||||
query_timeout,
|
||||
progress_clock,
|
||||
cancel_token.as_ref(),
|
||||
timeout_error.clone(),
|
||||
)
|
||||
.await;
|
||||
if result.as_ref().is_err_and(|error| error == &timeout_error || error == crate::query::QUERY_CANCELED) {
|
||||
cancel_postgres_query(pg_cancel_token, cancel_context.as_ref(), budget.cancel_timeout).await;
|
||||
}
|
||||
.await;
|
||||
|
||||
let result = match setup_result {
|
||||
Ok(()) => {
|
||||
let pg_cancel_token = client.cancel_token();
|
||||
let progress_clock = Arc::new(StreamProgressClock::new());
|
||||
let progress_clock_for_stream = progress_clock.clone();
|
||||
let mut on_stream_item = |item| {
|
||||
on_item(item)?;
|
||||
progress_clock_for_stream.mark();
|
||||
Ok(())
|
||||
};
|
||||
let result = await_stream_with_progress_timeout(
|
||||
stream_select_query_inner(&client, sql, row_limit, &mut on_stream_item),
|
||||
query_timeout,
|
||||
progress_clock,
|
||||
cancel_token.as_ref(),
|
||||
timeout_error.clone(),
|
||||
)
|
||||
.await;
|
||||
if result.as_ref().is_err_and(|error| error == &timeout_error || error == crate::query::QUERY_CANCELED) {
|
||||
cancel_postgres_query(pg_cancel_token, cancel_context.as_ref(), budget.cancel_timeout).await;
|
||||
}
|
||||
result
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
};
|
||||
|
||||
let result = if setup_transaction_started {
|
||||
let rollback_result =
|
||||
execute_postgres_infra_statement(&client, "ROLLBACK", budget.cleanup_timeout, "export_setup.rollback")
|
||||
.await;
|
||||
match (result, rollback_result) {
|
||||
(Ok(rows), Ok(_)) => Ok(rows),
|
||||
(Err(query_err), Ok(_)) => Err(query_err),
|
||||
(Ok(_), Err(rollback_err)) => Err(rollback_err),
|
||||
(Err(query_err), Err(rollback_err)) => Err(format!("{query_err}; {rollback_err}")),
|
||||
}
|
||||
} else {
|
||||
result
|
||||
};
|
||||
|
||||
if schema_was_set {
|
||||
let reset_result = reset_postgres_search_path(&client, budget.cleanup_timeout, start).await;
|
||||
|
||||
@@ -26,8 +26,10 @@ use crate::xlsx_export::{
|
||||
XlsxWorksheetData,
|
||||
};
|
||||
use serde_json::Value;
|
||||
use sqlparser::ast::{GroupByExpr, ObjectNamePart, OrderByKind, SelectItem, SetExpr, Statement, TableFactor};
|
||||
use sqlparser::dialect::GenericDialect;
|
||||
use sqlparser::ast::{
|
||||
GroupByExpr, ObjectName, ObjectNamePart, ObjectType, OrderByKind, SelectItem, SetExpr, Statement, TableFactor,
|
||||
};
|
||||
use sqlparser::dialect::{GenericDialect, PostgreSqlDialect};
|
||||
use sqlparser::parser::Parser;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
@@ -61,6 +63,8 @@ pub struct QueryResultExportRequest {
|
||||
pub schema: Option<String>,
|
||||
pub sql: String,
|
||||
pub query_base_sql: String,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub setup_sql: Vec<String>,
|
||||
pub database_type: DatabaseType,
|
||||
#[serde(default)]
|
||||
pub use_agent_cursor: bool,
|
||||
@@ -85,6 +89,33 @@ pub struct QueryResultExportRequest {
|
||||
pub date_time_format: Option<String>,
|
||||
}
|
||||
|
||||
fn safe_postgres_temp_setup_sql(setup_sql: &[String]) -> Option<Vec<String>> {
|
||||
if setup_sql.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let dialect = PostgreSqlDialect {};
|
||||
let mut temporary_tables: Vec<ObjectName> = Vec::new();
|
||||
for sql in setup_sql {
|
||||
let statements = Parser::parse_sql(&dialect, sql).ok()?;
|
||||
let [statement] = statements.as_slice() else {
|
||||
return None;
|
||||
};
|
||||
match statement {
|
||||
Statement::CreateTable(table) if table.temporary => temporary_tables.push(table.name.clone()),
|
||||
Statement::CreateIndex(index) if temporary_tables.iter().any(|name| name == &index.table_name) => {}
|
||||
Statement::Drop { object_type: ObjectType::Table, names, .. }
|
||||
if !names.is_empty() && names.iter().all(|name| temporary_tables.contains(name)) =>
|
||||
{
|
||||
temporary_tables.retain(|table| !names.contains(table));
|
||||
}
|
||||
_ => return None,
|
||||
}
|
||||
}
|
||||
|
||||
Some(setup_sql.to_vec())
|
||||
}
|
||||
|
||||
fn split_excel_cell_text(value: &str) -> Vec<String> {
|
||||
let mut chunks = Vec::new();
|
||||
let mut current = String::new();
|
||||
@@ -731,9 +762,11 @@ async fn try_export_postgres_query_result_stream(
|
||||
let budget = operation_budget_for_pool_key(state, &pool_key, query_export_timeout(request.timeout_secs)).await;
|
||||
let cancel_context = state.get_postgres_cancel_context(&pool_key).await;
|
||||
|
||||
let setup_sql = safe_postgres_temp_setup_sql(&request.setup_sql).unwrap_or_default();
|
||||
crate::db::postgres::stream_select_query_with_cancel(
|
||||
&pool,
|
||||
request.schema.as_deref(),
|
||||
&setup_sql,
|
||||
&request.sql,
|
||||
stream_row_limit,
|
||||
cancel_token,
|
||||
@@ -1428,6 +1461,35 @@ async fn try_export_sqlserver_query_result_stream(
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn postgres_temp_setup_accepts_only_session_local_table_operations() {
|
||||
let safe = vec![
|
||||
"CREATE TEMPORARY TABLE t1 AS SELECT 1 AS id".to_string(),
|
||||
"CREATE INDEX t1_id ON t1(id)".to_string(),
|
||||
"CREATE TEMP TABLE t2 AS SELECT id FROM t1".to_string(),
|
||||
"DROP TABLE t1".to_string(),
|
||||
];
|
||||
assert_eq!(safe_postgres_temp_setup_sql(&safe), Some(safe.clone()));
|
||||
|
||||
let parenthesized_ctas = vec![
|
||||
"CREATE TEMPORARY TABLE t1 AS (SELECT CURRENT_DATE AS \u{8d77}\u{4fdd}\u{65e5}\u{671f})".to_string(),
|
||||
"CREATE INDEX t1_1 ON t1(\u{8d77}\u{4fdd}\u{65e5}\u{671f}, \u{7ec8}\u{6b62}\u{65e5}\u{671f})".to_string(),
|
||||
];
|
||||
assert_eq!(safe_postgres_temp_setup_sql(&parenthesized_ctas), Some(parenthesized_ctas.clone()));
|
||||
|
||||
let persistent_create = vec!["CREATE TABLE users_copy AS SELECT * FROM users".to_string()];
|
||||
assert!(safe_postgres_temp_setup_sql(&persistent_create).is_none());
|
||||
|
||||
let persistent_write = vec![
|
||||
"CREATE TEMP TABLE t1 AS SELECT 1 AS id".to_string(),
|
||||
"INSERT INTO audit_log(message) VALUES ('export')".to_string(),
|
||||
];
|
||||
assert!(safe_postgres_temp_setup_sql(&persistent_write).is_none());
|
||||
|
||||
let persistent_index = vec!["CREATE INDEX users_name ON users(name)".to_string()];
|
||||
assert!(safe_postgres_temp_setup_sql(&persistent_index).is_none());
|
||||
}
|
||||
|
||||
fn request(format: &str, row_limit: Option<usize>, total_rows: Option<u64>) -> QueryResultExportRequest {
|
||||
QueryResultExportRequest {
|
||||
export_id: "export-1".to_string(),
|
||||
@@ -1436,6 +1498,7 @@ mod tests {
|
||||
schema: None,
|
||||
sql: "SELECT * FROM users".to_string(),
|
||||
query_base_sql: "SELECT * FROM users".to_string(),
|
||||
setup_sql: Vec::new(),
|
||||
database_type: DatabaseType::Postgres,
|
||||
use_agent_cursor: false,
|
||||
file_path: "out.csv".to_string(),
|
||||
|
||||
Reference in new issue
Block a user