fix(postgres): correct query and connection timeout handling under SSH tunnel

This commit is contained in:
Anubis authored and GitHub committed 2026-08-07 13:45:00 +08:00
1 parent b8144da5f0
commit 143fd96c3f
6 files changed
+287 -52

No files matched your search

+210 -42
View File
@@ -970,9 +970,13 @@ async fn execute_select_prepared(
sql: &str,
start: Instant,
row_limit: usize,
progress_clock: Option<&StreamProgressClock>,
) -> Result<PreparedSelectOutcome, tokio_postgres::Error> {
let prepared_start = Instant::now();
let (stmt, metadata) = prepare_select_with_metadata(client, sql).await?;
if let Some(progress_clock) = progress_clock {
progress_clock.mark();
}
log::info!(
"[postgres][select:prepare_cached:done] elapsed_ms={} total_ms={}",
prepared_start.elapsed().as_millis(),
@@ -986,6 +990,9 @@ async fn execute_select_prepared(
let params: Vec<&(dyn tokio_postgres::types::ToSql + Sync)> = Vec::new();
let query_start = Instant::now();
let stream = client.query_raw(&stmt, params).await?;
if let Some(progress_clock) = progress_clock {
progress_clock.mark();
}
log::info!(
"[postgres][select:query_raw:done] elapsed_ms={} total_ms={} column_count={}",
query_start.elapsed().as_millis(),
@@ -1005,6 +1012,9 @@ async fn execute_select_prepared(
let rows_start = Instant::now();
while let Some(row_result) = stream.next().await {
if let Some(progress_clock) = progress_clock {
progress_clock.mark();
}
if result_rows.len() >= row_limit {
truncated = true;
break;
@@ -1059,8 +1069,12 @@ async fn execute_select_text(
start: Instant,
row_limit: usize,
prepared_column_types: Option<Vec<String>>,
progress_clock: Option<&StreamProgressClock>,
) -> Result<QueryResult, String> {
let stream = client.simple_query_raw(sql).await.map_err(pg_error_to_string)?;
if let Some(progress_clock) = progress_clock {
progress_clock.mark();
}
tokio::pin!(stream);
let mut columns: Vec<String> = Vec::new();
let mut result_rows: Vec<Vec<serde_json::Value>> = Vec::new();
@@ -1076,6 +1090,9 @@ async fn execute_select_text(
let mut truncated = false;
while let Some(message) = stream.next().await {
if let Some(progress_clock) = progress_clock {
progress_clock.mark();
}
match message {
Ok(SimpleQueryMessage::RowDescription(cols)) => {
columns = cols.iter().map(|c| c.name().to_string()).collect();
@@ -1142,6 +1159,7 @@ async fn finish_prepared_select(
start: Instant,
row_limit: usize,
outcome: PreparedSelectOutcome,
progress_clock: Option<&StreamProgressClock>,
) -> Result<QueryResult, String> {
match outcome {
PreparedSelectOutcome::Complete(result) => Ok(result),
@@ -1150,7 +1168,7 @@ async fn finish_prepared_select(
"[postgres][select:text_fallback] unsupported_type={} switching_to=simple_query",
unsupported_type
);
execute_select_text(client, sql, start, row_limit, Some(column_types)).await
execute_select_text(client, sql, start, row_limit, Some(column_types), progress_clock).await
}
}
}
@@ -1161,24 +1179,34 @@ pub(crate) async fn execute_select_query(
start: Instant,
row_limit: usize,
) -> Result<QueryResult, String> {
match execute_select_prepared(client, sql, start, row_limit).await {
Ok(outcome) => finish_prepared_select(client, sql, start, row_limit, outcome).await,
execute_select_query_with_progress(client, sql, start, row_limit, None).await
}
async fn execute_select_query_with_progress(
client: &deadpool_postgres::Client,
sql: &str,
start: Instant,
row_limit: usize,
progress_clock: Option<&StreamProgressClock>,
) -> Result<QueryResult, String> {
match execute_select_prepared(client, sql, start, row_limit, progress_clock).await {
Ok(outcome) => finish_prepared_select(client, sql, start, row_limit, outcome, progress_clock).await,
Err(err) if should_retry_postgres_stale_cache(&err) => {
// The cached prepared statement is stale (e.g. the view or table
// schema changed since the statement was prepared). Evict the
// stale entry and retry with a fresh server-side prepare.
log::warn!("[postgres][select:stale_cache] evicting cached statement: {}", pg_error_to_string(err));
client.statement_cache.remove(sql, &[]);
match execute_select_prepared(client, sql, start, row_limit).await {
Ok(outcome) => finish_prepared_select(client, sql, start, row_limit, outcome).await,
match execute_select_prepared(client, sql, start, row_limit, progress_clock).await {
Ok(outcome) => finish_prepared_select(client, sql, start, row_limit, outcome, progress_clock).await,
Err(err) if should_retry_postgres_text_query(&err) => {
execute_select_text(client, sql, start, row_limit, None).await
execute_select_text(client, sql, start, row_limit, None, progress_clock).await
}
Err(err) => Err(pg_error_to_string(err)),
}
}
Err(err) if should_retry_postgres_text_query(&err) => {
execute_select_text(client, sql, start, row_limit, None).await
execute_select_text(client, sql, start, row_limit, None, progress_clock).await
}
Err(err) => Err(pg_error_to_string(err)),
}
@@ -1467,7 +1495,7 @@ async fn connect_with_optional_local_timezone(
let timeout = super::parse_connect_timeout_with_fallback(url, fallback_timeout);
super::with_connection_timeout("PostgreSQL", timeout, async {
let (pool, client) = super::with_connection_timeout("PostgreSQL", timeout, async {
let pg_config = tokio_postgres::Config::from_str(&postgres_url.url)
.map_err(|e| format!("Invalid PostgreSQL connection URL: {e}"))?;
@@ -1498,19 +1526,33 @@ async fn connect_with_optional_local_timezone(
.build()
.map_err(|e| format!("Failed to create PostgreSQL pool: {e}"))?;
// Verify connectivity and set timezone. Explicit connection options are
// handled by PostgreSQL during startup and must remain strict.
// Verify connectivity. Explicit connection options are handled by
// PostgreSQL during startup and must remain strict.
let client =
pool.get().await.map_err(|e| format!("PostgreSQL connection failed: {}", pg_pool_error_to_string(e)))?;
if !pg_url_has_timezone_setting(url) {
if let Some(timezone) = timezone {
set_automatic_postgres_timezone(&client, timezone).await?;
}
}
Ok(pool)
Ok((pool, client))
})
.await
.await?;
// Creating the physical connection and applying session defaults are two
// sequential network phases. Give each phase the configured connection
// timeout instead of sharing one deadline that can expire during the
// optional SET timezone round-trip on higher-latency tunnels.
if !pg_url_has_timezone_setting(url) {
if let Some(timezone) = timezone {
postgres_session_setup_with_timeout(timeout, set_automatic_postgres_timezone(&client, timezone)).await?;
}
}
drop(client);
Ok(pool)
}
async fn postgres_session_setup_with_timeout<T, F>(timeout: Duration, future: F) -> Result<T, String>
where
F: Future<Output = Result<T, String>>,
{
super::with_connection_timeout("PostgreSQL session setup", timeout, future).await
}
async fn set_automatic_postgres_timezone(client: &deadpool_postgres::Client, timezone: &str) -> Result<(), String> {
@@ -3135,6 +3177,7 @@ pub async fn get_redshift_columns(pool: &Pool, schema: &str, table: &str) -> Res
Instant::now(),
crate::query::MAX_ROWS,
None,
None,
)
.await?;
Ok(redshift_columns_from_query_result(result))
@@ -3312,14 +3355,15 @@ pub async fn execute_query_with_max_rows_and_cancel(
prefer_text_protocol: bool,
) -> Result<QueryResult, String> {
let client = checkout_postgres_client(pool, cancel_token.as_ref(), budget.checkout_timeout).await?;
let pg_cancel_token = client.cancel_token();
wait_postgres_query(
pg_cancel_token,
cancel_context,
execute_postgres_user_query(
&client,
sql,
max_rows,
cancel_token,
budget.query_timeout,
budget.cancel_timeout,
execute_query_with_max_rows_inner(&client, sql, max_rows, prefer_text_protocol),
cancel_context,
prefer_text_protocol,
)
.await
}
@@ -3386,14 +3430,15 @@ pub async fn execute_query_in_read_only_transaction_with_rollback(
execute_postgres_infra_statement(&client, &statement, budget.recycle_timeout, stage).await?;
}
let pg_cancel_token = client.cancel_token();
wait_postgres_query(
pg_cancel_token,
cancel_context,
execute_postgres_user_query(
&client,
sql,
max_rows,
cancel_token,
budget.query_timeout,
budget.cancel_timeout,
execute_query_with_max_rows_inner(&client, sql, max_rows, false),
cancel_context,
false,
)
.await
},
@@ -3534,7 +3579,7 @@ pub async fn execute_query_with_schema_and_max_rows(
"[postgres][execute_with_schema:skip-search-path] total_ms={} reason=transaction-recovery",
start.elapsed().as_millis()
);
return execute_query_with_max_rows_inner(&client, sql, max_rows, false).await;
return execute_query_with_max_rows_inner(&client, sql, max_rows, false, None).await;
}
let set_schema_start = Instant::now();
@@ -3546,7 +3591,7 @@ pub async fn execute_query_with_schema_and_max_rows(
);
let query_start = Instant::now();
let result = execute_query_with_max_rows_inner(&client, sql, max_rows, false).await;
let result = execute_query_with_max_rows_inner(&client, sql, max_rows, false, None).await;
if result.is_ok() {
clear_postgres_caches_after_ddl(pool, Some(&client), sql);
}
@@ -3585,14 +3630,15 @@ pub async fn execute_query_with_schema_and_max_rows_and_cancel(
"[postgres][execute_with_schema:skip-search-path] total_ms={} reason=transaction-recovery",
start.elapsed().as_millis()
);
let pg_cancel_token = client.cancel_token();
return wait_postgres_query(
pg_cancel_token,
cancel_context,
return execute_postgres_user_query(
&client,
sql,
max_rows,
cancel_token,
budget.query_timeout,
budget.cancel_timeout,
execute_query_with_max_rows_inner(&client, sql, max_rows, prefer_text_protocol),
cancel_context,
prefer_text_protocol,
)
.await;
}
@@ -3606,14 +3652,15 @@ pub async fn execute_query_with_schema_and_max_rows_and_cancel(
);
let query_start = Instant::now();
let pg_cancel_token = client.cancel_token();
let result = wait_postgres_query(
pg_cancel_token,
cancel_context,
let result = execute_postgres_user_query(
&client,
sql,
max_rows,
cancel_token,
budget.query_timeout,
budget.cancel_timeout,
execute_query_with_max_rows_inner(&client, sql, max_rows, prefer_text_protocol),
cancel_context,
prefer_text_protocol,
)
.await;
if result.is_ok() {
@@ -3747,6 +3794,52 @@ where
}
}
async fn execute_postgres_user_query(
client: &deadpool_postgres::Client,
sql: &str,
max_rows: Option<usize>,
cancel_token: Option<CancellationToken>,
timeout_duration: Option<Duration>,
cancel_timeout: Duration,
cancel_context: Option<PostgresCancelContext>,
prefer_text_protocol: bool,
) -> Result<QueryResult, String> {
let pg_cancel_token = client.cancel_token();
// Commands do not expose incremental results, so keep their timeout as a
// wall-clock deadline. Row-returning queries may spend longer transferring
// a bounded result through a slow tunnel; for those, the same setting is an
// inactivity budget that resets only after real PostgreSQL progress.
if !postgres_statement_returns_rows(sql) {
return wait_postgres_query(
pg_cancel_token,
cancel_context,
cancel_token,
timeout_duration,
cancel_timeout,
execute_query_with_max_rows_inner(client, sql, max_rows, prefer_text_protocol, None),
)
.await;
}
let timeout_error =
format!("Query timed out after {} seconds", timeout_duration.map_or(0, |timeout| timeout.as_secs()));
let progress_clock = Arc::new(StreamProgressClock::new());
let result = await_stream_with_progress_timeout(
execute_query_with_max_rows_inner(client, sql, max_rows, prefer_text_protocol, Some(progress_clock.clone())),
timeout_duration,
progress_clock,
cancel_token.as_ref(),
timeout_error.clone(),
)
.await;
if result.as_ref().is_err_and(|error| error == &timeout_error || error == crate::query::QUERY_CANCELED) {
cancel_postgres_query(pg_cancel_token, cancel_context.as_ref(), cancel_timeout).await;
}
result
}
/// PostgreSQL pool checkout with timeout and cancel token support.
/// When the checkout phase is stuck, the cancel token can terminate the wait early.
/// The timeout error message includes "checkout timed out" to ensure is_connection_error can classify it correctly.
@@ -3755,6 +3848,7 @@ pub async fn checkout_postgres_client(
cancel_token: Option<&CancellationToken>,
checkout_timeout: Duration,
) -> Result<deadpool_postgres::Object, String> {
let checkout_timeout = effective_postgres_checkout_timeout(pool, checkout_timeout);
let start = Instant::now();
let get_future = async {
tokio::time::timeout(checkout_timeout, pool.get())
@@ -3806,6 +3900,14 @@ pub async fn checkout_postgres_client(
result
}
fn effective_postgres_checkout_timeout(pool: &Pool, requested_timeout: Duration) -> Duration {
let configured = pool.timeouts();
[configured.wait, configured.create, configured.recycle]
.into_iter()
.flatten()
.fold(requested_timeout, std::cmp::max)
}
async fn cancel_postgres_query(
pg_cancel_token: tokio_postgres::CancelToken,
cancel_context: Option<&PostgresCancelContext>,
@@ -3853,15 +3955,16 @@ async fn execute_query_with_max_rows_inner(
sql: &str,
max_rows: Option<usize>,
prefer_text_protocol: bool,
progress_clock: Option<Arc<StreamProgressClock>>,
) -> Result<QueryResult, String> {
let start = Instant::now();
let row_limit = query_result_row_limit(max_rows);
if postgres_statement_returns_rows(sql) {
if prefer_text_protocol {
execute_select_text(client, sql, start, row_limit, None).await
execute_select_text(client, sql, start, row_limit, None, progress_clock.as_deref()).await
} else {
execute_select_query(client, sql, start, row_limit).await
execute_select_query_with_progress(client, sql, start, row_limit, progress_clock.as_deref()).await
}
} else {
let affected = client.execute(sql, &[]).await.map_err(pg_error_to_string)?;
@@ -5894,6 +5997,71 @@ mod tests {
);
}
#[test]
fn postgres_checkout_timeout_respects_pool_configuration() {
let manager = deadpool_postgres::Manager::new(tokio_postgres::Config::new(), NoTls);
let pool = Pool::builder(manager)
.runtime(Runtime::Tokio1)
.wait_timeout(Some(Duration::from_secs(10)))
.create_timeout(Some(Duration::from_secs(10)))
.recycle_timeout(Some(Duration::from_secs(10)))
.build()
.expect("build postgres pool");
assert_eq!(effective_postgres_checkout_timeout(&pool, Duration::from_secs(5)), Duration::from_secs(10));
assert_eq!(effective_postgres_checkout_timeout(&pool, Duration::from_secs(15)), Duration::from_secs(15));
}
#[tokio::test]
async fn postgres_session_setup_reports_its_own_timeout_stage() {
let error =
postgres_session_setup_with_timeout(Duration::from_millis(1), std::future::pending::<Result<(), String>>())
.await
.expect_err("pending session setup should time out");
assert!(error.starts_with("PostgreSQL session setup connection timed out"), "{error}");
}
#[tokio::test]
async fn postgres_row_query_timeout_tracks_inactivity_instead_of_total_duration() {
let progress_clock = Arc::new(StreamProgressClock::new());
let progress_clock_for_query = progress_clock.clone();
let future = async {
for _ in 0..4 {
tokio::time::sleep(Duration::from_millis(40)).await;
progress_clock_for_query.mark();
}
Ok::<_, String>(())
};
let result = await_stream_with_progress_timeout(
future,
Some(Duration::from_millis(100)),
progress_clock,
None,
"query inactivity timeout".to_string(),
)
.await;
assert_eq!(result, Ok(()));
}
#[tokio::test]
async fn postgres_row_query_timeout_still_fires_after_no_progress() {
let progress_clock = Arc::new(StreamProgressClock::new());
let error = await_stream_with_progress_timeout(
std::future::pending::<Result<(), String>>(),
Some(Duration::from_millis(10)),
progress_clock,
None,
"query inactivity timeout".to_string(),
)
.await
.expect_err("a stalled query should time out");
assert_eq!(error, "query inactivity timeout");
}
#[test]
fn postgres_cancel_context_omits_disabled_ssl_mode() {
assert!(build_postgres_cancel_context("postgres://localhost/app?sslmode=disable").is_none());