fix(postgres): correct query and connection timeout handling under SSH tunnel

This commit is contained in:
Anubis authored and GitHub committed 2026-08-07 13:45:00 +08:00
1 parent b8144da5f0
commit 143fd96c3f
6 files changed
+287 -52

No files matched your search

@@ -34,7 +34,7 @@ describe("connectionAttemptTimeout", () => {
expect(connectionAttemptTimeoutMs({ db_type: "access", connect_timeout_secs: 45, transport_layers: [] })).toBe(47_000); expect(connectionAttemptTimeoutMs({ db_type: "access", connect_timeout_secs: 45, transport_layers: [] })).toBe(47_000);
}); });
it("includes HTTP tunnel connection timeout values", () => { it("adds HTTP tunnel setup and tunneled endpoint probe budgets", () => {
expect( expect(
connectionAttemptTimeoutMs({ connectionAttemptTimeoutMs({
db_type: "mysql", db_type: "mysql",
@@ -48,6 +48,25 @@ describe("connectionAttemptTimeout", () => {
}, },
], ],
}), }),
).toBe(37_000);
});
it("adds SSH setup and tunneled endpoint probe budgets", () => {
expect(
connectionAttemptTimeoutMs({
db_type: "postgres",
connect_timeout_secs: 10,
transport_layers: [
{
type: "ssh",
id: "ssh",
host: "bastion.example.com",
port: 22,
user: "dbx",
connect_timeout_secs: 5,
},
],
}),
).toBe(27_000); ).toBe(27_000);
}); });
@@ -81,7 +100,7 @@ describe("connectionAttemptTimeout", () => {
}, },
(profileId) => (profileId === profile.id ? profile : undefined), (profileId) => (profileId === profile.id ? profile : undefined),
), ),
).toBe(42_000); ).toBe(52_000);
}); });
it("keeps disabled shared layers outside the attempt deadline", () => { it("keeps disabled shared layers outside the attempt deadline", () => {
+21
View File
@@ -0,0 +1,21 @@
import { describe, expect, it } from "vitest";
import { frontendQueryTimeoutSecsForSql, queryTimeoutSecsForConnection } from "@/lib/sql/queryTimeout";
describe("queryTimeout", () => {
it("lets PostgreSQL row queries use the backend inactivity timeout", () => {
expect(frontendQueryTimeoutSecsForSql("SELECT * FROM sample_records LIMIT 2000", "postgres", 30)).toBe(0);
expect(frontendQueryTimeoutSecsForSql("/* page */\nWITH rows AS (SELECT 1) SELECT * FROM rows", "postgres", 30)).toBe(0);
expect(frontendQueryTimeoutSecsForSql("UPDATE sample_records SET state = 'ready' RETURNING id", "postgres", 30)).toBe(0);
});
it("keeps the frontend guard for non-row PostgreSQL statements", () => {
expect(frontendQueryTimeoutSecsForSql("UPDATE sample_records SET state = 'ready'", "postgres", 30)).toBe(60);
expect(frontendQueryTimeoutSecsForSql("INSERT INTO sample_records(note) VALUES ('RETURNING is text')", "postgres", 30)).toBe(60);
expect(frontendQueryTimeoutSecsForSql("UPDATE sample_records SET note = 'ready' /* RETURNING */", "postgres", 30)).toBe(60);
});
it("keeps the existing frontend guard for other database types", () => {
expect(frontendQueryTimeoutSecsForSql("SELECT * FROM sample_records LIMIT 2000", "mysql", 30)).toBe(60);
expect(queryTimeoutSecsForConnection({ query_timeout_secs: undefined })).toBe(60);
});
});
@@ -65,16 +65,22 @@ function resolvedTimeoutLayer(layer: TransportLayerConfig, resolveTunnelProfile?
export function connectionAttemptTimeoutMs(config: Pick<ConnectionConfig, "connect_timeout_secs" | "transport_layers"> & Partial<Pick<ConnectionConfig, "db_type">>, resolveTunnelProfile?: TunnelProfileResolver): number { export function connectionAttemptTimeoutMs(config: Pick<ConnectionConfig, "connect_timeout_secs" | "transport_layers"> & Partial<Pick<ConnectionConfig, "db_type">>, resolveTunnelProfile?: TunnelProfileResolver): number {
const baseTimeoutSecs = positiveSeconds(config.connect_timeout_secs, DEFAULT_CONNECT_TIMEOUT_SECS); const baseTimeoutSecs = positiveSeconds(config.connect_timeout_secs, DEFAULT_CONNECT_TIMEOUT_SECS);
const agentMinTimeoutSecs = config.db_type === "access" ? ACCESS_AGENT_MIN_CONNECT_TIMEOUT_SECS : AGENT_DRIVER_MIN_CONNECT_TIMEOUT_SECS; const agentMinTimeoutSecs = config.db_type === "access" ? ACCESS_AGENT_MIN_CONNECT_TIMEOUT_SECS : AGENT_DRIVER_MIN_CONNECT_TIMEOUT_SECS;
const timeouts = [DRIVER_STARTUP_FLOOR_TYPES.has(config.db_type as DatabaseType) ? Math.max(baseTimeoutSecs, agentMinTimeoutSecs) : baseTimeoutSecs]; let timeoutSecs = DRIVER_STARTUP_FLOOR_TYPES.has(config.db_type as DatabaseType) ? Math.max(baseTimeoutSecs, agentMinTimeoutSecs) : baseTimeoutSecs;
let hasEnabledTransportLayer = false;
for (const unresolvedLayer of config.transport_layers ?? []) { for (const unresolvedLayer of config.transport_layers ?? []) {
const layer = resolvedTimeoutLayer(unresolvedLayer, resolveTunnelProfile); const layer = resolvedTimeoutLayer(unresolvedLayer, resolveTunnelProfile);
if (layer.enabled === false) continue; if (layer.enabled === false) continue;
hasEnabledTransportLayer = true;
if (layer.type === "ssh" || layer.type === "http_tunnel") { if (layer.type === "ssh" || layer.type === "http_tunnel") {
timeouts.push(positiveSeconds(layer.connect_timeout_secs, DEFAULT_CONNECT_TIMEOUT_SECS)); timeoutSecs += positiveSeconds(layer.connect_timeout_secs, DEFAULT_CONNECT_TIMEOUT_SECS);
} }
} }
// Transport setup, the tunneled endpoint probe, and the database connection
// happen sequentially in the backend. Keep the UI guard outside their total
// budget so it cannot cancel a connection attempt that is still progressing.
if (hasEnabledTransportLayer) timeoutSecs += baseTimeoutSecs;
const fallbackBuffer = config.db_type === "mongodb" ? MONGO_LEGACY_FALLBACK_TIMEOUT_BUFFER_MS : 0; const fallbackBuffer = config.db_type === "mongodb" ? MONGO_LEGACY_FALLBACK_TIMEOUT_BUFFER_MS : 0;
return Math.ceil(Math.max(...timeouts) * 1000 + CONNECTION_ATTEMPT_TIMEOUT_BUFFER_MS + fallbackBuffer); return Math.ceil(timeoutSecs * 1000 + CONNECTION_ATTEMPT_TIMEOUT_BUFFER_MS + fallbackBuffer);
} }
export function connectionAttemptTimeoutMessage(timeoutMs: number): string { export function connectionAttemptTimeoutMessage(timeoutMs: number): string {
+21
View File
@@ -1,8 +1,12 @@
import { splitSqlStatementRanges } from "@/lib/sql/sqlStatementRanges"; import { splitSqlStatementRanges } from "@/lib/sql/sqlStatementRanges";
import { tokenizeSqlSemantic } from "@/lib/sql/semantic/tokens";
import type { ConnectionConfig, DatabaseType } from "@/types/database"; import type { ConnectionConfig, DatabaseType } from "@/types/database";
export const DEFAULT_QUERY_TIMEOUT_SECS = 60; export const DEFAULT_QUERY_TIMEOUT_SECS = 60;
const POSTGRES_ROW_STATEMENT_KEYWORDS = new Set(["select", "show", "explain", "table", "with"]);
const POSTGRES_RETURNING_STATEMENT_KEYWORDS = new Set(["insert", "update", "delete", "merge"]);
export function queryTimeoutSecsForConnection(connection?: Pick<ConnectionConfig, "query_timeout_secs"> | null): number { export function queryTimeoutSecsForConnection(connection?: Pick<ConnectionConfig, "query_timeout_secs"> | null): number {
const value = Number(connection?.query_timeout_secs); const value = Number(connection?.query_timeout_secs);
return Number.isFinite(value) && value >= 0 ? value : DEFAULT_QUERY_TIMEOUT_SECS; return Number.isFinite(value) && value >= 0 ? value : DEFAULT_QUERY_TIMEOUT_SECS;
@@ -11,7 +15,24 @@ export function queryTimeoutSecsForConnection(connection?: Pick<ConnectionConfig
export function frontendQueryTimeoutSecsForSql(sql: string, databaseType: DatabaseType | undefined, queryTimeoutSecs: number): number { export function frontendQueryTimeoutSecsForSql(sql: string, databaseType: DatabaseType | undefined, queryTimeoutSecs: number): number {
if (queryTimeoutSecs === 0) return 0; if (queryTimeoutSecs === 0) return 0;
// PostgreSQL applies the configured timeout while the backend receives rows
// over a slow SSH tunnel. An absolute browser-side deadline can reject a
// successful, steadily progressing page fetch first, so let the backend own
// the timeout for row-returning PostgreSQL statements.
if (databaseType === "postgres" && postgresQueryMayReturnRows(sql, databaseType)) return 0;
const baseTimeoutSecs = Math.max(queryTimeoutSecs * 2, 60); const baseTimeoutSecs = Math.max(queryTimeoutSecs * 2, 60);
const statementCount = Math.max(splitSqlStatementRanges(sql, databaseType).length, 1); const statementCount = Math.max(splitSqlStatementRanges(sql, databaseType).length, 1);
return baseTimeoutSecs * statementCount; return baseTimeoutSecs * statementCount;
} }
function postgresQueryMayReturnRows(sql: string, databaseType: DatabaseType): boolean {
return splitSqlStatementRanges(sql, databaseType).some(({ sql: statement }) => {
const tokens = tokenizeSqlSemantic(statement, "postgres");
const firstKeyword = tokens.find((token) => token.kind === "word" && token.depth === 0)?.normalized;
if (!firstKeyword) return false;
if (POSTGRES_ROW_STATEMENT_KEYWORDS.has(firstKeyword)) return true;
if (!POSTGRES_RETURNING_STATEMENT_KEYWORDS.has(firstKeyword)) return false;
return tokens.some((token) => token.kind === "word" && token.depth === 0 && token.normalized === "returning");
});
}
@@ -199,7 +199,7 @@ describe("connectionStore timeout recovery", () => {
host: "bastion.example.com", host: "bastion.example.com",
port: 22, port: 22,
user: "dbx", user: "dbx",
connect_timeout_secs: 4, connect_timeout_secs: 1,
}, },
]; ];
const store = useConnectionStore(); const store = useConnectionStore();
@@ -213,7 +213,7 @@ describe("connectionStore timeout recovery", () => {
host: "", host: "",
port: 22, port: 22,
user: "root", user: "root",
connect_timeout_secs: 1, connect_timeout_secs: 4,
}, },
], ],
}); });
@@ -225,13 +225,13 @@ describe("connectionStore timeout recovery", () => {
settled = true; settled = true;
}); });
await vi.advanceTimersByTimeAsync(3001); await vi.advanceTimersByTimeAsync(4999);
expect(settled).toBe(false); expect(settled).toBe(false);
await vi.advanceTimersByTimeAsync(3000); await vi.advanceTimersByTimeAsync(1);
const error = await connect; const error = await connect;
expect(error).toBeInstanceOf(Error); expect(error).toBeInstanceOf(Error);
expect(error.message).toContain("timed out after 6s"); expect(error.message).toContain("timed out after 5s");
resolveConnect(connection.id); resolveConnect(connection.id);
await vi.advanceTimersByTimeAsync(1); await vi.advanceTimersByTimeAsync(1);
+210 -42
View File
@@ -970,9 +970,13 @@ async fn execute_select_prepared(
sql: &str, sql: &str,
start: Instant, start: Instant,
row_limit: usize, row_limit: usize,
progress_clock: Option<&StreamProgressClock>,
) -> Result<PreparedSelectOutcome, tokio_postgres::Error> { ) -> Result<PreparedSelectOutcome, tokio_postgres::Error> {
let prepared_start = Instant::now(); let prepared_start = Instant::now();
let (stmt, metadata) = prepare_select_with_metadata(client, sql).await?; let (stmt, metadata) = prepare_select_with_metadata(client, sql).await?;
if let Some(progress_clock) = progress_clock {
progress_clock.mark();
}
log::info!( log::info!(
"[postgres][select:prepare_cached:done] elapsed_ms={} total_ms={}", "[postgres][select:prepare_cached:done] elapsed_ms={} total_ms={}",
prepared_start.elapsed().as_millis(), prepared_start.elapsed().as_millis(),
@@ -986,6 +990,9 @@ async fn execute_select_prepared(
let params: Vec<&(dyn tokio_postgres::types::ToSql + Sync)> = Vec::new(); let params: Vec<&(dyn tokio_postgres::types::ToSql + Sync)> = Vec::new();
let query_start = Instant::now(); let query_start = Instant::now();
let stream = client.query_raw(&stmt, params).await?; let stream = client.query_raw(&stmt, params).await?;
if let Some(progress_clock) = progress_clock {
progress_clock.mark();
}
log::info!( log::info!(
"[postgres][select:query_raw:done] elapsed_ms={} total_ms={} column_count={}", "[postgres][select:query_raw:done] elapsed_ms={} total_ms={} column_count={}",
query_start.elapsed().as_millis(), query_start.elapsed().as_millis(),
@@ -1005,6 +1012,9 @@ async fn execute_select_prepared(
let rows_start = Instant::now(); let rows_start = Instant::now();
while let Some(row_result) = stream.next().await { while let Some(row_result) = stream.next().await {
if let Some(progress_clock) = progress_clock {
progress_clock.mark();
}
if result_rows.len() >= row_limit { if result_rows.len() >= row_limit {
truncated = true; truncated = true;
break; break;
@@ -1059,8 +1069,12 @@ async fn execute_select_text(
start: Instant, start: Instant,
row_limit: usize, row_limit: usize,
prepared_column_types: Option<Vec<String>>, prepared_column_types: Option<Vec<String>>,
progress_clock: Option<&StreamProgressClock>,
) -> Result<QueryResult, String> { ) -> Result<QueryResult, String> {
let stream = client.simple_query_raw(sql).await.map_err(pg_error_to_string)?; let stream = client.simple_query_raw(sql).await.map_err(pg_error_to_string)?;
if let Some(progress_clock) = progress_clock {
progress_clock.mark();
}
tokio::pin!(stream); tokio::pin!(stream);
let mut columns: Vec<String> = Vec::new(); let mut columns: Vec<String> = Vec::new();
let mut result_rows: Vec<Vec<serde_json::Value>> = Vec::new(); let mut result_rows: Vec<Vec<serde_json::Value>> = Vec::new();
@@ -1076,6 +1090,9 @@ async fn execute_select_text(
let mut truncated = false; let mut truncated = false;
while let Some(message) = stream.next().await { while let Some(message) = stream.next().await {
if let Some(progress_clock) = progress_clock {
progress_clock.mark();
}
match message { match message {
Ok(SimpleQueryMessage::RowDescription(cols)) => { Ok(SimpleQueryMessage::RowDescription(cols)) => {
columns = cols.iter().map(|c| c.name().to_string()).collect(); columns = cols.iter().map(|c| c.name().to_string()).collect();
@@ -1142,6 +1159,7 @@ async fn finish_prepared_select(
start: Instant, start: Instant,
row_limit: usize, row_limit: usize,
outcome: PreparedSelectOutcome, outcome: PreparedSelectOutcome,
progress_clock: Option<&StreamProgressClock>,
) -> Result<QueryResult, String> { ) -> Result<QueryResult, String> {
match outcome { match outcome {
PreparedSelectOutcome::Complete(result) => Ok(result), PreparedSelectOutcome::Complete(result) => Ok(result),
@@ -1150,7 +1168,7 @@ async fn finish_prepared_select(
"[postgres][select:text_fallback] unsupported_type={} switching_to=simple_query", "[postgres][select:text_fallback] unsupported_type={} switching_to=simple_query",
unsupported_type unsupported_type
); );
execute_select_text(client, sql, start, row_limit, Some(column_types)).await execute_select_text(client, sql, start, row_limit, Some(column_types), progress_clock).await
} }
} }
} }
@@ -1161,24 +1179,34 @@ pub(crate) async fn execute_select_query(
start: Instant, start: Instant,
row_limit: usize, row_limit: usize,
) -> Result<QueryResult, String> { ) -> Result<QueryResult, String> {
match execute_select_prepared(client, sql, start, row_limit).await { execute_select_query_with_progress(client, sql, start, row_limit, None).await
Ok(outcome) => finish_prepared_select(client, sql, start, row_limit, outcome).await, }
async fn execute_select_query_with_progress(
client: &deadpool_postgres::Client,
sql: &str,
start: Instant,
row_limit: usize,
progress_clock: Option<&StreamProgressClock>,
) -> Result<QueryResult, String> {
match execute_select_prepared(client, sql, start, row_limit, progress_clock).await {
Ok(outcome) => finish_prepared_select(client, sql, start, row_limit, outcome, progress_clock).await,
Err(err) if should_retry_postgres_stale_cache(&err) => { Err(err) if should_retry_postgres_stale_cache(&err) => {
// The cached prepared statement is stale (e.g. the view or table // The cached prepared statement is stale (e.g. the view or table
// schema changed since the statement was prepared). Evict the // schema changed since the statement was prepared). Evict the
// stale entry and retry with a fresh server-side prepare. // stale entry and retry with a fresh server-side prepare.
log::warn!("[postgres][select:stale_cache] evicting cached statement: {}", pg_error_to_string(err)); log::warn!("[postgres][select:stale_cache] evicting cached statement: {}", pg_error_to_string(err));
client.statement_cache.remove(sql, &[]); client.statement_cache.remove(sql, &[]);
match execute_select_prepared(client, sql, start, row_limit).await { match execute_select_prepared(client, sql, start, row_limit, progress_clock).await {
Ok(outcome) => finish_prepared_select(client, sql, start, row_limit, outcome).await, Ok(outcome) => finish_prepared_select(client, sql, start, row_limit, outcome, progress_clock).await,
Err(err) if should_retry_postgres_text_query(&err) => { Err(err) if should_retry_postgres_text_query(&err) => {
execute_select_text(client, sql, start, row_limit, None).await execute_select_text(client, sql, start, row_limit, None, progress_clock).await
} }
Err(err) => Err(pg_error_to_string(err)), Err(err) => Err(pg_error_to_string(err)),
} }
} }
Err(err) if should_retry_postgres_text_query(&err) => { Err(err) if should_retry_postgres_text_query(&err) => {
execute_select_text(client, sql, start, row_limit, None).await execute_select_text(client, sql, start, row_limit, None, progress_clock).await
} }
Err(err) => Err(pg_error_to_string(err)), Err(err) => Err(pg_error_to_string(err)),
} }
@@ -1467,7 +1495,7 @@ async fn connect_with_optional_local_timezone(
let timeout = super::parse_connect_timeout_with_fallback(url, fallback_timeout); let timeout = super::parse_connect_timeout_with_fallback(url, fallback_timeout);
super::with_connection_timeout("PostgreSQL", timeout, async { let (pool, client) = super::with_connection_timeout("PostgreSQL", timeout, async {
let pg_config = tokio_postgres::Config::from_str(&postgres_url.url) let pg_config = tokio_postgres::Config::from_str(&postgres_url.url)
.map_err(|e| format!("Invalid PostgreSQL connection URL: {e}"))?; .map_err(|e| format!("Invalid PostgreSQL connection URL: {e}"))?;
@@ -1498,19 +1526,33 @@ async fn connect_with_optional_local_timezone(
.build() .build()
.map_err(|e| format!("Failed to create PostgreSQL pool: {e}"))?; .map_err(|e| format!("Failed to create PostgreSQL pool: {e}"))?;
// Verify connectivity and set timezone. Explicit connection options are // Verify connectivity. Explicit connection options are handled by
// handled by PostgreSQL during startup and must remain strict. // PostgreSQL during startup and must remain strict.
let client = let client =
pool.get().await.map_err(|e| format!("PostgreSQL connection failed: {}", pg_pool_error_to_string(e)))?; pool.get().await.map_err(|e| format!("PostgreSQL connection failed: {}", pg_pool_error_to_string(e)))?;
if !pg_url_has_timezone_setting(url) { Ok((pool, client))
if let Some(timezone) = timezone {
set_automatic_postgres_timezone(&client, timezone).await?;
}
}
Ok(pool)
}) })
.await .await?;
// Creating the physical connection and applying session defaults are two
// sequential network phases. Give each phase the configured connection
// timeout instead of sharing one deadline that can expire during the
// optional SET timezone round-trip on higher-latency tunnels.
if !pg_url_has_timezone_setting(url) {
if let Some(timezone) = timezone {
postgres_session_setup_with_timeout(timeout, set_automatic_postgres_timezone(&client, timezone)).await?;
}
}
drop(client);
Ok(pool)
}
async fn postgres_session_setup_with_timeout<T, F>(timeout: Duration, future: F) -> Result<T, String>
where
F: Future<Output = Result<T, String>>,
{
super::with_connection_timeout("PostgreSQL session setup", timeout, future).await
} }
async fn set_automatic_postgres_timezone(client: &deadpool_postgres::Client, timezone: &str) -> Result<(), String> { async fn set_automatic_postgres_timezone(client: &deadpool_postgres::Client, timezone: &str) -> Result<(), String> {
@@ -3135,6 +3177,7 @@ pub async fn get_redshift_columns(pool: &Pool, schema: &str, table: &str) -> Res
Instant::now(), Instant::now(),
crate::query::MAX_ROWS, crate::query::MAX_ROWS,
None, None,
None,
) )
.await?; .await?;
Ok(redshift_columns_from_query_result(result)) Ok(redshift_columns_from_query_result(result))
@@ -3312,14 +3355,15 @@ pub async fn execute_query_with_max_rows_and_cancel(
prefer_text_protocol: bool, prefer_text_protocol: bool,
) -> Result<QueryResult, String> { ) -> Result<QueryResult, String> {
let client = checkout_postgres_client(pool, cancel_token.as_ref(), budget.checkout_timeout).await?; let client = checkout_postgres_client(pool, cancel_token.as_ref(), budget.checkout_timeout).await?;
let pg_cancel_token = client.cancel_token(); execute_postgres_user_query(
wait_postgres_query( &client,
pg_cancel_token, sql,
cancel_context, max_rows,
cancel_token, cancel_token,
budget.query_timeout, budget.query_timeout,
budget.cancel_timeout, budget.cancel_timeout,
execute_query_with_max_rows_inner(&client, sql, max_rows, prefer_text_protocol), cancel_context,
prefer_text_protocol,
) )
.await .await
} }
@@ -3386,14 +3430,15 @@ pub async fn execute_query_in_read_only_transaction_with_rollback(
execute_postgres_infra_statement(&client, &statement, budget.recycle_timeout, stage).await?; execute_postgres_infra_statement(&client, &statement, budget.recycle_timeout, stage).await?;
} }
let pg_cancel_token = client.cancel_token(); execute_postgres_user_query(
wait_postgres_query( &client,
pg_cancel_token, sql,
cancel_context, max_rows,
cancel_token, cancel_token,
budget.query_timeout, budget.query_timeout,
budget.cancel_timeout, budget.cancel_timeout,
execute_query_with_max_rows_inner(&client, sql, max_rows, false), cancel_context,
false,
) )
.await .await
}, },
@@ -3534,7 +3579,7 @@ pub async fn execute_query_with_schema_and_max_rows(
"[postgres][execute_with_schema:skip-search-path] total_ms={} reason=transaction-recovery", "[postgres][execute_with_schema:skip-search-path] total_ms={} reason=transaction-recovery",
start.elapsed().as_millis() start.elapsed().as_millis()
); );
return execute_query_with_max_rows_inner(&client, sql, max_rows, false).await; return execute_query_with_max_rows_inner(&client, sql, max_rows, false, None).await;
} }
let set_schema_start = Instant::now(); let set_schema_start = Instant::now();
@@ -3546,7 +3591,7 @@ pub async fn execute_query_with_schema_and_max_rows(
); );
let query_start = Instant::now(); let query_start = Instant::now();
let result = execute_query_with_max_rows_inner(&client, sql, max_rows, false).await; let result = execute_query_with_max_rows_inner(&client, sql, max_rows, false, None).await;
if result.is_ok() { if result.is_ok() {
clear_postgres_caches_after_ddl(pool, Some(&client), sql); clear_postgres_caches_after_ddl(pool, Some(&client), sql);
} }
@@ -3585,14 +3630,15 @@ pub async fn execute_query_with_schema_and_max_rows_and_cancel(
"[postgres][execute_with_schema:skip-search-path] total_ms={} reason=transaction-recovery", "[postgres][execute_with_schema:skip-search-path] total_ms={} reason=transaction-recovery",
start.elapsed().as_millis() start.elapsed().as_millis()
); );
let pg_cancel_token = client.cancel_token(); return execute_postgres_user_query(
return wait_postgres_query( &client,
pg_cancel_token, sql,
cancel_context, max_rows,
cancel_token, cancel_token,
budget.query_timeout, budget.query_timeout,
budget.cancel_timeout, budget.cancel_timeout,
execute_query_with_max_rows_inner(&client, sql, max_rows, prefer_text_protocol), cancel_context,
prefer_text_protocol,
) )
.await; .await;
} }
@@ -3606,14 +3652,15 @@ pub async fn execute_query_with_schema_and_max_rows_and_cancel(
); );
let query_start = Instant::now(); let query_start = Instant::now();
let pg_cancel_token = client.cancel_token(); let result = execute_postgres_user_query(
let result = wait_postgres_query( &client,
pg_cancel_token, sql,
cancel_context, max_rows,
cancel_token, cancel_token,
budget.query_timeout, budget.query_timeout,
budget.cancel_timeout, budget.cancel_timeout,
execute_query_with_max_rows_inner(&client, sql, max_rows, prefer_text_protocol), cancel_context,
prefer_text_protocol,
) )
.await; .await;
if result.is_ok() { if result.is_ok() {
@@ -3747,6 +3794,52 @@ where
} }
} }
async fn execute_postgres_user_query(
client: &deadpool_postgres::Client,
sql: &str,
max_rows: Option<usize>,
cancel_token: Option<CancellationToken>,
timeout_duration: Option<Duration>,
cancel_timeout: Duration,
cancel_context: Option<PostgresCancelContext>,
prefer_text_protocol: bool,
) -> Result<QueryResult, String> {
let pg_cancel_token = client.cancel_token();
// Commands do not expose incremental results, so keep their timeout as a
// wall-clock deadline. Row-returning queries may spend longer transferring
// a bounded result through a slow tunnel; for those, the same setting is an
// inactivity budget that resets only after real PostgreSQL progress.
if !postgres_statement_returns_rows(sql) {
return wait_postgres_query(
pg_cancel_token,
cancel_context,
cancel_token,
timeout_duration,
cancel_timeout,
execute_query_with_max_rows_inner(client, sql, max_rows, prefer_text_protocol, None),
)
.await;
}
let timeout_error =
format!("Query timed out after {} seconds", timeout_duration.map_or(0, |timeout| timeout.as_secs()));
let progress_clock = Arc::new(StreamProgressClock::new());
let result = await_stream_with_progress_timeout(
execute_query_with_max_rows_inner(client, sql, max_rows, prefer_text_protocol, Some(progress_clock.clone())),
timeout_duration,
progress_clock,
cancel_token.as_ref(),
timeout_error.clone(),
)
.await;
if result.as_ref().is_err_and(|error| error == &timeout_error || error == crate::query::QUERY_CANCELED) {
cancel_postgres_query(pg_cancel_token, cancel_context.as_ref(), cancel_timeout).await;
}
result
}
/// PostgreSQL pool checkout with timeout and cancel token support. /// PostgreSQL pool checkout with timeout and cancel token support.
/// When the checkout phase is stuck, the cancel token can terminate the wait early. /// When the checkout phase is stuck, the cancel token can terminate the wait early.
/// The timeout error message includes "checkout timed out" to ensure is_connection_error can classify it correctly. /// The timeout error message includes "checkout timed out" to ensure is_connection_error can classify it correctly.
@@ -3755,6 +3848,7 @@ pub async fn checkout_postgres_client(
cancel_token: Option<&CancellationToken>, cancel_token: Option<&CancellationToken>,
checkout_timeout: Duration, checkout_timeout: Duration,
) -> Result<deadpool_postgres::Object, String> { ) -> Result<deadpool_postgres::Object, String> {
let checkout_timeout = effective_postgres_checkout_timeout(pool, checkout_timeout);
let start = Instant::now(); let start = Instant::now();
let get_future = async { let get_future = async {
tokio::time::timeout(checkout_timeout, pool.get()) tokio::time::timeout(checkout_timeout, pool.get())
@@ -3806,6 +3900,14 @@ pub async fn checkout_postgres_client(
result result
} }
fn effective_postgres_checkout_timeout(pool: &Pool, requested_timeout: Duration) -> Duration {
let configured = pool.timeouts();
[configured.wait, configured.create, configured.recycle]
.into_iter()
.flatten()
.fold(requested_timeout, std::cmp::max)
}
async fn cancel_postgres_query( async fn cancel_postgres_query(
pg_cancel_token: tokio_postgres::CancelToken, pg_cancel_token: tokio_postgres::CancelToken,
cancel_context: Option<&PostgresCancelContext>, cancel_context: Option<&PostgresCancelContext>,
@@ -3853,15 +3955,16 @@ async fn execute_query_with_max_rows_inner(
sql: &str, sql: &str,
max_rows: Option<usize>, max_rows: Option<usize>,
prefer_text_protocol: bool, prefer_text_protocol: bool,
progress_clock: Option<Arc<StreamProgressClock>>,
) -> Result<QueryResult, String> { ) -> Result<QueryResult, String> {
let start = Instant::now(); let start = Instant::now();
let row_limit = query_result_row_limit(max_rows); let row_limit = query_result_row_limit(max_rows);
if postgres_statement_returns_rows(sql) { if postgres_statement_returns_rows(sql) {
if prefer_text_protocol { if prefer_text_protocol {
execute_select_text(client, sql, start, row_limit, None).await execute_select_text(client, sql, start, row_limit, None, progress_clock.as_deref()).await
} else { } else {
execute_select_query(client, sql, start, row_limit).await execute_select_query_with_progress(client, sql, start, row_limit, progress_clock.as_deref()).await
} }
} else { } else {
let affected = client.execute(sql, &[]).await.map_err(pg_error_to_string)?; let affected = client.execute(sql, &[]).await.map_err(pg_error_to_string)?;
@@ -5894,6 +5997,71 @@ mod tests {
); );
} }
#[test]
fn postgres_checkout_timeout_respects_pool_configuration() {
let manager = deadpool_postgres::Manager::new(tokio_postgres::Config::new(), NoTls);
let pool = Pool::builder(manager)
.runtime(Runtime::Tokio1)
.wait_timeout(Some(Duration::from_secs(10)))
.create_timeout(Some(Duration::from_secs(10)))
.recycle_timeout(Some(Duration::from_secs(10)))
.build()
.expect("build postgres pool");
assert_eq!(effective_postgres_checkout_timeout(&pool, Duration::from_secs(5)), Duration::from_secs(10));
assert_eq!(effective_postgres_checkout_timeout(&pool, Duration::from_secs(15)), Duration::from_secs(15));
}
#[tokio::test]
async fn postgres_session_setup_reports_its_own_timeout_stage() {
let error =
postgres_session_setup_with_timeout(Duration::from_millis(1), std::future::pending::<Result<(), String>>())
.await
.expect_err("pending session setup should time out");
assert!(error.starts_with("PostgreSQL session setup connection timed out"), "{error}");
}
#[tokio::test]
async fn postgres_row_query_timeout_tracks_inactivity_instead_of_total_duration() {
let progress_clock = Arc::new(StreamProgressClock::new());
let progress_clock_for_query = progress_clock.clone();
let future = async {
for _ in 0..4 {
tokio::time::sleep(Duration::from_millis(40)).await;
progress_clock_for_query.mark();
}
Ok::<_, String>(())
};
let result = await_stream_with_progress_timeout(
future,
Some(Duration::from_millis(100)),
progress_clock,
None,
"query inactivity timeout".to_string(),
)
.await;
assert_eq!(result, Ok(()));
}
#[tokio::test]
async fn postgres_row_query_timeout_still_fires_after_no_progress() {
let progress_clock = Arc::new(StreamProgressClock::new());
let error = await_stream_with_progress_timeout(
std::future::pending::<Result<(), String>>(),
Some(Duration::from_millis(10)),
progress_clock,
None,
"query inactivity timeout".to_string(),
)
.await
.expect_err("a stalled query should time out");
assert_eq!(error, "query inactivity timeout");
}
#[test] #[test]
fn postgres_cancel_context_omits_disabled_ssl_mode() { fn postgres_cancel_context_omits_disabled_ssl_mode() {
assert!(build_postgres_cancel_context("postgres://localhost/app?sslmode=disable").is_none()); assert!(build_postgres_cancel_context("postgres://localhost/app?sslmode=disable").is_none());