fix(postgres): correct query and connection timeout handling under SSH tunnel
This commit is contained in:
1 parent
b8144da5f0
commit
143fd96c3f
6 files changed
+287
-52
No files matched your search
+21
-2
@@ -34,7 +34,7 @@ describe("connectionAttemptTimeout", () => {
|
|||||||
expect(connectionAttemptTimeoutMs({ db_type: "access", connect_timeout_secs: 45, transport_layers: [] })).toBe(47_000);
|
expect(connectionAttemptTimeoutMs({ db_type: "access", connect_timeout_secs: 45, transport_layers: [] })).toBe(47_000);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("includes HTTP tunnel connection timeout values", () => {
|
it("adds HTTP tunnel setup and tunneled endpoint probe budgets", () => {
|
||||||
expect(
|
expect(
|
||||||
connectionAttemptTimeoutMs({
|
connectionAttemptTimeoutMs({
|
||||||
db_type: "mysql",
|
db_type: "mysql",
|
||||||
@@ -48,6 +48,25 @@ describe("connectionAttemptTimeout", () => {
|
|||||||
},
|
},
|
||||||
],
|
],
|
||||||
}),
|
}),
|
||||||
|
).toBe(37_000);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("adds SSH setup and tunneled endpoint probe budgets", () => {
|
||||||
|
expect(
|
||||||
|
connectionAttemptTimeoutMs({
|
||||||
|
db_type: "postgres",
|
||||||
|
connect_timeout_secs: 10,
|
||||||
|
transport_layers: [
|
||||||
|
{
|
||||||
|
type: "ssh",
|
||||||
|
id: "ssh",
|
||||||
|
host: "bastion.example.com",
|
||||||
|
port: 22,
|
||||||
|
user: "dbx",
|
||||||
|
connect_timeout_secs: 5,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}),
|
||||||
).toBe(27_000);
|
).toBe(27_000);
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -81,7 +100,7 @@ describe("connectionAttemptTimeout", () => {
|
|||||||
},
|
},
|
||||||
(profileId) => (profileId === profile.id ? profile : undefined),
|
(profileId) => (profileId === profile.id ? profile : undefined),
|
||||||
),
|
),
|
||||||
).toBe(42_000);
|
).toBe(52_000);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("keeps disabled shared layers outside the attempt deadline", () => {
|
it("keeps disabled shared layers outside the attempt deadline", () => {
|
||||||
|
|||||||
@@ -0,0 +1,21 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { frontendQueryTimeoutSecsForSql, queryTimeoutSecsForConnection } from "@/lib/sql/queryTimeout";
|
||||||
|
|
||||||
|
describe("queryTimeout", () => {
|
||||||
|
it("lets PostgreSQL row queries use the backend inactivity timeout", () => {
|
||||||
|
expect(frontendQueryTimeoutSecsForSql("SELECT * FROM sample_records LIMIT 2000", "postgres", 30)).toBe(0);
|
||||||
|
expect(frontendQueryTimeoutSecsForSql("/* page */\nWITH rows AS (SELECT 1) SELECT * FROM rows", "postgres", 30)).toBe(0);
|
||||||
|
expect(frontendQueryTimeoutSecsForSql("UPDATE sample_records SET state = 'ready' RETURNING id", "postgres", 30)).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps the frontend guard for non-row PostgreSQL statements", () => {
|
||||||
|
expect(frontendQueryTimeoutSecsForSql("UPDATE sample_records SET state = 'ready'", "postgres", 30)).toBe(60);
|
||||||
|
expect(frontendQueryTimeoutSecsForSql("INSERT INTO sample_records(note) VALUES ('RETURNING is text')", "postgres", 30)).toBe(60);
|
||||||
|
expect(frontendQueryTimeoutSecsForSql("UPDATE sample_records SET note = 'ready' /* RETURNING */", "postgres", 30)).toBe(60);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps the existing frontend guard for other database types", () => {
|
||||||
|
expect(frontendQueryTimeoutSecsForSql("SELECT * FROM sample_records LIMIT 2000", "mysql", 30)).toBe(60);
|
||||||
|
expect(queryTimeoutSecsForConnection({ query_timeout_secs: undefined })).toBe(60);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -65,16 +65,22 @@ function resolvedTimeoutLayer(layer: TransportLayerConfig, resolveTunnelProfile?
|
|||||||
export function connectionAttemptTimeoutMs(config: Pick<ConnectionConfig, "connect_timeout_secs" | "transport_layers"> & Partial<Pick<ConnectionConfig, "db_type">>, resolveTunnelProfile?: TunnelProfileResolver): number {
|
export function connectionAttemptTimeoutMs(config: Pick<ConnectionConfig, "connect_timeout_secs" | "transport_layers"> & Partial<Pick<ConnectionConfig, "db_type">>, resolveTunnelProfile?: TunnelProfileResolver): number {
|
||||||
const baseTimeoutSecs = positiveSeconds(config.connect_timeout_secs, DEFAULT_CONNECT_TIMEOUT_SECS);
|
const baseTimeoutSecs = positiveSeconds(config.connect_timeout_secs, DEFAULT_CONNECT_TIMEOUT_SECS);
|
||||||
const agentMinTimeoutSecs = config.db_type === "access" ? ACCESS_AGENT_MIN_CONNECT_TIMEOUT_SECS : AGENT_DRIVER_MIN_CONNECT_TIMEOUT_SECS;
|
const agentMinTimeoutSecs = config.db_type === "access" ? ACCESS_AGENT_MIN_CONNECT_TIMEOUT_SECS : AGENT_DRIVER_MIN_CONNECT_TIMEOUT_SECS;
|
||||||
const timeouts = [DRIVER_STARTUP_FLOOR_TYPES.has(config.db_type as DatabaseType) ? Math.max(baseTimeoutSecs, agentMinTimeoutSecs) : baseTimeoutSecs];
|
let timeoutSecs = DRIVER_STARTUP_FLOOR_TYPES.has(config.db_type as DatabaseType) ? Math.max(baseTimeoutSecs, agentMinTimeoutSecs) : baseTimeoutSecs;
|
||||||
|
let hasEnabledTransportLayer = false;
|
||||||
for (const unresolvedLayer of config.transport_layers ?? []) {
|
for (const unresolvedLayer of config.transport_layers ?? []) {
|
||||||
const layer = resolvedTimeoutLayer(unresolvedLayer, resolveTunnelProfile);
|
const layer = resolvedTimeoutLayer(unresolvedLayer, resolveTunnelProfile);
|
||||||
if (layer.enabled === false) continue;
|
if (layer.enabled === false) continue;
|
||||||
|
hasEnabledTransportLayer = true;
|
||||||
if (layer.type === "ssh" || layer.type === "http_tunnel") {
|
if (layer.type === "ssh" || layer.type === "http_tunnel") {
|
||||||
timeouts.push(positiveSeconds(layer.connect_timeout_secs, DEFAULT_CONNECT_TIMEOUT_SECS));
|
timeoutSecs += positiveSeconds(layer.connect_timeout_secs, DEFAULT_CONNECT_TIMEOUT_SECS);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Transport setup, the tunneled endpoint probe, and the database connection
|
||||||
|
// happen sequentially in the backend. Keep the UI guard outside their total
|
||||||
|
// budget so it cannot cancel a connection attempt that is still progressing.
|
||||||
|
if (hasEnabledTransportLayer) timeoutSecs += baseTimeoutSecs;
|
||||||
const fallbackBuffer = config.db_type === "mongodb" ? MONGO_LEGACY_FALLBACK_TIMEOUT_BUFFER_MS : 0;
|
const fallbackBuffer = config.db_type === "mongodb" ? MONGO_LEGACY_FALLBACK_TIMEOUT_BUFFER_MS : 0;
|
||||||
return Math.ceil(Math.max(...timeouts) * 1000 + CONNECTION_ATTEMPT_TIMEOUT_BUFFER_MS + fallbackBuffer);
|
return Math.ceil(timeoutSecs * 1000 + CONNECTION_ATTEMPT_TIMEOUT_BUFFER_MS + fallbackBuffer);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function connectionAttemptTimeoutMessage(timeoutMs: number): string {
|
export function connectionAttemptTimeoutMessage(timeoutMs: number): string {
|
||||||
|
|||||||
@@ -1,8 +1,12 @@
|
|||||||
import { splitSqlStatementRanges } from "@/lib/sql/sqlStatementRanges";
|
import { splitSqlStatementRanges } from "@/lib/sql/sqlStatementRanges";
|
||||||
|
import { tokenizeSqlSemantic } from "@/lib/sql/semantic/tokens";
|
||||||
import type { ConnectionConfig, DatabaseType } from "@/types/database";
|
import type { ConnectionConfig, DatabaseType } from "@/types/database";
|
||||||
|
|
||||||
export const DEFAULT_QUERY_TIMEOUT_SECS = 60;
|
export const DEFAULT_QUERY_TIMEOUT_SECS = 60;
|
||||||
|
|
||||||
|
const POSTGRES_ROW_STATEMENT_KEYWORDS = new Set(["select", "show", "explain", "table", "with"]);
|
||||||
|
const POSTGRES_RETURNING_STATEMENT_KEYWORDS = new Set(["insert", "update", "delete", "merge"]);
|
||||||
|
|
||||||
export function queryTimeoutSecsForConnection(connection?: Pick<ConnectionConfig, "query_timeout_secs"> | null): number {
|
export function queryTimeoutSecsForConnection(connection?: Pick<ConnectionConfig, "query_timeout_secs"> | null): number {
|
||||||
const value = Number(connection?.query_timeout_secs);
|
const value = Number(connection?.query_timeout_secs);
|
||||||
return Number.isFinite(value) && value >= 0 ? value : DEFAULT_QUERY_TIMEOUT_SECS;
|
return Number.isFinite(value) && value >= 0 ? value : DEFAULT_QUERY_TIMEOUT_SECS;
|
||||||
@@ -11,7 +15,24 @@ export function queryTimeoutSecsForConnection(connection?: Pick<ConnectionConfig
|
|||||||
export function frontendQueryTimeoutSecsForSql(sql: string, databaseType: DatabaseType | undefined, queryTimeoutSecs: number): number {
|
export function frontendQueryTimeoutSecsForSql(sql: string, databaseType: DatabaseType | undefined, queryTimeoutSecs: number): number {
|
||||||
if (queryTimeoutSecs === 0) return 0;
|
if (queryTimeoutSecs === 0) return 0;
|
||||||
|
|
||||||
|
// PostgreSQL applies the configured timeout while the backend receives rows
|
||||||
|
// over a slow SSH tunnel. An absolute browser-side deadline can reject a
|
||||||
|
// successful, steadily progressing page fetch first, so let the backend own
|
||||||
|
// the timeout for row-returning PostgreSQL statements.
|
||||||
|
if (databaseType === "postgres" && postgresQueryMayReturnRows(sql, databaseType)) return 0;
|
||||||
|
|
||||||
const baseTimeoutSecs = Math.max(queryTimeoutSecs * 2, 60);
|
const baseTimeoutSecs = Math.max(queryTimeoutSecs * 2, 60);
|
||||||
const statementCount = Math.max(splitSqlStatementRanges(sql, databaseType).length, 1);
|
const statementCount = Math.max(splitSqlStatementRanges(sql, databaseType).length, 1);
|
||||||
return baseTimeoutSecs * statementCount;
|
return baseTimeoutSecs * statementCount;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function postgresQueryMayReturnRows(sql: string, databaseType: DatabaseType): boolean {
|
||||||
|
return splitSqlStatementRanges(sql, databaseType).some(({ sql: statement }) => {
|
||||||
|
const tokens = tokenizeSqlSemantic(statement, "postgres");
|
||||||
|
const firstKeyword = tokens.find((token) => token.kind === "word" && token.depth === 0)?.normalized;
|
||||||
|
if (!firstKeyword) return false;
|
||||||
|
if (POSTGRES_ROW_STATEMENT_KEYWORDS.has(firstKeyword)) return true;
|
||||||
|
if (!POSTGRES_RETURNING_STATEMENT_KEYWORDS.has(firstKeyword)) return false;
|
||||||
|
return tokens.some((token) => token.kind === "word" && token.depth === 0 && token.normalized === "returning");
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -199,7 +199,7 @@ describe("connectionStore timeout recovery", () => {
|
|||||||
host: "bastion.example.com",
|
host: "bastion.example.com",
|
||||||
port: 22,
|
port: 22,
|
||||||
user: "dbx",
|
user: "dbx",
|
||||||
connect_timeout_secs: 4,
|
connect_timeout_secs: 1,
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
const store = useConnectionStore();
|
const store = useConnectionStore();
|
||||||
@@ -213,7 +213,7 @@ describe("connectionStore timeout recovery", () => {
|
|||||||
host: "",
|
host: "",
|
||||||
port: 22,
|
port: 22,
|
||||||
user: "root",
|
user: "root",
|
||||||
connect_timeout_secs: 1,
|
connect_timeout_secs: 4,
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
});
|
});
|
||||||
@@ -225,13 +225,13 @@ describe("connectionStore timeout recovery", () => {
|
|||||||
settled = true;
|
settled = true;
|
||||||
});
|
});
|
||||||
|
|
||||||
await vi.advanceTimersByTimeAsync(3001);
|
await vi.advanceTimersByTimeAsync(4999);
|
||||||
expect(settled).toBe(false);
|
expect(settled).toBe(false);
|
||||||
|
|
||||||
await vi.advanceTimersByTimeAsync(3000);
|
await vi.advanceTimersByTimeAsync(1);
|
||||||
const error = await connect;
|
const error = await connect;
|
||||||
expect(error).toBeInstanceOf(Error);
|
expect(error).toBeInstanceOf(Error);
|
||||||
expect(error.message).toContain("timed out after 6s");
|
expect(error.message).toContain("timed out after 5s");
|
||||||
|
|
||||||
resolveConnect(connection.id);
|
resolveConnect(connection.id);
|
||||||
await vi.advanceTimersByTimeAsync(1);
|
await vi.advanceTimersByTimeAsync(1);
|
||||||
|
|||||||
@@ -970,9 +970,13 @@ async fn execute_select_prepared(
|
|||||||
sql: &str,
|
sql: &str,
|
||||||
start: Instant,
|
start: Instant,
|
||||||
row_limit: usize,
|
row_limit: usize,
|
||||||
|
progress_clock: Option<&StreamProgressClock>,
|
||||||
) -> Result<PreparedSelectOutcome, tokio_postgres::Error> {
|
) -> Result<PreparedSelectOutcome, tokio_postgres::Error> {
|
||||||
let prepared_start = Instant::now();
|
let prepared_start = Instant::now();
|
||||||
let (stmt, metadata) = prepare_select_with_metadata(client, sql).await?;
|
let (stmt, metadata) = prepare_select_with_metadata(client, sql).await?;
|
||||||
|
if let Some(progress_clock) = progress_clock {
|
||||||
|
progress_clock.mark();
|
||||||
|
}
|
||||||
log::info!(
|
log::info!(
|
||||||
"[postgres][select:prepare_cached:done] elapsed_ms={} total_ms={}",
|
"[postgres][select:prepare_cached:done] elapsed_ms={} total_ms={}",
|
||||||
prepared_start.elapsed().as_millis(),
|
prepared_start.elapsed().as_millis(),
|
||||||
@@ -986,6 +990,9 @@ async fn execute_select_prepared(
|
|||||||
let params: Vec<&(dyn tokio_postgres::types::ToSql + Sync)> = Vec::new();
|
let params: Vec<&(dyn tokio_postgres::types::ToSql + Sync)> = Vec::new();
|
||||||
let query_start = Instant::now();
|
let query_start = Instant::now();
|
||||||
let stream = client.query_raw(&stmt, params).await?;
|
let stream = client.query_raw(&stmt, params).await?;
|
||||||
|
if let Some(progress_clock) = progress_clock {
|
||||||
|
progress_clock.mark();
|
||||||
|
}
|
||||||
log::info!(
|
log::info!(
|
||||||
"[postgres][select:query_raw:done] elapsed_ms={} total_ms={} column_count={}",
|
"[postgres][select:query_raw:done] elapsed_ms={} total_ms={} column_count={}",
|
||||||
query_start.elapsed().as_millis(),
|
query_start.elapsed().as_millis(),
|
||||||
@@ -1005,6 +1012,9 @@ async fn execute_select_prepared(
|
|||||||
|
|
||||||
let rows_start = Instant::now();
|
let rows_start = Instant::now();
|
||||||
while let Some(row_result) = stream.next().await {
|
while let Some(row_result) = stream.next().await {
|
||||||
|
if let Some(progress_clock) = progress_clock {
|
||||||
|
progress_clock.mark();
|
||||||
|
}
|
||||||
if result_rows.len() >= row_limit {
|
if result_rows.len() >= row_limit {
|
||||||
truncated = true;
|
truncated = true;
|
||||||
break;
|
break;
|
||||||
@@ -1059,8 +1069,12 @@ async fn execute_select_text(
|
|||||||
start: Instant,
|
start: Instant,
|
||||||
row_limit: usize,
|
row_limit: usize,
|
||||||
prepared_column_types: Option<Vec<String>>,
|
prepared_column_types: Option<Vec<String>>,
|
||||||
|
progress_clock: Option<&StreamProgressClock>,
|
||||||
) -> Result<QueryResult, String> {
|
) -> Result<QueryResult, String> {
|
||||||
let stream = client.simple_query_raw(sql).await.map_err(pg_error_to_string)?;
|
let stream = client.simple_query_raw(sql).await.map_err(pg_error_to_string)?;
|
||||||
|
if let Some(progress_clock) = progress_clock {
|
||||||
|
progress_clock.mark();
|
||||||
|
}
|
||||||
tokio::pin!(stream);
|
tokio::pin!(stream);
|
||||||
let mut columns: Vec<String> = Vec::new();
|
let mut columns: Vec<String> = Vec::new();
|
||||||
let mut result_rows: Vec<Vec<serde_json::Value>> = Vec::new();
|
let mut result_rows: Vec<Vec<serde_json::Value>> = Vec::new();
|
||||||
@@ -1076,6 +1090,9 @@ async fn execute_select_text(
|
|||||||
let mut truncated = false;
|
let mut truncated = false;
|
||||||
|
|
||||||
while let Some(message) = stream.next().await {
|
while let Some(message) = stream.next().await {
|
||||||
|
if let Some(progress_clock) = progress_clock {
|
||||||
|
progress_clock.mark();
|
||||||
|
}
|
||||||
match message {
|
match message {
|
||||||
Ok(SimpleQueryMessage::RowDescription(cols)) => {
|
Ok(SimpleQueryMessage::RowDescription(cols)) => {
|
||||||
columns = cols.iter().map(|c| c.name().to_string()).collect();
|
columns = cols.iter().map(|c| c.name().to_string()).collect();
|
||||||
@@ -1142,6 +1159,7 @@ async fn finish_prepared_select(
|
|||||||
start: Instant,
|
start: Instant,
|
||||||
row_limit: usize,
|
row_limit: usize,
|
||||||
outcome: PreparedSelectOutcome,
|
outcome: PreparedSelectOutcome,
|
||||||
|
progress_clock: Option<&StreamProgressClock>,
|
||||||
) -> Result<QueryResult, String> {
|
) -> Result<QueryResult, String> {
|
||||||
match outcome {
|
match outcome {
|
||||||
PreparedSelectOutcome::Complete(result) => Ok(result),
|
PreparedSelectOutcome::Complete(result) => Ok(result),
|
||||||
@@ -1150,7 +1168,7 @@ async fn finish_prepared_select(
|
|||||||
"[postgres][select:text_fallback] unsupported_type={} switching_to=simple_query",
|
"[postgres][select:text_fallback] unsupported_type={} switching_to=simple_query",
|
||||||
unsupported_type
|
unsupported_type
|
||||||
);
|
);
|
||||||
execute_select_text(client, sql, start, row_limit, Some(column_types)).await
|
execute_select_text(client, sql, start, row_limit, Some(column_types), progress_clock).await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1161,24 +1179,34 @@ pub(crate) async fn execute_select_query(
|
|||||||
start: Instant,
|
start: Instant,
|
||||||
row_limit: usize,
|
row_limit: usize,
|
||||||
) -> Result<QueryResult, String> {
|
) -> Result<QueryResult, String> {
|
||||||
match execute_select_prepared(client, sql, start, row_limit).await {
|
execute_select_query_with_progress(client, sql, start, row_limit, None).await
|
||||||
Ok(outcome) => finish_prepared_select(client, sql, start, row_limit, outcome).await,
|
}
|
||||||
|
|
||||||
|
async fn execute_select_query_with_progress(
|
||||||
|
client: &deadpool_postgres::Client,
|
||||||
|
sql: &str,
|
||||||
|
start: Instant,
|
||||||
|
row_limit: usize,
|
||||||
|
progress_clock: Option<&StreamProgressClock>,
|
||||||
|
) -> Result<QueryResult, String> {
|
||||||
|
match execute_select_prepared(client, sql, start, row_limit, progress_clock).await {
|
||||||
|
Ok(outcome) => finish_prepared_select(client, sql, start, row_limit, outcome, progress_clock).await,
|
||||||
Err(err) if should_retry_postgres_stale_cache(&err) => {
|
Err(err) if should_retry_postgres_stale_cache(&err) => {
|
||||||
// The cached prepared statement is stale (e.g. the view or table
|
// The cached prepared statement is stale (e.g. the view or table
|
||||||
// schema changed since the statement was prepared). Evict the
|
// schema changed since the statement was prepared). Evict the
|
||||||
// stale entry and retry with a fresh server-side prepare.
|
// stale entry and retry with a fresh server-side prepare.
|
||||||
log::warn!("[postgres][select:stale_cache] evicting cached statement: {}", pg_error_to_string(err));
|
log::warn!("[postgres][select:stale_cache] evicting cached statement: {}", pg_error_to_string(err));
|
||||||
client.statement_cache.remove(sql, &[]);
|
client.statement_cache.remove(sql, &[]);
|
||||||
match execute_select_prepared(client, sql, start, row_limit).await {
|
match execute_select_prepared(client, sql, start, row_limit, progress_clock).await {
|
||||||
Ok(outcome) => finish_prepared_select(client, sql, start, row_limit, outcome).await,
|
Ok(outcome) => finish_prepared_select(client, sql, start, row_limit, outcome, progress_clock).await,
|
||||||
Err(err) if should_retry_postgres_text_query(&err) => {
|
Err(err) if should_retry_postgres_text_query(&err) => {
|
||||||
execute_select_text(client, sql, start, row_limit, None).await
|
execute_select_text(client, sql, start, row_limit, None, progress_clock).await
|
||||||
}
|
}
|
||||||
Err(err) => Err(pg_error_to_string(err)),
|
Err(err) => Err(pg_error_to_string(err)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(err) if should_retry_postgres_text_query(&err) => {
|
Err(err) if should_retry_postgres_text_query(&err) => {
|
||||||
execute_select_text(client, sql, start, row_limit, None).await
|
execute_select_text(client, sql, start, row_limit, None, progress_clock).await
|
||||||
}
|
}
|
||||||
Err(err) => Err(pg_error_to_string(err)),
|
Err(err) => Err(pg_error_to_string(err)),
|
||||||
}
|
}
|
||||||
@@ -1467,7 +1495,7 @@ async fn connect_with_optional_local_timezone(
|
|||||||
|
|
||||||
let timeout = super::parse_connect_timeout_with_fallback(url, fallback_timeout);
|
let timeout = super::parse_connect_timeout_with_fallback(url, fallback_timeout);
|
||||||
|
|
||||||
super::with_connection_timeout("PostgreSQL", timeout, async {
|
let (pool, client) = super::with_connection_timeout("PostgreSQL", timeout, async {
|
||||||
let pg_config = tokio_postgres::Config::from_str(&postgres_url.url)
|
let pg_config = tokio_postgres::Config::from_str(&postgres_url.url)
|
||||||
.map_err(|e| format!("Invalid PostgreSQL connection URL: {e}"))?;
|
.map_err(|e| format!("Invalid PostgreSQL connection URL: {e}"))?;
|
||||||
|
|
||||||
@@ -1498,19 +1526,33 @@ async fn connect_with_optional_local_timezone(
|
|||||||
.build()
|
.build()
|
||||||
.map_err(|e| format!("Failed to create PostgreSQL pool: {e}"))?;
|
.map_err(|e| format!("Failed to create PostgreSQL pool: {e}"))?;
|
||||||
|
|
||||||
// Verify connectivity and set timezone. Explicit connection options are
|
// Verify connectivity. Explicit connection options are handled by
|
||||||
// handled by PostgreSQL during startup and must remain strict.
|
// PostgreSQL during startup and must remain strict.
|
||||||
let client =
|
let client =
|
||||||
pool.get().await.map_err(|e| format!("PostgreSQL connection failed: {}", pg_pool_error_to_string(e)))?;
|
pool.get().await.map_err(|e| format!("PostgreSQL connection failed: {}", pg_pool_error_to_string(e)))?;
|
||||||
if !pg_url_has_timezone_setting(url) {
|
Ok((pool, client))
|
||||||
if let Some(timezone) = timezone {
|
|
||||||
set_automatic_postgres_timezone(&client, timezone).await?;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(pool)
|
|
||||||
})
|
})
|
||||||
.await
|
.await?;
|
||||||
|
|
||||||
|
// Creating the physical connection and applying session defaults are two
|
||||||
|
// sequential network phases. Give each phase the configured connection
|
||||||
|
// timeout instead of sharing one deadline that can expire during the
|
||||||
|
// optional SET timezone round-trip on higher-latency tunnels.
|
||||||
|
if !pg_url_has_timezone_setting(url) {
|
||||||
|
if let Some(timezone) = timezone {
|
||||||
|
postgres_session_setup_with_timeout(timeout, set_automatic_postgres_timezone(&client, timezone)).await?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
drop(client);
|
||||||
|
Ok(pool)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn postgres_session_setup_with_timeout<T, F>(timeout: Duration, future: F) -> Result<T, String>
|
||||||
|
where
|
||||||
|
F: Future<Output = Result<T, String>>,
|
||||||
|
{
|
||||||
|
super::with_connection_timeout("PostgreSQL session setup", timeout, future).await
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn set_automatic_postgres_timezone(client: &deadpool_postgres::Client, timezone: &str) -> Result<(), String> {
|
async fn set_automatic_postgres_timezone(client: &deadpool_postgres::Client, timezone: &str) -> Result<(), String> {
|
||||||
@@ -3135,6 +3177,7 @@ pub async fn get_redshift_columns(pool: &Pool, schema: &str, table: &str) -> Res
|
|||||||
Instant::now(),
|
Instant::now(),
|
||||||
crate::query::MAX_ROWS,
|
crate::query::MAX_ROWS,
|
||||||
None,
|
None,
|
||||||
|
None,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
Ok(redshift_columns_from_query_result(result))
|
Ok(redshift_columns_from_query_result(result))
|
||||||
@@ -3312,14 +3355,15 @@ pub async fn execute_query_with_max_rows_and_cancel(
|
|||||||
prefer_text_protocol: bool,
|
prefer_text_protocol: bool,
|
||||||
) -> Result<QueryResult, String> {
|
) -> Result<QueryResult, String> {
|
||||||
let client = checkout_postgres_client(pool, cancel_token.as_ref(), budget.checkout_timeout).await?;
|
let client = checkout_postgres_client(pool, cancel_token.as_ref(), budget.checkout_timeout).await?;
|
||||||
let pg_cancel_token = client.cancel_token();
|
execute_postgres_user_query(
|
||||||
wait_postgres_query(
|
&client,
|
||||||
pg_cancel_token,
|
sql,
|
||||||
cancel_context,
|
max_rows,
|
||||||
cancel_token,
|
cancel_token,
|
||||||
budget.query_timeout,
|
budget.query_timeout,
|
||||||
budget.cancel_timeout,
|
budget.cancel_timeout,
|
||||||
execute_query_with_max_rows_inner(&client, sql, max_rows, prefer_text_protocol),
|
cancel_context,
|
||||||
|
prefer_text_protocol,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
@@ -3386,14 +3430,15 @@ pub async fn execute_query_in_read_only_transaction_with_rollback(
|
|||||||
execute_postgres_infra_statement(&client, &statement, budget.recycle_timeout, stage).await?;
|
execute_postgres_infra_statement(&client, &statement, budget.recycle_timeout, stage).await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
let pg_cancel_token = client.cancel_token();
|
execute_postgres_user_query(
|
||||||
wait_postgres_query(
|
&client,
|
||||||
pg_cancel_token,
|
sql,
|
||||||
cancel_context,
|
max_rows,
|
||||||
cancel_token,
|
cancel_token,
|
||||||
budget.query_timeout,
|
budget.query_timeout,
|
||||||
budget.cancel_timeout,
|
budget.cancel_timeout,
|
||||||
execute_query_with_max_rows_inner(&client, sql, max_rows, false),
|
cancel_context,
|
||||||
|
false,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
},
|
},
|
||||||
@@ -3534,7 +3579,7 @@ pub async fn execute_query_with_schema_and_max_rows(
|
|||||||
"[postgres][execute_with_schema:skip-search-path] total_ms={} reason=transaction-recovery",
|
"[postgres][execute_with_schema:skip-search-path] total_ms={} reason=transaction-recovery",
|
||||||
start.elapsed().as_millis()
|
start.elapsed().as_millis()
|
||||||
);
|
);
|
||||||
return execute_query_with_max_rows_inner(&client, sql, max_rows, false).await;
|
return execute_query_with_max_rows_inner(&client, sql, max_rows, false, None).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
let set_schema_start = Instant::now();
|
let set_schema_start = Instant::now();
|
||||||
@@ -3546,7 +3591,7 @@ pub async fn execute_query_with_schema_and_max_rows(
|
|||||||
);
|
);
|
||||||
|
|
||||||
let query_start = Instant::now();
|
let query_start = Instant::now();
|
||||||
let result = execute_query_with_max_rows_inner(&client, sql, max_rows, false).await;
|
let result = execute_query_with_max_rows_inner(&client, sql, max_rows, false, None).await;
|
||||||
if result.is_ok() {
|
if result.is_ok() {
|
||||||
clear_postgres_caches_after_ddl(pool, Some(&client), sql);
|
clear_postgres_caches_after_ddl(pool, Some(&client), sql);
|
||||||
}
|
}
|
||||||
@@ -3585,14 +3630,15 @@ pub async fn execute_query_with_schema_and_max_rows_and_cancel(
|
|||||||
"[postgres][execute_with_schema:skip-search-path] total_ms={} reason=transaction-recovery",
|
"[postgres][execute_with_schema:skip-search-path] total_ms={} reason=transaction-recovery",
|
||||||
start.elapsed().as_millis()
|
start.elapsed().as_millis()
|
||||||
);
|
);
|
||||||
let pg_cancel_token = client.cancel_token();
|
return execute_postgres_user_query(
|
||||||
return wait_postgres_query(
|
&client,
|
||||||
pg_cancel_token,
|
sql,
|
||||||
cancel_context,
|
max_rows,
|
||||||
cancel_token,
|
cancel_token,
|
||||||
budget.query_timeout,
|
budget.query_timeout,
|
||||||
budget.cancel_timeout,
|
budget.cancel_timeout,
|
||||||
execute_query_with_max_rows_inner(&client, sql, max_rows, prefer_text_protocol),
|
cancel_context,
|
||||||
|
prefer_text_protocol,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
@@ -3606,14 +3652,15 @@ pub async fn execute_query_with_schema_and_max_rows_and_cancel(
|
|||||||
);
|
);
|
||||||
|
|
||||||
let query_start = Instant::now();
|
let query_start = Instant::now();
|
||||||
let pg_cancel_token = client.cancel_token();
|
let result = execute_postgres_user_query(
|
||||||
let result = wait_postgres_query(
|
&client,
|
||||||
pg_cancel_token,
|
sql,
|
||||||
cancel_context,
|
max_rows,
|
||||||
cancel_token,
|
cancel_token,
|
||||||
budget.query_timeout,
|
budget.query_timeout,
|
||||||
budget.cancel_timeout,
|
budget.cancel_timeout,
|
||||||
execute_query_with_max_rows_inner(&client, sql, max_rows, prefer_text_protocol),
|
cancel_context,
|
||||||
|
prefer_text_protocol,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
if result.is_ok() {
|
if result.is_ok() {
|
||||||
@@ -3747,6 +3794,52 @@ where
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn execute_postgres_user_query(
|
||||||
|
client: &deadpool_postgres::Client,
|
||||||
|
sql: &str,
|
||||||
|
max_rows: Option<usize>,
|
||||||
|
cancel_token: Option<CancellationToken>,
|
||||||
|
timeout_duration: Option<Duration>,
|
||||||
|
cancel_timeout: Duration,
|
||||||
|
cancel_context: Option<PostgresCancelContext>,
|
||||||
|
prefer_text_protocol: bool,
|
||||||
|
) -> Result<QueryResult, String> {
|
||||||
|
let pg_cancel_token = client.cancel_token();
|
||||||
|
// Commands do not expose incremental results, so keep their timeout as a
|
||||||
|
// wall-clock deadline. Row-returning queries may spend longer transferring
|
||||||
|
// a bounded result through a slow tunnel; for those, the same setting is an
|
||||||
|
// inactivity budget that resets only after real PostgreSQL progress.
|
||||||
|
if !postgres_statement_returns_rows(sql) {
|
||||||
|
return wait_postgres_query(
|
||||||
|
pg_cancel_token,
|
||||||
|
cancel_context,
|
||||||
|
cancel_token,
|
||||||
|
timeout_duration,
|
||||||
|
cancel_timeout,
|
||||||
|
execute_query_with_max_rows_inner(client, sql, max_rows, prefer_text_protocol, None),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
|
let timeout_error =
|
||||||
|
format!("Query timed out after {} seconds", timeout_duration.map_or(0, |timeout| timeout.as_secs()));
|
||||||
|
let progress_clock = Arc::new(StreamProgressClock::new());
|
||||||
|
let result = await_stream_with_progress_timeout(
|
||||||
|
execute_query_with_max_rows_inner(client, sql, max_rows, prefer_text_protocol, Some(progress_clock.clone())),
|
||||||
|
timeout_duration,
|
||||||
|
progress_clock,
|
||||||
|
cancel_token.as_ref(),
|
||||||
|
timeout_error.clone(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
if result.as_ref().is_err_and(|error| error == &timeout_error || error == crate::query::QUERY_CANCELED) {
|
||||||
|
cancel_postgres_query(pg_cancel_token, cancel_context.as_ref(), cancel_timeout).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
result
|
||||||
|
}
|
||||||
|
|
||||||
/// PostgreSQL pool checkout with timeout and cancel token support.
|
/// PostgreSQL pool checkout with timeout and cancel token support.
|
||||||
/// When the checkout phase is stuck, the cancel token can terminate the wait early.
|
/// When the checkout phase is stuck, the cancel token can terminate the wait early.
|
||||||
/// The timeout error message includes "checkout timed out" to ensure is_connection_error can classify it correctly.
|
/// The timeout error message includes "checkout timed out" to ensure is_connection_error can classify it correctly.
|
||||||
@@ -3755,6 +3848,7 @@ pub async fn checkout_postgres_client(
|
|||||||
cancel_token: Option<&CancellationToken>,
|
cancel_token: Option<&CancellationToken>,
|
||||||
checkout_timeout: Duration,
|
checkout_timeout: Duration,
|
||||||
) -> Result<deadpool_postgres::Object, String> {
|
) -> Result<deadpool_postgres::Object, String> {
|
||||||
|
let checkout_timeout = effective_postgres_checkout_timeout(pool, checkout_timeout);
|
||||||
let start = Instant::now();
|
let start = Instant::now();
|
||||||
let get_future = async {
|
let get_future = async {
|
||||||
tokio::time::timeout(checkout_timeout, pool.get())
|
tokio::time::timeout(checkout_timeout, pool.get())
|
||||||
@@ -3806,6 +3900,14 @@ pub async fn checkout_postgres_client(
|
|||||||
result
|
result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn effective_postgres_checkout_timeout(pool: &Pool, requested_timeout: Duration) -> Duration {
|
||||||
|
let configured = pool.timeouts();
|
||||||
|
[configured.wait, configured.create, configured.recycle]
|
||||||
|
.into_iter()
|
||||||
|
.flatten()
|
||||||
|
.fold(requested_timeout, std::cmp::max)
|
||||||
|
}
|
||||||
|
|
||||||
async fn cancel_postgres_query(
|
async fn cancel_postgres_query(
|
||||||
pg_cancel_token: tokio_postgres::CancelToken,
|
pg_cancel_token: tokio_postgres::CancelToken,
|
||||||
cancel_context: Option<&PostgresCancelContext>,
|
cancel_context: Option<&PostgresCancelContext>,
|
||||||
@@ -3853,15 +3955,16 @@ async fn execute_query_with_max_rows_inner(
|
|||||||
sql: &str,
|
sql: &str,
|
||||||
max_rows: Option<usize>,
|
max_rows: Option<usize>,
|
||||||
prefer_text_protocol: bool,
|
prefer_text_protocol: bool,
|
||||||
|
progress_clock: Option<Arc<StreamProgressClock>>,
|
||||||
) -> Result<QueryResult, String> {
|
) -> Result<QueryResult, String> {
|
||||||
let start = Instant::now();
|
let start = Instant::now();
|
||||||
let row_limit = query_result_row_limit(max_rows);
|
let row_limit = query_result_row_limit(max_rows);
|
||||||
|
|
||||||
if postgres_statement_returns_rows(sql) {
|
if postgres_statement_returns_rows(sql) {
|
||||||
if prefer_text_protocol {
|
if prefer_text_protocol {
|
||||||
execute_select_text(client, sql, start, row_limit, None).await
|
execute_select_text(client, sql, start, row_limit, None, progress_clock.as_deref()).await
|
||||||
} else {
|
} else {
|
||||||
execute_select_query(client, sql, start, row_limit).await
|
execute_select_query_with_progress(client, sql, start, row_limit, progress_clock.as_deref()).await
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
let affected = client.execute(sql, &[]).await.map_err(pg_error_to_string)?;
|
let affected = client.execute(sql, &[]).await.map_err(pg_error_to_string)?;
|
||||||
@@ -5894,6 +5997,71 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn postgres_checkout_timeout_respects_pool_configuration() {
|
||||||
|
let manager = deadpool_postgres::Manager::new(tokio_postgres::Config::new(), NoTls);
|
||||||
|
let pool = Pool::builder(manager)
|
||||||
|
.runtime(Runtime::Tokio1)
|
||||||
|
.wait_timeout(Some(Duration::from_secs(10)))
|
||||||
|
.create_timeout(Some(Duration::from_secs(10)))
|
||||||
|
.recycle_timeout(Some(Duration::from_secs(10)))
|
||||||
|
.build()
|
||||||
|
.expect("build postgres pool");
|
||||||
|
|
||||||
|
assert_eq!(effective_postgres_checkout_timeout(&pool, Duration::from_secs(5)), Duration::from_secs(10));
|
||||||
|
assert_eq!(effective_postgres_checkout_timeout(&pool, Duration::from_secs(15)), Duration::from_secs(15));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn postgres_session_setup_reports_its_own_timeout_stage() {
|
||||||
|
let error =
|
||||||
|
postgres_session_setup_with_timeout(Duration::from_millis(1), std::future::pending::<Result<(), String>>())
|
||||||
|
.await
|
||||||
|
.expect_err("pending session setup should time out");
|
||||||
|
|
||||||
|
assert!(error.starts_with("PostgreSQL session setup connection timed out"), "{error}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn postgres_row_query_timeout_tracks_inactivity_instead_of_total_duration() {
|
||||||
|
let progress_clock = Arc::new(StreamProgressClock::new());
|
||||||
|
let progress_clock_for_query = progress_clock.clone();
|
||||||
|
let future = async {
|
||||||
|
for _ in 0..4 {
|
||||||
|
tokio::time::sleep(Duration::from_millis(40)).await;
|
||||||
|
progress_clock_for_query.mark();
|
||||||
|
}
|
||||||
|
Ok::<_, String>(())
|
||||||
|
};
|
||||||
|
|
||||||
|
let result = await_stream_with_progress_timeout(
|
||||||
|
future,
|
||||||
|
Some(Duration::from_millis(100)),
|
||||||
|
progress_clock,
|
||||||
|
None,
|
||||||
|
"query inactivity timeout".to_string(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert_eq!(result, Ok(()));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn postgres_row_query_timeout_still_fires_after_no_progress() {
|
||||||
|
let progress_clock = Arc::new(StreamProgressClock::new());
|
||||||
|
let error = await_stream_with_progress_timeout(
|
||||||
|
std::future::pending::<Result<(), String>>(),
|
||||||
|
Some(Duration::from_millis(10)),
|
||||||
|
progress_clock,
|
||||||
|
None,
|
||||||
|
"query inactivity timeout".to_string(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect_err("a stalled query should time out");
|
||||||
|
|
||||||
|
assert_eq!(error, "query inactivity timeout");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn postgres_cancel_context_omits_disabled_ssl_mode() {
|
fn postgres_cancel_context_omits_disabled_ssl_mode() {
|
||||||
assert!(build_postgres_cancel_context("postgres://localhost/app?sslmode=disable").is_none());
|
assert!(build_postgres_cancel_context("postgres://localhost/app?sslmode=disable").is_none());
|
||||||
|
|||||||
Reference in new issue
Block a user