fix(connection): 修复数据库连接超时后无法可靠恢复的问题

* feat(core): implement PIP-0001 database connection timeout recovery

- Add DbOperationBudget struct for unified execution budget model
- Add deadpool create/recycle timeout for PostgreSQL pools
- Add checkout_postgres_client helper replacing 14 bare pool.get() calls
  with timeout + cancel token support
- Fix PostgreSQL TLS cancel: rebuild TLS connector from stored context,
  fallback to NoTls on failure
- Add timeout to MySQL kill_query and kill_query_with_opts (5s)
- Extend is_connection_error to match deadpool pool timeout messages
- Unify keepalive default to 30s (was 60s Rust / 0s frontend mismatch)
- Inject PostgreSQL TCP keepalive params (idle=30s, interval=10s, retries=3)
- Inject MySQL tcp_keepalive=30s in OptsBuilder
- Make MySQL inactive_connection_ttl configurable from idle_timeout_secs
- Add stage-based structured logging for pool.checkout/health.check/cancel
- Store PostgresCancelContext in AppState for TLS cancel reconstruction
- Clean up cancel contexts on pool removal/drain
- Add unit tests for DbOperationBudget and pool timeout error detection

* fix(core): align timeout recovery with PIP-0001

* fix(core): handle timeout recovery review issues

* fix(core): complete connection timeout recovery

* fix(core): fall back postgres cancel connector build

* refactor: revert comment translations to English in connection.rs and query.rs

- Revert DbOperationBudget doc comments to English (query.rs)
- Revert PostgresCancelContext and TLS cancel comments to English (connection.rs)
- Keep Chinese error-matching strings in is_connection_error unchanged

---------

Co-authored-by: onceMisery <onceMisery@users.noreply.github.com>
This commit is contained in:
miracleandonceMisery authored and GitHub committed 2026-06-26 21:41:56 +08:00
1 parent 796b0ce5b7
commit d128604743
11 files changed
+1134 -169

No files matched your search

@@ -0,0 +1,120 @@
import { createPinia, setActivePinia } from "pinia";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { ConnectionConfig, TreeNode } from "@/types/database";
function installLocalStorage() {
const data = new Map<string, string>();
vi.stubGlobal("localStorage", {
getItem: vi.fn((key: string) => data.get(key) ?? null),
setItem: vi.fn((key: string, value: string) => data.set(key, value)),
removeItem: vi.fn((key: string) => data.delete(key)),
});
}
function postgresConnection(overrides: Partial<ConnectionConfig> = {}): ConnectionConfig {
return {
id: "pg-1",
name: "Postgres",
db_type: "postgres",
host: "127.0.0.1",
port: 5432,
username: "postgres",
password: "",
database: "app",
read_only: false,
...overrides,
} as ConnectionConfig;
}
describe("connectionStore timeout recovery", () => {
beforeEach(() => {
vi.useFakeTimers();
vi.resetModules();
vi.unstubAllGlobals();
installLocalStorage();
setActivePinia(createPinia());
});
afterEach(() => {
vi.useRealTimers();
});
it("times out connected health checks and falls back to reconnect", async () => {
const checkConnectionHealth = vi.fn(() => new Promise(() => undefined));
const connectDb = vi.fn().mockResolvedValue(undefined);
vi.doMock("@/lib/tauriRuntime", () => ({ isTauriRuntime: () => false }));
vi.doMock("@/lib/api", () => ({
checkConnectionHealth,
connectDb,
deleteSchemaCachePrefix: vi.fn().mockResolvedValue(undefined),
saveConnections: vi.fn().mockResolvedValue(undefined),
saveSidebarLayout: vi.fn().mockResolvedValue(undefined),
}));
const { useConnectionStore } = await import("@/stores/connectionStore");
const store = useConnectionStore();
const connection = postgresConnection({ connect_timeout_secs: 10 });
store.connections = [connection];
store.connectedIds.add(connection.id);
const ensure = store.ensureConnected(connection.id);
await vi.advanceTimersByTimeAsync(5001);
await ensure;
expect(checkConnectionHealth).toHaveBeenCalledWith(connection.id);
expect(connectDb).toHaveBeenCalledWith(connection);
expect(store.connectedIds.has(connection.id)).toBe(true);
}, 10_000);
it("normalizes missing keepalive interval to 30 seconds", async () => {
vi.doMock("@/lib/tauriRuntime", () => ({ isTauriRuntime: () => false }));
vi.doMock("@/lib/api", () => ({
deleteSchemaCachePrefix: vi.fn().mockResolvedValue(undefined),
saveConnections: vi.fn().mockResolvedValue(undefined),
saveSidebarLayout: vi.fn().mockResolvedValue(undefined),
}));
const { useConnectionStore } = await import("@/stores/connectionStore");
const store = useConnectionStore();
await store.addConnection(postgresConnection({ keepalive_interval_secs: undefined }));
expect(store.connections[0]?.keepalive_interval_secs).toBe(30);
});
it("clears connection node loading when health check timeout forces reconnect failure", async () => {
const checkConnectionHealth = vi.fn(() => new Promise(() => undefined));
const connectDb = vi.fn().mockRejectedValue(new Error("reconnect failed"));
vi.doMock("@/lib/tauriRuntime", () => ({ isTauriRuntime: () => false }));
vi.doMock("@/lib/api", () => ({
checkConnectionHealth,
connectDb,
deleteSchemaCachePrefix: vi.fn().mockResolvedValue(undefined),
saveConnections: vi.fn().mockResolvedValue(undefined),
saveSidebarLayout: vi.fn().mockResolvedValue(undefined),
}));
const { useConnectionStore } = await import("@/stores/connectionStore");
const store = useConnectionStore();
const connection = postgresConnection({ connect_timeout_secs: 10 });
const node: TreeNode = {
id: connection.id,
label: connection.name,
type: "connection",
connectionId: connection.id,
isLoading: true,
children: [],
};
store.connections = [connection];
store.connectedIds.add(connection.id);
store.treeNodes = [node];
const ensure = store.ensureConnected(connection.id).catch((error) => error);
await vi.advanceTimersByTimeAsync(5001);
const error = await ensure;
expect(error).toBeInstanceOf(Error);
expect(node.isLoading).toBe(false);
}, 10_000);
});
@@ -0,0 +1,51 @@
import { createPinia, setActivePinia } from "pinia";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
function installLocalStorage() {
const data = new Map<string, string>();
vi.stubGlobal("localStorage", {
getItem: vi.fn((key: string) => data.get(key) ?? null),
setItem: vi.fn((key: string, value: string) => data.set(key, value)),
removeItem: vi.fn((key: string) => data.delete(key)),
});
}
describe("queryStore cancel timeout recovery", () => {
beforeEach(() => {
vi.useFakeTimers();
vi.resetModules();
vi.unstubAllGlobals();
installLocalStorage();
setActivePinia(createPinia());
});
afterEach(() => {
vi.useRealTimers();
});
it("clears cancelling state when cancelQuery does not return", async () => {
const cancelQuery = vi.fn(() => new Promise(() => undefined));
vi.doMock("@/lib/api", () => ({ cancelQuery }));
const { useQueryStore } = await import("@/stores/queryStore");
const store = useQueryStore();
const tabId = store.createTab("pg-1", "app", "query_1");
store.setExecutingWithId(tabId, "exec-1");
const cancel = store.cancelTabExecution(tabId);
expect(store.tabs[0]?.isCancelling).toBe(true);
await vi.advanceTimersByTimeAsync(10_001);
const result = await cancel;
expect(result).toBe(false);
expect(cancelQuery).toHaveBeenCalledWith("exec-1");
expect(store.tabs[0]).toMatchObject({
isExecuting: false,
isCancelling: false,
executionId: undefined,
});
expect(store.tabs[0]?.result?.rows[0]?.[0]).toContain("Cancel request timed out");
}, 15_000);
});
+24 -2
View File
@@ -59,9 +59,11 @@ import { REDIS_SCAN_PAGE_SIZE_DEFAULT } from "@/lib/redisKeyPattern";
const PINNED_TREE_NODES_STORAGE_KEY = "dbx-pinned-tree-nodes";
const ACTIVE_CONNECTION_STORAGE_KEY = "dbx-active-connection";
const CONNECTION_HEALTH_CHECK_TTL_MS = 2000;
const CONNECTION_HEALTH_CHECK_TIMEOUT_MS = 5000;
const METADATA_LOAD_MIN_TIMEOUT_MS = 15_000;
const METADATA_LOAD_DISABLED_QUERY_TIMEOUT_MS = 60_000;
const DISCONNECT_REQUEST_TIMEOUT_MS = 5_000;
const DEFAULT_KEEPALIVE_INTERVAL_SECS = 30;
const MONGO_LEGACY_DRIVER_PROFILE = "mongodb-legacy";
const MONGO_LEGACY_DRIVER_LABEL = "MongoDB (Legacy)";
const SUPERSEDED_CONNECTION_ATTEMPT_MESSAGE = "Connection attempt was superseded by a newer attempt";
@@ -304,6 +306,25 @@ export const useConnectionStore = defineStore("connection", () => {
return Math.max(METADATA_LOAD_MIN_TIMEOUT_MS, boundedTimeoutSecs * 1000);
}
async function withConnectionHealthTimeout(connectionId: string, promise: Promise<void>): Promise<void> {
let timer: ReturnType<typeof setTimeout> | undefined;
try {
return await Promise.race([
promise,
new Promise<never>((_, reject) => {
timer = setTimeout(() => {
reject(new Error(`Connection health check timed out after ${Math.ceil(CONNECTION_HEALTH_CHECK_TIMEOUT_MS / 1000)}s.`));
}, CONNECTION_HEALTH_CHECK_TIMEOUT_MS);
}),
]);
} catch (error) {
clearConnectionNodeLoading(connectionId);
throw error;
} finally {
if (timer) clearTimeout(timer);
}
}
async function withMetadataLoadTimeout<T>(connectionId: string, promise: Promise<T>, label: string): Promise<T> {
const timeoutMs = metadataLoadTimeoutMs(getConfig(connectionId));
let timer: ReturnType<typeof setTimeout> | undefined;
@@ -486,7 +507,7 @@ export const useConnectionStore = defineStore("connection", () => {
connect_timeout_secs: config.connect_timeout_secs || 10,
query_timeout_secs: config.query_timeout_secs ?? 30,
idle_timeout_secs: config.idle_timeout_secs ?? 60,
keepalive_interval_secs: config.keepalive_interval_secs ?? 0,
keepalive_interval_secs: config.keepalive_interval_secs ?? DEFAULT_KEEPALIVE_INTERVAL_SECS,
};
}
@@ -1230,7 +1251,7 @@ export const useConnectionStore = defineStore("connection", () => {
if (hasRecentConnectionHealthCheck(connectionId)) return;
// Optimistic: verify backend pool is actually healthy
try {
await api.checkConnectionHealth(connectionId);
await withConnectionHealthTimeout(connectionId, api.checkConnectionHealth(connectionId));
markConnectionHealthChecked(connectionId);
return;
} catch {
@@ -1273,6 +1294,7 @@ export const useConnectionStore = defineStore("connection", () => {
return;
}
recordConnectionError(connectionId, e);
clearConnectionNodeLoading(connectionId);
throw e;
} finally {
if (connectInFlight.get(connectionId) === connectPromise) {
+19 -1
View File
@@ -49,6 +49,7 @@ const STORAGE_KEY = "dbx-open-tabs";
const ACTIVE_TAB_KEY = "dbx-active-tab";
const ORACLE_LIKE_METADATA_TYPES = new Set<string>(["oracle", "dameng", "oceanbase-oracle"]);
const BACKGROUND_CLIENT_SESSION_SUFFIXES = ["count", "explain", "export"] as const;
const CANCEL_QUERY_TIMEOUT_MS = 10_000;
interface BuildQueryResultExportRequestOptions {
exportId: string;
@@ -98,6 +99,20 @@ async function withFrontendQueryTimeout<T>(promise: Promise<T>, timeoutSecs: num
}
}
async function withCancelQueryTimeout<T>(promise: Promise<T>): Promise<T> {
let timer: ReturnType<typeof setTimeout> | undefined;
try {
return await Promise.race([
promise,
new Promise<never>((_, reject) => {
timer = setTimeout(() => reject(new Error("Cancel request timed out after 10s.")), CANCEL_QUERY_TIMEOUT_MS);
}),
]);
} finally {
if (timer) clearTimeout(timer);
}
}
function normalizeOracleLikeMetadataIdentifier(dbType: string, identifier: string | undefined, quoted?: boolean) {
if (!identifier || quoted || !ORACLE_LIKE_METADATA_TYPES.has(dbType)) return identifier;
return identifier.toUpperCase();
@@ -1946,7 +1961,7 @@ export const useQueryStore = defineStore("query", () => {
if (!executionId) return false;
tab.isCancelling = true;
try {
const canceled = await api.cancelQuery(executionId);
const canceled = await withCancelQueryTimeout(api.cancelQuery(executionId));
if (!canceled) {
const current = tabs.value.find((t) => t.id === id);
if (current && current.executionId === executionId) {
@@ -1962,7 +1977,10 @@ export const useQueryStore = defineStore("query", () => {
if (tab) useConnectionStore().recordConnectionLostError(tab.connectionId, e);
const current = tabs.value.find((t) => t.id === id);
if (current && current.executionId === executionId) {
current.isExecuting = false;
current.isCancelling = false;
current.executionId = undefined;
current.queryExecutionStartedAt = undefined;
current.result = toErrorResult(e);
}
return false;