fix(connection): 修复数据库连接超时后无法可靠恢复的问题
* feat(core): implement PIP-0001 database connection timeout recovery - Add DbOperationBudget struct for unified execution budget model - Add deadpool create/recycle timeout for PostgreSQL pools - Add checkout_postgres_client helper replacing 14 bare pool.get() calls with timeout + cancel token support - Fix PostgreSQL TLS cancel: rebuild TLS connector from stored context, fallback to NoTls on failure - Add timeout to MySQL kill_query and kill_query_with_opts (5s) - Extend is_connection_error to match deadpool pool timeout messages - Unify keepalive default to 30s (was 60s Rust / 0s frontend mismatch) - Inject PostgreSQL TCP keepalive params (idle=30s, interval=10s, retries=3) - Inject MySQL tcp_keepalive=30s in OptsBuilder - Make MySQL inactive_connection_ttl configurable from idle_timeout_secs - Add stage-based structured logging for pool.checkout/health.check/cancel - Store PostgresCancelContext in AppState for TLS cancel reconstruction - Clean up cancel contexts on pool removal/drain - Add unit tests for DbOperationBudget and pool timeout error detection * fix(core): align timeout recovery with PIP-0001 * fix(core): handle timeout recovery review issues * fix(core): complete connection timeout recovery * fix(core): fall back postgres cancel connector build * refactor: revert comment translations to English in connection.rs and query.rs - Revert DbOperationBudget doc comments to English (query.rs) - Revert PostgresCancelContext and TLS cancel comments to English (connection.rs) - Keep Chinese error-matching strings in is_connection_error unchanged --------- Co-authored-by: onceMisery <onceMisery@users.noreply.github.com>
This commit is contained in:
1 parent
796b0ce5b7
commit
d128604743
11 files changed
+1134
-169
No files matched your search
@@ -0,0 +1,120 @@
|
||||
import { createPinia, setActivePinia } from "pinia";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { ConnectionConfig, TreeNode } from "@/types/database";
|
||||
|
||||
function installLocalStorage() {
|
||||
const data = new Map<string, string>();
|
||||
vi.stubGlobal("localStorage", {
|
||||
getItem: vi.fn((key: string) => data.get(key) ?? null),
|
||||
setItem: vi.fn((key: string, value: string) => data.set(key, value)),
|
||||
removeItem: vi.fn((key: string) => data.delete(key)),
|
||||
});
|
||||
}
|
||||
|
||||
function postgresConnection(overrides: Partial<ConnectionConfig> = {}): ConnectionConfig {
|
||||
return {
|
||||
id: "pg-1",
|
||||
name: "Postgres",
|
||||
db_type: "postgres",
|
||||
host: "127.0.0.1",
|
||||
port: 5432,
|
||||
username: "postgres",
|
||||
password: "",
|
||||
database: "app",
|
||||
read_only: false,
|
||||
...overrides,
|
||||
} as ConnectionConfig;
|
||||
}
|
||||
|
||||
describe("connectionStore timeout recovery", () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
vi.resetModules();
|
||||
vi.unstubAllGlobals();
|
||||
installLocalStorage();
|
||||
setActivePinia(createPinia());
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("times out connected health checks and falls back to reconnect", async () => {
|
||||
const checkConnectionHealth = vi.fn(() => new Promise(() => undefined));
|
||||
const connectDb = vi.fn().mockResolvedValue(undefined);
|
||||
|
||||
vi.doMock("@/lib/tauriRuntime", () => ({ isTauriRuntime: () => false }));
|
||||
vi.doMock("@/lib/api", () => ({
|
||||
checkConnectionHealth,
|
||||
connectDb,
|
||||
deleteSchemaCachePrefix: vi.fn().mockResolvedValue(undefined),
|
||||
saveConnections: vi.fn().mockResolvedValue(undefined),
|
||||
saveSidebarLayout: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
const { useConnectionStore } = await import("@/stores/connectionStore");
|
||||
const store = useConnectionStore();
|
||||
const connection = postgresConnection({ connect_timeout_secs: 10 });
|
||||
store.connections = [connection];
|
||||
store.connectedIds.add(connection.id);
|
||||
|
||||
const ensure = store.ensureConnected(connection.id);
|
||||
await vi.advanceTimersByTimeAsync(5001);
|
||||
await ensure;
|
||||
|
||||
expect(checkConnectionHealth).toHaveBeenCalledWith(connection.id);
|
||||
expect(connectDb).toHaveBeenCalledWith(connection);
|
||||
expect(store.connectedIds.has(connection.id)).toBe(true);
|
||||
}, 10_000);
|
||||
|
||||
it("normalizes missing keepalive interval to 30 seconds", async () => {
|
||||
vi.doMock("@/lib/tauriRuntime", () => ({ isTauriRuntime: () => false }));
|
||||
vi.doMock("@/lib/api", () => ({
|
||||
deleteSchemaCachePrefix: vi.fn().mockResolvedValue(undefined),
|
||||
saveConnections: vi.fn().mockResolvedValue(undefined),
|
||||
saveSidebarLayout: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
const { useConnectionStore } = await import("@/stores/connectionStore");
|
||||
const store = useConnectionStore();
|
||||
await store.addConnection(postgresConnection({ keepalive_interval_secs: undefined }));
|
||||
|
||||
expect(store.connections[0]?.keepalive_interval_secs).toBe(30);
|
||||
});
|
||||
|
||||
it("clears connection node loading when health check timeout forces reconnect failure", async () => {
|
||||
const checkConnectionHealth = vi.fn(() => new Promise(() => undefined));
|
||||
const connectDb = vi.fn().mockRejectedValue(new Error("reconnect failed"));
|
||||
|
||||
vi.doMock("@/lib/tauriRuntime", () => ({ isTauriRuntime: () => false }));
|
||||
vi.doMock("@/lib/api", () => ({
|
||||
checkConnectionHealth,
|
||||
connectDb,
|
||||
deleteSchemaCachePrefix: vi.fn().mockResolvedValue(undefined),
|
||||
saveConnections: vi.fn().mockResolvedValue(undefined),
|
||||
saveSidebarLayout: vi.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
const { useConnectionStore } = await import("@/stores/connectionStore");
|
||||
const store = useConnectionStore();
|
||||
const connection = postgresConnection({ connect_timeout_secs: 10 });
|
||||
const node: TreeNode = {
|
||||
id: connection.id,
|
||||
label: connection.name,
|
||||
type: "connection",
|
||||
connectionId: connection.id,
|
||||
isLoading: true,
|
||||
children: [],
|
||||
};
|
||||
store.connections = [connection];
|
||||
store.connectedIds.add(connection.id);
|
||||
store.treeNodes = [node];
|
||||
|
||||
const ensure = store.ensureConnected(connection.id).catch((error) => error);
|
||||
await vi.advanceTimersByTimeAsync(5001);
|
||||
const error = await ensure;
|
||||
|
||||
expect(error).toBeInstanceOf(Error);
|
||||
expect(node.isLoading).toBe(false);
|
||||
}, 10_000);
|
||||
});
|
||||
@@ -0,0 +1,51 @@
|
||||
import { createPinia, setActivePinia } from "pinia";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
function installLocalStorage() {
|
||||
const data = new Map<string, string>();
|
||||
vi.stubGlobal("localStorage", {
|
||||
getItem: vi.fn((key: string) => data.get(key) ?? null),
|
||||
setItem: vi.fn((key: string, value: string) => data.set(key, value)),
|
||||
removeItem: vi.fn((key: string) => data.delete(key)),
|
||||
});
|
||||
}
|
||||
|
||||
describe("queryStore cancel timeout recovery", () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
vi.resetModules();
|
||||
vi.unstubAllGlobals();
|
||||
installLocalStorage();
|
||||
setActivePinia(createPinia());
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("clears cancelling state when cancelQuery does not return", async () => {
|
||||
const cancelQuery = vi.fn(() => new Promise(() => undefined));
|
||||
|
||||
vi.doMock("@/lib/api", () => ({ cancelQuery }));
|
||||
|
||||
const { useQueryStore } = await import("@/stores/queryStore");
|
||||
const store = useQueryStore();
|
||||
const tabId = store.createTab("pg-1", "app", "query_1");
|
||||
store.setExecutingWithId(tabId, "exec-1");
|
||||
|
||||
const cancel = store.cancelTabExecution(tabId);
|
||||
expect(store.tabs[0]?.isCancelling).toBe(true);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(10_001);
|
||||
const result = await cancel;
|
||||
|
||||
expect(result).toBe(false);
|
||||
expect(cancelQuery).toHaveBeenCalledWith("exec-1");
|
||||
expect(store.tabs[0]).toMatchObject({
|
||||
isExecuting: false,
|
||||
isCancelling: false,
|
||||
executionId: undefined,
|
||||
});
|
||||
expect(store.tabs[0]?.result?.rows[0]?.[0]).toContain("Cancel request timed out");
|
||||
}, 15_000);
|
||||
});
|
||||
@@ -59,9 +59,11 @@ import { REDIS_SCAN_PAGE_SIZE_DEFAULT } from "@/lib/redisKeyPattern";
|
||||
const PINNED_TREE_NODES_STORAGE_KEY = "dbx-pinned-tree-nodes";
|
||||
const ACTIVE_CONNECTION_STORAGE_KEY = "dbx-active-connection";
|
||||
const CONNECTION_HEALTH_CHECK_TTL_MS = 2000;
|
||||
const CONNECTION_HEALTH_CHECK_TIMEOUT_MS = 5000;
|
||||
const METADATA_LOAD_MIN_TIMEOUT_MS = 15_000;
|
||||
const METADATA_LOAD_DISABLED_QUERY_TIMEOUT_MS = 60_000;
|
||||
const DISCONNECT_REQUEST_TIMEOUT_MS = 5_000;
|
||||
const DEFAULT_KEEPALIVE_INTERVAL_SECS = 30;
|
||||
const MONGO_LEGACY_DRIVER_PROFILE = "mongodb-legacy";
|
||||
const MONGO_LEGACY_DRIVER_LABEL = "MongoDB (Legacy)";
|
||||
const SUPERSEDED_CONNECTION_ATTEMPT_MESSAGE = "Connection attempt was superseded by a newer attempt";
|
||||
@@ -304,6 +306,25 @@ export const useConnectionStore = defineStore("connection", () => {
|
||||
return Math.max(METADATA_LOAD_MIN_TIMEOUT_MS, boundedTimeoutSecs * 1000);
|
||||
}
|
||||
|
||||
async function withConnectionHealthTimeout(connectionId: string, promise: Promise<void>): Promise<void> {
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
return await Promise.race([
|
||||
promise,
|
||||
new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => {
|
||||
reject(new Error(`Connection health check timed out after ${Math.ceil(CONNECTION_HEALTH_CHECK_TIMEOUT_MS / 1000)}s.`));
|
||||
}, CONNECTION_HEALTH_CHECK_TIMEOUT_MS);
|
||||
}),
|
||||
]);
|
||||
} catch (error) {
|
||||
clearConnectionNodeLoading(connectionId);
|
||||
throw error;
|
||||
} finally {
|
||||
if (timer) clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
async function withMetadataLoadTimeout<T>(connectionId: string, promise: Promise<T>, label: string): Promise<T> {
|
||||
const timeoutMs = metadataLoadTimeoutMs(getConfig(connectionId));
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
@@ -486,7 +507,7 @@ export const useConnectionStore = defineStore("connection", () => {
|
||||
connect_timeout_secs: config.connect_timeout_secs || 10,
|
||||
query_timeout_secs: config.query_timeout_secs ?? 30,
|
||||
idle_timeout_secs: config.idle_timeout_secs ?? 60,
|
||||
keepalive_interval_secs: config.keepalive_interval_secs ?? 0,
|
||||
keepalive_interval_secs: config.keepalive_interval_secs ?? DEFAULT_KEEPALIVE_INTERVAL_SECS,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1230,7 +1251,7 @@ export const useConnectionStore = defineStore("connection", () => {
|
||||
if (hasRecentConnectionHealthCheck(connectionId)) return;
|
||||
// Optimistic: verify backend pool is actually healthy
|
||||
try {
|
||||
await api.checkConnectionHealth(connectionId);
|
||||
await withConnectionHealthTimeout(connectionId, api.checkConnectionHealth(connectionId));
|
||||
markConnectionHealthChecked(connectionId);
|
||||
return;
|
||||
} catch {
|
||||
@@ -1273,6 +1294,7 @@ export const useConnectionStore = defineStore("connection", () => {
|
||||
return;
|
||||
}
|
||||
recordConnectionError(connectionId, e);
|
||||
clearConnectionNodeLoading(connectionId);
|
||||
throw e;
|
||||
} finally {
|
||||
if (connectInFlight.get(connectionId) === connectPromise) {
|
||||
|
||||
@@ -49,6 +49,7 @@ const STORAGE_KEY = "dbx-open-tabs";
|
||||
const ACTIVE_TAB_KEY = "dbx-active-tab";
|
||||
const ORACLE_LIKE_METADATA_TYPES = new Set<string>(["oracle", "dameng", "oceanbase-oracle"]);
|
||||
const BACKGROUND_CLIENT_SESSION_SUFFIXES = ["count", "explain", "export"] as const;
|
||||
const CANCEL_QUERY_TIMEOUT_MS = 10_000;
|
||||
|
||||
interface BuildQueryResultExportRequestOptions {
|
||||
exportId: string;
|
||||
@@ -98,6 +99,20 @@ async function withFrontendQueryTimeout<T>(promise: Promise<T>, timeoutSecs: num
|
||||
}
|
||||
}
|
||||
|
||||
async function withCancelQueryTimeout<T>(promise: Promise<T>): Promise<T> {
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
try {
|
||||
return await Promise.race([
|
||||
promise,
|
||||
new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => reject(new Error("Cancel request timed out after 10s.")), CANCEL_QUERY_TIMEOUT_MS);
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
if (timer) clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeOracleLikeMetadataIdentifier(dbType: string, identifier: string | undefined, quoted?: boolean) {
|
||||
if (!identifier || quoted || !ORACLE_LIKE_METADATA_TYPES.has(dbType)) return identifier;
|
||||
return identifier.toUpperCase();
|
||||
@@ -1946,7 +1961,7 @@ export const useQueryStore = defineStore("query", () => {
|
||||
if (!executionId) return false;
|
||||
tab.isCancelling = true;
|
||||
try {
|
||||
const canceled = await api.cancelQuery(executionId);
|
||||
const canceled = await withCancelQueryTimeout(api.cancelQuery(executionId));
|
||||
if (!canceled) {
|
||||
const current = tabs.value.find((t) => t.id === id);
|
||||
if (current && current.executionId === executionId) {
|
||||
@@ -1962,7 +1977,10 @@ export const useQueryStore = defineStore("query", () => {
|
||||
if (tab) useConnectionStore().recordConnectionLostError(tab.connectionId, e);
|
||||
const current = tabs.value.find((t) => t.id === id);
|
||||
if (current && current.executionId === executionId) {
|
||||
current.isExecuting = false;
|
||||
current.isCancelling = false;
|
||||
current.executionId = undefined;
|
||||
current.queryExecutionStartedAt = undefined;
|
||||
current.result = toErrorResult(e);
|
||||
}
|
||||
return false;
|
||||
|
||||
Reference in new issue
Block a user