fix(metadata): bound cache maintenance and pool recovery

This commit is contained in:
miracle authored and GitHub committed 2026-08-19 11:10:12 +08:00
1 parent 21af5cf600
commit 8c2b232243
41 files changed
+2747 -350

No files matched your search

@@ -146,6 +146,7 @@ import { MAX_RESULT_PAGE_SIZE, MIN_RESULT_PAGE_SIZE } from "@/lib/dataGrid/pagin
import { MAX_QUERY_RESULT_MAX_ROWS } from "@/lib/dataGrid/queryResultRowLimit"; import { MAX_QUERY_RESULT_MAX_ROWS } from "@/lib/dataGrid/queryResultRowLimit";
import type { PromptTemplate } from "@/types/promptTemplate"; import type { PromptTemplate } from "@/types/promptTemplate";
import { GLOBAL_INSTRUCTIONS_MAX, PROMPT_TEMPLATE_CONTENT_MAX, PROMPT_TEMPLATE_NAME_MAX, promptTemplateCharacterCount } from "@/types/promptTemplate"; import { GLOBAL_INSTRUCTIONS_MAX, PROMPT_TEMPLATE_CONTENT_MAX, PROMPT_TEMPLATE_NAME_MAX, promptTemplateCharacterCount } from "@/types/promptTemplate";
import { METADATA_CACHE_HARD_MAX_MEMORY_MB, METADATA_CACHE_MIN_MEMORY_MB, normalizeMetadataCacheMemoryMb } from "@/lib/metadata/metadataRuntimeCache";
const { t } = useI18n(); const { t } = useI18n();
const { toast } = useToast(); const { toast } = useToast();
@@ -342,6 +343,7 @@ const editQuitOnClose = ref(settingsStore.desktopSettings.quit_on_close);
const desktopCloseBehaviorResetPending = ref(false); const desktopCloseBehaviorResetPending = ref(false);
const editIconTheme = ref<DesktopIconTheme>(settingsStore.desktopSettings.icon_theme); const editIconTheme = ref<DesktopIconTheme>(settingsStore.desktopSettings.icon_theme);
const editDebugLoggingEnabled = ref(settingsStore.desktopSettings.debug_logging_enabled); const editDebugLoggingEnabled = ref(settingsStore.desktopSettings.debug_logging_enabled);
const editMetadataCacheMaxMemoryMb = ref(settingsStore.desktopSettings.metadata_cache_max_memory_mb);
const editDuckDbWorkerProcessIsolation = ref(settingsStore.desktopSettings.duckdb_worker_process_isolation); const editDuckDbWorkerProcessIsolation = ref(settingsStore.desktopSettings.duckdb_worker_process_isolation);
const editDuckDbWorkerMaxProcesses = ref(settingsStore.desktopSettings.duckdb_worker_max_processes); const editDuckDbWorkerMaxProcesses = ref(settingsStore.desktopSettings.duckdb_worker_max_processes);
const startupDuckDbWorkerProcessIsolation = ref(settingsStore.desktopSettings.duckdb_worker_process_isolation); const startupDuckDbWorkerProcessIsolation = ref(settingsStore.desktopSettings.duckdb_worker_process_isolation);
@@ -887,6 +889,7 @@ watch(
editQuitOnClose.value = settingsStore.desktopSettings.quit_on_close; editQuitOnClose.value = settingsStore.desktopSettings.quit_on_close;
editIconTheme.value = settingsStore.desktopSettings.icon_theme; editIconTheme.value = settingsStore.desktopSettings.icon_theme;
editDebugLoggingEnabled.value = settingsStore.desktopSettings.debug_logging_enabled; editDebugLoggingEnabled.value = settingsStore.desktopSettings.debug_logging_enabled;
editMetadataCacheMaxMemoryMb.value = settingsStore.desktopSettings.metadata_cache_max_memory_mb;
editDuckDbWorkerProcessIsolation.value = settingsStore.desktopSettings.duckdb_worker_process_isolation; editDuckDbWorkerProcessIsolation.value = settingsStore.desktopSettings.duckdb_worker_process_isolation;
editSidebarTablePageSize.value = settingsStore.desktopSettings.sidebar_table_page_size ?? DEFAULT_SIDEBAR_TABLE_PAGE_SIZE; editSidebarTablePageSize.value = settingsStore.desktopSettings.sidebar_table_page_size ?? DEFAULT_SIDEBAR_TABLE_PAGE_SIZE;
} }
@@ -960,6 +963,7 @@ function hasChanges(): boolean {
editQuitOnClose.value !== settingsStore.desktopSettings.quit_on_close || editQuitOnClose.value !== settingsStore.desktopSettings.quit_on_close ||
editIconTheme.value !== settingsStore.desktopSettings.icon_theme || editIconTheme.value !== settingsStore.desktopSettings.icon_theme ||
editDebugLoggingEnabled.value !== settingsStore.desktopSettings.debug_logging_enabled || editDebugLoggingEnabled.value !== settingsStore.desktopSettings.debug_logging_enabled ||
editMetadataCacheMaxMemoryMb.value !== settingsStore.desktopSettings.metadata_cache_max_memory_mb ||
editDuckDbWorkerProcessIsolation.value !== settingsStore.desktopSettings.duckdb_worker_process_isolation || editDuckDbWorkerProcessIsolation.value !== settingsStore.desktopSettings.duckdb_worker_process_isolation ||
normalizeDuckDbWorkerMaxProcesses(editDuckDbWorkerMaxProcesses.value) !== settingsStore.desktopSettings.duckdb_worker_max_processes || normalizeDuckDbWorkerMaxProcesses(editDuckDbWorkerMaxProcesses.value) !== settingsStore.desktopSettings.duckdb_worker_max_processes ||
editSidebarTablePageSize.value !== (settingsStore.desktopSettings.sidebar_table_page_size ?? DEFAULT_SIDEBAR_TABLE_PAGE_SIZE) editSidebarTablePageSize.value !== (settingsStore.desktopSettings.sidebar_table_page_size ?? DEFAULT_SIDEBAR_TABLE_PAGE_SIZE)
@@ -976,16 +980,19 @@ async function persistSettings() {
await settingsStore.persistEditorSettings(); await settingsStore.persistEditorSettings();
editEditorSettingsBase.value = editorSettingsDraftFromSettings(settingsStore.editorSettings); editEditorSettingsBase.value = editorSettingsDraftFromSettings(settingsStore.editorSettings);
} }
const metadataCacheMaxMemoryMb = normalizeMetadataCacheMemoryMb(editMetadataCacheMaxMemoryMb.value);
await settingsStore.updateDesktopSettings({ await settingsStore.updateDesktopSettings({
show_tray_icon: editShowTrayIcon.value, show_tray_icon: editShowTrayIcon.value,
quit_on_close: editQuitOnClose.value, quit_on_close: editQuitOnClose.value,
close_action_prompted: desktopCloseBehaviorResetPending.value ? false : true, close_action_prompted: desktopCloseBehaviorResetPending.value ? false : true,
icon_theme: editIconTheme.value, icon_theme: editIconTheme.value,
debug_logging_enabled: editDebugLoggingEnabled.value, debug_logging_enabled: editDebugLoggingEnabled.value,
metadata_cache_max_memory_mb: metadataCacheMaxMemoryMb,
duckdb_worker_process_isolation: editDuckDbWorkerProcessIsolation.value, duckdb_worker_process_isolation: editDuckDbWorkerProcessIsolation.value,
duckdb_worker_max_processes: normalizeDuckDbWorkerMaxProcesses(editDuckDbWorkerMaxProcesses.value), duckdb_worker_max_processes: normalizeDuckDbWorkerMaxProcesses(editDuckDbWorkerMaxProcesses.value),
sidebar_table_page_size: editSidebarTablePageSize.value, sidebar_table_page_size: editSidebarTablePageSize.value,
}); });
editMetadataCacheMaxMemoryMb.value = settingsStore.desktopSettings.metadata_cache_max_memory_mb;
desktopCloseBehaviorResetPending.value = false; desktopCloseBehaviorResetPending.value = false;
if (sidebarObjectDisplayChanged) { if (sidebarObjectDisplayChanged) {
await connectionStore.refreshAllTree(); await connectionStore.refreshAllTree();
@@ -1081,6 +1088,7 @@ function resetDefaultsForTab(tab: SettingsCategory) {
desktopCloseBehaviorResetPending.value = true; desktopCloseBehaviorResetPending.value = true;
editIconTheme.value = DEFAULT_DESKTOP_SETTINGS.icon_theme; editIconTheme.value = DEFAULT_DESKTOP_SETTINGS.icon_theme;
editDebugLoggingEnabled.value = DEFAULT_DESKTOP_SETTINGS.debug_logging_enabled; editDebugLoggingEnabled.value = DEFAULT_DESKTOP_SETTINGS.debug_logging_enabled;
editMetadataCacheMaxMemoryMb.value = DEFAULT_DESKTOP_SETTINGS.metadata_cache_max_memory_mb;
} else if (tab === "navigation") { } else if (tab === "navigation") {
editSidebarTablePageSize.value = DEFAULT_SIDEBAR_TABLE_PAGE_SIZE; editSidebarTablePageSize.value = DEFAULT_SIDEBAR_TABLE_PAGE_SIZE;
editSidebarActivation.value = DEFAULT_EDITOR_SETTINGS.sidebarActivation; editSidebarActivation.value = DEFAULT_EDITOR_SETTINGS.sidebarActivation;
@@ -1176,6 +1184,7 @@ function resetAllDefaults() {
desktopCloseBehaviorResetPending.value = true; desktopCloseBehaviorResetPending.value = true;
editIconTheme.value = DEFAULT_DESKTOP_SETTINGS.icon_theme; editIconTheme.value = DEFAULT_DESKTOP_SETTINGS.icon_theme;
editDebugLoggingEnabled.value = DEFAULT_DESKTOP_SETTINGS.debug_logging_enabled; editDebugLoggingEnabled.value = DEFAULT_DESKTOP_SETTINGS.debug_logging_enabled;
editMetadataCacheMaxMemoryMb.value = DEFAULT_DESKTOP_SETTINGS.metadata_cache_max_memory_mb;
editDuckDbWorkerProcessIsolation.value = DEFAULT_DESKTOP_SETTINGS.duckdb_worker_process_isolation; editDuckDbWorkerProcessIsolation.value = DEFAULT_DESKTOP_SETTINGS.duckdb_worker_process_isolation;
editDuckDbWorkerMaxProcesses.value = DEFAULT_DESKTOP_SETTINGS.duckdb_worker_max_processes; editDuckDbWorkerMaxProcesses.value = DEFAULT_DESKTOP_SETTINGS.duckdb_worker_max_processes;
editSidebarTablePageSize.value = DEFAULT_SIDEBAR_TABLE_PAGE_SIZE; editSidebarTablePageSize.value = DEFAULT_SIDEBAR_TABLE_PAGE_SIZE;
@@ -2441,6 +2450,7 @@ watch(
editQuitOnClose.value = settingsStore.desktopSettings.quit_on_close; editQuitOnClose.value = settingsStore.desktopSettings.quit_on_close;
editIconTheme.value = settingsStore.desktopSettings.icon_theme; editIconTheme.value = settingsStore.desktopSettings.icon_theme;
editDebugLoggingEnabled.value = settingsStore.desktopSettings.debug_logging_enabled; editDebugLoggingEnabled.value = settingsStore.desktopSettings.debug_logging_enabled;
editMetadataCacheMaxMemoryMb.value = settingsStore.desktopSettings.metadata_cache_max_memory_mb;
editDuckDbWorkerProcessIsolation.value = settingsStore.desktopSettings.duckdb_worker_process_isolation; editDuckDbWorkerProcessIsolation.value = settingsStore.desktopSettings.duckdb_worker_process_isolation;
editDuckDbWorkerMaxProcesses.value = settingsStore.desktopSettings.duckdb_worker_max_processes; editDuckDbWorkerMaxProcesses.value = settingsStore.desktopSettings.duckdb_worker_max_processes;
if (!duckDbWorkerStartupCaptured.value) { if (!duckDbWorkerStartupCaptured.value) {
@@ -5365,6 +5375,26 @@ onUnmounted(() => {
<Separator /> <Separator />
<div data-settings-search-id="data-performance" :class="['space-y-3', settingsSearchTargetClass('data-performance')]">
<div class="text-sm font-medium text-muted-foreground">
{{ t("settings.performanceSection") }}
</div>
<div class="flex items-center justify-between gap-4 rounded-md border bg-muted/20 px-3 py-2">
<div class="min-w-0 space-y-1">
<Label for="metadata-cache-memory-limit">{{ t("settings.metadataCacheMemoryLimit") }}</Label>
<p class="text-xs text-muted-foreground">
{{ t("settings.metadataCacheMemoryLimitDescription") }}
</p>
</div>
<div class="flex shrink-0 items-center gap-2">
<Input id="metadata-cache-memory-limit" v-model.number="editMetadataCacheMaxMemoryMb" type="number" :min="METADATA_CACHE_MIN_MEMORY_MB" :max="METADATA_CACHE_HARD_MAX_MEMORY_MB" :step="16" class="h-8 w-24 text-right" />
<span class="text-xs text-muted-foreground">MB</span>
</div>
</div>
</div>
<Separator />
<div class="space-y-3"> <div class="space-y-3">
<div class="text-sm font-medium text-muted-foreground"> <div class="text-sm font-medium text-muted-foreground">
{{ t("settings.tableStructureSection") }} {{ t("settings.tableStructureSection") }}
@@ -2612,7 +2612,7 @@ async function resolveSqlHoverTooltip(currentView: EditorViewType, pos: number)
try { try {
const [columnsResult, indexesResult] = await Promise.all([ const [columnsResult, indexesResult] = await Promise.all([
loadObjectMetadataFacet(objectMetadataRequest, "columns", () => api.getColumns(props.connectionId!, hoverDatabase, hoverSchema, table.name, hoverScope.catalog)), loadObjectMetadataFacet(objectMetadataRequest, "columns", () => api.getColumns(props.connectionId!, hoverDatabase, hoverSchema, table.name, hoverScope.catalog)),
loadObjectMetadataFacet(objectMetadataRequest, "indexes", () => api.listIndexes(props.connectionId!, hoverDatabase, hoverSchema, table.name, hoverScope.catalog).catch(() => [])), loadObjectMetadataFacet(objectMetadataRequest, "indexes", () => api.listIndexes(props.connectionId!, hoverDatabase, hoverSchema, table.name, hoverScope.catalog)).catch(() => ({ value: [] as IndexInfo[], cacheStatus: "remote" as const })),
]); ]);
fullColumns = columnsResult.value; fullColumns = columnsResult.value;
fullIndexes = indexesResult.value; fullIndexes = indexesResult.value;
@@ -129,6 +129,8 @@ import { tableColumnDefaultDisplayValue } from "@/lib/table/tableColumnDefaultPr
import { gaussdbMTypeDisplayName } from "@/lib/table/postgresDataTypeHelp"; import { gaussdbMTypeDisplayName } from "@/lib/table/postgresDataTypeHelp";
import { cacheObjectBrowserRows, createObjectBrowserRowsCacheWriteToken, getCachedObjectBrowserRows, type ObjectBrowserRowsCacheScope, type ObjectBrowserRowsCacheWriteToken } from "@/lib/table/objectBrowserRowsCache"; import { cacheObjectBrowserRows, createObjectBrowserRowsCacheWriteToken, getCachedObjectBrowserRows, type ObjectBrowserRowsCacheScope, type ObjectBrowserRowsCacheWriteToken } from "@/lib/table/objectBrowserRowsCache";
import { createObjectBrowserRowsLoadGuard, type ObjectBrowserRowsLoadHandle } from "@/lib/table/objectBrowserRowsLoadGuard"; import { createObjectBrowserRowsLoadGuard, type ObjectBrowserRowsLoadHandle } from "@/lib/table/objectBrowserRowsLoadGuard";
import { loadObjectDdl, type ObjectDdlRequest } from "@/lib/metadata/objectDdlCache";
import { loadObjectMetadataFacet } from "@/lib/metadata/objectMetadataCache";
type ObjectFilter = ObjectBrowserFilter; type ObjectFilter = ObjectBrowserFilter;
type ObjectBrowserColumnKey = "select" | "name" | "type" | "estimatedRows" | "totalBytes" | "created_at" | "updated_at" | "comment"; type ObjectBrowserColumnKey = "select" | "name" | "type" | "estimatedRows" | "totalBytes" | "created_at" | "updated_at" | "comment";
@@ -183,14 +185,19 @@ const sidePanelMode = ref<"table-info" | "source" | "type-info">("source");
const tableInfoTab = ref<TableInfoTab>("ddl"); const tableInfoTab = ref<TableInfoTab>("ddl");
const tableColumns = ref<ColumnInfo[]>([]); const tableColumns = ref<ColumnInfo[]>([]);
const tableColumnsLoading = ref(false); const tableColumnsLoading = ref(false);
const tableColumnsLoaded = ref(false);
const tableDdlContent = ref(""); const tableDdlContent = ref("");
const tableDdlLoading = ref(false); const tableDdlLoading = ref(false);
const tableDdlLoaded = ref(false);
const tableIndexes = ref<IndexInfo[]>([]); const tableIndexes = ref<IndexInfo[]>([]);
const tableIndexesLoading = ref(false); const tableIndexesLoading = ref(false);
const tableIndexesLoaded = ref(false);
const tableForeignKeys = ref<ForeignKeyInfo[]>([]); const tableForeignKeys = ref<ForeignKeyInfo[]>([]);
const tableForeignKeysLoading = ref(false); const tableForeignKeysLoading = ref(false);
const tableForeignKeysLoaded = ref(false);
const tableTriggers = ref<TriggerInfo[]>([]); const tableTriggers = ref<TriggerInfo[]>([]);
const tableTriggersLoading = ref(false); const tableTriggersLoading = ref(false);
const tableTriggersLoaded = ref(false);
const tableInfoSearchQuery = ref(""); const tableInfoSearchQuery = ref("");
const tableInfoDdlPreRef = ref<HTMLPreElement | null>(null); const tableInfoDdlPreRef = ref<HTMLPreElement | null>(null);
const SIDE_PANEL_MIN_WIDTH = 280; const SIDE_PANEL_MIN_WIDTH = 280;
@@ -973,6 +980,11 @@ async function openTableInfo(row: ObjectBrowserRow, initialTab?: TableInfoTab) {
tableIndexes.value = []; tableIndexes.value = [];
tableForeignKeys.value = []; tableForeignKeys.value = [];
tableTriggers.value = []; tableTriggers.value = [];
tableColumnsLoaded.value = false;
tableDdlLoaded.value = false;
tableIndexesLoaded.value = false;
tableForeignKeysLoaded.value = false;
tableTriggersLoaded.value = false;
tableInfoSearchQuery.value = ""; tableInfoSearchQuery.value = "";
// Determine initial tab: explicit request > previously activated // Determine initial tab: explicit request > previously activated
const firstTab = initialTab ?? tableInfoTab.value; const firstTab = initialTab ?? tableInfoTab.value;
@@ -991,93 +1003,132 @@ async function selectTableInfoTab(tab: TableInfoTab) {
else if (nextTab === "triggers") await fetchTableTriggers(); else if (nextTab === "triggers") await fetchTableTriggers();
} }
async function fetchTableDdl() { function tableMetadataRequest(row: ObjectBrowserRow): ObjectDdlRequest {
return {
connectionId: props.connection.id,
database: props.database || "",
schema: row.schema || selectedSchema.value || props.database,
tableName: row.name,
objectType: tableDdlObjectType(row.type),
catalog: props.catalog,
};
}
async function fetchTableDdl(force = false) {
const row = sidePanelRow.value; const row = sidePanelRow.value;
if (!row) return; if (!row || (tableDdlLoaded.value && !force)) return;
const epoch = sidePanelGuard.capture(); const epoch = sidePanelGuard.capture();
tableDdlLoading.value = true; tableDdlLoading.value = true;
let loadedSuccessfully = false;
try { try {
const schema = row.schema || selectedSchema.value || props.database; const { ddl } = await loadObjectDdl(tableMetadataRequest(row), { force });
const ddl = await api.getTableDisplayDdl(props.connection.id, props.database || "", schema, row.name, tableDdlObjectType(row.type), props.catalog);
if (sidePanelGuard.isStale(epoch)) return; if (sidePanelGuard.isStale(epoch)) return;
tableDdlContent.value = ddl; tableDdlContent.value = ddl;
loadedSuccessfully = true;
} catch (e: any) { } catch (e: any) {
if (sidePanelGuard.isStale(epoch)) return; if (sidePanelGuard.isStale(epoch)) return;
tableDdlContent.value = `-- Error: ${e?.message || e}`; tableDdlContent.value = `-- Error: ${e?.message || e}`;
} finally { } finally {
if (sidePanelGuard.isFresh(epoch)) tableDdlLoading.value = false; if (sidePanelGuard.isFresh(epoch)) {
tableDdlLoaded.value = loadedSuccessfully;
tableDdlLoading.value = false;
}
} }
} }
async function fetchTableColumns() { async function fetchTableColumns(force = false) {
const row = sidePanelRow.value; const row = sidePanelRow.value;
if (!row || tableColumns.value.length > 0) return; if (!row || (tableColumnsLoaded.value && !force)) return;
const epoch = sidePanelGuard.capture(); const epoch = sidePanelGuard.capture();
tableColumnsLoading.value = true; tableColumnsLoading.value = true;
let loadedSuccessfully = false;
try { try {
const schema = row.schema || selectedSchema.value || props.database; const request = tableMetadataRequest(row);
const columns = await api.getColumns(props.connection.id, props.database || "", schema, row.name, props.catalog); const { value: columns } = await loadObjectMetadataFacet(request, "columns", () => api.getColumns(request.connectionId, request.database, request.schema || request.database, request.tableName, request.catalog), { force });
if (sidePanelGuard.isStale(epoch)) return; if (sidePanelGuard.isStale(epoch)) return;
tableColumns.value = columns; tableColumns.value = columns;
} catch { loadedSuccessfully = true;
} catch (error) {
if (sidePanelGuard.isStale(epoch)) return; if (sidePanelGuard.isStale(epoch)) return;
tableColumns.value = []; tableColumns.value = [];
toast(translateBackendError(t, error), 5000);
} finally { } finally {
if (sidePanelGuard.isFresh(epoch)) tableColumnsLoading.value = false; if (sidePanelGuard.isFresh(epoch)) {
tableColumnsLoaded.value = loadedSuccessfully;
tableColumnsLoading.value = false;
}
} }
} }
async function fetchTableIndexes() { async function fetchTableIndexes(force = false) {
const row = sidePanelRow.value; const row = sidePanelRow.value;
if (!row || tableIndexes.value.length > 0) return; if (!row || (tableIndexesLoaded.value && !force)) return;
const epoch = sidePanelGuard.capture(); const epoch = sidePanelGuard.capture();
tableIndexesLoading.value = true; tableIndexesLoading.value = true;
let loadedSuccessfully = false;
try { try {
const schema = row.schema || selectedSchema.value || props.database; const request = tableMetadataRequest(row);
const indexes = await api.listIndexes(props.connection.id, props.database || "", schema, row.name, props.catalog); const { value: indexes } = await loadObjectMetadataFacet(request, "indexes", () => api.listIndexes(request.connectionId, request.database, request.schema || request.database, request.tableName, request.catalog), { force });
if (sidePanelGuard.isStale(epoch)) return; if (sidePanelGuard.isStale(epoch)) return;
tableIndexes.value = indexes; tableIndexes.value = indexes;
} catch { loadedSuccessfully = true;
} catch (error) {
if (sidePanelGuard.isStale(epoch)) return; if (sidePanelGuard.isStale(epoch)) return;
tableIndexes.value = []; tableIndexes.value = [];
toast(translateBackendError(t, error), 5000);
} finally { } finally {
if (sidePanelGuard.isFresh(epoch)) tableIndexesLoading.value = false; if (sidePanelGuard.isFresh(epoch)) {
tableIndexesLoaded.value = loadedSuccessfully;
tableIndexesLoading.value = false;
}
} }
} }
async function fetchTableForeignKeys() { async function fetchTableForeignKeys(force = false) {
const row = sidePanelRow.value; const row = sidePanelRow.value;
if (!row || tableForeignKeys.value.length > 0) return; if (!row || (tableForeignKeysLoaded.value && !force)) return;
const epoch = sidePanelGuard.capture(); const epoch = sidePanelGuard.capture();
tableForeignKeysLoading.value = true; tableForeignKeysLoading.value = true;
let loadedSuccessfully = false;
try { try {
const schema = row.schema || selectedSchema.value || props.database; const request = tableMetadataRequest(row);
const fks = await api.listForeignKeys(props.connection.id, props.database || "", schema, row.name, props.catalog); const { value: fks } = await loadObjectMetadataFacet(request, "foreign-keys", () => api.listForeignKeys(request.connectionId, request.database, request.schema || request.database, request.tableName, request.catalog), { force });
if (sidePanelGuard.isStale(epoch)) return; if (sidePanelGuard.isStale(epoch)) return;
tableForeignKeys.value = fks; tableForeignKeys.value = fks;
} catch { loadedSuccessfully = true;
} catch (error) {
if (sidePanelGuard.isStale(epoch)) return; if (sidePanelGuard.isStale(epoch)) return;
tableForeignKeys.value = []; tableForeignKeys.value = [];
toast(translateBackendError(t, error), 5000);
} finally { } finally {
if (sidePanelGuard.isFresh(epoch)) tableForeignKeysLoading.value = false; if (sidePanelGuard.isFresh(epoch)) {
tableForeignKeysLoaded.value = loadedSuccessfully;
tableForeignKeysLoading.value = false;
}
} }
} }
async function fetchTableTriggers() { async function fetchTableTriggers(force = false) {
const row = sidePanelRow.value; const row = sidePanelRow.value;
if (!row || tableTriggers.value.length > 0) return; if (!row || (tableTriggersLoaded.value && !force)) return;
const epoch = sidePanelGuard.capture(); const epoch = sidePanelGuard.capture();
tableTriggersLoading.value = true; tableTriggersLoading.value = true;
let loadedSuccessfully = false;
try { try {
const schema = row.schema || selectedSchema.value || props.database; const request = tableMetadataRequest(row);
const triggers = await api.listTriggers(props.connection.id, props.database || "", schema, row.name, props.catalog); const { value: triggers } = await loadObjectMetadataFacet(request, "triggers", () => api.listTriggers(request.connectionId, request.database, request.schema || request.database, request.tableName, request.catalog), { force });
if (sidePanelGuard.isStale(epoch)) return; if (sidePanelGuard.isStale(epoch)) return;
tableTriggers.value = triggers; tableTriggers.value = triggers;
} catch { loadedSuccessfully = true;
} catch (error) {
if (sidePanelGuard.isStale(epoch)) return; if (sidePanelGuard.isStale(epoch)) return;
tableTriggers.value = []; tableTriggers.value = [];
toast(translateBackendError(t, error), 5000);
} finally { } finally {
if (sidePanelGuard.isFresh(epoch)) tableTriggersLoading.value = false; if (sidePanelGuard.isFresh(epoch)) {
tableTriggersLoaded.value = loadedSuccessfully;
tableTriggersLoading.value = false;
}
} }
} }
@@ -1087,19 +1138,24 @@ async function refreshActiveTableInfo() {
if (tableInfoTab.value === "ddl") { if (tableInfoTab.value === "ddl") {
tableDdlContent.value = ""; tableDdlContent.value = "";
await fetchTableDdl(); tableDdlLoaded.value = false;
await fetchTableDdl(true);
} else if (tableInfoTab.value === "columns") { } else if (tableInfoTab.value === "columns") {
tableColumns.value = []; tableColumns.value = [];
await fetchTableColumns(); tableColumnsLoaded.value = false;
await fetchTableColumns(true);
} else if (tableInfoTab.value === "indexes") { } else if (tableInfoTab.value === "indexes") {
tableIndexes.value = []; tableIndexes.value = [];
await fetchTableIndexes(); tableIndexesLoaded.value = false;
await fetchTableIndexes(true);
} else if (tableInfoTab.value === "foreignKeys") { } else if (tableInfoTab.value === "foreignKeys") {
tableForeignKeys.value = []; tableForeignKeys.value = [];
await fetchTableForeignKeys(); tableForeignKeysLoaded.value = false;
await fetchTableForeignKeys(true);
} else if (tableInfoTab.value === "triggers") { } else if (tableInfoTab.value === "triggers") {
tableTriggers.value = []; tableTriggers.value = [];
await fetchTableTriggers(); tableTriggersLoaded.value = false;
await fetchTableTriggers(true);
} }
} }
@@ -30,14 +30,33 @@ describe("ObjectBrowser table metadata refresh", () => {
expect(refreshTableInfo).toContain('sidePanelMode.value !== "table-info" || !sidePanelRow.value'); expect(refreshTableInfo).toContain('sidePanelMode.value !== "table-info" || !sidePanelRow.value');
expect(refreshTableInfo).toContain("sidePanelGuard.bump();"); expect(refreshTableInfo).toContain("sidePanelGuard.bump();");
expect(refreshTableInfo).toMatch(/tableInfoTab\.value === "ddl"[\s\S]*?tableDdlContent\.value = "";[\s\S]*?await fetchTableDdl\(\);/); expect(refreshTableInfo).toMatch(/tableInfoTab\.value === "ddl"[\s\S]*?tableDdlContent\.value = "";[\s\S]*?await fetchTableDdl\(true\);/);
expect(refreshTableInfo).toMatch(/tableInfoTab\.value === "columns"[\s\S]*?tableColumns\.value = \[\];[\s\S]*?await fetchTableColumns\(\);/); expect(refreshTableInfo).toMatch(/tableInfoTab\.value === "columns"[\s\S]*?tableColumns\.value = \[\];[\s\S]*?await fetchTableColumns\(true\);/);
expect(refreshTableInfo).toMatch(/tableInfoTab\.value === "indexes"[\s\S]*?tableIndexes\.value = \[\];[\s\S]*?await fetchTableIndexes\(\);/); expect(refreshTableInfo).toMatch(/tableInfoTab\.value === "indexes"[\s\S]*?tableIndexes\.value = \[\];[\s\S]*?await fetchTableIndexes\(true\);/);
expect(refreshTableInfo).toMatch(/tableInfoTab\.value === "foreignKeys"[\s\S]*?tableForeignKeys\.value = \[\];[\s\S]*?await fetchTableForeignKeys\(\);/); expect(refreshTableInfo).toMatch(/tableInfoTab\.value === "foreignKeys"[\s\S]*?tableForeignKeys\.value = \[\];[\s\S]*?await fetchTableForeignKeys\(true\);/);
expect(refreshTableInfo).toMatch(/tableInfoTab\.value === "triggers"[\s\S]*?tableTriggers\.value = \[\];[\s\S]*?await fetchTableTriggers\(\);/); expect(refreshTableInfo).toMatch(/tableInfoTab\.value === "triggers"[\s\S]*?tableTriggers\.value = \[\];[\s\S]*?await fetchTableTriggers\(true\);/);
});
it("routes table-info metadata through the object caches", () => {
expect(source).toContain('from "@/lib/metadata/objectDdlCache"');
expect(source).toContain('from "@/lib/metadata/objectMetadataCache"');
expect(functionBody("fetchTableDdl")).toContain("loadObjectDdl(");
for (const name of ["fetchTableColumns", "fetchTableIndexes", "fetchTableForeignKeys", "fetchTableTriggers"]) {
expect(functionBody(name)).toContain("loadObjectMetadataFacet(");
expect(functionBody(name)).not.toContain(".value.length > 0");
}
}); });
it("keeps automatic object reloads free of extra metadata requests", () => { it("keeps automatic object reloads free of extra metadata requests", () => {
expect(functionBody("reload")).not.toContain("refreshActiveTableInfo"); expect(functionBody("reload")).not.toContain("refreshActiveTableInfo");
}); });
it("leaves a failed facet eligible for retry", () => {
for (const name of ["fetchTableDdl", "fetchTableColumns", "fetchTableIndexes", "fetchTableForeignKeys", "fetchTableTriggers"]) {
const body = functionBody(name);
expect(body).toContain("let loadedSuccessfully = false;");
expect(body).toMatch(/loadedSuccessfully = true;/);
expect(body).toMatch(/Loaded\.value = loadedSuccessfully;/);
}
});
}); });
@@ -1505,7 +1505,7 @@ async function fetchTableCommentValue(connectionId: string, database: string, sc
} }
} }
function loadCachedTableComment(request: ReturnType<typeof ddlRequest>, force = false): Promise<{ value: string | undefined; cacheStatus: "disk" | "remote" }> { function loadCachedTableComment(request: ReturnType<typeof ddlRequest>, force = false): Promise<{ value: string | undefined; cacheStatus: "memory" | "disk" | "remote" }> {
return loadObjectMetadataFacet(request, "comment", () => fetchTableCommentValue(request.connectionId, request.database, request.schema, request.tableName, request.catalog), { force }); return loadObjectMetadataFacet(request, "comment", () => fetchTableCommentValue(request.connectionId, request.database, request.schema, request.tableName, request.catalog), { force });
} }
@@ -1547,22 +1547,22 @@ async function loadStructure(
const columnsPromise = scope.columns ? loadObjectMetadataFacet(metadataRequest, "columns", () => api.getColumns(connectionId, database, schema, tableName, catalog), { force: forceMetadata }).then((result) => result.value) : Promise.resolve(undefined); const columnsPromise = scope.columns ? loadObjectMetadataFacet(metadataRequest, "columns", () => api.getColumns(connectionId, database, schema, tableName, catalog), { force: forceMetadata }).then((result) => result.value) : Promise.resolve(undefined);
const indexesPromise = scope.indexes const indexesPromise = scope.indexes
? tableMetadataCapabilities.value.indexes ? tableMetadataCapabilities.value.indexes
? loadObjectMetadataFacet(metadataRequest, "indexes", () => api.listIndexes(connectionId, database, schema, tableName, catalog).catch(() => []), { force: forceMetadata }).then((result) => result.value) ? loadObjectMetadataFacet(metadataRequest, "indexes", () => api.listIndexes(connectionId, database, schema, tableName, catalog), { force: forceMetadata }).then((result) => result.value)
: Promise.resolve([]) : Promise.resolve([])
: Promise.resolve(undefined); : Promise.resolve(undefined);
const foreignKeysPromise = scope.foreignKeys const foreignKeysPromise = scope.foreignKeys
? tableMetadataCapabilities.value.foreignKeys ? tableMetadataCapabilities.value.foreignKeys
? loadObjectMetadataFacet(metadataRequest, "foreign-keys", () => api.listForeignKeys(connectionId, database, schema, tableName, catalog).catch(() => []), { force: forceMetadata }).then((result) => result.value) ? loadObjectMetadataFacet(metadataRequest, "foreign-keys", () => api.listForeignKeys(connectionId, database, schema, tableName, catalog), { force: forceMetadata }).then((result) => result.value)
: Promise.resolve([]) : Promise.resolve([])
: Promise.resolve(undefined); : Promise.resolve(undefined);
const constraintsPromise = scope.constraints const constraintsPromise = scope.constraints
? tableMetadataCapabilities.value.constraints ? tableMetadataCapabilities.value.constraints
? loadObjectMetadataFacet(metadataRequest, "constraints", () => api.listConstraints(connectionId, database, schema, tableName, catalog).catch(() => []), { force: forceMetadata }).then((result) => result.value) ? loadObjectMetadataFacet(metadataRequest, "constraints", () => api.listConstraints(connectionId, database, schema, tableName, catalog), { force: forceMetadata }).then((result) => result.value)
: Promise.resolve([]) : Promise.resolve([])
: Promise.resolve(undefined); : Promise.resolve(undefined);
const triggersPromise = scope.triggers const triggersPromise = scope.triggers
? tableMetadataCapabilities.value.triggers ? tableMetadataCapabilities.value.triggers
? loadObjectMetadataFacet(metadataRequest, "triggers", () => api.listTriggers(connectionId, database, schema, tableName, catalog).catch(() => []), { force: forceMetadata }).then((result) => result.value) ? loadObjectMetadataFacet(metadataRequest, "triggers", () => api.listTriggers(connectionId, database, schema, tableName, catalog), { force: forceMetadata }).then((result) => result.value)
: Promise.resolve([]) : Promise.resolve([])
: Promise.resolve(undefined); : Promise.resolve(undefined);
const tableCommentPromise = scope.tableComment && structureCapabilities.value.comment ? loadCachedTableComment(metadataRequest, forceMetadata).then((result) => result.value) : Promise.resolve(undefined); const tableCommentPromise = scope.tableComment && structureCapabilities.value.comment ? loadCachedTableComment(metadataRequest, forceMetadata).then((result) => result.value) : Promise.resolve(undefined);
@@ -1636,6 +1636,7 @@ async function loadStructure(
const secondaryMetadataPromise = applySecondaryMetadata() const secondaryMetadataPromise = applySecondaryMetadata()
.catch((error) => { .catch((error) => {
console.warn("[DBX][structure-editor:secondary-metadata-failed]", error); console.warn("[DBX][structure-editor:secondary-metadata-failed]", error);
if (showErrors && requestId === structureLoadRequestId) errorMessage.value = error?.message || String(error);
}) })
.finally(() => { .finally(() => {
if (requestId === structureLoadRequestId) setSecondaryMetadataLoading(scope, false); if (requestId === structureLoadRequestId) setSecondaryMetadataLoading(scope, false);
@@ -128,6 +128,33 @@ describe("useSqlExecution", () => {
setActivePinia(createPinia()); setActivePinia(createPinia());
}); });
it("invalidates object metadata after successful connection-level DDL", async () => {
const sql = "CREATE DATABASE app_db";
const activeTab = ref<QueryTab | undefined>({ ...queryTab(), sql });
const activeConnection = ref<ConnectionConfig | undefined>(connection("mysql"));
const activeOutputView = ref<"result" | "summary" | "explain" | "chart">("result");
const queryStore = useQueryStore();
const connectionStore = useConnectionStore();
vi.spyOn(queryStore, "executeCurrentSql").mockImplementation(async () => {
if (activeTab.value) activeTab.value.result = { columns: [], rows: [], affected_rows: 1, execution_time_ms: 1 };
});
vi.spyOn(useHistoryStore(), "add").mockResolvedValue(undefined);
const invalidateMetadata = vi.spyOn(connectionStore, "invalidateMetadataCache");
const loadDatabases = vi.spyOn(connectionStore, "loadDatabases").mockResolvedValue(undefined);
const execution = useSqlExecution({
activeTab: computed(() => activeTab.value),
activeConnection: computed(() => activeConnection.value),
executableSql: computed(() => sql),
activeOutputView,
});
await execution.tryExecute();
expect(invalidateMetadata).toHaveBeenCalledWith("conn-1");
expect(loadDatabases).toHaveBeenCalledWith("conn-1", { force: true });
});
it("sends every placeholder syntax and @set unchanged when substitution is disabled", async () => { it("sends every placeholder syntax and @set unchanged when substitution is disabled", async () => {
const sql = ["@set tenant = 42;", "SELECT ?, :named, ${shell}, #{mybatis}, @sqlserver, @tenant;"].join("\n"); const sql = ["@set tenant = 42;", "SELECT ?, :named, ${shell}, #{mybatis}, @sqlserver, @tenant;"].join("\n");
const activeTab = ref<QueryTab | undefined>({ ...queryTab("app"), sql }); const activeTab = ref<QueryTab | undefined>({ ...queryTab("app"), sql });
@@ -327,6 +327,7 @@ export function useSqlExecution(deps: {
if (success) { if (success) {
const refreshTarget = sqlMetadataRefreshTarget(sql, tab.schema); const refreshTarget = sqlMetadataRefreshTarget(sql, tab.schema);
if (refreshTarget.scope === "connection") { if (refreshTarget.scope === "connection") {
connectionStore.invalidateMetadataCache(tab.connectionId);
await connectionStore.loadDatabases(tab.connectionId, { force: true }); await connectionStore.loadDatabases(tab.connectionId, { force: true });
} else if (refreshTarget.scope === "database") { } else if (refreshTarget.scope === "database") {
await connectionStore.refreshObjectListTreeNode(tab.connectionId, tab.database, refreshTarget.schema); await connectionStore.refreshObjectListTreeNode(tab.connectionId, tab.database, refreshTarget.schema);
@@ -468,6 +469,7 @@ export function useSqlExecution(deps: {
if (success) { if (success) {
const refreshTarget = sqlMetadataRefreshTarget(sql, executionTab.schema); const refreshTarget = sqlMetadataRefreshTarget(sql, executionTab.schema);
if (refreshTarget.scope === "connection") { if (refreshTarget.scope === "connection") {
connectionStore.invalidateMetadataCache(executionTab.connectionId);
await connectionStore.loadDatabases(executionTab.connectionId, { force: true }); await connectionStore.loadDatabases(executionTab.connectionId, { force: true });
} else if (refreshTarget.scope === "database") { } else if (refreshTarget.scope === "database") {
await connectionStore.refreshObjectListTreeNode(executionTab.connectionId, executionTab.database, refreshTarget.schema); await connectionStore.refreshObjectListTreeNode(executionTab.connectionId, executionTab.database, refreshTarget.schema);
+3
View File
@@ -5851,6 +5851,8 @@ export default {
updateDownloadSourceCnb: "CNB", updateDownloadSourceCnb: "CNB",
debugLoggingEnabled: "Enable debug logs", debugLoggingEnabled: "Enable debug logs",
debugLoggingEnabledDescription: "Record more detailed local user-side diagnostic logs while troubleshooting reports. Disabled by default.", debugLoggingEnabledDescription: "Record more detailed local user-side diagnostic logs while troubleshooting reports. Disabled by default.",
metadataCacheMemoryLimit: "Metadata memory cache limit",
metadataCacheMemoryLimitDescription: "Defaults to 64 MB. Keep it at or below 256 MB when possible; the hard limit is 512 MB. Covers DDL, columns, indexes, foreign keys, triggers, and comments.",
duckDbWorkerProcessIsolation: "DuckDB process isolation mode (Experimental)", duckDbWorkerProcessIsolation: "DuckDB process isolation mode (Experimental)",
duckDbWorkerProcessIsolationDescription: "Runs DuckDB queries in a separate worker process. If a long query does not respond to cancellation, DBX can stop the worker process to release the database file lock.", duckDbWorkerProcessIsolationDescription: "Runs DuckDB queries in a separate worker process. If a long query does not respond to cancellation, DBX can stop the worker process to release the database file lock.",
duckDbWorkerMaxProcesses: "DuckDB max concurrent workers", duckDbWorkerMaxProcesses: "DuckDB max concurrent workers",
@@ -6172,6 +6174,7 @@ export default {
queryExportKeysetOptimizationEnabled: "Try keyset optimization", queryExportKeysetOptimizationEnabled: "Try keyset optimization",
queryExportKeysetOptimizationEnabledDescription: "Only applies to safely recognized single-table queries; complex queries automatically fall back.", queryExportKeysetOptimizationEnabledDescription: "Only applies to safely recognized single-table queries; complex queries automatically fall back.",
exportSection: "Export", exportSection: "Export",
performanceSection: "Performance",
shortcutExecuteSql: "Execute SQL", shortcutExecuteSql: "Execute SQL",
shortcutExecuteSqlInNewResultTab: "Execute SQL in new result tab", shortcutExecuteSqlInNewResultTab: "Execute SQL in new result tab",
shortcutFind: "Find", shortcutFind: "Find",
+3
View File
@@ -5592,6 +5592,8 @@ export default withEnglishFallback({
updateDownloadSourceCnb: "CNB", updateDownloadSourceCnb: "CNB",
debugLoggingEnabled: "Activar logs de depuración", debugLoggingEnabled: "Activar logs de depuración",
debugLoggingEnabledDescription: "Registra logs de diagnóstico locales del lado del usuario al investigar reportes. Desactivado por defecto.", debugLoggingEnabledDescription: "Registra logs de diagnóstico locales del lado del usuario al investigar reportes. Desactivado por defecto.",
metadataCacheMemoryLimit: "Límite de memoria de la caché de metadatos",
metadataCacheMemoryLimitDescription: "El valor predeterminado es 64 MB; se recomiendan 256 MB o menos y el máximo es 512 MB. Incluye la caché de DDL, columnas, índices, claves foráneas, disparadores y comentarios.",
duckDbWorkerProcessIsolation: "Modo de aislamiento de proceso de DuckDB (experimental)", duckDbWorkerProcessIsolation: "Modo de aislamiento de proceso de DuckDB (experimental)",
duckDbWorkerProcessIsolationDescription: "Ejecuta las consultas de DuckDB en un proceso worker independiente. Si una consulta larga no responde a la cancelación, DBX puede detener el proceso worker para liberar el bloqueo del archivo de base de datos.", duckDbWorkerProcessIsolationDescription: "Ejecuta las consultas de DuckDB en un proceso worker independiente. Si una consulta larga no responde a la cancelación, DBX puede detener el proceso worker para liberar el bloqueo del archivo de base de datos.",
duckDbWorkerMaxProcesses: "Workers concurrentes máximos de DuckDB", duckDbWorkerMaxProcesses: "Workers concurrentes máximos de DuckDB",
@@ -5864,6 +5866,7 @@ export default withEnglishFallback({
queryExportKeysetOptimizationEnabled: "Intentar optimización keyset", queryExportKeysetOptimizationEnabled: "Intentar optimización keyset",
queryExportKeysetOptimizationEnabledDescription: "Solo aplica a consultas de una sola tabla reconocidas de forma segura; las consultas complejas usan el método estándar automáticamente.", queryExportKeysetOptimizationEnabledDescription: "Solo aplica a consultas de una sola tabla reconocidas de forma segura; las consultas complejas usan el método estándar automáticamente.",
exportSection: "Exportar", exportSection: "Exportar",
performanceSection: "Rendimiento",
shortcutExecuteSql: "Ejecutar SQL", shortcutExecuteSql: "Ejecutar SQL",
shortcutExecuteSqlInNewResultTab: "Ejecutar SQL en una nueva pestaña de resultados", shortcutExecuteSqlInNewResultTab: "Ejecutar SQL en una nueva pestaña de resultados",
shortcutFind: "Buscar", shortcutFind: "Buscar",
+3
View File
@@ -5592,6 +5592,8 @@ export default withEnglishFallback({
updateDownloadSourceCnb: "CNB", updateDownloadSourceCnb: "CNB",
debugLoggingEnabled: "Abilita log di debug", debugLoggingEnabled: "Abilita log di debug",
debugLoggingEnabledDescription: "Registra log diagnostici locali lato utente durante l'analisi delle segnalazioni. Disattivato per impostazione predefinita.", debugLoggingEnabledDescription: "Registra log diagnostici locali lato utente durante l'analisi delle segnalazioni. Disattivato per impostazione predefinita.",
metadataCacheMemoryLimit: "Limite di memoria della cache dei metadati",
metadataCacheMemoryLimitDescription: "Il valore predefinito è 64 MB; si consigliano al massimo 256 MB e il limite massimo è 512 MB. Include DDL, colonne, indici, chiavi esterne, trigger e commenti.",
duckDbWorkerProcessIsolation: "Modalità di isolamento del processo DuckDB (sperimentale)", duckDbWorkerProcessIsolation: "Modalità di isolamento del processo DuckDB (sperimentale)",
duckDbWorkerProcessIsolationDescription: "Esegue le query DuckDB in un processo worker separato. Se una query lunga non risponde all'annullamento, DBX può arrestare il processo worker per rilasciare il blocco del file di database.", duckDbWorkerProcessIsolationDescription: "Esegue le query DuckDB in un processo worker separato. Se una query lunga non risponde all'annullamento, DBX può arrestare il processo worker per rilasciare il blocco del file di database.",
duckDbWorkerMaxProcesses: "Worker DuckDB simultanei massimi", duckDbWorkerMaxProcesses: "Worker DuckDB simultanei massimi",
@@ -5864,6 +5866,7 @@ export default withEnglishFallback({
queryExportKeysetOptimizationEnabled: "Prova ottimizzazione keyset", queryExportKeysetOptimizationEnabled: "Prova ottimizzazione keyset",
queryExportKeysetOptimizationEnabledDescription: "Si applica solo a query a tabella singola riconosciute in sicurezza; le query complesse ricadono automaticamente.", queryExportKeysetOptimizationEnabledDescription: "Si applica solo a query a tabella singola riconosciute in sicurezza; le query complesse ricadono automaticamente.",
exportSection: "Esportazione", exportSection: "Esportazione",
performanceSection: "Prestazioni",
shortcutExecuteSql: "Esegui SQL", shortcutExecuteSql: "Esegui SQL",
shortcutExecuteSqlInNewResultTab: "Esegui SQL in una nuova scheda dei risultati", shortcutExecuteSqlInNewResultTab: "Esegui SQL in una nuova scheda dei risultati",
shortcutFind: "Trova", shortcutFind: "Trova",
+3
View File
@@ -5627,6 +5627,8 @@ export default withEnglishFallback({
updateNotificationsEnabledDescription: "無効時、DBXはアプリやドライバーのアップデートを自動確認せず、アップデートバッジも表示しません。手動確認は引き続き利用可能です。", updateNotificationsEnabledDescription: "無効時、DBXはアプリやドライバーのアップデートを自動確認せず、アップデートバッジも表示しません。手動確認は引き続き利用可能です。",
debugLoggingEnabled: "デバッグログを有効にする", debugLoggingEnabled: "デバッグログを有効にする",
debugLoggingEnabledDescription: "トラブルシューティング時に、より詳細なローカル診断ログを記録します。デフォルトでは無効です。", debugLoggingEnabledDescription: "トラブルシューティング時に、より詳細なローカル診断ログを記録します。デフォルトでは無効です。",
metadataCacheMemoryLimit: "メタデータメモリキャッシュ上限",
metadataCacheMemoryLimitDescription: "デフォルトは64 MB、256 MB以下を推奨、最大512 MBです。DDL、フィールド、インデックス、外部キー、トリガー、コメントのキャッシュに使用されます。",
duckDbWorkerProcessIsolation: "DuckDB プロセス分離モード(実験的)", duckDbWorkerProcessIsolation: "DuckDB プロセス分離モード(実験的)",
duckDbWorkerProcessIsolationDescription: "DuckDB クエリを独立した worker プロセスで実行します。長時間クエリがキャンセルに応答しない場合、DBX は worker プロセスを停止してデータベースファイルのロックを解放できます。", duckDbWorkerProcessIsolationDescription: "DuckDB クエリを独立した worker プロセスで実行します。長時間クエリがキャンセルに応答しない場合、DBX は worker プロセスを停止してデータベースファイルのロックを解放できます。",
duckDbWorkerMaxProcesses: "DuckDB の最大同時 worker 数", duckDbWorkerMaxProcesses: "DuckDB の最大同時 worker 数",
@@ -5879,6 +5881,7 @@ export default withEnglishFallback({
exportBatchSize: "エクスポートバッチサイズ", exportBatchSize: "エクスポートバッチサイズ",
exportBatchSizeDescription: "テーブルデータエクスポート時のバッチあたりの取得行数(100〜100000)。", exportBatchSizeDescription: "テーブルデータエクスポート時のバッチあたりの取得行数(100〜100000)。",
exportSection: "エクスポート", exportSection: "エクスポート",
performanceSection: "パフォーマンス",
shortcutExecuteSql: "SQLを実行", shortcutExecuteSql: "SQLを実行",
shortcutExecuteSqlInNewResultTab: "新しい結果タブでSQLを実行", shortcutExecuteSqlInNewResultTab: "新しい結果タブでSQLを実行",
shortcutFind: "検索", shortcutFind: "検索",
+1
View File
@@ -5605,6 +5605,7 @@ export default withEnglishFallback({
queryExportKeysetOptimizationEnabled: "키셋 최적화 시도", queryExportKeysetOptimizationEnabled: "키셋 최적화 시도",
queryExportKeysetOptimizationEnabledDescription: "안전하게 인식된 단일 테이블 쿼리에만 적용되며 복잡한 쿼리는 자동으로 폴백됩니다.", queryExportKeysetOptimizationEnabledDescription: "안전하게 인식된 단일 테이블 쿼리에만 적용되며 복잡한 쿼리는 자동으로 폴백됩니다.",
exportSection: "내보내기", exportSection: "내보내기",
performanceSection: "성능",
shortcutExecuteSql: "SQL 실행", shortcutExecuteSql: "SQL 실행",
shortcutExecuteSqlInNewResultTab: "새 결과 탭에서 SQL 실행", shortcutExecuteSqlInNewResultTab: "새 결과 탭에서 SQL 실행",
shortcutFind: "찾기", shortcutFind: "찾기",
+3
View File
@@ -5594,6 +5594,8 @@ export default withEnglishFallback({
updateDownloadSourceCnb: "CNB", updateDownloadSourceCnb: "CNB",
debugLoggingEnabled: "Ativar logs de depuração", debugLoggingEnabled: "Ativar logs de depuração",
debugLoggingEnabledDescription: "Registra logs locais de diagnóstico do lado do usuário ao investigar relatos. Desativado por padrão.", debugLoggingEnabledDescription: "Registra logs locais de diagnóstico do lado do usuário ao investigar relatos. Desativado por padrão.",
metadataCacheMemoryLimit: "Limite de memória do cache de metadados",
metadataCacheMemoryLimitDescription: "O padrão é 64 MB; recomenda-se no máximo 256 MB, com limite máximo de 512 MB. Abrange o cache de DDL, colunas, índices, chaves estrangeiras, gatilhos e comentários.",
duckDbWorkerProcessIsolation: "Modo de isolamento de processo do DuckDB (experimental)", duckDbWorkerProcessIsolation: "Modo de isolamento de processo do DuckDB (experimental)",
duckDbWorkerProcessIsolationDescription: "Executa consultas DuckDB em um processo worker separado. Se uma consulta longa não responder ao cancelamento, o DBX pode encerrar o processo worker para liberar o bloqueio do arquivo de banco de dados.", duckDbWorkerProcessIsolationDescription: "Executa consultas DuckDB em um processo worker separado. Se uma consulta longa não responder ao cancelamento, o DBX pode encerrar o processo worker para liberar o bloqueio do arquivo de banco de dados.",
duckDbWorkerMaxProcesses: "Máximo de workers simultâneos do DuckDB", duckDbWorkerMaxProcesses: "Máximo de workers simultâneos do DuckDB",
@@ -5866,6 +5868,7 @@ export default withEnglishFallback({
queryExportKeysetOptimizationEnabled: "Tentar otimização keyset", queryExportKeysetOptimizationEnabled: "Tentar otimização keyset",
queryExportKeysetOptimizationEnabledDescription: "Aplica-se apenas a consultas de tabela única reconhecidas com segurança; consultas complexas recuam automaticamente.", queryExportKeysetOptimizationEnabledDescription: "Aplica-se apenas a consultas de tabela única reconhecidas com segurança; consultas complexas recuam automaticamente.",
exportSection: "Exportar", exportSection: "Exportar",
performanceSection: "Desempenho",
shortcutExecuteSql: "Executar SQL", shortcutExecuteSql: "Executar SQL",
shortcutExecuteSqlInNewResultTab: "Executar SQL em uma nova aba de resultados", shortcutExecuteSqlInNewResultTab: "Executar SQL em uma nova aba de resultados",
shortcutFind: "Localizar", shortcutFind: "Localizar",
+3
View File
@@ -5837,6 +5837,8 @@ export default withEnglishFallback({
updateDownloadSourceCnb: "CNB", updateDownloadSourceCnb: "CNB",
debugLoggingEnabled: "启用调试日志", debugLoggingEnabled: "启用调试日志",
debugLoggingEnabledDescription: "开启后在本机记录更详细的用户侧诊断日志,反馈问题时可临时开启;默认关闭。", debugLoggingEnabledDescription: "开启后在本机记录更详细的用户侧诊断日志,反馈问题时可临时开启;默认关闭。",
metadataCacheMemoryLimit: "元数据内存缓存上限",
metadataCacheMemoryLimitDescription: "默认 64 MB,建议不超过 256 MB,最大 512 MB。用于 DDL、字段、索引、外键、触发器和注释缓存。",
duckDbWorkerProcessIsolation: "DuckDB 进程隔离模式(实验性)", duckDbWorkerProcessIsolation: "DuckDB 进程隔离模式(实验性)",
duckDbWorkerProcessIsolationDescription: "将 DuckDB 查询放到独立 worker 进程中运行。停止长查询时,如果 DuckDB 未及时响应取消请求,DBX 会结束 worker 进程以释放数据库文件锁。", duckDbWorkerProcessIsolationDescription: "将 DuckDB 查询放到独立 worker 进程中运行。停止长查询时,如果 DuckDB 未及时响应取消请求,DBX 会结束 worker 进程以释放数据库文件锁。",
duckDbWorkerMaxProcesses: "DuckDB 最大并发 worker 数", duckDbWorkerMaxProcesses: "DuckDB 最大并发 worker 数",
@@ -6157,6 +6159,7 @@ export default withEnglishFallback({
queryExportKeysetOptimizationEnabled: "尝试 keyset 优化", queryExportKeysetOptimizationEnabled: "尝试 keyset 优化",
queryExportKeysetOptimizationEnabledDescription: "仅对可安全识别的单表查询生效,复杂查询会自动回退。", queryExportKeysetOptimizationEnabledDescription: "仅对可安全识别的单表查询生效,复杂查询会自动回退。",
exportSection: "导出", exportSection: "导出",
performanceSection: "性能",
shortcutExecuteSql: "执行 SQL", shortcutExecuteSql: "执行 SQL",
shortcutExecuteSqlInNewResultTab: "在新结果标签页中执行 SQL", shortcutExecuteSqlInNewResultTab: "在新结果标签页中执行 SQL",
shortcutFind: "查找", shortcutFind: "查找",
+3
View File
@@ -4914,6 +4914,8 @@ export default withEnglishFallback({
updateNotificationsEnabledDescription: "關閉後,DBX 不會自動檢查應用程式和驅動程式更新,也不會顯示更新紅點;仍可手動檢查更新。", updateNotificationsEnabledDescription: "關閉後,DBX 不會自動檢查應用程式和驅動程式更新,也不會顯示更新紅點;仍可手動檢查更新。",
debugLoggingEnabled: "啟用偵錯日誌", debugLoggingEnabled: "啟用偵錯日誌",
debugLoggingEnabledDescription: "開啟後在本機記錄更詳細的使用者側診斷日誌,回報問題時可暫時開啟;預設關閉。", debugLoggingEnabledDescription: "開啟後在本機記錄更詳細的使用者側診斷日誌,回報問題時可暫時開啟;預設關閉。",
metadataCacheMemoryLimit: "中繼資料記憶體快取上限",
metadataCacheMemoryLimitDescription: "預設為 64 MB,建議不超過 256 MB,最大為 512 MB。用於快取 DDL、欄位、索引、外部索引鍵、觸發程序及註解。",
duckDbWorkerProcessIsolation: "DuckDB 進程隔離模式(實驗性)", duckDbWorkerProcessIsolation: "DuckDB 進程隔離模式(實驗性)",
duckDbWorkerProcessIsolationDescription: "將 DuckDB 查詢放到獨立 worker 進程中執行。停止長查詢時,如果 DuckDB 未及時回應取消請求,DBX 會結束 worker 進程以釋放資料庫檔案鎖。", duckDbWorkerProcessIsolationDescription: "將 DuckDB 查詢放到獨立 worker 進程中執行。停止長查詢時,如果 DuckDB 未及時回應取消請求,DBX 會結束 worker 進程以釋放資料庫檔案鎖。",
duckDbWorkerMaxProcesses: "DuckDB 最大並行 worker 數", duckDbWorkerMaxProcesses: "DuckDB 最大並行 worker 數",
@@ -5431,6 +5433,7 @@ export default withEnglishFallback({
updateDownloadSourceOfficial: "官方來源(建議)", updateDownloadSourceOfficial: "官方來源(建議)",
updateDownloadSourceCnb: "CNB", updateDownloadSourceCnb: "CNB",
exportSection: "匯出", exportSection: "匯出",
performanceSection: "效能",
exportBatchSize: "匯出批次大小", exportBatchSize: "匯出批次大小",
dateTimeSection: "日期與時間", dateTimeSection: "日期與時間",
globalDateTimeDisplayFormat: "全域顯示格式", globalDateTimeDisplayFormat: "全域顯示格式",
@@ -5,7 +5,7 @@ const contentAreaSource = readFileSync(new URL("../../../components/layout/Conte
const connectionTreeSource = readFileSync(new URL("../../../components/sidebar/ConnectionTree.vue", import.meta.url), "utf8"); const connectionTreeSource = readFileSync(new URL("../../../components/sidebar/ConnectionTree.vue", import.meta.url), "utf8");
const ddlViewDialogSource = readFileSync(new URL("../../../components/objects/DdlViewDialog.vue", import.meta.url), "utf8"); const ddlViewDialogSource = readFileSync(new URL("../../../components/objects/DdlViewDialog.vue", import.meta.url), "utf8");
const objectBrowserSource = readFileSync(new URL("../../../components/objects/ObjectBrowser.vue", import.meta.url), "utf8"); const objectBrowserSource = readFileSync(new URL("../../../components/objects/ObjectBrowser.vue", import.meta.url), "utf8");
const fetchTableDdlSource = objectBrowserSource.match(/async function fetchTableDdl\(\)[\s\S]*?(?=\nasync function fetchTableColumns\()/)?.[0] ?? ""; const fetchTableDdlSource = objectBrowserSource.match(/async function fetchTableDdl(?:\([^)]*\))?[\s\S]*?(?=\nasync function fetchTableColumns\()/)?.[0] ?? "";
const exportStructureSource = objectBrowserSource.match(/async function exportStructure\([\s\S]*?(?=\nasync function exportDataLegacy\()/)?.[0] ?? ""; const exportStructureSource = objectBrowserSource.match(/async function exportStructure\([\s\S]*?(?=\nasync function exportDataLegacy\()/)?.[0] ?? "";
function openingTag(source: string, componentName: string): string { function openingTag(source: string, componentName: string): string {
@@ -47,8 +47,9 @@ describe("ContentArea object browser refresh wiring", () => {
}); });
describe("ObjectBrowser DDL API boundaries", () => { describe("ObjectBrowser DDL API boundaries", () => {
it("uses display DDL for the table information panel", () => { it("uses the cached display DDL boundary for the table information panel", () => {
expect(fetchTableDdlSource).toMatch(/api\.getTableDisplayDdl\([\s\S]*?props\.catalog\);/); expect(fetchTableDdlSource).toMatch(/loadObjectDdl\(tableMetadataRequest\(row\), \{ force \}\)/);
expect(objectBrowserSource).toMatch(/function tableMetadataRequest\([\s\S]*?catalog: props\.catalog/);
}); });
it("keeps structure exports on the portable base DDL", () => { it("keeps structure exports on the portable base DDL", () => {
+1
View File
@@ -232,6 +232,7 @@ const DEFAULT_DESKTOP_SETTINGS: DesktopSettings = {
quit_on_close: false, quit_on_close: false,
close_action_prompted: false, close_action_prompted: false,
debug_logging_enabled: false, debug_logging_enabled: false,
metadata_cache_max_memory_mb: 64,
duckdb_worker_process_isolation: false, duckdb_worker_process_isolation: false,
duckdb_worker_max_processes: 4, duckdb_worker_max_processes: 4,
saved_sql_sync_dir: null, saved_sql_sync_dir: null,
+1
View File
@@ -188,6 +188,7 @@ export interface DesktopSettings {
quit_on_close: boolean; quit_on_close: boolean;
close_action_prompted: boolean; close_action_prompted: boolean;
debug_logging_enabled: boolean; debug_logging_enabled: boolean;
metadata_cache_max_memory_mb: number;
duckdb_worker_process_isolation: boolean; duckdb_worker_process_isolation: boolean;
duckdb_worker_max_processes: number; duckdb_worker_max_processes: number;
saved_sql_sync_dir?: string | null; saved_sql_sync_dir?: string | null;
@@ -0,0 +1,19 @@
import { beforeEach, describe, expect, it } from "vitest";
import { resourceLifecycleDiagnostics } from "@/lib/diagnostics/resourceLifecycleDiagnostics";
import { clearMetadataRuntimeCache, setMetadataRuntimeCache } from "@/lib/metadata/metadataRuntimeCache";
describe("resourceLifecycleDiagnostics", () => {
beforeEach(() => {
clearMetadataRuntimeCache();
});
it("includes metadata cache diagnostics in the existing lifecycle snapshot", () => {
setMetadataRuntimeCache("metadata-key", { value: 1 }, "connection-1");
const diagnostics = resourceLifecycleDiagnostics([]);
expect(diagnostics.metadataCache.entries).toBe(1);
expect(diagnostics.metadataCache.bytes).toBeGreaterThan(0);
});
});
@@ -1,5 +1,6 @@
import type { QueryTab } from "@/types/database"; import type { QueryTab } from "@/types/database";
import { getResultCacheDiagnostics } from "@/lib/tabs/tabResultCache"; import { getResultCacheDiagnostics } from "@/lib/tabs/tabResultCache";
import { getMetadataRuntimeCacheDiagnostics } from "@/lib/metadata/metadataRuntimeCache";
const cancellation = { const cancellation = {
count: 0, count: 0,
@@ -15,11 +16,13 @@ export function recordQueryCancellationLatency(durationMs: number) {
export function resourceLifecycleDiagnostics(tabs: readonly QueryTab[]) { export function resourceLifecycleDiagnostics(tabs: readonly QueryTab[]) {
const cache = getResultCacheDiagnostics(); const cache = getResultCacheDiagnostics();
const metadataCache = getMetadataRuntimeCacheDiagnostics();
return { return {
activeTasks: tabs.filter((tab) => tab.isExecuting || tab.isExplaining || tab.isCancelling).length, activeTasks: tabs.filter((tab) => tab.isExecuting || tab.isExplaining || tab.isCancelling).length,
cancellationCount: cancellation.count, cancellationCount: cancellation.count,
averageCancellationLatencyMs: cancellation.count ? cancellation.totalLatencyMs / cancellation.count : 0, averageCancellationLatencyMs: cancellation.count ? cancellation.totalLatencyMs / cancellation.count : 0,
maxCancellationLatencyMs: cancellation.maxLatencyMs, maxCancellationLatencyMs: cancellation.maxLatencyMs,
...cache, ...cache,
metadataCache,
}; };
} }
@@ -0,0 +1,87 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import {
clearMetadataRuntimeCache,
clearMetadataRuntimeCacheForDatabase,
clearMetadataRuntimeCacheForConnection,
configureMetadataRuntimeCache,
getMetadataRuntimeCache,
getMetadataRuntimeCacheDiagnostics,
METADATA_CACHE_DEFAULT_MEMORY_MB,
normalizeMetadataCacheMemoryMb,
setMetadataRuntimeCache,
} from "@/lib/metadata/metadataRuntimeCache";
describe("metadataRuntimeCache", () => {
beforeEach(() => {
clearMetadataRuntimeCache();
configureMetadataRuntimeCache(METADATA_CACHE_DEFAULT_MEMORY_MB);
vi.restoreAllMocks();
});
it("normalizes the configured memory budget", () => {
expect(normalizeMetadataCacheMemoryMb(undefined)).toBe(64);
expect(normalizeMetadataCacheMemoryMb(1)).toBe(16);
expect(normalizeMetadataCacheMemoryMb(256.4)).toBe(256);
expect(normalizeMetadataCacheMemoryMb(512)).toBe(512);
expect(normalizeMetadataCacheMemoryMb(513)).toBe(64);
});
it("warns above the recommended limit and falls back above the hard limit", () => {
const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined);
expect(configureMetadataRuntimeCache(384)).toBe(384);
expect(configureMetadataRuntimeCache(513)).toBe(64);
expect(warn).toHaveBeenCalledTimes(2);
});
it("rejects a single entry larger than one megabyte", () => {
expect(setMetadataRuntimeCache("large", "x".repeat(600_000), "connection-1")).toBe(false);
expect(getMetadataRuntimeCacheDiagnostics().entries).toBe(0);
});
it("evicts the least recently used entry within a connection budget", () => {
configureMetadataRuntimeCache(16);
const value = "x".repeat(300_000);
for (let index = 0; index < 6; index += 1) {
expect(setMetadataRuntimeCache(`key-${index}`, value, "connection-1")).toBe(true);
}
expect(getMetadataRuntimeCache("key-0")).toBeDefined();
expect(setMetadataRuntimeCache("key-6", value, "connection-1")).toBe(true);
expect(getMetadataRuntimeCache("key-0")).toBeDefined();
expect(getMetadataRuntimeCache("key-1")).toBeUndefined();
expect(getMetadataRuntimeCacheDiagnostics().evictions).toBeGreaterThan(0);
});
it("reapplies the per-connection budget when the configured limit shrinks", () => {
const value = "x".repeat(300_000);
for (let index = 0; index < 10; index += 1) {
setMetadataRuntimeCache(`key-${index}`, value, "connection-1");
}
configureMetadataRuntimeCache(16);
expect(getMetadataRuntimeCacheDiagnostics().bytes).toBeLessThanOrEqual(4 * 1024 * 1024);
});
it("clears only entries owned by the requested connection", () => {
setMetadataRuntimeCache("a", { value: 1 }, "connection-1");
setMetadataRuntimeCache("b", { value: 2 }, "connection-2");
expect(clearMetadataRuntimeCacheForConnection("connection-1")).toBe(1);
expect(getMetadataRuntimeCache("a")).toBeUndefined();
expect(getMetadataRuntimeCache("b")?.value).toEqual({ value: 2 });
expect(getMetadataRuntimeCacheDiagnostics().connectionClears).toBe(1);
});
it("clears only entries owned by the requested database", () => {
setMetadataRuntimeCache("object-meta:v1:connection-1:database-1:schema:table:catalog:TABLE:columns:", { value: 1 }, "connection-1");
setMetadataRuntimeCache("object-meta:v1:connection-1:database-2:schema:table:catalog:TABLE:columns:", { value: 2 }, "connection-1");
setMetadataRuntimeCache("object-meta:v1:connection-2:database-1:schema:table:catalog:TABLE:columns:", { value: 3 }, "connection-2");
expect(clearMetadataRuntimeCacheForDatabase("connection-1", "database-1")).toBe(1);
expect(getMetadataRuntimeCache("object-meta:v1:connection-1:database-1:schema:table:catalog:TABLE:columns:")).toBeUndefined();
expect(getMetadataRuntimeCache("object-meta:v1:connection-1:database-2:schema:table:catalog:TABLE:columns:")).toBeDefined();
expect(getMetadataRuntimeCache("object-meta:v1:connection-2:database-1:schema:table:catalog:TABLE:columns:")).toBeDefined();
});
});
@@ -10,12 +10,14 @@ const mocks = vi.hoisted(() => ({
vi.mock("@/lib/backend/api", () => mocks); vi.mock("@/lib/backend/api", () => mocks);
import { invalidateObjectDdl, invalidateObjectDdlCache, loadObjectDdl, objectDdlCacheKey } from "@/lib/metadata/objectDdlCache"; import { cancelObjectDdlLoadsForConnection, getObjectDdlCacheDebugStateForTests, invalidateObjectDdl, invalidateObjectDdlCache, loadObjectDdl, objectDdlCacheKey } from "@/lib/metadata/objectDdlCache";
import { clearMetadataRuntimeCache } from "@/lib/metadata/metadataRuntimeCache";
const request = { connectionId: "c1", database: "app", schema: "public", tableName: "users", catalog: "analytics" } as const; const request = { connectionId: "c1", database: "app", schema: "public", tableName: "users", catalog: "analytics" } as const;
describe("objectDdlCache", () => { describe("objectDdlCache", () => {
beforeEach(() => { beforeEach(() => {
clearMetadataRuntimeCache();
vi.clearAllMocks(); vi.clearAllMocks();
mocks.persisted.clear(); mocks.persisted.clear();
mocks.loadSchemaCache.mockImplementation(async (cacheKey: string) => mocks.persisted.get(cacheKey) ?? null); mocks.loadSchemaCache.mockImplementation(async (cacheKey: string) => mocks.persisted.get(cacheKey) ?? null);
@@ -36,6 +38,23 @@ describe("objectDdlCache", () => {
expect(mocks.getTableDisplayDdl).not.toHaveBeenCalled(); expect(mocks.getTableDisplayDdl).not.toHaveBeenCalled();
}); });
it("backfills memory after a disk hit", async () => {
mocks.loadSchemaCache.mockResolvedValue({ version: 1, cachedAt: new Date().toISOString(), ddl: "CREATE TABLE users (id int)" });
await expect(loadObjectDdl(request)).resolves.toMatchObject({ cacheStatus: "disk" });
await expect(loadObjectDdl(request)).resolves.toMatchObject({ cacheStatus: "memory" });
expect(mocks.loadSchemaCache).toHaveBeenCalledTimes(1);
expect(mocks.getTableDisplayDdl).not.toHaveBeenCalled();
});
it("caches an empty DDL string in memory", async () => {
mocks.getTableDisplayDdl.mockResolvedValue("");
await expect(loadObjectDdl(request)).resolves.toEqual({ ddl: "", cacheStatus: "remote" });
await expect(loadObjectDdl(request)).resolves.toEqual({ ddl: "", cacheStatus: "memory" });
expect(mocks.getTableDisplayDdl).toHaveBeenCalledTimes(1);
});
it("persists a remote cache miss", async () => { it("persists a remote cache miss", async () => {
mocks.getTableDisplayDdl.mockResolvedValue("CREATE TABLE users (id bigint)"); mocks.getTableDisplayDdl.mockResolvedValue("CREATE TABLE users (id bigint)");
@@ -43,6 +62,41 @@ describe("objectDdlCache", () => {
expect(mocks.saveSchemaCache).toHaveBeenCalledWith(objectDdlCacheKey(request), expect.objectContaining({ version: 1, ddl: "CREATE TABLE users (id bigint)" })); expect(mocks.saveSchemaCache).toHaveBeenCalledWith(objectDdlCacheKey(request), expect.objectContaining({ version: 1, ddl: "CREATE TABLE users (id bigint)" }));
}); });
it("does not wait for a slow SQLite write before returning the remote result", async () => {
let releaseWrite: () => void = () => {};
mocks.saveSchemaCache.mockReturnValue(
new Promise<void>((resolve) => {
releaseWrite = resolve;
}),
);
mocks.getTableDisplayDdl.mockResolvedValue("CREATE TABLE users (id bigint)");
await expect(loadObjectDdl(request)).resolves.toMatchObject({ ddl: "CREATE TABLE users (id bigint)", cacheStatus: "remote" });
expect(mocks.saveSchemaCache).toHaveBeenCalledTimes(1);
releaseWrite();
await Promise.resolve();
await Promise.resolve();
});
it("waits for a pending SQLite write before deleting an invalidated entry", async () => {
let releaseWrite: () => void = () => {};
mocks.saveSchemaCache.mockReturnValue(
new Promise<void>((resolve) => {
releaseWrite = resolve;
}),
);
mocks.getTableDisplayDdl.mockResolvedValue("stale ddl");
await loadObjectDdl(request);
const invalidation = invalidateObjectDdl(request);
await Promise.resolve();
expect(mocks.deleteSchemaCachePrefix).not.toHaveBeenCalledWith(objectDdlCacheKey(request));
releaseWrite();
await invalidation;
expect(mocks.deleteSchemaCachePrefix).toHaveBeenCalledWith(objectDdlCacheKey(request));
});
it("deduplicates concurrent remote loads", async () => { it("deduplicates concurrent remote loads", async () => {
let release: (ddl: string) => void = () => {}; let release: (ddl: string) => void = () => {};
mocks.getTableDisplayDdl.mockReturnValue( mocks.getTableDisplayDdl.mockReturnValue(
@@ -68,6 +122,23 @@ describe("objectDdlCache", () => {
expect(mocks.saveSchemaCache).toHaveBeenCalledWith(objectDdlCacheKey(request), expect.objectContaining({ ddl: "new ddl" })); expect(mocks.saveSchemaCache).toHaveBeenCalledWith(objectDdlCacheKey(request), expect.objectContaining({ ddl: "new ddl" }));
}); });
it("force refresh bypasses memory and replaces it", async () => {
mocks.getTableDisplayDdl.mockResolvedValueOnce("old ddl").mockResolvedValueOnce("new ddl");
await expect(loadObjectDdl(request)).resolves.toEqual({ ddl: "old ddl", cacheStatus: "remote" });
await expect(loadObjectDdl(request, { force: true })).resolves.toEqual({ ddl: "new ddl", cacheStatus: "remote" });
await expect(loadObjectDdl(request)).resolves.toEqual({ ddl: "new ddl", cacheStatus: "memory" });
expect(mocks.getTableDisplayDdl).toHaveBeenCalledTimes(2);
});
it("ignores an expired persisted DDL", async () => {
mocks.loadSchemaCache.mockResolvedValue({ version: 1, cachedAt: new Date(Date.now() - 25 * 60 * 60 * 1000).toISOString(), ddl: "stale ddl" });
mocks.getTableDisplayDdl.mockResolvedValue("fresh ddl");
await expect(loadObjectDdl(request)).resolves.toEqual({ ddl: "fresh ddl", cacheStatus: "remote" });
expect(mocks.getTableDisplayDdl).toHaveBeenCalledTimes(1);
});
it("deletes the exact persisted entry", async () => { it("deletes the exact persisted entry", async () => {
await invalidateObjectDdl(request); await invalidateObjectDdl(request);
expect(mocks.deleteSchemaCachePrefix).toHaveBeenCalledWith(objectDdlCacheKey(request)); expect(mocks.deleteSchemaCachePrefix).toHaveBeenCalledWith(objectDdlCacheKey(request));
@@ -97,4 +168,99 @@ describe("objectDdlCache", () => {
await expect(load).resolves.toEqual({ ddl: "stale ddl", cacheStatus: "remote" }); await expect(load).resolves.toEqual({ ddl: "stale ddl", cacheStatus: "remote" });
expect(mocks.saveSchemaCache).not.toHaveBeenCalled(); expect(mocks.saveSchemaCache).not.toHaveBeenCalled();
}); });
it("does not resurrect a stale disk result when invalidated during the read", async () => {
let releaseDiskRead: (value: unknown) => void = () => {};
mocks.loadSchemaCache.mockReturnValueOnce(
new Promise((resolve) => {
releaseDiskRead = resolve;
}),
);
mocks.getTableDisplayDdl.mockResolvedValue("fresh ddl");
const load = loadObjectDdl(request);
await vi.waitFor(() => expect(mocks.loadSchemaCache).toHaveBeenCalledTimes(1));
const invalidation = invalidateObjectDdl(request);
releaseDiskRead({ version: 1, cachedAt: new Date().toISOString(), ddl: "stale ddl" });
await expect(load).resolves.toEqual({ ddl: "fresh ddl", cacheStatus: "remote" });
await invalidation;
expect(mocks.getTableDisplayDdl).toHaveBeenCalledTimes(1);
expect(mocks.saveSchemaCache).toHaveBeenCalledWith(objectDdlCacheKey(request), expect.objectContaining({ ddl: "fresh ddl" }));
});
it("refreshing object B does not invalidate object A's concurrent disk read", async () => {
const requestA = { ...request, tableName: "accounts" };
const requestB = { ...request, tableName: "billing" };
let releaseA: (value: unknown) => void = () => {};
let releaseB: (value: unknown) => void = () => {};
mocks.loadSchemaCache.mockImplementation((cacheKey: string) => {
return new Promise((resolve) => {
if (cacheKey === objectDdlCacheKey(requestA)) releaseA = resolve;
if (cacheKey === objectDdlCacheKey(requestB)) releaseB = resolve;
});
});
mocks.getTableDisplayDdl.mockImplementation(async (_connectionId: string, _database: string, _schema: string, tableName: string) => `remote ${tableName}`);
const loadA = loadObjectDdl(requestA);
const loadB = loadObjectDdl(requestB);
await vi.waitFor(() => expect(mocks.loadSchemaCache).toHaveBeenCalledTimes(2));
const invalidation = invalidateObjectDdl(requestB);
releaseA({ version: 1, cachedAt: new Date().toISOString(), ddl: "cached accounts" });
releaseB({ version: 1, cachedAt: new Date().toISOString(), ddl: "stale billing" });
await expect(loadA).resolves.toEqual({ ddl: "cached accounts", cacheStatus: "disk" });
await expect(loadB).resolves.toEqual({ ddl: "remote billing", cacheStatus: "remote" });
await invalidation;
expect(mocks.getTableDisplayDdl).toHaveBeenCalledTimes(1);
});
it("reclaims invalidation state after long-running multi-object refreshes", async () => {
for (let index = 0; index < 2_000; index += 1) {
await invalidateObjectDdlCache({ ...request, tableName: `history_${index}` });
}
expect(getObjectDdlCacheDebugStateForTests()).toEqual({ activeReads: 0 });
});
it("makes a concurrent normal read wait for a force refresh", async () => {
mocks.getTableDisplayDdl.mockResolvedValueOnce("old ddl");
await loadObjectDdl(request);
let releaseRemote: (value: string) => void = () => {};
mocks.getTableDisplayDdl.mockReturnValueOnce(
new Promise((resolve) => {
releaseRemote = resolve;
}),
);
const force = loadObjectDdl(request, { force: true });
const normal = loadObjectDdl(request);
await vi.waitFor(() => expect(mocks.getTableDisplayDdl).toHaveBeenCalledTimes(2));
releaseRemote("new ddl");
await expect(Promise.all([force, normal])).resolves.toEqual([
{ ddl: "new ddl", cacheStatus: "remote" },
{ ddl: "new ddl", cacheStatus: "remote" },
]);
expect(mocks.getTableDisplayDdl).toHaveBeenCalledTimes(2);
});
it("cancels an old connection load without repopulating the new session cache", async () => {
let releaseOld: (value: string) => void = () => {};
mocks.getTableDisplayDdl.mockReturnValueOnce(
new Promise((resolve) => {
releaseOld = resolve;
}),
);
const oldLoad = loadObjectDdl(request);
await vi.waitFor(() => expect(mocks.getTableDisplayDdl).toHaveBeenCalledTimes(1));
cancelObjectDdlLoadsForConnection(request.connectionId);
releaseOld("old session ddl");
await expect(oldLoad).resolves.toMatchObject({ ddl: "old session ddl" });
mocks.getTableDisplayDdl.mockResolvedValueOnce("new session ddl");
await expect(loadObjectDdl(request)).resolves.toMatchObject({ ddl: "new session ddl", cacheStatus: "remote" });
expect(mocks.getTableDisplayDdl).toHaveBeenCalledTimes(2);
expect(mocks.deleteSchemaCachePrefix).not.toHaveBeenCalledWith("object-ddl:v1:c1:");
});
}); });
@@ -9,12 +9,14 @@ const mocks = vi.hoisted(() => ({
vi.mock("@/lib/backend/api", () => mocks); vi.mock("@/lib/backend/api", () => mocks);
import { invalidateObjectMetadataCache, loadObjectMetadataFacet } from "@/lib/metadata/objectMetadataCache"; import { cancelObjectMetadataLoadsForConnection, getObjectMetadataCacheDebugStateForTests, invalidateObjectMetadataCache, loadObjectMetadataFacet } from "@/lib/metadata/objectMetadataCache";
import { clearMetadataRuntimeCache } from "@/lib/metadata/metadataRuntimeCache";
const request = { connectionId: "c1", database: "app", schema: "public", tableName: "users", catalog: "analytics" } as const; const request = { connectionId: "c1", database: "app", schema: "public", tableName: "users", catalog: "analytics" } as const;
describe("objectMetadataCache", () => { describe("objectMetadataCache", () => {
beforeEach(() => { beforeEach(() => {
clearMetadataRuntimeCache();
vi.clearAllMocks(); vi.clearAllMocks();
mocks.persisted.clear(); mocks.persisted.clear();
mocks.loadSchemaCache.mockImplementation(async (cacheKey: string) => mocks.persisted.get(cacheKey) ?? null); mocks.loadSchemaCache.mockImplementation(async (cacheKey: string) => mocks.persisted.get(cacheKey) ?? null);
@@ -36,6 +38,46 @@ describe("objectMetadataCache", () => {
expect(loader).not.toHaveBeenCalled(); expect(loader).not.toHaveBeenCalled();
}); });
it("backfills memory after a disk hit", async () => {
mocks.loadSchemaCache.mockResolvedValue({ version: 1, cachedAt: new Date().toISOString(), value: [] });
const loader = vi.fn();
await expect(loadObjectMetadataFacet(request, "columns", loader)).resolves.toEqual({ value: [], cacheStatus: "disk" });
await expect(loadObjectMetadataFacet(request, "columns", loader)).resolves.toEqual({ value: [], cacheStatus: "memory" });
expect(mocks.loadSchemaCache).toHaveBeenCalledTimes(1);
expect(loader).not.toHaveBeenCalled();
});
it("caches null and empty-array facet values", async () => {
const nullLoader = vi.fn().mockResolvedValue(null);
const emptyLoader = vi.fn().mockResolvedValue([]);
await expect(loadObjectMetadataFacet(request, "comment", nullLoader)).resolves.toMatchObject({ value: null, cacheStatus: "remote" });
await expect(loadObjectMetadataFacet(request, "comment", nullLoader)).resolves.toMatchObject({ value: null, cacheStatus: "memory" });
await expect(loadObjectMetadataFacet(request, "indexes", emptyLoader)).resolves.toMatchObject({ value: [], cacheStatus: "remote" });
await expect(loadObjectMetadataFacet(request, "indexes", emptyLoader)).resolves.toMatchObject({ value: [], cacheStatus: "memory" });
expect(nullLoader).toHaveBeenCalledTimes(1);
expect(emptyLoader).toHaveBeenCalledTimes(1);
});
it("does not cache undefined failure sentinels", async () => {
const loader = vi.fn().mockResolvedValueOnce(undefined).mockResolvedValueOnce("fresh comment");
await expect(loadObjectMetadataFacet(request, "comment", loader)).resolves.toMatchObject({ value: undefined, cacheStatus: "remote" });
await expect(loadObjectMetadataFacet(request, "comment", loader)).resolves.toMatchObject({ value: "fresh comment", cacheStatus: "remote" });
expect(loader).toHaveBeenCalledTimes(2);
});
it("isolates facets by object type", async () => {
const tableLoader = vi.fn().mockResolvedValue(["table"]);
const viewLoader = vi.fn().mockResolvedValue(["view"]);
await loadObjectMetadataFacet({ ...request, objectType: "TABLE" }, "columns", tableLoader);
await loadObjectMetadataFacet({ ...request, objectType: "VIEW" }, "columns", viewLoader);
expect(tableLoader).toHaveBeenCalledTimes(1);
expect(viewLoader).toHaveBeenCalledTimes(1);
});
it("persists a remote facet and force bypasses disk", async () => { it("persists a remote facet and force bypasses disk", async () => {
mocks.loadSchemaCache.mockResolvedValue({ version: 1, cachedAt: new Date().toISOString(), value: ["old"] }); mocks.loadSchemaCache.mockResolvedValue({ version: 1, cachedAt: new Date().toISOString(), value: ["old"] });
const loader = vi.fn().mockResolvedValue(["new"]); const loader = vi.fn().mockResolvedValue(["new"]);
@@ -43,7 +85,57 @@ describe("objectMetadataCache", () => {
await expect(loadObjectMetadataFacet(request, "indexes", loader, { force: true })).resolves.toEqual({ value: ["new"], cacheStatus: "remote" }); await expect(loadObjectMetadataFacet(request, "indexes", loader, { force: true })).resolves.toEqual({ value: ["new"], cacheStatus: "remote" });
expect(loader).toHaveBeenCalledTimes(1); expect(loader).toHaveBeenCalledTimes(1);
expect(mocks.loadSchemaCache).not.toHaveBeenCalled(); expect(mocks.loadSchemaCache).not.toHaveBeenCalled();
expect(mocks.saveSchemaCache).toHaveBeenCalledWith(expect.stringContaining("object-meta:v1:c1:app:public:users:analytics:indexes:"), expect.objectContaining({ value: ["new"] })); expect(mocks.saveSchemaCache).toHaveBeenCalledWith(expect.stringContaining("object-meta:v1:c1:app:public:users:analytics:TABLE:indexes:"), expect.objectContaining({ value: ["new"] }));
});
it("does not wait for a slow SQLite write before returning a remote facet", async () => {
let releaseWrite: () => void = () => {};
mocks.saveSchemaCache.mockReturnValue(
new Promise<void>((resolve) => {
releaseWrite = resolve;
}),
);
const loader = vi.fn().mockResolvedValue(["id"]);
await expect(loadObjectMetadataFacet(request, "columns", loader)).resolves.toMatchObject({ value: ["id"], cacheStatus: "remote" });
expect(mocks.saveSchemaCache).toHaveBeenCalledTimes(1);
releaseWrite();
});
it("deduplicates concurrent remote facet loads", async () => {
let release: (value: string[]) => void = () => {};
const loader = vi.fn().mockReturnValue(
new Promise<string[]>((resolve) => {
release = resolve;
}),
);
const first = loadObjectMetadataFacet(request, "columns", loader);
const second = loadObjectMetadataFacet(request, "columns", loader);
await vi.waitFor(() => expect(loader).toHaveBeenCalledTimes(1));
release(["id"]);
await expect(Promise.all([first, second])).resolves.toEqual([
{ value: ["id"], cacheStatus: "remote" },
{ value: ["id"], cacheStatus: "remote" },
]);
});
it("force refresh bypasses memory and replaces it", async () => {
const loader = vi.fn().mockResolvedValueOnce(["old"]).mockResolvedValueOnce(["new"]);
await expect(loadObjectMetadataFacet(request, "indexes", loader)).resolves.toMatchObject({ value: ["old"], cacheStatus: "remote" });
await expect(loadObjectMetadataFacet(request, "indexes", loader, { force: true })).resolves.toMatchObject({ value: ["new"], cacheStatus: "remote" });
await expect(loadObjectMetadataFacet(request, "indexes", loader)).resolves.toMatchObject({ value: ["new"], cacheStatus: "memory" });
expect(loader).toHaveBeenCalledTimes(2);
});
it("ignores an expired persisted facet", async () => {
mocks.loadSchemaCache.mockResolvedValue({ version: 1, cachedAt: new Date(Date.now() - 25 * 60 * 60 * 1000).toISOString(), value: ["stale"] });
const loader = vi.fn().mockResolvedValue(["fresh"]);
await expect(loadObjectMetadataFacet(request, "indexes", loader)).resolves.toMatchObject({ value: ["fresh"], cacheStatus: "remote" });
expect(loader).toHaveBeenCalledTimes(1);
}); });
it("invalidates only the requested object's facets", async () => { it("invalidates only the requested object's facets", async () => {
@@ -60,4 +152,104 @@ describe("objectMetadataCache", () => {
await expect(loadObjectMetadataFacet(request, "indexes", refreshedLoader)).resolves.toEqual({ value: ["new"], cacheStatus: "remote" }); await expect(loadObjectMetadataFacet(request, "indexes", refreshedLoader)).resolves.toEqual({ value: ["new"], cacheStatus: "remote" });
expect(refreshedLoader).toHaveBeenCalledTimes(1); expect(refreshedLoader).toHaveBeenCalledTimes(1);
}); });
it("does not resurrect a stale disk facet when invalidated during the read", async () => {
let releaseDiskRead: (value: unknown) => void = () => {};
mocks.loadSchemaCache.mockReturnValueOnce(
new Promise((resolve) => {
releaseDiskRead = resolve;
}),
);
const loader = vi.fn().mockResolvedValue(["fresh"]);
const load = loadObjectMetadataFacet(request, "columns", loader);
await vi.waitFor(() => expect(mocks.loadSchemaCache).toHaveBeenCalledTimes(1));
const invalidation = invalidateObjectMetadataCache(request);
releaseDiskRead({ version: 1, cachedAt: new Date().toISOString(), value: ["stale"] });
await expect(load).resolves.toEqual({ value: ["fresh"], cacheStatus: "remote" });
await invalidation;
expect(loader).toHaveBeenCalledTimes(1);
expect(mocks.saveSchemaCache).toHaveBeenCalledWith(expect.stringContaining("object-meta:v1:c1:app:public:users:"), expect.objectContaining({ value: ["fresh"] }));
});
it("refreshing object B does not invalidate object A's concurrent facet read", async () => {
const requestA = { ...request, tableName: "accounts" };
const requestB = { ...request, tableName: "billing" };
let releaseA: (value: unknown) => void = () => {};
let releaseB: (value: unknown) => void = () => {};
let diskReads = 0;
mocks.loadSchemaCache.mockImplementation((cacheKey: string) => {
diskReads += 1;
if (diskReads > 2) return Promise.resolve(null);
return new Promise((resolve) => {
if (cacheKey.includes(":accounts:")) releaseA = resolve;
if (cacheKey.includes(":billing:")) releaseB = resolve;
});
});
const loaderA = vi.fn().mockResolvedValue(["remote accounts"]);
const loaderB = vi.fn().mockResolvedValue(["remote billing"]);
const loadA = loadObjectMetadataFacet(requestA, "columns", loaderA);
const loadB = loadObjectMetadataFacet(requestB, "columns", loaderB);
await vi.waitFor(() => expect(mocks.loadSchemaCache).toHaveBeenCalledTimes(2));
const invalidation = invalidateObjectMetadataCache(requestB);
releaseA({ version: 1, cachedAt: new Date().toISOString(), value: ["cached accounts"] });
releaseB({ version: 1, cachedAt: new Date().toISOString(), value: ["stale billing"] });
await expect(loadA).resolves.toEqual({ value: ["cached accounts"], cacheStatus: "disk" });
await expect(loadB).resolves.toEqual({ value: ["remote billing"], cacheStatus: "remote" });
await invalidation;
expect(loaderA).not.toHaveBeenCalled();
expect(loaderB).toHaveBeenCalledTimes(1);
});
it("reclaims invalidation state after long-running multi-object refreshes", async () => {
for (let index = 0; index < 2_000; index += 1) {
await invalidateObjectMetadataCache({ ...request, tableName: `history_${index}` });
}
expect(getObjectMetadataCacheDebugStateForTests()).toEqual({ activeReads: 0 });
});
it("makes a concurrent normal facet read wait for a force refresh", async () => {
const loader = vi.fn().mockResolvedValueOnce(["old"]);
await loadObjectMetadataFacet(request, "indexes", loader);
let releaseRemote: (value: string[]) => void = () => {};
loader.mockReturnValueOnce(
new Promise((resolve) => {
releaseRemote = resolve;
}),
);
const force = loadObjectMetadataFacet(request, "indexes", loader, { force: true });
const normal = loadObjectMetadataFacet(request, "indexes", loader);
await vi.waitFor(() => expect(loader).toHaveBeenCalledTimes(2));
releaseRemote(["new"]);
await expect(Promise.all([force, normal])).resolves.toEqual([
{ value: ["new"], cacheStatus: "remote" },
{ value: ["new"], cacheStatus: "remote" },
]);
expect(loader).toHaveBeenCalledTimes(2);
});
it("cancels an old connection facet load without repopulating the new session cache", async () => {
let releaseOld: (value: string[]) => void = () => {};
const loader = vi.fn().mockReturnValueOnce(
new Promise((resolve) => {
releaseOld = resolve;
}),
);
const oldLoad = loadObjectMetadataFacet(request, "columns", loader);
await vi.waitFor(() => expect(loader).toHaveBeenCalledTimes(1));
cancelObjectMetadataLoadsForConnection(request.connectionId);
releaseOld(["old session"]);
await expect(oldLoad).resolves.toMatchObject({ value: ["old session"] });
loader.mockResolvedValueOnce(["new session"]);
await expect(loadObjectMetadataFacet(request, "columns", loader)).resolves.toMatchObject({ value: ["new session"], cacheStatus: "remote" });
expect(loader).toHaveBeenCalledTimes(2);
expect(mocks.deleteSchemaCachePrefix).not.toHaveBeenCalledWith("object-meta:v1:c1:");
});
}); });
@@ -0,0 +1,37 @@
export interface ActiveCacheReadToken {
readonly cacheKey: string;
invalidated: boolean;
}
/** Tracks only currently running L2 reads, so invalidation state is bounded by concurrency. */
export class ActiveCacheReadTracker {
private readonly readsByKey = new Map<string, Set<ActiveCacheReadToken>>();
private count = 0;
begin(cacheKey: string): ActiveCacheReadToken {
const token: ActiveCacheReadToken = { cacheKey, invalidated: false };
const existing = this.readsByKey.get(cacheKey);
if (existing) existing.add(token);
else this.readsByKey.set(cacheKey, new Set([token]));
this.count += 1;
return token;
}
finish(token: ActiveCacheReadToken): void {
const reads = this.readsByKey.get(token.cacheKey);
if (!reads?.delete(token)) return;
this.count -= 1;
if (reads.size === 0) this.readsByKey.delete(token.cacheKey);
}
invalidatePrefix(prefix: string): void {
for (const [cacheKey, reads] of this.readsByKey) {
if (!cacheKey.startsWith(prefix)) continue;
for (const token of reads) token.invalidated = true;
}
}
get activeCount(): number {
return this.count;
}
}
@@ -0,0 +1,284 @@
import { appendDebugLog } from "@/lib/backend/debugLog";
export const METADATA_CACHE_DEFAULT_MEMORY_MB = 64;
export const METADATA_CACHE_MIN_MEMORY_MB = 16;
export const METADATA_CACHE_RECOMMENDED_MAX_MEMORY_MB = 256;
export const METADATA_CACHE_HARD_MAX_MEMORY_MB = 512;
export const METADATA_CACHE_MAX_ENTRY_BYTES = 1024 * 1024;
const METADATA_CACHE_CONNECTION_SHARE = 0.25;
const METADATA_CACHE_KEY_PREFIX = "dbx-metadata-cache:";
interface RuntimeCacheEntry<T = unknown> {
key: string;
connectionId: string;
value: T;
sizeBytes: number;
touchedAt: number;
}
interface MetadataCacheLogContext {
connectionId: string;
database: string;
facet: string;
}
export interface MetadataRuntimeCacheHit<T> {
value: T;
sizeBytes: number;
}
export interface MetadataRuntimeCacheDiagnostics {
entries: number;
bytes: number;
maxMemoryBytes: number;
maxEntryBytes: number;
configuredMemoryMb: number;
l1Hits: number;
l1Misses: number;
l2Hits: number;
remoteMisses: number;
evictions: number;
connectionClears: number;
}
const entries = new Map<string, RuntimeCacheEntry>();
let configuredMemoryMb = METADATA_CACHE_DEFAULT_MEMORY_MB;
let totalBytes = 0;
let l1Hits = 0;
let l1Misses = 0;
let l2Hits = 0;
let remoteMisses = 0;
let evictions = 0;
let connectionClears = 0;
function now(): number {
return typeof performance !== "undefined" && typeof performance.now === "function" ? performance.now() : Date.now();
}
function memoryBudgetBytes(): number {
return configuredMemoryMb * 1024 * 1024;
}
function connectionBudgetBytes(): number {
return Math.floor(memoryBudgetBytes() * METADATA_CACHE_CONNECTION_SHARE);
}
function keyConnectionId(key: string, connectionId?: string): string {
if (connectionId) return connectionId;
const marker = `${METADATA_CACHE_KEY_PREFIX}`;
if (!key.startsWith(marker)) return "";
return key.slice(marker.length).split(":", 1)[0] ?? "";
}
function estimateValueBytes(value: unknown): number {
let serialized = "";
try {
serialized = JSON.stringify(value) ?? "null";
} catch {
serialized = String(value);
}
// JSON byte length is a lower bound for V8 heap usage. Account for UTF-16
// strings and object/array/Map entry overhead with a conservative multiplier.
const utf8Bytes = typeof TextEncoder !== "undefined" ? new TextEncoder().encode(serialized).byteLength : serialized.length;
return Math.max(256, utf8Bytes * 2 + 512);
}
function decodeCacheSegment(value: string | undefined): string {
if (!value) return "";
try {
return decodeURIComponent(value);
} catch {
return value;
}
}
function cacheLogContext(key: string, fallbackConnectionId = ""): MetadataCacheLogContext {
const parts = key.split(":");
if ((parts[0] === "object-ddl" || parts[0] === "object-meta") && parts[1] === "v1") {
return {
connectionId: decodeCacheSegment(parts[2]) || fallbackConnectionId,
database: decodeCacheSegment(parts[3]),
facet: parts[0] === "object-ddl" ? "ddl" : decodeCacheSegment(parts[8]),
};
}
return { connectionId: fallbackConnectionId, database: "", facet: "" };
}
function logCacheMetric(metric: string, key: string, details: Record<string, unknown> = {}, fallbackConnectionId = ""): void {
appendDebugLog("debug", "[DBX][metadata-cache]", {
metric,
...cacheLogContext(key, fallbackConnectionId),
...details,
cacheBytes: totalBytes,
cacheEntries: entries.size,
});
}
function removeEntry(key: string, evictionReason?: string): boolean {
const entry = entries.get(key);
if (!entry) return false;
entries.delete(key);
totalBytes = Math.max(0, totalBytes - entry.sizeBytes);
if (evictionReason) {
evictions++;
logCacheMetric("metadata_cache_eviction", key, { entrySizeBytes: entry.sizeBytes, evictionReason }, entry.connectionId);
}
return true;
}
function connectionBytes(connectionId: string): number {
let bytes = 0;
for (const entry of entries.values()) {
if (entry.connectionId === connectionId) bytes += entry.sizeBytes;
}
return bytes;
}
function evictOldest(predicate: (entry: RuntimeCacheEntry) => boolean, reason: string): boolean {
let oldest: RuntimeCacheEntry | undefined;
for (const entry of entries.values()) {
if (!predicate(entry)) continue;
if (!oldest || entry.touchedAt < oldest.touchedAt) oldest = entry;
}
return oldest ? removeEntry(oldest.key, reason) : false;
}
function evictForInsert(connectionId: string, sizeBytes: number): boolean {
const budget = memoryBudgetBytes();
const perConnectionBudget = connectionBudgetBytes();
while (connectionBytes(connectionId) + sizeBytes > perConnectionBudget) {
if (!evictOldest((entry) => entry.connectionId === connectionId, "connection-budget")) return false;
}
while (totalBytes + sizeBytes > budget) {
if (!evictOldest(() => true, "total-budget")) return false;
}
return true;
}
export function normalizeMetadataCacheMemoryMb(value: unknown): number {
if (typeof value !== "number" || !Number.isFinite(value)) return METADATA_CACHE_DEFAULT_MEMORY_MB;
const rounded = Math.round(value);
if (rounded > METADATA_CACHE_HARD_MAX_MEMORY_MB) return METADATA_CACHE_DEFAULT_MEMORY_MB;
return Math.max(METADATA_CACHE_MIN_MEMORY_MB, rounded);
}
export function configureMetadataRuntimeCache(memoryMb: unknown): number {
if (typeof memoryMb === "number" && Number.isFinite(memoryMb)) {
const rounded = Math.round(memoryMb);
if (rounded > METADATA_CACHE_HARD_MAX_MEMORY_MB) {
console.warn(`[metadata-cache] memory limit ${rounded} MB exceeds the hard limit; falling back to ${METADATA_CACHE_DEFAULT_MEMORY_MB} MB`);
} else if (rounded > METADATA_CACHE_RECOMMENDED_MAX_MEMORY_MB) {
console.warn(`[metadata-cache] memory limit ${rounded} MB exceeds the recommended ${METADATA_CACHE_RECOMMENDED_MAX_MEMORY_MB} MB`);
}
}
configuredMemoryMb = normalizeMetadataCacheMemoryMb(memoryMb);
const connectionIds = new Set([...entries.values()].map((entry) => entry.connectionId));
for (const connectionId of connectionIds) {
while (connectionBytes(connectionId) > connectionBudgetBytes()) {
if (!evictOldest((entry) => entry.connectionId === connectionId, "connection-budget-reconfigured")) break;
}
}
while (totalBytes > memoryBudgetBytes()) {
if (!evictOldest(() => true, "total-budget-reconfigured")) break;
}
return configuredMemoryMb;
}
export function getConfiguredMetadataCacheMemoryMb(): number {
return configuredMemoryMb;
}
export function getMetadataRuntimeCache<T>(key: string): MetadataRuntimeCacheHit<T> | undefined {
const entry = entries.get(key) as RuntimeCacheEntry<T> | undefined;
if (!entry) {
l1Misses++;
return undefined;
}
entries.delete(key);
entry.touchedAt = now();
entries.set(key, entry);
l1Hits++;
logCacheMetric("metadata_cache_l1_hit", key, { entrySizeBytes: entry.sizeBytes }, entry.connectionId);
return { value: entry.value, sizeBytes: entry.sizeBytes };
}
export function setMetadataRuntimeCache<T>(key: string, value: T, connectionId?: string): boolean {
const sizeBytes = estimateValueBytes(value);
if (sizeBytes > METADATA_CACHE_MAX_ENTRY_BYTES) {
logCacheMetric("metadata_cache_l1_skip", key, { entrySizeBytes: sizeBytes, evictionReason: "entry-too-large" }, connectionId);
return false;
}
removeEntry(key);
const owner = keyConnectionId(key, connectionId);
if (!evictForInsert(owner, sizeBytes)) return false;
entries.set(key, { key, connectionId: owner, value, sizeBytes, touchedAt: now() });
totalBytes += sizeBytes;
logCacheMetric("metadata_cache_store", key, { entrySizeBytes: sizeBytes }, owner);
return true;
}
export function invalidateMetadataRuntimeCachePrefix(prefix: string): number {
let removed = 0;
for (const key of [...entries.keys()]) {
if (key === prefix || key.startsWith(prefix)) removed += removeEntry(key) ? 1 : 0;
}
return removed;
}
export function clearMetadataRuntimeCacheForConnection(connectionId: string): number {
let removed = 0;
for (const [key, entry] of [...entries.entries()]) {
if (entry.connectionId === connectionId) removed += removeEntry(key) ? 1 : 0;
}
connectionClears++;
logCacheMetric("metadata_cache_connection_clear", "", { connectionId, removedEntries: removed }, connectionId);
return removed;
}
export function clearMetadataRuntimeCacheForDatabase(connectionId: string, database: string): number {
let removed = 0;
for (const [key, entry] of [...entries.entries()]) {
const context = cacheLogContext(key, entry.connectionId);
if (entry.connectionId === connectionId && context.database === database) removed += removeEntry(key) ? 1 : 0;
}
logCacheMetric("metadata_cache_database_clear", "", { connectionId, database, removedEntries: removed }, connectionId);
return removed;
}
export function clearMetadataRuntimeCache(): void {
entries.clear();
totalBytes = 0;
l1Hits = 0;
l1Misses = 0;
l2Hits = 0;
remoteMisses = 0;
evictions = 0;
connectionClears = 0;
}
export function recordMetadataCacheL2Hit(key: string, value: unknown): void {
l2Hits++;
logCacheMetric("metadata_cache_l2_hit", key, { entrySizeBytes: estimateValueBytes(value) });
}
export function recordMetadataCacheRemoteMiss(key: string): void {
remoteMisses++;
logCacheMetric("metadata_cache_remote_miss", key);
}
export function getMetadataRuntimeCacheDiagnostics(): MetadataRuntimeCacheDiagnostics {
return {
entries: entries.size,
bytes: totalBytes,
maxMemoryBytes: memoryBudgetBytes(),
maxEntryBytes: METADATA_CACHE_MAX_ENTRY_BYTES,
configuredMemoryMb,
l1Hits,
l1Misses,
l2Hits,
remoteMisses,
evictions,
connectionClears,
};
}
+113 -29
View File
@@ -1,10 +1,13 @@
import * as api from "@/lib/backend/api"; import * as api from "@/lib/backend/api";
import { ActiveCacheReadTracker } from "./activeCacheReadTracker";
import type { ObjectSourceKind } from "@/types/database"; import type { ObjectSourceKind } from "@/types/database";
import type { MetadataCacheInvalidation } from "./metadataResultCache"; import type { MetadataCacheInvalidation } from "./metadataResultCache";
import { invalidateObjectMetadataCache } from "./objectMetadataCache"; import { invalidateObjectMetadataCache } from "./objectMetadataCache";
import { getMetadataRuntimeCache, invalidateMetadataRuntimeCachePrefix, recordMetadataCacheL2Hit, recordMetadataCacheRemoteMiss, setMetadataRuntimeCache } from "./metadataRuntimeCache";
const OBJECT_DDL_CACHE_PREFIX = "object-ddl:v1"; const OBJECT_DDL_CACHE_PREFIX = "object-ddl:v1";
const MAX_PERSISTED_DDL_CHARS = 5 * 1024 * 1024; const MAX_PERSISTED_DDL_CHARS = 5 * 1024 * 1024;
const MAX_PERSISTED_DDL_AGE_MS = 24 * 60 * 60 * 1000;
interface ObjectDdlCacheEnvelope { interface ObjectDdlCacheEnvelope {
version: 1; version: 1;
@@ -29,11 +32,13 @@ export interface ObjectDdlRequest {
export interface ObjectDdlLoadResult { export interface ObjectDdlLoadResult {
ddl: string; ddl: string;
cacheStatus: "disk" | "remote"; cacheStatus: "memory" | "disk" | "remote";
} }
const remoteLoads = new Map<string, InFlightDdlLoad>(); const remoteLoads = new Map<string, InFlightDdlLoad>();
const activeCacheReads = new ActiveCacheReadTracker();
const pendingInvalidations = new Map<string, Promise<void>>(); const pendingInvalidations = new Map<string, Promise<void>>();
const pendingWrites = new Map<string, Promise<void>>();
async function loadSchemaCacheSafe<T>(cacheKey: string): Promise<T | null> { async function loadSchemaCacheSafe<T>(cacheKey: string): Promise<T | null> {
try { try {
@@ -83,7 +88,8 @@ function invalidationPrefix(match: MetadataCacheInvalidation): string {
function decodeCachedDdl(payload: unknown): string | null { function decodeCachedDdl(payload: unknown): string | null {
if (!payload || typeof payload !== "object") return null; if (!payload || typeof payload !== "object") return null;
const envelope = payload as Partial<ObjectDdlCacheEnvelope>; const envelope = payload as Partial<ObjectDdlCacheEnvelope>;
if (envelope.version !== 1 || typeof envelope.cachedAt !== "string" || !Number.isFinite(Date.parse(envelope.cachedAt)) || typeof envelope.ddl !== "string") return null; const cachedAt = typeof envelope.cachedAt === "string" ? Date.parse(envelope.cachedAt) : Number.NaN;
if (envelope.version !== 1 || !Number.isFinite(cachedAt) || Date.now() - cachedAt > MAX_PERSISTED_DDL_AGE_MS || typeof envelope.ddl !== "string") return null;
return envelope.ddl; return envelope.ddl;
} }
@@ -92,21 +98,62 @@ async function waitForPendingInvalidations(cacheKey: string): Promise<void> {
if (pending.length) await Promise.all(pending); if (pending.length) await Promise.all(pending);
} }
function invalidateRemoteLoad(cacheKey: string): void {
const entry = remoteLoads.get(cacheKey);
if (!entry) return;
entry.invalidated = true;
remoteLoads.delete(cacheKey);
}
function persistSchemaCache(cacheKey: string, payload: unknown, isInvalidated: () => boolean): void {
const previous = pendingWrites.get(cacheKey);
const write = (async () => {
if (previous) await previous;
if (!isInvalidated()) await saveSchemaCacheSafe(cacheKey, payload);
})().finally(() => {
if (pendingWrites.get(cacheKey) === write) pendingWrites.delete(cacheKey);
});
pendingWrites.set(cacheKey, write);
}
async function waitForPendingWrites(prefix: string): Promise<void> {
while (true) {
const pending = [...pendingWrites.entries()].filter(([cacheKey]) => cacheKey.startsWith(prefix)).map(([, promise]) => promise);
if (!pending.length) return;
await Promise.all(pending);
}
}
function invalidatePersistedPrefix(prefix: string): Promise<void> {
const existing = pendingInvalidations.get(prefix);
if (existing) return existing;
const deletion = (async () => {
await waitForPendingWrites(prefix);
await deleteSchemaCachePrefixSafe(prefix);
})().finally(() => {
if (pendingInvalidations.get(prefix) === deletion) pendingInvalidations.delete(prefix);
});
pendingInvalidations.set(prefix, deletion);
return deletion;
}
async function loadRemoteDdl(request: ObjectDdlRequest, cacheKey: string, force: boolean): Promise<string> { async function loadRemoteDdl(request: ObjectDdlRequest, cacheKey: string, force: boolean): Promise<string> {
const existing = remoteLoads.get(cacheKey); const existing = remoteLoads.get(cacheKey);
if (existing && (!force || existing.force)) return existing.promise; if (existing && (!force || existing.force)) return existing.promise;
if (existing) { if (existing) invalidateRemoteLoad(cacheKey);
existing.invalidated = true;
remoteLoads.delete(cacheKey);
}
const entry: InFlightDdlLoad = { force, invalidated: false, promise: Promise.resolve("") }; const entry: InFlightDdlLoad = { force, invalidated: false, promise: Promise.resolve("") };
recordMetadataCacheRemoteMiss(cacheKey);
entry.promise = api entry.promise = api
.getTableDisplayDdl(request.connectionId, request.database, request.schema, request.tableName, request.objectType, request.catalog) .getTableDisplayDdl(request.connectionId, request.database, request.schema, request.tableName, request.objectType, request.catalog)
.then(async (ddl) => { .then((ddl) => {
if (!entry.invalidated && ddl.length <= MAX_PERSISTED_DDL_CHARS) { const current = !entry.invalidated;
if (current && ddl.length <= MAX_PERSISTED_DDL_CHARS) {
const envelope: ObjectDdlCacheEnvelope = { version: 1, cachedAt: new Date().toISOString(), ddl }; const envelope: ObjectDdlCacheEnvelope = { version: 1, cachedAt: new Date().toISOString(), ddl };
await saveSchemaCacheSafe(cacheKey, envelope); setMetadataRuntimeCache(cacheKey, ddl, request.connectionId);
persistSchemaCache(cacheKey, envelope, () => entry.invalidated);
} else if (current) {
setMetadataRuntimeCache(cacheKey, ddl, request.connectionId);
} }
return ddl; return ddl;
}) })
@@ -119,11 +166,38 @@ async function loadRemoteDdl(request: ObjectDdlRequest, cacheKey: string, force:
export async function loadObjectDdl(request: ObjectDdlRequest, options?: { force?: boolean }): Promise<ObjectDdlLoadResult> { export async function loadObjectDdl(request: ObjectDdlRequest, options?: { force?: boolean }): Promise<ObjectDdlLoadResult> {
const cacheKey = objectDdlCacheKey(request); const cacheKey = objectDdlCacheKey(request);
await waitForPendingInvalidations(cacheKey); if (options?.force) {
const existing = remoteLoads.get(cacheKey);
if (existing?.force) return { ddl: await existing.promise, cacheStatus: "remote" };
invalidateRemoteLoad(cacheKey);
activeCacheReads.invalidatePrefix(cacheKey);
invalidateMetadataRuntimeCachePrefix(cacheKey);
const priorInvalidations = waitForPendingInvalidations(cacheKey);
const deletion = invalidatePersistedPrefix(cacheKey);
await Promise.all([priorInvalidations, deletion]);
} else {
await waitForPendingInvalidations(cacheKey);
}
if (!options?.force) { if (!options?.force) {
const cached = decodeCachedDdl(await loadSchemaCacheSafe<unknown>(cacheKey)); const runtime = getMetadataRuntimeCache<string>(cacheKey);
if (cached !== null) return { ddl: cached, cacheStatus: "disk" }; if (runtime) return { ddl: runtime.value, cacheStatus: "memory" };
const readToken = activeCacheReads.begin(cacheKey);
let cached: string | null;
try {
cached = decodeCachedDdl(await loadSchemaCacheSafe<unknown>(cacheKey));
} finally {
activeCacheReads.finish(readToken);
}
if (readToken.invalidated) {
await waitForPendingInvalidations(cacheKey);
return { ddl: await loadRemoteDdl(request, cacheKey, false), cacheStatus: "remote" };
}
if (cached !== null) {
recordMetadataCacheL2Hit(cacheKey, cached);
setMetadataRuntimeCache(cacheKey, cached, request.connectionId);
return { ddl: cached, cacheStatus: "disk" };
}
} }
return { ddl: await loadRemoteDdl(request, cacheKey, options?.force === true), cacheStatus: "remote" }; return { ddl: await loadRemoteDdl(request, cacheKey, options?.force === true), cacheStatus: "remote" };
@@ -131,30 +205,21 @@ export async function loadObjectDdl(request: ObjectDdlRequest, options?: { force
export async function invalidateObjectDdlCache(match: MetadataCacheInvalidation): Promise<void> { export async function invalidateObjectDdlCache(match: MetadataCacheInvalidation): Promise<void> {
const prefix = invalidationPrefix(match); const prefix = invalidationPrefix(match);
for (const [cacheKey, entry] of remoteLoads) { activeCacheReads.invalidatePrefix(prefix);
if (!cacheKey.startsWith(prefix)) continue; for (const cacheKey of [...remoteLoads.keys()]) if (cacheKey.startsWith(prefix)) invalidateRemoteLoad(cacheKey);
entry.invalidated = true; invalidateMetadataRuntimeCachePrefix(prefix);
remoteLoads.delete(cacheKey);
}
const existing = pendingInvalidations.get(prefix); const deletion = invalidatePersistedPrefix(prefix);
if (existing) return existing;
const deletion = deleteSchemaCachePrefixSafe(prefix).finally(() => {
if (pendingInvalidations.get(prefix) === deletion) pendingInvalidations.delete(prefix);
});
pendingInvalidations.set(prefix, deletion);
await Promise.all([deletion, invalidateObjectMetadataCache(match)]); await Promise.all([deletion, invalidateObjectMetadataCache(match)]);
} }
export async function invalidateObjectDdl(request: ObjectDdlRequest): Promise<void> { export async function invalidateObjectDdl(request: ObjectDdlRequest): Promise<void> {
const cacheKey = objectDdlCacheKey(request); const cacheKey = objectDdlCacheKey(request);
const entry = remoteLoads.get(cacheKey); activeCacheReads.invalidatePrefix(cacheKey);
if (entry) { invalidateRemoteLoad(cacheKey);
entry.invalidated = true; invalidateMetadataRuntimeCachePrefix(cacheKey);
remoteLoads.delete(cacheKey);
}
await Promise.all([ await Promise.all([
deleteSchemaCachePrefixSafe(cacheKey), invalidatePersistedPrefix(cacheKey),
invalidateObjectMetadataCache({ invalidateObjectMetadataCache({
connectionId: request.connectionId, connectionId: request.connectionId,
database: request.database, database: request.database,
@@ -163,3 +228,22 @@ export async function invalidateObjectDdl(request: ObjectDdlRequest): Promise<vo
}), }),
]); ]);
} }
/** Invalidate active loads for a disconnected connection without deleting its persisted snapshot. */
export function cancelObjectDdlLoadsForConnection(connectionId: string): void {
const prefix = `${OBJECT_DDL_CACHE_PREFIX}:${cacheSegment(connectionId)}:`;
activeCacheReads.invalidatePrefix(prefix);
for (const cacheKey of [...remoteLoads.keys()]) if (cacheKey.startsWith(prefix)) invalidateRemoteLoad(cacheKey);
invalidateMetadataRuntimeCachePrefix(prefix);
}
export function cancelObjectDdlLoadsForDatabase(connectionId: string, database: string): void {
const prefix = `${OBJECT_DDL_CACHE_PREFIX}:${cacheSegment(connectionId)}:${cacheSegment(database)}:`;
activeCacheReads.invalidatePrefix(prefix);
for (const cacheKey of [...remoteLoads.keys()]) if (cacheKey.startsWith(prefix)) invalidateRemoteLoad(cacheKey);
invalidateMetadataRuntimeCachePrefix(prefix);
}
export function getObjectDdlCacheDebugStateForTests(): { activeReads: number } {
return { activeReads: activeCacheReads.activeCount };
}
@@ -1,9 +1,12 @@
import * as api from "@/lib/backend/api"; import * as api from "@/lib/backend/api";
import { ActiveCacheReadTracker } from "./activeCacheReadTracker";
import type { MetadataCacheInvalidation } from "./metadataResultCache"; import type { MetadataCacheInvalidation } from "./metadataResultCache";
import type { ObjectDdlRequest } from "./objectDdlCache"; import type { ObjectDdlRequest } from "./objectDdlCache";
import { getMetadataRuntimeCache, invalidateMetadataRuntimeCachePrefix, recordMetadataCacheL2Hit, recordMetadataCacheRemoteMiss, setMetadataRuntimeCache } from "./metadataRuntimeCache";
const OBJECT_METADATA_CACHE_PREFIX = "object-meta:v1"; const OBJECT_METADATA_CACHE_PREFIX = "object-meta:v1";
const MAX_PERSISTED_METADATA_CHARS = 5 * 1024 * 1024; const MAX_PERSISTED_METADATA_CHARS = 5 * 1024 * 1024;
const MAX_PERSISTED_METADATA_AGE_MS = 24 * 60 * 60 * 1000;
interface ObjectMetadataCacheEnvelope<T> { interface ObjectMetadataCacheEnvelope<T> {
version: 1; version: 1;
@@ -18,7 +21,9 @@ interface InFlightLoad<T> {
} }
const inFlightLoads = new Map<string, InFlightLoad<unknown>>(); const inFlightLoads = new Map<string, InFlightLoad<unknown>>();
const activeCacheReads = new ActiveCacheReadTracker();
const pendingInvalidations = new Map<string, Promise<void>>(); const pendingInvalidations = new Map<string, Promise<void>>();
const pendingWrites = new Map<string, Promise<void>>();
async function loadSchemaCacheSafe<T>(cacheKey: string): Promise<T | null> { async function loadSchemaCacheSafe<T>(cacheKey: string): Promise<T | null> {
try { try {
@@ -51,7 +56,7 @@ function cacheSegment(value: string | undefined): string {
} }
function facetKey(request: ObjectDdlRequest, facet: ObjectMetadataFacet): string { function facetKey(request: ObjectDdlRequest, facet: ObjectMetadataFacet): string {
return `${[OBJECT_METADATA_CACHE_PREFIX, cacheSegment(request.connectionId), cacheSegment(request.database), cacheSegment(request.schema), cacheSegment(request.tableName), cacheSegment(request.catalog), facet].join(":")}:`; return `${[OBJECT_METADATA_CACHE_PREFIX, cacheSegment(request.connectionId), cacheSegment(request.database), cacheSegment(request.schema), cacheSegment(request.tableName), cacheSegment(request.catalog), cacheSegment(request.objectType ?? "TABLE"), facet].join(":")}:`;
} }
function invalidationPrefix(match: MetadataCacheInvalidation): string { function invalidationPrefix(match: MetadataCacheInvalidation): string {
@@ -67,11 +72,12 @@ function invalidationPrefix(match: MetadataCacheInvalidation): string {
return `${parts.join(":")}:`; return `${parts.join(":")}:`;
} }
function decodeEnvelope<T>(payload: unknown): T | null { function decodeEnvelope<T>(payload: unknown): { hit: true; value: T } | undefined {
if (!payload || typeof payload !== "object") return null; if (!payload || typeof payload !== "object") return undefined;
const envelope = payload as Partial<ObjectMetadataCacheEnvelope<T>>; const envelope = payload as Partial<ObjectMetadataCacheEnvelope<T>>;
if (envelope.version !== 1 || typeof envelope.cachedAt !== "string" || !Number.isFinite(Date.parse(envelope.cachedAt)) || !("value" in envelope)) return null; const cachedAt = typeof envelope.cachedAt === "string" ? Date.parse(envelope.cachedAt) : Number.NaN;
return envelope.value === undefined ? null : envelope.value; if (envelope.version !== 1 || !Number.isFinite(cachedAt) || Date.now() - cachedAt > MAX_PERSISTED_METADATA_AGE_MS || !("value" in envelope)) return undefined;
return { hit: true, value: envelope.value as T };
} }
async function waitForPendingInvalidations(cacheKey: string): Promise<void> { async function waitForPendingInvalidations(cacheKey: string): Promise<void> {
@@ -79,29 +85,99 @@ async function waitForPendingInvalidations(cacheKey: string): Promise<void> {
if (pending.length) await Promise.all(pending); if (pending.length) await Promise.all(pending);
} }
export async function loadObjectMetadataFacet<T>(request: ObjectDdlRequest, facet: ObjectMetadataFacet, loader: () => Promise<T>, options?: { force?: boolean }): Promise<{ value: T; cacheStatus: "disk" | "remote" }> { function invalidateInFlightLoad(cacheKey: string): void {
const entry = inFlightLoads.get(cacheKey);
if (!entry) return;
entry.invalidated = true;
inFlightLoads.delete(cacheKey);
}
function persistSchemaCache(cacheKey: string, payload: unknown, isInvalidated: () => boolean): void {
const previous = pendingWrites.get(cacheKey);
const write = (async () => {
if (previous) await previous;
if (!isInvalidated()) await saveSchemaCacheSafe(cacheKey, payload);
})().finally(() => {
if (pendingWrites.get(cacheKey) === write) pendingWrites.delete(cacheKey);
});
pendingWrites.set(cacheKey, write);
}
async function waitForPendingWrites(prefix: string): Promise<void> {
while (true) {
const pending = [...pendingWrites.entries()].filter(([cacheKey]) => cacheKey.startsWith(prefix)).map(([, promise]) => promise);
if (!pending.length) return;
await Promise.all(pending);
}
}
function invalidatePersistedPrefix(prefix: string): Promise<void> {
const existing = pendingInvalidations.get(prefix);
if (existing) return existing;
const deletion = (async () => {
await waitForPendingWrites(prefix);
await deleteSchemaCachePrefixSafe(prefix);
})().finally(() => {
if (pendingInvalidations.get(prefix) === deletion) pendingInvalidations.delete(prefix);
});
pendingInvalidations.set(prefix, deletion);
return deletion;
}
export async function loadObjectMetadataFacet<T>(request: ObjectDdlRequest, facet: ObjectMetadataFacet, loader: () => Promise<T>, options?: { force?: boolean }): Promise<{ value: T; cacheStatus: "memory" | "disk" | "remote" }> {
const cacheKey = facetKey(request, facet); const cacheKey = facetKey(request, facet);
await waitForPendingInvalidations(cacheKey); if (options?.force) {
const existing = inFlightLoads.get(cacheKey);
if (existing?.force) return { value: (await existing.promise) as T, cacheStatus: "remote" };
invalidateInFlightLoad(cacheKey);
activeCacheReads.invalidatePrefix(cacheKey);
invalidateMetadataRuntimeCachePrefix(cacheKey);
const priorInvalidations = waitForPendingInvalidations(cacheKey);
const deletion = invalidatePersistedPrefix(cacheKey);
await Promise.all([priorInvalidations, deletion]);
} else {
await waitForPendingInvalidations(cacheKey);
}
if (!options?.force) { if (!options?.force) {
const cached = decodeEnvelope<T>(await loadSchemaCacheSafe<unknown>(cacheKey)); const runtime = getMetadataRuntimeCache<T>(cacheKey);
if (cached !== null) return { value: cached, cacheStatus: "disk" }; if (runtime) return { value: runtime.value, cacheStatus: "memory" };
const readToken = activeCacheReads.begin(cacheKey);
let cached: ReturnType<typeof decodeEnvelope<T>>;
try {
cached = decodeEnvelope<T>(await loadSchemaCacheSafe<unknown>(cacheKey));
} finally {
activeCacheReads.finish(readToken);
}
if (readToken.invalidated) {
const value = await loadObjectMetadataFacet(request, facet, loader, { force: false });
return { value: value.value, cacheStatus: "remote" };
}
if (cached?.hit) {
recordMetadataCacheL2Hit(cacheKey, cached.value);
setMetadataRuntimeCache(cacheKey, cached.value, request.connectionId);
return { value: cached.value, cacheStatus: "disk" };
}
} }
const existing = inFlightLoads.get(cacheKey); const existing = inFlightLoads.get(cacheKey);
if (existing && (!options?.force || existing.force)) return { value: (await existing.promise) as T, cacheStatus: "remote" }; if (existing && (!options?.force || existing.force)) return { value: (await existing.promise) as T, cacheStatus: "remote" };
if (existing) { if (existing) invalidateInFlightLoad(cacheKey);
existing.invalidated = true;
inFlightLoads.delete(cacheKey);
}
const entry: InFlightLoad<T> = { force: options?.force === true, invalidated: false, promise: Promise.resolve(undefined as T) }; const entry: InFlightLoad<T> = { force: options?.force === true, invalidated: false, promise: Promise.resolve(undefined as T) };
recordMetadataCacheRemoteMiss(cacheKey);
entry.promise = loader() entry.promise = loader()
.then(async (value) => { .then((value) => {
// Callers use undefined to signal that all metadata fallbacks failed.
// Keep that result visible to the current request, but do not turn a
// transient failure into a successful cache hit.
if (value === undefined) return value;
const serialized = JSON.stringify(value); const serialized = JSON.stringify(value);
if (!entry.invalidated && serialized.length <= MAX_PERSISTED_METADATA_CHARS) { const current = !entry.invalidated;
if (current) setMetadataRuntimeCache(cacheKey, value, request.connectionId);
if (current && serialized !== undefined && serialized.length <= MAX_PERSISTED_METADATA_CHARS) {
const envelope: ObjectMetadataCacheEnvelope<T> = { version: 1, cachedAt: new Date().toISOString(), value }; const envelope: ObjectMetadataCacheEnvelope<T> = { version: 1, cachedAt: new Date().toISOString(), value };
await saveSchemaCacheSafe(cacheKey, envelope); persistSchemaCache(cacheKey, envelope, () => entry.invalidated);
} }
return value; return value;
}) })
@@ -114,17 +190,28 @@ export async function loadObjectMetadataFacet<T>(request: ObjectDdlRequest, face
export async function invalidateObjectMetadataCache(match: MetadataCacheInvalidation): Promise<void> { export async function invalidateObjectMetadataCache(match: MetadataCacheInvalidation): Promise<void> {
const prefix = invalidationPrefix(match); const prefix = invalidationPrefix(match);
for (const [cacheKey, entry] of inFlightLoads) { activeCacheReads.invalidatePrefix(prefix);
if (!cacheKey.startsWith(prefix)) continue; for (const cacheKey of [...inFlightLoads.keys()]) if (cacheKey.startsWith(prefix)) invalidateInFlightLoad(cacheKey);
entry.invalidated = true; invalidateMetadataRuntimeCachePrefix(prefix);
inFlightLoads.delete(cacheKey);
}
const existing = pendingInvalidations.get(prefix); return invalidatePersistedPrefix(prefix);
if (existing) return existing; }
const deletion = deleteSchemaCachePrefixSafe(prefix).finally(() => {
if (pendingInvalidations.get(prefix) === deletion) pendingInvalidations.delete(prefix); /** Invalidate active loads for a disconnected connection without deleting its persisted snapshot. */
}); export function cancelObjectMetadataLoadsForConnection(connectionId: string): void {
pendingInvalidations.set(prefix, deletion); const prefix = `${OBJECT_METADATA_CACHE_PREFIX}:${cacheSegment(connectionId)}:`;
return deletion; activeCacheReads.invalidatePrefix(prefix);
for (const cacheKey of [...inFlightLoads.keys()]) if (cacheKey.startsWith(prefix)) invalidateInFlightLoad(cacheKey);
invalidateMetadataRuntimeCachePrefix(prefix);
}
export function cancelObjectMetadataLoadsForDatabase(connectionId: string, database: string): void {
const prefix = `${OBJECT_METADATA_CACHE_PREFIX}:${cacheSegment(connectionId)}:${cacheSegment(database)}:`;
activeCacheReads.invalidatePrefix(prefix);
for (const cacheKey of [...inFlightLoads.keys()]) if (cacheKey.startsWith(prefix)) invalidateInFlightLoad(cacheKey);
invalidateMetadataRuntimeCachePrefix(prefix);
}
export function getObjectMetadataCacheDebugStateForTests(): { activeReads: number } {
return { activeReads: activeCacheReads.activeCount };
} }
@@ -127,6 +127,45 @@ describe("settings search", () => {
expect(searchSettings(entries, "hidden in the background", "en").map((entry) => entry.id)).toEqual(["appearance-tray"]); expect(searchSettings(entries, "hidden in the background", "en").map((entry) => entry.id)).toEqual(["appearance-tray"]);
}); });
it("indexes the metadata cache memory limit under data settings", () => {
expect(SETTINGS_SEARCH_DEFINITIONS).toContainEqual(
expect.objectContaining({
id: "data-performance",
category: "data",
titleKey: "settings.performanceSection",
targetId: "data-performance",
}),
);
expect(SETTINGS_SEARCH_DEFINITIONS).toContainEqual(
expect.objectContaining({
id: "data-metadata-cache",
category: "data",
titleKey: "settings.metadataCacheMemoryLimit",
targetId: "data-performance",
}),
);
});
it("renders the metadata cache memory limit in a performance section after export", () => {
const exportSectionStart = settingsDialogSource.indexOf('t("settings.exportSection")');
const performanceSectionStart = settingsDialogSource.indexOf('data-settings-search-id="data-performance"');
const tableStructureSectionStart = settingsDialogSource.indexOf('t("settings.tableStructureSection")');
const metadataCacheControl = settingsDialogSource.indexOf('id="metadata-cache-memory-limit"');
expect(exportSectionStart).toBeGreaterThan(-1);
expect(performanceSectionStart).toBeGreaterThan(exportSectionStart);
expect(tableStructureSectionStart).toBeGreaterThan(performanceSectionStart);
expect(metadataCacheControl).toBeGreaterThan(performanceSectionStart);
expect(metadataCacheControl).toBeLessThan(tableStructureSectionStart);
});
it("defines the performance section title in every supported locale", () => {
for (const locale of ["zh-CN", "zh-TW", "en", "es", "it", "ja", "ko", "pt-BR"]) {
const source = readFileSync(new URL(`../../../i18n/locales/${locale}.ts`, import.meta.url), "utf8");
expect(source, locale).toContain("performanceSection:");
}
});
it("activates result buttons through click for keyboard and assistive technology", () => { it("activates result buttons through click for keyboard and assistive technology", () => {
expect(settingsDialogSource).toMatch(/role="option"[\s\S]*?@mousedown\.prevent[\s\S]*?@click="void selectSettingsSearchResult\(result\)"/); expect(settingsDialogSource).toMatch(/role="option"[\s\S]*?@mousedown\.prevent[\s\S]*?@click="void selectSettingsSearchResult\(result\)"/);
}); });
@@ -200,6 +200,8 @@ export const SETTINGS_SEARCH_DEFINITIONS: readonly SettingsSearchDefinition[] =
{ id: "data-export-row-limit-enabled", category: "data", titleKey: "settings.exportRowLimitEnabled", descriptionKey: "settings.exportRowLimitEnabledDescription", targetId: "data" }, { id: "data-export-row-limit-enabled", category: "data", titleKey: "settings.exportRowLimitEnabled", descriptionKey: "settings.exportRowLimitEnabledDescription", targetId: "data" },
{ id: "data-export-row-limit", category: "data", titleKey: "settings.exportRowLimit", descriptionKey: "settings.exportRowLimitDescription", targetId: "data" }, { id: "data-export-row-limit", category: "data", titleKey: "settings.exportRowLimit", descriptionKey: "settings.exportRowLimitDescription", targetId: "data" },
{ id: "data-export-keyset", category: "data", titleKey: "settings.queryExportKeysetOptimizationEnabled", descriptionKey: "settings.queryExportKeysetOptimizationEnabledDescription", targetId: "data" }, { id: "data-export-keyset", category: "data", titleKey: "settings.queryExportKeysetOptimizationEnabled", descriptionKey: "settings.queryExportKeysetOptimizationEnabledDescription", targetId: "data" },
{ id: "data-performance", category: "data", titleKey: "settings.performanceSection", targetId: "data-performance" },
{ id: "data-metadata-cache", category: "data", titleKey: "settings.metadataCacheMemoryLimit", descriptionKey: "settings.metadataCacheMemoryLimitDescription", targetId: "data-performance" },
{ id: "data-table-template", category: "data", titleKey: "settings.tableColumnTemplateFields", descriptionKey: "settings.tableColumnTemplateFieldsDescription", targetId: "table-column-templates" }, { id: "data-table-template", category: "data", titleKey: "settings.tableColumnTemplateFields", descriptionKey: "settings.tableColumnTemplateFieldsDescription", targetId: "table-column-templates" },
{ id: "data-duckdb", category: "data", titleKey: "settings.duckDbWorkerProcessIsolation", descriptionKey: "settings.duckDbWorkerProcessIsolationDescription", targetId: "data", visible: desktopOnly }, { id: "data-duckdb", category: "data", titleKey: "settings.duckDbWorkerProcessIsolation", descriptionKey: "settings.duckDbWorkerProcessIsolationDescription", targetId: "data", visible: desktopOnly },
{ id: "data-duckdb-process-limit", category: "data", titleKey: "settings.duckDbWorkerMaxProcesses", descriptionKey: "settings.duckDbWorkerMaxProcessesDescription", targetId: "data", visible: desktopOnly }, { id: "data-duckdb-process-limit", category: "data", titleKey: "settings.duckDbWorkerMaxProcesses", descriptionKey: "settings.duckDbWorkerMaxProcessesDescription", targetId: "data", visible: desktopOnly },
@@ -345,6 +345,13 @@ describe("right sidebar panel transitions", () => {
}); });
describe("normalizeDesktopSettings", () => { describe("normalizeDesktopSettings", () => {
it("normalizes the metadata cache memory budget", () => {
expect(normalizeDesktopSettings({}).metadata_cache_max_memory_mb).toBe(64);
expect(normalizeDesktopSettings({ metadata_cache_max_memory_mb: 1 }).metadata_cache_max_memory_mb).toBe(16);
expect(normalizeDesktopSettings({ metadata_cache_max_memory_mb: 384 }).metadata_cache_max_memory_mb).toBe(384);
expect(normalizeDesktopSettings({ metadata_cache_max_memory_mb: 513 }).metadata_cache_max_memory_mb).toBe(64);
});
it("defaults DuckDB worker process isolation to disabled for old settings", () => { it("defaults DuckDB worker process isolation to disabled for old settings", () => {
expect(normalizeDesktopSettings({}).duckdb_worker_process_isolation).toBe(false); expect(normalizeDesktopSettings({}).duckdb_worker_process_isolation).toBe(false);
}); });
+14 -1
View File
@@ -130,7 +130,9 @@ import { createMetadataLoadTrace, logMetadataLoadTrace, MetadataLoadCoordinator,
import type { MetadataScopeInput } from "@/lib/metadata/metadataLoadScope"; import type { MetadataScopeInput } from "@/lib/metadata/metadataLoadScope";
import { MetadataResultCache, type MetadataCacheInvalidation } from "@/lib/metadata/metadataResultCache"; import { MetadataResultCache, type MetadataCacheInvalidation } from "@/lib/metadata/metadataResultCache";
import { invalidateTableMetadataCache } from "@/lib/metadata/tableMetadataCache"; import { invalidateTableMetadataCache } from "@/lib/metadata/tableMetadataCache";
import { invalidateObjectDdlCache } from "@/lib/metadata/objectDdlCache"; import { cancelObjectDdlLoadsForConnection, cancelObjectDdlLoadsForDatabase, invalidateObjectDdlCache } from "@/lib/metadata/objectDdlCache";
import { cancelObjectMetadataLoadsForConnection, cancelObjectMetadataLoadsForDatabase } from "@/lib/metadata/objectMetadataCache";
import { clearMetadataRuntimeCacheForConnection, clearMetadataRuntimeCacheForDatabase } from "@/lib/metadata/metadataRuntimeCache";
import { invalidateObjectBrowserRowsCache } from "@/lib/table/objectBrowserRowsCache"; import { invalidateObjectBrowserRowsCache } from "@/lib/table/objectBrowserRowsCache";
import { MetadataTaskLimiter } from "@/lib/metadata/metadataTaskLimiter"; import { MetadataTaskLimiter } from "@/lib/metadata/metadataTaskLimiter";
import { buildCustomTypeTreeChildren } from "@/lib/sidebar/customTypeTree"; import { buildCustomTypeTreeChildren } from "@/lib/sidebar/customTypeTree";
@@ -3322,6 +3324,8 @@ export const useConnectionStore = defineStore("connection", () => {
clearConnectionHealthCheck(connectionId); clearConnectionHealthCheck(connectionId);
if (activeConnectionId.value === connectionId) activeConnectionId.value = null; if (activeConnectionId.value === connectionId) activeConnectionId.value = null;
invalidateCompletionCache(connectionId); invalidateCompletionCache(connectionId);
cancelObjectDdlLoadsForConnection(connectionId);
cancelObjectMetadataLoadsForConnection(connectionId);
await disconnectRequest; await disconnectRequest;
return true; return true;
} }
@@ -3345,6 +3349,9 @@ export const useConnectionStore = defineStore("connection", () => {
node.children = []; node.children = [];
} }
clearConnectionRootMetadataLoad(connectionId); clearConnectionRootMetadataLoad(connectionId);
cancelObjectDdlLoadsForConnection(connectionId);
cancelObjectMetadataLoadsForConnection(connectionId);
clearMetadataRuntimeCacheForConnection(connectionId);
// Disconnecting only tears down the live session. Keep the schema snapshot so // Disconnecting only tears down the live session. Keep the schema snapshot so
// reconnecting can render databases and table names before the remote refresh. // reconnecting can render databases and table names before the remote refresh.
clearLoadedChildrenCache(connectionId, { deletePersisted: false }); clearLoadedChildrenCache(connectionId, { deletePersisted: false });
@@ -3387,7 +3394,10 @@ export const useConnectionStore = defineStore("connection", () => {
async function closeDatabaseConnection(connectionId: string, database: string) { async function closeDatabaseConnection(connectionId: string, database: string) {
if (hasSqlServerActivityTraceForConnection(connectionId, database)) await disposeSqlServerActivityTracesForConnection(connectionId, database); if (hasSqlServerActivityTraceForConnection(connectionId, database)) await disposeSqlServerActivityTracesForConnection(connectionId, database);
cancelObjectDdlLoadsForDatabase(connectionId, database);
cancelObjectMetadataLoadsForDatabase(connectionId, database);
await api.closeDatabaseConnection(connectionId, database); await api.closeDatabaseConnection(connectionId, database);
clearMetadataRuntimeCacheForDatabase(connectionId, database);
const { useQueryStore } = await import("@/stores/queryStore"); const { useQueryStore } = await import("@/stores/queryStore");
const queryStore = useQueryStore(); const queryStore = useQueryStore();
switch (settingsStore.editorSettings.disconnectTabHandlingMode) { switch (settingsStore.editorSettings.disconnectTabHandlingMode) {
@@ -3425,6 +3435,9 @@ export const useConnectionStore = defineStore("connection", () => {
return; return;
} catch { } catch {
// Backend pool is dead — remove from connectedIds and reconnect // Backend pool is dead — remove from connectedIds and reconnect
cancelObjectDdlLoadsForConnection(connectionId);
cancelObjectMetadataLoadsForConnection(connectionId);
clearMetadataRuntimeCacheForConnection(connectionId);
connectedIds.value.delete(connectionId); connectedIds.value.delete(connectionId);
clearPrimaryVisibleObjectNames(connectionId); clearPrimaryVisibleObjectNames(connectionId);
clearConnectionHealthCheck(connectionId); clearConnectionHealthCheck(connectionId);
+7
View File
@@ -23,6 +23,7 @@ import { normalizeSqlVariableSyntaxOverrides, type SqlVariableSyntaxOverrides }
import { DEFAULT_TABLE_COLUMN_TEMPLATE_FIELDS, normalizeTableColumnTemplateFields } from "@/lib/table/tableColumnTemplates"; import { DEFAULT_TABLE_COLUMN_TEMPLATE_FIELDS, normalizeTableColumnTemplateFields } from "@/lib/table/tableColumnTemplates";
import { type DataTabReuseMode, DEFAULT_DATA_TAB_REUSE_MODE, normalizeDataTabReuseMode } from "@/lib/tabs/dataTabReuseMode"; import { type DataTabReuseMode, DEFAULT_DATA_TAB_REUSE_MODE, normalizeDataTabReuseMode } from "@/lib/tabs/dataTabReuseMode";
import { normalizeCompletionTriggerMode, type SqlCompletionTriggerMode } from "@/lib/sql/sqlCompletionTriggerPolicy"; import { normalizeCompletionTriggerMode, type SqlCompletionTriggerMode } from "@/lib/sql/sqlCompletionTriggerPolicy";
import { configureMetadataRuntimeCache, METADATA_CACHE_DEFAULT_MEMORY_MB, normalizeMetadataCacheMemoryMb } from "@/lib/metadata/metadataRuntimeCache";
import type { AiApiStyle, AiAssistantMode, AiAuthMethod, AiChatSelectionState, AiConfig, AiConfigItem, AiConfiguredModel, AiEffortLevel, AiEffortSelection, AiModelEffortPreference, AiProvider, AiReasoningLevel, AiTestConnectionResult } from "@/types/ai"; import type { AiApiStyle, AiAssistantMode, AiAuthMethod, AiChatSelectionState, AiConfig, AiConfigItem, AiConfiguredModel, AiEffortLevel, AiEffortSelection, AiModelEffortPreference, AiProvider, AiReasoningLevel, AiTestConnectionResult } from "@/types/ai";
import type { SqlSnippet, TableInfoTab } from "@/types/database"; import type { SqlSnippet, TableInfoTab } from "@/types/database";
@@ -34,6 +35,7 @@ export interface DesktopSettings {
quit_on_close: boolean; quit_on_close: boolean;
close_action_prompted: boolean; close_action_prompted: boolean;
debug_logging_enabled: boolean; debug_logging_enabled: boolean;
metadata_cache_max_memory_mb: number;
duckdb_worker_process_isolation: boolean; duckdb_worker_process_isolation: boolean;
duckdb_worker_max_processes: number; duckdb_worker_max_processes: number;
saved_sql_sync_dir?: string | null; saved_sql_sync_dir?: string | null;
@@ -70,6 +72,7 @@ export const DEFAULT_DESKTOP_SETTINGS: DesktopSettings = {
quit_on_close: false, quit_on_close: false,
close_action_prompted: false, close_action_prompted: false,
debug_logging_enabled: false, debug_logging_enabled: false,
metadata_cache_max_memory_mb: METADATA_CACHE_DEFAULT_MEMORY_MB,
duckdb_worker_process_isolation: false, duckdb_worker_process_isolation: false,
duckdb_worker_max_processes: DUCKDB_WORKER_MAX_PROCESSES_DEFAULT, duckdb_worker_max_processes: DUCKDB_WORKER_MAX_PROCESSES_DEFAULT,
saved_sql_sync_dir: null, saved_sql_sync_dir: null,
@@ -105,6 +108,7 @@ export function normalizeDesktopSettings(settings: Partial<DesktopSettings> | nu
quit_on_close: settings?.quit_on_close ?? DEFAULT_DESKTOP_SETTINGS.quit_on_close, quit_on_close: settings?.quit_on_close ?? DEFAULT_DESKTOP_SETTINGS.quit_on_close,
close_action_prompted: settings?.close_action_prompted ?? DEFAULT_DESKTOP_SETTINGS.close_action_prompted, close_action_prompted: settings?.close_action_prompted ?? DEFAULT_DESKTOP_SETTINGS.close_action_prompted,
debug_logging_enabled: settings?.debug_logging_enabled ?? DEFAULT_DESKTOP_SETTINGS.debug_logging_enabled, debug_logging_enabled: settings?.debug_logging_enabled ?? DEFAULT_DESKTOP_SETTINGS.debug_logging_enabled,
metadata_cache_max_memory_mb: normalizeMetadataCacheMemoryMb(settings?.metadata_cache_max_memory_mb),
duckdb_worker_process_isolation: settings?.duckdb_worker_process_isolation ?? DEFAULT_DESKTOP_SETTINGS.duckdb_worker_process_isolation, duckdb_worker_process_isolation: settings?.duckdb_worker_process_isolation ?? DEFAULT_DESKTOP_SETTINGS.duckdb_worker_process_isolation,
duckdb_worker_max_processes: normalizeDuckDbWorkerMaxProcesses(settings?.duckdb_worker_max_processes), duckdb_worker_max_processes: normalizeDuckDbWorkerMaxProcesses(settings?.duckdb_worker_max_processes),
saved_sql_sync_dir: settings?.saved_sql_sync_dir?.trim() || DEFAULT_DESKTOP_SETTINGS.saved_sql_sync_dir, saved_sql_sync_dir: settings?.saved_sql_sync_dir?.trim() || DEFAULT_DESKTOP_SETTINGS.saved_sql_sync_dir,
@@ -1389,6 +1393,7 @@ export const useSettingsStore = defineStore("settings", () => {
if (isDesktopSettingsLoaded.value) return; if (isDesktopSettingsLoaded.value) return;
desktopSettings.value = normalizeDesktopSettings(await api.loadDesktopSettings().catch(() => null)); desktopSettings.value = normalizeDesktopSettings(await api.loadDesktopSettings().catch(() => null));
setDebugLoggingEnabled(desktopSettings.value.debug_logging_enabled); setDebugLoggingEnabled(desktopSettings.value.debug_logging_enabled);
configureMetadataRuntimeCache(desktopSettings.value.metadata_cache_max_memory_mb);
isDesktopSettingsLoaded.value = true; isDesktopSettingsLoaded.value = true;
} }
@@ -1400,11 +1405,13 @@ export const useSettingsStore = defineStore("settings", () => {
}; };
desktopSettings.value = normalizeDesktopSettings(next); desktopSettings.value = normalizeDesktopSettings(next);
setDebugLoggingEnabled(desktopSettings.value.debug_logging_enabled); setDebugLoggingEnabled(desktopSettings.value.debug_logging_enabled);
configureMetadataRuntimeCache(desktopSettings.value.metadata_cache_max_memory_mb);
try { try {
await api.saveDesktopSettings(desktopSettings.value); await api.saveDesktopSettings(desktopSettings.value);
} catch (error) { } catch (error) {
desktopSettings.value = previous; desktopSettings.value = previous;
setDebugLoggingEnabled(previous.debug_logging_enabled); setDebugLoggingEnabled(previous.debug_logging_enabled);
configureMetadataRuntimeCache(previous.metadata_cache_max_memory_mb);
throw error; throw error;
} }
} }
+200 -31
View File
@@ -3,7 +3,7 @@ use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::Arc; use std::sync::Arc;
use std::time::{Duration, Instant}; use std::time::{Duration, Instant};
use tokio::sync::{watch, Mutex, RwLock}; use tokio::sync::{watch, Mutex, OwnedSemaphorePermit, RwLock, Semaphore};
use mysql_async::prelude::Queryable; use mysql_async::prelude::Queryable;
use mysql_async::Row as MysqlRow; use mysql_async::Row as MysqlRow;
@@ -44,6 +44,9 @@ const DEFAULT_AGENT_CONNECT_TIMEOUT_SECS: u64 = 30;
const ACCESS_AGENT_CONNECT_TIMEOUT_SECS: u64 = 30; const ACCESS_AGENT_CONNECT_TIMEOUT_SECS: u64 = 30;
const POOL_CLOSE_TIMEOUT_SECS: u64 = 3; const POOL_CLOSE_TIMEOUT_SECS: u64 = 3;
const HEALTH_CHECK_POOL_ACQUIRE_TIMEOUT: Duration = Duration::from_millis(500); const HEALTH_CHECK_POOL_ACQUIRE_TIMEOUT: Duration = Duration::from_millis(500);
pub(crate) const METADATA_POOL_ACQUIRE_TIMEOUT: Duration = Duration::from_secs(5);
pub(crate) const METADATA_POOL_DEFAULT_LIMIT: usize = 6;
pub(crate) const METADATA_POOL_SQLSERVER_LIMIT: usize = 1;
mod duckdb_types { mod duckdb_types {
#[cfg(feature = "duckdb-sidecar")] #[cfg(feature = "duckdb-sidecar")]
@@ -115,6 +118,42 @@ pub enum PoolKind {
} }
impl PoolKind { impl PoolKind {
/// Clone only the handles used by metadata operations so the global pool
/// map lock can be released before any database I/O begins.
pub(crate) fn clone_for_metadata(&self) -> Option<Self> {
match self {
Self::Mysql(pool, mode) => Some(Self::Mysql(pool.clone(), *mode)),
Self::Postgres(pool) => Some(Self::Postgres(pool.clone())),
Self::Sqlite(pool) => Some(Self::Sqlite(pool.clone())),
Self::Rqlite(client) => Some(Self::Rqlite(client.clone())),
Self::Turso(client) => Some(Self::Turso(client.clone())),
Self::CloudflareD1(client) => Some(Self::CloudflareD1(client.clone())),
Self::DuckDbWorker(client) => Some(Self::DuckDbWorker(client.clone())),
Self::MongoDb(client) => Some(Self::MongoDb(client.clone())),
Self::ClickHouse(client) => Some(Self::ClickHouse(client.clone())),
Self::SqlServer(client) => Some(Self::SqlServer(client.clone())),
Self::Elasticsearch(client) => Some(Self::Elasticsearch(client.clone())),
Self::Easysearch(client) => Some(Self::Easysearch(client.clone())),
Self::Meilisearch(client) => Some(Self::Meilisearch(client.clone())),
Self::HBase(client) => Some(Self::HBase(client.clone())),
Self::VectorDb(client) => Some(Self::VectorDb(client.clone())),
Self::InfluxDb(client) => Some(Self::InfluxDb(client.clone())),
Self::VictoriaMetrics(client) => Some(Self::VictoriaMetrics(client.clone())),
Self::Agent(client) => Some(Self::Agent(client.clone())),
Self::ExternalDriver { driver_id, config, session } => Some(Self::ExternalDriver {
driver_id: driver_id.clone(),
config: config.clone(),
session: session.clone(),
}),
Self::MessageQueue => Some(Self::MessageQueue),
Self::Nacos => Some(Self::Nacos),
Self::Consul(client) => Some(Self::Consul(client.clone())),
#[cfg(feature = "mq-admin")]
Self::Mqtt(client) => Some(Self::Mqtt(client.clone())),
_ => None,
}
}
pub fn agent(client: db::agent_driver::AgentDriverClient) -> Self { pub fn agent(client: db::agent_driver::AgentDriverClient) -> Self {
Self::Agent(Arc::new(db::agent_driver::PooledAgentClient::new(client))) Self::Agent(Arc::new(db::agent_driver::PooledAgentClient::new(client)))
} }
@@ -221,6 +260,7 @@ pub struct AppState {
/// Web 端 owner 为已认证会话 token,不同登录会话互不可见。建池/池重建/ /// Web 端 owner 为已认证会话 token,不同登录会话互不可见。建池/池重建/
/// AI/元数据从它读取,前端通过状态接口查询。 /// AI/元数据从它读取,前端通过状态接口查询。
pub session_credentials: SessionCredentialStore, pub session_credentials: SessionCredentialStore,
metadata_gates: Arc<Mutex<HashMap<String, Arc<Semaphore>>>>,
#[cfg(feature = "mq-admin")] #[cfg(feature = "mq-admin")]
pub mq_registry: crate::mq::MqAdminRegistry, pub mq_registry: crate::mq::MqAdminRegistry,
} }
@@ -250,6 +290,32 @@ struct ConnectionAttemptState {
client_attempt: Option<u64>, client_attempt: Option<u64>,
} }
pub(crate) fn metadata_concurrency_limit(db_type: DatabaseType, max_connections: usize) -> usize {
if db_type == DatabaseType::SqlServer {
METADATA_POOL_SQLSERVER_LIMIT
} else {
max_connections.saturating_sub(2).clamp(1, METADATA_POOL_DEFAULT_LIMIT)
}
}
pub(crate) fn uses_metadata_gate(db_type: DatabaseType) -> bool {
matches!(db_type, DatabaseType::Mysql | DatabaseType::Postgres | DatabaseType::SqlServer)
}
fn metadata_gate_key(
connection_id: &str,
database: Option<&str>,
db_type: DatabaseType,
client_session_id: Option<&str>,
) -> String {
let session = if db_type == DatabaseType::SqlServer {
""
} else {
client_session_id.map(str::trim).filter(|session| !session.is_empty()).unwrap_or_default()
};
format!("{connection_id}\0{}\0{}", database.unwrap_or_default(), session)
}
struct PoolDrainGuard { struct PoolDrainGuard {
pool_key: String, pool_key: String,
draining_pools: Arc<std::sync::Mutex<HashMap<String, watch::Sender<bool>>>>, draining_pools: Arc<std::sync::Mutex<HashMap<String, watch::Sender<bool>>>>,
@@ -1091,11 +1157,65 @@ impl AppState {
postgres_cancel_contexts: Arc::new(RwLock::new(HashMap::new())), postgres_cancel_contexts: Arc::new(RwLock::new(HashMap::new())),
transaction_sessions: Arc::new(RwLock::new(HashMap::new())), transaction_sessions: Arc::new(RwLock::new(HashMap::new())),
session_credentials: SessionCredentialStore::new(), session_credentials: SessionCredentialStore::new(),
metadata_gates: Arc::new(Mutex::new(HashMap::new())),
#[cfg(feature = "mq-admin")] #[cfg(feature = "mq-admin")]
mq_registry: crate::mq::MqAdminRegistry::new(), mq_registry: crate::mq::MqAdminRegistry::new(),
} }
} }
pub(crate) async fn acquire_metadata_permit(
&self,
connection_id: &str,
database: Option<&str>,
db_type: DatabaseType,
client_session_id: Option<&str>,
) -> Result<OwnedSemaphorePermit, String> {
let key = metadata_gate_key(connection_id, database, db_type, client_session_id);
let max_connections =
if client_session_id.map(str::trim).is_some_and(|session| !session.is_empty()) { 1 } else { 10 };
let limit = metadata_concurrency_limit(db_type, max_connections);
let gate = {
let mut gates = self.metadata_gates.lock().await;
gates.entry(key.clone()).or_insert_with(|| Arc::new(Semaphore::new(limit))).clone()
};
let started = Instant::now();
let queued = gate.available_permits() == 0;
let permit = match tokio::time::timeout(METADATA_POOL_ACQUIRE_TIMEOUT, gate.acquire_owned()).await {
Ok(Ok(permit)) => permit,
Ok(Err(_)) => return Err(crate::query::METADATA_POOL_BUSY_ERROR.to_string()),
Err(_) => {
log::warn!(
"[metadata:pool:busy] connection_id={} database={} wait_ms={} limit={}",
connection_id,
database.unwrap_or_default(),
started.elapsed().as_millis(),
limit
);
return Err(crate::query::METADATA_POOL_BUSY_ERROR.to_string());
}
};
if queued {
log::debug!(
"[metadata:pool:acquired] connection_id={} database={} wait_ms={} limit={}",
connection_id,
database.unwrap_or_default(),
started.elapsed().as_millis(),
limit
);
}
Ok(permit)
}
async fn clear_metadata_gates_for_connection(&self, connection_id: &str) {
let prefix = format!("{connection_id}\0");
self.metadata_gates.lock().await.retain(|key, _| !key.starts_with(&prefix));
}
async fn clear_metadata_gate_for_database(&self, connection_id: &str, database: Option<&str>) {
let prefix = format!("{connection_id}\0{}\0", database.unwrap_or_default());
self.metadata_gates.lock().await.retain(|key, _| !key.starts_with(&prefix));
}
pub fn jdbc_unavailable_error(&self) -> String { pub fn jdbc_unavailable_error(&self) -> String {
match self.plugins.find_driver("jdbc") { match self.plugins.find_driver("jdbc") {
Ok(Some(_)) => "JDBC plugin is installed, but the connection could not be opened.".to_string(), Ok(Some(_)) => "JDBC plugin is installed, but the connection could not be opened.".to_string(),
@@ -3021,18 +3141,18 @@ impl AppState {
PoolKind::Mysql(pool, _) => { PoolKind::Mysql(pool, _) => {
let pool = pool.clone(); let pool = pool.clone();
drop(connections); drop(connections);
match tokio::time::timeout(HEALTH_CHECK_POOL_ACQUIRE_TIMEOUT, pool.get_conn()).await { match db::mysql::checkout_mysql_conn(&pool, HEALTH_CHECK_POOL_ACQUIRE_TIMEOUT).await {
// Pool saturation means active work, not a dead connection. Removing this pool would // Pool saturation means active work, not a dead connection. Removing this pool would
// start a competing reconnect while foreground queries and metadata are still running. // start a competing reconnect while foreground queries and metadata are still running.
Err(_) => { Err(err) if err.is_pool_saturation() => {
log::debug!("MySQL connection pool '{pool_key}' is busy; skipping health probe"); log::debug!("MySQL connection pool '{pool_key}' is busy; skipping health probe: {err}");
false false
} }
Ok(Err(err)) => { Err(err) => {
log::warn!("MySQL connection pool '{pool_key}' is stale: {err}"); log::warn!("MySQL connection pool '{pool_key}' is stale: {err}");
true true
} }
Ok(Ok(mut conn)) => { Ok(mut conn) => {
let timeout = crate::db::connection_timeout(); let timeout = crate::db::connection_timeout();
match tokio::time::timeout(timeout, conn.ping()).await { match tokio::time::timeout(timeout, conn.ping()).await {
Ok(Ok(())) => false, Ok(Ok(())) => false,
@@ -3052,8 +3172,14 @@ impl AppState {
let pool = pool.clone(); let pool = pool.clone();
drop(connections); drop(connections);
let timeout = crate::db::connection_timeout(); let timeout = crate::db::connection_timeout();
match tokio::time::timeout(HEALTH_CHECK_POOL_ACQUIRE_TIMEOUT, pool.get()).await { match db::postgres::checkout_postgres_client_classified(
Ok(Ok(client)) => match tokio::time::timeout(timeout, client.simple_query("SELECT 1")).await { &pool,
None,
HEALTH_CHECK_POOL_ACQUIRE_TIMEOUT,
)
.await
{
Ok(client) => match tokio::time::timeout(timeout, client.simple_query("SELECT 1")).await {
Ok(Ok(_)) => false, Ok(Ok(_)) => false,
Ok(Err(err)) => { Ok(Err(err)) => {
log::warn!("PostgreSQL connection pool '{pool_key}' is stale: {err}"); log::warn!("PostgreSQL connection pool '{pool_key}' is stale: {err}");
@@ -3064,14 +3190,16 @@ impl AppState {
true true
} }
}, },
Ok(Err(err)) => { Err(err) if err.is_pool_saturation() => {
log::debug!(
"PostgreSQL connection pool '{pool_key}' is busy; skipping health probe: {err}"
);
false
}
Err(err) => {
log::warn!("PostgreSQL connection pool '{pool_key}' is stale: {err}"); log::warn!("PostgreSQL connection pool '{pool_key}' is stale: {err}");
true true
} }
Err(_) => {
log::debug!("PostgreSQL connection pool '{pool_key}' is busy; skipping health probe");
false
}
} }
} }
PoolKind::SqlServer(client) => { PoolKind::SqlServer(client) => {
@@ -3742,6 +3870,7 @@ impl AppState {
} }
drop(conns); drop(conns);
let closed = !removed.is_empty(); let closed = !removed.is_empty();
self.clear_metadata_gate_for_database(connection_id, database).await;
for (key, pool) in removed { for (key, pool) in removed {
self.pool_routing_control().close_pool_with_timeout(key, pool).await; self.pool_routing_control().close_pool_with_timeout(key, pool).await;
} }
@@ -4034,32 +4163,38 @@ impl AppState {
let healthy = match pool { let healthy = match pool {
PoolKind::Mysql(p, _) => match db::mysql::get_conn_with_health_check(p).await { PoolKind::Mysql(p, _) => match db::mysql::get_conn_with_health_check(p).await {
Ok(_) => true, Ok(_) => true,
Err(e) if crate::query::is_pool_saturation_error(&e) => {
log::debug!("MySQL connection pool '{key}' is busy; skipping health probe: {e}");
true
}
Err(e) => { Err(e) => {
log::warn!("MySQL connection pool '{key}' is unhealthy: {e}"); log::warn!("MySQL connection pool '{key}' is unhealthy: {e}");
false false
} }
}, },
PoolKind::Postgres(p) => match tokio::time::timeout(timeout, p.get()).await { PoolKind::Postgres(p) => {
Ok(Ok(client)) => match tokio::time::timeout(timeout, client.simple_query("SELECT 1")).await { match db::postgres::checkout_postgres_client_classified(p, None, timeout).await {
Ok(Ok(_)) => true, Ok(client) => match tokio::time::timeout(timeout, client.simple_query("SELECT 1")).await {
Ok(Err(e)) => { Ok(Ok(_)) => true,
log::warn!("PostgreSQL connection pool '{key}' is unhealthy: {e}"); Ok(Err(e)) => {
log::warn!("PostgreSQL connection pool '{key}' is unhealthy: {e}");
false
}
Err(_) => {
log::warn!("PostgreSQL connection pool '{key}' is unhealthy: health check timed out");
false
}
},
Err(error) if error.is_pool_saturation() => {
log::debug!("PostgreSQL connection pool '{key}' is busy; skipping health probe: {error}");
true
}
Err(error) => {
log::warn!("PostgreSQL connection pool '{key}' is unhealthy: {error}");
false false
} }
Err(_) => {
log::warn!("PostgreSQL connection pool '{key}' is unhealthy: health check timed out");
false
}
},
Ok(Err(e)) => {
log::warn!("PostgreSQL connection pool '{key}' is unhealthy: {e}");
false
} }
Err(_) => { }
log::warn!("PostgreSQL connection pool '{key}' is unhealthy: get connection timed out");
false
}
},
PoolKind::SqlServer(client) => { PoolKind::SqlServer(client) => {
let mut client = client.lock().await; let mut client = client.lock().await;
match db::sqlserver::test_connection(&mut client).await { match db::sqlserver::test_connection(&mut client).await {
@@ -4280,11 +4415,13 @@ impl AppState {
pub async fn remove_connection_pools(&self, connection_id: &str) { pub async fn remove_connection_pools(&self, connection_id: &str) {
let removed = self.drain_connection_pools(connection_id).await; let removed = self.drain_connection_pools(connection_id).await;
self.clear_metadata_gates_for_connection(connection_id).await;
self.pool_routing_control().close_removed(removed).await; self.pool_routing_control().close_removed(removed).await;
} }
pub async fn remove_connection_pools_detached(&self, connection_id: &str) { pub async fn remove_connection_pools_detached(&self, connection_id: &str) {
let removed = self.drain_connection_pools(connection_id).await; let removed = self.drain_connection_pools(connection_id).await;
self.clear_metadata_gates_for_connection(connection_id).await;
self.pool_routing_control().close_removed_in_background(removed); self.pool_routing_control().close_removed_in_background(removed);
} }
@@ -4303,6 +4440,7 @@ impl AppState {
self.session_credentials.clear_pool_owners(); self.session_credentials.clear_pool_owners();
self.postgres_cancel_contexts.write().await.clear(); self.postgres_cancel_contexts.write().await.clear();
self.draining_pools.lock().unwrap_or_else(|error| error.into_inner()).clear(); self.draining_pools.lock().unwrap_or_else(|error| error.into_inner()).clear();
self.metadata_gates.lock().await.clear();
self.connections.write().await.drain().collect() self.connections.write().await.drain().collect()
} }
@@ -6859,6 +6997,37 @@ mod tests {
assert_eq!(super::mysql_pool_max_connections_for_session(Some("tab-1")), 1); assert_eq!(super::mysql_pool_max_connections_for_session(Some("tab-1")), 1);
} }
#[test]
fn metadata_concurrency_reserves_query_capacity() {
assert_eq!(super::metadata_concurrency_limit(DatabaseType::Mysql, 10), 6);
assert_eq!(super::metadata_concurrency_limit(DatabaseType::Postgres, 10), 6);
assert_eq!(super::metadata_concurrency_limit(DatabaseType::Mysql, 3), 1);
assert_eq!(super::metadata_concurrency_limit(DatabaseType::Postgres, 1), 1);
assert_eq!(super::metadata_concurrency_limit(DatabaseType::SqlServer, 10), 1);
}
#[test]
fn metadata_gate_is_limited_to_supported_pool_drivers() {
assert!(super::uses_metadata_gate(DatabaseType::Mysql));
assert!(super::uses_metadata_gate(DatabaseType::Postgres));
assert!(super::uses_metadata_gate(DatabaseType::SqlServer));
assert!(!super::uses_metadata_gate(DatabaseType::Oracle));
assert!(!super::uses_metadata_gate(DatabaseType::Sqlite));
assert!(!super::uses_metadata_gate(DatabaseType::MongoDb));
}
#[test]
fn sqlserver_metadata_gate_is_shared_across_client_sessions() {
assert_eq!(
super::metadata_gate_key("conn", Some("app"), DatabaseType::SqlServer, Some("metadata-1")),
super::metadata_gate_key("conn", Some("app"), DatabaseType::SqlServer, Some("metadata-2"))
);
assert_ne!(
super::metadata_gate_key("conn", Some("app"), DatabaseType::Mysql, Some("tab-1")),
super::metadata_gate_key("conn", Some("app"), DatabaseType::Mysql, Some("tab-2"))
);
}
#[test] #[test]
fn prestosql_uses_external_driver_pool_not_agent_pool() { fn prestosql_uses_external_driver_pool_not_agent_pool() {
assert!(!super::uses_agent_connection_pool(&DatabaseType::PrestoSql)); assert!(!super::uses_agent_connection_pool(&DatabaseType::PrestoSql));
+19 -5
View File
@@ -1848,6 +1848,17 @@ pub async fn export_database_sql_core(
state: &crate::connection::AppState, state: &crate::connection::AppState,
request: &DatabaseExportRequest, request: &DatabaseExportRequest,
on_progress: impl Fn(ExportProgress) + Sync, on_progress: impl Fn(ExportProgress) + Sync,
) -> Result<(), String> {
// Keep the large export state machine on the heap. Besides making the
// caller future small, this prevents the metadata-prefetch locals from
// exhausting the bounded stack used by test and runtime worker threads.
Box::pin(export_database_sql_core_inner(state, request, on_progress)).await
}
async fn export_database_sql_core_inner(
state: &crate::connection::AppState,
request: &DatabaseExportRequest,
on_progress: impl Fn(ExportProgress) + Sync,
) -> Result<(), String> { ) -> Result<(), String> {
let _snapshot_keep_alive = if let Some(snapshot_session_id) = request.snapshot_session_id.as_deref() { let _snapshot_keep_alive = if let Some(snapshot_session_id) = request.snapshot_session_id.as_deref() {
Some(crate::query::keep_manual_transaction_alive(state, snapshot_session_id).await?) Some(crate::query::keep_manual_transaction_alive(state, snapshot_session_id).await?)
@@ -1869,12 +1880,15 @@ pub async fn export_database_sql_core(
// 2. Get pool // 2. Get pool
let client_session_id = database_export_client_session_id(&request.export_id); let client_session_id = database_export_client_session_id(&request.export_id);
let pool_key = state let pool_key = Box::pin(state.get_or_create_pool_for_session(
.get_or_create_pool_for_session(&request.connection_id, Some(&request.database), Some(&client_session_id)) &request.connection_id,
.await?; Some(&request.database),
Some(&client_session_id),
))
.await?;
// 3. List tables // 3. List tables
let all_tables = crate::schema::list_tables_core( let all_tables = Box::pin(crate::schema::list_tables_core(
state, state,
&request.connection_id, &request.connection_id,
&request.database, &request.database,
@@ -1884,7 +1898,7 @@ pub async fn export_database_sql_core(
None, None,
None, None,
None, None,
) ))
.await?; .await?;
// 4. Create file // 4. Create file
let mut expected_destination_dev = None; let mut expected_destination_dev = None;
+57
View File
@@ -52,6 +52,7 @@ pub mod victoriametrics_driver;
pub mod wkb; pub mod wkb;
use reqwest::ClientBuilder; use reqwest::ClientBuilder;
use std::fmt;
use std::future::Future; use std::future::Future;
use std::time::Duration; use std::time::Duration;
@@ -62,6 +63,62 @@ pub use file_validator::validate_file_path;
pub const CONNECTION_TIMEOUT_SECS: u64 = 5; pub const CONNECTION_TIMEOUT_SECS: u64 = 5;
pub const TCP_PROBE_TIMEOUT_SECS: u64 = 3; pub const TCP_PROBE_TIMEOUT_SECS: u64 = 3;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PoolCheckoutStage {
Wait,
Create,
Recycle,
Unknown,
}
impl PoolCheckoutStage {
pub fn as_str(self) -> &'static str {
match self {
Self::Wait => "wait",
Self::Create => "create",
Self::Recycle => "recycle",
Self::Unknown => "unknown",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum PoolCheckoutError {
Timeout { database: &'static str, stage: PoolCheckoutStage, timeout: Duration },
Failed { database: &'static str, stage: PoolCheckoutStage, detail: String },
Canceled,
}
impl PoolCheckoutError {
pub fn stage(&self) -> Option<PoolCheckoutStage> {
match self {
Self::Timeout { stage, .. } | Self::Failed { stage, .. } => Some(*stage),
Self::Canceled => None,
}
}
pub fn is_pool_saturation(&self) -> bool {
matches!(self, Self::Timeout { stage: PoolCheckoutStage::Wait, .. })
}
}
impl fmt::Display for PoolCheckoutError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Timeout { database, stage, timeout } => write!(
formatter,
"{database} connection pool checkout timed out [stage={}, timeout_ms={}]",
stage.as_str(),
timeout.as_millis()
),
Self::Failed { database, stage, detail } => {
write!(formatter, "{database} connection pool checkout failed [stage={}]: {detail}", stage.as_str())
}
Self::Canceled => formatter.write_str(crate::query::QUERY_CANCELED),
}
}
}
pub fn connection_timeout() -> Duration { pub fn connection_timeout() -> Duration {
Duration::from_secs(CONNECTION_TIMEOUT_SECS) Duration::from_secs(CONNECTION_TIMEOUT_SECS)
} }
+270 -25
View File
@@ -9,7 +9,9 @@ use sqlparser::dialect::MySqlDialect;
use sqlparser::parser::Parser; use sqlparser::parser::Parser;
use std::borrow::Cow; use std::borrow::Cow;
use std::collections::{HashMap, HashSet}; use std::collections::{HashMap, HashSet};
#[cfg(test)]
use std::future::Future; use std::future::Future;
use std::ops::Deref;
use std::path::PathBuf; use std::path::PathBuf;
use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicBool;
use std::time::Duration; use std::time::Duration;
@@ -28,7 +30,62 @@ use crate::types::{
use super::file_validator::validate_file_path; use super::file_validator::validate_file_path;
pub type MySqlPool = mysql_async::Pool; /// DBX-owned MySQL pool handle. The driver does not expose its configured
/// maximum, so retain that value beside the driver pool for checkout phase
/// classification instead of guessing it from aggregate metrics.
#[derive(Debug, Clone)]
pub struct MySqlPool {
inner: mysql_async::Pool,
max_connections: usize,
}
impl MySqlPool {
pub(crate) fn new<O>(opts: O, max_connections: usize) -> Self
where
mysql_async::Opts: TryFrom<O>,
<mysql_async::Opts as TryFrom<O>>::Error: std::error::Error,
{
Self { inner: mysql_async::Pool::new(opts), max_connections: max_connections.max(1) }
}
pub async fn disconnect(self) -> Result<(), mysql_async::Error> {
self.inner.disconnect().await
}
}
impl Deref for MySqlPool {
type Target = mysql_async::Pool;
fn deref(&self) -> &Self::Target {
&self.inner
}
}
#[doc(hidden)]
pub trait MySqlPoolAccess {
fn driver_pool(&self) -> &mysql_async::Pool;
fn checkout_max_connections(&self) -> Option<usize>;
}
impl MySqlPoolAccess for MySqlPool {
fn driver_pool(&self) -> &mysql_async::Pool {
&self.inner
}
fn checkout_max_connections(&self) -> Option<usize> {
Some(self.max_connections)
}
}
impl MySqlPoolAccess for mysql_async::Pool {
fn driver_pool(&self) -> &mysql_async::Pool {
self
}
fn checkout_max_connections(&self) -> Option<usize> {
None
}
}
const MYSQL_TCP_KEEPALIVE_MS: u32 = 30_000; const MYSQL_TCP_KEEPALIVE_MS: u32 = 30_000;
const MYSQL_SQL_PACKET_MARGIN_MAX_BYTES: usize = 64 * 1024; const MYSQL_SQL_PACKET_MARGIN_MAX_BYTES: usize = 64 * 1024;
@@ -1242,7 +1299,7 @@ fn create_pool(
// to paths explicitly supplied by the user instead of enabling arbitrary reads. // to paths explicitly supplied by the user instead of enabling arbitrary reads.
builder = builder.local_infile_handler(Some(mysql_async::WhiteListFsHandler::new(local_infile_paths))); builder = builder.local_infile_handler(Some(mysql_async::WhiteListFsHandler::new(local_infile_paths)));
} }
Ok(MySqlPool::new(builder)) Ok(MySqlPool::new(builder, max_connections))
} }
fn mysql_async_tcp_host(host: &str) -> &str { fn mysql_async_tcp_host(host: &str) -> &str {
@@ -3603,7 +3660,10 @@ fn parse_mysql_enum_values(column_type: &str) -> Option<Vec<String>> {
} }
} }
pub async fn get_columns(pool: &MySqlPool, database: &str, table: &str) -> Result<Vec<ColumnInfo>, String> { pub async fn get_columns<P>(pool: &P, database: &str, table: &str) -> Result<Vec<ColumnInfo>, String>
where
P: MySqlPoolAccess + ?Sized,
{
let sql = columns_sql(database, table); let sql = columns_sql(database, table);
let mut conn = get_conn_with_health_check(pool).await?; let mut conn = get_conn_with_health_check(pool).await?;
let result = match conn.query_iter(&sql).await { let result = match conn.query_iter(&sql).await {
@@ -3675,7 +3735,10 @@ pub async fn get_columns(pool: &MySqlPool, database: &str, table: &str) -> Resul
Ok(columns) Ok(columns)
} }
pub async fn get_columns_show(pool: &MySqlPool, database: &str, table: &str) -> Result<Vec<ColumnInfo>, String> { pub async fn get_columns_show<P>(pool: &P, database: &str, table: &str) -> Result<Vec<ColumnInfo>, String>
where
P: MySqlPoolAccess + ?Sized,
{
let sql = show_columns_sql(database, table, true); let sql = show_columns_sql(database, table, true);
let mut conn = get_conn_with_health_check(pool).await?; let mut conn = get_conn_with_health_check(pool).await?;
let rows: Vec<mysql_async::Row> = match conn.query_iter(&sql).await { let rows: Vec<mysql_async::Row> = match conn.query_iter(&sql).await {
@@ -3852,23 +3915,32 @@ pub(super) fn skip_mysql_quoted(sql: &str, start: usize, quote: u8) -> usize {
/// Get a connection from the pool with a health check. If the connection is dead /// Get a connection from the pool with a health check. If the connection is dead
/// (e.g. after app was backgrounded), it tries again with a fresh connection. /// (e.g. after app was backgrounded), it tries again with a fresh connection.
pub async fn get_conn_with_health_check(pool: &MySqlPool) -> Result<mysql_async::Conn, String> { pub async fn get_conn_with_health_check<P>(pool: &P) -> Result<mysql_async::Conn, String>
where
P: MySqlPoolAccess + ?Sized,
{
get_conn_with_health_check_with_timeout(pool, super::connection_timeout()).await get_conn_with_health_check_with_timeout(pool, super::connection_timeout()).await
} }
pub async fn get_conn_with_health_check_with_timeout( pub async fn get_conn_with_health_check_with_timeout<P>(
pool: &MySqlPool, pool: &P,
timeout: Duration, timeout: Duration,
) -> Result<mysql_async::Conn, String> { ) -> Result<mysql_async::Conn, String>
where
P: MySqlPoolAccess + ?Sized,
{
get_conn_with_health_check_with_cancel(pool, timeout, timeout, None).await get_conn_with_health_check_with_cancel(pool, timeout, timeout, None).await
} }
pub async fn get_conn_with_health_check_with_cancel( pub async fn get_conn_with_health_check_with_cancel<P>(
pool: &MySqlPool, pool: &P,
timeout: Duration, timeout: Duration,
cleanup_timeout: Duration, cleanup_timeout: Duration,
cancel_token: Option<&CancellationToken>, cancel_token: Option<&CancellationToken>,
) -> Result<mysql_async::Conn, String> { ) -> Result<mysql_async::Conn, String>
where
P: MySqlPoolAccess + ?Sized,
{
let start = Instant::now(); let start = Instant::now();
let mut conn = get_conn_with_timeout_and_cancel(pool, timeout, cancel_token).await?; let mut conn = get_conn_with_timeout_and_cancel(pool, timeout, cancel_token).await?;
match ping_conn_with_timeout_and_cancel(&mut conn, timeout, cancel_token).await { match ping_conn_with_timeout_and_cancel(&mut conn, timeout, cancel_token).await {
@@ -3909,13 +3981,67 @@ pub async fn get_conn_with_health_check_with_cancel(
} }
} }
async fn get_conn_with_timeout_and_cancel( #[derive(Debug, Clone, Copy, PartialEq, Eq)]
pool: &MySqlPool, pub(crate) struct MysqlCheckoutSnapshot {
pub(crate) connection_count: usize,
pub(crate) connections_in_pool: usize,
pub(crate) max_connections: usize,
}
pub(crate) type MysqlCheckoutStage = super::PoolCheckoutStage;
pub(crate) fn classify_mysql_checkout_stage(snapshot: MysqlCheckoutSnapshot) -> MysqlCheckoutStage {
if snapshot.connections_in_pool > 0 {
MysqlCheckoutStage::Recycle
} else if snapshot.connection_count < snapshot.max_connections {
MysqlCheckoutStage::Create
} else {
MysqlCheckoutStage::Wait
}
}
fn mysql_checkout_snapshot<P>(pool: &P, max_connections: usize) -> MysqlCheckoutSnapshot
where
P: MySqlPoolAccess + ?Sized,
{
let metrics = pool.driver_pool().metrics();
use std::sync::atomic::Ordering;
MysqlCheckoutSnapshot {
connection_count: metrics.connection_count.load(Ordering::Relaxed),
connections_in_pool: metrics.connections_in_pool.load(Ordering::Relaxed),
max_connections,
}
}
pub(crate) async fn checkout_mysql_conn<P>(
pool: &P,
timeout: Duration,
) -> Result<mysql_async::Conn, super::PoolCheckoutError>
where
P: MySqlPoolAccess + ?Sized,
{
// Capture the phase before entering the driver future. The phase is tied
// to this checkout's capacity decision, not to a later aggregate metric
// snapshot that may already include another request.
let stage = pool.checkout_max_connections().map_or(MysqlCheckoutStage::Unknown, |max_connections| {
classify_mysql_checkout_stage(mysql_checkout_snapshot(pool, max_connections))
});
match tokio::time::timeout(timeout, pool.driver_pool().get_conn()).await {
Ok(Ok(conn)) => Ok(conn),
Ok(Err(error)) => Err(super::PoolCheckoutError::Failed { database: "MySQL", stage, detail: error.to_string() }),
Err(_) => Err(super::PoolCheckoutError::Timeout { database: "MySQL", stage, timeout }),
}
}
async fn get_conn_with_timeout_and_cancel<P>(
pool: &P,
timeout: Duration, timeout: Duration,
cancel_token: Option<&CancellationToken>, cancel_token: Option<&CancellationToken>,
) -> Result<mysql_async::Conn, String> { ) -> Result<mysql_async::Conn, String>
let get_future = where
connection_result_with_timeout(timeout, async { pool.get_conn().await.map_err(|error| error.to_string()) }); P: MySqlPoolAccess + ?Sized,
{
let get_future = async { checkout_mysql_conn(pool, timeout).await.map_err(|error| error.to_string()) };
match cancel_token { match cancel_token {
Some(token) => { Some(token) => {
@@ -3929,10 +4055,14 @@ async fn get_conn_with_timeout_and_cancel(
} }
} }
pub async fn get_conn_with_timeout(pool: &MySqlPool, timeout: Duration) -> Result<mysql_async::Conn, String> { pub async fn get_conn_with_timeout<P>(pool: &P, timeout: Duration) -> Result<mysql_async::Conn, String>
connection_result_with_timeout(timeout, async { pool.get_conn().await.map_err(|error| error.to_string()) }).await where
P: MySqlPoolAccess + ?Sized,
{
checkout_mysql_conn(pool, timeout).await.map_err(|error| error.to_string())
} }
#[cfg(test)]
async fn connection_result_with_timeout<T, F>(timeout: Duration, future: F) -> Result<T, String> async fn connection_result_with_timeout<T, F>(timeout: Duration, future: F) -> Result<T, String>
where where
F: Future<Output = Result<T, String>>, F: Future<Output = Result<T, String>>,
@@ -4564,13 +4694,16 @@ pub(crate) fn mysql_sql_statement_hard_limit(max_allowed_packet: u64) -> Option<
packet_bytes.checked_sub(margin).filter(|limit| *limit > 0) packet_bytes.checked_sub(margin).filter(|limit| *limit > 0)
} }
pub async fn execute_query_with_max_rows( pub async fn execute_query_with_max_rows<P>(
pool: &MySqlPool, pool: &P,
sql: &str, sql: &str,
bare: bool, bare: bool,
max_rows: Option<usize>, max_rows: Option<usize>,
dialect: MySqlQueryDialect, dialect: MySqlQueryDialect,
) -> Result<QueryResult, String> { ) -> Result<QueryResult, String>
where
P: MySqlPoolAccess + ?Sized,
{
let mut conn = get_conn_with_health_check(pool).await?; let mut conn = get_conn_with_health_check(pool).await?;
execute_query_on_conn_with_max_rows(&mut conn, sql, bare, max_rows, dialect).await execute_query_on_conn_with_max_rows(&mut conn, sql, bare, max_rows, dialect).await
} }
@@ -5357,10 +5490,10 @@ mod tests {
tokio::time::sleep(Duration::from_millis(10)).await; tokio::time::sleep(Duration::from_millis(10)).await;
Ok::<_, String>("connection") Ok::<_, String>("connection")
}); });
let configured_five = connection_result_with_timeout(Duration::from_millis(50), async { let configured_five = connection_result_with_timeout(
tokio::time::sleep(Duration::from_millis(60)).await; Duration::from_millis(10),
Ok::<_, String>("connection") std::future::pending::<Result<&'static str, String>>(),
}); );
let (exact, adjacent, configured_five) = tokio::join!(exact, adjacent, configured_five); let (exact, adjacent, configured_five) = tokio::join!(exact, adjacent, configured_five);
@@ -5369,6 +5502,118 @@ mod tests {
assert_eq!(configured_five, Err("MySQL get connection timed out".to_string())); assert_eq!(configured_five, Err("MySQL get connection timed out".to_string()));
} }
#[tokio::test]
async fn mysql_checkout_fault_injection_classifies_full_pool_as_wait() {
assert!(tokio::time::timeout(Duration::from_millis(1), std::future::pending::<()>()).await.is_err());
assert_eq!(
classify_mysql_checkout_stage(MysqlCheckoutSnapshot {
connection_count: 1,
connections_in_pool: 0,
max_connections: 1,
}),
MysqlCheckoutStage::Wait
);
}
#[tokio::test]
async fn mysql_checkout_fault_injection_classifies_hung_connection_create() {
assert!(tokio::time::timeout(Duration::from_millis(1), std::future::pending::<()>()).await.is_err());
assert_eq!(
classify_mysql_checkout_stage(MysqlCheckoutSnapshot {
connection_count: 0,
connections_in_pool: 0,
max_connections: 2,
}),
MysqlCheckoutStage::Create
);
}
#[tokio::test]
async fn mysql_checkout_fault_injection_preserves_create_stage_when_handshake_hangs() {
let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)).await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (_socket, _) = listener.accept().await.unwrap();
std::future::pending::<()>().await;
});
let pool_options =
mysql_async::PoolOpts::new().with_constraints(mysql_async::PoolConstraints::new(1, 2).unwrap());
let options = mysql_async::OptsBuilder::default()
.ip_or_hostname(address.ip().to_string())
.tcp_port(address.port())
.user(Some("fault-injection"))
.pass(Some("fault-injection"))
.pool_opts(Some(pool_options));
let pool = MySqlPool::new(options, 2);
let error = checkout_mysql_conn(&pool, Duration::from_millis(50)).await.unwrap_err();
assert!(matches!(error, super::super::PoolCheckoutError::Timeout { stage: MysqlCheckoutStage::Create, .. }));
server.abort();
let _ = tokio::time::timeout(Duration::from_secs(1), pool.disconnect()).await;
}
#[tokio::test]
async fn mysql_checkout_raw_driver_pool_timeout_keeps_stage_unknown() {
let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)).await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (_socket, _) = listener.accept().await.unwrap();
std::future::pending::<()>().await;
});
let options = mysql_async::OptsBuilder::default()
.ip_or_hostname(address.ip().to_string())
.tcp_port(address.port())
.user(Some("fault-injection"))
.pass(Some("fault-injection"));
let pool = mysql_async::Pool::new(options);
let error = checkout_mysql_conn(&pool, Duration::from_millis(50)).await.unwrap_err();
assert!(matches!(error, super::super::PoolCheckoutError::Timeout { stage: MysqlCheckoutStage::Unknown, .. }));
server.abort();
let _ = tokio::time::timeout(Duration::from_secs(1), pool.disconnect()).await;
}
#[tokio::test]
#[ignore = "requires DBX_REVIEW_MYSQL_* environment variables"]
async fn mysql_checkout_fault_injection_preserves_wait_stage_when_real_pool_is_full() {
let host = std::env::var("DBX_REVIEW_MYSQL_HOST").expect("DBX_REVIEW_MYSQL_HOST is required");
let port = std::env::var("DBX_REVIEW_MYSQL_PORT").expect("DBX_REVIEW_MYSQL_PORT is required");
let user = std::env::var("DBX_REVIEW_MYSQL_USER").expect("DBX_REVIEW_MYSQL_USER is required");
let password = std::env::var("DBX_REVIEW_MYSQL_PASSWORD").expect("DBX_REVIEW_MYSQL_PASSWORD is required");
let database = std::env::var("DBX_REVIEW_MYSQL_DATABASE").expect("DBX_REVIEW_MYSQL_DATABASE is required");
let pool_options =
mysql_async::PoolOpts::new().with_constraints(mysql_async::PoolConstraints::new(1, 1).unwrap());
let options = mysql_async::OptsBuilder::default()
.ip_or_hostname(host)
.tcp_port(port.parse().expect("DBX_REVIEW_MYSQL_PORT must be a valid port"))
.user(Some(user))
.pass(Some(password))
.db_name(Some(database))
.pool_opts(Some(pool_options));
let pool = MySqlPool::new(options, 1);
let held_connection = tokio::time::timeout(Duration::from_secs(10), pool.get_conn())
.await
.expect("real MySQL checkout timed out")
.expect("real MySQL checkout failed");
let error = checkout_mysql_conn(&pool, Duration::from_millis(50)).await.unwrap_err();
assert!(matches!(error, super::super::PoolCheckoutError::Timeout { stage: MysqlCheckoutStage::Wait, .. }));
drop(held_connection);
let _ = tokio::time::timeout(Duration::from_secs(2), pool.disconnect()).await;
}
#[test]
fn mysql_public_metadata_and_query_helpers_accept_driver_pool_for_compatibility() {
let pool = mysql_async::Pool::new(mysql_async::OptsBuilder::default());
let columns = get_columns(&pool, "database", "table");
let query = execute_query_with_max_rows(&pool, "SELECT 1", false, Some(1), MySqlQueryDialect::default());
drop((columns, query));
}
#[tokio::test] #[tokio::test]
async fn mysql_database_list_timeout_preserves_immediate_errors_and_fallback_plan() { async fn mysql_database_list_timeout_preserves_immediate_errors_and_fallback_plan() {
let error = let error =
+105 -26
View File
@@ -5900,40 +5900,70 @@ async fn execute_postgres_user_query(
/// PostgreSQL pool checkout with timeout and cancel token support. /// PostgreSQL pool checkout with timeout and cancel token support.
/// When the checkout phase is stuck, the cancel token can terminate the wait early. /// When the checkout phase is stuck, the cancel token can terminate the wait early.
/// The timeout error message includes "checkout timed out" to ensure is_connection_error can classify it correctly. pub(crate) type PostgresCheckoutStage = super::PoolCheckoutStage;
pub async fn checkout_postgres_client(
pub(crate) fn classify_postgres_checkout_stage(status: deadpool_postgres::Status) -> PostgresCheckoutStage {
if status.waiting > 0 || (status.available == 0 && status.size >= status.max_size) {
PostgresCheckoutStage::Wait
} else if status.available > 0 {
PostgresCheckoutStage::Recycle
} else if status.size < status.max_size {
PostgresCheckoutStage::Create
} else {
PostgresCheckoutStage::Unknown
}
}
fn postgres_pool_error_stage(error: &PoolError) -> PostgresCheckoutStage {
match error {
PoolError::Timeout(deadpool_postgres::TimeoutType::Wait) => PostgresCheckoutStage::Wait,
PoolError::Timeout(deadpool_postgres::TimeoutType::Create) => PostgresCheckoutStage::Create,
PoolError::Timeout(deadpool_postgres::TimeoutType::Recycle) => PostgresCheckoutStage::Recycle,
PoolError::Backend(_) | PoolError::PostCreateHook(_) => PostgresCheckoutStage::Create,
PoolError::Closed | PoolError::NoRuntimeSpecified => PostgresCheckoutStage::Unknown,
}
}
pub(crate) async fn checkout_postgres_client_classified(
pool: &Pool, pool: &Pool,
cancel_token: Option<&CancellationToken>, cancel_token: Option<&CancellationToken>,
checkout_timeout: Duration, checkout_timeout: Duration,
) -> Result<deadpool_postgres::Object, String> { ) -> Result<deadpool_postgres::Object, super::PoolCheckoutError> {
let checkout_timeout = effective_postgres_checkout_timeout(pool, checkout_timeout);
let start = Instant::now(); let start = Instant::now();
let get_future = async { let get_future = async {
tokio::time::timeout(checkout_timeout, pool.get()) match tokio::time::timeout(checkout_timeout, pool.get()).await {
.await Ok(Ok(client)) => Ok(client),
.map_err(|_| { Ok(Err(error)) => {
let elapsed = start.elapsed().as_millis(); let stage = postgres_pool_error_stage(&error);
let timed_out = matches!(&error, PoolError::Timeout(_));
let detail = pg_pool_error_to_string(error);
log::warn!( log::warn!(
"[db:pool.checkout:error] elapsed_ms={} timeout_ms={} error=checkout timed out", "[db:pool.checkout:error] elapsed_ms={} timeout_ms={} stage={} error={}",
elapsed, start.elapsed().as_millis(),
checkout_timeout.as_millis()
);
format!("PostgreSQL connection pool checkout timed out ({}s)", checkout_timeout.as_secs())
})?
.map_err(|e| {
let elapsed = start.elapsed().as_millis();
let err = pg_pool_error_to_string(e);
log::warn!(
"[db:pool.checkout:error] elapsed_ms={} timeout_ms={} error={}",
elapsed,
checkout_timeout.as_millis(), checkout_timeout.as_millis(),
err stage.as_str(),
detail
); );
format!("PostgreSQL connection pool checkout failed: {err}") if timed_out {
}) Err(super::PoolCheckoutError::Timeout { database: "PostgreSQL", stage, timeout: checkout_timeout })
} else {
Err(super::PoolCheckoutError::Failed { database: "PostgreSQL", stage, detail })
}
}
Err(_) => {
let stage = classify_postgres_checkout_stage(pool.status());
log::warn!(
"[db:pool.checkout:error] elapsed_ms={} timeout_ms={} stage={} error=checkout timed out",
start.elapsed().as_millis(),
checkout_timeout.as_millis(),
stage.as_str()
);
Err(super::PoolCheckoutError::Timeout { database: "PostgreSQL", stage, timeout: checkout_timeout })
}
}
}; };
let result = match cancel_token { match cancel_token {
Some(token) => tokio::select! { Some(token) => tokio::select! {
biased; biased;
_ = token.cancelled() => { _ = token.cancelled() => {
@@ -5942,12 +5972,24 @@ pub async fn checkout_postgres_client(
start.elapsed().as_millis(), start.elapsed().as_millis(),
checkout_timeout.as_millis() checkout_timeout.as_millis()
); );
return Err(crate::query::canceled_error()); Err(super::PoolCheckoutError::Canceled)
} }
result = get_future => result, result = get_future => result,
}, },
None => get_future.await, None => get_future.await,
}; }
}
pub async fn checkout_postgres_client(
pool: &Pool,
cancel_token: Option<&CancellationToken>,
checkout_timeout: Duration,
) -> Result<deadpool_postgres::Object, String> {
let checkout_timeout = effective_postgres_checkout_timeout(pool, checkout_timeout);
let start = Instant::now();
let result = checkout_postgres_client_classified(pool, cancel_token, checkout_timeout)
.await
.map_err(|error| error.to_string());
if let Ok(client) = &result { if let Ok(client) = &result {
log::debug!( log::debug!(
"[db:pool.checkout:done] elapsed_ms={} timeout_ms={}", "[db:pool.checkout:done] elapsed_ms={} timeout_ms={}",
@@ -8763,6 +8805,43 @@ mod tests {
assert_eq!(effective_postgres_checkout_timeout(&pool, Duration::from_secs(15)), Duration::from_secs(15)); assert_eq!(effective_postgres_checkout_timeout(&pool, Duration::from_secs(15)), Duration::from_secs(15));
} }
#[tokio::test]
async fn postgres_checkout_fault_injection_classifies_full_pool_as_wait() {
assert!(tokio::time::timeout(Duration::from_millis(1), std::future::pending::<()>()).await.is_err());
assert_eq!(
classify_postgres_checkout_stage(deadpool_postgres::Status {
max_size: 1,
size: 1,
available: 0,
waiting: 1
}),
PostgresCheckoutStage::Wait
);
}
#[tokio::test]
async fn postgres_checkout_fault_injection_preserves_create_and_recycle_stages() {
assert!(tokio::time::timeout(Duration::from_millis(1), std::future::pending::<()>()).await.is_err());
assert_eq!(
classify_postgres_checkout_stage(deadpool_postgres::Status {
max_size: 2,
size: 1,
available: 0,
waiting: 0
}),
PostgresCheckoutStage::Create
);
assert_eq!(
classify_postgres_checkout_stage(deadpool_postgres::Status {
max_size: 1,
size: 1,
available: 1,
waiting: 0
}),
PostgresCheckoutStage::Recycle
);
}
#[tokio::test] #[tokio::test]
async fn postgres_session_setup_reports_its_own_timeout_stage() { async fn postgres_session_setup_reports_its_own_timeout_stage() {
let error = let error =
+42 -6
View File
@@ -32,6 +32,16 @@ pub const QUERY_TIMEOUT: Duration = Duration::from_secs(30);
pub const MAX_ROWS: usize = 10000; pub const MAX_ROWS: usize = 10000;
pub const AGENT_PROTOCOL_MAX_ROWS: usize = i32::MAX as usize; pub const AGENT_PROTOCOL_MAX_ROWS: usize = i32::MAX as usize;
pub const QUERY_CANCELED: &str = "Query canceled"; pub const QUERY_CANCELED: &str = "Query canceled";
pub const METADATA_POOL_BUSY_ERROR: &str = "DBX metadata pool is busy; please retry";
/// Returns true when a metadata request failed because all pool/client slots
/// were temporarily occupied. This is deliberately separate from
/// `is_connection_error`: a saturated pool is healthy and must not trigger a
/// shared-pool reconnect.
pub fn is_pool_saturation_error(err: &str) -> bool {
let lower = err.to_lowercase();
lower.contains("connection pool checkout timed out [stage=wait") || lower.contains("dbx metadata pool is busy")
}
/// Fallback when a Mongo connection hits the generic SQL executor instead of the shell path. /// Fallback when a Mongo connection hits the generic SQL executor instead of the shell path.
/// Wording must match packages/mongo-shell `MONGO_SHELL_COMMAND_HINT` /// Wording must match packages/mongo-shell `MONGO_SHELL_COMMAND_HINT`
/// (desktop/CLI diagnose first; this is only the Rust SQL-executor backstop). /// (desktop/CLI diagnose first; this is only the Rust SQL-executor backstop).
@@ -1070,7 +1080,7 @@ pub fn agent_close_query_session_params(session_id: &str) -> serde_json::Value {
pub fn is_connection_error(err: &str) -> bool { pub fn is_connection_error(err: &str) -> bool {
let lower = err.to_lowercase(); let lower = err.to_lowercase();
if is_dbx_query_timeout_error(&lower) || is_agent_rpc_timeout_error(&lower) { if is_dbx_query_timeout_error(&lower) || is_agent_rpc_timeout_error(&lower) || is_pool_saturation_error(err) {
return false; return false;
} }
lower.contains("connection") lower.contains("connection")
@@ -3849,7 +3859,7 @@ pub async fn execute_statements_in_transaction_on_pool_typed(
/// Owned pool variants for safe dispatch across async boundaries. /// Owned pool variants for safe dispatch across async boundaries.
enum TxPath { enum TxPath {
Pg(deadpool_postgres::Pool), Pg(deadpool_postgres::Pool),
Mysql(mysql_async::Pool, bool), Mysql(db::mysql::MySqlPool, bool),
Sqlite(db::sqlite::SqliteHandle), Sqlite(db::sqlite::SqliteHandle),
CloudflareD1(db::cloudflare_d1_driver::CloudflareD1Client), CloudflareD1(db::cloudflare_d1_driver::CloudflareD1Client),
Agent(Arc<crate::db::agent_driver::PooledAgentClient>), Agent(Arc<crate::db::agent_driver::PooledAgentClient>),
@@ -3962,7 +3972,7 @@ async fn exec_tx_pg_statements(
async fn exec_tx_mysql_inner( async fn exec_tx_mysql_inner(
state: &AppState, state: &AppState,
pool_key: &str, pool_key: &str,
pool: mysql_async::Pool, pool: db::mysql::MySqlPool,
statements: &[String], statements: &[String],
start: std::time::Instant, start: std::time::Instant,
budget: DbOperationBudget, budget: DbOperationBudget,
@@ -6905,17 +6915,43 @@ for line in sys.stdin:
} }
#[test] #[test]
fn is_connection_error_detects_deadpool_pool_timeouts() { fn is_connection_error_distinguishes_checkout_saturation_from_connection_timeouts() {
// deadpool-postgres PoolError::Timeout messages (contain "pool" + "timeout" but not "timed out") // deadpool-postgres PoolError::Timeout messages (contain "pool" + "timeout" but not "timed out")
assert!(is_connection_error("pool wait timeout")); assert!(is_connection_error("pool wait timeout"));
assert!(is_connection_error("pool create timeout")); assert!(is_connection_error("pool create timeout"));
assert!(is_connection_error("pool recycle timeout")); assert!(is_connection_error("pool recycle timeout"));
// checkout helper timeout messages // checkout helper timeout messages
assert!(is_connection_error("PostgreSQL connection pool checkout timed out (5s)")); assert!(!is_connection_error("PostgreSQL connection pool checkout timed out [stage=wait, timeout_ms=5000]"));
assert!(is_connection_error("MySQL get connection timed out")); assert!(is_connection_error("PostgreSQL connection pool checkout timed out [stage=create, timeout_ms=5000]"));
assert!(is_connection_error("PostgreSQL connection pool checkout timed out [stage=recycle, timeout_ms=5000]"));
assert!(!is_connection_error("MySQL connection pool checkout timed out [stage=wait, timeout_ms=5000]"));
assert!(!is_connection_error(METADATA_POOL_BUSY_ERROR));
assert!(is_connection_error("MySQL ping timed out")); assert!(is_connection_error("MySQL ping timed out"));
assert!(is_connection_error("MySQL kill connection checkout timed out")); assert!(is_connection_error("MySQL kill connection checkout timed out"));
assert!(is_connection_error("MySQL KILL QUERY timed out")); assert!(is_connection_error("MySQL KILL QUERY timed out"));
assert!(is_pool_saturation_error(
"PostgreSQL connection pool checkout timed out [stage=wait, timeout_ms=5000]"
));
assert!(is_pool_saturation_error("MySQL connection pool checkout timed out [stage=wait, timeout_ms=5000]"));
assert!(!is_pool_saturation_error(
"PostgreSQL connection pool checkout timed out [stage=create, timeout_ms=5000]"
));
assert!(is_pool_saturation_error(METADATA_POOL_BUSY_ERROR));
assert_eq!(
pool_error_action(
Some(DatabaseType::Mysql),
"MySQL connection pool checkout timed out [stage=wait, timeout_ms=5000]"
),
PoolErrorAction::Keep
);
assert_eq!(
pool_error_action(
Some(DatabaseType::Postgres),
"PostgreSQL connection pool checkout timed out [stage=wait, timeout_ms=5000]"
),
PoolErrorAction::Keep
);
} }
#[test] #[test]
+315 -131
View File
@@ -1,7 +1,7 @@
use crate::agent_recovery::{RecoveryDecision, RecoveryPolicy, RecoveryScope}; use crate::agent_recovery::{RecoveryDecision, RecoveryPolicy, RecoveryScope};
use crate::connection::{ use crate::connection::{
connection_url_for_endpoint, database_connection_config, gaussdb_uses_m_jdbc_driver, task_client_session_id, connection_url_for_endpoint, database_connection_config, gaussdb_uses_m_jdbc_driver, task_client_session_id,
AppState, MysqlMode, PoolKind, uses_metadata_gate, AppState, MysqlMode, PoolKind, METADATA_POOL_ACQUIRE_TIMEOUT,
}; };
use crate::db; use crate::db;
use crate::models::connection::{ConnectionConfig, DatabaseType}; use crate::models::connection::{ConnectionConfig, DatabaseType};
@@ -33,6 +33,10 @@ macro_rules! dispatch_mysql {
}; };
} }
async fn clone_metadata_pool(state: &AppState, pool_key: &str) -> Option<PoolKind> {
state.connections.read().await.get(pool_key).and_then(PoolKind::clone_for_metadata)
}
struct EphemeralAgentMetadataSession { struct EphemeralAgentMetadataSession {
client_session_id: Option<String>, client_session_id: Option<String>,
cleanup_guard: Option<crate::connection::ClientSessionPoolCleanupGuard>, cleanup_guard: Option<crate::connection::ClientSessionPoolCleanupGuard>,
@@ -68,12 +72,25 @@ macro_rules! try_sqlserver {
($connections:expr, $pool_key:expr, $method:ident $(, $arg:expr)*) => { ($connections:expr, $pool_key:expr, $method:ident $(, $arg:expr)*) => {
if let Some(client) = extract_pool!(&$connections, $pool_key, SqlServer) { if let Some(client) = extract_pool!(&$connections, $pool_key, SqlServer) {
drop($connections); drop($connections);
let mut client = client.lock().await; let mut client = lock_sqlserver_metadata_client(&client).await?;
return db::sqlserver::$method(&mut client $(, $arg)*).await; return db::sqlserver::$method(&mut client $(, $arg)*).await;
} }
}; };
} }
async fn lock_sqlserver_metadata_client<'a>(
client: &'a Arc<tokio::sync::Mutex<db::sqlserver::SqlServerClient>>,
) -> Result<tokio::sync::MutexGuard<'a, db::sqlserver::SqlServerClient>, String> {
lock_metadata_mutex_with_timeout(client, METADATA_POOL_ACQUIRE_TIMEOUT).await
}
async fn lock_metadata_mutex_with_timeout<T>(
client: &Arc<tokio::sync::Mutex<T>>,
timeout: Duration,
) -> Result<tokio::sync::MutexGuard<'_, T>, String> {
tokio::time::timeout(timeout, client.lock()).await.map_err(|_| crate::query::METADATA_POOL_BUSY_ERROR.to_string())
}
const ORACLE_TABLE_COMMENT_BATCH_SIZE: usize = 500; const ORACLE_TABLE_COMMENT_BATCH_SIZE: usize = 500;
const TDENGINE_COMMENT_SEARCH_TIMEOUT: Duration = Duration::from_secs(5); const TDENGINE_COMMENT_SEARCH_TIMEOUT: Duration = Duration::from_secs(5);
const TDENGINE_COMMENT_SEARCH_CACHE_TTL: Duration = Duration::from_secs(10); const TDENGINE_COMMENT_SEARCH_CACHE_TTL: Duration = Duration::from_secs(10);
@@ -194,6 +211,17 @@ pub async fn list_database_storage_core(
state: &AppState, state: &AppState,
connection_id: &str, connection_id: &str,
database_names: &[String], database_names: &[String],
) -> Result<Vec<db::DatabaseStorageInfo>, String> {
retry_metadata_connection(state, connection_id, None, || {
list_database_storage_once(state, connection_id, database_names)
})
.await
}
async fn list_database_storage_once(
state: &AppState,
connection_id: &str,
database_names: &[String],
) -> Result<Vec<db::DatabaseStorageInfo>, String> { ) -> Result<Vec<db::DatabaseStorageInfo>, String> {
const DATABASE_STORAGE_TIMEOUT: Duration = Duration::from_secs(5); const DATABASE_STORAGE_TIMEOUT: Duration = Duration::from_secs(5);
const MAX_DATABASE_STORAGE_NAMES: usize = 2048; const MAX_DATABASE_STORAGE_NAMES: usize = 2048;
@@ -244,10 +272,11 @@ pub async fn list_sqlserver_linked_servers_core(
state: &AppState, state: &AppState,
connection_id: &str, connection_id: &str,
) -> Result<Vec<db::LinkedServerInfo>, String> { ) -> Result<Vec<db::LinkedServerInfo>, String> {
let _metadata_permit = state.acquire_metadata_permit(connection_id, None, DatabaseType::SqlServer, None).await?;
let connections = state.connections.read().await; let connections = state.connections.read().await;
if let Some(client) = extract_pool!(&connections, connection_id, SqlServer) { if let Some(client) = extract_pool!(&connections, connection_id, SqlServer) {
drop(connections); drop(connections);
let mut client = client.lock().await; let mut client = lock_sqlserver_metadata_client(&client).await?;
return db::sqlserver::list_linked_servers(&mut client).await; return db::sqlserver::list_linked_servers(&mut client).await;
} }
Ok(vec![]) Ok(vec![])
@@ -307,10 +336,11 @@ pub async fn list_sqlserver_linked_server_catalogs_core(
connection_id: &str, connection_id: &str,
server: &str, server: &str,
) -> Result<Vec<db::DatabaseInfo>, String> { ) -> Result<Vec<db::DatabaseInfo>, String> {
let _metadata_permit = state.acquire_metadata_permit(connection_id, None, DatabaseType::SqlServer, None).await?;
let connections = state.connections.read().await; let connections = state.connections.read().await;
if let Some(client) = extract_pool!(&connections, connection_id, SqlServer) { if let Some(client) = extract_pool!(&connections, connection_id, SqlServer) {
drop(connections); drop(connections);
let mut client = client.lock().await; let mut client = lock_sqlserver_metadata_client(&client).await?;
return db::sqlserver::list_linked_server_catalogs(&mut client, server).await; return db::sqlserver::list_linked_server_catalogs(&mut client, server).await;
} }
Ok(vec![]) Ok(vec![])
@@ -322,10 +352,11 @@ pub async fn list_sqlserver_linked_server_schemas_core(
server: &str, server: &str,
catalog: &str, catalog: &str,
) -> Result<Vec<String>, String> { ) -> Result<Vec<String>, String> {
let _metadata_permit = state.acquire_metadata_permit(connection_id, None, DatabaseType::SqlServer, None).await?;
let connections = state.connections.read().await; let connections = state.connections.read().await;
if let Some(client) = extract_pool!(&connections, connection_id, SqlServer) { if let Some(client) = extract_pool!(&connections, connection_id, SqlServer) {
drop(connections); drop(connections);
let mut client = client.lock().await; let mut client = lock_sqlserver_metadata_client(&client).await?;
return db::sqlserver::list_linked_server_schemas(&mut client, server, catalog).await; return db::sqlserver::list_linked_server_schemas(&mut client, server, catalog).await;
} }
Ok(vec![]) Ok(vec![])
@@ -341,10 +372,11 @@ pub async fn list_sqlserver_linked_server_tables_core(
limit: Option<usize>, limit: Option<usize>,
offset: Option<usize>, offset: Option<usize>,
) -> Result<Vec<db::TableInfo>, String> { ) -> Result<Vec<db::TableInfo>, String> {
let _metadata_permit = state.acquire_metadata_permit(connection_id, None, DatabaseType::SqlServer, None).await?;
let connections = state.connections.read().await; let connections = state.connections.read().await;
if let Some(client) = extract_pool!(&connections, connection_id, SqlServer) { if let Some(client) = extract_pool!(&connections, connection_id, SqlServer) {
drop(connections); drop(connections);
let mut client = client.lock().await; let mut client = lock_sqlserver_metadata_client(&client).await?;
return db::sqlserver::list_linked_server_tables(&mut client, server, catalog, schema, filter, limit, offset) return db::sqlserver::list_linked_server_tables(&mut client, server, catalog, schema, filter, limit, offset)
.await; .await;
} }
@@ -367,14 +399,17 @@ pub async fn list_sqlserver_linked_server_tables_core(
/// use the MySQL protocol, so this is a defensive no-op); the caller's /// use the MySQL protocol, so this is a defensive no-op); the caller's
/// flat-sidebar fallback then renders the standard database list. /// flat-sidebar fallback then renders the standard database list.
pub async fn list_doris_catalogs_core(state: &AppState, connection_id: &str) -> Result<Vec<db::CatalogInfo>, String> { pub async fn list_doris_catalogs_core(state: &AppState, connection_id: &str) -> Result<Vec<db::CatalogInfo>, String> {
retry_metadata_connection(state, connection_id, None, || list_doris_catalogs_once(state, connection_id)).await
}
async fn list_doris_catalogs_once(state: &AppState, connection_id: &str) -> Result<Vec<db::CatalogInfo>, String> {
let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, None, None).await?; let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, None, None).await?;
let db_config = connection_config(state, connection_id).await; let db_config = connection_config(state, connection_id).await;
let connections = state.connections.read().await; if let Some(PoolKind::Mysql(p, _)) = clone_metadata_pool(state, &pool_key).await {
if let Some(PoolKind::Mysql(p, _)) = connections.get(&pool_key) {
return if db_config.as_ref().is_some_and(db::starrocks::is_config) { return if db_config.as_ref().is_some_and(db::starrocks::is_config) {
db::starrocks::list_catalogs(p).await db::starrocks::list_catalogs(&p).await
} else { } else {
db::doris::list_catalogs(p).await db::doris::list_catalogs(&p).await
}; };
} }
Ok(vec![]) Ok(vec![])
@@ -389,12 +424,22 @@ pub async fn list_doris_catalog_databases_core(
state: &AppState, state: &AppState,
connection_id: &str, connection_id: &str,
catalog: &str, catalog: &str,
) -> Result<Vec<db::DatabaseInfo>, String> {
retry_metadata_connection(state, connection_id, None, || {
list_doris_catalog_databases_once(state, connection_id, catalog)
})
.await
}
async fn list_doris_catalog_databases_once(
state: &AppState,
connection_id: &str,
catalog: &str,
) -> Result<Vec<db::DatabaseInfo>, String> { ) -> Result<Vec<db::DatabaseInfo>, String> {
let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, None, None).await?; let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, None, None).await?;
let db_config = connection_config(state, connection_id).await; let db_config = connection_config(state, connection_id).await;
let connections = state.connections.read().await; let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
let pool = connections.get(&pool_key).ok_or("Pool not found")?; let PoolKind::Mysql(p, _) = &pool else {
let PoolKind::Mysql(p, _) = pool else {
return Ok(vec![]); return Ok(vec![]);
}; };
let databases = if db_config.as_ref().is_some_and(db::starrocks::is_config) { let databases = if db_config.as_ref().is_some_and(db::starrocks::is_config) {
@@ -433,12 +478,38 @@ pub async fn list_doris_catalog_tables_core(
offset: Option<usize>, offset: Option<usize>,
object_types: Option<&[String]>, object_types: Option<&[String]>,
table_name_filter: Option<&TableNameFilter>, table_name_filter: Option<&TableNameFilter>,
) -> Result<Vec<db::TableInfo>, String> {
retry_metadata_connection(state, connection_id, Some(database), || {
list_doris_catalog_tables_once(
state,
connection_id,
catalog,
database,
filter,
limit,
offset,
object_types,
table_name_filter,
)
})
.await
}
async fn list_doris_catalog_tables_once(
state: &AppState,
connection_id: &str,
catalog: &str,
database: &str,
filter: Option<&str>,
limit: Option<usize>,
offset: Option<usize>,
object_types: Option<&[String]>,
table_name_filter: Option<&TableNameFilter>,
) -> Result<Vec<db::TableInfo>, String> { ) -> Result<Vec<db::TableInfo>, String> {
let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, None, None).await?; let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, None, None).await?;
let db_config = connection_config(state, connection_id).await; let db_config = connection_config(state, connection_id).await;
let connections = state.connections.read().await; let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
let pool = connections.get(&pool_key).ok_or("Pool not found")?; let PoolKind::Mysql(p, _) = &pool else {
let PoolKind::Mysql(p, _) = pool else {
return Ok(vec![]); return Ok(vec![]);
}; };
let tables = if db_config.as_ref().is_some_and(db::starrocks::is_config) { let tables = if db_config.as_ref().is_some_and(db::starrocks::is_config) {
@@ -456,12 +527,24 @@ pub async fn get_doris_catalog_columns_core(
catalog: &str, catalog: &str,
database: &str, database: &str,
table: &str, table: &str,
) -> Result<Vec<db::ColumnInfo>, String> {
retry_metadata_connection(state, connection_id, Some(database), || {
get_doris_catalog_columns_once(state, connection_id, catalog, database, table)
})
.await
}
async fn get_doris_catalog_columns_once(
state: &AppState,
connection_id: &str,
catalog: &str,
database: &str,
table: &str,
) -> Result<Vec<db::ColumnInfo>, String> { ) -> Result<Vec<db::ColumnInfo>, String> {
let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, None, None).await?; let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, None, None).await?;
let db_config = connection_config(state, connection_id).await; let db_config = connection_config(state, connection_id).await;
let connections = state.connections.read().await; let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
let pool = connections.get(&pool_key).ok_or("Pool not found")?; let PoolKind::Mysql(p, _) = &pool else {
let PoolKind::Mysql(p, _) = pool else {
return Ok(vec![]); return Ok(vec![]);
}; };
let columns = if db_config.as_ref().is_some_and(db::starrocks::is_config) { let columns = if db_config.as_ref().is_some_and(db::starrocks::is_config) {
@@ -479,12 +562,24 @@ pub async fn get_doris_catalog_table_ddl_core(
catalog: &str, catalog: &str,
database: &str, database: &str,
table: &str, table: &str,
) -> Result<String, String> {
retry_metadata_connection(state, connection_id, Some(database), || {
get_doris_catalog_table_ddl_once(state, connection_id, catalog, database, table)
})
.await
}
async fn get_doris_catalog_table_ddl_once(
state: &AppState,
connection_id: &str,
catalog: &str,
database: &str,
table: &str,
) -> Result<String, String> { ) -> Result<String, String> {
let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, None, None).await?; let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, None, None).await?;
let db_config = connection_config(state, connection_id).await; let db_config = connection_config(state, connection_id).await;
let connections = state.connections.read().await; let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
let pool = connections.get(&pool_key).ok_or("Pool not found")?; let PoolKind::Mysql(p, _) = &pool else {
let PoolKind::Mysql(p, _) = pool else {
return Err("DDL not supported for this connection".to_string()); return Err("DDL not supported for this connection".to_string());
}; };
if db_config.as_ref().is_some_and(db::starrocks::is_config) { if db_config.as_ref().is_some_and(db::starrocks::is_config) {
@@ -501,12 +596,24 @@ pub async fn list_doris_catalog_indexes_core(
catalog: &str, catalog: &str,
database: &str, database: &str,
table: &str, table: &str,
) -> Result<Vec<db::IndexInfo>, String> {
retry_metadata_connection(state, connection_id, Some(database), || {
list_doris_catalog_indexes_once(state, connection_id, catalog, database, table)
})
.await
}
async fn list_doris_catalog_indexes_once(
state: &AppState,
connection_id: &str,
catalog: &str,
database: &str,
table: &str,
) -> Result<Vec<db::IndexInfo>, String> { ) -> Result<Vec<db::IndexInfo>, String> {
let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, None, None).await?; let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, None, None).await?;
let db_config = connection_config(state, connection_id).await; let db_config = connection_config(state, connection_id).await;
let connections = state.connections.read().await; let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
let pool = connections.get(&pool_key).ok_or("Pool not found")?; let PoolKind::Mysql(p, _) = &pool else {
let PoolKind::Mysql(p, _) = pool else {
return Ok(vec![]); return Ok(vec![]);
}; };
if db_config.as_ref().is_some_and(db::starrocks::is_config) { if db_config.as_ref().is_some_and(db::starrocks::is_config) {
@@ -603,8 +710,7 @@ async fn list_databases_once(state: &AppState, connection_id: &str) -> Result<Ve
} }
try_sqlserver!(connections, connection_id, list_databases); try_sqlserver!(connections, connection_id, list_databases);
if let Some(client) = extract_pool!(&connections, connection_id, Agent) { if let Some(client) = extract_pool!(&connections, connection_id, Agent) {
let is_mongo = let is_mongo = db_config.as_ref().is_some_and(|config| config.db_type == DatabaseType::MongoDb);
state.configs.read().await.get(connection_id).is_some_and(|c| c.db_type == DatabaseType::MongoDb);
if is_mongo { if is_mongo {
drop(connections); drop(connections);
let dbs = crate::mongo_ops::mongo_list_databases_core(state, connection_id).await?; let dbs = crate::mongo_ops::mongo_list_databases_core(state, connection_id).await?;
@@ -618,10 +724,9 @@ async fn list_databases_once(state: &AppState, connection_id: &str) -> Result<Ve
let db_config = connection_config(state, connection_id).await; let db_config = connection_config(state, connection_id).await;
let mysql_database_list_timeout = mysql_database_list_timeout(db_config.as_ref()); let mysql_database_list_timeout = mysql_database_list_timeout(db_config.as_ref());
let connections = state.connections.read().await; let pool = clone_metadata_pool(state, connection_id).await.ok_or("Connection not found")?;
let pool = connections.get(connection_id).ok_or("Connection not found")?;
match pool { match &pool {
PoolKind::Mysql(p, mode) PoolKind::Mysql(p, mode)
if *mode != MysqlMode::OceanBaseOracle && db_config.as_ref().is_some_and(db::dolt::is_config) => if *mode != MysqlMode::OceanBaseOracle && db_config.as_ref().is_some_and(db::dolt::is_config) =>
{ {
@@ -640,11 +745,7 @@ async fn list_databases_once(state: &AppState, connection_id: &str) -> Result<Ve
PoolKind::Turso(client) => db::turso_driver::list_databases(client).await, PoolKind::Turso(client) => db::turso_driver::list_databases(client).await,
PoolKind::HBase(client) => db::hbase_driver::list_namespaces(client).await, PoolKind::HBase(client) => db::hbase_driver::list_namespaces(client).await,
#[cfg(feature = "duckdb-sidecar")] #[cfg(feature = "duckdb-sidecar")]
PoolKind::DuckDbWorker(client) => { PoolKind::DuckDbWorker(client) => client.list_databases().await,
let client = client.clone();
drop(connections);
client.list_databases().await
}
PoolKind::CloudflareD1(client) => db::cloudflare_d1_driver::list_databases(client).await, PoolKind::CloudflareD1(client) => db::cloudflare_d1_driver::list_databases(client).await,
_ => Ok(vec![]), _ => Ok(vec![]),
} }
@@ -661,7 +762,7 @@ async fn list_database_metadata_once(state: &AppState, connection_id: &str) -> R
let connections = state.connections.read().await; let connections = state.connections.read().await;
if let Some(client) = extract_pool!(&connections, connection_id, SqlServer) { if let Some(client) = extract_pool!(&connections, connection_id, SqlServer) {
drop(connections); drop(connections);
let mut client = client.lock().await; let mut client = lock_sqlserver_metadata_client(&client).await?;
return db::sqlserver::list_database_metadata(&mut client).await; return db::sqlserver::list_database_metadata(&mut client).await;
} }
if let Some(PoolKind::Mysql(pool, mode)) = connections.get(connection_id) { if let Some(PoolKind::Mysql(pool, mode)) = connections.get(connection_id) {
@@ -717,11 +818,8 @@ async fn list_schema_infos_once(
let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, Some(database), None).await?; let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, Some(database), None).await?;
let db_config = connection_config(state, connection_id).await; let db_config = connection_config(state, connection_id).await;
let show_system_schemas = db_config.as_ref().is_some_and(|config| config.show_system_schemas); let show_system_schemas = db_config.as_ref().is_some_and(|config| config.show_system_schemas);
{ if let Some(PoolKind::Postgres(pool)) = clone_metadata_pool(state, &pool_key).await {
let connections = state.connections.read().await; return db::postgres::list_schema_infos_with_system(&pool, show_system_schemas).await;
if let Some(PoolKind::Postgres(pool)) = connections.get(&pool_key) {
return db::postgres::list_schema_infos_with_system(pool, show_system_schemas).await;
}
} }
let schemas = list_schemas_once(state, connection_id, database, false).await?; let schemas = list_schemas_once(state, connection_id, database, false).await?;
@@ -866,10 +964,9 @@ async fn list_schemas_once(
} }
} }
let connections = state.connections.read().await; let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
match pool { match &pool {
PoolKind::Mysql(p, mode) if *mode == MysqlMode::OceanBaseOracle => db::ob_oracle::list_schemas(p) PoolKind::Mysql(p, mode) if *mode == MysqlMode::OceanBaseOracle => db::ob_oracle::list_schemas(p)
.await .await
.map(|schemas| filter_visible_schema_names(schemas, visible_schema_filter.as_deref())), .map(|schemas| filter_visible_schema_names(schemas, visible_schema_filter.as_deref())),
@@ -878,9 +975,7 @@ async fn list_schemas_once(
.map(|schemas| filter_visible_schema_names(schemas, visible_schema_filter.as_deref())), .map(|schemas| filter_visible_schema_names(schemas, visible_schema_filter.as_deref())),
#[cfg(feature = "duckdb-sidecar")] #[cfg(feature = "duckdb-sidecar")]
PoolKind::DuckDbWorker(client) => { PoolKind::DuckDbWorker(client) => {
let client = client.clone();
let database = database.to_string(); let database = database.to_string();
drop(connections);
client client
.list_schemas(database) .list_schemas(database)
.await .await
@@ -1085,10 +1180,9 @@ async fn get_table_comment_core_for_session(
} }
} }
let connections = state.connections.read().await; let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
match pool { match &pool {
PoolKind::Mysql(p, mode) PoolKind::Mysql(p, mode)
if *mode != MysqlMode::OceanBaseOracle if *mode != MysqlMode::OceanBaseOracle
&& !db_config.as_ref().is_some_and(db::mysql_compatible::uses_show_metadata) && !db_config.as_ref().is_some_and(db::mysql_compatible::uses_show_metadata)
@@ -1955,7 +2049,7 @@ async fn list_tables_once(
if let Some(linked) = crate::sql_dialect::parse_sqlserver_linked_schema_ref(schema) { if let Some(linked) = crate::sql_dialect::parse_sqlserver_linked_schema_ref(schema) {
if let Some(client) = extract_pool!(&connections, &pool_key, SqlServer) { if let Some(client) = extract_pool!(&connections, &pool_key, SqlServer) {
drop(connections); drop(connections);
let mut client = client.lock().await; let mut client = lock_sqlserver_metadata_client(&client).await?;
return db::sqlserver::list_linked_server_tables( return db::sqlserver::list_linked_server_tables(
&mut client, &mut client,
&linked.server, &linked.server,
@@ -1972,7 +2066,7 @@ async fn list_tables_once(
if object_types.is_some() || table_name_filter.is_some_and(|filter| !filter.is_empty()) { if object_types.is_some() || table_name_filter.is_some_and(|filter| !filter.is_empty()) {
if let Some(client) = extract_pool!(&connections, &pool_key, SqlServer) { if let Some(client) = extract_pool!(&connections, &pool_key, SqlServer) {
drop(connections); drop(connections);
let mut client = client.lock().await; let mut client = lock_sqlserver_metadata_client(&client).await?;
return db::sqlserver::list_tables(&mut client, schema, filter, None, None) return db::sqlserver::list_tables(&mut client, schema, filter, None, None)
.await .await
.map(|tables| filter_table_infos(tables, filter, limit, offset, object_types, table_name_filter)); .map(|tables| filter_table_infos(tables, filter, limit, offset, object_types, table_name_filter));
@@ -2156,10 +2250,9 @@ async fn list_tables_once(
} }
} }
let connections = state.connections.read().await; let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
match pool { match &pool {
PoolKind::Mysql(p, _) if db_config.as_ref().is_some_and(db::starrocks::is_config) => { PoolKind::Mysql(p, _) if db_config.as_ref().is_some_and(db::starrocks::is_config) => {
db::starrocks::list_tables(p, database) db::starrocks::list_tables(p, database)
.await .await
@@ -3282,8 +3375,9 @@ mod tests {
} }
#[test] #[test]
fn metadata_retry_recovers_missing_pool_only_as_transient_state() { fn metadata_retry_excludes_pool_saturation_from_reconnects() {
assert!(is_retryable_metadata_error("Pool not found")); assert!(!is_retryable_metadata_error("Pool not found"));
assert!(!is_retryable_metadata_error(crate::query::METADATA_POOL_BUSY_ERROR));
assert!(is_retryable_metadata_error("connection reset by peer")); assert!(is_retryable_metadata_error("connection reset by peer"));
assert!(is_retryable_metadata_error("Agent RPC error (-1): dm.jdbc.driver.DMException: 网络通信异常")); assert!(is_retryable_metadata_error("Agent RPC error (-1): dm.jdbc.driver.DMException: 网络通信异常"));
assert!(is_retryable_metadata_error( assert!(is_retryable_metadata_error(
@@ -3302,6 +3396,78 @@ mod tests {
assert!(!is_retryable_metadata_error("Access denied for user")); assert!(!is_retryable_metadata_error("Access denied for user"));
} }
#[tokio::test]
async fn metadata_pool_races_retry_once_and_saturation_returns_busy() {
let dir = std::env::temp_dir().join(format!("dbx-schema-metadata-pool-race-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
let storage = crate::storage::Storage::open(&dir.join("storage.db")).await.unwrap();
let state = crate::connection::AppState::new(storage);
state.configs.write().await.insert("conn".to_string(), test_connection_config(DatabaseType::Mysql));
let mut recovered_attempts = 0;
let recovered = super::retry_metadata_connection_for_session(&state, "conn", Some("app"), None, || {
recovered_attempts += 1;
let attempt = recovered_attempts;
async move {
if attempt == 1 {
Err("Pool not found".to_string())
} else {
Ok("loaded")
}
}
})
.await;
assert_eq!(recovered, Ok("loaded"));
assert_eq!(recovered_attempts, 2);
let mut missing_attempts = 0;
let missing = super::retry_metadata_connection_for_session(&state, "conn", Some("app"), None, || {
missing_attempts += 1;
async { Err::<(), _>("Pool not found".to_string()) }
})
.await;
assert_eq!(missing.err().as_deref(), Some(crate::query::METADATA_POOL_BUSY_ERROR));
assert_eq!(missing_attempts, 2);
let mut saturation_attempts = 0;
let saturated = super::retry_metadata_connection_for_session(&state, "conn", Some("app"), None, || {
saturation_attempts += 1;
async { Err::<(), _>("MySQL connection pool checkout timed out [stage=wait, timeout_ms=500]".to_string()) }
})
.await;
assert_eq!(saturated.err().as_deref(), Some(crate::query::METADATA_POOL_BUSY_ERROR));
assert_eq!(saturation_attempts, 1);
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test]
async fn sqlserver_metadata_mutex_contention_returns_busy_after_timeout() {
let client = std::sync::Arc::new(tokio::sync::Mutex::new(()));
let _held = client.lock().await;
let result = super::lock_metadata_mutex_with_timeout(&client, Duration::from_millis(1)).await;
assert_eq!(result.err().as_deref(), Some(crate::query::METADATA_POOL_BUSY_ERROR));
}
#[tokio::test]
async fn metadata_pool_snapshot_releases_global_connections_lock() {
let dir = std::env::temp_dir().join(format!("dbx-schema-metadata-snapshot-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
let storage = crate::storage::Storage::open(&dir.join("storage.db")).await.unwrap();
let state = crate::connection::AppState::new(storage);
let pool = crate::db::sqlite::connect_path(":memory:").await.unwrap();
state.connections.write().await.insert("conn".to_string(), super::PoolKind::Sqlite(pool));
let snapshot = super::clone_metadata_pool(&state, "conn").await.expect("metadata pool snapshot");
let write_guard = state.connections.try_write().expect("snapshot must not retain the global pool-map lock");
assert!(matches!(snapshot, super::PoolKind::Sqlite(_)));
drop(write_guard);
drop(snapshot);
drop(state);
let _ = std::fs::remove_dir_all(dir);
}
#[test] #[test]
fn metadata_error_action_applies_fail_stop_to_every_attempt() { fn metadata_error_action_applies_fail_stop_to_every_attempt() {
let quarantine = "Agent RPC error (-1): connection lost\nDBX_AGENT_ERROR_DATA:{\"category\":\"connection\",\"sessionDisposition\":\"quarantine\"}"; let quarantine = "Agent RPC error (-1): connection lost\nDBX_AGENT_ERROR_DATA:{\"category\":\"connection\",\"sessionDisposition\":\"quarantine\"}";
@@ -4793,8 +4959,8 @@ pub async fn completion_assistant_search_core(
{ {
let connections = state.connections.read().await; let connections = state.connections.read().await;
if let Some(client) = extract_pool!(&connections, &pool_key, Agent) { if let Some(client) = extract_pool!(&connections, &pool_key, Agent) {
let db_config = connection_config(state, &request.connection_id).await;
drop(connections); drop(connections);
let db_config = connection_config(state, &request.connection_id).await;
let mut client = client.lock().await; let mut client = client.lock().await;
match client match client
.completion_assistant_search::<db::CompletionAssistantResponse>( .completion_assistant_search::<db::CompletionAssistantResponse>(
@@ -5030,8 +5196,9 @@ async fn list_object_statistics_once(
drop(connections); drop(connections);
return db::victoriametrics_driver::list_object_statistics(&client).await; return db::victoriametrics_driver::list_object_statistics(&client).await;
} }
let pool = connections.get(&pool_key).ok_or("Pool not found")?; drop(connections);
match pool { let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
match &pool {
PoolKind::Mysql(p, mode) => { PoolKind::Mysql(p, mode) => {
if *mode == MysqlMode::OceanBaseOracle || db_config.as_ref().is_some_and(db::manticoresearch::is_config) { if *mode == MysqlMode::OceanBaseOracle || db_config.as_ref().is_some_and(db::manticoresearch::is_config) {
Ok(vec![]) Ok(vec![])
@@ -5111,7 +5278,7 @@ async fn list_objects_once(
} }
if let Some(client) = extract_pool!(&connections, &pool_key, SqlServer) { if let Some(client) = extract_pool!(&connections, &pool_key, SqlServer) {
drop(connections); drop(connections);
let mut client = client.lock().await; let mut client = lock_sqlserver_metadata_client(&client).await?;
return db::sqlserver::list_objects(&mut client, schema).await.map(unpaged_object_list); return db::sqlserver::list_objects(&mut client, schema).await.map(unpaged_object_list);
} }
if let Some(client) = extract_pool!(&connections, &pool_key, Agent) { if let Some(client) = extract_pool!(&connections, &pool_key, Agent) {
@@ -5225,10 +5392,9 @@ async fn list_objects_once(
} }
} }
let connections = state.connections.read().await; let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
match pool { match &pool {
PoolKind::Mysql(p, mode) => { PoolKind::Mysql(p, mode) => {
// Note: mysql and ob_oracle take different second args (database vs schema) // Note: mysql and ob_oracle take different second args (database vs schema)
if *mode == MysqlMode::OceanBaseOracle { if *mode == MysqlMode::OceanBaseOracle {
@@ -5260,31 +5426,28 @@ async fn list_objects_once(
.await .await
.map(unpaged_object_list) .map(unpaged_object_list)
} }
_ => { _ => Ok(unpaged_object_list(
drop(connections); list_tables_core(state, connection_id, database, schema, None, None, None, None, None)
Ok(unpaged_object_list( .await?
list_tables_core(state, connection_id, database, schema, None, None, None, None, None) .into_iter()
.await? .map(|table| db::ObjectInfo {
.into_iter() name: table.name,
.map(|table| db::ObjectInfo { object_type: table.table_type,
name: table.name, schema: if schema.is_empty() { None } else { Some(schema.to_string()) },
object_type: table.table_type, valid: None,
schema: if schema.is_empty() { None } else { Some(schema.to_string()) }, signature: None,
valid: None, custom_type_kind: None,
signature: None, has_members: None,
custom_type_kind: None, comment: table.comment,
has_members: None, created_at: None,
comment: table.comment, updated_at: None,
created_at: None, parent_schema: table.parent_schema,
updated_at: None, parent_name: table.parent_name,
parent_schema: table.parent_schema, trigger: None,
parent_name: table.parent_name, xugu_type_members_expandable: None,
trigger: None, })
xugu_type_members_expandable: None, .collect(),
}) )),
.collect(),
))
}
} }
} }
@@ -5373,8 +5536,9 @@ async fn list_completion_objects_once(
return Ok(filter_completion_objects(objects)); return Ok(filter_completion_objects(objects));
} }
let pool = connections.get(&pool_key).ok_or("Pool not found")?; drop(connections);
match pool { let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
match &pool {
PoolKind::Mysql(p, mode) if *mode != MysqlMode::OceanBaseOracle => { PoolKind::Mysql(p, mode) if *mode != MysqlMode::OceanBaseOracle => {
db::mysql::list_completion_objects(p, database).await db::mysql::list_completion_objects(p, database).await
} }
@@ -5391,7 +5555,6 @@ async fn list_completion_objects_once(
db::postgres::list_objects(p, schema, true, true, false).await.map(filter_completion_objects) db::postgres::list_objects(p, schema, true, true, false).await.map(filter_completion_objects)
} }
PoolKind::SqlServer(_) => { PoolKind::SqlServer(_) => {
drop(connections);
let outcome = let outcome =
list_objects_once(state, connection_id, database, schema, None, None, None, None, None).await?; list_objects_once(state, connection_id, database, schema, None, None, None, None, None).await?;
Ok(filter_completion_objects(outcome.objects)) Ok(filter_completion_objects(outcome.objects))
@@ -5464,9 +5627,42 @@ where
let configs = state.configs.read().await; let configs = state.configs.read().await;
configs.get(connection_id).map(|config| config.db_type) configs.get(connection_id).map(|config| config.db_type)
}; };
let _metadata_permit = match db_type.filter(|db_type| uses_metadata_gate(*db_type)) {
Some(db_type) => {
Some(state.acquire_metadata_permit(connection_id, database, db_type, client_session_id).await?)
}
None => None,
};
let mut retried = false; let mut retried = false;
let mut missing_pool_retry = false;
loop { loop {
let result = operation().await; let result = operation().await;
if result.as_ref().err().is_some_and(|error| error == "Pool not found") {
if !missing_pool_retry {
missing_pool_retry = true;
log::debug!(
"[metadata:pool:missing-retry] connection_id={} database={}",
connection_id,
database.unwrap_or_default()
);
continue;
}
log::warn!(
"[metadata:pool:missing] connection_id={} database={}",
connection_id,
database.unwrap_or_default()
);
return Err(crate::query::METADATA_POOL_BUSY_ERROR.to_string());
}
if result.as_ref().err().is_some_and(|error| crate::query::is_pool_saturation_error(error)) {
log::warn!(
"[metadata:pool:saturation] connection_id={} database={} error={}",
connection_id,
database.unwrap_or_default(),
result.as_ref().err().map(String::as_str).unwrap_or_default()
);
return Err(crate::query::METADATA_POOL_BUSY_ERROR.to_string());
}
let recovery = let recovery =
result.as_ref().err().map(|error| metadata_recovery(db_type, error, retried)).unwrap_or_default(); result.as_ref().err().map(|error| metadata_recovery(db_type, error, retried)).unwrap_or_default();
match recovery.action { match recovery.action {
@@ -5592,7 +5788,7 @@ fn is_retryable_metadata_error(error: &str) -> bool {
return RecoveryPolicy::decide(&error, RecoveryScope::ReadOnlyMetadata { retried: false }) return RecoveryPolicy::decide(&error, RecoveryScope::ReadOnlyMetadata { retried: false })
== RecoveryDecision::RetryReadOnlyMetadata; == RecoveryDecision::RetryReadOnlyMetadata;
} }
error == "Pool not found" || crate::query::is_connection_error(error) crate::query::is_connection_error(error)
} }
pub async fn get_columns_core( pub async fn get_columns_core(
@@ -5748,7 +5944,7 @@ async fn get_columns_core_for_session_inner(
if let Some(linked) = crate::sql_dialect::parse_sqlserver_linked_schema_ref(schema) { if let Some(linked) = crate::sql_dialect::parse_sqlserver_linked_schema_ref(schema) {
if let Some(client) = extract_pool!(&connections, &pool_key, SqlServer) { if let Some(client) = extract_pool!(&connections, &pool_key, SqlServer) {
drop(connections); drop(connections);
let mut client = client.lock().await; let mut client = lock_sqlserver_metadata_client(&client).await?;
return db::sqlserver::get_linked_server_columns( return db::sqlserver::get_linked_server_columns(
&mut client, &mut client,
&linked.server, &linked.server,
@@ -5872,10 +6068,9 @@ async fn get_columns_core_for_session_inner(
} }
} }
let connections = state.connections.read().await; let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
match pool { match &pool {
PoolKind::Mysql(p, _) if db_config.as_ref().is_some_and(db::manticoresearch::is_config) => { PoolKind::Mysql(p, _) if db_config.as_ref().is_some_and(db::manticoresearch::is_config) => {
let metadata_database = mysql_show_metadata_database_for_config(db_config.as_ref(), database); let metadata_database = mysql_show_metadata_database_for_config(db_config.as_ref(), database);
db::manticoresearch::get_columns(p, metadata_database, table).await.map(deduplicate_column_infos) db::manticoresearch::get_columns(p, metadata_database, table).await.map(deduplicate_column_infos)
@@ -6072,10 +6267,9 @@ async fn list_indexes_core_for_session(
} }
} }
let connections = state.connections.read().await; let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
match pool { match &pool {
PoolKind::Mysql(p, mode) => { PoolKind::Mysql(p, mode) => {
if db_config.as_ref().is_some_and(db::manticoresearch::is_config) { if db_config.as_ref().is_some_and(db::manticoresearch::is_config) {
return db::manticoresearch::list_indexes(p, table).await; return db::manticoresearch::list_indexes(p, table).await;
@@ -6160,10 +6354,9 @@ async fn list_foreign_keys_core_for_session(
} }
} }
let connections = state.connections.read().await; let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
match pool { match &pool {
PoolKind::Mysql(p, mode) => { PoolKind::Mysql(p, mode) => {
if *mode == MysqlMode::OceanBaseOracle { if *mode == MysqlMode::OceanBaseOracle {
db::ob_oracle::list_foreign_keys(p, schema, table).await db::ob_oracle::list_foreign_keys(p, schema, table).await
@@ -6206,10 +6399,9 @@ pub async fn list_triggers_core(
} }
} }
let connections = state.connections.read().await; let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
match pool { match &pool {
PoolKind::Mysql(p, mode) => { PoolKind::Mysql(p, mode) => {
if *mode == MysqlMode::OceanBaseOracle { if *mode == MysqlMode::OceanBaseOracle {
db::ob_oracle::list_triggers(p, schema, table).await db::ob_oracle::list_triggers(p, schema, table).await
@@ -6361,10 +6553,9 @@ pub async fn list_functions_core(
) -> Result<Vec<db::FunctionInfo>, String> { ) -> Result<Vec<db::FunctionInfo>, String> {
retry_metadata_connection(state, connection_id, Some(database), || async { retry_metadata_connection(state, connection_id, Some(database), || async {
let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, Some(database), None).await?; let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, Some(database), None).await?;
let connections = state.connections.read().await; let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
match pool { match &pool {
PoolKind::Postgres(p) => db::postgres::list_functions(p, schema).await, PoolKind::Postgres(p) => db::postgres::list_functions(p, schema).await,
_ => Ok(vec![]), _ => Ok(vec![]),
} }
@@ -6382,10 +6573,9 @@ pub async fn list_sequences_core(
retry_metadata_connection(state, connection_id, Some(database), || async { retry_metadata_connection(state, connection_id, Some(database), || async {
let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, Some(database), None).await?; let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, Some(database), None).await?;
let db_config = connection_config(state, connection_id).await; let db_config = connection_config(state, connection_id).await;
let connections = state.connections.read().await; let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
match pool { match &pool {
PoolKind::Postgres(p) if db_config.as_ref().is_some_and(is_opengauss_family_config) => { PoolKind::Postgres(p) if db_config.as_ref().is_some_and(is_opengauss_family_config) => {
db::postgres::list_opengauss_sequences(p, schema, with_last_values).await db::postgres::list_opengauss_sequences(p, schema, with_last_values).await
} }
@@ -6404,10 +6594,9 @@ pub async fn list_rules_core(
) -> Result<Vec<db::RuleInfo>, String> { ) -> Result<Vec<db::RuleInfo>, String> {
retry_metadata_connection(state, connection_id, Some(database), || async { retry_metadata_connection(state, connection_id, Some(database), || async {
let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, Some(database), None).await?; let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, Some(database), None).await?;
let connections = state.connections.read().await; let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
match pool { match &pool {
PoolKind::Postgres(p) => db::postgres::list_rules(p, schema).await, PoolKind::Postgres(p) => db::postgres::list_rules(p, schema).await,
_ => Ok(vec![]), _ => Ok(vec![]),
} }
@@ -6433,10 +6622,9 @@ pub async fn list_extensions_core(
} }
} }
let connections = state.connections.read().await; let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
match pool { match &pool {
PoolKind::Postgres(p) => db::postgres::list_extensions(p, schema).await, PoolKind::Postgres(p) => db::postgres::list_extensions(p, schema).await,
_ => Ok(vec![]), _ => Ok(vec![]),
} }
@@ -6465,10 +6653,9 @@ pub async fn list_available_extensions_core(
} }
} }
let connections = state.connections.read().await; let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
match pool { match &pool {
PoolKind::Postgres(p) => db::postgres::list_available_extensions(p).await, PoolKind::Postgres(p) => db::postgres::list_available_extensions(p).await,
_ => Ok(vec![]), _ => Ok(vec![]),
} }
@@ -6484,10 +6671,9 @@ pub async fn list_owners_core(
) -> Result<Vec<db::OwnerInfo>, String> { ) -> Result<Vec<db::OwnerInfo>, String> {
retry_metadata_connection(state, connection_id, Some(database), || async { retry_metadata_connection(state, connection_id, Some(database), || async {
let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, Some(database), None).await?; let pool_key = state.get_or_create_metadata_pool_for_session(connection_id, Some(database), None).await?;
let connections = state.connections.read().await; let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
match pool { match &pool {
PoolKind::Postgres(p) => db::postgres::list_owners(p, schema).await, PoolKind::Postgres(p) => db::postgres::list_owners(p, schema).await,
_ => Ok(vec![]), _ => Ok(vec![]),
} }
@@ -6709,7 +6895,7 @@ async fn get_table_ddl_once(
} }
if let Some(client) = extract_pool!(&connections, &pool_key, SqlServer) { if let Some(client) = extract_pool!(&connections, &pool_key, SqlServer) {
drop(connections); drop(connections);
let mut client = client.lock().await; let mut client = lock_sqlserver_metadata_client(&client).await?;
return build_sqlserver_ddl(&mut client, schema, table).await; return build_sqlserver_ddl(&mut client, schema, table).await;
} }
if let Some(client) = extract_pool!(&connections, &pool_key, Agent) { if let Some(client) = extract_pool!(&connections, &pool_key, Agent) {
@@ -6771,10 +6957,9 @@ async fn get_table_ddl_once(
} }
} }
let connections = state.connections.read().await; let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
let pool = connections.get(&pool_key).ok_or("Pool not found")?;
match pool { match &pool {
PoolKind::Mysql(p, _) => mysql_ddl(p, mysql_table_metadata_catalog(database, schema), table).await, PoolKind::Mysql(p, _) => mysql_ddl(p, mysql_table_metadata_catalog(database, schema), table).await,
PoolKind::Postgres(p) if db_config.as_ref().is_some_and(is_opengauss_family_config) => { PoolKind::Postgres(p) if db_config.as_ref().is_some_and(is_opengauss_family_config) => {
match opengauss_table_ddl(p, schema, table).await { match opengauss_table_ddl(p, schema, table).await {
@@ -7567,9 +7752,8 @@ async fn get_custom_type_details_once(
.await; .await;
} }
} }
let connections = state.connections.read().await; let pool = clone_metadata_pool(state, &pool_key).await.ok_or("Pool not found")?;
let pool = connections.get(&pool_key).ok_or("Pool not found")?; match &pool {
match pool {
PoolKind::Postgres(p) => db::postgres::get_custom_type_details(p, schema, name).await, PoolKind::Postgres(p) => db::postgres::get_custom_type_details(p, schema, name).await,
_ => Err("custom type details are not supported for this connection type".to_string()), _ => Err("custom type details are not supported for this connection type".to_string()),
} }
@@ -7662,7 +7846,7 @@ async fn get_object_source_once(
} }
if let Some(client) = extract_pool!(&connections, &pool_key, SqlServer) { if let Some(client) = extract_pool!(&connections, &pool_key, SqlServer) {
drop(connections); drop(connections);
let mut client = client.lock().await; let mut client = lock_sqlserver_metadata_client(&client).await?;
let result = let result =
db::sqlserver::execute_query(&mut client, &sqlserver_object_source_sql(schema, name, &object_type)) db::sqlserver::execute_query(&mut client, &sqlserver_object_source_sql(schema, name, &object_type))
.await; .await;
+439 -14
View File
@@ -192,6 +192,8 @@ pub struct DesktopSettings {
pub close_action_prompted: bool, pub close_action_prompted: bool,
#[serde(default)] #[serde(default)]
pub debug_logging_enabled: bool, pub debug_logging_enabled: bool,
#[serde(default = "default_metadata_cache_max_memory_mb")]
pub metadata_cache_max_memory_mb: usize,
#[serde(default)] #[serde(default)]
pub duckdb_worker_process_isolation: bool, pub duckdb_worker_process_isolation: bool,
#[serde(default = "default_duckdb_worker_max_processes")] #[serde(default = "default_duckdb_worker_max_processes")]
@@ -244,6 +246,31 @@ pub const DUCKDB_WORKER_MAX_PROCESSES_MIN: usize = 1;
pub const DUCKDB_WORKER_MAX_PROCESSES_MAX: usize = 16; pub const DUCKDB_WORKER_MAX_PROCESSES_MAX: usize = 16;
pub const DUCKDB_WORKER_MAX_PROCESSES_DEFAULT: usize = 4; pub const DUCKDB_WORKER_MAX_PROCESSES_DEFAULT: usize = 4;
pub const METADATA_CACHE_MEMORY_MIN_MB: usize = 16;
pub const METADATA_CACHE_MEMORY_RECOMMENDED_MAX_MB: usize = 256;
pub const METADATA_CACHE_MEMORY_HARD_MAX_MB: usize = 512;
pub const METADATA_CACHE_MEMORY_DEFAULT_MB: usize = 64;
pub fn default_metadata_cache_max_memory_mb() -> usize {
METADATA_CACHE_MEMORY_DEFAULT_MB
}
pub fn normalize_metadata_cache_max_memory_mb(value: usize) -> usize {
if value > METADATA_CACHE_MEMORY_HARD_MAX_MB {
log::warn!(
"Metadata cache memory limit {value} MB exceeds the hard limit; falling back to {METADATA_CACHE_MEMORY_DEFAULT_MB} MB"
);
METADATA_CACHE_MEMORY_DEFAULT_MB
} else {
if value > METADATA_CACHE_MEMORY_RECOMMENDED_MAX_MB {
log::warn!(
"Metadata cache memory limit {value} MB exceeds the recommended {METADATA_CACHE_MEMORY_RECOMMENDED_MAX_MB} MB"
);
}
value.clamp(METADATA_CACHE_MEMORY_MIN_MB, METADATA_CACHE_MEMORY_HARD_MAX_MB)
}
}
pub fn default_duckdb_worker_max_processes() -> usize { pub fn default_duckdb_worker_max_processes() -> usize {
DUCKDB_WORKER_MAX_PROCESSES_DEFAULT DUCKDB_WORKER_MAX_PROCESSES_DEFAULT
} }
@@ -260,6 +287,7 @@ impl Default for DesktopSettings {
quit_on_close: false, quit_on_close: false,
close_action_prompted: false, close_action_prompted: false,
debug_logging_enabled: false, debug_logging_enabled: false,
metadata_cache_max_memory_mb: default_metadata_cache_max_memory_mb(),
duckdb_worker_process_isolation: false, duckdb_worker_process_isolation: false,
duckdb_worker_max_processes: default_duckdb_worker_max_processes(), duckdb_worker_max_processes: default_duckdb_worker_max_processes(),
saved_sql_sync_dir: None, saved_sql_sync_dir: None,
@@ -351,7 +379,11 @@ const SCHEMA_STATEMENTS: &[&str] = &[
"CREATE TABLE IF NOT EXISTS schema_cache ( "CREATE TABLE IF NOT EXISTS schema_cache (
cache_key TEXT PRIMARY KEY, cache_key TEXT PRIMARY KEY,
payload_json TEXT NOT NULL, payload_json TEXT NOT NULL,
updated_at TEXT NOT NULL updated_at TEXT NOT NULL,
updated_at_ms INTEGER NOT NULL DEFAULT 0,
last_accessed_at_ms INTEGER NOT NULL DEFAULT 0,
byte_size INTEGER NOT NULL DEFAULT 0,
owner_id TEXT NOT NULL DEFAULT ''
)", )",
"CREATE TABLE IF NOT EXISTS tab_runtime_cache ( "CREATE TABLE IF NOT EXISTS tab_runtime_cache (
cache_key TEXT PRIMARY KEY, cache_key TEXT PRIMARY KEY,
@@ -452,6 +484,7 @@ impl Storage {
ensure_history_columns_sync(conn)?; ensure_history_columns_sync(conn)?;
ensure_saved_sql_columns_sync(conn)?; ensure_saved_sql_columns_sync(conn)?;
ensure_tab_runtime_cache_columns_sync(conn)?; ensure_tab_runtime_cache_columns_sync(conn)?;
ensure_schema_cache_columns_sync(conn)?;
ensure_ai_configs_columns_sync(conn)?; ensure_ai_configs_columns_sync(conn)?;
ensure_state_store_columns_sync(conn)?; ensure_state_store_columns_sync(conn)?;
Ok(()) Ok(())
@@ -485,6 +518,52 @@ fn inspect_sqlite_db_file(path: &Path) -> Result<SqliteDbFileState, String> {
} }
} }
fn ensure_schema_cache_columns_sync(conn: &Connection) -> Result<(), String> {
let mut columns = HashSet::new();
let mut statement = conn.prepare("PRAGMA table_info(schema_cache)").map_err(|error| error.to_string())?;
let rows = statement.query_map([], |row| row.get::<_, String>(1)).map_err(|error| error.to_string())?;
for row in rows {
columns.insert(row.map_err(|error| error.to_string())?);
}
for (name, definition) in [
("updated_at_ms", "INTEGER NOT NULL DEFAULT 0"),
("last_accessed_at_ms", "INTEGER NOT NULL DEFAULT 0"),
("byte_size", "INTEGER NOT NULL DEFAULT 0"),
("owner_id", "TEXT NOT NULL DEFAULT ''"),
] {
if !columns.contains(name) {
conn.execute(&format!("ALTER TABLE schema_cache ADD COLUMN {name} {definition}"), [])
.map_err(|error| error.to_string())?;
}
}
conn.execute(
"UPDATE schema_cache
SET byte_size = length(payload_json)
WHERE byte_size = 0 AND payload_json IS NOT NULL",
[],
)
.map_err(|error| error.to_string())?;
conn.execute(
"UPDATE schema_cache
SET updated_at_ms = COALESCE(CAST(strftime('%s', updated_at) AS INTEGER) * 1000, 0),
last_accessed_at_ms = COALESCE(CAST(strftime('%s', updated_at) AS INTEGER) * 1000, 0)
WHERE updated_at_ms = 0",
[],
)
.map_err(|error| error.to_string())?;
conn.execute("CREATE INDEX IF NOT EXISTS idx_schema_cache_updated_at_ms ON schema_cache (updated_at_ms)", [])
.map_err(|error| error.to_string())?;
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_schema_cache_owner_lru
ON schema_cache (owner_id, last_accessed_at_ms, updated_at_ms, cache_key)",
[],
)
.map_err(|error| error.to_string())?;
Ok(())
}
fn open_read_only_sqlite(path: &Path) -> Result<Connection, String> { fn open_read_only_sqlite(path: &Path) -> Result<Connection, String> {
Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY) Connection::open_with_flags(path, OpenFlags::SQLITE_OPEN_READ_ONLY)
.map_err(|e| format!("Failed to open db read-only: {e}")) .map_err(|e| format!("Failed to open db read-only: {e}"))
@@ -1538,6 +1617,12 @@ impl Storage {
"debug_logging_enabled".to_string(), "debug_logging_enabled".to_string(),
serde_json::Value::Bool(desktop_settings.debug_logging_enabled), serde_json::Value::Bool(desktop_settings.debug_logging_enabled),
); );
settings.insert(
"metadata_cache_max_memory_mb".to_string(),
serde_json::Value::Number(serde_json::Number::from(normalize_metadata_cache_max_memory_mb(
desktop_settings.metadata_cache_max_memory_mb,
))),
);
settings.insert( settings.insert(
"duckdb_worker_process_isolation".to_string(), "duckdb_worker_process_isolation".to_string(),
serde_json::Value::Bool(desktop_settings.duckdb_worker_process_isolation), serde_json::Value::Bool(desktop_settings.duckdb_worker_process_isolation),
@@ -1608,6 +1693,12 @@ impl Storage {
.get("debug_logging_enabled") .get("debug_logging_enabled")
.and_then(|value| value.as_bool()) .and_then(|value| value.as_bool())
.unwrap_or_else(|| DesktopSettings::default().debug_logging_enabled), .unwrap_or_else(|| DesktopSettings::default().debug_logging_enabled),
metadata_cache_max_memory_mb: settings
.get("metadata_cache_max_memory_mb")
.and_then(|value| value.as_u64())
.and_then(|value| usize::try_from(value).ok())
.map(normalize_metadata_cache_max_memory_mb)
.unwrap_or_else(|| DesktopSettings::default().metadata_cache_max_memory_mb),
duckdb_worker_process_isolation: settings duckdb_worker_process_isolation: settings
.get("duckdb_worker_process_isolation") .get("duckdb_worker_process_isolation")
.and_then(|value| value.as_bool()) .and_then(|value| value.as_bool())
@@ -3342,31 +3433,167 @@ impl Storage {
// Schema cache // Schema cache
const SCHEMA_CACHE_MAX_TOTAL_BYTES: i64 = 256 * 1024 * 1024;
const SCHEMA_CACHE_MAX_CONNECTION_BYTES: i64 = 64 * 1024 * 1024;
const SCHEMA_CACHE_MAX_ENTRIES: usize = 50_000;
const SCHEMA_CACHE_MAX_AGE_MILLIS: i64 = 24 * 60 * 60 * 1000;
#[derive(Debug, Clone, Copy)]
struct SchemaCachePolicy {
max_total_bytes: i64,
max_connection_bytes: i64,
max_entries: usize,
max_age_millis: i64,
}
impl Default for SchemaCachePolicy {
fn default() -> Self {
Self {
max_total_bytes: SCHEMA_CACHE_MAX_TOTAL_BYTES,
max_connection_bytes: SCHEMA_CACHE_MAX_CONNECTION_BYTES,
max_entries: SCHEMA_CACHE_MAX_ENTRIES,
max_age_millis: SCHEMA_CACHE_MAX_AGE_MILLIS,
}
}
}
fn schema_cache_owner(cache_key: &str) -> &str {
let mut parts = cache_key.split(':');
match (parts.next(), parts.next(), parts.next()) {
(Some("object-ddl" | "object-meta"), Some("v1"), Some(owner)) => owner,
_ => "",
}
}
fn delete_oldest_schema_cache_entry(conn: &Connection, owner_id: Option<&str>) -> Result<bool, String> {
let deleted = match owner_id {
Some(owner_id) => conn.execute(
"DELETE FROM schema_cache
WHERE cache_key = (
SELECT cache_key FROM schema_cache WHERE owner_id = ?1
ORDER BY last_accessed_at_ms ASC, updated_at_ms ASC, cache_key ASC LIMIT 1
)",
[owner_id],
),
None => conn.execute(
"DELETE FROM schema_cache
WHERE cache_key = (
SELECT cache_key FROM schema_cache
ORDER BY last_accessed_at_ms ASC, updated_at_ms ASC, cache_key ASC LIMIT 1
)",
[],
),
}
.map_err(|error| error.to_string())?;
Ok(deleted > 0)
}
fn prune_schema_cache(conn: &Connection, policy: SchemaCachePolicy, now_ms: i64) -> Result<(), String> {
let expires_before = now_ms.saturating_sub(policy.max_age_millis.max(0));
conn.execute("DELETE FROM schema_cache WHERE updated_at_ms = 0 OR updated_at_ms <= ?1", [expires_before])
.map_err(|error| error.to_string())?;
loop {
let over_budget_owner: Option<String> = conn
.query_row(
"SELECT owner_id FROM schema_cache
GROUP BY owner_id
HAVING SUM(byte_size) > ?1
ORDER BY SUM(byte_size) DESC, owner_id ASC
LIMIT 1",
[policy.max_connection_bytes.max(0)],
|row| row.get(0),
)
.optional()
.map_err(|error| error.to_string())?;
let Some(owner_id) = over_budget_owner else {
break;
};
if !delete_oldest_schema_cache_entry(conn, Some(&owner_id))? {
break;
}
}
loop {
let (entry_count, total_bytes): (i64, i64) = conn
.query_row("SELECT COUNT(*), COALESCE(SUM(byte_size), 0) FROM schema_cache", [], |row| {
Ok((row.get(0)?, row.get(1)?))
})
.map_err(|error| error.to_string())?;
if entry_count <= policy.max_entries as i64 && total_bytes <= policy.max_total_bytes.max(0) {
break;
}
if !delete_oldest_schema_cache_entry(conn, None)? {
break;
}
}
Ok(())
}
impl Storage { impl Storage {
pub async fn save_schema_cache(&self, cache_key: &str, payload: &serde_json::Value) -> Result<(), String> { pub async fn save_schema_cache(&self, cache_key: &str, payload: &serde_json::Value) -> Result<(), String> {
self.save_schema_cache_with_policy(cache_key, payload, SchemaCachePolicy::default()).await
}
async fn save_schema_cache_with_policy(
&self,
cache_key: &str,
payload: &serde_json::Value,
policy: SchemaCachePolicy,
) -> Result<(), String> {
let cache_key = cache_key.to_string(); let cache_key = cache_key.to_string();
let json = serde_json::to_string(payload).map_err(|e| e.to_string())?; let json = serde_json::to_string(payload).map_err(|e| e.to_string())?;
let byte_size = json.len().min(i64::MAX as usize) as i64;
let owner_id = schema_cache_owner(&cache_key).to_string();
let now_ms = unix_timestamp_millis();
self.with_conn(move |conn| { self.with_conn(move |conn| {
conn.execute( let transaction = conn.transaction().map_err(|error| error.to_string())?;
"INSERT OR REPLACE INTO schema_cache (cache_key, payload_json, updated_at) \ transaction
VALUES (?1, ?2, datetime('now'))", .execute(
params![cache_key, json], "INSERT INTO schema_cache (
) cache_key, payload_json, updated_at, updated_at_ms, last_accessed_at_ms, byte_size, owner_id
.map(|_| ()) ) VALUES (?1, ?2, datetime('now'), ?3, ?3, ?4, ?5)
.map_err(|e| e.to_string()) ON CONFLICT(cache_key) DO UPDATE SET
payload_json = excluded.payload_json,
updated_at = excluded.updated_at,
updated_at_ms = excluded.updated_at_ms,
last_accessed_at_ms = excluded.last_accessed_at_ms,
byte_size = excluded.byte_size,
owner_id = excluded.owner_id",
params![cache_key, json, now_ms, byte_size, owner_id],
)
.map_err(|error| error.to_string())?;
prune_schema_cache(&transaction, policy, now_ms)?;
transaction.commit().map_err(|error| error.to_string())
}) })
.await .await
} }
pub async fn load_schema_cache(&self, cache_key: &str) -> Result<Option<serde_json::Value>, String> { pub async fn load_schema_cache(&self, cache_key: &str) -> Result<Option<serde_json::Value>, String> {
let cache_key = cache_key.to_string(); let cache_key = cache_key.to_string();
let now_ms = unix_timestamp_millis();
let json: Option<String> = self let json: Option<String> = self
.with_conn(move |conn| { .with_conn(move |conn| {
conn.query_row("SELECT payload_json FROM schema_cache WHERE cache_key = ?1", [cache_key], |row| { let transaction = conn.transaction().map_err(|error| error.to_string())?;
row.get(0) let json = transaction
}) .query_row(
.optional() "SELECT payload_json FROM schema_cache
.map_err(|e| e.to_string()) WHERE cache_key = ?1 AND updated_at_ms > ?2",
params![cache_key, now_ms.saturating_sub(SCHEMA_CACHE_MAX_AGE_MILLIS.max(0))],
|row| row.get(0),
)
.optional()
.map_err(|error| error.to_string())?;
if json.is_some() {
transaction
.execute(
"UPDATE schema_cache SET last_accessed_at_ms = ?2 WHERE cache_key = ?1",
params![cache_key, now_ms],
)
.map_err(|error| error.to_string())?;
}
transaction.commit().map_err(|error| error.to_string())?;
Ok(json)
}) })
.await?; .await?;
json.map(|value| serde_json::from_str(&value).map_err(|e| e.to_string())).transpose() json.map(|value| serde_json::from_str(&value).map_err(|e| e.to_string())).transpose()
@@ -4097,7 +4324,7 @@ mod tests {
}; };
use crate::saved_sql::SavedSqlFile; use crate::saved_sql::SavedSqlFile;
use rusqlite::Connection; use rusqlite::Connection;
use std::time::{SystemTime, UNIX_EPOCH}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
fn temp_db_path(name: &str) -> std::path::PathBuf { fn temp_db_path(name: &str) -> std::path::PathBuf {
let stamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos(); let stamp = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos();
@@ -5240,6 +5467,202 @@ mod tests {
); );
} }
#[test]
fn metadata_cache_memory_budget_is_bounded() {
assert_eq!(super::normalize_metadata_cache_max_memory_mb(1), 16);
assert_eq!(super::normalize_metadata_cache_max_memory_mb(256), 256);
assert_eq!(super::normalize_metadata_cache_max_memory_mb(512), 512);
assert_eq!(super::normalize_metadata_cache_max_memory_mb(513), 64);
}
#[tokio::test]
async fn schema_cache_prunes_expired_rows_on_write_without_maintaining_during_reads() {
let path = temp_db_path("schema-cache-ttl-prune");
let storage = Storage::open(&path).await.unwrap();
storage
.save_schema_cache(
"object-ddl:v1:conn-a:db:public:old::TABLE:",
&serde_json::json!({ "version": 1, "ddl": "old" }),
)
.await
.unwrap();
storage
.with_conn(|conn| {
conn.execute(
"UPDATE schema_cache SET updated_at = datetime('now', '-25 hours'), updated_at_ms = CAST(strftime('%s', 'now', '-25 hours') AS INTEGER) * 1000",
[],
)
.map(|_| ())
.map_err(|error| error.to_string())
})
.await
.unwrap();
assert_eq!(storage.load_schema_cache("object-ddl:v1:conn-a:db:public:old::TABLE:").await.unwrap(), None);
let remaining = storage
.with_conn(|conn| {
conn.query_row("SELECT COUNT(*) FROM schema_cache", [], |row| row.get::<_, i64>(0))
.map_err(|error| error.to_string())
})
.await
.unwrap();
assert_eq!(remaining, 1, "L2 reads must remain indexed point lookups without global maintenance");
storage
.save_schema_cache(
"object-ddl:v1:conn-a:db:public:new::TABLE:",
&serde_json::json!({ "version": 1, "ddl": "new" }),
)
.await
.unwrap();
let remaining = storage
.with_conn(|conn| {
conn.query_row("SELECT COUNT(*) FROM schema_cache", [], |row| row.get::<_, i64>(0))
.map_err(|error| error.to_string())
})
.await
.unwrap();
assert_eq!(remaining, 1, "the next write must prune the expired row");
let _ = std::fs::remove_file(path);
}
#[tokio::test]
async fn schema_cache_budget_covers_multiple_connections_objects_and_facets() {
let path = temp_db_path("schema-cache-capacity");
let storage = Storage::open(&path).await.unwrap();
let policy = super::SchemaCachePolicy {
max_total_bytes: 1_100,
max_connection_bytes: 700,
max_entries: 5,
max_age_millis: 86_400_000,
};
let payload = serde_json::json!({ "value": "x".repeat(180) });
let keys = [
"object-ddl:v1:conn-a:db:public:accounts::TABLE:",
"object-meta:v1:conn-a:db:public:accounts::TABLE:columns:",
"object-meta:v1:conn-a:db:public:billing::TABLE:indexes:",
"object-ddl:v1:conn-b:db:public:events::TABLE:",
"object-meta:v1:conn-b:db:public:events::TABLE:triggers:",
"object-meta:v1:conn-c:db:public:audit::TABLE:comment:",
];
for key in keys {
storage.save_schema_cache_with_policy(key, &payload, policy).await.unwrap();
}
let (entries, bytes, conn_a_bytes) = storage
.with_conn(|conn| {
conn.query_row(
"SELECT COUNT(*), COALESCE(SUM(byte_size), 0), COALESCE(SUM(CASE WHEN owner_id = 'conn-a' THEN byte_size ELSE 0 END), 0) FROM schema_cache",
[],
|row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?, row.get::<_, i64>(2)?)),
)
.map_err(|error| error.to_string())
})
.await
.unwrap();
assert!(entries <= policy.max_entries as i64);
assert!(bytes <= policy.max_total_bytes);
assert!(conn_a_bytes <= policy.max_connection_bytes);
assert!(storage.load_schema_cache(keys.last().unwrap()).await.unwrap().is_some());
let _ = std::fs::remove_file(path);
}
#[tokio::test]
async fn schema_cache_lru_keeps_recently_accessed_entries() {
let path = temp_db_path("schema-cache-lru");
let storage = Storage::open(&path).await.unwrap();
let policy = super::SchemaCachePolicy {
max_total_bytes: i64::MAX,
max_connection_bytes: i64::MAX,
max_entries: 2,
max_age_millis: 86_400_000,
};
let first = "object-ddl:v1:conn-a:db:public:first::TABLE:";
let second = "object-meta:v1:conn-b:db:public:second::TABLE:columns:";
let newest = "object-meta:v1:conn-c:db:public:newest::TABLE:indexes:";
storage.save_schema_cache_with_policy(first, &serde_json::json!({ "value": 1 }), policy).await.unwrap();
storage.save_schema_cache_with_policy(second, &serde_json::json!({ "value": 2 }), policy).await.unwrap();
storage
.with_conn(move |conn| {
conn.execute(
"UPDATE schema_cache SET last_accessed_at_ms = CASE cache_key WHEN ?1 THEN 1 ELSE 2 END",
rusqlite::params![first],
)
.map(|_| ())
.map_err(|error| error.to_string())
})
.await
.unwrap();
assert!(storage.load_schema_cache(first).await.unwrap().is_some());
storage.save_schema_cache_with_policy(newest, &serde_json::json!({ "value": 3 }), policy).await.unwrap();
assert!(storage.load_schema_cache(first).await.unwrap().is_some());
assert_eq!(storage.load_schema_cache(second).await.unwrap(), None);
assert!(storage.load_schema_cache(newest).await.unwrap().is_some());
let _ = std::fs::remove_file(path);
}
#[tokio::test]
async fn schema_cache_50k_point_reads_stay_below_performance_gate() {
const ENTRY_COUNT: usize = 50_000;
const SAMPLE_COUNT: usize = 40;
let path = temp_db_path("schema-cache-50k-read-performance");
let storage = Storage::open(&path).await.unwrap();
storage
.with_conn(|conn| {
let transaction = conn.transaction().map_err(|error| error.to_string())?;
{
let mut statement = transaction
.prepare(
"INSERT INTO schema_cache (
cache_key, payload_json, updated_at, updated_at_ms,
last_accessed_at_ms, byte_size, owner_id
) VALUES (?1, ?2, datetime('now'), ?3, ?3, ?4, ?5)",
)
.map_err(|error| error.to_string())?;
for index in 0..ENTRY_COUNT {
let cache_key =
format!("object-meta:v1:conn-{}:db:public:table-{index}::TABLE:columns:", index % 32);
let payload = format!(r#"{{"version":1,"value":{index}}}"#);
statement
.execute(rusqlite::params![
cache_key,
payload,
super::unix_timestamp_millis(),
32_i64,
format!("conn-{}", index % 32),
])
.map_err(|error| error.to_string())?;
}
}
transaction.commit().map_err(|error| error.to_string())
})
.await
.unwrap();
let mut samples = Vec::with_capacity(SAMPLE_COUNT);
for sample in 0..SAMPLE_COUNT {
let index = sample * (ENTRY_COUNT / SAMPLE_COUNT);
let cache_key = format!("object-meta:v1:conn-{}:db:public:table-{index}::TABLE:columns:", index % 32);
let started = Instant::now();
assert!(storage.load_schema_cache(&cache_key).await.unwrap().is_some());
samples.push(started.elapsed());
}
samples.sort_unstable();
let total = samples.iter().copied().sum::<Duration>();
let average = total / SAMPLE_COUNT as u32;
let p95 = samples[(SAMPLE_COUNT * 95 / 100).saturating_sub(1)];
eprintln!("schema_cache_50k_point_reads average={average:?} p95={p95:?}");
assert!(average < Duration::from_millis(20), "50k L2 point-read average {average:?} exceeded 20 ms");
assert!(p95 < Duration::from_millis(50), "50k L2 point-read P95 {p95:?} exceeded 50 ms");
let _ = std::fs::remove_file(path);
}
#[tokio::test] #[tokio::test]
async fn desktop_settings_preserve_existing_password_hash() { async fn desktop_settings_preserve_existing_password_hash() {
let path = temp_db_path("desktop-settings-preserve-password"); let path = temp_db_path("desktop-settings-preserve-password");
@@ -5253,6 +5676,7 @@ mod tests {
quit_on_close: true, quit_on_close: true,
close_action_prompted: false, close_action_prompted: false,
debug_logging_enabled: true, debug_logging_enabled: true,
metadata_cache_max_memory_mb: 128,
duckdb_worker_process_isolation: false, duckdb_worker_process_isolation: false,
duckdb_worker_max_processes: DesktopSettings::default().duckdb_worker_max_processes, duckdb_worker_max_processes: DesktopSettings::default().duckdb_worker_max_processes,
saved_sql_sync_dir: None, saved_sql_sync_dir: None,
@@ -5273,6 +5697,7 @@ mod tests {
quit_on_close: true, quit_on_close: true,
close_action_prompted: false, close_action_prompted: false,
debug_logging_enabled: true, debug_logging_enabled: true,
metadata_cache_max_memory_mb: 128,
duckdb_worker_process_isolation: false, duckdb_worker_process_isolation: false,
duckdb_worker_max_processes: DesktopSettings::default().duckdb_worker_max_processes, duckdb_worker_max_processes: DesktopSettings::default().duckdb_worker_max_processes,
saved_sql_sync_dir: None, saved_sql_sync_dir: None,
+4 -3
View File
@@ -704,9 +704,10 @@ async fn live_oceanbase_mysql_setup_applies_query_timeout() {
async fn live_mysql_query_cancel_kills_running_sleep() { async fn live_mysql_query_cancel_kills_running_sleep() {
let url = std::env::var("DBX_LIVE_MYSQL_CANCEL_URL").expect("DBX_LIVE_MYSQL_CANCEL_URL"); let url = std::env::var("DBX_LIVE_MYSQL_CANCEL_URL").expect("DBX_LIVE_MYSQL_CANCEL_URL");
let opts = mysql_async::OptsBuilder::from_opts(mysql_async::Opts::from_url(&url).unwrap()) let pool =
.pool_opts(mysql_async::PoolOpts::new().with_constraints(mysql_async::PoolConstraints::new(1, 1).unwrap())); dbx_core::db::mysql::connect_bare_with_pool_limit_and_setup(&url, std::time::Duration::from_secs(10), 1, &[])
let pool = mysql_async::Pool::new(opts); .await
.unwrap();
let mut conn = dbx_core::db::mysql::get_conn_with_health_check(&pool).await.unwrap(); let mut conn = dbx_core::db::mysql::get_conn_with_health_check(&pool).await.unwrap();
let connection_id = mysql_async::Conn::id(&conn); let connection_id = mysql_async::Conn::id(&conn);
let kill_opts = conn.opts().clone(); let kill_opts = conn.opts().clone();