feat: stream query result CSV/XLSX exports
* docs: add query result streaming export design * feat(core): stream query result exports * feat: expose query result streaming export APIs * feat(desktop): stream query result exports from grid * fix: route query result export cancellation through its command * fix: reduce query export batch size and cancel latency * opt: del local test files
This commit is contained in:
1 parent
849caf4b39
commit
588ba3352d
25 files changed
+1787
-47
No files matched your search
@@ -49,6 +49,12 @@ const ACTIVE_TAB_KEY = "dbx-active-tab";
|
||||
const ORACLE_LIKE_METADATA_TYPES = new Set<string>(["oracle", "dameng", "oceanbase-oracle"]);
|
||||
const BACKGROUND_CLIENT_SESSION_SUFFIXES = ["count", "explain", "export"] as const;
|
||||
|
||||
interface BuildQueryResultExportRequestOptions {
|
||||
exportId: string;
|
||||
filePath: string;
|
||||
format: "csv" | "xlsx";
|
||||
}
|
||||
|
||||
function tabClientSessionId(tab: Pick<QueryTab, "id">, suffix?: (typeof BACKGROUND_CLIENT_SESSION_SUFFIXES)[number]): string {
|
||||
return suffix ? `${tab.id}:${suffix}` : tab.id;
|
||||
}
|
||||
@@ -2186,6 +2192,7 @@ export const useQueryStore = defineStore("query", () => {
|
||||
const queryTimeoutSecs = queryTimeoutSecsForConnection(conn);
|
||||
const useAgentCursor = conn?.db_type === "jdbc" || supportsDatabaseFeature(conn?.db_type, "driverManagement");
|
||||
const queryBaseSql = tab.resultBaseSql ?? sql;
|
||||
const exportRowLimit = useSettingsStore().editorSettings.exportRowLimit;
|
||||
// Use the already-computed total row count as a progress estimate so the
|
||||
// export dialog shows a moving bar instead of a stuck 0 while paginating.
|
||||
const totalRows = typeof tab.resultTotalRowCount === "number" ? tab.resultTotalRowCount : null;
|
||||
@@ -2199,17 +2206,21 @@ export const useQueryStore = defineStore("query", () => {
|
||||
const exportExecutionId = uuid();
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
while (rows.length < exportRowLimit) {
|
||||
const remaining = exportRowLimit - rows.length;
|
||||
const effectivePageLimit = Math.min(pageLimit, remaining);
|
||||
const plan = await api.prepareQueryPaginationExecutionPlan({
|
||||
sql,
|
||||
queryBaseSql,
|
||||
databaseType: effectiveDbType,
|
||||
pagination: { limit: pageLimit, offset, sessionId },
|
||||
pagination: { limit: effectivePageLimit, offset, sessionId },
|
||||
useAgentCursor,
|
||||
firstPageUsesActualSql: true,
|
||||
});
|
||||
if (typeof plan.pageLimit !== "number" || typeof plan.pageOffset !== "number") return tab.result;
|
||||
const executionOptions = plan.useAgentResultSession
|
||||
? {
|
||||
maxRows: exportRowLimit,
|
||||
fetchSize: plan.pageLimit,
|
||||
pageSize: plan.pageLimit,
|
||||
resultSessionId: sessionId,
|
||||
@@ -2226,7 +2237,7 @@ export const useQueryStore = defineStore("query", () => {
|
||||
onProgress?.({ rowsExported: rows.length, totalRows });
|
||||
sessionId = result.session_id ?? undefined;
|
||||
const shouldFetchNextPage = plan.useAgentResultSession ? result.has_more === true : result.rows.length >= plan.pageLimit;
|
||||
if (!shouldFetchNextPage) break;
|
||||
if (!shouldFetchNextPage || rows.length >= exportRowLimit) break;
|
||||
offset += result.rows.length;
|
||||
}
|
||||
} finally {
|
||||
@@ -2244,6 +2255,45 @@ export const useQueryStore = defineStore("query", () => {
|
||||
};
|
||||
}
|
||||
|
||||
async function buildQueryResultExportRequest(id: string, options: BuildQueryResultExportRequestOptions) {
|
||||
const tab = tabs.value.find((t) => t.id === id);
|
||||
if (!tab?.result || tab.mode !== "query") return undefined;
|
||||
|
||||
const sql = tab.resultSortedSql ?? tab.resultBaseSql ?? tab.lastExecutedSql ?? tab.sql;
|
||||
if (!sql.trim()) return undefined;
|
||||
|
||||
const connStore = useConnectionStore();
|
||||
await connStore.ensureConnected(tab.connectionId);
|
||||
const conn = connStore.getConfig(tab.connectionId);
|
||||
const settings = useSettingsStore().editorSettings;
|
||||
const effectiveDbType = effectiveDatabaseTypeForConnection(conn);
|
||||
if (!effectiveDbType) return undefined;
|
||||
const useAgentCursor = conn?.db_type === "jdbc" || supportsDatabaseFeature(conn?.db_type, "driverManagement");
|
||||
const queryBaseSql = tab.resultBaseSql ?? sql;
|
||||
const totalRows = typeof tab.resultTotalRowCount === "number" ? tab.resultTotalRowCount : null;
|
||||
const clientSessionId = tabClientSessionId(tab, "export");
|
||||
|
||||
return {
|
||||
exportId: options.exportId,
|
||||
connectionId: tab.connectionId,
|
||||
database: tab.database,
|
||||
schema: tab.schema,
|
||||
sql,
|
||||
queryBaseSql,
|
||||
databaseType: effectiveDbType,
|
||||
useAgentCursor,
|
||||
filePath: options.filePath,
|
||||
format: options.format,
|
||||
pageSize: settings.exportBatchSize,
|
||||
rowLimit: settings.exportRowLimitEnabled ? settings.exportRowLimit : null,
|
||||
totalRows,
|
||||
timeoutSecs: queryTimeoutSecsForConnection(conn),
|
||||
keysetOptimizationEnabled: settings.queryExportKeysetOptimizationEnabled,
|
||||
clientSessionId,
|
||||
executionId: uuid(),
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
tabs,
|
||||
activeTabId,
|
||||
@@ -2301,6 +2351,7 @@ export const useQueryStore = defineStore("query", () => {
|
||||
exportResultArchive,
|
||||
importResultArchive,
|
||||
fetchTabResultForExport,
|
||||
buildQueryResultExportRequest,
|
||||
notifyConnectionMayBeLost,
|
||||
};
|
||||
});
|
||||
@@ -341,6 +341,9 @@ export interface EditorSettings {
|
||||
customColumnFormatters: Record<string, CustomColumnFormatterConfig>;
|
||||
snippets: SqlSnippet[];
|
||||
exportBatchSize: number;
|
||||
exportRowLimitEnabled: boolean;
|
||||
exportRowLimit: number;
|
||||
queryExportKeysetOptimizationEnabled: boolean;
|
||||
toolbarItems: ToolbarItems;
|
||||
}
|
||||
|
||||
@@ -440,12 +443,17 @@ export const DEFAULT_EDITOR_SETTINGS: EditorSettings = {
|
||||
columnFormatters: {},
|
||||
customColumnFormatters: {},
|
||||
snippets: DEFAULT_SQL_SNIPPETS,
|
||||
exportBatchSize: 10000,
|
||||
exportBatchSize: 2000,
|
||||
exportRowLimitEnabled: true,
|
||||
exportRowLimit: 100000,
|
||||
queryExportKeysetOptimizationEnabled: true,
|
||||
toolbarItems: { ...DEFAULT_TOOLBAR_ITEMS },
|
||||
};
|
||||
|
||||
export const STORAGE_KEY = "dbx-editor-settings";
|
||||
const OLD_FONT_SIZE_KEY = "dbx-query-editor-font-size";
|
||||
const EXPORT_BATCH_SIZE_DEFAULT_MIGRATION_KEY = "dbx-export-batch-size-default-migrated-v1";
|
||||
const LEGACY_DEFAULT_EXPORT_BATCH_SIZE = 10000;
|
||||
const MIN_UI_SCALE = 0.75;
|
||||
const MAX_UI_SCALE = 2;
|
||||
|
||||
@@ -606,6 +614,9 @@ export function normalizeEditorSettings(settings: Partial<EditorSettings>, exist
|
||||
customColumnFormatters: normalizeCustomColumnFormatters(settings.customColumnFormatters),
|
||||
snippets: normalizeSqlSnippets(settings.snippets, existing?.snippets),
|
||||
exportBatchSize: typeof settings.exportBatchSize === "number" && settings.exportBatchSize >= 100 && settings.exportBatchSize <= 100000 ? Math.round(settings.exportBatchSize) : DEFAULT_EDITOR_SETTINGS.exportBatchSize,
|
||||
exportRowLimitEnabled: typeof settings.exportRowLimitEnabled === "boolean" ? settings.exportRowLimitEnabled : DEFAULT_EDITOR_SETTINGS.exportRowLimitEnabled,
|
||||
exportRowLimit: typeof settings.exportRowLimit === "number" && settings.exportRowLimit >= 100 && settings.exportRowLimit <= 2147483647 ? Math.round(settings.exportRowLimit) : DEFAULT_EDITOR_SETTINGS.exportRowLimit,
|
||||
queryExportKeysetOptimizationEnabled: typeof settings.queryExportKeysetOptimizationEnabled === "boolean" ? settings.queryExportKeysetOptimizationEnabled : DEFAULT_EDITOR_SETTINGS.queryExportKeysetOptimizationEnabled,
|
||||
toolbarItems: normalizeToolbarItems(settings.toolbarItems),
|
||||
};
|
||||
}
|
||||
@@ -616,6 +627,13 @@ function loadEditorSettings(): EditorSettings {
|
||||
const raw = localStorage.getItem(STORAGE_KEY);
|
||||
if (raw) {
|
||||
const parsed = JSON.parse(raw) as Partial<EditorSettings>;
|
||||
if (parsed.exportBatchSize === LEGACY_DEFAULT_EXPORT_BATCH_SIZE && localStorage.getItem(EXPORT_BATCH_SIZE_DEFAULT_MIGRATION_KEY) !== "1") {
|
||||
parsed.exportBatchSize = DEFAULT_EDITOR_SETTINGS.exportBatchSize;
|
||||
localStorage.setItem(EXPORT_BATCH_SIZE_DEFAULT_MIGRATION_KEY, "1");
|
||||
const migrated = normalizeEditorSettings(parsed);
|
||||
saveEditorSettings(migrated);
|
||||
return migrated;
|
||||
}
|
||||
return normalizeEditorSettings(parsed);
|
||||
}
|
||||
} catch {
|
||||
@@ -767,6 +785,9 @@ export const useSettingsStore = defineStore("settings", () => {
|
||||
if (partial.customColumnFormatters !== undefined) editorSettings.value.customColumnFormatters = partial.customColumnFormatters;
|
||||
if (partial.snippets !== undefined) editorSettings.value.snippets = normalizeSqlSnippets(partial.snippets);
|
||||
if (partial.exportBatchSize !== undefined) editorSettings.value.exportBatchSize = Math.min(100000, Math.max(100, Math.round(partial.exportBatchSize)));
|
||||
if (partial.exportRowLimitEnabled !== undefined) editorSettings.value.exportRowLimitEnabled = partial.exportRowLimitEnabled;
|
||||
if (partial.exportRowLimit !== undefined) editorSettings.value.exportRowLimit = Math.min(2147483647, Math.max(100, Math.round(partial.exportRowLimit)));
|
||||
if (partial.queryExportKeysetOptimizationEnabled !== undefined) editorSettings.value.queryExportKeysetOptimizationEnabled = partial.queryExportKeysetOptimizationEnabled;
|
||||
if (partial.toolbarItems !== undefined) editorSettings.value.toolbarItems = normalizeToolbarItems(partial.toolbarItems);
|
||||
saveEditorSettings(editorSettings.value);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user