From 0e33c73035d5d18c7952800fd19927d9ae1b7840 Mon Sep 17 00:00:00 2001 From: miracle Date: Wed, 22 Jul 2026 16:56:49 +0800 Subject: [PATCH] perf(import): optimize large file imports --- Cargo.lock | 1 + .../components/import/TableImportDialog.vue | 191 +- apps/desktop/src/i18n/locales/en.ts | 9 + apps/desktop/src/i18n/locales/es.ts | 9 + apps/desktop/src/i18n/locales/it.ts | 9 + apps/desktop/src/i18n/locales/ja.ts | 9 + apps/desktop/src/i18n/locales/pt-BR.ts | 9 + apps/desktop/src/i18n/locales/zh-CN.ts | 9 + apps/desktop/src/i18n/locales/zh-TW.ts | 9 + .../lib/__tests__/table/tableImport.spec.ts | 37 +- apps/desktop/src/lib/backend/api.ts | 2 + apps/desktop/src/lib/backend/http.ts | 15 +- apps/desktop/src/lib/backend/tauri.ts | 28 +- apps/desktop/src/lib/table/tableImport.ts | 48 + crates/dbx-core/Cargo.toml | 1 + .../dbx-core/examples/table_import_bench.rs | 211 + crates/dbx-core/src/connection.rs | 78 +- crates/dbx-core/src/query.rs | 29 +- crates/dbx-core/src/table_import.rs | 4454 +++++++++++++++-- crates/dbx-core/src/task_supervisor.rs | 57 +- crates/dbx-core/src/transfer.rs | 72 + crates/dbx-web/src/main.rs | 3 + crates/dbx-web/src/routes/connection.rs | 1 + crates/dbx-web/src/routes/mongo.rs | 1 + crates/dbx-web/src/routes/table_import.rs | 240 +- crates/dbx-web/src/sse.rs | 19 + crates/dbx-web/src/state.rs | 3 +- package.json | 1 + src-tauri/src/commands/table_import.rs | 102 +- 29 files changed, 5032 insertions(+), 625 deletions(-) create mode 100644 crates/dbx-core/examples/table_import_bench.rs diff --git a/Cargo.lock b/Cargo.lock index 2a5907d019..f4c050630c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1914,6 +1914,7 @@ dependencies = [ "sqlparser", "ssfmt", "sysinfo", + "tempfile", "tiberius", "tokio", "tokio-postgres", diff --git a/apps/desktop/src/components/import/TableImportDialog.vue b/apps/desktop/src/components/import/TableImportDialog.vue index 88b22e6200..f966a75e2b 100644 --- a/apps/desktop/src/components/import/TableImportDialog.vue +++ b/apps/desktop/src/components/import/TableImportDialog.vue @@ -13,7 +13,19 @@ import { AlertTriangle, ArrowLeft, ArrowRight, Check, CheckCircle2, FileJson, Fi import { useConnectionStore } from "@/stores/connectionStore"; import { useSettingsStore } from "@/stores/settingsStore"; import { useToast } from "@/composables/useToast"; -import { autoMapImportColumns, buildTableImportParseOptions, nextTableImportWizardStep, previousTableImportWizardStep, requiredImportTargetColumns, suggestImportTargetDataTypes, validateImportMappings, type TableImportWizardStep } from "@/lib/table/tableImport"; +import { + autoMapImportColumns, + buildTableImportParseOptions, + formatTableImportElapsed, + nextTableImportWizardStep, + previousTableImportWizardStep, + requiredImportTargetColumns, + resolveTableImportElapsed, + suggestImportTargetDataTypes, + tableImportProgressPercent, + validateImportMappings, + type TableImportWizardStep, +} from "@/lib/table/tableImport"; import { getDataTypeOptions } from "@/lib/table/tableStructureEditorState"; import { tableStructureDatabaseTypeForConnection } from "@/lib/database/jdbcDialect"; import type { ColumnInfo } from "@/types/database"; @@ -83,7 +95,10 @@ const emptyStringAsNull = ref(true); const selectedSheet = ref(""); const jsonShape = ref("auto"); const previewLimit = ref(50); +const liveElapsedMs = ref(0); let previewReloadTimer: ReturnType | null = null; +let importElapsedTimer: ReturnType | null = null; +let importStartedAt = 0; let dataTypeOptionsRequestId = 0; let previewRequestId = 0; let batchEncodingRequestId = 0; @@ -158,11 +173,9 @@ const canGoNext = computed(() => { if (wizardStep.value === "mapping") return mappingValidation.value.valid; return false; }); -const progressPercent = computed(() => { - const p = progress.value; - if (!p || p.totalRows <= 0) return 0; - return Math.min(100, Math.round((p.rowsImported / p.totalRows) * 100)); -}); +const rawProgressPercent = computed(() => tableImportProgressPercent(progress.value)); +const progressPercentFloor = ref(0); +const progressPercent = computed(() => Math.max(rawProgressPercent.value, progressPercentFloor.value)); const currentStepIndex = computed(() => wizardSteps.findIndex((step) => step.value === wizardStep.value)); const targetLabel = computed(() => { const pieces = [selectedConnection.value?.name, props.prefillDatabase, props.prefillSchema, targetTableName.value].filter(Boolean); @@ -181,9 +194,43 @@ const createColumnSummaries = computed(() => })), ); const parseOptions = computed(() => taskParseOptions(sourceFormat.value, selectedSheet.value)); -const terminalStatus = computed(() => progress.value?.status && ["done", "error", "cancelled"].includes(progress.value.status)); +const terminalStatus = computed(() => !!progress.value?.status && ["done", "error", "cancelled"].includes(progress.value.status)); +const displayedElapsedMs = computed(() => resolveTableImportElapsed(liveElapsedMs.value, progress.value?.elapsedMs, terminalStatus.value)); +const progressLabelKey = computed(() => { + if (terminalStatus.value) return `tableImport.status_${progress.value?.status || "idle"}`; + return `tableImport.phase_${progress.value?.phase || "writing"}`; +}); + +function previewRowsLabel(currentPreview: api.TableImportPreview | null | undefined): string { + if (!currentPreview) return "-"; + if (currentPreview.totalRowsExact !== false) return currentPreview.totalRows.toLocaleString(); + return t("tableImport.previewPartial", { rows: currentPreview.rows.length }); +} + +function stopImportElapsedClock() { + if (importElapsedTimer) { + clearInterval(importElapsedTimer); + importElapsedTimer = null; + } +} + +function refreshImportElapsedClock() { + if (importStartedAt > 0) { + liveElapsedMs.value = Math.max(0, Math.round(performance.now() - importStartedAt)); + } +} + +function startImportElapsedClock() { + stopImportElapsedClock(); + importStartedAt = performance.now(); + liveElapsedMs.value = 0; + importElapsedTimer = setInterval(refreshImportElapsedClock, 100); +} function resetState() { + stopImportElapsedClock(); + importStartedAt = 0; + liveElapsedMs.value = 0; previewRequestId++; batchEncodingRequestId++; if (previewReloadTimer) { @@ -274,12 +321,23 @@ function taskParseOptions(format: api.TableImportSourceFormat, sheetName = ""): function importParseOptions(format: api.TableImportSourceFormat, currentPreview: api.TableImportPreview, sheetName = ""): api.TableImportParseOptions { const options = taskParseOptions(format, sheetName); - if (isDelimitedFormat(format) && options.encoding === "auto" && currentPreview.effectiveEncoding) { + if (isDelimitedFormat(format) && currentPreview.totalRowsExact !== false && options.encoding === "auto" && currentPreview.effectiveEncoding) { options.encoding = currentPreview.effectiveEncoding; } return options; } +function preparedImportSource(currentPreview: api.TableImportPreview): api.TableImportPreparedSource { + return { + fingerprint: currentPreview.sourceFingerprint, + columns: currentPreview.columns, + rows: currentPreview.rows, + totalRows: currentPreview.totalRows, + totalRowsExact: currentPreview.totalRowsExact !== false, + effectiveEncoding: currentPreview.effectiveEncoding ?? null, + }; +} + function mergeDataTypeOptions(...groups: readonly string[][]): string[] { const seen = new Set(); const result: string[] = []; @@ -358,7 +416,9 @@ async function loadTargetColumns() { } async function previewSelectedImportFile(fileOrPath: string | File) { - return api.previewTableImportFile(fileOrPath, { + const reusablePreview = preview.value?.sourceRef ? preview.value : null; + return api.previewTableImportFile(reusablePreview?.filePath || fileOrPath, { + sourceRef: reusablePreview?.sourceRef || null, sourceFormat: sourceFormat.value, parseOptions: parseOptions.value, previewLimit: Math.max(1, Number(previewLimit.value) || 50), @@ -445,20 +505,22 @@ async function prepareBatchSources(sources: ImportSource[]) { }); const sheets = format === "excel" && initialPreview.sheets?.length ? initialPreview.sheets : [""]; for (const sheetName of sheets) { - const taskPreview = - sheetName && sheetName !== initialPreview.sheets?.[0] - ? await api.previewTableImportFile(source, { - sourceFormat: format, - parseOptions: taskParseOptions(format, sheetName), - previewLimit: Math.max(1, Number(previewLimit.value) || 50), - }) - : initialPreview; + const reusableSource = initialPreview.sourceRef ? initialPreview.filePath : source; + const effectiveSheetName = sheetName && sheetName === initialPreview.sheets?.[0] ? "" : sheetName; + const taskPreview = effectiveSheetName + ? await api.previewTableImportFile(reusableSource, { + sourceRef: initialPreview.sourceRef || null, + sourceFormat: format, + parseOptions: taskParseOptions(format, effectiveSheetName), + previewLimit: Math.max(1, Number(previewLimit.value) || 50), + }) + : initialPreview; const tableBase = sheetName ? `${suggestedTableName(sourceName(source))}_${sheetName}` : sourceName(source); tasks.push({ id: uuid(), source, format, - sheetName, + sheetName: effectiveSheetName, tableName: uniqueTableName(tableBase, usedNames), preview: taskPreview, columnMapping: Object.fromEntries(taskPreview.columns.map((column) => [column, column])), @@ -591,15 +653,22 @@ async function startImport() { const tableName = targetTableName.value; if (!canImport.value || !currentPreview || !props.prefillConnectionId || !tableName) return; running.value = true; + progressPercentFloor.value = 0; cancelling.value = false; errorMessage.value = ""; wizardStep.value = "execution"; importId.value = uuid(); + startImportElapsedClock(); progress.value = { importId: importId.value, status: "running", + phase: "preparing", rowsImported: 0, totalRows: currentPreview.totalRows, + totalRowsExact: currentPreview.totalRowsExact !== false, + bytesRead: 0, + totalBytes: currentPreview.sizeBytes, + elapsedMs: 0, }; try { @@ -620,12 +689,13 @@ async function startImport() { createTable: targetMode.value === "create", batchSize: Math.max(1, Number(batchSize.value) || 500), dateTimeFormat: settingsStore.editorSettings.globalDateTimeImportFormat || undefined, + preparedSource: preparedImportSource(currentPreview), }, (nextProgress) => { - progress.value = nextProgress; + progress.value = { ...nextProgress, elapsedMs: nextProgress.elapsedMs ?? liveElapsedMs.value }; }, ); - progress.value = { importId: summary.importId, status: "done", rowsImported: summary.rowsImported, totalRows: summary.totalRows }; + progress.value = { importId: summary.importId, status: "done", rowsImported: summary.rowsImported, totalRows: summary.totalRows, elapsedMs: summary.elapsedMs }; toast(t("tableImport.success", { count: summary.rowsImported }), 2500); store.invalidateMetadataCache(props.prefillConnectionId, props.prefillDatabase || "", props.prefillSchema || undefined, tableName); if (targetMode.value === "create") { @@ -641,9 +711,12 @@ async function startImport() { status: progress.value?.status === "cancelled" ? "cancelled" : "error", rowsImported: progress.value?.rowsImported ?? 0, totalRows: progress.value?.totalRows ?? currentPreview.totalRows, + elapsedMs: progress.value?.elapsedMs ?? liveElapsedMs.value, error: message, }; } finally { + refreshImportElapsedClock(); + stopImportElapsedClock(); running.value = false; cancelling.value = false; } @@ -652,13 +725,18 @@ async function startImport() { async function startBatchImport() { if (!props.prefillConnectionId || !batchTasks.value.length || running.value) return; running.value = true; + progressPercentFloor.value = 0; cancelling.value = false; errorMessage.value = ""; wizardStep.value = "execution"; - const totalRows = batchTasks.value.reduce((sum, task) => sum + task.preview.totalRows, 0); + const totalRowsExact = batchTasks.value.every((task) => task.preview.totalRowsExact !== false); + const totalRows = totalRowsExact ? batchTasks.value.reduce((sum, task) => sum + task.preview.totalRows, 0) : 0; + const totalBytes = batchTasks.value.reduce((sum, task) => sum + task.preview.sizeBytes, 0); let completedRows = 0; + let completedBytes = 0; importId.value = uuid(); - progress.value = { importId: importId.value, status: "running", rowsImported: 0, totalRows }; + startImportElapsedClock(); + progress.value = { importId: importId.value, status: "running", phase: "preparing", rowsImported: 0, totalRows, totalRowsExact, bytesRead: 0, totalBytes, elapsedMs: 0 }; try { const tableNames = batchTasks.value.map((task) => task.tableName.trim().toLowerCase()); @@ -696,18 +774,35 @@ async function startBatchImport() { createTable: true, batchSize: Math.max(1, Number(batchSize.value) || 500), dateTimeFormat: settingsStore.editorSettings.globalDateTimeImportFormat || undefined, + preparedSource: preparedImportSource(task.preview), + retainSource: true, }, (nextProgress) => { task.rowsImported = nextProgress.rowsImported; - progress.value = { ...nextProgress, rowsImported: completedRows + nextProgress.rowsImported, totalRows }; + const hasRemainingTasks = index < batchTasks.value.length - 1; + const aggregateStatus = nextProgress.status === "done" && hasRemainingTasks ? "running" : nextProgress.status; + const aggregatePhase = nextProgress.status === "done" && hasRemainingTasks ? "preparing" : nextProgress.phase; + progress.value = { + ...nextProgress, + status: aggregateStatus, + phase: aggregatePhase, + rowsImported: completedRows + nextProgress.rowsImported, + totalRows, + totalRowsExact, + bytesRead: completedBytes + (nextProgress.bytesRead ?? 0), + totalBytes, + elapsedMs: liveElapsedMs.value, + }; }, ); task.status = "done"; task.rowsImported = summary.rowsImported; completedRows += summary.rowsImported; + completedBytes += task.preview.sizeBytes; store.invalidateMetadataCache(props.prefillConnectionId, props.prefillDatabase || "", props.prefillSchema || undefined, task.tableName); } - progress.value = { importId: importId.value, status: "done", rowsImported: completedRows, totalRows }; + refreshImportElapsedClock(); + progress.value = { importId: importId.value, status: "done", phase: "done", rowsImported: completedRows, totalRows: completedRows, totalRowsExact: true, bytesRead: totalBytes, totalBytes, elapsedMs: liveElapsedMs.value }; toast(t("tableImport.success", { count: completedRows }), 2500); store.refreshObjectListTreeNode(props.prefillConnectionId, props.prefillDatabase || "", props.prefillSchema || undefined).catch((error) => { console.warn("[DBX][table-import:refresh-created-table-failed]", error); @@ -725,14 +820,27 @@ async function startBatchImport() { status: progress.value?.status === "cancelled" ? "cancelled" : "error", rowsImported: progress.value?.rowsImported ?? completedRows, totalRows, + elapsedMs: progress.value?.elapsedMs ?? liveElapsedMs.value, error: message, }; } finally { + refreshImportElapsedClock(); + stopImportElapsedClock(); + await releaseTableImportSources(); running.value = false; cancelling.value = false; } } +async function releaseTableImportSources() { + const sourceRefs = new Set(); + if (preview.value?.sourceRef) sourceRefs.add(preview.value.sourceRef); + for (const task of batchTasks.value) { + if (task.preview.sourceRef) sourceRefs.add(task.preview.sourceRef); + } + await Promise.allSettled([...sourceRefs].map((sourceRef) => api.releaseTableImportSource(sourceRef))); +} + async function cancelImport() { if (!importId.value) return; cancelling.value = true; @@ -769,7 +877,9 @@ async function reloadBatchPreviewsForEncoding() { try { for (const task of batchTasks.value) { if (!isDelimitedFormat(task.format)) continue; - const nextPreview = await api.previewTableImportFile(task.source, { + const reusableSource = task.preview.sourceRef ? task.preview.filePath : task.source; + const nextPreview = await api.previewTableImportFile(reusableSource, { + sourceRef: task.preview.sourceRef || null, sourceFormat: task.format, parseOptions: taskParseOptions(task.format, task.sheetName), previewLimit: Math.max(1, Number(previewLimit.value) || 50), @@ -797,6 +907,8 @@ watch( resetState(); void loadTargetColumns(); void loadDataTypeOptions(); + } else if (!running.value) { + void releaseTableImportSources(); } }, { immediate: true }, @@ -818,6 +930,14 @@ watch(targetMode, (mode) => { void loadDataTypeOptions(); } }); + +watch(rawProgressPercent, (percent) => { + if (progress.value?.status === "running") { + progressPercentFloor.value = Math.max(progressPercentFloor.value, percent); + } else if (progress.value?.status === "done") { + progressPercentFloor.value = 100; + } +});