perf(export): optimize SQL export performance

This commit is contained in:
miracle authored and GitHub committed 2026-08-05 14:57:06 +08:00
1 parent 7fd4aef060
commit 605d20dd4f
14 files changed
+665 -293

No files matched your search

@@ -1,5 +1,5 @@
<script setup lang="ts">
import { computed, ref, watch } from "vue";
import { computed, onBeforeUnmount, onMounted, ref, watch } from "vue";
import { useI18n } from "vue-i18n";
import { Dialog, DialogHeader, DialogTitle, DialogFooter, DialogContent } from "@/components/ui/dialog";
import { Button } from "@/components/ui/button";
@@ -12,13 +12,13 @@ import * as api from "@/lib/backend/api";
import type { ExportProgress } from "@/lib/backend/api";
import { isSchemaAware } from "@/lib/database/databaseFeatureSupport";
import { databaseOptionsForConnection } from "@/composables/useDatabaseOptions";
import { buildAllDatabaseExportPlan, generateDatabaseExportId, runDatabaseExportUntilTerminal, type AllDatabaseExportPlanItem } from "@/lib/export/databaseExport";
import { buildAllDatabaseExportPlan, generateDatabaseExportId, runDatabaseExportUntilTerminal, runWithDatabaseBackupSnapshot, type AllDatabaseExportPlanItem } from "@/lib/export/databaseExport";
import { buildSelectedTablesPayload } from "@/lib/export/databaseExportSelection";
import { isTauriRuntime } from "@/lib/backend/tauriRuntime";
import { useToast } from "@/composables/useToast";
import { Input } from "@/components/ui/input";
import { Download, Square, CheckSquare, Search, X, Loader2 } from "@lucide/vue";
import { useExportTracker } from "@/composables/useExportTracker";
import { formatDataTransferDuration, useExportTracker } from "@/composables/useExportTracker";
const { t } = useI18n();
const { toast } = useToast();
@@ -73,6 +73,9 @@ const exportId = ref("");
const exportDone = ref(false);
const exportError = ref<string | null>(null);
const exportCancelled = ref(false);
const exportStartedAt = ref<number | null>(null);
const exportFinishedAt = ref<number | null>(null);
const currentTime = ref(Date.now());
const pendingPrefillTable = ref("");
const pendingPrefillTables = ref<string[]>([]);
const exportAllDatabases = ref(false);
@@ -81,6 +84,25 @@ const batchDatabaseTotal = ref(0);
const batchRowsExported = ref(0);
const activeDatabaseExportId = ref("");
let elapsedTimer: ReturnType<typeof setInterval> | undefined;
onMounted(() => {
elapsedTimer = setInterval(() => {
currentTime.value = Date.now();
}, 1000);
});
onBeforeUnmount(() => {
if (elapsedTimer) clearInterval(elapsedTimer);
});
function finishExportTiming() {
exportFinishedAt.value ??= Date.now();
}
const exportElapsedText = computed(() => {
if (exportStartedAt.value === null) return "";
return formatDataTransferDuration((exportFinishedAt.value ?? currentTime.value) - exportStartedAt.value);
});
const sqlConnections = computed(() => store.connections.filter((c) => !["redis", "mongodb", "elasticsearch", "easysearch", "qdrant", "milvus", "weaviate", "chromadb", "etcd", "zookeeper", "mq", "nacos"].includes(c.db_type)));
const canExport = computed(() => {
@@ -257,6 +279,8 @@ async function startExport() {
// Switch to the progress view only after the save dialog closes, and seed a
// preparing state so the dialog is never a blank panel while metadata loads.
isExporting.value = true;
exportStartedAt.value = Date.now();
exportFinishedAt.value = null;
exportDone.value = false;
exportError.value = null;
exportCancelled.value = false;
@@ -272,40 +296,54 @@ async function startExport() {
preparing: true,
};
const request: api.DatabaseExportRequest = {
exportId: exportId.value,
connectionId: connectionId.value,
database: database.value,
schema: schema.value,
filePath,
selectedTables: buildSelectedTablesPayload(tables.value, selectedTables.value),
includeStructure: includeStructure.value,
includeData: includeData.value,
includeObjects: includeObjects.value,
includeCreateDatabase: includeCreateDatabase.value,
dropTableIfExists: dropTableIfExists.value,
omitAutoIncrement: omitAutoIncrement.value,
batchSize: 1000,
};
addDatabaseExportTask(exportId.value, database.value || "database", filePath);
try {
await api.exportDatabaseSql(request, (progress) => {
exportProgress.value = { ...progress };
updateDatabaseExportTask(progress.exportId, progress);
if (progress.status === "Done") {
exportDone.value = true;
isExporting.value = false;
toast(t("databaseExport.exportSuccess"), 3000);
} else if (progress.status === "Error") {
exportError.value = progress.error;
isExporting.value = false;
} else if (progress.status === "Cancelled") {
exportCancelled.value = true;
isExporting.value = false;
}
});
const connectionType = store.getConfig(connectionId.value)?.db_type;
await runWithDatabaseBackupSnapshot(
{
connectionId: connectionId.value,
database: database.value,
enabled: includeData.value && (connectionType === "mysql" || connectionType === "postgres"),
},
async (snapshotSessionId) => {
const request: api.DatabaseExportRequest = {
exportId: exportId.value,
connectionId: connectionId.value,
database: database.value,
schema: schema.value,
filePath,
selectedTables: buildSelectedTablesPayload(tables.value, selectedTables.value),
includeStructure: includeStructure.value,
includeData: includeData.value,
includeObjects: includeObjects.value,
includeCreateDatabase: includeCreateDatabase.value,
dropTableIfExists: dropTableIfExists.value,
omitAutoIncrement: omitAutoIncrement.value,
snapshotSessionId,
batchSize: 1000,
};
await api.exportDatabaseSql(request, (progress) => {
exportProgress.value = { ...progress };
updateDatabaseExportTask(progress.exportId, progress);
if (progress.status === "Done") {
finishExportTiming();
exportDone.value = true;
isExporting.value = false;
toast(t("databaseExport.exportSuccess"), 3000);
} else if (progress.status === "Error") {
finishExportTiming();
exportError.value = progress.error;
isExporting.value = false;
} else if (progress.status === "Cancelled") {
finishExportTiming();
exportCancelled.value = true;
isExporting.value = false;
}
});
},
() => !exportError.value && !exportCancelled.value,
);
} catch (e: any) {
exportError.value = e?.message || String(e);
const lastProgress = exportProgress.value as api.ExportProgress | null;
@@ -320,6 +358,7 @@ async function startExport() {
error: exportError.value,
};
updateDatabaseExportTask(exportId.value, fallbackProgress);
finishExportTiming();
isExporting.value = false;
}
}
@@ -345,6 +384,8 @@ async function startAllDatabasesExport() {
}
isExporting.value = true;
exportStartedAt.value = Date.now();
exportFinishedAt.value = null;
exportDone.value = false;
exportError.value = null;
exportCancelled.value = false;
@@ -352,6 +393,7 @@ async function startAllDatabasesExport() {
batchRowsExported.value = 0;
const dbs = [...selectedDatabases.value];
const connectionType = store.getConfig(connectionId.value)?.db_type;
const batchId = generateDatabaseExportId();
exportId.value = batchId;
exportProgress.value = {
@@ -390,48 +432,62 @@ async function startAllDatabasesExport() {
const currentExportId = `${batchId}-${index + 1}`;
activeDatabaseExportId.value = currentExportId;
const filePath = isTauriRuntime() ? joinExportPath(directoryPath, `${sanitizeFileName(item.fileStem)}.sql`) : `__web_export_${currentExportId}.sql`;
const request: api.DatabaseExportRequest = {
exportId: currentExportId,
connectionId: connectionId.value,
database: item.database,
schema: item.schema,
filePath,
includeStructure: includeStructure.value,
includeData: includeData.value,
includeObjects: includeObjects.value,
includeCreateDatabase: includeCreateDatabase.value,
dropTableIfExists: dropTableIfExists.value,
omitAutoIncrement: omitAutoIncrement.value,
batchSize: 1000,
};
let currentDatabaseRowsExported = 0;
await runDatabaseExportUntilTerminal(request, (progress) => {
const nextRowsExported = Math.max(0, progress.rowsExported);
batchRowsExported.value += Math.max(0, nextRowsExported - currentDatabaseRowsExported);
currentDatabaseRowsExported = nextRowsExported;
exportProgress.value = {
...progress,
exportId: batchId,
currentObject: `${item.displayName}: ${progress.currentObject || item.displayName}`,
rowsExported: batchRowsExported.value,
};
updateDatabaseExportTask(batchId, {
...progress,
exportId: batchId,
currentObject: item.displayName,
objectIndex: index,
totalObjects: exportPlan.length,
rowsExported: batchRowsExported.value,
});
if (progress.status === "Error") {
exportError.value = progress.error;
isExporting.value = false;
} else if (progress.status === "Cancelled") {
exportCancelled.value = true;
isExporting.value = false;
}
});
await runWithDatabaseBackupSnapshot(
{
connectionId: connectionId.value,
database: item.database,
enabled: includeData.value && (connectionType === "mysql" || connectionType === "postgres"),
},
(snapshotSessionId) =>
runDatabaseExportUntilTerminal(
{
exportId: currentExportId,
connectionId: connectionId.value,
database: item.database,
schema: item.schema,
filePath,
includeStructure: includeStructure.value,
includeData: includeData.value,
includeObjects: includeObjects.value,
includeCreateDatabase: includeCreateDatabase.value,
dropTableIfExists: dropTableIfExists.value,
omitAutoIncrement: omitAutoIncrement.value,
snapshotSessionId,
batchSize: 1000,
},
(progress) => {
const nextRowsExported = Math.max(0, progress.rowsExported);
batchRowsExported.value += Math.max(0, nextRowsExported - currentDatabaseRowsExported);
currentDatabaseRowsExported = nextRowsExported;
exportProgress.value = {
...progress,
exportId: batchId,
currentObject: `${item.displayName}: ${progress.currentObject || item.displayName}`,
rowsExported: batchRowsExported.value,
};
updateDatabaseExportTask(batchId, {
...progress,
exportId: batchId,
currentObject: item.displayName,
objectIndex: index,
totalObjects: exportPlan.length,
rowsExported: batchRowsExported.value,
});
if (progress.status === "Error") {
finishExportTiming();
exportError.value = progress.error;
isExporting.value = false;
} else if (progress.status === "Cancelled") {
finishExportTiming();
exportCancelled.value = true;
isExporting.value = false;
}
},
),
(terminal) => terminal.status === "Done",
);
if (exportError.value || exportCancelled.value) break;
activeDatabaseExportId.value = "";
@@ -439,6 +495,7 @@ async function startAllDatabasesExport() {
if (!exportError.value && !exportCancelled.value) {
exportDone.value = true;
finishExportTiming();
isExporting.value = false;
const finalProgress: api.ExportProgress = {
exportId: batchId,
@@ -466,6 +523,7 @@ async function startAllDatabasesExport() {
status: "Error",
error: exportError.value,
});
finishExportTiming();
isExporting.value = false;
}
}
@@ -474,6 +532,7 @@ async function cancelExport() {
if (exportId.value) {
if (exportAllDatabases.value) {
exportCancelled.value = true;
finishExportTiming();
isExporting.value = false;
if (activeDatabaseExportId.value) {
await api.cancelDatabaseExport(activeDatabaseExportId.value);
@@ -509,6 +568,8 @@ function resetState() {
exportDone.value = false;
exportError.value = null;
exportCancelled.value = false;
exportStartedAt.value = null;
exportFinishedAt.value = null;
exportId.value = "";
batchDatabaseIndex.value = 0;
batchDatabaseTotal.value = 0;
@@ -802,6 +863,9 @@ watch(
<div v-if="exportProgress && !isPreparingExport" class="text-xs text-muted-foreground">
{{ exportAllDatabases ? t("databaseExport.allRowsExported", { count: exportProgress.rowsExported.toLocaleString() }) : t("databaseExport.rowsExported", { current: exportProgress.objectIndex, total: exportProgress.totalObjects, count: exportProgress.rowsExported.toLocaleString() }) }}
</div>
<div v-if="exportElapsedText" class="text-xs text-muted-foreground tabular-nums">
{{ t("exportProgress.elapsed", { duration: exportElapsedText }) }}
</div>
</div>
<!-- Status messages -->
@@ -1,5 +1,5 @@
<script setup lang="ts">
import { computed, ref } from "vue";
import { computed, onBeforeUnmount, onMounted, ref } from "vue";
import { useI18n } from "vue-i18n";
import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogFooter } from "@/components/ui/dialog";
import { Button } from "@/components/ui/button";
@@ -8,11 +8,22 @@ import { useToast } from "@/composables/useToast";
import { isTauriRuntime } from "@/lib/backend/tauriRuntime";
import * as api from "@/lib/backend/api";
import { translateBackendError } from "@/i18n/backend-errors";
import { formatDataTransferDuration } from "@/composables/useExportTracker";
const { t } = useI18n();
const { toast } = useToast();
const open = defineModel<boolean>("open", { default: false });
const isRevealing = ref(false);
const currentTime = ref(Date.now());
let elapsedTimer: ReturnType<typeof setInterval> | undefined;
onMounted(() => {
elapsedTimer = setInterval(() => {
currentTime.value = Date.now();
}, 1000);
});
onBeforeUnmount(() => {
if (elapsedTimer) clearInterval(elapsedTimer);
});
const props = defineProps<{
title: string;
@@ -23,6 +34,8 @@ const props = defineProps<{
status: string;
errorMessage: string | null;
filePath?: string | null;
startedAt?: number;
finishedAt?: number;
disableCancel?: boolean;
canMinimize?: boolean;
}>();
@@ -50,6 +63,10 @@ const rowsText = computed(() => {
}
return t("exportProgress.rowsExported", { count: props.rowsExported.toLocaleString() });
});
const elapsedText = computed(() => {
if (props.startedAt === undefined) return "";
return formatDataTransferDuration((props.finishedAt ?? currentTime.value) - props.startedAt);
});
async function revealExportFile() {
if (!props.filePath || isRevealing.value) return;
@@ -120,6 +137,7 @@ async function revealExportFile() {
<!-- Row count -->
<div class="text-xs text-muted-foreground tabular-nums">
{{ rowsText }}
<span v-if="elapsedText" class="ml-2">{{ t("exportProgress.elapsed", { duration: elapsedText }) }}</span>
</div>
</div>
@@ -1,5 +1,5 @@
<script setup lang="ts">
import { computed, ref } from "vue";
import { computed, onBeforeUnmount, onMounted, ref } from "vue";
import { useI18n } from "vue-i18n";
import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
import { Button } from "@/components/ui/button";
@@ -12,8 +12,19 @@ const { tasks, activeCount, hasActive, clearFinished, cancelTask, removeTask } =
const open = ref(false);
const showAll = ref(false);
const expandedFailureTaskIds = ref<string[]>([]);
const currentTime = ref(Date.now());
const MAX_VISIBLE = 5;
let elapsedTimer: ReturnType<typeof setInterval> | undefined;
onMounted(() => {
elapsedTimer = setInterval(() => {
currentTime.value = Date.now();
}, 1000);
});
onBeforeUnmount(() => {
if (elapsedTimer) clearInterval(elapsedTimer);
});
const reversedTasks = computed(() => {
return [...tasks.value].reverse();
});
@@ -96,6 +107,12 @@ const rowsText = (task: ExportTask) => {
return `${task.rowsExported.toLocaleString()} ${t("exportProgress.rowsShort")}`;
};
const elapsedText = (task: ExportTask) => {
if (task.startedAt === undefined) return "";
const finishedAt = task.finishedAt ?? currentTime.value;
return t("exportProgress.elapsed", { duration: formatDataTransferDuration(finishedAt - task.startedAt) });
};
const statusIcon = (task: ExportTask) => {
if (isActive(task.status)) {
if (task.kind === "database-export") return DatabaseBackup;
@@ -192,6 +209,7 @@ function failureDetailCount(task: ExportTask) {
<div class="min-w-0 text-muted-foreground">
<span class="break-words tabular-nums">{{ rowsText(task) }}</span>
<span v-if="task.kind !== 'data-transfer' && task.startedAt !== undefined" class="ml-1 tabular-nums">{{ elapsedText(task) }}</span>
<span v-if="task.status === 'Error' && task.errorMessage" class="mt-1 block whitespace-normal break-words text-destructive" :title="translateBackendError(t, task.errorMessage)">
{{ translateBackendError(t, task.errorMessage) }}
</span>
@@ -66,8 +66,8 @@ async function mountPopover() {
await nextTick();
}
describe("ExportProgressPopover data transfer duration", () => {
it("shows frozen transfer elapsed time without adding it to other task kinds", async () => {
describe("ExportProgressPopover task duration", () => {
it("shows frozen transfer elapsed time and live duration for table tasks", async () => {
const tracker = useExportTracker();
now = 1_000;
const transfer = tracker.addDataTransferTask("transfer", "users", 1);
@@ -89,7 +89,7 @@ describe("ExportProgressPopover data transfer duration", () => {
await mountPopover();
expect(document.body.textContent).toContain("Elapsed: 1m 5s");
expect(document.body.textContent?.match(/Elapsed:/g)).toHaveLength(1);
expect(document.body.textContent?.match(/Elapsed:/g)).toHaveLength(2);
});
it("expands complete per-table transfer failure details", async () => {