feat(export): split large sql exports into zip parts

This commit is contained in:
КГБ Lieu authored and GitHub committed 2026-09-11 15:41:28 +08:00
1 parent 339c96cc81
commit 16866293a3
30 files changed
+933 -57

No files matched your search

@@ -68,6 +68,10 @@ const includeObjects = ref(true);
const includeCreateDatabase = ref(false);
const dropTableIfExists = ref(false);
const omitAutoIncrement = ref(false);
const splitSqlOutput = ref(false);
const splitSqlPartMaxMb = ref(100);
const MIN_SPLIT_SQL_PART_MB = 1;
const MAX_SPLIT_SQL_PART_MB = 4096;
// `AUTO_INCREMENT` stripping is a MySQL-only DDL transform (backend gates on
// db_type == mysql, which also covers MariaDB / TiDB / OceanBase-MySQL-mode).
const isMysqlFamily = computed(() => store.getConfig(connectionId.value)?.db_type === "mysql");
@@ -161,6 +165,12 @@ function joinExportPath(directory: string, fileName: string): string {
return `${directory.replace(/[\\/]+$/, "")}${separator}${fileName}`;
}
function normalizedSplitSqlPartMaxMb(): number {
const value = Number(splitSqlPartMaxMb.value);
if (!Number.isFinite(value)) return 100;
return Math.min(MAX_SPLIT_SQL_PART_MB, Math.max(MIN_SPLIT_SQL_PART_MB, Math.round(value)));
}
// Lenient exports write per-object failures into the SQL file as `-- ERROR`
// comments and still finish; completion must warn instead of reporting plain
// success (#8184).
@@ -329,8 +339,8 @@ async function startExport() {
const { save } = await import("@tauri-apps/plugin-dialog");
const safeName = sanitizeFileName(database.value || "database");
const path = await save({
defaultPath: `${safeName}.sql`,
filters: [{ name: "SQL", extensions: ["sql"] }],
defaultPath: `${safeName}.${splitSqlOutput.value ? "zip" : "sql"}`,
filters: [{ name: splitSqlOutput.value ? "ZIP" : "SQL", extensions: [splitSqlOutput.value ? "zip" : "sql"] }],
});
if (!path) return;
filePath = path;
@@ -392,6 +402,7 @@ async function startExport() {
omitAutoIncrement: omitAutoIncrement.value,
snapshotSessionId,
batchSize: 1000,
splitMaxMb: splitSqlOutput.value ? normalizedSplitSqlPartMaxMb() : undefined,
};
return runDatabaseExportUntilTerminal(request, (progress) => {
exportProgress.value = { ...progress };
@@ -508,7 +519,7 @@ async function startAllDatabasesExport() {
batchDatabaseIndex.value = index + 1;
const currentExportId = `${batchId}-${index + 1}`;
activeDatabaseExportId.value = currentExportId;
const filePath = isTauriRuntime() ? joinExportPath(directoryPath, `${sanitizeFileName(item.fileStem)}.sql`) : `__web_export_${currentExportId}.sql`;
const filePath = isTauriRuntime() ? joinExportPath(directoryPath, `${sanitizeFileName(item.fileStem)}.${splitSqlOutput.value ? "zip" : "sql"}`) : `__web_export_${currentExportId}.${splitSqlOutput.value ? "zip" : "sql"}`;
let currentDatabaseRowsExported = 0;
const terminal = await runWithDatabaseBackupSnapshot(
@@ -533,6 +544,7 @@ async function startAllDatabasesExport() {
omitAutoIncrement: omitAutoIncrement.value,
snapshotSessionId,
batchSize: 1000,
splitMaxMb: splitSqlOutput.value ? normalizedSplitSqlPartMaxMb() : undefined,
},
(progress) => {
const nextRowsExported = Math.max(0, progress.rowsExported);
@@ -651,6 +663,8 @@ function resetState() {
includeCreateDatabase.value = false;
dropTableIfExists.value = false;
omitAutoIncrement.value = false;
splitSqlOutput.value = false;
splitSqlPartMaxMb.value = 100;
isExporting.value = false;
exportProgress.value = null;
exportDone.value = false;
@@ -938,6 +952,27 @@ watch(
<Square v-else class="w-3.5 h-3.5 text-muted-foreground/40 shrink-0" />
{{ t("databaseExport.includeObjects") }}
</div>
<div class="flex items-center justify-between gap-2 text-xs">
<button type="button" class="flex min-w-0 items-center gap-2 text-left" @click="splitSqlOutput = !splitSqlOutput">
<CheckSquare v-if="splitSqlOutput" class="w-3.5 h-3.5 text-primary shrink-0" />
<Square v-else class="w-3.5 h-3.5 text-muted-foreground/40 shrink-0" />
<span>{{ t("databaseExport.splitSqlOutput") }}</span>
</button>
<Input
v-if="splitSqlOutput"
v-model.number="splitSqlPartMaxMb"
type="number"
inputmode="numeric"
:min="MIN_SPLIT_SQL_PART_MB"
:max="MAX_SPLIT_SQL_PART_MB"
class="h-7 w-24 text-xs"
:aria-label="t('databaseExport.splitSqlPartMaxMb')"
@blur="splitSqlPartMaxMb = normalizedSplitSqlPartMaxMb()"
/>
</div>
<p v-if="splitSqlOutput" class="pl-5 text-[11px] text-muted-foreground">
{{ t("databaseExport.splitSqlOutputDescription", { min: MIN_SPLIT_SQL_PART_MB, max: MAX_SPLIT_SQL_PART_MB }) }}
</p>
</div>
</div>
@@ -3,22 +3,36 @@ import { ref } from "vue";
import { useI18n } from "vue-i18n";
import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogFooter } from "@/components/ui/dialog";
import { Button } from "@/components/ui/button";
import type { SqlInsertMode } from "@/lib/export/sqlInsertMode";
import type { SqlExportOptions, SqlInsertMode } from "@/lib/export/sqlInsertMode";
const { t } = useI18n();
const open = defineModel<boolean>("open", { default: false });
const props = defineProps<{ allowSplit?: boolean }>();
const selected = ref<SqlInsertMode>("batch");
const splitSqlOutput = ref(false);
const splitSqlPartMaxMb = ref(100);
const MIN_SPLIT_SQL_PART_MB = 1;
const MAX_SPLIT_SQL_PART_MB = 4096;
let outcomeEmitted = false;
const emit = defineEmits<{
confirm: [mode: SqlInsertMode];
confirm: [options: SqlExportOptions];
cancel: [];
}>();
function normalizedSplitSqlPartMaxMb(): number {
const value = Number(splitSqlPartMaxMb.value);
if (!Number.isFinite(value)) return 100;
return Math.min(MAX_SPLIT_SQL_PART_MB, Math.max(MIN_SPLIT_SQL_PART_MB, Math.round(value)));
}
function onConfirm() {
outcomeEmitted = true;
open.value = false;
emit("confirm", selected.value);
emit("confirm", {
insertMode: selected.value,
splitMaxMb: props.allowSplit && splitSqlOutput.value ? normalizedSplitSqlPartMaxMb() : undefined,
});
}
function onCancel() {
@@ -55,6 +69,19 @@ function onOpenChange(value: boolean) {
<span class="mt-1 block text-xs text-muted-foreground">{{ t("grid.sqlInsertModeSingleDescription") }}</span>
</span>
</label>
<div v-if="props.allowSplit" class="space-y-2 rounded-md border p-3">
<label class="flex cursor-pointer items-center gap-2 text-sm">
<input v-model="splitSqlOutput" type="checkbox" class="h-4 w-4" data-sql-split-output />
{{ t("databaseExport.splitSqlOutput") }}
</label>
<div v-if="splitSqlOutput" class="flex items-center justify-between gap-3 pl-6">
<span class="text-xs text-muted-foreground">{{ t("databaseExport.splitSqlPartMaxMb") }}</span>
<input v-model.number="splitSqlPartMaxMb" type="number" :min="MIN_SPLIT_SQL_PART_MB" :max="MAX_SPLIT_SQL_PART_MB" class="h-8 w-24 rounded-md border bg-background px-2 text-sm" data-sql-split-max-mb />
</div>
<p v-if="splitSqlOutput" class="pl-6 text-xs text-muted-foreground">
{{ t("databaseExport.splitSqlOutputDescription", { min: MIN_SPLIT_SQL_PART_MB, max: MAX_SPLIT_SQL_PART_MB }) }}
</p>
</div>
</div>
<DialogFooter>
<Button variant="outline" @click="onCancel">{{ t("common.cancel") }}</Button>
@@ -37,13 +37,13 @@ afterEach(() => {
i18n.global.locale.value = "en";
});
async function mountDialog(onConfirm = () => {}, onCancel = () => {}) {
async function mountDialog(onConfirm = () => {}, onCancel = () => {}, allowSplit = false) {
const container = document.createElement("div");
document.body.appendChild(container);
const app = createApp(
defineComponent({
setup() {
return () => h(SqlInsertModeDialog, { open: true, onConfirm, onCancel });
return () => h(SqlInsertModeDialog, { open: true, allowSplit, onConfirm, onCancel });
},
}),
);
@@ -68,7 +68,24 @@ describe("SqlInsertModeDialog", () => {
document.querySelector<HTMLButtonElement>("[data-sql-insert-mode-confirm]")?.click();
expect(onConfirm).toHaveBeenCalledOnce();
expect(onConfirm).toHaveBeenCalledWith("single");
expect(onConfirm).toHaveBeenCalledWith({ insertMode: "single", splitMaxMb: undefined });
});
it("enables split ZIP output only when the caller supports it", async () => {
const onConfirm = vi.fn();
await mountDialog(onConfirm, () => {}, true);
const split = document.querySelector<HTMLInputElement>("input[data-sql-split-output]");
expect(split).not.toBeNull();
split?.click();
await nextTick();
const size = document.querySelector<HTMLInputElement>("input[data-sql-split-max-mb]");
expect(size?.value).toBe("100");
if (size) size.value = "256";
size?.dispatchEvent(new Event("input", { bubbles: true }));
document.querySelector<HTMLButtonElement>("[data-sql-insert-mode-confirm]")?.click();
expect(onConfirm).toHaveBeenCalledWith({ insertMode: "batch", splitMaxMb: 256 });
});
it("renders the batch and single-row explanations", async () => {
@@ -2112,9 +2112,9 @@ async function exportData(row: ObjectBrowserRow, format: "csv" | "json" | "sql")
await exportDataLegacy(row, format);
return;
}
const insertMode = format === "sql" ? await showSqlInsertModeDialog() : undefined;
if (format === "sql" && insertMode === null) return;
await exportTableData(row, format, undefined, "name", true, insertMode ?? "batch");
const sqlExportOptions = format === "sql" ? await showSqlInsertModeDialog({ allowSplit: true }) : undefined;
if (format === "sql" && sqlExportOptions === null) return;
await exportTableData(row, format, undefined, "name", true, sqlExportOptions?.insertMode ?? "batch", sqlExportOptions?.splitMaxMb);
}
function showObjectBrowserXlsxHeaderDialog(hasComments: boolean): Promise<XlsxExportOptions | null> {
@@ -2157,17 +2157,18 @@ async function exportDataXlsx(row: ObjectBrowserRow) {
await exportTableData(row, "xlsx", columnInfos, exportOptions.headerMode, exportOptions.autoFilter);
}
async function exportTableData(row: ObjectBrowserRow, format: "csv" | "xlsx" | "sql", columnInfos?: ColumnInfo[], headerMode: XlsxHeaderMode = "name", autoFilter = true, insertMode: SqlInsertMode = "batch") {
async function exportTableData(row: ObjectBrowserRow, format: "csv" | "xlsx" | "sql", columnInfos?: ColumnInfo[], headerMode: XlsxHeaderMode = "name", autoFilter = true, insertMode: SqlInsertMode = "batch", splitMaxMb?: number) {
const schema = row.schema || selectedSchema.value;
const splitSqlOutput = format === "sql" && splitMaxMb !== undefined;
// Save dialog first
let filePath = "";
const defaultName = `${row.name}.${format}`;
const defaultName = `${row.name}.${splitSqlOutput ? "zip" : format}`;
if (isTauriRuntime()) {
try {
const { save } = await import("@tauri-apps/plugin-dialog");
const filter = format === "csv" ? { name: "CSV", extensions: ["csv"] } : format === "xlsx" ? { name: "Excel", extensions: ["xlsx"] } : { name: "SQL", extensions: ["sql"] };
const filter = format === "csv" ? { name: "CSV", extensions: ["csv"] } : format === "xlsx" ? { name: "Excel", extensions: ["xlsx"] } : splitSqlOutput ? { name: "ZIP", extensions: ["zip"] } : { name: "SQL", extensions: ["sql"] };
const path = await save({
defaultPath: defaultName,
filters: [filter],
@@ -2180,7 +2181,7 @@ async function exportTableData(row: ObjectBrowserRow, format: "csv" | "xlsx" | "
}
} else {
const webExportId = generateDatabaseExportId();
filePath = `__web_export_${webExportId}.${format}`;
filePath = `__web_export_${webExportId}.${splitSqlOutput ? "zip" : format}`;
}
let task: ExportTask | null = null;
@@ -2231,7 +2232,7 @@ async function exportTableData(row: ObjectBrowserRow, format: "csv" | "xlsx" | "
tableName: row.name,
filePath,
format,
...(format === "sql" ? { insertMode } : {}),
...(format === "sql" ? { insertMode, splitMaxMb } : {}),
csvQuoteMode: settingsStore.editorSettings.csvQuoteMode,
columns,
columnComments: format === "xlsx" ? columnComments : undefined,
@@ -30,9 +30,9 @@ describe("ObjectBrowser XLSX export", () => {
const exportData = functionBody("exportData");
const exportTableData = functionBody("exportTableData");
expect(exportData).toContain("await showSqlInsertModeDialog()");
expect(exportData.indexOf("await showSqlInsertModeDialog()")).toBeLessThan(exportData.indexOf("await exportTableData("));
expect(exportTableData).toContain('...(format === "sql" ? { insertMode } : {})');
expect(exportData).toContain("await showSqlInsertModeDialog({ allowSplit: true })");
expect(exportData.indexOf("await showSqlInsertModeDialog({ allowSplit: true })")).toBeLessThan(exportData.indexOf("await exportTableData("));
expect(exportTableData).toContain('...(format === "sql" ? { insertMode, splitMaxMb } : {})');
});
it("falls back to field-name headers when column metadata is unavailable", () => {