feat(export): split large sql exports into zip parts
This commit is contained in:
1 parent
339c96cc81
commit
16866293a3
30 files changed
+933
-57
No files matched your search
@@ -68,6 +68,10 @@ const includeObjects = ref(true);
|
||||
const includeCreateDatabase = ref(false);
|
||||
const dropTableIfExists = ref(false);
|
||||
const omitAutoIncrement = ref(false);
|
||||
const splitSqlOutput = ref(false);
|
||||
const splitSqlPartMaxMb = ref(100);
|
||||
const MIN_SPLIT_SQL_PART_MB = 1;
|
||||
const MAX_SPLIT_SQL_PART_MB = 4096;
|
||||
// `AUTO_INCREMENT` stripping is a MySQL-only DDL transform (backend gates on
|
||||
// db_type == mysql, which also covers MariaDB / TiDB / OceanBase-MySQL-mode).
|
||||
const isMysqlFamily = computed(() => store.getConfig(connectionId.value)?.db_type === "mysql");
|
||||
@@ -161,6 +165,12 @@ function joinExportPath(directory: string, fileName: string): string {
|
||||
return `${directory.replace(/[\\/]+$/, "")}${separator}${fileName}`;
|
||||
}
|
||||
|
||||
function normalizedSplitSqlPartMaxMb(): number {
|
||||
const value = Number(splitSqlPartMaxMb.value);
|
||||
if (!Number.isFinite(value)) return 100;
|
||||
return Math.min(MAX_SPLIT_SQL_PART_MB, Math.max(MIN_SPLIT_SQL_PART_MB, Math.round(value)));
|
||||
}
|
||||
|
||||
// Lenient exports write per-object failures into the SQL file as `-- ERROR`
|
||||
// comments and still finish; completion must warn instead of reporting plain
|
||||
// success (#8184).
|
||||
@@ -329,8 +339,8 @@ async function startExport() {
|
||||
const { save } = await import("@tauri-apps/plugin-dialog");
|
||||
const safeName = sanitizeFileName(database.value || "database");
|
||||
const path = await save({
|
||||
defaultPath: `${safeName}.sql`,
|
||||
filters: [{ name: "SQL", extensions: ["sql"] }],
|
||||
defaultPath: `${safeName}.${splitSqlOutput.value ? "zip" : "sql"}`,
|
||||
filters: [{ name: splitSqlOutput.value ? "ZIP" : "SQL", extensions: [splitSqlOutput.value ? "zip" : "sql"] }],
|
||||
});
|
||||
if (!path) return;
|
||||
filePath = path;
|
||||
@@ -392,6 +402,7 @@ async function startExport() {
|
||||
omitAutoIncrement: omitAutoIncrement.value,
|
||||
snapshotSessionId,
|
||||
batchSize: 1000,
|
||||
splitMaxMb: splitSqlOutput.value ? normalizedSplitSqlPartMaxMb() : undefined,
|
||||
};
|
||||
return runDatabaseExportUntilTerminal(request, (progress) => {
|
||||
exportProgress.value = { ...progress };
|
||||
@@ -508,7 +519,7 @@ async function startAllDatabasesExport() {
|
||||
batchDatabaseIndex.value = index + 1;
|
||||
const currentExportId = `${batchId}-${index + 1}`;
|
||||
activeDatabaseExportId.value = currentExportId;
|
||||
const filePath = isTauriRuntime() ? joinExportPath(directoryPath, `${sanitizeFileName(item.fileStem)}.sql`) : `__web_export_${currentExportId}.sql`;
|
||||
const filePath = isTauriRuntime() ? joinExportPath(directoryPath, `${sanitizeFileName(item.fileStem)}.${splitSqlOutput.value ? "zip" : "sql"}`) : `__web_export_${currentExportId}.${splitSqlOutput.value ? "zip" : "sql"}`;
|
||||
let currentDatabaseRowsExported = 0;
|
||||
|
||||
const terminal = await runWithDatabaseBackupSnapshot(
|
||||
@@ -533,6 +544,7 @@ async function startAllDatabasesExport() {
|
||||
omitAutoIncrement: omitAutoIncrement.value,
|
||||
snapshotSessionId,
|
||||
batchSize: 1000,
|
||||
splitMaxMb: splitSqlOutput.value ? normalizedSplitSqlPartMaxMb() : undefined,
|
||||
},
|
||||
(progress) => {
|
||||
const nextRowsExported = Math.max(0, progress.rowsExported);
|
||||
@@ -651,6 +663,8 @@ function resetState() {
|
||||
includeCreateDatabase.value = false;
|
||||
dropTableIfExists.value = false;
|
||||
omitAutoIncrement.value = false;
|
||||
splitSqlOutput.value = false;
|
||||
splitSqlPartMaxMb.value = 100;
|
||||
isExporting.value = false;
|
||||
exportProgress.value = null;
|
||||
exportDone.value = false;
|
||||
@@ -938,6 +952,27 @@ watch(
|
||||
<Square v-else class="w-3.5 h-3.5 text-muted-foreground/40 shrink-0" />
|
||||
{{ t("databaseExport.includeObjects") }}
|
||||
</div>
|
||||
<div class="flex items-center justify-between gap-2 text-xs">
|
||||
<button type="button" class="flex min-w-0 items-center gap-2 text-left" @click="splitSqlOutput = !splitSqlOutput">
|
||||
<CheckSquare v-if="splitSqlOutput" class="w-3.5 h-3.5 text-primary shrink-0" />
|
||||
<Square v-else class="w-3.5 h-3.5 text-muted-foreground/40 shrink-0" />
|
||||
<span>{{ t("databaseExport.splitSqlOutput") }}</span>
|
||||
</button>
|
||||
<Input
|
||||
v-if="splitSqlOutput"
|
||||
v-model.number="splitSqlPartMaxMb"
|
||||
type="number"
|
||||
inputmode="numeric"
|
||||
:min="MIN_SPLIT_SQL_PART_MB"
|
||||
:max="MAX_SPLIT_SQL_PART_MB"
|
||||
class="h-7 w-24 text-xs"
|
||||
:aria-label="t('databaseExport.splitSqlPartMaxMb')"
|
||||
@blur="splitSqlPartMaxMb = normalizedSplitSqlPartMaxMb()"
|
||||
/>
|
||||
</div>
|
||||
<p v-if="splitSqlOutput" class="pl-5 text-[11px] text-muted-foreground">
|
||||
{{ t("databaseExport.splitSqlOutputDescription", { min: MIN_SPLIT_SQL_PART_MB, max: MAX_SPLIT_SQL_PART_MB }) }}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -3,22 +3,36 @@ import { ref } from "vue";
|
||||
import { useI18n } from "vue-i18n";
|
||||
import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogFooter } from "@/components/ui/dialog";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import type { SqlInsertMode } from "@/lib/export/sqlInsertMode";
|
||||
import type { SqlExportOptions, SqlInsertMode } from "@/lib/export/sqlInsertMode";
|
||||
|
||||
const { t } = useI18n();
|
||||
const open = defineModel<boolean>("open", { default: false });
|
||||
const props = defineProps<{ allowSplit?: boolean }>();
|
||||
const selected = ref<SqlInsertMode>("batch");
|
||||
const splitSqlOutput = ref(false);
|
||||
const splitSqlPartMaxMb = ref(100);
|
||||
const MIN_SPLIT_SQL_PART_MB = 1;
|
||||
const MAX_SPLIT_SQL_PART_MB = 4096;
|
||||
let outcomeEmitted = false;
|
||||
|
||||
const emit = defineEmits<{
|
||||
confirm: [mode: SqlInsertMode];
|
||||
confirm: [options: SqlExportOptions];
|
||||
cancel: [];
|
||||
}>();
|
||||
|
||||
function normalizedSplitSqlPartMaxMb(): number {
|
||||
const value = Number(splitSqlPartMaxMb.value);
|
||||
if (!Number.isFinite(value)) return 100;
|
||||
return Math.min(MAX_SPLIT_SQL_PART_MB, Math.max(MIN_SPLIT_SQL_PART_MB, Math.round(value)));
|
||||
}
|
||||
|
||||
function onConfirm() {
|
||||
outcomeEmitted = true;
|
||||
open.value = false;
|
||||
emit("confirm", selected.value);
|
||||
emit("confirm", {
|
||||
insertMode: selected.value,
|
||||
splitMaxMb: props.allowSplit && splitSqlOutput.value ? normalizedSplitSqlPartMaxMb() : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
function onCancel() {
|
||||
@@ -55,6 +69,19 @@ function onOpenChange(value: boolean) {
|
||||
<span class="mt-1 block text-xs text-muted-foreground">{{ t("grid.sqlInsertModeSingleDescription") }}</span>
|
||||
</span>
|
||||
</label>
|
||||
<div v-if="props.allowSplit" class="space-y-2 rounded-md border p-3">
|
||||
<label class="flex cursor-pointer items-center gap-2 text-sm">
|
||||
<input v-model="splitSqlOutput" type="checkbox" class="h-4 w-4" data-sql-split-output />
|
||||
{{ t("databaseExport.splitSqlOutput") }}
|
||||
</label>
|
||||
<div v-if="splitSqlOutput" class="flex items-center justify-between gap-3 pl-6">
|
||||
<span class="text-xs text-muted-foreground">{{ t("databaseExport.splitSqlPartMaxMb") }}</span>
|
||||
<input v-model.number="splitSqlPartMaxMb" type="number" :min="MIN_SPLIT_SQL_PART_MB" :max="MAX_SPLIT_SQL_PART_MB" class="h-8 w-24 rounded-md border bg-background px-2 text-sm" data-sql-split-max-mb />
|
||||
</div>
|
||||
<p v-if="splitSqlOutput" class="pl-6 text-xs text-muted-foreground">
|
||||
{{ t("databaseExport.splitSqlOutputDescription", { min: MIN_SPLIT_SQL_PART_MB, max: MAX_SPLIT_SQL_PART_MB }) }}
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<DialogFooter>
|
||||
<Button variant="outline" @click="onCancel">{{ t("common.cancel") }}</Button>
|
||||
|
||||
@@ -37,13 +37,13 @@ afterEach(() => {
|
||||
i18n.global.locale.value = "en";
|
||||
});
|
||||
|
||||
async function mountDialog(onConfirm = () => {}, onCancel = () => {}) {
|
||||
async function mountDialog(onConfirm = () => {}, onCancel = () => {}, allowSplit = false) {
|
||||
const container = document.createElement("div");
|
||||
document.body.appendChild(container);
|
||||
const app = createApp(
|
||||
defineComponent({
|
||||
setup() {
|
||||
return () => h(SqlInsertModeDialog, { open: true, onConfirm, onCancel });
|
||||
return () => h(SqlInsertModeDialog, { open: true, allowSplit, onConfirm, onCancel });
|
||||
},
|
||||
}),
|
||||
);
|
||||
@@ -68,7 +68,24 @@ describe("SqlInsertModeDialog", () => {
|
||||
document.querySelector<HTMLButtonElement>("[data-sql-insert-mode-confirm]")?.click();
|
||||
|
||||
expect(onConfirm).toHaveBeenCalledOnce();
|
||||
expect(onConfirm).toHaveBeenCalledWith("single");
|
||||
expect(onConfirm).toHaveBeenCalledWith({ insertMode: "single", splitMaxMb: undefined });
|
||||
});
|
||||
|
||||
it("enables split ZIP output only when the caller supports it", async () => {
|
||||
const onConfirm = vi.fn();
|
||||
await mountDialog(onConfirm, () => {}, true);
|
||||
|
||||
const split = document.querySelector<HTMLInputElement>("input[data-sql-split-output]");
|
||||
expect(split).not.toBeNull();
|
||||
split?.click();
|
||||
await nextTick();
|
||||
const size = document.querySelector<HTMLInputElement>("input[data-sql-split-max-mb]");
|
||||
expect(size?.value).toBe("100");
|
||||
if (size) size.value = "256";
|
||||
size?.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
document.querySelector<HTMLButtonElement>("[data-sql-insert-mode-confirm]")?.click();
|
||||
|
||||
expect(onConfirm).toHaveBeenCalledWith({ insertMode: "batch", splitMaxMb: 256 });
|
||||
});
|
||||
|
||||
it("renders the batch and single-row explanations", async () => {
|
||||
|
||||
@@ -2112,9 +2112,9 @@ async function exportData(row: ObjectBrowserRow, format: "csv" | "json" | "sql")
|
||||
await exportDataLegacy(row, format);
|
||||
return;
|
||||
}
|
||||
const insertMode = format === "sql" ? await showSqlInsertModeDialog() : undefined;
|
||||
if (format === "sql" && insertMode === null) return;
|
||||
await exportTableData(row, format, undefined, "name", true, insertMode ?? "batch");
|
||||
const sqlExportOptions = format === "sql" ? await showSqlInsertModeDialog({ allowSplit: true }) : undefined;
|
||||
if (format === "sql" && sqlExportOptions === null) return;
|
||||
await exportTableData(row, format, undefined, "name", true, sqlExportOptions?.insertMode ?? "batch", sqlExportOptions?.splitMaxMb);
|
||||
}
|
||||
|
||||
function showObjectBrowserXlsxHeaderDialog(hasComments: boolean): Promise<XlsxExportOptions | null> {
|
||||
@@ -2157,17 +2157,18 @@ async function exportDataXlsx(row: ObjectBrowserRow) {
|
||||
await exportTableData(row, "xlsx", columnInfos, exportOptions.headerMode, exportOptions.autoFilter);
|
||||
}
|
||||
|
||||
async function exportTableData(row: ObjectBrowserRow, format: "csv" | "xlsx" | "sql", columnInfos?: ColumnInfo[], headerMode: XlsxHeaderMode = "name", autoFilter = true, insertMode: SqlInsertMode = "batch") {
|
||||
async function exportTableData(row: ObjectBrowserRow, format: "csv" | "xlsx" | "sql", columnInfos?: ColumnInfo[], headerMode: XlsxHeaderMode = "name", autoFilter = true, insertMode: SqlInsertMode = "batch", splitMaxMb?: number) {
|
||||
const schema = row.schema || selectedSchema.value;
|
||||
const splitSqlOutput = format === "sql" && splitMaxMb !== undefined;
|
||||
|
||||
// Save dialog first
|
||||
let filePath = "";
|
||||
const defaultName = `${row.name}.${format}`;
|
||||
const defaultName = `${row.name}.${splitSqlOutput ? "zip" : format}`;
|
||||
|
||||
if (isTauriRuntime()) {
|
||||
try {
|
||||
const { save } = await import("@tauri-apps/plugin-dialog");
|
||||
const filter = format === "csv" ? { name: "CSV", extensions: ["csv"] } : format === "xlsx" ? { name: "Excel", extensions: ["xlsx"] } : { name: "SQL", extensions: ["sql"] };
|
||||
const filter = format === "csv" ? { name: "CSV", extensions: ["csv"] } : format === "xlsx" ? { name: "Excel", extensions: ["xlsx"] } : splitSqlOutput ? { name: "ZIP", extensions: ["zip"] } : { name: "SQL", extensions: ["sql"] };
|
||||
const path = await save({
|
||||
defaultPath: defaultName,
|
||||
filters: [filter],
|
||||
@@ -2180,7 +2181,7 @@ async function exportTableData(row: ObjectBrowserRow, format: "csv" | "xlsx" | "
|
||||
}
|
||||
} else {
|
||||
const webExportId = generateDatabaseExportId();
|
||||
filePath = `__web_export_${webExportId}.${format}`;
|
||||
filePath = `__web_export_${webExportId}.${splitSqlOutput ? "zip" : format}`;
|
||||
}
|
||||
|
||||
let task: ExportTask | null = null;
|
||||
@@ -2231,7 +2232,7 @@ async function exportTableData(row: ObjectBrowserRow, format: "csv" | "xlsx" | "
|
||||
tableName: row.name,
|
||||
filePath,
|
||||
format,
|
||||
...(format === "sql" ? { insertMode } : {}),
|
||||
...(format === "sql" ? { insertMode, splitMaxMb } : {}),
|
||||
csvQuoteMode: settingsStore.editorSettings.csvQuoteMode,
|
||||
columns,
|
||||
columnComments: format === "xlsx" ? columnComments : undefined,
|
||||
|
||||
@@ -30,9 +30,9 @@ describe("ObjectBrowser XLSX export", () => {
|
||||
const exportData = functionBody("exportData");
|
||||
const exportTableData = functionBody("exportTableData");
|
||||
|
||||
expect(exportData).toContain("await showSqlInsertModeDialog()");
|
||||
expect(exportData.indexOf("await showSqlInsertModeDialog()")).toBeLessThan(exportData.indexOf("await exportTableData("));
|
||||
expect(exportTableData).toContain('...(format === "sql" ? { insertMode } : {})');
|
||||
expect(exportData).toContain("await showSqlInsertModeDialog({ allowSplit: true })");
|
||||
expect(exportData.indexOf("await showSqlInsertModeDialog({ allowSplit: true })")).toBeLessThan(exportData.indexOf("await exportTableData("));
|
||||
expect(exportTableData).toContain('...(format === "sql" ? { insertMode, splitMaxMb } : {})');
|
||||
});
|
||||
|
||||
it("falls back to field-name headers when column metadata is unavailable", () => {
|
||||
|
||||
Reference in new issue
Block a user