From 16866293a39e1b0d7f72ce11b5d67f25e4345e84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=93=D0=91=20Lieu?= <60683892+hanhvs@users.noreply.github.com> Date: Fri, 11 Sep 2026 14:41:28 +0700 Subject: [PATCH] feat(export): split large sql exports into zip parts --- .../export/DatabaseExportDialog.vue | 41 ++- .../components/export/SqlInsertModeDialog.vue | 33 +- .../__tests__/SqlInsertModeDialog.spec.ts | 23 +- .../src/components/objects/ObjectBrowser.vue | 17 +- .../__tests__/ObjectBrowserExport.spec.ts | 6 +- .../src/composables/useDataGridExport.ts | 26 +- apps/desktop/src/i18n/locales/az.ts | 3 + apps/desktop/src/i18n/locales/en.ts | 3 + apps/desktop/src/i18n/locales/es.ts | 3 + apps/desktop/src/i18n/locales/it.ts | 3 + apps/desktop/src/i18n/locales/ja.ts | 3 + apps/desktop/src/i18n/locales/ko.ts | 3 + apps/desktop/src/i18n/locales/pt-BR.ts | 3 + apps/desktop/src/i18n/locales/tr.ts | 3 + apps/desktop/src/i18n/locales/zh-CN.ts | 3 + apps/desktop/src/i18n/locales/zh-TW.ts | 3 + apps/desktop/src/lib/backend/tauri.ts | 2 + apps/desktop/src/lib/export/sqlInsertMode.ts | 14 +- crates/dbx-core/src/database_export.rs | 218 +++++++++++++- crates/dbx-core/src/export_split_zip.rs | 281 ++++++++++++++++++ crates/dbx-core/src/lib.rs | 1 + crates/dbx-core/src/table_export.rs | 265 ++++++++++++++++- .../tests/database_export_partition_ddl.rs | 1 + .../tests/database_export_prefetch.rs | 1 + .../dbx-core/tests/live_manual_transaction.rs | 3 + .../tests/live_mysql_database_export.rs | 5 + .../tests/live_mysql_export_table_order.rs | 1 + .../tests/live_postgres_all_schema_export.rs | 1 + crates/dbx-web/src/routes/database_export.rs | 17 +- crates/dbx-web/src/routes/table_export.rs | 4 +- 30 files changed, 933 insertions(+), 57 deletions(-) create mode 100644 crates/dbx-core/src/export_split_zip.rs diff --git a/apps/desktop/src/components/export/DatabaseExportDialog.vue b/apps/desktop/src/components/export/DatabaseExportDialog.vue index a3e413f4ce..bfce98ebb6 100644 --- a/apps/desktop/src/components/export/DatabaseExportDialog.vue +++ b/apps/desktop/src/components/export/DatabaseExportDialog.vue @@ -68,6 +68,10 @@ const includeObjects = ref(true); const includeCreateDatabase = ref(false); const dropTableIfExists = ref(false); const omitAutoIncrement = ref(false); +const splitSqlOutput = ref(false); +const splitSqlPartMaxMb = ref(100); +const MIN_SPLIT_SQL_PART_MB = 1; +const MAX_SPLIT_SQL_PART_MB = 4096; // `AUTO_INCREMENT` stripping is a MySQL-only DDL transform (backend gates on // db_type == mysql, which also covers MariaDB / TiDB / OceanBase-MySQL-mode). const isMysqlFamily = computed(() => store.getConfig(connectionId.value)?.db_type === "mysql"); @@ -161,6 +165,12 @@ function joinExportPath(directory: string, fileName: string): string { return `${directory.replace(/[\\/]+$/, "")}${separator}${fileName}`; } +function normalizedSplitSqlPartMaxMb(): number { + const value = Number(splitSqlPartMaxMb.value); + if (!Number.isFinite(value)) return 100; + return Math.min(MAX_SPLIT_SQL_PART_MB, Math.max(MIN_SPLIT_SQL_PART_MB, Math.round(value))); +} + // Lenient exports write per-object failures into the SQL file as `-- ERROR` // comments and still finish; completion must warn instead of reporting plain // success (#8184). @@ -329,8 +339,8 @@ async function startExport() { const { save } = await import("@tauri-apps/plugin-dialog"); const safeName = sanitizeFileName(database.value || "database"); const path = await save({ - defaultPath: `${safeName}.sql`, - filters: [{ name: "SQL", extensions: ["sql"] }], + defaultPath: `${safeName}.${splitSqlOutput.value ? "zip" : "sql"}`, + filters: [{ name: splitSqlOutput.value ? "ZIP" : "SQL", extensions: [splitSqlOutput.value ? "zip" : "sql"] }], }); if (!path) return; filePath = path; @@ -392,6 +402,7 @@ async function startExport() { omitAutoIncrement: omitAutoIncrement.value, snapshotSessionId, batchSize: 1000, + splitMaxMb: splitSqlOutput.value ? normalizedSplitSqlPartMaxMb() : undefined, }; return runDatabaseExportUntilTerminal(request, (progress) => { exportProgress.value = { ...progress }; @@ -508,7 +519,7 @@ async function startAllDatabasesExport() { batchDatabaseIndex.value = index + 1; const currentExportId = `${batchId}-${index + 1}`; activeDatabaseExportId.value = currentExportId; - const filePath = isTauriRuntime() ? joinExportPath(directoryPath, `${sanitizeFileName(item.fileStem)}.sql`) : `__web_export_${currentExportId}.sql`; + const filePath = isTauriRuntime() ? joinExportPath(directoryPath, `${sanitizeFileName(item.fileStem)}.${splitSqlOutput.value ? "zip" : "sql"}`) : `__web_export_${currentExportId}.${splitSqlOutput.value ? "zip" : "sql"}`; let currentDatabaseRowsExported = 0; const terminal = await runWithDatabaseBackupSnapshot( @@ -533,6 +544,7 @@ async function startAllDatabasesExport() { omitAutoIncrement: omitAutoIncrement.value, snapshotSessionId, batchSize: 1000, + splitMaxMb: splitSqlOutput.value ? normalizedSplitSqlPartMaxMb() : undefined, }, (progress) => { const nextRowsExported = Math.max(0, progress.rowsExported); @@ -651,6 +663,8 @@ function resetState() { includeCreateDatabase.value = false; dropTableIfExists.value = false; omitAutoIncrement.value = false; + splitSqlOutput.value = false; + splitSqlPartMaxMb.value = 100; isExporting.value = false; exportProgress.value = null; exportDone.value = false; @@ -938,6 +952,27 @@ watch( {{ t("databaseExport.includeObjects") }} +
+ + +
+

+ {{ t("databaseExport.splitSqlOutputDescription", { min: MIN_SPLIT_SQL_PART_MB, max: MAX_SPLIT_SQL_PART_MB }) }} +

diff --git a/apps/desktop/src/components/export/SqlInsertModeDialog.vue b/apps/desktop/src/components/export/SqlInsertModeDialog.vue index e2117953b0..29dd483ed7 100644 --- a/apps/desktop/src/components/export/SqlInsertModeDialog.vue +++ b/apps/desktop/src/components/export/SqlInsertModeDialog.vue @@ -3,22 +3,36 @@ import { ref } from "vue"; import { useI18n } from "vue-i18n"; import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogFooter } from "@/components/ui/dialog"; import { Button } from "@/components/ui/button"; -import type { SqlInsertMode } from "@/lib/export/sqlInsertMode"; +import type { SqlExportOptions, SqlInsertMode } from "@/lib/export/sqlInsertMode"; const { t } = useI18n(); const open = defineModel("open", { default: false }); +const props = defineProps<{ allowSplit?: boolean }>(); const selected = ref("batch"); +const splitSqlOutput = ref(false); +const splitSqlPartMaxMb = ref(100); +const MIN_SPLIT_SQL_PART_MB = 1; +const MAX_SPLIT_SQL_PART_MB = 4096; let outcomeEmitted = false; const emit = defineEmits<{ - confirm: [mode: SqlInsertMode]; + confirm: [options: SqlExportOptions]; cancel: []; }>(); +function normalizedSplitSqlPartMaxMb(): number { + const value = Number(splitSqlPartMaxMb.value); + if (!Number.isFinite(value)) return 100; + return Math.min(MAX_SPLIT_SQL_PART_MB, Math.max(MIN_SPLIT_SQL_PART_MB, Math.round(value))); +} + function onConfirm() { outcomeEmitted = true; open.value = false; - emit("confirm", selected.value); + emit("confirm", { + insertMode: selected.value, + splitMaxMb: props.allowSplit && splitSqlOutput.value ? normalizedSplitSqlPartMaxMb() : undefined, + }); } function onCancel() { @@ -55,6 +69,19 @@ function onOpenChange(value: boolean) { {{ t("grid.sqlInsertModeSingleDescription") }} +
+ +
+ {{ t("databaseExport.splitSqlPartMaxMb") }} + +
+

+ {{ t("databaseExport.splitSqlOutputDescription", { min: MIN_SPLIT_SQL_PART_MB, max: MAX_SPLIT_SQL_PART_MB }) }} +

+
diff --git a/apps/desktop/src/components/export/__tests__/SqlInsertModeDialog.spec.ts b/apps/desktop/src/components/export/__tests__/SqlInsertModeDialog.spec.ts index 2626200dff..d8e6c53e30 100644 --- a/apps/desktop/src/components/export/__tests__/SqlInsertModeDialog.spec.ts +++ b/apps/desktop/src/components/export/__tests__/SqlInsertModeDialog.spec.ts @@ -37,13 +37,13 @@ afterEach(() => { i18n.global.locale.value = "en"; }); -async function mountDialog(onConfirm = () => {}, onCancel = () => {}) { +async function mountDialog(onConfirm = () => {}, onCancel = () => {}, allowSplit = false) { const container = document.createElement("div"); document.body.appendChild(container); const app = createApp( defineComponent({ setup() { - return () => h(SqlInsertModeDialog, { open: true, onConfirm, onCancel }); + return () => h(SqlInsertModeDialog, { open: true, allowSplit, onConfirm, onCancel }); }, }), ); @@ -68,7 +68,24 @@ describe("SqlInsertModeDialog", () => { document.querySelector("[data-sql-insert-mode-confirm]")?.click(); expect(onConfirm).toHaveBeenCalledOnce(); - expect(onConfirm).toHaveBeenCalledWith("single"); + expect(onConfirm).toHaveBeenCalledWith({ insertMode: "single", splitMaxMb: undefined }); + }); + + it("enables split ZIP output only when the caller supports it", async () => { + const onConfirm = vi.fn(); + await mountDialog(onConfirm, () => {}, true); + + const split = document.querySelector("input[data-sql-split-output]"); + expect(split).not.toBeNull(); + split?.click(); + await nextTick(); + const size = document.querySelector("input[data-sql-split-max-mb]"); + expect(size?.value).toBe("100"); + if (size) size.value = "256"; + size?.dispatchEvent(new Event("input", { bubbles: true })); + document.querySelector("[data-sql-insert-mode-confirm]")?.click(); + + expect(onConfirm).toHaveBeenCalledWith({ insertMode: "batch", splitMaxMb: 256 }); }); it("renders the batch and single-row explanations", async () => { diff --git a/apps/desktop/src/components/objects/ObjectBrowser.vue b/apps/desktop/src/components/objects/ObjectBrowser.vue index d6ea061774..fe56730348 100644 --- a/apps/desktop/src/components/objects/ObjectBrowser.vue +++ b/apps/desktop/src/components/objects/ObjectBrowser.vue @@ -2112,9 +2112,9 @@ async function exportData(row: ObjectBrowserRow, format: "csv" | "json" | "sql") await exportDataLegacy(row, format); return; } - const insertMode = format === "sql" ? await showSqlInsertModeDialog() : undefined; - if (format === "sql" && insertMode === null) return; - await exportTableData(row, format, undefined, "name", true, insertMode ?? "batch"); + const sqlExportOptions = format === "sql" ? await showSqlInsertModeDialog({ allowSplit: true }) : undefined; + if (format === "sql" && sqlExportOptions === null) return; + await exportTableData(row, format, undefined, "name", true, sqlExportOptions?.insertMode ?? "batch", sqlExportOptions?.splitMaxMb); } function showObjectBrowserXlsxHeaderDialog(hasComments: boolean): Promise { @@ -2157,17 +2157,18 @@ async function exportDataXlsx(row: ObjectBrowserRow) { await exportTableData(row, "xlsx", columnInfos, exportOptions.headerMode, exportOptions.autoFilter); } -async function exportTableData(row: ObjectBrowserRow, format: "csv" | "xlsx" | "sql", columnInfos?: ColumnInfo[], headerMode: XlsxHeaderMode = "name", autoFilter = true, insertMode: SqlInsertMode = "batch") { +async function exportTableData(row: ObjectBrowserRow, format: "csv" | "xlsx" | "sql", columnInfos?: ColumnInfo[], headerMode: XlsxHeaderMode = "name", autoFilter = true, insertMode: SqlInsertMode = "batch", splitMaxMb?: number) { const schema = row.schema || selectedSchema.value; + const splitSqlOutput = format === "sql" && splitMaxMb !== undefined; // Save dialog first let filePath = ""; - const defaultName = `${row.name}.${format}`; + const defaultName = `${row.name}.${splitSqlOutput ? "zip" : format}`; if (isTauriRuntime()) { try { const { save } = await import("@tauri-apps/plugin-dialog"); - const filter = format === "csv" ? { name: "CSV", extensions: ["csv"] } : format === "xlsx" ? { name: "Excel", extensions: ["xlsx"] } : { name: "SQL", extensions: ["sql"] }; + const filter = format === "csv" ? { name: "CSV", extensions: ["csv"] } : format === "xlsx" ? { name: "Excel", extensions: ["xlsx"] } : splitSqlOutput ? { name: "ZIP", extensions: ["zip"] } : { name: "SQL", extensions: ["sql"] }; const path = await save({ defaultPath: defaultName, filters: [filter], @@ -2180,7 +2181,7 @@ async function exportTableData(row: ObjectBrowserRow, format: "csv" | "xlsx" | " } } else { const webExportId = generateDatabaseExportId(); - filePath = `__web_export_${webExportId}.${format}`; + filePath = `__web_export_${webExportId}.${splitSqlOutput ? "zip" : format}`; } let task: ExportTask | null = null; @@ -2231,7 +2232,7 @@ async function exportTableData(row: ObjectBrowserRow, format: "csv" | "xlsx" | " tableName: row.name, filePath, format, - ...(format === "sql" ? { insertMode } : {}), + ...(format === "sql" ? { insertMode, splitMaxMb } : {}), csvQuoteMode: settingsStore.editorSettings.csvQuoteMode, columns, columnComments: format === "xlsx" ? columnComments : undefined, diff --git a/apps/desktop/src/components/objects/__tests__/ObjectBrowserExport.spec.ts b/apps/desktop/src/components/objects/__tests__/ObjectBrowserExport.spec.ts index 5734256f5f..e578c96286 100644 --- a/apps/desktop/src/components/objects/__tests__/ObjectBrowserExport.spec.ts +++ b/apps/desktop/src/components/objects/__tests__/ObjectBrowserExport.spec.ts @@ -30,9 +30,9 @@ describe("ObjectBrowser XLSX export", () => { const exportData = functionBody("exportData"); const exportTableData = functionBody("exportTableData"); - expect(exportData).toContain("await showSqlInsertModeDialog()"); - expect(exportData.indexOf("await showSqlInsertModeDialog()")).toBeLessThan(exportData.indexOf("await exportTableData(")); - expect(exportTableData).toContain('...(format === "sql" ? { insertMode } : {})'); + expect(exportData).toContain("await showSqlInsertModeDialog({ allowSplit: true })"); + expect(exportData.indexOf("await showSqlInsertModeDialog({ allowSplit: true })")).toBeLessThan(exportData.indexOf("await exportTableData(")); + expect(exportTableData).toContain('...(format === "sql" ? { insertMode, splitMaxMb } : {})'); }); it("falls back to field-name headers when column metadata is unavailable", () => { diff --git a/apps/desktop/src/composables/useDataGridExport.ts b/apps/desktop/src/composables/useDataGridExport.ts index 711240b71b..b6eb87cc5a 100644 --- a/apps/desktop/src/composables/useDataGridExport.ts +++ b/apps/desktop/src/composables/useDataGridExport.ts @@ -15,7 +15,7 @@ import { copyToClipboard } from "@/lib/common/clipboard"; import { clearDataGridClipboardCopy, rememberDataGridClipboardCopy } from "@/lib/dataGrid/dataGridClipboard"; import { buildDataGridCopyInsertStatement, type DataGridCopyInsertMode, type DataGridTableMeta } from "@/lib/dataGrid/dataGridSql"; import { formatSqlInsert, formatTsv } from "@/lib/export/exportFormats"; -import { showSqlInsertModeDialog, type SqlInsertMode } from "@/lib/export/sqlInsertMode"; +import { showSqlInsertModeDialog, type SqlExportOptions, type SqlInsertMode } from "@/lib/export/sqlInsertMode"; import { summarizeExportRows } from "@/lib/export/exportDiagnostics"; import { appendDebugLog, appendNativeProcessMemoryLog, getBrowserMemorySnapshot, isDebugLoggingEnabled } from "@/lib/backend/debugLog"; import { uuid } from "@/lib/common/utils"; @@ -1164,7 +1164,7 @@ export function useDataGridExport(options: UseDataGridExportOptions) { await exportAllResultsXlsxResult(true); } - async function exportFullTableDataViaBackend(format: "csv" | "xlsx" | "json" | "markdown" | "sql" | "txt", rowIds?: number[], headerMode: XlsxHeaderMode = "name", autoFilter = true, insertMode?: SqlInsertMode): Promise { + async function exportFullTableDataViaBackend(format: "csv" | "xlsx" | "json" | "markdown" | "sql" | "txt", rowIds?: number[], headerMode: XlsxHeaderMode = "name", autoFilter = true, sqlExportOptions?: SqlExportOptions): Promise { const meta = tableMeta.value; // The backend table exporter currently builds two-part table names. External // Doris/StarRocks catalogs need the data-tab paginator's three-part SQL. @@ -1173,8 +1173,9 @@ export function useDataGridExport(options: UseDataGridExportOptions) { } const fmt = FORMAT_META[format]; - const extension = fmt?.ext ?? format; - const filterName = fmt?.label ?? format.toUpperCase(); + const splitSqlOutput = format === "sql" && sqlExportOptions?.splitMaxMb !== undefined; + const extension = splitSqlOutput ? "zip" : (fmt?.ext ?? format); + const filterName = splitSqlOutput ? "ZIP" : (fmt?.label ?? format.toUpperCase()); let outputPath = exportFileName(meta.tableName || "export", extension, { preferFallback: true }); if (isTauriRuntime()) { const { save } = await import("@tauri-apps/plugin-dialog"); @@ -1223,7 +1224,7 @@ export function useDataGridExport(options: UseDataGridExportOptions) { tableName: meta.tableName, filePath: outputPath, format, - ...(format === "sql" && insertMode ? { insertMode } : {}), + ...(format === "sql" && sqlExportOptions ? { insertMode: sqlExportOptions.insertMode, splitMaxMb: sqlExportOptions.splitMaxMb } : {}), csvQuoteMode: editorSettings.csvQuoteMode, columns: columns.value, columnTypes: columnTypes.value, @@ -1409,16 +1410,18 @@ export function useDataGridExport(options: UseDataGridExportOptions) { }, true, ); - const insertMode = await showSqlInsertModeDialog(); - if (insertMode === null) { + const selectedSqlExportOptions = (await showSqlInsertModeDialog({ allowSplit: rowIds === undefined && context.value === "table-data" })) as SqlExportOptions | SqlInsertMode | null; + if (selectedSqlExportOptions === null) { logExportStage("cancelled", { stage: "insert-mode-dialog" }); return; } - logExportStage("mode-selected", { insertMode }); + const sqlExportOptions = typeof selectedSqlExportOptions === "string" ? { insertMode: selectedSqlExportOptions } : selectedSqlExportOptions; + const insertMode = sqlExportOptions.insertMode; + logExportStage("mode-selected", { insertMode, splitMaxMb: sqlExportOptions.splitMaxMb }); try { // Step 1: table-data context — existing backend table export logExportStage("backend-export-start"); - const handledByBackend = await exportFullTableDataViaBackend("sql", rowIds, "name", true, insertMode); + const handledByBackend = await exportFullTableDataViaBackend("sql", rowIds, "name", true, sqlExportOptions); logExportStage("backend-export-finished", { handledByBackend }); if (handledByBackend) { logExportStage("done", { path: "backend" }); @@ -1490,8 +1493,9 @@ export function useDataGridExport(options: UseDataGridExportOptions) { async function exportCurrentPageSql() { await runExclusiveExport(async () => { - const insertMode = await showSqlInsertModeDialog(); - if (insertMode === null) return; + const selectedSqlExportOptions = (await showSqlInsertModeDialog()) as SqlExportOptions | SqlInsertMode | null; + if (selectedSqlExportOptions === null) return; + const insertMode = typeof selectedSqlExportOptions === "string" ? selectedSqlExportOptions : selectedSqlExportOptions.insertMode; try { const result = await resultToExport(undefined, undefined, false, false); const exportData = sqlInsertExportData(result); diff --git a/apps/desktop/src/i18n/locales/az.ts b/apps/desktop/src/i18n/locales/az.ts index f04cc14eb2..08db06c5e1 100644 --- a/apps/desktop/src/i18n/locales/az.ts +++ b/apps/desktop/src/i18n/locales/az.ts @@ -7881,6 +7881,9 @@ export default withEnglishFallback({ omitAutoIncrement: "AUTO_INCREMENT parametrini daxil etmə (yeni quraşdırma skriptləri üçün)", includeData: "Cədvəl məlumatları (INSERT)", includeObjects: "Görünüşlər / prosedurlar / funksiyalar / ardıcıllıqlar", + splitSqlOutput: "SQL ixracını ZIP arxivinə böl", + splitSqlPartMaxMb: "Hər SQL hissəsi üçün maksimum ölçü (MB)", + splitSqlOutputDescription: "Nömrələnmiş SQL hissələri və manifestdən ibarət ZIP arxivi yaradır. Hissələr tam SQL ifadələri arasında bölünür ({min}–{max} MB).", databaseSelection: "Verilənlər bazaları", selectedDatabases: "{selected}/{total} seçilib", selectAllDatabases: "Hamısını seç", diff --git a/apps/desktop/src/i18n/locales/en.ts b/apps/desktop/src/i18n/locales/en.ts index ddcfa48f3b..5b8d69856f 100644 --- a/apps/desktop/src/i18n/locales/en.ts +++ b/apps/desktop/src/i18n/locales/en.ts @@ -7879,6 +7879,9 @@ export default { omitAutoIncrement: "Omit AUTO_INCREMENT (for fresh-install scripts)", includeData: "Table data (INSERT)", includeObjects: "Views / Procedures / Functions / Sequences", + splitSqlOutput: "Split SQL export into a ZIP archive", + splitSqlPartMaxMb: "Maximum size per SQL part (MB)", + splitSqlOutputDescription: "Creates a ZIP archive containing numbered SQL parts and a manifest. Parts are split between complete SQL statements ({min}–{max} MB).", databaseSelection: "Databases", selectedDatabases: "{selected}/{total} selected", selectAllDatabases: "Select all", diff --git a/apps/desktop/src/i18n/locales/es.ts b/apps/desktop/src/i18n/locales/es.ts index 9268e78984..5cad5c7f7d 100644 --- a/apps/desktop/src/i18n/locales/es.ts +++ b/apps/desktop/src/i18n/locales/es.ts @@ -7441,6 +7441,9 @@ export default withEnglishFallback({ omitAutoIncrement: "Omitir AUTO_INCREMENT (para scripts de instalación nueva)", includeData: "Datos de tablas (INSERT)", includeObjects: "Vistas / Procedimientos / Funciones / Secuencias", + splitSqlOutput: "Dividir la exportación SQL en un archivo ZIP", + splitSqlPartMaxMb: "Tamaño máximo por parte SQL (MB)", + splitSqlOutputDescription: "Crea un archivo ZIP con partes SQL numeradas y un manifiesto. Las partes se dividen entre sentencias SQL completas ({min}–{max} MB).", databaseSelection: "Bases de datos", selectedDatabases: "{selected}/{total} seleccionadas", selectAllDatabases: "Seleccionar todo", diff --git a/apps/desktop/src/i18n/locales/it.ts b/apps/desktop/src/i18n/locales/it.ts index ef15cb233b..b25aaa74c6 100644 --- a/apps/desktop/src/i18n/locales/it.ts +++ b/apps/desktop/src/i18n/locales/it.ts @@ -7442,6 +7442,9 @@ export default withEnglishFallback({ omitAutoIncrement: "Ometti AUTO_INCREMENT (per script di installazione pulita)", includeData: "Dati tabella (INSERT)", includeObjects: "Viste / Procedure / Funzioni / Sequenze", + splitSqlOutput: "Dividere l'esportazione SQL in un archivio ZIP", + splitSqlPartMaxMb: "Dimensione massima per parte SQL (MB)", + splitSqlOutputDescription: "Crea un archivio ZIP con parti SQL numerate e un manifest. Le parti vengono divise tra istruzioni SQL complete ({min}–{max} MB).", databaseSelection: "Database", selectedDatabases: "{selected}/{total} selezionati", selectAllDatabases: "Seleziona tutti", diff --git a/apps/desktop/src/i18n/locales/ja.ts b/apps/desktop/src/i18n/locales/ja.ts index 39fcb4e61d..a9467478f0 100644 --- a/apps/desktop/src/i18n/locales/ja.ts +++ b/apps/desktop/src/i18n/locales/ja.ts @@ -7494,6 +7494,9 @@ export default withEnglishFallback({ omitAutoIncrement: "AUTO_INCREMENT を省略（新規インストール用スクリプト向け）", includeData: "テーブルデータ (INSERT)", includeObjects: "ビュー / プロシージャ / 関数 / シーケンス", + splitSqlOutput: "SQLエクスポートをZIPアーカイブに分割", + splitSqlPartMaxMb: "SQLパートあたりの最大サイズ（MB）", + splitSqlOutputDescription: "番号付きSQLパートとマニフェストを含むZIPアーカイブを作成します。パートは完全なSQL文の間で分割されます（{min}～{max} MB）。", databaseSelection: "データベース", selectedDatabases: "{selected}/{total}件選択中", selectAllDatabases: "すべて選択", diff --git a/apps/desktop/src/i18n/locales/ko.ts b/apps/desktop/src/i18n/locales/ko.ts index 48b838a4bf..3b54734777 100644 --- a/apps/desktop/src/i18n/locales/ko.ts +++ b/apps/desktop/src/i18n/locales/ko.ts @@ -7160,6 +7160,9 @@ export default withEnglishFallback({ omitAutoIncrement: "AUTO_INCREMENT 생략 (초기 설치 스크립트용)", includeData: "테이블 데이터 (INSERT)", includeObjects: "뷰 / 프로시저 / 함수 / 시퀀스", + splitSqlOutput: "SQL 내보내기를 ZIP 압축 파일로 분할", + splitSqlPartMaxMb: "SQL 파트당 최대 크기(MB)", + splitSqlOutputDescription: "번호가 지정된 SQL 파트와 매니페스트가 포함된 ZIP 압축 파일을 만듭니다. 파트는 완전한 SQL 문 사이에서 분할됩니다({min}–{max} MB).", databaseSelection: "데이터베이스", selectedDatabases: "{selected}/{total} 선택됨", selectAllDatabases: "모두 선택", diff --git a/apps/desktop/src/i18n/locales/pt-BR.ts b/apps/desktop/src/i18n/locales/pt-BR.ts index ddf2f705c6..ea696c63d9 100644 --- a/apps/desktop/src/i18n/locales/pt-BR.ts +++ b/apps/desktop/src/i18n/locales/pt-BR.ts @@ -7443,6 +7443,9 @@ export default withEnglishFallback({ omitAutoIncrement: "Omitir AUTO_INCREMENT (para scripts de instalação do zero)", includeData: "Dados da tabela (INSERT)", includeObjects: "Views / Procedures / Funções / Sequências", + splitSqlOutput: "Dividir exportação SQL em arquivo ZIP", + splitSqlPartMaxMb: "Tamanho máximo por parte SQL (MB)", + splitSqlOutputDescription: "Cria um arquivo ZIP com partes SQL numeradas e um manifesto. As partes são divididas entre instruções SQL completas ({min}–{max} MB).", databaseSelection: "Bancos de dados", selectedDatabases: "{selected}/{total} selecionados", selectAllDatabases: "Selecionar todos", diff --git a/apps/desktop/src/i18n/locales/tr.ts b/apps/desktop/src/i18n/locales/tr.ts index fff3b4a093..62facee42c 100644 --- a/apps/desktop/src/i18n/locales/tr.ts +++ b/apps/desktop/src/i18n/locales/tr.ts @@ -7733,6 +7733,9 @@ export default withEnglishFallback({ omitAutoIncrement: "AUTO_INCREMENT'i atla (sıfırdan kurulum betikleri için)", includeData: "Tablo verisi (INSERT)", includeObjects: "Görünümler / Yordamlar / İşlevler / Diziler", + splitSqlOutput: "SQL dışa aktarmayı ZIP arşivine böl", + splitSqlPartMaxMb: "SQL parçası başına maksimum boyut (MB)", + splitSqlOutputDescription: "Numaralı SQL parçaları ve bir bildirim içeren bir ZIP arşivi oluşturur. Parçalar tam SQL ifadeleri arasında bölünür ({min}–{max} MB).", databaseSelection: "Veritabanları", selectedDatabases: "{selected}/{total} seçildi", selectAllDatabases: "Tümünü seç", diff --git a/apps/desktop/src/i18n/locales/zh-CN.ts b/apps/desktop/src/i18n/locales/zh-CN.ts index 38c45c046a..d83f36591a 100644 --- a/apps/desktop/src/i18n/locales/zh-CN.ts +++ b/apps/desktop/src/i18n/locales/zh-CN.ts @@ -7860,6 +7860,9 @@ export default withEnglishFallback({ omitAutoIncrement: "省略 AUTO_INCREMENT（用于全新安装的初始化脚本）", includeData: "表数据 (INSERT)", includeObjects: "视图 / 存储过程 / 函数 / 序列", + splitSqlOutput: "将 SQL 导出拆分为 ZIP 压缩包", + splitSqlPartMaxMb: "每个 SQL 分片的最大大小（MB）", + splitSqlOutputDescription: "创建包含编号 SQL 分片和清单的 ZIP 压缩包。分片仅在完整 SQL 语句之间拆分（{min}–{max} MB）。", databaseSelection: "选择数据库", selectedDatabases: "已选择 {selected}/{total}", selectAllDatabases: "全选", diff --git a/apps/desktop/src/i18n/locales/zh-TW.ts b/apps/desktop/src/i18n/locales/zh-TW.ts index df1150af7a..e3da7633d6 100644 --- a/apps/desktop/src/i18n/locales/zh-TW.ts +++ b/apps/desktop/src/i18n/locales/zh-TW.ts @@ -6757,6 +6757,9 @@ export default withEnglishFallback({ omitAutoIncrement: "省略 AUTO_INCREMENT（用於全新安裝的初始化腳本）", includeData: "資料表資料 (INSERT)", includeObjects: "檢視 / 預存程序 / 函式 / 序列", + splitSqlOutput: "將 SQL 匯出分割為 ZIP 壓縮檔", + splitSqlPartMaxMb: "每個 SQL 分片的最大大小（MB）", + splitSqlOutputDescription: "建立包含編號 SQL 分片與資訊清單的 ZIP 壓縮檔。分片只會在完整 SQL 陳述式之間分割（{min}–{max} MB）。", databaseSelection: "選擇資料庫", selectedDatabases: "已選擇 {selected}/{total}", selectAllDatabases: "全選", diff --git a/apps/desktop/src/lib/backend/tauri.ts b/apps/desktop/src/lib/backend/tauri.ts index 78fbecced4..d39f644776 100644 --- a/apps/desktop/src/lib/backend/tauri.ts +++ b/apps/desktop/src/lib/backend/tauri.ts @@ -4891,6 +4891,7 @@ export interface DatabaseExportRequest { outputCompression?: "none" | "gzip"; snapshotSessionId?: string; batchSize: number; + splitMaxMb?: number; } export interface DatabaseBackupSnapshot { @@ -4941,6 +4942,7 @@ export interface TableExportRequest { dateTimeFormat?: string; numericColumnRightAlign?: boolean; autoFilter?: boolean; + splitMaxMb?: number; } export interface TableCsvExportOptions { diff --git a/apps/desktop/src/lib/export/sqlInsertMode.ts b/apps/desktop/src/lib/export/sqlInsertMode.ts index ffc61099b9..a20a424d7b 100644 --- a/apps/desktop/src/lib/export/sqlInsertMode.ts +++ b/apps/desktop/src/lib/export/sqlInsertMode.ts @@ -4,17 +4,22 @@ import i18n from "@/i18n"; export type SqlInsertMode = "batch" | "single"; +export interface SqlExportOptions { + insertMode: SqlInsertMode; + splitMaxMb?: number; +} + export const DEFAULT_SQL_INSERT_MODE: SqlInsertMode = "batch"; -export function showSqlInsertModeDialog(): Promise { - if (typeof document === "undefined") return Promise.resolve(DEFAULT_SQL_INSERT_MODE); +export function showSqlInsertModeDialog(options: { allowSplit?: boolean } = {}): Promise { + if (typeof document === "undefined") return Promise.resolve({ insertMode: DEFAULT_SQL_INSERT_MODE }); return new Promise((resolve) => { const container = document.createElement("div"); document.body.append(container); let settled = false; let app: ReturnType | null = null; - const finish = (value: SqlInsertMode | null) => { + const finish = (value: SqlExportOptions | null) => { if (settled) return; settled = true; resolve(value); @@ -23,7 +28,8 @@ export function showSqlInsertModeDialog(): Promise { }; app = createApp(SqlInsertModeDialog, { open: true, - onConfirm: (insertMode: SqlInsertMode) => finish(insertMode), + allowSplit: options.allowSplit === true, + onConfirm: (options: SqlExportOptions) => finish(options), onCancel: () => finish(null), }); app.use(i18n); diff --git a/crates/dbx-core/src/database_export.rs b/crates/dbx-core/src/database_export.rs index b4f290d066..0fcbae3ce8 100644 --- a/crates/dbx-core/src/database_export.rs +++ b/crates/dbx-core/src/database_export.rs @@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use std::collections::{HashMap, HashSet}; use std::future::Future; -use std::io::{BufWriter, Write}; +use std::io::{BufRead, BufWriter, Write}; use std::pin::Pin; use std::sync::{Arc, RwLock}; use std::time::Duration; @@ -126,11 +126,19 @@ pub struct DatabaseExportRequest { #[serde(default)] pub snapshot_session_id: Option, pub batch_size: usize, + /// When set, the export is packaged as a `.zip` archive containing + /// multiple `part-N.sql` entries (plus a `manifest.json`), each capped + /// at this many megabytes, instead of one unbounded `.sql`/`.sql.gz` + /// file. Mutually exclusive with `output_compression` -- a zip archive + /// is its own compressed container. + #[serde(default)] + pub split_max_mb: Option, } enum DatabaseExportWriter { Plain(BufWriter), Gzip(Box>>), + SplitZip(Box), } impl Write for DatabaseExportWriter { @@ -138,6 +146,7 @@ impl Write for DatabaseExportWriter { match self { Self::Plain(writer) => writer.write(buffer), Self::Gzip(writer) => writer.write(buffer), + Self::SplitZip(writer) => writer.write(buffer), } } @@ -145,17 +154,23 @@ impl Write for DatabaseExportWriter { match self { Self::Plain(writer) => writer.flush(), Self::Gzip(writer) => writer.flush(), + Self::SplitZip(writer) => writer.flush(), } } } impl DatabaseExportWriter { - fn finish(self) -> Result<(), String> { + fn finish(self, source_file_name: &str) -> Result<(), String> { match self { - Self::Plain(mut writer) => writer.flush(), - Self::Gzip(writer) => writer.finish().and_then(|mut output| output.flush()), + Self::Plain(mut writer) => { + writer.flush().map_err(|error| format!("Failed to finalize export file: {error}")) + } + Self::Gzip(writer) => writer + .finish() + .and_then(|mut output| output.flush()) + .map_err(|error| format!("Failed to finalize export file: {error}")), + Self::SplitZip(writer) => writer.finish(source_file_name), } - .map_err(|error| format!("Failed to finalize export file: {error}")) } } @@ -2574,6 +2589,15 @@ fn export_destination_parent_dir(file_path: &str) -> Option<&std::path::Path> { (!parent.as_os_str().is_empty()).then_some(parent) } +/// Name recorded inside `manifest.json` as the logical "source file" the +/// split parts represent, e.g. `mydb.zip` -> `mydb.sql`. When splitting is +/// disabled this is unused (the writer variant never calls it). +fn export_source_file_name(file_path: &str) -> String { + let path = std::path::Path::new(file_path); + let stem = path.file_stem().and_then(|stem| stem.to_str()).unwrap_or("export"); + format!("{stem}.sql") +} + async fn create_database_export_writer( state: &Arc, request: &DatabaseExportRequest, @@ -2582,6 +2606,32 @@ async fn create_database_export_writer( if let Some(parent) = export_destination_parent_dir(&request.file_path) { expected_destination_identity = ensure_export_destination_dir(state, parent).await?; } + if let Some(max_mb) = request.split_max_mb { + let zip_path = std::path::Path::new(&request.file_path); + let stem = zip_path.file_stem().and_then(|stem| stem.to_str()).unwrap_or("export"); + let writer = crate::export_split_zip::SplitZipExportWriter::create_with_overwrite( + zip_path, + max_mb, + stem, + "sql", + request.prevent_overwrite, + )?; + let opened_destination_identity = + std::fs::File::open(&request.file_path).ok().as_ref().and_then(export_destination_identity_for_file); + if export_destination_identity_mismatch( + expected_destination_identity.as_ref(), + opened_destination_identity.as_ref(), + ) { + let _ = std::fs::remove_file(&request.file_path); + return Err(format!( + "Backup destination for {} changed while opening the output file -- the directory now \ + resolves to a different filesystem than the one just verified. If a removable or network \ + drive was disconnected and reconnected, retry the backup.", + request.file_path + )); + } + return Ok(DatabaseExportWriter::SplitZip(Box::new(writer))); + } let file = std::fs::OpenOptions::new() .write(true) .create(true) @@ -2636,6 +2686,21 @@ fn postgres_create_schema_sql(schema: &str) -> String { format!("CREATE SCHEMA IF NOT EXISTS {};", quote_identifier(schema, &DatabaseType::Postgres)) } +// Copy one line at a time so a `SplitZipExportWriter` can only rotate between +// complete SQL statements; `std::io::copy` would feed it arbitrary 8KB chunks. +fn combine_schema_sql_export(source: &mut dyn BufRead, destination: &mut W) -> std::io::Result<()> { + let mut line = Vec::new(); + loop { + line.clear(); + let bytes_read = source.read_until(b'\n', &mut line)?; + if bytes_read == 0 { + break; + } + destination.write_all(&line)?; + } + Ok(()) +} + async fn export_postgres_all_schemas_sql_core( state: &Arc, request: &DatabaseExportRequest, @@ -2670,6 +2735,7 @@ async fn export_postgres_all_schemas_sql_core( schema_request.schema = schema_name.clone(); schema_request.file_path = temp_dir.path().join(format!("schema-{schema_index}.sql")).display().to_string(); schema_request.output_compression = DatabaseExportOutputCompression::None; + schema_request.split_max_mb = None; let terminal = Arc::new(std::sync::Mutex::new(None::)); let terminal_for_callback = terminal.clone(); @@ -2730,10 +2796,11 @@ async fn export_postgres_all_schemas_sql_core( let mut source = std::io::BufReader::new( std::fs::File::open(path).map_err(|e| format!("Failed to read temporary schema export: {e}"))?, ); - std::io::copy(&mut source, &mut file).map_err(|e| format!("Failed to combine schema export: {e}"))?; + combine_schema_sql_export(&mut source, &mut file) + .map_err(|e| format!("Failed to combine schema export: {e}"))?; writeln!(file).map_err(|e| format!("Failed to write file: {e}"))?; } - file.finish()?; + file.finish(&export_source_file_name(&request.file_path))?; on_progress(ExportProgress { export_id: request.export_id.clone(), current_object: request.database.clone(), @@ -3695,7 +3762,7 @@ async fn export_database_sql_core_inner( writeln!(file, "SET FOREIGN_KEY_CHECKS = 1;").map_err(|e| format!("Failed to write file: {e}"))?; } - file.finish()?; + file.finish(&export_source_file_name(&request.file_path))?; // Emit Done progress on_progress(ExportProgress { @@ -3758,7 +3825,7 @@ fn build_database_export_object_source_sql( #[cfg(test)] mod tests { use super::{ - await_export_operation, await_export_stream_operation, clear_export_cancelled, + await_export_operation, await_export_stream_operation, clear_export_cancelled, combine_schema_sql_export, concurrent_metadata_prefetch_allowed, database_export_metadata_prefetch_concurrency, emit_database_export_cancelled, postgres_create_schema_sql, postgres_export_schema_names, set_export_cancelled, snapshot_batch_cancelled, ExportStatus, EXPORT_CANCELLED_ERROR, @@ -3978,6 +4045,7 @@ mod tests { output_compression: Default::default(), snapshot_session_id: None, batch_size: 1000, + split_max_mb: None, } } @@ -3991,7 +4059,7 @@ mod tests { flate2::Compression::default(), ))); writer.write_all(b"SELECT 1;\n").unwrap(); - writer.finish().unwrap(); + writer.finish("backup.sql").unwrap(); let mut output = String::new(); flate2::read::GzDecoder::new(std::fs::File::open(path).unwrap()).read_to_string(&mut output).unwrap(); @@ -5471,6 +5539,136 @@ mod tests { ); } + #[test] + fn split_zip_export_writer_splits_across_insert_batches_into_valid_sql() { + let directory = tempfile::tempdir().unwrap(); + let zip_path = directory.path().join("orders.zip"); + let mut writer = crate::export_split_zip::SplitZipExportWriter::create( + &zip_path, + crate::export_split_zip::MIN_SPLIT_PART_MAX_MB, + "orders", + "sql", + ) + .unwrap(); + + // Each call is one full INSERT batch statement, exactly like the real + // write_database_export_rows call sites -- the writer must only cut + // between these calls, never inside one. + let long_value = "x".repeat(200_000); + for row_index in 0..20 { + write_database_export_rows( + &mut writer, + &[vec![json!(row_index), json!(long_value.clone())]], + &["id".to_string(), "payload".to_string()], + &[Some("bigint".to_string()), Some("text".to_string())], + &[None, None], + "orders", + "shop", + &DatabaseType::Postgres, + ) + .unwrap(); + } + writer.finish("orders.sql").unwrap(); + + let file = std::fs::File::open(&zip_path).unwrap(); + let mut archive = zip::ZipArchive::new(file).unwrap(); + let mut sql_parts = Vec::new(); + for index in 0..archive.len() { + let mut entry = archive.by_index(index).unwrap(); + if !entry.name().ends_with(".sql") { + continue; + } + let mut contents = String::new(); + entry.read_to_string(&mut contents).unwrap(); + sql_parts.push((entry.name().to_string(), contents)); + } + sql_parts.sort_by(|a, b| a.0.cmp(&b.0)); + + assert!( + sql_parts.len() > 1, + "expected the large export to be split into multiple parts, got {}", + sql_parts.len() + ); + for (name, contents) in &sql_parts { + assert!(!contents.is_empty(), "{name} must not be empty"); + // Every non-blank line must be a syntactically complete INSERT + // statement -- proof that no cut landed inside one. + for line in contents.lines().filter(|line| !line.trim().is_empty()) { + assert!( + line.trim_start().starts_with("INSERT INTO") && line.trim_end().ends_with(';'), + "{name} has a malformed line from a mid-statement cut: {line}" + ); + } + } + // Reassembling every part in order must reproduce all 20 rows. + let combined: String = sql_parts.iter().map(|(_, contents)| contents.as_str()).collect(); + assert_eq!(combined.matches("INSERT INTO").count(), 20); + } + + #[test] + fn all_schemas_combine_keeps_split_part_boundaries_statement_safe() { + // Mirror of the per-schema temporary files that + // `export_postgres_all_schemas_sql_core` combines: whole SQL + // statements, one per line, each newline-terminated. + let directory = tempfile::tempdir().unwrap(); + let schema_path = directory.path().join("schema-0.sql"); + let long_value = "x".repeat(200_000); + let mut schema_sql = String::new(); + for row_index in 0..20 { + schema_sql.push_str(&format!("INSERT INTO orders VALUES ({row_index}, '{long_value}');\n")); + } + std::fs::write(&schema_path, &schema_sql).unwrap(); + + let zip_path = directory.path().join("combined.zip"); + let mut writer = crate::export_split_zip::SplitZipExportWriter::create( + &zip_path, + crate::export_split_zip::MIN_SPLIT_PART_MAX_MB, + "combined", + "sql", + ) + .unwrap(); + let mut source = std::io::BufReader::new(std::fs::File::open(&schema_path).unwrap()); + combine_schema_sql_export(&mut source, &mut writer).unwrap(); + writer.finish("combined.sql").unwrap(); + + let file = std::fs::File::open(&zip_path).unwrap(); + let mut archive = zip::ZipArchive::new(file).unwrap(); + let mut sql_parts = Vec::new(); + for index in 0..archive.len() { + let mut entry = archive.by_index(index).unwrap(); + if !entry.name().ends_with(".sql") { + continue; + } + let mut contents = String::new(); + entry.read_to_string(&mut contents).unwrap(); + sql_parts.push((entry.name().to_string(), contents)); + } + sql_parts.sort_by(|a, b| a.0.cmp(&b.0)); + + assert!( + sql_parts.len() > 1, + "expected the combined export to be split into multiple parts, got {}", + sql_parts.len() + ); + for (name, contents) in &sql_parts { + assert!(!contents.is_empty(), "{name} must not be empty"); + // Every non-blank line must be a complete statement -- proof that + // the copy never cut inside one. + for line in contents.lines().filter(|line| !line.trim().is_empty()) { + assert!( + line.trim_start().starts_with("INSERT INTO") && line.trim_end().ends_with(';'), + "{name} has a malformed line from a mid-statement cut" + ); + } + } + // Reassembling the parts in order must reproduce the temporary file + // byte for byte: the line-by-line copy adds, drops, and alters + // nothing, including the trailing newline. + let combined: String = sql_parts.iter().map(|(_, contents)| contents.as_str()).collect(); + assert_eq!(combined, schema_sql); + assert_eq!(combined.matches("INSERT INTO").count(), 20); + } + #[test] fn dameng_identity_export_inserts_enable_identity_insert() { let statements = build_export_insert_statements(BuildExportInsertStatementsOptions { diff --git a/crates/dbx-core/src/export_split_zip.rs b/crates/dbx-core/src/export_split_zip.rs new file mode 100644 index 0000000000..15626b083c --- /dev/null +++ b/crates/dbx-core/src/export_split_zip.rs @@ -0,0 +1,281 @@ +//! Shared support for splitting a large generated SQL export into multiple +//! parts packaged inside a single `.zip` archive, instead of one unbounded +//! `.sql` file. Used by both the whole-database export (`database_export.rs`) +//! and the single-table SQL export (`table_export.rs`). +//! +//! # Cut boundary +//! The writer never splits mid-write: it only checks the size threshold +//! *before* starting a new `write()` call and, if exceeded, rotates to the +//! next zip entry first. Every caller in this codebase writes one complete +//! SQL statement (or comment line) per `write`/`writeln!` call, so a part +//! boundary can land between two INSERT batches of the same table but never +//! inside a single statement -- every part is independently valid, replayable +//! SQL. + +use std::io::Write; + +use serde::Serialize; +use zip::write::SimpleFileOptions; +use zip::{CompressionMethod, ZipWriter}; + +/// Default part size when a caller enables splitting without specifying a +/// size (should not normally happen -- the frontend always sends an explicit +/// value -- but keeps the backend safe against malformed requests). +pub const DEFAULT_SPLIT_PART_MAX_MB: u32 = 100; +pub const MIN_SPLIT_PART_MAX_MB: u32 = 1; +pub const MAX_SPLIT_PART_MAX_MB: u32 = 4096; + +/// Clamp a user-supplied "max MB per part" value to a sane range so a +/// malformed or hostile request cannot create an unbounded number of zip +/// entries (too-small) or a single-part export that defeats the point +/// (too-large is still allowed up to a generous ceiling). +pub fn clamp_split_part_max_mb(value: u32) -> u32 { + value.clamp(MIN_SPLIT_PART_MAX_MB, MAX_SPLIT_PART_MAX_MB) +} + +fn split_part_max_bytes(max_mb: u32) -> u64 { + u64::from(clamp_split_part_max_mb(max_mb)) * 1024 * 1024 +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct SplitExportManifest<'a> { + format: &'static str, + generated_at: String, + source_file_name: &'a str, + part_max_bytes: u64, + total_parts: u32, + parts: &'a [String], +} + +/// Writes SQL export content across multiple `part-N.sql` entries inside a +/// single `.zip` archive, rotating to a new entry once the current one +/// reaches `part_max_bytes`. Call [`SplitZipExportWriter::finish`] to write +/// the trailing `manifest.json` entry and flush the archive to disk. +pub struct SplitZipExportWriter { + zip: ZipWriter, + options: SimpleFileOptions, + part_max_bytes: u64, + current_part_bytes: u64, + part_index: u32, + part_stem: String, + part_extension: String, + part_names: Vec, +} + +impl SplitZipExportWriter { + /// `part_stem`/`part_extension` name each entry as `{stem}-part-{n}.{extension}` + /// (e.g. `mydb-part-1.sql`). + pub fn create( + zip_path: &std::path::Path, + max_mb: u32, + part_stem: &str, + part_extension: &str, + ) -> Result { + Self::create_with_overwrite(zip_path, max_mb, part_stem, part_extension, false) + } + + pub fn create_with_overwrite( + zip_path: &std::path::Path, + max_mb: u32, + part_stem: &str, + part_extension: &str, + prevent_overwrite: bool, + ) -> Result { + let file = std::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(!prevent_overwrite) + .create_new(prevent_overwrite) + .open(zip_path) + .map_err(|error| { + if prevent_overwrite && error.kind() == std::io::ErrorKind::AlreadyExists { + format!("Backup file already exists: {}", zip_path.display()) + } else { + format!("Failed to create zip file: {error}") + } + })?; + let zip = ZipWriter::new(file); + let options = SimpleFileOptions::default().compression_method(CompressionMethod::Deflated); + let mut writer = Self { + zip, + options, + part_max_bytes: split_part_max_bytes(max_mb), + current_part_bytes: 0, + part_index: 0, + part_stem: sanitize_zip_entry_component(part_stem), + part_extension: sanitize_zip_entry_component(part_extension), + part_names: Vec::new(), + }; + writer.start_next_part()?; + Ok(writer) + } + + fn start_next_part(&mut self) -> Result<(), String> { + self.part_index += 1; + let name = format!("{}-part-{}.{}", self.part_stem, self.part_index, self.part_extension); + self.zip + .start_file(&name, self.options) + .map_err(|error| format!("Failed to start zip entry {name}: {error}"))?; + self.part_names.push(name); + self.current_part_bytes = 0; + Ok(()) + } + + /// Writes the trailing `manifest.json` entry (listing every part in + /// order) and finalizes the zip archive. Consumes the writer because the + /// underlying `zip::ZipWriter::finish` does. + pub fn finish(mut self, source_file_name: &str) -> Result<(), String> { + let manifest = SplitExportManifest { + format: "dbx-sql-export-parts-v1", + generated_at: chrono::Local::now().to_rfc3339(), + source_file_name, + part_max_bytes: self.part_max_bytes, + total_parts: self.part_index, + parts: &self.part_names, + }; + let manifest_json = serde_json::to_vec_pretty(&manifest) + .map_err(|error| format!("Failed to encode export manifest: {error}"))?; + let stored_options = SimpleFileOptions::default().compression_method(CompressionMethod::Stored); + self.zip + .start_file("manifest.json", stored_options) + .map_err(|error| format!("Failed to start manifest entry: {error}"))?; + self.zip.write_all(&manifest_json).map_err(|error| format!("Failed to write export manifest: {error}"))?; + let mut file = self.zip.finish().map_err(|error| format!("Failed to finalize zip archive: {error}"))?; + file.flush().map_err(|error| format!("Failed to flush zip archive: {error}")) + } +} + +impl Write for SplitZipExportWriter { + fn write(&mut self, buffer: &[u8]) -> std::io::Result { + if self.current_part_bytes >= self.part_max_bytes { + self.start_next_part().map_err(std::io::Error::other)?; + } + let written = self.zip.write(buffer)?; + self.current_part_bytes += written as u64; + Ok(written) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.zip.flush() + } +} + +/// Zip entry names must not contain path separators or other characters that +/// could be misread as a directory traversal; export file stems come from +/// user-controlled database/table names, so sanitize defensively. +fn sanitize_zip_entry_component(value: &str) -> String { + let sanitized: String = value + .chars() + .map(|c| if c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.' { c } else { '_' }) + .collect(); + let trimmed = sanitized.trim_matches('_').to_string(); + if trimmed.is_empty() { + "export".to_string() + } else { + trimmed + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Read; + + fn read_zip_entries(path: &std::path::Path) -> Vec<(String, Vec)> { + let file = std::fs::File::open(path).unwrap(); + let mut archive = zip::ZipArchive::new(file).unwrap(); + let mut entries = Vec::new(); + for index in 0..archive.len() { + let mut entry = archive.by_index(index).unwrap(); + let mut contents = Vec::new(); + entry.read_to_end(&mut contents).unwrap(); + entries.push((entry.name().to_string(), contents)); + } + entries + } + + #[test] + fn clamps_split_part_max_mb_to_a_sane_range() { + assert_eq!(clamp_split_part_max_mb(0), MIN_SPLIT_PART_MAX_MB); + assert_eq!(clamp_split_part_max_mb(50), 50); + assert_eq!(clamp_split_part_max_mb(u32::MAX), MAX_SPLIT_PART_MAX_MB); + } + + #[test] + fn rotates_to_a_new_part_once_the_threshold_is_reached() { + let dir = std::env::temp_dir().join(format!("dbx-split-zip-test-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&dir).unwrap(); + let zip_path = dir.join("export.zip"); + + { + // 1 MB minimum threshold; each write is far smaller so every + // write should land in the very first part with no rotation. + let mut writer = SplitZipExportWriter::create(&zip_path, 1, "mydb", "sql").unwrap(); + for index in 0..5 { + writeln!(writer, "INSERT INTO t VALUES ({index});").unwrap(); + } + writer.finish("mydb.sql").unwrap(); + } + + let entries = read_zip_entries(&zip_path); + let sql_entries: Vec<_> = entries.iter().filter(|(name, _)| name.ends_with(".sql")).collect(); + assert_eq!(sql_entries.len(), 1); + assert_eq!(sql_entries[0].0, "mydb-part-1.sql"); + let manifest_entry = entries.iter().find(|(name, _)| name == "manifest.json").unwrap(); + let manifest: serde_json::Value = serde_json::from_slice(&manifest_entry.1).unwrap(); + assert_eq!(manifest["totalParts"], 1); + assert_eq!(manifest["parts"].as_array().unwrap().len(), 1); + + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn splits_into_multiple_parts_and_every_part_is_valid_statements() { + let dir = std::env::temp_dir().join(format!("dbx-split-zip-test-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&dir).unwrap(); + let zip_path = dir.join("export.zip"); + // Force a rotation by using a threshold smaller than a single write. + let long_statement = format!("INSERT INTO t VALUES ('{}');", "x".repeat(200)); + + { + let mut writer = SplitZipExportWriter::create(&zip_path, MIN_SPLIT_PART_MAX_MB, "mydb", "sql").unwrap(); + // Each write() call is one full statement; the threshold check + // happens before the call, so a small per-write payload combined + // with a below-1MB threshold still needs enough total bytes to + // exceed 1MB and trigger a second part. Use the internal minimum + // by writing repeatedly. + for index in 0..6000 { + writeln!(writer, "{long_statement} -- row {index}").unwrap(); + } + writer.finish("mydb.sql").unwrap(); + } + + let entries = read_zip_entries(&zip_path); + let mut sql_entries: Vec<_> = entries.iter().filter(|(name, _)| name.ends_with(".sql")).collect(); + sql_entries.sort_by(|a, b| a.0.cmp(&b.0)); + assert!(sql_entries.len() >= 2, "expected at least 2 parts, got {}", sql_entries.len()); + + for (name, contents) in &sql_entries { + let text = String::from_utf8(contents.clone()).unwrap(); + assert!(!text.is_empty(), "{name} must not be empty"); + for line in text.lines().filter(|line| !line.trim().is_empty()) { + assert!(line.contains(';'), "{name} contains a line that is not a complete statement: {line}"); + } + } + + let manifest_entry = entries.iter().find(|(name, _)| name == "manifest.json").unwrap(); + let manifest: serde_json::Value = serde_json::from_slice(&manifest_entry.1).unwrap(); + assert_eq!(manifest["totalParts"].as_u64().unwrap() as usize, sql_entries.len()); + assert_eq!(manifest["parts"].as_array().unwrap().len(), sql_entries.len()); + + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn sanitizes_zip_entry_stem_from_untrusted_names() { + assert_eq!(sanitize_zip_entry_component("../../etc/passwd"), ".._.._etc_passwd"); + assert_eq!(sanitize_zip_entry_component(""), "export"); + assert_eq!(sanitize_zip_entry_component("my db (prod)"), "my_db__prod"); + } +} diff --git a/crates/dbx-core/src/lib.rs b/crates/dbx-core/src/lib.rs index 570729f798..f222c45a22 100644 --- a/crates/dbx-core/src/lib.rs +++ b/crates/dbx-core/src/lib.rs @@ -48,6 +48,7 @@ pub mod docs; pub mod document_ops; pub mod driver_runtime; pub mod export_runtime; +pub mod export_split_zip; pub mod external; pub mod hbase_ops; pub mod history; diff --git a/crates/dbx-core/src/table_export.rs b/crates/dbx-core/src/table_export.rs index 8dde1cd0ee..4cf5a9181e 100644 --- a/crates/dbx-core/src/table_export.rs +++ b/crates/dbx-core/src/table_export.rs @@ -83,6 +83,12 @@ pub struct TableExportRequest { pub column_comments: Option>>, #[serde(default, skip_serializing_if = "Option::is_none")] pub auto_filter: Option, + /// SQL format only: when set, the export is packaged as a `.zip` archive + /// containing multiple `part-N.sql` entries (plus a `manifest.json`) + /// capped at this many megabytes each, instead of one unbounded `.sql` + /// file. Ignored for every other format. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub split_max_mb: Option, } #[derive(Debug, Clone, Serialize)] @@ -975,6 +981,53 @@ async fn stream_native_table_rows( } } +#[allow(clippy::too_many_arguments)] +/// Output writer for the single-table SQL export. `Plain` is the historical +/// unbounded `.sql` file; `SplitZip` packages the SQL into multiple +/// `part-N.sql` entries inside a `.zip` when `TableExportRequest::split_max_mb` +/// is set. +enum TableExportSqlWriter { + Plain(BufWriter), + SplitZip(Box), +} + +impl Write for TableExportSqlWriter { + fn write(&mut self, buffer: &[u8]) -> std::io::Result { + match self { + Self::Plain(writer) => writer.write(buffer), + Self::SplitZip(writer) => writer.write(buffer), + } + } + + fn flush(&mut self) -> std::io::Result<()> { + match self { + Self::Plain(writer) => writer.flush(), + Self::SplitZip(writer) => writer.flush(), + } + } +} + +impl TableExportSqlWriter { + fn finish(self, source_file_name: &str) -> Result<(), String> { + match self { + Self::Plain(mut writer) => writer.flush().map_err(|error| format!("Failed to flush export file: {error}")), + Self::SplitZip(writer) => writer.finish(source_file_name), + } + } +} + +fn create_table_export_sql_writer(request: &TableExportRequest) -> Result { + if let Some(max_mb) = request.split_max_mb { + let zip_path = std::path::Path::new(&request.file_path); + let stem = zip_path.file_stem().and_then(|stem| stem.to_str()).unwrap_or(&request.table_name); + let writer = crate::export_split_zip::SplitZipExportWriter::create(zip_path, max_mb, stem, "sql")?; + return Ok(TableExportSqlWriter::SplitZip(Box::new(writer))); + } + Ok(TableExportSqlWriter::Plain(BufWriter::new( + std::fs::File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?, + ))) +} + #[allow(clippy::too_many_arguments)] async fn try_export_native_table_stream( state: &AppState, @@ -1254,13 +1307,11 @@ async fn try_export_native_table_stream( result } "sql" => { - let mut file = BufWriter::new( - std::fs::File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?, - ); + let mut file = create_table_export_sql_writer(request)?; let mut pending_rows: Vec> = Vec::new(); let mut wrote_statements = false; let mut flush_pending = - |file: &mut BufWriter, pending_rows: &mut Vec>| -> Result<(), String> { + |file: &mut TableExportSqlWriter, pending_rows: &mut Vec>| -> Result<(), String> { if pending_rows.is_empty() { return Ok(()); } @@ -1321,7 +1372,7 @@ async fn try_export_native_table_stream( if wrote_statements { file.write_all(b"\n").map_err(|e| format!("Failed to write SQL: {e}"))?; } - file.flush().map_err(|e| format!("Failed to flush export file: {e}"))?; + file.finish(&format!("{}.sql", request.table_name))?; } result } @@ -1562,8 +1613,13 @@ async fn export_table_data_core_inner( } // 8. Create output file - let file = std::fs::File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?; - let mut file = BufWriter::new(file); + let mut file = if request.format.eq_ignore_ascii_case("sql") { + create_table_export_sql_writer(request)? + } else { + TableExportSqlWriter::Plain(BufWriter::new( + std::fs::File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?, + )) + }; let mut text_buffer = String::new(); let mut rows_exported: u64 = 0; @@ -2154,7 +2210,7 @@ async fn export_table_data_core_inner( } close_table_export_cursor_if_open(state, &pool_key, request, &mut cursor_session).await; - file.flush().map_err(|e| format!("Failed to flush export file: {e}"))?; + file.finish(&format!("{}.sql", request.table_name))?; // 8. Emit Done progress on_progress(TableExportProgress { @@ -2304,6 +2360,7 @@ mod tests { numeric_column_right_align: false, column_comments: None, auto_filter: None, + split_max_mb: None, }; ExternalDriverExportFixture { state, request, calls, output, dir } @@ -2466,6 +2523,7 @@ mod tests { numeric_column_right_align: false, column_comments: None, auto_filter: None, + split_max_mb: None, }; export_table_data_core(&state, &request, |_| {}).await.unwrap(); @@ -2631,6 +2689,7 @@ mod tests { numeric_column_right_align: false, column_comments: None, auto_filter: None, + split_max_mb: None, }; let context = table_export_sql_context(DatabaseType::Iotdb, None, request.schema.as_deref()); let columns = vec!["Time".to_string(), "root.test.device2.temperature".to_string()]; @@ -2689,6 +2748,7 @@ mod tests { numeric_column_right_align: false, column_comments: None, auto_filter: None, + split_max_mb: None, }; let context = table_export_sql_context(DatabaseType::Iotdb, None, request.schema.as_deref()); let columns = vec!["tImE".to_string(), "temperature".to_string()]; @@ -2725,6 +2785,7 @@ mod tests { numeric_column_right_align: false, column_comments: None, auto_filter: None, + split_max_mb: None, }; let context = table_export_sql_context(DatabaseType::Iotdb, None, request.schema.as_deref()); let error = table_export_query_columns(&request, &context, &["TIME".to_string()]).unwrap_err(); @@ -2756,6 +2817,7 @@ mod tests { numeric_column_right_align: false, column_comments: None, auto_filter: None, + split_max_mb: None, }; let context = table_export_sql_context(DatabaseType::Iotdb, None, request.schema.as_deref()); let columns = vec![ @@ -2795,6 +2857,7 @@ mod tests { numeric_column_right_align: false, column_comments: None, auto_filter: None, + split_max_mb: None, }; let columns = vec!["Time".to_string(), "value".to_string()]; @@ -2846,6 +2909,7 @@ mod tests { numeric_column_right_align: false, column_comments: None, auto_filter: None, + split_max_mb: None, }; let context = table_export_sql_context(DatabaseType::Oracle, None, request.schema.as_deref()); @@ -2891,6 +2955,7 @@ mod tests { numeric_column_right_align: false, column_comments: None, auto_filter: None, + split_max_mb: None, }; let columns = vec!["id".to_string(), "payload".to_string()]; let primary_keys = vec!["id".to_string()]; @@ -2964,6 +3029,7 @@ mod tests { numeric_column_right_align: false, column_comments: None, auto_filter: None, + split_max_mb: None, }; let columns = vec!["id".to_string(), "DisplayName".to_string()]; let primary_keys = vec!["id".to_string()]; @@ -3019,6 +3085,7 @@ mod tests { numeric_column_right_align: false, column_comments: None, auto_filter: None, + split_max_mb: None, }; let columns = vec!["id".to_string(), "geom".to_string(), "name".to_string()]; let column_types = vec![Some("int".to_string()), Some("geometry".to_string()), Some("varchar".to_string())]; @@ -3076,6 +3143,7 @@ mod tests { numeric_column_right_align: false, column_comments: None, auto_filter: None, + split_max_mb: None, }; let context = table_export_sql_context(DatabaseType::Oracle, None, request.schema.as_deref()); let sql = table_cursor_sql(&request, &context, &columns, &[], &primary_keys); @@ -3200,6 +3268,187 @@ mod tests { cleanup_external_driver_export_fixture(fixture); } + #[cfg(unix)] + #[test] + fn external_driver_sql_export_without_split_writes_plain_sql() { + // The fallback export future is too large for the default test-thread + // stack in debug builds; mirror the dedicated-thread pattern used by + // the other fallback-path tests. + let handle = std::thread::Builder::new() + .name("table-export-plain-sql".to_string()) + .stack_size(8 * 1024 * 1024) + .spawn(|| { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("build table export plain SQL test runtime") + .block_on(run_external_driver_sql_export_without_split_writes_plain_sql()); + }) + .expect("spawn table export plain SQL test thread"); + if let Err(panic) = handle.join() { + std::panic::resume_unwind(panic); + } + } + + #[cfg(unix)] + async fn run_external_driver_sql_export_without_split_writes_plain_sql() { + let fixture = external_driver_export_fixture( + r#" case "$line" in + *'"method":"executeQueryPage"'*) + echo executeQueryPage >> "$CALLS" + printf '{"id":%s,"result":{"columns":["id","name"],"rows":[[1,"Ada"],[2,"Grace"]],"affected_rows":0,"execution_time_ms":1,"session_id":null,"has_more":false}}\n' "$id" + ;; + esac"#, + 2, + None, + true, + ) + .await; + let output = fixture.dir.join("export.sql"); + let mut request = fixture.request.clone(); + request.format = "sql".to_string(); + request.file_path = output.to_string_lossy().into_owned(); + + let terminal = Arc::new(std::sync::Mutex::new(None::)); + let captured = terminal.clone(); + export_table_data_core(&fixture.state, &request, move |event| { + *captured.lock().unwrap() = Some(event.status); + }) + .await + .expect("plain JDBC SQL export should succeed"); + + let sql = std::fs::read_to_string(&output).unwrap(); + // Default batch insert mode emits one multi-row INSERT statement. + assert_eq!(sql.matches("INSERT INTO").count(), 1, "both rows should be exported: {sql}"); + assert!(sql.contains("Ada")); + assert!(sql.contains("Grace")); + assert!(sql.ends_with(";\n")); + // Without `split_max_mb` the fallback arm must keep producing a plain + // SQL file, not a zip archive. + assert!(!sql.starts_with("PK")); + assert_eq!(std::fs::read_to_string(&fixture.calls).unwrap(), "executeQueryPage\n"); + assert!(matches!(*terminal.lock().unwrap(), Some(ExportStatus::Done))); + + cleanup_external_driver_export_fixture(fixture); + } + + #[cfg(unix)] + #[test] + fn external_driver_sql_export_honors_split_max_mb_into_zip_parts() { + // See external_driver_sql_export_without_split_writes_plain_sql: the + // fallback export future needs a larger-than-default test stack. + let handle = std::thread::Builder::new() + .name("table-export-split-sql".to_string()) + .stack_size(8 * 1024 * 1024) + .spawn(|| { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("build table export split SQL test runtime") + .block_on(run_external_driver_sql_export_honors_split_max_mb_into_zip_parts()); + }) + .expect("spawn table export split SQL test thread"); + if let Err(panic) = handle.join() { + std::panic::resume_unwind(panic); + } + } + + #[cfg(unix)] + async fn run_external_driver_sql_export_honors_split_max_mb_into_zip_parts() { + // Two cursor pages: 200 wide rows (~1.2MB of INSERT statements) then a + // short 100-row page. The first page alone must exceed the 1MB part + // threshold, forcing the second page into a new zip part, and every + // part must still contain only complete statements. + let fixture = external_driver_export_fixture( + r#"LONG=$(head -c 6000 /dev/zero | tr '\0' 'x') +ROWS='' +i=1 +while [ $i -le 100 ]; do ROWS="$ROWS,[$i,\"$LONG\"]"; i=$((i+1)); done +ROWS=${ROWS#,} +case "$line" in + *'"method":"executeQueryPage"'*) + echo executeQueryPage >> "$CALLS" + printf '{"id":%s,"result":{"columns":["id","name"],"rows":[%s,%s],"affected_rows":0,"execution_time_ms":1,"session_id":"cursor-1","has_more":true}}\n' "$id" "$ROWS" "$ROWS" + ;; + *'"method":"fetchQueryPage"'*) + echo fetchQueryPage >> "$CALLS" + printf '{"id":%s,"result":{"columns":["id","name"],"rows":[%s],"affected_rows":0,"execution_time_ms":1,"session_id":null,"has_more":false}}\n' "$id" "$ROWS" + ;; + *'"method":"closeQuerySession"'*) + echo closeQuerySession >> "$CALLS" + printf '{"id":%s,"result":{"ok":true}}\n' "$id" + ;; +esac"#, + 200, + None, + true, + ) + .await; + let zip_path = fixture.dir.join("export.zip"); + let mut request = fixture.request.clone(); + request.format = "sql".to_string(); + request.file_path = zip_path.to_string_lossy().into_owned(); + request.split_max_mb = Some(crate::export_split_zip::MIN_SPLIT_PART_MAX_MB); + + let terminal = Arc::new(std::sync::Mutex::new(None::)); + let captured = terminal.clone(); + export_table_data_core(&fixture.state, &request, move |event| { + *captured.lock().unwrap() = Some(event.status); + }) + .await + .expect("split JDBC SQL export should succeed"); + + // The final page reports `has_more: false` with no session id, so the + // cursor is closed implicitly and no closeQuerySession RPC is issued. + assert_eq!(std::fs::read_to_string(&fixture.calls).unwrap(), "executeQueryPage\nfetchQueryPage\n"); + let file = std::fs::File::open(&zip_path).unwrap(); + let mut archive = zip::ZipArchive::new(file).unwrap(); + let mut sql_parts = Vec::new(); + let mut manifest = None; + for index in 0..archive.len() { + let mut entry = archive.by_index(index).unwrap(); + if entry.name() == "manifest.json" { + let mut contents = Vec::new(); + entry.read_to_end(&mut contents).unwrap(); + manifest = Some(serde_json::from_slice::(&contents).unwrap()); + continue; + } + let mut contents = String::new(); + entry.read_to_string(&mut contents).unwrap(); + sql_parts.push((entry.name().to_string(), contents)); + } + sql_parts.sort_by(|a, b| a.0.cmp(&b.0)); + + assert!( + sql_parts.len() >= 2, + "expected the fallback SQL export to be split into multiple parts, got {}", + sql_parts.len() + ); + for (name, contents) in &sql_parts { + assert!(!contents.is_empty(), "{name} must not be empty"); + // Every non-blank line must be a complete statement -- the part + // boundary may only land between the fallback arm's per-batch + // writes, never inside a statement. + for line in contents.lines().filter(|line| !line.trim().is_empty()) { + assert!( + line.trim_start().starts_with("INSERT INTO") && line.trim_end().ends_with(';'), + "{name} has a malformed line from a mid-statement cut" + ); + } + } + // All 300 rows must be present across the parts: every row value is + // `(n, 'xxxx…')`, so one `, '` occurrence per row. + let combined: String = sql_parts.iter().map(|(_, contents)| contents.as_str()).collect(); + assert_eq!(combined.matches(", '").count(), 300); + + let manifest = manifest.expect("split export must contain a manifest"); + assert_eq!(manifest["totalParts"].as_u64().unwrap() as usize, sql_parts.len()); + assert_eq!(manifest["sourceFileName"], "EXPORT_SAMPLE.sql"); + assert!(matches!(*terminal.lock().unwrap(), Some(ExportStatus::Done))); + + cleanup_external_driver_export_fixture(fixture); + } + #[cfg(unix)] #[tokio::test] async fn external_driver_table_export_does_not_repeat_legacy_one_shot_results() { diff --git a/crates/dbx-core/tests/database_export_partition_ddl.rs b/crates/dbx-core/tests/database_export_partition_ddl.rs index e19668f7f6..5e613f7c54 100644 --- a/crates/dbx-core/tests/database_export_partition_ddl.rs +++ b/crates/dbx-core/tests/database_export_partition_ddl.rs @@ -141,6 +141,7 @@ async fn run_database_export_of_partition_tree_has_no_duplicates_and_replays() { output_compression: Default::default(), snapshot_session_id: None, batch_size: 1000, + split_max_mb: None, }; export_database_sql_core(&state, &request, |_progress| {}).await.expect("export should succeed"); diff --git a/crates/dbx-core/tests/database_export_prefetch.rs b/crates/dbx-core/tests/database_export_prefetch.rs index ceb3cb3006..d6a1e34697 100644 --- a/crates/dbx-core/tests/database_export_prefetch.rs +++ b/crates/dbx-core/tests/database_export_prefetch.rs @@ -80,6 +80,7 @@ async fn run_database_export_writes_structure_and_data_for_all_tables() { output_compression: Default::default(), snapshot_session_id: None, batch_size: 1000, + split_max_mb: None, }; export_database_sql_core(&state, &request, |_progress| {}).await.expect("export should succeed"); diff --git a/crates/dbx-core/tests/live_manual_transaction.rs b/crates/dbx-core/tests/live_manual_transaction.rs index 51f4711d4e..e808ea3a0a 100644 --- a/crates/dbx-core/tests/live_manual_transaction.rs +++ b/crates/dbx-core/tests/live_manual_transaction.rs @@ -333,6 +333,7 @@ async fn live_postgres_backup_snapshot_exports_wide_jsonb_then_next_table_inner( output_compression: Default::default(), snapshot_session_id: Some(snapshot.session_id.clone()), batch_size: 1000, + split_max_mb: None, }; let terminal_rows = AtomicU64::new(u64::MAX); let export_result = export_database_sql_core(&state, &request, |progress| { @@ -447,6 +448,7 @@ async fn live_postgres_backup_snapshot_cancel_interrupts_pending_row_inner() { output_compression: Default::default(), snapshot_session_id: Some(snapshot.session_id.clone()), batch_size: 1, + split_max_mb: None, }; let rows_seen = Arc::new(AtomicU64::new(0)); let terminal_status = Arc::new(AtomicU64::new(0)); @@ -563,6 +565,7 @@ async fn live_mysql_database_backup_refreshes_an_idle_snapshot_before_export() { output_compression: Default::default(), snapshot_session_id: Some(snapshot.session_id.clone()), batch_size: 100, + split_max_mb: None, }; let export_result = export_database_sql_core(&state, &request, |_| {}).await; let rollback_result = rollback_manual_transaction(&state, &snapshot.session_id).await; diff --git a/crates/dbx-core/tests/live_mysql_database_export.rs b/crates/dbx-core/tests/live_mysql_database_export.rs index 1aa6b24a1f..79e74da41f 100644 --- a/crates/dbx-core/tests/live_mysql_database_export.rs +++ b/crates/dbx-core/tests/live_mysql_database_export.rs @@ -82,6 +82,7 @@ async fn live_mysql_database_export_restores_dependent_views() { output_compression: Default::default(), snapshot_session_id: None, batch_size: 1000, + split_max_mb: None, }; let test_result = async { export_database_sql_core(&state, &export_request, |_| {}).await?; @@ -217,6 +218,7 @@ async fn run_live_mysql_database_export_handles_many_tables_including_empty_tabl output_compression: Default::default(), snapshot_session_id: None, batch_size: 1000, + split_max_mb: None, }; export_database_sql_core(&state, &request, |_| {}).await?; @@ -311,6 +313,7 @@ async fn live_mysql_database_export_creates_missing_destination_directory() { output_compression: Default::default(), snapshot_session_id: None, batch_size: 1000, + split_max_mb: None, }; let result = export_database_sql_core(&state, &export_request, |_| {}).await; @@ -377,6 +380,7 @@ async fn live_mysql_database_export_refuses_to_recreate_a_destination_that_disap output_compression: Default::default(), snapshot_session_id: None, batch_size: 1000, + split_max_mb: None, }; export_database_sql_core(&state, &export_request, |_| {}).await.expect("first export should succeed"); @@ -464,6 +468,7 @@ async fn live_mysql_database_export_refuses_a_destination_that_vanished_before_i output_compression: Default::default(), snapshot_session_id: None, batch_size: 1000, + split_max_mb: None, }; // The mount disappears before the scheduler ever runs this schedule for diff --git a/crates/dbx-core/tests/live_mysql_export_table_order.rs b/crates/dbx-core/tests/live_mysql_export_table_order.rs index 110eaebd4f..177b897185 100644 --- a/crates/dbx-core/tests/live_mysql_export_table_order.rs +++ b/crates/dbx-core/tests/live_mysql_export_table_order.rs @@ -105,6 +105,7 @@ async fn live_mysql_database_export_table_order_is_not_alphabetical_when_fk_reor output_compression: Default::default(), snapshot_session_id: None, batch_size: 1000, + split_max_mb: None, }; let test_result = async { diff --git a/crates/dbx-core/tests/live_postgres_all_schema_export.rs b/crates/dbx-core/tests/live_postgres_all_schema_export.rs index cf74248203..30f3982a46 100644 --- a/crates/dbx-core/tests/live_postgres_all_schema_export.rs +++ b/crates/dbx-core/tests/live_postgres_all_schema_export.rs @@ -114,6 +114,7 @@ async fn live_postgres_all_schema_export_restores_one_sql_file() { output_compression: Default::default(), snapshot_session_id: None, batch_size: 1000, + split_max_mb: None, }, move |event| progress_sink.lock().expect("progress mutex poisoned").push(event), ) diff --git a/crates/dbx-web/src/routes/database_export.rs b/crates/dbx-web/src/routes/database_export.rs index b3009bc663..7830a1271a 100644 --- a/crates/dbx-web/src/routes/database_export.rs +++ b/crates/dbx-web/src/routes/database_export.rs @@ -44,7 +44,13 @@ pub async fn start_database_export( // endpoint after the export completes. let tmp_dir = state.data_dir.join("tmp"); std::fs::create_dir_all(&tmp_dir).map_err(|e| AppError::from(e.to_string()))?; - let extension = if req.output_compression == DatabaseExportOutputCompression::Gzip { "sql.gz" } else { "sql" }; + let extension = if req.split_max_mb.is_some() { + "zip" + } else if req.output_compression == DatabaseExportOutputCompression::Gzip { + "sql.gz" + } else { + "sql" + }; let tmp_file = tmp_dir.join(format!("database_export_{export_id}.{extension}")); let tmp_file_path = tmp_file.to_string_lossy().to_string(); req.file_path = tmp_file_path.clone(); @@ -170,8 +176,13 @@ pub async fn database_export_download( // Clean up temp file let _ = tokio::fs::remove_file(&export_file.file_path).await; - let content_type = - if export_file.format == "sql.gz" { "application/gzip" } else { "application/sql; charset=utf-8" }; + let content_type = if export_file.format == "zip" { + "application/zip" + } else if export_file.format == "sql.gz" { + "application/gzip" + } else { + "application/sql; charset=utf-8" + }; Response::builder() .status(StatusCode::OK) .header(header::CONTENT_TYPE, content_type) diff --git a/crates/dbx-web/src/routes/table_export.rs b/crates/dbx-web/src/routes/table_export.rs index 354bce206e..c9070e15ca 100644 --- a/crates/dbx-web/src/routes/table_export.rs +++ b/crates/dbx-web/src/routes/table_export.rs @@ -46,6 +46,7 @@ pub async fn start_table_export( "xlsx" => "xlsx", "json" => "json", "markdown" | "md" => "md", + "sql" if req.split_max_mb.is_some() => "zip", "sql" => "sql", _ => return Err(AppError::from(format!("Unsupported export format: {}", req.format))), }; @@ -59,7 +60,7 @@ pub async fn start_table_export( .export_files .write() .await - .insert(export_id.clone(), WebExportFile { file_path, download_filename, format: req.format.clone() }); + .insert(export_id.clone(), WebExportFile { file_path, download_filename, format: ext.to_string() }); let tx = { let mut channels = state.sse_channels.write().await; @@ -152,6 +153,7 @@ pub async fn table_export_download( "json" => "application/json; charset=utf-8", "markdown" | "md" => "text/markdown; charset=utf-8", "sql" => "application/sql; charset=utf-8", + "zip" => "application/zip", format => return Err(AppError::from(format!("Unknown format: {format}"))), };