From 705ea7464dd39c837da62f24c2fdb7ee6b958e4c Mon Sep 17 00:00:00 2001 From: ccq <76206374+caichangqing1120@users.noreply.github.com> Date: Sat, 26 Sep 2026 00:04:52 +0800 Subject: [PATCH] feat(export): run data dictionary PDF exports in background --- .../components/docs/DataDictionaryDialog.vue | 49 ++++-- .../export/ExportProgressPopover.vue | 16 +- .../__tests__/ExportProgressPopover.spec.ts | 18 ++ .../src/composables/useExportTracker.ts | 39 ++++- .../i18n/locales/dataDictionaryMessages.ts | 110 ++++++++++++ apps/desktop/src/lib/backend/api.ts | 1 + apps/desktop/src/lib/backend/http.ts | 5 + apps/desktop/src/lib/backend/tauri.ts | 13 ++ .../dataDictionaryExportTask.spec.ts | 159 +++++++++++++++++ .../src/lib/docs/dataDictionaryExportTask.ts | 123 +++++++++++++ .../src/lib/docs/dataDictionaryPdf.worker.ts | 12 ++ .../src/lib/docs/renderDataDictionaryPdf.ts | 18 ++ .../src/lib/docs/saveDataDictionaryFile.ts | 15 +- crates/dbx-core/src/data/docs/collector.rs | 165 ++++++++++-------- crates/dbx-core/src/data/docs/mod.rs | 2 +- crates/dbx-web/src/routes/docs.rs | 16 +- src-tauri/src/commands/docs.rs | 44 ++++- src-tauri/src/lib.rs | 1 + 18 files changed, 702 insertions(+), 104 deletions(-) create mode 100644 apps/desktop/src/lib/docs/__tests__/dataDictionaryExportTask.spec.ts create mode 100644 apps/desktop/src/lib/docs/dataDictionaryExportTask.ts create mode 100644 apps/desktop/src/lib/docs/dataDictionaryPdf.worker.ts create mode 100644 apps/desktop/src/lib/docs/renderDataDictionaryPdf.ts diff --git a/apps/desktop/src/components/docs/DataDictionaryDialog.vue b/apps/desktop/src/components/docs/DataDictionaryDialog.vue index dc0f8c85ae..757be66af8 100644 --- a/apps/desktop/src/components/docs/DataDictionaryDialog.vue +++ b/apps/desktop/src/components/docs/DataDictionaryDialog.vue @@ -31,7 +31,9 @@ import { type DictionaryTemplateId, } from "@/lib/docs/dataDictionary"; import { buildDataDictionaryPdf } from "@/lib/docs/dataDictionaryPdf"; -import { DictionaryFileExistsError, saveDataDictionaryFile } from "@/lib/docs/saveDataDictionaryFile"; +import { DictionaryFileExistsError, chooseDataDictionaryPath } from "@/lib/docs/saveDataDictionaryFile"; +import { hasActiveDictionaryExport, startDataDictionaryExport } from "@/lib/docs/dataDictionaryExportTask"; +import { useToast } from "@/composables/useToast"; import { isTauriRuntime } from "@/lib/backend/tauriRuntime"; import { useConnectionStore } from "@/stores/connectionStore"; @@ -46,14 +48,13 @@ const props = defineProps<{ const open = defineModel("open", { default: false }); const { t } = useI18n(); const connectionStore = useConnectionStore(); +const { toast } = useToast(); const step = ref(0); const loading = ref(false); const exporting = ref(false); const loadError = ref(null); const exportError = ref(null); -const generatedPath = ref(""); -const elapsed = ref(0); const databases = ref>([]); const schemas = ref>([]); const objects = ref([]); @@ -67,7 +68,6 @@ const continueOnError = ref(true); const outputPath = ref(""); const previewUrl = ref(""); let generation = 0; -let elapsedTimer = 0; const steps = computed(() => [t("dataDictionary.stepDatabases"), t("dataDictionary.stepObjects"), t("dataDictionary.stepTemplate"), t("dataDictionary.stepLayout"), t("dataDictionary.stepFile")]); const dbType = computed(() => connectionStore.getConfig(props.prefillConnectionId || "")?.db_type); @@ -312,22 +312,42 @@ async function browse(): Promise { } async function exportDictionary(): Promise { + const connectionId = props.prefillConnectionId; + if (!connectionId) return; + if (hasActiveDictionaryExport(connectionId)) { + exportError.value = t("dataDictionary.alreadyRunning"); + return; + } exporting.value = true; exportError.value = null; - generatedPath.value = ""; - elapsed.value = 0; - elapsedTimer = window.setInterval(() => (elapsed.value += 1), 1000); try { const fallback = dataDictionaryFileName(checkedDatabases.value[0] || "DataDictionary", false); const suggested = resolveDictionaryExportPath(outputPath.value, fallback, appendTimestamp.value); - const document = await collectSelection(); - const bytes = buildDataDictionaryPdf(document.tables, labels(), layout.value, continueOnError.value ? document.warnings : []); - const saved = await saveDataDictionaryFile(suggested, bytes, { overwrite: overwrite.value }); - if (saved) generatedPath.value = saved; + const path = await chooseDataDictionaryPath(suggested); + if (!path) return; + const started = startDataDictionaryExport({ + connectionId, + databases: [...checkedDatabases.value], + schemas: checkedSchemas.value.map((item) => ({ database: item.database, name: item.name })), + objects: orderedObjects.value.map((item) => ({ ...item })), + snapshot: props.snapshot, + layout: { ...layout.value }, + labels: labels(), + outputPath: path, + overwrite: overwrite.value, + continueOnError: continueOnError.value, + warningText: (warning) => warningText(warning), + skippedDatabase: (database, reason) => t("dataDictionary.warningTableSkipped", { table: database, reason }), + missingObject: (object) => t("dataDictionary.warningTableSkipped", { table: objectLabel(object), reason: t("dataDictionary.objectUnavailable") }), + onSuccess: (saved, warnings) => toast(t("dataDictionary.backgroundDone", { path: saved }) + (warnings ? ` · ${t("dataDictionary.warningCount", { count: warnings })}` : ""), 8000), + onFailure: (error) => toast(t("dataDictionary.backgroundFailed", { error: error instanceof DictionaryFileExistsError ? t("dataDictionary.fileExists") : error instanceof Error ? error.message : String(error) }), 8000), + }); + if (!started) throw new Error(t("dataDictionary.alreadyRunning")); + toast(t("dataDictionary.backgroundStarted")); + open.value = false; } catch (error) { exportError.value = error instanceof DictionaryFileExistsError ? t("dataDictionary.fileExists") : t("dataDictionary.exportFailed", { error: error instanceof Error ? error.message : String(error) }); } finally { - window.clearInterval(elapsedTimer); exporting.value = false; } } @@ -420,7 +440,6 @@ watch( return; } step.value = 0; - generatedPath.value = ""; exportError.value = null; outputPath.value = ""; appendTimestamp.value = false; @@ -434,7 +453,6 @@ watch( onBeforeUnmount(() => { generation += 1; - window.clearInterval(elapsedTimer); if (previewUrl.value) URL.revokeObjectURL(previewUrl.value); }); @@ -597,8 +615,7 @@ onBeforeUnmount(() => { >{{ t("dataDictionary.continueOnError") }} -

{{ t("dataDictionary.generating") }} {{ elapsed }}s

-

{{ t("dataDictionary.generated") }} {{ generatedPath }}

+

{{ t("dataDictionary.exporting") }}

{{ exportError }}

diff --git a/apps/desktop/src/components/export/ExportProgressPopover.vue b/apps/desktop/src/components/export/ExportProgressPopover.vue index 4410436277..a54766ce26 100644 --- a/apps/desktop/src/components/export/ExportProgressPopover.vue +++ b/apps/desktop/src/components/export/ExportProgressPopover.vue @@ -61,6 +61,9 @@ const progressPercent = (totalRows: number | null, rowsExported: number) => { }; const progressValue = (task: ExportTask) => { + if (task.kind === "data-dictionary") { + return task.dictionaryTotal ? Math.min(95, Math.round(((task.dictionaryCompleted ?? 0) / task.dictionaryTotal) * 95)) : 0; + } if (task.kind === "database-export") { if (task.overallPercent !== undefined) return task.overallPercent; if (!task.totalObjects || task.totalObjects <= 0) return 0; @@ -94,6 +97,7 @@ const taskFileName = (task: ExportTask) => { }; const taskTitle = (task: ExportTask) => { + if (task.kind === "data-dictionary") return `${t("dataDictionary.title")}: ${task.tableName}`; // database-export and sql-file keep their task labels on purpose: the former // names the database (filePath may be a directory), the latter already names // the executed script. Only table-export synthesizes a misleading @@ -125,6 +129,14 @@ const comparePhaseText = (phase?: string) => { }; const rowsText = (task: ExportTask) => { + if (task.kind === "data-dictionary") { + if (task.status === "Done") return [t("exportProgress.done"), task.dictionaryWarnings ? t("dataDictionary.warningCount", { count: task.dictionaryWarnings }) : ""].filter(Boolean).join(" · "); + if (task.status === "Error") return t("exportProgress.error"); + const phase = t(`dataDictionary.phase${(task.dictionaryPhase ?? "preparing").replace(/^./, (letter) => letter.toUpperCase())}`); + const count = task.dictionaryPhase === "collecting" && task.dictionaryTotal ? t("exportProgress.objectsCount", { current: (task.dictionaryCompleted ?? 0).toLocaleString(), total: task.dictionaryTotal.toLocaleString() }) : ""; + const warnings = task.dictionaryWarnings ? t("dataDictionary.warningCount", { count: task.dictionaryWarnings }) : ""; + return [phase, count, task.dictionaryCurrent, warnings].filter(Boolean).join(" · "); + } if (task.kind === "schema-diff") { if (isActive(task.status)) { const progress = task.compareTotal ? `${task.compareCurrent ?? 0}/${task.compareTotal}` : ""; @@ -217,6 +229,7 @@ const databaseObjectText = (task: ExportTask) => { const statusIcon = (task: ExportTask) => { if (isActive(task.status)) { if (task.kind === "database-export") return DatabaseBackup; + if (task.kind === "data-dictionary") return FileDown; if (task.kind === "sql-file") return FileCode2; if (task.kind === "data-transfer") return ArrowRightLeft; if (task.kind === "multi-db-execution") return Layers3; @@ -262,7 +275,7 @@ function toggleShowAll() { // Reveal is offered only for tasks that produce one local output file. // sql-file filePath can be a "; "-joined list of input scripts (not an // output), and data-transfer has no local file at all. -const canRevealTaskFile = (task: ExportTask) => (task.kind === "table-export" || task.kind === "database-export") && task.status === "Done" && !!task.filePath && isTauriRuntime(); +const canRevealTaskFile = (task: ExportTask) => (task.kind === "table-export" || task.kind === "database-export" || task.kind === "data-dictionary") && task.status === "Done" && !!task.filePath && isTauriRuntime(); async function revealTaskFile(task: ExportTask) { if (!canRevealTaskFile(task) || revealingTaskIds.value.includes(task.exportId)) return; @@ -359,6 +372,7 @@ function openTask(task: ExportTask): void { v-if=" task.totalRows || (task.kind === 'database-export' && (task.totalObjects || task.overallPercent !== undefined)) || + (task.kind === 'data-dictionary' && task.dictionaryPhase === 'collecting' && task.dictionaryProgressKnown && task.dictionaryTotal) || (task.kind === 'data-transfer' && task.totalTables) || (task.kind === 'multi-db-execution' && task.multiDbTotal) || ((task.kind === 'schema-diff' || task.kind === 'data-compare') && task.compareTotal) diff --git a/apps/desktop/src/components/export/__tests__/ExportProgressPopover.spec.ts b/apps/desktop/src/components/export/__tests__/ExportProgressPopover.spec.ts index 2c0838250b..34e3f0976c 100644 --- a/apps/desktop/src/components/export/__tests__/ExportProgressPopover.spec.ts +++ b/apps/desktop/src/components/export/__tests__/ExportProgressPopover.spec.ts @@ -94,6 +94,24 @@ describe("failure detail copy text", () => { }); }); +describe("data dictionary background task", () => { + it("shows actual collected-object progress and retains the finished output after the wizard closes", async () => { + const tracker = useExportTracker(); + tracker.addDataDictionaryTask("dictionary-task", "shop", 2); + tracker.updateDataDictionaryTask("dictionary-task", { dictionaryPhase: "collecting", dictionaryProgressKnown: true, dictionaryCompleted: 1, dictionaryTotal: 2 }); + await mountPopover(); + expect(document.body.textContent).toContain("Export Data Dictionary: shop"); + expect(document.body.textContent).toContain("1 / 2 objects"); + tracker.updateDataDictionaryTask("dictionary-task", { dictionaryPhase: "saving", status: "Writing" }); + await nextTick(); + expect(document.body.textContent).toContain("Saving file"); + tracker.updateDataDictionaryTask("dictionary-task", { status: "Done", filePath: "/tmp/shop.pdf" }); + await nextTick(); + expect(document.body.textContent).toContain("Export completed!"); + expect(document.querySelector('[title="Open containing folder"]')).not.toBeNull(); + }); +}); + describe("SQL file byte progress", () => { it("keeps legacy progress indeterminate instead of using successful statements as the total", async () => { const tracker = useExportTracker(); diff --git a/apps/desktop/src/composables/useExportTracker.ts b/apps/desktop/src/composables/useExportTracker.ts index 8bc463ae79..abde14a765 100644 --- a/apps/desktop/src/composables/useExportTracker.ts +++ b/apps/desktop/src/composables/useExportTracker.ts @@ -4,7 +4,7 @@ import { isTerminalTransferProgress } from "@/lib/backend/transferProgress"; import { uuid } from "@/lib/common/utils"; import { formatQueryDuration } from "@/lib/format/duration"; -export type BackgroundTaskKind = "table-export" | "database-export" | "sql-file" | "data-transfer" | "multi-db-execution" | "schema-diff" | "data-compare"; +export type BackgroundTaskKind = "table-export" | "database-export" | "data-dictionary" | "sql-file" | "data-transfer" | "multi-db-execution" | "schema-diff" | "data-compare"; export type BackgroundTaskStatus = "Running" | "Writing" | "Cancelling" | "Done" | "Error" | "Cancelled"; export type DatabaseExportSource = "manual" | "scheduled"; @@ -86,6 +86,12 @@ export interface ExportTask { compareAddedCount?: number; compareRemovedCount?: number; compareModifiedCount?: number; + dictionaryPhase?: "preparing" | "collecting" | "generating" | "saving"; + dictionaryCompleted?: number; + dictionaryTotal?: number; + dictionaryCurrent?: string; + dictionaryWarnings?: number; + dictionaryProgressKnown?: boolean; canCancel?: boolean; onOpen?: () => void; onRemove?: () => void; @@ -422,6 +428,35 @@ export function useExportTracker() { return task; } + function addDataDictionaryTask(exportId: string, label: string, total: number): ExportTask { + const task = reactive({ + exportId, + kind: "data-dictionary", + tableName: label, + format: "pdf", + filePath: "", + rowsExported: 0, + totalRows: null, + status: "Running", + errorMessage: null, + dictionaryPhase: "preparing", + dictionaryCompleted: 0, + dictionaryTotal: total, + dictionaryProgressKnown: false, + canCancel: false, + startedAt: Date.now(), + }); + taskMap.set(exportId, task); + return task; + } + + function updateDataDictionaryTask(exportId: string, changes: Partial): void { + const task = taskMap.get(exportId); + if (!task || task.kind !== "data-dictionary") return; + Object.assign(task, changes); + if (task.status === "Done" || task.status === "Error") finishExportTask(task); + } + function addSqlFileTask(executionId: string, fileName: string, filePath: string): ExportTask { sqlFileFailureStates.delete(executionId); const task = reactive({ @@ -771,6 +806,8 @@ export function useExportTracker() { hasActive, addTask, addDatabaseExportTask, + addDataDictionaryTask, + updateDataDictionaryTask, addSqlFileTask, addDataTransferTask, addSchemaDiffTask, diff --git a/apps/desktop/src/i18n/locales/dataDictionaryMessages.ts b/apps/desktop/src/i18n/locales/dataDictionaryMessages.ts index 633ceea9c3..9cceed9bb8 100644 --- a/apps/desktop/src/i18n/locales/dataDictionaryMessages.ts +++ b/apps/desktop/src/i18n/locales/dataDictionaryMessages.ts @@ -12,6 +12,7 @@ export const dataDictionaryEn = { loadFailed: "Could not read structure: {error}", warningsTitle: "This dictionary is incomplete", warningTableSkipped: "{table} was skipped: {reason}", + objectUnavailable: "The object was not found during export.", warningNoForeignKeys: "{engine} does not report foreign keys.", warningCommentsUnsupported: "{engine} does not support table or column comments.", empty: "No tables to export.", @@ -97,6 +98,15 @@ export const dataDictionaryEn = { continueOnError: "Continue on error", generating: "Generating", generated: "Data dictionary generated:", + phasePreparing: "Preparing objects", + phaseCollecting: "Reading metadata", + phaseGenerating: "Generating PDF", + phaseSaving: "Saving file", + warningCount: "{count} warning(s)", + alreadyRunning: "An export is already running for this connection.", + backgroundStarted: "Export started in background tasks.", + backgroundDone: "Data dictionary exported: {path}", + backgroundFailed: "Data dictionary export failed: {error}", fileExists: "The file already exists.", contentsHeading: "Contents", kindTable: "Table", @@ -123,6 +133,7 @@ export const dataDictionaryZhCN = { loadFailed: "无法读取结构:{error}", warningsTitle: "这份字典不完整", warningTableSkipped: "{table} 被跳过:{reason}", + objectUnavailable: "导出时未找到该对象。", warningNoForeignKeys: "{engine} 不报告外键。", warningCommentsUnsupported: "{engine} 不支持表或列注释。", empty: "没有可导出的表。", @@ -208,6 +219,15 @@ export const dataDictionaryZhCN = { continueOnError: "遇到错误时继续", generating: "正在生成", generated: "已生成数据字典:", + phasePreparing: "正在准备对象", + phaseCollecting: "正在读取元数据", + phaseGenerating: "正在生成 PDF", + phaseSaving: "正在保存文件", + warningCount: "{count} 条警告", + alreadyRunning: "该连接已有数据字典导出任务在运行。", + backgroundStarted: "导出已转入后台任务。", + backgroundDone: "数据字典已导出:{path}", + backgroundFailed: "数据字典导出失败:{error}", fileExists: "文件已存在。", contentsHeading: "目录", kindTable: "表", @@ -234,6 +254,7 @@ export const dataDictionaryZhTW = { loadFailed: "無法讀取結構:{error}", warningsTitle: "這份字典不完整", warningTableSkipped: "{table} 已略過:{reason}", + objectUnavailable: "匯出時找不到該物件。", warningNoForeignKeys: "{engine} 不回報外鍵。", warningCommentsUnsupported: "{engine} 不支援資料表或欄位註解。", empty: "沒有可匯出的資料表。", @@ -319,6 +340,15 @@ export const dataDictionaryZhTW = { continueOnError: "遇到錯誤時繼續", generating: "正在產生", generated: "已產生資料字典:", + phasePreparing: "正在準備物件", + phaseCollecting: "正在讀取中繼資料", + phaseGenerating: "正在產生 PDF", + phaseSaving: "正在儲存檔案", + warningCount: "{count} 條警告", + alreadyRunning: "此連線已有資料字典匯出任務在執行。", + backgroundStarted: "匯出已移至背景任務。", + backgroundDone: "資料字典已匯出:{path}", + backgroundFailed: "資料字典匯出失敗:{error}", fileExists: "檔案已存在。", contentsHeading: "目錄", kindTable: "資料表", @@ -345,6 +375,7 @@ export const dataDictionaryJa = { loadFailed: "構造を読み込めません: {error}", warningsTitle: "この辞書は不完全です", warningTableSkipped: "{table} はスキップされました: {reason}", + objectUnavailable: "The object was not found during export.", warningNoForeignKeys: "{engine} は外部キーを報告しません。", warningCommentsUnsupported: "{engine} はテーブルまたは列のコメントに対応していません。", empty: "エクスポートするテーブルがありません。", @@ -430,6 +461,15 @@ export const dataDictionaryJa = { continueOnError: "Continue on error", generating: "Generating", generated: "Data dictionary generated:", + phasePreparing: "Preparing objects", + phaseCollecting: "Reading metadata", + phaseGenerating: "Generating PDF", + phaseSaving: "Saving file", + warningCount: "{count} warning(s)", + alreadyRunning: "An export is already running for this connection.", + backgroundStarted: "Export started in background tasks.", + backgroundDone: "Data dictionary exported: {path}", + backgroundFailed: "Data dictionary export failed: {error}", fileExists: "The file already exists.", contentsHeading: "Contents", kindTable: "Table", @@ -456,6 +496,7 @@ export const dataDictionaryKo = { loadFailed: "구조를 읽을 수 없습니다: {error}", warningsTitle: "이 사전은 불완전합니다", warningTableSkipped: "{table}을(를) 건너뛰었습니다: {reason}", + objectUnavailable: "The object was not found during export.", warningNoForeignKeys: "{engine}은(는) 외래 키를 보고하지 않습니다.", warningCommentsUnsupported: "{engine}은(는) 테이블 또는 열 주석을 지원하지 않습니다.", empty: "내보낼 테이블이 없습니다.", @@ -541,6 +582,15 @@ export const dataDictionaryKo = { continueOnError: "Continue on error", generating: "Generating", generated: "Data dictionary generated:", + phasePreparing: "Preparing objects", + phaseCollecting: "Reading metadata", + phaseGenerating: "Generating PDF", + phaseSaving: "Saving file", + warningCount: "{count} warning(s)", + alreadyRunning: "An export is already running for this connection.", + backgroundStarted: "Export started in background tasks.", + backgroundDone: "Data dictionary exported: {path}", + backgroundFailed: "Data dictionary export failed: {error}", fileExists: "The file already exists.", contentsHeading: "Contents", kindTable: "Table", @@ -567,6 +617,7 @@ export const dataDictionaryEs = { loadFailed: "No se pudo leer la estructura: {error}", warningsTitle: "Este diccionario está incompleto", warningTableSkipped: "{table} se omitió: {reason}", + objectUnavailable: "The object was not found during export.", warningNoForeignKeys: "{engine} no informa claves foráneas.", warningCommentsUnsupported: "{engine} no admite comentarios de tabla o columna.", empty: "No hay tablas para exportar.", @@ -652,6 +703,15 @@ export const dataDictionaryEs = { continueOnError: "Continue on error", generating: "Generating", generated: "Data dictionary generated:", + phasePreparing: "Preparing objects", + phaseCollecting: "Reading metadata", + phaseGenerating: "Generating PDF", + phaseSaving: "Saving file", + warningCount: "{count} warning(s)", + alreadyRunning: "An export is already running for this connection.", + backgroundStarted: "Export started in background tasks.", + backgroundDone: "Data dictionary exported: {path}", + backgroundFailed: "Data dictionary export failed: {error}", fileExists: "The file already exists.", contentsHeading: "Contents", kindTable: "Table", @@ -678,6 +738,7 @@ export const dataDictionaryIt = { loadFailed: "Impossibile leggere la struttura: {error}", warningsTitle: "Questo dizionario è incompleto", warningTableSkipped: "{table} è stata ignorata: {reason}", + objectUnavailable: "The object was not found during export.", warningNoForeignKeys: "{engine} non riporta le chiavi esterne.", warningCommentsUnsupported: "{engine} non supporta i commenti di tabella o colonna.", empty: "Nessuna tabella da esportare.", @@ -763,6 +824,15 @@ export const dataDictionaryIt = { continueOnError: "Continue on error", generating: "Generating", generated: "Data dictionary generated:", + phasePreparing: "Preparing objects", + phaseCollecting: "Reading metadata", + phaseGenerating: "Generating PDF", + phaseSaving: "Saving file", + warningCount: "{count} warning(s)", + alreadyRunning: "An export is already running for this connection.", + backgroundStarted: "Export started in background tasks.", + backgroundDone: "Data dictionary exported: {path}", + backgroundFailed: "Data dictionary export failed: {error}", fileExists: "The file already exists.", contentsHeading: "Contents", kindTable: "Table", @@ -789,6 +859,7 @@ export const dataDictionaryPtBR = { loadFailed: "Não foi possível ler a estrutura: {error}", warningsTitle: "Este dicionário está incompleto", warningTableSkipped: "{table} foi ignorada: {reason}", + objectUnavailable: "The object was not found during export.", warningNoForeignKeys: "{engine} não informa chaves estrangeiras.", warningCommentsUnsupported: "{engine} não oferece comentários de tabela ou coluna.", empty: "Não há tabelas para exportar.", @@ -874,6 +945,15 @@ export const dataDictionaryPtBR = { continueOnError: "Continue on error", generating: "Generating", generated: "Data dictionary generated:", + phasePreparing: "Preparing objects", + phaseCollecting: "Reading metadata", + phaseGenerating: "Generating PDF", + phaseSaving: "Saving file", + warningCount: "{count} warning(s)", + alreadyRunning: "An export is already running for this connection.", + backgroundStarted: "Export started in background tasks.", + backgroundDone: "Data dictionary exported: {path}", + backgroundFailed: "Data dictionary export failed: {error}", fileExists: "The file already exists.", contentsHeading: "Contents", kindTable: "Table", @@ -900,6 +980,7 @@ export const dataDictionaryTr = { loadFailed: "Yapı okunamadı: {error}", warningsTitle: "Bu sözlük eksik", warningTableSkipped: "{table} atlandı: {reason}", + objectUnavailable: "The object was not found during export.", warningNoForeignKeys: "{engine} yabancı anahtar bildirmiyor.", warningCommentsUnsupported: "{engine} tablo veya sütun açıklamalarını desteklemiyor.", empty: "Dışa aktarılacak tablo yok.", @@ -985,6 +1066,15 @@ export const dataDictionaryTr = { continueOnError: "Continue on error", generating: "Generating", generated: "Data dictionary generated:", + phasePreparing: "Preparing objects", + phaseCollecting: "Reading metadata", + phaseGenerating: "Generating PDF", + phaseSaving: "Saving file", + warningCount: "{count} warning(s)", + alreadyRunning: "An export is already running for this connection.", + backgroundStarted: "Export started in background tasks.", + backgroundDone: "Data dictionary exported: {path}", + backgroundFailed: "Data dictionary export failed: {error}", fileExists: "The file already exists.", contentsHeading: "Contents", kindTable: "Table", @@ -1011,6 +1101,7 @@ export const dataDictionaryRu = { loadFailed: "Не удалось прочитать структуру: {error}", warningsTitle: "Этот словарь неполный", warningTableSkipped: "{table} пропущена: {reason}", + objectUnavailable: "The object was not found during export.", warningNoForeignKeys: "{engine} не сообщает внешние ключи.", warningCommentsUnsupported: "{engine} не поддерживает комментарии таблиц или столбцов.", empty: "Нет таблиц для экспорта.", @@ -1096,6 +1187,15 @@ export const dataDictionaryRu = { continueOnError: "Continue on error", generating: "Generating", generated: "Data dictionary generated:", + phasePreparing: "Preparing objects", + phaseCollecting: "Reading metadata", + phaseGenerating: "Generating PDF", + phaseSaving: "Saving file", + warningCount: "{count} warning(s)", + alreadyRunning: "An export is already running for this connection.", + backgroundStarted: "Export started in background tasks.", + backgroundDone: "Data dictionary exported: {path}", + backgroundFailed: "Data dictionary export failed: {error}", fileExists: "The file already exists.", contentsHeading: "Contents", kindTable: "Table", @@ -1122,6 +1222,7 @@ export const dataDictionaryAz = { loadFailed: "Strukturu oxumaq mümkün olmadı: {error}", warningsTitle: "Bu lüğət tam deyil", warningTableSkipped: "{table} ötürülüb: {reason}", + objectUnavailable: "The object was not found during export.", warningNoForeignKeys: "{engine} xarici açarları bildirmir.", warningCommentsUnsupported: "{engine} cədvəl və ya sütun şərhlərini dəstəkləmir.", empty: "İxrac ediləcək cədvəl yoxdur.", @@ -1207,6 +1308,15 @@ export const dataDictionaryAz = { continueOnError: "Continue on error", generating: "Generating", generated: "Data dictionary generated:", + phasePreparing: "Preparing objects", + phaseCollecting: "Reading metadata", + phaseGenerating: "Generating PDF", + phaseSaving: "Saving file", + warningCount: "{count} warning(s)", + alreadyRunning: "An export is already running for this connection.", + backgroundStarted: "Export started in background tasks.", + backgroundDone: "Data dictionary exported: {path}", + backgroundFailed: "Data dictionary export failed: {error}", fileExists: "The file already exists.", contentsHeading: "Contents", kindTable: "Table", diff --git a/apps/desktop/src/lib/backend/api.ts b/apps/desktop/src/lib/backend/api.ts index b842f139e3..789fcfbc4b 100644 --- a/apps/desktop/src/lib/backend/api.ts +++ b/apps/desktop/src/lib/backend/api.ts @@ -255,6 +255,7 @@ export const listDialectDataTypes = forward("listDialectDataTypes"); // Docs export const collectDocsSnapshot = forward("collectDocsSnapshot"); +export const collectDocsSnapshotForExport = forward("collectDocsSnapshotForExport"); export const loadDocsAnnotations = forward("loadDocsAnnotations"); export const applyDocsAnnotations = forward("applyDocsAnnotations"); export const saveDocsAnnotations = forward("saveDocsAnnotations"); diff --git a/apps/desktop/src/lib/backend/http.ts b/apps/desktop/src/lib/backend/http.ts index e582a24436..b5f29274c2 100644 --- a/apps/desktop/src/lib/backend/http.ts +++ b/apps/desktop/src/lib/backend/http.ts @@ -1352,6 +1352,11 @@ export async function collectDocsSnapshot(connectionId: string, database: string return post("/api/docs/snapshot", { connectionId, database, schemas, tables, projectName }); } +// HTTP snapshot transport is request/response; its progress remains indeterminate. +export async function collectDocsSnapshotForExport(connectionId: string, database: string, schemas: string[], tables: string[], _onProgress: (progress: import("./tauri").DocsCollectProgress) => void): Promise { + return post("/api/docs/snapshot", { connectionId, database, schemas, tables, projectName: database, maxConcurrentTables: 2 }); +} + export async function loadDocsAnnotations(connectionId: string): Promise { return post("/api/docs/annotations/load", { connectionId }); } diff --git a/apps/desktop/src/lib/backend/tauri.ts b/apps/desktop/src/lib/backend/tauri.ts index e8fd7c5e6b..3d992303b7 100644 --- a/apps/desktop/src/lib/backend/tauri.ts +++ b/apps/desktop/src/lib/backend/tauri.ts @@ -1,4 +1,5 @@ import { invoke as tauriInvoke } from "@tauri-apps/api/core"; +import { Channel } from "@tauri-apps/api/core"; import type { MongoDumpFormat, MongoDumpSourceInput, MongoDumpCatalog, MongoRestoreSourcePreview, MongoDatabaseDumpRequest, MongoDatabaseRestoreRequest, MongoDatabaseDumpProgress } from "./mongodbDumpTypes"; import type { MongoRestoreUpload, MongoSourceReadOptions } from "./mongodbDumpTypes"; import type { UserSkillRootSettings, UserSkillsListResult, UserSkillsReadResult } from "@/types/userSkills"; @@ -2465,6 +2466,18 @@ export async function collectDocsSnapshot(connectionId: string, database: string return invoke("docs_collect_snapshot", { connectionId, database, schemas, tables, projectName }); } +export interface DocsCollectProgress { + completed: number; + total: number; + current: string; +} + +export async function collectDocsSnapshotForExport(connectionId: string, database: string, schemas: string[], tables: string[], onProgress: (progress: DocsCollectProgress) => void): Promise { + const channel = new Channel(); + channel.onmessage = onProgress; + return invoke("docs_collect_snapshot_for_export", { connectionId, database, schemas, tables, projectName: database, onProgress: channel }); +} + export async function loadDocsAnnotations(connectionId: string): Promise { return invoke("docs_load_annotations", { connectionId }); } diff --git a/apps/desktop/src/lib/docs/__tests__/dataDictionaryExportTask.spec.ts b/apps/desktop/src/lib/docs/__tests__/dataDictionaryExportTask.spec.ts new file mode 100644 index 0000000000..9612b5b17e --- /dev/null +++ b/apps/desktop/src/lib/docs/__tests__/dataDictionaryExportTask.spec.ts @@ -0,0 +1,159 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { useExportTracker } from "@/composables/useExportTracker"; +import { hasActiveDictionaryExport, startDataDictionaryExport, type DictionaryExportRequest } from "@/lib/docs/dataDictionaryExportTask"; +import * as api from "@/lib/backend/api"; +import { renderDataDictionaryPdf } from "@/lib/docs/renderDataDictionaryPdf"; +import { saveDataDictionaryFile } from "@/lib/docs/saveDataDictionaryFile"; + +vi.mock("@/lib/backend/api", () => ({ + collectDocsSnapshotForExport: vi.fn(), + loadDocsAnnotations: vi.fn(), + applyDocsAnnotations: vi.fn(), +})); +vi.mock("@/lib/docs/renderDataDictionaryPdf", () => ({ renderDataDictionaryPdf: vi.fn() })); +vi.mock("@/lib/docs/saveDataDictionaryFile", () => ({ saveDataDictionaryFile: vi.fn() })); + +const tracker = useExportTracker(); +const snapshot = (database: string, names: string[], warnings: unknown[] = []) => ({ + project: { database }, + tables: names.map((name) => ({ name, schema: null, kind: "TABLE" })), + warnings, +}); + +function request(databases = ["shop"], names = ["a"]): DictionaryExportRequest { + return { + connectionId: "connection", + databases, + schemas: [], + objects: databases.flatMap((database) => names.map((name) => ({ database, schema: "", name, kind: "TABLE" as const }))), + layout: {} as DictionaryExportRequest["layout"], + labels: {} as DictionaryExportRequest["labels"], + outputPath: "/tmp/dictionary.pdf", + overwrite: true, + continueOnError: true, + warningText: (warning) => (warning.kind === "tableSkipped" ? (warning.reason ?? "skipped") : "warning"), + skippedDatabase: (database, reason) => `${database}: ${reason}`, + missingObject: (object) => `${object.database}.${object.name}: unavailable`, + onSuccess: vi.fn(), + onFailure: vi.fn(), + }; +} + +beforeEach(() => { + vi.mocked(api.collectDocsSnapshotForExport).mockReset(); + vi.mocked(api.loadDocsAnnotations).mockResolvedValue(null); + vi.mocked(api.applyDocsAnnotations).mockImplementation(async (_id, raw) => raw); + vi.mocked(renderDataDictionaryPdf).mockReset(); + vi.mocked(renderDataDictionaryPdf).mockResolvedValue(new Uint8Array([1, 2])); + vi.mocked(saveDataDictionaryFile).mockReset(); + vi.mocked(saveDataDictionaryFile).mockResolvedValue("/tmp/dictionary.pdf"); +}); + +afterEach(async () => { + await vi.waitFor(() => expect(hasActiveDictionaryExport("connection")).toBe(false)); + for (const task of tracker.tasks.value) tracker.removeTask(task.exportId); +}); + +describe("background dictionary export", () => { + it("survives its caller, tracks completed objects over multiple databases and finishes only after the file is written", async () => { + let resolveSave!: (path: string) => void; + vi.mocked(saveDataDictionaryFile).mockImplementation( + () => + new Promise((resolve) => { + resolveSave = resolve; + }), + ); + vi.mocked(api.collectDocsSnapshotForExport).mockImplementation(async (_id, database, _schemas, _names, progress) => { + progress({ completed: 0, total: 2, current: "" }); + progress({ completed: 1, total: 2, current: "a" }); + progress({ completed: 2, total: 2, current: "b" }); + return snapshot(database, ["a", "b"]) as never; + }); + const input = request(["shop", "archive"], ["a", "b"]); + expect(startDataDictionaryExport(input)).toBe(true); + expect(startDataDictionaryExport(input)).toBe(false); + await vi.waitFor(() => expect(saveDataDictionaryFile).toHaveBeenCalledOnce()); + const task = tracker.tasks.value[0]!; + expect(task.dictionaryCompleted).toBe(4); + expect(task.dictionaryTotal).toBe(4); + expect(task.status).toBe("Writing"); + expect(task.filePath).toBe(""); + expect(input.onSuccess).not.toHaveBeenCalled(); + resolveSave("/tmp/dictionary.pdf"); + await vi.waitFor(() => expect(task.status).toBe("Done")); + expect(task.filePath).toBe("/tmp/dictionary.pdf"); + expect(input.onSuccess).toHaveBeenCalledWith("/tmp/dictionary.pdf", 0); + expect(hasActiveDictionaryExport("connection")).toBe(false); + }); + + it("reports save failures, frees the connection slot and permits a retry", async () => { + vi.mocked(api.collectDocsSnapshotForExport).mockResolvedValue(snapshot("shop", ["a"]) as never); + vi.mocked(saveDataDictionaryFile).mockRejectedValueOnce(new Error("disk full")); + const input = request(); + startDataDictionaryExport(input); + await vi.waitFor(() => expect(tracker.tasks.value[0]?.status).toBe("Error")); + expect(tracker.tasks.value[0]?.errorMessage).toBe("disk full"); + expect(input.onSuccess).not.toHaveBeenCalled(); + expect(input.onFailure).toHaveBeenCalledOnce(); + expect(startDataDictionaryExport(request())).toBe(true); + await vi.waitFor(() => expect(tracker.tasks.value[1]?.status).toBe("Done")); + }); + + it("preserves skipped-object warnings only when continue-on-error is enabled", async () => { + vi.mocked(api.collectDocsSnapshotForExport).mockResolvedValue(snapshot("shop", ["a"], [{ kind: "tableSkipped", table: ".b", reason: "denied" }]) as never); + const input = request(["shop"], ["a", "b"]); + startDataDictionaryExport(input); + await vi.waitFor(() => expect(tracker.tasks.value[0]?.status).toBe("Done")); + expect(tracker.tasks.value[0]?.dictionaryWarnings).toBe(1); + expect(renderDataDictionaryPdf).toHaveBeenCalledWith(expect.any(Array), expect.anything(), expect.anything(), ["denied"]); + const strict = request(["shop"], ["a", "b"]); + strict.continueOnError = false; + startDataDictionaryExport(strict); + await vi.waitFor(() => expect(tracker.tasks.value[1]?.status).toBe("Error")); + expect(saveDataDictionaryFile).toHaveBeenCalledTimes(1); + }); + + it("does not silently export a partial PDF when a selected object disappears", async () => { + vi.mocked(api.collectDocsSnapshotForExport).mockResolvedValue(snapshot("shop", ["a"]) as never); + const input = request(["shop"], ["a", "b"]); + startDataDictionaryExport(input); + await vi.waitFor(() => expect(tracker.tasks.value[0]?.status).toBe("Done")); + expect(renderDataDictionaryPdf).toHaveBeenCalledWith(expect.any(Array), expect.anything(), expect.anything(), ["shop.b: unavailable"]); + expect(tracker.tasks.value[0]?.dictionaryWarnings).toBe(1); + + const strict = request(["shop"], ["a", "b"]); + strict.continueOnError = false; + startDataDictionaryExport(strict); + await vi.waitFor(() => expect(tracker.tasks.value[1]?.status).toBe("Error")); + expect(tracker.tasks.value[1]?.errorMessage).toContain("shop.b: unavailable"); + expect(saveDataDictionaryFile).toHaveBeenCalledTimes(1); + }); + + it("ignores stale and out-of-order progress callbacks", async () => { + let firstProgress!: Parameters[4]; + vi.mocked(api.collectDocsSnapshotForExport).mockImplementation(async (_id, database, _schemas, _names, progress) => { + if (database === "shop") { + firstProgress = progress; + progress({ completed: 2, total: 2, current: "b" }); + progress({ completed: 1, total: 2, current: "a" }); + } else { + firstProgress({ completed: 0, total: 2, current: "old" }); + progress({ completed: 1, total: 2, current: "new" }); + } + return snapshot(database, ["a", "b"]) as never; + }); + let resolveRender!: (bytes: Uint8Array) => void; + vi.mocked(renderDataDictionaryPdf).mockImplementation( + () => + new Promise((resolve) => { + resolveRender = resolve; + }), + ); + startDataDictionaryExport(request(["shop", "archive"], ["a", "b"])); + await vi.waitFor(() => expect(renderDataDictionaryPdf).toHaveBeenCalled()); + expect(tracker.tasks.value[0]?.dictionaryCompleted).toBe(4); + expect(tracker.tasks.value[0]?.dictionaryCurrent).toBe(""); + resolveRender(new Uint8Array([1])); + await vi.waitFor(() => expect(tracker.tasks.value[0]?.status).toBe("Done")); + }); +}); diff --git a/apps/desktop/src/lib/docs/dataDictionaryExportTask.ts b/apps/desktop/src/lib/docs/dataDictionaryExportTask.ts new file mode 100644 index 0000000000..ce3b322430 --- /dev/null +++ b/apps/desktop/src/lib/docs/dataDictionaryExportTask.ts @@ -0,0 +1,123 @@ +import * as api from "@/lib/backend/api"; +import { useExportTracker } from "@/composables/useExportTracker"; +import { emptyAnnotations } from "@/docs/annotationEdits"; +import type { SchemaSnapshot, SnapshotWarning } from "@/docs/types"; +import { uuid } from "@/lib/common/utils"; +import { exportBlockedBySkippedTables, objectKey, orderDictionaryTables, warningsForSelection, type DataDictionaryLabels, type DictionaryLayout, type DictionaryObjectRef, type DictionaryTable } from "./dataDictionary"; +import { renderDataDictionaryPdf } from "./renderDataDictionaryPdf"; +import { saveDataDictionaryFile } from "./saveDataDictionaryFile"; + +export interface DictionaryExportRequest { + connectionId: string; + databases: string[]; + schemas: Array<{ database: string; name: string }>; + objects: DictionaryObjectRef[]; + snapshot?: SchemaSnapshot | null; + layout: DictionaryLayout; + labels: DataDictionaryLabels; + outputPath: string; + overwrite: boolean; + continueOnError: boolean; + warningText: (warning: SnapshotWarning) => string; + skippedDatabase: (database: string, reason: string) => string; + missingObject: (object: DictionaryObjectRef) => string; + onSuccess: (path: string, warnings: number) => void; + onFailure: (error: unknown) => void; +} + +// Export lifetimes belong to the app, not the dialog instance. One job per +// connection also bounds the aggregate metadata pressure on its pool. +const activeConnections = new Set(); + +export function hasActiveDictionaryExport(connectionId: string): boolean { + return activeConnections.has(connectionId); +} + +export function startDataDictionaryExport(request: DictionaryExportRequest): boolean { + if (activeConnections.has(request.connectionId)) return false; + activeConnections.add(request.connectionId); + const tracker = useExportTracker(); + const id = uuid(); + tracker.addDataDictionaryTask(id, request.databases.join(", "), request.objects.length); + + void (async () => { + const warnings: string[] = []; + const tables: DictionaryTable[] = []; + const order = request.objects.map(objectKey); + let completed = 0; + try { + for (const database of request.databases) { + const selected = request.objects.filter((item) => item.database === database); + if (!selected.length) continue; + tracker.updateDataDictionaryTask(id, { dictionaryPhase: "preparing", dictionaryCompleted: completed, dictionaryCurrent: database }); + try { + let merged: SchemaSnapshot; + const fromSnapshot = + request.snapshot?.project.database === database + ? orderDictionaryTables( + request.snapshot.tables.map((table) => ({ ...table, database })), + selected.map(objectKey), + ) + : []; + if (request.snapshot && fromSnapshot.length === selected.length) { + merged = request.snapshot; + tracker.updateDataDictionaryTask(id, { dictionaryPhase: "collecting", dictionaryProgressKnown: true }); + } else { + const schemas = request.schemas.filter((item) => item.database === database).map((item) => item.name); + const names = selected.map((item) => (item.schema ? `${item.schema}.${item.name}` : item.name)); + const baseCompleted = completed; + let acceptingProgress = true; + let raw: SchemaSnapshot; + try { + raw = await api.collectDocsSnapshotForExport(request.connectionId, database, schemas, names, (progress) => { + if (!acceptingProgress) return; + const task = tracker.tasks.value.find((item) => item.exportId === id); + tracker.updateDataDictionaryTask(id, { + dictionaryPhase: "collecting", + dictionaryProgressKnown: progress.total > 0, + dictionaryCompleted: Math.max(task?.dictionaryCompleted ?? 0, baseCompleted + Math.min(progress.completed, selected.length)), + dictionaryCurrent: progress.current || database, + }); + }); + } finally { + acceptingProgress = false; + } + const file = (await api.loadDocsAnnotations(request.connectionId)) ?? emptyAnnotations(); + merged = await api.applyDocsAnnotations(request.connectionId, raw, file); + } + const relevant = warningsForSelection(merged.warnings, selected); + const collected = new Set(merged.tables.map((table) => objectKey({ database, schema: table.schema ?? "", name: table.name }))); + const missing = selected.filter((item) => !collected.has(objectKey(item))); + const missingWarnings = missing.filter((item) => !relevant.some((warning) => warning.kind === "tableSkipped" && (warning.table === "*" || warning.table === `${item.schema}.*` || warning.table === item.name || warning.table === `${item.schema}.${item.name}`))).map(request.missingObject); + if (exportBlockedBySkippedTables(relevant, request.continueOnError) || (!request.continueOnError && missingWarnings.length > 0)) { + throw new Error(relevant.map(request.warningText).find(Boolean) || missingWarnings[0] || database); + } + warnings.push(...relevant.map(request.warningText).filter(Boolean)); + warnings.push(...missingWarnings); + tables.push(...merged.tables.filter((table) => selected.some((item) => objectKey(item) === objectKey({ database, schema: table.schema ?? "", name: table.name }))).map((table) => ({ ...table, database }))); + } catch (error) { + if (!request.continueOnError) throw error; + warnings.push(request.skippedDatabase(database, error instanceof Error ? error.message : String(error))); + } + completed += selected.length; + tracker.updateDataDictionaryTask(id, { dictionaryCompleted: completed, dictionaryCurrent: "", dictionaryWarnings: warnings.length }); + } + if (!tables.length) throw new Error("No selected objects could be collected"); + tracker.updateDataDictionaryTask(id, { dictionaryPhase: "generating", dictionaryCurrent: "" }); + // Worker messages cannot clone Vue proxies; capture only serializable metadata. + const ordered = JSON.parse(JSON.stringify(orderDictionaryTables(tables, order))) as DictionaryTable[]; + const bytes = await renderDataDictionaryPdf(ordered, request.labels, request.layout, request.continueOnError ? warnings : []); + tracker.updateDataDictionaryTask(id, { dictionaryPhase: "saving", status: "Writing" }); + const path = await saveDataDictionaryFile(request.outputPath, bytes, { overwrite: request.overwrite, grantedPath: request.outputPath }); + if (!path) throw new Error("File was not saved"); + tracker.updateDataDictionaryTask(id, { status: "Done", filePath: path, dictionaryWarnings: warnings.length }); + request.onSuccess(path, warnings.length); + } catch (error) { + tracker.updateDataDictionaryTask(id, { status: "Error", errorMessage: error instanceof Error ? error.message : String(error) }); + request.onFailure(error); + } finally { + activeConnections.delete(request.connectionId); + } + })(); + return true; +} diff --git a/apps/desktop/src/lib/docs/dataDictionaryPdf.worker.ts b/apps/desktop/src/lib/docs/dataDictionaryPdf.worker.ts new file mode 100644 index 0000000000..9c74d2876b --- /dev/null +++ b/apps/desktop/src/lib/docs/dataDictionaryPdf.worker.ts @@ -0,0 +1,12 @@ +import { buildDataDictionaryPdf } from "./dataDictionaryPdf"; +import type { DataDictionaryLabels, DictionaryLayout, DictionaryTable } from "./dataDictionary"; + +self.onmessage = (event: MessageEvent<{ tables: DictionaryTable[]; labels: DataDictionaryLabels; layout: DictionaryLayout; warnings: string[] }>) => { + try { + const { tables, labels, layout, warnings } = event.data; + const bytes = buildDataDictionaryPdf(tables, labels, layout, warnings); + self.postMessage({ bytes }, { transfer: [bytes.buffer] }); + } catch (error) { + self.postMessage({ error: error instanceof Error ? error.message : String(error) }); + } +}; diff --git a/apps/desktop/src/lib/docs/renderDataDictionaryPdf.ts b/apps/desktop/src/lib/docs/renderDataDictionaryPdf.ts new file mode 100644 index 0000000000..efaf31a16b --- /dev/null +++ b/apps/desktop/src/lib/docs/renderDataDictionaryPdf.ts @@ -0,0 +1,18 @@ +import type { DataDictionaryLabels, DictionaryLayout, DictionaryTable } from "./dataDictionary"; + +export function renderDataDictionaryPdf(tables: DictionaryTable[], labels: DataDictionaryLabels, layout: DictionaryLayout, warnings: string[]): Promise { + return new Promise((resolve, reject) => { + const worker = new Worker(new URL("./dataDictionaryPdf.worker.ts", import.meta.url), { type: "module" }); + worker.onmessage = (event: MessageEvent<{ bytes?: Uint8Array; error?: string }>) => { + worker.terminate(); + if (event.data.error) reject(new Error(event.data.error)); + else if (event.data.bytes) resolve(event.data.bytes); + else reject(new Error("PDF worker returned no data")); + }; + worker.onerror = (event) => { + worker.terminate(); + reject(new Error(event.message)); + }; + worker.postMessage({ tables, labels, layout, warnings }); + }); +} diff --git a/apps/desktop/src/lib/docs/saveDataDictionaryFile.ts b/apps/desktop/src/lib/docs/saveDataDictionaryFile.ts index 35f0e95829..ad8f452ab1 100644 --- a/apps/desktop/src/lib/docs/saveDataDictionaryFile.ts +++ b/apps/desktop/src/lib/docs/saveDataDictionaryFile.ts @@ -7,18 +7,21 @@ export class DictionaryFileExistsError extends Error { } } +export async function chooseDataDictionaryPath(suggestedName: string): Promise { + if (!isTauriRuntime()) return suggestedName; + const { save } = await import("@tauri-apps/plugin-dialog"); + return save({ defaultPath: suggestedName, filters: [{ name: "PDF", extensions: ["pdf"] }] }); +} + /** * Suggested name is only the save dialog's defaultPath. The write uses the path * `save()` just returned. Tauri's dialog grant covers that exact path, not a * typed path or a timestamp appended after the dialog closes. */ -export async function saveDataDictionaryFile(suggestedName: string, content: Uint8Array, options: { overwrite: boolean }): Promise { +export async function saveDataDictionaryFile(suggestedName: string, content: Uint8Array, options: { overwrite: boolean; grantedPath?: string }): Promise { if (isTauriRuntime()) { - const [{ save }, fs] = await Promise.all([import("@tauri-apps/plugin-dialog"), import("@tauri-apps/plugin-fs")]); - const granted = await save({ - defaultPath: suggestedName, - filters: [{ name: "PDF", extensions: ["pdf"] }], - }); + const fs = await import("@tauri-apps/plugin-fs"); + const granted = options.grantedPath ?? (await chooseDataDictionaryPath(suggestedName)); if (!granted) return null; if (!options.overwrite && (await fs.exists(granted))) throw new DictionaryFileExistsError(); await fs.writeFile(granted, content); diff --git a/crates/dbx-core/src/data/docs/collector.rs b/crates/dbx-core/src/data/docs/collector.rs index ff6e702c75..cb9205ffc6 100644 --- a/crates/dbx-core/src/data/docs/collector.rs +++ b/crates/dbx-core/src/data/docs/collector.rs @@ -1,5 +1,5 @@ use std::collections::{BTreeMap, HashSet}; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use futures::stream::{self, StreamExt}; @@ -144,6 +144,18 @@ pub async fn collect_snapshot( options: &CollectOptions, progress: &(dyn Fn(CollectProgress) + Send + Sync), cancel: &AtomicBool, +) -> Result { + collect_snapshot_with_concurrency(state, connection, options, progress, cancel, MAX_CONCURRENT_TABLES).await +} + +/// Dictionary exports use a smaller limit to leave pool capacity for interactive queries. +pub async fn collect_snapshot_with_concurrency( + state: &AppState, + connection: &ConnectionConfig, + options: &CollectOptions, + progress: &(dyn Fn(CollectProgress) + Send + Sync), + cancel: &AtomicBool, + max_concurrent_tables: usize, ) -> Result { let mut warnings: Vec = Vec::new(); let engine = database_type_label(connection.db_type); @@ -199,64 +211,64 @@ pub async fn collect_snapshot( let total = targets.len(); + progress(CollectProgress { completed: 0, total, current: String::new() }); + let completed = AtomicUsize::new(0); let collected: Vec), SnapshotWarning>> = stream::iter(targets.into_iter().enumerate()) - .map(|(index, (schema_name, info))| { + .map(|(_index, (schema_name, info))| { let database = options.database.clone(); + let completed = &completed; async move { - if cancelled(cancel) { - return Err(SnapshotWarning::TableSkipped { - table: info.name.clone(), - reason: "cancelled".to_string(), - }); - } - - progress(CollectProgress { - completed: index, - total, - current: format!("{schema_name}.{}", info.name), - }); - - let columns = schema::get_columns_core(state, connection_id, &database, &schema_name, &info.name) - .await - .map_err(|error| SnapshotWarning::TableSkipped { - table: format!("{schema_name}.{}", info.name), - reason: error, - })?; - let column_notes = database_column_notes(&columns); - - let mut table_warnings = Vec::new(); - - // Indexes feed `relations.rs`'s uniqueness check, so a - // failure here must not look identical to "this table - // genuinely has no indexes" — that would silently - // downgrade a OneToOne relationship to ManyToOne. - let indexes = match schema::list_indexes_core( - state, - connection_id, - &database, - &schema_name, - &info.name, - ) - .await - { - Ok(indexes) => indexes, - Err(error) => { - table_warnings.push(SnapshotWarning::TableSkipped { - table: format!("{schema_name}.{}", info.name), - reason: format!("indexes unavailable: {error}"), + let current = format!("{schema_name}.{}", info.name); + let outcome = async { + if cancelled(cancel) { + return Err(SnapshotWarning::TableSkipped { + table: info.name.clone(), + reason: "cancelled".to_string(), }); - Vec::new() } - }; - // Foreign keys also degrade to empty rather than failing the - // table, but a real query failure must not look identical to - // "this table genuinely has no foreign keys" — it is reported - // as its own warning instead of being silently discarded. - let foreign_keys = - match schema::list_foreign_keys_core(state, connection_id, &database, &schema_name, &info.name) - .await + let columns = + schema::get_columns_core(state, connection_id, &database, &schema_name, &info.name) + .await + .map_err(|error| SnapshotWarning::TableSkipped { + table: format!("{schema_name}.{}", info.name), + reason: error, + })?; + let column_notes = database_column_notes(&columns); + + let mut table_warnings = Vec::new(); + + // Indexes feed `relations.rs`'s uniqueness check, so a + // failure here must not look identical to "this table + // genuinely has no indexes" — that would silently + // downgrade a OneToOne relationship to ManyToOne. + let indexes = + match schema::list_indexes_core(state, connection_id, &database, &schema_name, &info.name) + .await + { + Ok(indexes) => indexes, + Err(error) => { + table_warnings.push(SnapshotWarning::TableSkipped { + table: format!("{schema_name}.{}", info.name), + reason: format!("indexes unavailable: {error}"), + }); + Vec::new() + } + }; + + // Foreign keys also degrade to empty rather than failing the + // table, but a real query failure must not look identical to + // "this table genuinely has no foreign keys" — it is reported + // as its own warning instead of being silently discarded. + let foreign_keys = match schema::list_foreign_keys_core( + state, + connection_id, + &database, + &schema_name, + &info.name, + ) + .await { Ok(keys) => keys, Err(error) => { @@ -268,31 +280,36 @@ pub async fn collect_snapshot( } }; - Ok(( - DocTable { - schema: (!schema_name.is_empty()).then(|| schema_name.clone()), - name: info.name.clone(), - kind: table_kind_from(&info.table_type), - columns, - indexes, - foreign_keys, - group_id: None, - note: info.comment.clone().filter(|value| !value.trim().is_empty()), - note_source: if info.comment.as_deref().is_some_and(|v| !v.trim().is_empty()) { - NoteSource::Database - } else { - NoteSource::None + Ok(( + DocTable { + schema: (!schema_name.is_empty()).then(|| schema_name.clone()), + name: info.name.clone(), + kind: table_kind_from(&info.table_type), + columns, + indexes, + foreign_keys, + group_id: None, + note: info.comment.clone().filter(|value| !value.trim().is_empty()), + note_source: if info.comment.as_deref().is_some_and(|v| !v.trim().is_empty()) { + NoteSource::Database + } else { + NoteSource::None + }, + shadowed_note: None, + column_notes, + estimated_rows: None, + view_definition: None, }, - shadowed_note: None, - column_notes, - estimated_rows: None, - view_definition: None, - }, - table_warnings, - )) + table_warnings, + )) + } + .await; + let done = completed.fetch_add(1, Ordering::SeqCst) + 1; + progress(CollectProgress { completed: done, total, current }); + outcome } }) - .buffer_unordered(MAX_CONCURRENT_TABLES) + .buffer_unordered(max_concurrent_tables.max(1)) .collect() .await; diff --git a/crates/dbx-core/src/data/docs/mod.rs b/crates/dbx-core/src/data/docs/mod.rs index d4f8497a23..7e799ff2e7 100644 --- a/crates/dbx-core/src/data/docs/mod.rs +++ b/crates/dbx-core/src/data/docs/mod.rs @@ -7,7 +7,7 @@ pub mod keys; pub mod relations; pub mod snapshot; -pub use collector::{collect_snapshot, CollectOptions, CollectProgress}; +pub use collector::{collect_snapshot, collect_snapshot_with_concurrency, CollectOptions, CollectProgress}; pub use color::hue_to_hex; pub use dbml::{to_dbml, DbmlOutput}; pub use export::{to_standalone_html, EXPORT_LANGUAGES}; diff --git a/crates/dbx-web/src/routes/docs.rs b/crates/dbx-web/src/routes/docs.rs index 0dadaedd33..8ecfedd62c 100644 --- a/crates/dbx-web/src/routes/docs.rs +++ b/crates/dbx-web/src/routes/docs.rs @@ -21,6 +21,8 @@ pub struct DocsSnapshotRequest { pub tables: Vec, #[serde(default)] pub project_name: Option, + #[serde(default)] + pub max_concurrent_tables: Option, } async fn load_connection(state: &Arc, connection_id: &str) -> Result { @@ -48,10 +50,16 @@ pub async fn collect_snapshot( project_name: request.project_name.clone().unwrap_or_else(|| connection.name.clone()), }; - let snapshot = - dbx_core::docs::collect_snapshot(&state.app, &connection, &options, &|_progress| {}, &AtomicBool::new(false)) - .await - .map_err(AppError::from)?; + let snapshot = dbx_core::docs::collect_snapshot_with_concurrency( + &state.app, + &connection, + &options, + &|_progress| {}, + &AtomicBool::new(false), + request.max_concurrent_tables.unwrap_or(8).clamp(1, 8), + ) + .await + .map_err(AppError::from)?; Ok(Json(snapshot)) } diff --git a/src-tauri/src/commands/docs.rs b/src-tauri/src/commands/docs.rs index 8d9dba7c2d..a0845f7d90 100644 --- a/src-tauri/src/commands/docs.rs +++ b/src-tauri/src/commands/docs.rs @@ -8,10 +8,22 @@ use dbx_core::connection::AppState; use dbx_core::docs::annotations::{ apply_annotations, load_annotations, resolve_notes_path, save_annotations, AnnotationFile, }; -use dbx_core::docs::{collect_snapshot, to_standalone_html, CollectOptions, SchemaSnapshot}; +use dbx_core::docs::{ + collect_snapshot, collect_snapshot_with_concurrency, to_standalone_html, CollectOptions, SchemaSnapshot, +}; use dbx_core::models::connection::ConnectionConfig; +use serde::Serialize; +use tauri::ipc::Channel; use tauri::State; +#[derive(Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DocsCollectProgress { + completed: usize, + total: usize, + current: String, +} + async fn connection_of(state: &Arc, connection_id: &str) -> Result { let configs = state.configs.read().await; configs.get(connection_id).cloned().ok_or_else(|| format!("Connection {connection_id} not found.")) @@ -45,6 +57,36 @@ pub async fn docs_collect_snapshot( collect_snapshot(&state, &config, &options, &|_progress| {}, &AtomicBool::new(false)).await } +#[tauri::command] +pub async fn docs_collect_snapshot_for_export( + state: State<'_, Arc>, + connection_id: String, + database: String, + schemas: Vec, + tables: Vec, + project_name: Option, + on_progress: Channel, +) -> Result { + let config = connection_of(&state, &connection_id).await?; + let options = + CollectOptions { database, schemas, tables, project_name: project_name.unwrap_or_else(|| config.name.clone()) }; + collect_snapshot_with_concurrency( + &state, + &config, + &options, + &|event| { + let _ = on_progress.send(DocsCollectProgress { + completed: event.completed, + total: event.total, + current: event.current, + }); + }, + &AtomicBool::new(false), + 2, + ) + .await +} + #[tauri::command] pub async fn docs_load_annotations( state: State<'_, Arc>, diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 746f4ee26e..352ff8624e 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -2412,6 +2412,7 @@ pub fn run() { commands::mongo_cmd::mongo_rename_collection, commands::mongo_cmd::mongo_clone_collection, commands::docs::docs_collect_snapshot, + commands::docs::docs_collect_snapshot_for_export, commands::docs::docs_load_annotations, commands::docs::docs_apply_annotations, commands::docs::docs_save_annotations,