feat(export): run data dictionary PDF exports in background

This commit is contained in:
ccq authored and GitHub committed 2026-09-26 00:04:52 +08:00
1 parent 4baec21fb7
commit 705ea7464d
18 files changed
+702 -104

No files matched your search

+1
View File
@@ -255,6 +255,7 @@ export const listDialectDataTypes = forward("listDialectDataTypes");
// Docs
export const collectDocsSnapshot = forward("collectDocsSnapshot");
export const collectDocsSnapshotForExport = forward("collectDocsSnapshotForExport");
export const loadDocsAnnotations = forward("loadDocsAnnotations");
export const applyDocsAnnotations = forward("applyDocsAnnotations");
export const saveDocsAnnotations = forward("saveDocsAnnotations");
+5
View File
@@ -1352,6 +1352,11 @@ export async function collectDocsSnapshot(connectionId: string, database: string
return post("/api/docs/snapshot", { connectionId, database, schemas, tables, projectName });
}
// HTTP snapshot transport is request/response; its progress remains indeterminate.
export async function collectDocsSnapshotForExport(connectionId: string, database: string, schemas: string[], tables: string[], _onProgress: (progress: import("./tauri").DocsCollectProgress) => void): Promise<SchemaSnapshot> {
return post("/api/docs/snapshot", { connectionId, database, schemas, tables, projectName: database, maxConcurrentTables: 2 });
}
export async function loadDocsAnnotations(connectionId: string): Promise<AnnotationFile | null> {
return post("/api/docs/annotations/load", { connectionId });
}
+13
View File
@@ -1,4 +1,5 @@
import { invoke as tauriInvoke } from "@tauri-apps/api/core";
import { Channel } from "@tauri-apps/api/core";
import type { MongoDumpFormat, MongoDumpSourceInput, MongoDumpCatalog, MongoRestoreSourcePreview, MongoDatabaseDumpRequest, MongoDatabaseRestoreRequest, MongoDatabaseDumpProgress } from "./mongodbDumpTypes";
import type { MongoRestoreUpload, MongoSourceReadOptions } from "./mongodbDumpTypes";
import type { UserSkillRootSettings, UserSkillsListResult, UserSkillsReadResult } from "@/types/userSkills";
@@ -2465,6 +2466,18 @@ export async function collectDocsSnapshot(connectionId: string, database: string
return invoke("docs_collect_snapshot", { connectionId, database, schemas, tables, projectName });
}
export interface DocsCollectProgress {
completed: number;
total: number;
current: string;
}
export async function collectDocsSnapshotForExport(connectionId: string, database: string, schemas: string[], tables: string[], onProgress: (progress: DocsCollectProgress) => void): Promise<SchemaSnapshot> {
const channel = new Channel<DocsCollectProgress>();
channel.onmessage = onProgress;
return invoke("docs_collect_snapshot_for_export", { connectionId, database, schemas, tables, projectName: database, onProgress: channel });
}
export async function loadDocsAnnotations(connectionId: string): Promise<AnnotationFile | null> {
return invoke("docs_load_annotations", { connectionId });
}
@@ -0,0 +1,159 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { useExportTracker } from "@/composables/useExportTracker";
import { hasActiveDictionaryExport, startDataDictionaryExport, type DictionaryExportRequest } from "@/lib/docs/dataDictionaryExportTask";
import * as api from "@/lib/backend/api";
import { renderDataDictionaryPdf } from "@/lib/docs/renderDataDictionaryPdf";
import { saveDataDictionaryFile } from "@/lib/docs/saveDataDictionaryFile";
vi.mock("@/lib/backend/api", () => ({
collectDocsSnapshotForExport: vi.fn(),
loadDocsAnnotations: vi.fn(),
applyDocsAnnotations: vi.fn(),
}));
vi.mock("@/lib/docs/renderDataDictionaryPdf", () => ({ renderDataDictionaryPdf: vi.fn() }));
vi.mock("@/lib/docs/saveDataDictionaryFile", () => ({ saveDataDictionaryFile: vi.fn() }));
const tracker = useExportTracker();
const snapshot = (database: string, names: string[], warnings: unknown[] = []) => ({
project: { database },
tables: names.map((name) => ({ name, schema: null, kind: "TABLE" })),
warnings,
});
function request(databases = ["shop"], names = ["a"]): DictionaryExportRequest {
return {
connectionId: "connection",
databases,
schemas: [],
objects: databases.flatMap((database) => names.map((name) => ({ database, schema: "", name, kind: "TABLE" as const }))),
layout: {} as DictionaryExportRequest["layout"],
labels: {} as DictionaryExportRequest["labels"],
outputPath: "/tmp/dictionary.pdf",
overwrite: true,
continueOnError: true,
warningText: (warning) => (warning.kind === "tableSkipped" ? (warning.reason ?? "skipped") : "warning"),
skippedDatabase: (database, reason) => `${database}: ${reason}`,
missingObject: (object) => `${object.database}.${object.name}: unavailable`,
onSuccess: vi.fn(),
onFailure: vi.fn(),
};
}
beforeEach(() => {
vi.mocked(api.collectDocsSnapshotForExport).mockReset();
vi.mocked(api.loadDocsAnnotations).mockResolvedValue(null);
vi.mocked(api.applyDocsAnnotations).mockImplementation(async (_id, raw) => raw);
vi.mocked(renderDataDictionaryPdf).mockReset();
vi.mocked(renderDataDictionaryPdf).mockResolvedValue(new Uint8Array([1, 2]));
vi.mocked(saveDataDictionaryFile).mockReset();
vi.mocked(saveDataDictionaryFile).mockResolvedValue("/tmp/dictionary.pdf");
});
afterEach(async () => {
await vi.waitFor(() => expect(hasActiveDictionaryExport("connection")).toBe(false));
for (const task of tracker.tasks.value) tracker.removeTask(task.exportId);
});
describe("background dictionary export", () => {
it("survives its caller, tracks completed objects over multiple databases and finishes only after the file is written", async () => {
let resolveSave!: (path: string) => void;
vi.mocked(saveDataDictionaryFile).mockImplementation(
() =>
new Promise((resolve) => {
resolveSave = resolve;
}),
);
vi.mocked(api.collectDocsSnapshotForExport).mockImplementation(async (_id, database, _schemas, _names, progress) => {
progress({ completed: 0, total: 2, current: "" });
progress({ completed: 1, total: 2, current: "a" });
progress({ completed: 2, total: 2, current: "b" });
return snapshot(database, ["a", "b"]) as never;
});
const input = request(["shop", "archive"], ["a", "b"]);
expect(startDataDictionaryExport(input)).toBe(true);
expect(startDataDictionaryExport(input)).toBe(false);
await vi.waitFor(() => expect(saveDataDictionaryFile).toHaveBeenCalledOnce());
const task = tracker.tasks.value[0]!;
expect(task.dictionaryCompleted).toBe(4);
expect(task.dictionaryTotal).toBe(4);
expect(task.status).toBe("Writing");
expect(task.filePath).toBe("");
expect(input.onSuccess).not.toHaveBeenCalled();
resolveSave("/tmp/dictionary.pdf");
await vi.waitFor(() => expect(task.status).toBe("Done"));
expect(task.filePath).toBe("/tmp/dictionary.pdf");
expect(input.onSuccess).toHaveBeenCalledWith("/tmp/dictionary.pdf", 0);
expect(hasActiveDictionaryExport("connection")).toBe(false);
});
it("reports save failures, frees the connection slot and permits a retry", async () => {
vi.mocked(api.collectDocsSnapshotForExport).mockResolvedValue(snapshot("shop", ["a"]) as never);
vi.mocked(saveDataDictionaryFile).mockRejectedValueOnce(new Error("disk full"));
const input = request();
startDataDictionaryExport(input);
await vi.waitFor(() => expect(tracker.tasks.value[0]?.status).toBe("Error"));
expect(tracker.tasks.value[0]?.errorMessage).toBe("disk full");
expect(input.onSuccess).not.toHaveBeenCalled();
expect(input.onFailure).toHaveBeenCalledOnce();
expect(startDataDictionaryExport(request())).toBe(true);
await vi.waitFor(() => expect(tracker.tasks.value[1]?.status).toBe("Done"));
});
it("preserves skipped-object warnings only when continue-on-error is enabled", async () => {
vi.mocked(api.collectDocsSnapshotForExport).mockResolvedValue(snapshot("shop", ["a"], [{ kind: "tableSkipped", table: ".b", reason: "denied" }]) as never);
const input = request(["shop"], ["a", "b"]);
startDataDictionaryExport(input);
await vi.waitFor(() => expect(tracker.tasks.value[0]?.status).toBe("Done"));
expect(tracker.tasks.value[0]?.dictionaryWarnings).toBe(1);
expect(renderDataDictionaryPdf).toHaveBeenCalledWith(expect.any(Array), expect.anything(), expect.anything(), ["denied"]);
const strict = request(["shop"], ["a", "b"]);
strict.continueOnError = false;
startDataDictionaryExport(strict);
await vi.waitFor(() => expect(tracker.tasks.value[1]?.status).toBe("Error"));
expect(saveDataDictionaryFile).toHaveBeenCalledTimes(1);
});
it("does not silently export a partial PDF when a selected object disappears", async () => {
vi.mocked(api.collectDocsSnapshotForExport).mockResolvedValue(snapshot("shop", ["a"]) as never);
const input = request(["shop"], ["a", "b"]);
startDataDictionaryExport(input);
await vi.waitFor(() => expect(tracker.tasks.value[0]?.status).toBe("Done"));
expect(renderDataDictionaryPdf).toHaveBeenCalledWith(expect.any(Array), expect.anything(), expect.anything(), ["shop.b: unavailable"]);
expect(tracker.tasks.value[0]?.dictionaryWarnings).toBe(1);
const strict = request(["shop"], ["a", "b"]);
strict.continueOnError = false;
startDataDictionaryExport(strict);
await vi.waitFor(() => expect(tracker.tasks.value[1]?.status).toBe("Error"));
expect(tracker.tasks.value[1]?.errorMessage).toContain("shop.b: unavailable");
expect(saveDataDictionaryFile).toHaveBeenCalledTimes(1);
});
it("ignores stale and out-of-order progress callbacks", async () => {
let firstProgress!: Parameters<typeof api.collectDocsSnapshotForExport>[4];
vi.mocked(api.collectDocsSnapshotForExport).mockImplementation(async (_id, database, _schemas, _names, progress) => {
if (database === "shop") {
firstProgress = progress;
progress({ completed: 2, total: 2, current: "b" });
progress({ completed: 1, total: 2, current: "a" });
} else {
firstProgress({ completed: 0, total: 2, current: "old" });
progress({ completed: 1, total: 2, current: "new" });
}
return snapshot(database, ["a", "b"]) as never;
});
let resolveRender!: (bytes: Uint8Array) => void;
vi.mocked(renderDataDictionaryPdf).mockImplementation(
() =>
new Promise((resolve) => {
resolveRender = resolve;
}),
);
startDataDictionaryExport(request(["shop", "archive"], ["a", "b"]));
await vi.waitFor(() => expect(renderDataDictionaryPdf).toHaveBeenCalled());
expect(tracker.tasks.value[0]?.dictionaryCompleted).toBe(4);
expect(tracker.tasks.value[0]?.dictionaryCurrent).toBe("");
resolveRender(new Uint8Array([1]));
await vi.waitFor(() => expect(tracker.tasks.value[0]?.status).toBe("Done"));
});
});
@@ -0,0 +1,123 @@
import * as api from "@/lib/backend/api";
import { useExportTracker } from "@/composables/useExportTracker";
import { emptyAnnotations } from "@/docs/annotationEdits";
import type { SchemaSnapshot, SnapshotWarning } from "@/docs/types";
import { uuid } from "@/lib/common/utils";
import { exportBlockedBySkippedTables, objectKey, orderDictionaryTables, warningsForSelection, type DataDictionaryLabels, type DictionaryLayout, type DictionaryObjectRef, type DictionaryTable } from "./dataDictionary";
import { renderDataDictionaryPdf } from "./renderDataDictionaryPdf";
import { saveDataDictionaryFile } from "./saveDataDictionaryFile";
export interface DictionaryExportRequest {
connectionId: string;
databases: string[];
schemas: Array<{ database: string; name: string }>;
objects: DictionaryObjectRef[];
snapshot?: SchemaSnapshot | null;
layout: DictionaryLayout;
labels: DataDictionaryLabels;
outputPath: string;
overwrite: boolean;
continueOnError: boolean;
warningText: (warning: SnapshotWarning) => string;
skippedDatabase: (database: string, reason: string) => string;
missingObject: (object: DictionaryObjectRef) => string;
onSuccess: (path: string, warnings: number) => void;
onFailure: (error: unknown) => void;
}
// Export lifetimes belong to the app, not the dialog instance. One job per
// connection also bounds the aggregate metadata pressure on its pool.
const activeConnections = new Set<string>();
export function hasActiveDictionaryExport(connectionId: string): boolean {
return activeConnections.has(connectionId);
}
export function startDataDictionaryExport(request: DictionaryExportRequest): boolean {
if (activeConnections.has(request.connectionId)) return false;
activeConnections.add(request.connectionId);
const tracker = useExportTracker();
const id = uuid();
tracker.addDataDictionaryTask(id, request.databases.join(", "), request.objects.length);
void (async () => {
const warnings: string[] = [];
const tables: DictionaryTable[] = [];
const order = request.objects.map(objectKey);
let completed = 0;
try {
for (const database of request.databases) {
const selected = request.objects.filter((item) => item.database === database);
if (!selected.length) continue;
tracker.updateDataDictionaryTask(id, { dictionaryPhase: "preparing", dictionaryCompleted: completed, dictionaryCurrent: database });
try {
let merged: SchemaSnapshot;
const fromSnapshot =
request.snapshot?.project.database === database
? orderDictionaryTables(
request.snapshot.tables.map((table) => ({ ...table, database })),
selected.map(objectKey),
)
: [];
if (request.snapshot && fromSnapshot.length === selected.length) {
merged = request.snapshot;
tracker.updateDataDictionaryTask(id, { dictionaryPhase: "collecting", dictionaryProgressKnown: true });
} else {
const schemas = request.schemas.filter((item) => item.database === database).map((item) => item.name);
const names = selected.map((item) => (item.schema ? `${item.schema}.${item.name}` : item.name));
const baseCompleted = completed;
let acceptingProgress = true;
let raw: SchemaSnapshot;
try {
raw = await api.collectDocsSnapshotForExport(request.connectionId, database, schemas, names, (progress) => {
if (!acceptingProgress) return;
const task = tracker.tasks.value.find((item) => item.exportId === id);
tracker.updateDataDictionaryTask(id, {
dictionaryPhase: "collecting",
dictionaryProgressKnown: progress.total > 0,
dictionaryCompleted: Math.max(task?.dictionaryCompleted ?? 0, baseCompleted + Math.min(progress.completed, selected.length)),
dictionaryCurrent: progress.current || database,
});
});
} finally {
acceptingProgress = false;
}
const file = (await api.loadDocsAnnotations(request.connectionId)) ?? emptyAnnotations();
merged = await api.applyDocsAnnotations(request.connectionId, raw, file);
}
const relevant = warningsForSelection(merged.warnings, selected);
const collected = new Set(merged.tables.map((table) => objectKey({ database, schema: table.schema ?? "", name: table.name })));
const missing = selected.filter((item) => !collected.has(objectKey(item)));
const missingWarnings = missing.filter((item) => !relevant.some((warning) => warning.kind === "tableSkipped" && (warning.table === "*" || warning.table === `${item.schema}.*` || warning.table === item.name || warning.table === `${item.schema}.${item.name}`))).map(request.missingObject);
if (exportBlockedBySkippedTables(relevant, request.continueOnError) || (!request.continueOnError && missingWarnings.length > 0)) {
throw new Error(relevant.map(request.warningText).find(Boolean) || missingWarnings[0] || database);
}
warnings.push(...relevant.map(request.warningText).filter(Boolean));
warnings.push(...missingWarnings);
tables.push(...merged.tables.filter((table) => selected.some((item) => objectKey(item) === objectKey({ database, schema: table.schema ?? "", name: table.name }))).map((table) => ({ ...table, database })));
} catch (error) {
if (!request.continueOnError) throw error;
warnings.push(request.skippedDatabase(database, error instanceof Error ? error.message : String(error)));
}
completed += selected.length;
tracker.updateDataDictionaryTask(id, { dictionaryCompleted: completed, dictionaryCurrent: "", dictionaryWarnings: warnings.length });
}
if (!tables.length) throw new Error("No selected objects could be collected");
tracker.updateDataDictionaryTask(id, { dictionaryPhase: "generating", dictionaryCurrent: "" });
// Worker messages cannot clone Vue proxies; capture only serializable metadata.
const ordered = JSON.parse(JSON.stringify(orderDictionaryTables(tables, order))) as DictionaryTable[];
const bytes = await renderDataDictionaryPdf(ordered, request.labels, request.layout, request.continueOnError ? warnings : []);
tracker.updateDataDictionaryTask(id, { dictionaryPhase: "saving", status: "Writing" });
const path = await saveDataDictionaryFile(request.outputPath, bytes, { overwrite: request.overwrite, grantedPath: request.outputPath });
if (!path) throw new Error("File was not saved");
tracker.updateDataDictionaryTask(id, { status: "Done", filePath: path, dictionaryWarnings: warnings.length });
request.onSuccess(path, warnings.length);
} catch (error) {
tracker.updateDataDictionaryTask(id, { status: "Error", errorMessage: error instanceof Error ? error.message : String(error) });
request.onFailure(error);
} finally {
activeConnections.delete(request.connectionId);
}
})();
return true;
}
@@ -0,0 +1,12 @@
import { buildDataDictionaryPdf } from "./dataDictionaryPdf";
import type { DataDictionaryLabels, DictionaryLayout, DictionaryTable } from "./dataDictionary";
self.onmessage = (event: MessageEvent<{ tables: DictionaryTable[]; labels: DataDictionaryLabels; layout: DictionaryLayout; warnings: string[] }>) => {
try {
const { tables, labels, layout, warnings } = event.data;
const bytes = buildDataDictionaryPdf(tables, labels, layout, warnings);
self.postMessage({ bytes }, { transfer: [bytes.buffer] });
} catch (error) {
self.postMessage({ error: error instanceof Error ? error.message : String(error) });
}
};
@@ -0,0 +1,18 @@
import type { DataDictionaryLabels, DictionaryLayout, DictionaryTable } from "./dataDictionary";
export function renderDataDictionaryPdf(tables: DictionaryTable[], labels: DataDictionaryLabels, layout: DictionaryLayout, warnings: string[]): Promise<Uint8Array> {
return new Promise((resolve, reject) => {
const worker = new Worker(new URL("./dataDictionaryPdf.worker.ts", import.meta.url), { type: "module" });
worker.onmessage = (event: MessageEvent<{ bytes?: Uint8Array; error?: string }>) => {
worker.terminate();
if (event.data.error) reject(new Error(event.data.error));
else if (event.data.bytes) resolve(event.data.bytes);
else reject(new Error("PDF worker returned no data"));
};
worker.onerror = (event) => {
worker.terminate();
reject(new Error(event.message));
};
worker.postMessage({ tables, labels, layout, warnings });
});
}
@@ -7,18 +7,21 @@ export class DictionaryFileExistsError extends Error {
}
}
export async function chooseDataDictionaryPath(suggestedName: string): Promise<string | null> {
if (!isTauriRuntime()) return suggestedName;
const { save } = await import("@tauri-apps/plugin-dialog");
return save({ defaultPath: suggestedName, filters: [{ name: "PDF", extensions: ["pdf"] }] });
}
/**
* Suggested name is only the save dialog's defaultPath. The write uses the path
* `save()` just returned. Tauri's dialog grant covers that exact path, not a
* typed path or a timestamp appended after the dialog closes.
*/
export async function saveDataDictionaryFile(suggestedName: string, content: Uint8Array, options: { overwrite: boolean }): Promise<string | null> {
export async function saveDataDictionaryFile(suggestedName: string, content: Uint8Array, options: { overwrite: boolean; grantedPath?: string }): Promise<string | null> {
if (isTauriRuntime()) {
const [{ save }, fs] = await Promise.all([import("@tauri-apps/plugin-dialog"), import("@tauri-apps/plugin-fs")]);
const granted = await save({
defaultPath: suggestedName,
filters: [{ name: "PDF", extensions: ["pdf"] }],
});
const fs = await import("@tauri-apps/plugin-fs");
const granted = options.grantedPath ?? (await chooseDataDictionaryPath(suggestedName));
if (!granted) return null;
if (!options.overwrite && (await fs.exists(granted))) throw new DictionaryFileExistsError();
await fs.writeFile(granted, content);