feat(export): run data dictionary PDF exports in background
This commit is contained in:
18 files changed
+702
-104
No files matched your search
@@ -0,0 +1,159 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { useExportTracker } from "@/composables/useExportTracker";
|
||||
import { hasActiveDictionaryExport, startDataDictionaryExport, type DictionaryExportRequest } from "@/lib/docs/dataDictionaryExportTask";
|
||||
import * as api from "@/lib/backend/api";
|
||||
import { renderDataDictionaryPdf } from "@/lib/docs/renderDataDictionaryPdf";
|
||||
import { saveDataDictionaryFile } from "@/lib/docs/saveDataDictionaryFile";
|
||||
|
||||
vi.mock("@/lib/backend/api", () => ({
|
||||
collectDocsSnapshotForExport: vi.fn(),
|
||||
loadDocsAnnotations: vi.fn(),
|
||||
applyDocsAnnotations: vi.fn(),
|
||||
}));
|
||||
vi.mock("@/lib/docs/renderDataDictionaryPdf", () => ({ renderDataDictionaryPdf: vi.fn() }));
|
||||
vi.mock("@/lib/docs/saveDataDictionaryFile", () => ({ saveDataDictionaryFile: vi.fn() }));
|
||||
|
||||
const tracker = useExportTracker();
|
||||
const snapshot = (database: string, names: string[], warnings: unknown[] = []) => ({
|
||||
project: { database },
|
||||
tables: names.map((name) => ({ name, schema: null, kind: "TABLE" })),
|
||||
warnings,
|
||||
});
|
||||
|
||||
function request(databases = ["shop"], names = ["a"]): DictionaryExportRequest {
|
||||
return {
|
||||
connectionId: "connection",
|
||||
databases,
|
||||
schemas: [],
|
||||
objects: databases.flatMap((database) => names.map((name) => ({ database, schema: "", name, kind: "TABLE" as const }))),
|
||||
layout: {} as DictionaryExportRequest["layout"],
|
||||
labels: {} as DictionaryExportRequest["labels"],
|
||||
outputPath: "/tmp/dictionary.pdf",
|
||||
overwrite: true,
|
||||
continueOnError: true,
|
||||
warningText: (warning) => (warning.kind === "tableSkipped" ? (warning.reason ?? "skipped") : "warning"),
|
||||
skippedDatabase: (database, reason) => `${database}: ${reason}`,
|
||||
missingObject: (object) => `${object.database}.${object.name}: unavailable`,
|
||||
onSuccess: vi.fn(),
|
||||
onFailure: vi.fn(),
|
||||
};
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.mocked(api.collectDocsSnapshotForExport).mockReset();
|
||||
vi.mocked(api.loadDocsAnnotations).mockResolvedValue(null);
|
||||
vi.mocked(api.applyDocsAnnotations).mockImplementation(async (_id, raw) => raw);
|
||||
vi.mocked(renderDataDictionaryPdf).mockReset();
|
||||
vi.mocked(renderDataDictionaryPdf).mockResolvedValue(new Uint8Array([1, 2]));
|
||||
vi.mocked(saveDataDictionaryFile).mockReset();
|
||||
vi.mocked(saveDataDictionaryFile).mockResolvedValue("/tmp/dictionary.pdf");
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await vi.waitFor(() => expect(hasActiveDictionaryExport("connection")).toBe(false));
|
||||
for (const task of tracker.tasks.value) tracker.removeTask(task.exportId);
|
||||
});
|
||||
|
||||
describe("background dictionary export", () => {
|
||||
it("survives its caller, tracks completed objects over multiple databases and finishes only after the file is written", async () => {
|
||||
let resolveSave!: (path: string) => void;
|
||||
vi.mocked(saveDataDictionaryFile).mockImplementation(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
resolveSave = resolve;
|
||||
}),
|
||||
);
|
||||
vi.mocked(api.collectDocsSnapshotForExport).mockImplementation(async (_id, database, _schemas, _names, progress) => {
|
||||
progress({ completed: 0, total: 2, current: "" });
|
||||
progress({ completed: 1, total: 2, current: "a" });
|
||||
progress({ completed: 2, total: 2, current: "b" });
|
||||
return snapshot(database, ["a", "b"]) as never;
|
||||
});
|
||||
const input = request(["shop", "archive"], ["a", "b"]);
|
||||
expect(startDataDictionaryExport(input)).toBe(true);
|
||||
expect(startDataDictionaryExport(input)).toBe(false);
|
||||
await vi.waitFor(() => expect(saveDataDictionaryFile).toHaveBeenCalledOnce());
|
||||
const task = tracker.tasks.value[0]!;
|
||||
expect(task.dictionaryCompleted).toBe(4);
|
||||
expect(task.dictionaryTotal).toBe(4);
|
||||
expect(task.status).toBe("Writing");
|
||||
expect(task.filePath).toBe("");
|
||||
expect(input.onSuccess).not.toHaveBeenCalled();
|
||||
resolveSave("/tmp/dictionary.pdf");
|
||||
await vi.waitFor(() => expect(task.status).toBe("Done"));
|
||||
expect(task.filePath).toBe("/tmp/dictionary.pdf");
|
||||
expect(input.onSuccess).toHaveBeenCalledWith("/tmp/dictionary.pdf", 0);
|
||||
expect(hasActiveDictionaryExport("connection")).toBe(false);
|
||||
});
|
||||
|
||||
it("reports save failures, frees the connection slot and permits a retry", async () => {
|
||||
vi.mocked(api.collectDocsSnapshotForExport).mockResolvedValue(snapshot("shop", ["a"]) as never);
|
||||
vi.mocked(saveDataDictionaryFile).mockRejectedValueOnce(new Error("disk full"));
|
||||
const input = request();
|
||||
startDataDictionaryExport(input);
|
||||
await vi.waitFor(() => expect(tracker.tasks.value[0]?.status).toBe("Error"));
|
||||
expect(tracker.tasks.value[0]?.errorMessage).toBe("disk full");
|
||||
expect(input.onSuccess).not.toHaveBeenCalled();
|
||||
expect(input.onFailure).toHaveBeenCalledOnce();
|
||||
expect(startDataDictionaryExport(request())).toBe(true);
|
||||
await vi.waitFor(() => expect(tracker.tasks.value[1]?.status).toBe("Done"));
|
||||
});
|
||||
|
||||
it("preserves skipped-object warnings only when continue-on-error is enabled", async () => {
|
||||
vi.mocked(api.collectDocsSnapshotForExport).mockResolvedValue(snapshot("shop", ["a"], [{ kind: "tableSkipped", table: ".b", reason: "denied" }]) as never);
|
||||
const input = request(["shop"], ["a", "b"]);
|
||||
startDataDictionaryExport(input);
|
||||
await vi.waitFor(() => expect(tracker.tasks.value[0]?.status).toBe("Done"));
|
||||
expect(tracker.tasks.value[0]?.dictionaryWarnings).toBe(1);
|
||||
expect(renderDataDictionaryPdf).toHaveBeenCalledWith(expect.any(Array), expect.anything(), expect.anything(), ["denied"]);
|
||||
const strict = request(["shop"], ["a", "b"]);
|
||||
strict.continueOnError = false;
|
||||
startDataDictionaryExport(strict);
|
||||
await vi.waitFor(() => expect(tracker.tasks.value[1]?.status).toBe("Error"));
|
||||
expect(saveDataDictionaryFile).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not silently export a partial PDF when a selected object disappears", async () => {
|
||||
vi.mocked(api.collectDocsSnapshotForExport).mockResolvedValue(snapshot("shop", ["a"]) as never);
|
||||
const input = request(["shop"], ["a", "b"]);
|
||||
startDataDictionaryExport(input);
|
||||
await vi.waitFor(() => expect(tracker.tasks.value[0]?.status).toBe("Done"));
|
||||
expect(renderDataDictionaryPdf).toHaveBeenCalledWith(expect.any(Array), expect.anything(), expect.anything(), ["shop.b: unavailable"]);
|
||||
expect(tracker.tasks.value[0]?.dictionaryWarnings).toBe(1);
|
||||
|
||||
const strict = request(["shop"], ["a", "b"]);
|
||||
strict.continueOnError = false;
|
||||
startDataDictionaryExport(strict);
|
||||
await vi.waitFor(() => expect(tracker.tasks.value[1]?.status).toBe("Error"));
|
||||
expect(tracker.tasks.value[1]?.errorMessage).toContain("shop.b: unavailable");
|
||||
expect(saveDataDictionaryFile).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("ignores stale and out-of-order progress callbacks", async () => {
|
||||
let firstProgress!: Parameters<typeof api.collectDocsSnapshotForExport>[4];
|
||||
vi.mocked(api.collectDocsSnapshotForExport).mockImplementation(async (_id, database, _schemas, _names, progress) => {
|
||||
if (database === "shop") {
|
||||
firstProgress = progress;
|
||||
progress({ completed: 2, total: 2, current: "b" });
|
||||
progress({ completed: 1, total: 2, current: "a" });
|
||||
} else {
|
||||
firstProgress({ completed: 0, total: 2, current: "old" });
|
||||
progress({ completed: 1, total: 2, current: "new" });
|
||||
}
|
||||
return snapshot(database, ["a", "b"]) as never;
|
||||
});
|
||||
let resolveRender!: (bytes: Uint8Array) => void;
|
||||
vi.mocked(renderDataDictionaryPdf).mockImplementation(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
resolveRender = resolve;
|
||||
}),
|
||||
);
|
||||
startDataDictionaryExport(request(["shop", "archive"], ["a", "b"]));
|
||||
await vi.waitFor(() => expect(renderDataDictionaryPdf).toHaveBeenCalled());
|
||||
expect(tracker.tasks.value[0]?.dictionaryCompleted).toBe(4);
|
||||
expect(tracker.tasks.value[0]?.dictionaryCurrent).toBe("");
|
||||
resolveRender(new Uint8Array([1]));
|
||||
await vi.waitFor(() => expect(tracker.tasks.value[0]?.status).toBe("Done"));
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,123 @@
|
||||
import * as api from "@/lib/backend/api";
|
||||
import { useExportTracker } from "@/composables/useExportTracker";
|
||||
import { emptyAnnotations } from "@/docs/annotationEdits";
|
||||
import type { SchemaSnapshot, SnapshotWarning } from "@/docs/types";
|
||||
import { uuid } from "@/lib/common/utils";
|
||||
import { exportBlockedBySkippedTables, objectKey, orderDictionaryTables, warningsForSelection, type DataDictionaryLabels, type DictionaryLayout, type DictionaryObjectRef, type DictionaryTable } from "./dataDictionary";
|
||||
import { renderDataDictionaryPdf } from "./renderDataDictionaryPdf";
|
||||
import { saveDataDictionaryFile } from "./saveDataDictionaryFile";
|
||||
|
||||
export interface DictionaryExportRequest {
|
||||
connectionId: string;
|
||||
databases: string[];
|
||||
schemas: Array<{ database: string; name: string }>;
|
||||
objects: DictionaryObjectRef[];
|
||||
snapshot?: SchemaSnapshot | null;
|
||||
layout: DictionaryLayout;
|
||||
labels: DataDictionaryLabels;
|
||||
outputPath: string;
|
||||
overwrite: boolean;
|
||||
continueOnError: boolean;
|
||||
warningText: (warning: SnapshotWarning) => string;
|
||||
skippedDatabase: (database: string, reason: string) => string;
|
||||
missingObject: (object: DictionaryObjectRef) => string;
|
||||
onSuccess: (path: string, warnings: number) => void;
|
||||
onFailure: (error: unknown) => void;
|
||||
}
|
||||
|
||||
// Export lifetimes belong to the app, not the dialog instance. One job per
|
||||
// connection also bounds the aggregate metadata pressure on its pool.
|
||||
const activeConnections = new Set<string>();
|
||||
|
||||
export function hasActiveDictionaryExport(connectionId: string): boolean {
|
||||
return activeConnections.has(connectionId);
|
||||
}
|
||||
|
||||
export function startDataDictionaryExport(request: DictionaryExportRequest): boolean {
|
||||
if (activeConnections.has(request.connectionId)) return false;
|
||||
activeConnections.add(request.connectionId);
|
||||
const tracker = useExportTracker();
|
||||
const id = uuid();
|
||||
tracker.addDataDictionaryTask(id, request.databases.join(", "), request.objects.length);
|
||||
|
||||
void (async () => {
|
||||
const warnings: string[] = [];
|
||||
const tables: DictionaryTable[] = [];
|
||||
const order = request.objects.map(objectKey);
|
||||
let completed = 0;
|
||||
try {
|
||||
for (const database of request.databases) {
|
||||
const selected = request.objects.filter((item) => item.database === database);
|
||||
if (!selected.length) continue;
|
||||
tracker.updateDataDictionaryTask(id, { dictionaryPhase: "preparing", dictionaryCompleted: completed, dictionaryCurrent: database });
|
||||
try {
|
||||
let merged: SchemaSnapshot;
|
||||
const fromSnapshot =
|
||||
request.snapshot?.project.database === database
|
||||
? orderDictionaryTables(
|
||||
request.snapshot.tables.map((table) => ({ ...table, database })),
|
||||
selected.map(objectKey),
|
||||
)
|
||||
: [];
|
||||
if (request.snapshot && fromSnapshot.length === selected.length) {
|
||||
merged = request.snapshot;
|
||||
tracker.updateDataDictionaryTask(id, { dictionaryPhase: "collecting", dictionaryProgressKnown: true });
|
||||
} else {
|
||||
const schemas = request.schemas.filter((item) => item.database === database).map((item) => item.name);
|
||||
const names = selected.map((item) => (item.schema ? `${item.schema}.${item.name}` : item.name));
|
||||
const baseCompleted = completed;
|
||||
let acceptingProgress = true;
|
||||
let raw: SchemaSnapshot;
|
||||
try {
|
||||
raw = await api.collectDocsSnapshotForExport(request.connectionId, database, schemas, names, (progress) => {
|
||||
if (!acceptingProgress) return;
|
||||
const task = tracker.tasks.value.find((item) => item.exportId === id);
|
||||
tracker.updateDataDictionaryTask(id, {
|
||||
dictionaryPhase: "collecting",
|
||||
dictionaryProgressKnown: progress.total > 0,
|
||||
dictionaryCompleted: Math.max(task?.dictionaryCompleted ?? 0, baseCompleted + Math.min(progress.completed, selected.length)),
|
||||
dictionaryCurrent: progress.current || database,
|
||||
});
|
||||
});
|
||||
} finally {
|
||||
acceptingProgress = false;
|
||||
}
|
||||
const file = (await api.loadDocsAnnotations(request.connectionId)) ?? emptyAnnotations();
|
||||
merged = await api.applyDocsAnnotations(request.connectionId, raw, file);
|
||||
}
|
||||
const relevant = warningsForSelection(merged.warnings, selected);
|
||||
const collected = new Set(merged.tables.map((table) => objectKey({ database, schema: table.schema ?? "", name: table.name })));
|
||||
const missing = selected.filter((item) => !collected.has(objectKey(item)));
|
||||
const missingWarnings = missing.filter((item) => !relevant.some((warning) => warning.kind === "tableSkipped" && (warning.table === "*" || warning.table === `${item.schema}.*` || warning.table === item.name || warning.table === `${item.schema}.${item.name}`))).map(request.missingObject);
|
||||
if (exportBlockedBySkippedTables(relevant, request.continueOnError) || (!request.continueOnError && missingWarnings.length > 0)) {
|
||||
throw new Error(relevant.map(request.warningText).find(Boolean) || missingWarnings[0] || database);
|
||||
}
|
||||
warnings.push(...relevant.map(request.warningText).filter(Boolean));
|
||||
warnings.push(...missingWarnings);
|
||||
tables.push(...merged.tables.filter((table) => selected.some((item) => objectKey(item) === objectKey({ database, schema: table.schema ?? "", name: table.name }))).map((table) => ({ ...table, database })));
|
||||
} catch (error) {
|
||||
if (!request.continueOnError) throw error;
|
||||
warnings.push(request.skippedDatabase(database, error instanceof Error ? error.message : String(error)));
|
||||
}
|
||||
completed += selected.length;
|
||||
tracker.updateDataDictionaryTask(id, { dictionaryCompleted: completed, dictionaryCurrent: "", dictionaryWarnings: warnings.length });
|
||||
}
|
||||
if (!tables.length) throw new Error("No selected objects could be collected");
|
||||
tracker.updateDataDictionaryTask(id, { dictionaryPhase: "generating", dictionaryCurrent: "" });
|
||||
// Worker messages cannot clone Vue proxies; capture only serializable metadata.
|
||||
const ordered = JSON.parse(JSON.stringify(orderDictionaryTables(tables, order))) as DictionaryTable[];
|
||||
const bytes = await renderDataDictionaryPdf(ordered, request.labels, request.layout, request.continueOnError ? warnings : []);
|
||||
tracker.updateDataDictionaryTask(id, { dictionaryPhase: "saving", status: "Writing" });
|
||||
const path = await saveDataDictionaryFile(request.outputPath, bytes, { overwrite: request.overwrite, grantedPath: request.outputPath });
|
||||
if (!path) throw new Error("File was not saved");
|
||||
tracker.updateDataDictionaryTask(id, { status: "Done", filePath: path, dictionaryWarnings: warnings.length });
|
||||
request.onSuccess(path, warnings.length);
|
||||
} catch (error) {
|
||||
tracker.updateDataDictionaryTask(id, { status: "Error", errorMessage: error instanceof Error ? error.message : String(error) });
|
||||
request.onFailure(error);
|
||||
} finally {
|
||||
activeConnections.delete(request.connectionId);
|
||||
}
|
||||
})();
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import { buildDataDictionaryPdf } from "./dataDictionaryPdf";
|
||||
import type { DataDictionaryLabels, DictionaryLayout, DictionaryTable } from "./dataDictionary";
|
||||
|
||||
self.onmessage = (event: MessageEvent<{ tables: DictionaryTable[]; labels: DataDictionaryLabels; layout: DictionaryLayout; warnings: string[] }>) => {
|
||||
try {
|
||||
const { tables, labels, layout, warnings } = event.data;
|
||||
const bytes = buildDataDictionaryPdf(tables, labels, layout, warnings);
|
||||
self.postMessage({ bytes }, { transfer: [bytes.buffer] });
|
||||
} catch (error) {
|
||||
self.postMessage({ error: error instanceof Error ? error.message : String(error) });
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,18 @@
|
||||
import type { DataDictionaryLabels, DictionaryLayout, DictionaryTable } from "./dataDictionary";
|
||||
|
||||
export function renderDataDictionaryPdf(tables: DictionaryTable[], labels: DataDictionaryLabels, layout: DictionaryLayout, warnings: string[]): Promise<Uint8Array> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const worker = new Worker(new URL("./dataDictionaryPdf.worker.ts", import.meta.url), { type: "module" });
|
||||
worker.onmessage = (event: MessageEvent<{ bytes?: Uint8Array; error?: string }>) => {
|
||||
worker.terminate();
|
||||
if (event.data.error) reject(new Error(event.data.error));
|
||||
else if (event.data.bytes) resolve(event.data.bytes);
|
||||
else reject(new Error("PDF worker returned no data"));
|
||||
};
|
||||
worker.onerror = (event) => {
|
||||
worker.terminate();
|
||||
reject(new Error(event.message));
|
||||
};
|
||||
worker.postMessage({ tables, labels, layout, warnings });
|
||||
});
|
||||
}
|
||||
@@ -7,18 +7,21 @@ export class DictionaryFileExistsError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
export async function chooseDataDictionaryPath(suggestedName: string): Promise<string | null> {
|
||||
if (!isTauriRuntime()) return suggestedName;
|
||||
const { save } = await import("@tauri-apps/plugin-dialog");
|
||||
return save({ defaultPath: suggestedName, filters: [{ name: "PDF", extensions: ["pdf"] }] });
|
||||
}
|
||||
|
||||
/**
|
||||
* Suggested name is only the save dialog's defaultPath. The write uses the path
|
||||
* `save()` just returned. Tauri's dialog grant covers that exact path, not a
|
||||
* typed path or a timestamp appended after the dialog closes.
|
||||
*/
|
||||
export async function saveDataDictionaryFile(suggestedName: string, content: Uint8Array, options: { overwrite: boolean }): Promise<string | null> {
|
||||
export async function saveDataDictionaryFile(suggestedName: string, content: Uint8Array, options: { overwrite: boolean; grantedPath?: string }): Promise<string | null> {
|
||||
if (isTauriRuntime()) {
|
||||
const [{ save }, fs] = await Promise.all([import("@tauri-apps/plugin-dialog"), import("@tauri-apps/plugin-fs")]);
|
||||
const granted = await save({
|
||||
defaultPath: suggestedName,
|
||||
filters: [{ name: "PDF", extensions: ["pdf"] }],
|
||||
});
|
||||
const fs = await import("@tauri-apps/plugin-fs");
|
||||
const granted = options.grantedPath ?? (await chooseDataDictionaryPath(suggestedName));
|
||||
if (!granted) return null;
|
||||
if (!options.overwrite && (await fs.exists(granted))) throw new DictionaryFileExistsError();
|
||||
await fs.writeFile(granted, content);
|
||||
|
||||
Reference in new issue
Block a user