feat(mongodb): bson dumps and database backup/restore
This commit is contained in:
Hand Sonic authored and GitHub committed
1 parent
ecc1135de3
commit
f5673ca0a1
54 files changed
+6355
-106
No files matched your search
@@ -0,0 +1,50 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { prepareMongodbRestoreSource, restoreMongodbDatabase } from "../http";
|
||||
|
||||
function file(name: string, size = 0) {
|
||||
const value = new File([], name);
|
||||
Object.defineProperty(value, "webkitRelativePath", { value: `dump/source/${name}` });
|
||||
Object.defineProperty(value, "size", { value: size });
|
||||
return value;
|
||||
}
|
||||
|
||||
afterEach(() => vi.unstubAllGlobals());
|
||||
|
||||
describe("MongoDB metadata-first transport", () => {
|
||||
it("previews a multi-gigabyte directory without uploading any BSON bodies", async () => {
|
||||
const fetch = vi.fn().mockResolvedValue(new Response(JSON.stringify({ sourceRef: "catalog", databases: ["source"], collections: [] })));
|
||||
vi.stubGlobal("fetch", fetch);
|
||||
const data = file("large.bson.gz", 5 * 1024 ** 3);
|
||||
const metadata = file("large.metadata.json.gz", 200);
|
||||
await prepareMongodbRestoreSource([data, metadata], "directory", true);
|
||||
expect(fetch).toHaveBeenCalledTimes(1);
|
||||
const form = fetch.mock.calls[0]![1].body as FormData;
|
||||
expect(form.get("format")).toBe("directory");
|
||||
expect(form.get("gzip")).toBe("true");
|
||||
expect(JSON.parse(form.get("manifest") as string)).toEqual([
|
||||
{ path: "dump/source/large.bson.gz", sizeBytes: 5 * 1024 ** 3 },
|
||||
{ path: "dump/source/large.metadata.json.gz", sizeBytes: 200 },
|
||||
]);
|
||||
expect(form.getAll("file")).toHaveLength(1);
|
||||
expect((form.get("file") as File).name).toBe("dump/source/large.metadata.json.gz");
|
||||
});
|
||||
|
||||
it("checks the effective server limit before the confirmed data upload", async () => {
|
||||
const fetch = vi.fn().mockResolvedValue(new Response(JSON.stringify(1024 ** 3)));
|
||||
vi.stubGlobal("fetch", fetch);
|
||||
const request = { taskId: "task", connectionId: "conn", database: "target", sourceDatabase: "source", sourceRef: "catalog", dropExisting: true, restoreOptions: true, restoreIndexes: true, stopOnError: true, batchSize: 500 };
|
||||
await expect(restoreMongodbDatabase(request, vi.fn(), { files: [file("large.bson", 5 * 1024 ** 3)], gzip: false })).rejects.toThrow("exceeds");
|
||||
expect(fetch).toHaveBeenCalledTimes(1);
|
||||
expect(fetch.mock.calls[0]![0]).toContain("/upload-limit");
|
||||
});
|
||||
|
||||
it("passes cancellation to catalog upload without a separate restore request", async () => {
|
||||
const controller = new AbortController();
|
||||
const fetch = vi.fn().mockRejectedValue(new DOMException("Aborted", "AbortError"));
|
||||
vi.stubGlobal("fetch", fetch);
|
||||
controller.abort();
|
||||
await expect(prepareMongodbRestoreSource([file("a.bson")], "directory", false, { signal: controller.signal })).rejects.toThrow("Aborted");
|
||||
expect(fetch.mock.calls[0]![1].signal).toBe(controller.signal);
|
||||
expect(fetch).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -495,6 +495,13 @@ export const cancelMongodbImport = forward("cancelMongodbImport");
|
||||
export const releaseMongodbImportSource = forward("releaseMongodbImportSource");
|
||||
export const exportMongodbQuery = forward("exportMongodbQuery");
|
||||
export const cancelMongodbExport = forward("cancelMongodbExport");
|
||||
export const inspectMongodbDatabaseDump = forward("inspectMongodbDatabaseDump");
|
||||
export const prepareMongodbRestoreSource = forward("prepareMongodbRestoreSource");
|
||||
export const releaseMongodbRestoreSource = forward("releaseMongodbRestoreSource");
|
||||
export const dumpMongodbDatabase = forward("dumpMongodbDatabase");
|
||||
export const restoreMongodbDatabase = forward("restoreMongodbDatabase");
|
||||
export const cancelMongodbDatabaseDump = forward("cancelMongodbDatabaseDump");
|
||||
export type { MongoDumpFormat, MongoDumpSourceInput, MongoDumpCatalog, MongoDumpCollection, MongoRestoreSourcePreview, MongoDatabaseDumpRequest, MongoDatabaseRestoreRequest, MongoDatabaseDumpProgress } from "./mongodbDumpTypes";
|
||||
|
||||
// Database Export
|
||||
export const beginDatabaseBackupSnapshot = forward("beginDatabaseBackupSnapshot");
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import type { MongoDumpFormat, MongoDumpSourceInput, MongoDumpCatalog, MongoRestoreSourcePreview, MongoDatabaseDumpRequest, MongoDatabaseRestoreRequest, MongoDatabaseDumpProgress } from "./mongodbDumpTypes";
|
||||
import type { MongoRestoreUpload, MongoSourceReadOptions } from "./mongodbDumpTypes";
|
||||
import type {
|
||||
ConnectionConfig,
|
||||
ConnectionTestResult,
|
||||
@@ -2674,6 +2676,143 @@ export async function releaseTableImportSource(sourceRef: string): Promise<boole
|
||||
return result.released;
|
||||
}
|
||||
|
||||
export function inspectMongodbDatabaseDump(connectionId: string, database: string): Promise<MongoDumpCatalog> {
|
||||
return post("/api/mongo/dump/catalog", { connectionId, database });
|
||||
}
|
||||
async function checkMongoUploadSize(files: File[]) {
|
||||
const limit = await get<number>("/api/mongo/dump/upload-limit");
|
||||
const size = files.reduce((total, file) => total + file.size, 0);
|
||||
if (size > limit) throw new Error(`MongoDB upload: ${(size / 1024 ** 3).toFixed(2)} GiB exceeds ${(limit / 1024 ** 3).toFixed(2)} GiB (DBX_MAX_UPLOAD_MB)`);
|
||||
}
|
||||
|
||||
function uploadMongoForm(url: string, form: FormData, options?: MongoSourceReadOptions): Promise<MongoRestoreSourcePreview> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const xhr = new XMLHttpRequest();
|
||||
const abort = () => xhr.abort();
|
||||
const cleanup = () => options?.signal?.removeEventListener("abort", abort);
|
||||
if (options?.signal?.aborted) {
|
||||
reject(new DOMException("Cancelled", "AbortError"));
|
||||
return;
|
||||
}
|
||||
xhr.open("POST", apiUrl(url));
|
||||
xhr.upload.onprogress = (event) => options?.onUploadProgress?.(event.loaded, event.lengthComputable ? event.total : 0);
|
||||
xhr.onload = async () => {
|
||||
cleanup();
|
||||
try {
|
||||
if (xhr.status < 200 || xhr.status >= 300) throw await backendResponseError(new Response(xhr.responseText, { status: xhr.status }));
|
||||
resolve(JSON.parse(xhr.responseText));
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
}
|
||||
};
|
||||
xhr.onerror = () => {
|
||||
cleanup();
|
||||
reject(new Error("MongoDB upload connection failed"));
|
||||
};
|
||||
xhr.onabort = () => {
|
||||
cleanup();
|
||||
reject(new DOMException("Cancelled", "AbortError"));
|
||||
};
|
||||
options?.signal?.addEventListener("abort", abort, { once: true });
|
||||
xhr.send(form);
|
||||
});
|
||||
}
|
||||
|
||||
export async function prepareMongodbRestoreSource(source: MongoDumpSourceInput, format: MongoDumpFormat, gzip: boolean, options?: MongoSourceReadOptions): Promise<MongoRestoreSourcePreview> {
|
||||
if (typeof source === "string") throw new Error("Web restores require uploaded files");
|
||||
const files = Array.isArray(source) ? source : [source];
|
||||
const form = new FormData();
|
||||
form.append("format", format);
|
||||
form.append("gzip", String(gzip));
|
||||
if (format === "directory") {
|
||||
form.append("manifest", JSON.stringify(files.map((file) => ({ path: file.webkitRelativePath || file.name, sizeBytes: file.size }))));
|
||||
for (const file of files) if (/\.metadata\.json(?:\.gz)?$/.test(file.name)) form.append("file", file, file.webkitRelativePath || file.name);
|
||||
} else {
|
||||
await checkMongoUploadSize(files);
|
||||
for (const file of files) form.append("file", file, file.name);
|
||||
return uploadMongoForm("/api/mongo/dump/source", form, options);
|
||||
}
|
||||
const response = await fetch(apiUrl("/api/mongo/dump/source"), { method: "POST", body: form, signal: options?.signal });
|
||||
if (!response.ok) throw await backendResponseError(response);
|
||||
return response.json();
|
||||
}
|
||||
export async function releaseMongodbRestoreSource(sourceRef: string): Promise<boolean> {
|
||||
return (await post<{ released: boolean }>("/api/mongo/dump/source/release", { sourceRef })).released;
|
||||
}
|
||||
async function runMongodbDatabaseTask(mode: "export" | "restore", request: MongoDatabaseDumpRequest | MongoDatabaseRestoreRequest, onProgress: (progress: MongoDatabaseDumpProgress) => void): Promise<MongoDatabaseDumpProgress> {
|
||||
await post(`/api/mongo/dump/${mode}`, { request });
|
||||
return new Promise((resolve, reject) => {
|
||||
const events = new EventSource(apiUrl(`/api/mongo/dump/progress/${request.taskId}`));
|
||||
events.onmessage = (event) => {
|
||||
const progress: MongoDatabaseDumpProgress = JSON.parse(event.data);
|
||||
onProgress(progress);
|
||||
if (progress.status === "running") return;
|
||||
events.close();
|
||||
if (progress.status !== "done") {
|
||||
reject(new Error(progress.errorMessage || "MongoDB database task failed"));
|
||||
return;
|
||||
}
|
||||
if (mode === "export") {
|
||||
const anchor = document.createElement("a");
|
||||
anchor.href = apiUrl(`/api/mongo/export/download/${request.taskId}`);
|
||||
anchor.click();
|
||||
}
|
||||
resolve(progress);
|
||||
};
|
||||
events.onerror = () => {
|
||||
events.close();
|
||||
reject(new Error("MongoDB database progress connection failed"));
|
||||
};
|
||||
});
|
||||
}
|
||||
export function dumpMongodbDatabase(request: MongoDatabaseDumpRequest, onProgress: (progress: MongoDatabaseDumpProgress) => void) {
|
||||
return runMongodbDatabaseTask("export", request, onProgress);
|
||||
}
|
||||
export async function restoreMongodbDatabase(request: MongoDatabaseRestoreRequest, onProgress: (progress: MongoDatabaseDumpProgress) => void, upload?: MongoRestoreUpload) {
|
||||
let acquired: string | undefined;
|
||||
try {
|
||||
if (upload) {
|
||||
await checkMongoUploadSize(upload.files);
|
||||
const form = new FormData();
|
||||
form.append("format", "directory");
|
||||
form.append("gzip", String(upload.gzip));
|
||||
form.append("request", JSON.stringify(request));
|
||||
for (const file of upload.files) form.append("file", file, file.webkitRelativePath || file.name);
|
||||
const started = Date.now();
|
||||
const preview = await uploadMongoForm("/api/mongo/dump/source/upload", form, {
|
||||
signal: upload.signal,
|
||||
onUploadProgress: (loaded, total) =>
|
||||
onProgress({
|
||||
taskId: request.taskId,
|
||||
status: "running",
|
||||
phase: "uploading",
|
||||
collection: null,
|
||||
collectionsDone: 0,
|
||||
collectionsTotal: request.collections?.length ?? 0,
|
||||
documentsRead: 0,
|
||||
documentsWritten: 0,
|
||||
documentsFailed: 0,
|
||||
indexesCreated: 0,
|
||||
elapsedMs: Date.now() - started,
|
||||
errorMessage: null,
|
||||
filePath: null,
|
||||
bytesProcessed: loaded,
|
||||
bytesTotal: total,
|
||||
}),
|
||||
});
|
||||
acquired = preview.sourceRef;
|
||||
if (upload.signal?.aborted) throw new DOMException("Cancelled", "AbortError");
|
||||
request = { ...request, sourceRef: acquired };
|
||||
}
|
||||
return await runMongodbDatabaseTask("restore", request, onProgress);
|
||||
} finally {
|
||||
if (acquired) void releaseMongodbRestoreSource(acquired).catch(() => {});
|
||||
}
|
||||
}
|
||||
export async function cancelMongodbDatabaseDump(taskId: string): Promise<boolean> {
|
||||
return (await post<{ cancelled: boolean }>("/api/mongo/dump/cancel", { taskId })).cancelled;
|
||||
}
|
||||
|
||||
export async function previewMongodbImportFile(fileOrPath: string | File | MongoImportPreviewRequest, options: Partial<MongoImportPreviewRequest> = {}): Promise<MongoImportPreview> {
|
||||
if (typeof fileOrPath === "object" && !(fileOrPath instanceof File)) {
|
||||
throw new Error("previewMongodbImportFile in web mode requires a File object for upload previews");
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
export type MongoDumpFormat = "directory" | "archive";
|
||||
export type MongoDumpSourceInput = string | File | File[];
|
||||
|
||||
export interface MongoDumpCollection {
|
||||
database: string;
|
||||
name: string;
|
||||
kind: string;
|
||||
documents: number | null;
|
||||
sizeBytes: number;
|
||||
indexes: number;
|
||||
sourceFiles?: string[];
|
||||
}
|
||||
export interface MongoDumpCatalog {
|
||||
databases: string[];
|
||||
collections: MongoDumpCollection[];
|
||||
}
|
||||
export interface MongoRestoreSourcePreview extends MongoDumpCatalog {
|
||||
sourceRef: string;
|
||||
}
|
||||
export interface MongoDatabaseDumpRequest {
|
||||
taskId: string;
|
||||
connectionId: string;
|
||||
database: string;
|
||||
filePath: string;
|
||||
format: MongoDumpFormat;
|
||||
gzip: boolean;
|
||||
collections?: string[];
|
||||
}
|
||||
export interface MongoDatabaseRestoreRequest {
|
||||
taskId: string;
|
||||
connectionId: string;
|
||||
database: string;
|
||||
sourceDatabase: string;
|
||||
sourceRef: string;
|
||||
collections?: string[];
|
||||
dropExisting: boolean;
|
||||
restoreOptions: boolean;
|
||||
restoreIndexes: boolean;
|
||||
stopOnError: boolean;
|
||||
objcheck?: boolean;
|
||||
batchSize: number;
|
||||
executionId?: string;
|
||||
}
|
||||
export interface MongoDatabaseDumpProgress {
|
||||
taskId: string;
|
||||
status: "running" | "done" | "error" | "cancelled";
|
||||
phase: "preparing" | "uploading" | "validating" | "data" | "indexes" | "views" | "archive" | "done";
|
||||
collection: string | null;
|
||||
collectionsDone: number;
|
||||
collectionsTotal: number;
|
||||
documentsRead: number;
|
||||
documentsWritten: number;
|
||||
documentsFailed: number;
|
||||
indexesCreated: number;
|
||||
elapsedMs: number;
|
||||
errorMessage: string | null;
|
||||
filePath: string | null;
|
||||
bytesProcessed?: number;
|
||||
bytesTotal?: number;
|
||||
documentsValidated?: number;
|
||||
}
|
||||
|
||||
export interface MongoRestoreUpload {
|
||||
files: File[];
|
||||
gzip: boolean;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface MongoSourceReadOptions {
|
||||
signal?: AbortSignal;
|
||||
onUploadProgress?: (loaded: number, total: number) => void;
|
||||
}
|
||||
@@ -1,4 +1,6 @@
|
||||
import { invoke as tauriInvoke } from "@tauri-apps/api/core";
|
||||
import type { MongoDumpFormat, MongoDumpSourceInput, MongoDumpCatalog, MongoRestoreSourcePreview, MongoDatabaseDumpRequest, MongoDatabaseRestoreRequest, MongoDatabaseDumpProgress } from "./mongodbDumpTypes";
|
||||
import type { MongoRestoreUpload, MongoSourceReadOptions } from "./mongodbDumpTypes";
|
||||
import { assertUpdateAllowsCommand } from "@/lib/app/updatePreparation";
|
||||
import { collectBrowserSupportInfo } from "@/lib/app/supportInfo";
|
||||
|
||||
@@ -5070,12 +5072,42 @@ export async function releaseTableImportSource(_sourceRef: string): Promise<bool
|
||||
return false;
|
||||
}
|
||||
|
||||
export type MongoImportFormat = "csv" | "json" | "ndjson";
|
||||
export function inspectMongodbDatabaseDump(connectionId: string, database: string): Promise<MongoDumpCatalog> {
|
||||
return invoke("inspect_mongodb_database_dump", { connectionId, database });
|
||||
}
|
||||
export function prepareMongodbRestoreSource(source: MongoDumpSourceInput, format: MongoDumpFormat, gzip: boolean, _options?: MongoSourceReadOptions): Promise<MongoRestoreSourcePreview> {
|
||||
if (typeof source !== "string") throw new Error("Desktop restores require a file or directory path");
|
||||
return invoke("prepare_mongodb_restore_source", { request: { path: source, format, gzip } });
|
||||
}
|
||||
export function releaseMongodbRestoreSource(sourceRef: string): Promise<boolean> {
|
||||
return invoke("release_mongodb_restore_source", { sourceRef });
|
||||
}
|
||||
async function runMongodbDatabaseTask(command: string, request: MongoDatabaseDumpRequest | MongoDatabaseRestoreRequest, onProgress: (progress: MongoDatabaseDumpProgress) => void): Promise<MongoDatabaseDumpProgress> {
|
||||
const unlisten = await listen<MongoDatabaseDumpProgress>("mongo-database-dump-progress", (event) => {
|
||||
if (event.payload.taskId === request.taskId) onProgress(event.payload);
|
||||
});
|
||||
try {
|
||||
return await invoke(command, { request });
|
||||
} finally {
|
||||
unlisten();
|
||||
}
|
||||
}
|
||||
export function dumpMongodbDatabase(request: MongoDatabaseDumpRequest, onProgress: (progress: MongoDatabaseDumpProgress) => void) {
|
||||
return runMongodbDatabaseTask("dump_mongodb_database", request, onProgress);
|
||||
}
|
||||
export function restoreMongodbDatabase(request: MongoDatabaseRestoreRequest, onProgress: (progress: MongoDatabaseDumpProgress) => void, _upload?: MongoRestoreUpload) {
|
||||
return runMongodbDatabaseTask("restore_mongodb_database", request, onProgress);
|
||||
}
|
||||
export function cancelMongodbDatabaseDump(taskId: string): Promise<boolean> {
|
||||
return invoke("cancel_mongodb_database_dump", { taskId });
|
||||
}
|
||||
|
||||
export type MongoImportFormat = "csv" | "json" | "ndjson" | "bson";
|
||||
export type MongoImportTypeMode = "string" | "auto" | "extendedJson";
|
||||
export type MongoImportInferredType = "boolean" | "integer" | "decimal" | "date" | "objectId" | "object" | "array" | "mixed" | "string";
|
||||
export type MongoImportStatus = "running" | "done" | "error" | "cancelled";
|
||||
export type MongoImportPhase = "preparing" | "parsing" | "writing" | "done";
|
||||
export type MongoExportFormat = "csv" | "ndjson";
|
||||
export type MongoExportFormat = "csv" | "ndjson" | "bson";
|
||||
export type MongoExportStatus = "running" | "done" | "error" | "cancelled";
|
||||
|
||||
export interface MongoImportIssue {
|
||||
@@ -5176,6 +5208,7 @@ export interface MongoExportRequest {
|
||||
collation?: string | null;
|
||||
format: MongoExportFormat;
|
||||
includeHeader?: boolean;
|
||||
gzip?: boolean;
|
||||
filePath: string;
|
||||
executionId?: string | null;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user