feat(mongodb): bson dumps and database backup/restore

This commit is contained in:
Hand Sonic
2026-09-16 22:46:48 +08:00
committed by GitHub
parent ecc1135de3
commit f5673ca0a1
54 changed files with 6355 additions and 106 deletions
Generated
+19
View File
@@ -1701,6 +1701,21 @@ dependencies = [
"libc",
]
[[package]]
name = "crc"
version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d"
dependencies = [
"crc-catalog",
]
[[package]]
name = "crc-catalog"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853"
[[package]]
name = "crc16"
version = "0.4.0"
@@ -2091,6 +2106,7 @@ dependencies = [
"calamine",
"cbc 0.1.2",
"chrono",
"crc",
"csv",
"dbx-sqlite-worker",
"deadpool-postgres",
@@ -2134,6 +2150,7 @@ dependencies = [
"serde_json",
"serde_yaml",
"serde_yaml_ng",
"sha1 0.10.7",
"sha2 0.10.9",
"sqlparser",
"ssfmt",
@@ -2215,6 +2232,7 @@ dependencies = [
"serde",
"serde_json",
"sha2 0.10.9",
"tempfile",
"tokio",
"tokio-util",
"tower-http",
@@ -6899,6 +6917,7 @@ dependencies = [
"js-sys",
"log",
"mime",
"mime_guess",
"native-tls",
"percent-encoding",
"pin-project-lite",
+17
View File
@@ -954,6 +954,21 @@ dependencies = [
"libc",
]
[[package]]
name = "crc"
version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d"
dependencies = [
"crc-catalog",
]
[[package]]
name = "crc-catalog"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853"
[[package]]
name = "crc16"
version = "0.4.0"
@@ -1243,6 +1258,7 @@ dependencies = [
"calamine",
"cbc 0.1.2",
"chrono",
"crc",
"csv",
"dbx-sqlite-worker",
"deadpool-postgres",
@@ -1282,6 +1298,7 @@ dependencies = [
"serde_json",
"serde_yaml",
"serde_yaml_ng",
"sha1 0.10.7",
"sha2 0.10.9",
"sqlparser",
"ssfmt",
@@ -0,0 +1,438 @@
<script setup lang="ts">
import { computed, nextTick, onBeforeUnmount, ref, watch } from "vue";
import { useI18n } from "vue-i18n";
import { ArrowLeft, ArrowRight, DatabaseBackup, FolderOpen, Loader2, Play, Square, X } from "@lucide/vue";
import { Dialog, DialogFooter, DialogHeader, DialogScrollContent, DialogTitle } from "@/components/ui/dialog";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
import { useConnectionStore } from "@/stores/connectionStore";
import { connectionIsEffectivelyReadOnly } from "@/lib/database/readOnlyWriteAccess";
import { executeWithProductionContextGuard } from "@/lib/database/productionExecutionGuard";
import { isTauriRuntime } from "@/lib/backend/tauriRuntime";
import { uuid } from "@/lib/common/utils";
import * as api from "@/lib/backend/api";
const props = defineProps<{ connectionId: string; database: string; mode: "dump" | "restore" }>();
const open = defineModel<boolean>("open", { default: false });
const { t } = useI18n();
const store = useConnectionStore();
const desktop = isTauriRuntime();
const restoring = computed(() => props.mode === "restore");
const connection = computed(() => store.getConfig(props.connectionId));
const readonly = computed(() => restoring.value && connectionIsEffectivelyReadOnly(connection.value));
const format = ref<api.MongoDumpFormat>("archive");
const gzip = ref(false);
const catalog = ref<api.MongoDumpCatalog | null>(null);
const sourceRef = ref<string | null>(null);
const source = ref<api.MongoDumpSourceInput | null>(null);
const sourceName = ref("");
const sourceDatabase = ref("");
const targetDatabase = ref(props.database);
const outputPath = ref(desktop ? "" : `${props.database}.archive`);
const selected = ref<string[]>([]);
const dropExisting = ref(false);
const restoreOptions = ref(true);
const restoreIndexes = ref(true);
const stopOnError = ref(true);
const objcheck = ref(false);
const batchSize = ref(500);
const loading = ref(false);
const confirming = ref(false);
const running = ref(false);
const cancelling = ref(false);
const error = ref("");
const step = ref<"configure" | "confirm" | "result">("configure");
const progress = ref<api.MongoDatabaseDumpProgress | null>(null);
const taskId = ref("");
const fileInput = ref<HTMLInputElement | null>(null);
let generation = 0;
let readController: AbortController | null = null;
let uploadController: AbortController | null = null;
const readUpload = ref<{ loaded: number; total: number } | null>(null);
function gib(value: number) {
return (value / 1024 ** 3).toFixed(2);
}
const entries = computed(() => catalog.value?.collections.filter((entry) => !restoring.value || entry.database === sourceDatabase.value) ?? []);
const allSelected = computed(() => entries.value.length > 0 && selected.value.length === entries.value.length);
const canReview = computed(
() =>
!!catalog.value &&
!loading.value &&
!confirming.value &&
!running.value &&
!readonly.value &&
(entries.value.length === 0 || selected.value.length > 0) &&
(restoring.value ? !!sourceRef.value && !!targetDatabase.value.trim() && (!catalog.value.databases.length || catalog.value.databases.includes(sourceDatabase.value)) : !!outputPath.value.trim()),
);
function releaseSource() {
const reference = sourceRef.value;
sourceRef.value = null;
if (reference) void api.releaseMongodbRestoreSource(reference);
}
function invalidate() {
generation += 1;
releaseSource();
catalog.value = null;
selected.value = [];
error.value = "";
step.value = "configure";
}
function cleanup() {
generation += 1;
readController?.abort();
if (!running.value) releaseSource();
}
onBeforeUnmount(cleanup);
watch(
open,
async (value) => {
if (!value) {
cleanup();
return;
}
if (restoring.value) return;
const current = ++generation;
loading.value = true;
try {
await store.ensureConnected(props.connectionId);
const result = await api.inspectMongodbDatabaseDump(props.connectionId, props.database);
if (current !== generation) return;
catalog.value = result;
selected.value = result.collections.map((entry) => entry.name);
} catch (cause) {
if (current === generation) error.value = String(cause instanceof Error ? cause.message : cause);
} finally {
if (current === generation) loading.value = false;
}
},
{ immediate: true },
);
watch([format, gzip], () => {
if (restoring.value) invalidate();
else outputPath.value = desktop ? "" : `${props.database}.archive${gzip.value ? ".gz" : ""}`;
});
watch(format, () => {
if (restoring.value) {
source.value = null;
sourceName.value = "";
}
});
function setFormat(value: unknown) {
if (value === "directory" || value === "archive") format.value = value;
}
function setSourceDatabase(value: unknown) {
if (typeof value !== "string") return;
sourceDatabase.value = value;
selected.value = entries.value.map((entry) => entry.name);
}
function selectAll(event: Event) {
selected.value = (event.target as HTMLInputElement).checked ? entries.value.map((entry) => entry.name) : [];
}
function selectCollection(name: string, event: Event) {
selected.value = (event.target as HTMLInputElement).checked ? [...selected.value, name] : selected.value.filter((value) => value !== name);
}
async function chooseSource() {
if (!desktop) {
fileInput.value?.click();
return;
}
const { open: choose } = await import("@tauri-apps/plugin-dialog");
const path = await choose({ directory: format.value === "directory", multiple: false });
if (typeof path === "string") {
invalidate();
source.value = path;
sourceName.value = path;
}
}
function chooseFiles(event: Event) {
const input = event.target as HTMLInputElement;
const files = Array.from(input.files ?? []);
input.value = "";
if (!files.length) return;
invalidate();
source.value = format.value === "directory" ? files : files[0]!;
sourceName.value = format.value === "directory" ? `${(files[0]!.webkitRelativePath || files[0]!.name).split("/")[0]} (${files.length})` : files[0]!.name;
}
async function readSource() {
if (!source.value) return;
releaseSource();
const current = ++generation;
loading.value = true;
error.value = "";
catalog.value = null;
readUpload.value = null;
readController = new AbortController();
try {
const result = await api.prepareMongodbRestoreSource(source.value, format.value, gzip.value, {
signal: readController.signal,
onUploadProgress: (loaded, total) => {
readUpload.value = { loaded, total };
},
});
if (current !== generation) {
void api.releaseMongodbRestoreSource(result.sourceRef);
return;
}
sourceRef.value = result.sourceRef;
catalog.value = result;
setSourceDatabase(result.databases.length === 1 ? result.databases[0] : "");
} catch (cause) {
if (current === generation) error.value = String(cause instanceof Error ? cause.message : cause);
} finally {
if (current === generation) loading.value = false;
readController = null;
}
}
async function chooseDestination() {
const { open: choose, save } = await import("@tauri-apps/plugin-dialog");
if (format.value === "directory") {
const parent = await choose({ directory: true, multiple: false });
if (typeof parent === "string") outputPath.value = `${parent.replace(/[\\/]$/, "")}/${props.database}-dump`;
} else {
const path = await save({ defaultPath: `${props.database}.archive${gzip.value ? ".gz" : ""}`, filters: [{ name: "MongoDB archive", extensions: [gzip.value ? "gz" : "archive"] }] });
if (path) outputPath.value = path;
}
}
async function refreshDatabase() {
await store.loadMongoDatabases(props.connectionId);
await store.loadMongoCollections(props.connectionId, targetDatabase.value);
for (const collection of selected.value) {
store.mongoImportCompleted = { connectionId: props.connectionId, database: targetDatabase.value, collection, at: Date.now() };
await nextTick();
}
}
async function execute() {
if (!canReview.value) return;
confirming.value = true;
const current = generation;
try {
targetDatabase.value = targetDatabase.value.trim();
if (restoring.value) {
const allowed = await executeWithProductionContextGuard({
connection: connection.value,
database: targetDatabase.value.trim(),
source: t("mongoDump.restoreTitle"),
reviewText: `${dropExisting.value ? "DROP selected collections; " : ""}RESTORE ${JSON.stringify(sourceDatabase.value)} INTO ${JSON.stringify(targetDatabase.value.trim())}: ${selected.value.map((name) => JSON.stringify(name)).join(", ")}`,
execute: async () => true,
});
if (!allowed) return;
}
if (current !== generation || !open.value) return;
running.value = true;
confirming.value = false;
step.value = "result";
error.value = "";
progress.value = null;
taskId.value = uuid();
const onProgress = (value: api.MongoDatabaseDumpProgress) => {
progress.value = value;
};
await store.ensureConnected(props.connectionId);
if (restoring.value) {
uploadController = new AbortController();
const selectedPaths = new Set(entries.value.filter((entry) => selected.value.includes(entry.name)).flatMap((entry) => entry.sourceFiles ?? []));
const upload = !desktop && format.value === "directory" && Array.isArray(source.value) ? { files: source.value.filter((file) => selectedPaths.has(file.webkitRelativePath || file.name)), gzip: gzip.value, signal: uploadController.signal } : undefined;
progress.value = await api.restoreMongodbDatabase(
{
taskId: taskId.value,
connectionId: props.connectionId,
database: targetDatabase.value.trim(),
sourceDatabase: sourceDatabase.value,
sourceRef: sourceRef.value!,
collections: [...selected.value],
dropExisting: dropExisting.value,
restoreOptions: restoreOptions.value,
restoreIndexes: restoreIndexes.value,
stopOnError: stopOnError.value,
objcheck: objcheck.value,
batchSize: Math.max(100, Math.min(5000, Math.trunc(Number(batchSize.value)) || 500)),
},
onProgress,
upload,
);
void refreshDatabase().catch((cause) => console.error("Refresh after MongoDB restore failed", cause));
} else {
progress.value = await api.dumpMongodbDatabase({ taskId: taskId.value, connectionId: props.connectionId, database: props.database, filePath: outputPath.value.trim(), format: format.value, gzip: gzip.value, collections: [...selected.value] }, onProgress);
}
} catch (cause) {
error.value = String(cause instanceof Error ? cause.message : cause);
if (cause instanceof DOMException && cause.name === "AbortError") {
progress.value = { taskId: taskId.value, status: "cancelled", phase: "done", collection: null, collectionsDone: 0, collectionsTotal: selected.value.length, documentsRead: 0, documentsWritten: 0, documentsFailed: 0, indexesCreated: 0, elapsedMs: 0, errorMessage: null, filePath: null };
error.value = "";
}
} finally {
confirming.value = false;
running.value = false;
cancelling.value = false;
uploadController = null;
}
}
async function cancel() {
cancelling.value = true;
uploadController?.abort();
try {
await api.cancelMongodbDatabaseDump(taskId.value);
} catch (cause) {
cancelling.value = false;
error.value = String(cause instanceof Error ? cause.message : cause);
}
}
function close() {
if (!running.value && !confirming.value) open.value = false;
}
</script>
<template>
<Dialog
:open="open"
@update:open="
(value) => {
if (!value) close();
}
"
>
<DialogScrollContent class="flex max-h-[calc(var(--dbx-viewport-height)-4rem)] min-h-0 flex-col overflow-hidden sm:max-w-[860px]">
<DialogHeader class="shrink-0 pr-8"
><DialogTitle class="flex items-center gap-2 text-base"><DatabaseBackup class="h-4 w-4" />{{ t(restoring ? "mongoDump.restoreTitle" : "mongoDump.dumpTitle") }}</DialogTitle></DialogHeader
>
<div class="min-h-0 flex-1 space-y-4 overflow-y-auto py-3 text-sm">
<div class="border-b pb-2 font-medium break-all">{{ connection?.name }} / {{ props.database }}</div>
<div v-if="readonly" class="text-xs text-amber-600">{{ t("mongoDump.readonly") }}</div>
<template v-if="step === 'configure'">
<div class="grid grid-cols-1 gap-3 sm:grid-cols-2">
<div class="space-y-1">
<Label>{{ t("mongoDump.format") }}</Label
><Select :model-value="format" :disabled="loading" @update:model-value="setFormat"
><SelectTrigger class="h-8"><SelectValue /></SelectTrigger
><SelectContent
><SelectItem value="archive">{{ t("mongoDump.archive") }}</SelectItem
><SelectItem v-if="desktop || restoring" value="directory">{{ t("mongoDump.directory") }}</SelectItem></SelectContent
></Select
>
</div>
<label class="flex min-h-8 items-center gap-2 self-end"><input v-model="gzip" type="checkbox" :disabled="loading" />{{ t("mongoDump.gzip") }}</label>
</div>
<div v-if="restoring" class="space-y-2">
<Label>{{ t("mongoDump.source") }}</Label>
<input ref="fileInput" type="file" class="hidden" :multiple="format === 'directory'" :webkitdirectory="format === 'directory' ? '' : undefined" @change="chooseFiles" />
<div class="flex flex-wrap items-center gap-2">
<div class="min-w-0 flex-1 basis-full break-all text-xs sm:basis-0">{{ sourceName || t("tableImport.noFileSelected") }}</div>
<Button variant="outline" size="sm" :disabled="loading" @click="chooseSource"><FolderOpen class="mr-1.5 h-4 w-4" />{{ t("tableImport.selectFile") }}</Button
><Button size="sm" :disabled="!source || loading" @click="readSource"><Loader2 v-if="loading" class="mr-1.5 h-4 w-4 animate-spin" /><Play v-else class="mr-1.5 h-4 w-4" />{{ t("mongoDump.readSource") }}</Button>
</div>
<div class="grid grid-cols-1 gap-3 sm:grid-cols-2">
<div class="space-y-1">
<Label>{{ t("mongoDump.sourceDatabase") }}</Label
><Select :model-value="sourceDatabase" :disabled="!catalog || loading" @update:model-value="setSourceDatabase"
><SelectTrigger class="h-8 w-full min-w-0 [&>span]:min-w-0 [&>span]:truncate"><SelectValue /></SelectTrigger
><SelectContent
><SelectItem v-for="db in catalog?.databases ?? []" :key="db" :value="db">{{ db }}</SelectItem></SelectContent
></Select
>
</div>
<div class="space-y-1">
<Label>{{ t("mongoDump.targetDatabase") }}</Label
><Input v-model="targetDatabase" class="h-8" :disabled="loading" />
</div>
</div>
</div>
<div v-else class="space-y-1">
<Label>{{ t("mongoDump.destination") }}</Label>
<div class="flex gap-2">
<Input v-model="outputPath" class="h-8 min-w-0" :readonly="desktop" /><Button v-if="desktop" variant="outline" size="icon" class="h-8 w-8 shrink-0" :title="t('mongoDump.chooseDestination')" :aria-label="t('mongoDump.chooseDestination')" @click="chooseDestination"
><FolderOpen class="h-4 w-4"
/></Button>
</div>
</div>
<div v-if="loading" class="flex items-center gap-2 text-xs text-muted-foreground"><Loader2 class="h-4 w-4 animate-spin" />{{ t(restoring ? "mongoDump.preparing" : "mongoDump.loading") }}</div>
<div v-if="loading && readUpload" class="text-xs tabular-nums">{{ t("mongoDump.uploadBytes", { done: gib(readUpload.loaded), total: gib(readUpload.total) }) }}</div>
<div v-if="catalog" class="space-y-2">
<div class="text-xs text-muted-foreground">{{ t("mongoDump.selected", { count: selected.length }) }}</div>
<div class="max-h-64 overflow-auto border-y">
<table class="w-full min-w-[480px] table-fixed text-xs">
<thead class="sticky top-0 bg-background">
<tr>
<th class="w-8 p-2"><input type="checkbox" :checked="allSelected" :aria-label="t('mongoDump.selectAll')" @change="selectAll" /></th>
<th class="w-2/5 p-2 text-left">{{ t("mongoDump.collection") }}</th>
<th class="p-2 text-left">{{ t("mongoDump.kind") }}</th>
<th class="p-2 text-right">{{ t("mongoDump.documents") }}</th>
<th class="p-2 text-right">{{ t("mongoDump.indexes") }}</th>
</tr>
</thead>
<tbody>
<tr v-for="entry in entries" :key="entry.name" class="border-t">
<td class="p-2"><input type="checkbox" :checked="selected.includes(entry.name)" :aria-label="entry.name" @change="selectCollection(entry.name, $event)" /></td>
<td class="p-2 break-all">{{ entry.name }}</td>
<td class="p-2 break-all">{{ entry.kind }}</td>
<td class="p-2 text-right">{{ entry.documents ?? "-" }}</td>
<td class="p-2 text-right">{{ entry.indexes }}</td>
</tr>
<tr v-if="!entries.length">
<td colspan="5" class="p-4 text-center text-muted-foreground">{{ t("mongoDump.empty") }}</td>
</tr>
</tbody>
</table>
</div>
</div>
<div v-if="restoring" class="space-y-3 border-t pt-3">
<label class="flex items-center gap-2"><input v-model="dropExisting" type="checkbox" />{{ t("mongoDump.dropExisting") }}</label>
<div class="grid grid-cols-1 gap-3 sm:grid-cols-2">
<label class="flex items-center gap-2"><input v-model="restoreOptions" type="checkbox" />{{ t("mongoDump.restoreOptions") }}</label
><label class="flex items-center gap-2"><input v-model="restoreIndexes" type="checkbox" />{{ t("mongoDump.restoreIndexes") }}</label
><label class="flex items-center gap-2"><input v-model="stopOnError" type="checkbox" />{{ t("mongoDump.stopOnError") }}</label>
<label class="flex items-center gap-2"><input v-model="objcheck" type="checkbox" />{{ t("mongoDump.objcheck") }}</label>
<div class="flex items-center gap-2">
<Label class="shrink-0">{{ t("mongoDump.batchSize") }}</Label
><Input v-model.number="batchSize" type="number" min="100" max="5000" class="h-8 w-24" />
</div>
</div>
</div>
</template>
<template v-else-if="step === 'confirm'"
><p>{{ t(restoring ? "mongoDump.confirmRestore" : "mongoDump.confirmDump", { count: selected.length, database: restoring ? targetDatabase : props.database }) }}</p>
<p v-if="restoring" :class="dropExisting ? 'text-destructive' : 'text-muted-foreground'">{{ t(dropExisting ? "mongoDump.confirmDrop" : "mongoDump.confirmAppend") }}</p>
<div class="max-h-56 overflow-auto border-y py-2 text-xs">
<div v-for="name in selected" :key="name" class="py-1 break-all">{{ name }}</div>
</div>
<div v-if="!restoring" class="break-all text-xs text-muted-foreground">{{ outputPath }}</div></template
>
<template v-else
><div class="flex items-center gap-2 font-medium">
<Loader2 v-if="running" class="h-4 w-4 animate-spin" />{{ running ? t(`mongoDump.phase.${progress?.phase ?? "preparing"}`) : t(progress?.status === "done" ? "mongoDump.done" : progress?.status === "cancelled" ? "mongoDump.cancelled" : "mongoDump.failed") }}
</div>
<div class="break-all text-xs">{{ progress?.collection }}</div>
<div v-if="progress?.phase === 'uploading'" class="text-xs tabular-nums">{{ t("mongoDump.uploadBytes", { done: gib(progress.bytesProcessed ?? 0), total: gib(progress.bytesTotal ?? 0) }) }}</div>
<div v-if="restoring && progress?.phase === 'data'" class="text-xs tabular-nums">{{ t("mongoDump.readBytes", { size: gib(progress.bytesProcessed ?? 0) }) }}</div>
<div v-if="progress?.phase === 'validating'" class="text-xs tabular-nums">{{ t("mongoDump.validationBytes", { size: gib(progress.bytesProcessed ?? 0), count: progress.documentsValidated ?? 0 }) }}</div>
<progress class="h-2 w-full" :value="progress?.collectionsDone ?? 0" :max="Math.max(1, progress?.collectionsTotal ?? selected.length)" />
<div class="text-xs">{{ t("mongoDump.progress", { done: progress?.collectionsDone ?? 0, total: progress?.collectionsTotal ?? selected.length }) }}</div>
<dl class="grid grid-cols-2 gap-3 border-y py-3 text-xs">
<dt>{{ t("mongoDump.read") }}</dt>
<dd class="text-right tabular-nums">{{ progress?.documentsRead ?? 0 }}</dd>
<dt v-if="restoring">{{ t("mongoDump.written") }}</dt>
<dd v-if="restoring" class="text-right tabular-nums">{{ progress?.documentsWritten ?? 0 }}</dd>
<dt>{{ t("mongoDump.rejected") }}</dt>
<dd class="text-right tabular-nums">{{ progress?.documentsFailed ?? 0 }}</dd>
<dt v-if="restoring">{{ t("mongoDump.createdIndexes") }}</dt>
<dd v-if="restoring" class="text-right tabular-nums">{{ progress?.indexesCreated ?? 0 }}</dd>
</dl></template
>
<div v-if="error" class="break-words border-l-2 border-destructive pl-3 text-xs text-destructive" role="alert">{{ error }}</div>
</div>
<DialogFooter class="shrink-0"
><Button v-if="running" variant="destructive" :disabled="cancelling" @click="cancel"><Square class="mr-1.5 h-4 w-4" />{{ t("mongoDump.cancel") }}</Button
><Button v-else variant="outline" :disabled="confirming" @click="close"><X class="mr-1.5 h-4 w-4" />{{ t("mongoDump.close") }}</Button
><Button v-if="step !== 'configure' && !running" variant="outline" :disabled="confirming" @click="step = 'configure'"><ArrowLeft class="mr-1.5 h-4 w-4" />{{ t("mongoDump.back") }}</Button
><Button v-if="step === 'configure'" :disabled="!canReview" @click="step = 'confirm'"><ArrowRight class="mr-1.5 h-4 w-4" />{{ t("mongoDump.review") }}</Button
><Button v-if="step === 'confirm'" :disabled="!canReview" @click="execute"><Loader2 v-if="confirming" class="mr-1.5 h-4 w-4 animate-spin" /><Play v-else class="mr-1.5 h-4 w-4" />{{ t(restoring ? "mongoDump.restore" : "mongoDump.dump") }}</Button></DialogFooter
>
</DialogScrollContent>
</Dialog>
</template>
@@ -1,5 +1,5 @@
<script setup lang="ts">
import { computed, ref, watch } from "vue";
import { computed, onBeforeUnmount, ref, watch } from "vue";
import { uuid } from "@/lib/common/utils";
import { useI18n } from "vue-i18n";
import { isTauriRuntime } from "@/lib/backend/tauriRuntime";
@@ -14,7 +14,7 @@ import { useConnectionStore } from "@/stores/connectionStore";
import { connectionIsEffectivelyReadOnly } from "@/lib/database/readOnlyWriteAccess";
import { executeWithProductionContextGuard } from "@/lib/database/productionExecutionGuard";
import { TABLE_IMPORT_ENCODING_OPTIONS } from "@/lib/table/tableImport";
import { importPreviewInput, importSourceDisplayName, importTextDelimiterForName, uploadedImportSourceFromPreview, type UploadedImportSource } from "@/lib/import/importSource";
import { importPreviewInput, importSourceDisplayName, uploadedImportSourceFromPreview, type UploadedImportSource } from "@/lib/import/importSource";
import { useToast } from "@/composables/useToast";
import { translateBackendError } from "@/i18n/backend-errors";
import * as api from "@/lib/backend/api";
@@ -93,13 +93,6 @@ const parseOptions = computed((): api.MongoImportParseOptions => {
};
});
function formatFromName(name: string): api.MongoImportFormat {
const lower = name.toLowerCase();
if (lower.endsWith(".ndjson") || lower.endsWith(".jsonl")) return "ndjson";
if (lower.endsWith(".json")) return "json";
return "csv";
}
function releasePreviewSource() {
const sourceRef = uploadedSource?.sourceRef;
uploadedSource = null;
@@ -111,10 +104,7 @@ function assignSource(source: string | File) {
releasePreviewSource();
selectedSource.value = source;
sourceName.value = importSourceDisplayName(source);
format.value = formatFromName(sourceName.value);
typeMode.value = format.value === "csv" ? "auto" : "extendedJson";
columnTypeOverrides.value = {};
delimiter.value = importTextDelimiterForName(sourceName.value);
wizardStep.value = "source";
queuePreview();
}
@@ -127,7 +117,7 @@ async function selectFile() {
const { open: openDialog } = await import("@tauri-apps/plugin-dialog");
const selected = await openDialog({
multiple: false,
filters: [{ name: t("mongo.import.fileFilter"), extensions: ["csv", "json", "ndjson", "jsonl", "tsv"] }],
filters: [{ name: t("mongo.import.fileFilter"), extensions: ["bson", "gz", "csv", "json", "ndjson", "jsonl", "tsv"] }],
});
if (!selected || Array.isArray(selected)) return;
assignSource(selected);
@@ -143,15 +133,17 @@ function handleFileInputChange(event: Event) {
function queuePreview() {
if (!selectedSource.value) return;
if (previewReloadTimer) clearTimeout(previewReloadTimer);
const requestId = ++previewRequestId;
loadingPreview.value = true;
previewReloadTimer = setTimeout(() => {
void loadPreview();
previewReloadTimer = null;
void loadPreview(requestId);
}, 200);
}
async function loadPreview() {
async function loadPreview(requestId: number) {
const source = selectedSource.value;
if (!source) return;
const requestId = ++previewRequestId;
const keepExistingPreview = !!preview.value;
if (!keepExistingPreview) {
loadingPreview.value = true;
@@ -165,7 +157,10 @@ async function loadPreview() {
previewLimit: previewLimit.value,
sourceRef: input.sourceRef,
});
if (requestId !== previewRequestId) return;
if (requestId !== previewRequestId) {
if (next.sourceRef && next.sourceRef !== uploadedSource?.sourceRef) void api.releaseMongodbImportSource(next.sourceRef);
return;
}
preview.value = next;
previewError.value = "";
uploadedSource = uploadedImportSourceFromPreview(next) ?? uploadedSource;
@@ -178,13 +173,21 @@ async function loadPreview() {
}
}
watch([format, hasHeader, typeMode], () => {
if (Object.keys(columnTypeOverrides.value).length) columnTypeOverrides.value = {};
});
watch(
[format, hasHeader, typeMode],
() => {
if (Object.keys(columnTypeOverrides.value).length) columnTypeOverrides.value = {};
},
{ flush: "sync" },
);
watch([format, encoding, delimiter, hasHeader, trim, emptyAsNull, typeMode, recognizeObjectIdHex, previewLimit, columnTypeOverrides], () => {
if (selectedSource.value) queuePreview();
});
watch(
[format, encoding, delimiter, hasHeader, trim, emptyAsNull, typeMode, recognizeObjectIdHex, previewLimit, columnTypeOverrides],
() => {
if (selectedSource.value) queuePreview();
},
{ flush: "sync" },
);
watch(open, (value) => {
if (value) return;
@@ -197,6 +200,10 @@ watch(open, (value) => {
});
function reset() {
previewRequestId += 1;
if (previewReloadTimer) clearTimeout(previewReloadTimer);
previewReloadTimer = null;
loadingPreview.value = false;
releasePreviewSource();
selectedSource.value = null;
sourceName.value = "";
@@ -211,8 +218,12 @@ function reset() {
importId.value = "";
}
onBeforeUnmount(() => {
if (!running.value) reset();
});
function canConfirm() {
return !!preview.value && !loadingPreview.value && !previewError.value && !effectivelyReadOnly.value && (!preview.value.errors.length || skipErrorRows.value);
return !!preview.value && preview.value.format === format.value && !loadingPreview.value && !previewError.value && !effectivelyReadOnly.value && (!preview.value.errors.length || skipErrorRows.value);
}
async function startImport() {
@@ -287,7 +298,7 @@ function formatPreviewCell(value: unknown) {
}
function setFormat(value: unknown) {
if (value === "csv" || value === "json" || value === "ndjson") format.value = value;
if (value === "csv" || value === "json" || value === "ndjson" || value === "bson") format.value = value;
}
function setEncoding(value: unknown) {
@@ -343,7 +354,7 @@ function requestClose() {
</DialogHeader>
<div class="min-h-0 flex-1 space-y-4 overflow-y-auto py-2 pr-1 text-sm">
<div class="grid grid-cols-[minmax(0,1fr)_auto] items-center gap-2">
<input ref="fileInput" type="file" class="hidden" accept=".csv,.json,.ndjson,.jsonl,.tsv" @change="handleFileInputChange" />
<input ref="fileInput" type="file" class="hidden" accept=".bson,.bson.gz,.csv,.json,.ndjson,.jsonl,.tsv" @change="handleFileInputChange" />
<div class="flex h-10 min-w-0 items-center gap-2 rounded-md border bg-muted/20 px-3">
<span class="shrink-0 text-xs text-muted-foreground">{{ t("mongo.import.target") }}</span>
<span class="min-w-0 truncate text-sm font-medium">{{ targetLabel }}</span>
@@ -366,10 +377,11 @@ function requestClose() {
<SelectItem value="csv">CSV</SelectItem>
<SelectItem value="json">JSON</SelectItem>
<SelectItem value="ndjson">NDJSON</SelectItem>
<SelectItem value="bson">BSON dump</SelectItem>
</SelectContent>
</Select>
</div>
<div class="space-y-1.5">
<div v-if="format !== 'bson'" class="space-y-1.5">
<Label class="text-xs">{{ t("tableImport.encoding") }}</Label>
<Select :model-value="encoding" @update:model-value="setEncoding">
<SelectTrigger class="h-8 text-xs"><SelectValue /></SelectTrigger>
@@ -382,7 +394,7 @@ function requestClose() {
<Label class="text-xs">{{ t("tableImport.delimiter") }}</Label>
<Input v-model="delimiter" class="h-8 text-xs font-mono" :aria-label="t('tableImport.delimiter')" />
</div>
<div class="space-y-1.5">
<div v-if="format !== 'bson'" class="space-y-1.5">
<Label class="text-xs">{{ t("mongo.import.typeMode") }}</Label>
<Select :model-value="typeMode" :aria-label="t('mongo.import.typeMode')" @update:model-value="setTypeMode">
<SelectTrigger class="h-8 text-xs"><SelectValue /></SelectTrigger>
@@ -405,13 +417,14 @@ function requestClose() {
</div>
</div>
<div class="grid grid-cols-2 gap-x-4 gap-y-2 rounded-md border p-3">
<div v-if="format !== 'bson'" class="grid grid-cols-2 gap-x-4 gap-y-2 rounded-md border p-3">
<label class="flex items-center gap-2 text-xs"><input v-model="hasHeader" type="checkbox" class="h-3.5 w-3.5 accent-primary" :aria-label="t('tableImport.hasHeader')" /> {{ t("tableImport.hasHeader") }}</label>
<label class="flex items-center gap-2 text-xs"><input v-model="trim" type="checkbox" class="h-3.5 w-3.5 accent-primary" /> {{ t("tableImport.trimValues") }}</label>
<label class="flex items-center gap-2 text-xs"><input v-model="emptyAsNull" type="checkbox" class="h-3.5 w-3.5 accent-primary" /> {{ t("mongo.import.emptyAsNull") }}</label>
<label class="flex items-center gap-2 text-xs"><input v-model="recognizeObjectIdHex" type="checkbox" class="h-3.5 w-3.5 accent-primary" /> {{ t("mongo.import.recognizeObjectIdHex") }}</label>
<label class="flex items-center gap-2 text-xs"><input v-model="skipErrorRows" type="checkbox" class="h-3.5 w-3.5 accent-primary" /> {{ t("mongo.import.skipErrorRows") }}</label>
</div>
<label v-else class="flex items-center gap-2 text-xs"><input v-model="skipErrorRows" type="checkbox" class="h-3.5 w-3.5 accent-primary" /> {{ t("mongo.import.skipErrorRows") }}</label>
<div v-if="loadingPreview && !preview" class="flex items-center gap-2 text-xs text-muted-foreground">
<Loader2 class="h-3.5 w-3.5 animate-spin" />
@@ -438,7 +451,7 @@ function requestClose() {
check-position="right"
@update:model-value="(value) => setColumnType(column.name, value)"
/>
<span v-else-if="format !== 'csv'" class="text-muted-foreground">({{ column.inferredType }})</span>
<span v-else-if="format !== 'csv' && format !== 'bson'" class="text-muted-foreground">({{ column.inferredType }})</span>
</th>
</tr>
</thead>
@@ -0,0 +1,233 @@
// @vitest-environment happy-dom
import { createApp, nextTick, type App } from "vue";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({
api: { inspectMongodbDatabaseDump: vi.fn(), prepareMongodbRestoreSource: vi.fn(), releaseMongodbRestoreSource: vi.fn(), dumpMongodbDatabase: vi.fn(), restoreMongodbDatabase: vi.fn(), cancelMongodbDatabaseDump: vi.fn() },
store: { getConfig: vi.fn(), ensureConnected: vi.fn(), loadMongoDatabases: vi.fn(), loadMongoCollections: vi.fn(), mongoImportCompleted: null },
guard: vi.fn(),
}));
vi.mock("@/lib/backend/api", () => mocks.api);
vi.mock("@/lib/backend/tauriRuntime", () => ({ isTauriRuntime: () => false }));
vi.mock("@/stores/connectionStore", () => ({ useConnectionStore: () => mocks.store }));
vi.mock("vue-i18n", async (original) => ({ ...(await original<typeof import("vue-i18n")>()), useI18n: () => ({ t: (key: string) => key }) }));
vi.mock("@/lib/database/productionExecutionGuard", () => ({ executeWithProductionContextGuard: mocks.guard }));
import MongoDatabaseDumpDialog from "../MongoDatabaseDumpDialog.vue";
const catalog = {
databases: ["source"],
collections: [
{ database: "source", name: "records", kind: "collection", documents: 2, sizeBytes: 30, indexes: 2 },
{ database: "source", name: "empty", kind: "collection", documents: 0, sizeBytes: 0, indexes: 0 },
],
};
const done = { taskId: "task", status: "done", phase: "done", collection: "records", collectionsDone: 2, collectionsTotal: 2, documentsRead: 2, documentsWritten: 2, documentsFailed: 0, indexesCreated: 2, elapsedMs: 10, errorMessage: null, filePath: null };
let app: App | null;
async function flush() {
for (let i = 0; i < 5; i++) {
await Promise.resolve();
await nextTick();
}
}
function button(key: string): HTMLButtonElement {
const button = Array.from(document.querySelectorAll("button")).find((button) => button.textContent?.trim() === key);
if (!button) throw new Error(`Missing button ${key}`);
return button;
}
function checkbox(key: string): HTMLInputElement {
const label = Array.from(document.querySelectorAll("label")).find((label) => label.textContent?.trim() === key);
const checkbox = label?.querySelector<HTMLInputElement>("input[type=checkbox]");
if (!checkbox) throw new Error(`Missing checkbox ${key}`);
return checkbox;
}
async function mount(mode: "dump" | "restore") {
app = createApp(MongoDatabaseDumpDialog, { open: true, connectionId: "c1", database: "target", mode });
app.mount(document.body);
await flush();
}
async function filesSelected(files: File[]) {
const input = document.querySelector<HTMLInputElement>("input[type=file]")!;
Object.defineProperty(input, "files", { configurable: true, value: files });
input.dispatchEvent(new Event("change"));
await flush();
}
async function formatSelected(value: string) {
document.querySelector<HTMLElement>("[role=combobox]")!.dispatchEvent(new KeyboardEvent("keydown", { key: "ArrowDown", bubbles: true, cancelable: true }));
await vi.advanceTimersByTimeAsync(10);
await flush();
const option = Array.from(document.querySelectorAll<HTMLElement>("[role=option]")).find((option) => option.textContent?.trim() === value)!;
option.focus();
option.dispatchEvent(new KeyboardEvent("keydown", { key: "Enter", bubbles: true, cancelable: true }));
await vi.advanceTimersByTimeAsync(10);
await flush();
}
describe("MongoDatabaseDumpDialog", () => {
beforeEach(() => {
vi.useFakeTimers();
vi.clearAllMocks();
mocks.store.getConfig.mockReturnValue({ name: "MongoDB", read_only: false });
mocks.store.ensureConnected.mockResolvedValue(undefined);
mocks.api.inspectMongodbDatabaseDump.mockResolvedValue(catalog);
mocks.api.prepareMongodbRestoreSource.mockResolvedValue({ ...catalog, sourceRef: "prepared-one" });
mocks.api.releaseMongodbRestoreSource.mockResolvedValue(true);
mocks.api.dumpMongodbDatabase.mockResolvedValue(done);
mocks.api.restoreMongodbDatabase.mockResolvedValue(done);
mocks.guard.mockImplementation(async ({ execute }) => execute());
});
afterEach(() => {
app?.unmount();
app = null;
document.body.innerHTML = "";
vi.useRealTimers();
});
it("exports selected collections using the explicit archive and gzip options", async () => {
await mount("dump");
checkbox("mongoDump.gzip").click();
document.querySelector<HTMLInputElement>('input[aria-label="empty"]')!.click();
await flush();
button("mongoDump.review").click();
await flush();
button("mongoDump.dump").click();
await flush();
expect(mocks.api.dumpMongodbDatabase).toHaveBeenCalledWith(expect.objectContaining({ connectionId: "c1", database: "target", format: "archive", gzip: true, collections: ["records"], filePath: "target.archive.gz" }), expect.any(Function));
expect(mocks.guard).not.toHaveBeenCalled();
});
it("restores a directory into the chosen database only after drop confirmation", async () => {
await mount("restore");
await formatSelected("mongoDump.directory");
checkbox("mongoDump.gzip").click();
await filesSelected([new File([], "records.bson.gz")]);
button("mongoDump.readSource").click();
await flush();
expect(mocks.api.prepareMongodbRestoreSource).toHaveBeenCalledWith(expect.any(Array), "directory", true, expect.objectContaining({ signal: expect.any(AbortSignal) }));
checkbox("mongoDump.dropExisting").click();
await flush();
button("mongoDump.review").click();
await flush();
expect(document.body.textContent).toContain("mongoDump.confirmDrop");
expect(mocks.api.restoreMongodbDatabase).not.toHaveBeenCalled();
button("mongoDump.restore").click();
await flush();
expect(mocks.api.restoreMongodbDatabase).toHaveBeenCalledWith(
expect.objectContaining({ database: "target", sourceDatabase: "source", sourceRef: "prepared-one", dropExisting: true, restoreOptions: true, restoreIndexes: true, objcheck: false, batchSize: 500 }),
expect.any(Function),
expect.objectContaining({ gzip: true }),
);
expect(mocks.guard).toHaveBeenCalledWith(expect.objectContaining({ database: "target", reviewText: expect.stringContaining("DROP selected collections") }));
});
it("passes optional objcheck without a separate prevalidation request", async () => {
await mount("restore");
expect(checkbox("mongoDump.objcheck").checked).toBe(false);
await filesSelected([new File([], "backup.archive")]);
button("mongoDump.readSource").click();
await flush();
checkbox("mongoDump.objcheck").click();
button("mongoDump.review").click();
await flush();
button("mongoDump.restore").click();
await flush();
expect(mocks.api.restoreMongodbDatabase).toHaveBeenCalledWith(expect.objectContaining({ objcheck: true }), expect.any(Function), undefined);
expect(mocks.api.prepareMongodbRestoreSource).toHaveBeenCalledTimes(1);
});
it("starts only one restore when confirmation is pending and the button is clicked twice", async () => {
let allow!: (value: boolean) => void;
mocks.guard.mockReturnValue(
new Promise<boolean>((resolve) => {
allow = resolve;
}),
);
await mount("restore");
await filesSelected([new File([], "backup.archive")]);
button("mongoDump.readSource").click();
await flush();
button("mongoDump.review").click();
await flush();
const restore = button("mongoDump.restore");
restore.click();
restore.click();
await flush();
expect(mocks.guard).toHaveBeenCalledTimes(1);
expect(button("mongoDump.back").disabled).toBe(true);
expect(button("mongoDump.close").disabled).toBe(true);
expect(mocks.api.restoreMongodbDatabase).not.toHaveBeenCalled();
allow(true);
await flush();
expect(mocks.api.restoreMongodbDatabase).toHaveBeenCalledTimes(1);
});
it("does not restore if the dialog is unmounted while confirmation is pending", async () => {
let allow!: (value: boolean) => void;
mocks.guard.mockReturnValue(
new Promise<boolean>((resolve) => {
allow = resolve;
}),
);
await mount("restore");
await filesSelected([new File([], "backup.archive")]);
button("mongoDump.readSource").click();
await flush();
button("mongoDump.review").click();
await flush();
button("mongoDump.restore").click();
await flush();
app!.unmount();
app = null;
allow(true);
await flush();
expect(mocks.api.restoreMongodbDatabase).not.toHaveBeenCalled();
expect(mocks.store.ensureConnected).not.toHaveBeenCalled();
});
it("unlocks the dialog when production confirmation is declined", async () => {
mocks.guard.mockResolvedValueOnce(false);
await mount("restore");
await filesSelected([new File([], "backup.archive")]);
button("mongoDump.readSource").click();
await flush();
button("mongoDump.review").click();
await flush();
button("mongoDump.restore").click();
await flush();
expect(mocks.api.restoreMongodbDatabase).not.toHaveBeenCalled();
expect(button("mongoDump.restore").disabled).toBe(false);
button("mongoDump.restore").click();
await flush();
expect(mocks.api.restoreMongodbDatabase).toHaveBeenCalledTimes(1);
});
it("invalidates a prepared source when gzip changes without inferring the format from its name", async () => {
await mount("restore");
await filesSelected([new File([], "misleading.bson.gz")]);
button("mongoDump.readSource").click();
await flush();
expect(mocks.api.prepareMongodbRestoreSource).toHaveBeenCalledWith(expect.any(File), "archive", false, expect.any(Object));
checkbox("mongoDump.gzip").click();
await flush();
expect(mocks.api.releaseMongodbRestoreSource).toHaveBeenCalledWith("prepared-one");
expect(button("mongoDump.review").disabled).toBe(true);
expect(mocks.api.prepareMongodbRestoreSource).toHaveBeenCalledTimes(1);
});
it("releases a late preview after the dialog is unmounted", async () => {
let resolve!: (value: unknown) => void;
mocks.api.prepareMongodbRestoreSource.mockReturnValueOnce(
new Promise((done) => {
resolve = done;
}),
);
await mount("restore");
await filesSelected([new File([], "backup.archive")]);
button("mongoDump.readSource").click();
await flush();
app!.unmount();
app = null;
resolve({ ...catalog, sourceRef: "late-source" });
await flush();
expect(mocks.api.releaseMongodbRestoreSource).toHaveBeenCalledWith("late-source");
});
});
@@ -143,6 +143,53 @@ describe("MongoImportDialog", () => {
app.unmount();
});
it("does not override the selected format from the file extension", async () => {
const app = createApp(
defineComponent({
setup() {
return () => h(MongoImportDialog, { open: true, connectionId: "c1", database: "shop", collection: "orders" });
},
}),
);
app.mount(document.body);
await nextTick();
const input = document.querySelector("input[type='file']") as HTMLInputElement;
const file = new File([new Uint8Array([5, 0, 0, 0, 0])], "orders.bson.gz", { type: "application/gzip" });
Object.defineProperty(input, "files", { value: [file] });
input.dispatchEvent(new Event("change"));
await vi.advanceTimersByTimeAsync(200);
await Promise.resolve();
await nextTick();
expect(api.previewMongodbImportFile.mock.calls[0]?.[1]).toMatchObject({
format: "csv",
parseOptions: { delimiter: ",", typeMode: "auto" },
});
expect(document.body.textContent).toContain("tableImport.encoding");
expect(document.body.textContent).toContain("mongo.import.typeMode");
app.unmount();
});
it("keeps an explicitly selected BSON format when changing files", async () => {
api.previewMongodbImportFile.mockResolvedValue(previewResult({ format: "bson" }));
const app = createApp(MongoImportDialog, { open: true, connectionId: "c1", database: "shop", collection: "orders" });
app.mount(document.body);
await nextTick();
await setLabeledSelect("tableImport.sourceFormat", "bson");
expect(document.body.textContent).not.toContain("tableImport.encoding");
for (const name of ["orders.bson", "renamed.json"]) {
const input = document.querySelector("input[type='file']") as HTMLInputElement;
Object.defineProperty(input, "files", { configurable: true, value: [new File([new Uint8Array([5, 0, 0, 0, 0])], name)] });
input.dispatchEvent(new Event("change"));
await vi.advanceTimersByTimeAsync(200);
await nextTick();
expect(api.previewMongodbImportFile.mock.lastCall?.[1]).toMatchObject({ format: "bson" });
}
app.unmount();
});
it("reuses the uploaded sourceRef on later previews and releases it on close", async () => {
const open = ref(true);
const app = createApp(
@@ -180,6 +227,9 @@ describe("MongoImportDialog", () => {
const header = document.querySelector("input[type='checkbox']") as HTMLInputElement;
header.click();
await nextTick();
const next = Array.from(document.querySelectorAll("button")).find((button) => button.textContent?.includes("tableImport.next"));
expect(next?.disabled).toBe(true);
await vi.advanceTimersByTimeAsync(200);
await Promise.resolve();
await nextTick();
@@ -194,7 +244,7 @@ describe("MongoImportDialog", () => {
app.unmount();
});
async function mountWithFile(fileName: string, preview: MongoImportPreview) {
async function mountWithFile(fileName: string, preview: MongoImportPreview, format: MongoImportPreview["format"] = "csv") {
api.previewMongodbImportFile.mockResolvedValue(preview);
const app = createApp(
defineComponent({
@@ -211,6 +261,7 @@ describe("MongoImportDialog", () => {
);
app.mount(document.body);
await nextTick();
if (format !== "csv") await setLabeledSelect("tableImport.sourceFormat", format);
const input = document.querySelector("input[type='file']") as HTMLInputElement;
const file = new File(["id\n1"], fileName, { type: "text/csv" });
Object.defineProperty(input, "files", { configurable: true, value: [file] });
@@ -317,6 +368,7 @@ describe("MongoImportDialog", () => {
fileName: "orders.json",
columns: [{ name: "id", inferredType: "string" }],
}),
"json",
);
expect(columnTypeTrigger()).toBeNull();
expect(document.body.textContent).toContain("(string)");
@@ -14,6 +14,7 @@ const SchemaDiagramDialog = defineAsyncComponent(() => import("@/components/diag
const DatabaseDocsDialog = defineAsyncComponent(() => import("@/components/docs/DatabaseDocsDialog.vue"));
const TableImportDialog = defineAsyncComponent(() => import("@/components/import/TableImportDialog.vue"));
const MongoImportDialog = defineAsyncComponent(() => import("@/components/document/MongoImportDialog.vue"));
const MongoDatabaseDumpDialog = defineAsyncComponent(() => import("@/components/document/MongoDatabaseDumpDialog.vue"));
const FieldLineageDialog = defineAsyncComponent(() => import("@/components/lineage/FieldLineageDialog.vue"));
const ConfigPassphraseDialog = defineAsyncComponent(() => import("@/components/config/ConfigPassphraseDialog.vue"));
const ConfigConnectionSelectDialog = defineAsyncComponent(() => import("@/components/config/ConfigConnectionSelectDialog.vue"));
@@ -301,6 +302,13 @@ watch(
:prefill-table="dialogs.tableImportPrefillTable.value"
/>
<MongoImportDialog v-model:open="dialogs.showMongoImportDialog.value" :connection-id="dialogs.mongoImportPrefillConnectionId.value" :database="dialogs.mongoImportPrefillDatabase.value" :collection="dialogs.mongoImportPrefillCollection.value" />
<MongoDatabaseDumpDialog
v-if="dialogs.showMongoDatabaseDumpDialog.value"
v-model:open="dialogs.showMongoDatabaseDumpDialog.value"
:connection-id="dialogs.mongoDatabaseDumpPrefillConnectionId.value"
:database="dialogs.mongoDatabaseDumpPrefillDatabase.value"
:mode="dialogs.mongoDatabaseDumpMode.value"
/>
<DataGenerateDialog
v-if="dialogs.showTableDataGenerateDialog.value"
v-model:open="dialogs.showTableDataGenerateDialog.value"
@@ -410,15 +410,31 @@ const { copyStructureAs, copyStructureDocText, copyStructurePreview, exportData,
acceptedSelectionIds: () => acceptedSelectionIds,
});
const { openAllDatabasesExport, openDataCompare, openDatabaseExport, openDatabaseSearch, openDiagram, openDocs, openFieldLineage, openMongoImport, openScheduledBackups, openSchemaDiff, openSchemaDiffForRoutine, openSqlFileExecution, openStructureEditor, openTableImport, openTransfer } =
useSidebarTreeToolRuntime({
activeNode,
connectionStore,
queryStore,
settingsStore,
tableChildObjectName: tableChildDropObjectName,
acceptedSelectionIds: () => acceptedSelectionIds,
});
const {
openAllDatabasesExport,
openDataCompare,
openDatabaseExport,
openDatabaseSearch,
openDiagram,
openDocs,
openFieldLineage,
openMongoImport,
openMongoDatabaseDump,
openScheduledBackups,
openSchemaDiff,
openSchemaDiffForRoutine,
openSqlFileExecution,
openStructureEditor,
openTableImport,
openTransfer,
} = useSidebarTreeToolRuntime({
activeNode,
connectionStore,
queryStore,
settingsStore,
tableChildObjectName: tableChildDropObjectName,
acceptedSelectionIds: () => acceptedSelectionIds,
});
const emit = defineEmits<{
"rename-started": [];
@@ -5870,6 +5886,8 @@ function buildSpecialSidebarMenu(context: SidebarMenuFactoryContext): boolean {
if (node.type === "mongo-db") {
items.push({ label: "", separator: true });
items.push({ label: t("transfer.dataTransfer"), action: openTransfer, icon: ArrowRightLeft });
items.push({ label: t("mongoDump.menuDump"), action: () => openMongoDatabaseDump("dump"), icon: Upload });
items.push({ label: t("mongoDump.menuRestore"), action: () => openMongoDatabaseDump("restore"), icon: Download });
}
if (node.type === "redis-db") {
items.push({ label: "", separator: true });
@@ -5950,6 +5968,8 @@ function buildSpecialSidebarMenu(context: SidebarMenuFactoryContext): boolean {
children: [
{ label: "CSV", action: () => void exportMongoCollection("csv") },
{ label: "NDJSON", action: () => void exportMongoCollection("ndjson") },
{ label: "BSON dump", action: () => void exportMongoCollection("bson") },
{ label: "BSON dump (gzip)", action: () => void exportMongoCollection("bsonGzip") },
],
});
if (canDropMongoCollection.value) {
@@ -158,6 +158,39 @@ describe("useSidebarTreeExportRuntime", () => {
expect(toastMock).toHaveBeenCalledWith("grid.exported");
});
it.each(["bson", "bsonGzip"] as const)("exports an official-compatible %s collection dump", async (mode) => {
apiMock.exportMongodbQuery.mockImplementation(async (_request, onProgress) => {
onProgress({ exportId: "export-1", status: "done", documentsRead: 2, bytesWritten: 64, elapsedMs: 4 });
return { exportId: "export-1", documentsExported: 2, filePath: "orders.bson.gz", elapsedMs: 4 };
});
const activeNode = shallowRef({ id: "col-1", type: "mongo-collection", label: "orders", connectionId: "conn-1", database: "shop", children: [] } as TreeNode);
const connectionStore = {
ensureConnected: vi.fn(),
getConfig: vi.fn(() => ({ db_type: "mongodb" })),
treeNodes: [],
selectedTreeNodeIds: [],
};
const runtime = useSidebarTreeExportRuntime({
activeNode,
connectionStore: connectionStore as never,
settingsStore: exportSettings() as never,
acceptedSelectionIds: () => null,
});
await runtime.exportMongoCollection(mode);
expect(apiMock.exportMongodbQuery).toHaveBeenCalledWith(
expect.objectContaining({
database: "shop",
collection: "orders",
format: "bson",
gzip: mode === "bsonGzip",
filePath: mode === "bsonGzip" ? "orders.bson.gz" : "orders.bson",
}),
expect.any(Function),
);
});
it("loads and joins every selected DDL in tree order", async () => {
apiMock.getTableDdl.mockResolvedValueOnce("CREATE TABLE one (id INT)").mockResolvedValueOnce("CREATE VIEW two AS SELECT 1;");
const first = { id: "table-1", type: "table", label: "one", connectionId: "conn-1", database: "db", schema: "main" } as TreeNode;
@@ -11,6 +11,7 @@ function setup(node: Partial<TreeNode>, options: { treeNodes?: TreeNode[]; selec
diagramSource: null as unknown,
databaseExportSource: null as unknown,
mongoImportSource: undefined as unknown,
mongoDatabaseDumpSource: null as unknown,
schemaDiffSource: null as unknown,
treeNodes: options.treeNodes ?? [],
selectedTreeNodeIds: options.selectedTreeNodeIds ?? [],
@@ -129,6 +130,29 @@ describe("useSidebarTreeToolRuntime mongo import", () => {
});
});
describe("useSidebarTreeToolRuntime MongoDB dump and restore", () => {
it.each(["dump", "restore"] as const)("opens %s for the selected MongoDB database", (mode) => {
const { connectionStore, runtime } = setup({ type: "mongo-db", connectionId: "conn-1", database: "shop" });
runtime.openMongoDatabaseDump(mode);
expect(connectionStore.mongoDatabaseDumpSource).toEqual({ connectionId: "conn-1", database: "shop", mode });
});
it.each<Partial<TreeNode>>([
{ type: "table", connectionId: "conn-1", database: "shop" },
{ type: "mongo-collection", connectionId: "conn-1", database: "shop" },
{ type: "mongo-db", database: "shop" },
{ type: "mongo-db", connectionId: "conn-1" },
])("does not open a database restore from an invalid context: %j", (node) => {
const { connectionStore, runtime } = setup(node);
runtime.openMongoDatabaseDump("restore");
expect(connectionStore.mongoDatabaseDumpSource).toBeNull();
});
});
describe("useSidebarTreeToolRuntime openSchemaDiffForRoutine", () => {
it("prefills schema diff with a signature-aware routine key", () => {
const { connectionStore, runtime } = setup({
@@ -15,6 +15,10 @@ const showDiagramDialog = ref(false);
const showDocsDialog = ref(false);
const showTableImportDialog = ref(false);
const showMongoImportDialog = ref(false);
const showMongoDatabaseDumpDialog = ref(false);
const mongoDatabaseDumpPrefillConnectionId = ref("");
const mongoDatabaseDumpPrefillDatabase = ref("");
const mongoDatabaseDumpMode = ref<"dump" | "restore">("dump");
const showTableDataGenerateDialog = ref(false);
const showFieldLineageDialog = ref(false);
const showDatabaseSearchDialog = ref(false);
@@ -263,6 +267,18 @@ export function useDialogSources() {
},
);
watch(
() => connectionStore.mongoDatabaseDumpSource,
(source) => {
if (!source) return;
mongoDatabaseDumpPrefillConnectionId.value = source.connectionId;
mongoDatabaseDumpPrefillDatabase.value = source.database;
mongoDatabaseDumpMode.value = source.mode;
showMongoDatabaseDumpDialog.value = true;
connectionStore.mongoDatabaseDumpSource = null;
},
);
watch(
() => connectionStore.tableDataGenerateSource,
(v) => {
@@ -519,6 +535,10 @@ export function useDialogSources() {
showDocsDialog,
showTableImportDialog,
showMongoImportDialog,
showMongoDatabaseDumpDialog,
mongoDatabaseDumpPrefillConnectionId,
mongoDatabaseDumpPrefillDatabase,
mongoDatabaseDumpMode,
showTableDataGenerateDialog,
showFieldLineageDialog,
showDatabaseSearchDialog,
@@ -274,6 +274,7 @@ export function useSidebarTreeExportRuntime(options: SidebarTreeExportRuntimeOpt
if (format === "csv") return "CSV";
if (format === "json") return "JSON";
if (format === "ndjson") return "NDJSON";
if (format === "bson" || format === "bson.gz") return "MongoDB BSON dump";
if (format === "xlsx") return "Excel";
return "SQL";
}
@@ -287,7 +288,7 @@ export function useSidebarTreeExportRuntime(options: SidebarTreeExportRuntimeOpt
const { save } = await import("@tauri-apps/plugin-dialog");
const path = await save({
defaultPath: fileName,
filters: [{ name: exportFilterName(format), extensions: [format] }],
filters: [{ name: exportFilterName(format), extensions: [format === "bson.gz" ? "gz" : format] }],
});
return path ? String(path) : null;
}
@@ -472,10 +473,12 @@ export function useSidebarTreeExportRuntime(options: SidebarTreeExportRuntimeOpt
return pickTableExportDirectory();
}
async function exportMongoCollection(format: "csv" | "ndjson") {
async function exportMongoCollection(outputFormat: "csv" | "ndjson" | "bson" | "bsonGzip") {
const node = activeNode.value;
if (node.type !== "mongo-collection" || !node.connectionId || !node.database) return;
const outputPath = await resolveExportOutputPath(node.label, format);
const format: api.MongoExportFormat = outputFormat === "bsonGzip" ? "bson" : outputFormat;
const fileExtension = outputFormat === "bsonGzip" ? "bson.gz" : outputFormat;
const outputPath = await resolveExportOutputPath(node.label, fileExtension);
if (!outputPath) return;
let task: ExportTask | null = null;
try {
@@ -490,6 +493,7 @@ export function useSidebarTreeExportRuntime(options: SidebarTreeExportRuntimeOpt
collection: node.label,
format,
includeHeader: true,
gzip: outputFormat === "bsonGzip",
filePath: outputPath,
},
(progress) => {
@@ -149,6 +149,12 @@ export function useSidebarTreeToolRuntime(options: SidebarTreeToolRuntimeOptions
};
}
function openMongoDatabaseDump(mode: "dump" | "restore") {
const node = activeNode.value;
if (node.type !== "mongo-db" || !node.connectionId || !node.database) return;
connectionStore.mongoDatabaseDumpSource = { connectionId: node.connectionId, database: node.database, mode };
}
function openStructureEditor() {
const node = activeNode.value;
if (!node.connectionId || !node.database) return;
@@ -191,6 +197,7 @@ export function useSidebarTreeToolRuntime(options: SidebarTreeToolRuntimeOptions
openDocs,
openFieldLineage,
openMongoImport,
openMongoDatabaseDump,
openScheduledBackups,
openSchemaDiff,
openSchemaDiffForRoutine,
@@ -0,0 +1,26 @@
import { describe, expect, it } from "vitest";
import * as messages from "../locales/mongodbDatabaseDump";
function leaves(value: Record<string, unknown>, prefix = ""): Record<string, string> {
return Object.fromEntries(
Object.entries(value).flatMap(([key, child]) => {
const path = prefix ? `${prefix}.${key}` : key;
if (typeof child === "string") return [[path, child]];
return Object.entries(leaves(child as Record<string, unknown>, path));
}),
);
}
describe("MongoDB dump translations", () => {
const source = leaves(messages.mongodbDatabaseDumpEn);
for (const [locale, value] of Object.entries(messages)) {
it(`${locale} includes every key and preserves placeholders`, () => {
const translated = leaves(value);
expect(Object.keys(translated).sort()).toEqual(Object.keys(source).sort());
for (const [key, text] of Object.entries(translated)) {
expect(text.trim(), key).not.toBe("");
expect((text.match(/\{\w+\}/g) ?? []).sort(), key).toEqual((source[key]!.match(/\{\w+\}/g) ?? []).sort());
}
});
}
});
+2
View File
@@ -3,6 +3,7 @@ import docs from "./docs/en";
import { consulUiMessages } from "./consulUi";
import { sqlServerTraceMessages as sqlServerTrace } from "./sqlServerTraceMessages";
import { meilisearchManagementEn } from "./meilisearchManagement";
import { mongodbDatabaseDumpEn as mongoDump } from "./mongodbDatabaseDump";
const consul = {
...consulUiMessages,
@@ -5482,6 +5483,7 @@ export default {
},
},
},
mongoDump,
mongo: {
documents: "{count} documents",
addField: "Add Field",
+2
View File
@@ -1,4 +1,5 @@
import { withEnglishFallback } from "./fallback";
import { mongodbDatabaseDumpEs as mongoDump } from "./mongodbDatabaseDump";
import { meilisearchManagementEs } from "./meilisearchManagement";
import { redisGroupingEs as redisGrouping } from "./redisGrouping";
import docs from "./docs/es";
@@ -83,6 +84,7 @@ const consul = {
};
export default withEnglishFallback({
mongoDump,
cellTransform: {
title: "Vista previa de conversión",
description: "Convierte el valor actual para verlo o copiarlo. El valor original y el borrador no cambian.",
+2
View File
@@ -1,4 +1,5 @@
import { withEnglishFallback } from "./fallback";
import { mongodbDatabaseDumpIt as mongoDump } from "./mongodbDatabaseDump";
import { meilisearchManagementIt } from "./meilisearchManagement";
import { redisGroupingIt as redisGrouping } from "./redisGrouping";
import docs from "./docs/it";
@@ -82,6 +83,7 @@ const consul = {
};
export default withEnglishFallback({
mongoDump,
cellTransform: {
title: "Anteprima conversione",
description: "Converte il valore corrente per visualizzarlo o copiarlo. Il valore originale e la bozza restano invariati.",
+2
View File
@@ -1,4 +1,5 @@
import { withEnglishFallback } from "./fallback";
import { mongodbDatabaseDumpJa as mongoDump } from "./mongodbDatabaseDump";
import docs from "./docs/ja";
import { consulJaMessages } from "./consulJa";
import { nacosAccessControlMessages, nacosAccessControlTabs } from "./nacosAccessControl";
@@ -83,6 +84,7 @@ const consul = {
};
export default withEnglishFallback({
mongoDump,
cellTransform: {
title: "変換プレビュー",
description: "現在の値を変換して表示またはコピーします。元の値と編集中の内容は変更されません。",
+2
View File
@@ -1,4 +1,5 @@
import { withEnglishFallback } from "./fallback";
import { mongodbDatabaseDumpKo as mongoDump } from "./mongodbDatabaseDump";
import { meilisearchManagementKo } from "./meilisearchManagement";
import { redisGroupingKo as redisGrouping } from "./redisGrouping";
import docs from "./docs/ko";
@@ -82,6 +83,7 @@ const consul = {
};
export default withEnglishFallback({
mongoDump,
cellTransform: {
title: "변환 미리보기",
description: "현재 값을 변환하여 보거나 복사합니다. 원래 값과 편집 초안은 변경되지 않습니다.",
@@ -0,0 +1,439 @@
export const mongodbDatabaseDumpEn = {
selectAll: "Select all collections",
menuDump: "Dump database...",
menuRestore: "Restore database...",
dumpTitle: "MongoDB Database Dump",
restoreTitle: "MongoDB Database Restore",
format: "Format",
archive: "MongoDB archive",
directory: "MongoDB dump directory",
gzip: "Gzip",
source: "Backup source",
readSource: "Read backup",
sourceDatabase: "Source database",
targetDatabase: "Target database",
destination: "Destination",
chooseDestination: "Choose destination",
collection: "Collection",
kind: "Type",
documents: "Documents",
indexes: "Indexes",
dropExisting: "Drop selected target collections before restoring",
restoreOptions: "Restore collection options",
restoreIndexes: "Restore indexes",
stopOnError: "Stop on write errors",
objcheck: "Check BSON documents before insertion (objcheck)",
readBytes: "Read: {size} GiB decoded",
batchSize: "Batch size",
selected: "{count} collections selected",
empty: "No collections",
preparing: "Reading backup...",
loading: "Loading collections...",
dump: "Start dump",
restore: "Start restore",
review: "Review",
back: "Back",
close: "Close",
cancel: "Cancel task",
confirmDump: "Dump {count} collections from {database}.",
confirmRestore: "Restore {count} collections into {database}.",
confirmDrop: "The selected target collections will be deleted and recreated.",
confirmAppend: "Documents will be appended to existing collections.",
readonly: "This connection is read-only.",
done: "Completed",
cancelled: "Cancelled",
failed: "Failed",
progress: "Collections: {done} / {total}",
read: "Documents read",
written: "Documents restored",
rejected: "Failed documents",
createdIndexes: "Indexes restored",
uploadBytes: "Uploaded: {done} / {total} GiB",
validationBytes: "Validated: {size} GiB decoded, {count} documents",
phase: { preparing: "Preparing", uploading: "Uploading selected files", validating: "Validating backup", data: "Collection data", indexes: "Restoring indexes", views: "Restoring views", archive: "Writing backup", done: "Finished" },
};
export const mongodbDatabaseDumpZhCN = {
selectAll: "选择全部集合",
menuDump: "备份数据库...",
menuRestore: "恢复数据库...",
dumpTitle: "MongoDB 数据库备份",
restoreTitle: "MongoDB 数据库恢复",
format: "格式",
archive: "MongoDB archive",
directory: "MongoDB dump 目录",
gzip: "Gzip 压缩",
source: "备份源",
readSource: "读取备份",
sourceDatabase: "源数据库",
targetDatabase: "目标数据库",
destination: "保存位置",
chooseDestination: "选择保存位置",
collection: "集合",
kind: "类型",
documents: "文档数",
indexes: "索引数",
dropExisting: "恢复前删除选中的目标同名集合",
restoreOptions: "恢复集合选项",
restoreIndexes: "恢复索引",
stopOnError: "遇到写入错误时停止",
objcheck: "写入前检查 BSON 文档(objcheck)",
readBytes: "已读取:解压后 {size} GiB",
batchSize: "批大小",
selected: "已选择 {count} 个集合",
empty: "没有集合",
preparing: "正在读取备份...",
loading: "正在读取集合...",
dump: "开始备份",
restore: "开始恢复",
review: "确认操作",
back: "返回",
close: "关闭",
cancel: "取消任务",
confirmDump: "备份数据库 {database} 中的 {count} 个集合。",
confirmRestore: "将 {count} 个集合恢复到数据库 {database}。",
confirmDrop: "选中的目标同名集合将被删除并重建。",
confirmAppend: "文档将追加到现有集合。",
readonly: "该连接处于只读模式。",
done: "已完成",
cancelled: "已取消",
failed: "失败",
progress: "集合:{done} / {total}",
read: "已读取文档",
written: "已恢复文档",
rejected: "失败文档",
createdIndexes: "已恢复索引",
uploadBytes: "已上传:{done} / {total} GiB",
validationBytes: "已校验:解压后 {size} GiB,{count} 个文档",
phase: { preparing: "准备中", uploading: "上传选中的文件", validating: "校验备份", data: "集合数据", indexes: "恢复索引", views: "恢复视图", archive: "写入备份", done: "已结束" },
};
export const mongodbDatabaseDumpEs = {
selectAll: "Seleccionar todas las colecciones",
menuDump: "Crear copia de la base de datos...",
menuRestore: "Restaurar base de datos...",
dumpTitle: "Copia de seguridad de MongoDB",
restoreTitle: "Restauración de MongoDB",
format: "Formato",
archive: "MongoDB archive",
directory: "Directorio de copia de MongoDB",
gzip: "Gzip",
source: "Origen de la copia",
readSource: "Leer copia",
sourceDatabase: "Base de datos de origen",
targetDatabase: "Base de datos de destino",
destination: "Destino",
chooseDestination: "Elegir destino",
collection: "Colección",
kind: "Tipo",
documents: "Documentos",
indexes: "Índices",
dropExisting: "Eliminar las colecciones de destino seleccionadas antes de restaurar",
restoreOptions: "Restaurar opciones de colección",
restoreIndexes: "Restaurar índices",
stopOnError: "Detener al producirse errores de escritura",
objcheck: "Comprobar documentos BSON antes de insertarlos (objcheck)",
readBytes: "Leídos: {size} GiB descomprimidos",
batchSize: "Tamaño del lote",
selected: "{count} colecciones seleccionadas",
empty: "No hay colecciones",
preparing: "Leyendo copia...",
loading: "Cargando colecciones...",
dump: "Iniciar copia",
restore: "Iniciar restauración",
review: "Revisar",
back: "Volver",
close: "Cerrar",
cancel: "Cancelar tarea",
confirmDump: "Crear una copia de {count} colecciones de {database}.",
confirmRestore: "Restaurar {count} colecciones en {database}.",
confirmDrop: "Las colecciones de destino seleccionadas se eliminarán y se volverán a crear.",
confirmAppend: "Los documentos se añadirán a las colecciones existentes.",
readonly: "Esta conexión es de solo lectura.",
done: "Completado",
cancelled: "Cancelado",
failed: "Error",
progress: "Colecciones: {done} / {total}",
read: "Documentos leídos",
written: "Documentos restaurados",
rejected: "Documentos fallidos",
createdIndexes: "Índices restaurados",
uploadBytes: "Subidos: {done} / {total} GiB",
validationBytes: "Validados: {size} GiB descomprimidos, {count} documentos",
phase: { preparing: "Preparando", uploading: "Subiendo archivos seleccionados", validating: "Validando copia", data: "Datos de las colecciones", indexes: "Restaurando índices", views: "Restaurando vistas", archive: "Escribiendo copia", done: "Finalizado" },
};
export const mongodbDatabaseDumpIt = {
selectAll: "Seleziona tutte le raccolte",
menuDump: "Backup del database...",
menuRestore: "Ripristina database...",
dumpTitle: "Backup del database MongoDB",
restoreTitle: "Ripristino del database MongoDB",
format: "Formato",
archive: "MongoDB archive",
directory: "Directory di backup MongoDB",
gzip: "Gzip",
source: "Origine del backup",
readSource: "Leggi backup",
sourceDatabase: "Database di origine",
targetDatabase: "Database di destinazione",
destination: "Destinazione",
chooseDestination: "Scegli destinazione",
collection: "Raccolta",
kind: "Tipo",
documents: "Documenti",
indexes: "Indici",
dropExisting: "Elimina le raccolte di destinazione selezionate prima del ripristino",
restoreOptions: "Ripristina opzioni delle raccolte",
restoreIndexes: "Ripristina indici",
stopOnError: "Interrompi in caso di errori di scrittura",
objcheck: "Verifica i documenti BSON prima di inserirli (objcheck)",
readBytes: "Letti: {size} GiB decompressi",
batchSize: "Dimensione del batch",
selected: "{count} raccolte selezionate",
empty: "Nessuna raccolta",
preparing: "Lettura del backup...",
loading: "Caricamento delle raccolte...",
dump: "Avvia backup",
restore: "Avvia ripristino",
review: "Verifica",
back: "Indietro",
close: "Chiudi",
cancel: "Annulla attività",
confirmDump: "Esegui il backup di {count} raccolte da {database}.",
confirmRestore: "Ripristina {count} raccolte in {database}.",
confirmDrop: "Le raccolte di destinazione selezionate verranno eliminate e ricreate.",
confirmAppend: "I documenti verranno aggiunti alle raccolte esistenti.",
readonly: "Questa connessione è di sola lettura.",
done: "Completato",
cancelled: "Annullato",
failed: "Non riuscito",
progress: "Raccolte: {done} / {total}",
read: "Documenti letti",
written: "Documenti ripristinati",
rejected: "Documenti non riusciti",
createdIndexes: "Indici ripristinati",
uploadBytes: "Caricati: {done} / {total} GiB",
validationBytes: "Verificati: {size} GiB decompressi, {count} documenti",
phase: { preparing: "Preparazione", uploading: "Caricamento dei file selezionati", validating: "Verifica del backup", data: "Dati delle raccolte", indexes: "Ripristino degli indici", views: "Ripristino delle viste", archive: "Scrittura del backup", done: "Terminato" },
};
export const mongodbDatabaseDumpJa = {
selectAll: "すべてのコレクションを選択",
menuDump: "データベースをバックアップ...",
menuRestore: "データベースを復元...",
dumpTitle: "MongoDB データベースのバックアップ",
restoreTitle: "MongoDB データベースの復元",
format: "形式",
archive: "MongoDB archive",
directory: "MongoDB dump ディレクトリ",
gzip: "Gzip 圧縮",
source: "バックアップ元",
readSource: "バックアップを読み込む",
sourceDatabase: "復元元データベース",
targetDatabase: "復元先データベース",
destination: "保存先",
chooseDestination: "保存先を選択",
collection: "コレクション",
kind: "種類",
documents: "ドキュメント数",
indexes: "インデックス数",
dropExisting: "復元前に選択した復元先の同名コレクションを削除",
restoreOptions: "コレクションのオプションを復元",
restoreIndexes: "インデックスを復元",
stopOnError: "書き込みエラーが発生したら停止",
objcheck: "書き込み前に BSON ドキュメントを検証(objcheck)",
readBytes: "読み込み済み: 展開後 {size} GiB",
batchSize: "バッチサイズ",
selected: "{count} 個のコレクションを選択済み",
empty: "コレクションがありません",
preparing: "バックアップを読み込み中...",
loading: "コレクションを読み込み中...",
dump: "バックアップ開始",
restore: "復元開始",
review: "操作を確認",
back: "戻る",
close: "閉じる",
cancel: "タスクをキャンセル",
confirmDump: "{database} の {count} 個のコレクションをバックアップします。",
confirmRestore: "{count} 個のコレクションを {database} に復元します。",
confirmDrop: "選択した復元先の同名コレクションは削除され、再作成されます。",
confirmAppend: "ドキュメントは既存のコレクションに追加されます。",
readonly: "この接続は読み取り専用です。",
done: "完了",
cancelled: "キャンセル済み",
failed: "失敗",
progress: "コレクション: {done} / {total}",
read: "読み込んだドキュメント",
written: "復元したドキュメント",
rejected: "失敗したドキュメント",
createdIndexes: "復元したインデックス",
uploadBytes: "アップロード済み: {done} / {total} GiB",
validationBytes: "検証済み: 展開後 {size} GiB、{count} ドキュメント",
phase: { preparing: "準備中", uploading: "選択したファイルをアップロード中", validating: "バックアップを検証中", data: "コレクションデータ", indexes: "インデックスを復元中", views: "ビューを復元中", archive: "バックアップを書き込み中", done: "終了" },
};
export const mongodbDatabaseDumpKo = {
selectAll: "모든 컬렉션 선택",
menuDump: "데이터베이스 백업...",
menuRestore: "데이터베이스 복원...",
dumpTitle: "MongoDB 데이터베이스 백업",
restoreTitle: "MongoDB 데이터베이스 복원",
format: "형식",
archive: "MongoDB archive",
directory: "MongoDB dump 디렉터리",
gzip: "Gzip 압축",
source: "백업 원본",
readSource: "백업 읽기",
sourceDatabase: "원본 데이터베이스",
targetDatabase: "대상 데이터베이스",
destination: "저장 위치",
chooseDestination: "저장 위치 선택",
collection: "컬렉션",
kind: "유형",
documents: "문서 수",
indexes: "인덱스 수",
dropExisting: "복원 전 선택한 대상의 동명 컬렉션 삭제",
restoreOptions: "컬렉션 옵션 복원",
restoreIndexes: "인덱스 복원",
stopOnError: "쓰기 오류 발생 시 중지",
objcheck: "쓰기 전 BSON 문서 검사 (objcheck)",
readBytes: "읽음: 압축 해제 후 {size} GiB",
batchSize: "배치 크기",
selected: "컬렉션 {count}개 선택됨",
empty: "컬렉션 없음",
preparing: "백업 읽는 중...",
loading: "컬렉션 불러오는 중...",
dump: "백업 시작",
restore: "복원 시작",
review: "작업 확인",
back: "뒤로",
close: "닫기",
cancel: "작업 취소",
confirmDump: "{database}의 컬렉션 {count}개를 백업합니다.",
confirmRestore: "컬렉션 {count}개를 {database}에 복원합니다.",
confirmDrop: "선택한 대상의 동명 컬렉션이 삭제된 후 다시 생성됩니다.",
confirmAppend: "기존 컬렉션에 문서가 추가됩니다.",
readonly: "이 연결은 읽기 전용입니다.",
done: "완료",
cancelled: "취소됨",
failed: "실패",
progress: "컬렉션: {done} / {total}",
read: "읽은 문서",
written: "복원된 문서",
rejected: "실패한 문서",
createdIndexes: "복원된 인덱스",
uploadBytes: "업로드됨: {done} / {total} GiB",
validationBytes: "검증됨: 압축 해제 후 {size} GiB, 문서 {count}개",
phase: { preparing: "준비 중", uploading: "선택한 파일 업로드 중", validating: "백업 검증 중", data: "컬렉션 데이터", indexes: "인덱스 복원 중", views: "뷰 복원 중", archive: "백업 쓰는 중", done: "종료됨" },
};
export const mongodbDatabaseDumpPtBR = {
selectAll: "Selecionar todas as coleções",
menuDump: "Fazer backup do banco de dados...",
menuRestore: "Restaurar banco de dados...",
dumpTitle: "Backup do banco de dados MongoDB",
restoreTitle: "Restauração do banco de dados MongoDB",
format: "Formato",
archive: "MongoDB archive",
directory: "Diretório de backup do MongoDB",
gzip: "Gzip",
source: "Origem do backup",
readSource: "Ler backup",
sourceDatabase: "Banco de dados de origem",
targetDatabase: "Banco de dados de destino",
destination: "Destino",
chooseDestination: "Escolher destino",
collection: "Coleção",
kind: "Tipo",
documents: "Documentos",
indexes: "Índices",
dropExisting: "Excluir as coleções de destino selecionadas antes de restaurar",
restoreOptions: "Restaurar opções das coleções",
restoreIndexes: "Restaurar índices",
stopOnError: "Parar ao ocorrerem erros de gravação",
objcheck: "Verificar documentos BSON antes da inserção (objcheck)",
readBytes: "Lidos: {size} GiB descompactados",
batchSize: "Tamanho do lote",
selected: "{count} coleções selecionadas",
empty: "Nenhuma coleção",
preparing: "Lendo backup...",
loading: "Carregando coleções...",
dump: "Iniciar backup",
restore: "Iniciar restauração",
review: "Revisar",
back: "Voltar",
close: "Fechar",
cancel: "Cancelar tarefa",
confirmDump: "Fazer backup de {count} coleções de {database}.",
confirmRestore: "Restaurar {count} coleções em {database}.",
confirmDrop: "As coleções de destino selecionadas serão excluídas e recriadas.",
confirmAppend: "Os documentos serão adicionados às coleções existentes.",
readonly: "Esta conexão é somente leitura.",
done: "Concluído",
cancelled: "Cancelado",
failed: "Falhou",
progress: "Coleções: {done} / {total}",
read: "Documentos lidos",
written: "Documentos restaurados",
rejected: "Documentos com falha",
createdIndexes: "Índices restaurados",
uploadBytes: "Enviados: {done} / {total} GiB",
validationBytes: "Validados: {size} GiB descompactados, {count} documentos",
phase: { preparing: "Preparando", uploading: "Enviando arquivos selecionados", validating: "Validando backup", data: "Dados das coleções", indexes: "Restaurando índices", views: "Restaurando visualizações", archive: "Gravando backup", done: "Finalizado" },
};
export const mongodbDatabaseDumpZhTW = {
selectAll: "選取所有集合",
menuDump: "備份資料庫...",
menuRestore: "還原資料庫...",
dumpTitle: "MongoDB 資料庫備份",
restoreTitle: "MongoDB 資料庫還原",
format: "格式",
archive: "MongoDB archive",
directory: "MongoDB dump 目錄",
gzip: "Gzip 壓縮",
source: "備份來源",
readSource: "讀取備份",
sourceDatabase: "來源資料庫",
targetDatabase: "目標資料庫",
destination: "儲存位置",
chooseDestination: "選擇儲存位置",
collection: "集合",
kind: "類型",
documents: "文件數",
indexes: "索引數",
dropExisting: "還原前刪除選取的目標同名集合",
restoreOptions: "還原集合選項",
restoreIndexes: "還原索引",
stopOnError: "遇到寫入錯誤時停止",
objcheck: "寫入前檢查 BSON 文件(objcheck)",
readBytes: "已讀取:解壓縮後 {size} GiB",
batchSize: "批次大小",
selected: "已選取 {count} 個集合",
empty: "沒有集合",
preparing: "正在讀取備份...",
loading: "正在載入集合...",
dump: "開始備份",
restore: "開始還原",
review: "確認操作",
back: "返回",
close: "關閉",
cancel: "取消任務",
confirmDump: "備份資料庫 {database} 中的 {count} 個集合。",
confirmRestore: "將 {count} 個集合還原至資料庫 {database}。",
confirmDrop: "選取的目標同名集合將被刪除並重建。",
confirmAppend: "文件將附加至現有集合。",
readonly: "此連線處於唯讀模式。",
done: "已完成",
cancelled: "已取消",
failed: "失敗",
progress: "集合:{done} / {total}",
read: "已讀取文件",
written: "已還原文件",
rejected: "失敗文件",
createdIndexes: "已還原索引",
uploadBytes: "已上傳:{done} / {total} GiB",
validationBytes: "已驗證:解壓縮後 {size} GiB,{count} 個文件",
phase: { preparing: "準備中", uploading: "上傳選取的檔案", validating: "驗證備份", data: "集合資料", indexes: "還原索引", views: "還原檢視表", archive: "寫入備份", done: "已結束" },
};
+2
View File
@@ -1,4 +1,5 @@
import { withEnglishFallback } from "./fallback";
import { mongodbDatabaseDumpPtBR as mongoDump } from "./mongodbDatabaseDump";
import { meilisearchManagementPtBR } from "./meilisearchManagement";
import { redisGroupingPtBR as redisGrouping } from "./redisGrouping";
import docs from "./docs/pt-BR";
@@ -83,6 +84,7 @@ const consul = {
};
export default withEnglishFallback({
mongoDump,
cellTransform: {
title: "Prévia da conversão",
description: "Converte o valor atual para visualizar ou copiar. O valor original e o rascunho permanecem inalterados.",
+2
View File
@@ -3,6 +3,7 @@ import { withEnglishFallback } from "./fallback";
import docs from "./docs/zh-CN";
import { consul } from "./consulZhCN";
import { meilisearchManagementZhCN } from "./meilisearchManagement";
import { mongodbDatabaseDumpZhCN as mongoDump } from "./mongodbDatabaseDump";
export default withEnglishFallback({
cellTransform: {
@@ -5457,6 +5458,7 @@ export default withEnglishFallback({
},
},
},
mongoDump,
mongo: {
documents: "{count} 个文档",
addField: "添加字段",
+2
View File
@@ -1,5 +1,6 @@
import { redisGroupingZhTW as redisGrouping } from "./redisGrouping";
import { withEnglishFallback } from "./fallback";
import { mongodbDatabaseDumpZhTW as mongoDump } from "./mongodbDatabaseDump";
import { meilisearchManagementZhTW } from "./meilisearchManagement";
import docs from "./docs/zh-TW";
import { consulZhTWMessages } from "./consulZhTW";
@@ -82,6 +83,7 @@ const consul = {
};
export default withEnglishFallback({
mongoDump,
cellTransform: {
title: "轉換預覽",
description: "將目前內容轉換後檢視或複製,不改變原值和編輯草稿。",
@@ -0,0 +1,50 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { prepareMongodbRestoreSource, restoreMongodbDatabase } from "../http";
function file(name: string, size = 0) {
const value = new File([], name);
Object.defineProperty(value, "webkitRelativePath", { value: `dump/source/${name}` });
Object.defineProperty(value, "size", { value: size });
return value;
}
afterEach(() => vi.unstubAllGlobals());
describe("MongoDB metadata-first transport", () => {
it("previews a multi-gigabyte directory without uploading any BSON bodies", async () => {
const fetch = vi.fn().mockResolvedValue(new Response(JSON.stringify({ sourceRef: "catalog", databases: ["source"], collections: [] })));
vi.stubGlobal("fetch", fetch);
const data = file("large.bson.gz", 5 * 1024 ** 3);
const metadata = file("large.metadata.json.gz", 200);
await prepareMongodbRestoreSource([data, metadata], "directory", true);
expect(fetch).toHaveBeenCalledTimes(1);
const form = fetch.mock.calls[0]![1].body as FormData;
expect(form.get("format")).toBe("directory");
expect(form.get("gzip")).toBe("true");
expect(JSON.parse(form.get("manifest") as string)).toEqual([
{ path: "dump/source/large.bson.gz", sizeBytes: 5 * 1024 ** 3 },
{ path: "dump/source/large.metadata.json.gz", sizeBytes: 200 },
]);
expect(form.getAll("file")).toHaveLength(1);
expect((form.get("file") as File).name).toBe("dump/source/large.metadata.json.gz");
});
it("checks the effective server limit before the confirmed data upload", async () => {
const fetch = vi.fn().mockResolvedValue(new Response(JSON.stringify(1024 ** 3)));
vi.stubGlobal("fetch", fetch);
const request = { taskId: "task", connectionId: "conn", database: "target", sourceDatabase: "source", sourceRef: "catalog", dropExisting: true, restoreOptions: true, restoreIndexes: true, stopOnError: true, batchSize: 500 };
await expect(restoreMongodbDatabase(request, vi.fn(), { files: [file("large.bson", 5 * 1024 ** 3)], gzip: false })).rejects.toThrow("exceeds");
expect(fetch).toHaveBeenCalledTimes(1);
expect(fetch.mock.calls[0]![0]).toContain("/upload-limit");
});
it("passes cancellation to catalog upload without a separate restore request", async () => {
const controller = new AbortController();
const fetch = vi.fn().mockRejectedValue(new DOMException("Aborted", "AbortError"));
vi.stubGlobal("fetch", fetch);
controller.abort();
await expect(prepareMongodbRestoreSource([file("a.bson")], "directory", false, { signal: controller.signal })).rejects.toThrow("Aborted");
expect(fetch.mock.calls[0]![1].signal).toBe(controller.signal);
expect(fetch).toHaveBeenCalledTimes(1);
});
});
+7
View File
@@ -495,6 +495,13 @@ export const cancelMongodbImport = forward("cancelMongodbImport");
export const releaseMongodbImportSource = forward("releaseMongodbImportSource");
export const exportMongodbQuery = forward("exportMongodbQuery");
export const cancelMongodbExport = forward("cancelMongodbExport");
export const inspectMongodbDatabaseDump = forward("inspectMongodbDatabaseDump");
export const prepareMongodbRestoreSource = forward("prepareMongodbRestoreSource");
export const releaseMongodbRestoreSource = forward("releaseMongodbRestoreSource");
export const dumpMongodbDatabase = forward("dumpMongodbDatabase");
export const restoreMongodbDatabase = forward("restoreMongodbDatabase");
export const cancelMongodbDatabaseDump = forward("cancelMongodbDatabaseDump");
export type { MongoDumpFormat, MongoDumpSourceInput, MongoDumpCatalog, MongoDumpCollection, MongoRestoreSourcePreview, MongoDatabaseDumpRequest, MongoDatabaseRestoreRequest, MongoDatabaseDumpProgress } from "./mongodbDumpTypes";
// Database Export
export const beginDatabaseBackupSnapshot = forward("beginDatabaseBackupSnapshot");
+139
View File
@@ -1,3 +1,5 @@
import type { MongoDumpFormat, MongoDumpSourceInput, MongoDumpCatalog, MongoRestoreSourcePreview, MongoDatabaseDumpRequest, MongoDatabaseRestoreRequest, MongoDatabaseDumpProgress } from "./mongodbDumpTypes";
import type { MongoRestoreUpload, MongoSourceReadOptions } from "./mongodbDumpTypes";
import type {
ConnectionConfig,
ConnectionTestResult,
@@ -2674,6 +2676,143 @@ export async function releaseTableImportSource(sourceRef: string): Promise<boole
return result.released;
}
export function inspectMongodbDatabaseDump(connectionId: string, database: string): Promise<MongoDumpCatalog> {
return post("/api/mongo/dump/catalog", { connectionId, database });
}
async function checkMongoUploadSize(files: File[]) {
const limit = await get<number>("/api/mongo/dump/upload-limit");
const size = files.reduce((total, file) => total + file.size, 0);
if (size > limit) throw new Error(`MongoDB upload: ${(size / 1024 ** 3).toFixed(2)} GiB exceeds ${(limit / 1024 ** 3).toFixed(2)} GiB (DBX_MAX_UPLOAD_MB)`);
}
function uploadMongoForm(url: string, form: FormData, options?: MongoSourceReadOptions): Promise<MongoRestoreSourcePreview> {
return new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
const abort = () => xhr.abort();
const cleanup = () => options?.signal?.removeEventListener("abort", abort);
if (options?.signal?.aborted) {
reject(new DOMException("Cancelled", "AbortError"));
return;
}
xhr.open("POST", apiUrl(url));
xhr.upload.onprogress = (event) => options?.onUploadProgress?.(event.loaded, event.lengthComputable ? event.total : 0);
xhr.onload = async () => {
cleanup();
try {
if (xhr.status < 200 || xhr.status >= 300) throw await backendResponseError(new Response(xhr.responseText, { status: xhr.status }));
resolve(JSON.parse(xhr.responseText));
} catch (error) {
reject(error);
}
};
xhr.onerror = () => {
cleanup();
reject(new Error("MongoDB upload connection failed"));
};
xhr.onabort = () => {
cleanup();
reject(new DOMException("Cancelled", "AbortError"));
};
options?.signal?.addEventListener("abort", abort, { once: true });
xhr.send(form);
});
}
export async function prepareMongodbRestoreSource(source: MongoDumpSourceInput, format: MongoDumpFormat, gzip: boolean, options?: MongoSourceReadOptions): Promise<MongoRestoreSourcePreview> {
if (typeof source === "string") throw new Error("Web restores require uploaded files");
const files = Array.isArray(source) ? source : [source];
const form = new FormData();
form.append("format", format);
form.append("gzip", String(gzip));
if (format === "directory") {
form.append("manifest", JSON.stringify(files.map((file) => ({ path: file.webkitRelativePath || file.name, sizeBytes: file.size }))));
for (const file of files) if (/\.metadata\.json(?:\.gz)?$/.test(file.name)) form.append("file", file, file.webkitRelativePath || file.name);
} else {
await checkMongoUploadSize(files);
for (const file of files) form.append("file", file, file.name);
return uploadMongoForm("/api/mongo/dump/source", form, options);
}
const response = await fetch(apiUrl("/api/mongo/dump/source"), { method: "POST", body: form, signal: options?.signal });
if (!response.ok) throw await backendResponseError(response);
return response.json();
}
export async function releaseMongodbRestoreSource(sourceRef: string): Promise<boolean> {
return (await post<{ released: boolean }>("/api/mongo/dump/source/release", { sourceRef })).released;
}
async function runMongodbDatabaseTask(mode: "export" | "restore", request: MongoDatabaseDumpRequest | MongoDatabaseRestoreRequest, onProgress: (progress: MongoDatabaseDumpProgress) => void): Promise<MongoDatabaseDumpProgress> {
await post(`/api/mongo/dump/${mode}`, { request });
return new Promise((resolve, reject) => {
const events = new EventSource(apiUrl(`/api/mongo/dump/progress/${request.taskId}`));
events.onmessage = (event) => {
const progress: MongoDatabaseDumpProgress = JSON.parse(event.data);
onProgress(progress);
if (progress.status === "running") return;
events.close();
if (progress.status !== "done") {
reject(new Error(progress.errorMessage || "MongoDB database task failed"));
return;
}
if (mode === "export") {
const anchor = document.createElement("a");
anchor.href = apiUrl(`/api/mongo/export/download/${request.taskId}`);
anchor.click();
}
resolve(progress);
};
events.onerror = () => {
events.close();
reject(new Error("MongoDB database progress connection failed"));
};
});
}
export function dumpMongodbDatabase(request: MongoDatabaseDumpRequest, onProgress: (progress: MongoDatabaseDumpProgress) => void) {
return runMongodbDatabaseTask("export", request, onProgress);
}
export async function restoreMongodbDatabase(request: MongoDatabaseRestoreRequest, onProgress: (progress: MongoDatabaseDumpProgress) => void, upload?: MongoRestoreUpload) {
let acquired: string | undefined;
try {
if (upload) {
await checkMongoUploadSize(upload.files);
const form = new FormData();
form.append("format", "directory");
form.append("gzip", String(upload.gzip));
form.append("request", JSON.stringify(request));
for (const file of upload.files) form.append("file", file, file.webkitRelativePath || file.name);
const started = Date.now();
const preview = await uploadMongoForm("/api/mongo/dump/source/upload", form, {
signal: upload.signal,
onUploadProgress: (loaded, total) =>
onProgress({
taskId: request.taskId,
status: "running",
phase: "uploading",
collection: null,
collectionsDone: 0,
collectionsTotal: request.collections?.length ?? 0,
documentsRead: 0,
documentsWritten: 0,
documentsFailed: 0,
indexesCreated: 0,
elapsedMs: Date.now() - started,
errorMessage: null,
filePath: null,
bytesProcessed: loaded,
bytesTotal: total,
}),
});
acquired = preview.sourceRef;
if (upload.signal?.aborted) throw new DOMException("Cancelled", "AbortError");
request = { ...request, sourceRef: acquired };
}
return await runMongodbDatabaseTask("restore", request, onProgress);
} finally {
if (acquired) void releaseMongodbRestoreSource(acquired).catch(() => {});
}
}
export async function cancelMongodbDatabaseDump(taskId: string): Promise<boolean> {
return (await post<{ cancelled: boolean }>("/api/mongo/dump/cancel", { taskId })).cancelled;
}
export async function previewMongodbImportFile(fileOrPath: string | File | MongoImportPreviewRequest, options: Partial<MongoImportPreviewRequest> = {}): Promise<MongoImportPreview> {
if (typeof fileOrPath === "object" && !(fileOrPath instanceof File)) {
throw new Error("previewMongodbImportFile in web mode requires a File object for upload previews");
@@ -0,0 +1,72 @@
export type MongoDumpFormat = "directory" | "archive";
export type MongoDumpSourceInput = string | File | File[];
export interface MongoDumpCollection {
database: string;
name: string;
kind: string;
documents: number | null;
sizeBytes: number;
indexes: number;
sourceFiles?: string[];
}
export interface MongoDumpCatalog {
databases: string[];
collections: MongoDumpCollection[];
}
export interface MongoRestoreSourcePreview extends MongoDumpCatalog {
sourceRef: string;
}
export interface MongoDatabaseDumpRequest {
taskId: string;
connectionId: string;
database: string;
filePath: string;
format: MongoDumpFormat;
gzip: boolean;
collections?: string[];
}
export interface MongoDatabaseRestoreRequest {
taskId: string;
connectionId: string;
database: string;
sourceDatabase: string;
sourceRef: string;
collections?: string[];
dropExisting: boolean;
restoreOptions: boolean;
restoreIndexes: boolean;
stopOnError: boolean;
objcheck?: boolean;
batchSize: number;
executionId?: string;
}
export interface MongoDatabaseDumpProgress {
taskId: string;
status: "running" | "done" | "error" | "cancelled";
phase: "preparing" | "uploading" | "validating" | "data" | "indexes" | "views" | "archive" | "done";
collection: string | null;
collectionsDone: number;
collectionsTotal: number;
documentsRead: number;
documentsWritten: number;
documentsFailed: number;
indexesCreated: number;
elapsedMs: number;
errorMessage: string | null;
filePath: string | null;
bytesProcessed?: number;
bytesTotal?: number;
documentsValidated?: number;
}
export interface MongoRestoreUpload {
files: File[];
gzip: boolean;
signal?: AbortSignal;
}
export interface MongoSourceReadOptions {
signal?: AbortSignal;
onUploadProgress?: (loaded: number, total: number) => void;
}
+35 -2
View File
@@ -1,4 +1,6 @@
import { invoke as tauriInvoke } from "@tauri-apps/api/core";
import type { MongoDumpFormat, MongoDumpSourceInput, MongoDumpCatalog, MongoRestoreSourcePreview, MongoDatabaseDumpRequest, MongoDatabaseRestoreRequest, MongoDatabaseDumpProgress } from "./mongodbDumpTypes";
import type { MongoRestoreUpload, MongoSourceReadOptions } from "./mongodbDumpTypes";
import { assertUpdateAllowsCommand } from "@/lib/app/updatePreparation";
import { collectBrowserSupportInfo } from "@/lib/app/supportInfo";
@@ -5070,12 +5072,42 @@ export async function releaseTableImportSource(_sourceRef: string): Promise<bool
return false;
}
export type MongoImportFormat = "csv" | "json" | "ndjson";
export function inspectMongodbDatabaseDump(connectionId: string, database: string): Promise<MongoDumpCatalog> {
return invoke("inspect_mongodb_database_dump", { connectionId, database });
}
export function prepareMongodbRestoreSource(source: MongoDumpSourceInput, format: MongoDumpFormat, gzip: boolean, _options?: MongoSourceReadOptions): Promise<MongoRestoreSourcePreview> {
if (typeof source !== "string") throw new Error("Desktop restores require a file or directory path");
return invoke("prepare_mongodb_restore_source", { request: { path: source, format, gzip } });
}
export function releaseMongodbRestoreSource(sourceRef: string): Promise<boolean> {
return invoke("release_mongodb_restore_source", { sourceRef });
}
async function runMongodbDatabaseTask(command: string, request: MongoDatabaseDumpRequest | MongoDatabaseRestoreRequest, onProgress: (progress: MongoDatabaseDumpProgress) => void): Promise<MongoDatabaseDumpProgress> {
const unlisten = await listen<MongoDatabaseDumpProgress>("mongo-database-dump-progress", (event) => {
if (event.payload.taskId === request.taskId) onProgress(event.payload);
});
try {
return await invoke(command, { request });
} finally {
unlisten();
}
}
export function dumpMongodbDatabase(request: MongoDatabaseDumpRequest, onProgress: (progress: MongoDatabaseDumpProgress) => void) {
return runMongodbDatabaseTask("dump_mongodb_database", request, onProgress);
}
export function restoreMongodbDatabase(request: MongoDatabaseRestoreRequest, onProgress: (progress: MongoDatabaseDumpProgress) => void, _upload?: MongoRestoreUpload) {
return runMongodbDatabaseTask("restore_mongodb_database", request, onProgress);
}
export function cancelMongodbDatabaseDump(taskId: string): Promise<boolean> {
return invoke("cancel_mongodb_database_dump", { taskId });
}
export type MongoImportFormat = "csv" | "json" | "ndjson" | "bson";
export type MongoImportTypeMode = "string" | "auto" | "extendedJson";
export type MongoImportInferredType = "boolean" | "integer" | "decimal" | "date" | "objectId" | "object" | "array" | "mixed" | "string";
export type MongoImportStatus = "running" | "done" | "error" | "cancelled";
export type MongoImportPhase = "preparing" | "parsing" | "writing" | "done";
export type MongoExportFormat = "csv" | "ndjson";
export type MongoExportFormat = "csv" | "ndjson" | "bson";
export type MongoExportStatus = "running" | "done" | "error" | "cancelled";
export interface MongoImportIssue {
@@ -5176,6 +5208,7 @@ export interface MongoExportRequest {
collation?: string | null;
format: MongoExportFormat;
includeHeader?: boolean;
gzip?: boolean;
filePath: string;
executionId?: string | null;
}
@@ -559,6 +559,7 @@ export const useConnectionStore = defineStore("connection", () => {
schema?: string;
tableName?: string;
} | null>(null);
const mongoDatabaseDumpSource = ref<{ connectionId: string; database: string; mode: "dump" | "restore" } | null>(null);
const mongoImportSource = ref<{
connectionId: string;
database: string;
@@ -9409,6 +9410,7 @@ export const useConnectionStore = defineStore("connection", () => {
diagramSource,
docsSource,
tableImportSource,
mongoDatabaseDumpSource,
mongoImportSource,
mongoImportCompleted,
tableDataGenerateSource,
+2
View File
@@ -83,6 +83,8 @@ bytes = "1"
font-kit = { version = "0.14.3", optional = true }
zip = { version = "4", default-features = false, features = ["deflate"] }
flate2 = "1"
crc = "3"
sha1 = "0.10"
tar = "0.4"
zstd = "0.13"
sysinfo = { version = "0.32", features = ["system"] }
+3 -3
View File
@@ -2579,17 +2579,17 @@ pub struct MongoInsertOutcome {
pub errors: Vec<MongoBulkWriteError>,
}
pub async fn insert_bson_documents(
pub async fn insert_bson_documents<T: serde::Serialize + Send + Sync>(
client: &Client,
database: &str,
collection: &str,
documents: Vec<Document>,
documents: Vec<T>,
) -> Result<MongoInsertOutcome, MongoBulkWriteError> {
if documents.is_empty() {
return Ok(MongoInsertOutcome::default());
}
let total = documents.len() as u64;
let col = client.database(database).collection::<Document>(collection);
let col = client.database(database).collection::<T>(collection);
// Unordered: one rejected document must not abandon the rest of the batch, and the server
// then reports every rejection instead of stopping at the first.
match col.insert_many(documents).ordered(false).await {
+1
View File
@@ -58,6 +58,7 @@ pub mod models;
pub mod mongo_oidc;
pub mod mongo_ops;
pub mod mongo_shell;
pub mod mongodb_dump;
pub mod mongodb_import_export;
#[cfg(feature = "mq-admin")]
pub mod mq;
+639
View File
@@ -0,0 +1,639 @@
//! Database-scoped MongoDB dump/restore; transport adapters own dialogs and uploads.
mod archive;
mod directory;
mod metadata;
mod source;
mod stream;
pub use source::{
attach_mongodb_restore_upload, mongodb_restore_upload_files, prepare_mongodb_directory_catalog,
prepare_owned_mongodb_restore_source, MongoDumpFile,
};
use std::collections::{BTreeSet, HashMap, HashSet};
use std::future::Future;
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::sync::{
atomic::{AtomicBool, Ordering},
Arc,
};
use std::time::{Duration, Instant};
use futures::TryStreamExt;
use mongodb::bson::{doc, Document};
use serde::{Deserialize, Serialize};
use crate::connection::{AppState, PoolKind};
use crate::mongodb_import_export::{export_mongodb_query_core, MongoExportFormat, MongoExportRequest};
use metadata::CollectionMetadata;
type CancelFuture = Pin<Box<dyn Future<Output = bool> + Send>>;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum MongoDumpFormat {
Directory,
Archive,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MongoDumpCollection {
pub database: String,
pub name: String,
pub kind: String,
pub documents: Option<u64>,
pub size_bytes: u64,
pub indexes: usize,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub source_files: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MongoDumpCatalog {
pub databases: Vec<String>,
pub collections: Vec<MongoDumpCollection>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MongoRestoreSourceRequest {
pub path: String,
pub format: MongoDumpFormat,
#[serde(default)]
pub gzip: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MongoRestoreSourcePreview {
pub source_ref: String,
#[serde(flatten)]
pub catalog: MongoDumpCatalog,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MongoDatabaseDumpRequest {
pub task_id: String,
pub connection_id: String,
pub database: String,
pub file_path: String,
pub format: MongoDumpFormat,
#[serde(default)]
pub gzip: bool,
#[serde(default)]
pub collections: Option<Vec<String>>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MongoDatabaseRestoreRequest {
pub task_id: String,
pub connection_id: String,
pub database: String,
pub source_database: String,
pub source_ref: String,
#[serde(default)]
pub collections: Option<Vec<String>>,
#[serde(default)]
pub drop_existing: bool,
#[serde(default = "yes")]
pub restore_options: bool,
#[serde(default = "yes")]
pub restore_indexes: bool,
#[serde(default = "yes")]
pub stop_on_error: bool,
#[serde(default)]
pub objcheck: bool,
#[serde(default = "batch_size")]
pub batch_size: usize,
#[serde(default)]
pub execution_id: Option<String>,
}
fn yes() -> bool {
true
}
fn batch_size() -> usize {
500
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MongoDatabaseDumpProgress {
pub task_id: String,
pub status: String,
pub phase: String,
pub collection: Option<String>,
pub collections_done: usize,
pub collections_total: usize,
pub documents_read: u64,
pub documents_written: u64,
pub documents_failed: u64,
pub indexes_created: usize,
pub elapsed_ms: u128,
pub error_message: Option<String>,
pub file_path: Option<String>,
#[serde(default)]
pub bytes_processed: u64,
#[serde(default)]
pub documents_validated: u64,
}
impl MongoDatabaseDumpProgress {
pub fn initial(task_id: &str) -> Self {
Self {
task_id: task_id.into(),
status: "running".into(),
phase: "preparing".into(),
collection: None,
collections_done: 0,
collections_total: 0,
documents_read: 0,
documents_written: 0,
documents_failed: 0,
indexes_created: 0,
elapsed_ms: 0,
error_message: None,
file_path: None,
bytes_processed: 0,
documents_validated: 0,
}
}
fn emit(&mut self, started: Instant, callback: &mut impl FnMut(Self)) {
self.elapsed_ms = started.elapsed().as_millis();
callback(self.clone());
}
}
#[derive(Clone)]
struct PreparedCollection {
database: String,
metadata: CollectionMetadata,
path: Option<PathBuf>,
documents: u64,
size_bytes: u64,
metadata_path: Option<PathBuf>,
}
pub fn release_mongodb_restore_source(source_ref: &str) -> bool {
source::release(source_ref)
}
pub async fn prepare_mongodb_restore_source(
request: MongoRestoreSourceRequest,
) -> Result<MongoRestoreSourcePreview, String> {
source::prepare(request, None, None).await
}
fn catalog(entries: &[PreparedCollection], exact: bool) -> MongoDumpCatalog {
MongoDumpCatalog {
databases: entries.iter().map(|entry| entry.database.clone()).collect::<BTreeSet<_>>().into_iter().collect(),
collections: entries
.iter()
.map(|entry| MongoDumpCollection {
database: entry.database.clone(),
name: entry.metadata.collection_name.clone(),
kind: entry.metadata.kind.clone(),
documents: if exact { Some(entry.documents) } else { None },
size_bytes: entry.size_bytes,
indexes: entry.metadata.restore_indexes().len(),
source_files: Vec::new(),
})
.collect(),
}
}
async fn native_client(state: &AppState, id: &str, database: &str) -> Result<mongodb::Client, String> {
metadata::validate_database(database)?;
state.get_or_create_pool(id, Some(database)).await?;
match state.pool_handle(id).await {
Some(PoolKind::MongoDb(client)) => Ok(client.clone()),
_ => Err("Database dump/restore requires the native MongoDB driver".into()),
}
}
async fn database_entries(client: &mongodb::Client, database: &str) -> Result<Vec<PreparedCollection>, String> {
let db = client.database(database);
let mut cursor = db.list_collections().await.map_err(|e| e.to_string())?;
let mut entries = Vec::new();
while let Some(spec) = cursor.try_next().await.map_err(|e| e.to_string())? {
if spec.name.starts_with("system.") && spec.name != "system.js" {
continue;
}
let mut metadata = CollectionMetadata::empty(&spec.name);
metadata.kind =
crate::db::mongo_driver::MongoCollectionKind::from_driver_type(&spec.collection_type).as_str().into();
metadata.options = mongodb::bson::to_document(&spec.options).map_err(|e| e.to_string())?;
metadata.validate()?;
if !metadata.is_view() {
let mut indexes = db.collection::<Document>(&spec.name).list_indexes().await.map_err(|e| e.to_string())?;
while let Some(index) = indexes.try_next().await.map_err(|e| e.to_string())? {
metadata.indexes.push(mongodb::bson::to_document(&index).map_err(|e| e.to_string())?);
}
}
metadata.validate()?;
entries.push(PreparedCollection {
database: database.into(),
metadata,
path: None,
documents: 0,
size_bytes: 0,
metadata_path: None,
});
}
entries.sort_by(|a, b| a.metadata.collection_name.cmp(&b.metadata.collection_name));
Ok(entries)
}
pub async fn inspect_mongodb_database_dump(
state: &AppState,
id: &str,
database: &str,
) -> Result<MongoDumpCatalog, String> {
let client = native_client(state, id, database).await?;
let entries = database_entries(&client, database).await?;
let mut catalog = catalog(&entries, false);
catalog.databases = vec![database.into()];
Ok(catalog)
}
fn select_entries(
entries: &[PreparedCollection],
selected: &Option<Vec<String>>,
) -> Result<Vec<PreparedCollection>, String> {
match selected {
None => Ok(entries.to_vec()),
Some(names) => {
let mut requested: HashSet<_> = names.iter().collect();
let chosen =
entries.iter().filter(|entry| requested.remove(&entry.metadata.collection_name)).cloned().collect();
if !requested.is_empty() {
return Err("A selected collection is not present in the dump plan".into());
}
Ok(chosen)
}
}
}
fn ordered_entries(entries: &[PreparedCollection]) -> Result<Vec<PreparedCollection>, String> {
let names: HashMap<_, _> = entries
.iter()
.enumerate()
.map(|(i, entry)| ((entry.database.clone(), entry.metadata.collection_name.clone()), i))
.collect();
let mut ordered = Vec::with_capacity(entries.len());
// 0 = unseen, 1 = active dependency, 2 = emitted. Uploaded metadata may
// contain arbitrarily deep chains, so keep the traversal off the call stack.
let mut states = vec![0u8; entries.len()];
for (i, entry) in entries.iter().enumerate().filter(|(_, e)| !e.metadata.is_view()) {
states[i] = 2;
ordered.push(entry.clone());
}
for i in 0..entries.len() {
let mut pending = vec![(i, false)];
while let Some((index, finish)) = pending.pop() {
if finish {
states[index] = 2;
ordered.push(entries[index].clone());
continue;
}
match states[index] {
2 => continue,
1 => return Err("Cyclic view dependencies in dump metadata".into()),
_ => {}
}
states[index] = 1;
pending.push((index, true));
let entry = &entries[index];
if let Ok(base) = entry.metadata.options.get_str("viewOn") {
if let Some(&parent) = names.get(&(entry.database.clone(), base.into())) {
pending.push((parent, false));
}
}
}
}
Ok(ordered)
}
fn cancelled(flag: &AtomicBool) -> Result<(), String> {
if flag.load(Ordering::Relaxed) {
Err("MongoDB dump/restore cancelled".into())
} else {
Ok(())
}
}
async fn blocking_work<T: Send + 'static>(
task_id: &str,
is_cancelled: &mut impl FnMut(&str) -> CancelFuture,
work: impl FnOnce(Arc<AtomicBool>) -> Result<T, String> + Send + 'static,
) -> Result<T, String> {
if is_cancelled(task_id).await {
return Err("MongoDB dump/restore cancelled".into());
}
struct CancelOnDrop(Arc<AtomicBool>);
impl Drop for CancelOnDrop {
fn drop(&mut self) {
self.0.store(true, Ordering::Relaxed);
}
}
let flag = Arc::new(AtomicBool::new(false));
let _guard = CancelOnDrop(flag.clone());
let signal = flag.clone();
let mut task = tokio::task::spawn_blocking(move || work(signal));
let mut interval = tokio::time::interval(Duration::from_millis(100));
loop {
tokio::select! {
result = &mut task => return result.map_err(|e| e.to_string())?,
_ = interval.tick() => if is_cancelled(task_id).await { flag.store(true, Ordering::Relaxed); },
}
}
}
pub async fn dump_mongodb_database<C, F>(
state: &AppState,
request: &MongoDatabaseDumpRequest,
mut is_cancelled: C,
mut on_progress: F,
) -> Result<MongoDatabaseDumpProgress, String>
where
C: FnMut(&str) -> CancelFuture,
F: FnMut(MongoDatabaseDumpProgress),
{
let started = Instant::now();
let mut progress = MongoDatabaseDumpProgress::initial(&request.task_id);
progress.emit(started, &mut on_progress);
let result: Result<(), String> = async {
if is_cancelled(&request.task_id).await {
return Err("MongoDB dump/restore cancelled".into());
}
let client = native_client(state, &request.connection_id, &request.database).await?;
let mut entries = select_entries(&database_entries(&client, &request.database).await?, &request.collections)?;
progress.collections_total = entries.len();
let target = PathBuf::from(&request.file_path);
if target.exists() {
return Err(match request.format {
MongoDumpFormat::Directory => "Dump destination already exists; choose a new directory",
MongoDumpFormat::Archive => "Dump destination already exists; choose a new archive file",
}
.into());
}
let parent = target.parent().filter(|p| !p.as_os_str().is_empty()).unwrap_or(Path::new("."));
std::fs::create_dir_all(parent).map_err(|e| e.to_string())?;
let working =
tempfile::Builder::new().prefix(".dbx-mongo-dump-").tempdir_in(parent).map_err(|e| e.to_string())?;
for (index, entry) in entries.iter_mut().enumerate() {
if is_cancelled(&request.task_id).await {
return Err("MongoDB dump/restore cancelled".into());
}
progress.collection = Some(entry.metadata.collection_name.clone());
progress.phase = "data".into();
progress.emit(started, &mut on_progress);
if !entry.metadata.is_view() {
let path = working.path().join(format!("{index}.bson"));
let child = MongoExportRequest {
export_id: request.task_id.clone(),
connection_id: request.connection_id.clone(),
database: request.database.clone(),
collection: entry.metadata.collection_name.clone(),
filter: None,
sort: None,
projection: None,
collation: None,
format: MongoExportFormat::Bson,
include_header: false,
gzip: false,
file_path: path.to_string_lossy().into_owned(),
execution_id: None,
};
let base = progress.documents_read;
let summary = export_mongodb_query_core(
state,
&child,
|id| is_cancelled(id),
|child| {
progress.documents_read = base + child.documents_read;
progress.emit(started, &mut on_progress);
},
)
.await?;
entry.documents = summary.documents_exported;
entry.size_bytes = std::fs::metadata(&path).map_err(|e| e.to_string())?.len();
entry.path = Some(path);
}
progress.collections_done += 1;
}
progress.phase = "archive".into();
progress.emit(started, &mut on_progress);
let server_version = client
.database(&request.database)
.run_command(doc! { "buildInfo": 1 })
.await
.map_err(|e| e.to_string())?
.get_str("version")
.unwrap_or("")
.to_string();
let database = request.database.clone();
let gzip = request.gzip;
let format = request.format;
blocking_work(&request.task_id, &mut is_cancelled, move |flag| {
let _working = working;
let parent = target.parent().filter(|p| !p.as_os_str().is_empty()).unwrap_or(Path::new("."));
match format {
MongoDumpFormat::Archive => {
let temp = tempfile::NamedTempFile::new_in(parent).map_err(|e| e.to_string())?;
directory::write_file(temp.path(), gzip, |writer| {
archive::pack(writer, &entries, &server_version, || cancelled(&flag))
})?;
cancelled(&flag)?;
if target.exists() {
return Err("Dump destination already exists".into());
}
temp.persist(&target).map_err(|e| e.to_string())?;
}
MongoDumpFormat::Directory => {
let output = tempfile::Builder::new()
.prefix(".dbx-dump-output-")
.tempdir_in(parent)
.map_err(|e| e.to_string())?;
directory::publish_files(output.path(), &database, &entries, gzip, || cancelled(&flag))?;
cancelled(&flag)?;
if target.exists() {
return Err("Dump destination already exists".into());
}
std::fs::rename(output.path(), &target).map_err(|e| e.to_string())?;
}
}
Ok(())
})
.await?;
progress.file_path = Some(request.file_path.clone());
Ok(())
}
.await;
finish_progress(progress, result, started, &mut on_progress)
}
pub async fn restore_mongodb_database<C, F>(
state: &AppState,
request: &MongoDatabaseRestoreRequest,
mut is_cancelled: C,
mut on_progress: F,
) -> Result<MongoDatabaseDumpProgress, String>
where
C: FnMut(&str) -> CancelFuture,
F: FnMut(MongoDatabaseDumpProgress),
{
let started = Instant::now();
let mut progress = MongoDatabaseDumpProgress::initial(&request.task_id);
progress.emit(started, &mut on_progress);
let result: Result<(), String> = async {
if let Some(name) = crate::query::connection_readonly_name(state, &request.connection_id).await {
return Err(format!("Read-only connection '{name}': restore blocked"));
}
if is_cancelled(&request.task_id).await {
return Err("MongoDB dump/restore cancelled".into());
}
crate::mongodb_import_export::clamp_batch_size(request.batch_size).map_err(|e| e.display_message())?;
let source = source::get(&request.source_ref)?;
let source_entries: Vec<_> =
source.entries.iter().filter(|entry| entry.database == request.source_database).cloned().collect();
if source_entries.is_empty() && !source.entries.is_empty() {
return Err("Source database is not present in the dump".into());
}
let entries = ordered_entries(&select_entries(&source_entries, &request.collections)?)?;
progress.collections_total = entries.len();
source.check_ready(&entries)?;
let client = native_client(state, &request.connection_id, &request.database).await?;
let db = client.database(&request.database);
let existing = db
.list_collections()
.await
.map_err(|e| e.to_string())?
.try_collect::<Vec<_>>()
.await
.map_err(|e| e.to_string())?;
for entry in &entries {
entry.metadata.validate()?;
if let Some(target) = existing.iter().find(|target| target.name == entry.metadata.collection_name) {
if !request.drop_existing
&& (entry.metadata.is_view()
|| !matches!(target.collection_type, mongodb::results::CollectionType::Collection))
{
return Err(format!(
"{}: replacing an existing view or non-regular collection requires dropExisting",
target.name
));
}
}
if let Ok(base) = entry.metadata.options.get_str("viewOn") {
if !entries.iter().any(|e| e.metadata.collection_name == base)
&& !existing.iter().any(|e| e.name == base)
{
return Err(format!(
"Missing backing collection for view {}: {base}",
entry.metadata.collection_name
));
}
}
}
let existing_names = existing.iter().map(|e| e.name.clone()).collect::<HashSet<_>>();
stream::restore_data(
state,
&client,
request,
source,
&entries,
&existing_names,
&mut progress,
started,
&mut is_cancelled,
&mut on_progress,
)
.await?;
for entry in &entries {
if is_cancelled(&request.task_id).await {
return Err("MongoDB dump/restore cancelled".into());
}
stream::writable(state, &request.connection_id).await?;
let name = &entry.metadata.collection_name;
progress.collection = Some(name.clone());
progress.phase = if entry.metadata.is_view() { "views" } else { "indexes" }.into();
progress.emit(started, &mut on_progress);
if entry.metadata.is_view() {
stream::initialize_collection(&client, request, entry, existing_names.contains(name)).await?;
progress.collections_done += 1;
} else if request.restore_indexes {
for index in entry.metadata.restore_indexes() {
if is_cancelled(&request.task_id).await {
return Err("MongoDB dump/restore cancelled".into());
}
db.run_command(doc! { "createIndexes": name, "indexes": [index] })
.await
.map_err(|e| e.to_string())?;
progress.indexes_created += 1;
}
}
}
Ok(())
}
.await;
finish_progress(progress, result, started, &mut on_progress)
}
fn finish_progress(
mut progress: MongoDatabaseDumpProgress,
result: Result<(), String>,
started: Instant,
callback: &mut impl FnMut(MongoDatabaseDumpProgress),
) -> Result<MongoDatabaseDumpProgress, String> {
progress.phase = "done".into();
progress.status = match &result {
Ok(()) => "done",
Err(error)
if matches!(error.as_str(), "MongoDB dump/restore cancelled" | "Import cancelled" | "Export cancelled") =>
{
"cancelled"
}
Err(_) => "error",
}
.into();
progress.error_message = result.as_ref().err().cloned();
progress.emit(started, callback);
result.map(|_| progress)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn view_ordering_handles_deep_dependency_chains() {
let count = 20_000;
let entries = (0..count)
.map(|index| {
let mut metadata = CollectionMetadata::empty(&format!("view_{index}"));
if index + 1 < count {
metadata.kind = "view".into();
metadata.options = doc! { "viewOn": format!("view_{}", index + 1), "pipeline": [] };
}
PreparedCollection {
database: "source".into(),
metadata,
path: None,
documents: 0,
size_bytes: 0,
metadata_path: None,
}
})
.collect::<Vec<_>>();
let ordered = ordered_entries(&entries).unwrap();
assert_eq!(ordered.len(), count);
for (entry, index) in ordered.iter().zip((0..count).rev()) {
assert_eq!(entry.metadata.collection_name, format!("view_{index}"));
}
}
}
+290
View File
@@ -0,0 +1,290 @@
//! MongoDB Database Tools archive format 0.1, see common/archive/spec.md upstream.
use std::collections::HashMap;
use std::fs::File;
use std::io::{BufReader, Read, Write};
use crc::{Crc, Digest, CRC_64_XZ};
use mongodb::bson::{doc, Document};
use super::{metadata::CollectionMetadata, PreparedCollection};
const MAGIC: u32 = 0x8199_e26d;
const TERMINATOR: [u8; 4] = [255; 4];
// Go hash/crc64 with the ECMA table reflects bits and complements the initial/final state.
const CRC: Crc<u64> = Crc::<u64>::new(&CRC_64_XZ);
pub(super) const MAX_DOCUMENT: usize = 16 * 1024 * 1024;
pub(super) enum Frame {
End,
Terminator,
Document(Vec<u8>),
}
pub(super) fn read_frame(reader: &mut dyn Read) -> Result<Frame, String> {
let frame = read_raw_frame(reader)?;
if let Frame::Document(bytes) = &frame {
mongodb::bson::from_slice::<Document>(bytes).map_err(|e| format!("Invalid BSON document: {e}"))?;
}
Ok(frame)
}
pub(super) fn read_raw_frame(reader: &mut dyn Read) -> Result<Frame, String> {
let mut header = [0; 4];
loop {
match reader.read(&mut header[..1]) {
Ok(0) => return Ok(Frame::End),
Ok(_) => break,
Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
Err(e) => return Err(e.to_string()),
}
}
reader.read_exact(&mut header[1..]).map_err(|e| format!("Truncated BSON header: {e}"))?;
if header == TERMINATOR {
return Ok(Frame::Terminator);
}
let length = i32::from_le_bytes(header);
if !(5..=MAX_DOCUMENT as i32).contains(&length) {
return Err(format!("Invalid BSON frame length: {length}"));
}
let mut bytes = vec![0; length as usize];
bytes[..4].copy_from_slice(&header);
reader.read_exact(&mut bytes[4..]).map_err(|e| format!("Truncated BSON document: {e}"))?;
mongodb::bson::RawDocument::from_bytes(&bytes).map_err(|e| format!("Invalid BSON document: {e}"))?;
Ok(Frame::Document(bytes))
}
fn document(frame: Frame) -> Result<Document, String> {
match frame {
Frame::Document(bytes) => mongodb::bson::from_slice(&bytes).map_err(|e| e.to_string()),
_ => Err("Expected an archive BSON header".into()),
}
}
fn string(doc: &Document, key: &str) -> Result<String, String> {
doc.get_str(key).map(String::from).map_err(|_| format!("Archive header is missing {key}"))
}
fn write_document(writer: &mut dyn Write, doc: &Document) -> Result<(), String> {
let bytes = mongodb::bson::to_vec(doc).map_err(|e| e.to_string())?;
if bytes.len() > MAX_DOCUMENT {
return Err("Archive metadata exceeds the BSON document limit".into());
}
writer.write_all(&bytes).map_err(|e| e.to_string())
}
pub(super) fn inspect(
reader: &mut dyn Read,
mut check: impl FnMut() -> Result<(), String>,
) -> Result<Vec<PreparedCollection>, String> {
let mut magic = [0; 4];
reader.read_exact(&mut magic).map_err(|e| format!("Invalid MongoDB archive: {e}"))?;
if u32::from_le_bytes(magic) != MAGIC {
return Err("Not a MongoDB --archive file".into());
}
let header = document(read_frame(reader)?)?;
if header.get_str("version") != Ok("0.1") {
return Err("Unsupported MongoDB archive version".into());
}
let mut entries = Vec::<PreparedCollection>::new();
let mut namespaces = HashMap::new();
let mut metadata_bytes = 0usize;
loop {
check()?;
let frame = read_frame(reader)?;
if matches!(frame, Frame::Terminator) {
break;
}
let header = document(frame)?;
let database = string(&header, "db")?;
super::metadata::validate_database(&database)?;
let name = string(&header, "collection")?;
let json = string(&header, "metadata")?;
metadata_bytes += json.len();
if entries.len() >= 100_000 || metadata_bytes > 128 * 1024 * 1024 {
return Err("Archive metadata exceeds the supported limit".into());
}
let metadata = if json.is_empty() {
CollectionMetadata::empty(&name)
} else {
CollectionMetadata::from_json(&json, &name)?
};
metadata.validate()?;
if metadata.collection_name != name {
return Err("Archive namespace does not match collection metadata".into());
}
if header.get_str("type").is_ok_and(|kind| kind == "timeseries") {
return Err("Time-series archives require server-specific restore support".into());
}
let index = entries.len();
if namespaces.insert((database.clone(), name), index).is_some() {
return Err("Duplicate archive namespace".into());
}
entries.push(PreparedCollection {
database,
metadata,
path: None,
documents: 0,
size_bytes: 0,
metadata_path: None,
});
}
Ok(entries)
}
pub(super) enum StreamEvent {
Document(usize, Vec<u8>),
End(usize),
}
pub(super) fn stream(
reader: &mut dyn Read,
expected: &[PreparedCollection],
mut check: impl FnMut() -> Result<(), String>,
mut emit: impl FnMut(StreamEvent) -> Result<(), String>,
) -> Result<(), String> {
let entries = inspect(reader, &mut check)?;
let namespaces: HashMap<_, _> = entries
.iter()
.enumerate()
.map(|(i, e)| ((e.database.clone(), e.metadata.collection_name.clone()), i))
.collect();
if entries.len() != expected.len() {
return Err("Archive metadata changed after preview".into());
}
let mut mapped = Vec::with_capacity(entries.len());
for entry in &entries {
let index = expected
.iter()
.position(|e| e.database == entry.database && e.metadata.collection_name == entry.metadata.collection_name)
.ok_or("Archive metadata changed after preview")?;
if expected[index].metadata.to_json()? != entry.metadata.to_json()? {
return Err("Archive metadata changed after preview".into());
}
mapped.push(index);
}
let mut digests: Vec<Digest<'_, u64>> = entries.iter().map(|_| CRC.digest()).collect();
let mut finished = vec![false; entries.len()];
loop {
check()?;
let frame = read_frame(reader)?;
if matches!(frame, Frame::End) {
break;
}
let header = document(frame)?;
let namespace = (string(&header, "db")?, string(&header, "collection")?);
let index = *namespaces.get(&namespace).ok_or_else(|| "Unknown namespace in archive data".to_string())?;
if finished[index] {
return Err("Archive contains data after namespace EOF".into());
}
let eof = header.get_bool("EOF").map_err(|_| "Archive header is missing EOF".to_string())?;
let checksum = header.get_i64("CRC").map_err(|_| "Archive header is missing CRC".to_string())? as u64;
if eof {
if !matches!(read_frame(reader)?, Frame::Terminator) {
return Err("Invalid archive EOF terminator".into());
}
if digests[index].clone().finalize() != checksum {
return Err(format!("Archive checksum mismatch: {}.{}", namespace.0, namespace.1));
}
finished[index] = true;
emit(StreamEvent::End(mapped[index]))?;
continue;
}
if checksum != 0 {
return Err("Non-EOF archive header contains a checksum".into());
}
let entry = &entries[index];
loop {
check()?;
match read_raw_frame(reader)? {
Frame::Document(bytes) => {
if entry.metadata.is_view() {
return Err("Archive contains BSON documents for a view".into());
}
digests[index].update(&bytes);
emit(StreamEvent::Document(mapped[index], bytes))?;
}
Frame::Terminator => break,
Frame::End => return Err("Truncated archive namespace segment".into()),
}
}
}
for (index, entry) in entries.iter().enumerate() {
if !finished[index] {
return Err(format!("Missing archive EOF: {}.{}", entry.database, entry.metadata.collection_name));
}
}
Ok(())
}
pub(super) fn pack(
writer: &mut dyn Write,
entries: &[PreparedCollection],
server_version: &str,
mut check: impl FnMut() -> Result<(), String>,
) -> Result<(), String> {
writer.write_all(&MAGIC.to_le_bytes()).map_err(|e| e.to_string())?;
write_document(
writer,
&doc! { "concurrent_collections": 1i32, "version": "0.1", "server_version": server_version, "tool_version": concat!("DBX ", env!("CARGO_PKG_VERSION")) },
)?;
for entry in entries {
check()?;
write_document(
writer,
&doc! { "db": &entry.database, "collection": &entry.metadata.collection_name, "metadata": entry.metadata.to_json()?, "size": entry.documents as i64, "type": if entry.metadata.is_view() { "view" } else { "" } },
)?;
}
writer.write_all(&TERMINATOR).map_err(|e| e.to_string())?;
for entry in entries {
check()?;
if entry.metadata.is_view() {
write_document(
writer,
&doc! { "db": &entry.database, "collection": &entry.metadata.collection_name, "EOF": true, "CRC": 0i64 },
)?;
writer.write_all(&TERMINATOR).map_err(|e| e.to_string())?;
continue;
}
let mut digest = CRC.digest();
let path = entry.path.as_ref().ok_or_else(|| "Missing collection BSON file".to_string())?;
let mut reader = BufReader::new(File::open(path).map_err(|e| e.to_string())?);
let mut opened = false;
loop {
check()?;
match read_frame(&mut reader)? {
Frame::End => break,
Frame::Terminator => return Err("Unexpected terminator in BSON file".into()),
Frame::Document(bytes) => {
if !opened {
write_document(
writer,
&doc! { "db": &entry.database, "collection": &entry.metadata.collection_name, "EOF": false, "CRC": 0i64 },
)?;
opened = true;
}
writer.write_all(&bytes).map_err(|e| e.to_string())?;
digest.update(&bytes);
}
}
}
if opened {
writer.write_all(&TERMINATOR).map_err(|e| e.to_string())?;
}
write_document(
writer,
&doc! { "db": &entry.database, "collection": &entry.metadata.collection_name, "EOF": true, "CRC": digest.finalize() as i64 },
)?;
writer.write_all(&TERMINATOR).map_err(|e| e.to_string())?;
}
writer.flush().map_err(|e| e.to_string())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn checksum_matches_go_crc64_ecma() {
assert_eq!(CRC.checksum(b"123456789"), 0x995d_c9bb_df19_39fa);
}
}
@@ -0,0 +1,302 @@
use std::collections::BTreeMap;
use std::fs::File;
use std::io::{BufReader, Read, Write};
use std::path::{Path, PathBuf};
use flate2::{read::MultiGzDecoder, write::GzEncoder, Compression};
use super::{
archive::MAX_DOCUMENT,
metadata::{self, CollectionMetadata},
PreparedCollection,
};
pub(super) fn reader(path: &Path, gzip: bool) -> Result<Box<dyn Read>, String> {
regular_file(path)?;
let reader = BufReader::new(File::open(path).map_err(|e| e.to_string())?);
if gzip {
Ok(Box::new(BufReader::with_capacity(64 * 1024, MultiGzDecoder::new(reader))))
} else {
Ok(Box::new(reader))
}
}
fn reject_link(path: &Path) -> Result<std::fs::Metadata, String> {
let metadata = std::fs::symlink_metadata(path).map_err(|e| e.to_string())?;
let is_link = metadata.file_type().is_symlink();
#[cfg(windows)]
let is_link = {
use std::os::windows::fs::MetadataExt;
is_link || metadata.file_attributes() & 0x400 != 0
};
if is_link {
return Err(format!("Dump sources cannot contain links: {}", path.display()));
}
Ok(metadata)
}
pub(super) fn regular_file(path: &Path) -> Result<(), String> {
if !reject_link(path)?.is_file() {
return Err(format!("Not a regular dump file: {}", path.display()));
}
Ok(())
}
pub(super) fn copy_stream(
reader: &mut dyn Read,
writer: &mut dyn Write,
check: &mut impl FnMut() -> Result<(), String>,
) -> Result<(), String> {
let mut buffer = [0; 64 * 1024];
loop {
check()?;
let length = reader.read(&mut buffer).map_err(|e| e.to_string())?;
if length == 0 {
break;
}
writer.write_all(&buffer[..length]).map_err(|e| e.to_string())?;
}
Ok(())
}
pub(super) fn write_file(
path: &Path,
gzip: bool,
write: impl FnOnce(&mut dyn Write) -> Result<(), String>,
) -> Result<(), String> {
let file = std::io::BufWriter::new(File::create(path).map_err(|e| e.to_string())?);
if gzip {
let mut writer = GzEncoder::new(file, Compression::default());
write(&mut writer)?;
writer.finish().map_err(|e| e.to_string())?.flush().map_err(|e| e.to_string())
} else {
let mut writer = file;
write(&mut writer)?;
writer.flush().map_err(|e| e.to_string())
}
}
#[derive(Default)]
struct Files {
data: Option<PathBuf>,
metadata: Option<PathBuf>,
}
fn collect(
path: &Path,
depth: usize,
gzip: bool,
groups: &mut BTreeMap<(String, String), Files>,
) -> Result<(), String> {
if depth > 3 || !reject_link(path)?.is_dir() {
return Err("Invalid dump directory layout".into());
}
let database =
path.file_name().and_then(|name| name.to_str()).ok_or_else(|| "Invalid database directory name".to_string())?;
let mut subdirectories = Vec::new();
let mut has_data = false;
for entry in std::fs::read_dir(path).map_err(|e| e.to_string())? {
let entry = entry.map_err(|e| e.to_string())?;
let path = entry.path();
let info = reject_link(&path)?;
if info.is_dir() {
subdirectories.push(path);
continue;
}
if !info.is_file() {
return Err("Unsupported dump directory entry".into());
}
let filename = entry.file_name().to_str().ok_or_else(|| "Invalid dump filename".to_string())?.to_string();
let (base, metadata_file, compressed) = if let Some(base) = filename.strip_suffix(".metadata.json.gz") {
(base, true, true)
} else if let Some(base) = filename.strip_suffix(".bson.gz") {
(base, false, true)
} else if let Some(base) = filename.strip_suffix(".metadata.json") {
(base, true, false)
} else if let Some(base) = filename.strip_suffix(".bson") {
(base, false, false)
} else {
continue;
};
if compressed != gzip {
return Err(format!("Gzip option does not match dump file {filename}"));
}
metadata::validate_database(database)?;
has_data = true;
let group = groups.entry((database.into(), base.into())).or_default();
let slot = if metadata_file { &mut group.metadata } else { &mut group.data };
if slot.replace(path).is_some() {
return Err("Duplicate dump filename".into());
}
if groups.len() > 100_000 {
return Err("Too many dump collections".into());
}
}
if has_data && !subdirectories.is_empty() {
return Err(
"Mixed database files and directories; select a single database directory or a dump root without oplog"
.into(),
);
}
for directory in subdirectories {
collect(&directory, depth + 1, gzip, groups)?;
}
Ok(())
}
pub(super) fn inspect(
path: &Path,
gzip: bool,
manifest: Option<&[super::MongoDumpFile]>,
) -> Result<Vec<PreparedCollection>, String> {
let mut groups = BTreeMap::new();
if let Some(manifest) = manifest {
let mut seen = std::collections::HashSet::new();
for file in manifest {
let relative = super::source::relative_path(&file.path)?;
if !seen.insert(file.path.clone()) {
return Err("Duplicate dump manifest path".into());
}
let database = relative
.parent()
.and_then(Path::file_name)
.and_then(|s| s.to_str())
.ok_or("Directory uploads require database-relative paths")?;
metadata::validate_database(database)?;
let filename = relative.file_name().unwrap().to_str().ok_or("Invalid dump filename")?;
let plain = filename.strip_suffix(".gz").unwrap_or(filename);
let (base, is_metadata) = if let Some(base) = plain.strip_suffix(".metadata.json") {
(base, true)
} else if let Some(base) = plain.strip_suffix(".bson") {
(base, false)
} else {
continue;
};
if filename.ends_with(".gz") != gzip {
return Err(format!("Gzip option does not match dump file {filename}"));
}
let group = groups.entry((database.to_string(), base.to_string())).or_insert_with(Files::default);
let slot = if is_metadata { &mut group.metadata } else { &mut group.data };
if slot.replace(path.join(relative)).is_some() {
return Err("Duplicate dump namespace".into());
}
}
} else {
collect(path, 0, gzip, &mut groups)?;
}
let mut entries = Vec::new();
let root = path;
let mut namespaces = std::collections::HashSet::new();
let mut metadata_bytes = 0usize;
for ((database, basename), files) in groups {
// Truncated official filenames may end mid-escape; collectionName is authoritative.
let fallback = metadata::unescape_collection(&basename).unwrap_or_default();
let metadata = if let Some(path) = &files.metadata {
let mut json = String::new();
reader(path, gzip)?.take(MAX_DOCUMENT as u64 + 1).read_to_string(&mut json).map_err(|e| e.to_string())?;
if json.len() > MAX_DOCUMENT {
return Err("Collection metadata exceeds 16 MiB".into());
}
metadata_bytes = metadata_bytes.saturating_add(json.len());
if metadata_bytes > 128 * 1024 * 1024 {
return Err("Dump metadata exceeds 128 MiB".into());
}
if json.trim().is_empty() {
CollectionMetadata::empty(&fallback)
} else {
CollectionMetadata::from_json(&json, &fallback)?
}
} else {
CollectionMetadata::empty(&fallback)
};
metadata.validate()?;
if !namespaces.insert((database.clone(), metadata.collection_name.clone())) {
return Err("Duplicate dump namespace".into());
}
let mut entry = PreparedCollection {
database,
metadata,
path: None,
documents: 0,
size_bytes: 0,
metadata_path: files.metadata,
};
match files.data {
Some(path) => {
if entry.metadata.is_view() {
return Err("View dump unexpectedly contains BSON data".into());
}
entry.size_bytes = if let Some(manifest) = manifest {
manifest.iter().find(|f| path == root.join(&f.path)).map(|f| f.size_bytes).unwrap_or(0)
} else {
regular_file(&path)?;
std::fs::metadata(&path).map_err(|e| e.to_string())?.len()
};
entry.path = Some(path);
}
None if !entry.metadata.is_view() => {
return Err(format!("Missing BSON file for {}", entry.metadata.collection_name))
}
None => {}
}
entries.push(entry);
}
Ok(entries)
}
pub(super) fn publish_files(
root: &Path,
database: &str,
entries: &[PreparedCollection],
gzip: bool,
mut check: impl FnMut() -> Result<(), String>,
) -> Result<(), String> {
let database_dir = root.join(database);
std::fs::create_dir_all(&database_dir).map_err(|e| e.to_string())?;
let mut names = std::collections::HashSet::new();
for entry in entries {
check()?;
let basename = metadata::escaped_collection(&entry.metadata.collection_name);
let filesystem_key = if cfg!(windows) { basename.to_lowercase() } else { basename.clone() };
if !names.insert(filesystem_key) {
return Err("Collection names collide on this filesystem; use archive format".into());
}
let suffix = if gzip { ".gz" } else { "" };
write_file(&database_dir.join(format!("{basename}.metadata.json{suffix}")), gzip, |writer| {
writer.write_all(entry.metadata.to_json()?.as_bytes()).map_err(|e| e.to_string())
})?;
if let Some(path) = &entry.path {
write_file(&database_dir.join(format!("{basename}.bson{suffix}")), gzip, |writer| {
copy_stream(&mut *reader(path, false)?, writer, &mut check)
})?;
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn regular_file_accepts_files_and_rejects_directories() {
let root = tempfile::tempdir().unwrap();
let file = root.path().join("records.bson");
std::fs::write(&file, []).unwrap();
regular_file(&file).unwrap();
assert!(regular_file(root.path()).unwrap_err().contains("Not a regular dump file"));
}
#[cfg(unix)]
#[test]
fn reject_link_rejects_file_and_directory_symlinks() {
let root = tempfile::tempdir().unwrap();
let file = root.path().join("records.bson");
std::fs::write(&file, []).unwrap();
for (name, target) in [("file-link", file.as_path()), ("directory-link", root.path())] {
let link = root.path().join(name);
std::os::unix::fs::symlink(target, &link).unwrap();
assert!(reject_link(&link).unwrap_err().contains("Dump sources cannot contain links"));
}
}
}
@@ -0,0 +1,190 @@
use base64::Engine;
use mongodb::bson::{doc, Bson, Document};
use serde::{Deserialize, Serialize};
use sha1::{Digest, Sha1};
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct CollectionMetadata {
#[serde(default)]
pub options: Document,
#[serde(default)]
pub indexes: Vec<Document>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub uuid: Option<String>,
#[serde(default)]
pub collection_name: String,
#[serde(default, rename = "type")]
pub kind: String,
}
impl CollectionMetadata {
pub fn empty(name: &str) -> Self {
Self {
options: Document::new(),
indexes: Vec::new(),
uuid: None,
collection_name: name.into(),
kind: "collection".into(),
}
}
pub fn from_json(json: &str, name: &str) -> Result<Self, String> {
let value = serde_json::from_str(json).map_err(|e| format!("Invalid metadata for {name}: {e}"))?;
let document = crate::db::mongo_driver::json_object_to_document_extended_json(&value)?;
let mut metadata: Self =
mongodb::bson::from_document(document).map_err(|e| format!("Invalid metadata for {name}: {e}"))?;
if metadata.collection_name.is_empty() {
metadata.collection_name = name.into();
}
if metadata.kind.is_empty() {
metadata.kind = if metadata.options.contains_key("viewOn") { "view" } else { "collection" }.into();
}
metadata.validate()?;
Ok(metadata)
}
pub fn to_json(&self) -> Result<String, String> {
let document = mongodb::bson::to_document(self).map_err(|e| e.to_string())?;
let json = crate::db::mongo_driver::document_to_canonical_extended_json(&document).to_string();
if json.len() > super::archive::MAX_DOCUMENT {
return Err("Collection metadata exceeds 16 MiB".into());
}
Ok(json)
}
pub fn is_view(&self) -> bool {
self.kind == "view"
}
pub fn validate(&self) -> Result<(), String> {
validate_collection(&self.collection_name)?;
let parsed_options: mongodb::options::CreateCollectionOptions =
mongodb::bson::from_document(self.options.clone())
.map_err(|e| format!("Invalid collection options for {}: {e}", self.collection_name))?;
let known_options = mongodb::bson::to_document(&parsed_options).map_err(|e| e.to_string())?;
for key in self.options.keys() {
if !known_options.contains_key(key) && key != "autoIndexId" && key != "recordIdsReplicated" {
return Err(format!("Unsupported collection option for {}: {key}", self.collection_name));
}
}
if self.kind == "timeseries"
|| self.options.contains_key("timeseries")
|| self.options.contains_key("encryptedFields")
{
return Err(format!(
"{}: time-series and encrypted collection dumps require server-specific restore support",
self.collection_name
));
}
if self.kind != "collection" && self.kind != "view" {
return Err(format!("Unsupported collection type: {}", self.kind));
}
if self.is_view() {
self.options
.get_str("viewOn")
.map_err(|_| format!("{}: view metadata is missing viewOn", self.collection_name))?;
if !self.indexes.is_empty() {
return Err("View metadata cannot contain indexes".into());
}
}
for index in &self.indexes {
index.get_document("key").map_err(|_| "Index metadata is missing key".to_string())?;
index.get_str("name").map_err(|_| "Index metadata is missing name".to_string())?;
}
Ok(())
}
pub fn create_command(&self, restore_options: bool) -> Document {
let mut command = doc! { "create": &self.collection_name };
if restore_options || self.is_view() {
let mut options = self.options.clone();
options.remove("uuid");
options.remove("recordIdsReplicated");
options.remove("autoIndexId");
// Keep create as the first command key and never accept it from metadata.
options.remove("create");
command.extend(options);
}
command
}
pub fn restore_indexes(&self) -> Vec<Document> {
self.indexes
.iter()
.filter(|index| {
let key = index.get_document("key").expect("validated index");
!(key.len() == 1
&& matches!(key.get("_id"), Some(Bson::Int32(1) | Bson::Int64(1)) | Some(Bson::Double(1.0))))
})
.map(|index| {
let mut index = index.clone();
index.remove("ns");
index.remove("v");
index.remove("background");
index
})
.collect()
}
}
pub fn validate_database(name: &str) -> Result<(), String> {
if name.is_empty() || name.len() > 63 || name.chars().any(|c| c.is_control() || " /\\.\"$*<>:|?".contains(c)) {
return Err("Invalid MongoDB database name".into());
}
if matches!(name, "admin" | "config" | "local") {
return Err(
"Internal MongoDB databases and user/role migration are not supported by database dump/restore".into()
);
}
Ok(())
}
pub fn validate_collection(name: &str) -> Result<(), String> {
if name.is_empty()
|| name.contains('\0')
|| name.contains('$')
|| (name.starts_with("system.") && name != "system.js")
{
return Err(format!("Unsupported MongoDB collection name: {name}"));
}
Ok(())
}
pub fn escaped_collection(name: &str) -> String {
const ESCAPE: &percent_encoding::AsciiSet =
&percent_encoding::NON_ALPHANUMERIC.remove(b'-').remove(b'_').remove(b'.').remove(b'~');
let mut escaped = percent_encoding::utf8_percent_encode(name, ESCAPE).to_string().replace("%20", "+");
let stem = escaped.split('.').next().unwrap_or_default().to_ascii_uppercase();
if matches!(stem.as_str(), "CON" | "PRN" | "AUX" | "NUL")
|| (stem.len() == 4
&& (stem.starts_with("COM") || stem.starts_with("LPT"))
&& matches!(stem.as_bytes()[3], b'1'..=b'9'))
{
escaped = format!("%{:02X}{}", escaped.as_bytes()[0], &escaped[1..]);
}
if escaped.len() > 238 {
let hash = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(Sha1::digest(name.as_bytes()));
escaped = format!("{}%24{hash}", &escaped[..208]);
}
escaped
}
pub fn unescape_collection(name: &str) -> Result<String, String> {
let bytes = name.as_bytes();
let mut position = 0;
while position < bytes.len() {
if bytes[position] == b'%' {
if position + 2 >= bytes.len() || !bytes[position + 1..position + 3].iter().all(u8::is_ascii_hexdigit) {
return Err("Invalid escaped collection filename".into());
}
position += 3;
} else {
position += 1;
}
}
percent_encoding::percent_decode_str(&name.replace('+', " "))
.decode_utf8()
.map(|v| v.into_owned())
.map_err(|e| e.to_string())
}
+247
View File
@@ -0,0 +1,247 @@
use std::collections::HashMap;
use std::io::Read;
use std::path::{Component, Path, PathBuf};
use std::sync::{Arc, Mutex, OnceLock};
use std::time::{Duration, Instant, SystemTime};
use serde::{Deserialize, Serialize};
use super::{
archive, catalog, directory, ordered_entries, select_entries, MongoDatabaseRestoreRequest, MongoDumpFormat,
MongoRestoreSourcePreview, MongoRestoreSourceRequest, PreparedCollection,
};
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MongoDumpFile {
pub path: String,
pub size_bytes: u64,
}
pub(super) struct CatalogSource {
_owner: Option<tempfile::TempDir>,
request: MongoRestoreSourceRequest,
pub entries: Vec<PreparedCollection>,
identities: HashMap<PathBuf, (u64, Option<SystemTime>)>,
manifest: Option<Vec<MongoDumpFile>>,
created: Instant,
}
static SOURCES: OnceLock<Mutex<HashMap<String, Arc<CatalogSource>>>> = OnceLock::new();
fn sources() -> &'static Mutex<HashMap<String, Arc<CatalogSource>>> {
SOURCES.get_or_init(Default::default)
}
pub(super) fn get(reference: &str) -> Result<Arc<CatalogSource>, String> {
let source = sources()
.lock()
.unwrap_or_else(|e| e.into_inner())
.get(reference)
.cloned()
.ok_or("Restore source is no longer available")?;
if source.created.elapsed() >= Duration::from_secs(86400) {
return Err("Restore source has expired".into());
}
Ok(source)
}
pub(super) fn release(reference: &str) -> bool {
sources().lock().unwrap_or_else(|e| e.into_inner()).remove(reference).is_some()
}
pub(super) fn relative_path(value: &str) -> Result<&Path, String> {
let path = Path::new(value);
if value.is_empty()
|| value.contains(['\\', ':', '\0'])
|| path.components().count() > 4
|| path.components().any(|c| !matches!(c, Component::Normal(_)))
{
return Err("Invalid dump upload path".into());
}
Ok(path)
}
fn identity(path: &Path) -> Result<(u64, Option<SystemTime>), String> {
directory::regular_file(path)?;
let info = std::fs::metadata(path).map_err(|e| e.to_string())?;
Ok((info.len(), info.modified().ok()))
}
pub(super) async fn prepare(
request: MongoRestoreSourceRequest,
owner: Option<tempfile::TempDir>,
manifest: Option<Vec<MongoDumpFile>>,
) -> Result<MongoRestoreSourcePreview, String> {
tokio::task::spawn_blocking(move || {
if manifest.as_ref().is_some_and(|m| m.len() > 100_000) {
return Err("Too many dump files".into());
}
let path = Path::new(&request.path);
let entries = match request.format {
MongoDumpFormat::Directory => directory::inspect(path, request.gzip, manifest.as_deref())?,
MongoDumpFormat::Archive => archive::inspect(&mut *directory::reader(path, request.gzip)?, || Ok(()))?,
};
ordered_entries(&entries)?;
let mut catalog = catalog(&entries, false);
let mut identities = HashMap::new();
for (entry, preview) in entries.iter().zip(catalog.collections.iter_mut()) {
for file in entry.path.iter().chain(entry.metadata_path.iter()) {
if manifest.is_none() || entry.metadata_path.as_ref() == Some(file) {
identities.insert(file.clone(), identity(file)?);
}
if let Ok(relative) = file.strip_prefix(path) {
preview.source_files.push(relative.to_string_lossy().replace('\\', "/"));
}
}
}
if request.format == MongoDumpFormat::Archive {
identities.insert(path.to_owned(), identity(path)?);
}
let source =
Arc::new(CatalogSource { _owner: owner, request, entries, identities, manifest, created: Instant::now() });
let mut sources = sources().lock().unwrap_or_else(|e| e.into_inner());
sources.retain(|_, s| s.created.elapsed() < Duration::from_secs(86400));
if sources.len() >= 32 {
return Err("Too many prepared restores; close or release an existing source".into());
}
let source_ref = uuid::Uuid::new_v4().to_string();
sources.insert(source_ref.clone(), source);
Ok(MongoRestoreSourcePreview { source_ref, catalog })
})
.await
.map_err(|e| e.to_string())?
}
pub async fn prepare_owned_mongodb_restore_source(
request: MongoRestoreSourceRequest,
owner: tempfile::TempDir,
) -> Result<MongoRestoreSourcePreview, String> {
prepare(request, Some(owner), None).await
}
pub async fn prepare_mongodb_directory_catalog(
owner: tempfile::TempDir,
manifest: Vec<MongoDumpFile>,
gzip: bool,
) -> Result<MongoRestoreSourcePreview, String> {
let request = MongoRestoreSourceRequest {
path: owner.path().to_string_lossy().into_owned(),
format: MongoDumpFormat::Directory,
gzip,
};
prepare(request, Some(owner), Some(manifest)).await
}
fn same_metadata(a: &PreparedCollection, b: &PreparedCollection) -> Result<bool, String> {
Ok(a.database == b.database && a.metadata.to_json()? == b.metadata.to_json()? && a.size_bytes == b.size_bytes)
}
/// The server, not the upload payload, determines which files the confirmed selection requires.
pub fn mongodb_restore_upload_files(request: &MongoDatabaseRestoreRequest) -> Result<Vec<MongoDumpFile>, String> {
let source = get(&request.source_ref)?;
let manifest = source.manifest.as_ref().ok_or("Source does not require a directory upload")?;
let paths = selected(&source, request)?
.into_iter()
.flat_map(|e| e.path.into_iter().chain(e.metadata_path))
.collect::<Vec<_>>();
Ok(manifest.iter().filter(|f| paths.contains(&Path::new(&source.request.path).join(&f.path))).cloned().collect())
}
/// Bind the complete selected upload to the reviewed catalog.
pub async fn attach_mongodb_restore_upload(
request: &MongoDatabaseRestoreRequest,
owner: tempfile::TempDir,
) -> Result<MongoRestoreSourcePreview, String> {
let original = get(&request.source_ref)?;
if original.manifest.is_none() {
return Err("Source does not require a directory upload".into());
}
let selected = selected(&original, request)?;
let next = prepare_owned_mongodb_restore_source(
MongoRestoreSourceRequest {
path: owner.path().to_string_lossy().into_owned(),
format: MongoDumpFormat::Directory,
gzip: original.request.gzip,
},
owner,
)
.await?;
let uploaded = get(&next.source_ref)?;
let matches =
(|| -> Result<bool, String> {
if selected.len() != uploaded.entries.len() {
return Ok(false);
}
for entry in &selected {
let Some(found) = uploaded.entries.iter().find(|e| {
e.database == entry.database && e.metadata.collection_name == entry.metadata.collection_name
}) else {
return Ok(false);
};
if !same_metadata(entry, found)? {
return Ok(false);
}
let expected = entry
.path
.iter()
.chain(entry.metadata_path.iter())
.map(|p| p.strip_prefix(&original.request.path).unwrap().to_owned())
.collect::<Vec<_>>();
let actual = found
.path
.iter()
.chain(found.metadata_path.iter())
.map(|p| p.strip_prefix(&uploaded.request.path).unwrap().to_owned())
.collect::<Vec<_>>();
if expected != actual {
return Ok(false);
}
}
Ok(true)
})();
if matches != Ok(true) {
release(&next.source_ref);
return Err("Uploaded files differ from the confirmed catalog; read the backup again".into());
}
Ok(next)
}
fn selected(source: &CatalogSource, request: &MongoDatabaseRestoreRequest) -> Result<Vec<PreparedCollection>, String> {
let entries: Vec<_> = source.entries.iter().filter(|e| e.database == request.source_database).cloned().collect();
select_entries(&entries, &request.collections)
}
fn unchanged(source: &CatalogSource, path: &Path) -> Result<(), String> {
if source.identities.get(path) != Some(&identity(path)?) {
return Err("Backup source changed; read the backup again".into());
}
Ok(())
}
impl CatalogSource {
pub(super) fn format(&self) -> MongoDumpFormat {
self.request.format
}
pub(super) fn check_ready(&self, entries: &[PreparedCollection]) -> Result<(), String> {
if self.manifest.is_some() {
return Err("Selected directory data has not been uploaded".into());
}
if self.request.format == MongoDumpFormat::Archive {
unchanged(self, Path::new(&self.request.path))?;
} else {
for entry in entries {
for path in entry.path.iter().chain(entry.metadata_path.iter()) {
unchanged(self, path)?;
}
}
}
Ok(())
}
pub(super) fn reader(&self, path: Option<&Path>) -> Result<Box<dyn Read>, String> {
let path = path.unwrap_or_else(|| Path::new(&self.request.path));
unchanged(self, path)?;
directory::reader(path, self.request.gzip)
}
}
+377
View File
@@ -0,0 +1,377 @@
use std::collections::HashSet;
use std::sync::{
atomic::{AtomicBool, AtomicU64, Ordering},
Arc,
};
use std::time::{Duration, Instant};
use mongodb::bson::{Document, RawDocumentBuf};
use tokio::sync::mpsc;
use super::{
archive, cancelled, source::CatalogSource, CancelFuture, MongoDatabaseDumpProgress, MongoDatabaseRestoreRequest,
MongoDumpFormat, PreparedCollection,
};
use crate::{connection::AppState, db::mongo_driver::insert_bson_documents};
const BATCH_BYTES: usize = 8 * 1024 * 1024;
enum Event {
Batch { index: usize, documents: Vec<RawDocumentBuf> },
End(usize),
}
struct Batcher {
tx: mpsc::Sender<Event>,
index: usize,
documents: Vec<RawDocumentBuf>,
bytes: usize,
limit: usize,
objcheck: bool,
}
impl Batcher {
fn flush(&mut self) -> Result<(), String> {
if self.documents.is_empty() {
return Ok(());
}
let documents = std::mem::take(&mut self.documents);
self.bytes = 0;
self.tx
.blocking_send(Event::Batch { index: self.index, documents })
.map_err(|_| "MongoDB dump/restore cancelled".into())
}
fn push(&mut self, index: usize, bytes: Vec<u8>) -> Result<(), String> {
if index != self.index || self.bytes.saturating_add(bytes.len()) > BATCH_BYTES {
self.flush()?;
}
self.index = index;
let document = checked_document(bytes, self.objcheck)?;
self.bytes += document.as_bytes().len();
self.documents.push(document);
if self.documents.len() >= self.limit {
self.flush()?;
}
Ok(())
}
fn end(&mut self, index: usize) -> Result<(), String> {
self.flush()?;
self.tx.blocking_send(Event::End(index)).map_err(|_| "MongoDB dump/restore cancelled".into())
}
}
// Like mongorestore's objcheck, deep parsing is optional and occurs in the stream,
// immediately before a document is queued for insertion. No whole-backup pass.
fn checked_document(bytes: Vec<u8>, objcheck: bool) -> Result<RawDocumentBuf, String> {
let raw = RawDocumentBuf::from_bytes(bytes).map_err(|e| format!("Invalid BSON document: {e}"))?;
if objcheck {
mongodb::bson::from_slice::<Document>(raw.as_bytes()).map_err(|e| format!("Invalid BSON document: {e}"))?;
}
Ok(raw)
}
fn produce(
source: Arc<CatalogSource>,
entries: Vec<PreparedCollection>,
flag: Arc<AtomicBool>,
bytes_read: Arc<AtomicU64>,
mut batcher: Batcher,
) -> Result<(), String> {
let result = (|| {
source.check_ready(&entries)?;
if source.format() == MongoDumpFormat::Archive {
let indices = source
.entries
.iter()
.map(|original| {
entries.iter().position(|e| {
e.database == original.database
&& e.metadata.collection_name == original.metadata.collection_name
&& !e.metadata.is_view()
})
})
.collect::<Vec<_>>();
archive::stream(
&mut *source.reader(None)?,
&source.entries,
|| cancelled(&flag),
|event| {
match event {
archive::StreamEvent::Document(i, bytes) => {
bytes_read.fetch_add(bytes.len() as u64, Ordering::Relaxed);
if let Some(index) = indices[i] {
batcher.push(index, bytes)?;
}
}
archive::StreamEvent::End(i) => {
if let Some(index) = indices[i] {
batcher.end(index)?;
}
}
}
Ok(())
},
)?;
} else {
for (index, entry) in entries.iter().enumerate().filter(|(_, e)| !e.metadata.is_view()) {
cancelled(&flag)?;
let path = entry.path.as_deref().ok_or("Missing collection BSON file")?;
let mut reader = source.reader(Some(path))?;
loop {
cancelled(&flag)?;
match archive::read_raw_frame(&mut *reader)? {
archive::Frame::End => break,
archive::Frame::Terminator => return Err("Unexpected terminator in collection BSON".into()),
archive::Frame::Document(bytes) => {
bytes_read.fetch_add(bytes.len() as u64, Ordering::Relaxed);
batcher.push(index, bytes)?;
}
}
}
batcher.end(index)?;
}
}
Ok(())
})();
// Deliver valid earlier documents even if a later frame is corrupt, matching
// the streaming import contract and reporting partial writes accurately.
let flushed = batcher.flush();
result.and(flushed)
}
pub(super) async fn initialize_collection(
client: &mongodb::Client,
request: &MongoDatabaseRestoreRequest,
entry: &PreparedCollection,
exists: bool,
) -> Result<(), String> {
let db = client.database(&request.database);
if request.drop_existing && exists {
db.collection::<Document>(&entry.metadata.collection_name).drop().await.map_err(|e| e.to_string())?;
}
if !exists || request.drop_existing {
db.run_command(entry.metadata.create_command(request.restore_options)).await.map_err(|e| e.to_string())?;
}
Ok(())
}
pub(super) async fn writable(state: &AppState, id: &str) -> Result<(), String> {
if let Some(name) = crate::query::connection_readonly_name(state, id).await {
return Err(format!("Read-only connection '{name}': restore blocked"));
}
Ok(())
}
#[allow(clippy::too_many_arguments)]
pub(super) async fn restore_data(
state: &AppState,
client: &mongodb::Client,
request: &MongoDatabaseRestoreRequest,
source: Arc<CatalogSource>,
entries: &[PreparedCollection],
existing: &HashSet<String>,
progress: &mut MongoDatabaseDumpProgress,
started: Instant,
is_cancelled: &mut impl FnMut(&str) -> CancelFuture,
on_progress: &mut impl FnMut(MongoDatabaseDumpProgress),
) -> Result<(), String> {
struct StopOnDrop(Arc<AtomicBool>);
impl Drop for StopOnDrop {
fn drop(&mut self) {
self.0.store(true, Ordering::Relaxed);
}
}
let flag = Arc::new(AtomicBool::new(false));
let _guard = StopOnDrop(flag.clone());
let (tx, mut rx) = mpsc::channel(2);
let batcher = Batcher {
tx,
index: 0,
documents: Vec::new(),
bytes: 0,
limit: request.batch_size.max(1),
objcheck: request.objcheck,
};
let producer_entries = entries.to_vec();
let signal = flag.clone();
let bytes_read = Arc::new(AtomicU64::new(0));
let reader_progress = bytes_read.clone();
let producer =
tokio::task::spawn_blocking(move || produce(source, producer_entries, signal, reader_progress, batcher));
progress.phase = "data".into();
progress.emit(started, on_progress);
let result = async {
let mut initialized = HashSet::new();
let mut tick = tokio::time::interval(Duration::from_millis(100));
loop {
let event = tokio::select! {
event = rx.recv() => match event { Some(event) => event, None => break },
_ = tick.tick() => {
if is_cancelled(&request.task_id).await { return Err("MongoDB dump/restore cancelled".into()); }
progress.bytes_processed = bytes_read.load(Ordering::Relaxed);
progress.emit(started, on_progress);
continue;
},
};
if is_cancelled(&request.task_id).await {
return Err("MongoDB dump/restore cancelled".into());
}
writable(state, &request.connection_id).await?;
let index = match &event {
Event::Batch { index, .. } | Event::End(index) => *index,
};
let entry = &entries[index];
progress.phase = "data".into();
progress.collection = Some(entry.metadata.collection_name.clone());
progress.emit(started, on_progress);
if initialized.insert(index) {
initialize_collection(client, request, entry, existing.contains(&entry.metadata.collection_name))
.await?;
}
match event {
Event::Batch { documents, .. } => {
progress.documents_read += documents.len() as u64;
progress.bytes_processed = bytes_read.load(Ordering::Relaxed);
if request.objcheck {
progress.documents_validated += documents.len() as u64;
}
let outcome =
insert_bson_documents(client, &request.database, &entry.metadata.collection_name, documents)
.await
.map_err(|e| e.message)?;
progress.documents_written += outcome.inserted;
progress.documents_failed += outcome.errors.len() as u64;
progress.emit(started, on_progress);
if let Some(error) = outcome.errors.first().filter(|_| request.stop_on_error) {
return Err(error.message.clone());
}
}
Event::End(_) => {
progress.collections_done += 1;
progress.emit(started, on_progress);
}
}
}
Ok(())
}
.await;
flag.store(true, Ordering::Relaxed);
drop(rx);
let produced = producer.await.map_err(|e| e.to_string()).and_then(|r| r);
result.and(produced)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn objcheck_uses_official_bson_parser_only_when_enabled() {
// Valid outer framing but invalid boolean payload.
let bytes = vec![9, 0, 0, 0, 8, b'x', 0, 2, 0];
assert!(checked_document(bytes.clone(), false).is_ok());
assert!(checked_document(bytes, true).is_err());
assert!(checked_document(vec![5, 0, 0, 0, 1], false).is_err());
}
#[test]
fn raw_bson_preserves_duplicate_keys_and_exact_bytes() {
let bytes = vec![19, 0, 0, 0, 16, b'x', 0, 1, 0, 0, 0, 16, b'x', 0, 2, 0, 0, 0, 0];
for objcheck in [false, true] {
let raw = checked_document(bytes.clone(), objcheck).unwrap();
assert_eq!(raw.as_bytes(), bytes);
assert_eq!(mongodb::bson::to_vec(&raw).unwrap(), bytes);
}
}
#[tokio::test]
async fn reader_delivers_earlier_batches_before_reporting_corruption() {
use crate::mongodb_dump::{
prepare_mongodb_restore_source, release_mongodb_restore_source, MongoRestoreSourceRequest,
};
let dir = tempfile::tempdir().unwrap();
let db = dir.path().join("source");
std::fs::create_dir(&db).unwrap();
let mut bytes = mongodb::bson::to_vec(&mongodb::bson::doc! { "_id": 1 }).unwrap();
bytes.extend_from_slice(b"invalid BSON");
std::fs::write(db.join("records.bson"), bytes).unwrap();
let preview = prepare_mongodb_restore_source(MongoRestoreSourceRequest {
path: db.to_string_lossy().into_owned(),
format: MongoDumpFormat::Directory,
gzip: false,
})
.await
.unwrap();
let source = super::super::source::get(&preview.source_ref).unwrap();
let entries = source.entries.clone();
let (tx, mut rx) = mpsc::channel(1);
let batcher = Batcher { tx, index: 0, documents: Vec::new(), bytes: 0, limit: 1, objcheck: false };
let worker = tokio::task::spawn_blocking(move || {
produce(source, entries, Arc::new(AtomicBool::new(false)), Arc::new(AtomicU64::new(0)), batcher)
});
assert!(matches!(rx.recv().await, Some(Event::Batch { documents, .. }) if documents.len() == 1));
assert!(worker.await.unwrap().unwrap_err().contains("BSON"));
assert!(rx.recv().await.is_none());
release_mongodb_restore_source(&preview.source_ref);
}
#[test]
fn batches_are_limited_by_bytes_as_well_as_document_count() {
let bytes = mongodb::bson::to_vec(&mongodb::bson::doc! { "payload": "x".repeat(5 * 1024 * 1024) }).unwrap();
let (tx, mut rx) = mpsc::channel(2);
let mut batcher = Batcher { tx, index: 0, documents: Vec::new(), bytes: 0, limit: 500, objcheck: false };
batcher.push(0, bytes.clone()).unwrap();
batcher.push(0, bytes).unwrap();
batcher.flush().unwrap();
for _ in 0..2 {
assert!(matches!(rx.try_recv().unwrap(), Event::Batch { documents, .. } if documents.len() == 1));
}
}
#[test]
#[ignore = "opt-in compressed BSON parser throughput smoke test"]
fn compressed_bson_parser_throughput() {
use flate2::{read::MultiGzDecoder, write::GzEncoder, Compression};
use std::io::{BufReader, Cursor, Write};
let mut document = Document::new();
document.insert("_id", 1i64);
for i in 0..64 {
document.insert(format!("field_{i}"), "value".repeat(8));
}
let encoded = mongodb::bson::to_vec(&document).unwrap();
let documents = 8192;
let mut encoder = GzEncoder::new(Vec::new(), Compression::fast());
for _ in 0..documents {
encoder.write_all(&encoded).unwrap();
}
let compressed = encoder.finish().unwrap();
for (mode, objcheck, passes) in
[("raw-stream", false, 1), ("objcheck-stream", true, 1), ("old-double-parse", true, 2)]
{
let started = Instant::now();
let mut count = 0;
for _ in 0..passes {
let mut input = BufReader::with_capacity(64 * 1024, MultiGzDecoder::new(Cursor::new(&compressed)));
loop {
match archive::read_raw_frame(&mut input).unwrap() {
archive::Frame::Document(bytes) => {
checked_document(bytes, objcheck).unwrap();
count += 1;
}
archive::Frame::End => break,
_ => panic!("unexpected terminator"),
}
}
}
assert_eq!(count, documents * passes);
eprintln!(
"{mode}: logical_mib={:.2}, elapsed_ms={}, logical_mib_s={:.2}",
(documents * encoded.len()) as f64 / 1048576.0,
started.elapsed().as_millis(),
(documents * encoded.len()) as f64 / 1048576.0 / started.elapsed().as_secs_f64()
);
}
}
}
+344 -3
View File
@@ -7,6 +7,7 @@ use std::path::{Path, PathBuf};
use std::time::Instant;
use chrono::{DateTime as ChronoDateTime, NaiveDate, Utc};
use flate2::{read::MultiGzDecoder, write::GzEncoder, Compression};
use mongodb::bson::{oid::ObjectId, Bson, DateTime, Decimal128, Document};
use serde::{Deserialize, Serialize};
@@ -26,6 +27,7 @@ pub const MAX_BATCH_SIZE: usize = 5000;
/// Rows sampled for CSV type inference, independent of the preview window so that the
/// preview and the import always agree on column types.
pub const TYPE_SAMPLE_ROWS: usize = 1000;
const BSON_MAX_DOCUMENT_BYTES: usize = 16 * 1024 * 1024;
const TYPE_STRING: u8 = 1 << 0;
const TYPE_BOOLEAN: u8 = 1 << 1;
@@ -45,6 +47,7 @@ pub enum MongoImportFormat {
Csv,
Json,
Ndjson,
Bson,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
@@ -341,14 +344,16 @@ struct CsvParseConfig {
pub fn format_from_path(path: &str) -> Result<MongoImportFormat, String> {
let lower = path.to_lowercase();
if lower.ends_with(".csv") || lower.ends_with(".tsv") || lower.ends_with(".txt") {
if lower.ends_with(".bson") || lower.ends_with(".bson.gz") {
Ok(MongoImportFormat::Bson)
} else if lower.ends_with(".csv") || lower.ends_with(".tsv") || lower.ends_with(".txt") {
Ok(MongoImportFormat::Csv)
} else if lower.ends_with(".ndjson") || lower.ends_with(".jsonl") {
Ok(MongoImportFormat::Ndjson)
} else if lower.ends_with(".json") {
Ok(MongoImportFormat::Json)
} else {
Err("Unsupported MongoDB import file type; use .csv, .json, or .ndjson".to_string())
Err("Unsupported MongoDB import file type; use .bson, .bson.gz, .csv, .json, or .ndjson".to_string())
}
}
@@ -1348,6 +1353,72 @@ fn file_size(path: &str) -> u64 {
std::fs::metadata(path).map(|metadata| metadata.len()).unwrap_or(0)
}
fn open_bson_dump_reader(path: &str) -> Result<Box<dyn Read>, MongoImportIssue> {
let file = File::open(path).map_err(|error| MongoImportIssue::new("FILE_UNREADABLE", error.to_string()))?;
let reader = BufReader::new(file);
if path.to_lowercase().ends_with(".gz") {
Ok(Box::new(MultiGzDecoder::new(reader)))
} else {
Ok(Box::new(reader))
}
}
fn read_bson_dump_document<R: Read + ?Sized>(reader: &mut R, row: u64) -> Result<Option<Document>, MongoImportIssue> {
let mut length_bytes = [0u8; 4];
loop {
match reader.read(&mut length_bytes[..1]) {
Ok(0) => return Ok(None),
Ok(_) => break,
Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue,
Err(error) => return Err(MongoImportIssue::new("FILE_UNREADABLE", error.to_string()).with_row(row)),
}
}
reader.read_exact(&mut length_bytes[1..]).map_err(|error| {
MongoImportIssue::new("BSON_STRUCTURE", format!("Truncated BSON document length: {error}")).with_row(row)
})?;
let document_length = i32::from_le_bytes(length_bytes);
if document_length < 5 || document_length as usize > BSON_MAX_DOCUMENT_BYTES {
return Err(MongoImportIssue::new(
"BSON_STRUCTURE",
format!("Invalid BSON document length {document_length}; expected 5..={BSON_MAX_DOCUMENT_BYTES} bytes"),
)
.with_row(row));
}
let mut bytes = vec![0u8; document_length as usize];
bytes[..4].copy_from_slice(&length_bytes);
reader.read_exact(&mut bytes[4..]).map_err(|error| {
MongoImportIssue::new(
"BSON_STRUCTURE",
format!("Truncated BSON document; expected {document_length} bytes: {error}"),
)
.with_row(row)
})?;
let document = mongodb::bson::from_slice::<Document>(&bytes).map_err(|error| {
MongoImportIssue::new("BSON_STRUCTURE", format!("Invalid BSON document: {error}")).with_row(row)
})?;
Ok(Some(document))
}
fn parse_bson_preview(
path: &str,
preview_limit: usize,
) -> Result<(Vec<ParsedMongoDocument>, u64, bool), MongoImportIssue> {
let mut reader = open_bson_dump_reader(path)?;
let mut documents = Vec::with_capacity(preview_limit.min(DEFAULT_PREVIEW_LIMIT));
let mut row = 1u64;
while documents.len() <= preview_limit {
let Some(document) = read_bson_dump_document(&mut *reader, row)? else {
return Ok((documents, row - 1, true));
};
documents.push(parsed_document(row, document, true));
row += 1;
}
documents.truncate(preview_limit);
Ok((documents, preview_limit as u64, false))
}
pub fn preview_mongodb_import_file(
request: &MongoImportPreviewRequest,
) -> Result<MongoImportPreview, MongoImportIssue> {
@@ -1406,6 +1477,25 @@ pub fn preview_mongodb_import_file(
estimated_rows_exact,
})
}
MongoImportFormat::Bson => {
let (documents, estimated_rows, estimated_rows_exact) =
parse_bson_preview(&request.file_path, preview_limit)?;
Ok(MongoImportPreview {
source_ref: request.source_ref.clone(),
format: request.format,
detected_encoding: None,
file_name: file_name(&request.file_path),
file_path: request.file_path.clone(),
size_bytes: file_size(&request.file_path),
columns: columns_from_documents(&documents),
row_numbers: documents.iter().map(|document| document.row).collect(),
rows: documents.into_iter().map(|document| document.extended_json).collect(),
warnings: Vec::new(),
errors: Vec::new(),
estimated_rows: Some(estimated_rows),
estimated_rows_exact,
})
}
}
}
@@ -1419,6 +1509,7 @@ pub fn preview_mongodb_import_bytes(
MongoImportFormat::Csv => "csv",
MongoImportFormat::Json => "json",
MongoImportFormat::Ndjson => "ndjson",
MongoImportFormat::Bson => "bson",
};
let dir = std::env::temp_dir().join(format!("dbx-mongo-import-preview-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).map_err(|error| MongoImportIssue::new("FILE_UNREADABLE", error.to_string()))?;
@@ -1517,6 +1608,20 @@ where
Ok(())
}
fn stream_bson_file<F>(path: &str, mut on_document: F) -> Result<(), MongoImportIssue>
where
F: FnMut(Result<ParsedMongoDocument, MongoImportIssue>) -> Result<(), MongoImportIssue>,
{
let mut reader = open_bson_dump_reader(path)?;
let mut row = 1u64;
loop {
// A corrupt frame has no reliable next-document boundary, even when skipping row errors.
let Some(document) = read_bson_dump_document(&mut *reader, row)? else { return Ok(()) };
on_document(Ok(parsed_document(row, document, false)))?;
row += 1;
}
}
pub fn for_each_mongodb_import_document<F>(
path: &str,
format: MongoImportFormat,
@@ -1531,6 +1636,7 @@ where
MongoImportFormat::Csv => stream_csv_file(path, options, on_document),
MongoImportFormat::Json => stream_json_file(path, options, false, on_document),
MongoImportFormat::Ndjson => stream_json_file(path, options, true, on_document),
MongoImportFormat::Bson => stream_bson_file(path, on_document),
}
}
@@ -1929,6 +2035,7 @@ pub fn mongodb_export_client_session_id(export_id: &str) -> String {
pub enum MongoExportFormat {
Csv,
Ndjson,
Bson,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
@@ -1958,6 +2065,8 @@ pub struct MongoExportRequest {
pub format: MongoExportFormat,
#[serde(default = "default_true")]
pub include_header: bool,
#[serde(default)]
pub gzip: bool,
pub file_path: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub execution_id: Option<String>,
@@ -1992,7 +2101,7 @@ pub struct MongoExportSummary {
fn temp_export_path(target: &Path) -> PathBuf {
let name = target.file_name().and_then(|name| name.to_str()).unwrap_or("export");
target.with_file_name(format!(".{name}.dbx-export.tmp"))
target.with_file_name(format!(".{name}.{}.dbx-export.tmp", uuid::Uuid::new_v4()))
}
fn cleanup_temp(path: &Path) {
@@ -2214,6 +2323,9 @@ where
{
let started_at = Instant::now();
on_progress(export_progress(&request.export_id, MongoExportStatus::Running, 0, 0, None, None, started_at));
if request.gzip && request.format != MongoExportFormat::Bson {
return Err("Gzip compression is only supported for BSON dump exports".to_string());
}
if is_cancelled(&request.export_id).await {
on_progress(export_progress(
&request.export_id,
@@ -2250,6 +2362,9 @@ where
MongoExportFormat::Csv => {
export_csv(state, request, &temp, total_documents, started_at, &mut is_cancelled, &mut on_progress).await
}
MongoExportFormat::Bson => {
export_bson(state, request, &temp, total_documents, started_at, &mut is_cancelled, &mut on_progress).await
}
};
match result {
@@ -2685,6 +2800,120 @@ where
Ok((documents_read, bytes_written))
}
enum BsonExportWriter {
Plain(BufWriter<File>),
Gzip(Box<GzEncoder<BufWriter<File>>>),
}
impl BsonExportWriter {
fn finish(self) -> Result<(), String> {
match self {
Self::Plain(mut writer) => writer.flush().map_err(|error| error.to_string()),
Self::Gzip(writer) => {
let mut writer = writer.finish().map_err(|error| error.to_string())?;
writer.flush().map_err(|error| error.to_string())
}
}
}
}
impl Write for BsonExportWriter {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
match self {
Self::Plain(writer) => writer.write(buf),
Self::Gzip(writer) => writer.write(buf),
}
}
fn flush(&mut self) -> std::io::Result<()> {
match self {
Self::Plain(writer) => writer.flush(),
Self::Gzip(writer) => writer.flush(),
}
}
}
async fn export_bson<C, F>(
state: &AppState,
request: &MongoExportRequest,
temp: &Path,
total_documents: Option<u64>,
started_at: Instant,
is_cancelled: &mut C,
on_progress: &mut F,
) -> Result<(u64, u64), String>
where
C: FnMut(&str) -> std::pin::Pin<Box<dyn std::future::Future<Output = bool> + Send>>,
F: FnMut(MongoExportProgress),
{
if is_cancelled(&request.export_id).await {
return Err("Export cancelled".to_string());
}
// BSON dumps must carry native BSON types, so they are written straight from the native
// driver's documents; the legacy agent's canonical Extended JSON round trip does not apply.
let client = match state.pool_handle(&request.connection_id).await {
Some(PoolKind::MongoDb(client)) => client,
_ => return Err("BSON dump export requires the native MongoDB driver".into()),
};
let file = File::create(temp).map_err(|error| error.to_string())?;
let buffered = BufWriter::new(file);
let mut writer = if request.gzip {
BsonExportWriter::Gzip(Box::new(GzEncoder::new(buffered, Compression::default())))
} else {
BsonExportWriter::Plain(buffered)
};
let mut documents_read = 0u64;
let mut bytes_written = 0u64;
let export = for_each_find_document(
&client,
&request.database,
&request.collection,
request.filter.as_deref(),
request.projection.as_deref(),
request.sort.as_deref(),
request.collation.as_deref(),
DEFAULT_EXPORT_BATCH_SIZE,
|document| {
let bytes = mongodb::bson::to_vec(&document).map_err(|error| error.to_string())?;
writer.write_all(&bytes).map_err(|error| error.to_string())?;
bytes_written += bytes.len() as u64;
documents_read += 1;
if documents_read == 1 || documents_read.is_multiple_of(500) {
on_progress(export_progress(
&request.export_id,
MongoExportStatus::Running,
documents_read,
bytes_written,
total_documents,
None,
started_at,
));
}
Ok(())
},
);
{
tokio::pin!(export);
let mut poll_cancel = tokio::time::interval(std::time::Duration::from_millis(100));
loop {
tokio::select! {
result = &mut export => { result?; break; }
_ = poll_cancel.tick() => {
if is_cancelled(&request.export_id).await {
return Err("Export cancelled".to_string());
}
}
}
}
}
writer.finish()?;
if is_cancelled(&request.export_id).await {
return Err("Export cancelled".to_string());
}
Ok((documents_read, bytes_written))
}
const CSV_FIELD_DISCOVERY_DOCS: usize = 10_000;
async fn export_csv<C, F>(
@@ -2847,6 +3076,112 @@ mod tests {
Ok(output)
}
fn bson_dump_bytes(documents: &[Document]) -> Vec<u8> {
let mut output = Vec::new();
for document in documents {
output.extend_from_slice(&mongodb::bson::to_vec(document).unwrap());
}
output
}
#[test]
fn bson_export_writer_finishes_plain_and_gzip_dumps() {
let root = tempfile::tempdir().unwrap();
let documents = vec![doc! { "_id": 1, "name": "first" }, doc! { "_id": 2, "value": 42i64 }];
for expected in [Vec::new(), bson_dump_bytes(&documents)] {
for gzip in [false, true] {
let path = root.path().join(if gzip { "records.bson.gz" } else { "records.bson" });
let buffered = BufWriter::new(File::create(&path).unwrap());
let mut writer = if gzip {
BsonExportWriter::Gzip(Box::new(GzEncoder::new(buffered, Compression::default())))
} else {
BsonExportWriter::Plain(buffered)
};
writer.write_all(&expected).unwrap();
writer.finish().unwrap();
let bytes = std::fs::read(&path).unwrap();
let actual = if gzip {
let mut decoded = Vec::new();
MultiGzDecoder::new(bytes.as_slice()).read_to_end(&mut decoded).unwrap();
decoded
} else {
bytes
};
assert_eq!(actual, expected);
}
}
}
#[test]
fn bson_dump_preview_and_import_preserve_native_types() {
use mongodb::bson::spec::BinarySubtype;
use mongodb::bson::{Binary, DateTime};
let documents = vec![
doc! {
"_id": ObjectId::parse_str("507f1f77bcf86cd799439011").unwrap(),
"createdAt": DateTime::from_millis(1_609_459_200_000),
"payload": Bson::Binary(Binary { subtype: BinarySubtype::Generic, bytes: vec![1, 2, 3] }),
"nested": { "count": 42i64 },
},
doc! { "_id": 2i32, "name": "second" },
];
let bytes = bson_dump_bytes(&documents);
let preview =
preview_mongodb_import_bytes(&bytes, MongoImportFormat::Bson, &MongoImportParseOptions::default(), 10)
.unwrap();
assert_eq!(preview.rows.len(), 2);
assert_eq!(preview.estimated_rows, Some(2));
assert!(preview.estimated_rows_exact);
assert_eq!(preview.rows[0]["_id"]["$oid"], "507f1f77bcf86cd799439011");
assert!(preview.rows[0]["payload"].get("$binary").is_some());
let imported = execute_source(&bytes, "bson", MongoImportFormat::Bson, &MongoImportParseOptions::default());
assert_eq!(imported, preview.rows);
}
#[test]
fn gzip_bson_dump_import_matches_plain_dump() {
let documents = vec![doc! { "_id": 1i32, "name": "Ada" }, doc! { "_id": 2i32, "name": "Bob" }];
let plain = bson_dump_bytes(&documents);
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
encoder.write_all(&plain).unwrap();
let compressed = encoder.finish().unwrap();
let imported =
execute_source(&compressed, "bson.gz", MongoImportFormat::Bson, &MongoImportParseOptions::default());
assert_eq!(imported.len(), 2);
assert_eq!(imported[0]["name"], "Ada");
assert_eq!(imported[1]["name"], "Bob");
}
#[test]
fn bson_dump_rejects_truncated_and_invalid_lengths() {
let mut truncated = mongodb::bson::to_vec(&doc! { "name": "Ada" }).unwrap();
truncated.pop();
let error =
preview_mongodb_import_bytes(&truncated, MongoImportFormat::Bson, &MongoImportParseOptions::default(), 10)
.unwrap_err();
assert_eq!(error.code, "BSON_STRUCTURE");
assert_eq!(error.row, Some(1));
let error = preview_mongodb_import_bytes(
&(BSON_MAX_DOCUMENT_BYTES as i32 + 1).to_le_bytes(),
MongoImportFormat::Bson,
&MongoImportParseOptions::default(),
10,
)
.unwrap_err();
assert_eq!(error.code, "BSON_STRUCTURE");
}
#[test]
fn bson_dump_format_is_detected_from_plain_and_gzip_paths() {
assert_eq!(format_from_path("users.bson").unwrap(), MongoImportFormat::Bson);
assert_eq!(format_from_path("users.BSON.GZ").unwrap(), MongoImportFormat::Bson);
}
#[test]
fn csv_quoted_comma_newline_and_escaped_quotes() {
let csv = "name,note\n\"Ada, Lovelace\",\"line1\nline2\"\n\"quotes\",\"she said \"\"hi\"\"\"\n";
@@ -3806,6 +4141,7 @@ def handle(request):
collation: None,
format: MongoExportFormat::Ndjson,
include_header: true,
gzip: false,
file_path: target.to_string_lossy().to_string(),
execution_id: None,
},
@@ -3857,6 +4193,7 @@ def handle(request):
collation: None,
format: MongoExportFormat::Ndjson,
include_header: true,
gzip: false,
file_path: target.to_string_lossy().to_string(),
execution_id: None,
},
@@ -3928,6 +4265,7 @@ def handle(request):
collation: None,
format: MongoExportFormat::Ndjson,
include_header: true,
gzip: false,
file_path: target.to_string_lossy().to_string(),
execution_id: None,
},
@@ -3979,6 +4317,7 @@ def handle(request):
collation: None,
format: MongoExportFormat::Ndjson,
include_header: true,
gzip: false,
file_path: target.to_string_lossy().to_string(),
execution_id: None,
},
@@ -4035,6 +4374,7 @@ def handle(request):
collation: None,
format: MongoExportFormat::Ndjson,
include_header: true,
gzip: false,
file_path: target.to_string_lossy().to_string(),
execution_id: None,
},
@@ -4085,6 +4425,7 @@ def handle(request):
collation: None,
format: MongoExportFormat::Ndjson,
include_header: true,
gzip: false,
file_path: target.to_string_lossy().to_string(),
execution_id: None,
},
@@ -0,0 +1,822 @@
use std::io::Write;
use std::path::Path;
use dbx_core::mongodb_import_export::{
for_each_mongodb_import_document, preview_mongodb_import_file, MongoImportFormat, MongoImportIssue,
MongoImportParseOptions, MongoImportPreviewRequest, ParsedMongoDocument,
};
use flate2::{write::GzEncoder, Compression};
use mongodb::bson::{doc, Bson, Document};
fn encode(documents: &[Document], gzip: bool) -> Vec<u8> {
let bytes = documents.iter().flat_map(|document| mongodb::bson::to_vec(document).unwrap()).collect::<Vec<_>>();
if !gzip {
return bytes;
}
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
encoder.write_all(&bytes).unwrap();
encoder.finish().unwrap()
}
fn read(path: &Path) -> Result<Vec<Document>, Box<MongoImportIssue>> {
let mut documents = Vec::new();
#[expect(clippy::result_large_err, reason = "The public import callback requires MongoImportIssue by value")]
let collect = |parsed: Result<ParsedMongoDocument, MongoImportIssue>| {
documents.push(parsed?.document);
Ok(())
};
for_each_mongodb_import_document(
path.to_str().unwrap(),
MongoImportFormat::Bson,
&MongoImportParseOptions::default(),
collect,
)
.map_err(Box::new)?;
Ok(documents)
}
#[test]
fn bson_preview_is_bounded_and_preserves_explicit_format() {
let directory = tempfile::tempdir().unwrap();
// The selected format, not the filename, chooses the parser.
let path = directory.path().join("renamed.json");
let documents = (0..80).map(|index| doc! { "_id": index, "smallLong": 1i64 }).collect::<Vec<_>>();
std::fs::write(&path, encode(&documents, false)).unwrap();
let mut request = MongoImportPreviewRequest {
file_path: path.to_str().unwrap().into(),
source_ref: None,
format: MongoImportFormat::Bson,
parse_options: MongoImportParseOptions::default(),
preview_limit: Some(3),
};
let preview = preview_mongodb_import_file(&request).unwrap();
assert_eq!(preview.rows.len(), 3);
assert!(!preview.estimated_rows_exact);
assert_eq!(preview.detected_encoding, None);
assert_eq!(preview.rows[0]["smallLong"]["$numberLong"], "1");
assert_eq!(read(&path).unwrap(), documents);
std::fs::write(&path, []).unwrap();
request.preview_limit = Some(1);
let preview = preview_mongodb_import_file(&request).unwrap();
assert_eq!(preview.estimated_rows, Some(0));
assert!(preview.estimated_rows_exact);
assert!(read(&path).unwrap().is_empty());
}
#[test]
fn gzip_reads_all_members_and_rejects_corrupt_or_truncated_streams() {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("records.bson.gz");
let first = doc! { "_id": 1 };
let second = doc! { "_id": 2 };
let mut bytes = encode(std::slice::from_ref(&first), true);
bytes.extend(encode(std::slice::from_ref(&second), true));
std::fs::write(&path, &bytes).unwrap();
assert_eq!(read(&path).unwrap(), vec![first.clone(), second]);
let valid = encode(&[first], true);
for length in [0, 1, valid.len() - 1, valid.len() - 8] {
std::fs::write(&path, &valid[..length]).unwrap();
assert!(read(&path).is_err(), "accepted truncated gzip at {length}");
}
let mut corrupt = valid.clone();
let crc_offset = corrupt.len() - 8;
corrupt[crc_offset] ^= 1;
std::fs::write(&path, corrupt).unwrap();
assert!(read(&path).is_err(), "accepted invalid gzip checksum");
let mut trailing = valid;
trailing.extend_from_slice(b"junk");
std::fs::write(&path, trailing).unwrap();
assert!(read(&path).is_err(), "ignored trailing compressed data");
std::fs::write(&path, encode(&[], true)).unwrap();
assert!(read(&path).unwrap().is_empty());
}
#[test]
fn invalid_bson_is_fatal_even_when_callback_skips_errors() {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("records.bson");
let prefix = encode(&[doc! { "_id": 1 }], false);
for tail in [
vec![1],
vec![5, 0, 0],
4i32.to_le_bytes().to_vec(),
(-1i32).to_le_bytes().to_vec(),
(16 * 1024 * 1024 + 1i32).to_le_bytes().to_vec(),
vec![5, 0, 0, 0, 1],
] {
let mut bytes = prefix.clone();
bytes.extend(tail);
std::fs::write(&path, bytes).unwrap();
#[expect(clippy::result_large_err, reason = "The public import callback requires MongoImportIssue by value")]
let ignore = |_: Result<ParsedMongoDocument, MongoImportIssue>| Ok(());
let error = for_each_mongodb_import_document(
path.to_str().unwrap(),
MongoImportFormat::Bson,
&MongoImportParseOptions { skip_error_rows: Some(true), ..Default::default() },
ignore,
)
.unwrap_err();
assert_eq!(error.code, "BSON_STRUCTURE");
assert_eq!(error.row, Some(2));
}
}
fn run_tool(tools: &Path, name: &str, args: &[&str]) {
let logs = tempfile::tempdir().unwrap();
let log = logs.path().join("stderr.log");
let mut child = std::process::Command::new(tools.join(format!("{name}{}", std::env::consts::EXE_SUFFIX)))
.args(args)
.stdout(std::process::Stdio::null())
.stderr(std::fs::File::create(&log).unwrap())
.spawn()
.unwrap_or_else(|error| panic!("cannot run {name}: {error}"));
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60);
loop {
if let Some(status) = child.try_wait().unwrap() {
assert!(status.success(), "{name}: {}", std::fs::read_to_string(&log).unwrap());
return;
}
if std::time::Instant::now() >= deadline {
let _ = child.kill();
let _ = child.wait();
panic!("{name} timed out: {}", std::fs::read_to_string(&log).unwrap());
}
std::thread::sleep(std::time::Duration::from_millis(10));
}
}
async fn collection_bytes(client: &mongodb::Client, database: &str, collection: &str) -> Vec<Vec<u8>> {
use futures::TryStreamExt;
client
.database(database)
.collection::<Document>(collection)
.find(doc! {})
.sort(doc! { "_id": 1 })
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap()
.iter()
.map(|document| mongodb::bson::to_vec(document).unwrap())
.collect()
}
#[tokio::test]
#[ignore = "opt-in: DBX_MONGO_DUMP_TEST_URI and DBX_MONGO_TOOLS_DIR; creates a temporary database"]
async fn official_tools_round_trip_through_dbx_core() {
use dbx_core::connection::{AppState, PoolKind};
use dbx_core::models::connection::ConnectionConfig;
use dbx_core::mongodb_import_export::{
export_mongodb_query_core, import_mongodb_file_core, MongoExportFormat, MongoExportRequest, MongoExportStatus,
MongoImportRequest, MongoImportStatus,
};
use dbx_core::storage::Storage;
use mongodb::bson::{oid::ObjectId, spec::BinarySubtype, Binary, DateTime, Decimal128, Regex, Timestamp};
use std::sync::{
atomic::{AtomicBool, Ordering},
Arc,
};
let uri = std::env::var("DBX_MONGO_DUMP_TEST_URI").expect("DBX_MONGO_DUMP_TEST_URI");
let tools = std::path::PathBuf::from(std::env::var("DBX_MONGO_TOOLS_DIR").expect("DBX_MONGO_TOOLS_DIR"));
let directory = tempfile::tempdir().unwrap();
let client = mongodb::Client::with_uri_str(&uri).await.unwrap();
let database = format!("dbx_dump_test_{}", uuid::Uuid::new_v4().simple());
let storage = Storage::open(&directory.path().join("storage.db")).await.unwrap();
let state = AppState::new(storage);
let connection_id = "mongo-dump-test";
let config: ConnectionConfig = serde_json::from_value(serde_json::json!({
"id": connection_id, "name": "Dump test", "db_type": "mongodb", "host": "127.0.0.1",
"port": 27090, "username": "", "password": "", "database": database,
"connection_string": uri, "driver_profile": "mongodb-native"
}))
.unwrap();
state.configs.write().await.insert(connection_id.into(), config);
state
.update_connection_pools(|pools| {
pools.insert(connection_id.into(), PoolKind::MongoDb(client.clone()));
})
.await;
let documents = (0..1207)
.map(|index| {
doc! {
"_id": index,
"objectId": ObjectId::parse_str("507f1f77bcf86cd799439011").unwrap(),
"smallLong": 1i64, "maxLong": i64::MAX, "double": 1.5f64,
"decimal": "123.45".parse::<Decimal128>().unwrap(),
"date": DateTime::from_millis(-1234567890),
"binary": Binary { subtype: BinarySubtype::UserDefined(0x80), bytes: vec![0, 1, 255] },
"regex": Regex { pattern: "^test".into(), options: "im".into() },
"timestamp": Timestamp { time: 12345, increment: 42 },
"min": Bson::MinKey, "max": Bson::MaxKey, "undefined": Bson::Undefined,
"code": Bson::JavaScriptCode("return 1".into()),
"nested": { "array": [Bson::Null, Bson::Boolean(true), Bson::String("1".into())] },
}
})
.collect::<Vec<_>>();
client.database(&database).collection("source").insert_many(documents).await.unwrap();
let expected = collection_bytes(&client, &database, "source").await;
assert_eq!(expected.len(), 1207);
for gzip in [false, true] {
let extension = if gzip { "bson.gz" } else { "bson" };
let dump_dir = directory.path().join(if gzip { "official-gzip" } else { "official-plain" });
let mut args = vec![
"--uri",
uri.as_str(),
"--db",
&database,
"--collection",
"source",
"--out",
dump_dir.to_str().unwrap(),
];
if gzip {
args.push("--gzip");
}
run_tool(&tools, "mongodump", &args);
let dumped = dump_dir.join(&database).join(format!("source.{extension}"));
assert_eq!(read(&dumped).unwrap().len(), expected.len());
let preview = preview_mongodb_import_file(&MongoImportPreviewRequest {
file_path: dumped.to_str().unwrap().into(),
source_ref: None,
format: MongoImportFormat::Bson,
parse_options: Default::default(),
preview_limit: Some(5),
})
.unwrap();
assert_eq!(preview.rows.len(), 5);
let imported = if gzip { "dbx_gzip" } else { "dbx_plain" };
let request = MongoImportRequest {
import_id: uuid::Uuid::new_v4().to_string(),
connection_id: connection_id.into(),
database: database.clone(),
collection: imported.into(),
file_path: dumped.to_str().unwrap().into(),
source_ref: None,
format: MongoImportFormat::Bson,
parse_options: Default::default(),
batch_size: 500,
execution_id: None,
};
let summary = import_mongodb_file_core(&state, &request, |_| Box::pin(async { false }), |_| {}).await.unwrap();
assert_eq!(summary.rows_inserted, 1207);
assert_eq!(summary.rows_failed, 0);
assert_eq!(summary.batches_committed, 3);
assert_eq!(collection_bytes(&client, &database, imported).await, expected);
// This file is produced by DBX, with no official metadata sidecar beside it.
let exported = directory.path().join(format!("dbx-output.{extension}"));
let export = MongoExportRequest {
export_id: uuid::Uuid::new_v4().to_string(),
connection_id: connection_id.into(),
database: database.clone(),
collection: imported.into(),
filter: None,
sort: Some("{\"_id\":1}".into()),
projection: None,
collation: None,
format: MongoExportFormat::Bson,
include_header: false,
gzip,
file_path: exported.to_str().unwrap().into(),
execution_id: None,
};
let summary = export_mongodb_query_core(&state, &export, |_| Box::pin(async { false }), |_| {}).await.unwrap();
assert_eq!(summary.documents_exported, 1207);
assert_eq!(
read(&exported).unwrap().iter().map(|d| mongodb::bson::to_vec(d).unwrap()).collect::<Vec<_>>(),
expected
);
let restored = if gzip { "official_gzip" } else { "official_plain" };
let mut args = vec!["--uri", uri.as_str(), "--db", &database, "--collection", restored];
if gzip {
args.push("--gzip");
}
args.push(exported.to_str().unwrap());
run_tool(&tools, "mongorestore", &args);
assert_eq!(collection_bytes(&client, &database, restored).await, expected);
println!("{extension}: official -> DBX -> official, 1207 documents, BSON bytes match");
let mut empty_export = export.clone();
empty_export.collection = "empty".into();
empty_export.file_path = directory.path().join(format!("empty.{extension}")).to_str().unwrap().into();
let empty =
export_mongodb_query_core(&state, &empty_export, |_| Box::pin(async { false }), |_| {}).await.unwrap();
assert_eq!(empty.documents_exported, 0);
assert!(read(Path::new(&empty.file_path)).unwrap().is_empty());
let cancelled = Arc::new(AtomicBool::new(false));
let check = cancelled.clone();
let original = std::fs::read(&exported).unwrap();
let result = export_mongodb_query_core(
&state,
&export,
move |_| {
let check = check.clone();
Box::pin(async move { check.load(Ordering::SeqCst) })
},
|progress| {
if progress.status == MongoExportStatus::Running && progress.documents_read > 0 {
cancelled.store(true, Ordering::SeqCst);
}
},
)
.await;
assert_eq!(result.unwrap_err(), "Export cancelled");
assert_eq!(std::fs::read(&exported).unwrap(), original, "cancel replaced the previous export");
let damaged = directory.path().join("damaged.bson");
std::fs::write(&damaged, [1]).unwrap();
let mut request = request;
request.file_path = damaged.to_str().unwrap().into();
request.parse_options.skip_error_rows = Some(true);
let mut status = MongoImportStatus::Running;
let result =
import_mongodb_file_core(&state, &request, |_| Box::pin(async { false }), |p| status = p.status).await;
assert!(result.is_err());
assert_eq!(status, MongoImportStatus::Error);
}
client.database(&database).drop().await.unwrap();
}
async fn specs(client: &mongodb::Client, db: &str, collection: &str) -> (Document, Vec<Document>) {
use futures::TryStreamExt;
let specification = client
.database(db)
.list_collections()
.filter(doc! { "name": collection })
.await
.unwrap()
.try_next()
.await
.unwrap()
.unwrap();
let options = mongodb::bson::to_document(&specification.options).unwrap();
let mut indexes = Vec::new();
if !matches!(specification.collection_type, mongodb::results::CollectionType::View) {
let mut cursor = client.database(db).collection::<Document>(collection).list_indexes().await.unwrap();
while let Some(index) = cursor.try_next().await.unwrap() {
let mut index = mongodb::bson::to_document(&index).unwrap();
index.remove("ns");
index.remove("v");
indexes.push(index);
}
indexes.sort_by_key(|index| index.get_str("name").unwrap().to_string());
}
(options, indexes)
}
#[tokio::test]
#[ignore = "opt-in: DBX_MONGO_DUMP_TEST_URI and DBX_MONGO_TOOLS_DIR; creates temporary databases"]
async fn official_database_tools_round_trip_through_dbx() {
use dbx_core::{
connection::{AppState, PoolKind},
models::connection::ConnectionConfig,
mongodb_dump::*,
storage::Storage,
};
let uri = std::env::var("DBX_MONGO_DUMP_TEST_URI").expect("DBX_MONGO_DUMP_TEST_URI");
let tools = std::path::PathBuf::from(std::env::var("DBX_MONGO_TOOLS_DIR").expect("DBX_MONGO_TOOLS_DIR"));
let files = tempfile::tempdir().unwrap();
let client = mongodb::Client::with_uri_str(&uri).await.unwrap();
let database = format!("dbx_db_{}", uuid::Uuid::new_v4().simple());
let db = client.database(&database);
db.run_command(doc! { "create": "records", "validator": { "score": { "$gte": 0 } }, "validationLevel": "strict", "collation": { "locale": "en", "strength": 2 } }).await.unwrap();
db.collection("records")
.insert_many(vec![
doc! { "_id": 1, "score": 1i64, "name": "first" },
doc! { "_id": 2, "score": 2i64, "name": "second" },
])
.await
.unwrap();
db.run_command(doc! { "createIndexes": "records", "indexes": [ { "key": { "name": 1, "score": -1 }, "name": "compound_unique", "unique": true }, { "key": { "expires": 1 }, "name": "ttl", "expireAfterSeconds": 3600 }] }).await.unwrap();
db.create_collection("empty").await.unwrap();
db.run_command(doc! { "create": "capped", "capped": true, "size": 65536i64, "max": 100i64 }).await.unwrap();
db.collection("capped").insert_one(doc! { "_id": 1, "n": 1 }).await.unwrap();
db.collection("odd/name").insert_one(doc! { "_id": 1, "binary": mongodb::bson::Binary { subtype: mongodb::bson::spec::BinarySubtype::Generic, bytes: vec![1, 2, 3] } }).await.unwrap();
db.run_command(doc! { "create": "z_view", "viewOn": "records", "pipeline": [ { "$match": { "score": { "$gte": 1 } } } ], "collation": { "locale": "en", "strength": 2 } }).await.unwrap();
db.run_command(doc! { "create": "a_nested", "viewOn": "z_view", "pipeline": [], "collation": { "locale": "en", "strength": 2 } }).await.unwrap();
let collections = ["records", "empty", "capped", "odd/name", "z_view", "a_nested"];
let state = AppState::new(Storage::open(&files.path().join("storage.db")).await.unwrap());
let connection_id = "database-dump-test";
let config: ConnectionConfig = serde_json::from_value(serde_json::json!({ "id": connection_id, "name": "Database dump test", "db_type": "mongodb", "host": "127.0.0.1", "port": 27090, "username": "", "password": "", "database": database, "connection_string": uri, "driver_profile": "mongodb-native" })).unwrap();
state.configs.write().await.insert(connection_id.into(), config);
state
.update_connection_pools(|pools| {
pools.insert(connection_id.into(), PoolKind::MongoDb(client.clone()));
})
.await;
let mut databases = vec![database.clone()];
for format in [MongoDumpFormat::Directory, MongoDumpFormat::Archive] {
for gzip in [false, true] {
let tag = format!("{format:?}_{gzip}");
let official_path = files.path().join(format!("official-{tag}"));
let archive_arg = format!("--archive={}", official_path.display());
let mut args = vec!["--uri", uri.as_str(), "--db", &database];
if format == MongoDumpFormat::Archive {
args.push(&archive_arg);
} else {
args.extend(["--out", official_path.to_str().unwrap()]);
}
if gzip {
args.push("--gzip");
}
run_tool(&tools, "mongodump", &args);
let preview = prepare_mongodb_restore_source(MongoRestoreSourceRequest {
path: official_path.to_str().unwrap().into(),
format,
gzip,
})
.await
.unwrap();
assert_eq!(preview.catalog.collections.len(), 6);
assert!(preview.catalog.collections.iter().all(|c| c.documents.is_none()));
let restored = format!("{database}_{tag}");
databases.push(restored.clone());
let request = MongoDatabaseRestoreRequest {
task_id: uuid::Uuid::new_v4().to_string(),
connection_id: connection_id.into(),
database: restored.clone(),
source_database: database.clone(),
source_ref: preview.source_ref.clone(),
collections: None,
drop_existing: false,
restore_options: true,
restore_indexes: true,
stop_on_error: true,
objcheck: gzip,
batch_size: 500,
execution_id: None,
};
let mut saw_data = false;
let result = restore_mongodb_database(
&state,
&request,
|_| Box::pin(async { false }),
|p| {
assert_ne!(p.phase, "validating");
saw_data |= p.phase == "data";
},
)
.await
.unwrap();
assert!(saw_data);
assert_eq!(result.documents_validated, if gzip { 4 } else { 0 });
assert_eq!(result.collections_done, 6);
assert_eq!(result.documents_written, 4);
for collection in collections {
assert_eq!(
collection_bytes(&client, &database, collection).await,
collection_bytes(&client, &restored, collection).await,
"data: {tag} {collection}"
);
assert_eq!(
specs(&client, &database, collection).await,
specs(&client, &restored, collection).await,
"metadata: {tag} {collection}"
);
}
client.database(&restored).collection("unselected").insert_one(doc! { "_id": "keep" }).await.unwrap();
client
.database(&restored)
.collection("records")
.insert_one(doc! { "_id": 999, "score": 999, "name": "remove" })
.await
.unwrap();
let mut selected = request.clone();
selected.drop_existing = true;
selected.collections = Some(vec!["records".into()]);
restore_mongodb_database(&state, &selected, |_| Box::pin(async { false }), |_| {}).await.unwrap();
assert_eq!(
collection_bytes(&client, &database, "records").await,
collection_bytes(&client, &restored, "records").await
);
assert_eq!(collection_bytes(&client, &restored, "unselected").await.len(), 1);
if format == MongoDumpFormat::Directory && !gzip {
let cancel_dir = files.path().join("cancel_source");
std::fs::create_dir(&cancel_dir).unwrap();
let documents = (0..201).map(|id| doc! { "_id": id }).collect::<Vec<_>>();
std::fs::write(cancel_dir.join("records.bson"), encode(&documents, false)).unwrap();
let cancel_source = prepare_mongodb_restore_source(MongoRestoreSourceRequest {
path: cancel_dir.to_str().unwrap().into(),
format: MongoDumpFormat::Directory,
gzip: false,
})
.await
.unwrap();
let cancel = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
let signal = cancel.clone();
let mut cancel_request = selected.clone();
cancel_request.source_ref = cancel_source.source_ref.clone();
cancel_request.source_database = "cancel_source".into();
cancel_request.batch_size = 100;
let mut final_progress = None;
let cancelled = tokio::time::timeout(
std::time::Duration::from_secs(10),
restore_mongodb_database(
&state,
&cancel_request,
|_| {
let signal = signal.clone();
Box::pin(async move { signal.load(std::sync::atomic::Ordering::Relaxed) })
},
|p| {
if p.documents_written >= 100 {
cancel.store(true, std::sync::atomic::Ordering::Relaxed);
}
final_progress = Some(p);
},
),
)
.await
.expect("cancellation must stop and join the reader")
.unwrap_err();
assert!(cancelled.contains("cancelled"), "{cancelled}");
let final_progress = final_progress.unwrap();
assert_eq!(final_progress.status, "cancelled");
assert_eq!(final_progress.documents_written, 100);
assert_eq!(collection_bytes(&client, &restored, "records").await.len(), 100);
let mut duplicate_request = cancel_request.clone();
duplicate_request.batch_size = 500;
duplicate_request.drop_existing = false;
let mut partial_progress = None;
let error = restore_mongodb_database(
&state,
&duplicate_request,
|_| Box::pin(async { false }),
|p| partial_progress = Some(p),
)
.await
.unwrap_err();
assert!(error.contains("duplicate key"), "{error}");
let partial_progress = partial_progress.unwrap();
assert_eq!(partial_progress.documents_written, 101);
assert_eq!(partial_progress.documents_failed, 100);
duplicate_request.stop_on_error = false;
let continued =
restore_mongodb_database(&state, &duplicate_request, |_| Box::pin(async { false }), |_| {})
.await
.unwrap();
assert_eq!(continued.documents_written, 0);
assert_eq!(continued.documents_failed, 201);
release_mongodb_restore_source(&cancel_source.source_ref);
let broken_dir = files.path().join("broken_source");
std::fs::create_dir(&broken_dir).unwrap();
std::fs::write(broken_dir.join("records.bson"), encode(&[doc! { "_id": "replacement" }], false))
.unwrap();
std::fs::write(broken_dir.join("zz_broken.bson"), b"invalid BSON").unwrap();
let broken = prepare_mongodb_restore_source(MongoRestoreSourceRequest {
path: broken_dir.to_str().unwrap().into(),
format,
gzip,
})
.await
.unwrap();
let mut invalid = selected.clone();
invalid.source_ref = broken.source_ref.clone();
invalid.source_database = "broken_source".into();
invalid.collections = None;
let mut partial = 0;
let error = restore_mongodb_database(&state, &invalid, |_| Box::pin(async { false }), |_| {})
.await
.unwrap_err();
assert!(error.contains("BSON"), "{error}");
assert_eq!(
collection_bytes(&client, &restored, "records").await,
vec![mongodb::bson::to_vec(&doc! { "_id": "replacement" }).unwrap()],
"valid earlier collections are restored without a full pre-scan"
);
invalid.objcheck = true;
assert!(restore_mongodb_database(
&state,
&invalid,
|_| Box::pin(async { false }),
|p| partial = p.documents_written
)
.await
.is_err());
assert_eq!(partial, 1);
release_mongodb_restore_source(&broken.source_ref);
let mut no_metadata = request.clone();
no_metadata.database = format!("{database}_no_options");
databases.push(no_metadata.database.clone());
no_metadata.collections = Some(vec!["records".into()]);
no_metadata.restore_options = false;
no_metadata.restore_indexes = false;
restore_mongodb_database(&state, &no_metadata, |_| Box::pin(async { false }), |_| {}).await.unwrap();
let (options, indexes) = specs(&client, &no_metadata.database, "records").await;
assert!(options.is_empty());
assert_eq!(indexes.len(), 1);
state.configs.write().await.get_mut(connection_id).unwrap().read_only = true;
let blocked = restore_mongodb_database(&state, &no_metadata, |_| Box::pin(async { false }), |_| {})
.await
.unwrap_err();
assert!(blocked.contains("Read-only"));
state.configs.write().await.get_mut(connection_id).unwrap().read_only = false;
let cancelled = restore_mongodb_database(&state, &no_metadata, |_| Box::pin(async { true }), |_| {})
.await
.unwrap_err();
assert!(cancelled.contains("cancelled"));
}
assert!(release_mongodb_restore_source(&preview.source_ref));
let output = files.path().join(format!("dbx-{tag}"));
let dump = MongoDatabaseDumpRequest {
task_id: uuid::Uuid::new_v4().to_string(),
connection_id: connection_id.into(),
database: database.clone(),
file_path: output.to_str().unwrap().into(),
format,
gzip,
collections: None,
};
dump_mongodb_database(&state, &dump, |_| Box::pin(async { false }), |_| {}).await.unwrap();
let target = format!("{database}_out_{tag}");
databases.push(target.clone());
let from = format!("{database}.*");
let to = format!("{target}.*");
let archive_arg = format!("--archive={}", output.display());
let mut args = vec!["--uri", uri.as_str(), "--nsFrom", &from, "--nsTo", &to];
if gzip {
args.push("--gzip");
}
if format == MongoDumpFormat::Archive {
args.push(&archive_arg);
} else {
args.push(output.to_str().unwrap());
}
run_tool(&tools, "mongorestore", &args);
for collection in collections {
assert_eq!(
collection_bytes(&client, &database, collection).await,
collection_bytes(&client, &target, collection).await,
"exported data: {tag} {collection}"
);
assert_eq!(
specs(&client, &database, collection).await,
specs(&client, &target, collection).await,
"exported metadata: {tag} {collection}"
);
}
println!("{tag}: six collections/views, BSON, options and indexes match in both directions");
if format == MongoDumpFormat::Archive && !gzip {
let mut bytes = std::fs::read(&output).unwrap();
let offset = bytes.windows(5).position(|bytes| bytes == b"first").unwrap();
bytes[offset] = b'F';
let corrupt = files.path().join("corrupt.archive");
std::fs::write(&corrupt, &bytes).unwrap();
let corrupt_preview = prepare_mongodb_restore_source(MongoRestoreSourceRequest {
path: corrupt.to_str().unwrap().into(),
format,
gzip: false,
})
.await
.unwrap();
let mut corrupt_request = request.clone();
corrupt_request.source_ref = corrupt_preview.source_ref.clone();
corrupt_request.database = target.clone();
corrupt_request.drop_existing = true;
let mut partial = 0;
let error = restore_mongodb_database(
&state,
&corrupt_request,
|_| Box::pin(async { false }),
|p| partial = p.documents_written,
)
.await
.unwrap_err();
assert!(error.contains("checksum"), "{error}");
assert!(partial > 0, "archive CRC is checked as data streams, not in a separate pre-pass");
release_mongodb_restore_source(&corrupt_preview.source_ref);
bytes = std::fs::read(&output).unwrap();
bytes.truncate(bytes.len() - 1);
std::fs::write(&corrupt, bytes).unwrap();
let truncated = prepare_mongodb_restore_source(MongoRestoreSourceRequest {
path: corrupt.to_str().unwrap().into(),
format,
gzip: false,
})
.await
.unwrap();
corrupt_request.source_ref = truncated.source_ref.clone();
assert!(restore_mongodb_database(&state, &corrupt_request, |_| Box::pin(async { false }), |_| {})
.await
.is_err());
assert_eq!(
collection_bytes(&client, &database, "records").await,
collection_bytes(&client, &target, "records").await
);
release_mongodb_restore_source(&truncated.source_ref);
}
}
}
for database in databases {
client.database(&database).drop().await.unwrap();
}
}
#[tokio::test]
async fn database_source_rejects_invalid_metadata_and_view_cycles() {
use dbx_core::mongodb_dump::*;
let directory = tempfile::tempdir().unwrap();
let source = directory.path().join("source");
std::fs::create_dir(&source).unwrap();
std::fs::write(source.join("records.bson"), []).unwrap();
let request = MongoRestoreSourceRequest {
path: source.to_str().unwrap().into(),
format: MongoDumpFormat::Directory,
gzip: false,
};
for metadata in [
r#"{"collectionName":"records","options":{"$db":"outside"},"indexes":[]}"#,
r#"{"collectionName":"records","options":{},"indexes":[{"name":"bad"}]}"#,
r#"{"collectionName":"records","type":"timeseries","options":{},"indexes":[]}"#,
] {
std::fs::write(source.join("records.metadata.json"), metadata).unwrap();
assert!(prepare_mongodb_restore_source(request.clone()).await.is_err());
}
std::fs::write(source.join("records.metadata.json"), r#"{"collectionName":"records","options":{},"indexes":[]}"#)
.unwrap();
std::fs::write(
source.join("one.metadata.json"),
r#"{"collectionName":"one","type":"view","options":{"viewOn":"two","pipeline":[]},"indexes":[]}"#,
)
.unwrap();
std::fs::write(
source.join("two.metadata.json"),
r#"{"collectionName":"two","type":"view","options":{"viewOn":"one","pipeline":[]},"indexes":[]}"#,
)
.unwrap();
let error = prepare_mongodb_restore_source(request).await.unwrap_err();
assert!(error.contains("Cyclic"));
}
#[tokio::test]
async fn directory_preview_does_not_read_bson_and_source_identity_is_checked_before_restore() {
use dbx_core::{connection::AppState, mongodb_dump::*, storage::Storage};
let files = tempfile::tempdir().unwrap();
let source = files.path().join("source");
std::fs::create_dir(&source).unwrap();
// Preview must succeed without opening/decoding this invalid gzip data.
std::fs::write(source.join("records.bson.gz"), b"not gzip or BSON").unwrap();
let preview = prepare_mongodb_restore_source(MongoRestoreSourceRequest {
path: source.to_str().unwrap().into(),
format: MongoDumpFormat::Directory,
gzip: true,
})
.await
.unwrap();
assert_eq!(preview.catalog.collections[0].documents, None);
assert_eq!(preview.catalog.collections[0].size_bytes, 16);
let state = AppState::new(Storage::open(&files.path().join("state.db")).await.unwrap());
let request: MongoDatabaseRestoreRequest = serde_json::from_value(serde_json::json!({
"taskId":"validation-test", "connectionId":"missing", "database":"target", "sourceDatabase":"source",
"sourceRef":preview.source_ref, "dropExisting":true
}))
.unwrap();
let error = restore_mongodb_database(&state, &request, |_| Box::pin(async { false }), |_| {}).await.unwrap_err();
assert!(
!error.to_lowercase().contains("gzip"),
"no pre-scan should run before the missing connection fails: {error}"
);
assert!(!request.objcheck, "objcheck is opt-in");
std::fs::write(source.join("records.bson.gz"), []).unwrap();
let error = restore_mongodb_database(&state, &request, |_| Box::pin(async { false }), |_| {}).await.unwrap_err();
assert!(error.contains("changed"), "{error}");
release_mongodb_restore_source(&preview.source_ref);
}
#[tokio::test]
async fn directory_catalog_accepts_large_manifest_without_data_and_binds_only_selected_upload() {
use dbx_core::mongodb_dump::*;
let owner = tempfile::tempdir().unwrap();
let manifest = vec![
MongoDumpFile { path: "source/selected.bson".into(), size_bytes: 0 },
MongoDumpFile { path: "source/huge.bson".into(), size_bytes: 5 * 1024 * 1024 * 1024 },
];
let preview = prepare_mongodb_directory_catalog(owner, manifest, false).await.unwrap();
assert_eq!(preview.catalog.collections.len(), 2);
assert!(preview.catalog.collections.iter().all(|c| c.documents.is_none()));
let request: MongoDatabaseRestoreRequest = serde_json::from_value(serde_json::json!({
"taskId":"upload-test", "connectionId":"missing", "database":"target", "sourceDatabase":"source",
"sourceRef":preview.source_ref, "collections":["selected"]
}))
.unwrap();
let upload = tempfile::tempdir().unwrap();
std::fs::create_dir(upload.path().join("source")).unwrap();
std::fs::write(upload.path().join("source/selected.bson"), []).unwrap();
let acquired = attach_mongodb_restore_upload(&request, upload).await.unwrap();
assert_eq!(acquired.catalog.collections.len(), 1);
release_mongodb_restore_source(&acquired.source_ref);
let changed = tempfile::tempdir().unwrap();
std::fs::create_dir(changed.path().join("source")).unwrap();
std::fs::write(changed.path().join("source/selected.bson"), b"changed").unwrap();
assert!(attach_mongodb_restore_upload(&request, changed).await.unwrap_err().contains("differ"));
release_mongodb_restore_source(&preview.source_ref);
}
+5 -1
View File
@@ -34,10 +34,14 @@ tracing = "0.1"
async-stream = "0.3"
futures = "0.3"
rustls = { version = "0.23", features = ["aws-lc-rs"] }
tokio-util = "0.7"
tokio-util = { version = "0.7", features = ["io"] }
reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "rustls-tls"] }
chrono = "0.4"
aes-gcm = "0.10"
base64 = "0.22"
pbkdf2 = "0.12"
sha2 = "0.10"
tempfile = "3"
[dev-dependencies]
reqwest = { version = "0.12", default-features = false, features = ["multipart"] }
+19
View File
@@ -988,6 +988,25 @@ async fn main() {
.route("/mongo/export/progress/{exportId}", get(routes::mongodb_import_export::export_progress))
.route("/mongo/export/download/{exportId}", get(routes::mongodb_import_export::export_download))
.route("/mongo/export/cancel", post(routes::mongodb_import_export::cancel_export))
.route("/mongo/dump/catalog", post(routes::mongodb_dump::catalog))
.route(
"/mongo/dump/source",
post(routes::mongodb_dump::prepare_source).layer(DefaultBodyLimit::max(
routes::table_import::import_request_body_limit_for_upload(web_body_limit_bytes()),
)),
)
.route("/mongo/dump/source/release", post(routes::mongodb_dump::release_source))
.route("/mongo/dump/upload-limit", get(routes::mongodb_dump::upload_limit))
.route(
"/mongo/dump/source/upload",
post(routes::mongodb_dump::upload_restore_source).layer(DefaultBodyLimit::max(
routes::table_import::import_request_body_limit_for_upload(web_body_limit_bytes()),
)),
)
.route("/mongo/dump/export", post(routes::mongodb_dump::start_dump))
.route("/mongo/dump/restore", post(routes::mongodb_dump::start_restore))
.route("/mongo/dump/progress/{taskId}", get(routes::mongodb_dump::progress))
.route("/mongo/dump/cancel", post(routes::mongodb_dump::cancel))
// History
.route("/history", get(routes::history::load_history).delete(routes::history::clear_history))
.route("/history/save", post(routes::history::save_history))
+1
View File
@@ -17,6 +17,7 @@ pub mod jdbc;
pub mod layout;
pub mod mcp_policy;
pub mod mongo;
pub mod mongodb_dump;
pub mod mongodb_import_export;
#[cfg(feature = "mq-admin")]
pub mod mq;
+402
View File
@@ -0,0 +1,402 @@
use std::path::{Component, Path as FilePath, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use axum::{
extract::{Multipart, Path, State},
response::Sse,
Json,
};
use dbx_core::{
mongodb_dump::{
self, MongoDatabaseDumpProgress, MongoDatabaseDumpRequest, MongoDatabaseRestoreRequest, MongoDumpFormat,
MongoRestoreSourceRequest,
},
transfer,
};
use serde::Deserialize;
use tokio::io::AsyncWriteExt;
use super::export_download::export_download_filename;
use crate::{
error::AppError,
sse::{TransferProgressChannel, TransferReplayEventKind},
state::{WebExportFile, WebState},
};
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct CatalogRequest {
connection_id: String,
database: String,
}
pub async fn catalog(
State(state): State<Arc<WebState>>,
Json(request): Json<CatalogRequest>,
) -> Result<Json<serde_json::Value>, AppError> {
let catalog = mongodb_dump::inspect_mongodb_database_dump(&state.app, &request.connection_id, &request.database)
.await
.map_err(AppError::from)?;
Ok(Json(serde_json::to_value(catalog).map_err(|e| AppError::from(e.to_string()))?))
}
fn upload_path(root: &FilePath, filename: &str) -> Result<PathBuf, AppError> {
let path = FilePath::new(filename);
if filename.is_empty()
|| filename.contains(['\\', ':', '\0'])
|| path.components().any(|c| !matches!(c, Component::Normal(_)))
{
return Err(AppError::from("Invalid dump upload path".to_string()));
}
if path.components().count() > 4 {
return Err(AppError::from("Dump directory nesting is too deep".to_string()));
}
Ok(root.join(path))
}
pub async fn prepare_source(
State(state): State<Arc<WebState>>,
multipart: Multipart,
) -> Result<Json<serde_json::Value>, AppError> {
receive_source(state, multipart, false).await
}
pub async fn upload_restore_source(
State(state): State<Arc<WebState>>,
multipart: Multipart,
) -> Result<Json<serde_json::Value>, AppError> {
receive_source(state, multipart, true).await
}
pub async fn upload_limit() -> Json<usize> {
Json(crate::web_body_limit_bytes())
}
async fn field_text(mut field: axum::extract::multipart::Field<'_>, limit: usize) -> Result<String, AppError> {
let mut bytes = Vec::new();
while let Some(chunk) = field.chunk().await.map_err(|e| AppError::bad_request(e.to_string()))? {
if bytes.len().saturating_add(chunk.len()) > limit {
return Err(AppError::bad_request("Dump metadata field is too large"));
}
bytes.extend_from_slice(&chunk);
}
String::from_utf8(bytes).map_err(|e| AppError::bad_request(e.to_string()))
}
async fn receive_source(
state: Arc<WebState>,
mut multipart: Multipart,
acquiring: bool,
) -> Result<Json<serde_json::Value>, AppError> {
let tmp = state.data_dir.join("tmp");
tokio::fs::create_dir_all(&tmp).await.map_err(|e| AppError::from(e.to_string()))?;
let upload =
tempfile::Builder::new().prefix("mongo-upload-").tempdir_in(&tmp).map_err(|e| AppError::from(e.to_string()))?;
let mut format = None;
let mut gzip = false;
let mut paths = Vec::new();
let mut bytes = 0usize;
let mut manifest: Option<Vec<mongodb_dump::MongoDumpFile>> = None;
let mut request: Option<MongoDatabaseRestoreRequest> = None;
let mut expected_upload = std::collections::HashMap::<String, u64>::new();
while let Some(mut field) = multipart.next_field().await.map_err(|e| AppError::from(e.to_string()))? {
match field.name().unwrap_or("") {
"file" => {
if paths.len() >= 100_000 {
return Err(AppError::from("Too many dump files".to_string()));
}
let filename =
field.file_name().ok_or_else(|| AppError::from("Missing dump filename".to_string()))?.to_string();
if format.is_none() {
return Err(AppError::bad_request("Dump format must precede files"));
}
if !acquiring
&& format == Some(MongoDumpFormat::Directory)
&& !filename.ends_with(".metadata.json")
&& !filename.ends_with(".metadata.json.gz")
{
return Err(AppError::bad_request(
"Directory preview accepts metadata only; upload BSON after confirmation",
));
}
if acquiring && request.is_none() {
return Err(AppError::bad_request("Restore plan must precede files"));
}
let expected_size = if acquiring {
Some(
expected_upload
.remove(&filename)
.ok_or_else(|| AppError::bad_request("File is not in the selected restore plan"))?,
)
} else {
None
};
let mut file_bytes = 0u64;
let path = upload_path(upload.path(), &filename)?;
tokio::fs::create_dir_all(path.parent().unwrap()).await.map_err(|e| AppError::from(e.to_string()))?;
let mut output = tokio::fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(&path)
.await
.map_err(|e| AppError::from(e.to_string()))?;
while let Some(chunk) = field.chunk().await.map_err(|e| AppError::from(e.to_string()))? {
if let Some(request) = &request {
if transfer::is_cancelled(&request.task_id).await {
return Err(AppError::bad_request("MongoDB dump/restore cancelled"));
}
}
bytes = bytes.saturating_add(chunk.len());
file_bytes = file_bytes.saturating_add(chunk.len() as u64);
if expected_size.is_some_and(|size| file_bytes > size) {
return Err(AppError::bad_request("File size changed after preview"));
}
let limit = if !acquiring && format == Some(MongoDumpFormat::Directory) {
32 * 1024 * 1024
} else {
crate::web_body_limit_bytes()
};
if bytes > limit {
let mut error =
AppError::from(format!("Dump upload exceeds the configured {} byte limit", limit));
error.status = axum::http::StatusCode::PAYLOAD_TOO_LARGE;
return Err(error);
}
output.write_all(&chunk).await.map_err(|e| AppError::from(e.to_string()))?;
}
output.flush().await.map_err(|e| AppError::from(e.to_string()))?;
if expected_size.is_some_and(|size| file_bytes != size) {
return Err(AppError::bad_request("File size changed after preview"));
}
paths.push((filename, path));
}
"format" => {
let value = field_text(field, 32).await?;
format = Some(
serde_json::from_value::<MongoDumpFormat>(serde_json::Value::String(value))
.map_err(|e| AppError::from(e.to_string()))?,
);
}
"gzip" => {
let value = field_text(field, 16).await?;
gzip = value.parse::<bool>().map_err(|_| AppError::from("Invalid gzip option".to_string()))?;
}
"manifest" if !acquiring => {
let value = field_text(field, 16 * 1024 * 1024).await?;
manifest = Some(serde_json::from_str(&value).map_err(|e| AppError::bad_request(e.to_string()))?);
}
"request" if acquiring => {
let value = field_text(field, 1024 * 1024).await?;
if request.is_some() {
return Err(AppError::bad_request("Duplicate restore plan"));
}
request = Some(serde_json::from_str(&value).map_err(|e| AppError::bad_request(e.to_string()))?);
expected_upload = mongodb_dump::mongodb_restore_upload_files(request.as_ref().unwrap())
.map_err(AppError::from)?
.into_iter()
.map(|f| (f.path, f.size_bytes))
.collect();
}
_ => return Err(AppError::from("Unexpected dump upload field".to_string())),
}
}
let format = format.ok_or_else(|| AppError::from("Dump format is required".to_string()))?;
if paths.is_empty() && manifest.is_none() {
return Err(AppError::from("No dump files uploaded".to_string()));
}
if acquiring {
if !expected_upload.is_empty() {
return Err(AppError::bad_request("Selected restore files are missing"));
}
let request = request.ok_or_else(|| AppError::bad_request("Missing restore plan"))?;
if transfer::is_cancelled(&request.task_id).await {
return Err(AppError::bad_request("MongoDB dump/restore cancelled"));
}
let preview = mongodb_dump::attach_mongodb_restore_upload(&request, upload).await.map_err(AppError::from)?;
return Ok(Json(serde_json::to_value(preview).map_err(|e| AppError::from(e.to_string()))?));
}
if format == MongoDumpFormat::Directory {
let manifest = manifest.ok_or_else(|| AppError::bad_request("Directory preview requires a file manifest"))?;
let expected = manifest
.iter()
.filter(|f| f.path.ends_with(".metadata.json") || f.path.ends_with(".metadata.json.gz"))
.collect::<Vec<_>>();
if expected.len() != paths.len()
|| paths.iter().any(|(name, path)| {
!expected
.iter()
.any(|f| f.path == *name && std::fs::metadata(path).is_ok_and(|m| m.len() == f.size_bytes))
})
{
return Err(AppError::bad_request("Uploaded metadata does not match the file manifest"));
}
let preview =
mongodb_dump::prepare_mongodb_directory_catalog(upload, manifest, gzip).await.map_err(AppError::from)?;
return Ok(Json(serde_json::to_value(preview).map_err(|e| AppError::from(e.to_string()))?));
}
let path = match format {
MongoDumpFormat::Archive if paths.len() == 1 => paths[0].1.clone(),
MongoDumpFormat::Archive => return Err(AppError::from("Archive restore accepts one file".to_string())),
MongoDumpFormat::Directory => {
if paths.iter().any(|(name, _)| FilePath::new(name).components().count() < 2) {
return Err(AppError::from("Directory uploads require database-relative paths".to_string()));
}
upload.path().to_owned()
}
};
let preview = mongodb_dump::prepare_owned_mongodb_restore_source(
MongoRestoreSourceRequest { path: path.to_string_lossy().into_owned(), format, gzip },
upload,
)
.await
.map_err(AppError::from)?;
Ok(Json(serde_json::to_value(preview).map_err(|e| AppError::from(e.to_string()))?))
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct SourceRef {
source_ref: String,
}
pub async fn release_source(Json(request): Json<SourceRef>) -> Json<serde_json::Value> {
Json(serde_json::json!({ "released": mongodb_dump::release_mongodb_restore_source(&request.source_ref) }))
}
#[derive(Deserialize)]
pub struct DumpWrapper {
request: MongoDatabaseDumpRequest,
}
#[derive(Deserialize)]
pub struct RestoreWrapper {
request: MongoDatabaseRestoreRequest,
}
enum Task {
Dump(MongoDatabaseDumpRequest, WebExportFile),
Restore(MongoDatabaseRestoreRequest),
}
async fn start_task(state: Arc<WebState>, task: Task) -> Result<Json<serde_json::Value>, AppError> {
let id = match &task {
Task::Dump(request, _) => &request.task_id,
Task::Restore(request) => &request.task_id,
}
.clone();
let key = format!("mongo-database:{id}");
let channel = Arc::new(TransferProgressChannel::new());
{
let mut channels = state.transfer_progress_channels.write().await;
if channels.contains_key(&key) {
return Err(AppError::from("Database dump/restore task already exists".to_string()));
}
channels.insert(key.clone(), channel.clone());
}
let task_id = id.clone();
dbx_core::export_runtime::spawn_export_task(async move {
let mut latest = MongoDatabaseDumpProgress::initial(&task_id);
let mut callback = |progress: MongoDatabaseDumpProgress| {
latest = progress;
if latest.status == "running" {
if let Ok(json) = serde_json::to_string(&latest) {
channel.send(json, TransferReplayEventKind::Progress);
}
}
};
let result = match task {
Task::Dump(request, file) => {
let result = mongodb_dump::dump_mongodb_database(
&state.app,
&request,
|id| {
let id = id.to_string();
Box::pin(async move { transfer::is_cancelled(&id).await })
},
&mut callback,
)
.await;
if result.is_ok() {
state.export_files.write().await.insert(task_id.clone(), file);
}
result
}
Task::Restore(request) => {
mongodb_dump::restore_mongodb_database(
&state.app,
&request,
|id| {
let id = id.to_string();
Box::pin(async move { transfer::is_cancelled(&id).await })
},
&mut callback,
)
.await
}
};
if let Err(error) = result {
latest.error_message = Some(error);
if latest.status == "running" {
latest.status = "error".into();
latest.phase = "done".into();
}
}
if let Ok(json) = serde_json::to_string(&latest) {
channel.send(json, TransferReplayEventKind::Terminal);
}
transfer::clear_cancelled(&task_id).await;
tokio::spawn(async move {
tokio::time::sleep(Duration::from_secs(60)).await;
state.transfer_progress_channels.write().await.remove(&key);
});
});
Ok(Json(serde_json::json!({ "taskId": id })))
}
pub async fn start_dump(
State(state): State<Arc<WebState>>,
Json(body): Json<DumpWrapper>,
) -> Result<Json<serde_json::Value>, AppError> {
let mut request = body.request;
if request.format != MongoDumpFormat::Archive {
return Err(AppError::from("Web database exports use MongoDB archive format".to_string()));
}
let tmp = state.data_dir.join("tmp");
tokio::fs::create_dir_all(&tmp).await.map_err(|e| AppError::from(e.to_string()))?;
let ext = if request.gzip { "archive.gz" } else { "archive" };
let filename = export_download_filename(&request.file_path, &request.database, ext);
request.file_path =
tmp.join(format!("mongo_database_{}.{ext}", uuid::Uuid::new_v4())).to_string_lossy().into_owned();
let file = WebExportFile { file_path: request.file_path.clone(), download_filename: filename, format: ext.into() };
start_task(state, Task::Dump(request, file)).await
}
pub async fn start_restore(
State(state): State<Arc<WebState>>,
Json(body): Json<RestoreWrapper>,
) -> Result<Json<serde_json::Value>, AppError> {
start_task(state, Task::Restore(body.request)).await
}
pub async fn progress(
State(state): State<Arc<WebState>>,
Path(id): Path<String>,
) -> Result<Sse<impl futures::Stream<Item = Result<axum::response::sse::Event, std::convert::Infallible>>>, AppError> {
let channel = state
.transfer_progress_channels
.read()
.await
.get(&format!("mongo-database:{id}"))
.cloned()
.ok_or_else(|| AppError::from("Database task not found".to_string()))?;
Ok(crate::sse::sse_from_transfer_channel(channel))
}
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct CancelRequest {
task_id: String,
}
pub async fn cancel(Json(request): Json<CancelRequest>) -> Json<serde_json::Value> {
transfer::set_cancelled(&request.task_id).await;
Json(serde_json::json!({ "cancelled": true }))
}
@@ -17,13 +17,40 @@ use futures::stream::Stream;
use futures::StreamExt;
use serde::Deserialize;
use tokio::io::AsyncWriteExt;
use tokio_util::io::ReaderStream;
use crate::error::AppError;
use crate::routes::export_download::{attachment_content_disposition, export_download_filename};
use crate::sse::{TransferProgressChannel, TransferReplayEventKind};
use crate::state::{WebExportFile, WebState};
const MONGO_IMPORT_PROGRESS_TTL: Duration = Duration::from_secs(30);
struct RemoveFileOnDrop(PathBuf);
impl Drop for RemoveFileOnDrop {
fn drop(&mut self) {
let _ = std::fs::remove_file(&self.0);
}
}
struct ExportDownloadStream {
// Fields are dropped in declaration order, so Windows closes the file before deleting it.
chunks: ReaderStream<tokio::fs::File>,
_cleanup: RemoveFileOnDrop,
}
impl Stream for ExportDownloadStream {
type Item = std::io::Result<Bytes>;
fn poll_next(
self: std::pin::Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Option<Self::Item>> {
std::pin::Pin::new(&mut self.get_mut().chunks).poll_next(cx)
}
}
fn initial_import_progress(import_id: &str, started_at: Instant) -> MongoImportProgress {
MongoImportProgress {
import_id: import_id.to_string(),
@@ -153,11 +180,11 @@ pub async fn preview_import(
if let Some((source_ref, file_path)) = uploaded_file {
let file_path_str = file_path.to_string_lossy().to_string();
let format = match format.or_else(|| mongodb_import_export::format_from_path(&file_path_str).ok()) {
let format = match format {
Some(format) => format,
None => {
cleanup_uploaded_import_path(&file_path).await;
return Err(AppError::from("Unsupported MongoDB import file type".to_string()));
return Err(AppError::from("MongoDB import format is required".to_string()));
}
};
let preview = mongodb_import_export::preview_mongodb_import_file(&MongoImportPreviewRequest {
@@ -269,7 +296,7 @@ pub async fn execute_import(
let state_clone = state.clone();
tokio::spawn(async move {
let tx_clone = tx.clone();
let _result = mongodb_import_export::import_mongodb_file_core(
let result = mongodb_import_export::import_mongodb_file_core(
&app,
&req,
|id| {
@@ -279,6 +306,17 @@ pub async fn execute_import(
|progress| send_import_progress(&tx_clone, &progress),
)
.await;
if let Err(error) = result {
let current = tx.borrow().clone();
if let Ok(mut progress) = serde_json::from_str::<MongoImportProgress>(&current) {
if progress.status == MongoImportStatus::Running {
progress.phase = MongoImportPhase::Done;
progress.status = MongoImportStatus::Error;
progress.error_message = Some(error.display_message());
send_import_progress(&tx, &progress);
}
}
}
cleanup_uploaded_import_source(&req.file_path).await;
schedule_import_progress_cleanup(state_clone, req.import_id.clone());
});
@@ -402,26 +440,35 @@ pub async fn start_export(
let ext = match req.format {
MongoExportFormat::Csv => "csv",
MongoExportFormat::Ndjson => "ndjson",
MongoExportFormat::Bson if req.gzip => "bson.gz",
MongoExportFormat::Bson => "bson",
};
let tmp_file = tmp_dir.join(format!("mongo_export_{export_id}.{ext}"));
let tmp_file = tmp_dir.join(format!("mongo_export_{}.{ext}", uuid::Uuid::new_v4()));
let file_path = tmp_file.to_string_lossy().to_string();
let download_filename = export_download_filename(&req.file_path, &req.collection, ext);
req.file_path = file_path.clone();
state
.export_files
.write()
.await
.insert(export_id.clone(), WebExportFile { file_path, download_filename, format: ext.to_string() });
let export_file = WebExportFile { file_path, download_filename, format: ext.to_string() };
let channel_key = format!("mongo-export:{export_id}");
let tx = {
let mut channels = state.sse_channels.write().await;
channels.entry(export_id.clone()).or_insert_with(|| tokio::sync::broadcast::channel::<String>(256).0).clone()
let mut channels = state.transfer_progress_channels.write().await;
let channel = Arc::new(TransferProgressChannel::new());
channels.insert(channel_key.clone(), channel.clone());
channel
};
let app = state.app.clone();
let state_clone = state.clone();
dbx_core::export_runtime::spawn_export_task(async move {
let mut latest = MongoExportProgress {
export_id: req.export_id.clone(),
status: MongoExportStatus::Running,
documents_read: 0,
bytes_written: 0,
total_documents: None,
error_message: None,
elapsed_ms: 0,
};
let result = mongodb_import_export::export_mongodb_query_core(
&app,
&req,
@@ -430,32 +477,38 @@ pub async fn start_export(
Box::pin(async move { transfer::is_cancelled(&id).await })
},
|progress| {
if let Ok(json) = serde_json::to_string(&progress) {
let _ = tx.send(json);
latest = progress;
if latest.status == MongoExportStatus::Running {
if let Ok(json) = serde_json::to_string(&latest) {
tx.send(json, TransferReplayEventKind::Progress);
}
}
},
)
.await;
if let Err(error) = result {
let _ = tokio::fs::remove_file(&req.file_path).await;
state_clone.export_files.write().await.remove(&req.export_id);
let progress = MongoExportProgress {
export_id: req.export_id.clone(),
status: MongoExportStatus::Error,
documents_read: 0,
bytes_written: 0,
total_documents: None,
error_message: Some(error),
elapsed_ms: 0,
};
if let Ok(json) = serde_json::to_string(&progress) {
let _ = tx.send(json);
match result {
Ok(_) => {
// The download must exist before a terminal event can reach the browser.
state_clone.export_files.write().await.insert(req.export_id.clone(), export_file);
}
Err(error) => {
let _ = tokio::fs::remove_file(&req.file_path).await;
if latest.status != MongoExportStatus::Cancelled {
latest.status = MongoExportStatus::Error;
}
latest.error_message = Some(error);
}
}
if let Ok(json) = serde_json::to_string(&latest) {
tx.send(json, TransferReplayEventKind::Terminal);
}
transfer::clear_cancelled(&req.export_id).await;
tokio::time::sleep(Duration::from_secs(5)).await;
state_clone.remove_sse_channel(&req.export_id).await;
tokio::spawn(async move {
tokio::time::sleep(Duration::from_secs(30)).await;
state_clone.transfer_progress_channels.write().await.remove(&channel_key);
});
});
Ok(Json(serde_json::json!({ "exportId": export_id })))
@@ -465,12 +518,14 @@ pub async fn export_progress(
State(state): State<Arc<WebState>>,
Path(export_id): Path<String>,
) -> Result<Sse<impl Stream<Item = Result<axum::response::sse::Event, std::convert::Infallible>>>, AppError> {
let tx = {
let mut channels = state.sse_channels.write().await;
channels.entry(export_id).or_insert_with(|| tokio::sync::broadcast::channel::<String>(256).0).clone()
};
let rx = tx.subscribe();
Ok(crate::sse::sse_from_channel(rx))
let channel = state
.transfer_progress_channels
.read()
.await
.get(&format!("mongo-export:{export_id}"))
.cloned()
.ok_or_else(|| AppError::from("Export not found".to_string()))?;
Ok(crate::sse::sse_from_transfer_channel(channel))
}
pub async fn cancel_export(
@@ -491,17 +546,27 @@ pub async fn export_download(
.await
.remove(&export_id)
.ok_or_else(|| AppError::from("Export file not found".to_string()))?;
let data = tokio::fs::read(&export_file.file_path).await.map_err(|e| AppError::from(e.to_string()))?;
let _ = tokio::fs::remove_file(&export_file.file_path).await;
export_file_response(export_file).await
}
pub(crate) async fn export_file_response(export_file: WebExportFile) -> Result<Response, AppError> {
let file = tokio::fs::File::open(&export_file.file_path).await.map_err(|e| AppError::from(e.to_string()))?;
let content_type = match export_file.format.as_str() {
"csv" => "text/csv; charset=utf-8",
_ => "application/x-ndjson; charset=utf-8",
"ndjson" => "application/x-ndjson; charset=utf-8",
"bson.gz" | "archive.gz" => "application/gzip",
"bson" => "application/bson",
_ => "application/octet-stream",
};
let download = ExportDownloadStream {
chunks: ReaderStream::new(file),
_cleanup: RemoveFileOnDrop(PathBuf::from(&export_file.file_path)),
};
Response::builder()
.status(StatusCode::OK)
.header(header::CONTENT_TYPE, content_type)
.header(header::CONTENT_DISPOSITION, attachment_content_disposition(&export_file.download_filename))
.body(Body::from(data))
.body(Body::from_stream(download))
.map_err(|error| AppError::from(error.to_string()))
}
@@ -509,6 +574,152 @@ pub async fn export_download(
mod tests {
use super::*;
#[tokio::test]
async fn download_keeps_file_until_body_is_consumed_or_abandoned() {
for extension in ["bson", "bson.gz"] {
for consume in [true, false] {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join(format!("records.{extension}"));
let bytes = vec![42u8; 100_000];
std::fs::write(&path, &bytes).unwrap();
let response = export_file_response(WebExportFile {
file_path: path.to_str().unwrap().into(),
download_filename: format!("records.{extension}"),
format: extension.into(),
})
.await
.unwrap_or_else(|error| panic!("{}", error.message));
assert!(path.exists(), "download file was removed before streaming");
assert_eq!(
response.headers()[header::CONTENT_TYPE],
if extension == "bson" { "application/bson" } else { "application/gzip" }
);
assert!(response.headers()[header::CONTENT_DISPOSITION]
.to_str()
.unwrap()
.contains(&format!("records.{extension}")));
if consume {
assert_eq!(axum::body::to_bytes(response.into_body(), bytes.len()).await.unwrap(), bytes);
} else {
let mut stream = response.into_body().into_data_stream();
assert!(!stream.next().await.unwrap().unwrap().is_empty());
drop(stream);
}
assert!(!path.exists(), "download file leaked after the response ended");
}
}
}
#[tokio::test]
async fn export_failure_and_cancellation_are_replayed_to_late_subscribers() {
use axum::response::IntoResponse;
use dbx_core::{connection::AppState, storage::Storage};
for cancelled in [false, true] {
let directory = tempfile::tempdir().unwrap();
let storage = Storage::open(&directory.path().join("storage.db")).await.unwrap();
let state = Arc::new(WebState::for_tests(Arc::new(AppState::new(storage)), directory.path().into()));
let id = uuid::Uuid::new_v4().to_string();
let expected = if cancelled { MongoExportStatus::Cancelled } else { MongoExportStatus::Error };
if cancelled {
transfer::set_cancelled(&id).await;
}
let Json(started) = start_export(
State(state.clone()),
Json(StartExportRequest {
request: MongoExportRequest {
export_id: id.clone(),
connection_id: "missing".into(),
database: "test".into(),
collection: "empty".into(),
filter: None,
projection: None,
sort: None,
collation: None,
format: MongoExportFormat::Bson,
gzip: false,
include_header: false,
file_path: "empty.bson".into(),
execution_id: None,
},
}),
)
.await
.unwrap_or_else(|error| panic!("{}", error.message));
assert_eq!(started["exportId"], id);
let key = format!("mongo-export:{id}");
tokio::time::timeout(Duration::from_secs(5), async {
loop {
let latest = state.transfer_progress_channels.read().await.get(&key).unwrap().latest();
if latest.is_some_and(|json| {
serde_json::from_str::<MongoExportProgress>(&json).unwrap().status == expected
}) {
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.unwrap();
assert!(!state.export_files.read().await.contains_key(&id));
let response = export_progress(State(state.clone()), Path(id))
.await
.unwrap_or_else(|error| panic!("{}", error.message));
let mut stream = response.into_response().into_body().into_data_stream();
let chunk = tokio::time::timeout(Duration::from_secs(1), stream.next()).await.unwrap().unwrap().unwrap();
assert!(String::from_utf8_lossy(&chunk)
.contains(&format!("\"status\":{}", serde_json::to_string(&expected).unwrap())));
}
}
#[tokio::test]
async fn early_import_failure_finishes_progress_and_cleans_source() {
use dbx_core::{connection::AppState, storage::Storage};
let directory = tempfile::tempdir().unwrap();
let storage = Storage::open(&directory.path().join("storage.db")).await.unwrap();
let state = Arc::new(WebState::for_tests(Arc::new(AppState::new(storage)), directory.path().into()));
let upload_dir = import_upload_dir(directory.path());
std::fs::create_dir_all(&upload_dir).unwrap();
let path = upload_dir.join("empty.bson");
std::fs::write(&path, []).unwrap();
let id = uuid::Uuid::new_v4().to_string();
let Json(started) = execute_import(
State(state.clone()),
Json(ExecuteImportWrapper {
request: MongoImportRequest {
import_id: id.clone(),
connection_id: "missing".into(),
database: "test".into(),
collection: "empty".into(),
file_path: path.to_str().unwrap().into(),
source_ref: None,
format: mongodb_import_export::MongoImportFormat::Bson,
parse_options: Default::default(),
batch_size: 1,
execution_id: None,
},
}),
)
.await
.unwrap_or_else(|error| panic!("{}", error.message));
assert_eq!(started["importId"], id);
tokio::time::timeout(Duration::from_secs(5), async {
loop {
let progress = state.table_import_channels.read().await.get(&id).unwrap().borrow().clone();
let progress: MongoImportProgress = serde_json::from_str(&progress).unwrap();
if progress.status == MongoImportStatus::Error && !path.exists() {
assert_eq!(progress.phase, MongoImportPhase::Done);
assert!(progress.error_message.unwrap().contains("Batch size"));
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.unwrap();
}
#[test]
fn source_ref_rejects_path_traversal() {
let data_dir = std::env::temp_dir().join(format!("dbx-mongo-import-data-{}", uuid::Uuid::new_v4()));
+118
View File
@@ -0,0 +1,118 @@
use futures::StreamExt;
use reqwest::multipart::{Form, Part};
use std::process::Stdio;
use std::time::Duration;
#[tokio::test]
async fn database_dump_upload_and_progress_http_contract() {
let directory = tempfile::tempdir().unwrap();
let port = std::net::TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port();
let log_path = directory.path().join("server.log");
let mut server = tokio::process::Command::new(env!("CARGO_BIN_EXE_dbx-web"))
.env("DBX_DATA_DIR", directory.path())
.env("DBX_PORT", port.to_string())
.env("DBX_DISABLE_PASSWORD", "1")
.kill_on_drop(true)
.stdout(Stdio::null())
.stderr(std::fs::File::create(&log_path).unwrap())
.spawn()
.unwrap();
let base = format!("http://127.0.0.1:{port}/api/mongo/dump");
let client = reqwest::Client::new();
tokio::time::timeout(Duration::from_secs(20), async {
loop {
if client.get(format!("http://127.0.0.1:{port}/api/auth/check")).send().await.is_ok() {
break;
}
if let Some(status) = server.try_wait().unwrap() {
panic!("Web server exited {status}: {}", std::fs::read_to_string(&log_path).unwrap());
}
tokio::time::sleep(Duration::from_millis(50)).await;
}
})
.await
.unwrap();
let bad = Form::new()
.text("format", "archive")
.text("gzip", "false")
.part("file", Part::bytes(vec![0]).file_name("../escape.archive"));
let response = client.post(format!("{base}/source")).multipart(bad).send().await.unwrap();
assert!(!response.status().is_success());
assert!(response.text().await.unwrap().contains("Invalid dump upload path"));
let form = Form::new()
.text("format", "directory")
.text("gzip", "false")
.text("manifest", serde_json::json!([
{"path":"source/records.bson","sizeBytes":0},
{"path":"source/records.metadata.json","sizeBytes":r#"{"collectionName":"records","options":{},"indexes":[],"type":"collection"}"#.len()}
]).to_string())
.part(
"file",
Part::text(r#"{"collectionName":"records","options":{},"indexes":[],"type":"collection"}"#)
.file_name("source/records.metadata.json"),
);
let response = client.post(format!("{base}/source")).multipart(form).send().await.unwrap();
assert!(response.status().is_success(), "{}", response.text().await.unwrap());
let preview: serde_json::Value = response.json().await.unwrap();
assert_eq!(preview["databases"], serde_json::json!(["source"]));
assert_eq!(preview["collections"][0]["name"], "records");
assert!(preview["collections"][0]["documents"].is_null());
assert_eq!(preview["collections"][0]["sourceFiles"].as_array().unwrap().len(), 2);
let source_ref = preview["sourceRef"].as_str().unwrap();
let request = serde_json::json!({ "taskId":"acquire", "connectionId":"missing", "database":"target", "sourceDatabase":"source", "sourceRef":source_ref, "collections":["records"] });
let acquisition = Form::new()
.text("format", "directory")
.text("gzip", "false")
.text("request", request.to_string())
.part("file", Part::bytes(Vec::new()).file_name("source/records.bson"))
.part(
"file",
Part::text(r#"{"collectionName":"records","options":{},"indexes":[],"type":"collection"}"#)
.file_name("source/records.metadata.json"),
);
let response = client.post(format!("{base}/source/upload")).multipart(acquisition).send().await.unwrap();
assert!(response.status().is_success(), "{}", response.text().await.unwrap());
let acquired: serde_json::Value = response.json().await.unwrap();
let acquired_ref = acquired["sourceRef"].as_str().unwrap();
let id = uuid::Uuid::new_v4().to_string();
let response = client.post(format!("{base}/restore")).json(&serde_json::json!({ "request": {
"taskId": id, "connectionId": "missing", "database": "target", "sourceDatabase": "source", "sourceRef": acquired_ref
}})).send().await.unwrap();
assert!(response.status().is_success());
tokio::time::sleep(Duration::from_millis(100)).await;
let events = client.get(format!("{base}/progress/{id}")).send().await.unwrap();
assert!(events.status().is_success());
tokio::time::timeout(Duration::from_secs(5), async {
let mut stream = events.bytes_stream();
let mut text = String::new();
while let Some(chunk) = stream.next().await {
text.push_str(&String::from_utf8_lossy(&chunk.unwrap()));
if text.contains("\"status\":\"error\"") {
return;
}
}
panic!("Missing terminal restore event: {text}");
})
.await
.unwrap();
let response: serde_json::Value = client
.post(format!("{base}/source/release"))
.json(&serde_json::json!({ "sourceRef": source_ref }))
.send()
.await
.unwrap()
.json()
.await
.unwrap();
assert_eq!(response["released"], true);
client
.post(format!("{base}/source/release"))
.json(&serde_json::json!({"sourceRef":acquired_ref}))
.send()
.await
.unwrap();
server.kill().await.unwrap();
server.wait().await.unwrap();
}
+20 -7
View File
@@ -101,26 +101,39 @@ GridFS 工作台用于管理 Bucket 和文件:
## 导入与导出
Collection 文档提供独立的 CSV/JSON 导入和 CSV/NDJSON 导出,和关系型 Table Import 不是同一条工作流。CSV 每一行生成一个文档,嵌套值保持嵌套,`$oid`、`$date` 等 Extended JSON 类型可以保留。
Collection 文档提供独立的 CSV/JSON/NDJSON 与 BSON 导入导出,和关系型 Table Import 不是同一条工作流。CSV 每一行生成一个文档,嵌套值保持嵌套,BSON 备份保留原生 BSON 类型。
### 导入数据
1. 在集合上右键选择「导入数据」。
2. 选择 `.csv`、`.json` 或 `.ndjson` 文件。桌面端读取本地路径,Web 端上传到临时文件。
3. 对照预览调整编码、分隔符、表头、空值和类型策略。
2. 先明确选择格式,再选择 `.csv`、`.json`、`.ndjson`/`.jsonl`、`.bson` 或 `.bson.gz` 文件。桌面端读取本地路径,Web 端上传到临时文件;更换文件不会覆盖已选择的格式。
3. 对照预览检查数据。文本格式支持编码和类型设置,CSV 可逐列覆盖推断类型;BSON 导入不应用 CSV 文本转换。
4. 确认以追加方式写入。重复 `_id` 会停止导入,已提交的批次不会回滚。
CSV 按 MongoDB Compass:首行表头,嵌套字段用点号路径(`address.city`),数组索引(`tags[0]`、`matrix[0][1]`),ObjectId 写成 24 位十六进制,日期为 ISO-8601,空单元格为 null。导入采样前 1000 行推断列类型;`_id` 的十六进制会还原为 ObjectId,`recognizeObjectIdHex` 可将此规则扩展到其他列。JSON/NDJSON 使用 Extended JSON(`$oid`、`$date`)。全部按字符串会把每个单元格当文本。
类型保真:`Double` 导出再导入会变为 `Decimal128`(十进制文本无损)。非 `_id` 的 ObjectId 字段会往返为 String,除非打开 `recognizeObjectIdHex`。小的 `Int64` 可能收窄为 `Int32`;ISO 日期格式的字符串会变为 `DateTime`。Extended JSON 类型(`$binary`、`$regularExpression`、`$timestamp`、`$minKey`、`$maxKey`、`$code`)通过 JSON 文本单元格往返。空数组 `[]` 和空对象 `{}` 往返正确。
CSV 类型保真:`Double` 导出再导入会变为 `Decimal128`(十进制文本无损)。非 `_id` 的 ObjectId 字段会往返为 String,除非打开 `recognizeObjectIdHex`。小的 `Int64` 可能收窄为 `Int32`;ISO 日期格式的字符串会变为 `DateTime`。Extended JSON 类型(`$binary`、`$regularExpression`、`$timestamp`、`$minKey`、`$maxKey`、`$code`)通过 JSON 文本单元格往返。空数组 `[]` 和空对象 `{}` 往返正确。
首期限制:只支持追加(无清空或 upsert),无按字段类型覆盖,CSV 导出从前 10000 个文档发现字段(之后的字段会被静默丢弃;变化 schema 请改用 NDJSON),无 JSON 数组导出格式,不支持 Excel/GridFS/BSON 二进制导入,失败批次不会自动重试。
集合导入限制:只支持追加(无清空或 upsert),CSV 导出从前 10000 个文档发现字段(之后的字段会被静默丢弃;变化 schema 请改用 NDJSON 或 BSON),无 JSON 数组导出格式,不支持 Excel/GridFS 导入,失败批次不会自动重试。集合级 BSON 导入只恢复文档,不恢复配套元数据和索引。
### 导出数据
- 在集合上右键选择「导出数据」,再选 CSV 或 NDJSON,会像表导出一样直接打开保存文件对话框。
- 在集合上右键选择「导出数据」,再选 CSV、NDJSON、BSON 或 Gzip BSON,会像表导出一样直接打开保存文件对话框。
- CSV 嵌套对象写为点号列(`address.city`),数组写为索引列(`tags[0]`、`tags[1]`)。NDJSON 每行一个规范 Extended JSON 文档。
- 字段数超过 256 的 Collection 请改用 NDJSON。
- 字段数超过 256 的 Collection 请改用 NDJSON 或 BSON。
### 数据库备份与恢复
使用 MongoDB 原生驱动时,在数据库节点右键选择「备份数据库」或「恢复数据库」。数据库备份包含选中集合、集合选项、索引和视图。
- 桌面端支持官方目录布局与 MongoDB archive 格式,均可使用 gzip。Web 端导出 archive 文件,恢复时支持 archive 或上传目录。
- 目录备份包含 `<数据库>/<集合>.bson` 和 `.metadata.json`,或对应的 `.gz` 文件。MongoDB archive 是独立的流式格式,不是 ZIP 或 tar。
- 格式与 gzip 选项需明确选择。目录恢复的「读取备份」只读取清单和元数据;确认源库、目标库和集合后,才上传选中的数据文件。
- 恢复按单遍流式读取和写入进行。可选的 `objcheck` 默认关闭,开启后在同一遍读取中逐文档校验;BSON 边界、gzip 完整性和 archive 校验和检查始终保留。
- 默认追加写入。「恢复前删除选中的目标同名集合」只删除并重建这些集合。已有 `_id` 可能导致 `E11000`;失败或取消不会回滚此前的写入。
- Web 上传复用服务端 `DBX_MAX_UPLOAD_MB` 配置。桌面端直接读取本地文件,无需 HTTP 上传。
上述格式可与官方 `mongodump`/`mongorestore` 对支持的集合双向互通。本流程不提供 oplog 回放、时间点一致性保证、UUID 保留,也不处理内部数据库、时间序列集合和加密集合;这些场景请使用官方工具。
如果只需要当前可见选择,仍可从表格复制当前文档、选中文档或字段值为普通 JSON 或 Extended JSON。
+20 -7
View File
@@ -101,26 +101,39 @@ Dropping a bucket removes its files and chunks. Verify the bucket name and backu
## Import and Export
Collection documents have a dedicated CSV/JSON import and CSV/NDJSON export workflow. This is separate from relational Table Import: each CSV row becomes a document, nested values stay nested, and Extended JSON types such as `$oid` and `$date` can be preserved.
Collection documents have dedicated CSV/JSON/NDJSON and BSON import/export workflows. This is separate from relational Table Import: each CSV row becomes a document, nested values stay nested, and BSON dumps preserve native BSON types.
### Import data
1. Right-click a collection and choose **Import data**.
2. Select a `.csv`, `.json`, or `.ndjson` file. Desktop reads the local path; Web uploads it to a temporary server file.
3. Review encoding, delimiter, header, empty-value, and type settings against the preview.
2. Choose the format explicitly, then select a `.csv`, `.json`, `.ndjson`/`.jsonl`, `.bson`, or `.bson.gz` file. Desktop reads the local path; Web uploads it to a temporary server file. Changing the file does not override the selected format.
3. Review the preview. Text formats expose encoding and type settings; CSV supports per-column type overrides. BSON imports do not apply CSV text conversions.
4. Confirm append-only insert. Duplicate `_id` values stop the import. Batches that already committed are not rolled back.
CSV follows MongoDB Compass: a header row, nested fields as dotted paths (`address.city`), array indices (`tags[0]`, `matrix[0][1]`), ObjectId as 24-character hex, dates as ISO-8601, and empty cells as null. Import samples the first 1000 rows to infer column types; `_id` hex is restored as ObjectId, and `recognizeObjectIdHex` extends this to other columns. JSON/NDJSON use Extended JSON (`$oid`, `$date`). All strings keeps every cell as text.
Type fidelity: `Double` exports and reimports as `Decimal128` (lossless for decimal text). Non-`_id` ObjectId fields round-trip as String unless `recognizeObjectIdHex` is on. Small `Int64` may narrow to `Int32`; ISO-date-looking strings become `DateTime`. Extended JSON types (`$binary`, `$regularExpression`, `$timestamp`, `$minKey`, `$maxKey`, `$code`) round-trip through JSON-text cells. Empty array `[]` and empty object `{}` round-trip correctly.
CSV type fidelity: `Double` exports and reimports as `Decimal128` (lossless for decimal text). Non-`_id` ObjectId fields round-trip as String unless `recognizeObjectIdHex` is on. Small `Int64` may narrow to `Int32`; ISO-date-looking strings become `DateTime`. Extended JSON types (`$binary`, `$regularExpression`, `$timestamp`, `$minKey`, `$maxKey`, `$code`) round-trip through JSON-text cells. Empty array `[]` and empty object `{}` round-trip correctly.
First-release limits: append only (no truncate or upsert), no per-field type override, CSV export discovers fields from the first 10000 documents (later fields are silently dropped; use NDJSON for variable schemas), no JSON-array export format, no Excel/GridFS/BSON binary import, and no automatic retry of failed batches.
Collection-import limits: append only (no truncate or upsert), CSV export discovers fields from the first 10000 documents (later fields are silently dropped; use NDJSON or BSON for variable schemas), no JSON-array export format, no Excel/GridFS import, and no automatic retry of failed batches. Collection-level BSON import restores documents only, not the companion metadata or indexes.
### Export data
- Right-click a collection and choose **Export data**, then CSV or NDJSON. This opens the save-file dialog immediately, the same as table export.
- Right-click a collection and choose **Export data**, then CSV, NDJSON, BSON, or gzip BSON. This opens the save-file dialog immediately, the same as table export.
- CSV writes nested objects as dotted columns (`address.city`) and arrays as indexed columns (`tags[0]`, `tags[1]`). NDJSON writes one canonical Extended JSON document per line.
- Collections with more than 256 distinct fields should use NDJSON.
- Collections with more than 256 distinct fields should use NDJSON or BSON.
### Database backups
With the native MongoDB driver, right-click a database and choose **Back up database** or **Restore database**. Database backups include selected collections, collection options, indexes and views.
- Desktop supports the official dump directory layout and MongoDB archive format, optionally gzip-compressed. Web exports archive files and restores either archives or uploaded directories.
- A dump directory contains `<database>/<collection>.bson` and `.metadata.json`, or their `.gz` variants. MongoDB archive is a separate stream format, not ZIP or tar.
- Choose the format and gzip option explicitly. For directory restore, **Read backup** reads the catalog and metadata; selected data files are uploaded only after confirming the source database, target database and collections.
- Restore reads and writes data in one pass. Optional `objcheck` validates each BSON document in that same pass and defaults off. Framing, gzip and archive checksum checks remain enabled.
- Restore appends by default. **Drop selected target collections before restoring** deletes and recreates only those collections. Existing `_id` values can cause `E11000`; failures or cancellation do not roll back earlier writes.
- Web uploads use the shared `DBX_MAX_UPLOAD_MB` server setting. Desktop reads local files without an HTTP upload.
These formats interoperate with `mongodump`/`mongorestore` for supported collections. Oplog replay, point-in-time guarantees, UUID preservation, internal databases, time-series collections and encrypted collections are outside this workflow; use the official tools for those cases.
Copy the current document, selected documents, or field values as plain JSON or Extended JSON from the grid when you only need the visible selection.
+133
View File
@@ -0,0 +1,133 @@
# MongoDB Database Dump and Restore
Restore preparation is superseded by [Lightweight Preview](mongodb-database-restore-v2.md)
and [Streaming Restore](mongodb-database-restore-streaming.md): catalog discovery
reads metadata only, and data is checked while restoring, without a separate full
validation pass. The verification history below describes earlier implementations.
## Goal and Formats
Add database-scoped dump/restore to the existing MongoDB database tree node. Keep
the collection data workflow added in f21a47725 and reuse its BSON import/export.
Support the official Database Tools 100.18.0 formats:
- Directory: `<database>/<escaped-collection>.bson` and `.metadata.json`.
- Gzip directory: `.bson.gz` and `.metadata.json.gz`.
- Official `mongodump --archive` format 0.1, optionally gzip compressed.
Archive is the single-file transport for Web downloads. It is not a ZIP file.
Web restores accept an archive or a selected dump directory; desktop exports and
restores support both formats. Format and gzip are explicit user options.
References:
- https://www.mongodb.com/docs/database-tools/mongodump/
- https://www.mongodb.com/docs/database-tools/mongorestore/
- https://github.com/mongodb/mongo-tools/blob/100.18.0/common/archive/spec.md
- Official source: `common/archive`, `mongodump/metadata_dump.go`,
`mongodump/prepare.go`, `mongorestore/metadata.go`, `mongorestore/restore.go`.
## Ownership
- `dbx-core::mongodb_dump`: metadata, source preparation, database plans, progress,
cancellation, collection options, indexes and views.
- `dbx-core::mongodb_dump::archive`: official prelude, namespace segments,
terminators and per-namespace CRC64 validation. BSON and gzip use the existing
libraries; CRC uses a maintained implementation, not handwritten arithmetic.
- Existing `mongodb_import_export`: bounded BSON streaming and batched inserts.
- Web routes and Tauri commands: transport, writable-connection checks, task
notifications and source/download cleanup. No database logic in adapters.
- `MongoDatabaseDumpDialog`: source/destination, collection selection, restore
policy, confirmation, progress and cancellation.
## Export
1. Resolve the native MongoDB connection and list collection specifications.
2. Collect canonical Extended JSON metadata including options, index key order,
index options and collection type. Views have metadata but no BSON data file.
3. Dump each regular collection, including empty collections, into a private
temporary directory using the existing BSON exporter.
4. Publish the directory or stream it into an official archive. Archives include
the official header and CRC64 checksums over the original BSON bytes.
5. Publish only after success. Cancellation or errors discard temporary output.
## Restore
1. Explicitly select directory/archive and gzip. Directory preview sends a manifest
and metadata only; archive preview reads the prelude of the uploaded archive.
2. Select one source database and a target database; select all or some collections.
3. Confirm the selection and policies, then acquire the selected data. Uploads stay within owned staging
directories. Reject links, unsafe paths, duplicate namespaces, malformed BSON,
corrupt gzip, unsupported archive versions and CRC mismatches.
Use the same owned input for validation and restoration. Validate all selected
directory BSON files or full archive framing/checksums before any target writes.
Exact document counts are unknown during preview and populated during validation.
4. Default to appending documents. `dropExisting` applies only to selected target
collections, never to the whole target database. Enforce readonly and existing
production-execution guards in both transports/UI.
5. Create missing collections with saved options when enabled; existing collections
retain their options unless explicitly dropped. Never preserve source UUIDs or
issue `applyOps`. Strip obsolete index version/namespace fields before creating
indexes; let collection creation establish its automatic `_id` index.
6. Import documents through the existing batched BSON importer, then restore indexes.
Restore views in dependency order after their backing collections.
7. Report partial counts on failure/cancellation. Completed writes are not rolled
back. Releasing a source cannot delete data still owned by an active task.
## Initial Compatibility Boundary
This change covers ordinary, capped and clustered collections, GridFS collections,
validators, collations, index definitions and views. It is not a replica-set
snapshot: concurrent writes follow the normal no-oplog dump behavior.
Time-series/raw bucket restoration, Queryable Encryption, oplog replay, UUID
preservation and user/role migration require separate server-specific handling.
Reject unsupported collection types/options before writing; do not silently omit
them. Internal admin/config/local databases are outside this user-database workflow.
Ordinary user databases skip internal `system.*` collections except `system.js`.
## Verification
- Unit tests: canonical metadata, escaped names, archive CRC/EOF/interleaving,
corrupt/truncated input, unsafe directory entries, view dependencies.
- Official tool tests on an isolated MongoDB server: multiple collections, an empty
collection, capped/validated collections, compound/unique/TTL indexes and views;
both directions for directory/archive, plain/gzip, with BSON and metadata checks.
- Restore policies: append, selected-collection drop, unselected target preservation,
optional indexes/options, cancellation and readonly rejection.
- Web tests: multipart paths, source ownership, delayed progress subscription and
streaming downloads. Frontend tests: database menu, explicit format/options,
confirmation and backend dispatch. Inspect the running Web UI before delivery.
## Implementation Order
1. Core metadata/source plan and official archive codec.
2. Database export/restore orchestration and focused integration tests.
3. Web/Tauri adapters and lifecycle handling.
4. Database menu/dialog, localization and frontend tests.
5. Official tool round trips, UI inspection and final review.
## Verified Results
Verified against MongoDB Server 8.0.17 and Database Tools 100.18.0 on Windows:
- Both directions for all four combinations: directory/archive, plain/gzip.
- Six collections/views, including an empty collection, capped data, an escaped
collection name, a validator and collation, compound unique and TTL indexes,
and a view that depends on another view. BSON data and collection/index options
match the source after each restore.
- Selected-collection drop preserves unselected target data; disabling options
and indexes works; readonly and cancellation checks reject writes; prepared
changed original inputs are rejected and restore uses an owned input snapshot.
- Invalid metadata, cyclic views, truncated archives and bad archive CRC are rejected.
- The HTTP test launches the real Web binary and verifies multipart path rejection,
directory preparation, source release and replay of late task notifications.
- Desktop and Web compile checks, frontend type checking, focused UI tests and
desktop/mobile Playwright inspection completed. Screenshots are in
`output/playwright/database-{dump,restore}-desktop.png` and
`output/playwright/database-restore-mobile.png`.
The opt-in database test is `official_database_tools_round_trip_through_dbx` in
`crates/dbx-core/tests/mongodb_dump_restore.rs`. Set `DBX_MONGO_DUMP_TEST_URI` and
`DBX_MONGO_TOOLS_DIR` to run it against an isolated server.
+111
View File
@@ -0,0 +1,111 @@
# MongoDB Streaming Restore and Object Checks
This supersedes the mandatory full-validation stage in
`mongodb-database-restore-v2.md`. Metadata-first previews remain unchanged.
## Official Reference
Compared against the locally checked-out MongoDB Database Tools 100.18.0 sources:
- `common/db/bson_stream.go`: reads BSON frames into byte buffers.
- `mongorestore/restore.go`: `LoadNext` feeds raw BSON to buffered insertion workers.
When `objCheck` is enabled, the worker parses each document before inserting it.
- `mongorestore/mongorestore.go`: enables object checks when `--objcheck` is set.
Object checks are part of the single restore pass. They are not a separate full
backup scan. Server-side collection validators are a different concern and are
not disabled by this change.
## DBX Behavior
- Directory previews upload only the manifest and metadata. After confirmation,
upload the selected data files. Web retains the existing upload limit setting.
- Archive previews read only the prelude after uploading the archive.
- Confirmed restoration checks source identity and metadata, then reads, decompresses
and inserts data in one pass. Local desktop inputs are read directly; Web inputs
remain owned uploads held alive until the restore reader finishes.
- Restore data uses the existing official Rust BSON library's `RawDocumentBuf`
and the existing MongoDB driver's insert helper, generalized for serializable
BSON values. Raw bytes are not converted through JSON or an owned Document by default.
- `objcheck` is an optional boolean, default false. If enabled, the same official
BSON library parses each selected document in the reader before it enters the
insertion queue. The original bytes, not the decoded/re-encoded object, are inserted.
- A bounded two-batch queue connects blocking file/decoder work to asynchronous
writes. Each batch is capped by document count and an 8 MiB byte target; a single
valid BSON document may exceed that target up to the existing document-size cap.
- Archive namespace segments are demultiplexed directly into this queue, including
interleaved namespaces. There is no archive extraction/recompression/spool pass.
- Frame boundaries, truncation, gzip integrity and archive namespace EOF/CRC checks
remain mandatory while reading. They are not controlled by `objcheck`.
- Indexes and dependency-ordered views are restored after the data stream completes.
- Stream errors terminate the task and preserve partial write counts. Data preceding
a late error may already be restored; this no longer promises prevalidation before
selected-collection replacement. No rollback or automatic retry of the whole task.
- Cancellation closes the bounded queue, signals the reader and joins it. Progress
includes decoded bytes read even while skipping unselected archive namespaces.
## Ownership
- `mongodb_dump/source.rs`: catalog references, upload binding and file identity checks.
- `mongodb_dump/archive.rs`: official framing and streaming namespace/CRC handling.
- `mongodb_dump/stream.rs`: bounded raw BSON batches, optional SDK object checks and
lifecycle coordination with the native MongoDB insert helper.
- `mongodb_dump.rs`: metadata preflight, selected target initialization, indexes,
views and task summaries.
- Web/Tauri transport: unchanged restore command with the additional `objcheck` flag.
- Dialog: one optional object-check checkbox, default off. No additional warning,
prevalidation toggle or confirmation step is introduced.
- The execution entry is locked before awaiting production confirmation. A late
confirmation after dialog disposal cannot start a task. View dependency sorting
uses an explicit stack, so deep metadata chains cannot overflow the call stack.
## Verification
Focused tests cover default-off and enabled object checks, exact raw BSON bytes,
metadata-only previews, partial restore on late corruption, gzip/archive integrity,
official-tool round trips, read-only mode and cancellation. An opt-in compressed
parser throughput smoke test compares raw streaming, streaming with object checks
and the former double-parse pattern without database/network overhead.
### Local Results (2026-09-15)
- 7 focused core tests passed, including the opt-in parser benchmark and ordering
a 20,000-entry dependency chain without recursion.
- 8 integration tests passed against isolated MongoDB 8.0.17 and Database Tools
100.18.0. Directory/archive, with and without gzip, round-tripped in both
directions with matching BSON, collection options, indexes and views.
- Cancellation stopped after the first 100-document batch of a 201-document
fixture. Duplicate-key tests verified partial success counts and stop/continue
behavior. Late BSON corruption, archive CRC failure and truncation preserved
partial write counts instead of requiring an initial full scan.
- Collection-level BSON and gzip regressions each round-tripped 1,207 documents
through official tools with matching BSON bytes.
- 32 frontend tests and Vue type checking passed, including double-submit,
disposed-dialog confirmation and declined-confirmation regressions. The Web
backend build and HTTP upload/progress smoke test also passed.
Parser-only smoke test: Windows x64, unoptimized Rust test build, an in-memory
gzip fixture containing 8,192 documents (27.56 MiB decoded, 64 string fields per
document). The old pattern is simulated by two object-parsing passes, not by
running an older DBX binary.
| Path | Passes | Elapsed |
| --- | --- | --- |
| Raw streaming, default | 1 | 129 ms |
| Streaming with object checks | 1 | 851 ms |
| Simulated former double parse | 2 | 2,315 ms |
These figures exclude upload, disk I/O, MongoDB writes and index building. They
are not an end-to-end restore speed guarantee; the user's 5.28 GiB backup has not
been benchmarked with this implementation.
No active user task or configured data source was used by these tests.
## Duplicate Keys
Restore inserts documents; it does not upsert or overwrite matching IDs. Without
`dropExisting`, existing target documents (including prior partial restores) may
cause `E11000`. Object checks do not disable server-side unique indexes. With
`dropExisting`, only selected target collections are deleted and recreated.
The confirmation-entry race was reproduced and fixed, but that alone does not
identify the cause of a particular duplicate-key report.
+238
View File
@@ -0,0 +1,238 @@
# MongoDB Database Restore: Lightweight Preview
The full-prevalidation/snapshot sections below are historical and superseded by
[Streaming Restore and Object Checks](mongodb-database-restore-streaming.md).
Metadata-first preview remains in use; restore no longer scans the backup twice.
Status: metadata-first restore flow implemented. This supersedes the restore preparation
flow in `mongodb-database-dump-restore.md`, not its dump formats or compatibility
boundary. Database export and collection import/export are unchanged.
## Implemented Scope
- Directory catalog requests contain only a bounded file manifest and metadata.
Selected BSON files are uploaded after the user confirms the restore options.
- Desktop catalog discovery reads metadata/stat information without copying BSON.
Archive catalog discovery stops after the prelude, without certifying payloads.
- Full validation now runs inside the confirmed restore task before target access
or writes. Directory validation retains compressed inputs; archive validation
writes only selected namespaces as compressed, multi-member BSON streams.
- Upload byte progress, validation byte/document progress, cancellation, source
identity checks and a temporary disk free-space reserve are implemented.
- The existing upload environment setting is exposed to the frontend for prechecks;
SQL and other table upload configurations are unchanged.
The broader lifecycle proposals below are not all part of this implementation:
there is no persistent task resume/background-task UI, abandoned-task startup
cleanup, or per-collection opt-out for unsupported metadata types yet. Unsupported
types retain the existing strict rejection policy. Directory upload is followed
by an explicit restore-start request, rather than a persistent upload/seal job.
## Problem
The current `prepare_mongodb_restore_source` uploads the entire directory,
decompresses and validates every BSON document, and writes an uncompressed
snapshot before presenting the source database and collection choices. The UI
waits for a single HTTP response without preparation progress or cancellation.
A 211-file gzip backup totaling 5,672,833,827 bytes exposed the problem: preparation
spent tens of minutes doing disk and CPU work before confirmation. Process I/O
counters confirmed continuing writes; directory file sizes on Windows did not
reliably reflect writes through still-open handles. No target database writes
occurred in that preparation phase.
## User Flow
1. Show the target connection/database immediately, prefilled from the tree node.
Select directory/archive and gzip explicitly; do not infer or change options.
2. Read the backup catalog only. Populate source database, collection names,
indexes/options availability, types and source file sizes. Unknown document
counts stay unknown; do not scan BSON to obtain an exact count.
3. Choose one source database, target database and collections. Configure append
or selected-collection replacement, options, indexes and write-error policy.
4. Confirm the frozen selection and destructive policy. Apply the existing
production guard. Changing any choice requires a new confirmation.
5. Acquire the selected data, validate it, then restore. Show each phase separately.
No target mutation is allowed before validation of the selected restore set
completes successfully.
### Directory
- Web: enumerate `File` references and relative paths locally. Send a bounded file
manifest plus `.metadata.json`/`.metadata.json.gz` contents for core parsing.
Do not upload or read `.bson`/`.bson.gz` bodies before confirmation. Metadata
uploads are also size-limited; gzip metadata has a decompressed size limit.
- Desktop: enumerate local paths and read metadata directly, without copying or
decompressing collection data during preview.
- Missing optional metadata permits data-only restore with an explicit warning;
names come from the existing official filename decoder. Reject ambiguous names,
duplicate namespaces, unsafe paths, missing required BSON and option mismatch.
- After confirmation, Web uploads only the selected BSON and associated metadata.
Validate the uploaded manifest against the confirmed catalog. Extra, missing,
duplicate or changed metadata requires correction/review, not silent changes.
- Unselected directory collection data is not opened or validated. Unsupported
collections can be displayed as unavailable without blocking unrelated supported
selections. Unsafe source layout still rejects the source as a whole.
### Archive
- Web retains one full upload of the original archive in this iteration, with
byte progress and cancellation. This transport cost happens before selection,
but it does not include a full BSON scan or full archive extraction.
- Desktop reads only the local archive prelude during preview.
- Core reads magic, version and bounded collection metadata up to the prelude
terminator, then stops. For gzip this decompresses only the prefix needed for
the prelude. Preview does not certify archive EOF or gzip integrity.
- Collection data can be interleaved. After confirmation, validation must traverse
the archive, including framing, namespace EOFs and checksums, even when only some
collections are selected. Do not promise random access or selected-only reads
for an archive. Restore only the selected namespaces.
- Whole-archive integrity errors still fail validation. Unsupported but unselected
collection metadata need not fail an otherwise supported selection.
The archive layout above follows the checked-out official Database Tools 100.18.0
source, `common/archive/spec.md`: metadata is in the prelude, followed by interleaved
namespace segments and namespace EOF/checksum records.
## Source Ownership and Validation
Split the current single prepared-source concept into two internal states:
- `CatalogSource`: metadata, file manifest and source identity; not validated data.
- `ValidatedRestoreSource`: owned input plus validated selected namespaces/counts,
bound to a frozen restore plan. Only this state may enter the write phase.
Keep opaque references on transports; do not accept arbitrary Web server paths.
Never let a catalog reference or client-provided `validated` flag bypass validation.
Cap catalog/session counts and metadata memory independently of BSON upload limits.
For directory input, retain an immutable snapshot in its original representation:
gzip stays gzip. Validate by streaming through it and discarding decoded bytes;
restore rereads the same owned snapshot. This intentionally trades a second gzip
decode for avoiding a potentially enormous uncompressed temporary copy. Reuse the
existing BSON importer and multi-member gzip support.
For archive input, validation may spool selected namespaces as compressed BSON
files for the existing collection importer. Use bounded buffers and bounded open
writers; complete gzip members per namespace segment allow interleaved namespaces
without an unbounded file-handle cache. Do not spool unselected namespaces or retain
an additional whole uncompressed archive. Finish all checks before using the spool.
Web owns uploaded inputs. Desktop acquires an owned snapshot after confirmation;
it must not validate a mutable original and later restore different bytes from it.
Check metadata/source identity against preview and fail/review changes. After
snapshot acquisition, validation and restoration use that same snapshot.
Before writes, recheck target writability, collection types, selected view
dependencies and the confirmed options. Missing dependencies must be resolved
explicitly or rejected; never silently add collections to the selection.
Corrupt BSON, gzip or archive framing is fatal regardless of the write-error policy.
## Job and Transport Contract
Use one restore task identity and freeze the plan at confirmation:
`catalog -> confirmed -> acquiring -> validating -> restoring -> indexes/views -> done`
Any active phase can become `failed` or `cancelled`. Target writes only start in
`restoring`. The metadata catalog endpoint stays bounded; it must not synchronously
wait for a full restore-source validation pass.
- Web catalog inspection accepts a directory manifest/metadata or an uploaded
archive reference. Raw BSON is never treated as catalog metadata.
- Confirmation creates a task with the selected source, target and policies.
Upload selected directory files into that task, then seal its input. Missing files
or incomplete uploads prevent validation from starting.
- Repeated submit/seal requests for the same task cannot start a second restore.
- Reuse the existing replayable task progress transport. Reconnecting must recover
the latest state and terminal result, not automatically re-execute writes.
- Cancellation is an explicit backend operation checked during upload, decode,
validation, writes and index operations. Browser abort alone is not sufficient.
- Closing an active dialog offers cancel or continue in background. A transient
connection loss does not silently cancel or retry database writes. Server restart
interrupts a task; automatic mid-restore resume is outside this change.
- Adapt Web and Tauri to the same core plan/validation phases. Do not move MongoDB
metadata parsing or restore policy into Vue or HTTP handlers.
Progress includes phase, current collection, collection counts, input bytes read,
validated uncompressed bytes, documents validated/restored and elapsed time.
Upload has a byte total; directory validation uses selected physical input size.
Archive scan progress uses the whole physical archive size, not selected sizes.
Do not compare compressed input totals to uncompressed output or display invented
overall percentages. Report partial writes on cancellation/failure; no rollback is
promised after the write phase begins.
## Resource Limits
- Reuse `DBX_MAX_UPLOAD_MB` for Web uploads. The current test service uses 8192 MiB;
this design does not change the project default or SQL upload settings.
- Expose the effective upload limit for frontend checks before sending data. Apply
it to the selected directory upload total or the whole archive, plus a separate
bounded allowance for multipart framing. Enforce actual received bytes too and
return structured HTTP 413 errors.
- Use server-owned `data_dir/tmp` storage for Web inputs and validation spools;
avoid silently expanding large backups into a different OS temporary volume.
- Check available disk space before known-size acquisition and periodically while
spooling. Keep an explicit free-space reserve and bounded decompression/metadata
memory. Compressed file size and gzip trailer size are not safe expansion bounds.
- On failure, cancellation or expiry, remove only owned temporary data after active
workers release it. Add startup cleanup for abandoned task directories. Never
remove user originals, and do not change source ownership while a task is active.
- Desktop has no Web upload cap, but shares disk, format and cancellation checks.
## Code Boundaries
- `dbx-core/src/mongodb_dump/directory.rs`: separate metadata catalog discovery from
selected input validation. No data reader calls during directory inspection.
- `dbx-core/src/mongodb_dump/archive.rs`: separate prelude inspection from full
validation/demultiplexing; preserve official CRC and interleaving compatibility.
- `dbx-core/src/mongodb_dump.rs`: source states, frozen plans, acquisition ownership,
preflight validation and restore phase transitions.
- Web routes/Tauri commands: file transport, task lifecycle, progress/cancel and
limits. Keep existing export tasks and collection workflows unchanged.
- `MongoDatabaseDumpDialog.vue`: immediate target selection, lightweight catalog,
confirmation and visible phase-specific progress. Format selection stays explicit.
## Implementation Order and Acceptance
1. Split catalog inspection from validation, with tests proving directory preview
never reads BSON bodies and archive preview stops at the prelude terminator.
2. Introduce confirmed plans and selected acquisition/validation, avoiding whole
uncompressed temporary snapshots. Validate all selected data before any drops.
3. Add Web metadata-first directory transport, task progress, cancellation, replay
and upload prechecks. Connect the same phases to Tauri.
4. Update the dialog and run official-tool compatibility plus large-file tests.
Required regressions:
- 211-file directory preview sends only manifest/metadata; a huge BSON file does
not increase preview work except for filename/size enumeration.
- Explicit gzip mismatch is reported before BSON upload when filenames identify it.
- Confirming a subset uploads/restores only that subset; confirm/drop never affects
unselected target collections. Unknown counts remain unknown until validation.
- Corruption in the last selected collection fails before any target mutation.
- Interleaved archive and multi-member gzip still round-trip with official tools.
- Highly compressible input does not produce a whole uncompressed temporary copy;
memory remains bounded and progress advances during validation.
- Cancel upload/validation, disconnect/reconnect, duplicate submit and disk-full
cases do not leak workers or start unintended writes. Later write failures report
partial completion honestly.
- Modified desktop inputs or changed uploaded metadata cannot invalidate the
confirmed selection silently. SQL/table imports and database exports regress cleanly.
## Verification Results
- Eight core/integration tests passed, including official MongoDB Tools 100.18.0
round trips for directory/archive with and without gzip, unchanged BSON values,
options/indexes/views and rejection of corrupt data before selected-collection drops.
- Metadata-only directory preview accepts a 5 GiB manifest without BSON bodies;
changed source files and mismatched confirmed uploads are rejected.
- A synthetic 211-file directory manifest representing 5 GiB sent 8,080 bytes of
gzip metadata and returned 105 collections in 253 ms through the real HTTP route.
This was not a benchmark of the user's original backup, which was unavailable.
- Twenty-eight focused frontend tests passed. A browser workflow against an
isolated MongoDB instance previewed an official gzip directory, selected one
collection, confirmed a different target database and restored two documents
and two indexes. Desktop/mobile screenshots are under `output/playwright/metadata-v2-*`.
- Final Web HTTP integration, Vue type checking and Tauri compile checking passed.
The main local Web service was restarted with `DBX_MAX_UPLOAD_MB=8192`.
+1
View File
@@ -30,6 +30,7 @@ pub mod mcp;
pub mod mcp_bridge;
pub mod mcp_http_server;
pub mod mongo_cmd;
pub mod mongodb_dump;
pub mod mongodb_import_export;
#[cfg(feature = "mq-admin")]
pub mod mq_cmd;
+96
View File
@@ -0,0 +1,96 @@
use std::sync::Arc;
use tauri::{AppHandle, Emitter, State};
use crate::commands::connection::{ensure_connection_writable, AppState};
use dbx_core::mongodb_dump::{
self, MongoDatabaseDumpProgress, MongoDatabaseDumpRequest, MongoDatabaseRestoreRequest, MongoDumpCatalog,
MongoRestoreSourcePreview, MongoRestoreSourceRequest,
};
#[tauri::command]
pub async fn inspect_mongodb_database_dump(
state: State<'_, Arc<AppState>>,
connection_id: String,
database: String,
) -> Result<MongoDumpCatalog, String> {
mongodb_dump::inspect_mongodb_database_dump(&state, &connection_id, &database).await
}
#[tauri::command]
pub async fn prepare_mongodb_restore_source(
request: MongoRestoreSourceRequest,
) -> Result<MongoRestoreSourcePreview, String> {
mongodb_dump::prepare_mongodb_restore_source(request).await
}
#[tauri::command]
pub fn release_mongodb_restore_source(source_ref: String) -> bool {
mongodb_dump::release_mongodb_restore_source(&source_ref)
}
#[tauri::command]
pub async fn dump_mongodb_database(
app: AppHandle,
state: State<'_, Arc<AppState>>,
request: MongoDatabaseDumpRequest,
) -> Result<MongoDatabaseDumpProgress, String> {
let state = state.inner().clone();
let runtime = tokio::runtime::Handle::current();
tokio::task::spawn_blocking(move || {
runtime.block_on(async move {
let result = mongodb_dump::dump_mongodb_database(
&state,
&request,
|id| {
let id = id.to_string();
Box::pin(async move { dbx_core::transfer::is_cancelled(&id).await })
},
|progress| {
let _ = app.emit("mongo-database-dump-progress", progress);
},
)
.await;
dbx_core::transfer::clear_cancelled(&request.task_id).await;
result
})
})
.await
.map_err(|e| e.to_string())?
}
#[tauri::command]
pub async fn restore_mongodb_database(
app: AppHandle,
state: State<'_, Arc<AppState>>,
request: MongoDatabaseRestoreRequest,
) -> Result<MongoDatabaseDumpProgress, String> {
ensure_connection_writable(&state, &request.connection_id, "Restore").await?;
let state = state.inner().clone();
let runtime = tokio::runtime::Handle::current();
tokio::task::spawn_blocking(move || {
runtime.block_on(async move {
let result = mongodb_dump::restore_mongodb_database(
&state,
&request,
|id| {
let id = id.to_string();
Box::pin(async move { dbx_core::transfer::is_cancelled(&id).await })
},
|progress| {
let _ = app.emit("mongo-database-dump-progress", progress);
},
)
.await;
dbx_core::transfer::clear_cancelled(&request.task_id).await;
result
})
})
.await
.map_err(|e| e.to_string())?
}
#[tauri::command]
pub async fn cancel_mongodb_database_dump(task_id: String) -> bool {
dbx_core::transfer::set_cancelled(&task_id).await;
true
}
+6
View File
@@ -2060,6 +2060,12 @@ pub fn run() {
commands::mongodb_import_export::cancel_mongodb_import,
commands::mongodb_import_export::export_mongodb_query,
commands::mongodb_import_export::cancel_mongodb_export,
commands::mongodb_dump::inspect_mongodb_database_dump,
commands::mongodb_dump::prepare_mongodb_restore_source,
commands::mongodb_dump::release_mongodb_restore_source,
commands::mongodb_dump::dump_mongodb_database,
commands::mongodb_dump::restore_mongodb_database,
commands::mongodb_dump::cancel_mongodb_database_dump,
commands::redis_cmd::redis_list_databases,
commands::redis_cmd::redis_scan_keys,
commands::redis_cmd::redis_scan_keys_batch,