mirror of
https://github.com/t8y2/dbx.git
synced 2026-10-19 16:16:45 +00:00
feat(mongodb): bson dumps and database backup/restore
This commit is contained in:
Generated
+19
@@ -1701,6 +1701,21 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crc"
|
||||
version = "3.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d"
|
||||
dependencies = [
|
||||
"crc-catalog",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crc-catalog"
|
||||
version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853"
|
||||
|
||||
[[package]]
|
||||
name = "crc16"
|
||||
version = "0.4.0"
|
||||
@@ -2091,6 +2106,7 @@ dependencies = [
|
||||
"calamine",
|
||||
"cbc 0.1.2",
|
||||
"chrono",
|
||||
"crc",
|
||||
"csv",
|
||||
"dbx-sqlite-worker",
|
||||
"deadpool-postgres",
|
||||
@@ -2134,6 +2150,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"serde_yaml",
|
||||
"serde_yaml_ng",
|
||||
"sha1 0.10.7",
|
||||
"sha2 0.10.9",
|
||||
"sqlparser",
|
||||
"ssfmt",
|
||||
@@ -2215,6 +2232,7 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
"tempfile",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tower-http",
|
||||
@@ -6899,6 +6917,7 @@ dependencies = [
|
||||
"js-sys",
|
||||
"log",
|
||||
"mime",
|
||||
"mime_guess",
|
||||
"native-tls",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
|
||||
Generated
+17
@@ -954,6 +954,21 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crc"
|
||||
version = "3.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d"
|
||||
dependencies = [
|
||||
"crc-catalog",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crc-catalog"
|
||||
version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853"
|
||||
|
||||
[[package]]
|
||||
name = "crc16"
|
||||
version = "0.4.0"
|
||||
@@ -1243,6 +1258,7 @@ dependencies = [
|
||||
"calamine",
|
||||
"cbc 0.1.2",
|
||||
"chrono",
|
||||
"crc",
|
||||
"csv",
|
||||
"dbx-sqlite-worker",
|
||||
"deadpool-postgres",
|
||||
@@ -1282,6 +1298,7 @@ dependencies = [
|
||||
"serde_json",
|
||||
"serde_yaml",
|
||||
"serde_yaml_ng",
|
||||
"sha1 0.10.7",
|
||||
"sha2 0.10.9",
|
||||
"sqlparser",
|
||||
"ssfmt",
|
||||
|
||||
@@ -0,0 +1,438 @@
|
||||
<script setup lang="ts">
|
||||
import { computed, nextTick, onBeforeUnmount, ref, watch } from "vue";
|
||||
import { useI18n } from "vue-i18n";
|
||||
import { ArrowLeft, ArrowRight, DatabaseBackup, FolderOpen, Loader2, Play, Square, X } from "@lucide/vue";
|
||||
import { Dialog, DialogFooter, DialogHeader, DialogScrollContent, DialogTitle } from "@/components/ui/dialog";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Input } from "@/components/ui/input";
|
||||
import { Label } from "@/components/ui/label";
|
||||
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
|
||||
import { useConnectionStore } from "@/stores/connectionStore";
|
||||
import { connectionIsEffectivelyReadOnly } from "@/lib/database/readOnlyWriteAccess";
|
||||
import { executeWithProductionContextGuard } from "@/lib/database/productionExecutionGuard";
|
||||
import { isTauriRuntime } from "@/lib/backend/tauriRuntime";
|
||||
import { uuid } from "@/lib/common/utils";
|
||||
import * as api from "@/lib/backend/api";
|
||||
|
||||
const props = defineProps<{ connectionId: string; database: string; mode: "dump" | "restore" }>();
|
||||
const open = defineModel<boolean>("open", { default: false });
|
||||
const { t } = useI18n();
|
||||
const store = useConnectionStore();
|
||||
const desktop = isTauriRuntime();
|
||||
const restoring = computed(() => props.mode === "restore");
|
||||
const connection = computed(() => store.getConfig(props.connectionId));
|
||||
const readonly = computed(() => restoring.value && connectionIsEffectivelyReadOnly(connection.value));
|
||||
const format = ref<api.MongoDumpFormat>("archive");
|
||||
const gzip = ref(false);
|
||||
const catalog = ref<api.MongoDumpCatalog | null>(null);
|
||||
const sourceRef = ref<string | null>(null);
|
||||
const source = ref<api.MongoDumpSourceInput | null>(null);
|
||||
const sourceName = ref("");
|
||||
const sourceDatabase = ref("");
|
||||
const targetDatabase = ref(props.database);
|
||||
const outputPath = ref(desktop ? "" : `${props.database}.archive`);
|
||||
const selected = ref<string[]>([]);
|
||||
const dropExisting = ref(false);
|
||||
const restoreOptions = ref(true);
|
||||
const restoreIndexes = ref(true);
|
||||
const stopOnError = ref(true);
|
||||
const objcheck = ref(false);
|
||||
const batchSize = ref(500);
|
||||
const loading = ref(false);
|
||||
const confirming = ref(false);
|
||||
const running = ref(false);
|
||||
const cancelling = ref(false);
|
||||
const error = ref("");
|
||||
const step = ref<"configure" | "confirm" | "result">("configure");
|
||||
const progress = ref<api.MongoDatabaseDumpProgress | null>(null);
|
||||
const taskId = ref("");
|
||||
const fileInput = ref<HTMLInputElement | null>(null);
|
||||
let generation = 0;
|
||||
let readController: AbortController | null = null;
|
||||
let uploadController: AbortController | null = null;
|
||||
const readUpload = ref<{ loaded: number; total: number } | null>(null);
|
||||
function gib(value: number) {
|
||||
return (value / 1024 ** 3).toFixed(2);
|
||||
}
|
||||
|
||||
const entries = computed(() => catalog.value?.collections.filter((entry) => !restoring.value || entry.database === sourceDatabase.value) ?? []);
|
||||
const allSelected = computed(() => entries.value.length > 0 && selected.value.length === entries.value.length);
|
||||
const canReview = computed(
|
||||
() =>
|
||||
!!catalog.value &&
|
||||
!loading.value &&
|
||||
!confirming.value &&
|
||||
!running.value &&
|
||||
!readonly.value &&
|
||||
(entries.value.length === 0 || selected.value.length > 0) &&
|
||||
(restoring.value ? !!sourceRef.value && !!targetDatabase.value.trim() && (!catalog.value.databases.length || catalog.value.databases.includes(sourceDatabase.value)) : !!outputPath.value.trim()),
|
||||
);
|
||||
|
||||
function releaseSource() {
|
||||
const reference = sourceRef.value;
|
||||
sourceRef.value = null;
|
||||
if (reference) void api.releaseMongodbRestoreSource(reference);
|
||||
}
|
||||
function invalidate() {
|
||||
generation += 1;
|
||||
releaseSource();
|
||||
catalog.value = null;
|
||||
selected.value = [];
|
||||
error.value = "";
|
||||
step.value = "configure";
|
||||
}
|
||||
function cleanup() {
|
||||
generation += 1;
|
||||
readController?.abort();
|
||||
if (!running.value) releaseSource();
|
||||
}
|
||||
onBeforeUnmount(cleanup);
|
||||
watch(
|
||||
open,
|
||||
async (value) => {
|
||||
if (!value) {
|
||||
cleanup();
|
||||
return;
|
||||
}
|
||||
if (restoring.value) return;
|
||||
const current = ++generation;
|
||||
loading.value = true;
|
||||
try {
|
||||
await store.ensureConnected(props.connectionId);
|
||||
const result = await api.inspectMongodbDatabaseDump(props.connectionId, props.database);
|
||||
if (current !== generation) return;
|
||||
catalog.value = result;
|
||||
selected.value = result.collections.map((entry) => entry.name);
|
||||
} catch (cause) {
|
||||
if (current === generation) error.value = String(cause instanceof Error ? cause.message : cause);
|
||||
} finally {
|
||||
if (current === generation) loading.value = false;
|
||||
}
|
||||
},
|
||||
{ immediate: true },
|
||||
);
|
||||
watch([format, gzip], () => {
|
||||
if (restoring.value) invalidate();
|
||||
else outputPath.value = desktop ? "" : `${props.database}.archive${gzip.value ? ".gz" : ""}`;
|
||||
});
|
||||
watch(format, () => {
|
||||
if (restoring.value) {
|
||||
source.value = null;
|
||||
sourceName.value = "";
|
||||
}
|
||||
});
|
||||
|
||||
function setFormat(value: unknown) {
|
||||
if (value === "directory" || value === "archive") format.value = value;
|
||||
}
|
||||
function setSourceDatabase(value: unknown) {
|
||||
if (typeof value !== "string") return;
|
||||
sourceDatabase.value = value;
|
||||
selected.value = entries.value.map((entry) => entry.name);
|
||||
}
|
||||
function selectAll(event: Event) {
|
||||
selected.value = (event.target as HTMLInputElement).checked ? entries.value.map((entry) => entry.name) : [];
|
||||
}
|
||||
function selectCollection(name: string, event: Event) {
|
||||
selected.value = (event.target as HTMLInputElement).checked ? [...selected.value, name] : selected.value.filter((value) => value !== name);
|
||||
}
|
||||
|
||||
async function chooseSource() {
|
||||
if (!desktop) {
|
||||
fileInput.value?.click();
|
||||
return;
|
||||
}
|
||||
const { open: choose } = await import("@tauri-apps/plugin-dialog");
|
||||
const path = await choose({ directory: format.value === "directory", multiple: false });
|
||||
if (typeof path === "string") {
|
||||
invalidate();
|
||||
source.value = path;
|
||||
sourceName.value = path;
|
||||
}
|
||||
}
|
||||
function chooseFiles(event: Event) {
|
||||
const input = event.target as HTMLInputElement;
|
||||
const files = Array.from(input.files ?? []);
|
||||
input.value = "";
|
||||
if (!files.length) return;
|
||||
invalidate();
|
||||
source.value = format.value === "directory" ? files : files[0]!;
|
||||
sourceName.value = format.value === "directory" ? `${(files[0]!.webkitRelativePath || files[0]!.name).split("/")[0]} (${files.length})` : files[0]!.name;
|
||||
}
|
||||
async function readSource() {
|
||||
if (!source.value) return;
|
||||
releaseSource();
|
||||
const current = ++generation;
|
||||
loading.value = true;
|
||||
error.value = "";
|
||||
catalog.value = null;
|
||||
readUpload.value = null;
|
||||
readController = new AbortController();
|
||||
try {
|
||||
const result = await api.prepareMongodbRestoreSource(source.value, format.value, gzip.value, {
|
||||
signal: readController.signal,
|
||||
onUploadProgress: (loaded, total) => {
|
||||
readUpload.value = { loaded, total };
|
||||
},
|
||||
});
|
||||
if (current !== generation) {
|
||||
void api.releaseMongodbRestoreSource(result.sourceRef);
|
||||
return;
|
||||
}
|
||||
sourceRef.value = result.sourceRef;
|
||||
catalog.value = result;
|
||||
setSourceDatabase(result.databases.length === 1 ? result.databases[0] : "");
|
||||
} catch (cause) {
|
||||
if (current === generation) error.value = String(cause instanceof Error ? cause.message : cause);
|
||||
} finally {
|
||||
if (current === generation) loading.value = false;
|
||||
readController = null;
|
||||
}
|
||||
}
|
||||
async function chooseDestination() {
|
||||
const { open: choose, save } = await import("@tauri-apps/plugin-dialog");
|
||||
if (format.value === "directory") {
|
||||
const parent = await choose({ directory: true, multiple: false });
|
||||
if (typeof parent === "string") outputPath.value = `${parent.replace(/[\\/]$/, "")}/${props.database}-dump`;
|
||||
} else {
|
||||
const path = await save({ defaultPath: `${props.database}.archive${gzip.value ? ".gz" : ""}`, filters: [{ name: "MongoDB archive", extensions: [gzip.value ? "gz" : "archive"] }] });
|
||||
if (path) outputPath.value = path;
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshDatabase() {
|
||||
await store.loadMongoDatabases(props.connectionId);
|
||||
await store.loadMongoCollections(props.connectionId, targetDatabase.value);
|
||||
for (const collection of selected.value) {
|
||||
store.mongoImportCompleted = { connectionId: props.connectionId, database: targetDatabase.value, collection, at: Date.now() };
|
||||
await nextTick();
|
||||
}
|
||||
}
|
||||
async function execute() {
|
||||
if (!canReview.value) return;
|
||||
confirming.value = true;
|
||||
const current = generation;
|
||||
try {
|
||||
targetDatabase.value = targetDatabase.value.trim();
|
||||
if (restoring.value) {
|
||||
const allowed = await executeWithProductionContextGuard({
|
||||
connection: connection.value,
|
||||
database: targetDatabase.value.trim(),
|
||||
source: t("mongoDump.restoreTitle"),
|
||||
reviewText: `${dropExisting.value ? "DROP selected collections; " : ""}RESTORE ${JSON.stringify(sourceDatabase.value)} INTO ${JSON.stringify(targetDatabase.value.trim())}: ${selected.value.map((name) => JSON.stringify(name)).join(", ")}`,
|
||||
execute: async () => true,
|
||||
});
|
||||
if (!allowed) return;
|
||||
}
|
||||
if (current !== generation || !open.value) return;
|
||||
running.value = true;
|
||||
confirming.value = false;
|
||||
step.value = "result";
|
||||
error.value = "";
|
||||
progress.value = null;
|
||||
taskId.value = uuid();
|
||||
const onProgress = (value: api.MongoDatabaseDumpProgress) => {
|
||||
progress.value = value;
|
||||
};
|
||||
await store.ensureConnected(props.connectionId);
|
||||
if (restoring.value) {
|
||||
uploadController = new AbortController();
|
||||
const selectedPaths = new Set(entries.value.filter((entry) => selected.value.includes(entry.name)).flatMap((entry) => entry.sourceFiles ?? []));
|
||||
const upload = !desktop && format.value === "directory" && Array.isArray(source.value) ? { files: source.value.filter((file) => selectedPaths.has(file.webkitRelativePath || file.name)), gzip: gzip.value, signal: uploadController.signal } : undefined;
|
||||
progress.value = await api.restoreMongodbDatabase(
|
||||
{
|
||||
taskId: taskId.value,
|
||||
connectionId: props.connectionId,
|
||||
database: targetDatabase.value.trim(),
|
||||
sourceDatabase: sourceDatabase.value,
|
||||
sourceRef: sourceRef.value!,
|
||||
collections: [...selected.value],
|
||||
dropExisting: dropExisting.value,
|
||||
restoreOptions: restoreOptions.value,
|
||||
restoreIndexes: restoreIndexes.value,
|
||||
stopOnError: stopOnError.value,
|
||||
objcheck: objcheck.value,
|
||||
batchSize: Math.max(100, Math.min(5000, Math.trunc(Number(batchSize.value)) || 500)),
|
||||
},
|
||||
onProgress,
|
||||
upload,
|
||||
);
|
||||
void refreshDatabase().catch((cause) => console.error("Refresh after MongoDB restore failed", cause));
|
||||
} else {
|
||||
progress.value = await api.dumpMongodbDatabase({ taskId: taskId.value, connectionId: props.connectionId, database: props.database, filePath: outputPath.value.trim(), format: format.value, gzip: gzip.value, collections: [...selected.value] }, onProgress);
|
||||
}
|
||||
} catch (cause) {
|
||||
error.value = String(cause instanceof Error ? cause.message : cause);
|
||||
if (cause instanceof DOMException && cause.name === "AbortError") {
|
||||
progress.value = { taskId: taskId.value, status: "cancelled", phase: "done", collection: null, collectionsDone: 0, collectionsTotal: selected.value.length, documentsRead: 0, documentsWritten: 0, documentsFailed: 0, indexesCreated: 0, elapsedMs: 0, errorMessage: null, filePath: null };
|
||||
error.value = "";
|
||||
}
|
||||
} finally {
|
||||
confirming.value = false;
|
||||
running.value = false;
|
||||
cancelling.value = false;
|
||||
uploadController = null;
|
||||
}
|
||||
}
|
||||
async function cancel() {
|
||||
cancelling.value = true;
|
||||
uploadController?.abort();
|
||||
try {
|
||||
await api.cancelMongodbDatabaseDump(taskId.value);
|
||||
} catch (cause) {
|
||||
cancelling.value = false;
|
||||
error.value = String(cause instanceof Error ? cause.message : cause);
|
||||
}
|
||||
}
|
||||
function close() {
|
||||
if (!running.value && !confirming.value) open.value = false;
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<Dialog
|
||||
:open="open"
|
||||
@update:open="
|
||||
(value) => {
|
||||
if (!value) close();
|
||||
}
|
||||
"
|
||||
>
|
||||
<DialogScrollContent class="flex max-h-[calc(var(--dbx-viewport-height)-4rem)] min-h-0 flex-col overflow-hidden sm:max-w-[860px]">
|
||||
<DialogHeader class="shrink-0 pr-8"
|
||||
><DialogTitle class="flex items-center gap-2 text-base"><DatabaseBackup class="h-4 w-4" />{{ t(restoring ? "mongoDump.restoreTitle" : "mongoDump.dumpTitle") }}</DialogTitle></DialogHeader
|
||||
>
|
||||
<div class="min-h-0 flex-1 space-y-4 overflow-y-auto py-3 text-sm">
|
||||
<div class="border-b pb-2 font-medium break-all">{{ connection?.name }} / {{ props.database }}</div>
|
||||
<div v-if="readonly" class="text-xs text-amber-600">{{ t("mongoDump.readonly") }}</div>
|
||||
<template v-if="step === 'configure'">
|
||||
<div class="grid grid-cols-1 gap-3 sm:grid-cols-2">
|
||||
<div class="space-y-1">
|
||||
<Label>{{ t("mongoDump.format") }}</Label
|
||||
><Select :model-value="format" :disabled="loading" @update:model-value="setFormat"
|
||||
><SelectTrigger class="h-8"><SelectValue /></SelectTrigger
|
||||
><SelectContent
|
||||
><SelectItem value="archive">{{ t("mongoDump.archive") }}</SelectItem
|
||||
><SelectItem v-if="desktop || restoring" value="directory">{{ t("mongoDump.directory") }}</SelectItem></SelectContent
|
||||
></Select
|
||||
>
|
||||
</div>
|
||||
<label class="flex min-h-8 items-center gap-2 self-end"><input v-model="gzip" type="checkbox" :disabled="loading" />{{ t("mongoDump.gzip") }}</label>
|
||||
</div>
|
||||
<div v-if="restoring" class="space-y-2">
|
||||
<Label>{{ t("mongoDump.source") }}</Label>
|
||||
<input ref="fileInput" type="file" class="hidden" :multiple="format === 'directory'" :webkitdirectory="format === 'directory' ? '' : undefined" @change="chooseFiles" />
|
||||
<div class="flex flex-wrap items-center gap-2">
|
||||
<div class="min-w-0 flex-1 basis-full break-all text-xs sm:basis-0">{{ sourceName || t("tableImport.noFileSelected") }}</div>
|
||||
<Button variant="outline" size="sm" :disabled="loading" @click="chooseSource"><FolderOpen class="mr-1.5 h-4 w-4" />{{ t("tableImport.selectFile") }}</Button
|
||||
><Button size="sm" :disabled="!source || loading" @click="readSource"><Loader2 v-if="loading" class="mr-1.5 h-4 w-4 animate-spin" /><Play v-else class="mr-1.5 h-4 w-4" />{{ t("mongoDump.readSource") }}</Button>
|
||||
</div>
|
||||
<div class="grid grid-cols-1 gap-3 sm:grid-cols-2">
|
||||
<div class="space-y-1">
|
||||
<Label>{{ t("mongoDump.sourceDatabase") }}</Label
|
||||
><Select :model-value="sourceDatabase" :disabled="!catalog || loading" @update:model-value="setSourceDatabase"
|
||||
><SelectTrigger class="h-8 w-full min-w-0 [&>span]:min-w-0 [&>span]:truncate"><SelectValue /></SelectTrigger
|
||||
><SelectContent
|
||||
><SelectItem v-for="db in catalog?.databases ?? []" :key="db" :value="db">{{ db }}</SelectItem></SelectContent
|
||||
></Select
|
||||
>
|
||||
</div>
|
||||
<div class="space-y-1">
|
||||
<Label>{{ t("mongoDump.targetDatabase") }}</Label
|
||||
><Input v-model="targetDatabase" class="h-8" :disabled="loading" />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div v-else class="space-y-1">
|
||||
<Label>{{ t("mongoDump.destination") }}</Label>
|
||||
<div class="flex gap-2">
|
||||
<Input v-model="outputPath" class="h-8 min-w-0" :readonly="desktop" /><Button v-if="desktop" variant="outline" size="icon" class="h-8 w-8 shrink-0" :title="t('mongoDump.chooseDestination')" :aria-label="t('mongoDump.chooseDestination')" @click="chooseDestination"
|
||||
><FolderOpen class="h-4 w-4"
|
||||
/></Button>
|
||||
</div>
|
||||
</div>
|
||||
<div v-if="loading" class="flex items-center gap-2 text-xs text-muted-foreground"><Loader2 class="h-4 w-4 animate-spin" />{{ t(restoring ? "mongoDump.preparing" : "mongoDump.loading") }}</div>
|
||||
<div v-if="loading && readUpload" class="text-xs tabular-nums">{{ t("mongoDump.uploadBytes", { done: gib(readUpload.loaded), total: gib(readUpload.total) }) }}</div>
|
||||
<div v-if="catalog" class="space-y-2">
|
||||
<div class="text-xs text-muted-foreground">{{ t("mongoDump.selected", { count: selected.length }) }}</div>
|
||||
<div class="max-h-64 overflow-auto border-y">
|
||||
<table class="w-full min-w-[480px] table-fixed text-xs">
|
||||
<thead class="sticky top-0 bg-background">
|
||||
<tr>
|
||||
<th class="w-8 p-2"><input type="checkbox" :checked="allSelected" :aria-label="t('mongoDump.selectAll')" @change="selectAll" /></th>
|
||||
<th class="w-2/5 p-2 text-left">{{ t("mongoDump.collection") }}</th>
|
||||
<th class="p-2 text-left">{{ t("mongoDump.kind") }}</th>
|
||||
<th class="p-2 text-right">{{ t("mongoDump.documents") }}</th>
|
||||
<th class="p-2 text-right">{{ t("mongoDump.indexes") }}</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr v-for="entry in entries" :key="entry.name" class="border-t">
|
||||
<td class="p-2"><input type="checkbox" :checked="selected.includes(entry.name)" :aria-label="entry.name" @change="selectCollection(entry.name, $event)" /></td>
|
||||
<td class="p-2 break-all">{{ entry.name }}</td>
|
||||
<td class="p-2 break-all">{{ entry.kind }}</td>
|
||||
<td class="p-2 text-right">{{ entry.documents ?? "-" }}</td>
|
||||
<td class="p-2 text-right">{{ entry.indexes }}</td>
|
||||
</tr>
|
||||
<tr v-if="!entries.length">
|
||||
<td colspan="5" class="p-4 text-center text-muted-foreground">{{ t("mongoDump.empty") }}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
<div v-if="restoring" class="space-y-3 border-t pt-3">
|
||||
<label class="flex items-center gap-2"><input v-model="dropExisting" type="checkbox" />{{ t("mongoDump.dropExisting") }}</label>
|
||||
<div class="grid grid-cols-1 gap-3 sm:grid-cols-2">
|
||||
<label class="flex items-center gap-2"><input v-model="restoreOptions" type="checkbox" />{{ t("mongoDump.restoreOptions") }}</label
|
||||
><label class="flex items-center gap-2"><input v-model="restoreIndexes" type="checkbox" />{{ t("mongoDump.restoreIndexes") }}</label
|
||||
><label class="flex items-center gap-2"><input v-model="stopOnError" type="checkbox" />{{ t("mongoDump.stopOnError") }}</label>
|
||||
<label class="flex items-center gap-2"><input v-model="objcheck" type="checkbox" />{{ t("mongoDump.objcheck") }}</label>
|
||||
<div class="flex items-center gap-2">
|
||||
<Label class="shrink-0">{{ t("mongoDump.batchSize") }}</Label
|
||||
><Input v-model.number="batchSize" type="number" min="100" max="5000" class="h-8 w-24" />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
<template v-else-if="step === 'confirm'"
|
||||
><p>{{ t(restoring ? "mongoDump.confirmRestore" : "mongoDump.confirmDump", { count: selected.length, database: restoring ? targetDatabase : props.database }) }}</p>
|
||||
<p v-if="restoring" :class="dropExisting ? 'text-destructive' : 'text-muted-foreground'">{{ t(dropExisting ? "mongoDump.confirmDrop" : "mongoDump.confirmAppend") }}</p>
|
||||
<div class="max-h-56 overflow-auto border-y py-2 text-xs">
|
||||
<div v-for="name in selected" :key="name" class="py-1 break-all">{{ name }}</div>
|
||||
</div>
|
||||
<div v-if="!restoring" class="break-all text-xs text-muted-foreground">{{ outputPath }}</div></template
|
||||
>
|
||||
<template v-else
|
||||
><div class="flex items-center gap-2 font-medium">
|
||||
<Loader2 v-if="running" class="h-4 w-4 animate-spin" />{{ running ? t(`mongoDump.phase.${progress?.phase ?? "preparing"}`) : t(progress?.status === "done" ? "mongoDump.done" : progress?.status === "cancelled" ? "mongoDump.cancelled" : "mongoDump.failed") }}
|
||||
</div>
|
||||
<div class="break-all text-xs">{{ progress?.collection }}</div>
|
||||
<div v-if="progress?.phase === 'uploading'" class="text-xs tabular-nums">{{ t("mongoDump.uploadBytes", { done: gib(progress.bytesProcessed ?? 0), total: gib(progress.bytesTotal ?? 0) }) }}</div>
|
||||
<div v-if="restoring && progress?.phase === 'data'" class="text-xs tabular-nums">{{ t("mongoDump.readBytes", { size: gib(progress.bytesProcessed ?? 0) }) }}</div>
|
||||
<div v-if="progress?.phase === 'validating'" class="text-xs tabular-nums">{{ t("mongoDump.validationBytes", { size: gib(progress.bytesProcessed ?? 0), count: progress.documentsValidated ?? 0 }) }}</div>
|
||||
<progress class="h-2 w-full" :value="progress?.collectionsDone ?? 0" :max="Math.max(1, progress?.collectionsTotal ?? selected.length)" />
|
||||
<div class="text-xs">{{ t("mongoDump.progress", { done: progress?.collectionsDone ?? 0, total: progress?.collectionsTotal ?? selected.length }) }}</div>
|
||||
<dl class="grid grid-cols-2 gap-3 border-y py-3 text-xs">
|
||||
<dt>{{ t("mongoDump.read") }}</dt>
|
||||
<dd class="text-right tabular-nums">{{ progress?.documentsRead ?? 0 }}</dd>
|
||||
<dt v-if="restoring">{{ t("mongoDump.written") }}</dt>
|
||||
<dd v-if="restoring" class="text-right tabular-nums">{{ progress?.documentsWritten ?? 0 }}</dd>
|
||||
<dt>{{ t("mongoDump.rejected") }}</dt>
|
||||
<dd class="text-right tabular-nums">{{ progress?.documentsFailed ?? 0 }}</dd>
|
||||
<dt v-if="restoring">{{ t("mongoDump.createdIndexes") }}</dt>
|
||||
<dd v-if="restoring" class="text-right tabular-nums">{{ progress?.indexesCreated ?? 0 }}</dd>
|
||||
</dl></template
|
||||
>
|
||||
<div v-if="error" class="break-words border-l-2 border-destructive pl-3 text-xs text-destructive" role="alert">{{ error }}</div>
|
||||
</div>
|
||||
<DialogFooter class="shrink-0"
|
||||
><Button v-if="running" variant="destructive" :disabled="cancelling" @click="cancel"><Square class="mr-1.5 h-4 w-4" />{{ t("mongoDump.cancel") }}</Button
|
||||
><Button v-else variant="outline" :disabled="confirming" @click="close"><X class="mr-1.5 h-4 w-4" />{{ t("mongoDump.close") }}</Button
|
||||
><Button v-if="step !== 'configure' && !running" variant="outline" :disabled="confirming" @click="step = 'configure'"><ArrowLeft class="mr-1.5 h-4 w-4" />{{ t("mongoDump.back") }}</Button
|
||||
><Button v-if="step === 'configure'" :disabled="!canReview" @click="step = 'confirm'"><ArrowRight class="mr-1.5 h-4 w-4" />{{ t("mongoDump.review") }}</Button
|
||||
><Button v-if="step === 'confirm'" :disabled="!canReview" @click="execute"><Loader2 v-if="confirming" class="mr-1.5 h-4 w-4 animate-spin" /><Play v-else class="mr-1.5 h-4 w-4" />{{ t(restoring ? "mongoDump.restore" : "mongoDump.dump") }}</Button></DialogFooter
|
||||
>
|
||||
</DialogScrollContent>
|
||||
</Dialog>
|
||||
</template>
|
||||
@@ -1,5 +1,5 @@
|
||||
<script setup lang="ts">
|
||||
import { computed, ref, watch } from "vue";
|
||||
import { computed, onBeforeUnmount, ref, watch } from "vue";
|
||||
import { uuid } from "@/lib/common/utils";
|
||||
import { useI18n } from "vue-i18n";
|
||||
import { isTauriRuntime } from "@/lib/backend/tauriRuntime";
|
||||
@@ -14,7 +14,7 @@ import { useConnectionStore } from "@/stores/connectionStore";
|
||||
import { connectionIsEffectivelyReadOnly } from "@/lib/database/readOnlyWriteAccess";
|
||||
import { executeWithProductionContextGuard } from "@/lib/database/productionExecutionGuard";
|
||||
import { TABLE_IMPORT_ENCODING_OPTIONS } from "@/lib/table/tableImport";
|
||||
import { importPreviewInput, importSourceDisplayName, importTextDelimiterForName, uploadedImportSourceFromPreview, type UploadedImportSource } from "@/lib/import/importSource";
|
||||
import { importPreviewInput, importSourceDisplayName, uploadedImportSourceFromPreview, type UploadedImportSource } from "@/lib/import/importSource";
|
||||
import { useToast } from "@/composables/useToast";
|
||||
import { translateBackendError } from "@/i18n/backend-errors";
|
||||
import * as api from "@/lib/backend/api";
|
||||
@@ -93,13 +93,6 @@ const parseOptions = computed((): api.MongoImportParseOptions => {
|
||||
};
|
||||
});
|
||||
|
||||
function formatFromName(name: string): api.MongoImportFormat {
|
||||
const lower = name.toLowerCase();
|
||||
if (lower.endsWith(".ndjson") || lower.endsWith(".jsonl")) return "ndjson";
|
||||
if (lower.endsWith(".json")) return "json";
|
||||
return "csv";
|
||||
}
|
||||
|
||||
function releasePreviewSource() {
|
||||
const sourceRef = uploadedSource?.sourceRef;
|
||||
uploadedSource = null;
|
||||
@@ -111,10 +104,7 @@ function assignSource(source: string | File) {
|
||||
releasePreviewSource();
|
||||
selectedSource.value = source;
|
||||
sourceName.value = importSourceDisplayName(source);
|
||||
format.value = formatFromName(sourceName.value);
|
||||
typeMode.value = format.value === "csv" ? "auto" : "extendedJson";
|
||||
columnTypeOverrides.value = {};
|
||||
delimiter.value = importTextDelimiterForName(sourceName.value);
|
||||
wizardStep.value = "source";
|
||||
queuePreview();
|
||||
}
|
||||
@@ -127,7 +117,7 @@ async function selectFile() {
|
||||
const { open: openDialog } = await import("@tauri-apps/plugin-dialog");
|
||||
const selected = await openDialog({
|
||||
multiple: false,
|
||||
filters: [{ name: t("mongo.import.fileFilter"), extensions: ["csv", "json", "ndjson", "jsonl", "tsv"] }],
|
||||
filters: [{ name: t("mongo.import.fileFilter"), extensions: ["bson", "gz", "csv", "json", "ndjson", "jsonl", "tsv"] }],
|
||||
});
|
||||
if (!selected || Array.isArray(selected)) return;
|
||||
assignSource(selected);
|
||||
@@ -143,15 +133,17 @@ function handleFileInputChange(event: Event) {
|
||||
function queuePreview() {
|
||||
if (!selectedSource.value) return;
|
||||
if (previewReloadTimer) clearTimeout(previewReloadTimer);
|
||||
const requestId = ++previewRequestId;
|
||||
loadingPreview.value = true;
|
||||
previewReloadTimer = setTimeout(() => {
|
||||
void loadPreview();
|
||||
previewReloadTimer = null;
|
||||
void loadPreview(requestId);
|
||||
}, 200);
|
||||
}
|
||||
|
||||
async function loadPreview() {
|
||||
async function loadPreview(requestId: number) {
|
||||
const source = selectedSource.value;
|
||||
if (!source) return;
|
||||
const requestId = ++previewRequestId;
|
||||
const keepExistingPreview = !!preview.value;
|
||||
if (!keepExistingPreview) {
|
||||
loadingPreview.value = true;
|
||||
@@ -165,7 +157,10 @@ async function loadPreview() {
|
||||
previewLimit: previewLimit.value,
|
||||
sourceRef: input.sourceRef,
|
||||
});
|
||||
if (requestId !== previewRequestId) return;
|
||||
if (requestId !== previewRequestId) {
|
||||
if (next.sourceRef && next.sourceRef !== uploadedSource?.sourceRef) void api.releaseMongodbImportSource(next.sourceRef);
|
||||
return;
|
||||
}
|
||||
preview.value = next;
|
||||
previewError.value = "";
|
||||
uploadedSource = uploadedImportSourceFromPreview(next) ?? uploadedSource;
|
||||
@@ -178,13 +173,21 @@ async function loadPreview() {
|
||||
}
|
||||
}
|
||||
|
||||
watch([format, hasHeader, typeMode], () => {
|
||||
if (Object.keys(columnTypeOverrides.value).length) columnTypeOverrides.value = {};
|
||||
});
|
||||
watch(
|
||||
[format, hasHeader, typeMode],
|
||||
() => {
|
||||
if (Object.keys(columnTypeOverrides.value).length) columnTypeOverrides.value = {};
|
||||
},
|
||||
{ flush: "sync" },
|
||||
);
|
||||
|
||||
watch([format, encoding, delimiter, hasHeader, trim, emptyAsNull, typeMode, recognizeObjectIdHex, previewLimit, columnTypeOverrides], () => {
|
||||
if (selectedSource.value) queuePreview();
|
||||
});
|
||||
watch(
|
||||
[format, encoding, delimiter, hasHeader, trim, emptyAsNull, typeMode, recognizeObjectIdHex, previewLimit, columnTypeOverrides],
|
||||
() => {
|
||||
if (selectedSource.value) queuePreview();
|
||||
},
|
||||
{ flush: "sync" },
|
||||
);
|
||||
|
||||
watch(open, (value) => {
|
||||
if (value) return;
|
||||
@@ -197,6 +200,10 @@ watch(open, (value) => {
|
||||
});
|
||||
|
||||
function reset() {
|
||||
previewRequestId += 1;
|
||||
if (previewReloadTimer) clearTimeout(previewReloadTimer);
|
||||
previewReloadTimer = null;
|
||||
loadingPreview.value = false;
|
||||
releasePreviewSource();
|
||||
selectedSource.value = null;
|
||||
sourceName.value = "";
|
||||
@@ -211,8 +218,12 @@ function reset() {
|
||||
importId.value = "";
|
||||
}
|
||||
|
||||
onBeforeUnmount(() => {
|
||||
if (!running.value) reset();
|
||||
});
|
||||
|
||||
function canConfirm() {
|
||||
return !!preview.value && !loadingPreview.value && !previewError.value && !effectivelyReadOnly.value && (!preview.value.errors.length || skipErrorRows.value);
|
||||
return !!preview.value && preview.value.format === format.value && !loadingPreview.value && !previewError.value && !effectivelyReadOnly.value && (!preview.value.errors.length || skipErrorRows.value);
|
||||
}
|
||||
|
||||
async function startImport() {
|
||||
@@ -287,7 +298,7 @@ function formatPreviewCell(value: unknown) {
|
||||
}
|
||||
|
||||
function setFormat(value: unknown) {
|
||||
if (value === "csv" || value === "json" || value === "ndjson") format.value = value;
|
||||
if (value === "csv" || value === "json" || value === "ndjson" || value === "bson") format.value = value;
|
||||
}
|
||||
|
||||
function setEncoding(value: unknown) {
|
||||
@@ -343,7 +354,7 @@ function requestClose() {
|
||||
</DialogHeader>
|
||||
<div class="min-h-0 flex-1 space-y-4 overflow-y-auto py-2 pr-1 text-sm">
|
||||
<div class="grid grid-cols-[minmax(0,1fr)_auto] items-center gap-2">
|
||||
<input ref="fileInput" type="file" class="hidden" accept=".csv,.json,.ndjson,.jsonl,.tsv" @change="handleFileInputChange" />
|
||||
<input ref="fileInput" type="file" class="hidden" accept=".bson,.bson.gz,.csv,.json,.ndjson,.jsonl,.tsv" @change="handleFileInputChange" />
|
||||
<div class="flex h-10 min-w-0 items-center gap-2 rounded-md border bg-muted/20 px-3">
|
||||
<span class="shrink-0 text-xs text-muted-foreground">{{ t("mongo.import.target") }}</span>
|
||||
<span class="min-w-0 truncate text-sm font-medium">{{ targetLabel }}</span>
|
||||
@@ -366,10 +377,11 @@ function requestClose() {
|
||||
<SelectItem value="csv">CSV</SelectItem>
|
||||
<SelectItem value="json">JSON</SelectItem>
|
||||
<SelectItem value="ndjson">NDJSON</SelectItem>
|
||||
<SelectItem value="bson">BSON dump</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
<div class="space-y-1.5">
|
||||
<div v-if="format !== 'bson'" class="space-y-1.5">
|
||||
<Label class="text-xs">{{ t("tableImport.encoding") }}</Label>
|
||||
<Select :model-value="encoding" @update:model-value="setEncoding">
|
||||
<SelectTrigger class="h-8 text-xs"><SelectValue /></SelectTrigger>
|
||||
@@ -382,7 +394,7 @@ function requestClose() {
|
||||
<Label class="text-xs">{{ t("tableImport.delimiter") }}</Label>
|
||||
<Input v-model="delimiter" class="h-8 text-xs font-mono" :aria-label="t('tableImport.delimiter')" />
|
||||
</div>
|
||||
<div class="space-y-1.5">
|
||||
<div v-if="format !== 'bson'" class="space-y-1.5">
|
||||
<Label class="text-xs">{{ t("mongo.import.typeMode") }}</Label>
|
||||
<Select :model-value="typeMode" :aria-label="t('mongo.import.typeMode')" @update:model-value="setTypeMode">
|
||||
<SelectTrigger class="h-8 text-xs"><SelectValue /></SelectTrigger>
|
||||
@@ -405,13 +417,14 @@ function requestClose() {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="grid grid-cols-2 gap-x-4 gap-y-2 rounded-md border p-3">
|
||||
<div v-if="format !== 'bson'" class="grid grid-cols-2 gap-x-4 gap-y-2 rounded-md border p-3">
|
||||
<label class="flex items-center gap-2 text-xs"><input v-model="hasHeader" type="checkbox" class="h-3.5 w-3.5 accent-primary" :aria-label="t('tableImport.hasHeader')" /> {{ t("tableImport.hasHeader") }}</label>
|
||||
<label class="flex items-center gap-2 text-xs"><input v-model="trim" type="checkbox" class="h-3.5 w-3.5 accent-primary" /> {{ t("tableImport.trimValues") }}</label>
|
||||
<label class="flex items-center gap-2 text-xs"><input v-model="emptyAsNull" type="checkbox" class="h-3.5 w-3.5 accent-primary" /> {{ t("mongo.import.emptyAsNull") }}</label>
|
||||
<label class="flex items-center gap-2 text-xs"><input v-model="recognizeObjectIdHex" type="checkbox" class="h-3.5 w-3.5 accent-primary" /> {{ t("mongo.import.recognizeObjectIdHex") }}</label>
|
||||
<label class="flex items-center gap-2 text-xs"><input v-model="skipErrorRows" type="checkbox" class="h-3.5 w-3.5 accent-primary" /> {{ t("mongo.import.skipErrorRows") }}</label>
|
||||
</div>
|
||||
<label v-else class="flex items-center gap-2 text-xs"><input v-model="skipErrorRows" type="checkbox" class="h-3.5 w-3.5 accent-primary" /> {{ t("mongo.import.skipErrorRows") }}</label>
|
||||
|
||||
<div v-if="loadingPreview && !preview" class="flex items-center gap-2 text-xs text-muted-foreground">
|
||||
<Loader2 class="h-3.5 w-3.5 animate-spin" />
|
||||
@@ -438,7 +451,7 @@ function requestClose() {
|
||||
check-position="right"
|
||||
@update:model-value="(value) => setColumnType(column.name, value)"
|
||||
/>
|
||||
<span v-else-if="format !== 'csv'" class="text-muted-foreground">({{ column.inferredType }})</span>
|
||||
<span v-else-if="format !== 'csv' && format !== 'bson'" class="text-muted-foreground">({{ column.inferredType }})</span>
|
||||
</th>
|
||||
</tr>
|
||||
</thead>
|
||||
|
||||
+233
@@ -0,0 +1,233 @@
|
||||
// @vitest-environment happy-dom
|
||||
import { createApp, nextTick, type App } from "vue";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
api: { inspectMongodbDatabaseDump: vi.fn(), prepareMongodbRestoreSource: vi.fn(), releaseMongodbRestoreSource: vi.fn(), dumpMongodbDatabase: vi.fn(), restoreMongodbDatabase: vi.fn(), cancelMongodbDatabaseDump: vi.fn() },
|
||||
store: { getConfig: vi.fn(), ensureConnected: vi.fn(), loadMongoDatabases: vi.fn(), loadMongoCollections: vi.fn(), mongoImportCompleted: null },
|
||||
guard: vi.fn(),
|
||||
}));
|
||||
vi.mock("@/lib/backend/api", () => mocks.api);
|
||||
vi.mock("@/lib/backend/tauriRuntime", () => ({ isTauriRuntime: () => false }));
|
||||
vi.mock("@/stores/connectionStore", () => ({ useConnectionStore: () => mocks.store }));
|
||||
vi.mock("vue-i18n", async (original) => ({ ...(await original<typeof import("vue-i18n")>()), useI18n: () => ({ t: (key: string) => key }) }));
|
||||
vi.mock("@/lib/database/productionExecutionGuard", () => ({ executeWithProductionContextGuard: mocks.guard }));
|
||||
import MongoDatabaseDumpDialog from "../MongoDatabaseDumpDialog.vue";
|
||||
|
||||
const catalog = {
|
||||
databases: ["source"],
|
||||
collections: [
|
||||
{ database: "source", name: "records", kind: "collection", documents: 2, sizeBytes: 30, indexes: 2 },
|
||||
{ database: "source", name: "empty", kind: "collection", documents: 0, sizeBytes: 0, indexes: 0 },
|
||||
],
|
||||
};
|
||||
const done = { taskId: "task", status: "done", phase: "done", collection: "records", collectionsDone: 2, collectionsTotal: 2, documentsRead: 2, documentsWritten: 2, documentsFailed: 0, indexesCreated: 2, elapsedMs: 10, errorMessage: null, filePath: null };
|
||||
let app: App | null;
|
||||
async function flush() {
|
||||
for (let i = 0; i < 5; i++) {
|
||||
await Promise.resolve();
|
||||
await nextTick();
|
||||
}
|
||||
}
|
||||
function button(key: string): HTMLButtonElement {
|
||||
const button = Array.from(document.querySelectorAll("button")).find((button) => button.textContent?.trim() === key);
|
||||
if (!button) throw new Error(`Missing button ${key}`);
|
||||
return button;
|
||||
}
|
||||
function checkbox(key: string): HTMLInputElement {
|
||||
const label = Array.from(document.querySelectorAll("label")).find((label) => label.textContent?.trim() === key);
|
||||
const checkbox = label?.querySelector<HTMLInputElement>("input[type=checkbox]");
|
||||
if (!checkbox) throw new Error(`Missing checkbox ${key}`);
|
||||
return checkbox;
|
||||
}
|
||||
async function mount(mode: "dump" | "restore") {
|
||||
app = createApp(MongoDatabaseDumpDialog, { open: true, connectionId: "c1", database: "target", mode });
|
||||
app.mount(document.body);
|
||||
await flush();
|
||||
}
|
||||
async function filesSelected(files: File[]) {
|
||||
const input = document.querySelector<HTMLInputElement>("input[type=file]")!;
|
||||
Object.defineProperty(input, "files", { configurable: true, value: files });
|
||||
input.dispatchEvent(new Event("change"));
|
||||
await flush();
|
||||
}
|
||||
async function formatSelected(value: string) {
|
||||
document.querySelector<HTMLElement>("[role=combobox]")!.dispatchEvent(new KeyboardEvent("keydown", { key: "ArrowDown", bubbles: true, cancelable: true }));
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
await flush();
|
||||
const option = Array.from(document.querySelectorAll<HTMLElement>("[role=option]")).find((option) => option.textContent?.trim() === value)!;
|
||||
option.focus();
|
||||
option.dispatchEvent(new KeyboardEvent("keydown", { key: "Enter", bubbles: true, cancelable: true }));
|
||||
await vi.advanceTimersByTimeAsync(10);
|
||||
await flush();
|
||||
}
|
||||
|
||||
describe("MongoDatabaseDumpDialog", () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers();
|
||||
vi.clearAllMocks();
|
||||
mocks.store.getConfig.mockReturnValue({ name: "MongoDB", read_only: false });
|
||||
mocks.store.ensureConnected.mockResolvedValue(undefined);
|
||||
mocks.api.inspectMongodbDatabaseDump.mockResolvedValue(catalog);
|
||||
mocks.api.prepareMongodbRestoreSource.mockResolvedValue({ ...catalog, sourceRef: "prepared-one" });
|
||||
mocks.api.releaseMongodbRestoreSource.mockResolvedValue(true);
|
||||
mocks.api.dumpMongodbDatabase.mockResolvedValue(done);
|
||||
mocks.api.restoreMongodbDatabase.mockResolvedValue(done);
|
||||
mocks.guard.mockImplementation(async ({ execute }) => execute());
|
||||
});
|
||||
afterEach(() => {
|
||||
app?.unmount();
|
||||
app = null;
|
||||
document.body.innerHTML = "";
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("exports selected collections using the explicit archive and gzip options", async () => {
|
||||
await mount("dump");
|
||||
checkbox("mongoDump.gzip").click();
|
||||
document.querySelector<HTMLInputElement>('input[aria-label="empty"]')!.click();
|
||||
await flush();
|
||||
button("mongoDump.review").click();
|
||||
await flush();
|
||||
button("mongoDump.dump").click();
|
||||
await flush();
|
||||
expect(mocks.api.dumpMongodbDatabase).toHaveBeenCalledWith(expect.objectContaining({ connectionId: "c1", database: "target", format: "archive", gzip: true, collections: ["records"], filePath: "target.archive.gz" }), expect.any(Function));
|
||||
expect(mocks.guard).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("restores a directory into the chosen database only after drop confirmation", async () => {
|
||||
await mount("restore");
|
||||
await formatSelected("mongoDump.directory");
|
||||
checkbox("mongoDump.gzip").click();
|
||||
await filesSelected([new File([], "records.bson.gz")]);
|
||||
button("mongoDump.readSource").click();
|
||||
await flush();
|
||||
expect(mocks.api.prepareMongodbRestoreSource).toHaveBeenCalledWith(expect.any(Array), "directory", true, expect.objectContaining({ signal: expect.any(AbortSignal) }));
|
||||
checkbox("mongoDump.dropExisting").click();
|
||||
await flush();
|
||||
button("mongoDump.review").click();
|
||||
await flush();
|
||||
expect(document.body.textContent).toContain("mongoDump.confirmDrop");
|
||||
expect(mocks.api.restoreMongodbDatabase).not.toHaveBeenCalled();
|
||||
button("mongoDump.restore").click();
|
||||
await flush();
|
||||
expect(mocks.api.restoreMongodbDatabase).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ database: "target", sourceDatabase: "source", sourceRef: "prepared-one", dropExisting: true, restoreOptions: true, restoreIndexes: true, objcheck: false, batchSize: 500 }),
|
||||
expect.any(Function),
|
||||
expect.objectContaining({ gzip: true }),
|
||||
);
|
||||
expect(mocks.guard).toHaveBeenCalledWith(expect.objectContaining({ database: "target", reviewText: expect.stringContaining("DROP selected collections") }));
|
||||
});
|
||||
|
||||
it("passes optional objcheck without a separate prevalidation request", async () => {
|
||||
await mount("restore");
|
||||
expect(checkbox("mongoDump.objcheck").checked).toBe(false);
|
||||
await filesSelected([new File([], "backup.archive")]);
|
||||
button("mongoDump.readSource").click();
|
||||
await flush();
|
||||
checkbox("mongoDump.objcheck").click();
|
||||
button("mongoDump.review").click();
|
||||
await flush();
|
||||
button("mongoDump.restore").click();
|
||||
await flush();
|
||||
expect(mocks.api.restoreMongodbDatabase).toHaveBeenCalledWith(expect.objectContaining({ objcheck: true }), expect.any(Function), undefined);
|
||||
expect(mocks.api.prepareMongodbRestoreSource).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("starts only one restore when confirmation is pending and the button is clicked twice", async () => {
|
||||
let allow!: (value: boolean) => void;
|
||||
mocks.guard.mockReturnValue(
|
||||
new Promise<boolean>((resolve) => {
|
||||
allow = resolve;
|
||||
}),
|
||||
);
|
||||
await mount("restore");
|
||||
await filesSelected([new File([], "backup.archive")]);
|
||||
button("mongoDump.readSource").click();
|
||||
await flush();
|
||||
button("mongoDump.review").click();
|
||||
await flush();
|
||||
const restore = button("mongoDump.restore");
|
||||
restore.click();
|
||||
restore.click();
|
||||
await flush();
|
||||
expect(mocks.guard).toHaveBeenCalledTimes(1);
|
||||
expect(button("mongoDump.back").disabled).toBe(true);
|
||||
expect(button("mongoDump.close").disabled).toBe(true);
|
||||
expect(mocks.api.restoreMongodbDatabase).not.toHaveBeenCalled();
|
||||
allow(true);
|
||||
await flush();
|
||||
expect(mocks.api.restoreMongodbDatabase).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not restore if the dialog is unmounted while confirmation is pending", async () => {
|
||||
let allow!: (value: boolean) => void;
|
||||
mocks.guard.mockReturnValue(
|
||||
new Promise<boolean>((resolve) => {
|
||||
allow = resolve;
|
||||
}),
|
||||
);
|
||||
await mount("restore");
|
||||
await filesSelected([new File([], "backup.archive")]);
|
||||
button("mongoDump.readSource").click();
|
||||
await flush();
|
||||
button("mongoDump.review").click();
|
||||
await flush();
|
||||
button("mongoDump.restore").click();
|
||||
await flush();
|
||||
app!.unmount();
|
||||
app = null;
|
||||
allow(true);
|
||||
await flush();
|
||||
expect(mocks.api.restoreMongodbDatabase).not.toHaveBeenCalled();
|
||||
expect(mocks.store.ensureConnected).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("unlocks the dialog when production confirmation is declined", async () => {
|
||||
mocks.guard.mockResolvedValueOnce(false);
|
||||
await mount("restore");
|
||||
await filesSelected([new File([], "backup.archive")]);
|
||||
button("mongoDump.readSource").click();
|
||||
await flush();
|
||||
button("mongoDump.review").click();
|
||||
await flush();
|
||||
button("mongoDump.restore").click();
|
||||
await flush();
|
||||
expect(mocks.api.restoreMongodbDatabase).not.toHaveBeenCalled();
|
||||
expect(button("mongoDump.restore").disabled).toBe(false);
|
||||
button("mongoDump.restore").click();
|
||||
await flush();
|
||||
expect(mocks.api.restoreMongodbDatabase).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("invalidates a prepared source when gzip changes without inferring the format from its name", async () => {
|
||||
await mount("restore");
|
||||
await filesSelected([new File([], "misleading.bson.gz")]);
|
||||
button("mongoDump.readSource").click();
|
||||
await flush();
|
||||
expect(mocks.api.prepareMongodbRestoreSource).toHaveBeenCalledWith(expect.any(File), "archive", false, expect.any(Object));
|
||||
checkbox("mongoDump.gzip").click();
|
||||
await flush();
|
||||
expect(mocks.api.releaseMongodbRestoreSource).toHaveBeenCalledWith("prepared-one");
|
||||
expect(button("mongoDump.review").disabled).toBe(true);
|
||||
expect(mocks.api.prepareMongodbRestoreSource).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("releases a late preview after the dialog is unmounted", async () => {
|
||||
let resolve!: (value: unknown) => void;
|
||||
mocks.api.prepareMongodbRestoreSource.mockReturnValueOnce(
|
||||
new Promise((done) => {
|
||||
resolve = done;
|
||||
}),
|
||||
);
|
||||
await mount("restore");
|
||||
await filesSelected([new File([], "backup.archive")]);
|
||||
button("mongoDump.readSource").click();
|
||||
await flush();
|
||||
app!.unmount();
|
||||
app = null;
|
||||
resolve({ ...catalog, sourceRef: "late-source" });
|
||||
await flush();
|
||||
expect(mocks.api.releaseMongodbRestoreSource).toHaveBeenCalledWith("late-source");
|
||||
});
|
||||
});
|
||||
@@ -143,6 +143,53 @@ describe("MongoImportDialog", () => {
|
||||
app.unmount();
|
||||
});
|
||||
|
||||
it("does not override the selected format from the file extension", async () => {
|
||||
const app = createApp(
|
||||
defineComponent({
|
||||
setup() {
|
||||
return () => h(MongoImportDialog, { open: true, connectionId: "c1", database: "shop", collection: "orders" });
|
||||
},
|
||||
}),
|
||||
);
|
||||
app.mount(document.body);
|
||||
await nextTick();
|
||||
|
||||
const input = document.querySelector("input[type='file']") as HTMLInputElement;
|
||||
const file = new File([new Uint8Array([5, 0, 0, 0, 0])], "orders.bson.gz", { type: "application/gzip" });
|
||||
Object.defineProperty(input, "files", { value: [file] });
|
||||
input.dispatchEvent(new Event("change"));
|
||||
await vi.advanceTimersByTimeAsync(200);
|
||||
await Promise.resolve();
|
||||
await nextTick();
|
||||
|
||||
expect(api.previewMongodbImportFile.mock.calls[0]?.[1]).toMatchObject({
|
||||
format: "csv",
|
||||
parseOptions: { delimiter: ",", typeMode: "auto" },
|
||||
});
|
||||
expect(document.body.textContent).toContain("tableImport.encoding");
|
||||
expect(document.body.textContent).toContain("mongo.import.typeMode");
|
||||
app.unmount();
|
||||
});
|
||||
|
||||
it("keeps an explicitly selected BSON format when changing files", async () => {
|
||||
api.previewMongodbImportFile.mockResolvedValue(previewResult({ format: "bson" }));
|
||||
const app = createApp(MongoImportDialog, { open: true, connectionId: "c1", database: "shop", collection: "orders" });
|
||||
app.mount(document.body);
|
||||
await nextTick();
|
||||
await setLabeledSelect("tableImport.sourceFormat", "bson");
|
||||
expect(document.body.textContent).not.toContain("tableImport.encoding");
|
||||
|
||||
for (const name of ["orders.bson", "renamed.json"]) {
|
||||
const input = document.querySelector("input[type='file']") as HTMLInputElement;
|
||||
Object.defineProperty(input, "files", { configurable: true, value: [new File([new Uint8Array([5, 0, 0, 0, 0])], name)] });
|
||||
input.dispatchEvent(new Event("change"));
|
||||
await vi.advanceTimersByTimeAsync(200);
|
||||
await nextTick();
|
||||
expect(api.previewMongodbImportFile.mock.lastCall?.[1]).toMatchObject({ format: "bson" });
|
||||
}
|
||||
app.unmount();
|
||||
});
|
||||
|
||||
it("reuses the uploaded sourceRef on later previews and releases it on close", async () => {
|
||||
const open = ref(true);
|
||||
const app = createApp(
|
||||
@@ -180,6 +227,9 @@ describe("MongoImportDialog", () => {
|
||||
|
||||
const header = document.querySelector("input[type='checkbox']") as HTMLInputElement;
|
||||
header.click();
|
||||
await nextTick();
|
||||
const next = Array.from(document.querySelectorAll("button")).find((button) => button.textContent?.includes("tableImport.next"));
|
||||
expect(next?.disabled).toBe(true);
|
||||
await vi.advanceTimersByTimeAsync(200);
|
||||
await Promise.resolve();
|
||||
await nextTick();
|
||||
@@ -194,7 +244,7 @@ describe("MongoImportDialog", () => {
|
||||
app.unmount();
|
||||
});
|
||||
|
||||
async function mountWithFile(fileName: string, preview: MongoImportPreview) {
|
||||
async function mountWithFile(fileName: string, preview: MongoImportPreview, format: MongoImportPreview["format"] = "csv") {
|
||||
api.previewMongodbImportFile.mockResolvedValue(preview);
|
||||
const app = createApp(
|
||||
defineComponent({
|
||||
@@ -211,6 +261,7 @@ describe("MongoImportDialog", () => {
|
||||
);
|
||||
app.mount(document.body);
|
||||
await nextTick();
|
||||
if (format !== "csv") await setLabeledSelect("tableImport.sourceFormat", format);
|
||||
const input = document.querySelector("input[type='file']") as HTMLInputElement;
|
||||
const file = new File(["id\n1"], fileName, { type: "text/csv" });
|
||||
Object.defineProperty(input, "files", { configurable: true, value: [file] });
|
||||
@@ -317,6 +368,7 @@ describe("MongoImportDialog", () => {
|
||||
fileName: "orders.json",
|
||||
columns: [{ name: "id", inferredType: "string" }],
|
||||
}),
|
||||
"json",
|
||||
);
|
||||
expect(columnTypeTrigger()).toBeNull();
|
||||
expect(document.body.textContent).toContain("(string)");
|
||||
|
||||
@@ -14,6 +14,7 @@ const SchemaDiagramDialog = defineAsyncComponent(() => import("@/components/diag
|
||||
const DatabaseDocsDialog = defineAsyncComponent(() => import("@/components/docs/DatabaseDocsDialog.vue"));
|
||||
const TableImportDialog = defineAsyncComponent(() => import("@/components/import/TableImportDialog.vue"));
|
||||
const MongoImportDialog = defineAsyncComponent(() => import("@/components/document/MongoImportDialog.vue"));
|
||||
const MongoDatabaseDumpDialog = defineAsyncComponent(() => import("@/components/document/MongoDatabaseDumpDialog.vue"));
|
||||
const FieldLineageDialog = defineAsyncComponent(() => import("@/components/lineage/FieldLineageDialog.vue"));
|
||||
const ConfigPassphraseDialog = defineAsyncComponent(() => import("@/components/config/ConfigPassphraseDialog.vue"));
|
||||
const ConfigConnectionSelectDialog = defineAsyncComponent(() => import("@/components/config/ConfigConnectionSelectDialog.vue"));
|
||||
@@ -301,6 +302,13 @@ watch(
|
||||
:prefill-table="dialogs.tableImportPrefillTable.value"
|
||||
/>
|
||||
<MongoImportDialog v-model:open="dialogs.showMongoImportDialog.value" :connection-id="dialogs.mongoImportPrefillConnectionId.value" :database="dialogs.mongoImportPrefillDatabase.value" :collection="dialogs.mongoImportPrefillCollection.value" />
|
||||
<MongoDatabaseDumpDialog
|
||||
v-if="dialogs.showMongoDatabaseDumpDialog.value"
|
||||
v-model:open="dialogs.showMongoDatabaseDumpDialog.value"
|
||||
:connection-id="dialogs.mongoDatabaseDumpPrefillConnectionId.value"
|
||||
:database="dialogs.mongoDatabaseDumpPrefillDatabase.value"
|
||||
:mode="dialogs.mongoDatabaseDumpMode.value"
|
||||
/>
|
||||
<DataGenerateDialog
|
||||
v-if="dialogs.showTableDataGenerateDialog.value"
|
||||
v-model:open="dialogs.showTableDataGenerateDialog.value"
|
||||
|
||||
@@ -410,15 +410,31 @@ const { copyStructureAs, copyStructureDocText, copyStructurePreview, exportData,
|
||||
acceptedSelectionIds: () => acceptedSelectionIds,
|
||||
});
|
||||
|
||||
const { openAllDatabasesExport, openDataCompare, openDatabaseExport, openDatabaseSearch, openDiagram, openDocs, openFieldLineage, openMongoImport, openScheduledBackups, openSchemaDiff, openSchemaDiffForRoutine, openSqlFileExecution, openStructureEditor, openTableImport, openTransfer } =
|
||||
useSidebarTreeToolRuntime({
|
||||
activeNode,
|
||||
connectionStore,
|
||||
queryStore,
|
||||
settingsStore,
|
||||
tableChildObjectName: tableChildDropObjectName,
|
||||
acceptedSelectionIds: () => acceptedSelectionIds,
|
||||
});
|
||||
const {
|
||||
openAllDatabasesExport,
|
||||
openDataCompare,
|
||||
openDatabaseExport,
|
||||
openDatabaseSearch,
|
||||
openDiagram,
|
||||
openDocs,
|
||||
openFieldLineage,
|
||||
openMongoImport,
|
||||
openMongoDatabaseDump,
|
||||
openScheduledBackups,
|
||||
openSchemaDiff,
|
||||
openSchemaDiffForRoutine,
|
||||
openSqlFileExecution,
|
||||
openStructureEditor,
|
||||
openTableImport,
|
||||
openTransfer,
|
||||
} = useSidebarTreeToolRuntime({
|
||||
activeNode,
|
||||
connectionStore,
|
||||
queryStore,
|
||||
settingsStore,
|
||||
tableChildObjectName: tableChildDropObjectName,
|
||||
acceptedSelectionIds: () => acceptedSelectionIds,
|
||||
});
|
||||
|
||||
const emit = defineEmits<{
|
||||
"rename-started": [];
|
||||
@@ -5870,6 +5886,8 @@ function buildSpecialSidebarMenu(context: SidebarMenuFactoryContext): boolean {
|
||||
if (node.type === "mongo-db") {
|
||||
items.push({ label: "", separator: true });
|
||||
items.push({ label: t("transfer.dataTransfer"), action: openTransfer, icon: ArrowRightLeft });
|
||||
items.push({ label: t("mongoDump.menuDump"), action: () => openMongoDatabaseDump("dump"), icon: Upload });
|
||||
items.push({ label: t("mongoDump.menuRestore"), action: () => openMongoDatabaseDump("restore"), icon: Download });
|
||||
}
|
||||
if (node.type === "redis-db") {
|
||||
items.push({ label: "", separator: true });
|
||||
@@ -5950,6 +5968,8 @@ function buildSpecialSidebarMenu(context: SidebarMenuFactoryContext): boolean {
|
||||
children: [
|
||||
{ label: "CSV", action: () => void exportMongoCollection("csv") },
|
||||
{ label: "NDJSON", action: () => void exportMongoCollection("ndjson") },
|
||||
{ label: "BSON dump", action: () => void exportMongoCollection("bson") },
|
||||
{ label: "BSON dump (gzip)", action: () => void exportMongoCollection("bsonGzip") },
|
||||
],
|
||||
});
|
||||
if (canDropMongoCollection.value) {
|
||||
|
||||
@@ -158,6 +158,39 @@ describe("useSidebarTreeExportRuntime", () => {
|
||||
expect(toastMock).toHaveBeenCalledWith("grid.exported");
|
||||
});
|
||||
|
||||
it.each(["bson", "bsonGzip"] as const)("exports an official-compatible %s collection dump", async (mode) => {
|
||||
apiMock.exportMongodbQuery.mockImplementation(async (_request, onProgress) => {
|
||||
onProgress({ exportId: "export-1", status: "done", documentsRead: 2, bytesWritten: 64, elapsedMs: 4 });
|
||||
return { exportId: "export-1", documentsExported: 2, filePath: "orders.bson.gz", elapsedMs: 4 };
|
||||
});
|
||||
const activeNode = shallowRef({ id: "col-1", type: "mongo-collection", label: "orders", connectionId: "conn-1", database: "shop", children: [] } as TreeNode);
|
||||
const connectionStore = {
|
||||
ensureConnected: vi.fn(),
|
||||
getConfig: vi.fn(() => ({ db_type: "mongodb" })),
|
||||
treeNodes: [],
|
||||
selectedTreeNodeIds: [],
|
||||
};
|
||||
const runtime = useSidebarTreeExportRuntime({
|
||||
activeNode,
|
||||
connectionStore: connectionStore as never,
|
||||
settingsStore: exportSettings() as never,
|
||||
acceptedSelectionIds: () => null,
|
||||
});
|
||||
|
||||
await runtime.exportMongoCollection(mode);
|
||||
|
||||
expect(apiMock.exportMongodbQuery).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
database: "shop",
|
||||
collection: "orders",
|
||||
format: "bson",
|
||||
gzip: mode === "bsonGzip",
|
||||
filePath: mode === "bsonGzip" ? "orders.bson.gz" : "orders.bson",
|
||||
}),
|
||||
expect.any(Function),
|
||||
);
|
||||
});
|
||||
|
||||
it("loads and joins every selected DDL in tree order", async () => {
|
||||
apiMock.getTableDdl.mockResolvedValueOnce("CREATE TABLE one (id INT)").mockResolvedValueOnce("CREATE VIEW two AS SELECT 1;");
|
||||
const first = { id: "table-1", type: "table", label: "one", connectionId: "conn-1", database: "db", schema: "main" } as TreeNode;
|
||||
|
||||
+24
@@ -11,6 +11,7 @@ function setup(node: Partial<TreeNode>, options: { treeNodes?: TreeNode[]; selec
|
||||
diagramSource: null as unknown,
|
||||
databaseExportSource: null as unknown,
|
||||
mongoImportSource: undefined as unknown,
|
||||
mongoDatabaseDumpSource: null as unknown,
|
||||
schemaDiffSource: null as unknown,
|
||||
treeNodes: options.treeNodes ?? [],
|
||||
selectedTreeNodeIds: options.selectedTreeNodeIds ?? [],
|
||||
@@ -129,6 +130,29 @@ describe("useSidebarTreeToolRuntime mongo import", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("useSidebarTreeToolRuntime MongoDB dump and restore", () => {
|
||||
it.each(["dump", "restore"] as const)("opens %s for the selected MongoDB database", (mode) => {
|
||||
const { connectionStore, runtime } = setup({ type: "mongo-db", connectionId: "conn-1", database: "shop" });
|
||||
|
||||
runtime.openMongoDatabaseDump(mode);
|
||||
|
||||
expect(connectionStore.mongoDatabaseDumpSource).toEqual({ connectionId: "conn-1", database: "shop", mode });
|
||||
});
|
||||
|
||||
it.each<Partial<TreeNode>>([
|
||||
{ type: "table", connectionId: "conn-1", database: "shop" },
|
||||
{ type: "mongo-collection", connectionId: "conn-1", database: "shop" },
|
||||
{ type: "mongo-db", database: "shop" },
|
||||
{ type: "mongo-db", connectionId: "conn-1" },
|
||||
])("does not open a database restore from an invalid context: %j", (node) => {
|
||||
const { connectionStore, runtime } = setup(node);
|
||||
|
||||
runtime.openMongoDatabaseDump("restore");
|
||||
|
||||
expect(connectionStore.mongoDatabaseDumpSource).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("useSidebarTreeToolRuntime openSchemaDiffForRoutine", () => {
|
||||
it("prefills schema diff with a signature-aware routine key", () => {
|
||||
const { connectionStore, runtime } = setup({
|
||||
|
||||
@@ -15,6 +15,10 @@ const showDiagramDialog = ref(false);
|
||||
const showDocsDialog = ref(false);
|
||||
const showTableImportDialog = ref(false);
|
||||
const showMongoImportDialog = ref(false);
|
||||
const showMongoDatabaseDumpDialog = ref(false);
|
||||
const mongoDatabaseDumpPrefillConnectionId = ref("");
|
||||
const mongoDatabaseDumpPrefillDatabase = ref("");
|
||||
const mongoDatabaseDumpMode = ref<"dump" | "restore">("dump");
|
||||
const showTableDataGenerateDialog = ref(false);
|
||||
const showFieldLineageDialog = ref(false);
|
||||
const showDatabaseSearchDialog = ref(false);
|
||||
@@ -263,6 +267,18 @@ export function useDialogSources() {
|
||||
},
|
||||
);
|
||||
|
||||
watch(
|
||||
() => connectionStore.mongoDatabaseDumpSource,
|
||||
(source) => {
|
||||
if (!source) return;
|
||||
mongoDatabaseDumpPrefillConnectionId.value = source.connectionId;
|
||||
mongoDatabaseDumpPrefillDatabase.value = source.database;
|
||||
mongoDatabaseDumpMode.value = source.mode;
|
||||
showMongoDatabaseDumpDialog.value = true;
|
||||
connectionStore.mongoDatabaseDumpSource = null;
|
||||
},
|
||||
);
|
||||
|
||||
watch(
|
||||
() => connectionStore.tableDataGenerateSource,
|
||||
(v) => {
|
||||
@@ -519,6 +535,10 @@ export function useDialogSources() {
|
||||
showDocsDialog,
|
||||
showTableImportDialog,
|
||||
showMongoImportDialog,
|
||||
showMongoDatabaseDumpDialog,
|
||||
mongoDatabaseDumpPrefillConnectionId,
|
||||
mongoDatabaseDumpPrefillDatabase,
|
||||
mongoDatabaseDumpMode,
|
||||
showTableDataGenerateDialog,
|
||||
showFieldLineageDialog,
|
||||
showDatabaseSearchDialog,
|
||||
|
||||
@@ -274,6 +274,7 @@ export function useSidebarTreeExportRuntime(options: SidebarTreeExportRuntimeOpt
|
||||
if (format === "csv") return "CSV";
|
||||
if (format === "json") return "JSON";
|
||||
if (format === "ndjson") return "NDJSON";
|
||||
if (format === "bson" || format === "bson.gz") return "MongoDB BSON dump";
|
||||
if (format === "xlsx") return "Excel";
|
||||
return "SQL";
|
||||
}
|
||||
@@ -287,7 +288,7 @@ export function useSidebarTreeExportRuntime(options: SidebarTreeExportRuntimeOpt
|
||||
const { save } = await import("@tauri-apps/plugin-dialog");
|
||||
const path = await save({
|
||||
defaultPath: fileName,
|
||||
filters: [{ name: exportFilterName(format), extensions: [format] }],
|
||||
filters: [{ name: exportFilterName(format), extensions: [format === "bson.gz" ? "gz" : format] }],
|
||||
});
|
||||
return path ? String(path) : null;
|
||||
}
|
||||
@@ -472,10 +473,12 @@ export function useSidebarTreeExportRuntime(options: SidebarTreeExportRuntimeOpt
|
||||
return pickTableExportDirectory();
|
||||
}
|
||||
|
||||
async function exportMongoCollection(format: "csv" | "ndjson") {
|
||||
async function exportMongoCollection(outputFormat: "csv" | "ndjson" | "bson" | "bsonGzip") {
|
||||
const node = activeNode.value;
|
||||
if (node.type !== "mongo-collection" || !node.connectionId || !node.database) return;
|
||||
const outputPath = await resolveExportOutputPath(node.label, format);
|
||||
const format: api.MongoExportFormat = outputFormat === "bsonGzip" ? "bson" : outputFormat;
|
||||
const fileExtension = outputFormat === "bsonGzip" ? "bson.gz" : outputFormat;
|
||||
const outputPath = await resolveExportOutputPath(node.label, fileExtension);
|
||||
if (!outputPath) return;
|
||||
let task: ExportTask | null = null;
|
||||
try {
|
||||
@@ -490,6 +493,7 @@ export function useSidebarTreeExportRuntime(options: SidebarTreeExportRuntimeOpt
|
||||
collection: node.label,
|
||||
format,
|
||||
includeHeader: true,
|
||||
gzip: outputFormat === "bsonGzip",
|
||||
filePath: outputPath,
|
||||
},
|
||||
(progress) => {
|
||||
|
||||
@@ -149,6 +149,12 @@ export function useSidebarTreeToolRuntime(options: SidebarTreeToolRuntimeOptions
|
||||
};
|
||||
}
|
||||
|
||||
function openMongoDatabaseDump(mode: "dump" | "restore") {
|
||||
const node = activeNode.value;
|
||||
if (node.type !== "mongo-db" || !node.connectionId || !node.database) return;
|
||||
connectionStore.mongoDatabaseDumpSource = { connectionId: node.connectionId, database: node.database, mode };
|
||||
}
|
||||
|
||||
function openStructureEditor() {
|
||||
const node = activeNode.value;
|
||||
if (!node.connectionId || !node.database) return;
|
||||
@@ -191,6 +197,7 @@ export function useSidebarTreeToolRuntime(options: SidebarTreeToolRuntimeOptions
|
||||
openDocs,
|
||||
openFieldLineage,
|
||||
openMongoImport,
|
||||
openMongoDatabaseDump,
|
||||
openScheduledBackups,
|
||||
openSchemaDiff,
|
||||
openSchemaDiffForRoutine,
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import * as messages from "../locales/mongodbDatabaseDump";
|
||||
|
||||
function leaves(value: Record<string, unknown>, prefix = ""): Record<string, string> {
|
||||
return Object.fromEntries(
|
||||
Object.entries(value).flatMap(([key, child]) => {
|
||||
const path = prefix ? `${prefix}.${key}` : key;
|
||||
if (typeof child === "string") return [[path, child]];
|
||||
return Object.entries(leaves(child as Record<string, unknown>, path));
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
describe("MongoDB dump translations", () => {
|
||||
const source = leaves(messages.mongodbDatabaseDumpEn);
|
||||
for (const [locale, value] of Object.entries(messages)) {
|
||||
it(`${locale} includes every key and preserves placeholders`, () => {
|
||||
const translated = leaves(value);
|
||||
expect(Object.keys(translated).sort()).toEqual(Object.keys(source).sort());
|
||||
for (const [key, text] of Object.entries(translated)) {
|
||||
expect(text.trim(), key).not.toBe("");
|
||||
expect((text.match(/\{\w+\}/g) ?? []).sort(), key).toEqual((source[key]!.match(/\{\w+\}/g) ?? []).sort());
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
@@ -3,6 +3,7 @@ import docs from "./docs/en";
|
||||
import { consulUiMessages } from "./consulUi";
|
||||
import { sqlServerTraceMessages as sqlServerTrace } from "./sqlServerTraceMessages";
|
||||
import { meilisearchManagementEn } from "./meilisearchManagement";
|
||||
import { mongodbDatabaseDumpEn as mongoDump } from "./mongodbDatabaseDump";
|
||||
|
||||
const consul = {
|
||||
...consulUiMessages,
|
||||
@@ -5482,6 +5483,7 @@ export default {
|
||||
},
|
||||
},
|
||||
},
|
||||
mongoDump,
|
||||
mongo: {
|
||||
documents: "{count} documents",
|
||||
addField: "Add Field",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { withEnglishFallback } from "./fallback";
|
||||
import { mongodbDatabaseDumpEs as mongoDump } from "./mongodbDatabaseDump";
|
||||
import { meilisearchManagementEs } from "./meilisearchManagement";
|
||||
import { redisGroupingEs as redisGrouping } from "./redisGrouping";
|
||||
import docs from "./docs/es";
|
||||
@@ -83,6 +84,7 @@ const consul = {
|
||||
};
|
||||
|
||||
export default withEnglishFallback({
|
||||
mongoDump,
|
||||
cellTransform: {
|
||||
title: "Vista previa de conversión",
|
||||
description: "Convierte el valor actual para verlo o copiarlo. El valor original y el borrador no cambian.",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { withEnglishFallback } from "./fallback";
|
||||
import { mongodbDatabaseDumpIt as mongoDump } from "./mongodbDatabaseDump";
|
||||
import { meilisearchManagementIt } from "./meilisearchManagement";
|
||||
import { redisGroupingIt as redisGrouping } from "./redisGrouping";
|
||||
import docs from "./docs/it";
|
||||
@@ -82,6 +83,7 @@ const consul = {
|
||||
};
|
||||
|
||||
export default withEnglishFallback({
|
||||
mongoDump,
|
||||
cellTransform: {
|
||||
title: "Anteprima conversione",
|
||||
description: "Converte il valore corrente per visualizzarlo o copiarlo. Il valore originale e la bozza restano invariati.",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { withEnglishFallback } from "./fallback";
|
||||
import { mongodbDatabaseDumpJa as mongoDump } from "./mongodbDatabaseDump";
|
||||
import docs from "./docs/ja";
|
||||
import { consulJaMessages } from "./consulJa";
|
||||
import { nacosAccessControlMessages, nacosAccessControlTabs } from "./nacosAccessControl";
|
||||
@@ -83,6 +84,7 @@ const consul = {
|
||||
};
|
||||
|
||||
export default withEnglishFallback({
|
||||
mongoDump,
|
||||
cellTransform: {
|
||||
title: "変換プレビュー",
|
||||
description: "現在の値を変換して表示またはコピーします。元の値と編集中の内容は変更されません。",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { withEnglishFallback } from "./fallback";
|
||||
import { mongodbDatabaseDumpKo as mongoDump } from "./mongodbDatabaseDump";
|
||||
import { meilisearchManagementKo } from "./meilisearchManagement";
|
||||
import { redisGroupingKo as redisGrouping } from "./redisGrouping";
|
||||
import docs from "./docs/ko";
|
||||
@@ -82,6 +83,7 @@ const consul = {
|
||||
};
|
||||
|
||||
export default withEnglishFallback({
|
||||
mongoDump,
|
||||
cellTransform: {
|
||||
title: "변환 미리보기",
|
||||
description: "현재 값을 변환하여 보거나 복사합니다. 원래 값과 편집 초안은 변경되지 않습니다.",
|
||||
|
||||
@@ -0,0 +1,439 @@
|
||||
export const mongodbDatabaseDumpEn = {
|
||||
selectAll: "Select all collections",
|
||||
menuDump: "Dump database...",
|
||||
menuRestore: "Restore database...",
|
||||
dumpTitle: "MongoDB Database Dump",
|
||||
restoreTitle: "MongoDB Database Restore",
|
||||
format: "Format",
|
||||
archive: "MongoDB archive",
|
||||
directory: "MongoDB dump directory",
|
||||
gzip: "Gzip",
|
||||
source: "Backup source",
|
||||
readSource: "Read backup",
|
||||
sourceDatabase: "Source database",
|
||||
targetDatabase: "Target database",
|
||||
destination: "Destination",
|
||||
chooseDestination: "Choose destination",
|
||||
collection: "Collection",
|
||||
kind: "Type",
|
||||
documents: "Documents",
|
||||
indexes: "Indexes",
|
||||
dropExisting: "Drop selected target collections before restoring",
|
||||
restoreOptions: "Restore collection options",
|
||||
restoreIndexes: "Restore indexes",
|
||||
stopOnError: "Stop on write errors",
|
||||
objcheck: "Check BSON documents before insertion (objcheck)",
|
||||
readBytes: "Read: {size} GiB decoded",
|
||||
batchSize: "Batch size",
|
||||
selected: "{count} collections selected",
|
||||
empty: "No collections",
|
||||
preparing: "Reading backup...",
|
||||
loading: "Loading collections...",
|
||||
dump: "Start dump",
|
||||
restore: "Start restore",
|
||||
review: "Review",
|
||||
back: "Back",
|
||||
close: "Close",
|
||||
cancel: "Cancel task",
|
||||
confirmDump: "Dump {count} collections from {database}.",
|
||||
confirmRestore: "Restore {count} collections into {database}.",
|
||||
confirmDrop: "The selected target collections will be deleted and recreated.",
|
||||
confirmAppend: "Documents will be appended to existing collections.",
|
||||
readonly: "This connection is read-only.",
|
||||
done: "Completed",
|
||||
cancelled: "Cancelled",
|
||||
failed: "Failed",
|
||||
progress: "Collections: {done} / {total}",
|
||||
read: "Documents read",
|
||||
written: "Documents restored",
|
||||
rejected: "Failed documents",
|
||||
createdIndexes: "Indexes restored",
|
||||
uploadBytes: "Uploaded: {done} / {total} GiB",
|
||||
validationBytes: "Validated: {size} GiB decoded, {count} documents",
|
||||
phase: { preparing: "Preparing", uploading: "Uploading selected files", validating: "Validating backup", data: "Collection data", indexes: "Restoring indexes", views: "Restoring views", archive: "Writing backup", done: "Finished" },
|
||||
};
|
||||
|
||||
export const mongodbDatabaseDumpZhCN = {
|
||||
selectAll: "选择全部集合",
|
||||
menuDump: "备份数据库...",
|
||||
menuRestore: "恢复数据库...",
|
||||
dumpTitle: "MongoDB 数据库备份",
|
||||
restoreTitle: "MongoDB 数据库恢复",
|
||||
format: "格式",
|
||||
archive: "MongoDB archive",
|
||||
directory: "MongoDB dump 目录",
|
||||
gzip: "Gzip 压缩",
|
||||
source: "备份源",
|
||||
readSource: "读取备份",
|
||||
sourceDatabase: "源数据库",
|
||||
targetDatabase: "目标数据库",
|
||||
destination: "保存位置",
|
||||
chooseDestination: "选择保存位置",
|
||||
collection: "集合",
|
||||
kind: "类型",
|
||||
documents: "文档数",
|
||||
indexes: "索引数",
|
||||
dropExisting: "恢复前删除选中的目标同名集合",
|
||||
restoreOptions: "恢复集合选项",
|
||||
restoreIndexes: "恢复索引",
|
||||
stopOnError: "遇到写入错误时停止",
|
||||
objcheck: "写入前检查 BSON 文档(objcheck)",
|
||||
readBytes: "已读取:解压后 {size} GiB",
|
||||
batchSize: "批大小",
|
||||
selected: "已选择 {count} 个集合",
|
||||
empty: "没有集合",
|
||||
preparing: "正在读取备份...",
|
||||
loading: "正在读取集合...",
|
||||
dump: "开始备份",
|
||||
restore: "开始恢复",
|
||||
review: "确认操作",
|
||||
back: "返回",
|
||||
close: "关闭",
|
||||
cancel: "取消任务",
|
||||
confirmDump: "备份数据库 {database} 中的 {count} 个集合。",
|
||||
confirmRestore: "将 {count} 个集合恢复到数据库 {database}。",
|
||||
confirmDrop: "选中的目标同名集合将被删除并重建。",
|
||||
confirmAppend: "文档将追加到现有集合。",
|
||||
readonly: "该连接处于只读模式。",
|
||||
done: "已完成",
|
||||
cancelled: "已取消",
|
||||
failed: "失败",
|
||||
progress: "集合:{done} / {total}",
|
||||
read: "已读取文档",
|
||||
written: "已恢复文档",
|
||||
rejected: "失败文档",
|
||||
createdIndexes: "已恢复索引",
|
||||
uploadBytes: "已上传:{done} / {total} GiB",
|
||||
validationBytes: "已校验:解压后 {size} GiB,{count} 个文档",
|
||||
phase: { preparing: "准备中", uploading: "上传选中的文件", validating: "校验备份", data: "集合数据", indexes: "恢复索引", views: "恢复视图", archive: "写入备份", done: "已结束" },
|
||||
};
|
||||
|
||||
export const mongodbDatabaseDumpEs = {
|
||||
selectAll: "Seleccionar todas las colecciones",
|
||||
menuDump: "Crear copia de la base de datos...",
|
||||
menuRestore: "Restaurar base de datos...",
|
||||
dumpTitle: "Copia de seguridad de MongoDB",
|
||||
restoreTitle: "Restauración de MongoDB",
|
||||
format: "Formato",
|
||||
archive: "MongoDB archive",
|
||||
directory: "Directorio de copia de MongoDB",
|
||||
gzip: "Gzip",
|
||||
source: "Origen de la copia",
|
||||
readSource: "Leer copia",
|
||||
sourceDatabase: "Base de datos de origen",
|
||||
targetDatabase: "Base de datos de destino",
|
||||
destination: "Destino",
|
||||
chooseDestination: "Elegir destino",
|
||||
collection: "Colección",
|
||||
kind: "Tipo",
|
||||
documents: "Documentos",
|
||||
indexes: "Índices",
|
||||
dropExisting: "Eliminar las colecciones de destino seleccionadas antes de restaurar",
|
||||
restoreOptions: "Restaurar opciones de colección",
|
||||
restoreIndexes: "Restaurar índices",
|
||||
stopOnError: "Detener al producirse errores de escritura",
|
||||
objcheck: "Comprobar documentos BSON antes de insertarlos (objcheck)",
|
||||
readBytes: "Leídos: {size} GiB descomprimidos",
|
||||
batchSize: "Tamaño del lote",
|
||||
selected: "{count} colecciones seleccionadas",
|
||||
empty: "No hay colecciones",
|
||||
preparing: "Leyendo copia...",
|
||||
loading: "Cargando colecciones...",
|
||||
dump: "Iniciar copia",
|
||||
restore: "Iniciar restauración",
|
||||
review: "Revisar",
|
||||
back: "Volver",
|
||||
close: "Cerrar",
|
||||
cancel: "Cancelar tarea",
|
||||
confirmDump: "Crear una copia de {count} colecciones de {database}.",
|
||||
confirmRestore: "Restaurar {count} colecciones en {database}.",
|
||||
confirmDrop: "Las colecciones de destino seleccionadas se eliminarán y se volverán a crear.",
|
||||
confirmAppend: "Los documentos se añadirán a las colecciones existentes.",
|
||||
readonly: "Esta conexión es de solo lectura.",
|
||||
done: "Completado",
|
||||
cancelled: "Cancelado",
|
||||
failed: "Error",
|
||||
progress: "Colecciones: {done} / {total}",
|
||||
read: "Documentos leídos",
|
||||
written: "Documentos restaurados",
|
||||
rejected: "Documentos fallidos",
|
||||
createdIndexes: "Índices restaurados",
|
||||
uploadBytes: "Subidos: {done} / {total} GiB",
|
||||
validationBytes: "Validados: {size} GiB descomprimidos, {count} documentos",
|
||||
phase: { preparing: "Preparando", uploading: "Subiendo archivos seleccionados", validating: "Validando copia", data: "Datos de las colecciones", indexes: "Restaurando índices", views: "Restaurando vistas", archive: "Escribiendo copia", done: "Finalizado" },
|
||||
};
|
||||
|
||||
export const mongodbDatabaseDumpIt = {
|
||||
selectAll: "Seleziona tutte le raccolte",
|
||||
menuDump: "Backup del database...",
|
||||
menuRestore: "Ripristina database...",
|
||||
dumpTitle: "Backup del database MongoDB",
|
||||
restoreTitle: "Ripristino del database MongoDB",
|
||||
format: "Formato",
|
||||
archive: "MongoDB archive",
|
||||
directory: "Directory di backup MongoDB",
|
||||
gzip: "Gzip",
|
||||
source: "Origine del backup",
|
||||
readSource: "Leggi backup",
|
||||
sourceDatabase: "Database di origine",
|
||||
targetDatabase: "Database di destinazione",
|
||||
destination: "Destinazione",
|
||||
chooseDestination: "Scegli destinazione",
|
||||
collection: "Raccolta",
|
||||
kind: "Tipo",
|
||||
documents: "Documenti",
|
||||
indexes: "Indici",
|
||||
dropExisting: "Elimina le raccolte di destinazione selezionate prima del ripristino",
|
||||
restoreOptions: "Ripristina opzioni delle raccolte",
|
||||
restoreIndexes: "Ripristina indici",
|
||||
stopOnError: "Interrompi in caso di errori di scrittura",
|
||||
objcheck: "Verifica i documenti BSON prima di inserirli (objcheck)",
|
||||
readBytes: "Letti: {size} GiB decompressi",
|
||||
batchSize: "Dimensione del batch",
|
||||
selected: "{count} raccolte selezionate",
|
||||
empty: "Nessuna raccolta",
|
||||
preparing: "Lettura del backup...",
|
||||
loading: "Caricamento delle raccolte...",
|
||||
dump: "Avvia backup",
|
||||
restore: "Avvia ripristino",
|
||||
review: "Verifica",
|
||||
back: "Indietro",
|
||||
close: "Chiudi",
|
||||
cancel: "Annulla attività",
|
||||
confirmDump: "Esegui il backup di {count} raccolte da {database}.",
|
||||
confirmRestore: "Ripristina {count} raccolte in {database}.",
|
||||
confirmDrop: "Le raccolte di destinazione selezionate verranno eliminate e ricreate.",
|
||||
confirmAppend: "I documenti verranno aggiunti alle raccolte esistenti.",
|
||||
readonly: "Questa connessione è di sola lettura.",
|
||||
done: "Completato",
|
||||
cancelled: "Annullato",
|
||||
failed: "Non riuscito",
|
||||
progress: "Raccolte: {done} / {total}",
|
||||
read: "Documenti letti",
|
||||
written: "Documenti ripristinati",
|
||||
rejected: "Documenti non riusciti",
|
||||
createdIndexes: "Indici ripristinati",
|
||||
uploadBytes: "Caricati: {done} / {total} GiB",
|
||||
validationBytes: "Verificati: {size} GiB decompressi, {count} documenti",
|
||||
phase: { preparing: "Preparazione", uploading: "Caricamento dei file selezionati", validating: "Verifica del backup", data: "Dati delle raccolte", indexes: "Ripristino degli indici", views: "Ripristino delle viste", archive: "Scrittura del backup", done: "Terminato" },
|
||||
};
|
||||
|
||||
export const mongodbDatabaseDumpJa = {
|
||||
selectAll: "すべてのコレクションを選択",
|
||||
menuDump: "データベースをバックアップ...",
|
||||
menuRestore: "データベースを復元...",
|
||||
dumpTitle: "MongoDB データベースのバックアップ",
|
||||
restoreTitle: "MongoDB データベースの復元",
|
||||
format: "形式",
|
||||
archive: "MongoDB archive",
|
||||
directory: "MongoDB dump ディレクトリ",
|
||||
gzip: "Gzip 圧縮",
|
||||
source: "バックアップ元",
|
||||
readSource: "バックアップを読み込む",
|
||||
sourceDatabase: "復元元データベース",
|
||||
targetDatabase: "復元先データベース",
|
||||
destination: "保存先",
|
||||
chooseDestination: "保存先を選択",
|
||||
collection: "コレクション",
|
||||
kind: "種類",
|
||||
documents: "ドキュメント数",
|
||||
indexes: "インデックス数",
|
||||
dropExisting: "復元前に選択した復元先の同名コレクションを削除",
|
||||
restoreOptions: "コレクションのオプションを復元",
|
||||
restoreIndexes: "インデックスを復元",
|
||||
stopOnError: "書き込みエラーが発生したら停止",
|
||||
objcheck: "書き込み前に BSON ドキュメントを検証(objcheck)",
|
||||
readBytes: "読み込み済み: 展開後 {size} GiB",
|
||||
batchSize: "バッチサイズ",
|
||||
selected: "{count} 個のコレクションを選択済み",
|
||||
empty: "コレクションがありません",
|
||||
preparing: "バックアップを読み込み中...",
|
||||
loading: "コレクションを読み込み中...",
|
||||
dump: "バックアップ開始",
|
||||
restore: "復元開始",
|
||||
review: "操作を確認",
|
||||
back: "戻る",
|
||||
close: "閉じる",
|
||||
cancel: "タスクをキャンセル",
|
||||
confirmDump: "{database} の {count} 個のコレクションをバックアップします。",
|
||||
confirmRestore: "{count} 個のコレクションを {database} に復元します。",
|
||||
confirmDrop: "選択した復元先の同名コレクションは削除され、再作成されます。",
|
||||
confirmAppend: "ドキュメントは既存のコレクションに追加されます。",
|
||||
readonly: "この接続は読み取り専用です。",
|
||||
done: "完了",
|
||||
cancelled: "キャンセル済み",
|
||||
failed: "失敗",
|
||||
progress: "コレクション: {done} / {total}",
|
||||
read: "読み込んだドキュメント",
|
||||
written: "復元したドキュメント",
|
||||
rejected: "失敗したドキュメント",
|
||||
createdIndexes: "復元したインデックス",
|
||||
uploadBytes: "アップロード済み: {done} / {total} GiB",
|
||||
validationBytes: "検証済み: 展開後 {size} GiB、{count} ドキュメント",
|
||||
phase: { preparing: "準備中", uploading: "選択したファイルをアップロード中", validating: "バックアップを検証中", data: "コレクションデータ", indexes: "インデックスを復元中", views: "ビューを復元中", archive: "バックアップを書き込み中", done: "終了" },
|
||||
};
|
||||
|
||||
export const mongodbDatabaseDumpKo = {
|
||||
selectAll: "모든 컬렉션 선택",
|
||||
menuDump: "데이터베이스 백업...",
|
||||
menuRestore: "데이터베이스 복원...",
|
||||
dumpTitle: "MongoDB 데이터베이스 백업",
|
||||
restoreTitle: "MongoDB 데이터베이스 복원",
|
||||
format: "형식",
|
||||
archive: "MongoDB archive",
|
||||
directory: "MongoDB dump 디렉터리",
|
||||
gzip: "Gzip 압축",
|
||||
source: "백업 원본",
|
||||
readSource: "백업 읽기",
|
||||
sourceDatabase: "원본 데이터베이스",
|
||||
targetDatabase: "대상 데이터베이스",
|
||||
destination: "저장 위치",
|
||||
chooseDestination: "저장 위치 선택",
|
||||
collection: "컬렉션",
|
||||
kind: "유형",
|
||||
documents: "문서 수",
|
||||
indexes: "인덱스 수",
|
||||
dropExisting: "복원 전 선택한 대상의 동명 컬렉션 삭제",
|
||||
restoreOptions: "컬렉션 옵션 복원",
|
||||
restoreIndexes: "인덱스 복원",
|
||||
stopOnError: "쓰기 오류 발생 시 중지",
|
||||
objcheck: "쓰기 전 BSON 문서 검사 (objcheck)",
|
||||
readBytes: "읽음: 압축 해제 후 {size} GiB",
|
||||
batchSize: "배치 크기",
|
||||
selected: "컬렉션 {count}개 선택됨",
|
||||
empty: "컬렉션 없음",
|
||||
preparing: "백업 읽는 중...",
|
||||
loading: "컬렉션 불러오는 중...",
|
||||
dump: "백업 시작",
|
||||
restore: "복원 시작",
|
||||
review: "작업 확인",
|
||||
back: "뒤로",
|
||||
close: "닫기",
|
||||
cancel: "작업 취소",
|
||||
confirmDump: "{database}의 컬렉션 {count}개를 백업합니다.",
|
||||
confirmRestore: "컬렉션 {count}개를 {database}에 복원합니다.",
|
||||
confirmDrop: "선택한 대상의 동명 컬렉션이 삭제된 후 다시 생성됩니다.",
|
||||
confirmAppend: "기존 컬렉션에 문서가 추가됩니다.",
|
||||
readonly: "이 연결은 읽기 전용입니다.",
|
||||
done: "완료",
|
||||
cancelled: "취소됨",
|
||||
failed: "실패",
|
||||
progress: "컬렉션: {done} / {total}",
|
||||
read: "읽은 문서",
|
||||
written: "복원된 문서",
|
||||
rejected: "실패한 문서",
|
||||
createdIndexes: "복원된 인덱스",
|
||||
uploadBytes: "업로드됨: {done} / {total} GiB",
|
||||
validationBytes: "검증됨: 압축 해제 후 {size} GiB, 문서 {count}개",
|
||||
phase: { preparing: "준비 중", uploading: "선택한 파일 업로드 중", validating: "백업 검증 중", data: "컬렉션 데이터", indexes: "인덱스 복원 중", views: "뷰 복원 중", archive: "백업 쓰는 중", done: "종료됨" },
|
||||
};
|
||||
|
||||
export const mongodbDatabaseDumpPtBR = {
|
||||
selectAll: "Selecionar todas as coleções",
|
||||
menuDump: "Fazer backup do banco de dados...",
|
||||
menuRestore: "Restaurar banco de dados...",
|
||||
dumpTitle: "Backup do banco de dados MongoDB",
|
||||
restoreTitle: "Restauração do banco de dados MongoDB",
|
||||
format: "Formato",
|
||||
archive: "MongoDB archive",
|
||||
directory: "Diretório de backup do MongoDB",
|
||||
gzip: "Gzip",
|
||||
source: "Origem do backup",
|
||||
readSource: "Ler backup",
|
||||
sourceDatabase: "Banco de dados de origem",
|
||||
targetDatabase: "Banco de dados de destino",
|
||||
destination: "Destino",
|
||||
chooseDestination: "Escolher destino",
|
||||
collection: "Coleção",
|
||||
kind: "Tipo",
|
||||
documents: "Documentos",
|
||||
indexes: "Índices",
|
||||
dropExisting: "Excluir as coleções de destino selecionadas antes de restaurar",
|
||||
restoreOptions: "Restaurar opções das coleções",
|
||||
restoreIndexes: "Restaurar índices",
|
||||
stopOnError: "Parar ao ocorrerem erros de gravação",
|
||||
objcheck: "Verificar documentos BSON antes da inserção (objcheck)",
|
||||
readBytes: "Lidos: {size} GiB descompactados",
|
||||
batchSize: "Tamanho do lote",
|
||||
selected: "{count} coleções selecionadas",
|
||||
empty: "Nenhuma coleção",
|
||||
preparing: "Lendo backup...",
|
||||
loading: "Carregando coleções...",
|
||||
dump: "Iniciar backup",
|
||||
restore: "Iniciar restauração",
|
||||
review: "Revisar",
|
||||
back: "Voltar",
|
||||
close: "Fechar",
|
||||
cancel: "Cancelar tarefa",
|
||||
confirmDump: "Fazer backup de {count} coleções de {database}.",
|
||||
confirmRestore: "Restaurar {count} coleções em {database}.",
|
||||
confirmDrop: "As coleções de destino selecionadas serão excluídas e recriadas.",
|
||||
confirmAppend: "Os documentos serão adicionados às coleções existentes.",
|
||||
readonly: "Esta conexão é somente leitura.",
|
||||
done: "Concluído",
|
||||
cancelled: "Cancelado",
|
||||
failed: "Falhou",
|
||||
progress: "Coleções: {done} / {total}",
|
||||
read: "Documentos lidos",
|
||||
written: "Documentos restaurados",
|
||||
rejected: "Documentos com falha",
|
||||
createdIndexes: "Índices restaurados",
|
||||
uploadBytes: "Enviados: {done} / {total} GiB",
|
||||
validationBytes: "Validados: {size} GiB descompactados, {count} documentos",
|
||||
phase: { preparing: "Preparando", uploading: "Enviando arquivos selecionados", validating: "Validando backup", data: "Dados das coleções", indexes: "Restaurando índices", views: "Restaurando visualizações", archive: "Gravando backup", done: "Finalizado" },
|
||||
};
|
||||
|
||||
export const mongodbDatabaseDumpZhTW = {
|
||||
selectAll: "選取所有集合",
|
||||
menuDump: "備份資料庫...",
|
||||
menuRestore: "還原資料庫...",
|
||||
dumpTitle: "MongoDB 資料庫備份",
|
||||
restoreTitle: "MongoDB 資料庫還原",
|
||||
format: "格式",
|
||||
archive: "MongoDB archive",
|
||||
directory: "MongoDB dump 目錄",
|
||||
gzip: "Gzip 壓縮",
|
||||
source: "備份來源",
|
||||
readSource: "讀取備份",
|
||||
sourceDatabase: "來源資料庫",
|
||||
targetDatabase: "目標資料庫",
|
||||
destination: "儲存位置",
|
||||
chooseDestination: "選擇儲存位置",
|
||||
collection: "集合",
|
||||
kind: "類型",
|
||||
documents: "文件數",
|
||||
indexes: "索引數",
|
||||
dropExisting: "還原前刪除選取的目標同名集合",
|
||||
restoreOptions: "還原集合選項",
|
||||
restoreIndexes: "還原索引",
|
||||
stopOnError: "遇到寫入錯誤時停止",
|
||||
objcheck: "寫入前檢查 BSON 文件(objcheck)",
|
||||
readBytes: "已讀取:解壓縮後 {size} GiB",
|
||||
batchSize: "批次大小",
|
||||
selected: "已選取 {count} 個集合",
|
||||
empty: "沒有集合",
|
||||
preparing: "正在讀取備份...",
|
||||
loading: "正在載入集合...",
|
||||
dump: "開始備份",
|
||||
restore: "開始還原",
|
||||
review: "確認操作",
|
||||
back: "返回",
|
||||
close: "關閉",
|
||||
cancel: "取消任務",
|
||||
confirmDump: "備份資料庫 {database} 中的 {count} 個集合。",
|
||||
confirmRestore: "將 {count} 個集合還原至資料庫 {database}。",
|
||||
confirmDrop: "選取的目標同名集合將被刪除並重建。",
|
||||
confirmAppend: "文件將附加至現有集合。",
|
||||
readonly: "此連線處於唯讀模式。",
|
||||
done: "已完成",
|
||||
cancelled: "已取消",
|
||||
failed: "失敗",
|
||||
progress: "集合:{done} / {total}",
|
||||
read: "已讀取文件",
|
||||
written: "已還原文件",
|
||||
rejected: "失敗文件",
|
||||
createdIndexes: "已還原索引",
|
||||
uploadBytes: "已上傳:{done} / {total} GiB",
|
||||
validationBytes: "已驗證:解壓縮後 {size} GiB,{count} 個文件",
|
||||
phase: { preparing: "準備中", uploading: "上傳選取的檔案", validating: "驗證備份", data: "集合資料", indexes: "還原索引", views: "還原檢視表", archive: "寫入備份", done: "已結束" },
|
||||
};
|
||||
@@ -1,4 +1,5 @@
|
||||
import { withEnglishFallback } from "./fallback";
|
||||
import { mongodbDatabaseDumpPtBR as mongoDump } from "./mongodbDatabaseDump";
|
||||
import { meilisearchManagementPtBR } from "./meilisearchManagement";
|
||||
import { redisGroupingPtBR as redisGrouping } from "./redisGrouping";
|
||||
import docs from "./docs/pt-BR";
|
||||
@@ -83,6 +84,7 @@ const consul = {
|
||||
};
|
||||
|
||||
export default withEnglishFallback({
|
||||
mongoDump,
|
||||
cellTransform: {
|
||||
title: "Prévia da conversão",
|
||||
description: "Converte o valor atual para visualizar ou copiar. O valor original e o rascunho permanecem inalterados.",
|
||||
|
||||
@@ -3,6 +3,7 @@ import { withEnglishFallback } from "./fallback";
|
||||
import docs from "./docs/zh-CN";
|
||||
import { consul } from "./consulZhCN";
|
||||
import { meilisearchManagementZhCN } from "./meilisearchManagement";
|
||||
import { mongodbDatabaseDumpZhCN as mongoDump } from "./mongodbDatabaseDump";
|
||||
|
||||
export default withEnglishFallback({
|
||||
cellTransform: {
|
||||
@@ -5457,6 +5458,7 @@ export default withEnglishFallback({
|
||||
},
|
||||
},
|
||||
},
|
||||
mongoDump,
|
||||
mongo: {
|
||||
documents: "{count} 个文档",
|
||||
addField: "添加字段",
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { redisGroupingZhTW as redisGrouping } from "./redisGrouping";
|
||||
import { withEnglishFallback } from "./fallback";
|
||||
import { mongodbDatabaseDumpZhTW as mongoDump } from "./mongodbDatabaseDump";
|
||||
import { meilisearchManagementZhTW } from "./meilisearchManagement";
|
||||
import docs from "./docs/zh-TW";
|
||||
import { consulZhTWMessages } from "./consulZhTW";
|
||||
@@ -82,6 +83,7 @@ const consul = {
|
||||
};
|
||||
|
||||
export default withEnglishFallback({
|
||||
mongoDump,
|
||||
cellTransform: {
|
||||
title: "轉換預覽",
|
||||
description: "將目前內容轉換後檢視或複製,不改變原值和編輯草稿。",
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { prepareMongodbRestoreSource, restoreMongodbDatabase } from "../http";
|
||||
|
||||
function file(name: string, size = 0) {
|
||||
const value = new File([], name);
|
||||
Object.defineProperty(value, "webkitRelativePath", { value: `dump/source/${name}` });
|
||||
Object.defineProperty(value, "size", { value: size });
|
||||
return value;
|
||||
}
|
||||
|
||||
afterEach(() => vi.unstubAllGlobals());
|
||||
|
||||
describe("MongoDB metadata-first transport", () => {
|
||||
it("previews a multi-gigabyte directory without uploading any BSON bodies", async () => {
|
||||
const fetch = vi.fn().mockResolvedValue(new Response(JSON.stringify({ sourceRef: "catalog", databases: ["source"], collections: [] })));
|
||||
vi.stubGlobal("fetch", fetch);
|
||||
const data = file("large.bson.gz", 5 * 1024 ** 3);
|
||||
const metadata = file("large.metadata.json.gz", 200);
|
||||
await prepareMongodbRestoreSource([data, metadata], "directory", true);
|
||||
expect(fetch).toHaveBeenCalledTimes(1);
|
||||
const form = fetch.mock.calls[0]![1].body as FormData;
|
||||
expect(form.get("format")).toBe("directory");
|
||||
expect(form.get("gzip")).toBe("true");
|
||||
expect(JSON.parse(form.get("manifest") as string)).toEqual([
|
||||
{ path: "dump/source/large.bson.gz", sizeBytes: 5 * 1024 ** 3 },
|
||||
{ path: "dump/source/large.metadata.json.gz", sizeBytes: 200 },
|
||||
]);
|
||||
expect(form.getAll("file")).toHaveLength(1);
|
||||
expect((form.get("file") as File).name).toBe("dump/source/large.metadata.json.gz");
|
||||
});
|
||||
|
||||
it("checks the effective server limit before the confirmed data upload", async () => {
|
||||
const fetch = vi.fn().mockResolvedValue(new Response(JSON.stringify(1024 ** 3)));
|
||||
vi.stubGlobal("fetch", fetch);
|
||||
const request = { taskId: "task", connectionId: "conn", database: "target", sourceDatabase: "source", sourceRef: "catalog", dropExisting: true, restoreOptions: true, restoreIndexes: true, stopOnError: true, batchSize: 500 };
|
||||
await expect(restoreMongodbDatabase(request, vi.fn(), { files: [file("large.bson", 5 * 1024 ** 3)], gzip: false })).rejects.toThrow("exceeds");
|
||||
expect(fetch).toHaveBeenCalledTimes(1);
|
||||
expect(fetch.mock.calls[0]![0]).toContain("/upload-limit");
|
||||
});
|
||||
|
||||
it("passes cancellation to catalog upload without a separate restore request", async () => {
|
||||
const controller = new AbortController();
|
||||
const fetch = vi.fn().mockRejectedValue(new DOMException("Aborted", "AbortError"));
|
||||
vi.stubGlobal("fetch", fetch);
|
||||
controller.abort();
|
||||
await expect(prepareMongodbRestoreSource([file("a.bson")], "directory", false, { signal: controller.signal })).rejects.toThrow("Aborted");
|
||||
expect(fetch.mock.calls[0]![1].signal).toBe(controller.signal);
|
||||
expect(fetch).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -495,6 +495,13 @@ export const cancelMongodbImport = forward("cancelMongodbImport");
|
||||
export const releaseMongodbImportSource = forward("releaseMongodbImportSource");
|
||||
export const exportMongodbQuery = forward("exportMongodbQuery");
|
||||
export const cancelMongodbExport = forward("cancelMongodbExport");
|
||||
export const inspectMongodbDatabaseDump = forward("inspectMongodbDatabaseDump");
|
||||
export const prepareMongodbRestoreSource = forward("prepareMongodbRestoreSource");
|
||||
export const releaseMongodbRestoreSource = forward("releaseMongodbRestoreSource");
|
||||
export const dumpMongodbDatabase = forward("dumpMongodbDatabase");
|
||||
export const restoreMongodbDatabase = forward("restoreMongodbDatabase");
|
||||
export const cancelMongodbDatabaseDump = forward("cancelMongodbDatabaseDump");
|
||||
export type { MongoDumpFormat, MongoDumpSourceInput, MongoDumpCatalog, MongoDumpCollection, MongoRestoreSourcePreview, MongoDatabaseDumpRequest, MongoDatabaseRestoreRequest, MongoDatabaseDumpProgress } from "./mongodbDumpTypes";
|
||||
|
||||
// Database Export
|
||||
export const beginDatabaseBackupSnapshot = forward("beginDatabaseBackupSnapshot");
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import type { MongoDumpFormat, MongoDumpSourceInput, MongoDumpCatalog, MongoRestoreSourcePreview, MongoDatabaseDumpRequest, MongoDatabaseRestoreRequest, MongoDatabaseDumpProgress } from "./mongodbDumpTypes";
|
||||
import type { MongoRestoreUpload, MongoSourceReadOptions } from "./mongodbDumpTypes";
|
||||
import type {
|
||||
ConnectionConfig,
|
||||
ConnectionTestResult,
|
||||
@@ -2674,6 +2676,143 @@ export async function releaseTableImportSource(sourceRef: string): Promise<boole
|
||||
return result.released;
|
||||
}
|
||||
|
||||
export function inspectMongodbDatabaseDump(connectionId: string, database: string): Promise<MongoDumpCatalog> {
|
||||
return post("/api/mongo/dump/catalog", { connectionId, database });
|
||||
}
|
||||
async function checkMongoUploadSize(files: File[]) {
|
||||
const limit = await get<number>("/api/mongo/dump/upload-limit");
|
||||
const size = files.reduce((total, file) => total + file.size, 0);
|
||||
if (size > limit) throw new Error(`MongoDB upload: ${(size / 1024 ** 3).toFixed(2)} GiB exceeds ${(limit / 1024 ** 3).toFixed(2)} GiB (DBX_MAX_UPLOAD_MB)`);
|
||||
}
|
||||
|
||||
function uploadMongoForm(url: string, form: FormData, options?: MongoSourceReadOptions): Promise<MongoRestoreSourcePreview> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const xhr = new XMLHttpRequest();
|
||||
const abort = () => xhr.abort();
|
||||
const cleanup = () => options?.signal?.removeEventListener("abort", abort);
|
||||
if (options?.signal?.aborted) {
|
||||
reject(new DOMException("Cancelled", "AbortError"));
|
||||
return;
|
||||
}
|
||||
xhr.open("POST", apiUrl(url));
|
||||
xhr.upload.onprogress = (event) => options?.onUploadProgress?.(event.loaded, event.lengthComputable ? event.total : 0);
|
||||
xhr.onload = async () => {
|
||||
cleanup();
|
||||
try {
|
||||
if (xhr.status < 200 || xhr.status >= 300) throw await backendResponseError(new Response(xhr.responseText, { status: xhr.status }));
|
||||
resolve(JSON.parse(xhr.responseText));
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
}
|
||||
};
|
||||
xhr.onerror = () => {
|
||||
cleanup();
|
||||
reject(new Error("MongoDB upload connection failed"));
|
||||
};
|
||||
xhr.onabort = () => {
|
||||
cleanup();
|
||||
reject(new DOMException("Cancelled", "AbortError"));
|
||||
};
|
||||
options?.signal?.addEventListener("abort", abort, { once: true });
|
||||
xhr.send(form);
|
||||
});
|
||||
}
|
||||
|
||||
export async function prepareMongodbRestoreSource(source: MongoDumpSourceInput, format: MongoDumpFormat, gzip: boolean, options?: MongoSourceReadOptions): Promise<MongoRestoreSourcePreview> {
|
||||
if (typeof source === "string") throw new Error("Web restores require uploaded files");
|
||||
const files = Array.isArray(source) ? source : [source];
|
||||
const form = new FormData();
|
||||
form.append("format", format);
|
||||
form.append("gzip", String(gzip));
|
||||
if (format === "directory") {
|
||||
form.append("manifest", JSON.stringify(files.map((file) => ({ path: file.webkitRelativePath || file.name, sizeBytes: file.size }))));
|
||||
for (const file of files) if (/\.metadata\.json(?:\.gz)?$/.test(file.name)) form.append("file", file, file.webkitRelativePath || file.name);
|
||||
} else {
|
||||
await checkMongoUploadSize(files);
|
||||
for (const file of files) form.append("file", file, file.name);
|
||||
return uploadMongoForm("/api/mongo/dump/source", form, options);
|
||||
}
|
||||
const response = await fetch(apiUrl("/api/mongo/dump/source"), { method: "POST", body: form, signal: options?.signal });
|
||||
if (!response.ok) throw await backendResponseError(response);
|
||||
return response.json();
|
||||
}
|
||||
export async function releaseMongodbRestoreSource(sourceRef: string): Promise<boolean> {
|
||||
return (await post<{ released: boolean }>("/api/mongo/dump/source/release", { sourceRef })).released;
|
||||
}
|
||||
async function runMongodbDatabaseTask(mode: "export" | "restore", request: MongoDatabaseDumpRequest | MongoDatabaseRestoreRequest, onProgress: (progress: MongoDatabaseDumpProgress) => void): Promise<MongoDatabaseDumpProgress> {
|
||||
await post(`/api/mongo/dump/${mode}`, { request });
|
||||
return new Promise((resolve, reject) => {
|
||||
const events = new EventSource(apiUrl(`/api/mongo/dump/progress/${request.taskId}`));
|
||||
events.onmessage = (event) => {
|
||||
const progress: MongoDatabaseDumpProgress = JSON.parse(event.data);
|
||||
onProgress(progress);
|
||||
if (progress.status === "running") return;
|
||||
events.close();
|
||||
if (progress.status !== "done") {
|
||||
reject(new Error(progress.errorMessage || "MongoDB database task failed"));
|
||||
return;
|
||||
}
|
||||
if (mode === "export") {
|
||||
const anchor = document.createElement("a");
|
||||
anchor.href = apiUrl(`/api/mongo/export/download/${request.taskId}`);
|
||||
anchor.click();
|
||||
}
|
||||
resolve(progress);
|
||||
};
|
||||
events.onerror = () => {
|
||||
events.close();
|
||||
reject(new Error("MongoDB database progress connection failed"));
|
||||
};
|
||||
});
|
||||
}
|
||||
export function dumpMongodbDatabase(request: MongoDatabaseDumpRequest, onProgress: (progress: MongoDatabaseDumpProgress) => void) {
|
||||
return runMongodbDatabaseTask("export", request, onProgress);
|
||||
}
|
||||
export async function restoreMongodbDatabase(request: MongoDatabaseRestoreRequest, onProgress: (progress: MongoDatabaseDumpProgress) => void, upload?: MongoRestoreUpload) {
|
||||
let acquired: string | undefined;
|
||||
try {
|
||||
if (upload) {
|
||||
await checkMongoUploadSize(upload.files);
|
||||
const form = new FormData();
|
||||
form.append("format", "directory");
|
||||
form.append("gzip", String(upload.gzip));
|
||||
form.append("request", JSON.stringify(request));
|
||||
for (const file of upload.files) form.append("file", file, file.webkitRelativePath || file.name);
|
||||
const started = Date.now();
|
||||
const preview = await uploadMongoForm("/api/mongo/dump/source/upload", form, {
|
||||
signal: upload.signal,
|
||||
onUploadProgress: (loaded, total) =>
|
||||
onProgress({
|
||||
taskId: request.taskId,
|
||||
status: "running",
|
||||
phase: "uploading",
|
||||
collection: null,
|
||||
collectionsDone: 0,
|
||||
collectionsTotal: request.collections?.length ?? 0,
|
||||
documentsRead: 0,
|
||||
documentsWritten: 0,
|
||||
documentsFailed: 0,
|
||||
indexesCreated: 0,
|
||||
elapsedMs: Date.now() - started,
|
||||
errorMessage: null,
|
||||
filePath: null,
|
||||
bytesProcessed: loaded,
|
||||
bytesTotal: total,
|
||||
}),
|
||||
});
|
||||
acquired = preview.sourceRef;
|
||||
if (upload.signal?.aborted) throw new DOMException("Cancelled", "AbortError");
|
||||
request = { ...request, sourceRef: acquired };
|
||||
}
|
||||
return await runMongodbDatabaseTask("restore", request, onProgress);
|
||||
} finally {
|
||||
if (acquired) void releaseMongodbRestoreSource(acquired).catch(() => {});
|
||||
}
|
||||
}
|
||||
export async function cancelMongodbDatabaseDump(taskId: string): Promise<boolean> {
|
||||
return (await post<{ cancelled: boolean }>("/api/mongo/dump/cancel", { taskId })).cancelled;
|
||||
}
|
||||
|
||||
export async function previewMongodbImportFile(fileOrPath: string | File | MongoImportPreviewRequest, options: Partial<MongoImportPreviewRequest> = {}): Promise<MongoImportPreview> {
|
||||
if (typeof fileOrPath === "object" && !(fileOrPath instanceof File)) {
|
||||
throw new Error("previewMongodbImportFile in web mode requires a File object for upload previews");
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
export type MongoDumpFormat = "directory" | "archive";
|
||||
export type MongoDumpSourceInput = string | File | File[];
|
||||
|
||||
export interface MongoDumpCollection {
|
||||
database: string;
|
||||
name: string;
|
||||
kind: string;
|
||||
documents: number | null;
|
||||
sizeBytes: number;
|
||||
indexes: number;
|
||||
sourceFiles?: string[];
|
||||
}
|
||||
export interface MongoDumpCatalog {
|
||||
databases: string[];
|
||||
collections: MongoDumpCollection[];
|
||||
}
|
||||
export interface MongoRestoreSourcePreview extends MongoDumpCatalog {
|
||||
sourceRef: string;
|
||||
}
|
||||
export interface MongoDatabaseDumpRequest {
|
||||
taskId: string;
|
||||
connectionId: string;
|
||||
database: string;
|
||||
filePath: string;
|
||||
format: MongoDumpFormat;
|
||||
gzip: boolean;
|
||||
collections?: string[];
|
||||
}
|
||||
export interface MongoDatabaseRestoreRequest {
|
||||
taskId: string;
|
||||
connectionId: string;
|
||||
database: string;
|
||||
sourceDatabase: string;
|
||||
sourceRef: string;
|
||||
collections?: string[];
|
||||
dropExisting: boolean;
|
||||
restoreOptions: boolean;
|
||||
restoreIndexes: boolean;
|
||||
stopOnError: boolean;
|
||||
objcheck?: boolean;
|
||||
batchSize: number;
|
||||
executionId?: string;
|
||||
}
|
||||
export interface MongoDatabaseDumpProgress {
|
||||
taskId: string;
|
||||
status: "running" | "done" | "error" | "cancelled";
|
||||
phase: "preparing" | "uploading" | "validating" | "data" | "indexes" | "views" | "archive" | "done";
|
||||
collection: string | null;
|
||||
collectionsDone: number;
|
||||
collectionsTotal: number;
|
||||
documentsRead: number;
|
||||
documentsWritten: number;
|
||||
documentsFailed: number;
|
||||
indexesCreated: number;
|
||||
elapsedMs: number;
|
||||
errorMessage: string | null;
|
||||
filePath: string | null;
|
||||
bytesProcessed?: number;
|
||||
bytesTotal?: number;
|
||||
documentsValidated?: number;
|
||||
}
|
||||
|
||||
export interface MongoRestoreUpload {
|
||||
files: File[];
|
||||
gzip: boolean;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface MongoSourceReadOptions {
|
||||
signal?: AbortSignal;
|
||||
onUploadProgress?: (loaded: number, total: number) => void;
|
||||
}
|
||||
@@ -1,4 +1,6 @@
|
||||
import { invoke as tauriInvoke } from "@tauri-apps/api/core";
|
||||
import type { MongoDumpFormat, MongoDumpSourceInput, MongoDumpCatalog, MongoRestoreSourcePreview, MongoDatabaseDumpRequest, MongoDatabaseRestoreRequest, MongoDatabaseDumpProgress } from "./mongodbDumpTypes";
|
||||
import type { MongoRestoreUpload, MongoSourceReadOptions } from "./mongodbDumpTypes";
|
||||
import { assertUpdateAllowsCommand } from "@/lib/app/updatePreparation";
|
||||
import { collectBrowserSupportInfo } from "@/lib/app/supportInfo";
|
||||
|
||||
@@ -5070,12 +5072,42 @@ export async function releaseTableImportSource(_sourceRef: string): Promise<bool
|
||||
return false;
|
||||
}
|
||||
|
||||
export type MongoImportFormat = "csv" | "json" | "ndjson";
|
||||
export function inspectMongodbDatabaseDump(connectionId: string, database: string): Promise<MongoDumpCatalog> {
|
||||
return invoke("inspect_mongodb_database_dump", { connectionId, database });
|
||||
}
|
||||
export function prepareMongodbRestoreSource(source: MongoDumpSourceInput, format: MongoDumpFormat, gzip: boolean, _options?: MongoSourceReadOptions): Promise<MongoRestoreSourcePreview> {
|
||||
if (typeof source !== "string") throw new Error("Desktop restores require a file or directory path");
|
||||
return invoke("prepare_mongodb_restore_source", { request: { path: source, format, gzip } });
|
||||
}
|
||||
export function releaseMongodbRestoreSource(sourceRef: string): Promise<boolean> {
|
||||
return invoke("release_mongodb_restore_source", { sourceRef });
|
||||
}
|
||||
async function runMongodbDatabaseTask(command: string, request: MongoDatabaseDumpRequest | MongoDatabaseRestoreRequest, onProgress: (progress: MongoDatabaseDumpProgress) => void): Promise<MongoDatabaseDumpProgress> {
|
||||
const unlisten = await listen<MongoDatabaseDumpProgress>("mongo-database-dump-progress", (event) => {
|
||||
if (event.payload.taskId === request.taskId) onProgress(event.payload);
|
||||
});
|
||||
try {
|
||||
return await invoke(command, { request });
|
||||
} finally {
|
||||
unlisten();
|
||||
}
|
||||
}
|
||||
export function dumpMongodbDatabase(request: MongoDatabaseDumpRequest, onProgress: (progress: MongoDatabaseDumpProgress) => void) {
|
||||
return runMongodbDatabaseTask("dump_mongodb_database", request, onProgress);
|
||||
}
|
||||
export function restoreMongodbDatabase(request: MongoDatabaseRestoreRequest, onProgress: (progress: MongoDatabaseDumpProgress) => void, _upload?: MongoRestoreUpload) {
|
||||
return runMongodbDatabaseTask("restore_mongodb_database", request, onProgress);
|
||||
}
|
||||
export function cancelMongodbDatabaseDump(taskId: string): Promise<boolean> {
|
||||
return invoke("cancel_mongodb_database_dump", { taskId });
|
||||
}
|
||||
|
||||
export type MongoImportFormat = "csv" | "json" | "ndjson" | "bson";
|
||||
export type MongoImportTypeMode = "string" | "auto" | "extendedJson";
|
||||
export type MongoImportInferredType = "boolean" | "integer" | "decimal" | "date" | "objectId" | "object" | "array" | "mixed" | "string";
|
||||
export type MongoImportStatus = "running" | "done" | "error" | "cancelled";
|
||||
export type MongoImportPhase = "preparing" | "parsing" | "writing" | "done";
|
||||
export type MongoExportFormat = "csv" | "ndjson";
|
||||
export type MongoExportFormat = "csv" | "ndjson" | "bson";
|
||||
export type MongoExportStatus = "running" | "done" | "error" | "cancelled";
|
||||
|
||||
export interface MongoImportIssue {
|
||||
@@ -5176,6 +5208,7 @@ export interface MongoExportRequest {
|
||||
collation?: string | null;
|
||||
format: MongoExportFormat;
|
||||
includeHeader?: boolean;
|
||||
gzip?: boolean;
|
||||
filePath: string;
|
||||
executionId?: string | null;
|
||||
}
|
||||
|
||||
@@ -559,6 +559,7 @@ export const useConnectionStore = defineStore("connection", () => {
|
||||
schema?: string;
|
||||
tableName?: string;
|
||||
} | null>(null);
|
||||
const mongoDatabaseDumpSource = ref<{ connectionId: string; database: string; mode: "dump" | "restore" } | null>(null);
|
||||
const mongoImportSource = ref<{
|
||||
connectionId: string;
|
||||
database: string;
|
||||
@@ -9409,6 +9410,7 @@ export const useConnectionStore = defineStore("connection", () => {
|
||||
diagramSource,
|
||||
docsSource,
|
||||
tableImportSource,
|
||||
mongoDatabaseDumpSource,
|
||||
mongoImportSource,
|
||||
mongoImportCompleted,
|
||||
tableDataGenerateSource,
|
||||
|
||||
@@ -83,6 +83,8 @@ bytes = "1"
|
||||
font-kit = { version = "0.14.3", optional = true }
|
||||
zip = { version = "4", default-features = false, features = ["deflate"] }
|
||||
flate2 = "1"
|
||||
crc = "3"
|
||||
sha1 = "0.10"
|
||||
tar = "0.4"
|
||||
zstd = "0.13"
|
||||
sysinfo = { version = "0.32", features = ["system"] }
|
||||
|
||||
@@ -2579,17 +2579,17 @@ pub struct MongoInsertOutcome {
|
||||
pub errors: Vec<MongoBulkWriteError>,
|
||||
}
|
||||
|
||||
pub async fn insert_bson_documents(
|
||||
pub async fn insert_bson_documents<T: serde::Serialize + Send + Sync>(
|
||||
client: &Client,
|
||||
database: &str,
|
||||
collection: &str,
|
||||
documents: Vec<Document>,
|
||||
documents: Vec<T>,
|
||||
) -> Result<MongoInsertOutcome, MongoBulkWriteError> {
|
||||
if documents.is_empty() {
|
||||
return Ok(MongoInsertOutcome::default());
|
||||
}
|
||||
let total = documents.len() as u64;
|
||||
let col = client.database(database).collection::<Document>(collection);
|
||||
let col = client.database(database).collection::<T>(collection);
|
||||
// Unordered: one rejected document must not abandon the rest of the batch, and the server
|
||||
// then reports every rejection instead of stopping at the first.
|
||||
match col.insert_many(documents).ordered(false).await {
|
||||
|
||||
@@ -58,6 +58,7 @@ pub mod models;
|
||||
pub mod mongo_oidc;
|
||||
pub mod mongo_ops;
|
||||
pub mod mongo_shell;
|
||||
pub mod mongodb_dump;
|
||||
pub mod mongodb_import_export;
|
||||
#[cfg(feature = "mq-admin")]
|
||||
pub mod mq;
|
||||
|
||||
@@ -0,0 +1,639 @@
|
||||
//! Database-scoped MongoDB dump/restore; transport adapters own dialogs and uploads.
|
||||
mod archive;
|
||||
mod directory;
|
||||
mod metadata;
|
||||
mod source;
|
||||
mod stream;
|
||||
|
||||
pub use source::{
|
||||
attach_mongodb_restore_upload, mongodb_restore_upload_files, prepare_mongodb_directory_catalog,
|
||||
prepare_owned_mongodb_restore_source, MongoDumpFile,
|
||||
};
|
||||
|
||||
use std::collections::{BTreeSet, HashMap, HashSet};
|
||||
use std::future::Future;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::pin::Pin;
|
||||
use std::sync::{
|
||||
atomic::{AtomicBool, Ordering},
|
||||
Arc,
|
||||
};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use futures::TryStreamExt;
|
||||
use mongodb::bson::{doc, Document};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::connection::{AppState, PoolKind};
|
||||
use crate::mongodb_import_export::{export_mongodb_query_core, MongoExportFormat, MongoExportRequest};
|
||||
use metadata::CollectionMetadata;
|
||||
|
||||
type CancelFuture = Pin<Box<dyn Future<Output = bool> + Send>>;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum MongoDumpFormat {
|
||||
Directory,
|
||||
Archive,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct MongoDumpCollection {
|
||||
pub database: String,
|
||||
pub name: String,
|
||||
pub kind: String,
|
||||
pub documents: Option<u64>,
|
||||
pub size_bytes: u64,
|
||||
pub indexes: usize,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub source_files: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct MongoDumpCatalog {
|
||||
pub databases: Vec<String>,
|
||||
pub collections: Vec<MongoDumpCollection>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct MongoRestoreSourceRequest {
|
||||
pub path: String,
|
||||
pub format: MongoDumpFormat,
|
||||
#[serde(default)]
|
||||
pub gzip: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct MongoRestoreSourcePreview {
|
||||
pub source_ref: String,
|
||||
#[serde(flatten)]
|
||||
pub catalog: MongoDumpCatalog,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct MongoDatabaseDumpRequest {
|
||||
pub task_id: String,
|
||||
pub connection_id: String,
|
||||
pub database: String,
|
||||
pub file_path: String,
|
||||
pub format: MongoDumpFormat,
|
||||
#[serde(default)]
|
||||
pub gzip: bool,
|
||||
#[serde(default)]
|
||||
pub collections: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct MongoDatabaseRestoreRequest {
|
||||
pub task_id: String,
|
||||
pub connection_id: String,
|
||||
pub database: String,
|
||||
pub source_database: String,
|
||||
pub source_ref: String,
|
||||
#[serde(default)]
|
||||
pub collections: Option<Vec<String>>,
|
||||
#[serde(default)]
|
||||
pub drop_existing: bool,
|
||||
#[serde(default = "yes")]
|
||||
pub restore_options: bool,
|
||||
#[serde(default = "yes")]
|
||||
pub restore_indexes: bool,
|
||||
#[serde(default = "yes")]
|
||||
pub stop_on_error: bool,
|
||||
#[serde(default)]
|
||||
pub objcheck: bool,
|
||||
#[serde(default = "batch_size")]
|
||||
pub batch_size: usize,
|
||||
#[serde(default)]
|
||||
pub execution_id: Option<String>,
|
||||
}
|
||||
fn yes() -> bool {
|
||||
true
|
||||
}
|
||||
fn batch_size() -> usize {
|
||||
500
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct MongoDatabaseDumpProgress {
|
||||
pub task_id: String,
|
||||
pub status: String,
|
||||
pub phase: String,
|
||||
pub collection: Option<String>,
|
||||
pub collections_done: usize,
|
||||
pub collections_total: usize,
|
||||
pub documents_read: u64,
|
||||
pub documents_written: u64,
|
||||
pub documents_failed: u64,
|
||||
pub indexes_created: usize,
|
||||
pub elapsed_ms: u128,
|
||||
pub error_message: Option<String>,
|
||||
pub file_path: Option<String>,
|
||||
#[serde(default)]
|
||||
pub bytes_processed: u64,
|
||||
#[serde(default)]
|
||||
pub documents_validated: u64,
|
||||
}
|
||||
|
||||
impl MongoDatabaseDumpProgress {
|
||||
pub fn initial(task_id: &str) -> Self {
|
||||
Self {
|
||||
task_id: task_id.into(),
|
||||
status: "running".into(),
|
||||
phase: "preparing".into(),
|
||||
collection: None,
|
||||
collections_done: 0,
|
||||
collections_total: 0,
|
||||
documents_read: 0,
|
||||
documents_written: 0,
|
||||
documents_failed: 0,
|
||||
indexes_created: 0,
|
||||
elapsed_ms: 0,
|
||||
error_message: None,
|
||||
file_path: None,
|
||||
bytes_processed: 0,
|
||||
documents_validated: 0,
|
||||
}
|
||||
}
|
||||
fn emit(&mut self, started: Instant, callback: &mut impl FnMut(Self)) {
|
||||
self.elapsed_ms = started.elapsed().as_millis();
|
||||
callback(self.clone());
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct PreparedCollection {
|
||||
database: String,
|
||||
metadata: CollectionMetadata,
|
||||
path: Option<PathBuf>,
|
||||
documents: u64,
|
||||
size_bytes: u64,
|
||||
metadata_path: Option<PathBuf>,
|
||||
}
|
||||
|
||||
pub fn release_mongodb_restore_source(source_ref: &str) -> bool {
|
||||
source::release(source_ref)
|
||||
}
|
||||
|
||||
pub async fn prepare_mongodb_restore_source(
|
||||
request: MongoRestoreSourceRequest,
|
||||
) -> Result<MongoRestoreSourcePreview, String> {
|
||||
source::prepare(request, None, None).await
|
||||
}
|
||||
|
||||
fn catalog(entries: &[PreparedCollection], exact: bool) -> MongoDumpCatalog {
|
||||
MongoDumpCatalog {
|
||||
databases: entries.iter().map(|entry| entry.database.clone()).collect::<BTreeSet<_>>().into_iter().collect(),
|
||||
collections: entries
|
||||
.iter()
|
||||
.map(|entry| MongoDumpCollection {
|
||||
database: entry.database.clone(),
|
||||
name: entry.metadata.collection_name.clone(),
|
||||
kind: entry.metadata.kind.clone(),
|
||||
documents: if exact { Some(entry.documents) } else { None },
|
||||
size_bytes: entry.size_bytes,
|
||||
indexes: entry.metadata.restore_indexes().len(),
|
||||
source_files: Vec::new(),
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn native_client(state: &AppState, id: &str, database: &str) -> Result<mongodb::Client, String> {
|
||||
metadata::validate_database(database)?;
|
||||
state.get_or_create_pool(id, Some(database)).await?;
|
||||
match state.pool_handle(id).await {
|
||||
Some(PoolKind::MongoDb(client)) => Ok(client.clone()),
|
||||
_ => Err("Database dump/restore requires the native MongoDB driver".into()),
|
||||
}
|
||||
}
|
||||
|
||||
async fn database_entries(client: &mongodb::Client, database: &str) -> Result<Vec<PreparedCollection>, String> {
|
||||
let db = client.database(database);
|
||||
let mut cursor = db.list_collections().await.map_err(|e| e.to_string())?;
|
||||
let mut entries = Vec::new();
|
||||
while let Some(spec) = cursor.try_next().await.map_err(|e| e.to_string())? {
|
||||
if spec.name.starts_with("system.") && spec.name != "system.js" {
|
||||
continue;
|
||||
}
|
||||
let mut metadata = CollectionMetadata::empty(&spec.name);
|
||||
metadata.kind =
|
||||
crate::db::mongo_driver::MongoCollectionKind::from_driver_type(&spec.collection_type).as_str().into();
|
||||
metadata.options = mongodb::bson::to_document(&spec.options).map_err(|e| e.to_string())?;
|
||||
metadata.validate()?;
|
||||
if !metadata.is_view() {
|
||||
let mut indexes = db.collection::<Document>(&spec.name).list_indexes().await.map_err(|e| e.to_string())?;
|
||||
while let Some(index) = indexes.try_next().await.map_err(|e| e.to_string())? {
|
||||
metadata.indexes.push(mongodb::bson::to_document(&index).map_err(|e| e.to_string())?);
|
||||
}
|
||||
}
|
||||
metadata.validate()?;
|
||||
entries.push(PreparedCollection {
|
||||
database: database.into(),
|
||||
metadata,
|
||||
path: None,
|
||||
documents: 0,
|
||||
size_bytes: 0,
|
||||
metadata_path: None,
|
||||
});
|
||||
}
|
||||
entries.sort_by(|a, b| a.metadata.collection_name.cmp(&b.metadata.collection_name));
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
pub async fn inspect_mongodb_database_dump(
|
||||
state: &AppState,
|
||||
id: &str,
|
||||
database: &str,
|
||||
) -> Result<MongoDumpCatalog, String> {
|
||||
let client = native_client(state, id, database).await?;
|
||||
let entries = database_entries(&client, database).await?;
|
||||
let mut catalog = catalog(&entries, false);
|
||||
catalog.databases = vec![database.into()];
|
||||
Ok(catalog)
|
||||
}
|
||||
|
||||
fn select_entries(
|
||||
entries: &[PreparedCollection],
|
||||
selected: &Option<Vec<String>>,
|
||||
) -> Result<Vec<PreparedCollection>, String> {
|
||||
match selected {
|
||||
None => Ok(entries.to_vec()),
|
||||
Some(names) => {
|
||||
let mut requested: HashSet<_> = names.iter().collect();
|
||||
let chosen =
|
||||
entries.iter().filter(|entry| requested.remove(&entry.metadata.collection_name)).cloned().collect();
|
||||
if !requested.is_empty() {
|
||||
return Err("A selected collection is not present in the dump plan".into());
|
||||
}
|
||||
Ok(chosen)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn ordered_entries(entries: &[PreparedCollection]) -> Result<Vec<PreparedCollection>, String> {
|
||||
let names: HashMap<_, _> = entries
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, entry)| ((entry.database.clone(), entry.metadata.collection_name.clone()), i))
|
||||
.collect();
|
||||
let mut ordered = Vec::with_capacity(entries.len());
|
||||
// 0 = unseen, 1 = active dependency, 2 = emitted. Uploaded metadata may
|
||||
// contain arbitrarily deep chains, so keep the traversal off the call stack.
|
||||
let mut states = vec![0u8; entries.len()];
|
||||
for (i, entry) in entries.iter().enumerate().filter(|(_, e)| !e.metadata.is_view()) {
|
||||
states[i] = 2;
|
||||
ordered.push(entry.clone());
|
||||
}
|
||||
for i in 0..entries.len() {
|
||||
let mut pending = vec![(i, false)];
|
||||
while let Some((index, finish)) = pending.pop() {
|
||||
if finish {
|
||||
states[index] = 2;
|
||||
ordered.push(entries[index].clone());
|
||||
continue;
|
||||
}
|
||||
match states[index] {
|
||||
2 => continue,
|
||||
1 => return Err("Cyclic view dependencies in dump metadata".into()),
|
||||
_ => {}
|
||||
}
|
||||
states[index] = 1;
|
||||
pending.push((index, true));
|
||||
let entry = &entries[index];
|
||||
if let Ok(base) = entry.metadata.options.get_str("viewOn") {
|
||||
if let Some(&parent) = names.get(&(entry.database.clone(), base.into())) {
|
||||
pending.push((parent, false));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(ordered)
|
||||
}
|
||||
|
||||
fn cancelled(flag: &AtomicBool) -> Result<(), String> {
|
||||
if flag.load(Ordering::Relaxed) {
|
||||
Err("MongoDB dump/restore cancelled".into())
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
async fn blocking_work<T: Send + 'static>(
|
||||
task_id: &str,
|
||||
is_cancelled: &mut impl FnMut(&str) -> CancelFuture,
|
||||
work: impl FnOnce(Arc<AtomicBool>) -> Result<T, String> + Send + 'static,
|
||||
) -> Result<T, String> {
|
||||
if is_cancelled(task_id).await {
|
||||
return Err("MongoDB dump/restore cancelled".into());
|
||||
}
|
||||
struct CancelOnDrop(Arc<AtomicBool>);
|
||||
impl Drop for CancelOnDrop {
|
||||
fn drop(&mut self) {
|
||||
self.0.store(true, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
let flag = Arc::new(AtomicBool::new(false));
|
||||
let _guard = CancelOnDrop(flag.clone());
|
||||
let signal = flag.clone();
|
||||
let mut task = tokio::task::spawn_blocking(move || work(signal));
|
||||
let mut interval = tokio::time::interval(Duration::from_millis(100));
|
||||
loop {
|
||||
tokio::select! {
|
||||
result = &mut task => return result.map_err(|e| e.to_string())?,
|
||||
_ = interval.tick() => if is_cancelled(task_id).await { flag.store(true, Ordering::Relaxed); },
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn dump_mongodb_database<C, F>(
|
||||
state: &AppState,
|
||||
request: &MongoDatabaseDumpRequest,
|
||||
mut is_cancelled: C,
|
||||
mut on_progress: F,
|
||||
) -> Result<MongoDatabaseDumpProgress, String>
|
||||
where
|
||||
C: FnMut(&str) -> CancelFuture,
|
||||
F: FnMut(MongoDatabaseDumpProgress),
|
||||
{
|
||||
let started = Instant::now();
|
||||
let mut progress = MongoDatabaseDumpProgress::initial(&request.task_id);
|
||||
progress.emit(started, &mut on_progress);
|
||||
let result: Result<(), String> = async {
|
||||
if is_cancelled(&request.task_id).await {
|
||||
return Err("MongoDB dump/restore cancelled".into());
|
||||
}
|
||||
let client = native_client(state, &request.connection_id, &request.database).await?;
|
||||
let mut entries = select_entries(&database_entries(&client, &request.database).await?, &request.collections)?;
|
||||
progress.collections_total = entries.len();
|
||||
let target = PathBuf::from(&request.file_path);
|
||||
if target.exists() {
|
||||
return Err(match request.format {
|
||||
MongoDumpFormat::Directory => "Dump destination already exists; choose a new directory",
|
||||
MongoDumpFormat::Archive => "Dump destination already exists; choose a new archive file",
|
||||
}
|
||||
.into());
|
||||
}
|
||||
let parent = target.parent().filter(|p| !p.as_os_str().is_empty()).unwrap_or(Path::new("."));
|
||||
std::fs::create_dir_all(parent).map_err(|e| e.to_string())?;
|
||||
let working =
|
||||
tempfile::Builder::new().prefix(".dbx-mongo-dump-").tempdir_in(parent).map_err(|e| e.to_string())?;
|
||||
for (index, entry) in entries.iter_mut().enumerate() {
|
||||
if is_cancelled(&request.task_id).await {
|
||||
return Err("MongoDB dump/restore cancelled".into());
|
||||
}
|
||||
progress.collection = Some(entry.metadata.collection_name.clone());
|
||||
progress.phase = "data".into();
|
||||
progress.emit(started, &mut on_progress);
|
||||
if !entry.metadata.is_view() {
|
||||
let path = working.path().join(format!("{index}.bson"));
|
||||
let child = MongoExportRequest {
|
||||
export_id: request.task_id.clone(),
|
||||
connection_id: request.connection_id.clone(),
|
||||
database: request.database.clone(),
|
||||
collection: entry.metadata.collection_name.clone(),
|
||||
filter: None,
|
||||
sort: None,
|
||||
projection: None,
|
||||
collation: None,
|
||||
format: MongoExportFormat::Bson,
|
||||
include_header: false,
|
||||
gzip: false,
|
||||
file_path: path.to_string_lossy().into_owned(),
|
||||
execution_id: None,
|
||||
};
|
||||
let base = progress.documents_read;
|
||||
let summary = export_mongodb_query_core(
|
||||
state,
|
||||
&child,
|
||||
|id| is_cancelled(id),
|
||||
|child| {
|
||||
progress.documents_read = base + child.documents_read;
|
||||
progress.emit(started, &mut on_progress);
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
entry.documents = summary.documents_exported;
|
||||
entry.size_bytes = std::fs::metadata(&path).map_err(|e| e.to_string())?.len();
|
||||
entry.path = Some(path);
|
||||
}
|
||||
progress.collections_done += 1;
|
||||
}
|
||||
progress.phase = "archive".into();
|
||||
progress.emit(started, &mut on_progress);
|
||||
let server_version = client
|
||||
.database(&request.database)
|
||||
.run_command(doc! { "buildInfo": 1 })
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
.get_str("version")
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let database = request.database.clone();
|
||||
let gzip = request.gzip;
|
||||
let format = request.format;
|
||||
blocking_work(&request.task_id, &mut is_cancelled, move |flag| {
|
||||
let _working = working;
|
||||
let parent = target.parent().filter(|p| !p.as_os_str().is_empty()).unwrap_or(Path::new("."));
|
||||
match format {
|
||||
MongoDumpFormat::Archive => {
|
||||
let temp = tempfile::NamedTempFile::new_in(parent).map_err(|e| e.to_string())?;
|
||||
directory::write_file(temp.path(), gzip, |writer| {
|
||||
archive::pack(writer, &entries, &server_version, || cancelled(&flag))
|
||||
})?;
|
||||
cancelled(&flag)?;
|
||||
if target.exists() {
|
||||
return Err("Dump destination already exists".into());
|
||||
}
|
||||
temp.persist(&target).map_err(|e| e.to_string())?;
|
||||
}
|
||||
MongoDumpFormat::Directory => {
|
||||
let output = tempfile::Builder::new()
|
||||
.prefix(".dbx-dump-output-")
|
||||
.tempdir_in(parent)
|
||||
.map_err(|e| e.to_string())?;
|
||||
directory::publish_files(output.path(), &database, &entries, gzip, || cancelled(&flag))?;
|
||||
cancelled(&flag)?;
|
||||
if target.exists() {
|
||||
return Err("Dump destination already exists".into());
|
||||
}
|
||||
std::fs::rename(output.path(), &target).map_err(|e| e.to_string())?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await?;
|
||||
progress.file_path = Some(request.file_path.clone());
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
finish_progress(progress, result, started, &mut on_progress)
|
||||
}
|
||||
|
||||
pub async fn restore_mongodb_database<C, F>(
|
||||
state: &AppState,
|
||||
request: &MongoDatabaseRestoreRequest,
|
||||
mut is_cancelled: C,
|
||||
mut on_progress: F,
|
||||
) -> Result<MongoDatabaseDumpProgress, String>
|
||||
where
|
||||
C: FnMut(&str) -> CancelFuture,
|
||||
F: FnMut(MongoDatabaseDumpProgress),
|
||||
{
|
||||
let started = Instant::now();
|
||||
let mut progress = MongoDatabaseDumpProgress::initial(&request.task_id);
|
||||
progress.emit(started, &mut on_progress);
|
||||
let result: Result<(), String> = async {
|
||||
if let Some(name) = crate::query::connection_readonly_name(state, &request.connection_id).await {
|
||||
return Err(format!("Read-only connection '{name}': restore blocked"));
|
||||
}
|
||||
if is_cancelled(&request.task_id).await {
|
||||
return Err("MongoDB dump/restore cancelled".into());
|
||||
}
|
||||
crate::mongodb_import_export::clamp_batch_size(request.batch_size).map_err(|e| e.display_message())?;
|
||||
let source = source::get(&request.source_ref)?;
|
||||
let source_entries: Vec<_> =
|
||||
source.entries.iter().filter(|entry| entry.database == request.source_database).cloned().collect();
|
||||
if source_entries.is_empty() && !source.entries.is_empty() {
|
||||
return Err("Source database is not present in the dump".into());
|
||||
}
|
||||
let entries = ordered_entries(&select_entries(&source_entries, &request.collections)?)?;
|
||||
progress.collections_total = entries.len();
|
||||
source.check_ready(&entries)?;
|
||||
let client = native_client(state, &request.connection_id, &request.database).await?;
|
||||
let db = client.database(&request.database);
|
||||
let existing = db
|
||||
.list_collections()
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
.try_collect::<Vec<_>>()
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
for entry in &entries {
|
||||
entry.metadata.validate()?;
|
||||
if let Some(target) = existing.iter().find(|target| target.name == entry.metadata.collection_name) {
|
||||
if !request.drop_existing
|
||||
&& (entry.metadata.is_view()
|
||||
|| !matches!(target.collection_type, mongodb::results::CollectionType::Collection))
|
||||
{
|
||||
return Err(format!(
|
||||
"{}: replacing an existing view or non-regular collection requires dropExisting",
|
||||
target.name
|
||||
));
|
||||
}
|
||||
}
|
||||
if let Ok(base) = entry.metadata.options.get_str("viewOn") {
|
||||
if !entries.iter().any(|e| e.metadata.collection_name == base)
|
||||
&& !existing.iter().any(|e| e.name == base)
|
||||
{
|
||||
return Err(format!(
|
||||
"Missing backing collection for view {}: {base}",
|
||||
entry.metadata.collection_name
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
let existing_names = existing.iter().map(|e| e.name.clone()).collect::<HashSet<_>>();
|
||||
stream::restore_data(
|
||||
state,
|
||||
&client,
|
||||
request,
|
||||
source,
|
||||
&entries,
|
||||
&existing_names,
|
||||
&mut progress,
|
||||
started,
|
||||
&mut is_cancelled,
|
||||
&mut on_progress,
|
||||
)
|
||||
.await?;
|
||||
for entry in &entries {
|
||||
if is_cancelled(&request.task_id).await {
|
||||
return Err("MongoDB dump/restore cancelled".into());
|
||||
}
|
||||
stream::writable(state, &request.connection_id).await?;
|
||||
let name = &entry.metadata.collection_name;
|
||||
progress.collection = Some(name.clone());
|
||||
progress.phase = if entry.metadata.is_view() { "views" } else { "indexes" }.into();
|
||||
progress.emit(started, &mut on_progress);
|
||||
if entry.metadata.is_view() {
|
||||
stream::initialize_collection(&client, request, entry, existing_names.contains(name)).await?;
|
||||
progress.collections_done += 1;
|
||||
} else if request.restore_indexes {
|
||||
for index in entry.metadata.restore_indexes() {
|
||||
if is_cancelled(&request.task_id).await {
|
||||
return Err("MongoDB dump/restore cancelled".into());
|
||||
}
|
||||
db.run_command(doc! { "createIndexes": name, "indexes": [index] })
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
progress.indexes_created += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
finish_progress(progress, result, started, &mut on_progress)
|
||||
}
|
||||
|
||||
fn finish_progress(
|
||||
mut progress: MongoDatabaseDumpProgress,
|
||||
result: Result<(), String>,
|
||||
started: Instant,
|
||||
callback: &mut impl FnMut(MongoDatabaseDumpProgress),
|
||||
) -> Result<MongoDatabaseDumpProgress, String> {
|
||||
progress.phase = "done".into();
|
||||
progress.status = match &result {
|
||||
Ok(()) => "done",
|
||||
Err(error)
|
||||
if matches!(error.as_str(), "MongoDB dump/restore cancelled" | "Import cancelled" | "Export cancelled") =>
|
||||
{
|
||||
"cancelled"
|
||||
}
|
||||
Err(_) => "error",
|
||||
}
|
||||
.into();
|
||||
progress.error_message = result.as_ref().err().cloned();
|
||||
progress.emit(started, callback);
|
||||
result.map(|_| progress)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn view_ordering_handles_deep_dependency_chains() {
|
||||
let count = 20_000;
|
||||
let entries = (0..count)
|
||||
.map(|index| {
|
||||
let mut metadata = CollectionMetadata::empty(&format!("view_{index}"));
|
||||
if index + 1 < count {
|
||||
metadata.kind = "view".into();
|
||||
metadata.options = doc! { "viewOn": format!("view_{}", index + 1), "pipeline": [] };
|
||||
}
|
||||
PreparedCollection {
|
||||
database: "source".into(),
|
||||
metadata,
|
||||
path: None,
|
||||
documents: 0,
|
||||
size_bytes: 0,
|
||||
metadata_path: None,
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let ordered = ordered_entries(&entries).unwrap();
|
||||
assert_eq!(ordered.len(), count);
|
||||
for (entry, index) in ordered.iter().zip((0..count).rev()) {
|
||||
assert_eq!(entry.metadata.collection_name, format!("view_{index}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,290 @@
|
||||
//! MongoDB Database Tools archive format 0.1, see common/archive/spec.md upstream.
|
||||
use std::collections::HashMap;
|
||||
use std::fs::File;
|
||||
use std::io::{BufReader, Read, Write};
|
||||
|
||||
use crc::{Crc, Digest, CRC_64_XZ};
|
||||
use mongodb::bson::{doc, Document};
|
||||
|
||||
use super::{metadata::CollectionMetadata, PreparedCollection};
|
||||
|
||||
const MAGIC: u32 = 0x8199_e26d;
|
||||
const TERMINATOR: [u8; 4] = [255; 4];
|
||||
// Go hash/crc64 with the ECMA table reflects bits and complements the initial/final state.
|
||||
const CRC: Crc<u64> = Crc::<u64>::new(&CRC_64_XZ);
|
||||
pub(super) const MAX_DOCUMENT: usize = 16 * 1024 * 1024;
|
||||
|
||||
pub(super) enum Frame {
|
||||
End,
|
||||
Terminator,
|
||||
Document(Vec<u8>),
|
||||
}
|
||||
|
||||
pub(super) fn read_frame(reader: &mut dyn Read) -> Result<Frame, String> {
|
||||
let frame = read_raw_frame(reader)?;
|
||||
if let Frame::Document(bytes) = &frame {
|
||||
mongodb::bson::from_slice::<Document>(bytes).map_err(|e| format!("Invalid BSON document: {e}"))?;
|
||||
}
|
||||
Ok(frame)
|
||||
}
|
||||
|
||||
pub(super) fn read_raw_frame(reader: &mut dyn Read) -> Result<Frame, String> {
|
||||
let mut header = [0; 4];
|
||||
loop {
|
||||
match reader.read(&mut header[..1]) {
|
||||
Ok(0) => return Ok(Frame::End),
|
||||
Ok(_) => break,
|
||||
Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
|
||||
Err(e) => return Err(e.to_string()),
|
||||
}
|
||||
}
|
||||
reader.read_exact(&mut header[1..]).map_err(|e| format!("Truncated BSON header: {e}"))?;
|
||||
if header == TERMINATOR {
|
||||
return Ok(Frame::Terminator);
|
||||
}
|
||||
let length = i32::from_le_bytes(header);
|
||||
if !(5..=MAX_DOCUMENT as i32).contains(&length) {
|
||||
return Err(format!("Invalid BSON frame length: {length}"));
|
||||
}
|
||||
let mut bytes = vec![0; length as usize];
|
||||
bytes[..4].copy_from_slice(&header);
|
||||
reader.read_exact(&mut bytes[4..]).map_err(|e| format!("Truncated BSON document: {e}"))?;
|
||||
mongodb::bson::RawDocument::from_bytes(&bytes).map_err(|e| format!("Invalid BSON document: {e}"))?;
|
||||
Ok(Frame::Document(bytes))
|
||||
}
|
||||
|
||||
fn document(frame: Frame) -> Result<Document, String> {
|
||||
match frame {
|
||||
Frame::Document(bytes) => mongodb::bson::from_slice(&bytes).map_err(|e| e.to_string()),
|
||||
_ => Err("Expected an archive BSON header".into()),
|
||||
}
|
||||
}
|
||||
|
||||
fn string(doc: &Document, key: &str) -> Result<String, String> {
|
||||
doc.get_str(key).map(String::from).map_err(|_| format!("Archive header is missing {key}"))
|
||||
}
|
||||
|
||||
fn write_document(writer: &mut dyn Write, doc: &Document) -> Result<(), String> {
|
||||
let bytes = mongodb::bson::to_vec(doc).map_err(|e| e.to_string())?;
|
||||
if bytes.len() > MAX_DOCUMENT {
|
||||
return Err("Archive metadata exceeds the BSON document limit".into());
|
||||
}
|
||||
writer.write_all(&bytes).map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
pub(super) fn inspect(
|
||||
reader: &mut dyn Read,
|
||||
mut check: impl FnMut() -> Result<(), String>,
|
||||
) -> Result<Vec<PreparedCollection>, String> {
|
||||
let mut magic = [0; 4];
|
||||
reader.read_exact(&mut magic).map_err(|e| format!("Invalid MongoDB archive: {e}"))?;
|
||||
if u32::from_le_bytes(magic) != MAGIC {
|
||||
return Err("Not a MongoDB --archive file".into());
|
||||
}
|
||||
let header = document(read_frame(reader)?)?;
|
||||
if header.get_str("version") != Ok("0.1") {
|
||||
return Err("Unsupported MongoDB archive version".into());
|
||||
}
|
||||
let mut entries = Vec::<PreparedCollection>::new();
|
||||
let mut namespaces = HashMap::new();
|
||||
let mut metadata_bytes = 0usize;
|
||||
loop {
|
||||
check()?;
|
||||
let frame = read_frame(reader)?;
|
||||
if matches!(frame, Frame::Terminator) {
|
||||
break;
|
||||
}
|
||||
let header = document(frame)?;
|
||||
let database = string(&header, "db")?;
|
||||
super::metadata::validate_database(&database)?;
|
||||
let name = string(&header, "collection")?;
|
||||
let json = string(&header, "metadata")?;
|
||||
metadata_bytes += json.len();
|
||||
if entries.len() >= 100_000 || metadata_bytes > 128 * 1024 * 1024 {
|
||||
return Err("Archive metadata exceeds the supported limit".into());
|
||||
}
|
||||
let metadata = if json.is_empty() {
|
||||
CollectionMetadata::empty(&name)
|
||||
} else {
|
||||
CollectionMetadata::from_json(&json, &name)?
|
||||
};
|
||||
metadata.validate()?;
|
||||
if metadata.collection_name != name {
|
||||
return Err("Archive namespace does not match collection metadata".into());
|
||||
}
|
||||
if header.get_str("type").is_ok_and(|kind| kind == "timeseries") {
|
||||
return Err("Time-series archives require server-specific restore support".into());
|
||||
}
|
||||
let index = entries.len();
|
||||
if namespaces.insert((database.clone(), name), index).is_some() {
|
||||
return Err("Duplicate archive namespace".into());
|
||||
}
|
||||
entries.push(PreparedCollection {
|
||||
database,
|
||||
metadata,
|
||||
path: None,
|
||||
documents: 0,
|
||||
size_bytes: 0,
|
||||
metadata_path: None,
|
||||
});
|
||||
}
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
pub(super) enum StreamEvent {
|
||||
Document(usize, Vec<u8>),
|
||||
End(usize),
|
||||
}
|
||||
|
||||
pub(super) fn stream(
|
||||
reader: &mut dyn Read,
|
||||
expected: &[PreparedCollection],
|
||||
mut check: impl FnMut() -> Result<(), String>,
|
||||
mut emit: impl FnMut(StreamEvent) -> Result<(), String>,
|
||||
) -> Result<(), String> {
|
||||
let entries = inspect(reader, &mut check)?;
|
||||
let namespaces: HashMap<_, _> = entries
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, e)| ((e.database.clone(), e.metadata.collection_name.clone()), i))
|
||||
.collect();
|
||||
if entries.len() != expected.len() {
|
||||
return Err("Archive metadata changed after preview".into());
|
||||
}
|
||||
let mut mapped = Vec::with_capacity(entries.len());
|
||||
for entry in &entries {
|
||||
let index = expected
|
||||
.iter()
|
||||
.position(|e| e.database == entry.database && e.metadata.collection_name == entry.metadata.collection_name)
|
||||
.ok_or("Archive metadata changed after preview")?;
|
||||
if expected[index].metadata.to_json()? != entry.metadata.to_json()? {
|
||||
return Err("Archive metadata changed after preview".into());
|
||||
}
|
||||
mapped.push(index);
|
||||
}
|
||||
let mut digests: Vec<Digest<'_, u64>> = entries.iter().map(|_| CRC.digest()).collect();
|
||||
let mut finished = vec![false; entries.len()];
|
||||
loop {
|
||||
check()?;
|
||||
let frame = read_frame(reader)?;
|
||||
if matches!(frame, Frame::End) {
|
||||
break;
|
||||
}
|
||||
let header = document(frame)?;
|
||||
let namespace = (string(&header, "db")?, string(&header, "collection")?);
|
||||
let index = *namespaces.get(&namespace).ok_or_else(|| "Unknown namespace in archive data".to_string())?;
|
||||
if finished[index] {
|
||||
return Err("Archive contains data after namespace EOF".into());
|
||||
}
|
||||
let eof = header.get_bool("EOF").map_err(|_| "Archive header is missing EOF".to_string())?;
|
||||
let checksum = header.get_i64("CRC").map_err(|_| "Archive header is missing CRC".to_string())? as u64;
|
||||
if eof {
|
||||
if !matches!(read_frame(reader)?, Frame::Terminator) {
|
||||
return Err("Invalid archive EOF terminator".into());
|
||||
}
|
||||
if digests[index].clone().finalize() != checksum {
|
||||
return Err(format!("Archive checksum mismatch: {}.{}", namespace.0, namespace.1));
|
||||
}
|
||||
finished[index] = true;
|
||||
emit(StreamEvent::End(mapped[index]))?;
|
||||
continue;
|
||||
}
|
||||
if checksum != 0 {
|
||||
return Err("Non-EOF archive header contains a checksum".into());
|
||||
}
|
||||
let entry = &entries[index];
|
||||
loop {
|
||||
check()?;
|
||||
match read_raw_frame(reader)? {
|
||||
Frame::Document(bytes) => {
|
||||
if entry.metadata.is_view() {
|
||||
return Err("Archive contains BSON documents for a view".into());
|
||||
}
|
||||
digests[index].update(&bytes);
|
||||
emit(StreamEvent::Document(mapped[index], bytes))?;
|
||||
}
|
||||
Frame::Terminator => break,
|
||||
Frame::End => return Err("Truncated archive namespace segment".into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
for (index, entry) in entries.iter().enumerate() {
|
||||
if !finished[index] {
|
||||
return Err(format!("Missing archive EOF: {}.{}", entry.database, entry.metadata.collection_name));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn pack(
|
||||
writer: &mut dyn Write,
|
||||
entries: &[PreparedCollection],
|
||||
server_version: &str,
|
||||
mut check: impl FnMut() -> Result<(), String>,
|
||||
) -> Result<(), String> {
|
||||
writer.write_all(&MAGIC.to_le_bytes()).map_err(|e| e.to_string())?;
|
||||
write_document(
|
||||
writer,
|
||||
&doc! { "concurrent_collections": 1i32, "version": "0.1", "server_version": server_version, "tool_version": concat!("DBX ", env!("CARGO_PKG_VERSION")) },
|
||||
)?;
|
||||
for entry in entries {
|
||||
check()?;
|
||||
write_document(
|
||||
writer,
|
||||
&doc! { "db": &entry.database, "collection": &entry.metadata.collection_name, "metadata": entry.metadata.to_json()?, "size": entry.documents as i64, "type": if entry.metadata.is_view() { "view" } else { "" } },
|
||||
)?;
|
||||
}
|
||||
writer.write_all(&TERMINATOR).map_err(|e| e.to_string())?;
|
||||
for entry in entries {
|
||||
check()?;
|
||||
if entry.metadata.is_view() {
|
||||
write_document(
|
||||
writer,
|
||||
&doc! { "db": &entry.database, "collection": &entry.metadata.collection_name, "EOF": true, "CRC": 0i64 },
|
||||
)?;
|
||||
writer.write_all(&TERMINATOR).map_err(|e| e.to_string())?;
|
||||
continue;
|
||||
}
|
||||
let mut digest = CRC.digest();
|
||||
let path = entry.path.as_ref().ok_or_else(|| "Missing collection BSON file".to_string())?;
|
||||
let mut reader = BufReader::new(File::open(path).map_err(|e| e.to_string())?);
|
||||
let mut opened = false;
|
||||
loop {
|
||||
check()?;
|
||||
match read_frame(&mut reader)? {
|
||||
Frame::End => break,
|
||||
Frame::Terminator => return Err("Unexpected terminator in BSON file".into()),
|
||||
Frame::Document(bytes) => {
|
||||
if !opened {
|
||||
write_document(
|
||||
writer,
|
||||
&doc! { "db": &entry.database, "collection": &entry.metadata.collection_name, "EOF": false, "CRC": 0i64 },
|
||||
)?;
|
||||
opened = true;
|
||||
}
|
||||
writer.write_all(&bytes).map_err(|e| e.to_string())?;
|
||||
digest.update(&bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
if opened {
|
||||
writer.write_all(&TERMINATOR).map_err(|e| e.to_string())?;
|
||||
}
|
||||
write_document(
|
||||
writer,
|
||||
&doc! { "db": &entry.database, "collection": &entry.metadata.collection_name, "EOF": true, "CRC": digest.finalize() as i64 },
|
||||
)?;
|
||||
writer.write_all(&TERMINATOR).map_err(|e| e.to_string())?;
|
||||
}
|
||||
writer.flush().map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn checksum_matches_go_crc64_ecma() {
|
||||
assert_eq!(CRC.checksum(b"123456789"), 0x995d_c9bb_df19_39fa);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,302 @@
|
||||
use std::collections::BTreeMap;
|
||||
use std::fs::File;
|
||||
use std::io::{BufReader, Read, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use flate2::{read::MultiGzDecoder, write::GzEncoder, Compression};
|
||||
|
||||
use super::{
|
||||
archive::MAX_DOCUMENT,
|
||||
metadata::{self, CollectionMetadata},
|
||||
PreparedCollection,
|
||||
};
|
||||
|
||||
pub(super) fn reader(path: &Path, gzip: bool) -> Result<Box<dyn Read>, String> {
|
||||
regular_file(path)?;
|
||||
let reader = BufReader::new(File::open(path).map_err(|e| e.to_string())?);
|
||||
if gzip {
|
||||
Ok(Box::new(BufReader::with_capacity(64 * 1024, MultiGzDecoder::new(reader))))
|
||||
} else {
|
||||
Ok(Box::new(reader))
|
||||
}
|
||||
}
|
||||
|
||||
fn reject_link(path: &Path) -> Result<std::fs::Metadata, String> {
|
||||
let metadata = std::fs::symlink_metadata(path).map_err(|e| e.to_string())?;
|
||||
let is_link = metadata.file_type().is_symlink();
|
||||
#[cfg(windows)]
|
||||
let is_link = {
|
||||
use std::os::windows::fs::MetadataExt;
|
||||
is_link || metadata.file_attributes() & 0x400 != 0
|
||||
};
|
||||
if is_link {
|
||||
return Err(format!("Dump sources cannot contain links: {}", path.display()));
|
||||
}
|
||||
Ok(metadata)
|
||||
}
|
||||
|
||||
pub(super) fn regular_file(path: &Path) -> Result<(), String> {
|
||||
if !reject_link(path)?.is_file() {
|
||||
return Err(format!("Not a regular dump file: {}", path.display()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn copy_stream(
|
||||
reader: &mut dyn Read,
|
||||
writer: &mut dyn Write,
|
||||
check: &mut impl FnMut() -> Result<(), String>,
|
||||
) -> Result<(), String> {
|
||||
let mut buffer = [0; 64 * 1024];
|
||||
loop {
|
||||
check()?;
|
||||
let length = reader.read(&mut buffer).map_err(|e| e.to_string())?;
|
||||
if length == 0 {
|
||||
break;
|
||||
}
|
||||
writer.write_all(&buffer[..length]).map_err(|e| e.to_string())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn write_file(
|
||||
path: &Path,
|
||||
gzip: bool,
|
||||
write: impl FnOnce(&mut dyn Write) -> Result<(), String>,
|
||||
) -> Result<(), String> {
|
||||
let file = std::io::BufWriter::new(File::create(path).map_err(|e| e.to_string())?);
|
||||
if gzip {
|
||||
let mut writer = GzEncoder::new(file, Compression::default());
|
||||
write(&mut writer)?;
|
||||
writer.finish().map_err(|e| e.to_string())?.flush().map_err(|e| e.to_string())
|
||||
} else {
|
||||
let mut writer = file;
|
||||
write(&mut writer)?;
|
||||
writer.flush().map_err(|e| e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Files {
|
||||
data: Option<PathBuf>,
|
||||
metadata: Option<PathBuf>,
|
||||
}
|
||||
|
||||
fn collect(
|
||||
path: &Path,
|
||||
depth: usize,
|
||||
gzip: bool,
|
||||
groups: &mut BTreeMap<(String, String), Files>,
|
||||
) -> Result<(), String> {
|
||||
if depth > 3 || !reject_link(path)?.is_dir() {
|
||||
return Err("Invalid dump directory layout".into());
|
||||
}
|
||||
let database =
|
||||
path.file_name().and_then(|name| name.to_str()).ok_or_else(|| "Invalid database directory name".to_string())?;
|
||||
let mut subdirectories = Vec::new();
|
||||
let mut has_data = false;
|
||||
for entry in std::fs::read_dir(path).map_err(|e| e.to_string())? {
|
||||
let entry = entry.map_err(|e| e.to_string())?;
|
||||
let path = entry.path();
|
||||
let info = reject_link(&path)?;
|
||||
if info.is_dir() {
|
||||
subdirectories.push(path);
|
||||
continue;
|
||||
}
|
||||
if !info.is_file() {
|
||||
return Err("Unsupported dump directory entry".into());
|
||||
}
|
||||
let filename = entry.file_name().to_str().ok_or_else(|| "Invalid dump filename".to_string())?.to_string();
|
||||
let (base, metadata_file, compressed) = if let Some(base) = filename.strip_suffix(".metadata.json.gz") {
|
||||
(base, true, true)
|
||||
} else if let Some(base) = filename.strip_suffix(".bson.gz") {
|
||||
(base, false, true)
|
||||
} else if let Some(base) = filename.strip_suffix(".metadata.json") {
|
||||
(base, true, false)
|
||||
} else if let Some(base) = filename.strip_suffix(".bson") {
|
||||
(base, false, false)
|
||||
} else {
|
||||
continue;
|
||||
};
|
||||
if compressed != gzip {
|
||||
return Err(format!("Gzip option does not match dump file {filename}"));
|
||||
}
|
||||
metadata::validate_database(database)?;
|
||||
has_data = true;
|
||||
let group = groups.entry((database.into(), base.into())).or_default();
|
||||
let slot = if metadata_file { &mut group.metadata } else { &mut group.data };
|
||||
if slot.replace(path).is_some() {
|
||||
return Err("Duplicate dump filename".into());
|
||||
}
|
||||
if groups.len() > 100_000 {
|
||||
return Err("Too many dump collections".into());
|
||||
}
|
||||
}
|
||||
if has_data && !subdirectories.is_empty() {
|
||||
return Err(
|
||||
"Mixed database files and directories; select a single database directory or a dump root without oplog"
|
||||
.into(),
|
||||
);
|
||||
}
|
||||
for directory in subdirectories {
|
||||
collect(&directory, depth + 1, gzip, groups)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn inspect(
|
||||
path: &Path,
|
||||
gzip: bool,
|
||||
manifest: Option<&[super::MongoDumpFile]>,
|
||||
) -> Result<Vec<PreparedCollection>, String> {
|
||||
let mut groups = BTreeMap::new();
|
||||
if let Some(manifest) = manifest {
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
for file in manifest {
|
||||
let relative = super::source::relative_path(&file.path)?;
|
||||
if !seen.insert(file.path.clone()) {
|
||||
return Err("Duplicate dump manifest path".into());
|
||||
}
|
||||
let database = relative
|
||||
.parent()
|
||||
.and_then(Path::file_name)
|
||||
.and_then(|s| s.to_str())
|
||||
.ok_or("Directory uploads require database-relative paths")?;
|
||||
metadata::validate_database(database)?;
|
||||
let filename = relative.file_name().unwrap().to_str().ok_or("Invalid dump filename")?;
|
||||
let plain = filename.strip_suffix(".gz").unwrap_or(filename);
|
||||
let (base, is_metadata) = if let Some(base) = plain.strip_suffix(".metadata.json") {
|
||||
(base, true)
|
||||
} else if let Some(base) = plain.strip_suffix(".bson") {
|
||||
(base, false)
|
||||
} else {
|
||||
continue;
|
||||
};
|
||||
if filename.ends_with(".gz") != gzip {
|
||||
return Err(format!("Gzip option does not match dump file {filename}"));
|
||||
}
|
||||
let group = groups.entry((database.to_string(), base.to_string())).or_insert_with(Files::default);
|
||||
let slot = if is_metadata { &mut group.metadata } else { &mut group.data };
|
||||
if slot.replace(path.join(relative)).is_some() {
|
||||
return Err("Duplicate dump namespace".into());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
collect(path, 0, gzip, &mut groups)?;
|
||||
}
|
||||
let mut entries = Vec::new();
|
||||
let root = path;
|
||||
let mut namespaces = std::collections::HashSet::new();
|
||||
let mut metadata_bytes = 0usize;
|
||||
for ((database, basename), files) in groups {
|
||||
// Truncated official filenames may end mid-escape; collectionName is authoritative.
|
||||
let fallback = metadata::unescape_collection(&basename).unwrap_or_default();
|
||||
let metadata = if let Some(path) = &files.metadata {
|
||||
let mut json = String::new();
|
||||
reader(path, gzip)?.take(MAX_DOCUMENT as u64 + 1).read_to_string(&mut json).map_err(|e| e.to_string())?;
|
||||
if json.len() > MAX_DOCUMENT {
|
||||
return Err("Collection metadata exceeds 16 MiB".into());
|
||||
}
|
||||
metadata_bytes = metadata_bytes.saturating_add(json.len());
|
||||
if metadata_bytes > 128 * 1024 * 1024 {
|
||||
return Err("Dump metadata exceeds 128 MiB".into());
|
||||
}
|
||||
if json.trim().is_empty() {
|
||||
CollectionMetadata::empty(&fallback)
|
||||
} else {
|
||||
CollectionMetadata::from_json(&json, &fallback)?
|
||||
}
|
||||
} else {
|
||||
CollectionMetadata::empty(&fallback)
|
||||
};
|
||||
metadata.validate()?;
|
||||
if !namespaces.insert((database.clone(), metadata.collection_name.clone())) {
|
||||
return Err("Duplicate dump namespace".into());
|
||||
}
|
||||
let mut entry = PreparedCollection {
|
||||
database,
|
||||
metadata,
|
||||
path: None,
|
||||
documents: 0,
|
||||
size_bytes: 0,
|
||||
metadata_path: files.metadata,
|
||||
};
|
||||
match files.data {
|
||||
Some(path) => {
|
||||
if entry.metadata.is_view() {
|
||||
return Err("View dump unexpectedly contains BSON data".into());
|
||||
}
|
||||
entry.size_bytes = if let Some(manifest) = manifest {
|
||||
manifest.iter().find(|f| path == root.join(&f.path)).map(|f| f.size_bytes).unwrap_or(0)
|
||||
} else {
|
||||
regular_file(&path)?;
|
||||
std::fs::metadata(&path).map_err(|e| e.to_string())?.len()
|
||||
};
|
||||
entry.path = Some(path);
|
||||
}
|
||||
None if !entry.metadata.is_view() => {
|
||||
return Err(format!("Missing BSON file for {}", entry.metadata.collection_name))
|
||||
}
|
||||
None => {}
|
||||
}
|
||||
entries.push(entry);
|
||||
}
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
pub(super) fn publish_files(
|
||||
root: &Path,
|
||||
database: &str,
|
||||
entries: &[PreparedCollection],
|
||||
gzip: bool,
|
||||
mut check: impl FnMut() -> Result<(), String>,
|
||||
) -> Result<(), String> {
|
||||
let database_dir = root.join(database);
|
||||
std::fs::create_dir_all(&database_dir).map_err(|e| e.to_string())?;
|
||||
let mut names = std::collections::HashSet::new();
|
||||
for entry in entries {
|
||||
check()?;
|
||||
let basename = metadata::escaped_collection(&entry.metadata.collection_name);
|
||||
let filesystem_key = if cfg!(windows) { basename.to_lowercase() } else { basename.clone() };
|
||||
if !names.insert(filesystem_key) {
|
||||
return Err("Collection names collide on this filesystem; use archive format".into());
|
||||
}
|
||||
let suffix = if gzip { ".gz" } else { "" };
|
||||
write_file(&database_dir.join(format!("{basename}.metadata.json{suffix}")), gzip, |writer| {
|
||||
writer.write_all(entry.metadata.to_json()?.as_bytes()).map_err(|e| e.to_string())
|
||||
})?;
|
||||
if let Some(path) = &entry.path {
|
||||
write_file(&database_dir.join(format!("{basename}.bson{suffix}")), gzip, |writer| {
|
||||
copy_stream(&mut *reader(path, false)?, writer, &mut check)
|
||||
})?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn regular_file_accepts_files_and_rejects_directories() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let file = root.path().join("records.bson");
|
||||
std::fs::write(&file, []).unwrap();
|
||||
regular_file(&file).unwrap();
|
||||
assert!(regular_file(root.path()).unwrap_err().contains("Not a regular dump file"));
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn reject_link_rejects_file_and_directory_symlinks() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let file = root.path().join("records.bson");
|
||||
std::fs::write(&file, []).unwrap();
|
||||
for (name, target) in [("file-link", file.as_path()), ("directory-link", root.path())] {
|
||||
let link = root.path().join(name);
|
||||
std::os::unix::fs::symlink(target, &link).unwrap();
|
||||
assert!(reject_link(&link).unwrap_err().contains("Dump sources cannot contain links"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
use base64::Engine;
|
||||
use mongodb::bson::{doc, Bson, Document};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha1::{Digest, Sha1};
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct CollectionMetadata {
|
||||
#[serde(default)]
|
||||
pub options: Document,
|
||||
#[serde(default)]
|
||||
pub indexes: Vec<Document>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub uuid: Option<String>,
|
||||
#[serde(default)]
|
||||
pub collection_name: String,
|
||||
#[serde(default, rename = "type")]
|
||||
pub kind: String,
|
||||
}
|
||||
|
||||
impl CollectionMetadata {
|
||||
pub fn empty(name: &str) -> Self {
|
||||
Self {
|
||||
options: Document::new(),
|
||||
indexes: Vec::new(),
|
||||
uuid: None,
|
||||
collection_name: name.into(),
|
||||
kind: "collection".into(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_json(json: &str, name: &str) -> Result<Self, String> {
|
||||
let value = serde_json::from_str(json).map_err(|e| format!("Invalid metadata for {name}: {e}"))?;
|
||||
let document = crate::db::mongo_driver::json_object_to_document_extended_json(&value)?;
|
||||
let mut metadata: Self =
|
||||
mongodb::bson::from_document(document).map_err(|e| format!("Invalid metadata for {name}: {e}"))?;
|
||||
if metadata.collection_name.is_empty() {
|
||||
metadata.collection_name = name.into();
|
||||
}
|
||||
if metadata.kind.is_empty() {
|
||||
metadata.kind = if metadata.options.contains_key("viewOn") { "view" } else { "collection" }.into();
|
||||
}
|
||||
metadata.validate()?;
|
||||
Ok(metadata)
|
||||
}
|
||||
|
||||
pub fn to_json(&self) -> Result<String, String> {
|
||||
let document = mongodb::bson::to_document(self).map_err(|e| e.to_string())?;
|
||||
let json = crate::db::mongo_driver::document_to_canonical_extended_json(&document).to_string();
|
||||
if json.len() > super::archive::MAX_DOCUMENT {
|
||||
return Err("Collection metadata exceeds 16 MiB".into());
|
||||
}
|
||||
Ok(json)
|
||||
}
|
||||
|
||||
pub fn is_view(&self) -> bool {
|
||||
self.kind == "view"
|
||||
}
|
||||
|
||||
pub fn validate(&self) -> Result<(), String> {
|
||||
validate_collection(&self.collection_name)?;
|
||||
let parsed_options: mongodb::options::CreateCollectionOptions =
|
||||
mongodb::bson::from_document(self.options.clone())
|
||||
.map_err(|e| format!("Invalid collection options for {}: {e}", self.collection_name))?;
|
||||
let known_options = mongodb::bson::to_document(&parsed_options).map_err(|e| e.to_string())?;
|
||||
for key in self.options.keys() {
|
||||
if !known_options.contains_key(key) && key != "autoIndexId" && key != "recordIdsReplicated" {
|
||||
return Err(format!("Unsupported collection option for {}: {key}", self.collection_name));
|
||||
}
|
||||
}
|
||||
if self.kind == "timeseries"
|
||||
|| self.options.contains_key("timeseries")
|
||||
|| self.options.contains_key("encryptedFields")
|
||||
{
|
||||
return Err(format!(
|
||||
"{}: time-series and encrypted collection dumps require server-specific restore support",
|
||||
self.collection_name
|
||||
));
|
||||
}
|
||||
if self.kind != "collection" && self.kind != "view" {
|
||||
return Err(format!("Unsupported collection type: {}", self.kind));
|
||||
}
|
||||
if self.is_view() {
|
||||
self.options
|
||||
.get_str("viewOn")
|
||||
.map_err(|_| format!("{}: view metadata is missing viewOn", self.collection_name))?;
|
||||
if !self.indexes.is_empty() {
|
||||
return Err("View metadata cannot contain indexes".into());
|
||||
}
|
||||
}
|
||||
for index in &self.indexes {
|
||||
index.get_document("key").map_err(|_| "Index metadata is missing key".to_string())?;
|
||||
index.get_str("name").map_err(|_| "Index metadata is missing name".to_string())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn create_command(&self, restore_options: bool) -> Document {
|
||||
let mut command = doc! { "create": &self.collection_name };
|
||||
if restore_options || self.is_view() {
|
||||
let mut options = self.options.clone();
|
||||
options.remove("uuid");
|
||||
options.remove("recordIdsReplicated");
|
||||
options.remove("autoIndexId");
|
||||
// Keep create as the first command key and never accept it from metadata.
|
||||
options.remove("create");
|
||||
command.extend(options);
|
||||
}
|
||||
command
|
||||
}
|
||||
|
||||
pub fn restore_indexes(&self) -> Vec<Document> {
|
||||
self.indexes
|
||||
.iter()
|
||||
.filter(|index| {
|
||||
let key = index.get_document("key").expect("validated index");
|
||||
!(key.len() == 1
|
||||
&& matches!(key.get("_id"), Some(Bson::Int32(1) | Bson::Int64(1)) | Some(Bson::Double(1.0))))
|
||||
})
|
||||
.map(|index| {
|
||||
let mut index = index.clone();
|
||||
index.remove("ns");
|
||||
index.remove("v");
|
||||
index.remove("background");
|
||||
index
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn validate_database(name: &str) -> Result<(), String> {
|
||||
if name.is_empty() || name.len() > 63 || name.chars().any(|c| c.is_control() || " /\\.\"$*<>:|?".contains(c)) {
|
||||
return Err("Invalid MongoDB database name".into());
|
||||
}
|
||||
if matches!(name, "admin" | "config" | "local") {
|
||||
return Err(
|
||||
"Internal MongoDB databases and user/role migration are not supported by database dump/restore".into()
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn validate_collection(name: &str) -> Result<(), String> {
|
||||
if name.is_empty()
|
||||
|| name.contains('\0')
|
||||
|| name.contains('$')
|
||||
|| (name.starts_with("system.") && name != "system.js")
|
||||
{
|
||||
return Err(format!("Unsupported MongoDB collection name: {name}"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn escaped_collection(name: &str) -> String {
|
||||
const ESCAPE: &percent_encoding::AsciiSet =
|
||||
&percent_encoding::NON_ALPHANUMERIC.remove(b'-').remove(b'_').remove(b'.').remove(b'~');
|
||||
let mut escaped = percent_encoding::utf8_percent_encode(name, ESCAPE).to_string().replace("%20", "+");
|
||||
let stem = escaped.split('.').next().unwrap_or_default().to_ascii_uppercase();
|
||||
if matches!(stem.as_str(), "CON" | "PRN" | "AUX" | "NUL")
|
||||
|| (stem.len() == 4
|
||||
&& (stem.starts_with("COM") || stem.starts_with("LPT"))
|
||||
&& matches!(stem.as_bytes()[3], b'1'..=b'9'))
|
||||
{
|
||||
escaped = format!("%{:02X}{}", escaped.as_bytes()[0], &escaped[1..]);
|
||||
}
|
||||
if escaped.len() > 238 {
|
||||
let hash = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(Sha1::digest(name.as_bytes()));
|
||||
escaped = format!("{}%24{hash}", &escaped[..208]);
|
||||
}
|
||||
escaped
|
||||
}
|
||||
|
||||
pub fn unescape_collection(name: &str) -> Result<String, String> {
|
||||
let bytes = name.as_bytes();
|
||||
let mut position = 0;
|
||||
while position < bytes.len() {
|
||||
if bytes[position] == b'%' {
|
||||
if position + 2 >= bytes.len() || !bytes[position + 1..position + 3].iter().all(u8::is_ascii_hexdigit) {
|
||||
return Err("Invalid escaped collection filename".into());
|
||||
}
|
||||
position += 3;
|
||||
} else {
|
||||
position += 1;
|
||||
}
|
||||
}
|
||||
percent_encoding::percent_decode_str(&name.replace('+', " "))
|
||||
.decode_utf8()
|
||||
.map(|v| v.into_owned())
|
||||
.map_err(|e| e.to_string())
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
use std::collections::HashMap;
|
||||
use std::io::Read;
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
use std::sync::{Arc, Mutex, OnceLock};
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::{
|
||||
archive, catalog, directory, ordered_entries, select_entries, MongoDatabaseRestoreRequest, MongoDumpFormat,
|
||||
MongoRestoreSourcePreview, MongoRestoreSourceRequest, PreparedCollection,
|
||||
};
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct MongoDumpFile {
|
||||
pub path: String,
|
||||
pub size_bytes: u64,
|
||||
}
|
||||
|
||||
pub(super) struct CatalogSource {
|
||||
_owner: Option<tempfile::TempDir>,
|
||||
request: MongoRestoreSourceRequest,
|
||||
pub entries: Vec<PreparedCollection>,
|
||||
identities: HashMap<PathBuf, (u64, Option<SystemTime>)>,
|
||||
manifest: Option<Vec<MongoDumpFile>>,
|
||||
created: Instant,
|
||||
}
|
||||
|
||||
static SOURCES: OnceLock<Mutex<HashMap<String, Arc<CatalogSource>>>> = OnceLock::new();
|
||||
fn sources() -> &'static Mutex<HashMap<String, Arc<CatalogSource>>> {
|
||||
SOURCES.get_or_init(Default::default)
|
||||
}
|
||||
|
||||
pub(super) fn get(reference: &str) -> Result<Arc<CatalogSource>, String> {
|
||||
let source = sources()
|
||||
.lock()
|
||||
.unwrap_or_else(|e| e.into_inner())
|
||||
.get(reference)
|
||||
.cloned()
|
||||
.ok_or("Restore source is no longer available")?;
|
||||
if source.created.elapsed() >= Duration::from_secs(86400) {
|
||||
return Err("Restore source has expired".into());
|
||||
}
|
||||
Ok(source)
|
||||
}
|
||||
|
||||
pub(super) fn release(reference: &str) -> bool {
|
||||
sources().lock().unwrap_or_else(|e| e.into_inner()).remove(reference).is_some()
|
||||
}
|
||||
|
||||
pub(super) fn relative_path(value: &str) -> Result<&Path, String> {
|
||||
let path = Path::new(value);
|
||||
if value.is_empty()
|
||||
|| value.contains(['\\', ':', '\0'])
|
||||
|| path.components().count() > 4
|
||||
|| path.components().any(|c| !matches!(c, Component::Normal(_)))
|
||||
{
|
||||
return Err("Invalid dump upload path".into());
|
||||
}
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
fn identity(path: &Path) -> Result<(u64, Option<SystemTime>), String> {
|
||||
directory::regular_file(path)?;
|
||||
let info = std::fs::metadata(path).map_err(|e| e.to_string())?;
|
||||
Ok((info.len(), info.modified().ok()))
|
||||
}
|
||||
|
||||
pub(super) async fn prepare(
|
||||
request: MongoRestoreSourceRequest,
|
||||
owner: Option<tempfile::TempDir>,
|
||||
manifest: Option<Vec<MongoDumpFile>>,
|
||||
) -> Result<MongoRestoreSourcePreview, String> {
|
||||
tokio::task::spawn_blocking(move || {
|
||||
if manifest.as_ref().is_some_and(|m| m.len() > 100_000) {
|
||||
return Err("Too many dump files".into());
|
||||
}
|
||||
let path = Path::new(&request.path);
|
||||
let entries = match request.format {
|
||||
MongoDumpFormat::Directory => directory::inspect(path, request.gzip, manifest.as_deref())?,
|
||||
MongoDumpFormat::Archive => archive::inspect(&mut *directory::reader(path, request.gzip)?, || Ok(()))?,
|
||||
};
|
||||
ordered_entries(&entries)?;
|
||||
let mut catalog = catalog(&entries, false);
|
||||
let mut identities = HashMap::new();
|
||||
for (entry, preview) in entries.iter().zip(catalog.collections.iter_mut()) {
|
||||
for file in entry.path.iter().chain(entry.metadata_path.iter()) {
|
||||
if manifest.is_none() || entry.metadata_path.as_ref() == Some(file) {
|
||||
identities.insert(file.clone(), identity(file)?);
|
||||
}
|
||||
if let Ok(relative) = file.strip_prefix(path) {
|
||||
preview.source_files.push(relative.to_string_lossy().replace('\\', "/"));
|
||||
}
|
||||
}
|
||||
}
|
||||
if request.format == MongoDumpFormat::Archive {
|
||||
identities.insert(path.to_owned(), identity(path)?);
|
||||
}
|
||||
let source =
|
||||
Arc::new(CatalogSource { _owner: owner, request, entries, identities, manifest, created: Instant::now() });
|
||||
let mut sources = sources().lock().unwrap_or_else(|e| e.into_inner());
|
||||
sources.retain(|_, s| s.created.elapsed() < Duration::from_secs(86400));
|
||||
if sources.len() >= 32 {
|
||||
return Err("Too many prepared restores; close or release an existing source".into());
|
||||
}
|
||||
let source_ref = uuid::Uuid::new_v4().to_string();
|
||||
sources.insert(source_ref.clone(), source);
|
||||
Ok(MongoRestoreSourcePreview { source_ref, catalog })
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
}
|
||||
|
||||
pub async fn prepare_owned_mongodb_restore_source(
|
||||
request: MongoRestoreSourceRequest,
|
||||
owner: tempfile::TempDir,
|
||||
) -> Result<MongoRestoreSourcePreview, String> {
|
||||
prepare(request, Some(owner), None).await
|
||||
}
|
||||
|
||||
pub async fn prepare_mongodb_directory_catalog(
|
||||
owner: tempfile::TempDir,
|
||||
manifest: Vec<MongoDumpFile>,
|
||||
gzip: bool,
|
||||
) -> Result<MongoRestoreSourcePreview, String> {
|
||||
let request = MongoRestoreSourceRequest {
|
||||
path: owner.path().to_string_lossy().into_owned(),
|
||||
format: MongoDumpFormat::Directory,
|
||||
gzip,
|
||||
};
|
||||
prepare(request, Some(owner), Some(manifest)).await
|
||||
}
|
||||
|
||||
fn same_metadata(a: &PreparedCollection, b: &PreparedCollection) -> Result<bool, String> {
|
||||
Ok(a.database == b.database && a.metadata.to_json()? == b.metadata.to_json()? && a.size_bytes == b.size_bytes)
|
||||
}
|
||||
|
||||
/// The server, not the upload payload, determines which files the confirmed selection requires.
|
||||
pub fn mongodb_restore_upload_files(request: &MongoDatabaseRestoreRequest) -> Result<Vec<MongoDumpFile>, String> {
|
||||
let source = get(&request.source_ref)?;
|
||||
let manifest = source.manifest.as_ref().ok_or("Source does not require a directory upload")?;
|
||||
let paths = selected(&source, request)?
|
||||
.into_iter()
|
||||
.flat_map(|e| e.path.into_iter().chain(e.metadata_path))
|
||||
.collect::<Vec<_>>();
|
||||
Ok(manifest.iter().filter(|f| paths.contains(&Path::new(&source.request.path).join(&f.path))).cloned().collect())
|
||||
}
|
||||
|
||||
/// Bind the complete selected upload to the reviewed catalog.
|
||||
pub async fn attach_mongodb_restore_upload(
|
||||
request: &MongoDatabaseRestoreRequest,
|
||||
owner: tempfile::TempDir,
|
||||
) -> Result<MongoRestoreSourcePreview, String> {
|
||||
let original = get(&request.source_ref)?;
|
||||
if original.manifest.is_none() {
|
||||
return Err("Source does not require a directory upload".into());
|
||||
}
|
||||
let selected = selected(&original, request)?;
|
||||
let next = prepare_owned_mongodb_restore_source(
|
||||
MongoRestoreSourceRequest {
|
||||
path: owner.path().to_string_lossy().into_owned(),
|
||||
format: MongoDumpFormat::Directory,
|
||||
gzip: original.request.gzip,
|
||||
},
|
||||
owner,
|
||||
)
|
||||
.await?;
|
||||
let uploaded = get(&next.source_ref)?;
|
||||
let matches =
|
||||
(|| -> Result<bool, String> {
|
||||
if selected.len() != uploaded.entries.len() {
|
||||
return Ok(false);
|
||||
}
|
||||
for entry in &selected {
|
||||
let Some(found) = uploaded.entries.iter().find(|e| {
|
||||
e.database == entry.database && e.metadata.collection_name == entry.metadata.collection_name
|
||||
}) else {
|
||||
return Ok(false);
|
||||
};
|
||||
if !same_metadata(entry, found)? {
|
||||
return Ok(false);
|
||||
}
|
||||
let expected = entry
|
||||
.path
|
||||
.iter()
|
||||
.chain(entry.metadata_path.iter())
|
||||
.map(|p| p.strip_prefix(&original.request.path).unwrap().to_owned())
|
||||
.collect::<Vec<_>>();
|
||||
let actual = found
|
||||
.path
|
||||
.iter()
|
||||
.chain(found.metadata_path.iter())
|
||||
.map(|p| p.strip_prefix(&uploaded.request.path).unwrap().to_owned())
|
||||
.collect::<Vec<_>>();
|
||||
if expected != actual {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
})();
|
||||
if matches != Ok(true) {
|
||||
release(&next.source_ref);
|
||||
return Err("Uploaded files differ from the confirmed catalog; read the backup again".into());
|
||||
}
|
||||
Ok(next)
|
||||
}
|
||||
|
||||
fn selected(source: &CatalogSource, request: &MongoDatabaseRestoreRequest) -> Result<Vec<PreparedCollection>, String> {
|
||||
let entries: Vec<_> = source.entries.iter().filter(|e| e.database == request.source_database).cloned().collect();
|
||||
select_entries(&entries, &request.collections)
|
||||
}
|
||||
|
||||
fn unchanged(source: &CatalogSource, path: &Path) -> Result<(), String> {
|
||||
if source.identities.get(path) != Some(&identity(path)?) {
|
||||
return Err("Backup source changed; read the backup again".into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl CatalogSource {
|
||||
pub(super) fn format(&self) -> MongoDumpFormat {
|
||||
self.request.format
|
||||
}
|
||||
|
||||
pub(super) fn check_ready(&self, entries: &[PreparedCollection]) -> Result<(), String> {
|
||||
if self.manifest.is_some() {
|
||||
return Err("Selected directory data has not been uploaded".into());
|
||||
}
|
||||
if self.request.format == MongoDumpFormat::Archive {
|
||||
unchanged(self, Path::new(&self.request.path))?;
|
||||
} else {
|
||||
for entry in entries {
|
||||
for path in entry.path.iter().chain(entry.metadata_path.iter()) {
|
||||
unchanged(self, path)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn reader(&self, path: Option<&Path>) -> Result<Box<dyn Read>, String> {
|
||||
let path = path.unwrap_or_else(|| Path::new(&self.request.path));
|
||||
unchanged(self, path)?;
|
||||
directory::reader(path, self.request.gzip)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,377 @@
|
||||
use std::collections::HashSet;
|
||||
use std::sync::{
|
||||
atomic::{AtomicBool, AtomicU64, Ordering},
|
||||
Arc,
|
||||
};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use mongodb::bson::{Document, RawDocumentBuf};
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use super::{
|
||||
archive, cancelled, source::CatalogSource, CancelFuture, MongoDatabaseDumpProgress, MongoDatabaseRestoreRequest,
|
||||
MongoDumpFormat, PreparedCollection,
|
||||
};
|
||||
use crate::{connection::AppState, db::mongo_driver::insert_bson_documents};
|
||||
|
||||
const BATCH_BYTES: usize = 8 * 1024 * 1024;
|
||||
|
||||
enum Event {
|
||||
Batch { index: usize, documents: Vec<RawDocumentBuf> },
|
||||
End(usize),
|
||||
}
|
||||
|
||||
struct Batcher {
|
||||
tx: mpsc::Sender<Event>,
|
||||
index: usize,
|
||||
documents: Vec<RawDocumentBuf>,
|
||||
bytes: usize,
|
||||
limit: usize,
|
||||
objcheck: bool,
|
||||
}
|
||||
|
||||
impl Batcher {
|
||||
fn flush(&mut self) -> Result<(), String> {
|
||||
if self.documents.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let documents = std::mem::take(&mut self.documents);
|
||||
self.bytes = 0;
|
||||
self.tx
|
||||
.blocking_send(Event::Batch { index: self.index, documents })
|
||||
.map_err(|_| "MongoDB dump/restore cancelled".into())
|
||||
}
|
||||
|
||||
fn push(&mut self, index: usize, bytes: Vec<u8>) -> Result<(), String> {
|
||||
if index != self.index || self.bytes.saturating_add(bytes.len()) > BATCH_BYTES {
|
||||
self.flush()?;
|
||||
}
|
||||
self.index = index;
|
||||
let document = checked_document(bytes, self.objcheck)?;
|
||||
self.bytes += document.as_bytes().len();
|
||||
self.documents.push(document);
|
||||
if self.documents.len() >= self.limit {
|
||||
self.flush()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn end(&mut self, index: usize) -> Result<(), String> {
|
||||
self.flush()?;
|
||||
self.tx.blocking_send(Event::End(index)).map_err(|_| "MongoDB dump/restore cancelled".into())
|
||||
}
|
||||
}
|
||||
|
||||
// Like mongorestore's objcheck, deep parsing is optional and occurs in the stream,
|
||||
// immediately before a document is queued for insertion. No whole-backup pass.
|
||||
fn checked_document(bytes: Vec<u8>, objcheck: bool) -> Result<RawDocumentBuf, String> {
|
||||
let raw = RawDocumentBuf::from_bytes(bytes).map_err(|e| format!("Invalid BSON document: {e}"))?;
|
||||
if objcheck {
|
||||
mongodb::bson::from_slice::<Document>(raw.as_bytes()).map_err(|e| format!("Invalid BSON document: {e}"))?;
|
||||
}
|
||||
Ok(raw)
|
||||
}
|
||||
|
||||
fn produce(
|
||||
source: Arc<CatalogSource>,
|
||||
entries: Vec<PreparedCollection>,
|
||||
flag: Arc<AtomicBool>,
|
||||
bytes_read: Arc<AtomicU64>,
|
||||
mut batcher: Batcher,
|
||||
) -> Result<(), String> {
|
||||
let result = (|| {
|
||||
source.check_ready(&entries)?;
|
||||
if source.format() == MongoDumpFormat::Archive {
|
||||
let indices = source
|
||||
.entries
|
||||
.iter()
|
||||
.map(|original| {
|
||||
entries.iter().position(|e| {
|
||||
e.database == original.database
|
||||
&& e.metadata.collection_name == original.metadata.collection_name
|
||||
&& !e.metadata.is_view()
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
archive::stream(
|
||||
&mut *source.reader(None)?,
|
||||
&source.entries,
|
||||
|| cancelled(&flag),
|
||||
|event| {
|
||||
match event {
|
||||
archive::StreamEvent::Document(i, bytes) => {
|
||||
bytes_read.fetch_add(bytes.len() as u64, Ordering::Relaxed);
|
||||
if let Some(index) = indices[i] {
|
||||
batcher.push(index, bytes)?;
|
||||
}
|
||||
}
|
||||
archive::StreamEvent::End(i) => {
|
||||
if let Some(index) = indices[i] {
|
||||
batcher.end(index)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
)?;
|
||||
} else {
|
||||
for (index, entry) in entries.iter().enumerate().filter(|(_, e)| !e.metadata.is_view()) {
|
||||
cancelled(&flag)?;
|
||||
let path = entry.path.as_deref().ok_or("Missing collection BSON file")?;
|
||||
let mut reader = source.reader(Some(path))?;
|
||||
loop {
|
||||
cancelled(&flag)?;
|
||||
match archive::read_raw_frame(&mut *reader)? {
|
||||
archive::Frame::End => break,
|
||||
archive::Frame::Terminator => return Err("Unexpected terminator in collection BSON".into()),
|
||||
archive::Frame::Document(bytes) => {
|
||||
bytes_read.fetch_add(bytes.len() as u64, Ordering::Relaxed);
|
||||
batcher.push(index, bytes)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
batcher.end(index)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
// Deliver valid earlier documents even if a later frame is corrupt, matching
|
||||
// the streaming import contract and reporting partial writes accurately.
|
||||
let flushed = batcher.flush();
|
||||
result.and(flushed)
|
||||
}
|
||||
|
||||
pub(super) async fn initialize_collection(
|
||||
client: &mongodb::Client,
|
||||
request: &MongoDatabaseRestoreRequest,
|
||||
entry: &PreparedCollection,
|
||||
exists: bool,
|
||||
) -> Result<(), String> {
|
||||
let db = client.database(&request.database);
|
||||
if request.drop_existing && exists {
|
||||
db.collection::<Document>(&entry.metadata.collection_name).drop().await.map_err(|e| e.to_string())?;
|
||||
}
|
||||
if !exists || request.drop_existing {
|
||||
db.run_command(entry.metadata.create_command(request.restore_options)).await.map_err(|e| e.to_string())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn writable(state: &AppState, id: &str) -> Result<(), String> {
|
||||
if let Some(name) = crate::query::connection_readonly_name(state, id).await {
|
||||
return Err(format!("Read-only connection '{name}': restore blocked"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) async fn restore_data(
|
||||
state: &AppState,
|
||||
client: &mongodb::Client,
|
||||
request: &MongoDatabaseRestoreRequest,
|
||||
source: Arc<CatalogSource>,
|
||||
entries: &[PreparedCollection],
|
||||
existing: &HashSet<String>,
|
||||
progress: &mut MongoDatabaseDumpProgress,
|
||||
started: Instant,
|
||||
is_cancelled: &mut impl FnMut(&str) -> CancelFuture,
|
||||
on_progress: &mut impl FnMut(MongoDatabaseDumpProgress),
|
||||
) -> Result<(), String> {
|
||||
struct StopOnDrop(Arc<AtomicBool>);
|
||||
impl Drop for StopOnDrop {
|
||||
fn drop(&mut self) {
|
||||
self.0.store(true, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
let flag = Arc::new(AtomicBool::new(false));
|
||||
let _guard = StopOnDrop(flag.clone());
|
||||
let (tx, mut rx) = mpsc::channel(2);
|
||||
let batcher = Batcher {
|
||||
tx,
|
||||
index: 0,
|
||||
documents: Vec::new(),
|
||||
bytes: 0,
|
||||
limit: request.batch_size.max(1),
|
||||
objcheck: request.objcheck,
|
||||
};
|
||||
let producer_entries = entries.to_vec();
|
||||
let signal = flag.clone();
|
||||
let bytes_read = Arc::new(AtomicU64::new(0));
|
||||
let reader_progress = bytes_read.clone();
|
||||
let producer =
|
||||
tokio::task::spawn_blocking(move || produce(source, producer_entries, signal, reader_progress, batcher));
|
||||
progress.phase = "data".into();
|
||||
progress.emit(started, on_progress);
|
||||
let result = async {
|
||||
let mut initialized = HashSet::new();
|
||||
let mut tick = tokio::time::interval(Duration::from_millis(100));
|
||||
loop {
|
||||
let event = tokio::select! {
|
||||
event = rx.recv() => match event { Some(event) => event, None => break },
|
||||
_ = tick.tick() => {
|
||||
if is_cancelled(&request.task_id).await { return Err("MongoDB dump/restore cancelled".into()); }
|
||||
progress.bytes_processed = bytes_read.load(Ordering::Relaxed);
|
||||
progress.emit(started, on_progress);
|
||||
continue;
|
||||
},
|
||||
};
|
||||
if is_cancelled(&request.task_id).await {
|
||||
return Err("MongoDB dump/restore cancelled".into());
|
||||
}
|
||||
writable(state, &request.connection_id).await?;
|
||||
let index = match &event {
|
||||
Event::Batch { index, .. } | Event::End(index) => *index,
|
||||
};
|
||||
let entry = &entries[index];
|
||||
progress.phase = "data".into();
|
||||
progress.collection = Some(entry.metadata.collection_name.clone());
|
||||
progress.emit(started, on_progress);
|
||||
if initialized.insert(index) {
|
||||
initialize_collection(client, request, entry, existing.contains(&entry.metadata.collection_name))
|
||||
.await?;
|
||||
}
|
||||
match event {
|
||||
Event::Batch { documents, .. } => {
|
||||
progress.documents_read += documents.len() as u64;
|
||||
progress.bytes_processed = bytes_read.load(Ordering::Relaxed);
|
||||
if request.objcheck {
|
||||
progress.documents_validated += documents.len() as u64;
|
||||
}
|
||||
let outcome =
|
||||
insert_bson_documents(client, &request.database, &entry.metadata.collection_name, documents)
|
||||
.await
|
||||
.map_err(|e| e.message)?;
|
||||
progress.documents_written += outcome.inserted;
|
||||
progress.documents_failed += outcome.errors.len() as u64;
|
||||
progress.emit(started, on_progress);
|
||||
if let Some(error) = outcome.errors.first().filter(|_| request.stop_on_error) {
|
||||
return Err(error.message.clone());
|
||||
}
|
||||
}
|
||||
Event::End(_) => {
|
||||
progress.collections_done += 1;
|
||||
progress.emit(started, on_progress);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
.await;
|
||||
flag.store(true, Ordering::Relaxed);
|
||||
drop(rx);
|
||||
let produced = producer.await.map_err(|e| e.to_string()).and_then(|r| r);
|
||||
result.and(produced)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn objcheck_uses_official_bson_parser_only_when_enabled() {
|
||||
// Valid outer framing but invalid boolean payload.
|
||||
let bytes = vec![9, 0, 0, 0, 8, b'x', 0, 2, 0];
|
||||
assert!(checked_document(bytes.clone(), false).is_ok());
|
||||
assert!(checked_document(bytes, true).is_err());
|
||||
assert!(checked_document(vec![5, 0, 0, 0, 1], false).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_bson_preserves_duplicate_keys_and_exact_bytes() {
|
||||
let bytes = vec![19, 0, 0, 0, 16, b'x', 0, 1, 0, 0, 0, 16, b'x', 0, 2, 0, 0, 0, 0];
|
||||
for objcheck in [false, true] {
|
||||
let raw = checked_document(bytes.clone(), objcheck).unwrap();
|
||||
assert_eq!(raw.as_bytes(), bytes);
|
||||
assert_eq!(mongodb::bson::to_vec(&raw).unwrap(), bytes);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reader_delivers_earlier_batches_before_reporting_corruption() {
|
||||
use crate::mongodb_dump::{
|
||||
prepare_mongodb_restore_source, release_mongodb_restore_source, MongoRestoreSourceRequest,
|
||||
};
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let db = dir.path().join("source");
|
||||
std::fs::create_dir(&db).unwrap();
|
||||
let mut bytes = mongodb::bson::to_vec(&mongodb::bson::doc! { "_id": 1 }).unwrap();
|
||||
bytes.extend_from_slice(b"invalid BSON");
|
||||
std::fs::write(db.join("records.bson"), bytes).unwrap();
|
||||
let preview = prepare_mongodb_restore_source(MongoRestoreSourceRequest {
|
||||
path: db.to_string_lossy().into_owned(),
|
||||
format: MongoDumpFormat::Directory,
|
||||
gzip: false,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let source = super::super::source::get(&preview.source_ref).unwrap();
|
||||
let entries = source.entries.clone();
|
||||
let (tx, mut rx) = mpsc::channel(1);
|
||||
let batcher = Batcher { tx, index: 0, documents: Vec::new(), bytes: 0, limit: 1, objcheck: false };
|
||||
let worker = tokio::task::spawn_blocking(move || {
|
||||
produce(source, entries, Arc::new(AtomicBool::new(false)), Arc::new(AtomicU64::new(0)), batcher)
|
||||
});
|
||||
assert!(matches!(rx.recv().await, Some(Event::Batch { documents, .. }) if documents.len() == 1));
|
||||
assert!(worker.await.unwrap().unwrap_err().contains("BSON"));
|
||||
assert!(rx.recv().await.is_none());
|
||||
release_mongodb_restore_source(&preview.source_ref);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batches_are_limited_by_bytes_as_well_as_document_count() {
|
||||
let bytes = mongodb::bson::to_vec(&mongodb::bson::doc! { "payload": "x".repeat(5 * 1024 * 1024) }).unwrap();
|
||||
let (tx, mut rx) = mpsc::channel(2);
|
||||
let mut batcher = Batcher { tx, index: 0, documents: Vec::new(), bytes: 0, limit: 500, objcheck: false };
|
||||
batcher.push(0, bytes.clone()).unwrap();
|
||||
batcher.push(0, bytes).unwrap();
|
||||
batcher.flush().unwrap();
|
||||
for _ in 0..2 {
|
||||
assert!(matches!(rx.try_recv().unwrap(), Event::Batch { documents, .. } if documents.len() == 1));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "opt-in compressed BSON parser throughput smoke test"]
|
||||
fn compressed_bson_parser_throughput() {
|
||||
use flate2::{read::MultiGzDecoder, write::GzEncoder, Compression};
|
||||
use std::io::{BufReader, Cursor, Write};
|
||||
let mut document = Document::new();
|
||||
document.insert("_id", 1i64);
|
||||
for i in 0..64 {
|
||||
document.insert(format!("field_{i}"), "value".repeat(8));
|
||||
}
|
||||
let encoded = mongodb::bson::to_vec(&document).unwrap();
|
||||
let documents = 8192;
|
||||
let mut encoder = GzEncoder::new(Vec::new(), Compression::fast());
|
||||
for _ in 0..documents {
|
||||
encoder.write_all(&encoded).unwrap();
|
||||
}
|
||||
let compressed = encoder.finish().unwrap();
|
||||
for (mode, objcheck, passes) in
|
||||
[("raw-stream", false, 1), ("objcheck-stream", true, 1), ("old-double-parse", true, 2)]
|
||||
{
|
||||
let started = Instant::now();
|
||||
let mut count = 0;
|
||||
for _ in 0..passes {
|
||||
let mut input = BufReader::with_capacity(64 * 1024, MultiGzDecoder::new(Cursor::new(&compressed)));
|
||||
loop {
|
||||
match archive::read_raw_frame(&mut input).unwrap() {
|
||||
archive::Frame::Document(bytes) => {
|
||||
checked_document(bytes, objcheck).unwrap();
|
||||
count += 1;
|
||||
}
|
||||
archive::Frame::End => break,
|
||||
_ => panic!("unexpected terminator"),
|
||||
}
|
||||
}
|
||||
}
|
||||
assert_eq!(count, documents * passes);
|
||||
eprintln!(
|
||||
"{mode}: logical_mib={:.2}, elapsed_ms={}, logical_mib_s={:.2}",
|
||||
(documents * encoded.len()) as f64 / 1048576.0,
|
||||
started.elapsed().as_millis(),
|
||||
(documents * encoded.len()) as f64 / 1048576.0 / started.elapsed().as_secs_f64()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ use std::path::{Path, PathBuf};
|
||||
use std::time::Instant;
|
||||
|
||||
use chrono::{DateTime as ChronoDateTime, NaiveDate, Utc};
|
||||
use flate2::{read::MultiGzDecoder, write::GzEncoder, Compression};
|
||||
use mongodb::bson::{oid::ObjectId, Bson, DateTime, Decimal128, Document};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
@@ -26,6 +27,7 @@ pub const MAX_BATCH_SIZE: usize = 5000;
|
||||
/// Rows sampled for CSV type inference, independent of the preview window so that the
|
||||
/// preview and the import always agree on column types.
|
||||
pub const TYPE_SAMPLE_ROWS: usize = 1000;
|
||||
const BSON_MAX_DOCUMENT_BYTES: usize = 16 * 1024 * 1024;
|
||||
|
||||
const TYPE_STRING: u8 = 1 << 0;
|
||||
const TYPE_BOOLEAN: u8 = 1 << 1;
|
||||
@@ -45,6 +47,7 @@ pub enum MongoImportFormat {
|
||||
Csv,
|
||||
Json,
|
||||
Ndjson,
|
||||
Bson,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
@@ -341,14 +344,16 @@ struct CsvParseConfig {
|
||||
|
||||
pub fn format_from_path(path: &str) -> Result<MongoImportFormat, String> {
|
||||
let lower = path.to_lowercase();
|
||||
if lower.ends_with(".csv") || lower.ends_with(".tsv") || lower.ends_with(".txt") {
|
||||
if lower.ends_with(".bson") || lower.ends_with(".bson.gz") {
|
||||
Ok(MongoImportFormat::Bson)
|
||||
} else if lower.ends_with(".csv") || lower.ends_with(".tsv") || lower.ends_with(".txt") {
|
||||
Ok(MongoImportFormat::Csv)
|
||||
} else if lower.ends_with(".ndjson") || lower.ends_with(".jsonl") {
|
||||
Ok(MongoImportFormat::Ndjson)
|
||||
} else if lower.ends_with(".json") {
|
||||
Ok(MongoImportFormat::Json)
|
||||
} else {
|
||||
Err("Unsupported MongoDB import file type; use .csv, .json, or .ndjson".to_string())
|
||||
Err("Unsupported MongoDB import file type; use .bson, .bson.gz, .csv, .json, or .ndjson".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1348,6 +1353,72 @@ fn file_size(path: &str) -> u64 {
|
||||
std::fs::metadata(path).map(|metadata| metadata.len()).unwrap_or(0)
|
||||
}
|
||||
|
||||
fn open_bson_dump_reader(path: &str) -> Result<Box<dyn Read>, MongoImportIssue> {
|
||||
let file = File::open(path).map_err(|error| MongoImportIssue::new("FILE_UNREADABLE", error.to_string()))?;
|
||||
let reader = BufReader::new(file);
|
||||
if path.to_lowercase().ends_with(".gz") {
|
||||
Ok(Box::new(MultiGzDecoder::new(reader)))
|
||||
} else {
|
||||
Ok(Box::new(reader))
|
||||
}
|
||||
}
|
||||
|
||||
fn read_bson_dump_document<R: Read + ?Sized>(reader: &mut R, row: u64) -> Result<Option<Document>, MongoImportIssue> {
|
||||
let mut length_bytes = [0u8; 4];
|
||||
loop {
|
||||
match reader.read(&mut length_bytes[..1]) {
|
||||
Ok(0) => return Ok(None),
|
||||
Ok(_) => break,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue,
|
||||
Err(error) => return Err(MongoImportIssue::new("FILE_UNREADABLE", error.to_string()).with_row(row)),
|
||||
}
|
||||
}
|
||||
reader.read_exact(&mut length_bytes[1..]).map_err(|error| {
|
||||
MongoImportIssue::new("BSON_STRUCTURE", format!("Truncated BSON document length: {error}")).with_row(row)
|
||||
})?;
|
||||
|
||||
let document_length = i32::from_le_bytes(length_bytes);
|
||||
if document_length < 5 || document_length as usize > BSON_MAX_DOCUMENT_BYTES {
|
||||
return Err(MongoImportIssue::new(
|
||||
"BSON_STRUCTURE",
|
||||
format!("Invalid BSON document length {document_length}; expected 5..={BSON_MAX_DOCUMENT_BYTES} bytes"),
|
||||
)
|
||||
.with_row(row));
|
||||
}
|
||||
|
||||
let mut bytes = vec![0u8; document_length as usize];
|
||||
bytes[..4].copy_from_slice(&length_bytes);
|
||||
reader.read_exact(&mut bytes[4..]).map_err(|error| {
|
||||
MongoImportIssue::new(
|
||||
"BSON_STRUCTURE",
|
||||
format!("Truncated BSON document; expected {document_length} bytes: {error}"),
|
||||
)
|
||||
.with_row(row)
|
||||
})?;
|
||||
let document = mongodb::bson::from_slice::<Document>(&bytes).map_err(|error| {
|
||||
MongoImportIssue::new("BSON_STRUCTURE", format!("Invalid BSON document: {error}")).with_row(row)
|
||||
})?;
|
||||
Ok(Some(document))
|
||||
}
|
||||
|
||||
fn parse_bson_preview(
|
||||
path: &str,
|
||||
preview_limit: usize,
|
||||
) -> Result<(Vec<ParsedMongoDocument>, u64, bool), MongoImportIssue> {
|
||||
let mut reader = open_bson_dump_reader(path)?;
|
||||
let mut documents = Vec::with_capacity(preview_limit.min(DEFAULT_PREVIEW_LIMIT));
|
||||
let mut row = 1u64;
|
||||
while documents.len() <= preview_limit {
|
||||
let Some(document) = read_bson_dump_document(&mut *reader, row)? else {
|
||||
return Ok((documents, row - 1, true));
|
||||
};
|
||||
documents.push(parsed_document(row, document, true));
|
||||
row += 1;
|
||||
}
|
||||
documents.truncate(preview_limit);
|
||||
Ok((documents, preview_limit as u64, false))
|
||||
}
|
||||
|
||||
pub fn preview_mongodb_import_file(
|
||||
request: &MongoImportPreviewRequest,
|
||||
) -> Result<MongoImportPreview, MongoImportIssue> {
|
||||
@@ -1406,6 +1477,25 @@ pub fn preview_mongodb_import_file(
|
||||
estimated_rows_exact,
|
||||
})
|
||||
}
|
||||
MongoImportFormat::Bson => {
|
||||
let (documents, estimated_rows, estimated_rows_exact) =
|
||||
parse_bson_preview(&request.file_path, preview_limit)?;
|
||||
Ok(MongoImportPreview {
|
||||
source_ref: request.source_ref.clone(),
|
||||
format: request.format,
|
||||
detected_encoding: None,
|
||||
file_name: file_name(&request.file_path),
|
||||
file_path: request.file_path.clone(),
|
||||
size_bytes: file_size(&request.file_path),
|
||||
columns: columns_from_documents(&documents),
|
||||
row_numbers: documents.iter().map(|document| document.row).collect(),
|
||||
rows: documents.into_iter().map(|document| document.extended_json).collect(),
|
||||
warnings: Vec::new(),
|
||||
errors: Vec::new(),
|
||||
estimated_rows: Some(estimated_rows),
|
||||
estimated_rows_exact,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1419,6 +1509,7 @@ pub fn preview_mongodb_import_bytes(
|
||||
MongoImportFormat::Csv => "csv",
|
||||
MongoImportFormat::Json => "json",
|
||||
MongoImportFormat::Ndjson => "ndjson",
|
||||
MongoImportFormat::Bson => "bson",
|
||||
};
|
||||
let dir = std::env::temp_dir().join(format!("dbx-mongo-import-preview-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::create_dir_all(&dir).map_err(|error| MongoImportIssue::new("FILE_UNREADABLE", error.to_string()))?;
|
||||
@@ -1517,6 +1608,20 @@ where
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn stream_bson_file<F>(path: &str, mut on_document: F) -> Result<(), MongoImportIssue>
|
||||
where
|
||||
F: FnMut(Result<ParsedMongoDocument, MongoImportIssue>) -> Result<(), MongoImportIssue>,
|
||||
{
|
||||
let mut reader = open_bson_dump_reader(path)?;
|
||||
let mut row = 1u64;
|
||||
loop {
|
||||
// A corrupt frame has no reliable next-document boundary, even when skipping row errors.
|
||||
let Some(document) = read_bson_dump_document(&mut *reader, row)? else { return Ok(()) };
|
||||
on_document(Ok(parsed_document(row, document, false)))?;
|
||||
row += 1;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn for_each_mongodb_import_document<F>(
|
||||
path: &str,
|
||||
format: MongoImportFormat,
|
||||
@@ -1531,6 +1636,7 @@ where
|
||||
MongoImportFormat::Csv => stream_csv_file(path, options, on_document),
|
||||
MongoImportFormat::Json => stream_json_file(path, options, false, on_document),
|
||||
MongoImportFormat::Ndjson => stream_json_file(path, options, true, on_document),
|
||||
MongoImportFormat::Bson => stream_bson_file(path, on_document),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1929,6 +2035,7 @@ pub fn mongodb_export_client_session_id(export_id: &str) -> String {
|
||||
pub enum MongoExportFormat {
|
||||
Csv,
|
||||
Ndjson,
|
||||
Bson,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
@@ -1958,6 +2065,8 @@ pub struct MongoExportRequest {
|
||||
pub format: MongoExportFormat,
|
||||
#[serde(default = "default_true")]
|
||||
pub include_header: bool,
|
||||
#[serde(default)]
|
||||
pub gzip: bool,
|
||||
pub file_path: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub execution_id: Option<String>,
|
||||
@@ -1992,7 +2101,7 @@ pub struct MongoExportSummary {
|
||||
|
||||
fn temp_export_path(target: &Path) -> PathBuf {
|
||||
let name = target.file_name().and_then(|name| name.to_str()).unwrap_or("export");
|
||||
target.with_file_name(format!(".{name}.dbx-export.tmp"))
|
||||
target.with_file_name(format!(".{name}.{}.dbx-export.tmp", uuid::Uuid::new_v4()))
|
||||
}
|
||||
|
||||
fn cleanup_temp(path: &Path) {
|
||||
@@ -2214,6 +2323,9 @@ where
|
||||
{
|
||||
let started_at = Instant::now();
|
||||
on_progress(export_progress(&request.export_id, MongoExportStatus::Running, 0, 0, None, None, started_at));
|
||||
if request.gzip && request.format != MongoExportFormat::Bson {
|
||||
return Err("Gzip compression is only supported for BSON dump exports".to_string());
|
||||
}
|
||||
if is_cancelled(&request.export_id).await {
|
||||
on_progress(export_progress(
|
||||
&request.export_id,
|
||||
@@ -2250,6 +2362,9 @@ where
|
||||
MongoExportFormat::Csv => {
|
||||
export_csv(state, request, &temp, total_documents, started_at, &mut is_cancelled, &mut on_progress).await
|
||||
}
|
||||
MongoExportFormat::Bson => {
|
||||
export_bson(state, request, &temp, total_documents, started_at, &mut is_cancelled, &mut on_progress).await
|
||||
}
|
||||
};
|
||||
|
||||
match result {
|
||||
@@ -2685,6 +2800,120 @@ where
|
||||
Ok((documents_read, bytes_written))
|
||||
}
|
||||
|
||||
enum BsonExportWriter {
|
||||
Plain(BufWriter<File>),
|
||||
Gzip(Box<GzEncoder<BufWriter<File>>>),
|
||||
}
|
||||
|
||||
impl BsonExportWriter {
|
||||
fn finish(self) -> Result<(), String> {
|
||||
match self {
|
||||
Self::Plain(mut writer) => writer.flush().map_err(|error| error.to_string()),
|
||||
Self::Gzip(writer) => {
|
||||
let mut writer = writer.finish().map_err(|error| error.to_string())?;
|
||||
writer.flush().map_err(|error| error.to_string())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for BsonExportWriter {
|
||||
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
||||
match self {
|
||||
Self::Plain(writer) => writer.write(buf),
|
||||
Self::Gzip(writer) => writer.write(buf),
|
||||
}
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
match self {
|
||||
Self::Plain(writer) => writer.flush(),
|
||||
Self::Gzip(writer) => writer.flush(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn export_bson<C, F>(
|
||||
state: &AppState,
|
||||
request: &MongoExportRequest,
|
||||
temp: &Path,
|
||||
total_documents: Option<u64>,
|
||||
started_at: Instant,
|
||||
is_cancelled: &mut C,
|
||||
on_progress: &mut F,
|
||||
) -> Result<(u64, u64), String>
|
||||
where
|
||||
C: FnMut(&str) -> std::pin::Pin<Box<dyn std::future::Future<Output = bool> + Send>>,
|
||||
F: FnMut(MongoExportProgress),
|
||||
{
|
||||
if is_cancelled(&request.export_id).await {
|
||||
return Err("Export cancelled".to_string());
|
||||
}
|
||||
// BSON dumps must carry native BSON types, so they are written straight from the native
|
||||
// driver's documents; the legacy agent's canonical Extended JSON round trip does not apply.
|
||||
let client = match state.pool_handle(&request.connection_id).await {
|
||||
Some(PoolKind::MongoDb(client)) => client,
|
||||
_ => return Err("BSON dump export requires the native MongoDB driver".into()),
|
||||
};
|
||||
let file = File::create(temp).map_err(|error| error.to_string())?;
|
||||
let buffered = BufWriter::new(file);
|
||||
let mut writer = if request.gzip {
|
||||
BsonExportWriter::Gzip(Box::new(GzEncoder::new(buffered, Compression::default())))
|
||||
} else {
|
||||
BsonExportWriter::Plain(buffered)
|
||||
};
|
||||
let mut documents_read = 0u64;
|
||||
let mut bytes_written = 0u64;
|
||||
|
||||
let export = for_each_find_document(
|
||||
&client,
|
||||
&request.database,
|
||||
&request.collection,
|
||||
request.filter.as_deref(),
|
||||
request.projection.as_deref(),
|
||||
request.sort.as_deref(),
|
||||
request.collation.as_deref(),
|
||||
DEFAULT_EXPORT_BATCH_SIZE,
|
||||
|document| {
|
||||
let bytes = mongodb::bson::to_vec(&document).map_err(|error| error.to_string())?;
|
||||
writer.write_all(&bytes).map_err(|error| error.to_string())?;
|
||||
bytes_written += bytes.len() as u64;
|
||||
documents_read += 1;
|
||||
if documents_read == 1 || documents_read.is_multiple_of(500) {
|
||||
on_progress(export_progress(
|
||||
&request.export_id,
|
||||
MongoExportStatus::Running,
|
||||
documents_read,
|
||||
bytes_written,
|
||||
total_documents,
|
||||
None,
|
||||
started_at,
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
);
|
||||
{
|
||||
tokio::pin!(export);
|
||||
let mut poll_cancel = tokio::time::interval(std::time::Duration::from_millis(100));
|
||||
loop {
|
||||
tokio::select! {
|
||||
result = &mut export => { result?; break; }
|
||||
_ = poll_cancel.tick() => {
|
||||
if is_cancelled(&request.export_id).await {
|
||||
return Err("Export cancelled".to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
writer.finish()?;
|
||||
if is_cancelled(&request.export_id).await {
|
||||
return Err("Export cancelled".to_string());
|
||||
}
|
||||
Ok((documents_read, bytes_written))
|
||||
}
|
||||
|
||||
const CSV_FIELD_DISCOVERY_DOCS: usize = 10_000;
|
||||
|
||||
async fn export_csv<C, F>(
|
||||
@@ -2847,6 +3076,112 @@ mod tests {
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
fn bson_dump_bytes(documents: &[Document]) -> Vec<u8> {
|
||||
let mut output = Vec::new();
|
||||
for document in documents {
|
||||
output.extend_from_slice(&mongodb::bson::to_vec(document).unwrap());
|
||||
}
|
||||
output
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bson_export_writer_finishes_plain_and_gzip_dumps() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let documents = vec![doc! { "_id": 1, "name": "first" }, doc! { "_id": 2, "value": 42i64 }];
|
||||
for expected in [Vec::new(), bson_dump_bytes(&documents)] {
|
||||
for gzip in [false, true] {
|
||||
let path = root.path().join(if gzip { "records.bson.gz" } else { "records.bson" });
|
||||
let buffered = BufWriter::new(File::create(&path).unwrap());
|
||||
let mut writer = if gzip {
|
||||
BsonExportWriter::Gzip(Box::new(GzEncoder::new(buffered, Compression::default())))
|
||||
} else {
|
||||
BsonExportWriter::Plain(buffered)
|
||||
};
|
||||
writer.write_all(&expected).unwrap();
|
||||
writer.finish().unwrap();
|
||||
|
||||
let bytes = std::fs::read(&path).unwrap();
|
||||
let actual = if gzip {
|
||||
let mut decoded = Vec::new();
|
||||
MultiGzDecoder::new(bytes.as_slice()).read_to_end(&mut decoded).unwrap();
|
||||
decoded
|
||||
} else {
|
||||
bytes
|
||||
};
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bson_dump_preview_and_import_preserve_native_types() {
|
||||
use mongodb::bson::spec::BinarySubtype;
|
||||
use mongodb::bson::{Binary, DateTime};
|
||||
|
||||
let documents = vec![
|
||||
doc! {
|
||||
"_id": ObjectId::parse_str("507f1f77bcf86cd799439011").unwrap(),
|
||||
"createdAt": DateTime::from_millis(1_609_459_200_000),
|
||||
"payload": Bson::Binary(Binary { subtype: BinarySubtype::Generic, bytes: vec![1, 2, 3] }),
|
||||
"nested": { "count": 42i64 },
|
||||
},
|
||||
doc! { "_id": 2i32, "name": "second" },
|
||||
];
|
||||
let bytes = bson_dump_bytes(&documents);
|
||||
let preview =
|
||||
preview_mongodb_import_bytes(&bytes, MongoImportFormat::Bson, &MongoImportParseOptions::default(), 10)
|
||||
.unwrap();
|
||||
assert_eq!(preview.rows.len(), 2);
|
||||
assert_eq!(preview.estimated_rows, Some(2));
|
||||
assert!(preview.estimated_rows_exact);
|
||||
assert_eq!(preview.rows[0]["_id"]["$oid"], "507f1f77bcf86cd799439011");
|
||||
assert!(preview.rows[0]["payload"].get("$binary").is_some());
|
||||
|
||||
let imported = execute_source(&bytes, "bson", MongoImportFormat::Bson, &MongoImportParseOptions::default());
|
||||
assert_eq!(imported, preview.rows);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gzip_bson_dump_import_matches_plain_dump() {
|
||||
let documents = vec![doc! { "_id": 1i32, "name": "Ada" }, doc! { "_id": 2i32, "name": "Bob" }];
|
||||
let plain = bson_dump_bytes(&documents);
|
||||
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
|
||||
encoder.write_all(&plain).unwrap();
|
||||
let compressed = encoder.finish().unwrap();
|
||||
|
||||
let imported =
|
||||
execute_source(&compressed, "bson.gz", MongoImportFormat::Bson, &MongoImportParseOptions::default());
|
||||
assert_eq!(imported.len(), 2);
|
||||
assert_eq!(imported[0]["name"], "Ada");
|
||||
assert_eq!(imported[1]["name"], "Bob");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bson_dump_rejects_truncated_and_invalid_lengths() {
|
||||
let mut truncated = mongodb::bson::to_vec(&doc! { "name": "Ada" }).unwrap();
|
||||
truncated.pop();
|
||||
let error =
|
||||
preview_mongodb_import_bytes(&truncated, MongoImportFormat::Bson, &MongoImportParseOptions::default(), 10)
|
||||
.unwrap_err();
|
||||
assert_eq!(error.code, "BSON_STRUCTURE");
|
||||
assert_eq!(error.row, Some(1));
|
||||
|
||||
let error = preview_mongodb_import_bytes(
|
||||
&(BSON_MAX_DOCUMENT_BYTES as i32 + 1).to_le_bytes(),
|
||||
MongoImportFormat::Bson,
|
||||
&MongoImportParseOptions::default(),
|
||||
10,
|
||||
)
|
||||
.unwrap_err();
|
||||
assert_eq!(error.code, "BSON_STRUCTURE");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bson_dump_format_is_detected_from_plain_and_gzip_paths() {
|
||||
assert_eq!(format_from_path("users.bson").unwrap(), MongoImportFormat::Bson);
|
||||
assert_eq!(format_from_path("users.BSON.GZ").unwrap(), MongoImportFormat::Bson);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn csv_quoted_comma_newline_and_escaped_quotes() {
|
||||
let csv = "name,note\n\"Ada, Lovelace\",\"line1\nline2\"\n\"quotes\",\"she said \"\"hi\"\"\"\n";
|
||||
@@ -3806,6 +4141,7 @@ def handle(request):
|
||||
collation: None,
|
||||
format: MongoExportFormat::Ndjson,
|
||||
include_header: true,
|
||||
gzip: false,
|
||||
file_path: target.to_string_lossy().to_string(),
|
||||
execution_id: None,
|
||||
},
|
||||
@@ -3857,6 +4193,7 @@ def handle(request):
|
||||
collation: None,
|
||||
format: MongoExportFormat::Ndjson,
|
||||
include_header: true,
|
||||
gzip: false,
|
||||
file_path: target.to_string_lossy().to_string(),
|
||||
execution_id: None,
|
||||
},
|
||||
@@ -3928,6 +4265,7 @@ def handle(request):
|
||||
collation: None,
|
||||
format: MongoExportFormat::Ndjson,
|
||||
include_header: true,
|
||||
gzip: false,
|
||||
file_path: target.to_string_lossy().to_string(),
|
||||
execution_id: None,
|
||||
},
|
||||
@@ -3979,6 +4317,7 @@ def handle(request):
|
||||
collation: None,
|
||||
format: MongoExportFormat::Ndjson,
|
||||
include_header: true,
|
||||
gzip: false,
|
||||
file_path: target.to_string_lossy().to_string(),
|
||||
execution_id: None,
|
||||
},
|
||||
@@ -4035,6 +4374,7 @@ def handle(request):
|
||||
collation: None,
|
||||
format: MongoExportFormat::Ndjson,
|
||||
include_header: true,
|
||||
gzip: false,
|
||||
file_path: target.to_string_lossy().to_string(),
|
||||
execution_id: None,
|
||||
},
|
||||
@@ -4085,6 +4425,7 @@ def handle(request):
|
||||
collation: None,
|
||||
format: MongoExportFormat::Ndjson,
|
||||
include_header: true,
|
||||
gzip: false,
|
||||
file_path: target.to_string_lossy().to_string(),
|
||||
execution_id: None,
|
||||
},
|
||||
|
||||
@@ -0,0 +1,822 @@
|
||||
use std::io::Write;
|
||||
use std::path::Path;
|
||||
|
||||
use dbx_core::mongodb_import_export::{
|
||||
for_each_mongodb_import_document, preview_mongodb_import_file, MongoImportFormat, MongoImportIssue,
|
||||
MongoImportParseOptions, MongoImportPreviewRequest, ParsedMongoDocument,
|
||||
};
|
||||
use flate2::{write::GzEncoder, Compression};
|
||||
use mongodb::bson::{doc, Bson, Document};
|
||||
|
||||
fn encode(documents: &[Document], gzip: bool) -> Vec<u8> {
|
||||
let bytes = documents.iter().flat_map(|document| mongodb::bson::to_vec(document).unwrap()).collect::<Vec<_>>();
|
||||
if !gzip {
|
||||
return bytes;
|
||||
}
|
||||
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
|
||||
encoder.write_all(&bytes).unwrap();
|
||||
encoder.finish().unwrap()
|
||||
}
|
||||
|
||||
fn read(path: &Path) -> Result<Vec<Document>, Box<MongoImportIssue>> {
|
||||
let mut documents = Vec::new();
|
||||
#[expect(clippy::result_large_err, reason = "The public import callback requires MongoImportIssue by value")]
|
||||
let collect = |parsed: Result<ParsedMongoDocument, MongoImportIssue>| {
|
||||
documents.push(parsed?.document);
|
||||
Ok(())
|
||||
};
|
||||
for_each_mongodb_import_document(
|
||||
path.to_str().unwrap(),
|
||||
MongoImportFormat::Bson,
|
||||
&MongoImportParseOptions::default(),
|
||||
collect,
|
||||
)
|
||||
.map_err(Box::new)?;
|
||||
Ok(documents)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bson_preview_is_bounded_and_preserves_explicit_format() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
// The selected format, not the filename, chooses the parser.
|
||||
let path = directory.path().join("renamed.json");
|
||||
let documents = (0..80).map(|index| doc! { "_id": index, "smallLong": 1i64 }).collect::<Vec<_>>();
|
||||
std::fs::write(&path, encode(&documents, false)).unwrap();
|
||||
let mut request = MongoImportPreviewRequest {
|
||||
file_path: path.to_str().unwrap().into(),
|
||||
source_ref: None,
|
||||
format: MongoImportFormat::Bson,
|
||||
parse_options: MongoImportParseOptions::default(),
|
||||
preview_limit: Some(3),
|
||||
};
|
||||
let preview = preview_mongodb_import_file(&request).unwrap();
|
||||
assert_eq!(preview.rows.len(), 3);
|
||||
assert!(!preview.estimated_rows_exact);
|
||||
assert_eq!(preview.detected_encoding, None);
|
||||
assert_eq!(preview.rows[0]["smallLong"]["$numberLong"], "1");
|
||||
assert_eq!(read(&path).unwrap(), documents);
|
||||
|
||||
std::fs::write(&path, []).unwrap();
|
||||
request.preview_limit = Some(1);
|
||||
let preview = preview_mongodb_import_file(&request).unwrap();
|
||||
assert_eq!(preview.estimated_rows, Some(0));
|
||||
assert!(preview.estimated_rows_exact);
|
||||
assert!(read(&path).unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gzip_reads_all_members_and_rejects_corrupt_or_truncated_streams() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let path = directory.path().join("records.bson.gz");
|
||||
let first = doc! { "_id": 1 };
|
||||
let second = doc! { "_id": 2 };
|
||||
let mut bytes = encode(std::slice::from_ref(&first), true);
|
||||
bytes.extend(encode(std::slice::from_ref(&second), true));
|
||||
std::fs::write(&path, &bytes).unwrap();
|
||||
assert_eq!(read(&path).unwrap(), vec![first.clone(), second]);
|
||||
|
||||
let valid = encode(&[first], true);
|
||||
for length in [0, 1, valid.len() - 1, valid.len() - 8] {
|
||||
std::fs::write(&path, &valid[..length]).unwrap();
|
||||
assert!(read(&path).is_err(), "accepted truncated gzip at {length}");
|
||||
}
|
||||
let mut corrupt = valid.clone();
|
||||
let crc_offset = corrupt.len() - 8;
|
||||
corrupt[crc_offset] ^= 1;
|
||||
std::fs::write(&path, corrupt).unwrap();
|
||||
assert!(read(&path).is_err(), "accepted invalid gzip checksum");
|
||||
let mut trailing = valid;
|
||||
trailing.extend_from_slice(b"junk");
|
||||
std::fs::write(&path, trailing).unwrap();
|
||||
assert!(read(&path).is_err(), "ignored trailing compressed data");
|
||||
|
||||
std::fs::write(&path, encode(&[], true)).unwrap();
|
||||
assert!(read(&path).unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_bson_is_fatal_even_when_callback_skips_errors() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let path = directory.path().join("records.bson");
|
||||
let prefix = encode(&[doc! { "_id": 1 }], false);
|
||||
for tail in [
|
||||
vec![1],
|
||||
vec![5, 0, 0],
|
||||
4i32.to_le_bytes().to_vec(),
|
||||
(-1i32).to_le_bytes().to_vec(),
|
||||
(16 * 1024 * 1024 + 1i32).to_le_bytes().to_vec(),
|
||||
vec![5, 0, 0, 0, 1],
|
||||
] {
|
||||
let mut bytes = prefix.clone();
|
||||
bytes.extend(tail);
|
||||
std::fs::write(&path, bytes).unwrap();
|
||||
#[expect(clippy::result_large_err, reason = "The public import callback requires MongoImportIssue by value")]
|
||||
let ignore = |_: Result<ParsedMongoDocument, MongoImportIssue>| Ok(());
|
||||
let error = for_each_mongodb_import_document(
|
||||
path.to_str().unwrap(),
|
||||
MongoImportFormat::Bson,
|
||||
&MongoImportParseOptions { skip_error_rows: Some(true), ..Default::default() },
|
||||
ignore,
|
||||
)
|
||||
.unwrap_err();
|
||||
assert_eq!(error.code, "BSON_STRUCTURE");
|
||||
assert_eq!(error.row, Some(2));
|
||||
}
|
||||
}
|
||||
|
||||
fn run_tool(tools: &Path, name: &str, args: &[&str]) {
|
||||
let logs = tempfile::tempdir().unwrap();
|
||||
let log = logs.path().join("stderr.log");
|
||||
let mut child = std::process::Command::new(tools.join(format!("{name}{}", std::env::consts::EXE_SUFFIX)))
|
||||
.args(args)
|
||||
.stdout(std::process::Stdio::null())
|
||||
.stderr(std::fs::File::create(&log).unwrap())
|
||||
.spawn()
|
||||
.unwrap_or_else(|error| panic!("cannot run {name}: {error}"));
|
||||
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60);
|
||||
loop {
|
||||
if let Some(status) = child.try_wait().unwrap() {
|
||||
assert!(status.success(), "{name}: {}", std::fs::read_to_string(&log).unwrap());
|
||||
return;
|
||||
}
|
||||
if std::time::Instant::now() >= deadline {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
panic!("{name} timed out: {}", std::fs::read_to_string(&log).unwrap());
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(10));
|
||||
}
|
||||
}
|
||||
|
||||
async fn collection_bytes(client: &mongodb::Client, database: &str, collection: &str) -> Vec<Vec<u8>> {
|
||||
use futures::TryStreamExt;
|
||||
client
|
||||
.database(database)
|
||||
.collection::<Document>(collection)
|
||||
.find(doc! {})
|
||||
.sort(doc! { "_id": 1 })
|
||||
.await
|
||||
.unwrap()
|
||||
.try_collect::<Vec<_>>()
|
||||
.await
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|document| mongodb::bson::to_vec(document).unwrap())
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "opt-in: DBX_MONGO_DUMP_TEST_URI and DBX_MONGO_TOOLS_DIR; creates a temporary database"]
|
||||
async fn official_tools_round_trip_through_dbx_core() {
|
||||
use dbx_core::connection::{AppState, PoolKind};
|
||||
use dbx_core::models::connection::ConnectionConfig;
|
||||
use dbx_core::mongodb_import_export::{
|
||||
export_mongodb_query_core, import_mongodb_file_core, MongoExportFormat, MongoExportRequest, MongoExportStatus,
|
||||
MongoImportRequest, MongoImportStatus,
|
||||
};
|
||||
use dbx_core::storage::Storage;
|
||||
use mongodb::bson::{oid::ObjectId, spec::BinarySubtype, Binary, DateTime, Decimal128, Regex, Timestamp};
|
||||
use std::sync::{
|
||||
atomic::{AtomicBool, Ordering},
|
||||
Arc,
|
||||
};
|
||||
|
||||
let uri = std::env::var("DBX_MONGO_DUMP_TEST_URI").expect("DBX_MONGO_DUMP_TEST_URI");
|
||||
let tools = std::path::PathBuf::from(std::env::var("DBX_MONGO_TOOLS_DIR").expect("DBX_MONGO_TOOLS_DIR"));
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let client = mongodb::Client::with_uri_str(&uri).await.unwrap();
|
||||
let database = format!("dbx_dump_test_{}", uuid::Uuid::new_v4().simple());
|
||||
let storage = Storage::open(&directory.path().join("storage.db")).await.unwrap();
|
||||
let state = AppState::new(storage);
|
||||
let connection_id = "mongo-dump-test";
|
||||
let config: ConnectionConfig = serde_json::from_value(serde_json::json!({
|
||||
"id": connection_id, "name": "Dump test", "db_type": "mongodb", "host": "127.0.0.1",
|
||||
"port": 27090, "username": "", "password": "", "database": database,
|
||||
"connection_string": uri, "driver_profile": "mongodb-native"
|
||||
}))
|
||||
.unwrap();
|
||||
state.configs.write().await.insert(connection_id.into(), config);
|
||||
state
|
||||
.update_connection_pools(|pools| {
|
||||
pools.insert(connection_id.into(), PoolKind::MongoDb(client.clone()));
|
||||
})
|
||||
.await;
|
||||
let documents = (0..1207)
|
||||
.map(|index| {
|
||||
doc! {
|
||||
"_id": index,
|
||||
"objectId": ObjectId::parse_str("507f1f77bcf86cd799439011").unwrap(),
|
||||
"smallLong": 1i64, "maxLong": i64::MAX, "double": 1.5f64,
|
||||
"decimal": "123.45".parse::<Decimal128>().unwrap(),
|
||||
"date": DateTime::from_millis(-1234567890),
|
||||
"binary": Binary { subtype: BinarySubtype::UserDefined(0x80), bytes: vec![0, 1, 255] },
|
||||
"regex": Regex { pattern: "^test".into(), options: "im".into() },
|
||||
"timestamp": Timestamp { time: 12345, increment: 42 },
|
||||
"min": Bson::MinKey, "max": Bson::MaxKey, "undefined": Bson::Undefined,
|
||||
"code": Bson::JavaScriptCode("return 1".into()),
|
||||
"nested": { "array": [Bson::Null, Bson::Boolean(true), Bson::String("1".into())] },
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
client.database(&database).collection("source").insert_many(documents).await.unwrap();
|
||||
let expected = collection_bytes(&client, &database, "source").await;
|
||||
assert_eq!(expected.len(), 1207);
|
||||
|
||||
for gzip in [false, true] {
|
||||
let extension = if gzip { "bson.gz" } else { "bson" };
|
||||
let dump_dir = directory.path().join(if gzip { "official-gzip" } else { "official-plain" });
|
||||
let mut args = vec![
|
||||
"--uri",
|
||||
uri.as_str(),
|
||||
"--db",
|
||||
&database,
|
||||
"--collection",
|
||||
"source",
|
||||
"--out",
|
||||
dump_dir.to_str().unwrap(),
|
||||
];
|
||||
if gzip {
|
||||
args.push("--gzip");
|
||||
}
|
||||
run_tool(&tools, "mongodump", &args);
|
||||
let dumped = dump_dir.join(&database).join(format!("source.{extension}"));
|
||||
assert_eq!(read(&dumped).unwrap().len(), expected.len());
|
||||
let preview = preview_mongodb_import_file(&MongoImportPreviewRequest {
|
||||
file_path: dumped.to_str().unwrap().into(),
|
||||
source_ref: None,
|
||||
format: MongoImportFormat::Bson,
|
||||
parse_options: Default::default(),
|
||||
preview_limit: Some(5),
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(preview.rows.len(), 5);
|
||||
|
||||
let imported = if gzip { "dbx_gzip" } else { "dbx_plain" };
|
||||
let request = MongoImportRequest {
|
||||
import_id: uuid::Uuid::new_v4().to_string(),
|
||||
connection_id: connection_id.into(),
|
||||
database: database.clone(),
|
||||
collection: imported.into(),
|
||||
file_path: dumped.to_str().unwrap().into(),
|
||||
source_ref: None,
|
||||
format: MongoImportFormat::Bson,
|
||||
parse_options: Default::default(),
|
||||
batch_size: 500,
|
||||
execution_id: None,
|
||||
};
|
||||
let summary = import_mongodb_file_core(&state, &request, |_| Box::pin(async { false }), |_| {}).await.unwrap();
|
||||
assert_eq!(summary.rows_inserted, 1207);
|
||||
assert_eq!(summary.rows_failed, 0);
|
||||
assert_eq!(summary.batches_committed, 3);
|
||||
assert_eq!(collection_bytes(&client, &database, imported).await, expected);
|
||||
|
||||
// This file is produced by DBX, with no official metadata sidecar beside it.
|
||||
let exported = directory.path().join(format!("dbx-output.{extension}"));
|
||||
let export = MongoExportRequest {
|
||||
export_id: uuid::Uuid::new_v4().to_string(),
|
||||
connection_id: connection_id.into(),
|
||||
database: database.clone(),
|
||||
collection: imported.into(),
|
||||
filter: None,
|
||||
sort: Some("{\"_id\":1}".into()),
|
||||
projection: None,
|
||||
collation: None,
|
||||
format: MongoExportFormat::Bson,
|
||||
include_header: false,
|
||||
gzip,
|
||||
file_path: exported.to_str().unwrap().into(),
|
||||
execution_id: None,
|
||||
};
|
||||
let summary = export_mongodb_query_core(&state, &export, |_| Box::pin(async { false }), |_| {}).await.unwrap();
|
||||
assert_eq!(summary.documents_exported, 1207);
|
||||
assert_eq!(
|
||||
read(&exported).unwrap().iter().map(|d| mongodb::bson::to_vec(d).unwrap()).collect::<Vec<_>>(),
|
||||
expected
|
||||
);
|
||||
let restored = if gzip { "official_gzip" } else { "official_plain" };
|
||||
let mut args = vec!["--uri", uri.as_str(), "--db", &database, "--collection", restored];
|
||||
if gzip {
|
||||
args.push("--gzip");
|
||||
}
|
||||
args.push(exported.to_str().unwrap());
|
||||
run_tool(&tools, "mongorestore", &args);
|
||||
assert_eq!(collection_bytes(&client, &database, restored).await, expected);
|
||||
println!("{extension}: official -> DBX -> official, 1207 documents, BSON bytes match");
|
||||
|
||||
let mut empty_export = export.clone();
|
||||
empty_export.collection = "empty".into();
|
||||
empty_export.file_path = directory.path().join(format!("empty.{extension}")).to_str().unwrap().into();
|
||||
let empty =
|
||||
export_mongodb_query_core(&state, &empty_export, |_| Box::pin(async { false }), |_| {}).await.unwrap();
|
||||
assert_eq!(empty.documents_exported, 0);
|
||||
assert!(read(Path::new(&empty.file_path)).unwrap().is_empty());
|
||||
|
||||
let cancelled = Arc::new(AtomicBool::new(false));
|
||||
let check = cancelled.clone();
|
||||
let original = std::fs::read(&exported).unwrap();
|
||||
let result = export_mongodb_query_core(
|
||||
&state,
|
||||
&export,
|
||||
move |_| {
|
||||
let check = check.clone();
|
||||
Box::pin(async move { check.load(Ordering::SeqCst) })
|
||||
},
|
||||
|progress| {
|
||||
if progress.status == MongoExportStatus::Running && progress.documents_read > 0 {
|
||||
cancelled.store(true, Ordering::SeqCst);
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert_eq!(result.unwrap_err(), "Export cancelled");
|
||||
assert_eq!(std::fs::read(&exported).unwrap(), original, "cancel replaced the previous export");
|
||||
|
||||
let damaged = directory.path().join("damaged.bson");
|
||||
std::fs::write(&damaged, [1]).unwrap();
|
||||
let mut request = request;
|
||||
request.file_path = damaged.to_str().unwrap().into();
|
||||
request.parse_options.skip_error_rows = Some(true);
|
||||
let mut status = MongoImportStatus::Running;
|
||||
let result =
|
||||
import_mongodb_file_core(&state, &request, |_| Box::pin(async { false }), |p| status = p.status).await;
|
||||
assert!(result.is_err());
|
||||
assert_eq!(status, MongoImportStatus::Error);
|
||||
}
|
||||
client.database(&database).drop().await.unwrap();
|
||||
}
|
||||
|
||||
async fn specs(client: &mongodb::Client, db: &str, collection: &str) -> (Document, Vec<Document>) {
|
||||
use futures::TryStreamExt;
|
||||
let specification = client
|
||||
.database(db)
|
||||
.list_collections()
|
||||
.filter(doc! { "name": collection })
|
||||
.await
|
||||
.unwrap()
|
||||
.try_next()
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let options = mongodb::bson::to_document(&specification.options).unwrap();
|
||||
let mut indexes = Vec::new();
|
||||
if !matches!(specification.collection_type, mongodb::results::CollectionType::View) {
|
||||
let mut cursor = client.database(db).collection::<Document>(collection).list_indexes().await.unwrap();
|
||||
while let Some(index) = cursor.try_next().await.unwrap() {
|
||||
let mut index = mongodb::bson::to_document(&index).unwrap();
|
||||
index.remove("ns");
|
||||
index.remove("v");
|
||||
indexes.push(index);
|
||||
}
|
||||
indexes.sort_by_key(|index| index.get_str("name").unwrap().to_string());
|
||||
}
|
||||
(options, indexes)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "opt-in: DBX_MONGO_DUMP_TEST_URI and DBX_MONGO_TOOLS_DIR; creates temporary databases"]
|
||||
async fn official_database_tools_round_trip_through_dbx() {
|
||||
use dbx_core::{
|
||||
connection::{AppState, PoolKind},
|
||||
models::connection::ConnectionConfig,
|
||||
mongodb_dump::*,
|
||||
storage::Storage,
|
||||
};
|
||||
let uri = std::env::var("DBX_MONGO_DUMP_TEST_URI").expect("DBX_MONGO_DUMP_TEST_URI");
|
||||
let tools = std::path::PathBuf::from(std::env::var("DBX_MONGO_TOOLS_DIR").expect("DBX_MONGO_TOOLS_DIR"));
|
||||
let files = tempfile::tempdir().unwrap();
|
||||
let client = mongodb::Client::with_uri_str(&uri).await.unwrap();
|
||||
let database = format!("dbx_db_{}", uuid::Uuid::new_v4().simple());
|
||||
let db = client.database(&database);
|
||||
db.run_command(doc! { "create": "records", "validator": { "score": { "$gte": 0 } }, "validationLevel": "strict", "collation": { "locale": "en", "strength": 2 } }).await.unwrap();
|
||||
db.collection("records")
|
||||
.insert_many(vec![
|
||||
doc! { "_id": 1, "score": 1i64, "name": "first" },
|
||||
doc! { "_id": 2, "score": 2i64, "name": "second" },
|
||||
])
|
||||
.await
|
||||
.unwrap();
|
||||
db.run_command(doc! { "createIndexes": "records", "indexes": [ { "key": { "name": 1, "score": -1 }, "name": "compound_unique", "unique": true }, { "key": { "expires": 1 }, "name": "ttl", "expireAfterSeconds": 3600 }] }).await.unwrap();
|
||||
db.create_collection("empty").await.unwrap();
|
||||
db.run_command(doc! { "create": "capped", "capped": true, "size": 65536i64, "max": 100i64 }).await.unwrap();
|
||||
db.collection("capped").insert_one(doc! { "_id": 1, "n": 1 }).await.unwrap();
|
||||
db.collection("odd/name").insert_one(doc! { "_id": 1, "binary": mongodb::bson::Binary { subtype: mongodb::bson::spec::BinarySubtype::Generic, bytes: vec![1, 2, 3] } }).await.unwrap();
|
||||
db.run_command(doc! { "create": "z_view", "viewOn": "records", "pipeline": [ { "$match": { "score": { "$gte": 1 } } } ], "collation": { "locale": "en", "strength": 2 } }).await.unwrap();
|
||||
db.run_command(doc! { "create": "a_nested", "viewOn": "z_view", "pipeline": [], "collation": { "locale": "en", "strength": 2 } }).await.unwrap();
|
||||
let collections = ["records", "empty", "capped", "odd/name", "z_view", "a_nested"];
|
||||
let state = AppState::new(Storage::open(&files.path().join("storage.db")).await.unwrap());
|
||||
let connection_id = "database-dump-test";
|
||||
let config: ConnectionConfig = serde_json::from_value(serde_json::json!({ "id": connection_id, "name": "Database dump test", "db_type": "mongodb", "host": "127.0.0.1", "port": 27090, "username": "", "password": "", "database": database, "connection_string": uri, "driver_profile": "mongodb-native" })).unwrap();
|
||||
state.configs.write().await.insert(connection_id.into(), config);
|
||||
state
|
||||
.update_connection_pools(|pools| {
|
||||
pools.insert(connection_id.into(), PoolKind::MongoDb(client.clone()));
|
||||
})
|
||||
.await;
|
||||
let mut databases = vec![database.clone()];
|
||||
for format in [MongoDumpFormat::Directory, MongoDumpFormat::Archive] {
|
||||
for gzip in [false, true] {
|
||||
let tag = format!("{format:?}_{gzip}");
|
||||
let official_path = files.path().join(format!("official-{tag}"));
|
||||
let archive_arg = format!("--archive={}", official_path.display());
|
||||
let mut args = vec!["--uri", uri.as_str(), "--db", &database];
|
||||
if format == MongoDumpFormat::Archive {
|
||||
args.push(&archive_arg);
|
||||
} else {
|
||||
args.extend(["--out", official_path.to_str().unwrap()]);
|
||||
}
|
||||
if gzip {
|
||||
args.push("--gzip");
|
||||
}
|
||||
run_tool(&tools, "mongodump", &args);
|
||||
let preview = prepare_mongodb_restore_source(MongoRestoreSourceRequest {
|
||||
path: official_path.to_str().unwrap().into(),
|
||||
format,
|
||||
gzip,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(preview.catalog.collections.len(), 6);
|
||||
assert!(preview.catalog.collections.iter().all(|c| c.documents.is_none()));
|
||||
let restored = format!("{database}_{tag}");
|
||||
databases.push(restored.clone());
|
||||
let request = MongoDatabaseRestoreRequest {
|
||||
task_id: uuid::Uuid::new_v4().to_string(),
|
||||
connection_id: connection_id.into(),
|
||||
database: restored.clone(),
|
||||
source_database: database.clone(),
|
||||
source_ref: preview.source_ref.clone(),
|
||||
collections: None,
|
||||
drop_existing: false,
|
||||
restore_options: true,
|
||||
restore_indexes: true,
|
||||
stop_on_error: true,
|
||||
objcheck: gzip,
|
||||
batch_size: 500,
|
||||
execution_id: None,
|
||||
};
|
||||
let mut saw_data = false;
|
||||
let result = restore_mongodb_database(
|
||||
&state,
|
||||
&request,
|
||||
|_| Box::pin(async { false }),
|
||||
|p| {
|
||||
assert_ne!(p.phase, "validating");
|
||||
saw_data |= p.phase == "data";
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(saw_data);
|
||||
assert_eq!(result.documents_validated, if gzip { 4 } else { 0 });
|
||||
assert_eq!(result.collections_done, 6);
|
||||
assert_eq!(result.documents_written, 4);
|
||||
for collection in collections {
|
||||
assert_eq!(
|
||||
collection_bytes(&client, &database, collection).await,
|
||||
collection_bytes(&client, &restored, collection).await,
|
||||
"data: {tag} {collection}"
|
||||
);
|
||||
assert_eq!(
|
||||
specs(&client, &database, collection).await,
|
||||
specs(&client, &restored, collection).await,
|
||||
"metadata: {tag} {collection}"
|
||||
);
|
||||
}
|
||||
client.database(&restored).collection("unselected").insert_one(doc! { "_id": "keep" }).await.unwrap();
|
||||
client
|
||||
.database(&restored)
|
||||
.collection("records")
|
||||
.insert_one(doc! { "_id": 999, "score": 999, "name": "remove" })
|
||||
.await
|
||||
.unwrap();
|
||||
let mut selected = request.clone();
|
||||
selected.drop_existing = true;
|
||||
selected.collections = Some(vec!["records".into()]);
|
||||
restore_mongodb_database(&state, &selected, |_| Box::pin(async { false }), |_| {}).await.unwrap();
|
||||
assert_eq!(
|
||||
collection_bytes(&client, &database, "records").await,
|
||||
collection_bytes(&client, &restored, "records").await
|
||||
);
|
||||
assert_eq!(collection_bytes(&client, &restored, "unselected").await.len(), 1);
|
||||
if format == MongoDumpFormat::Directory && !gzip {
|
||||
let cancel_dir = files.path().join("cancel_source");
|
||||
std::fs::create_dir(&cancel_dir).unwrap();
|
||||
let documents = (0..201).map(|id| doc! { "_id": id }).collect::<Vec<_>>();
|
||||
std::fs::write(cancel_dir.join("records.bson"), encode(&documents, false)).unwrap();
|
||||
let cancel_source = prepare_mongodb_restore_source(MongoRestoreSourceRequest {
|
||||
path: cancel_dir.to_str().unwrap().into(),
|
||||
format: MongoDumpFormat::Directory,
|
||||
gzip: false,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let cancel = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let signal = cancel.clone();
|
||||
let mut cancel_request = selected.clone();
|
||||
cancel_request.source_ref = cancel_source.source_ref.clone();
|
||||
cancel_request.source_database = "cancel_source".into();
|
||||
cancel_request.batch_size = 100;
|
||||
let mut final_progress = None;
|
||||
let cancelled = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(10),
|
||||
restore_mongodb_database(
|
||||
&state,
|
||||
&cancel_request,
|
||||
|_| {
|
||||
let signal = signal.clone();
|
||||
Box::pin(async move { signal.load(std::sync::atomic::Ordering::Relaxed) })
|
||||
},
|
||||
|p| {
|
||||
if p.documents_written >= 100 {
|
||||
cancel.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
final_progress = Some(p);
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("cancellation must stop and join the reader")
|
||||
.unwrap_err();
|
||||
assert!(cancelled.contains("cancelled"), "{cancelled}");
|
||||
let final_progress = final_progress.unwrap();
|
||||
assert_eq!(final_progress.status, "cancelled");
|
||||
assert_eq!(final_progress.documents_written, 100);
|
||||
assert_eq!(collection_bytes(&client, &restored, "records").await.len(), 100);
|
||||
let mut duplicate_request = cancel_request.clone();
|
||||
duplicate_request.batch_size = 500;
|
||||
duplicate_request.drop_existing = false;
|
||||
let mut partial_progress = None;
|
||||
let error = restore_mongodb_database(
|
||||
&state,
|
||||
&duplicate_request,
|
||||
|_| Box::pin(async { false }),
|
||||
|p| partial_progress = Some(p),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(error.contains("duplicate key"), "{error}");
|
||||
let partial_progress = partial_progress.unwrap();
|
||||
assert_eq!(partial_progress.documents_written, 101);
|
||||
assert_eq!(partial_progress.documents_failed, 100);
|
||||
duplicate_request.stop_on_error = false;
|
||||
let continued =
|
||||
restore_mongodb_database(&state, &duplicate_request, |_| Box::pin(async { false }), |_| {})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(continued.documents_written, 0);
|
||||
assert_eq!(continued.documents_failed, 201);
|
||||
release_mongodb_restore_source(&cancel_source.source_ref);
|
||||
let broken_dir = files.path().join("broken_source");
|
||||
std::fs::create_dir(&broken_dir).unwrap();
|
||||
std::fs::write(broken_dir.join("records.bson"), encode(&[doc! { "_id": "replacement" }], false))
|
||||
.unwrap();
|
||||
std::fs::write(broken_dir.join("zz_broken.bson"), b"invalid BSON").unwrap();
|
||||
let broken = prepare_mongodb_restore_source(MongoRestoreSourceRequest {
|
||||
path: broken_dir.to_str().unwrap().into(),
|
||||
format,
|
||||
gzip,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let mut invalid = selected.clone();
|
||||
invalid.source_ref = broken.source_ref.clone();
|
||||
invalid.source_database = "broken_source".into();
|
||||
invalid.collections = None;
|
||||
let mut partial = 0;
|
||||
let error = restore_mongodb_database(&state, &invalid, |_| Box::pin(async { false }), |_| {})
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(error.contains("BSON"), "{error}");
|
||||
assert_eq!(
|
||||
collection_bytes(&client, &restored, "records").await,
|
||||
vec![mongodb::bson::to_vec(&doc! { "_id": "replacement" }).unwrap()],
|
||||
"valid earlier collections are restored without a full pre-scan"
|
||||
);
|
||||
invalid.objcheck = true;
|
||||
assert!(restore_mongodb_database(
|
||||
&state,
|
||||
&invalid,
|
||||
|_| Box::pin(async { false }),
|
||||
|p| partial = p.documents_written
|
||||
)
|
||||
.await
|
||||
.is_err());
|
||||
assert_eq!(partial, 1);
|
||||
release_mongodb_restore_source(&broken.source_ref);
|
||||
let mut no_metadata = request.clone();
|
||||
no_metadata.database = format!("{database}_no_options");
|
||||
databases.push(no_metadata.database.clone());
|
||||
no_metadata.collections = Some(vec!["records".into()]);
|
||||
no_metadata.restore_options = false;
|
||||
no_metadata.restore_indexes = false;
|
||||
restore_mongodb_database(&state, &no_metadata, |_| Box::pin(async { false }), |_| {}).await.unwrap();
|
||||
let (options, indexes) = specs(&client, &no_metadata.database, "records").await;
|
||||
assert!(options.is_empty());
|
||||
assert_eq!(indexes.len(), 1);
|
||||
state.configs.write().await.get_mut(connection_id).unwrap().read_only = true;
|
||||
let blocked = restore_mongodb_database(&state, &no_metadata, |_| Box::pin(async { false }), |_| {})
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(blocked.contains("Read-only"));
|
||||
state.configs.write().await.get_mut(connection_id).unwrap().read_only = false;
|
||||
let cancelled = restore_mongodb_database(&state, &no_metadata, |_| Box::pin(async { true }), |_| {})
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(cancelled.contains("cancelled"));
|
||||
}
|
||||
assert!(release_mongodb_restore_source(&preview.source_ref));
|
||||
|
||||
let output = files.path().join(format!("dbx-{tag}"));
|
||||
let dump = MongoDatabaseDumpRequest {
|
||||
task_id: uuid::Uuid::new_v4().to_string(),
|
||||
connection_id: connection_id.into(),
|
||||
database: database.clone(),
|
||||
file_path: output.to_str().unwrap().into(),
|
||||
format,
|
||||
gzip,
|
||||
collections: None,
|
||||
};
|
||||
dump_mongodb_database(&state, &dump, |_| Box::pin(async { false }), |_| {}).await.unwrap();
|
||||
let target = format!("{database}_out_{tag}");
|
||||
databases.push(target.clone());
|
||||
let from = format!("{database}.*");
|
||||
let to = format!("{target}.*");
|
||||
let archive_arg = format!("--archive={}", output.display());
|
||||
let mut args = vec!["--uri", uri.as_str(), "--nsFrom", &from, "--nsTo", &to];
|
||||
if gzip {
|
||||
args.push("--gzip");
|
||||
}
|
||||
if format == MongoDumpFormat::Archive {
|
||||
args.push(&archive_arg);
|
||||
} else {
|
||||
args.push(output.to_str().unwrap());
|
||||
}
|
||||
run_tool(&tools, "mongorestore", &args);
|
||||
for collection in collections {
|
||||
assert_eq!(
|
||||
collection_bytes(&client, &database, collection).await,
|
||||
collection_bytes(&client, &target, collection).await,
|
||||
"exported data: {tag} {collection}"
|
||||
);
|
||||
assert_eq!(
|
||||
specs(&client, &database, collection).await,
|
||||
specs(&client, &target, collection).await,
|
||||
"exported metadata: {tag} {collection}"
|
||||
);
|
||||
}
|
||||
println!("{tag}: six collections/views, BSON, options and indexes match in both directions");
|
||||
if format == MongoDumpFormat::Archive && !gzip {
|
||||
let mut bytes = std::fs::read(&output).unwrap();
|
||||
let offset = bytes.windows(5).position(|bytes| bytes == b"first").unwrap();
|
||||
bytes[offset] = b'F';
|
||||
let corrupt = files.path().join("corrupt.archive");
|
||||
std::fs::write(&corrupt, &bytes).unwrap();
|
||||
let corrupt_preview = prepare_mongodb_restore_source(MongoRestoreSourceRequest {
|
||||
path: corrupt.to_str().unwrap().into(),
|
||||
format,
|
||||
gzip: false,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let mut corrupt_request = request.clone();
|
||||
corrupt_request.source_ref = corrupt_preview.source_ref.clone();
|
||||
corrupt_request.database = target.clone();
|
||||
corrupt_request.drop_existing = true;
|
||||
let mut partial = 0;
|
||||
let error = restore_mongodb_database(
|
||||
&state,
|
||||
&corrupt_request,
|
||||
|_| Box::pin(async { false }),
|
||||
|p| partial = p.documents_written,
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(error.contains("checksum"), "{error}");
|
||||
assert!(partial > 0, "archive CRC is checked as data streams, not in a separate pre-pass");
|
||||
release_mongodb_restore_source(&corrupt_preview.source_ref);
|
||||
bytes = std::fs::read(&output).unwrap();
|
||||
bytes.truncate(bytes.len() - 1);
|
||||
std::fs::write(&corrupt, bytes).unwrap();
|
||||
let truncated = prepare_mongodb_restore_source(MongoRestoreSourceRequest {
|
||||
path: corrupt.to_str().unwrap().into(),
|
||||
format,
|
||||
gzip: false,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
corrupt_request.source_ref = truncated.source_ref.clone();
|
||||
assert!(restore_mongodb_database(&state, &corrupt_request, |_| Box::pin(async { false }), |_| {})
|
||||
.await
|
||||
.is_err());
|
||||
assert_eq!(
|
||||
collection_bytes(&client, &database, "records").await,
|
||||
collection_bytes(&client, &target, "records").await
|
||||
);
|
||||
release_mongodb_restore_source(&truncated.source_ref);
|
||||
}
|
||||
}
|
||||
}
|
||||
for database in databases {
|
||||
client.database(&database).drop().await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn database_source_rejects_invalid_metadata_and_view_cycles() {
|
||||
use dbx_core::mongodb_dump::*;
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let source = directory.path().join("source");
|
||||
std::fs::create_dir(&source).unwrap();
|
||||
std::fs::write(source.join("records.bson"), []).unwrap();
|
||||
let request = MongoRestoreSourceRequest {
|
||||
path: source.to_str().unwrap().into(),
|
||||
format: MongoDumpFormat::Directory,
|
||||
gzip: false,
|
||||
};
|
||||
for metadata in [
|
||||
r#"{"collectionName":"records","options":{"$db":"outside"},"indexes":[]}"#,
|
||||
r#"{"collectionName":"records","options":{},"indexes":[{"name":"bad"}]}"#,
|
||||
r#"{"collectionName":"records","type":"timeseries","options":{},"indexes":[]}"#,
|
||||
] {
|
||||
std::fs::write(source.join("records.metadata.json"), metadata).unwrap();
|
||||
assert!(prepare_mongodb_restore_source(request.clone()).await.is_err());
|
||||
}
|
||||
std::fs::write(source.join("records.metadata.json"), r#"{"collectionName":"records","options":{},"indexes":[]}"#)
|
||||
.unwrap();
|
||||
std::fs::write(
|
||||
source.join("one.metadata.json"),
|
||||
r#"{"collectionName":"one","type":"view","options":{"viewOn":"two","pipeline":[]},"indexes":[]}"#,
|
||||
)
|
||||
.unwrap();
|
||||
std::fs::write(
|
||||
source.join("two.metadata.json"),
|
||||
r#"{"collectionName":"two","type":"view","options":{"viewOn":"one","pipeline":[]},"indexes":[]}"#,
|
||||
)
|
||||
.unwrap();
|
||||
let error = prepare_mongodb_restore_source(request).await.unwrap_err();
|
||||
assert!(error.contains("Cyclic"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn directory_preview_does_not_read_bson_and_source_identity_is_checked_before_restore() {
|
||||
use dbx_core::{connection::AppState, mongodb_dump::*, storage::Storage};
|
||||
let files = tempfile::tempdir().unwrap();
|
||||
let source = files.path().join("source");
|
||||
std::fs::create_dir(&source).unwrap();
|
||||
// Preview must succeed without opening/decoding this invalid gzip data.
|
||||
std::fs::write(source.join("records.bson.gz"), b"not gzip or BSON").unwrap();
|
||||
let preview = prepare_mongodb_restore_source(MongoRestoreSourceRequest {
|
||||
path: source.to_str().unwrap().into(),
|
||||
format: MongoDumpFormat::Directory,
|
||||
gzip: true,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(preview.catalog.collections[0].documents, None);
|
||||
assert_eq!(preview.catalog.collections[0].size_bytes, 16);
|
||||
let state = AppState::new(Storage::open(&files.path().join("state.db")).await.unwrap());
|
||||
let request: MongoDatabaseRestoreRequest = serde_json::from_value(serde_json::json!({
|
||||
"taskId":"validation-test", "connectionId":"missing", "database":"target", "sourceDatabase":"source",
|
||||
"sourceRef":preview.source_ref, "dropExisting":true
|
||||
}))
|
||||
.unwrap();
|
||||
let error = restore_mongodb_database(&state, &request, |_| Box::pin(async { false }), |_| {}).await.unwrap_err();
|
||||
assert!(
|
||||
!error.to_lowercase().contains("gzip"),
|
||||
"no pre-scan should run before the missing connection fails: {error}"
|
||||
);
|
||||
assert!(!request.objcheck, "objcheck is opt-in");
|
||||
std::fs::write(source.join("records.bson.gz"), []).unwrap();
|
||||
let error = restore_mongodb_database(&state, &request, |_| Box::pin(async { false }), |_| {}).await.unwrap_err();
|
||||
assert!(error.contains("changed"), "{error}");
|
||||
release_mongodb_restore_source(&preview.source_ref);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn directory_catalog_accepts_large_manifest_without_data_and_binds_only_selected_upload() {
|
||||
use dbx_core::mongodb_dump::*;
|
||||
let owner = tempfile::tempdir().unwrap();
|
||||
let manifest = vec![
|
||||
MongoDumpFile { path: "source/selected.bson".into(), size_bytes: 0 },
|
||||
MongoDumpFile { path: "source/huge.bson".into(), size_bytes: 5 * 1024 * 1024 * 1024 },
|
||||
];
|
||||
let preview = prepare_mongodb_directory_catalog(owner, manifest, false).await.unwrap();
|
||||
assert_eq!(preview.catalog.collections.len(), 2);
|
||||
assert!(preview.catalog.collections.iter().all(|c| c.documents.is_none()));
|
||||
let request: MongoDatabaseRestoreRequest = serde_json::from_value(serde_json::json!({
|
||||
"taskId":"upload-test", "connectionId":"missing", "database":"target", "sourceDatabase":"source",
|
||||
"sourceRef":preview.source_ref, "collections":["selected"]
|
||||
}))
|
||||
.unwrap();
|
||||
let upload = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir(upload.path().join("source")).unwrap();
|
||||
std::fs::write(upload.path().join("source/selected.bson"), []).unwrap();
|
||||
let acquired = attach_mongodb_restore_upload(&request, upload).await.unwrap();
|
||||
assert_eq!(acquired.catalog.collections.len(), 1);
|
||||
release_mongodb_restore_source(&acquired.source_ref);
|
||||
let changed = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir(changed.path().join("source")).unwrap();
|
||||
std::fs::write(changed.path().join("source/selected.bson"), b"changed").unwrap();
|
||||
assert!(attach_mongodb_restore_upload(&request, changed).await.unwrap_err().contains("differ"));
|
||||
release_mongodb_restore_source(&preview.source_ref);
|
||||
}
|
||||
@@ -34,10 +34,14 @@ tracing = "0.1"
|
||||
async-stream = "0.3"
|
||||
futures = "0.3"
|
||||
rustls = { version = "0.23", features = ["aws-lc-rs"] }
|
||||
tokio-util = "0.7"
|
||||
tokio-util = { version = "0.7", features = ["io"] }
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "rustls-tls"] }
|
||||
chrono = "0.4"
|
||||
aes-gcm = "0.10"
|
||||
base64 = "0.22"
|
||||
pbkdf2 = "0.12"
|
||||
sha2 = "0.10"
|
||||
tempfile = "3"
|
||||
|
||||
[dev-dependencies]
|
||||
reqwest = { version = "0.12", default-features = false, features = ["multipart"] }
|
||||
|
||||
@@ -988,6 +988,25 @@ async fn main() {
|
||||
.route("/mongo/export/progress/{exportId}", get(routes::mongodb_import_export::export_progress))
|
||||
.route("/mongo/export/download/{exportId}", get(routes::mongodb_import_export::export_download))
|
||||
.route("/mongo/export/cancel", post(routes::mongodb_import_export::cancel_export))
|
||||
.route("/mongo/dump/catalog", post(routes::mongodb_dump::catalog))
|
||||
.route(
|
||||
"/mongo/dump/source",
|
||||
post(routes::mongodb_dump::prepare_source).layer(DefaultBodyLimit::max(
|
||||
routes::table_import::import_request_body_limit_for_upload(web_body_limit_bytes()),
|
||||
)),
|
||||
)
|
||||
.route("/mongo/dump/source/release", post(routes::mongodb_dump::release_source))
|
||||
.route("/mongo/dump/upload-limit", get(routes::mongodb_dump::upload_limit))
|
||||
.route(
|
||||
"/mongo/dump/source/upload",
|
||||
post(routes::mongodb_dump::upload_restore_source).layer(DefaultBodyLimit::max(
|
||||
routes::table_import::import_request_body_limit_for_upload(web_body_limit_bytes()),
|
||||
)),
|
||||
)
|
||||
.route("/mongo/dump/export", post(routes::mongodb_dump::start_dump))
|
||||
.route("/mongo/dump/restore", post(routes::mongodb_dump::start_restore))
|
||||
.route("/mongo/dump/progress/{taskId}", get(routes::mongodb_dump::progress))
|
||||
.route("/mongo/dump/cancel", post(routes::mongodb_dump::cancel))
|
||||
// History
|
||||
.route("/history", get(routes::history::load_history).delete(routes::history::clear_history))
|
||||
.route("/history/save", post(routes::history::save_history))
|
||||
|
||||
@@ -17,6 +17,7 @@ pub mod jdbc;
|
||||
pub mod layout;
|
||||
pub mod mcp_policy;
|
||||
pub mod mongo;
|
||||
pub mod mongodb_dump;
|
||||
pub mod mongodb_import_export;
|
||||
#[cfg(feature = "mq-admin")]
|
||||
pub mod mq;
|
||||
|
||||
@@ -0,0 +1,402 @@
|
||||
use std::path::{Component, Path as FilePath, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use axum::{
|
||||
extract::{Multipart, Path, State},
|
||||
response::Sse,
|
||||
Json,
|
||||
};
|
||||
use dbx_core::{
|
||||
mongodb_dump::{
|
||||
self, MongoDatabaseDumpProgress, MongoDatabaseDumpRequest, MongoDatabaseRestoreRequest, MongoDumpFormat,
|
||||
MongoRestoreSourceRequest,
|
||||
},
|
||||
transfer,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
use super::export_download::export_download_filename;
|
||||
use crate::{
|
||||
error::AppError,
|
||||
sse::{TransferProgressChannel, TransferReplayEventKind},
|
||||
state::{WebExportFile, WebState},
|
||||
};
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct CatalogRequest {
|
||||
connection_id: String,
|
||||
database: String,
|
||||
}
|
||||
|
||||
pub async fn catalog(
|
||||
State(state): State<Arc<WebState>>,
|
||||
Json(request): Json<CatalogRequest>,
|
||||
) -> Result<Json<serde_json::Value>, AppError> {
|
||||
let catalog = mongodb_dump::inspect_mongodb_database_dump(&state.app, &request.connection_id, &request.database)
|
||||
.await
|
||||
.map_err(AppError::from)?;
|
||||
Ok(Json(serde_json::to_value(catalog).map_err(|e| AppError::from(e.to_string()))?))
|
||||
}
|
||||
|
||||
fn upload_path(root: &FilePath, filename: &str) -> Result<PathBuf, AppError> {
|
||||
let path = FilePath::new(filename);
|
||||
if filename.is_empty()
|
||||
|| filename.contains(['\\', ':', '\0'])
|
||||
|| path.components().any(|c| !matches!(c, Component::Normal(_)))
|
||||
{
|
||||
return Err(AppError::from("Invalid dump upload path".to_string()));
|
||||
}
|
||||
if path.components().count() > 4 {
|
||||
return Err(AppError::from("Dump directory nesting is too deep".to_string()));
|
||||
}
|
||||
Ok(root.join(path))
|
||||
}
|
||||
|
||||
pub async fn prepare_source(
|
||||
State(state): State<Arc<WebState>>,
|
||||
multipart: Multipart,
|
||||
) -> Result<Json<serde_json::Value>, AppError> {
|
||||
receive_source(state, multipart, false).await
|
||||
}
|
||||
|
||||
pub async fn upload_restore_source(
|
||||
State(state): State<Arc<WebState>>,
|
||||
multipart: Multipart,
|
||||
) -> Result<Json<serde_json::Value>, AppError> {
|
||||
receive_source(state, multipart, true).await
|
||||
}
|
||||
|
||||
pub async fn upload_limit() -> Json<usize> {
|
||||
Json(crate::web_body_limit_bytes())
|
||||
}
|
||||
|
||||
async fn field_text(mut field: axum::extract::multipart::Field<'_>, limit: usize) -> Result<String, AppError> {
|
||||
let mut bytes = Vec::new();
|
||||
while let Some(chunk) = field.chunk().await.map_err(|e| AppError::bad_request(e.to_string()))? {
|
||||
if bytes.len().saturating_add(chunk.len()) > limit {
|
||||
return Err(AppError::bad_request("Dump metadata field is too large"));
|
||||
}
|
||||
bytes.extend_from_slice(&chunk);
|
||||
}
|
||||
String::from_utf8(bytes).map_err(|e| AppError::bad_request(e.to_string()))
|
||||
}
|
||||
|
||||
async fn receive_source(
|
||||
state: Arc<WebState>,
|
||||
mut multipart: Multipart,
|
||||
acquiring: bool,
|
||||
) -> Result<Json<serde_json::Value>, AppError> {
|
||||
let tmp = state.data_dir.join("tmp");
|
||||
tokio::fs::create_dir_all(&tmp).await.map_err(|e| AppError::from(e.to_string()))?;
|
||||
let upload =
|
||||
tempfile::Builder::new().prefix("mongo-upload-").tempdir_in(&tmp).map_err(|e| AppError::from(e.to_string()))?;
|
||||
let mut format = None;
|
||||
let mut gzip = false;
|
||||
let mut paths = Vec::new();
|
||||
let mut bytes = 0usize;
|
||||
let mut manifest: Option<Vec<mongodb_dump::MongoDumpFile>> = None;
|
||||
let mut request: Option<MongoDatabaseRestoreRequest> = None;
|
||||
let mut expected_upload = std::collections::HashMap::<String, u64>::new();
|
||||
while let Some(mut field) = multipart.next_field().await.map_err(|e| AppError::from(e.to_string()))? {
|
||||
match field.name().unwrap_or("") {
|
||||
"file" => {
|
||||
if paths.len() >= 100_000 {
|
||||
return Err(AppError::from("Too many dump files".to_string()));
|
||||
}
|
||||
let filename =
|
||||
field.file_name().ok_or_else(|| AppError::from("Missing dump filename".to_string()))?.to_string();
|
||||
if format.is_none() {
|
||||
return Err(AppError::bad_request("Dump format must precede files"));
|
||||
}
|
||||
if !acquiring
|
||||
&& format == Some(MongoDumpFormat::Directory)
|
||||
&& !filename.ends_with(".metadata.json")
|
||||
&& !filename.ends_with(".metadata.json.gz")
|
||||
{
|
||||
return Err(AppError::bad_request(
|
||||
"Directory preview accepts metadata only; upload BSON after confirmation",
|
||||
));
|
||||
}
|
||||
if acquiring && request.is_none() {
|
||||
return Err(AppError::bad_request("Restore plan must precede files"));
|
||||
}
|
||||
let expected_size = if acquiring {
|
||||
Some(
|
||||
expected_upload
|
||||
.remove(&filename)
|
||||
.ok_or_else(|| AppError::bad_request("File is not in the selected restore plan"))?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut file_bytes = 0u64;
|
||||
let path = upload_path(upload.path(), &filename)?;
|
||||
tokio::fs::create_dir_all(path.parent().unwrap()).await.map_err(|e| AppError::from(e.to_string()))?;
|
||||
let mut output = tokio::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(&path)
|
||||
.await
|
||||
.map_err(|e| AppError::from(e.to_string()))?;
|
||||
while let Some(chunk) = field.chunk().await.map_err(|e| AppError::from(e.to_string()))? {
|
||||
if let Some(request) = &request {
|
||||
if transfer::is_cancelled(&request.task_id).await {
|
||||
return Err(AppError::bad_request("MongoDB dump/restore cancelled"));
|
||||
}
|
||||
}
|
||||
bytes = bytes.saturating_add(chunk.len());
|
||||
file_bytes = file_bytes.saturating_add(chunk.len() as u64);
|
||||
if expected_size.is_some_and(|size| file_bytes > size) {
|
||||
return Err(AppError::bad_request("File size changed after preview"));
|
||||
}
|
||||
let limit = if !acquiring && format == Some(MongoDumpFormat::Directory) {
|
||||
32 * 1024 * 1024
|
||||
} else {
|
||||
crate::web_body_limit_bytes()
|
||||
};
|
||||
if bytes > limit {
|
||||
let mut error =
|
||||
AppError::from(format!("Dump upload exceeds the configured {} byte limit", limit));
|
||||
error.status = axum::http::StatusCode::PAYLOAD_TOO_LARGE;
|
||||
return Err(error);
|
||||
}
|
||||
output.write_all(&chunk).await.map_err(|e| AppError::from(e.to_string()))?;
|
||||
}
|
||||
output.flush().await.map_err(|e| AppError::from(e.to_string()))?;
|
||||
if expected_size.is_some_and(|size| file_bytes != size) {
|
||||
return Err(AppError::bad_request("File size changed after preview"));
|
||||
}
|
||||
paths.push((filename, path));
|
||||
}
|
||||
"format" => {
|
||||
let value = field_text(field, 32).await?;
|
||||
format = Some(
|
||||
serde_json::from_value::<MongoDumpFormat>(serde_json::Value::String(value))
|
||||
.map_err(|e| AppError::from(e.to_string()))?,
|
||||
);
|
||||
}
|
||||
"gzip" => {
|
||||
let value = field_text(field, 16).await?;
|
||||
gzip = value.parse::<bool>().map_err(|_| AppError::from("Invalid gzip option".to_string()))?;
|
||||
}
|
||||
"manifest" if !acquiring => {
|
||||
let value = field_text(field, 16 * 1024 * 1024).await?;
|
||||
manifest = Some(serde_json::from_str(&value).map_err(|e| AppError::bad_request(e.to_string()))?);
|
||||
}
|
||||
"request" if acquiring => {
|
||||
let value = field_text(field, 1024 * 1024).await?;
|
||||
if request.is_some() {
|
||||
return Err(AppError::bad_request("Duplicate restore plan"));
|
||||
}
|
||||
request = Some(serde_json::from_str(&value).map_err(|e| AppError::bad_request(e.to_string()))?);
|
||||
expected_upload = mongodb_dump::mongodb_restore_upload_files(request.as_ref().unwrap())
|
||||
.map_err(AppError::from)?
|
||||
.into_iter()
|
||||
.map(|f| (f.path, f.size_bytes))
|
||||
.collect();
|
||||
}
|
||||
_ => return Err(AppError::from("Unexpected dump upload field".to_string())),
|
||||
}
|
||||
}
|
||||
let format = format.ok_or_else(|| AppError::from("Dump format is required".to_string()))?;
|
||||
if paths.is_empty() && manifest.is_none() {
|
||||
return Err(AppError::from("No dump files uploaded".to_string()));
|
||||
}
|
||||
if acquiring {
|
||||
if !expected_upload.is_empty() {
|
||||
return Err(AppError::bad_request("Selected restore files are missing"));
|
||||
}
|
||||
let request = request.ok_or_else(|| AppError::bad_request("Missing restore plan"))?;
|
||||
if transfer::is_cancelled(&request.task_id).await {
|
||||
return Err(AppError::bad_request("MongoDB dump/restore cancelled"));
|
||||
}
|
||||
let preview = mongodb_dump::attach_mongodb_restore_upload(&request, upload).await.map_err(AppError::from)?;
|
||||
return Ok(Json(serde_json::to_value(preview).map_err(|e| AppError::from(e.to_string()))?));
|
||||
}
|
||||
if format == MongoDumpFormat::Directory {
|
||||
let manifest = manifest.ok_or_else(|| AppError::bad_request("Directory preview requires a file manifest"))?;
|
||||
let expected = manifest
|
||||
.iter()
|
||||
.filter(|f| f.path.ends_with(".metadata.json") || f.path.ends_with(".metadata.json.gz"))
|
||||
.collect::<Vec<_>>();
|
||||
if expected.len() != paths.len()
|
||||
|| paths.iter().any(|(name, path)| {
|
||||
!expected
|
||||
.iter()
|
||||
.any(|f| f.path == *name && std::fs::metadata(path).is_ok_and(|m| m.len() == f.size_bytes))
|
||||
})
|
||||
{
|
||||
return Err(AppError::bad_request("Uploaded metadata does not match the file manifest"));
|
||||
}
|
||||
let preview =
|
||||
mongodb_dump::prepare_mongodb_directory_catalog(upload, manifest, gzip).await.map_err(AppError::from)?;
|
||||
return Ok(Json(serde_json::to_value(preview).map_err(|e| AppError::from(e.to_string()))?));
|
||||
}
|
||||
let path = match format {
|
||||
MongoDumpFormat::Archive if paths.len() == 1 => paths[0].1.clone(),
|
||||
MongoDumpFormat::Archive => return Err(AppError::from("Archive restore accepts one file".to_string())),
|
||||
MongoDumpFormat::Directory => {
|
||||
if paths.iter().any(|(name, _)| FilePath::new(name).components().count() < 2) {
|
||||
return Err(AppError::from("Directory uploads require database-relative paths".to_string()));
|
||||
}
|
||||
upload.path().to_owned()
|
||||
}
|
||||
};
|
||||
let preview = mongodb_dump::prepare_owned_mongodb_restore_source(
|
||||
MongoRestoreSourceRequest { path: path.to_string_lossy().into_owned(), format, gzip },
|
||||
upload,
|
||||
)
|
||||
.await
|
||||
.map_err(AppError::from)?;
|
||||
Ok(Json(serde_json::to_value(preview).map_err(|e| AppError::from(e.to_string()))?))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct SourceRef {
|
||||
source_ref: String,
|
||||
}
|
||||
pub async fn release_source(Json(request): Json<SourceRef>) -> Json<serde_json::Value> {
|
||||
Json(serde_json::json!({ "released": mongodb_dump::release_mongodb_restore_source(&request.source_ref) }))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct DumpWrapper {
|
||||
request: MongoDatabaseDumpRequest,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
pub struct RestoreWrapper {
|
||||
request: MongoDatabaseRestoreRequest,
|
||||
}
|
||||
|
||||
enum Task {
|
||||
Dump(MongoDatabaseDumpRequest, WebExportFile),
|
||||
Restore(MongoDatabaseRestoreRequest),
|
||||
}
|
||||
|
||||
async fn start_task(state: Arc<WebState>, task: Task) -> Result<Json<serde_json::Value>, AppError> {
|
||||
let id = match &task {
|
||||
Task::Dump(request, _) => &request.task_id,
|
||||
Task::Restore(request) => &request.task_id,
|
||||
}
|
||||
.clone();
|
||||
let key = format!("mongo-database:{id}");
|
||||
let channel = Arc::new(TransferProgressChannel::new());
|
||||
{
|
||||
let mut channels = state.transfer_progress_channels.write().await;
|
||||
if channels.contains_key(&key) {
|
||||
return Err(AppError::from("Database dump/restore task already exists".to_string()));
|
||||
}
|
||||
channels.insert(key.clone(), channel.clone());
|
||||
}
|
||||
let task_id = id.clone();
|
||||
dbx_core::export_runtime::spawn_export_task(async move {
|
||||
let mut latest = MongoDatabaseDumpProgress::initial(&task_id);
|
||||
let mut callback = |progress: MongoDatabaseDumpProgress| {
|
||||
latest = progress;
|
||||
if latest.status == "running" {
|
||||
if let Ok(json) = serde_json::to_string(&latest) {
|
||||
channel.send(json, TransferReplayEventKind::Progress);
|
||||
}
|
||||
}
|
||||
};
|
||||
let result = match task {
|
||||
Task::Dump(request, file) => {
|
||||
let result = mongodb_dump::dump_mongodb_database(
|
||||
&state.app,
|
||||
&request,
|
||||
|id| {
|
||||
let id = id.to_string();
|
||||
Box::pin(async move { transfer::is_cancelled(&id).await })
|
||||
},
|
||||
&mut callback,
|
||||
)
|
||||
.await;
|
||||
if result.is_ok() {
|
||||
state.export_files.write().await.insert(task_id.clone(), file);
|
||||
}
|
||||
result
|
||||
}
|
||||
Task::Restore(request) => {
|
||||
mongodb_dump::restore_mongodb_database(
|
||||
&state.app,
|
||||
&request,
|
||||
|id| {
|
||||
let id = id.to_string();
|
||||
Box::pin(async move { transfer::is_cancelled(&id).await })
|
||||
},
|
||||
&mut callback,
|
||||
)
|
||||
.await
|
||||
}
|
||||
};
|
||||
if let Err(error) = result {
|
||||
latest.error_message = Some(error);
|
||||
if latest.status == "running" {
|
||||
latest.status = "error".into();
|
||||
latest.phase = "done".into();
|
||||
}
|
||||
}
|
||||
if let Ok(json) = serde_json::to_string(&latest) {
|
||||
channel.send(json, TransferReplayEventKind::Terminal);
|
||||
}
|
||||
transfer::clear_cancelled(&task_id).await;
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(Duration::from_secs(60)).await;
|
||||
state.transfer_progress_channels.write().await.remove(&key);
|
||||
});
|
||||
});
|
||||
Ok(Json(serde_json::json!({ "taskId": id })))
|
||||
}
|
||||
|
||||
pub async fn start_dump(
|
||||
State(state): State<Arc<WebState>>,
|
||||
Json(body): Json<DumpWrapper>,
|
||||
) -> Result<Json<serde_json::Value>, AppError> {
|
||||
let mut request = body.request;
|
||||
if request.format != MongoDumpFormat::Archive {
|
||||
return Err(AppError::from("Web database exports use MongoDB archive format".to_string()));
|
||||
}
|
||||
let tmp = state.data_dir.join("tmp");
|
||||
tokio::fs::create_dir_all(&tmp).await.map_err(|e| AppError::from(e.to_string()))?;
|
||||
let ext = if request.gzip { "archive.gz" } else { "archive" };
|
||||
let filename = export_download_filename(&request.file_path, &request.database, ext);
|
||||
request.file_path =
|
||||
tmp.join(format!("mongo_database_{}.{ext}", uuid::Uuid::new_v4())).to_string_lossy().into_owned();
|
||||
let file = WebExportFile { file_path: request.file_path.clone(), download_filename: filename, format: ext.into() };
|
||||
start_task(state, Task::Dump(request, file)).await
|
||||
}
|
||||
|
||||
pub async fn start_restore(
|
||||
State(state): State<Arc<WebState>>,
|
||||
Json(body): Json<RestoreWrapper>,
|
||||
) -> Result<Json<serde_json::Value>, AppError> {
|
||||
start_task(state, Task::Restore(body.request)).await
|
||||
}
|
||||
|
||||
pub async fn progress(
|
||||
State(state): State<Arc<WebState>>,
|
||||
Path(id): Path<String>,
|
||||
) -> Result<Sse<impl futures::Stream<Item = Result<axum::response::sse::Event, std::convert::Infallible>>>, AppError> {
|
||||
let channel = state
|
||||
.transfer_progress_channels
|
||||
.read()
|
||||
.await
|
||||
.get(&format!("mongo-database:{id}"))
|
||||
.cloned()
|
||||
.ok_or_else(|| AppError::from("Database task not found".to_string()))?;
|
||||
Ok(crate::sse::sse_from_transfer_channel(channel))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct CancelRequest {
|
||||
task_id: String,
|
||||
}
|
||||
pub async fn cancel(Json(request): Json<CancelRequest>) -> Json<serde_json::Value> {
|
||||
transfer::set_cancelled(&request.task_id).await;
|
||||
Json(serde_json::json!({ "cancelled": true }))
|
||||
}
|
||||
@@ -17,13 +17,40 @@ use futures::stream::Stream;
|
||||
use futures::StreamExt;
|
||||
use serde::Deserialize;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use tokio_util::io::ReaderStream;
|
||||
|
||||
use crate::error::AppError;
|
||||
use crate::routes::export_download::{attachment_content_disposition, export_download_filename};
|
||||
use crate::sse::{TransferProgressChannel, TransferReplayEventKind};
|
||||
use crate::state::{WebExportFile, WebState};
|
||||
|
||||
const MONGO_IMPORT_PROGRESS_TTL: Duration = Duration::from_secs(30);
|
||||
|
||||
struct RemoveFileOnDrop(PathBuf);
|
||||
|
||||
impl Drop for RemoveFileOnDrop {
|
||||
fn drop(&mut self) {
|
||||
let _ = std::fs::remove_file(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
struct ExportDownloadStream {
|
||||
// Fields are dropped in declaration order, so Windows closes the file before deleting it.
|
||||
chunks: ReaderStream<tokio::fs::File>,
|
||||
_cleanup: RemoveFileOnDrop,
|
||||
}
|
||||
|
||||
impl Stream for ExportDownloadStream {
|
||||
type Item = std::io::Result<Bytes>;
|
||||
|
||||
fn poll_next(
|
||||
self: std::pin::Pin<&mut Self>,
|
||||
cx: &mut std::task::Context<'_>,
|
||||
) -> std::task::Poll<Option<Self::Item>> {
|
||||
std::pin::Pin::new(&mut self.get_mut().chunks).poll_next(cx)
|
||||
}
|
||||
}
|
||||
|
||||
fn initial_import_progress(import_id: &str, started_at: Instant) -> MongoImportProgress {
|
||||
MongoImportProgress {
|
||||
import_id: import_id.to_string(),
|
||||
@@ -153,11 +180,11 @@ pub async fn preview_import(
|
||||
|
||||
if let Some((source_ref, file_path)) = uploaded_file {
|
||||
let file_path_str = file_path.to_string_lossy().to_string();
|
||||
let format = match format.or_else(|| mongodb_import_export::format_from_path(&file_path_str).ok()) {
|
||||
let format = match format {
|
||||
Some(format) => format,
|
||||
None => {
|
||||
cleanup_uploaded_import_path(&file_path).await;
|
||||
return Err(AppError::from("Unsupported MongoDB import file type".to_string()));
|
||||
return Err(AppError::from("MongoDB import format is required".to_string()));
|
||||
}
|
||||
};
|
||||
let preview = mongodb_import_export::preview_mongodb_import_file(&MongoImportPreviewRequest {
|
||||
@@ -269,7 +296,7 @@ pub async fn execute_import(
|
||||
let state_clone = state.clone();
|
||||
tokio::spawn(async move {
|
||||
let tx_clone = tx.clone();
|
||||
let _result = mongodb_import_export::import_mongodb_file_core(
|
||||
let result = mongodb_import_export::import_mongodb_file_core(
|
||||
&app,
|
||||
&req,
|
||||
|id| {
|
||||
@@ -279,6 +306,17 @@ pub async fn execute_import(
|
||||
|progress| send_import_progress(&tx_clone, &progress),
|
||||
)
|
||||
.await;
|
||||
if let Err(error) = result {
|
||||
let current = tx.borrow().clone();
|
||||
if let Ok(mut progress) = serde_json::from_str::<MongoImportProgress>(¤t) {
|
||||
if progress.status == MongoImportStatus::Running {
|
||||
progress.phase = MongoImportPhase::Done;
|
||||
progress.status = MongoImportStatus::Error;
|
||||
progress.error_message = Some(error.display_message());
|
||||
send_import_progress(&tx, &progress);
|
||||
}
|
||||
}
|
||||
}
|
||||
cleanup_uploaded_import_source(&req.file_path).await;
|
||||
schedule_import_progress_cleanup(state_clone, req.import_id.clone());
|
||||
});
|
||||
@@ -402,26 +440,35 @@ pub async fn start_export(
|
||||
let ext = match req.format {
|
||||
MongoExportFormat::Csv => "csv",
|
||||
MongoExportFormat::Ndjson => "ndjson",
|
||||
MongoExportFormat::Bson if req.gzip => "bson.gz",
|
||||
MongoExportFormat::Bson => "bson",
|
||||
};
|
||||
let tmp_file = tmp_dir.join(format!("mongo_export_{export_id}.{ext}"));
|
||||
let tmp_file = tmp_dir.join(format!("mongo_export_{}.{ext}", uuid::Uuid::new_v4()));
|
||||
let file_path = tmp_file.to_string_lossy().to_string();
|
||||
let download_filename = export_download_filename(&req.file_path, &req.collection, ext);
|
||||
req.file_path = file_path.clone();
|
||||
|
||||
state
|
||||
.export_files
|
||||
.write()
|
||||
.await
|
||||
.insert(export_id.clone(), WebExportFile { file_path, download_filename, format: ext.to_string() });
|
||||
|
||||
let export_file = WebExportFile { file_path, download_filename, format: ext.to_string() };
|
||||
let channel_key = format!("mongo-export:{export_id}");
|
||||
let tx = {
|
||||
let mut channels = state.sse_channels.write().await;
|
||||
channels.entry(export_id.clone()).or_insert_with(|| tokio::sync::broadcast::channel::<String>(256).0).clone()
|
||||
let mut channels = state.transfer_progress_channels.write().await;
|
||||
let channel = Arc::new(TransferProgressChannel::new());
|
||||
channels.insert(channel_key.clone(), channel.clone());
|
||||
channel
|
||||
};
|
||||
|
||||
let app = state.app.clone();
|
||||
let state_clone = state.clone();
|
||||
dbx_core::export_runtime::spawn_export_task(async move {
|
||||
let mut latest = MongoExportProgress {
|
||||
export_id: req.export_id.clone(),
|
||||
status: MongoExportStatus::Running,
|
||||
documents_read: 0,
|
||||
bytes_written: 0,
|
||||
total_documents: None,
|
||||
error_message: None,
|
||||
elapsed_ms: 0,
|
||||
};
|
||||
let result = mongodb_import_export::export_mongodb_query_core(
|
||||
&app,
|
||||
&req,
|
||||
@@ -430,32 +477,38 @@ pub async fn start_export(
|
||||
Box::pin(async move { transfer::is_cancelled(&id).await })
|
||||
},
|
||||
|progress| {
|
||||
if let Ok(json) = serde_json::to_string(&progress) {
|
||||
let _ = tx.send(json);
|
||||
latest = progress;
|
||||
if latest.status == MongoExportStatus::Running {
|
||||
if let Ok(json) = serde_json::to_string(&latest) {
|
||||
tx.send(json, TransferReplayEventKind::Progress);
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
if let Err(error) = result {
|
||||
let _ = tokio::fs::remove_file(&req.file_path).await;
|
||||
state_clone.export_files.write().await.remove(&req.export_id);
|
||||
let progress = MongoExportProgress {
|
||||
export_id: req.export_id.clone(),
|
||||
status: MongoExportStatus::Error,
|
||||
documents_read: 0,
|
||||
bytes_written: 0,
|
||||
total_documents: None,
|
||||
error_message: Some(error),
|
||||
elapsed_ms: 0,
|
||||
};
|
||||
if let Ok(json) = serde_json::to_string(&progress) {
|
||||
let _ = tx.send(json);
|
||||
match result {
|
||||
Ok(_) => {
|
||||
// The download must exist before a terminal event can reach the browser.
|
||||
state_clone.export_files.write().await.insert(req.export_id.clone(), export_file);
|
||||
}
|
||||
Err(error) => {
|
||||
let _ = tokio::fs::remove_file(&req.file_path).await;
|
||||
if latest.status != MongoExportStatus::Cancelled {
|
||||
latest.status = MongoExportStatus::Error;
|
||||
}
|
||||
latest.error_message = Some(error);
|
||||
}
|
||||
}
|
||||
if let Ok(json) = serde_json::to_string(&latest) {
|
||||
tx.send(json, TransferReplayEventKind::Terminal);
|
||||
}
|
||||
transfer::clear_cancelled(&req.export_id).await;
|
||||
|
||||
tokio::time::sleep(Duration::from_secs(5)).await;
|
||||
state_clone.remove_sse_channel(&req.export_id).await;
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(Duration::from_secs(30)).await;
|
||||
state_clone.transfer_progress_channels.write().await.remove(&channel_key);
|
||||
});
|
||||
});
|
||||
|
||||
Ok(Json(serde_json::json!({ "exportId": export_id })))
|
||||
@@ -465,12 +518,14 @@ pub async fn export_progress(
|
||||
State(state): State<Arc<WebState>>,
|
||||
Path(export_id): Path<String>,
|
||||
) -> Result<Sse<impl Stream<Item = Result<axum::response::sse::Event, std::convert::Infallible>>>, AppError> {
|
||||
let tx = {
|
||||
let mut channels = state.sse_channels.write().await;
|
||||
channels.entry(export_id).or_insert_with(|| tokio::sync::broadcast::channel::<String>(256).0).clone()
|
||||
};
|
||||
let rx = tx.subscribe();
|
||||
Ok(crate::sse::sse_from_channel(rx))
|
||||
let channel = state
|
||||
.transfer_progress_channels
|
||||
.read()
|
||||
.await
|
||||
.get(&format!("mongo-export:{export_id}"))
|
||||
.cloned()
|
||||
.ok_or_else(|| AppError::from("Export not found".to_string()))?;
|
||||
Ok(crate::sse::sse_from_transfer_channel(channel))
|
||||
}
|
||||
|
||||
pub async fn cancel_export(
|
||||
@@ -491,17 +546,27 @@ pub async fn export_download(
|
||||
.await
|
||||
.remove(&export_id)
|
||||
.ok_or_else(|| AppError::from("Export file not found".to_string()))?;
|
||||
let data = tokio::fs::read(&export_file.file_path).await.map_err(|e| AppError::from(e.to_string()))?;
|
||||
let _ = tokio::fs::remove_file(&export_file.file_path).await;
|
||||
export_file_response(export_file).await
|
||||
}
|
||||
|
||||
pub(crate) async fn export_file_response(export_file: WebExportFile) -> Result<Response, AppError> {
|
||||
let file = tokio::fs::File::open(&export_file.file_path).await.map_err(|e| AppError::from(e.to_string()))?;
|
||||
let content_type = match export_file.format.as_str() {
|
||||
"csv" => "text/csv; charset=utf-8",
|
||||
_ => "application/x-ndjson; charset=utf-8",
|
||||
"ndjson" => "application/x-ndjson; charset=utf-8",
|
||||
"bson.gz" | "archive.gz" => "application/gzip",
|
||||
"bson" => "application/bson",
|
||||
_ => "application/octet-stream",
|
||||
};
|
||||
let download = ExportDownloadStream {
|
||||
chunks: ReaderStream::new(file),
|
||||
_cleanup: RemoveFileOnDrop(PathBuf::from(&export_file.file_path)),
|
||||
};
|
||||
Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header(header::CONTENT_TYPE, content_type)
|
||||
.header(header::CONTENT_DISPOSITION, attachment_content_disposition(&export_file.download_filename))
|
||||
.body(Body::from(data))
|
||||
.body(Body::from_stream(download))
|
||||
.map_err(|error| AppError::from(error.to_string()))
|
||||
}
|
||||
|
||||
@@ -509,6 +574,152 @@ pub async fn export_download(
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn download_keeps_file_until_body_is_consumed_or_abandoned() {
|
||||
for extension in ["bson", "bson.gz"] {
|
||||
for consume in [true, false] {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let path = directory.path().join(format!("records.{extension}"));
|
||||
let bytes = vec![42u8; 100_000];
|
||||
std::fs::write(&path, &bytes).unwrap();
|
||||
let response = export_file_response(WebExportFile {
|
||||
file_path: path.to_str().unwrap().into(),
|
||||
download_filename: format!("records.{extension}"),
|
||||
format: extension.into(),
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("{}", error.message));
|
||||
assert!(path.exists(), "download file was removed before streaming");
|
||||
assert_eq!(
|
||||
response.headers()[header::CONTENT_TYPE],
|
||||
if extension == "bson" { "application/bson" } else { "application/gzip" }
|
||||
);
|
||||
assert!(response.headers()[header::CONTENT_DISPOSITION]
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.contains(&format!("records.{extension}")));
|
||||
if consume {
|
||||
assert_eq!(axum::body::to_bytes(response.into_body(), bytes.len()).await.unwrap(), bytes);
|
||||
} else {
|
||||
let mut stream = response.into_body().into_data_stream();
|
||||
assert!(!stream.next().await.unwrap().unwrap().is_empty());
|
||||
drop(stream);
|
||||
}
|
||||
assert!(!path.exists(), "download file leaked after the response ended");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn export_failure_and_cancellation_are_replayed_to_late_subscribers() {
|
||||
use axum::response::IntoResponse;
|
||||
use dbx_core::{connection::AppState, storage::Storage};
|
||||
|
||||
for cancelled in [false, true] {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let storage = Storage::open(&directory.path().join("storage.db")).await.unwrap();
|
||||
let state = Arc::new(WebState::for_tests(Arc::new(AppState::new(storage)), directory.path().into()));
|
||||
let id = uuid::Uuid::new_v4().to_string();
|
||||
let expected = if cancelled { MongoExportStatus::Cancelled } else { MongoExportStatus::Error };
|
||||
if cancelled {
|
||||
transfer::set_cancelled(&id).await;
|
||||
}
|
||||
let Json(started) = start_export(
|
||||
State(state.clone()),
|
||||
Json(StartExportRequest {
|
||||
request: MongoExportRequest {
|
||||
export_id: id.clone(),
|
||||
connection_id: "missing".into(),
|
||||
database: "test".into(),
|
||||
collection: "empty".into(),
|
||||
filter: None,
|
||||
projection: None,
|
||||
sort: None,
|
||||
collation: None,
|
||||
format: MongoExportFormat::Bson,
|
||||
gzip: false,
|
||||
include_header: false,
|
||||
file_path: "empty.bson".into(),
|
||||
execution_id: None,
|
||||
},
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("{}", error.message));
|
||||
assert_eq!(started["exportId"], id);
|
||||
let key = format!("mongo-export:{id}");
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
let latest = state.transfer_progress_channels.read().await.get(&key).unwrap().latest();
|
||||
if latest.is_some_and(|json| {
|
||||
serde_json::from_str::<MongoExportProgress>(&json).unwrap().status == expected
|
||||
}) {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(!state.export_files.read().await.contains_key(&id));
|
||||
let response = export_progress(State(state.clone()), Path(id))
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("{}", error.message));
|
||||
let mut stream = response.into_response().into_body().into_data_stream();
|
||||
let chunk = tokio::time::timeout(Duration::from_secs(1), stream.next()).await.unwrap().unwrap().unwrap();
|
||||
assert!(String::from_utf8_lossy(&chunk)
|
||||
.contains(&format!("\"status\":{}", serde_json::to_string(&expected).unwrap())));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn early_import_failure_finishes_progress_and_cleans_source() {
|
||||
use dbx_core::{connection::AppState, storage::Storage};
|
||||
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let storage = Storage::open(&directory.path().join("storage.db")).await.unwrap();
|
||||
let state = Arc::new(WebState::for_tests(Arc::new(AppState::new(storage)), directory.path().into()));
|
||||
let upload_dir = import_upload_dir(directory.path());
|
||||
std::fs::create_dir_all(&upload_dir).unwrap();
|
||||
let path = upload_dir.join("empty.bson");
|
||||
std::fs::write(&path, []).unwrap();
|
||||
let id = uuid::Uuid::new_v4().to_string();
|
||||
let Json(started) = execute_import(
|
||||
State(state.clone()),
|
||||
Json(ExecuteImportWrapper {
|
||||
request: MongoImportRequest {
|
||||
import_id: id.clone(),
|
||||
connection_id: "missing".into(),
|
||||
database: "test".into(),
|
||||
collection: "empty".into(),
|
||||
file_path: path.to_str().unwrap().into(),
|
||||
source_ref: None,
|
||||
format: mongodb_import_export::MongoImportFormat::Bson,
|
||||
parse_options: Default::default(),
|
||||
batch_size: 1,
|
||||
execution_id: None,
|
||||
},
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|error| panic!("{}", error.message));
|
||||
assert_eq!(started["importId"], id);
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
let progress = state.table_import_channels.read().await.get(&id).unwrap().borrow().clone();
|
||||
let progress: MongoImportProgress = serde_json::from_str(&progress).unwrap();
|
||||
if progress.status == MongoImportStatus::Error && !path.exists() {
|
||||
assert_eq!(progress.phase, MongoImportPhase::Done);
|
||||
assert!(progress.error_message.unwrap().contains("Batch size"));
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_ref_rejects_path_traversal() {
|
||||
let data_dir = std::env::temp_dir().join(format!("dbx-mongo-import-data-{}", uuid::Uuid::new_v4()));
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
use futures::StreamExt;
|
||||
use reqwest::multipart::{Form, Part};
|
||||
use std::process::Stdio;
|
||||
use std::time::Duration;
|
||||
|
||||
#[tokio::test]
|
||||
async fn database_dump_upload_and_progress_http_contract() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let port = std::net::TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port();
|
||||
let log_path = directory.path().join("server.log");
|
||||
let mut server = tokio::process::Command::new(env!("CARGO_BIN_EXE_dbx-web"))
|
||||
.env("DBX_DATA_DIR", directory.path())
|
||||
.env("DBX_PORT", port.to_string())
|
||||
.env("DBX_DISABLE_PASSWORD", "1")
|
||||
.kill_on_drop(true)
|
||||
.stdout(Stdio::null())
|
||||
.stderr(std::fs::File::create(&log_path).unwrap())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
let base = format!("http://127.0.0.1:{port}/api/mongo/dump");
|
||||
let client = reqwest::Client::new();
|
||||
tokio::time::timeout(Duration::from_secs(20), async {
|
||||
loop {
|
||||
if client.get(format!("http://127.0.0.1:{port}/api/auth/check")).send().await.is_ok() {
|
||||
break;
|
||||
}
|
||||
if let Some(status) = server.try_wait().unwrap() {
|
||||
panic!("Web server exited {status}: {}", std::fs::read_to_string(&log_path).unwrap());
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let bad = Form::new()
|
||||
.text("format", "archive")
|
||||
.text("gzip", "false")
|
||||
.part("file", Part::bytes(vec![0]).file_name("../escape.archive"));
|
||||
let response = client.post(format!("{base}/source")).multipart(bad).send().await.unwrap();
|
||||
assert!(!response.status().is_success());
|
||||
assert!(response.text().await.unwrap().contains("Invalid dump upload path"));
|
||||
|
||||
let form = Form::new()
|
||||
.text("format", "directory")
|
||||
.text("gzip", "false")
|
||||
.text("manifest", serde_json::json!([
|
||||
{"path":"source/records.bson","sizeBytes":0},
|
||||
{"path":"source/records.metadata.json","sizeBytes":r#"{"collectionName":"records","options":{},"indexes":[],"type":"collection"}"#.len()}
|
||||
]).to_string())
|
||||
.part(
|
||||
"file",
|
||||
Part::text(r#"{"collectionName":"records","options":{},"indexes":[],"type":"collection"}"#)
|
||||
.file_name("source/records.metadata.json"),
|
||||
);
|
||||
let response = client.post(format!("{base}/source")).multipart(form).send().await.unwrap();
|
||||
assert!(response.status().is_success(), "{}", response.text().await.unwrap());
|
||||
let preview: serde_json::Value = response.json().await.unwrap();
|
||||
assert_eq!(preview["databases"], serde_json::json!(["source"]));
|
||||
assert_eq!(preview["collections"][0]["name"], "records");
|
||||
assert!(preview["collections"][0]["documents"].is_null());
|
||||
assert_eq!(preview["collections"][0]["sourceFiles"].as_array().unwrap().len(), 2);
|
||||
let source_ref = preview["sourceRef"].as_str().unwrap();
|
||||
let request = serde_json::json!({ "taskId":"acquire", "connectionId":"missing", "database":"target", "sourceDatabase":"source", "sourceRef":source_ref, "collections":["records"] });
|
||||
let acquisition = Form::new()
|
||||
.text("format", "directory")
|
||||
.text("gzip", "false")
|
||||
.text("request", request.to_string())
|
||||
.part("file", Part::bytes(Vec::new()).file_name("source/records.bson"))
|
||||
.part(
|
||||
"file",
|
||||
Part::text(r#"{"collectionName":"records","options":{},"indexes":[],"type":"collection"}"#)
|
||||
.file_name("source/records.metadata.json"),
|
||||
);
|
||||
let response = client.post(format!("{base}/source/upload")).multipart(acquisition).send().await.unwrap();
|
||||
assert!(response.status().is_success(), "{}", response.text().await.unwrap());
|
||||
let acquired: serde_json::Value = response.json().await.unwrap();
|
||||
let acquired_ref = acquired["sourceRef"].as_str().unwrap();
|
||||
let id = uuid::Uuid::new_v4().to_string();
|
||||
let response = client.post(format!("{base}/restore")).json(&serde_json::json!({ "request": {
|
||||
"taskId": id, "connectionId": "missing", "database": "target", "sourceDatabase": "source", "sourceRef": acquired_ref
|
||||
}})).send().await.unwrap();
|
||||
assert!(response.status().is_success());
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
let events = client.get(format!("{base}/progress/{id}")).send().await.unwrap();
|
||||
assert!(events.status().is_success());
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
let mut stream = events.bytes_stream();
|
||||
let mut text = String::new();
|
||||
while let Some(chunk) = stream.next().await {
|
||||
text.push_str(&String::from_utf8_lossy(&chunk.unwrap()));
|
||||
if text.contains("\"status\":\"error\"") {
|
||||
return;
|
||||
}
|
||||
}
|
||||
panic!("Missing terminal restore event: {text}");
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let response: serde_json::Value = client
|
||||
.post(format!("{base}/source/release"))
|
||||
.json(&serde_json::json!({ "sourceRef": source_ref }))
|
||||
.send()
|
||||
.await
|
||||
.unwrap()
|
||||
.json()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response["released"], true);
|
||||
client
|
||||
.post(format!("{base}/source/release"))
|
||||
.json(&serde_json::json!({"sourceRef":acquired_ref}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
server.kill().await.unwrap();
|
||||
server.wait().await.unwrap();
|
||||
}
|
||||
@@ -101,26 +101,39 @@ GridFS 工作台用于管理 Bucket 和文件:
|
||||
|
||||
## 导入与导出
|
||||
|
||||
Collection 文档提供独立的 CSV/JSON 导入和 CSV/NDJSON 导出,和关系型 Table Import 不是同一条工作流。CSV 每一行生成一个文档,嵌套值保持嵌套,`$oid`、`$date` 等 Extended JSON 类型可以保留。
|
||||
Collection 文档提供独立的 CSV/JSON/NDJSON 与 BSON 导入导出,和关系型 Table Import 不是同一条工作流。CSV 每一行生成一个文档,嵌套值保持嵌套,BSON 备份保留原生 BSON 类型。
|
||||
|
||||
### 导入数据
|
||||
|
||||
1. 在集合上右键选择「导入数据」。
|
||||
2. 选择 `.csv`、`.json` 或 `.ndjson` 文件。桌面端读取本地路径,Web 端上传到临时文件。
|
||||
3. 对照预览调整编码、分隔符、表头、空值和类型策略。
|
||||
2. 先明确选择格式,再选择 `.csv`、`.json`、`.ndjson`/`.jsonl`、`.bson` 或 `.bson.gz` 文件。桌面端读取本地路径,Web 端上传到临时文件;更换文件不会覆盖已选择的格式。
|
||||
3. 对照预览检查数据。文本格式支持编码和类型设置,CSV 可逐列覆盖推断类型;BSON 导入不应用 CSV 文本转换。
|
||||
4. 确认以追加方式写入。重复 `_id` 会停止导入,已提交的批次不会回滚。
|
||||
|
||||
CSV 按 MongoDB Compass:首行表头,嵌套字段用点号路径(`address.city`),数组索引(`tags[0]`、`matrix[0][1]`),ObjectId 写成 24 位十六进制,日期为 ISO-8601,空单元格为 null。导入采样前 1000 行推断列类型;`_id` 的十六进制会还原为 ObjectId,`recognizeObjectIdHex` 可将此规则扩展到其他列。JSON/NDJSON 使用 Extended JSON(`$oid`、`$date`)。全部按字符串会把每个单元格当文本。
|
||||
|
||||
类型保真:`Double` 导出再导入会变为 `Decimal128`(十进制文本无损)。非 `_id` 的 ObjectId 字段会往返为 String,除非打开 `recognizeObjectIdHex`。小的 `Int64` 可能收窄为 `Int32`;ISO 日期格式的字符串会变为 `DateTime`。Extended JSON 类型(`$binary`、`$regularExpression`、`$timestamp`、`$minKey`、`$maxKey`、`$code`)通过 JSON 文本单元格往返。空数组 `[]` 和空对象 `{}` 往返正确。
|
||||
CSV 类型保真:`Double` 导出再导入会变为 `Decimal128`(十进制文本无损)。非 `_id` 的 ObjectId 字段会往返为 String,除非打开 `recognizeObjectIdHex`。小的 `Int64` 可能收窄为 `Int32`;ISO 日期格式的字符串会变为 `DateTime`。Extended JSON 类型(`$binary`、`$regularExpression`、`$timestamp`、`$minKey`、`$maxKey`、`$code`)通过 JSON 文本单元格往返。空数组 `[]` 和空对象 `{}` 往返正确。
|
||||
|
||||
首期限制:只支持追加(无清空或 upsert),无按字段类型覆盖,CSV 导出从前 10000 个文档发现字段(之后的字段会被静默丢弃;变化 schema 请改用 NDJSON),无 JSON 数组导出格式,不支持 Excel/GridFS/BSON 二进制导入,失败批次不会自动重试。
|
||||
集合导入限制:只支持追加(无清空或 upsert),CSV 导出从前 10000 个文档发现字段(之后的字段会被静默丢弃;变化 schema 请改用 NDJSON 或 BSON),无 JSON 数组导出格式,不支持 Excel/GridFS 导入,失败批次不会自动重试。集合级 BSON 导入只恢复文档,不恢复配套元数据和索引。
|
||||
|
||||
### 导出数据
|
||||
|
||||
- 在集合上右键选择「导出数据」,再选 CSV 或 NDJSON,会像表导出一样直接打开保存文件对话框。
|
||||
- 在集合上右键选择「导出数据」,再选 CSV、NDJSON、BSON 或 Gzip BSON,会像表导出一样直接打开保存文件对话框。
|
||||
- CSV 嵌套对象写为点号列(`address.city`),数组写为索引列(`tags[0]`、`tags[1]`)。NDJSON 每行一个规范 Extended JSON 文档。
|
||||
- 字段数超过 256 的 Collection 请改用 NDJSON。
|
||||
- 字段数超过 256 的 Collection 请改用 NDJSON 或 BSON。
|
||||
|
||||
### 数据库备份与恢复
|
||||
|
||||
使用 MongoDB 原生驱动时,在数据库节点右键选择「备份数据库」或「恢复数据库」。数据库备份包含选中集合、集合选项、索引和视图。
|
||||
|
||||
- 桌面端支持官方目录布局与 MongoDB archive 格式,均可使用 gzip。Web 端导出 archive 文件,恢复时支持 archive 或上传目录。
|
||||
- 目录备份包含 `<数据库>/<集合>.bson` 和 `.metadata.json`,或对应的 `.gz` 文件。MongoDB archive 是独立的流式格式,不是 ZIP 或 tar。
|
||||
- 格式与 gzip 选项需明确选择。目录恢复的「读取备份」只读取清单和元数据;确认源库、目标库和集合后,才上传选中的数据文件。
|
||||
- 恢复按单遍流式读取和写入进行。可选的 `objcheck` 默认关闭,开启后在同一遍读取中逐文档校验;BSON 边界、gzip 完整性和 archive 校验和检查始终保留。
|
||||
- 默认追加写入。「恢复前删除选中的目标同名集合」只删除并重建这些集合。已有 `_id` 可能导致 `E11000`;失败或取消不会回滚此前的写入。
|
||||
- Web 上传复用服务端 `DBX_MAX_UPLOAD_MB` 配置。桌面端直接读取本地文件,无需 HTTP 上传。
|
||||
|
||||
上述格式可与官方 `mongodump`/`mongorestore` 对支持的集合双向互通。本流程不提供 oplog 回放、时间点一致性保证、UUID 保留,也不处理内部数据库、时间序列集合和加密集合;这些场景请使用官方工具。
|
||||
|
||||
如果只需要当前可见选择,仍可从表格复制当前文档、选中文档或字段值为普通 JSON 或 Extended JSON。
|
||||
|
||||
|
||||
@@ -101,26 +101,39 @@ Dropping a bucket removes its files and chunks. Verify the bucket name and backu
|
||||
|
||||
## Import and Export
|
||||
|
||||
Collection documents have a dedicated CSV/JSON import and CSV/NDJSON export workflow. This is separate from relational Table Import: each CSV row becomes a document, nested values stay nested, and Extended JSON types such as `$oid` and `$date` can be preserved.
|
||||
Collection documents have dedicated CSV/JSON/NDJSON and BSON import/export workflows. This is separate from relational Table Import: each CSV row becomes a document, nested values stay nested, and BSON dumps preserve native BSON types.
|
||||
|
||||
### Import data
|
||||
|
||||
1. Right-click a collection and choose **Import data**.
|
||||
2. Select a `.csv`, `.json`, or `.ndjson` file. Desktop reads the local path; Web uploads it to a temporary server file.
|
||||
3. Review encoding, delimiter, header, empty-value, and type settings against the preview.
|
||||
2. Choose the format explicitly, then select a `.csv`, `.json`, `.ndjson`/`.jsonl`, `.bson`, or `.bson.gz` file. Desktop reads the local path; Web uploads it to a temporary server file. Changing the file does not override the selected format.
|
||||
3. Review the preview. Text formats expose encoding and type settings; CSV supports per-column type overrides. BSON imports do not apply CSV text conversions.
|
||||
4. Confirm append-only insert. Duplicate `_id` values stop the import. Batches that already committed are not rolled back.
|
||||
|
||||
CSV follows MongoDB Compass: a header row, nested fields as dotted paths (`address.city`), array indices (`tags[0]`, `matrix[0][1]`), ObjectId as 24-character hex, dates as ISO-8601, and empty cells as null. Import samples the first 1000 rows to infer column types; `_id` hex is restored as ObjectId, and `recognizeObjectIdHex` extends this to other columns. JSON/NDJSON use Extended JSON (`$oid`, `$date`). All strings keeps every cell as text.
|
||||
|
||||
Type fidelity: `Double` exports and reimports as `Decimal128` (lossless for decimal text). Non-`_id` ObjectId fields round-trip as String unless `recognizeObjectIdHex` is on. Small `Int64` may narrow to `Int32`; ISO-date-looking strings become `DateTime`. Extended JSON types (`$binary`, `$regularExpression`, `$timestamp`, `$minKey`, `$maxKey`, `$code`) round-trip through JSON-text cells. Empty array `[]` and empty object `{}` round-trip correctly.
|
||||
CSV type fidelity: `Double` exports and reimports as `Decimal128` (lossless for decimal text). Non-`_id` ObjectId fields round-trip as String unless `recognizeObjectIdHex` is on. Small `Int64` may narrow to `Int32`; ISO-date-looking strings become `DateTime`. Extended JSON types (`$binary`, `$regularExpression`, `$timestamp`, `$minKey`, `$maxKey`, `$code`) round-trip through JSON-text cells. Empty array `[]` and empty object `{}` round-trip correctly.
|
||||
|
||||
First-release limits: append only (no truncate or upsert), no per-field type override, CSV export discovers fields from the first 10000 documents (later fields are silently dropped; use NDJSON for variable schemas), no JSON-array export format, no Excel/GridFS/BSON binary import, and no automatic retry of failed batches.
|
||||
Collection-import limits: append only (no truncate or upsert), CSV export discovers fields from the first 10000 documents (later fields are silently dropped; use NDJSON or BSON for variable schemas), no JSON-array export format, no Excel/GridFS import, and no automatic retry of failed batches. Collection-level BSON import restores documents only, not the companion metadata or indexes.
|
||||
|
||||
### Export data
|
||||
|
||||
- Right-click a collection and choose **Export data**, then CSV or NDJSON. This opens the save-file dialog immediately, the same as table export.
|
||||
- Right-click a collection and choose **Export data**, then CSV, NDJSON, BSON, or gzip BSON. This opens the save-file dialog immediately, the same as table export.
|
||||
- CSV writes nested objects as dotted columns (`address.city`) and arrays as indexed columns (`tags[0]`, `tags[1]`). NDJSON writes one canonical Extended JSON document per line.
|
||||
- Collections with more than 256 distinct fields should use NDJSON.
|
||||
- Collections with more than 256 distinct fields should use NDJSON or BSON.
|
||||
|
||||
### Database backups
|
||||
|
||||
With the native MongoDB driver, right-click a database and choose **Back up database** or **Restore database**. Database backups include selected collections, collection options, indexes and views.
|
||||
|
||||
- Desktop supports the official dump directory layout and MongoDB archive format, optionally gzip-compressed. Web exports archive files and restores either archives or uploaded directories.
|
||||
- A dump directory contains `<database>/<collection>.bson` and `.metadata.json`, or their `.gz` variants. MongoDB archive is a separate stream format, not ZIP or tar.
|
||||
- Choose the format and gzip option explicitly. For directory restore, **Read backup** reads the catalog and metadata; selected data files are uploaded only after confirming the source database, target database and collections.
|
||||
- Restore reads and writes data in one pass. Optional `objcheck` validates each BSON document in that same pass and defaults off. Framing, gzip and archive checksum checks remain enabled.
|
||||
- Restore appends by default. **Drop selected target collections before restoring** deletes and recreates only those collections. Existing `_id` values can cause `E11000`; failures or cancellation do not roll back earlier writes.
|
||||
- Web uploads use the shared `DBX_MAX_UPLOAD_MB` server setting. Desktop reads local files without an HTTP upload.
|
||||
|
||||
These formats interoperate with `mongodump`/`mongorestore` for supported collections. Oplog replay, point-in-time guarantees, UUID preservation, internal databases, time-series collections and encrypted collections are outside this workflow; use the official tools for those cases.
|
||||
|
||||
Copy the current document, selected documents, or field values as plain JSON or Extended JSON from the grid when you only need the visible selection.
|
||||
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
# MongoDB Database Dump and Restore
|
||||
|
||||
Restore preparation is superseded by [Lightweight Preview](mongodb-database-restore-v2.md)
|
||||
and [Streaming Restore](mongodb-database-restore-streaming.md): catalog discovery
|
||||
reads metadata only, and data is checked while restoring, without a separate full
|
||||
validation pass. The verification history below describes earlier implementations.
|
||||
|
||||
## Goal and Formats
|
||||
|
||||
Add database-scoped dump/restore to the existing MongoDB database tree node. Keep
|
||||
the collection data workflow added in f21a47725 and reuse its BSON import/export.
|
||||
|
||||
Support the official Database Tools 100.18.0 formats:
|
||||
|
||||
- Directory: `<database>/<escaped-collection>.bson` and `.metadata.json`.
|
||||
- Gzip directory: `.bson.gz` and `.metadata.json.gz`.
|
||||
- Official `mongodump --archive` format 0.1, optionally gzip compressed.
|
||||
|
||||
Archive is the single-file transport for Web downloads. It is not a ZIP file.
|
||||
Web restores accept an archive or a selected dump directory; desktop exports and
|
||||
restores support both formats. Format and gzip are explicit user options.
|
||||
|
||||
References:
|
||||
|
||||
- https://www.mongodb.com/docs/database-tools/mongodump/
|
||||
- https://www.mongodb.com/docs/database-tools/mongorestore/
|
||||
- https://github.com/mongodb/mongo-tools/blob/100.18.0/common/archive/spec.md
|
||||
- Official source: `common/archive`, `mongodump/metadata_dump.go`,
|
||||
`mongodump/prepare.go`, `mongorestore/metadata.go`, `mongorestore/restore.go`.
|
||||
|
||||
## Ownership
|
||||
|
||||
- `dbx-core::mongodb_dump`: metadata, source preparation, database plans, progress,
|
||||
cancellation, collection options, indexes and views.
|
||||
- `dbx-core::mongodb_dump::archive`: official prelude, namespace segments,
|
||||
terminators and per-namespace CRC64 validation. BSON and gzip use the existing
|
||||
libraries; CRC uses a maintained implementation, not handwritten arithmetic.
|
||||
- Existing `mongodb_import_export`: bounded BSON streaming and batched inserts.
|
||||
- Web routes and Tauri commands: transport, writable-connection checks, task
|
||||
notifications and source/download cleanup. No database logic in adapters.
|
||||
- `MongoDatabaseDumpDialog`: source/destination, collection selection, restore
|
||||
policy, confirmation, progress and cancellation.
|
||||
|
||||
## Export
|
||||
|
||||
1. Resolve the native MongoDB connection and list collection specifications.
|
||||
2. Collect canonical Extended JSON metadata including options, index key order,
|
||||
index options and collection type. Views have metadata but no BSON data file.
|
||||
3. Dump each regular collection, including empty collections, into a private
|
||||
temporary directory using the existing BSON exporter.
|
||||
4. Publish the directory or stream it into an official archive. Archives include
|
||||
the official header and CRC64 checksums over the original BSON bytes.
|
||||
5. Publish only after success. Cancellation or errors discard temporary output.
|
||||
|
||||
## Restore
|
||||
|
||||
1. Explicitly select directory/archive and gzip. Directory preview sends a manifest
|
||||
and metadata only; archive preview reads the prelude of the uploaded archive.
|
||||
2. Select one source database and a target database; select all or some collections.
|
||||
3. Confirm the selection and policies, then acquire the selected data. Uploads stay within owned staging
|
||||
directories. Reject links, unsafe paths, duplicate namespaces, malformed BSON,
|
||||
corrupt gzip, unsupported archive versions and CRC mismatches.
|
||||
Use the same owned input for validation and restoration. Validate all selected
|
||||
directory BSON files or full archive framing/checksums before any target writes.
|
||||
Exact document counts are unknown during preview and populated during validation.
|
||||
4. Default to appending documents. `dropExisting` applies only to selected target
|
||||
collections, never to the whole target database. Enforce readonly and existing
|
||||
production-execution guards in both transports/UI.
|
||||
5. Create missing collections with saved options when enabled; existing collections
|
||||
retain their options unless explicitly dropped. Never preserve source UUIDs or
|
||||
issue `applyOps`. Strip obsolete index version/namespace fields before creating
|
||||
indexes; let collection creation establish its automatic `_id` index.
|
||||
6. Import documents through the existing batched BSON importer, then restore indexes.
|
||||
Restore views in dependency order after their backing collections.
|
||||
7. Report partial counts on failure/cancellation. Completed writes are not rolled
|
||||
back. Releasing a source cannot delete data still owned by an active task.
|
||||
|
||||
## Initial Compatibility Boundary
|
||||
|
||||
This change covers ordinary, capped and clustered collections, GridFS collections,
|
||||
validators, collations, index definitions and views. It is not a replica-set
|
||||
snapshot: concurrent writes follow the normal no-oplog dump behavior.
|
||||
|
||||
Time-series/raw bucket restoration, Queryable Encryption, oplog replay, UUID
|
||||
preservation and user/role migration require separate server-specific handling.
|
||||
Reject unsupported collection types/options before writing; do not silently omit
|
||||
them. Internal admin/config/local databases are outside this user-database workflow.
|
||||
Ordinary user databases skip internal `system.*` collections except `system.js`.
|
||||
|
||||
## Verification
|
||||
|
||||
- Unit tests: canonical metadata, escaped names, archive CRC/EOF/interleaving,
|
||||
corrupt/truncated input, unsafe directory entries, view dependencies.
|
||||
- Official tool tests on an isolated MongoDB server: multiple collections, an empty
|
||||
collection, capped/validated collections, compound/unique/TTL indexes and views;
|
||||
both directions for directory/archive, plain/gzip, with BSON and metadata checks.
|
||||
- Restore policies: append, selected-collection drop, unselected target preservation,
|
||||
optional indexes/options, cancellation and readonly rejection.
|
||||
- Web tests: multipart paths, source ownership, delayed progress subscription and
|
||||
streaming downloads. Frontend tests: database menu, explicit format/options,
|
||||
confirmation and backend dispatch. Inspect the running Web UI before delivery.
|
||||
|
||||
## Implementation Order
|
||||
|
||||
1. Core metadata/source plan and official archive codec.
|
||||
2. Database export/restore orchestration and focused integration tests.
|
||||
3. Web/Tauri adapters and lifecycle handling.
|
||||
4. Database menu/dialog, localization and frontend tests.
|
||||
5. Official tool round trips, UI inspection and final review.
|
||||
|
||||
## Verified Results
|
||||
|
||||
Verified against MongoDB Server 8.0.17 and Database Tools 100.18.0 on Windows:
|
||||
|
||||
- Both directions for all four combinations: directory/archive, plain/gzip.
|
||||
- Six collections/views, including an empty collection, capped data, an escaped
|
||||
collection name, a validator and collation, compound unique and TTL indexes,
|
||||
and a view that depends on another view. BSON data and collection/index options
|
||||
match the source after each restore.
|
||||
- Selected-collection drop preserves unselected target data; disabling options
|
||||
and indexes works; readonly and cancellation checks reject writes; prepared
|
||||
changed original inputs are rejected and restore uses an owned input snapshot.
|
||||
- Invalid metadata, cyclic views, truncated archives and bad archive CRC are rejected.
|
||||
- The HTTP test launches the real Web binary and verifies multipart path rejection,
|
||||
directory preparation, source release and replay of late task notifications.
|
||||
- Desktop and Web compile checks, frontend type checking, focused UI tests and
|
||||
desktop/mobile Playwright inspection completed. Screenshots are in
|
||||
`output/playwright/database-{dump,restore}-desktop.png` and
|
||||
`output/playwright/database-restore-mobile.png`.
|
||||
|
||||
The opt-in database test is `official_database_tools_round_trip_through_dbx` in
|
||||
`crates/dbx-core/tests/mongodb_dump_restore.rs`. Set `DBX_MONGO_DUMP_TEST_URI` and
|
||||
`DBX_MONGO_TOOLS_DIR` to run it against an isolated server.
|
||||
@@ -0,0 +1,111 @@
|
||||
# MongoDB Streaming Restore and Object Checks
|
||||
|
||||
This supersedes the mandatory full-validation stage in
|
||||
`mongodb-database-restore-v2.md`. Metadata-first previews remain unchanged.
|
||||
|
||||
## Official Reference
|
||||
|
||||
Compared against the locally checked-out MongoDB Database Tools 100.18.0 sources:
|
||||
|
||||
- `common/db/bson_stream.go`: reads BSON frames into byte buffers.
|
||||
- `mongorestore/restore.go`: `LoadNext` feeds raw BSON to buffered insertion workers.
|
||||
When `objCheck` is enabled, the worker parses each document before inserting it.
|
||||
- `mongorestore/mongorestore.go`: enables object checks when `--objcheck` is set.
|
||||
|
||||
Object checks are part of the single restore pass. They are not a separate full
|
||||
backup scan. Server-side collection validators are a different concern and are
|
||||
not disabled by this change.
|
||||
|
||||
## DBX Behavior
|
||||
|
||||
- Directory previews upload only the manifest and metadata. After confirmation,
|
||||
upload the selected data files. Web retains the existing upload limit setting.
|
||||
- Archive previews read only the prelude after uploading the archive.
|
||||
- Confirmed restoration checks source identity and metadata, then reads, decompresses
|
||||
and inserts data in one pass. Local desktop inputs are read directly; Web inputs
|
||||
remain owned uploads held alive until the restore reader finishes.
|
||||
- Restore data uses the existing official Rust BSON library's `RawDocumentBuf`
|
||||
and the existing MongoDB driver's insert helper, generalized for serializable
|
||||
BSON values. Raw bytes are not converted through JSON or an owned Document by default.
|
||||
- `objcheck` is an optional boolean, default false. If enabled, the same official
|
||||
BSON library parses each selected document in the reader before it enters the
|
||||
insertion queue. The original bytes, not the decoded/re-encoded object, are inserted.
|
||||
- A bounded two-batch queue connects blocking file/decoder work to asynchronous
|
||||
writes. Each batch is capped by document count and an 8 MiB byte target; a single
|
||||
valid BSON document may exceed that target up to the existing document-size cap.
|
||||
- Archive namespace segments are demultiplexed directly into this queue, including
|
||||
interleaved namespaces. There is no archive extraction/recompression/spool pass.
|
||||
- Frame boundaries, truncation, gzip integrity and archive namespace EOF/CRC checks
|
||||
remain mandatory while reading. They are not controlled by `objcheck`.
|
||||
- Indexes and dependency-ordered views are restored after the data stream completes.
|
||||
- Stream errors terminate the task and preserve partial write counts. Data preceding
|
||||
a late error may already be restored; this no longer promises prevalidation before
|
||||
selected-collection replacement. No rollback or automatic retry of the whole task.
|
||||
- Cancellation closes the bounded queue, signals the reader and joins it. Progress
|
||||
includes decoded bytes read even while skipping unselected archive namespaces.
|
||||
|
||||
## Ownership
|
||||
|
||||
- `mongodb_dump/source.rs`: catalog references, upload binding and file identity checks.
|
||||
- `mongodb_dump/archive.rs`: official framing and streaming namespace/CRC handling.
|
||||
- `mongodb_dump/stream.rs`: bounded raw BSON batches, optional SDK object checks and
|
||||
lifecycle coordination with the native MongoDB insert helper.
|
||||
- `mongodb_dump.rs`: metadata preflight, selected target initialization, indexes,
|
||||
views and task summaries.
|
||||
- Web/Tauri transport: unchanged restore command with the additional `objcheck` flag.
|
||||
- Dialog: one optional object-check checkbox, default off. No additional warning,
|
||||
prevalidation toggle or confirmation step is introduced.
|
||||
- The execution entry is locked before awaiting production confirmation. A late
|
||||
confirmation after dialog disposal cannot start a task. View dependency sorting
|
||||
uses an explicit stack, so deep metadata chains cannot overflow the call stack.
|
||||
|
||||
## Verification
|
||||
|
||||
Focused tests cover default-off and enabled object checks, exact raw BSON bytes,
|
||||
metadata-only previews, partial restore on late corruption, gzip/archive integrity,
|
||||
official-tool round trips, read-only mode and cancellation. An opt-in compressed
|
||||
parser throughput smoke test compares raw streaming, streaming with object checks
|
||||
and the former double-parse pattern without database/network overhead.
|
||||
|
||||
### Local Results (2026-09-15)
|
||||
|
||||
- 7 focused core tests passed, including the opt-in parser benchmark and ordering
|
||||
a 20,000-entry dependency chain without recursion.
|
||||
- 8 integration tests passed against isolated MongoDB 8.0.17 and Database Tools
|
||||
100.18.0. Directory/archive, with and without gzip, round-tripped in both
|
||||
directions with matching BSON, collection options, indexes and views.
|
||||
- Cancellation stopped after the first 100-document batch of a 201-document
|
||||
fixture. Duplicate-key tests verified partial success counts and stop/continue
|
||||
behavior. Late BSON corruption, archive CRC failure and truncation preserved
|
||||
partial write counts instead of requiring an initial full scan.
|
||||
- Collection-level BSON and gzip regressions each round-tripped 1,207 documents
|
||||
through official tools with matching BSON bytes.
|
||||
- 32 frontend tests and Vue type checking passed, including double-submit,
|
||||
disposed-dialog confirmation and declined-confirmation regressions. The Web
|
||||
backend build and HTTP upload/progress smoke test also passed.
|
||||
|
||||
Parser-only smoke test: Windows x64, unoptimized Rust test build, an in-memory
|
||||
gzip fixture containing 8,192 documents (27.56 MiB decoded, 64 string fields per
|
||||
document). The old pattern is simulated by two object-parsing passes, not by
|
||||
running an older DBX binary.
|
||||
|
||||
| Path | Passes | Elapsed |
|
||||
| --- | --- | --- |
|
||||
| Raw streaming, default | 1 | 129 ms |
|
||||
| Streaming with object checks | 1 | 851 ms |
|
||||
| Simulated former double parse | 2 | 2,315 ms |
|
||||
|
||||
These figures exclude upload, disk I/O, MongoDB writes and index building. They
|
||||
are not an end-to-end restore speed guarantee; the user's 5.28 GiB backup has not
|
||||
been benchmarked with this implementation.
|
||||
|
||||
No active user task or configured data source was used by these tests.
|
||||
|
||||
## Duplicate Keys
|
||||
|
||||
Restore inserts documents; it does not upsert or overwrite matching IDs. Without
|
||||
`dropExisting`, existing target documents (including prior partial restores) may
|
||||
cause `E11000`. Object checks do not disable server-side unique indexes. With
|
||||
`dropExisting`, only selected target collections are deleted and recreated.
|
||||
The confirmation-entry race was reproduced and fixed, but that alone does not
|
||||
identify the cause of a particular duplicate-key report.
|
||||
@@ -0,0 +1,238 @@
|
||||
# MongoDB Database Restore: Lightweight Preview
|
||||
|
||||
The full-prevalidation/snapshot sections below are historical and superseded by
|
||||
[Streaming Restore and Object Checks](mongodb-database-restore-streaming.md).
|
||||
Metadata-first preview remains in use; restore no longer scans the backup twice.
|
||||
|
||||
Status: metadata-first restore flow implemented. This supersedes the restore preparation
|
||||
flow in `mongodb-database-dump-restore.md`, not its dump formats or compatibility
|
||||
boundary. Database export and collection import/export are unchanged.
|
||||
|
||||
## Implemented Scope
|
||||
|
||||
- Directory catalog requests contain only a bounded file manifest and metadata.
|
||||
Selected BSON files are uploaded after the user confirms the restore options.
|
||||
- Desktop catalog discovery reads metadata/stat information without copying BSON.
|
||||
Archive catalog discovery stops after the prelude, without certifying payloads.
|
||||
- Full validation now runs inside the confirmed restore task before target access
|
||||
or writes. Directory validation retains compressed inputs; archive validation
|
||||
writes only selected namespaces as compressed, multi-member BSON streams.
|
||||
- Upload byte progress, validation byte/document progress, cancellation, source
|
||||
identity checks and a temporary disk free-space reserve are implemented.
|
||||
- The existing upload environment setting is exposed to the frontend for prechecks;
|
||||
SQL and other table upload configurations are unchanged.
|
||||
|
||||
The broader lifecycle proposals below are not all part of this implementation:
|
||||
there is no persistent task resume/background-task UI, abandoned-task startup
|
||||
cleanup, or per-collection opt-out for unsupported metadata types yet. Unsupported
|
||||
types retain the existing strict rejection policy. Directory upload is followed
|
||||
by an explicit restore-start request, rather than a persistent upload/seal job.
|
||||
|
||||
## Problem
|
||||
|
||||
The current `prepare_mongodb_restore_source` uploads the entire directory,
|
||||
decompresses and validates every BSON document, and writes an uncompressed
|
||||
snapshot before presenting the source database and collection choices. The UI
|
||||
waits for a single HTTP response without preparation progress or cancellation.
|
||||
|
||||
A 211-file gzip backup totaling 5,672,833,827 bytes exposed the problem: preparation
|
||||
spent tens of minutes doing disk and CPU work before confirmation. Process I/O
|
||||
counters confirmed continuing writes; directory file sizes on Windows did not
|
||||
reliably reflect writes through still-open handles. No target database writes
|
||||
occurred in that preparation phase.
|
||||
|
||||
## User Flow
|
||||
|
||||
1. Show the target connection/database immediately, prefilled from the tree node.
|
||||
Select directory/archive and gzip explicitly; do not infer or change options.
|
||||
2. Read the backup catalog only. Populate source database, collection names,
|
||||
indexes/options availability, types and source file sizes. Unknown document
|
||||
counts stay unknown; do not scan BSON to obtain an exact count.
|
||||
3. Choose one source database, target database and collections. Configure append
|
||||
or selected-collection replacement, options, indexes and write-error policy.
|
||||
4. Confirm the frozen selection and destructive policy. Apply the existing
|
||||
production guard. Changing any choice requires a new confirmation.
|
||||
5. Acquire the selected data, validate it, then restore. Show each phase separately.
|
||||
No target mutation is allowed before validation of the selected restore set
|
||||
completes successfully.
|
||||
|
||||
### Directory
|
||||
|
||||
- Web: enumerate `File` references and relative paths locally. Send a bounded file
|
||||
manifest plus `.metadata.json`/`.metadata.json.gz` contents for core parsing.
|
||||
Do not upload or read `.bson`/`.bson.gz` bodies before confirmation. Metadata
|
||||
uploads are also size-limited; gzip metadata has a decompressed size limit.
|
||||
- Desktop: enumerate local paths and read metadata directly, without copying or
|
||||
decompressing collection data during preview.
|
||||
- Missing optional metadata permits data-only restore with an explicit warning;
|
||||
names come from the existing official filename decoder. Reject ambiguous names,
|
||||
duplicate namespaces, unsafe paths, missing required BSON and option mismatch.
|
||||
- After confirmation, Web uploads only the selected BSON and associated metadata.
|
||||
Validate the uploaded manifest against the confirmed catalog. Extra, missing,
|
||||
duplicate or changed metadata requires correction/review, not silent changes.
|
||||
- Unselected directory collection data is not opened or validated. Unsupported
|
||||
collections can be displayed as unavailable without blocking unrelated supported
|
||||
selections. Unsafe source layout still rejects the source as a whole.
|
||||
|
||||
### Archive
|
||||
|
||||
- Web retains one full upload of the original archive in this iteration, with
|
||||
byte progress and cancellation. This transport cost happens before selection,
|
||||
but it does not include a full BSON scan or full archive extraction.
|
||||
- Desktop reads only the local archive prelude during preview.
|
||||
- Core reads magic, version and bounded collection metadata up to the prelude
|
||||
terminator, then stops. For gzip this decompresses only the prefix needed for
|
||||
the prelude. Preview does not certify archive EOF or gzip integrity.
|
||||
- Collection data can be interleaved. After confirmation, validation must traverse
|
||||
the archive, including framing, namespace EOFs and checksums, even when only some
|
||||
collections are selected. Do not promise random access or selected-only reads
|
||||
for an archive. Restore only the selected namespaces.
|
||||
- Whole-archive integrity errors still fail validation. Unsupported but unselected
|
||||
collection metadata need not fail an otherwise supported selection.
|
||||
|
||||
The archive layout above follows the checked-out official Database Tools 100.18.0
|
||||
source, `common/archive/spec.md`: metadata is in the prelude, followed by interleaved
|
||||
namespace segments and namespace EOF/checksum records.
|
||||
|
||||
## Source Ownership and Validation
|
||||
|
||||
Split the current single prepared-source concept into two internal states:
|
||||
|
||||
- `CatalogSource`: metadata, file manifest and source identity; not validated data.
|
||||
- `ValidatedRestoreSource`: owned input plus validated selected namespaces/counts,
|
||||
bound to a frozen restore plan. Only this state may enter the write phase.
|
||||
|
||||
Keep opaque references on transports; do not accept arbitrary Web server paths.
|
||||
Never let a catalog reference or client-provided `validated` flag bypass validation.
|
||||
Cap catalog/session counts and metadata memory independently of BSON upload limits.
|
||||
|
||||
For directory input, retain an immutable snapshot in its original representation:
|
||||
gzip stays gzip. Validate by streaming through it and discarding decoded bytes;
|
||||
restore rereads the same owned snapshot. This intentionally trades a second gzip
|
||||
decode for avoiding a potentially enormous uncompressed temporary copy. Reuse the
|
||||
existing BSON importer and multi-member gzip support.
|
||||
|
||||
For archive input, validation may spool selected namespaces as compressed BSON
|
||||
files for the existing collection importer. Use bounded buffers and bounded open
|
||||
writers; complete gzip members per namespace segment allow interleaved namespaces
|
||||
without an unbounded file-handle cache. Do not spool unselected namespaces or retain
|
||||
an additional whole uncompressed archive. Finish all checks before using the spool.
|
||||
|
||||
Web owns uploaded inputs. Desktop acquires an owned snapshot after confirmation;
|
||||
it must not validate a mutable original and later restore different bytes from it.
|
||||
Check metadata/source identity against preview and fail/review changes. After
|
||||
snapshot acquisition, validation and restoration use that same snapshot.
|
||||
|
||||
Before writes, recheck target writability, collection types, selected view
|
||||
dependencies and the confirmed options. Missing dependencies must be resolved
|
||||
explicitly or rejected; never silently add collections to the selection.
|
||||
Corrupt BSON, gzip or archive framing is fatal regardless of the write-error policy.
|
||||
|
||||
## Job and Transport Contract
|
||||
|
||||
Use one restore task identity and freeze the plan at confirmation:
|
||||
|
||||
`catalog -> confirmed -> acquiring -> validating -> restoring -> indexes/views -> done`
|
||||
|
||||
Any active phase can become `failed` or `cancelled`. Target writes only start in
|
||||
`restoring`. The metadata catalog endpoint stays bounded; it must not synchronously
|
||||
wait for a full restore-source validation pass.
|
||||
|
||||
- Web catalog inspection accepts a directory manifest/metadata or an uploaded
|
||||
archive reference. Raw BSON is never treated as catalog metadata.
|
||||
- Confirmation creates a task with the selected source, target and policies.
|
||||
Upload selected directory files into that task, then seal its input. Missing files
|
||||
or incomplete uploads prevent validation from starting.
|
||||
- Repeated submit/seal requests for the same task cannot start a second restore.
|
||||
- Reuse the existing replayable task progress transport. Reconnecting must recover
|
||||
the latest state and terminal result, not automatically re-execute writes.
|
||||
- Cancellation is an explicit backend operation checked during upload, decode,
|
||||
validation, writes and index operations. Browser abort alone is not sufficient.
|
||||
- Closing an active dialog offers cancel or continue in background. A transient
|
||||
connection loss does not silently cancel or retry database writes. Server restart
|
||||
interrupts a task; automatic mid-restore resume is outside this change.
|
||||
- Adapt Web and Tauri to the same core plan/validation phases. Do not move MongoDB
|
||||
metadata parsing or restore policy into Vue or HTTP handlers.
|
||||
|
||||
Progress includes phase, current collection, collection counts, input bytes read,
|
||||
validated uncompressed bytes, documents validated/restored and elapsed time.
|
||||
Upload has a byte total; directory validation uses selected physical input size.
|
||||
Archive scan progress uses the whole physical archive size, not selected sizes.
|
||||
Do not compare compressed input totals to uncompressed output or display invented
|
||||
overall percentages. Report partial writes on cancellation/failure; no rollback is
|
||||
promised after the write phase begins.
|
||||
|
||||
## Resource Limits
|
||||
|
||||
- Reuse `DBX_MAX_UPLOAD_MB` for Web uploads. The current test service uses 8192 MiB;
|
||||
this design does not change the project default or SQL upload settings.
|
||||
- Expose the effective upload limit for frontend checks before sending data. Apply
|
||||
it to the selected directory upload total or the whole archive, plus a separate
|
||||
bounded allowance for multipart framing. Enforce actual received bytes too and
|
||||
return structured HTTP 413 errors.
|
||||
- Use server-owned `data_dir/tmp` storage for Web inputs and validation spools;
|
||||
avoid silently expanding large backups into a different OS temporary volume.
|
||||
- Check available disk space before known-size acquisition and periodically while
|
||||
spooling. Keep an explicit free-space reserve and bounded decompression/metadata
|
||||
memory. Compressed file size and gzip trailer size are not safe expansion bounds.
|
||||
- On failure, cancellation or expiry, remove only owned temporary data after active
|
||||
workers release it. Add startup cleanup for abandoned task directories. Never
|
||||
remove user originals, and do not change source ownership while a task is active.
|
||||
- Desktop has no Web upload cap, but shares disk, format and cancellation checks.
|
||||
|
||||
## Code Boundaries
|
||||
|
||||
- `dbx-core/src/mongodb_dump/directory.rs`: separate metadata catalog discovery from
|
||||
selected input validation. No data reader calls during directory inspection.
|
||||
- `dbx-core/src/mongodb_dump/archive.rs`: separate prelude inspection from full
|
||||
validation/demultiplexing; preserve official CRC and interleaving compatibility.
|
||||
- `dbx-core/src/mongodb_dump.rs`: source states, frozen plans, acquisition ownership,
|
||||
preflight validation and restore phase transitions.
|
||||
- Web routes/Tauri commands: file transport, task lifecycle, progress/cancel and
|
||||
limits. Keep existing export tasks and collection workflows unchanged.
|
||||
- `MongoDatabaseDumpDialog.vue`: immediate target selection, lightweight catalog,
|
||||
confirmation and visible phase-specific progress. Format selection stays explicit.
|
||||
|
||||
## Implementation Order and Acceptance
|
||||
|
||||
1. Split catalog inspection from validation, with tests proving directory preview
|
||||
never reads BSON bodies and archive preview stops at the prelude terminator.
|
||||
2. Introduce confirmed plans and selected acquisition/validation, avoiding whole
|
||||
uncompressed temporary snapshots. Validate all selected data before any drops.
|
||||
3. Add Web metadata-first directory transport, task progress, cancellation, replay
|
||||
and upload prechecks. Connect the same phases to Tauri.
|
||||
4. Update the dialog and run official-tool compatibility plus large-file tests.
|
||||
|
||||
Required regressions:
|
||||
|
||||
- 211-file directory preview sends only manifest/metadata; a huge BSON file does
|
||||
not increase preview work except for filename/size enumeration.
|
||||
- Explicit gzip mismatch is reported before BSON upload when filenames identify it.
|
||||
- Confirming a subset uploads/restores only that subset; confirm/drop never affects
|
||||
unselected target collections. Unknown counts remain unknown until validation.
|
||||
- Corruption in the last selected collection fails before any target mutation.
|
||||
- Interleaved archive and multi-member gzip still round-trip with official tools.
|
||||
- Highly compressible input does not produce a whole uncompressed temporary copy;
|
||||
memory remains bounded and progress advances during validation.
|
||||
- Cancel upload/validation, disconnect/reconnect, duplicate submit and disk-full
|
||||
cases do not leak workers or start unintended writes. Later write failures report
|
||||
partial completion honestly.
|
||||
- Modified desktop inputs or changed uploaded metadata cannot invalidate the
|
||||
confirmed selection silently. SQL/table imports and database exports regress cleanly.
|
||||
|
||||
## Verification Results
|
||||
|
||||
- Eight core/integration tests passed, including official MongoDB Tools 100.18.0
|
||||
round trips for directory/archive with and without gzip, unchanged BSON values,
|
||||
options/indexes/views and rejection of corrupt data before selected-collection drops.
|
||||
- Metadata-only directory preview accepts a 5 GiB manifest without BSON bodies;
|
||||
changed source files and mismatched confirmed uploads are rejected.
|
||||
- A synthetic 211-file directory manifest representing 5 GiB sent 8,080 bytes of
|
||||
gzip metadata and returned 105 collections in 253 ms through the real HTTP route.
|
||||
This was not a benchmark of the user's original backup, which was unavailable.
|
||||
- Twenty-eight focused frontend tests passed. A browser workflow against an
|
||||
isolated MongoDB instance previewed an official gzip directory, selected one
|
||||
collection, confirmed a different target database and restored two documents
|
||||
and two indexes. Desktop/mobile screenshots are under `output/playwright/metadata-v2-*`.
|
||||
- Final Web HTTP integration, Vue type checking and Tauri compile checking passed.
|
||||
The main local Web service was restarted with `DBX_MAX_UPLOAD_MB=8192`.
|
||||
@@ -30,6 +30,7 @@ pub mod mcp;
|
||||
pub mod mcp_bridge;
|
||||
pub mod mcp_http_server;
|
||||
pub mod mongo_cmd;
|
||||
pub mod mongodb_dump;
|
||||
pub mod mongodb_import_export;
|
||||
#[cfg(feature = "mq-admin")]
|
||||
pub mod mq_cmd;
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
use std::sync::Arc;
|
||||
use tauri::{AppHandle, Emitter, State};
|
||||
|
||||
use crate::commands::connection::{ensure_connection_writable, AppState};
|
||||
use dbx_core::mongodb_dump::{
|
||||
self, MongoDatabaseDumpProgress, MongoDatabaseDumpRequest, MongoDatabaseRestoreRequest, MongoDumpCatalog,
|
||||
MongoRestoreSourcePreview, MongoRestoreSourceRequest,
|
||||
};
|
||||
|
||||
#[tauri::command]
|
||||
pub async fn inspect_mongodb_database_dump(
|
||||
state: State<'_, Arc<AppState>>,
|
||||
connection_id: String,
|
||||
database: String,
|
||||
) -> Result<MongoDumpCatalog, String> {
|
||||
mongodb_dump::inspect_mongodb_database_dump(&state, &connection_id, &database).await
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
pub async fn prepare_mongodb_restore_source(
|
||||
request: MongoRestoreSourceRequest,
|
||||
) -> Result<MongoRestoreSourcePreview, String> {
|
||||
mongodb_dump::prepare_mongodb_restore_source(request).await
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
pub fn release_mongodb_restore_source(source_ref: String) -> bool {
|
||||
mongodb_dump::release_mongodb_restore_source(&source_ref)
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
pub async fn dump_mongodb_database(
|
||||
app: AppHandle,
|
||||
state: State<'_, Arc<AppState>>,
|
||||
request: MongoDatabaseDumpRequest,
|
||||
) -> Result<MongoDatabaseDumpProgress, String> {
|
||||
let state = state.inner().clone();
|
||||
let runtime = tokio::runtime::Handle::current();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
runtime.block_on(async move {
|
||||
let result = mongodb_dump::dump_mongodb_database(
|
||||
&state,
|
||||
&request,
|
||||
|id| {
|
||||
let id = id.to_string();
|
||||
Box::pin(async move { dbx_core::transfer::is_cancelled(&id).await })
|
||||
},
|
||||
|progress| {
|
||||
let _ = app.emit("mongo-database-dump-progress", progress);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
dbx_core::transfer::clear_cancelled(&request.task_id).await;
|
||||
result
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
pub async fn restore_mongodb_database(
|
||||
app: AppHandle,
|
||||
state: State<'_, Arc<AppState>>,
|
||||
request: MongoDatabaseRestoreRequest,
|
||||
) -> Result<MongoDatabaseDumpProgress, String> {
|
||||
ensure_connection_writable(&state, &request.connection_id, "Restore").await?;
|
||||
let state = state.inner().clone();
|
||||
let runtime = tokio::runtime::Handle::current();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
runtime.block_on(async move {
|
||||
let result = mongodb_dump::restore_mongodb_database(
|
||||
&state,
|
||||
&request,
|
||||
|id| {
|
||||
let id = id.to_string();
|
||||
Box::pin(async move { dbx_core::transfer::is_cancelled(&id).await })
|
||||
},
|
||||
|progress| {
|
||||
let _ = app.emit("mongo-database-dump-progress", progress);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
dbx_core::transfer::clear_cancelled(&request.task_id).await;
|
||||
result
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
pub async fn cancel_mongodb_database_dump(task_id: String) -> bool {
|
||||
dbx_core::transfer::set_cancelled(&task_id).await;
|
||||
true
|
||||
}
|
||||
@@ -2060,6 +2060,12 @@ pub fn run() {
|
||||
commands::mongodb_import_export::cancel_mongodb_import,
|
||||
commands::mongodb_import_export::export_mongodb_query,
|
||||
commands::mongodb_import_export::cancel_mongodb_export,
|
||||
commands::mongodb_dump::inspect_mongodb_database_dump,
|
||||
commands::mongodb_dump::prepare_mongodb_restore_source,
|
||||
commands::mongodb_dump::release_mongodb_restore_source,
|
||||
commands::mongodb_dump::dump_mongodb_database,
|
||||
commands::mongodb_dump::restore_mongodb_database,
|
||||
commands::mongodb_dump::cancel_mongodb_database_dump,
|
||||
commands::redis_cmd::redis_list_databases,
|
||||
commands::redis_cmd::redis_scan_keys,
|
||||
commands::redis_cmd::redis_scan_keys_batch,
|
||||
|
||||
Reference in New Issue
Block a user