Files
dbx/crates/dbx-core/src/data/mongodb_dump.rs
T

844 lines
33 KiB
Rust

//! Database-scoped MongoDB dump/restore; transport adapters own dialogs and uploads.
mod archive;
mod directory;
mod metadata;
mod source;
mod stream;
pub use source::{
attach_mongodb_restore_upload, mongodb_restore_upload_files, prepare_mongodb_directory_catalog,
prepare_owned_mongodb_restore_source, MongoDumpFile,
};
use std::collections::{BTreeSet, HashMap, HashSet};
use std::future::Future;
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::sync::{
atomic::{AtomicBool, Ordering},
Arc,
};
use std::time::{Duration, Instant};
use futures::TryStreamExt;
use mongodb::bson::{doc, Document, RawDocumentBuf};
use serde::{Deserialize, Serialize};
use crate::connection::{AppState, PoolKind};
use crate::db::agent_driver::{AgentCapability, PooledAgentClient};
use crate::db::mongo_driver::{
agent_insert_outcome, document_to_canonical_extended_json, insert_bson_documents,
json_object_to_document_extended_json, MongoCollectionKind, MongoDocumentResult, MongoInsertOutcome,
};
use crate::mongodb_import_export::{export_mongodb_query_core, MongoExportFormat, MongoExportRequest};
use metadata::CollectionMetadata;
type CancelFuture = Pin<Box<dyn Future<Output = bool> + Send>>;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum MongoDumpFormat {
Directory,
Archive,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MongoDumpCollection {
pub database: String,
pub name: String,
pub kind: String,
pub documents: Option<u64>,
pub size_bytes: u64,
pub indexes: usize,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub source_files: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MongoDumpCatalog {
pub databases: Vec<String>,
pub collections: Vec<MongoDumpCollection>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MongoRestoreSourceRequest {
pub path: String,
pub format: MongoDumpFormat,
#[serde(default)]
pub gzip: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MongoRestoreSourcePreview {
pub source_ref: String,
#[serde(flatten)]
pub catalog: MongoDumpCatalog,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MongoDatabaseDumpRequest {
pub task_id: String,
pub connection_id: String,
pub database: String,
pub file_path: String,
pub format: MongoDumpFormat,
#[serde(default)]
pub gzip: bool,
#[serde(default)]
pub collections: Option<Vec<String>>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MongoDatabaseRestoreRequest {
pub task_id: String,
pub connection_id: String,
pub database: String,
pub source_database: String,
pub source_ref: String,
#[serde(default)]
pub collections: Option<Vec<String>>,
#[serde(default)]
pub drop_existing: bool,
#[serde(default = "yes")]
pub restore_options: bool,
#[serde(default = "yes")]
pub restore_indexes: bool,
#[serde(default = "yes")]
pub stop_on_error: bool,
#[serde(default)]
pub objcheck: bool,
#[serde(default = "batch_size")]
pub batch_size: usize,
#[serde(default)]
pub execution_id: Option<String>,
}
fn yes() -> bool {
true
}
fn batch_size() -> usize {
500
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MongoDatabaseDumpProgress {
pub task_id: String,
pub status: String,
pub phase: String,
pub collection: Option<String>,
pub collections_done: usize,
pub collections_total: usize,
pub documents_read: u64,
pub documents_written: u64,
pub documents_failed: u64,
pub indexes_created: usize,
pub elapsed_ms: u128,
pub error_message: Option<String>,
pub file_path: Option<String>,
#[serde(default)]
pub bytes_processed: u64,
#[serde(default)]
pub documents_validated: u64,
}
impl MongoDatabaseDumpProgress {
pub fn initial(task_id: &str) -> Self {
Self {
task_id: task_id.into(),
status: "running".into(),
phase: "preparing".into(),
collection: None,
collections_done: 0,
collections_total: 0,
documents_read: 0,
documents_written: 0,
documents_failed: 0,
indexes_created: 0,
elapsed_ms: 0,
error_message: None,
file_path: None,
bytes_processed: 0,
documents_validated: 0,
}
}
fn emit(&mut self, started: Instant, callback: &mut impl FnMut(Self)) {
self.elapsed_ms = started.elapsed().as_millis();
callback(self.clone());
}
}
#[derive(Clone)]
struct PreparedCollection {
database: String,
metadata: CollectionMetadata,
path: Option<PathBuf>,
documents: u64,
size_bytes: u64,
metadata_path: Option<PathBuf>,
}
pub fn release_mongodb_restore_source(source_ref: &str) -> bool {
source::release(source_ref)
}
pub async fn prepare_mongodb_restore_source(
request: MongoRestoreSourceRequest,
) -> Result<MongoRestoreSourcePreview, String> {
source::prepare(request, None, None).await
}
fn catalog(entries: &[PreparedCollection], exact: bool) -> MongoDumpCatalog {
MongoDumpCatalog {
databases: entries.iter().map(|entry| entry.database.clone()).collect::<BTreeSet<_>>().into_iter().collect(),
collections: entries
.iter()
.map(|entry| MongoDumpCollection {
database: entry.database.clone(),
name: entry.metadata.collection_name.clone(),
kind: entry.metadata.kind.clone(),
documents: if exact { Some(entry.documents) } else { None },
size_bytes: entry.size_bytes,
indexes: entry.metadata.restore_indexes().len(),
source_files: Vec::new(),
})
.collect(),
}
}
/// A `listCollections` entry with what dump/restore needs from it.
pub(crate) struct DumpCollectionSpec {
pub name: String,
pub kind: MongoCollectionKind,
pub options: Document,
}
/// The connection a dump or restore runs over. Both drivers expose the same handful of
/// commands; the legacy agent runs them through its generic `runCommand` so nothing has to
/// be added to the agent itself.
pub(crate) enum DumpClient {
Native(mongodb::Client),
Agent(Arc<PooledAgentClient>),
}
impl DumpClient {
/// A dump reads documents through the agent's find cursor; gate it here, before any
/// work starts, so an outdated agent is refused rather than failing at the first
/// collection. Restore and preview never read a find cursor and stay ungated here.
async fn require_find_cursor(&self) -> Result<(), String> {
if let Self::Agent(client) = self {
if !client.lock().await.supports_capability(AgentCapability::MongoFindCursor) {
return Err(
"MongoDB Legacy Agent does not support database dump; upgrade or reinstall the MongoDB Legacy driver"
.into(),
);
}
}
Ok(())
}
async fn run_command(&self, database: &str, command: Document) -> Result<Document, String> {
match self {
Self::Native(client) => client.database(database).run_command(command).await.map_err(|e| e.to_string()),
Self::Agent(client) => {
let mut client = client.lock().await;
let result: MongoDocumentResult = client
.mongo_run_command(serde_json::json!({
"database": database,
"command_json": document_to_canonical_extended_json(&command).to_string(),
}))
.await?;
let response = result
.extended_documents
.as_ref()
.and_then(|documents| documents.first())
.or_else(|| result.documents.first())
.ok_or("MongoDB Legacy Agent returned an empty runCommand response")?;
json_object_to_document_extended_json(response)
}
}
}
/// Drains a command cursor (`listCollections`, `listIndexes`) through `getMore` so a database
/// with more entries than one batch is still read completely on the agent path.
async fn command_cursor(&self, database: &str, command: Document) -> Result<Vec<Document>, String> {
let mut response = self.run_command(database, command).await?;
let mut documents = Vec::new();
loop {
let cursor = response.get_document_mut("cursor").map_err(|e| e.to_string())?;
let batch = if cursor.contains_key("firstBatch") { "firstBatch" } else { "nextBatch" };
if let Ok(entries) = cursor.get_array_mut(batch) {
documents.extend(std::mem::take(entries).into_iter().filter_map(|entry| match entry {
mongodb::bson::Bson::Document(document) => Some(document),
_ => None,
}));
}
let id = command_cursor_id(cursor)?;
if id == 0 {
return Ok(documents);
}
let namespace = cursor.get_str("ns").map_err(|e| e.to_string())?;
let collection =
namespace.strip_prefix(database).and_then(|rest| rest.strip_prefix('.')).unwrap_or(namespace);
response = self.run_command(database, doc! { "getMore": id, "collection": collection }).await?;
}
}
pub(crate) async fn list_collections(&self, database: &str) -> Result<Vec<DumpCollectionSpec>, String> {
match self {
Self::Native(client) => {
let mut cursor = client.database(database).list_collections().await.map_err(|e| e.to_string())?;
let mut specs = Vec::new();
while let Some(spec) = cursor.try_next().await.map_err(|e| e.to_string())? {
specs.push(DumpCollectionSpec {
name: spec.name,
kind: MongoCollectionKind::from_driver_type(&spec.collection_type),
options: mongodb::bson::to_document(&spec.options).map_err(|e| e.to_string())?,
});
}
Ok(specs)
}
Self::Agent(_) => self
.command_cursor(database, doc! { "listCollections": 1 })
.await?
.into_iter()
.map(|mut spec| {
let name = spec.get_str("name").map_err(|e| format!("listCollections entry: {e}"))?.to_string();
let kind = MongoCollectionKind::from_metadata_kind(spec.get_str("type").ok());
let options = spec.remove("options").and_then(|o| o.as_document().cloned()).unwrap_or_default();
Ok(DumpCollectionSpec { name, kind, options })
})
.collect(),
}
}
async fn list_indexes(&self, database: &str, collection: &str) -> Result<Vec<Document>, String> {
match self {
Self::Native(client) => {
let mut indexes = client
.database(database)
.collection::<Document>(collection)
.list_indexes()
.await
.map_err(|e| e.to_string())?;
let mut documents = Vec::new();
while let Some(index) = indexes.try_next().await.map_err(|e| e.to_string())? {
documents.push(mongodb::bson::to_document(&index).map_err(|e| e.to_string())?);
}
Ok(documents)
}
Self::Agent(_) => self.command_cursor(database, doc! { "listIndexes": collection }).await,
}
}
pub(crate) async fn drop_collection(&self, database: &str, collection: &str) -> Result<(), String> {
match self {
Self::Native(client) => {
client.database(database).collection::<Document>(collection).drop().await.map_err(|e| e.to_string())
}
Self::Agent(_) => self.run_command(database, doc! { "drop": collection }).await.map(|_| ()),
}
}
pub(crate) async fn create_collection(&self, database: &str, command: Document) -> Result<(), String> {
self.run_command(database, command).await.map(|_| ())
}
/// Inserts one restore batch. The agent takes documents as canonical Extended JSON, the same
/// lossless form collection-level BSON import already uses on that path.
pub(crate) async fn insert_documents(
&self,
database: &str,
collection: &str,
documents: Vec<RawDocumentBuf>,
) -> Result<MongoInsertOutcome, String> {
match self {
Self::Native(client) => {
insert_bson_documents(client, database, collection, documents).await.map_err(|e| e.message)
}
Self::Agent(client) => {
let documents = documents
.iter()
.map(|raw| {
Document::try_from(raw.as_ref()).map(|document| document_to_canonical_extended_json(&document))
})
.collect::<Result<Vec<_>, _>>()
.map_err(|e| e.to_string())?;
let docs_json = serde_json::to_string(&documents).map_err(|e| e.to_string())?;
let mut client = client.lock().await;
let result: serde_json::Value = client
.mongo_insert_documents(serde_json::json!({
"database": database,
"collection": collection,
"docs_json": docs_json,
"ordered": false,
}))
.await?;
agent_insert_outcome(&result)
}
}
}
async fn create_index(&self, database: &str, collection: &str, index: Document) -> Result<(), String> {
self.run_command(database, doc! { "createIndexes": collection, "indexes": [index] }).await.map(|_| ())
}
async fn server_version(&self, database: &str) -> Result<String, String> {
Ok(self.run_command(database, doc! { "buildInfo": 1 }).await?.get_str("version").unwrap_or("").to_string())
}
}
/// A cursor id we cannot read must not look like "no more batches": that would end the listing
/// early and produce a dump that is silently missing collections or indexes.
fn command_cursor_id(cursor: &Document) -> Result<i64, String> {
match cursor.get("id") {
Some(mongodb::bson::Bson::Int64(id)) => Ok(*id),
Some(mongodb::bson::Bson::Int32(id)) => Ok(i64::from(*id)),
None => Err("Cursor id missing from command response".into()),
Some(other) => Err(format!("Unexpected cursor id in command response: {other:?}")),
}
}
async fn dump_client(state: &AppState, id: &str, database: &str) -> Result<DumpClient, String> {
metadata::validate_database(database)?;
// Propagate pool resolution errors (e.g. connection establishment failures) instead of
// swallowing them into the generic message below, so users see the real cause.
let pool = crate::mongodb_import_export::mongo_pool_for_database(state, id, database).await?;
match pool {
PoolKind::MongoDb(client) => Ok(DumpClient::Native(client.clone())),
PoolKind::Agent(client) => {
// Check what every dump, restore, or preview shares up front, so an outdated
// agent is refused before any work starts rather than mid-operation. The find
// cursor only the dump's BSON export relies on is gated on that path instead.
let supported = {
let client = client.lock().await;
[AgentCapability::MongoRunCommand, AgentCapability::MongoInsertDocuments]
.into_iter()
.all(|capability| client.supports_capability(capability))
};
if !supported {
return Err("MongoDB Legacy Agent does not support database dump/restore; upgrade or reinstall the MongoDB Legacy driver".into());
}
Ok(DumpClient::Agent(client.clone()))
}
_ => Err("Database dump/restore requires a MongoDB connection".into()),
}
}
async fn database_entries(client: &DumpClient, database: &str) -> Result<Vec<PreparedCollection>, String> {
let mut entries = Vec::new();
for spec in client.list_collections(database).await? {
if spec.name.starts_with("system.") && spec.name != "system.js" {
continue;
}
let mut metadata = CollectionMetadata::empty(&spec.name);
metadata.kind = spec.kind.as_str().into();
metadata.options = spec.options;
metadata.validate()?;
if !metadata.is_view() {
metadata.indexes = client.list_indexes(database, &spec.name).await?;
}
metadata.validate()?;
entries.push(PreparedCollection {
database: database.into(),
metadata,
path: None,
documents: 0,
size_bytes: 0,
metadata_path: None,
});
}
entries.sort_by(|a, b| a.metadata.collection_name.cmp(&b.metadata.collection_name));
Ok(entries)
}
pub async fn inspect_mongodb_database_dump(
state: &AppState,
id: &str,
database: &str,
) -> Result<MongoDumpCatalog, String> {
let client = dump_client(state, id, database).await?;
let entries = database_entries(&client, database).await?;
let mut catalog = catalog(&entries, false);
catalog.databases = vec![database.into()];
Ok(catalog)
}
fn select_entries(
entries: &[PreparedCollection],
selected: &Option<Vec<String>>,
) -> Result<Vec<PreparedCollection>, String> {
match selected {
None => Ok(entries.to_vec()),
Some(names) => {
let mut requested: HashSet<_> = names.iter().collect();
let chosen =
entries.iter().filter(|entry| requested.remove(&entry.metadata.collection_name)).cloned().collect();
if !requested.is_empty() {
return Err("A selected collection is not present in the dump plan".into());
}
Ok(chosen)
}
}
}
fn ordered_entries(entries: &[PreparedCollection]) -> Result<Vec<PreparedCollection>, String> {
let names: HashMap<_, _> = entries
.iter()
.enumerate()
.map(|(i, entry)| ((entry.database.clone(), entry.metadata.collection_name.clone()), i))
.collect();
let mut ordered = Vec::with_capacity(entries.len());
// 0 = unseen, 1 = active dependency, 2 = emitted. Uploaded metadata may
// contain arbitrarily deep chains, so keep the traversal off the call stack.
let mut states = vec![0u8; entries.len()];
for (i, entry) in entries.iter().enumerate().filter(|(_, e)| !e.metadata.is_view()) {
states[i] = 2;
ordered.push(entry.clone());
}
for i in 0..entries.len() {
let mut pending = vec![(i, false)];
while let Some((index, finish)) = pending.pop() {
if finish {
states[index] = 2;
ordered.push(entries[index].clone());
continue;
}
match states[index] {
2 => continue,
1 => return Err("Cyclic view dependencies in dump metadata".into()),
_ => {}
}
states[index] = 1;
pending.push((index, true));
let entry = &entries[index];
if let Ok(base) = entry.metadata.options.get_str("viewOn") {
if let Some(&parent) = names.get(&(entry.database.clone(), base.into())) {
pending.push((parent, false));
}
}
}
}
Ok(ordered)
}
fn cancelled(flag: &AtomicBool) -> Result<(), String> {
if flag.load(Ordering::Relaxed) {
Err("MongoDB dump/restore cancelled".into())
} else {
Ok(())
}
}
async fn blocking_work<T: Send + 'static>(
task_id: &str,
is_cancelled: &mut impl FnMut(&str) -> CancelFuture,
work: impl FnOnce(Arc<AtomicBool>) -> Result<T, String> + Send + 'static,
) -> Result<T, String> {
if is_cancelled(task_id).await {
return Err("MongoDB dump/restore cancelled".into());
}
struct CancelOnDrop(Arc<AtomicBool>);
impl Drop for CancelOnDrop {
fn drop(&mut self) {
self.0.store(true, Ordering::Relaxed);
}
}
let flag = Arc::new(AtomicBool::new(false));
let _guard = CancelOnDrop(flag.clone());
let signal = flag.clone();
let mut task = tokio::task::spawn_blocking(move || work(signal));
let mut interval = tokio::time::interval(Duration::from_millis(100));
loop {
tokio::select! {
result = &mut task => return result.map_err(|e| e.to_string())?,
_ = interval.tick() => if is_cancelled(task_id).await { flag.store(true, Ordering::Relaxed); },
}
}
}
pub async fn dump_mongodb_database<C, F>(
state: &AppState,
request: &MongoDatabaseDumpRequest,
mut is_cancelled: C,
mut on_progress: F,
) -> Result<MongoDatabaseDumpProgress, String>
where
C: FnMut(&str) -> CancelFuture,
F: FnMut(MongoDatabaseDumpProgress),
{
let started = Instant::now();
let mut progress = MongoDatabaseDumpProgress::initial(&request.task_id);
progress.emit(started, &mut on_progress);
let result: Result<(), String> = async {
if is_cancelled(&request.task_id).await {
return Err("MongoDB dump/restore cancelled".into());
}
let client = dump_client(state, &request.connection_id, &request.database).await?;
client.require_find_cursor().await?;
let mut entries = select_entries(&database_entries(&client, &request.database).await?, &request.collections)?;
progress.collections_total = entries.len();
let target = PathBuf::from(&request.file_path);
if target.exists() {
return Err(match request.format {
MongoDumpFormat::Directory => "Dump destination already exists; choose a new directory",
MongoDumpFormat::Archive => "Dump destination already exists; choose a new archive file",
}
.into());
}
let parent = target.parent().filter(|p| !p.as_os_str().is_empty()).unwrap_or(Path::new("."));
std::fs::create_dir_all(parent).map_err(|e| e.to_string())?;
let working =
tempfile::Builder::new().prefix(".dbx-mongo-dump-").tempdir_in(parent).map_err(|e| e.to_string())?;
for (index, entry) in entries.iter_mut().enumerate() {
if is_cancelled(&request.task_id).await {
return Err("MongoDB dump/restore cancelled".into());
}
progress.collection = Some(entry.metadata.collection_name.clone());
progress.phase = "data".into();
progress.emit(started, &mut on_progress);
if !entry.metadata.is_view() {
let path = working.path().join(format!("{index}.bson"));
let child = MongoExportRequest {
export_id: request.task_id.clone(),
connection_id: request.connection_id.clone(),
database: request.database.clone(),
collection: entry.metadata.collection_name.clone(),
filter: None,
sort: None,
projection: None,
collation: None,
format: MongoExportFormat::Bson,
include_header: false,
gzip: false,
file_path: path.to_string_lossy().into_owned(),
execution_id: None,
};
let base = progress.documents_read;
let summary = export_mongodb_query_core(
state,
&child,
|id| is_cancelled(id),
|child| {
progress.documents_read = base + child.documents_read;
progress.emit(started, &mut on_progress);
},
)
.await?;
entry.documents = summary.documents_exported;
entry.size_bytes = std::fs::metadata(&path).map_err(|e| e.to_string())?.len();
entry.path = Some(path);
}
progress.collections_done += 1;
}
progress.phase = "archive".into();
progress.emit(started, &mut on_progress);
let server_version = client.server_version(&request.database).await?;
let database = request.database.clone();
let gzip = request.gzip;
let format = request.format;
blocking_work(&request.task_id, &mut is_cancelled, move |flag| {
let _working = working;
let parent = target.parent().filter(|p| !p.as_os_str().is_empty()).unwrap_or(Path::new("."));
match format {
MongoDumpFormat::Archive => {
let temp = tempfile::NamedTempFile::new_in(parent).map_err(|e| e.to_string())?;
directory::write_file(temp.path(), gzip, |writer| {
archive::pack(writer, &entries, &server_version, || cancelled(&flag))
})?;
cancelled(&flag)?;
if target.exists() {
return Err("Dump destination already exists".into());
}
temp.persist(&target).map_err(|e| e.to_string())?;
}
MongoDumpFormat::Directory => {
let output = tempfile::Builder::new()
.prefix(".dbx-dump-output-")
.tempdir_in(parent)
.map_err(|e| e.to_string())?;
directory::publish_files(output.path(), &database, &entries, gzip, || cancelled(&flag))?;
cancelled(&flag)?;
if target.exists() {
return Err("Dump destination already exists".into());
}
std::fs::rename(output.path(), &target).map_err(|e| e.to_string())?;
}
}
Ok(())
})
.await?;
progress.file_path = Some(request.file_path.clone());
Ok(())
}
.await;
finish_progress(progress, result, started, &mut on_progress)
}
pub async fn restore_mongodb_database<C, F>(
state: &AppState,
request: &MongoDatabaseRestoreRequest,
mut is_cancelled: C,
mut on_progress: F,
) -> Result<MongoDatabaseDumpProgress, String>
where
C: FnMut(&str) -> CancelFuture,
F: FnMut(MongoDatabaseDumpProgress),
{
let started = Instant::now();
let mut progress = MongoDatabaseDumpProgress::initial(&request.task_id);
progress.emit(started, &mut on_progress);
let result: Result<(), String> = async {
if let Some(name) = crate::query::connection_readonly_name(state, &request.connection_id).await {
return Err(format!("Read-only connection '{name}': restore blocked"));
}
if is_cancelled(&request.task_id).await {
return Err("MongoDB dump/restore cancelled".into());
}
crate::mongodb_import_export::clamp_batch_size(request.batch_size).map_err(|e| e.display_message())?;
let source = source::get(&request.source_ref)?;
let source_entries: Vec<_> =
source.entries.iter().filter(|entry| entry.database == request.source_database).cloned().collect();
if source_entries.is_empty() && !source.entries.is_empty() {
return Err("Source database is not present in the dump".into());
}
let entries = ordered_entries(&select_entries(&source_entries, &request.collections)?)?;
progress.collections_total = entries.len();
source.check_ready(&entries)?;
let client = dump_client(state, &request.connection_id, &request.database).await?;
let existing = client.list_collections(&request.database).await?;
for entry in &entries {
entry.metadata.validate()?;
if let Some(target) = existing.iter().find(|target| target.name == entry.metadata.collection_name) {
if !request.drop_existing
&& (entry.metadata.is_view() || target.kind != MongoCollectionKind::Collection)
{
return Err(format!(
"{}: replacing an existing view or non-regular collection requires dropExisting",
target.name
));
}
}
if let Ok(base) = entry.metadata.options.get_str("viewOn") {
if !entries.iter().any(|e| e.metadata.collection_name == base)
&& !existing.iter().any(|e| e.name == base)
{
return Err(format!(
"Missing backing collection for view {}: {base}",
entry.metadata.collection_name
));
}
}
}
let existing_names = existing.iter().map(|e| e.name.clone()).collect::<HashSet<_>>();
stream::restore_data(
state,
&client,
request,
source,
&entries,
&existing_names,
&mut progress,
started,
&mut is_cancelled,
&mut on_progress,
)
.await?;
for entry in &entries {
if is_cancelled(&request.task_id).await {
return Err("MongoDB dump/restore cancelled".into());
}
stream::writable(state, &request.connection_id).await?;
let name = &entry.metadata.collection_name;
progress.collection = Some(name.clone());
progress.phase = if entry.metadata.is_view() { "views" } else { "indexes" }.into();
progress.emit(started, &mut on_progress);
if entry.metadata.is_view() {
stream::initialize_collection(&client, request, entry, existing_names.contains(name)).await?;
progress.collections_done += 1;
} else if request.restore_indexes {
for index in entry.metadata.restore_indexes() {
if is_cancelled(&request.task_id).await {
return Err("MongoDB dump/restore cancelled".into());
}
client.create_index(&request.database, name, index).await?;
progress.indexes_created += 1;
}
}
}
Ok(())
}
.await;
finish_progress(progress, result, started, &mut on_progress)
}
fn finish_progress(
mut progress: MongoDatabaseDumpProgress,
result: Result<(), String>,
started: Instant,
callback: &mut impl FnMut(MongoDatabaseDumpProgress),
) -> Result<MongoDatabaseDumpProgress, String> {
progress.phase = "done".into();
progress.status = match &result {
Ok(()) => "done",
Err(error)
if matches!(error.as_str(), "MongoDB dump/restore cancelled" | "Import cancelled" | "Export cancelled") =>
{
"cancelled"
}
Err(_) => "error",
}
.into();
progress.error_message = result.as_ref().err().cloned();
progress.emit(started, callback);
result.map(|_| progress)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn command_cursor_id_accepts_integers_and_refuses_anything_else() {
assert_eq!(command_cursor_id(&doc! { "id": 0i64 }), Ok(0));
assert_eq!(command_cursor_id(&doc! { "id": 42i32 }), Ok(42));
assert_eq!(command_cursor_id(&doc! { "id": 1234567890123i64 }), Ok(1234567890123));
// Neither a missing id nor a non-integer one may be mistaken for an exhausted cursor.
assert!(command_cursor_id(&doc! {}).is_err());
assert!(command_cursor_id(&doc! { "id": "7" }).is_err());
assert!(command_cursor_id(&doc! { "id": 1.5 }).is_err());
}
#[test]
fn view_ordering_handles_deep_dependency_chains() {
let count = 20_000;
let entries = (0..count)
.map(|index| {
let mut metadata = CollectionMetadata::empty(&format!("view_{index}"));
if index + 1 < count {
metadata.kind = "view".into();
metadata.options = doc! { "viewOn": format!("view_{}", index + 1), "pipeline": [] };
}
PreparedCollection {
database: "source".into(),
metadata,
path: None,
documents: 0,
size_bytes: 0,
metadata_path: None,
}
})
.collect::<Vec<_>>();
let ordered = ordered_entries(&entries).unwrap();
assert_eq!(ordered.len(), count);
for (entry, index) in ordered.iter().zip((0..count).rev()) {
assert_eq!(entry.metadata.collection_name, format!("view_{index}"));
}
}
}