//! Database-scoped MongoDB dump/restore; transport adapters own dialogs and uploads. mod archive; mod directory; mod metadata; mod source; mod stream; pub use source::{ attach_mongodb_restore_upload, mongodb_restore_upload_files, prepare_mongodb_directory_catalog, prepare_owned_mongodb_restore_source, MongoDumpFile, }; use std::collections::{BTreeSet, HashMap, HashSet}; use std::future::Future; use std::path::{Path, PathBuf}; use std::pin::Pin; use std::sync::{ atomic::{AtomicBool, Ordering}, Arc, }; use std::time::{Duration, Instant}; use futures::TryStreamExt; use mongodb::bson::{doc, Document}; use serde::{Deserialize, Serialize}; use crate::connection::{AppState, PoolKind}; use crate::mongodb_import_export::{export_mongodb_query_core, MongoExportFormat, MongoExportRequest}; use metadata::CollectionMetadata; type CancelFuture = Pin + Send>>; #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub enum MongoDumpFormat { Directory, Archive, } #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct MongoDumpCollection { pub database: String, pub name: String, pub kind: String, pub documents: Option, pub size_bytes: u64, pub indexes: usize, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub source_files: Vec, } #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct MongoDumpCatalog { pub databases: Vec, pub collections: Vec, } #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct MongoRestoreSourceRequest { pub path: String, pub format: MongoDumpFormat, #[serde(default)] pub gzip: bool, } #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct MongoRestoreSourcePreview { pub source_ref: String, #[serde(flatten)] pub catalog: MongoDumpCatalog, } #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct MongoDatabaseDumpRequest { pub task_id: String, pub connection_id: String, pub database: String, pub file_path: String, pub format: MongoDumpFormat, #[serde(default)] pub gzip: bool, #[serde(default)] pub collections: Option>, } #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct MongoDatabaseRestoreRequest { pub task_id: String, pub connection_id: String, pub database: String, pub source_database: String, pub source_ref: String, #[serde(default)] pub collections: Option>, #[serde(default)] pub drop_existing: bool, #[serde(default = "yes")] pub restore_options: bool, #[serde(default = "yes")] pub restore_indexes: bool, #[serde(default = "yes")] pub stop_on_error: bool, #[serde(default)] pub objcheck: bool, #[serde(default = "batch_size")] pub batch_size: usize, #[serde(default)] pub execution_id: Option, } fn yes() -> bool { true } fn batch_size() -> usize { 500 } #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct MongoDatabaseDumpProgress { pub task_id: String, pub status: String, pub phase: String, pub collection: Option, pub collections_done: usize, pub collections_total: usize, pub documents_read: u64, pub documents_written: u64, pub documents_failed: u64, pub indexes_created: usize, pub elapsed_ms: u128, pub error_message: Option, pub file_path: Option, #[serde(default)] pub bytes_processed: u64, #[serde(default)] pub documents_validated: u64, } impl MongoDatabaseDumpProgress { pub fn initial(task_id: &str) -> Self { Self { task_id: task_id.into(), status: "running".into(), phase: "preparing".into(), collection: None, collections_done: 0, collections_total: 0, documents_read: 0, documents_written: 0, documents_failed: 0, indexes_created: 0, elapsed_ms: 0, error_message: None, file_path: None, bytes_processed: 0, documents_validated: 0, } } fn emit(&mut self, started: Instant, callback: &mut impl FnMut(Self)) { self.elapsed_ms = started.elapsed().as_millis(); callback(self.clone()); } } #[derive(Clone)] struct PreparedCollection { database: String, metadata: CollectionMetadata, path: Option, documents: u64, size_bytes: u64, metadata_path: Option, } pub fn release_mongodb_restore_source(source_ref: &str) -> bool { source::release(source_ref) } pub async fn prepare_mongodb_restore_source( request: MongoRestoreSourceRequest, ) -> Result { source::prepare(request, None, None).await } fn catalog(entries: &[PreparedCollection], exact: bool) -> MongoDumpCatalog { MongoDumpCatalog { databases: entries.iter().map(|entry| entry.database.clone()).collect::>().into_iter().collect(), collections: entries .iter() .map(|entry| MongoDumpCollection { database: entry.database.clone(), name: entry.metadata.collection_name.clone(), kind: entry.metadata.kind.clone(), documents: if exact { Some(entry.documents) } else { None }, size_bytes: entry.size_bytes, indexes: entry.metadata.restore_indexes().len(), source_files: Vec::new(), }) .collect(), } } async fn native_client(state: &AppState, id: &str, database: &str) -> Result { metadata::validate_database(database)?; state.get_or_create_pool(id, Some(database)).await?; match state.pool_handle(id).await { Some(PoolKind::MongoDb(client)) => Ok(client.clone()), _ => Err("Database dump/restore requires the native MongoDB driver".into()), } } async fn database_entries(client: &mongodb::Client, database: &str) -> Result, String> { let db = client.database(database); let mut cursor = db.list_collections().await.map_err(|e| e.to_string())?; let mut entries = Vec::new(); while let Some(spec) = cursor.try_next().await.map_err(|e| e.to_string())? { if spec.name.starts_with("system.") && spec.name != "system.js" { continue; } let mut metadata = CollectionMetadata::empty(&spec.name); metadata.kind = crate::db::mongo_driver::MongoCollectionKind::from_driver_type(&spec.collection_type).as_str().into(); metadata.options = mongodb::bson::to_document(&spec.options).map_err(|e| e.to_string())?; metadata.validate()?; if !metadata.is_view() { let mut indexes = db.collection::(&spec.name).list_indexes().await.map_err(|e| e.to_string())?; while let Some(index) = indexes.try_next().await.map_err(|e| e.to_string())? { metadata.indexes.push(mongodb::bson::to_document(&index).map_err(|e| e.to_string())?); } } metadata.validate()?; entries.push(PreparedCollection { database: database.into(), metadata, path: None, documents: 0, size_bytes: 0, metadata_path: None, }); } entries.sort_by(|a, b| a.metadata.collection_name.cmp(&b.metadata.collection_name)); Ok(entries) } pub async fn inspect_mongodb_database_dump( state: &AppState, id: &str, database: &str, ) -> Result { let client = native_client(state, id, database).await?; let entries = database_entries(&client, database).await?; let mut catalog = catalog(&entries, false); catalog.databases = vec![database.into()]; Ok(catalog) } fn select_entries( entries: &[PreparedCollection], selected: &Option>, ) -> Result, String> { match selected { None => Ok(entries.to_vec()), Some(names) => { let mut requested: HashSet<_> = names.iter().collect(); let chosen = entries.iter().filter(|entry| requested.remove(&entry.metadata.collection_name)).cloned().collect(); if !requested.is_empty() { return Err("A selected collection is not present in the dump plan".into()); } Ok(chosen) } } } fn ordered_entries(entries: &[PreparedCollection]) -> Result, String> { let names: HashMap<_, _> = entries .iter() .enumerate() .map(|(i, entry)| ((entry.database.clone(), entry.metadata.collection_name.clone()), i)) .collect(); let mut ordered = Vec::with_capacity(entries.len()); // 0 = unseen, 1 = active dependency, 2 = emitted. Uploaded metadata may // contain arbitrarily deep chains, so keep the traversal off the call stack. let mut states = vec![0u8; entries.len()]; for (i, entry) in entries.iter().enumerate().filter(|(_, e)| !e.metadata.is_view()) { states[i] = 2; ordered.push(entry.clone()); } for i in 0..entries.len() { let mut pending = vec![(i, false)]; while let Some((index, finish)) = pending.pop() { if finish { states[index] = 2; ordered.push(entries[index].clone()); continue; } match states[index] { 2 => continue, 1 => return Err("Cyclic view dependencies in dump metadata".into()), _ => {} } states[index] = 1; pending.push((index, true)); let entry = &entries[index]; if let Ok(base) = entry.metadata.options.get_str("viewOn") { if let Some(&parent) = names.get(&(entry.database.clone(), base.into())) { pending.push((parent, false)); } } } } Ok(ordered) } fn cancelled(flag: &AtomicBool) -> Result<(), String> { if flag.load(Ordering::Relaxed) { Err("MongoDB dump/restore cancelled".into()) } else { Ok(()) } } async fn blocking_work( task_id: &str, is_cancelled: &mut impl FnMut(&str) -> CancelFuture, work: impl FnOnce(Arc) -> Result + Send + 'static, ) -> Result { if is_cancelled(task_id).await { return Err("MongoDB dump/restore cancelled".into()); } struct CancelOnDrop(Arc); impl Drop for CancelOnDrop { fn drop(&mut self) { self.0.store(true, Ordering::Relaxed); } } let flag = Arc::new(AtomicBool::new(false)); let _guard = CancelOnDrop(flag.clone()); let signal = flag.clone(); let mut task = tokio::task::spawn_blocking(move || work(signal)); let mut interval = tokio::time::interval(Duration::from_millis(100)); loop { tokio::select! { result = &mut task => return result.map_err(|e| e.to_string())?, _ = interval.tick() => if is_cancelled(task_id).await { flag.store(true, Ordering::Relaxed); }, } } } pub async fn dump_mongodb_database( state: &AppState, request: &MongoDatabaseDumpRequest, mut is_cancelled: C, mut on_progress: F, ) -> Result where C: FnMut(&str) -> CancelFuture, F: FnMut(MongoDatabaseDumpProgress), { let started = Instant::now(); let mut progress = MongoDatabaseDumpProgress::initial(&request.task_id); progress.emit(started, &mut on_progress); let result: Result<(), String> = async { if is_cancelled(&request.task_id).await { return Err("MongoDB dump/restore cancelled".into()); } let client = native_client(state, &request.connection_id, &request.database).await?; let mut entries = select_entries(&database_entries(&client, &request.database).await?, &request.collections)?; progress.collections_total = entries.len(); let target = PathBuf::from(&request.file_path); if target.exists() { return Err(match request.format { MongoDumpFormat::Directory => "Dump destination already exists; choose a new directory", MongoDumpFormat::Archive => "Dump destination already exists; choose a new archive file", } .into()); } let parent = target.parent().filter(|p| !p.as_os_str().is_empty()).unwrap_or(Path::new(".")); std::fs::create_dir_all(parent).map_err(|e| e.to_string())?; let working = tempfile::Builder::new().prefix(".dbx-mongo-dump-").tempdir_in(parent).map_err(|e| e.to_string())?; for (index, entry) in entries.iter_mut().enumerate() { if is_cancelled(&request.task_id).await { return Err("MongoDB dump/restore cancelled".into()); } progress.collection = Some(entry.metadata.collection_name.clone()); progress.phase = "data".into(); progress.emit(started, &mut on_progress); if !entry.metadata.is_view() { let path = working.path().join(format!("{index}.bson")); let child = MongoExportRequest { export_id: request.task_id.clone(), connection_id: request.connection_id.clone(), database: request.database.clone(), collection: entry.metadata.collection_name.clone(), filter: None, sort: None, projection: None, collation: None, format: MongoExportFormat::Bson, include_header: false, gzip: false, file_path: path.to_string_lossy().into_owned(), execution_id: None, }; let base = progress.documents_read; let summary = export_mongodb_query_core( state, &child, |id| is_cancelled(id), |child| { progress.documents_read = base + child.documents_read; progress.emit(started, &mut on_progress); }, ) .await?; entry.documents = summary.documents_exported; entry.size_bytes = std::fs::metadata(&path).map_err(|e| e.to_string())?.len(); entry.path = Some(path); } progress.collections_done += 1; } progress.phase = "archive".into(); progress.emit(started, &mut on_progress); let server_version = client .database(&request.database) .run_command(doc! { "buildInfo": 1 }) .await .map_err(|e| e.to_string())? .get_str("version") .unwrap_or("") .to_string(); let database = request.database.clone(); let gzip = request.gzip; let format = request.format; blocking_work(&request.task_id, &mut is_cancelled, move |flag| { let _working = working; let parent = target.parent().filter(|p| !p.as_os_str().is_empty()).unwrap_or(Path::new(".")); match format { MongoDumpFormat::Archive => { let temp = tempfile::NamedTempFile::new_in(parent).map_err(|e| e.to_string())?; directory::write_file(temp.path(), gzip, |writer| { archive::pack(writer, &entries, &server_version, || cancelled(&flag)) })?; cancelled(&flag)?; if target.exists() { return Err("Dump destination already exists".into()); } temp.persist(&target).map_err(|e| e.to_string())?; } MongoDumpFormat::Directory => { let output = tempfile::Builder::new() .prefix(".dbx-dump-output-") .tempdir_in(parent) .map_err(|e| e.to_string())?; directory::publish_files(output.path(), &database, &entries, gzip, || cancelled(&flag))?; cancelled(&flag)?; if target.exists() { return Err("Dump destination already exists".into()); } std::fs::rename(output.path(), &target).map_err(|e| e.to_string())?; } } Ok(()) }) .await?; progress.file_path = Some(request.file_path.clone()); Ok(()) } .await; finish_progress(progress, result, started, &mut on_progress) } pub async fn restore_mongodb_database( state: &AppState, request: &MongoDatabaseRestoreRequest, mut is_cancelled: C, mut on_progress: F, ) -> Result where C: FnMut(&str) -> CancelFuture, F: FnMut(MongoDatabaseDumpProgress), { let started = Instant::now(); let mut progress = MongoDatabaseDumpProgress::initial(&request.task_id); progress.emit(started, &mut on_progress); let result: Result<(), String> = async { if let Some(name) = crate::query::connection_readonly_name(state, &request.connection_id).await { return Err(format!("Read-only connection '{name}': restore blocked")); } if is_cancelled(&request.task_id).await { return Err("MongoDB dump/restore cancelled".into()); } crate::mongodb_import_export::clamp_batch_size(request.batch_size).map_err(|e| e.display_message())?; let source = source::get(&request.source_ref)?; let source_entries: Vec<_> = source.entries.iter().filter(|entry| entry.database == request.source_database).cloned().collect(); if source_entries.is_empty() && !source.entries.is_empty() { return Err("Source database is not present in the dump".into()); } let entries = ordered_entries(&select_entries(&source_entries, &request.collections)?)?; progress.collections_total = entries.len(); source.check_ready(&entries)?; let client = native_client(state, &request.connection_id, &request.database).await?; let db = client.database(&request.database); let existing = db .list_collections() .await .map_err(|e| e.to_string())? .try_collect::>() .await .map_err(|e| e.to_string())?; for entry in &entries { entry.metadata.validate()?; if let Some(target) = existing.iter().find(|target| target.name == entry.metadata.collection_name) { if !request.drop_existing && (entry.metadata.is_view() || !matches!(target.collection_type, mongodb::results::CollectionType::Collection)) { return Err(format!( "{}: replacing an existing view or non-regular collection requires dropExisting", target.name )); } } if let Ok(base) = entry.metadata.options.get_str("viewOn") { if !entries.iter().any(|e| e.metadata.collection_name == base) && !existing.iter().any(|e| e.name == base) { return Err(format!( "Missing backing collection for view {}: {base}", entry.metadata.collection_name )); } } } let existing_names = existing.iter().map(|e| e.name.clone()).collect::>(); stream::restore_data( state, &client, request, source, &entries, &existing_names, &mut progress, started, &mut is_cancelled, &mut on_progress, ) .await?; for entry in &entries { if is_cancelled(&request.task_id).await { return Err("MongoDB dump/restore cancelled".into()); } stream::writable(state, &request.connection_id).await?; let name = &entry.metadata.collection_name; progress.collection = Some(name.clone()); progress.phase = if entry.metadata.is_view() { "views" } else { "indexes" }.into(); progress.emit(started, &mut on_progress); if entry.metadata.is_view() { stream::initialize_collection(&client, request, entry, existing_names.contains(name)).await?; progress.collections_done += 1; } else if request.restore_indexes { for index in entry.metadata.restore_indexes() { if is_cancelled(&request.task_id).await { return Err("MongoDB dump/restore cancelled".into()); } db.run_command(doc! { "createIndexes": name, "indexes": [index] }) .await .map_err(|e| e.to_string())?; progress.indexes_created += 1; } } } Ok(()) } .await; finish_progress(progress, result, started, &mut on_progress) } fn finish_progress( mut progress: MongoDatabaseDumpProgress, result: Result<(), String>, started: Instant, callback: &mut impl FnMut(MongoDatabaseDumpProgress), ) -> Result { progress.phase = "done".into(); progress.status = match &result { Ok(()) => "done", Err(error) if matches!(error.as_str(), "MongoDB dump/restore cancelled" | "Import cancelled" | "Export cancelled") => { "cancelled" } Err(_) => "error", } .into(); progress.error_message = result.as_ref().err().cloned(); progress.emit(started, callback); result.map(|_| progress) } #[cfg(test)] mod tests { use super::*; #[test] fn view_ordering_handles_deep_dependency_chains() { let count = 20_000; let entries = (0..count) .map(|index| { let mut metadata = CollectionMetadata::empty(&format!("view_{index}")); if index + 1 < count { metadata.kind = "view".into(); metadata.options = doc! { "viewOn": format!("view_{}", index + 1), "pipeline": [] }; } PreparedCollection { database: "source".into(), metadata, path: None, documents: 0, size_bytes: 0, metadata_path: None, } }) .collect::>(); let ordered = ordered_entries(&entries).unwrap(); assert_eq!(ordered.len(), count); for (entry, index) in ordered.iter().zip((0..count).rev()) { assert_eq!(entry.metadata.collection_name, format!("view_{index}")); } } }