feat(mongodb): support database dump and restore over legacy agent
This commit is contained in:
1 parent
06fc0df234
commit
a67bbb4ed5
6 files changed
+453
-98
No files matched your search
@@ -21,10 +21,15 @@ use std::sync::{
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use futures::TryStreamExt;
|
||||
use mongodb::bson::{doc, Document};
|
||||
use mongodb::bson::{doc, Document, RawDocumentBuf};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::connection::{AppState, PoolKind};
|
||||
use crate::db::agent_driver::{AgentCapability, PooledAgentClient};
|
||||
use crate::db::mongo_driver::{
|
||||
agent_insert_outcome, document_to_canonical_extended_json, insert_bson_documents,
|
||||
json_object_to_document_extended_json, MongoCollectionKind, MongoDocumentResult, MongoInsertOutcome,
|
||||
};
|
||||
use crate::mongodb_import_export::{export_mongodb_query_core, MongoExportFormat, MongoExportRequest};
|
||||
use metadata::CollectionMetadata;
|
||||
|
||||
@@ -206,33 +211,205 @@ fn catalog(entries: &[PreparedCollection], exact: bool) -> MongoDumpCatalog {
|
||||
}
|
||||
}
|
||||
|
||||
async fn native_client(state: &AppState, id: &str, database: &str) -> Result<mongodb::Client, String> {
|
||||
metadata::validate_database(database)?;
|
||||
state.get_or_create_pool(id, Some(database)).await?;
|
||||
match state.pool_handle(id).await {
|
||||
Some(PoolKind::MongoDb(client)) => Ok(client.clone()),
|
||||
_ => Err("Database dump/restore requires the native MongoDB driver".into()),
|
||||
/// A `listCollections` entry with what dump/restore needs from it.
|
||||
pub(crate) struct DumpCollectionSpec {
|
||||
pub name: String,
|
||||
pub kind: MongoCollectionKind,
|
||||
pub options: Document,
|
||||
}
|
||||
|
||||
/// The connection a dump or restore runs over. Both drivers expose the same handful of
|
||||
/// commands; the legacy agent runs them through its generic `runCommand` so nothing has to
|
||||
/// be added to the agent itself.
|
||||
pub(crate) enum DumpClient {
|
||||
Native(mongodb::Client),
|
||||
Agent(Arc<PooledAgentClient>),
|
||||
}
|
||||
|
||||
impl DumpClient {
|
||||
async fn run_command(&self, database: &str, command: Document) -> Result<Document, String> {
|
||||
match self {
|
||||
Self::Native(client) => client.database(database).run_command(command).await.map_err(|e| e.to_string()),
|
||||
Self::Agent(client) => {
|
||||
let mut client = client.lock().await;
|
||||
let result: MongoDocumentResult = client
|
||||
.mongo_run_command(serde_json::json!({
|
||||
"database": database,
|
||||
"command_json": document_to_canonical_extended_json(&command).to_string(),
|
||||
}))
|
||||
.await?;
|
||||
let response = result
|
||||
.extended_documents
|
||||
.as_ref()
|
||||
.and_then(|documents| documents.first())
|
||||
.or_else(|| result.documents.first())
|
||||
.ok_or("MongoDB Legacy Agent returned an empty runCommand response")?;
|
||||
json_object_to_document_extended_json(response)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Drains a command cursor (`listCollections`, `listIndexes`) through `getMore` so a database
|
||||
/// with more entries than one batch is still read completely on the agent path.
|
||||
async fn command_cursor(&self, database: &str, command: Document) -> Result<Vec<Document>, String> {
|
||||
let mut response = self.run_command(database, command).await?;
|
||||
let mut documents = Vec::new();
|
||||
loop {
|
||||
let cursor = response.get_document_mut("cursor").map_err(|e| e.to_string())?;
|
||||
let batch = if cursor.contains_key("firstBatch") { "firstBatch" } else { "nextBatch" };
|
||||
if let Ok(entries) = cursor.get_array_mut(batch) {
|
||||
documents.extend(std::mem::take(entries).into_iter().filter_map(|entry| match entry {
|
||||
mongodb::bson::Bson::Document(document) => Some(document),
|
||||
_ => None,
|
||||
}));
|
||||
}
|
||||
let id = cursor.get_i64("id").unwrap_or(0);
|
||||
if id == 0 {
|
||||
return Ok(documents);
|
||||
}
|
||||
let namespace = cursor.get_str("ns").map_err(|e| e.to_string())?;
|
||||
let collection =
|
||||
namespace.strip_prefix(database).and_then(|rest| rest.strip_prefix('.')).unwrap_or(namespace);
|
||||
response = self.run_command(database, doc! { "getMore": id, "collection": collection }).await?;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn list_collections(&self, database: &str) -> Result<Vec<DumpCollectionSpec>, String> {
|
||||
match self {
|
||||
Self::Native(client) => {
|
||||
let mut cursor = client.database(database).list_collections().await.map_err(|e| e.to_string())?;
|
||||
let mut specs = Vec::new();
|
||||
while let Some(spec) = cursor.try_next().await.map_err(|e| e.to_string())? {
|
||||
specs.push(DumpCollectionSpec {
|
||||
name: spec.name,
|
||||
kind: MongoCollectionKind::from_driver_type(&spec.collection_type),
|
||||
options: mongodb::bson::to_document(&spec.options).map_err(|e| e.to_string())?,
|
||||
});
|
||||
}
|
||||
Ok(specs)
|
||||
}
|
||||
Self::Agent(_) => self
|
||||
.command_cursor(database, doc! { "listCollections": 1 })
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|mut spec| {
|
||||
let name = spec.get_str("name").map_err(|e| format!("listCollections entry: {e}"))?.to_string();
|
||||
let kind = MongoCollectionKind::from_metadata_kind(spec.get_str("type").ok());
|
||||
let options = spec.remove("options").and_then(|o| o.as_document().cloned()).unwrap_or_default();
|
||||
Ok(DumpCollectionSpec { name, kind, options })
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn list_indexes(&self, database: &str, collection: &str) -> Result<Vec<Document>, String> {
|
||||
match self {
|
||||
Self::Native(client) => {
|
||||
let mut indexes = client
|
||||
.database(database)
|
||||
.collection::<Document>(collection)
|
||||
.list_indexes()
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
let mut documents = Vec::new();
|
||||
while let Some(index) = indexes.try_next().await.map_err(|e| e.to_string())? {
|
||||
documents.push(mongodb::bson::to_document(&index).map_err(|e| e.to_string())?);
|
||||
}
|
||||
Ok(documents)
|
||||
}
|
||||
Self::Agent(_) => self.command_cursor(database, doc! { "listIndexes": collection }).await,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn drop_collection(&self, database: &str, collection: &str) -> Result<(), String> {
|
||||
match self {
|
||||
Self::Native(client) => {
|
||||
client.database(database).collection::<Document>(collection).drop().await.map_err(|e| e.to_string())
|
||||
}
|
||||
Self::Agent(_) => self.run_command(database, doc! { "drop": collection }).await.map(|_| ()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn create_collection(&self, database: &str, command: Document) -> Result<(), String> {
|
||||
self.run_command(database, command).await.map(|_| ())
|
||||
}
|
||||
|
||||
/// Inserts one restore batch. The agent takes documents as canonical Extended JSON, the same
|
||||
/// lossless form collection-level BSON import already uses on that path.
|
||||
pub(crate) async fn insert_documents(
|
||||
&self,
|
||||
database: &str,
|
||||
collection: &str,
|
||||
documents: Vec<RawDocumentBuf>,
|
||||
) -> Result<MongoInsertOutcome, String> {
|
||||
match self {
|
||||
Self::Native(client) => {
|
||||
insert_bson_documents(client, database, collection, documents).await.map_err(|e| e.message)
|
||||
}
|
||||
Self::Agent(client) => {
|
||||
let documents = documents
|
||||
.iter()
|
||||
.map(|raw| {
|
||||
Document::try_from(raw.as_ref()).map(|document| document_to_canonical_extended_json(&document))
|
||||
})
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.map_err(|e| e.to_string())?;
|
||||
let docs_json = serde_json::to_string(&documents).map_err(|e| e.to_string())?;
|
||||
let mut client = client.lock().await;
|
||||
let result: serde_json::Value = client
|
||||
.mongo_insert_documents(serde_json::json!({
|
||||
"database": database,
|
||||
"collection": collection,
|
||||
"docs_json": docs_json,
|
||||
"ordered": false,
|
||||
}))
|
||||
.await?;
|
||||
agent_insert_outcome(&result)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn create_index(&self, database: &str, collection: &str, index: Document) -> Result<(), String> {
|
||||
self.run_command(database, doc! { "createIndexes": collection, "indexes": [index] }).await.map(|_| ())
|
||||
}
|
||||
|
||||
async fn server_version(&self, database: &str) -> Result<String, String> {
|
||||
Ok(self.run_command(database, doc! { "buildInfo": 1 }).await?.get_str("version").unwrap_or("").to_string())
|
||||
}
|
||||
}
|
||||
|
||||
async fn database_entries(client: &mongodb::Client, database: &str) -> Result<Vec<PreparedCollection>, String> {
|
||||
let db = client.database(database);
|
||||
let mut cursor = db.list_collections().await.map_err(|e| e.to_string())?;
|
||||
async fn dump_client(state: &AppState, id: &str, database: &str) -> Result<DumpClient, String> {
|
||||
metadata::validate_database(database)?;
|
||||
state.get_or_create_pool(id, Some(database)).await?;
|
||||
match state.pool_handle(id).await {
|
||||
Some(PoolKind::MongoDb(client)) => Ok(DumpClient::Native(client.clone())),
|
||||
Some(PoolKind::Agent(client)) => {
|
||||
let supported = {
|
||||
let client = client.lock().await;
|
||||
client.supports_capability(AgentCapability::MongoRunCommand)
|
||||
&& client.supports_capability(AgentCapability::MongoInsertDocuments)
|
||||
};
|
||||
if !supported {
|
||||
return Err("MongoDB Legacy Agent does not support database dump/restore; upgrade or reinstall the MongoDB Legacy driver".into());
|
||||
}
|
||||
Ok(DumpClient::Agent(client.clone()))
|
||||
}
|
||||
_ => Err("Database dump/restore requires a MongoDB connection".into()),
|
||||
}
|
||||
}
|
||||
|
||||
async fn database_entries(client: &DumpClient, database: &str) -> Result<Vec<PreparedCollection>, String> {
|
||||
let mut entries = Vec::new();
|
||||
while let Some(spec) = cursor.try_next().await.map_err(|e| e.to_string())? {
|
||||
for spec in client.list_collections(database).await? {
|
||||
if spec.name.starts_with("system.") && spec.name != "system.js" {
|
||||
continue;
|
||||
}
|
||||
let mut metadata = CollectionMetadata::empty(&spec.name);
|
||||
metadata.kind =
|
||||
crate::db::mongo_driver::MongoCollectionKind::from_driver_type(&spec.collection_type).as_str().into();
|
||||
metadata.options = mongodb::bson::to_document(&spec.options).map_err(|e| e.to_string())?;
|
||||
metadata.kind = spec.kind.as_str().into();
|
||||
metadata.options = spec.options;
|
||||
metadata.validate()?;
|
||||
if !metadata.is_view() {
|
||||
let mut indexes = db.collection::<Document>(&spec.name).list_indexes().await.map_err(|e| e.to_string())?;
|
||||
while let Some(index) = indexes.try_next().await.map_err(|e| e.to_string())? {
|
||||
metadata.indexes.push(mongodb::bson::to_document(&index).map_err(|e| e.to_string())?);
|
||||
}
|
||||
metadata.indexes = client.list_indexes(database, &spec.name).await?;
|
||||
}
|
||||
metadata.validate()?;
|
||||
entries.push(PreparedCollection {
|
||||
@@ -253,7 +430,7 @@ pub async fn inspect_mongodb_database_dump(
|
||||
id: &str,
|
||||
database: &str,
|
||||
) -> Result<MongoDumpCatalog, String> {
|
||||
let client = native_client(state, id, database).await?;
|
||||
let client = dump_client(state, id, database).await?;
|
||||
let entries = database_entries(&client, database).await?;
|
||||
let mut catalog = catalog(&entries, false);
|
||||
catalog.databases = vec![database.into()];
|
||||
@@ -370,7 +547,7 @@ where
|
||||
if is_cancelled(&request.task_id).await {
|
||||
return Err("MongoDB dump/restore cancelled".into());
|
||||
}
|
||||
let client = native_client(state, &request.connection_id, &request.database).await?;
|
||||
let client = dump_client(state, &request.connection_id, &request.database).await?;
|
||||
let mut entries = select_entries(&database_entries(&client, &request.database).await?, &request.collections)?;
|
||||
progress.collections_total = entries.len();
|
||||
let target = PathBuf::from(&request.file_path);
|
||||
@@ -428,14 +605,7 @@ where
|
||||
}
|
||||
progress.phase = "archive".into();
|
||||
progress.emit(started, &mut on_progress);
|
||||
let server_version = client
|
||||
.database(&request.database)
|
||||
.run_command(doc! { "buildInfo": 1 })
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
.get_str("version")
|
||||
.unwrap_or("")
|
||||
.to_string();
|
||||
let server_version = client.server_version(&request.database).await?;
|
||||
let database = request.database.clone();
|
||||
let gzip = request.gzip;
|
||||
let format = request.format;
|
||||
@@ -507,21 +677,13 @@ where
|
||||
let entries = ordered_entries(&select_entries(&source_entries, &request.collections)?)?;
|
||||
progress.collections_total = entries.len();
|
||||
source.check_ready(&entries)?;
|
||||
let client = native_client(state, &request.connection_id, &request.database).await?;
|
||||
let db = client.database(&request.database);
|
||||
let existing = db
|
||||
.list_collections()
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
.try_collect::<Vec<_>>()
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
let client = dump_client(state, &request.connection_id, &request.database).await?;
|
||||
let existing = client.list_collections(&request.database).await?;
|
||||
for entry in &entries {
|
||||
entry.metadata.validate()?;
|
||||
if let Some(target) = existing.iter().find(|target| target.name == entry.metadata.collection_name) {
|
||||
if !request.drop_existing
|
||||
&& (entry.metadata.is_view()
|
||||
|| !matches!(target.collection_type, mongodb::results::CollectionType::Collection))
|
||||
&& (entry.metadata.is_view() || target.kind != MongoCollectionKind::Collection)
|
||||
{
|
||||
return Err(format!(
|
||||
"{}: replacing an existing view or non-regular collection requires dropExisting",
|
||||
@@ -571,9 +733,7 @@ where
|
||||
if is_cancelled(&request.task_id).await {
|
||||
return Err("MongoDB dump/restore cancelled".into());
|
||||
}
|
||||
db.run_command(doc! { "createIndexes": name, "indexes": [index] })
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
client.create_index(&request.database, name, index).await?;
|
||||
progress.indexes_created += 1;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,10 +9,10 @@ use mongodb::bson::{Document, RawDocumentBuf};
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use super::{
|
||||
archive, cancelled, source::CatalogSource, CancelFuture, MongoDatabaseDumpProgress, MongoDatabaseRestoreRequest,
|
||||
MongoDumpFormat, PreparedCollection,
|
||||
archive, cancelled, source::CatalogSource, CancelFuture, DumpClient, MongoDatabaseDumpProgress,
|
||||
MongoDatabaseRestoreRequest, MongoDumpFormat, PreparedCollection,
|
||||
};
|
||||
use crate::{connection::AppState, db::mongo_driver::insert_bson_documents};
|
||||
use crate::connection::AppState;
|
||||
|
||||
const BATCH_BYTES: usize = 8 * 1024 * 1024;
|
||||
|
||||
@@ -142,17 +142,16 @@ fn produce(
|
||||
}
|
||||
|
||||
pub(super) async fn initialize_collection(
|
||||
client: &mongodb::Client,
|
||||
client: &DumpClient,
|
||||
request: &MongoDatabaseRestoreRequest,
|
||||
entry: &PreparedCollection,
|
||||
exists: bool,
|
||||
) -> Result<(), String> {
|
||||
let db = client.database(&request.database);
|
||||
if request.drop_existing && exists {
|
||||
db.collection::<Document>(&entry.metadata.collection_name).drop().await.map_err(|e| e.to_string())?;
|
||||
client.drop_collection(&request.database, &entry.metadata.collection_name).await?;
|
||||
}
|
||||
if !exists || request.drop_existing {
|
||||
db.run_command(entry.metadata.create_command(request.restore_options)).await.map_err(|e| e.to_string())?;
|
||||
client.create_collection(&request.database, entry.metadata.create_command(request.restore_options)).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -167,7 +166,7 @@ pub(super) async fn writable(state: &AppState, id: &str) -> Result<(), String> {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) async fn restore_data(
|
||||
state: &AppState,
|
||||
client: &mongodb::Client,
|
||||
client: &DumpClient,
|
||||
request: &MongoDatabaseRestoreRequest,
|
||||
source: Arc<CatalogSource>,
|
||||
entries: &[PreparedCollection],
|
||||
@@ -238,9 +237,7 @@ pub(super) async fn restore_data(
|
||||
progress.documents_validated += documents.len() as u64;
|
||||
}
|
||||
let outcome =
|
||||
insert_bson_documents(client, &request.database, &entry.metadata.collection_name, documents)
|
||||
.await
|
||||
.map_err(|e| e.message)?;
|
||||
client.insert_documents(&request.database, &entry.metadata.collection_name, documents).await?;
|
||||
progress.documents_written += outcome.inserted;
|
||||
progress.documents_failed += outcome.errors.len() as u64;
|
||||
progress.emit(started, on_progress);
|
||||
|
||||
@@ -2849,12 +2849,7 @@ where
|
||||
if is_cancelled(&request.export_id).await {
|
||||
return Err("Export cancelled".to_string());
|
||||
}
|
||||
// BSON dumps must carry native BSON types, so they are written straight from the native
|
||||
// driver's documents; the legacy agent's canonical Extended JSON round trip does not apply.
|
||||
let client = match state.pool_handle(&request.connection_id).await {
|
||||
Some(PoolKind::MongoDb(client)) => client,
|
||||
_ => return Err("BSON dump export requires the native MongoDB driver".into()),
|
||||
};
|
||||
let pool = state.pool_handle(&request.connection_id).await.ok_or_else(|| "Not found".to_string())?;
|
||||
let file = File::create(temp).map_err(|error| error.to_string())?;
|
||||
let buffered = BufWriter::new(file);
|
||||
let mut writer = if request.gzip {
|
||||
@@ -2864,48 +2859,64 @@ where
|
||||
};
|
||||
let mut documents_read = 0u64;
|
||||
let mut bytes_written = 0u64;
|
||||
let mut write_document = |document: Document| -> Result<(), String> {
|
||||
let bytes = mongodb::bson::to_vec(&document).map_err(|error| error.to_string())?;
|
||||
writer.write_all(&bytes).map_err(|error| error.to_string())?;
|
||||
bytes_written += bytes.len() as u64;
|
||||
documents_read += 1;
|
||||
if documents_read == 1 || documents_read.is_multiple_of(500) {
|
||||
on_progress(export_progress(
|
||||
&request.export_id,
|
||||
MongoExportStatus::Running,
|
||||
documents_read,
|
||||
bytes_written,
|
||||
total_documents,
|
||||
None,
|
||||
started_at,
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
|
||||
let export = for_each_find_document(
|
||||
&client,
|
||||
&request.database,
|
||||
&request.collection,
|
||||
request.filter.as_deref(),
|
||||
request.projection.as_deref(),
|
||||
request.sort.as_deref(),
|
||||
request.collation.as_deref(),
|
||||
DEFAULT_EXPORT_BATCH_SIZE,
|
||||
|document| {
|
||||
let bytes = mongodb::bson::to_vec(&document).map_err(|error| error.to_string())?;
|
||||
writer.write_all(&bytes).map_err(|error| error.to_string())?;
|
||||
bytes_written += bytes.len() as u64;
|
||||
documents_read += 1;
|
||||
if documents_read == 1 || documents_read.is_multiple_of(500) {
|
||||
on_progress(export_progress(
|
||||
&request.export_id,
|
||||
MongoExportStatus::Running,
|
||||
documents_read,
|
||||
bytes_written,
|
||||
total_documents,
|
||||
None,
|
||||
started_at,
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
);
|
||||
{
|
||||
tokio::pin!(export);
|
||||
let mut poll_cancel = tokio::time::interval(std::time::Duration::from_millis(100));
|
||||
loop {
|
||||
tokio::select! {
|
||||
result = &mut export => { result?; break; }
|
||||
_ = poll_cancel.tick() => {
|
||||
if is_cancelled(&request.export_id).await {
|
||||
return Err("Export cancelled".to_string());
|
||||
match &pool {
|
||||
PoolKind::MongoDb(client) => {
|
||||
let export = for_each_find_document(
|
||||
client,
|
||||
&request.database,
|
||||
&request.collection,
|
||||
request.filter.as_deref(),
|
||||
request.projection.as_deref(),
|
||||
request.sort.as_deref(),
|
||||
request.collation.as_deref(),
|
||||
DEFAULT_EXPORT_BATCH_SIZE,
|
||||
&mut write_document,
|
||||
);
|
||||
tokio::pin!(export);
|
||||
let mut poll_cancel = tokio::time::interval(std::time::Duration::from_millis(100));
|
||||
loop {
|
||||
tokio::select! {
|
||||
result = &mut export => { result?; break; }
|
||||
_ = poll_cancel.tick() => {
|
||||
if is_cancelled(&request.export_id).await {
|
||||
return Err("Export cancelled".to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
PoolKind::Agent(client) => {
|
||||
// A BSON dump must keep every BSON type. The agent's find cursor streams canonical
|
||||
// Extended JSON, which round-trips losslessly; the older page-based fallback is
|
||||
// relaxed JSON (Int32/Int64 collapse), so it is not accepted here.
|
||||
if !client.lock().await.supports_capability(AgentCapability::MongoFindCursor) {
|
||||
return Err("BSON export over the MongoDB Legacy Agent needs find-cursor support; upgrade or reinstall the MongoDB Legacy driver".into());
|
||||
}
|
||||
for_each_agent_export_document(client, request, is_cancelled, |value| {
|
||||
write_document(mongo_driver::json_object_to_document_extended_json(&value)?)
|
||||
})
|
||||
.await?;
|
||||
}
|
||||
_ => return Err("Not a MongoDB connection".to_string()),
|
||||
}
|
||||
writer.finish()?;
|
||||
if is_cancelled(&request.export_id).await {
|
||||
|
||||
@@ -721,6 +721,193 @@ async fn official_database_tools_round_trip_through_dbx() {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "opt-in: DBX_MONGO_LEGACY_DUMP_TEST_HOST (host:port, MongoDB 3.6+ without auth) and an installed MongoDB Legacy Agent; creates temporary databases"]
|
||||
async fn legacy_agent_database_round_trip_through_dbx() {
|
||||
use dbx_core::{
|
||||
connection::{AppState, PoolKind},
|
||||
models::connection::ConnectionConfig,
|
||||
mongo_ops::mongo_run_command_core,
|
||||
mongodb_dump::*,
|
||||
storage::Storage,
|
||||
};
|
||||
// Everything, including seeding and verification, goes through the legacy agent: the native
|
||||
// Rust driver refuses servers older than 4.2, which is the very case the agent exists for.
|
||||
async fn command(state: &AppState, id: &str, database: &str, command: Document) -> Document {
|
||||
let json = mongodb::bson::Bson::Document(command).into_canonical_extjson().to_string();
|
||||
let result = mongo_run_command_core(state, id, database, &json).await.unwrap();
|
||||
let response = result.extended_documents.unwrap().remove(0);
|
||||
let response = dbx_core::db::mongo_driver::json_object_to_document_extended_json(&response).unwrap();
|
||||
assert_eq!(
|
||||
response.get_f64("ok").ok().or_else(|| response.get_i32("ok").ok().map(f64::from)),
|
||||
Some(1.0),
|
||||
"{response}"
|
||||
);
|
||||
response
|
||||
}
|
||||
fn batch(response: &Document) -> Vec<Document> {
|
||||
let cursor = response.get_document("cursor").unwrap();
|
||||
cursor.get_array("firstBatch").unwrap().iter().map(|b| b.as_document().unwrap().clone()).collect()
|
||||
}
|
||||
async fn documents(state: &AppState, id: &str, database: &str, collection: &str) -> Vec<Vec<u8>> {
|
||||
batch(&command(state, id, database, doc! { "find": collection, "sort": { "_id": 1 } }).await)
|
||||
.iter()
|
||||
.map(|document| mongodb::bson::to_vec(document).unwrap())
|
||||
.collect()
|
||||
}
|
||||
async fn specs(state: &AppState, id: &str, database: &str, collection: &str) -> (Document, Vec<Document>) {
|
||||
let listed =
|
||||
batch(&command(state, id, database, doc! { "listCollections": 1, "filter": { "name": collection } }).await);
|
||||
let spec = listed.first().unwrap_or_else(|| panic!("{database}.{collection} is missing"));
|
||||
let options = spec.get_document("options").cloned().unwrap_or_default();
|
||||
let mut indexes = Vec::new();
|
||||
if spec.get_str("type").unwrap_or("collection") != "view" {
|
||||
for mut index in batch(&command(state, id, database, doc! { "listIndexes": collection }).await) {
|
||||
index.remove("ns");
|
||||
index.remove("v");
|
||||
indexes.push(index);
|
||||
}
|
||||
indexes.sort_by_key(|index| index.get_str("name").unwrap().to_string());
|
||||
}
|
||||
(options, indexes)
|
||||
}
|
||||
|
||||
let endpoint = std::env::var("DBX_MONGO_LEGACY_DUMP_TEST_HOST").expect("DBX_MONGO_LEGACY_DUMP_TEST_HOST");
|
||||
let (host, port) = endpoint.split_once(':').expect("host:port");
|
||||
let files = tempfile::tempdir().unwrap();
|
||||
let database = format!("dbx_legacy_{}", uuid::Uuid::new_v4().simple());
|
||||
let state = AppState::new(Storage::open(&files.path().join("storage.db")).await.unwrap());
|
||||
let id = "legacy-dump-test";
|
||||
let config: ConnectionConfig = serde_json::from_value(serde_json::json!({ "id": id, "name": "Legacy dump test", "db_type": "mongodb", "host": host, "port": port.parse::<u16>().unwrap(), "username": "", "password": "", "database": database, "driver_profile": "mongodb-legacy" })).unwrap();
|
||||
state.configs.write().await.insert(id.into(), config);
|
||||
let key = state.get_or_create_pool(id, Some(&database)).await.unwrap();
|
||||
let kind = match state.pool_handle(&key).await {
|
||||
Some(PoolKind::Agent(_)) => "agent",
|
||||
Some(PoolKind::MongoDb(_)) => "native",
|
||||
Some(_) => "other",
|
||||
None => "none",
|
||||
};
|
||||
assert_eq!(kind, "agent", "test must run over the legacy agent (pool key {key})");
|
||||
|
||||
command(&state, id, &database, doc! { "create": "records", "validator": { "score": { "$gte": 0 } }, "validationLevel": "strict", "collation": { "locale": "en", "strength": 2 } }).await;
|
||||
command(&state, id, &database, doc! { "insert": "records", "documents": [
|
||||
{ "_id": 1, "score": 1i64, "name": "first", "when": mongodb::bson::DateTime::from_millis(1_700_000_000_000) },
|
||||
{ "_id": 2, "score": 2i64, "name": "second", "ratio": 0.5f64, "tags": ["a", "b"] },
|
||||
] }).await;
|
||||
command(&state, id, &database, doc! { "createIndexes": "records", "indexes": [ { "key": { "name": 1, "score": -1 }, "name": "compound_unique", "unique": true }, { "key": { "expires": 1 }, "name": "ttl", "expireAfterSeconds": 3600 }] }).await;
|
||||
command(&state, id, &database, doc! { "create": "empty" }).await;
|
||||
command(&state, id, &database, doc! { "create": "capped", "capped": true, "size": 65536i64, "max": 100i64 }).await;
|
||||
command(&state, id, &database, doc! { "insert": "capped", "documents": [ { "_id": 1, "n": 1 } ] }).await;
|
||||
command(&state, id, &database, doc! { "insert": "odd/name", "documents": [ { "_id": mongodb::bson::oid::ObjectId::new(), "binary": mongodb::bson::Binary { subtype: mongodb::bson::spec::BinarySubtype::Generic, bytes: vec![1, 2, 3] }, "big": i64::MAX } ] }).await;
|
||||
command(&state, id, &database, doc! { "create": "z_view", "viewOn": "records", "pipeline": [ { "$match": { "score": { "$gte": 1 } } } ], "collation": { "locale": "en", "strength": 2 } }).await;
|
||||
command(&state, id, &database, doc! { "create": "a_nested", "viewOn": "z_view", "pipeline": [], "collation": { "locale": "en", "strength": 2 } }).await;
|
||||
let collections = ["records", "empty", "capped", "odd/name", "z_view", "a_nested"];
|
||||
|
||||
let catalog = inspect_mongodb_database_dump(&state, id, &database).await.unwrap();
|
||||
assert_eq!(catalog.collections.len(), 6);
|
||||
|
||||
let mut databases = vec![database.clone()];
|
||||
for format in [MongoDumpFormat::Directory, MongoDumpFormat::Archive] {
|
||||
for gzip in [false, true] {
|
||||
let tag = format!("{format:?}_{gzip}");
|
||||
let output = files.path().join(format!("legacy-{tag}"));
|
||||
let dump = MongoDatabaseDumpRequest {
|
||||
task_id: uuid::Uuid::new_v4().to_string(),
|
||||
connection_id: id.into(),
|
||||
database: database.clone(),
|
||||
file_path: output.to_str().unwrap().into(),
|
||||
format,
|
||||
gzip,
|
||||
collections: None,
|
||||
};
|
||||
let dumped = dump_mongodb_database(&state, &dump, |_| Box::pin(async { false }), |_| {}).await.unwrap();
|
||||
assert_eq!(dumped.collections_done, 6);
|
||||
assert_eq!(dumped.documents_read, 4);
|
||||
|
||||
let preview = prepare_mongodb_restore_source(MongoRestoreSourceRequest {
|
||||
path: output.to_str().unwrap().into(),
|
||||
format,
|
||||
gzip,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let restored = format!("{database}_{tag}");
|
||||
databases.push(restored.clone());
|
||||
let request = MongoDatabaseRestoreRequest {
|
||||
task_id: uuid::Uuid::new_v4().to_string(),
|
||||
connection_id: id.into(),
|
||||
database: restored.clone(),
|
||||
source_database: database.clone(),
|
||||
source_ref: preview.source_ref.clone(),
|
||||
collections: None,
|
||||
drop_existing: false,
|
||||
restore_options: true,
|
||||
restore_indexes: true,
|
||||
stop_on_error: true,
|
||||
objcheck: gzip,
|
||||
batch_size: 500,
|
||||
execution_id: None,
|
||||
};
|
||||
let result =
|
||||
restore_mongodb_database(&state, &request, |_| Box::pin(async { false }), |_| {}).await.unwrap();
|
||||
assert_eq!(result.collections_done, 6);
|
||||
assert_eq!(result.documents_written, 4);
|
||||
assert_eq!(result.documents_failed, 0);
|
||||
assert_eq!(result.indexes_created, 2, "records: compound_unique + ttl; _id_ indexes are never recreated");
|
||||
for collection in collections {
|
||||
assert_eq!(
|
||||
documents(&state, id, &database, collection).await,
|
||||
documents(&state, id, &restored, collection).await,
|
||||
"data: {tag} {collection}"
|
||||
);
|
||||
assert_eq!(
|
||||
specs(&state, id, &database, collection).await,
|
||||
specs(&state, id, &restored, collection).await,
|
||||
"metadata: {tag} {collection}"
|
||||
);
|
||||
}
|
||||
|
||||
// Selective restore with dropExisting replaces only the chosen collection.
|
||||
command(&state, id, &restored, doc! { "insert": "unselected", "documents": [ { "_id": "keep" } ] }).await;
|
||||
command(
|
||||
&state,
|
||||
id,
|
||||
&restored,
|
||||
doc! { "insert": "records", "documents": [ { "_id": 999, "score": 999, "name": "remove" } ] },
|
||||
)
|
||||
.await;
|
||||
let mut selected = request.clone();
|
||||
selected.drop_existing = true;
|
||||
selected.collections = Some(vec!["records".into()]);
|
||||
restore_mongodb_database(&state, &selected, |_| Box::pin(async { false }), |_| {}).await.unwrap();
|
||||
assert_eq!(
|
||||
documents(&state, id, &database, "records").await,
|
||||
documents(&state, id, &restored, "records").await
|
||||
);
|
||||
assert_eq!(documents(&state, id, &restored, "unselected").await.len(), 1);
|
||||
|
||||
// Appending onto existing _ids reports each duplicate instead of silently succeeding.
|
||||
let mut duplicate = request.clone();
|
||||
duplicate.collections = Some(vec!["records".into()]);
|
||||
duplicate.stop_on_error = false;
|
||||
let appended =
|
||||
restore_mongodb_database(&state, &duplicate, |_| Box::pin(async { false }), |_| {}).await.unwrap();
|
||||
assert_eq!(appended.documents_written, 0);
|
||||
assert_eq!(appended.documents_failed, 2);
|
||||
assert_eq!(
|
||||
documents(&state, id, &database, "records").await,
|
||||
documents(&state, id, &restored, "records").await
|
||||
);
|
||||
|
||||
assert!(release_mongodb_restore_source(&preview.source_ref));
|
||||
println!("legacy agent {tag}: six collections/views, BSON, options and indexes match");
|
||||
}
|
||||
}
|
||||
for database in databases {
|
||||
command(&state, id, &database, doc! { "dropDatabase": 1 }).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn database_source_rejects_invalid_metadata_and_view_cycles() {
|
||||
use dbx_core::mongodb_dump::*;
|
||||
|
||||
Reference in new issue
Block a user