feat(mongo): add collection clone
This commit is contained in:
1 parent
e80942de17
commit
5fe3b5b435
34 files changed
+1217
-21
No files matched your search
@@ -972,6 +972,7 @@ pub enum AgentCapability {
|
||||
EtcdLease,
|
||||
EtcdAuth,
|
||||
MongoDropDatabase,
|
||||
MongoCloneCollection,
|
||||
MultiSession,
|
||||
StructuredErrorV1,
|
||||
}
|
||||
@@ -1054,7 +1055,7 @@ fn parse_agent_rpc_error_header(header: &str) -> (Option<i64>, String) {
|
||||
}
|
||||
|
||||
impl AgentCapability {
|
||||
pub const ALL: [Self; 21] = [
|
||||
pub const ALL: [Self; 22] = [
|
||||
Self::Connect,
|
||||
Self::TestConnection,
|
||||
Self::Metadata,
|
||||
@@ -1074,6 +1075,7 @@ impl AgentCapability {
|
||||
Self::EtcdLease,
|
||||
Self::EtcdAuth,
|
||||
Self::MongoDropDatabase,
|
||||
Self::MongoCloneCollection,
|
||||
Self::MultiSession,
|
||||
Self::StructuredErrorV1,
|
||||
];
|
||||
@@ -1099,6 +1101,7 @@ impl AgentCapability {
|
||||
Self::EtcdLease => "etcd_lease",
|
||||
Self::EtcdAuth => "etcd_auth",
|
||||
Self::MongoDropDatabase => "mongo_drop_database",
|
||||
Self::MongoCloneCollection => "mongo_clone_collection",
|
||||
Self::MultiSession => "multi_session",
|
||||
Self::StructuredErrorV1 => "structured_error_v1",
|
||||
}
|
||||
@@ -1282,6 +1285,7 @@ pub enum MongoAgentMethod {
|
||||
CreateUser,
|
||||
DropIndexes,
|
||||
DropCollection,
|
||||
CloneCollection,
|
||||
DropDatabase,
|
||||
InsertDocument,
|
||||
UpdateDocument,
|
||||
@@ -1291,7 +1295,7 @@ pub enum MongoAgentMethod {
|
||||
}
|
||||
|
||||
impl MongoAgentMethod {
|
||||
pub const ALL: [Self; 19] = [
|
||||
pub const ALL: [Self; 20] = [
|
||||
Self::ListDatabases,
|
||||
Self::ListCollections,
|
||||
Self::FindDocuments,
|
||||
@@ -1305,6 +1309,7 @@ impl MongoAgentMethod {
|
||||
Self::CreateUser,
|
||||
Self::DropIndexes,
|
||||
Self::DropCollection,
|
||||
Self::CloneCollection,
|
||||
Self::DropDatabase,
|
||||
Self::InsertDocument,
|
||||
Self::UpdateDocument,
|
||||
@@ -1328,6 +1333,7 @@ impl MongoAgentMethod {
|
||||
Self::CreateUser => "create_user",
|
||||
Self::DropIndexes => "drop_indexes",
|
||||
Self::DropCollection => "drop_collection",
|
||||
Self::CloneCollection => "clone_collection",
|
||||
Self::DropDatabase => "drop_database",
|
||||
Self::InsertDocument => "insert_document",
|
||||
Self::UpdateDocument => "update_document",
|
||||
@@ -2544,6 +2550,13 @@ impl AgentDriverClient {
|
||||
self.call_mongo_method(MongoAgentMethod::DropCollection, params).await
|
||||
}
|
||||
|
||||
pub async fn mongo_clone_collection<T: DeserializeOwned + Send + 'static>(
|
||||
&mut self,
|
||||
params: Value,
|
||||
) -> Result<T, String> {
|
||||
self.call_mongo_method(MongoAgentMethod::CloneCollection, params).await
|
||||
}
|
||||
|
||||
pub async fn mongo_drop_database<T: DeserializeOwned + Send + 'static>(
|
||||
&mut self,
|
||||
params: Value,
|
||||
@@ -2724,6 +2737,7 @@ pub fn agent_supports_capability(handshake: Option<&AgentHandshake>, capability:
|
||||
| AgentCapability::EtcdLease
|
||||
| AgentCapability::EtcdAuth
|
||||
| AgentCapability::MongoDropDatabase
|
||||
| AgentCapability::MongoCloneCollection
|
||||
) {
|
||||
return handshake.map(|value| value.supports(capability)).unwrap_or(false);
|
||||
}
|
||||
@@ -4307,9 +4321,10 @@ for line in sys.stdin:
|
||||
assert_eq!(AgentCapability::EtcdLease.as_str(), "etcd_lease");
|
||||
assert_eq!(AgentCapability::EtcdAuth.as_str(), "etcd_auth");
|
||||
assert_eq!(AgentCapability::MongoDropDatabase.as_str(), "mongo_drop_database");
|
||||
assert_eq!(AgentCapability::MongoCloneCollection.as_str(), "mongo_clone_collection");
|
||||
assert_eq!(AgentCapability::MultiSession.as_str(), "multi_session");
|
||||
assert_eq!(AgentCapability::StructuredErrorV1.as_str(), "structured_error_v1");
|
||||
assert_eq!(AgentCapability::ALL.len(), 21);
|
||||
assert_eq!(AgentCapability::ALL.len(), 22);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -4361,6 +4376,7 @@ for line in sys.stdin:
|
||||
assert_eq!(MongoAgentMethod::CreateUser.as_str(), "create_user");
|
||||
assert_eq!(MongoAgentMethod::DropIndexes.as_str(), "drop_indexes");
|
||||
assert_eq!(MongoAgentMethod::DropCollection.as_str(), "drop_collection");
|
||||
assert_eq!(MongoAgentMethod::CloneCollection.as_str(), "clone_collection");
|
||||
assert_eq!(MongoAgentMethod::DropDatabase.as_str(), "drop_database");
|
||||
assert_eq!(MongoAgentMethod::InsertDocument.as_str(), "insert_document");
|
||||
assert_eq!(MongoAgentMethod::UpdateDocument.as_str(), "update_document");
|
||||
@@ -4429,6 +4445,7 @@ for line in sys.stdin:
|
||||
let _mongo_create_index = AgentDriverClient::mongo_create_index::<serde_json::Value>;
|
||||
let _mongo_drop_indexes = AgentDriverClient::mongo_drop_indexes::<serde_json::Value>;
|
||||
let _mongo_drop_collection = AgentDriverClient::mongo_drop_collection::<serde_json::Value>;
|
||||
let _mongo_clone_collection = AgentDriverClient::mongo_clone_collection::<serde_json::Value>;
|
||||
let _mongo_drop_database = AgentDriverClient::mongo_drop_database::<serde_json::Value>;
|
||||
let _mongo_insert_document = AgentDriverClient::mongo_insert_document::<serde_json::Value>;
|
||||
let _mongo_update_document = AgentDriverClient::mongo_update_document::<serde_json::Value>;
|
||||
@@ -4662,10 +4679,18 @@ for line in sys.stdin:
|
||||
assert!(!agent_supports_capability(Some(&handshake), AgentCapability::KvHistory));
|
||||
assert!(!agent_supports_capability(None, AgentCapability::MongoDropDatabase));
|
||||
assert!(!agent_supports_capability(Some(&handshake), AgentCapability::MongoDropDatabase));
|
||||
assert!(!agent_supports_capability(None, AgentCapability::MongoCloneCollection));
|
||||
assert!(!agent_supports_capability(Some(&handshake), AgentCapability::MongoCloneCollection));
|
||||
|
||||
let mongo_handshake =
|
||||
AgentHandshake { capabilities: vec![AgentCapability::MongoDropDatabase.as_str().to_string()], ..handshake };
|
||||
assert!(agent_supports_capability(Some(&mongo_handshake), AgentCapability::MongoDropDatabase));
|
||||
|
||||
let mongo_clone_handshake = AgentHandshake {
|
||||
capabilities: vec![AgentCapability::MongoCloneCollection.as_str().to_string()],
|
||||
..mongo_handshake
|
||||
};
|
||||
assert!(agent_supports_capability(Some(&mongo_clone_handshake), AgentCapability::MongoCloneCollection));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -58,6 +58,13 @@ pub struct MongoCollectionStatsResult {
|
||||
pub nindexes: serde_json::Value,
|
||||
}
|
||||
|
||||
/// Result counts returned after cloning a regular MongoDB collection.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct MongoCloneCollectionResult {
|
||||
pub documents_copied: u64,
|
||||
pub indexes_copied: u64,
|
||||
}
|
||||
|
||||
pub async fn connect(url: &str, timeout: Duration, idle_timeout: Duration) -> Result<Client, String> {
|
||||
let url = normalize_mongo_uri_direct_connection(url);
|
||||
let is_multi_host = is_multi_host_mongo_uri(&url);
|
||||
@@ -645,6 +652,122 @@ pub async fn rename_collection(client: &Client, database: &str, old_name: &str,
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Clone a regular collection within one database without relying on MongoDB's
|
||||
/// deprecated clone commands. The individual commands used here have been
|
||||
/// available across the MongoDB versions supported by the native driver.
|
||||
pub async fn clone_collection(
|
||||
client: &Client,
|
||||
database: &str,
|
||||
source_name: &str,
|
||||
target_name: &str,
|
||||
) -> Result<MongoCloneCollectionResult, String> {
|
||||
validate_clone_collection_names(database, source_name, target_name)?;
|
||||
|
||||
let database = client.database(database);
|
||||
let source_spec = find_collection_specification(&database, source_name).await?;
|
||||
if !matches!(source_spec.collection_type, mongodb::results::CollectionType::Collection) {
|
||||
return Err(
|
||||
"Only regular MongoDB collections can be cloned; views and time-series collections are not supported"
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
// Create explicitly before copying data so an existing target fails rather
|
||||
// than being silently merged with or overwritten by the source documents.
|
||||
database
|
||||
.create_collection(target_name)
|
||||
.with_options(source_spec.options.clone())
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
|
||||
let source = database.collection::<Document>(source_name);
|
||||
let target = database.collection::<Document>(target_name);
|
||||
let needs_validation_bypass = source_spec.options.validator.is_some()
|
||||
|| source_spec.options.validation_level.is_some()
|
||||
|| source_spec.options.validation_action.is_some();
|
||||
let mut cursor = source.find(doc! {}).await.map_err(|error| error.to_string())?;
|
||||
let mut batch = Vec::with_capacity(1_000);
|
||||
let mut documents_copied = 0_u64;
|
||||
|
||||
while let Some(document) = cursor.try_next().await.map_err(|error| error.to_string())? {
|
||||
batch.push(document);
|
||||
if batch.len() == 1_000 {
|
||||
documents_copied += insert_clone_batch(&target, &mut batch, needs_validation_bypass).await?;
|
||||
}
|
||||
}
|
||||
if !batch.is_empty() {
|
||||
documents_copied += insert_clone_batch(&target, &mut batch, needs_validation_bypass).await?;
|
||||
}
|
||||
|
||||
// The target gets its _id index during createCollection. Recreating every
|
||||
// other source index after the data copy avoids needless index maintenance.
|
||||
let mut index_cursor = source.list_indexes().await.map_err(|error| error.to_string())?;
|
||||
let mut indexes_copied = 0_u64;
|
||||
while let Some(index) = index_cursor.try_next().await.map_err(|error| error.to_string())? {
|
||||
if is_automatic_id_index(&index) {
|
||||
continue;
|
||||
}
|
||||
target.create_index(index).await.map_err(|error| error.to_string())?;
|
||||
indexes_copied += 1;
|
||||
}
|
||||
|
||||
Ok(MongoCloneCollectionResult { documents_copied, indexes_copied })
|
||||
}
|
||||
|
||||
async fn find_collection_specification(
|
||||
database: &Database,
|
||||
source_name: &str,
|
||||
) -> Result<mongodb::results::CollectionSpecification, String> {
|
||||
let mut cursor = database.list_collections().await.map_err(|error| error.to_string())?;
|
||||
while let Some(specification) = cursor.try_next().await.map_err(|error| error.to_string())? {
|
||||
if specification.name == source_name {
|
||||
return Ok(specification);
|
||||
}
|
||||
}
|
||||
Err(format!("MongoDB collection '{source_name}' was not found"))
|
||||
}
|
||||
|
||||
async fn insert_clone_batch(
|
||||
target: &mongodb::Collection<Document>,
|
||||
batch: &mut Vec<Document>,
|
||||
bypass_document_validation: bool,
|
||||
) -> Result<u64, String> {
|
||||
let documents = std::mem::take(batch);
|
||||
let result = if bypass_document_validation {
|
||||
target.insert_many(documents).bypass_document_validation(true).await
|
||||
} else {
|
||||
target.insert_many(documents).await
|
||||
}
|
||||
.map_err(|error| error.to_string())?;
|
||||
Ok(result.inserted_ids.len() as u64)
|
||||
}
|
||||
|
||||
/// `createCollection` always creates the `_id` index, but it need not be named
|
||||
/// `_id_` (for example, on a clustered collection). Compare the key instead
|
||||
/// of the name so cloning never attempts to create it a second time.
|
||||
fn is_automatic_id_index(index: &IndexModel) -> bool {
|
||||
index.keys.len() == 1
|
||||
&& (matches!(index.keys.get("_id"), Some(Bson::Int32(1) | Bson::Int64(1)))
|
||||
|| matches!(index.keys.get("_id"), Some(Bson::Double(value)) if *value == 1.0))
|
||||
}
|
||||
|
||||
pub(crate) fn validate_clone_collection_names(
|
||||
database: &str,
|
||||
source_name: &str,
|
||||
target_name: &str,
|
||||
) -> Result<(), String> {
|
||||
validate_mongo_namespace_name(database, "Database")?;
|
||||
validate_mongo_namespace_name(source_name, "Source collection")?;
|
||||
validate_mongo_namespace_name(target_name, "Target collection")?;
|
||||
if source_name == target_name {
|
||||
return Err("Target collection name must differ from the source collection name".to_string());
|
||||
}
|
||||
if source_name.starts_with("system.") || target_name.starts_with("system.") {
|
||||
return Err("System collections cannot be cloned".to_string());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn list_indexes(client: &Client, database: &str, collection: &str) -> Result<Vec<IndexInfo>, String> {
|
||||
let col = client.database(database).collection::<Document>(collection);
|
||||
let mut cursor = col.list_indexes().await.map_err(|e| e.to_string())?;
|
||||
@@ -2568,6 +2691,33 @@ mod tests {
|
||||
assert!(!command.contains_key("dropTarget"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clone_collection_validation_preserves_identifiers_and_rejects_unsafe_targets() {
|
||||
validate_clone_collection_names("app", " users ", " users_backup ").unwrap();
|
||||
assert!(validate_clone_collection_names("app", "users", "users").unwrap_err().contains("differ"));
|
||||
assert!(validate_clone_collection_names("app", "system.users", "users_backup")
|
||||
.unwrap_err()
|
||||
.contains("System collections"));
|
||||
assert!(validate_clone_collection_names("app", "users", "system.users_backup")
|
||||
.unwrap_err()
|
||||
.contains("System collections"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clone_collection_skips_the_automatic_id_index_by_key_not_name() {
|
||||
let automatic_id = IndexModel::builder()
|
||||
.keys(doc! { "_id": 1 })
|
||||
.options(IndexOptions::builder().name("custom_id_name".to_string()).build())
|
||||
.build();
|
||||
let ordinary_index = IndexModel::builder()
|
||||
.keys(doc! { "external_id": 1 })
|
||||
.options(IndexOptions::builder().name("external_id_1".to_string()).build())
|
||||
.build();
|
||||
|
||||
assert!(is_automatic_id_index(&automatic_id));
|
||||
assert!(!is_automatic_id_index(&ordinary_index));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mongo_collection_kind_as_str_is_stable_wire_value() {
|
||||
assert_eq!(MongoCollectionKind::Collection.as_str(), "collection");
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
use crate::connection::{AppState, PoolKind};
|
||||
use crate::db::agent_driver::AgentCapability;
|
||||
use crate::db::mongo_driver::{
|
||||
self, MongoCollectionStatsResult, MongoDocumentResult, MongoDropIndexFailure, MongoDropIndexesResult,
|
||||
self, MongoCloneCollectionResult, MongoCollectionStatsResult, MongoDocumentResult, MongoDropIndexFailure,
|
||||
MongoDropIndexesResult,
|
||||
};
|
||||
use crate::document_ops::CollectionInfo;
|
||||
use crate::mongo_shell::MongoCommand;
|
||||
@@ -96,6 +97,40 @@ pub async fn mongo_rename_collection_core(
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn mongo_clone_collection_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
database: &str,
|
||||
source_collection: &str,
|
||||
target_collection: &str,
|
||||
) -> Result<MongoCloneCollectionResult, String> {
|
||||
mongo_driver::validate_clone_collection_names(database, source_collection, target_collection)?;
|
||||
ensure_document_pool(state, connection_id).await?;
|
||||
let connections = state.connections.read().await;
|
||||
match connections.get(connection_id).ok_or("Not found")? {
|
||||
PoolKind::MongoDb(client) => {
|
||||
mongo_driver::clone_collection(client, database, source_collection, target_collection).await
|
||||
}
|
||||
PoolKind::Agent(client) => {
|
||||
let mut client = client.lock().await;
|
||||
if !client.supports_capability(AgentCapability::MongoCloneCollection) {
|
||||
return Err(
|
||||
"MongoDB Legacy Agent does not support clone collection; upgrade or reinstall the MongoDB Legacy driver"
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
client
|
||||
.mongo_clone_collection(serde_json::json!({
|
||||
"database": database,
|
||||
"source_collection": source_collection,
|
||||
"target_collection": target_collection,
|
||||
}))
|
||||
.await
|
||||
}
|
||||
_ => Err("Not a MongoDB connection".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn mongo_server_version_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
@@ -1328,6 +1363,47 @@ for line in sys.stdin:
|
||||
mongo_drop_collection_core(&state, "legacy", "app", "users").await.unwrap();
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn mongo_clone_collection_routes_legacy_connections_to_the_agent() {
|
||||
let (state, _directory) = legacy_mongo_state_with_capabilities(
|
||||
"clone_collection",
|
||||
serde_json::json!({
|
||||
"database": "app",
|
||||
"source_collection": "users",
|
||||
"target_collection": "users_copy",
|
||||
}),
|
||||
serde_json::json!({ "documents_copied": 2, "indexes_copied": 1 }),
|
||||
&[AgentCapability::MongoCloneCollection.as_str()],
|
||||
)
|
||||
.await;
|
||||
|
||||
let result = mongo_clone_collection_core(&state, "legacy", "app", "users", "users_copy").await.unwrap();
|
||||
|
||||
assert_eq!(result, MongoCloneCollectionResult { documents_copied: 2, indexes_copied: 1 });
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn mongo_clone_collection_requires_an_explicit_legacy_agent_capability() {
|
||||
let (state, _directory) = legacy_mongo_state_with_capabilities(
|
||||
"clone_collection",
|
||||
serde_json::json!({
|
||||
"database": "app",
|
||||
"source_collection": "users",
|
||||
"target_collection": "users_copy",
|
||||
}),
|
||||
serde_json::json!({ "documents_copied": 2, "indexes_copied": 1 }),
|
||||
&[],
|
||||
)
|
||||
.await;
|
||||
|
||||
let error = mongo_clone_collection_core(&state, "legacy", "app", "users", "users_copy").await.unwrap_err();
|
||||
|
||||
assert!(error.contains("upgrade or reinstall"), "{error}");
|
||||
assert!(!error.contains("Unknown method"), "{error}");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn mongo_list_collections_requests_legacy_collection_types() {
|
||||
|
||||
Reference in new issue
Block a user