feat(mongo): add collection clone

This commit is contained in:
onenewcode authored and GitHub committed 2026-08-10 15:53:00 +08:00
1 parent e80942de17
commit 5fe3b5b435
34 files changed
+1217 -21

No files matched your search

+28 -3
View File
@@ -972,6 +972,7 @@ pub enum AgentCapability {
EtcdLease,
EtcdAuth,
MongoDropDatabase,
MongoCloneCollection,
MultiSession,
StructuredErrorV1,
}
@@ -1054,7 +1055,7 @@ fn parse_agent_rpc_error_header(header: &str) -> (Option<i64>, String) {
}
impl AgentCapability {
pub const ALL: [Self; 21] = [
pub const ALL: [Self; 22] = [
Self::Connect,
Self::TestConnection,
Self::Metadata,
@@ -1074,6 +1075,7 @@ impl AgentCapability {
Self::EtcdLease,
Self::EtcdAuth,
Self::MongoDropDatabase,
Self::MongoCloneCollection,
Self::MultiSession,
Self::StructuredErrorV1,
];
@@ -1099,6 +1101,7 @@ impl AgentCapability {
Self::EtcdLease => "etcd_lease",
Self::EtcdAuth => "etcd_auth",
Self::MongoDropDatabase => "mongo_drop_database",
Self::MongoCloneCollection => "mongo_clone_collection",
Self::MultiSession => "multi_session",
Self::StructuredErrorV1 => "structured_error_v1",
}
@@ -1282,6 +1285,7 @@ pub enum MongoAgentMethod {
CreateUser,
DropIndexes,
DropCollection,
CloneCollection,
DropDatabase,
InsertDocument,
UpdateDocument,
@@ -1291,7 +1295,7 @@ pub enum MongoAgentMethod {
}
impl MongoAgentMethod {
pub const ALL: [Self; 19] = [
pub const ALL: [Self; 20] = [
Self::ListDatabases,
Self::ListCollections,
Self::FindDocuments,
@@ -1305,6 +1309,7 @@ impl MongoAgentMethod {
Self::CreateUser,
Self::DropIndexes,
Self::DropCollection,
Self::CloneCollection,
Self::DropDatabase,
Self::InsertDocument,
Self::UpdateDocument,
@@ -1328,6 +1333,7 @@ impl MongoAgentMethod {
Self::CreateUser => "create_user",
Self::DropIndexes => "drop_indexes",
Self::DropCollection => "drop_collection",
Self::CloneCollection => "clone_collection",
Self::DropDatabase => "drop_database",
Self::InsertDocument => "insert_document",
Self::UpdateDocument => "update_document",
@@ -2544,6 +2550,13 @@ impl AgentDriverClient {
self.call_mongo_method(MongoAgentMethod::DropCollection, params).await
}
pub async fn mongo_clone_collection<T: DeserializeOwned + Send + 'static>(
&mut self,
params: Value,
) -> Result<T, String> {
self.call_mongo_method(MongoAgentMethod::CloneCollection, params).await
}
pub async fn mongo_drop_database<T: DeserializeOwned + Send + 'static>(
&mut self,
params: Value,
@@ -2724,6 +2737,7 @@ pub fn agent_supports_capability(handshake: Option<&AgentHandshake>, capability:
| AgentCapability::EtcdLease
| AgentCapability::EtcdAuth
| AgentCapability::MongoDropDatabase
| AgentCapability::MongoCloneCollection
) {
return handshake.map(|value| value.supports(capability)).unwrap_or(false);
}
@@ -4307,9 +4321,10 @@ for line in sys.stdin:
assert_eq!(AgentCapability::EtcdLease.as_str(), "etcd_lease");
assert_eq!(AgentCapability::EtcdAuth.as_str(), "etcd_auth");
assert_eq!(AgentCapability::MongoDropDatabase.as_str(), "mongo_drop_database");
assert_eq!(AgentCapability::MongoCloneCollection.as_str(), "mongo_clone_collection");
assert_eq!(AgentCapability::MultiSession.as_str(), "multi_session");
assert_eq!(AgentCapability::StructuredErrorV1.as_str(), "structured_error_v1");
assert_eq!(AgentCapability::ALL.len(), 21);
assert_eq!(AgentCapability::ALL.len(), 22);
}
#[test]
@@ -4361,6 +4376,7 @@ for line in sys.stdin:
assert_eq!(MongoAgentMethod::CreateUser.as_str(), "create_user");
assert_eq!(MongoAgentMethod::DropIndexes.as_str(), "drop_indexes");
assert_eq!(MongoAgentMethod::DropCollection.as_str(), "drop_collection");
assert_eq!(MongoAgentMethod::CloneCollection.as_str(), "clone_collection");
assert_eq!(MongoAgentMethod::DropDatabase.as_str(), "drop_database");
assert_eq!(MongoAgentMethod::InsertDocument.as_str(), "insert_document");
assert_eq!(MongoAgentMethod::UpdateDocument.as_str(), "update_document");
@@ -4429,6 +4445,7 @@ for line in sys.stdin:
let _mongo_create_index = AgentDriverClient::mongo_create_index::<serde_json::Value>;
let _mongo_drop_indexes = AgentDriverClient::mongo_drop_indexes::<serde_json::Value>;
let _mongo_drop_collection = AgentDriverClient::mongo_drop_collection::<serde_json::Value>;
let _mongo_clone_collection = AgentDriverClient::mongo_clone_collection::<serde_json::Value>;
let _mongo_drop_database = AgentDriverClient::mongo_drop_database::<serde_json::Value>;
let _mongo_insert_document = AgentDriverClient::mongo_insert_document::<serde_json::Value>;
let _mongo_update_document = AgentDriverClient::mongo_update_document::<serde_json::Value>;
@@ -4662,10 +4679,18 @@ for line in sys.stdin:
assert!(!agent_supports_capability(Some(&handshake), AgentCapability::KvHistory));
assert!(!agent_supports_capability(None, AgentCapability::MongoDropDatabase));
assert!(!agent_supports_capability(Some(&handshake), AgentCapability::MongoDropDatabase));
assert!(!agent_supports_capability(None, AgentCapability::MongoCloneCollection));
assert!(!agent_supports_capability(Some(&handshake), AgentCapability::MongoCloneCollection));
let mongo_handshake =
AgentHandshake { capabilities: vec![AgentCapability::MongoDropDatabase.as_str().to_string()], ..handshake };
assert!(agent_supports_capability(Some(&mongo_handshake), AgentCapability::MongoDropDatabase));
let mongo_clone_handshake = AgentHandshake {
capabilities: vec![AgentCapability::MongoCloneCollection.as_str().to_string()],
..mongo_handshake
};
assert!(agent_supports_capability(Some(&mongo_clone_handshake), AgentCapability::MongoCloneCollection));
}
#[test]
+150
View File
@@ -58,6 +58,13 @@ pub struct MongoCollectionStatsResult {
pub nindexes: serde_json::Value,
}
/// Result counts returned after cloning a regular MongoDB collection.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct MongoCloneCollectionResult {
pub documents_copied: u64,
pub indexes_copied: u64,
}
pub async fn connect(url: &str, timeout: Duration, idle_timeout: Duration) -> Result<Client, String> {
let url = normalize_mongo_uri_direct_connection(url);
let is_multi_host = is_multi_host_mongo_uri(&url);
@@ -645,6 +652,122 @@ pub async fn rename_collection(client: &Client, database: &str, old_name: &str,
Ok(())
}
/// Clone a regular collection within one database without relying on MongoDB's
/// deprecated clone commands. The individual commands used here have been
/// available across the MongoDB versions supported by the native driver.
pub async fn clone_collection(
client: &Client,
database: &str,
source_name: &str,
target_name: &str,
) -> Result<MongoCloneCollectionResult, String> {
validate_clone_collection_names(database, source_name, target_name)?;
let database = client.database(database);
let source_spec = find_collection_specification(&database, source_name).await?;
if !matches!(source_spec.collection_type, mongodb::results::CollectionType::Collection) {
return Err(
"Only regular MongoDB collections can be cloned; views and time-series collections are not supported"
.to_string(),
);
}
// Create explicitly before copying data so an existing target fails rather
// than being silently merged with or overwritten by the source documents.
database
.create_collection(target_name)
.with_options(source_spec.options.clone())
.await
.map_err(|error| error.to_string())?;
let source = database.collection::<Document>(source_name);
let target = database.collection::<Document>(target_name);
let needs_validation_bypass = source_spec.options.validator.is_some()
|| source_spec.options.validation_level.is_some()
|| source_spec.options.validation_action.is_some();
let mut cursor = source.find(doc! {}).await.map_err(|error| error.to_string())?;
let mut batch = Vec::with_capacity(1_000);
let mut documents_copied = 0_u64;
while let Some(document) = cursor.try_next().await.map_err(|error| error.to_string())? {
batch.push(document);
if batch.len() == 1_000 {
documents_copied += insert_clone_batch(&target, &mut batch, needs_validation_bypass).await?;
}
}
if !batch.is_empty() {
documents_copied += insert_clone_batch(&target, &mut batch, needs_validation_bypass).await?;
}
// The target gets its _id index during createCollection. Recreating every
// other source index after the data copy avoids needless index maintenance.
let mut index_cursor = source.list_indexes().await.map_err(|error| error.to_string())?;
let mut indexes_copied = 0_u64;
while let Some(index) = index_cursor.try_next().await.map_err(|error| error.to_string())? {
if is_automatic_id_index(&index) {
continue;
}
target.create_index(index).await.map_err(|error| error.to_string())?;
indexes_copied += 1;
}
Ok(MongoCloneCollectionResult { documents_copied, indexes_copied })
}
async fn find_collection_specification(
database: &Database,
source_name: &str,
) -> Result<mongodb::results::CollectionSpecification, String> {
let mut cursor = database.list_collections().await.map_err(|error| error.to_string())?;
while let Some(specification) = cursor.try_next().await.map_err(|error| error.to_string())? {
if specification.name == source_name {
return Ok(specification);
}
}
Err(format!("MongoDB collection '{source_name}' was not found"))
}
async fn insert_clone_batch(
target: &mongodb::Collection<Document>,
batch: &mut Vec<Document>,
bypass_document_validation: bool,
) -> Result<u64, String> {
let documents = std::mem::take(batch);
let result = if bypass_document_validation {
target.insert_many(documents).bypass_document_validation(true).await
} else {
target.insert_many(documents).await
}
.map_err(|error| error.to_string())?;
Ok(result.inserted_ids.len() as u64)
}
/// `createCollection` always creates the `_id` index, but it need not be named
/// `_id_` (for example, on a clustered collection). Compare the key instead
/// of the name so cloning never attempts to create it a second time.
fn is_automatic_id_index(index: &IndexModel) -> bool {
index.keys.len() == 1
&& (matches!(index.keys.get("_id"), Some(Bson::Int32(1) | Bson::Int64(1)))
|| matches!(index.keys.get("_id"), Some(Bson::Double(value)) if *value == 1.0))
}
pub(crate) fn validate_clone_collection_names(
database: &str,
source_name: &str,
target_name: &str,
) -> Result<(), String> {
validate_mongo_namespace_name(database, "Database")?;
validate_mongo_namespace_name(source_name, "Source collection")?;
validate_mongo_namespace_name(target_name, "Target collection")?;
if source_name == target_name {
return Err("Target collection name must differ from the source collection name".to_string());
}
if source_name.starts_with("system.") || target_name.starts_with("system.") {
return Err("System collections cannot be cloned".to_string());
}
Ok(())
}
pub async fn list_indexes(client: &Client, database: &str, collection: &str) -> Result<Vec<IndexInfo>, String> {
let col = client.database(database).collection::<Document>(collection);
let mut cursor = col.list_indexes().await.map_err(|e| e.to_string())?;
@@ -2568,6 +2691,33 @@ mod tests {
assert!(!command.contains_key("dropTarget"));
}
#[test]
fn clone_collection_validation_preserves_identifiers_and_rejects_unsafe_targets() {
validate_clone_collection_names("app", " users ", " users_backup ").unwrap();
assert!(validate_clone_collection_names("app", "users", "users").unwrap_err().contains("differ"));
assert!(validate_clone_collection_names("app", "system.users", "users_backup")
.unwrap_err()
.contains("System collections"));
assert!(validate_clone_collection_names("app", "users", "system.users_backup")
.unwrap_err()
.contains("System collections"));
}
#[test]
fn clone_collection_skips_the_automatic_id_index_by_key_not_name() {
let automatic_id = IndexModel::builder()
.keys(doc! { "_id": 1 })
.options(IndexOptions::builder().name("custom_id_name".to_string()).build())
.build();
let ordinary_index = IndexModel::builder()
.keys(doc! { "external_id": 1 })
.options(IndexOptions::builder().name("external_id_1".to_string()).build())
.build();
assert!(is_automatic_id_index(&automatic_id));
assert!(!is_automatic_id_index(&ordinary_index));
}
#[test]
fn mongo_collection_kind_as_str_is_stable_wire_value() {
assert_eq!(MongoCollectionKind::Collection.as_str(), "collection");
+77 -1
View File
@@ -1,7 +1,8 @@
use crate::connection::{AppState, PoolKind};
use crate::db::agent_driver::AgentCapability;
use crate::db::mongo_driver::{
self, MongoCollectionStatsResult, MongoDocumentResult, MongoDropIndexFailure, MongoDropIndexesResult,
self, MongoCloneCollectionResult, MongoCollectionStatsResult, MongoDocumentResult, MongoDropIndexFailure,
MongoDropIndexesResult,
};
use crate::document_ops::CollectionInfo;
use crate::mongo_shell::MongoCommand;
@@ -96,6 +97,40 @@ pub async fn mongo_rename_collection_core(
}
}
pub async fn mongo_clone_collection_core(
state: &AppState,
connection_id: &str,
database: &str,
source_collection: &str,
target_collection: &str,
) -> Result<MongoCloneCollectionResult, String> {
mongo_driver::validate_clone_collection_names(database, source_collection, target_collection)?;
ensure_document_pool(state, connection_id).await?;
let connections = state.connections.read().await;
match connections.get(connection_id).ok_or("Not found")? {
PoolKind::MongoDb(client) => {
mongo_driver::clone_collection(client, database, source_collection, target_collection).await
}
PoolKind::Agent(client) => {
let mut client = client.lock().await;
if !client.supports_capability(AgentCapability::MongoCloneCollection) {
return Err(
"MongoDB Legacy Agent does not support clone collection; upgrade or reinstall the MongoDB Legacy driver"
.to_string(),
);
}
client
.mongo_clone_collection(serde_json::json!({
"database": database,
"source_collection": source_collection,
"target_collection": target_collection,
}))
.await
}
_ => Err("Not a MongoDB connection".to_string()),
}
}
pub async fn mongo_server_version_core(
state: &AppState,
connection_id: &str,
@@ -1328,6 +1363,47 @@ for line in sys.stdin:
mongo_drop_collection_core(&state, "legacy", "app", "users").await.unwrap();
}
#[cfg(unix)]
#[tokio::test]
async fn mongo_clone_collection_routes_legacy_connections_to_the_agent() {
let (state, _directory) = legacy_mongo_state_with_capabilities(
"clone_collection",
serde_json::json!({
"database": "app",
"source_collection": "users",
"target_collection": "users_copy",
}),
serde_json::json!({ "documents_copied": 2, "indexes_copied": 1 }),
&[AgentCapability::MongoCloneCollection.as_str()],
)
.await;
let result = mongo_clone_collection_core(&state, "legacy", "app", "users", "users_copy").await.unwrap();
assert_eq!(result, MongoCloneCollectionResult { documents_copied: 2, indexes_copied: 1 });
}
#[cfg(unix)]
#[tokio::test]
async fn mongo_clone_collection_requires_an_explicit_legacy_agent_capability() {
let (state, _directory) = legacy_mongo_state_with_capabilities(
"clone_collection",
serde_json::json!({
"database": "app",
"source_collection": "users",
"target_collection": "users_copy",
}),
serde_json::json!({ "documents_copied": 2, "indexes_copied": 1 }),
&[],
)
.await;
let error = mongo_clone_collection_core(&state, "legacy", "app", "users", "users_copy").await.unwrap_err();
assert!(error.contains("upgrade or reinstall"), "{error}");
assert!(!error.contains("Unknown method"), "{error}");
}
#[cfg(unix)]
#[tokio::test]
async fn mongo_list_collections_requests_legacy_collection_types() {