fix(mongodb): support aggregate options and parse errors
This commit is contained in:
1 parent
782615586c
commit
bb613b1712
25 files changed
+952
-486
No files matched your search
@@ -1,7 +1,7 @@
|
||||
use mongodb::{
|
||||
bson::{doc, oid::ObjectId, Bson, DateTime, Document},
|
||||
options::{ClientOptions, GridFsBucketOptions, IndexOptions, UpdateModifications},
|
||||
Client, Database, IndexModel,
|
||||
Client, Cursor, Database, IndexModel,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
@@ -734,22 +734,86 @@ pub async fn find_documents_extended_json(
|
||||
Ok(MongoDocumentResult { extended_documents: Some(documents.clone()), documents, raw_documents: None, total })
|
||||
}
|
||||
|
||||
/// Run `db.collection.aggregate(pipeline, options)`.
|
||||
///
|
||||
/// One execution model for every non-explain aggregate (empty or free-form options):
|
||||
/// build the server [`aggregate`](https://www.mongodb.com/docs/manual/reference/command/aggregate/)
|
||||
/// command and open it with [`Database::run_cursor_command`] so session-scoped getMore/killCursors
|
||||
/// stay with the driver cursor. Options are forwarded as-is (`allowDiskUse`, `cursor.batchSize`,
|
||||
/// `maxTimeMS`, collation, hint, comment, let, …).
|
||||
///
|
||||
/// `explain: true` is the only special case: the server returns a plan document (no cursor), so
|
||||
/// that path uses plain `run_command`.
|
||||
///
|
||||
/// Note: this intentionally uses the command/cursor path rather than `Collection::aggregate`, so
|
||||
/// default read/write concern inheritance matches other free-form shell options (explicit options
|
||||
/// on the command document only).
|
||||
pub async fn aggregate_documents(
|
||||
client: &Client,
|
||||
database: &str,
|
||||
collection: &str,
|
||||
pipeline_json: &str,
|
||||
max_rows: Option<usize>,
|
||||
options_json: Option<&str>,
|
||||
) -> Result<MongoDocumentResult, String> {
|
||||
let json: serde_json::Value =
|
||||
serde_json::from_str(pipeline_json).map_err(|e| format!("Invalid pipeline JSON: {e}"))?;
|
||||
let pipeline_values = json.as_array().ok_or_else(|| "Aggregate pipeline must be a JSON array".to_string())?;
|
||||
let pipeline = pipeline_values
|
||||
let pipeline_docs = pipeline_values
|
||||
.iter()
|
||||
.map(|value| json_object_to_document(value).map_err(|e| format!("Invalid pipeline stage: {e}")))
|
||||
.collect::<Result<Vec<Document>, String>>()?;
|
||||
let col = client.database(database).collection::<Document>(collection);
|
||||
let mut cursor = col.aggregate(pipeline).await.map_err(|e| e.to_string())?;
|
||||
|
||||
let options = parse_aggregate_options_document(options_json)?;
|
||||
let (command, explain) = build_aggregate_command(collection, pipeline_docs, options)?;
|
||||
let db = client.database(database);
|
||||
|
||||
if explain {
|
||||
let result = db.run_command(command).await.map_err(|e| e.to_string())?;
|
||||
let document = bson_to_json(&Bson::Document(result.clone()));
|
||||
let extended = Bson::Document(result).into_relaxed_extjson();
|
||||
return Ok(MongoDocumentResult {
|
||||
documents: vec![document],
|
||||
raw_documents: None,
|
||||
extended_documents: Some(vec![extended]),
|
||||
total: 1,
|
||||
});
|
||||
}
|
||||
|
||||
// Driver cursor owns the implicit session across aggregate + getMore + killCursors.
|
||||
let mut cursor = db.run_cursor_command(command).await.map_err(|e| e.to_string())?;
|
||||
drain_document_cursor(&mut cursor, max_rows).await
|
||||
}
|
||||
|
||||
/// Build the server aggregate command document, preserving free-form options.
|
||||
/// Returns `(command, explain)` so callers branch only on the explain flag.
|
||||
fn build_aggregate_command(
|
||||
collection: &str,
|
||||
pipeline: Vec<Document>,
|
||||
options: Document,
|
||||
) -> Result<(Document, bool), String> {
|
||||
let explain = aggregate_options_explain(&options)?;
|
||||
let pipeline_bson: Vec<Bson> = pipeline.into_iter().map(Bson::Document).collect();
|
||||
let mut command = doc! {
|
||||
"aggregate": collection,
|
||||
"pipeline": pipeline_bson,
|
||||
};
|
||||
for (key, value) in options {
|
||||
command.insert(key, value);
|
||||
}
|
||||
// Server requires `cursor` unless `explain` is true.
|
||||
if !explain && !command.contains_key("cursor") {
|
||||
command.insert("cursor", doc! {});
|
||||
}
|
||||
Ok((command, explain))
|
||||
}
|
||||
|
||||
/// Drain a driver cursor up to `max_rows`, peeking one extra document so callers can detect
|
||||
/// truncation: when more rows exist, `total == max_rows + 1` while `documents.len() == max_rows`.
|
||||
async fn drain_document_cursor(
|
||||
cursor: &mut Cursor<Document>,
|
||||
max_rows: Option<usize>,
|
||||
) -> Result<MongoDocumentResult, String> {
|
||||
let max_rows = max_rows.unwrap_or(100);
|
||||
let fetch_limit = max_rows.saturating_add(1);
|
||||
let mut documents = Vec::new();
|
||||
@@ -767,6 +831,28 @@ pub async fn aggregate_documents(
|
||||
Ok(MongoDocumentResult { documents, raw_documents: None, extended_documents: Some(extended_documents), total })
|
||||
}
|
||||
|
||||
fn parse_aggregate_options_document(options_json: Option<&str>) -> Result<Document, String> {
|
||||
let Some(raw) = options_json.map(str::trim).filter(|value| !value.is_empty()) else {
|
||||
return Ok(Document::new());
|
||||
};
|
||||
let value: serde_json::Value =
|
||||
serde_json::from_str(raw).map_err(|e| format!("Invalid aggregate options JSON: {e}"))?;
|
||||
if !value.is_object() {
|
||||
return Err("Aggregate options must be a JSON object".to_string());
|
||||
}
|
||||
// Free-form object conversion keeps every official aggregate option (nested
|
||||
// docs, hint strings, maxTimeMS numbers, comments, let vars, …) intact.
|
||||
json_object_to_document(&value).map_err(|e| format!("Invalid aggregate options: {e}"))
|
||||
}
|
||||
|
||||
fn aggregate_options_explain(options: &Document) -> Result<bool, String> {
|
||||
match options.get("explain") {
|
||||
None => Ok(false),
|
||||
Some(Bson::Boolean(flag)) => Ok(*flag),
|
||||
Some(_) => Err("aggregate options.explain must be a boolean (for example { explain: true })".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Distinct values of a field, matching the mongo shell's `db.coll.distinct(field, filter)`:
|
||||
/// array fields contribute their elements rather than the whole array, and the server may
|
||||
/// answer from an index with a DISTINCT_SCAN. Values are returned in the `documents` slot as
|
||||
@@ -1676,6 +1762,83 @@ fn expand_object_id_string_array(items: &[serde_json::Value]) -> Bson {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_aggregate_options_document_keeps_official_fields() {
|
||||
let doc = parse_aggregate_options_document(Some(
|
||||
r#"{
|
||||
"explain": true,
|
||||
"allowDiskUse": true,
|
||||
"cursor": { "batchSize": 25 },
|
||||
"maxTimeMS": 1000,
|
||||
"bypassDocumentValidation": true,
|
||||
"collation": { "locale": "en" },
|
||||
"hint": "status_1",
|
||||
"comment": "agg-test",
|
||||
"let": { "year": 2024 },
|
||||
"readConcern": { "level": "local" }
|
||||
}"#,
|
||||
))
|
||||
.unwrap();
|
||||
assert_eq!(doc.get_bool("explain").ok(), Some(true));
|
||||
assert_eq!(doc.get_bool("allowDiskUse").ok(), Some(true));
|
||||
assert_eq!(doc.get_i64("maxTimeMS").ok(), Some(1000));
|
||||
assert_eq!(doc.get_str("hint").ok(), Some("status_1"));
|
||||
assert_eq!(doc.get_str("comment").ok(), Some("agg-test"));
|
||||
assert_eq!(doc.get_document("cursor").ok().and_then(|c| c.get_i64("batchSize").ok()), Some(25));
|
||||
assert_eq!(doc.get_document("collation").ok().and_then(|c| c.get_str("locale").ok()), Some("en"));
|
||||
assert_eq!(doc.get_document("let").ok().and_then(|c| c.get_i64("year").ok()), Some(2024));
|
||||
assert!(aggregate_options_explain(&doc).unwrap());
|
||||
|
||||
let no_explain = parse_aggregate_options_document(Some(r#"{"allowDiskUse":false}"#)).unwrap();
|
||||
assert!(!aggregate_options_explain(&no_explain).unwrap());
|
||||
|
||||
let err = aggregate_options_explain(
|
||||
&parse_aggregate_options_document(Some(r#"{"explain":"executionStats"}"#)).unwrap(),
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(err.contains("boolean"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_aggregate_command_adds_cursor_unless_explain() {
|
||||
let pipeline = vec![doc! { "$match": { "active": true } }];
|
||||
let (with_cursor, explain_flag) = build_aggregate_command(
|
||||
"products",
|
||||
pipeline.clone(),
|
||||
parse_aggregate_options_document(Some(r#"{"allowDiskUse":true,"cursor":{"batchSize":1}}"#)).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(!explain_flag);
|
||||
assert_eq!(with_cursor.get_str("aggregate").ok(), Some("products"));
|
||||
assert_eq!(with_cursor.get_bool("allowDiskUse").ok(), Some(true));
|
||||
assert_eq!(with_cursor.get_document("cursor").ok().and_then(|c| c.get_i64("batchSize").ok()), Some(1));
|
||||
|
||||
let (default_cursor, _) = build_aggregate_command(
|
||||
"products",
|
||||
pipeline.clone(),
|
||||
parse_aggregate_options_document(Some(r#"{"comment":"agg"}"#)).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(default_cursor.get_document("cursor").is_ok(), "non-explain aggregate requires cursor");
|
||||
assert_eq!(default_cursor.get_str("comment").ok(), Some("agg"));
|
||||
|
||||
// Empty options still produce a cursor command (single path with run_cursor_command).
|
||||
let (empty_options, empty_explain) =
|
||||
build_aggregate_command("products", pipeline.clone(), Document::new()).unwrap();
|
||||
assert!(!empty_explain);
|
||||
assert!(empty_options.get_document("cursor").is_ok());
|
||||
|
||||
let (explain, is_explain) = build_aggregate_command(
|
||||
"products",
|
||||
pipeline,
|
||||
parse_aggregate_options_document(Some(r#"{"explain":true,"allowDiskUse":true}"#)).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(is_explain);
|
||||
assert!(explain.get("cursor").is_none(), "explain path must not inject cursor");
|
||||
assert_eq!(explain.get_bool("explain").ok(), Some(true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn update_modifications_accept_document_and_pipeline() {
|
||||
let document = json_update_to_modifications(&serde_json::json!({ "$set": { "status": "done" } })).unwrap();
|
||||
|
||||
@@ -245,12 +245,13 @@ pub async fn mongo_aggregate_documents_core(
|
||||
collection: &str,
|
||||
pipeline_json: &str,
|
||||
max_rows: Option<usize>,
|
||||
options_json: Option<&str>,
|
||||
) -> Result<MongoDocumentResult, String> {
|
||||
ensure_document_pool(state, connection_id).await?;
|
||||
let connections = state.connections.read().await;
|
||||
match connections.get(connection_id).ok_or("Not found")? {
|
||||
PoolKind::MongoDb(client) => {
|
||||
mongo_driver::aggregate_documents(client, database, collection, pipeline_json, max_rows).await
|
||||
mongo_driver::aggregate_documents(client, database, collection, pipeline_json, max_rows, options_json).await
|
||||
}
|
||||
PoolKind::Agent(_) => Err("MongoDB legacy agent does not support aggregate".to_string()),
|
||||
_ => Err("Not a MongoDB connection".to_string()),
|
||||
|
||||
@@ -34,6 +34,10 @@ use crate::sql::{split_sql_batches, split_sql_statements, starts_with_executable
|
||||
pub const QUERY_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
pub const MAX_ROWS: usize = 10000;
|
||||
pub const QUERY_CANCELED: &str = "Query canceled";
|
||||
/// Fallback when a Mongo connection hits the generic SQL executor instead of the shell path.
|
||||
/// Wording must match packages/mongo-shell `MONGO_SHELL_COMMAND_HINT`
|
||||
/// (desktop/CLI diagnose first; this is only the Rust SQL-executor backstop).
|
||||
const MONGO_SHELL_COMMAND_HINT: &str = "Use MongoDB shell-style commands, for example: db.collection.find({}).limit(100), db.collection.aggregate([]), db.collection.aggregate([], { explain: true }), db.version(), db.collection.countDocuments({}), db.collection.distinct(\"field\"), db.collection.getIndexes(), db.collection.createIndex({...}), or db.collection.insertOne({...}).";
|
||||
const SQL_OMITTED_ERROR_CONTEXT: &str =
|
||||
"SQL text omitted from user-facing error; enable debug SQL diagnostics for a redacted statement.";
|
||||
#[cfg(feature = "duckdb-bundled")]
|
||||
@@ -1487,7 +1491,7 @@ pub async fn do_execute(
|
||||
result
|
||||
}
|
||||
PoolKind::Redis(_) => Err("Use Redis-specific commands".to_string()),
|
||||
PoolKind::MongoDb(_) => Err("Use MongoDB-specific commands".to_string()),
|
||||
PoolKind::MongoDb(_) => Err(MONGO_SHELL_COMMAND_HINT.to_string()),
|
||||
PoolKind::MessageQueue => Err("Use Message Queue-specific commands".to_string()),
|
||||
PoolKind::Nacos => Err("Use Nacos-specific commands".to_string()),
|
||||
PoolKind::InfluxDb(client) => {
|
||||
@@ -1712,7 +1716,7 @@ pub async fn execute_sql_statement_with_options(
|
||||
// (e.g. typos) must be rejected before any pool/key creation so that
|
||||
// session-scoped pools do not leak MongoDB Clients and SSH tunnels.
|
||||
if connection_is_mongodb(state, connection_id).await {
|
||||
return Err("Use MongoDB-specific commands".to_string());
|
||||
return Err(MONGO_SHELL_COMMAND_HINT.to_string());
|
||||
}
|
||||
|
||||
let db_type = connection_database_type(state, connection_id).await;
|
||||
@@ -1934,7 +1938,7 @@ pub async fn execute_multi_core_with_options_for_client(
|
||||
) -> Result<Vec<ExecuteMultiResult>, String> {
|
||||
// Reject MongoDB queries that fall through to the generic executor.
|
||||
if connection_is_mongodb(state, connection_id).await {
|
||||
return Err("Use MongoDB-specific commands".to_string());
|
||||
return Err(MONGO_SHELL_COMMAND_HINT.to_string());
|
||||
}
|
||||
|
||||
let pool_key = if database.is_empty() {
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
//! Live MongoDB aggregate regressions.
|
||||
//!
|
||||
//! Run with a writable server:
|
||||
//! ```text
|
||||
//! DBX_LIVE_MONGODB_URL='mongodb://127.0.0.1:27017' \
|
||||
//! cargo test -p dbx-core --test live_mongodb_aggregate -- --ignored --nocapture
|
||||
//! ```
|
||||
//!
|
||||
//! `aggregate_with_small_batch_size_reads_multiple_cursor_batches` is the session/getMore
|
||||
//! guard: `cursor.batchSize: 1` forces multiple batches; a broken session fails or truncates.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use dbx_core::db::mongo_driver;
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DBX_LIVE_MONGODB_URL pointing at a writable MongoDB database"]
|
||||
async fn aggregate_with_small_batch_size_reads_multiple_cursor_batches() {
|
||||
let url = std::env::var("DBX_LIVE_MONGODB_URL").expect("DBX_LIVE_MONGODB_URL");
|
||||
let client = mongo_driver::connect(&url, Duration::from_secs(10), Duration::from_secs(60)).await.unwrap();
|
||||
let database = "dbx_live_aggregate";
|
||||
let collection = format!("batch_pages_{}", std::process::id());
|
||||
|
||||
let docs: Vec<serde_json::Value> =
|
||||
(0..5).map(|n| serde_json::json!({ "n": n, "label": format!("row-{n}") })).collect();
|
||||
mongo_driver::insert_documents(&client, database, &collection, &serde_json::to_string(&docs).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// batchSize:1 forces firstBatch + multiple getMore round-trips. Those getMore calls must share
|
||||
// the driver's cursor session (run_cursor_command), or pagination fails / returns partial data.
|
||||
let result = mongo_driver::aggregate_documents(
|
||||
&client,
|
||||
database,
|
||||
&collection,
|
||||
r#"[{"$sort":{"n":1}},{"$project":{"_id":0,"n":1,"label":1}}]"#,
|
||||
Some(10),
|
||||
Some(r#"{"allowDiskUse":true,"cursor":{"batchSize":1}}"#),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result.total, 5, "expected all rows across multiple cursor batches: {:?}", result.documents);
|
||||
let numbers: Vec<i64> = result
|
||||
.documents
|
||||
.iter()
|
||||
.map(|doc| doc["n"].as_i64().or_else(|| doc["n"].as_u64().map(|v| v as i64)).expect("n"))
|
||||
.collect();
|
||||
assert_eq!(numbers, vec![0, 1, 2, 3, 4]);
|
||||
|
||||
mongo_driver::drop_collection(&client, database, &collection).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DBX_LIVE_MONGODB_URL pointing at a writable MongoDB database"]
|
||||
async fn aggregate_with_options_respects_max_rows_across_batches() {
|
||||
let url = std::env::var("DBX_LIVE_MONGODB_URL").expect("DBX_LIVE_MONGODB_URL");
|
||||
let client = mongo_driver::connect(&url, Duration::from_secs(10), Duration::from_secs(60)).await.unwrap();
|
||||
let database = "dbx_live_aggregate";
|
||||
let collection = format!("batch_limit_{}", std::process::id());
|
||||
|
||||
let docs: Vec<serde_json::Value> = (0..8).map(|n| serde_json::json!({ "n": n })).collect();
|
||||
mongo_driver::insert_documents(&client, database, &collection, &serde_json::to_string(&docs).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let result = mongo_driver::aggregate_documents(
|
||||
&client,
|
||||
database,
|
||||
&collection,
|
||||
r#"[{"$sort":{"n":1}},{"$project":{"_id":0,"n":1}}]"#,
|
||||
Some(3),
|
||||
Some(r#"{"cursor":{"batchSize":1}}"#),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result.documents.len(), 3);
|
||||
// drain peeks one past max_rows so total > documents.len() signals more rows available.
|
||||
assert_eq!(result.total, 4);
|
||||
assert_eq!(
|
||||
result.documents[0]["n"].as_i64().or_else(|| result.documents[0]["n"].as_u64().map(|v| v as i64)),
|
||||
Some(0)
|
||||
);
|
||||
assert_eq!(
|
||||
result.documents[2]["n"].as_i64().or_else(|| result.documents[2]["n"].as_u64().map(|v| v as i64)),
|
||||
Some(2)
|
||||
);
|
||||
|
||||
mongo_driver::drop_collection(&client, database, &collection).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DBX_LIVE_MONGODB_URL pointing at a writable MongoDB database"]
|
||||
async fn aggregate_without_options_uses_same_cursor_path() {
|
||||
let url = std::env::var("DBX_LIVE_MONGODB_URL").expect("DBX_LIVE_MONGODB_URL");
|
||||
let client = mongo_driver::connect(&url, Duration::from_secs(10), Duration::from_secs(60)).await.unwrap();
|
||||
let database = "dbx_live_aggregate";
|
||||
let collection = format!("no_options_{}", std::process::id());
|
||||
|
||||
let docs: Vec<serde_json::Value> = (0..3).map(|n| serde_json::json!({ "n": n })).collect();
|
||||
mongo_driver::insert_documents(&client, database, &collection, &serde_json::to_string(&docs).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Empty options still go through run_cursor_command (not Collection::aggregate).
|
||||
let result = mongo_driver::aggregate_documents(
|
||||
&client,
|
||||
database,
|
||||
&collection,
|
||||
r#"[{"$sort":{"n":1}},{"$project":{"_id":0,"n":1}}]"#,
|
||||
Some(10),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result.total, 3);
|
||||
assert_eq!(result.documents.len(), 3);
|
||||
|
||||
mongo_driver::drop_collection(&client, database, &collection).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DBX_LIVE_MONGODB_URL pointing at a writable MongoDB database"]
|
||||
async fn aggregate_explain_option_returns_plan_document() {
|
||||
let url = std::env::var("DBX_LIVE_MONGODB_URL").expect("DBX_LIVE_MONGODB_URL");
|
||||
let client = mongo_driver::connect(&url, Duration::from_secs(10), Duration::from_secs(60)).await.unwrap();
|
||||
let database = "dbx_live_aggregate";
|
||||
let collection = format!("explain_{}", std::process::id());
|
||||
|
||||
mongo_driver::insert_documents(&client, database, &collection, r#"[{"n":1}]"#).await.unwrap();
|
||||
|
||||
let result = mongo_driver::aggregate_documents(
|
||||
&client,
|
||||
database,
|
||||
&collection,
|
||||
r#"[{"$match":{"n":1}}]"#,
|
||||
Some(10),
|
||||
Some(r#"{"explain":true}"#),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result.total, 1);
|
||||
assert_eq!(result.documents.len(), 1);
|
||||
// Explain plan shape varies by server version; just require a non-empty object.
|
||||
assert!(result.documents[0].is_object(), "explain should return a plan object");
|
||||
|
||||
mongo_driver::drop_collection(&client, database, &collection).await.unwrap();
|
||||
}
|
||||
Reference in new issue
Block a user