fix(elasticsearch): pass routing for document writes
This commit is contained in:
1 parent
910fde75e1
commit
299a9843f3
16 files changed
+912
-351
No files matched your search
@@ -27,6 +27,9 @@ const ELASTICSEARCH_PATH_SEGMENT_ENCODE_SET: &AsciiSet = &CONTROLS
|
||||
.add(b'|')
|
||||
.add(b'}');
|
||||
|
||||
const ELASTICSEARCH_QUERY_VALUE_ENCODE_SET: &AsciiSet =
|
||||
&CONTROLS.add(b' ').add(b'"').add(b'#').add(b'%').add(b'&').add(b'+').add(b'/').add(b'=').add(b'?');
|
||||
|
||||
pub struct EsClient {
|
||||
http: HttpClient,
|
||||
base_url: String,
|
||||
@@ -189,6 +192,19 @@ fn elasticsearch_path_segment(value: &str) -> String {
|
||||
utf8_percent_encode(value, ELASTICSEARCH_PATH_SEGMENT_ENCODE_SET).to_string()
|
||||
}
|
||||
|
||||
fn elasticsearch_query_value(value: &str) -> String {
|
||||
utf8_percent_encode(value, ELASTICSEARCH_QUERY_VALUE_ENCODE_SET).to_string()
|
||||
}
|
||||
|
||||
fn elasticsearch_document_path(index: &str, id: &str, routing: Option<&str>) -> String {
|
||||
let base = format!("/{}/_doc/{}", elasticsearch_path_segment(index), elasticsearch_path_segment(id));
|
||||
if let Some(routing) = routing.map(str::trim).filter(|value| !value.is_empty()) {
|
||||
format!("{base}?routing={}&refresh=true", elasticsearch_query_value(routing))
|
||||
} else {
|
||||
format!("{base}?refresh=true")
|
||||
}
|
||||
}
|
||||
|
||||
fn redact_elasticsearch_url(url: &str) -> String {
|
||||
let Ok(mut parsed) = reqwest::Url::parse(url) else {
|
||||
return url.to_string();
|
||||
@@ -364,6 +380,8 @@ impl<'de> Deserialize<'de> for HitsTotal {
|
||||
struct SearchHit {
|
||||
#[serde(rename = "_id")]
|
||||
id: String,
|
||||
#[serde(rename = "_routing")]
|
||||
routing: Option<String>,
|
||||
#[serde(rename = "_source")]
|
||||
source: serde_json::Value,
|
||||
}
|
||||
@@ -398,6 +416,9 @@ pub async fn find_documents(
|
||||
_ => serde_json::Map::new(),
|
||||
};
|
||||
doc.insert("_id".to_string(), serde_json::Value::String(hit.id));
|
||||
if let Some(routing) = hit.routing {
|
||||
doc.insert("_routing".to_string(), serde_json::Value::String(routing));
|
||||
}
|
||||
serde_json::Value::Object(doc)
|
||||
})
|
||||
.collect();
|
||||
@@ -634,10 +655,16 @@ pub async fn insert_document(client: &EsClient, index: &str, doc_json: &str) ->
|
||||
Ok(result["_id"].as_str().unwrap_or("").to_string())
|
||||
}
|
||||
|
||||
pub async fn update_document(client: &EsClient, index: &str, id: &str, doc_json: &str) -> Result<u64, String> {
|
||||
let doc = elasticsearch_document_body_from_json(doc_json)?;
|
||||
pub async fn update_document(
|
||||
client: &EsClient,
|
||||
index: &str,
|
||||
id: &str,
|
||||
doc_json: &str,
|
||||
routing: Option<&str>,
|
||||
) -> Result<u64, String> {
|
||||
let (doc, routing) = elasticsearch_document_body_and_routing_from_json(doc_json, routing)?;
|
||||
|
||||
let path = format!("/{}/_doc/{}?refresh=true", elasticsearch_path_segment(index), elasticsearch_path_segment(id));
|
||||
let path = elasticsearch_document_path(index, id, routing.as_deref());
|
||||
let resp = client.put(&path).json(&doc).send().await.map_err(|e| format!("Elasticsearch request failed: {e}"))?;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
@@ -649,15 +676,37 @@ pub async fn update_document(client: &EsClient, index: &str, id: &str, doc_json:
|
||||
}
|
||||
|
||||
fn elasticsearch_document_body_from_json(doc_json: &str) -> Result<serde_json::Value, String> {
|
||||
let mut doc: serde_json::Value = serde_json::from_str(doc_json).map_err(|e| format!("Invalid JSON: {e}"))?;
|
||||
if let serde_json::Value::Object(map) = &mut doc {
|
||||
map.remove("_id");
|
||||
}
|
||||
Ok(doc)
|
||||
elasticsearch_document_body_and_routing_from_json(doc_json, None).map(|(doc, _)| doc)
|
||||
}
|
||||
|
||||
pub async fn delete_document(client: &EsClient, index: &str, id: &str) -> Result<u64, String> {
|
||||
let path = format!("/{}/_doc/{}?refresh=true", elasticsearch_path_segment(index), elasticsearch_path_segment(id));
|
||||
fn elasticsearch_document_body_and_routing_from_json(
|
||||
doc_json: &str,
|
||||
routing: Option<&str>,
|
||||
) -> Result<(serde_json::Value, Option<String>), String> {
|
||||
let mut doc: serde_json::Value = serde_json::from_str(doc_json).map_err(|e| format!("Invalid JSON: {e}"))?;
|
||||
let mut routing = routing.map(str::trim).filter(|value| !value.is_empty()).map(str::to_string);
|
||||
if let serde_json::Value::Object(map) = &mut doc {
|
||||
map.remove("_id");
|
||||
if routing.is_none() {
|
||||
routing = map.get("_routing").and_then(elasticsearch_routing_from_value);
|
||||
}
|
||||
map.remove("_routing");
|
||||
}
|
||||
Ok((doc, routing))
|
||||
}
|
||||
|
||||
fn elasticsearch_routing_from_value(value: &serde_json::Value) -> Option<String> {
|
||||
match value {
|
||||
serde_json::Value::String(value) => {
|
||||
let trimmed = value.trim();
|
||||
(!trimmed.is_empty()).then(|| trimmed.to_string())
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn delete_document(client: &EsClient, index: &str, id: &str, routing: Option<&str>) -> Result<u64, String> {
|
||||
let path = elasticsearch_document_path(index, id, routing);
|
||||
let resp = client.delete(&path).send().await.map_err(|e| format!("Elasticsearch request failed: {e}"))?;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
@@ -807,12 +856,15 @@ fn parse_elasticsearch_response(
|
||||
.iter()
|
||||
.map(|hit| {
|
||||
let mut row = serde_json::Map::new();
|
||||
row.insert("_id".to_string(), hit.get("_id").cloned().unwrap_or(serde_json::Value::Null));
|
||||
if let Some(source) = hit.get("_source").and_then(|s| s.as_object()) {
|
||||
for (k, v) in source {
|
||||
row.insert(k.clone(), v.clone());
|
||||
}
|
||||
}
|
||||
row.insert("_id".to_string(), hit.get("_id").cloned().unwrap_or(serde_json::Value::Null));
|
||||
if let Some(routing) = hit.get("_routing") {
|
||||
row.insert("_routing".to_string(), routing.clone());
|
||||
}
|
||||
for k in row.keys() {
|
||||
if !all_keys.contains(k) {
|
||||
all_keys.push(k.clone());
|
||||
@@ -1475,6 +1527,15 @@ mod tests {
|
||||
assert_eq!(super::elasticsearch_path_segment("a%b/c"), "a%25b%2Fc");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_elasticsearch_document_path_with_routing() {
|
||||
assert_eq!(
|
||||
super::elasticsearch_document_path("orders/2026", "a%b/c", Some("tenant/a&b")),
|
||||
"/orders%2F2026/_doc/a%25b%2Fc?routing=tenant%2Fa%26b&refresh=true"
|
||||
);
|
||||
assert_eq!(super::elasticsearch_document_path("orders", "1", None), "/orders/_doc/1?refresh=true");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn elasticsearch_sql_detection_does_not_treat_rest_methods_as_sql() {
|
||||
assert!(super::is_elasticsearch_sql_query("SELECT * FROM index_task_v1"));
|
||||
@@ -1594,13 +1655,72 @@ mod tests {
|
||||
assert_eq!(response.hits.total.value(), 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_elasticsearch_hit_routing_metadata() {
|
||||
let response: SearchResponse = serde_json::from_value(json!({
|
||||
"hits": {
|
||||
"total": { "value": 1, "relation": "eq" },
|
||||
"hits": [
|
||||
{ "_id": "doc-1", "_routing": "tenant-1", "_source": { "name": "Alice" } }
|
||||
]
|
||||
}
|
||||
}))
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(response.hits.hits[0].routing.as_deref(), Some("tenant-1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_search_response_rows_with_routing_metadata() {
|
||||
let result = super::parse_elasticsearch_response(
|
||||
200,
|
||||
json!({
|
||||
"hits": {
|
||||
"hits": [
|
||||
{ "_id": "doc-1", "_routing": "tenant-1", "_source": { "name": "Alice" } }
|
||||
]
|
||||
}
|
||||
}),
|
||||
std::time::Instant::now(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let routing_idx = result.columns.iter().position(|column| column == "_routing").unwrap();
|
||||
assert_eq!(result.rows[0][routing_idx], json!("tenant-1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn document_body_removes_elasticsearch_id_metadata() {
|
||||
let doc = super::elasticsearch_document_body_from_json(r#"{"_id":"abc","name":"Alice"}"#).unwrap();
|
||||
let doc = super::elasticsearch_document_body_from_json(r#"{"_id":"abc","_routing":"tenant-1","name":"Alice"}"#)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(doc, json!({ "name": "Alice" }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn document_body_extracts_elasticsearch_routing_metadata() {
|
||||
let (doc, routing) = super::elasticsearch_document_body_and_routing_from_json(
|
||||
r#"{"_id":"abc","_routing":"tenant-1","name":"Alice"}"#,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(doc, json!({ "name": "Alice" }));
|
||||
assert_eq!(routing.as_deref(), Some("tenant-1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_elasticsearch_routing_overrides_document_metadata() {
|
||||
let (doc, routing) = super::elasticsearch_document_body_and_routing_from_json(
|
||||
r#"{"_id":"abc","_routing":"tenant-1","name":"Alice"}"#,
|
||||
Some("tenant-2"),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(doc, json!({ "name": "Alice" }));
|
||||
assert_eq!(routing.as_deref(), Some("tenant-2"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn document_body_preserves_user_field_order() {
|
||||
let doc = super::elasticsearch_document_body_from_json(r#"{"z":1,"_id":"abc","a":2}"#).unwrap();
|
||||
|
||||
@@ -0,0 +1,266 @@
|
||||
use crate::connection::{AppState, PoolKind};
|
||||
use crate::db::agent_driver::mongo_document_id_params;
|
||||
use crate::db::mongo_driver::MongoDocumentResult;
|
||||
use crate::db::{elasticsearch_driver, mongo_driver, vector_driver};
|
||||
|
||||
pub use crate::db::vector_driver::CollectionInfo;
|
||||
|
||||
fn sort_names(mut names: Vec<String>) -> Vec<String> {
|
||||
names.sort_by(|left, right| {
|
||||
let left_lower = left.to_lowercase();
|
||||
let right_lower = right.to_lowercase();
|
||||
left_lower.cmp(&right_lower).then_with(|| left.cmp(right))
|
||||
});
|
||||
names
|
||||
}
|
||||
|
||||
async fn ensure_document_pool(state: &AppState, connection_id: &str) -> Result<(), String> {
|
||||
state.get_or_create_pool(connection_id, None).await.map(|_| ())
|
||||
}
|
||||
|
||||
pub async fn list_databases_core(state: &AppState, connection_id: &str) -> Result<Vec<String>, String> {
|
||||
ensure_document_pool(state, connection_id).await?;
|
||||
let fallback_database = configured_mongo_database(state, connection_id).await;
|
||||
let connections = state.connections.read().await;
|
||||
match connections.get(connection_id).ok_or("Not found")? {
|
||||
PoolKind::MongoDb(client) => match mongo_driver::list_databases(client).await {
|
||||
Ok(databases) => Ok(sort_names(databases)),
|
||||
Err(error) if mongo_list_databases_unauthorized(&error) => {
|
||||
fallback_mongo_database(&error, fallback_database)
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
},
|
||||
PoolKind::Elasticsearch(_) | PoolKind::VectorDb(_) => Ok(vec!["default".to_string()]),
|
||||
PoolKind::Agent(client) => {
|
||||
let mut client = client.lock().await;
|
||||
match client.mongo_list_databases::<Vec<serde_json::Value>>().await {
|
||||
Ok(result) => {
|
||||
Ok(sort_names(result.iter().filter_map(|v| v.get("name")?.as_str().map(String::from)).collect()))
|
||||
}
|
||||
Err(error) if mongo_list_databases_unauthorized(&error) => {
|
||||
fallback_mongo_database(&error, fallback_database)
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
_ => Err("Not a MongoDB/Elasticsearch/vector connection".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
async fn configured_mongo_database(state: &AppState, connection_id: &str) -> Option<String> {
|
||||
let configs = state.configs.read().await;
|
||||
configs.get(connection_id).and_then(|config| config.effective_database().map(str::to_string))
|
||||
}
|
||||
|
||||
fn fallback_mongo_database(error: &str, fallback_database: Option<String>) -> Result<Vec<String>, String> {
|
||||
fallback_database.map(|database| vec![database]).ok_or_else(|| error.to_string())
|
||||
}
|
||||
|
||||
fn mongo_list_databases_unauthorized(error: &str) -> bool {
|
||||
let lower = error.to_lowercase();
|
||||
lower.contains("not authorized") && lower.contains("listdatabases")
|
||||
}
|
||||
|
||||
pub async fn list_collections_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
database: &str,
|
||||
) -> Result<Vec<CollectionInfo>, String> {
|
||||
ensure_document_pool(state, connection_id).await?;
|
||||
let connections = state.connections.read().await;
|
||||
match connections.get(connection_id).ok_or("Not found")? {
|
||||
PoolKind::MongoDb(client) => {
|
||||
let names = sort_names(mongo_driver::list_collections(client, database).await?);
|
||||
Ok(names.into_iter().map(|n| CollectionInfo { name: n.clone(), id: n, dimension: None }).collect())
|
||||
}
|
||||
PoolKind::Elasticsearch(client) => {
|
||||
let names = sort_names(elasticsearch_driver::list_indices(client).await?);
|
||||
Ok(names.into_iter().map(|n| CollectionInfo { name: n.clone(), id: n, dimension: None }).collect())
|
||||
}
|
||||
PoolKind::VectorDb(client) => vector_driver::list_collections_with_db(&client, database).await,
|
||||
PoolKind::Agent(client) => {
|
||||
let mut client = client.lock().await;
|
||||
let names = sort_names(client.mongo_list_collections(database).await?);
|
||||
Ok(names.into_iter().map(|n| CollectionInfo { name: n.clone(), id: n, dimension: None }).collect())
|
||||
}
|
||||
_ => Err("Not a MongoDB/Elasticsearch/vector connection".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn find_documents_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
database: &str,
|
||||
collection: &str,
|
||||
skip: u64,
|
||||
limit: i64,
|
||||
filter: Option<&str>,
|
||||
projection: Option<&str>,
|
||||
sort: Option<&str>,
|
||||
) -> Result<MongoDocumentResult, String> {
|
||||
ensure_document_pool(state, connection_id).await?;
|
||||
let connections = state.connections.read().await;
|
||||
match connections.get(connection_id).ok_or("Not found")? {
|
||||
PoolKind::MongoDb(client) => {
|
||||
mongo_driver::find_documents(client, database, collection, skip, limit, filter, projection, sort).await
|
||||
}
|
||||
PoolKind::Elasticsearch(client) => {
|
||||
let client = client.clone();
|
||||
drop(connections);
|
||||
elasticsearch_driver::find_documents(&client, collection, skip, limit, filter, sort).await
|
||||
}
|
||||
PoolKind::VectorDb(client) => {
|
||||
let client = client.clone();
|
||||
drop(connections);
|
||||
let _ = (database, filter, sort);
|
||||
vector_driver::find_documents(&client, collection, skip, limit).await
|
||||
}
|
||||
PoolKind::Agent(client) => {
|
||||
let mut client = client.lock().await;
|
||||
let mut params = serde_json::json!({
|
||||
"database": database,
|
||||
"collection": collection,
|
||||
"skip": skip,
|
||||
"limit": limit,
|
||||
"filter": filter,
|
||||
"sort": sort,
|
||||
});
|
||||
if let Some(projection) = projection {
|
||||
params["projection"] = serde_json::json!(projection);
|
||||
}
|
||||
client.mongo_find_documents(params).await
|
||||
}
|
||||
_ => Err("Not a MongoDB/Elasticsearch/vector connection".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn insert_document_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
database: &str,
|
||||
collection: &str,
|
||||
doc_json: &str,
|
||||
) -> Result<String, String> {
|
||||
ensure_document_pool(state, connection_id).await?;
|
||||
let connections = state.connections.read().await;
|
||||
match connections.get(connection_id).ok_or("Not found")? {
|
||||
PoolKind::MongoDb(client) => mongo_driver::insert_document(client, database, collection, doc_json).await,
|
||||
PoolKind::Elasticsearch(client) => {
|
||||
let client = client.clone();
|
||||
drop(connections);
|
||||
elasticsearch_driver::insert_document(&client, collection, doc_json).await
|
||||
}
|
||||
PoolKind::Agent(client) => {
|
||||
let mut client = client.lock().await;
|
||||
let result: serde_json::Value = client
|
||||
.mongo_insert_document(serde_json::json!({
|
||||
"database": database,
|
||||
"collection": collection,
|
||||
"doc_json": doc_json,
|
||||
}))
|
||||
.await?;
|
||||
Ok(result.get("inserted_id").and_then(|v| v.as_str()).unwrap_or("").to_string())
|
||||
}
|
||||
_ => Err("Not a MongoDB/Elasticsearch connection".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn update_document_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
database: &str,
|
||||
collection: &str,
|
||||
id: &str,
|
||||
doc_json: &str,
|
||||
routing: Option<&str>,
|
||||
) -> Result<u64, String> {
|
||||
ensure_document_pool(state, connection_id).await?;
|
||||
let connections = state.connections.read().await;
|
||||
match connections.get(connection_id).ok_or("Not found")? {
|
||||
PoolKind::MongoDb(client) => mongo_driver::update_document(client, database, collection, id, doc_json).await,
|
||||
PoolKind::Elasticsearch(client) => {
|
||||
let client = client.clone();
|
||||
drop(connections);
|
||||
// Elasticsearch requires the same custom routing value for writes
|
||||
// as was used to index the document.
|
||||
elasticsearch_driver::update_document(&client, collection, id, doc_json, routing).await
|
||||
}
|
||||
PoolKind::Agent(client) => {
|
||||
let mut client = client.lock().await;
|
||||
let result: serde_json::Value = client
|
||||
.mongo_update_document(serde_json::json!({
|
||||
"database": database,
|
||||
"collection": collection,
|
||||
"id": id,
|
||||
"doc_json": doc_json,
|
||||
}))
|
||||
.await?;
|
||||
Ok(result.get("modified_count").and_then(|v| v.as_u64()).unwrap_or(0))
|
||||
}
|
||||
_ => Err("Not a MongoDB/Elasticsearch connection".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn delete_document_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
database: &str,
|
||||
collection: &str,
|
||||
id: &str,
|
||||
routing: Option<&str>,
|
||||
) -> Result<u64, String> {
|
||||
ensure_document_pool(state, connection_id).await?;
|
||||
let connections = state.connections.read().await;
|
||||
match connections.get(connection_id).ok_or("Not found")? {
|
||||
PoolKind::MongoDb(client) => mongo_driver::delete_document(client, database, collection, id).await,
|
||||
PoolKind::Elasticsearch(client) => {
|
||||
let client = client.clone();
|
||||
drop(connections);
|
||||
// Elasticsearch requires the same custom routing value for writes
|
||||
// as was used to index the document.
|
||||
elasticsearch_driver::delete_document(&client, collection, id, routing).await
|
||||
}
|
||||
PoolKind::Agent(client) => {
|
||||
let mut client = client.lock().await;
|
||||
let result: serde_json::Value =
|
||||
client.mongo_delete_document(mongo_document_id_params(database, collection, id)).await?;
|
||||
Ok(result.get("deleted_count").and_then(|v| v.as_u64()).unwrap_or(0))
|
||||
}
|
||||
_ => Err("Not a MongoDB/Elasticsearch connection".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{fallback_mongo_database, mongo_list_databases_unauthorized, sort_names};
|
||||
|
||||
#[test]
|
||||
fn sorts_names_case_insensitively() {
|
||||
let sorted = sort_names(vec![
|
||||
"movies".to_string(),
|
||||
"Comments".to_string(),
|
||||
"users".to_string(),
|
||||
"embedded_movies".to_string(),
|
||||
]);
|
||||
|
||||
assert_eq!(sorted, vec!["Comments", "embedded_movies", "movies", "users"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_mongo_list_databases_unauthorized_errors() {
|
||||
assert!(mongo_list_databases_unauthorized(
|
||||
"Command failed with error 13 (Unauthorized): not authorized on admin to execute command { listDatabases: 1 }",
|
||||
));
|
||||
assert!(!mongo_list_databases_unauthorized("not authorized to execute command { find: \"orders\" }"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn falls_back_to_configured_mongo_database() {
|
||||
assert_eq!(
|
||||
fallback_mongo_database("not authorized", Some("app".to_string())).unwrap(),
|
||||
vec!["app".to_string()],
|
||||
);
|
||||
assert_eq!(fallback_mongo_database("not authorized", None).unwrap_err(), "not authorized");
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,7 @@ pub mod database_export;
|
||||
pub mod database_search_sql;
|
||||
pub mod db;
|
||||
pub mod db_admin_sql;
|
||||
pub mod document_ops;
|
||||
pub mod driver_runtime;
|
||||
pub mod external;
|
||||
pub mod history;
|
||||
|
||||
@@ -1,91 +1,21 @@
|
||||
use crate::connection::{AppState, PoolKind};
|
||||
use crate::db::agent_driver::mongo_document_id_params;
|
||||
use crate::db::elasticsearch_driver;
|
||||
use crate::db::mongo_driver::{self, MongoDocumentResult};
|
||||
use crate::db::vector_driver;
|
||||
|
||||
fn sort_names(mut names: Vec<String>) -> Vec<String> {
|
||||
names.sort_by(|left, right| {
|
||||
let left_lower = left.to_lowercase();
|
||||
let right_lower = right.to_lowercase();
|
||||
left_lower.cmp(&right_lower).then_with(|| left.cmp(right))
|
||||
});
|
||||
names
|
||||
}
|
||||
use crate::document_ops::CollectionInfo;
|
||||
|
||||
async fn ensure_document_pool(state: &AppState, connection_id: &str) -> Result<(), String> {
|
||||
state.get_or_create_pool(connection_id, None).await.map(|_| ())
|
||||
}
|
||||
|
||||
pub async fn mongo_list_databases_core(state: &AppState, connection_id: &str) -> Result<Vec<String>, String> {
|
||||
ensure_document_pool(state, connection_id).await?;
|
||||
let fallback_database = configured_mongo_database(state, connection_id).await;
|
||||
let connections = state.connections.read().await;
|
||||
match connections.get(connection_id).ok_or("Not found")? {
|
||||
PoolKind::MongoDb(client) => match mongo_driver::list_databases(client).await {
|
||||
Ok(databases) => Ok(sort_names(databases)),
|
||||
Err(error) if mongo_list_databases_unauthorized(&error) => {
|
||||
fallback_mongo_database(&error, fallback_database)
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
},
|
||||
PoolKind::Elasticsearch(_) | PoolKind::VectorDb(_) => Ok(vec!["default".to_string()]),
|
||||
PoolKind::Agent(client) => {
|
||||
let mut client = client.lock().await;
|
||||
match client.mongo_list_databases::<Vec<serde_json::Value>>().await {
|
||||
Ok(result) => {
|
||||
Ok(sort_names(result.iter().filter_map(|v| v.get("name")?.as_str().map(String::from)).collect()))
|
||||
}
|
||||
Err(error) if mongo_list_databases_unauthorized(&error) => {
|
||||
fallback_mongo_database(&error, fallback_database)
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
_ => Err("Not a MongoDB/Elasticsearch/vector connection".to_string()),
|
||||
}
|
||||
crate::document_ops::list_databases_core(state, connection_id).await
|
||||
}
|
||||
|
||||
async fn configured_mongo_database(state: &AppState, connection_id: &str) -> Option<String> {
|
||||
let configs = state.configs.read().await;
|
||||
configs.get(connection_id).and_then(|config| config.effective_database().map(str::to_string))
|
||||
}
|
||||
|
||||
fn fallback_mongo_database(error: &str, fallback_database: Option<String>) -> Result<Vec<String>, String> {
|
||||
fallback_database.map(|database| vec![database]).ok_or_else(|| error.to_string())
|
||||
}
|
||||
|
||||
fn mongo_list_databases_unauthorized(error: &str) -> bool {
|
||||
let lower = error.to_lowercase();
|
||||
lower.contains("not authorized") && lower.contains("listdatabases")
|
||||
}
|
||||
|
||||
use crate::db::vector_driver::CollectionInfo;
|
||||
|
||||
pub async fn mongo_list_collections_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
database: &str,
|
||||
) -> Result<Vec<CollectionInfo>, String> {
|
||||
ensure_document_pool(state, connection_id).await?;
|
||||
let connections = state.connections.read().await;
|
||||
match connections.get(connection_id).ok_or("Not found")? {
|
||||
PoolKind::MongoDb(client) => {
|
||||
let names = sort_names(mongo_driver::list_collections(client, database).await?);
|
||||
Ok(names.into_iter().map(|n| CollectionInfo { name: n.clone(), id: n, dimension: None }).collect())
|
||||
}
|
||||
PoolKind::Elasticsearch(client) => {
|
||||
let names = sort_names(elasticsearch_driver::list_indices(client).await?);
|
||||
Ok(names.into_iter().map(|n| CollectionInfo { name: n.clone(), id: n, dimension: None }).collect())
|
||||
}
|
||||
PoolKind::VectorDb(client) => vector_driver::list_collections_with_db(&client, database).await,
|
||||
PoolKind::Agent(client) => {
|
||||
let mut client = client.lock().await;
|
||||
let names = sort_names(client.mongo_list_collections(database).await?);
|
||||
Ok(names.into_iter().map(|n| CollectionInfo { name: n.clone(), id: n, dimension: None }).collect())
|
||||
}
|
||||
_ => Err("Not a MongoDB/Elasticsearch/vector connection".to_string()),
|
||||
}
|
||||
crate::document_ops::list_collections_core(state, connection_id, database).await
|
||||
}
|
||||
|
||||
pub async fn mongo_create_database_core(state: &AppState, connection_id: &str, database: &str) -> Result<(), String> {
|
||||
@@ -140,54 +70,6 @@ pub async fn mongo_server_version_core(
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn document_find_documents_core(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
database: &str,
|
||||
collection: &str,
|
||||
skip: u64,
|
||||
limit: i64,
|
||||
filter: Option<&str>,
|
||||
projection: Option<&str>,
|
||||
sort: Option<&str>,
|
||||
) -> Result<MongoDocumentResult, String> {
|
||||
ensure_document_pool(state, connection_id).await?;
|
||||
let connections = state.connections.read().await;
|
||||
match connections.get(connection_id).ok_or("Not found")? {
|
||||
PoolKind::MongoDb(client) => {
|
||||
mongo_driver::find_documents(client, database, collection, skip, limit, filter, projection, sort).await
|
||||
}
|
||||
PoolKind::Elasticsearch(client) => {
|
||||
let client = client.clone();
|
||||
drop(connections);
|
||||
elasticsearch_driver::find_documents(&client, collection, skip, limit, filter, sort).await
|
||||
}
|
||||
PoolKind::VectorDb(client) => {
|
||||
let client = client.clone();
|
||||
drop(connections);
|
||||
let _ = (database, filter, sort);
|
||||
vector_driver::find_documents(&client, collection, skip, limit).await
|
||||
}
|
||||
PoolKind::Agent(client) => {
|
||||
let mut client = client.lock().await;
|
||||
let mut params = serde_json::json!({
|
||||
"database": database,
|
||||
"collection": collection,
|
||||
"skip": skip,
|
||||
"limit": limit,
|
||||
"filter": filter,
|
||||
"sort": sort,
|
||||
});
|
||||
if let Some(projection) = projection {
|
||||
params["projection"] = serde_json::json!(projection);
|
||||
}
|
||||
client.mongo_find_documents(params).await
|
||||
}
|
||||
_ => Err("Not a MongoDB/Elasticsearch/vector connection".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn mongo_find_documents_core(
|
||||
state: &AppState,
|
||||
@@ -200,8 +82,18 @@ pub async fn mongo_find_documents_core(
|
||||
projection: Option<&str>,
|
||||
sort: Option<&str>,
|
||||
) -> Result<MongoDocumentResult, String> {
|
||||
document_find_documents_core(state, connection_id, database, collection, skip, limit, filter, projection, sort)
|
||||
.await
|
||||
crate::document_ops::find_documents_core(
|
||||
state,
|
||||
connection_id,
|
||||
database,
|
||||
collection,
|
||||
skip,
|
||||
limit,
|
||||
filter,
|
||||
projection,
|
||||
sort,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn mongo_aggregate_documents_core(
|
||||
@@ -249,28 +141,7 @@ pub async fn mongo_insert_document_core(
|
||||
collection: &str,
|
||||
doc_json: &str,
|
||||
) -> Result<String, String> {
|
||||
ensure_document_pool(state, connection_id).await?;
|
||||
let connections = state.connections.read().await;
|
||||
match connections.get(connection_id).ok_or("Not found")? {
|
||||
PoolKind::MongoDb(client) => mongo_driver::insert_document(client, database, collection, doc_json).await,
|
||||
PoolKind::Elasticsearch(client) => {
|
||||
let client = client.clone();
|
||||
drop(connections);
|
||||
elasticsearch_driver::insert_document(&client, collection, doc_json).await
|
||||
}
|
||||
PoolKind::Agent(client) => {
|
||||
let mut client = client.lock().await;
|
||||
let result: serde_json::Value = client
|
||||
.mongo_insert_document(serde_json::json!({
|
||||
"database": database,
|
||||
"collection": collection,
|
||||
"doc_json": doc_json,
|
||||
}))
|
||||
.await?;
|
||||
Ok(result.get("inserted_id").and_then(|v| v.as_str()).unwrap_or("").to_string())
|
||||
}
|
||||
_ => Err("Not a MongoDB/Elasticsearch connection".to_string()),
|
||||
}
|
||||
crate::document_ops::insert_document_core(state, connection_id, database, collection, doc_json).await
|
||||
}
|
||||
|
||||
pub async fn mongo_insert_documents_core(
|
||||
@@ -296,30 +167,9 @@ pub async fn mongo_update_document_core(
|
||||
collection: &str,
|
||||
id: &str,
|
||||
doc_json: &str,
|
||||
routing: Option<&str>,
|
||||
) -> Result<u64, String> {
|
||||
ensure_document_pool(state, connection_id).await?;
|
||||
let connections = state.connections.read().await;
|
||||
match connections.get(connection_id).ok_or("Not found")? {
|
||||
PoolKind::MongoDb(client) => mongo_driver::update_document(client, database, collection, id, doc_json).await,
|
||||
PoolKind::Elasticsearch(client) => {
|
||||
let client = client.clone();
|
||||
drop(connections);
|
||||
elasticsearch_driver::update_document(&client, collection, id, doc_json).await
|
||||
}
|
||||
PoolKind::Agent(client) => {
|
||||
let mut client = client.lock().await;
|
||||
let result: serde_json::Value = client
|
||||
.mongo_update_document(serde_json::json!({
|
||||
"database": database,
|
||||
"collection": collection,
|
||||
"id": id,
|
||||
"doc_json": doc_json,
|
||||
}))
|
||||
.await?;
|
||||
Ok(result.get("modified_count").and_then(|v| v.as_u64()).unwrap_or(0))
|
||||
}
|
||||
_ => Err("Not a MongoDB/Elasticsearch connection".to_string()),
|
||||
}
|
||||
crate::document_ops::update_document_core(state, connection_id, database, collection, id, doc_json, routing).await
|
||||
}
|
||||
|
||||
pub async fn mongo_update_documents_core(
|
||||
@@ -348,24 +198,9 @@ pub async fn mongo_delete_document_core(
|
||||
database: &str,
|
||||
collection: &str,
|
||||
id: &str,
|
||||
routing: Option<&str>,
|
||||
) -> Result<u64, String> {
|
||||
ensure_document_pool(state, connection_id).await?;
|
||||
let connections = state.connections.read().await;
|
||||
match connections.get(connection_id).ok_or("Not found")? {
|
||||
PoolKind::MongoDb(client) => mongo_driver::delete_document(client, database, collection, id).await,
|
||||
PoolKind::Elasticsearch(client) => {
|
||||
let client = client.clone();
|
||||
drop(connections);
|
||||
elasticsearch_driver::delete_document(&client, collection, id).await
|
||||
}
|
||||
PoolKind::Agent(client) => {
|
||||
let mut client = client.lock().await;
|
||||
let result: serde_json::Value =
|
||||
client.mongo_delete_document(mongo_document_id_params(database, collection, id)).await?;
|
||||
Ok(result.get("deleted_count").and_then(|v| v.as_u64()).unwrap_or(0))
|
||||
}
|
||||
_ => Err("Not a MongoDB/Elasticsearch connection".to_string()),
|
||||
}
|
||||
crate::document_ops::delete_document_core(state, connection_id, database, collection, id, routing).await
|
||||
}
|
||||
|
||||
pub async fn mongo_delete_documents_core(
|
||||
@@ -386,37 +221,3 @@ pub async fn mongo_delete_documents_core(
|
||||
_ => Err("Not a MongoDB connection".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{fallback_mongo_database, mongo_list_databases_unauthorized, sort_names};
|
||||
|
||||
#[test]
|
||||
fn sorts_names_case_insensitively() {
|
||||
let sorted = sort_names(vec![
|
||||
"movies".to_string(),
|
||||
"Comments".to_string(),
|
||||
"users".to_string(),
|
||||
"embedded_movies".to_string(),
|
||||
]);
|
||||
|
||||
assert_eq!(sorted, vec!["Comments", "embedded_movies", "movies", "users"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_mongo_list_databases_unauthorized_errors() {
|
||||
assert!(mongo_list_databases_unauthorized(
|
||||
"Command failed with error 13 (Unauthorized): not authorized on admin to execute command { listDatabases: 1 }",
|
||||
));
|
||||
assert!(!mongo_list_databases_unauthorized("not authorized to execute command { find: \"orders\" }"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn falls_back_to_configured_mongo_database() {
|
||||
assert_eq!(
|
||||
fallback_mongo_database("not authorized", Some("app".to_string())).unwrap(),
|
||||
vec!["app".to_string()],
|
||||
);
|
||||
assert_eq!(fallback_mongo_database("not authorized", None).unwrap_err(), "not authorized");
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user