fix(elasticsearch): correct totals and CAT JSON output

This commit is contained in:
onenewcode authored and GitHub committed 2026-07-23 17:06:23 +08:00
1 parent a8e5e8d8f9
commit f07509b5ad
27 files changed
+855 -70

No files matched your search

+50
View File
@@ -0,0 +1,50 @@
use serde::{Deserialize, Serialize};
/// Common result shape returned by document-store queries.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DocumentQueryResult {
pub documents: Vec<serde_json::Value>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub raw_documents: Option<Vec<String>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub extended_documents: Option<Vec<serde_json::Value>>,
pub total: u64,
// Older document stores always return an exact total. Keep that wire shape
// unchanged and only send the flag when Elasticsearch reports a lower bound.
#[serde(default = "default_total_is_exact", skip_serializing_if = "is_true")]
pub total_is_exact: bool,
}
fn default_total_is_exact() -> bool {
true
}
fn is_true(value: &bool) -> bool {
*value
}
#[cfg(test)]
mod tests {
use super::DocumentQueryResult;
#[test]
fn exact_results_keep_the_existing_wire_shape() {
let result = DocumentQueryResult {
documents: Vec::new(),
raw_documents: None,
extended_documents: None,
total: 1,
total_is_exact: true,
};
let serialized = serde_json::to_value(result).unwrap();
assert!(serialized.get("total_is_exact").is_none());
let deserialized: DocumentQueryResult = serde_json::from_value(serde_json::json!({
"documents": [],
"total": 1,
}))
.unwrap();
assert!(deserialized.total_is_exact);
}
}
+390 -18
View File
@@ -1,4 +1,4 @@
use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS};
use percent_encoding::{percent_decode_str, utf8_percent_encode, AsciiSet, CONTROLS};
use reqwest::{Client as HttpClient, Method, StatusCode};
use serde::Deserialize;
use serde_json::Value;
@@ -7,7 +7,7 @@ use std::error::Error;
use std::time::Duration;
use super::{http_client_builder, with_connection_timeout};
use crate::db::mongo_driver::MongoDocumentResult;
use crate::db::document_result::DocumentQueryResult;
const ELASTICSEARCH_PATH_SEGMENT_ENCODE_SET: &AsciiSet = &CONTROLS
.add(b' ')
@@ -421,6 +421,12 @@ fn push_mapping_column(
#[derive(Deserialize)]
struct SearchResponse {
hits: SearchHits,
#[serde(rename = "_shards")]
shards: Option<ElasticsearchShards>,
#[serde(default)]
timed_out: bool,
#[serde(default)]
terminated_early: bool,
}
#[derive(Deserialize)]
@@ -431,13 +437,20 @@ struct SearchHits {
enum HitsTotal {
Count(u64),
Value { value: u64 },
Value { value: u64, is_exact: bool },
}
impl HitsTotal {
fn value(&self) -> u64 {
match self {
Self::Count(value) | Self::Value { value } => *value,
Self::Count(value) | Self::Value { value, .. } => *value,
}
}
fn is_exact(&self) -> bool {
match self {
Self::Count(_) => true,
Self::Value { is_exact, .. } => *is_exact,
}
}
}
@@ -452,7 +465,10 @@ impl<'de> Deserialize<'de> for HitsTotal {
return Ok(Self::Count(count));
}
if let Some(count) = value.get("value").and_then(serde_json::Value::as_u64) {
return Ok(Self::Value { value: count });
// Object totals are exact only when Elasticsearch explicitly says
// so. Treat an absent or unknown relation conservatively.
let is_exact = value.get("relation").and_then(serde_json::Value::as_str) == Some("eq");
return Ok(Self::Value { value: count, is_exact });
}
Err(serde::de::Error::custom("expected hits.total as a number or an object with value"))
}
@@ -475,7 +491,7 @@ pub async fn find_documents(
limit: i64,
filter: Option<&str>,
sort: Option<&str>,
) -> Result<MongoDocumentResult, String> {
) -> Result<DocumentQueryResult, String> {
let body = build_find_documents_body(skip, limit, filter, sort)?;
let path = elasticsearch_index_path(index, "_search");
@@ -491,7 +507,58 @@ pub async fn find_documents(
search_response_to_document_result(result)
}
fn search_response_to_document_result(result: SearchResponse) -> Result<MongoDocumentResult, String> {
#[derive(Deserialize)]
struct CountResponse {
count: u64,
#[serde(rename = "_shards")]
shards: ElasticsearchShards,
}
#[derive(Deserialize)]
struct ElasticsearchShards {
total: u64,
successful: u64,
#[serde(default)]
skipped: u64,
failed: u64,
}
impl ElasticsearchShards {
fn is_complete(&self) -> bool {
// Elasticsearch only skips shards that cannot match, so successful and
// skipped shards together must account for every requested shard.
self.failed == 0 && self.successful.checked_add(self.skipped) == Some(self.total)
}
}
pub async fn count_documents(client: &EsClient, index: &str, filter: Option<&str>) -> Result<u64, String> {
let body = build_count_documents_body(filter)?;
let path = elasticsearch_index_path(index, "_count");
let resp = client.post(&path).json(&body).send().await.map_err(|e| format!("Elasticsearch request failed: {e}"))?;
if !client.response_status(&resp).is_success() {
let body = resp.text().await.unwrap_or_default();
return Err(format!("Elasticsearch error: {body}"));
}
let result: CountResponse = resp.json().await.map_err(|e| format!("Elasticsearch count parse error: {e}"))?;
if !result.shards.is_complete() {
return Err(format!(
"Elasticsearch count returned an incomplete shard response: {} successful, {} skipped, {} failed of {} shards",
result.shards.successful, result.shards.skipped, result.shards.failed, result.shards.total,
));
}
Ok(result.count)
}
fn search_response_to_document_result(result: SearchResponse) -> Result<DocumentQueryResult, String> {
// A 200 search response can still omit failed shards. Only expose an
// exact total when both the total relation and shard metadata agree.
let total_is_exact = !result.timed_out
&& !result.terminated_early
&& result.hits.total.is_exact()
&& result.shards.as_ref().is_some_and(ElasticsearchShards::is_complete);
let total = result.hits.total.value();
let documents: Vec<serde_json::Value> = result
.hits
.hits
@@ -518,11 +585,12 @@ fn search_response_to_document_result(result: SearchResponse) -> Result<MongoDoc
.collect::<Result<Vec<_>, _>>()
.map_err(|e| format!("Elasticsearch document serialization failed: {e}"))?;
Ok(MongoDocumentResult {
Ok(DocumentQueryResult {
documents,
raw_documents: Some(raw_documents),
extended_documents: None,
total: result.hits.total.value(),
total,
total_is_exact,
})
}
@@ -544,6 +612,14 @@ fn build_find_documents_body(
Ok(serde_json::Value::Object(body))
}
fn build_count_documents_body(filter: Option<&str>) -> Result<serde_json::Value, String> {
let mut body = serde_json::Map::new();
if let Some(query) = elasticsearch_query_from_document_filter(filter)? {
body.insert("query".to_string(), query);
}
Ok(serde_json::Value::Object(body))
}
fn elasticsearch_query_from_document_filter(filter: Option<&str>) -> Result<Option<serde_json::Value>, String> {
let Some(filter) = filter.map(str::trim).filter(|value| !value.is_empty()) else {
return Ok(None);
@@ -865,6 +941,32 @@ fn is_elasticsearch_ndjson_path(path: &str) -> bool {
|| path.ends_with("/_msearch/template")
}
fn is_elasticsearch_cat_path(path: &str) -> bool {
let path = path.split('?').next().unwrap_or(path).trim_end_matches('/');
path == "/_cat" || path.starts_with("/_cat/")
}
fn elasticsearch_query_parameter(path: &str, name: &str) -> Option<String> {
let query = path.split_once('?')?.1;
query.split('&').find_map(|part| {
let (key, value) = part.split_once('=').unwrap_or((part, ""));
let key = percent_decode_str(key).decode_utf8_lossy();
if key.eq_ignore_ascii_case(name) {
Some(percent_decode_str(value).decode_utf8_lossy().into_owned())
} else {
None
}
})
}
fn add_default_cat_json_format(mut request: ElasticsearchRestRequest) -> ElasticsearchRestRequest {
if is_elasticsearch_cat_path(&request.path) && elasticsearch_query_parameter(&request.path, "format").is_none() {
request.path.push(if request.path.contains('?') { '&' } else { '?' });
request.path.push_str("format=json");
}
request
}
fn normalize_elasticsearch_rest_path(path: &str) -> String {
let (path_part, query) = path.split_once('?').map_or((path, None), |(path, query)| (path, Some(query)));
let mut normalized = String::with_capacity(path.len());
@@ -998,7 +1100,9 @@ pub async fn execute_rest_query(client: &EsClient, input: &str) -> Result<crate:
return execute_sql_query(client, input, start).await;
}
let request = parse_elasticsearch_rest_request(input)?;
// CAT APIs default to text, so request JSON for an unformatted CAT call.
// The frontend renders the returned HTTP body in its JSON response panel.
let request = add_default_cat_json_format(parse_elasticsearch_rest_request(input)?);
let mut builder = client.request(request.method, &request.path);
if let Some(body) = request.body {
builder = match request.body_kind {
@@ -1721,8 +1825,8 @@ fn parse_aggregations(aggs: &serde_json::Map<String, serde_json::Value>) -> (Vec
#[cfg(test)]
mod tests {
use super::{
build_find_documents_body, elasticsearch_accept_invalid_certs, elasticsearch_base_url_fallbacks,
redact_elasticsearch_url, EsClient, SearchResponse,
build_count_documents_body, build_find_documents_body, elasticsearch_accept_invalid_certs,
elasticsearch_base_url_fallbacks, redact_elasticsearch_url, EsClient, SearchResponse,
};
use serde_json::json;
use std::time::Duration;
@@ -1947,6 +2051,7 @@ mod tests {
"sort": [{ "created_at": { "order": "desc" } }]
})
);
assert!(body.get("track_total_hits").is_none());
}
#[test]
@@ -1981,6 +2086,95 @@ mod tests {
);
}
#[test]
fn builds_elasticsearch_count_body_with_the_same_native_query_filter() {
let body = build_count_documents_body(Some(r#"{"$esQuery":{"term":{"status":"active"}}}"#)).unwrap();
assert_eq!(body, json!({ "query": { "term": { "status": "active" } } }));
assert!(body.get("from").is_none());
assert!(body.get("size").is_none());
assert!(body.get("sort").is_none());
}
#[test]
fn builds_elasticsearch_count_body_with_structured_filter_operators() {
let filter = r#"{"$and":[{"city":{"$ne":"上海"}},{"age":{"$gte":18}}]}"#;
assert_eq!(
build_count_documents_body(Some(filter)).unwrap(),
json!({
"query": {
"bool": {
"filter": [
{ "bool": { "must_not": [{ "term": { "city": "上海" } }] } },
{ "range": { "age": { "gte": 18 } } }
]
}
}
})
);
}
#[test]
fn accepts_only_complete_elasticsearch_count_shards() {
assert!(super::ElasticsearchShards { total: 3, successful: 2, skipped: 1, failed: 0 }.is_complete());
assert!(!super::ElasticsearchShards { total: 3, successful: 2, skipped: 0, failed: 0 }.is_complete());
assert!(!super::ElasticsearchShards { total: 3, successful: 2, skipped: 0, failed: 1 }.is_complete());
}
#[tokio::test]
async fn counts_documents_with_the_translated_filter() {
use tokio::io::AsyncWriteExt;
let response_body = r#"{"count":552033,"_shards":{"total":3,"successful":3,"skipped":0,"failed":0}}"#;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let request = read_http_request(&mut socket).await;
assert!(request.starts_with("POST /orders/_count "));
let body = request.split_once("\r\n\r\n").unwrap().1;
assert_eq!(
serde_json::from_str::<serde_json::Value>(body).unwrap(),
json!({ "query": { "term": { "status": "active" } } })
);
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
response_body.len(),
response_body,
);
socket.write_all(response.as_bytes()).await.unwrap();
});
let client = EsClient::new(&format!("http://{addr}"), None, None, false, Duration::from_secs(1));
assert_eq!(super::count_documents(&client, "orders", Some(r#"{"status":"active"}"#)).await.unwrap(), 552_033);
server.await.unwrap();
}
#[tokio::test]
async fn rejects_partial_elasticsearch_document_count() {
use tokio::io::AsyncWriteExt;
let response_body = r#"{"count":4,"_shards":{"total":3,"successful":2,"skipped":0,"failed":1}}"#;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let request = read_http_request(&mut socket).await;
assert!(request.starts_with("POST /orders/_count "));
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
response_body.len(),
response_body,
);
socket.write_all(response.as_bytes()).await.unwrap();
});
let client = EsClient::new(&format!("http://{addr}"), None, None, false, Duration::from_secs(1));
let error = super::count_documents(&client, "orders", None).await.unwrap_err();
assert!(error.contains("incomplete shard response: 2 successful, 0 skipped, 1 failed of 3 shards"));
server.await.unwrap();
}
#[test]
fn builds_elasticsearch_find_body_with_structured_filter_operators() {
let body = build_find_documents_body(
@@ -2053,6 +2247,7 @@ mod tests {
#[test]
fn parses_search_total_from_elasticsearch_6_number_shape() {
let response: SearchResponse = serde_json::from_value(json!({
"_shards": { "total": 1, "successful": 1, "skipped": 0, "failed": 0 },
"hits": {
"total": 5,
"hits": []
@@ -2061,11 +2256,13 @@ mod tests {
.unwrap();
assert_eq!(response.hits.total.value(), 5);
assert!(response.hits.total.is_exact());
}
#[test]
fn parses_search_total_from_elasticsearch_7_object_shape() {
let response: SearchResponse = serde_json::from_value(json!({
"_shards": { "total": 1, "successful": 1, "skipped": 0, "failed": 0 },
"hits": {
"total": { "value": 5, "relation": "eq" },
"hits": []
@@ -2074,6 +2271,78 @@ mod tests {
.unwrap();
assert_eq!(response.hits.total.value(), 5);
assert!(response.hits.total.is_exact());
let result = super::search_response_to_document_result(response).unwrap();
assert!(result.total_is_exact);
}
#[test]
fn preserves_elasticsearch_lower_bound_total_relation() {
let response: SearchResponse = serde_json::from_value(json!({
"hits": {
"total": { "value": 10_000, "relation": "gte" },
"hits": []
}
}))
.unwrap();
assert_eq!(response.hits.total.value(), 10_000);
assert!(!response.hits.total.is_exact());
let result = super::search_response_to_document_result(response).unwrap();
assert_eq!(result.total, 10_000);
assert!(!result.total_is_exact);
}
#[test]
fn treats_search_total_as_a_lower_bound_when_a_shard_failed() {
let response: SearchResponse = serde_json::from_value(json!({
"_shards": { "total": 3, "successful": 2, "skipped": 0, "failed": 1 },
"hits": {
"total": { "value": 5, "relation": "eq" },
"hits": []
}
}))
.unwrap();
let result = super::search_response_to_document_result(response).unwrap();
assert_eq!(result.total, 5);
assert!(!result.total_is_exact);
}
#[test]
fn treats_timed_out_or_terminated_searches_as_lower_bounds() {
for response in [
json!({
"timed_out": true,
"_shards": { "total": 1, "successful": 1, "skipped": 0, "failed": 0 },
"hits": { "total": { "value": 5, "relation": "eq" }, "hits": [] }
}),
json!({
"terminated_early": true,
"_shards": { "total": 1, "successful": 1, "skipped": 0, "failed": 0 },
"hits": { "total": { "value": 5, "relation": "eq" }, "hits": [] }
}),
] {
let response: SearchResponse = serde_json::from_value(response).unwrap();
let result = super::search_response_to_document_result(response).unwrap();
assert_eq!(result.total, 5);
assert!(!result.total_is_exact);
}
}
#[test]
fn treats_search_total_without_shard_metadata_as_a_lower_bound() {
let response: SearchResponse = serde_json::from_value(json!({
"hits": {
"total": { "value": 5, "relation": "eq" },
"hits": []
}
}))
.unwrap();
let result = super::search_response_to_document_result(response).unwrap();
assert_eq!(result.total, 5);
assert!(!result.total_is_exact);
}
#[test]
@@ -2191,6 +2460,22 @@ mod tests {
assert_eq!(result.affected_rows, 2);
}
#[test]
fn adds_json_format_only_when_cat_format_is_not_explicit() {
let plain =
super::add_default_cat_json_format(super::parse_elasticsearch_rest_request("GET /_cat/indices").unwrap());
assert_eq!(plain.path, "/_cat/indices?format=json");
let defaulted =
super::add_default_cat_json_format(super::parse_elasticsearch_rest_request("GET /_cat/indices?v").unwrap());
assert_eq!(defaulted.path, "/_cat/indices?v&format=json");
let explicit_text = super::add_default_cat_json_format(
super::parse_elasticsearch_rest_request("GET /_cat/indices?format=txt").unwrap(),
);
assert_eq!(explicit_text.path, "/_cat/indices?format=txt");
}
#[test]
fn preserves_http_status_for_plain_text_rest_errors() {
let result =
@@ -2223,11 +2508,10 @@ mod tests {
}
#[tokio::test]
async fn execute_rest_query_keeps_plain_text_response_body() {
async fn execute_rest_query_preserves_index_specific_cat_json_response() {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let body =
"health status index docs.count store.size\ngreen open app-log-2026-07 42 10mb\n";
let body = r#"[{"health":"green","index":"app-log-2026-07","docs.count":"42"}]"#;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
@@ -2235,7 +2519,67 @@ mod tests {
let mut request = [0_u8; 1024];
let read = socket.read(&mut request).await.unwrap();
let request = String::from_utf8_lossy(&request[..read]);
assert!(request.starts_with("GET /_cat/indices "));
assert!(request.starts_with("GET /_cat/indices/data_pack_and_box_index_v1?format=json "));
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
body.len(),
body
);
socket.write_all(response.as_bytes()).await.unwrap();
});
let client = EsClient::new(&format!("http://{addr}"), None, None, false, Duration::from_secs(1));
let result = super::execute_rest_query(&client, "GET /_cat/indices/data_pack_and_box_index_v1").await.unwrap();
server.await.unwrap();
assert_eq!(result.columns, vec!["status", "response"]);
assert_eq!(result.rows, vec![vec![json!(200), json!(body)]]);
}
#[tokio::test]
async fn execute_rest_query_preserves_explicit_cat_json_response() {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let body = r#"[{"index":"data_pack_and_box_index_v1","docs.count":"42"}]"#;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let mut request = [0_u8; 1024];
let read = socket.read(&mut request).await.unwrap();
let request = String::from_utf8_lossy(&request[..read]);
assert!(request.starts_with("GET /_cat/indices/data_pack_and_box_index_v1?format=json "));
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
body.len(),
body
);
socket.write_all(response.as_bytes()).await.unwrap();
});
let client = EsClient::new(&format!("http://{addr}"), None, None, false, Duration::from_secs(1));
let result = super::execute_rest_query(&client, "GET /_cat/indices/data_pack_and_box_index_v1?format=json")
.await
.unwrap();
server.await.unwrap();
assert_eq!(result.columns, vec!["status", "response"]);
assert_eq!(result.rows, vec![vec![json!(200), json!(body)]])
}
#[tokio::test]
async fn execute_rest_query_keeps_default_cat_text_when_server_does_not_return_json() {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let body = "health status\ngreen open\n";
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let mut request = [0_u8; 1024];
let read = socket.read(&mut request).await.unwrap();
let request = String::from_utf8_lossy(&request[..read]);
assert!(request.starts_with("GET /_cat/indices?format=json "));
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
body.len(),
@@ -2249,8 +2593,36 @@ mod tests {
server.await.unwrap();
assert_eq!(result.columns, vec!["response"]);
assert_eq!(result.rows.len(), 2);
assert_eq!(result.rows[1][0], json!("green open app-log-2026-07 42 10mb"));
assert_eq!(result.rows, vec![vec![json!("health status")], vec![json!("green open")]]);
}
#[tokio::test]
async fn execute_rest_query_keeps_explicit_text_cat_response() {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let body = "health status\ngreen open\n";
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let mut request = [0_u8; 1024];
let read = socket.read(&mut request).await.unwrap();
let request = String::from_utf8_lossy(&request[..read]);
assert!(request.starts_with("GET /_cat/indices?format=txt "));
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
body.len(),
body
);
socket.write_all(response.as_bytes()).await.unwrap();
});
let client = EsClient::new(&format!("http://{addr}"), None, None, false, Duration::from_secs(1));
let result = super::execute_rest_query(&client, "GET /_cat/indices?format=txt").await.unwrap();
server.await.unwrap();
assert_eq!(result.columns, vec!["response"]);
assert_eq!(result.rows, vec![vec![json!("health status")], vec![json!("green open")]]);
}
#[tokio::test]
+1
View File
@@ -2,6 +2,7 @@ pub mod agent_driver;
pub mod clickhouse_driver;
pub mod cloudflare_d1;
pub use cloudflare_d1 as cloudflare_d1_driver;
pub mod document_result;
pub mod duckdb_driver;
pub mod duckdb_sql;
#[cfg(feature = "duckdb-bundled")]
+34 -13
View File
@@ -12,15 +12,9 @@ use futures::{io::AsyncReadExt, io::AsyncWriteExt, TryStreamExt};
use percent_encoding::percent_decode_str;
use std::{collections::HashSet, time::Duration};
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MongoDocumentResult {
pub documents: Vec<serde_json::Value>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub raw_documents: Option<Vec<String>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub extended_documents: Option<Vec<serde_json::Value>>,
pub total: u64,
}
pub use super::document_result::DocumentQueryResult;
/// Backward-compatible name for callers of Mongo-specific APIs.
pub type MongoDocumentResult = DocumentQueryResult;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MongoDropIndexesResult {
@@ -676,7 +670,13 @@ pub async fn find_documents(
extended_documents.push(Bson::Document(doc).into_relaxed_extjson());
}
Ok(MongoDocumentResult { documents, raw_documents: None, extended_documents: Some(extended_documents), total })
Ok(MongoDocumentResult {
documents,
raw_documents: None,
extended_documents: Some(extended_documents),
total,
total_is_exact: true,
})
}
#[derive(Debug, Default, Deserialize)]
@@ -812,7 +812,13 @@ pub async fn find_documents_extended_json(
documents.push(bson_to_browser_json(&Bson::Document(doc)));
}
Ok(MongoDocumentResult { extended_documents: Some(documents.clone()), documents, raw_documents: None, total })
Ok(MongoDocumentResult {
extended_documents: Some(documents.clone()),
documents,
raw_documents: None,
total,
total_is_exact: true,
})
}
/// Run `db.collection.aggregate(pipeline, options)`.
@@ -858,6 +864,7 @@ pub async fn aggregate_documents(
raw_documents: None,
extended_documents: Some(vec![extended]),
total: 1,
total_is_exact: true,
});
}
@@ -909,7 +916,13 @@ async fn drain_document_cursor(
documents.truncate(max_rows);
extended_documents.truncate(max_rows);
}
Ok(MongoDocumentResult { documents, raw_documents: None, extended_documents: Some(extended_documents), total })
Ok(MongoDocumentResult {
documents,
raw_documents: None,
extended_documents: Some(extended_documents),
total,
total_is_exact: true,
})
}
fn parse_aggregate_options_document(options_json: Option<&str>) -> Result<Document, String> {
@@ -963,7 +976,13 @@ pub async fn distinct(
let extended_documents = values.into_iter().map(|value| value.into_relaxed_extjson()).collect::<Vec<_>>();
let total = documents.len() as u64;
Ok(MongoDocumentResult { documents, raw_documents: None, extended_documents: Some(extended_documents), total })
Ok(MongoDocumentResult {
documents,
raw_documents: None,
extended_documents: Some(extended_documents),
total,
total_is_exact: true,
})
}
pub async fn create_index(
@@ -1329,12 +1348,14 @@ fn single_document_result(document: Option<Document>) -> MongoDocumentResult {
raw_documents: None,
extended_documents: Some(vec![Bson::Document(document).into_relaxed_extjson()]),
total: 1,
total_is_exact: true,
},
None => MongoDocumentResult {
documents: Vec::new(),
raw_documents: None,
extended_documents: Some(Vec::new()),
total: 0,
total_is_exact: true,
},
}
}
+5 -3
View File
@@ -483,7 +483,7 @@ pub async fn find_documents(
collection: &str,
skip: u64,
limit: i64,
) -> Result<crate::db::mongo_driver::MongoDocumentResult, String> {
) -> Result<crate::db::document_result::DocumentQueryResult, String> {
if client.kind == VectorDbKind::ChromaDb {
let start = std::time::Instant::now();
let url = format!(
@@ -520,11 +520,12 @@ pub async fn find_documents(
Value::Object(map)
})
.collect();
return Ok(crate::db::mongo_driver::MongoDocumentResult {
return Ok(crate::db::document_result::DocumentQueryResult {
documents,
raw_documents: None,
extended_documents: None,
total: result.affected_rows,
total_is_exact: true,
});
}
@@ -567,11 +568,12 @@ pub async fn find_documents(
Value::Object(map)
})
.collect();
Ok(crate::db::mongo_driver::MongoDocumentResult {
Ok(crate::db::document_result::DocumentQueryResult {
documents,
raw_documents: None,
extended_documents: None,
total: result.affected_rows,
total_is_exact: true,
})
}
+20 -2
View File
@@ -1,6 +1,6 @@
use crate::connection::{AppState, PoolKind};
use crate::db::agent_driver::mongo_document_id_params;
use crate::db::mongo_driver::MongoDocumentResult;
use crate::db::document_result::DocumentQueryResult;
use crate::db::{elasticsearch_driver, mongo_driver, vector_driver};
pub use crate::db::vector_driver::CollectionInfo;
@@ -377,7 +377,7 @@ pub async fn find_documents_core(
filter: Option<&str>,
projection: Option<&str>,
sort: Option<&str>,
) -> Result<MongoDocumentResult, String> {
) -> Result<DocumentQueryResult, String> {
ensure_document_pool(state, connection_id).await?;
let connections = state.connections.read().await;
match connections.get(connection_id).ok_or("Not found")? {
@@ -425,6 +425,24 @@ pub async fn find_documents_core(
}
}
pub async fn count_elasticsearch_documents_core(
state: &AppState,
connection_id: &str,
index: &str,
filter: Option<&str>,
) -> Result<u64, String> {
ensure_document_pool(state, connection_id).await?;
let connections = state.connections.read().await;
match connections.get(connection_id).ok_or("Not found")? {
PoolKind::Elasticsearch(client) => {
let client = client.clone();
drop(connections);
elasticsearch_driver::count_documents(&client, index, filter).await
}
_ => Err("Not an Elasticsearch connection".to_string()),
}
}
fn is_unknown_agent_method_error(error: &str, method: &str) -> bool {
let lower = error.to_ascii_lowercase();
lower.contains(method) && (lower.contains("unknown method") || lower.contains("method not found"))