Files
dbx/crates/dbx-core/src/db/elasticsearch_driver.rs
T

3816 lines
147 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
use percent_encoding::{percent_decode_str, utf8_percent_encode, AsciiSet, CONTROLS};
use regex::Regex;
use reqwest::{Client as HttpClient, Method, StatusCode};
use serde::Deserialize;
use serde_json::Value;
use std::collections::{BTreeMap, HashMap, HashSet};
use std::error::Error;
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
use std::time::Duration;
use super::{http_client_builder, with_connection_timeout};
use crate::db::document_result::DocumentQueryResult;
use crate::types::QueryResult;
const ELASTICSEARCH_PATH_SEGMENT_ENCODE_SET: &AsciiSet = &CONTROLS
.add(b' ')
.add(b'"')
.add(b'#')
.add(b'%')
.add(b'/')
.add(b'<')
.add(b'>')
.add(b'?')
.add(b'[')
.add(b'\\')
.add(b']')
.add(b'^')
.add(b'`')
.add(b'{')
.add(b'|')
.add(b'}');
const ELASTICSEARCH_QUERY_VALUE_ENCODE_SET: &AsciiSet =
&CONTROLS.add(b' ').add(b'"').add(b'#').add(b'%').add(b'&').add(b'+').add(b'/').add(b'=').add(b'?');
const KIBANA_PROXY_STATUS_HEADER: &str = "x-console-proxy-status-code";
const ELASTICSEARCH_REST_TABLE_MAX_BODY_BYTES: usize = 8 * 1024 * 1024;
const ELASTICSEARCH_REST_TABLE_MAX_ROWS: usize = 2_000;
const ELASTICSEARCH_REST_TABLE_MAX_CELLS: usize = 200_000;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ElasticsearchTransportMode {
Direct,
KibanaProxy,
}
pub struct EsClient {
http: HttpClient,
base_url: String,
fallback_base_urls: Vec<String>,
auth: Option<(String, String)>,
transport_mode: ElasticsearchTransportMode,
/// GET path used for connect / health / test (default "/").
connectivity_check_path: String,
/// 为 true 时完全跳过连通性检查(test_connection 直接返回 Ok)。
/// 用于账号连 `/` 或任何检查路径都无权限、且集群间权限不统一的场景。
connectivity_check_disabled: bool,
/// 正则:把易变的时间/滚动后缀折叠成 `*`,将同一前缀的滚动索引聚合成一个
/// pattern 节点。`None` 表示关闭聚合,展示原始索引名。
index_grouping: Option<Regex>,
}
impl EsClient {
pub fn new(
url: &str,
username: Option<&str>,
password: Option<&str>,
accept_invalid_certs: bool,
timeout: Duration,
) -> Self {
Self::new_with_mode(
url,
username,
password,
accept_invalid_certs,
timeout,
ElasticsearchTransportMode::Direct,
"/".to_string(),
false,
None,
)
}
fn new_with_mode(
url: &str,
username: Option<&str>,
password: Option<&str>,
accept_invalid_certs: bool,
timeout: Duration,
transport_mode: ElasticsearchTransportMode,
connectivity_check_path: String,
connectivity_check_disabled: bool,
index_grouping: Option<Regex>,
) -> Self {
let base_url = url.trim_end_matches('/').to_string();
let auth = match (username, password) {
(Some(u), Some(p)) if !u.is_empty() => Some((u.to_string(), p.to_string())),
_ => None,
};
let mut builder = http_client_builder(timeout).danger_accept_invalid_certs(accept_invalid_certs);
if let Some(addrs) = elasticsearch_localhost_resolve_addrs(&base_url, connectivity_check_disabled) {
builder = builder.resolve_to_addrs("localhost", &addrs);
}
let http = builder.build().unwrap_or_else(|_| HttpClient::new());
let fallback_base_urls = elasticsearch_base_url_fallbacks(&base_url);
Self {
http,
base_url,
fallback_base_urls,
auth,
transport_mode,
connectivity_check_path,
connectivity_check_disabled,
index_grouping,
}
}
pub fn from_config(
url: &str,
username: Option<&str>,
password: Option<&str>,
tls_enabled: bool,
url_params: Option<&str>,
external_config: Option<&Value>,
timeout: Duration,
) -> Self {
let kibana_base_path = elasticsearch_kibana_base_path(external_config);
let transport_mode = if kibana_base_path.is_some() {
ElasticsearchTransportMode::KibanaProxy
} else {
ElasticsearchTransportMode::Direct
};
let base_url = format!("{}{}", url.trim_end_matches('/'), kibana_base_path.as_deref().unwrap_or(""));
let connectivity_check_path = elasticsearch_connectivity_check_path(external_config);
let connectivity_check_disabled = elasticsearch_connectivity_check_disabled(external_config);
let index_grouping = elasticsearch_index_grouping(external_config);
Self::new_with_mode(
&base_url,
username,
password,
elasticsearch_accept_invalid_certs(tls_enabled, url_params),
timeout,
transport_mode,
connectivity_check_path,
connectivity_check_disabled,
index_grouping,
)
}
fn get(&self, path: &str) -> reqwest::RequestBuilder {
self.request(Method::GET, path)
}
fn post(&self, path: &str) -> reqwest::RequestBuilder {
self.request(Method::POST, path)
}
fn put(&self, path: &str) -> reqwest::RequestBuilder {
self.request(Method::PUT, path)
}
fn delete(&self, path: &str) -> reqwest::RequestBuilder {
self.request(Method::DELETE, path)
}
fn request(&self, method: Method, path: &str) -> reqwest::RequestBuilder {
let req = match self.transport_mode {
ElasticsearchTransportMode::Direct => self.http.request(method, format!("{}{}", self.base_url, path)),
ElasticsearchTransportMode::KibanaProxy => self
.http
.post(format!("{}/api/console/proxy", self.base_url))
.query(&[("path", path), ("method", method.as_str())])
.header("kbn-xsrf", "true"),
};
self.with_auth(req)
}
fn with_auth(&self, req: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
if let Some((ref user, ref pass)) = self.auth {
req.basic_auth(user, Some(pass))
} else {
req
}
}
fn response_status(&self, response: &reqwest::Response) -> StatusCode {
if self.transport_mode == ElasticsearchTransportMode::KibanaProxy {
if let Some(status) = response
.headers()
.get(KIBANA_PROXY_STATUS_HEADER)
.and_then(|value| value.to_str().ok())
.and_then(|value| value.parse::<u16>().ok())
.and_then(|value| StatusCode::from_u16(value).ok())
{
return status;
}
}
response.status()
}
}
impl Clone for EsClient {
fn clone(&self) -> Self {
Self {
http: self.http.clone(),
base_url: self.base_url.clone(),
fallback_base_urls: self.fallback_base_urls.clone(),
auth: self.auth.clone(),
transport_mode: self.transport_mode,
connectivity_check_path: self.connectivity_check_path.clone(),
connectivity_check_disabled: self.connectivity_check_disabled,
index_grouping: self.index_grouping.clone(),
}
}
}
fn elasticsearch_kibana_base_path(external_config: Option<&Value>) -> Option<String> {
let config = external_config?.as_object()?;
let mode = config.get("mode").and_then(Value::as_str)?;
if !mode.eq_ignore_ascii_case("kibana") {
return None;
}
let base_path = config.get("kibanaBasePath").and_then(Value::as_str).unwrap_or("").trim().trim_matches('/');
Some(if base_path.is_empty() { String::new() } else { format!("/{base_path}") })
}
/// Path used for connectivity checks (test / open / health). Defaults to `/`.
/// Accepts bare paths (`my-index/_search`) or a single-line `GET path` paste.
pub fn elasticsearch_connectivity_check_path(external_config: Option<&Value>) -> String {
let raw = external_config
.and_then(Value::as_object)
.and_then(|config| config.get("connectivityCheckPath"))
.and_then(Value::as_str)
.unwrap_or("")
.trim();
if raw.is_empty() {
return "/".to_string();
}
// First line only — ignore accidental body lines from console paste.
let line = raw.lines().next().unwrap_or("").trim();
let without_method = line
.strip_prefix("GET ")
.or_else(|| line.strip_prefix("get "))
.or_else(|| line.strip_prefix("Get "))
.unwrap_or(line)
.trim();
if without_method.is_empty() || without_method == "/" {
return "/".to_string();
}
if without_method.starts_with('/') {
without_method.to_string()
} else {
format!("/{without_method}")
}
}
/// 解析连接配置里的索引聚合正则。**默认关闭**(命名格式因环境而异,不做全局假设)。
/// - 缺省/空/`off`/`none`/`false` → 关闭聚合,展示原始索引名。
/// - 其它 → 作为自定义正则;编译失败同样视为关闭,避免意外折叠。
///
/// 语义:用 `${1}*` 模板替换匹配区间——正则带捕获组 1 时保留其内容做前缀,
/// 不带捕获组时即把匹配到的“易变尾巴”替换成 `*`。
/// 是否完全跳过连通性检查。读取配置 `connectivityCheckDisabled`(布尔)。
/// 也兼容字符串 "true"/"1"/"yes"/"on"。用于账号无任何集群/索引探活权限的场景。
pub fn elasticsearch_connectivity_check_disabled(external_config: Option<&Value>) -> bool {
let Some(value) = external_config.and_then(Value::as_object).and_then(|c| c.get("connectivityCheckDisabled"))
else {
return false;
};
match value {
Value::Bool(b) => *b,
Value::String(s) => {
let s = s.trim();
s.eq_ignore_ascii_case("true") || s == "1" || s.eq_ignore_ascii_case("yes") || s.eq_ignore_ascii_case("on")
}
_ => false,
}
}
pub fn elasticsearch_index_grouping(external_config: Option<&Value>) -> Option<Regex> {
let raw = external_config
.and_then(Value::as_object)
.and_then(|config| config.get("indexGroupingPattern"))
.and_then(Value::as_str)
.unwrap_or("")
.trim();
if raw.is_empty()
|| raw.eq_ignore_ascii_case("off")
|| raw.eq_ignore_ascii_case("none")
|| raw.eq_ignore_ascii_case("false")
{
return None;
}
Regex::new(raw).ok()
}
/// 用分组正则把索引名聚合成 pattern。`None` 表示关闭聚合,原样返回。
fn group_index_names(names: Vec<String>, pattern: Option<&Regex>) -> Vec<String> {
let Some(re) = pattern else {
return names;
};
let mut buckets: BTreeMap<String, Vec<String>> = BTreeMap::new();
for name in names {
// `${1}*`:有捕获组则保留组1做前缀,无组则把匹配尾巴替换成 `*`。
let key = re.replace(&name, "${1}*").into_owned();
buckets.entry(key).or_default().push(name);
}
let mut out: Vec<String> = buckets.into_keys().collect();
out.sort();
out.dedup();
out
}
pub async fn test_connection(client: &mut EsClient, timeout: Duration) -> Result<(), String> {
// 用户显式关闭连通性检查:不发探活请求,直接视为可连。
if client.connectivity_check_disabled {
return Ok(());
}
let mut errors = Vec::new();
let urls = std::iter::once(client.base_url.clone()).chain(client.fallback_base_urls.clone());
let check_path = client.connectivity_check_path.clone();
for base_url in urls {
client.base_url = base_url.clone();
let path = check_path.clone();
let resp = with_connection_timeout("Elasticsearch", timeout, async {
client.get(&path).send().await.map_err(|e| {
format!(
"Elasticsearch connection failed for {} ({}): {}",
redact_elasticsearch_url(&base_url),
path,
format_reqwest_error(&e)
)
})
})
.await;
let resp = match resp {
Ok(resp) => resp,
Err(err) => {
errors.push(err);
continue;
}
};
let status = client.response_status(&resp);
if !status.is_success() {
let body = resp.text().await.unwrap_or_default();
return Err(format!("Elasticsearch error ({status}) for {check_path}: {body}"));
}
return Ok(());
}
if errors.is_empty() {
Err("Elasticsearch connection failed: no URL candidates".to_string())
} else {
Err(errors.join("; "))
}
}
pub fn elasticsearch_accept_invalid_certs(tls_enabled: bool, url_params: Option<&str>) -> bool {
tls_enabled
|| elasticsearch_url_params_flag(url_params, "sslmode", &["disable", "allow"])
|| elasticsearch_url_params_flag(url_params, "tlsverify", &["false", "0", "no", "off"])
|| elasticsearch_url_params_flag(url_params, "verify", &["false", "0", "no", "off"])
|| elasticsearch_url_params_flag(url_params, "insecure", &["true", "1", "yes", "on"])
|| elasticsearch_url_params_flag(url_params, "accept_invalid_certs", &["true", "1", "yes", "on"])
}
fn elasticsearch_url_params_flag(params: Option<&str>, key: &str, expected_values: &[&str]) -> bool {
params.unwrap_or("").trim().trim_start_matches('?').split('&').filter_map(|pair| pair.split_once('=')).any(
|(k, v)| {
k.trim().eq_ignore_ascii_case(key)
&& expected_values.iter().any(|expected| v.trim().eq_ignore_ascii_case(expected))
},
)
}
fn elasticsearch_base_url_fallbacks(base_url: &str) -> Vec<String> {
let Ok(parsed) = reqwest::Url::parse(base_url) else {
return Vec::new();
};
let Some(host) = parsed.host_str() else {
return Vec::new();
};
if !host.eq_ignore_ascii_case("localhost") {
return Vec::new();
}
let mut fallback = parsed;
if fallback.set_host(Some("127.0.0.1")).is_ok() {
vec![fallback.as_str().trim_end_matches('/').to_string()]
} else {
Vec::new()
}
}
fn elasticsearch_localhost_resolve_addrs(base_url: &str, connectivity_check_disabled: bool) -> Option<[SocketAddr; 2]> {
if !connectivity_check_disabled {
return None;
}
let parsed = reqwest::Url::parse(base_url).ok()?;
if !parsed.host_str()?.eq_ignore_ascii_case("localhost") {
return None;
}
Some([SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 0), SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)])
}
fn elasticsearch_index_path(index: &str, endpoint: &str) -> String {
format!("/{}/{}", elasticsearch_path_segment(index), endpoint.trim_start_matches('/'))
}
fn elasticsearch_path_segment(value: &str) -> String {
utf8_percent_encode(value, ELASTICSEARCH_PATH_SEGMENT_ENCODE_SET).to_string()
}
fn elasticsearch_query_value(value: &str) -> String {
utf8_percent_encode(value, ELASTICSEARCH_QUERY_VALUE_ENCODE_SET).to_string()
}
fn elasticsearch_document_path(index: &str, id: &str, document_type: Option<&str>, routing: Option<&str>) -> String {
let document_type = document_type.map(str::trim).filter(|value| !value.is_empty()).unwrap_or("_doc");
let base = format!(
"/{}/{}/{}",
elasticsearch_path_segment(index),
elasticsearch_path_segment(document_type),
elasticsearch_path_segment(id)
);
elasticsearch_path_with_routing_refresh(base, routing)
}
/// Auto-id index path: `POST /{index}/_doc` with optional custom routing.
fn elasticsearch_auto_id_document_path(index: &str, routing: Option<&str>) -> String {
let base = format!("/{}/_doc", elasticsearch_path_segment(index));
elasticsearch_path_with_routing_refresh(base, routing)
}
fn elasticsearch_path_with_routing_refresh(base: String, routing: Option<&str>) -> String {
if let Some(routing) = routing.map(str::trim).filter(|value| !value.is_empty()) {
format!("{base}?routing={}&refresh=true", elasticsearch_query_value(routing))
} else {
format!("{base}?refresh=true")
}
}
fn redact_elasticsearch_url(url: &str) -> String {
let Ok(mut parsed) = reqwest::Url::parse(url) else {
return url.to_string();
};
if !parsed.username().is_empty() {
let _ = parsed.set_username("user");
}
if parsed.password().is_some() {
let _ = parsed.set_password(Some("password"));
}
parsed.as_str().trim_end_matches('/').to_string()
}
fn format_reqwest_error(err: &reqwest::Error) -> String {
let mut parts = vec![err.to_string()];
let mut source = err.source();
while let Some(err) = source {
let text = err.to_string();
if !text.is_empty() && !parts.iter().any(|part| part == &text) {
parts.push(text);
}
source = err.source();
}
parts.join(": ")
}
#[derive(Deserialize)]
struct CatIndex {
index: String,
}
#[derive(Deserialize)]
struct ResolveIndexResponse {
#[serde(default)]
indices: Vec<ResolveNamed>,
#[serde(default)]
data_streams: Vec<ResolveNamed>,
}
#[derive(Deserialize)]
struct ResolveNamed {
name: String,
}
/// 去掉 ES 内部索引(以 `.` 开头),排序并去重后返回可见索引名。
fn normalize_index_names(names: impl Iterator<Item = String>) -> Vec<String> {
let mut names: Vec<String> = names.filter(|name| !name.starts_with('.')).collect();
names.sort();
names.dedup();
names
}
pub async fn list_indices(client: &EsClient) -> Result<Vec<String>, String> {
let names = list_raw_index_names(client).await?;
Ok(group_index_names(names, client.index_grouping.as_ref()))
}
async fn list_raw_index_names(client: &EsClient) -> Result<Vec<String>, String> {
// 主路径 `_cat/indices` 需要集群级 `monitor` 权限。仅有索引级权限的账号
// (例如日志采集用户)会在这里拿到 401/403,此时降级到索引级元数据端点。
let resp = client
.get("/_cat/indices?format=json&h=index")
.send()
.await
.map_err(|e| format!("Elasticsearch request failed: {e}"))?;
let status = client.response_status(&resp);
if status.is_success() {
let indices: Vec<CatIndex> = resp.json().await.map_err(|e| format!("Elasticsearch parse error: {e}"))?;
return Ok(normalize_index_names(indices.into_iter().map(|i| i.index)));
}
if status == StatusCode::FORBIDDEN || status == StatusCode::UNAUTHORIZED {
return list_indices_via_metadata(client).await;
}
let body = resp.text().await.unwrap_or_default();
Err(format!("Elasticsearch error: {body}"))
}
/// 集群 `monitor` 不可用时的降级:`_resolve/index` 与 `_alias` 属于
/// `indices:admin/*` 动作,`view_index_metadata`/`read` 索引权限即可访问,
/// 且 ES 安全层会把结果过滤为当前账号可见的索引。
async fn list_indices_via_metadata(client: &EsClient) -> Result<Vec<String>, String> {
// 优先 `_resolve/index`:同时覆盖普通索引与数据流(data stream)。
if let Some(names) = resolve_index_names(client).await? {
return Ok(names);
}
// 再退回 `_alias`:以对象 key 形式返回具体索引名。
alias_index_names(client).await
}
/// 通过 `GET /_resolve/index/*` 列举索引。该端点缺权限时返回 `Ok(None)`,
/// 以便继续尝试 `_alias`;其它错误如实上抛。
async fn resolve_index_names(client: &EsClient) -> Result<Option<Vec<String>>, String> {
let resp =
client.get("/_resolve/index/*").send().await.map_err(|e| format!("Elasticsearch request failed: {e}"))?;
let status = client.response_status(&resp);
if status == StatusCode::FORBIDDEN || status == StatusCode::UNAUTHORIZED {
return Ok(None);
}
if !status.is_success() {
let body = resp.text().await.unwrap_or_default();
return Err(format!("Elasticsearch error: {body}"));
}
let body: ResolveIndexResponse = resp.json().await.map_err(|e| format!("Elasticsearch parse error: {e}"))?;
let names = body.indices.into_iter().chain(body.data_streams).map(|item| item.name);
Ok(Some(normalize_index_names(names)))
}
/// 通过 `GET /_alias` 列举索引(对象 key 即索引名)。
async fn alias_index_names(client: &EsClient) -> Result<Vec<String>, String> {
let resp = client.get("/_alias").send().await.map_err(|e| format!("Elasticsearch request failed: {e}"))?;
if !client.response_status(&resp).is_success() {
let body = resp.text().await.unwrap_or_default();
return Err(format!("Elasticsearch error: {body}"));
}
let body: serde_json::Map<String, Value> =
resp.json().await.map_err(|e| format!("Elasticsearch parse error: {e}"))?;
Ok(normalize_index_names(body.into_iter().map(|(name, _)| name)))
}
pub async fn get_columns(client: &EsClient, index: &str) -> Result<Vec<crate::db::ColumnInfo>, String> {
let path = elasticsearch_index_path(index, "_mapping");
let resp = client.get(&path).send().await.map_err(|e| format!("Elasticsearch request failed: {e}"))?;
if !client.response_status(&resp).is_success() {
let body = resp.text().await.unwrap_or_default();
return Err(format!("Elasticsearch error: {body}"));
}
let body: Value = resp.json().await.map_err(|e| format!("Elasticsearch parse error: {e}"))?;
let mut seen = HashSet::new();
let mut columns = Vec::new();
if let Some(indices) = body.as_object() {
for index_mapping in indices.values() {
if let Some(properties) = mapping_properties(index_mapping) {
collect_mapping_columns("", properties, &mut seen, &mut columns);
}
}
}
columns.sort_by(|left, right| left.name.cmp(&right.name));
Ok(columns)
}
fn mapping_properties(mapping: &Value) -> Option<&serde_json::Map<String, Value>> {
if let Some(properties) = mapping.pointer("/mappings/properties").and_then(Value::as_object) {
return Some(properties);
}
mapping
.get("mappings")
.and_then(Value::as_object)?
.values()
.find_map(|typed_mapping| typed_mapping.get("properties").and_then(Value::as_object))
}
fn collect_mapping_columns(
prefix: &str,
properties: &serde_json::Map<String, Value>,
seen: &mut HashSet<String>,
columns: &mut Vec<crate::db::ColumnInfo>,
) {
for (name, definition) in properties {
let field_name = if prefix.is_empty() { name.clone() } else { format!("{prefix}.{name}") };
let field_type = definition.get("type").and_then(Value::as_str);
if let Some(data_type) = field_type {
push_mapping_column(&field_name, data_type, seen, columns);
}
if let Some(fields) = definition.get("fields").and_then(Value::as_object) {
collect_mapping_columns(&field_name, fields, seen, columns);
}
if let Some(children) = definition.get("properties").and_then(Value::as_object) {
collect_mapping_columns(&field_name, children, seen, columns);
}
}
}
fn push_mapping_column(
name: &str,
data_type: &str,
seen: &mut HashSet<String>,
columns: &mut Vec<crate::db::ColumnInfo>,
) {
if !seen.insert(name.to_string()) {
return;
}
columns.push(crate::db::ColumnInfo {
name: name.to_string(),
data_type: data_type.to_string(),
is_nullable: true,
column_default: None,
is_primary_key: false,
extra: None,
comment: None,
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
..Default::default()
});
}
#[derive(Deserialize)]
struct SearchResponse {
hits: SearchHits,
#[serde(rename = "_shards")]
shards: Option<ElasticsearchShards>,
#[serde(default)]
timed_out: bool,
#[serde(default)]
terminated_early: bool,
}
#[derive(Deserialize)]
struct SearchHits {
total: HitsTotal,
hits: Vec<SearchHit>,
}
enum HitsTotal {
Count(u64),
Value { value: u64, is_exact: bool },
}
impl HitsTotal {
fn value(&self) -> u64 {
match self {
Self::Count(value) | Self::Value { value, .. } => *value,
}
}
fn is_exact(&self) -> bool {
match self {
Self::Count(_) => true,
Self::Value { is_exact, .. } => *is_exact,
}
}
}
impl<'de> Deserialize<'de> for HitsTotal {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
let value = serde_json::Value::deserialize(deserializer)?;
if let Some(count) = value.as_u64() {
return Ok(Self::Count(count));
}
if let Some(count) = value.get("value").and_then(serde_json::Value::as_u64) {
// Object totals are exact only when Elasticsearch explicitly says
// so. Treat an absent or unknown relation conservatively.
let is_exact = value.get("relation").and_then(serde_json::Value::as_str) == Some("eq");
return Ok(Self::Value { value: count, is_exact });
}
Err(serde::de::Error::custom("expected hits.total as a number or an object with value"))
}
}
#[derive(Deserialize)]
struct SearchHit {
#[serde(rename = "_id")]
id: String,
#[serde(rename = "_type")]
document_type: Option<String>,
#[serde(rename = "_routing")]
routing: Option<String>,
#[serde(rename = "_source")]
source: serde_json::Value,
}
pub async fn find_documents(
client: &EsClient,
index: &str,
skip: u64,
limit: i64,
filter: Option<&str>,
sort: Option<&str>,
) -> Result<DocumentQueryResult, String> {
let body = build_find_documents_body(skip, limit, filter, sort)?;
let path = elasticsearch_index_path(index, "_search");
let resp = client.post(&path).json(&body).send().await.map_err(|e| format!("Elasticsearch request failed: {e}"))?;
if !client.response_status(&resp).is_success() {
let body = resp.text().await.unwrap_or_default();
return Err(format!("Elasticsearch error: {body}"));
}
let result: SearchResponse = resp.json().await.map_err(|e| format!("Elasticsearch parse error: {e}"))?;
search_response_to_document_result(result)
}
#[derive(Deserialize)]
struct CountResponse {
count: u64,
#[serde(rename = "_shards")]
shards: ElasticsearchShards,
}
#[derive(Deserialize)]
struct ElasticsearchShards {
total: u64,
successful: u64,
#[serde(default)]
skipped: u64,
failed: u64,
}
impl ElasticsearchShards {
fn is_complete(&self) -> bool {
// Elasticsearch only skips shards that cannot match, so successful and
// skipped shards together must account for every requested shard.
self.failed == 0 && self.successful.checked_add(self.skipped) == Some(self.total)
}
}
pub async fn count_documents(client: &EsClient, index: &str, filter: Option<&str>) -> Result<u64, String> {
let body = build_count_documents_body(filter)?;
let path = elasticsearch_index_path(index, "_count");
let resp = client.post(&path).json(&body).send().await.map_err(|e| format!("Elasticsearch request failed: {e}"))?;
if !client.response_status(&resp).is_success() {
let body = resp.text().await.unwrap_or_default();
return Err(format!("Elasticsearch error: {body}"));
}
let result: CountResponse = resp.json().await.map_err(|e| format!("Elasticsearch count parse error: {e}"))?;
if !result.shards.is_complete() {
return Err(format!(
"Elasticsearch count returned an incomplete shard response: {} successful, {} skipped, {} failed of {} shards",
result.shards.successful, result.shards.skipped, result.shards.failed, result.shards.total,
));
}
Ok(result.count)
}
fn search_response_to_document_result(result: SearchResponse) -> Result<DocumentQueryResult, String> {
// A 200 search response can still omit failed shards. Only expose an
// exact total when both the total relation and shard metadata agree.
let total_is_exact = !result.timed_out
&& !result.terminated_early
&& result.hits.total.is_exact()
&& result.shards.as_ref().is_some_and(ElasticsearchShards::is_complete);
let total = result.hits.total.value();
let documents: Vec<serde_json::Value> = result
.hits
.hits
.into_iter()
.map(|hit| {
let mut doc = match hit.source {
serde_json::Value::Object(map) => map,
_ => serde_json::Map::new(),
};
doc.insert("_id".to_string(), serde_json::Value::String(hit.id));
if let Some(document_type) = hit.document_type.filter(|value| value != "_doc") {
doc.insert("_type".to_string(), serde_json::Value::String(document_type));
}
if let Some(routing) = hit.routing {
doc.insert("_routing".to_string(), serde_json::Value::String(routing));
}
serde_json::Value::Object(doc)
})
.collect();
// Keep the exact JSON numeric tokens alongside the compatibility value tree.
// Desktop IPC and browser JSON parsing otherwise coerce ES long values through
// JavaScript Number before the document editor can serialize them again.
let raw_documents = documents
.iter()
.map(serde_json::to_string)
.collect::<Result<Vec<_>, _>>()
.map_err(|e| format!("Elasticsearch document serialization failed: {e}"))?;
Ok(DocumentQueryResult {
documents,
raw_documents: Some(raw_documents),
extended_documents: None,
total,
total_is_exact,
})
}
fn build_find_documents_body(
skip: u64,
limit: i64,
filter: Option<&str>,
sort: Option<&str>,
) -> Result<serde_json::Value, String> {
let mut body = serde_json::Map::new();
body.insert("from".to_string(), serde_json::json!(skip));
body.insert("size".to_string(), serde_json::json!(limit));
if let Some(query) = elasticsearch_query_from_document_filter(filter)? {
body.insert("query".to_string(), query);
}
body.insert("sort".to_string(), elasticsearch_sort_from_document_sort(sort)?);
Ok(serde_json::Value::Object(body))
}
fn build_count_documents_body(filter: Option<&str>) -> Result<serde_json::Value, String> {
let mut body = serde_json::Map::new();
if let Some(query) = elasticsearch_query_from_document_filter(filter)? {
body.insert("query".to_string(), query);
}
Ok(serde_json::Value::Object(body))
}
fn elasticsearch_query_from_document_filter(filter: Option<&str>) -> Result<Option<serde_json::Value>, String> {
let Some(filter) = filter.map(str::trim).filter(|value| !value.is_empty()) else {
return Ok(None);
};
let value: serde_json::Value = serde_json::from_str(filter).map_err(|e| format!("Invalid filter JSON: {e}"))?;
let query = translate_document_filter_value(&value)?;
Ok(query)
}
fn translate_document_filter_value(value: &serde_json::Value) -> Result<Option<serde_json::Value>, String> {
let Some(object) = value.as_object() else {
return Err("Elasticsearch filter must be a JSON object".to_string());
};
if object.is_empty() {
return Ok(None);
}
let mut must = Vec::new();
for (key, value) in object {
match key.as_str() {
"$and" => must.extend(translate_logical_filter_array("$and", value)?),
"$or" => {
let should = translate_logical_filter_array("$or", value)?;
if !should.is_empty() {
must.push(serde_json::json!({ "bool": { "should": should, "minimum_should_match": 1 } }));
}
}
"$esQuery" => {
if !value.is_object() {
return Err("$esQuery must be an object".to_string());
}
must.push(value.clone());
}
key if key.starts_with('$') => {
return Err(format!("Unsupported Elasticsearch filter operator: {key}"));
}
field => must.push(translate_field_filter(field, value)?),
}
}
Ok(single_or_bool_filter(must))
}
fn translate_logical_filter_array(operator: &str, value: &serde_json::Value) -> Result<Vec<serde_json::Value>, String> {
let items = value.as_array().ok_or_else(|| format!("{operator} must be an array"))?;
let mut queries = Vec::new();
for item in items {
if let Some(query) = translate_document_filter_value(item)? {
queries.push(query);
}
}
Ok(queries)
}
fn translate_field_filter(field: &str, value: &serde_json::Value) -> Result<serde_json::Value, String> {
let Some(object) = value.as_object() else {
return Ok(term_or_null_query(field, value));
};
if object.keys().any(|key| key.starts_with('$')) {
return translate_field_operator_filter(field, object);
}
Ok(term_or_null_query(field, value))
}
fn translate_field_operator_filter(
field: &str,
object: &serde_json::Map<String, serde_json::Value>,
) -> Result<serde_json::Value, String> {
let mut must = Vec::new();
let mut must_not = Vec::new();
let mut range = serde_json::Map::new();
for (operator, value) in object {
match operator.as_str() {
"$options" => {}
"$ne" => {
if value.is_null() {
must.push(serde_json::json!({ "exists": { "field": field } }));
} else {
must_not.push(serde_json::json!({ "term": { field: value.clone() } }));
}
}
"$gt" => {
range.insert("gt".to_string(), value.clone());
}
"$gte" => {
range.insert("gte".to_string(), value.clone());
}
"$lt" => {
range.insert("lt".to_string(), value.clone());
}
"$lte" => {
range.insert("lte".to_string(), value.clone());
}
"$regex" => {
must.push(regex_like_query(field, value, object.get("$options"))?);
}
"$not" => {
let Some(inner) = value.as_object() else {
return Err("$not must be a JSON object".to_string());
};
if let Some(regex) = inner.get("$regex") {
must_not.push(regex_like_query(
field,
regex,
inner.get("$options").or_else(|| object.get("$options")),
)?);
} else {
return Err("Unsupported Elasticsearch $not filter".to_string());
}
}
other => return Err(format!("Unsupported Elasticsearch field filter operator: {other}")),
}
}
if !range.is_empty() {
must.push(serde_json::json!({ "range": { field: serde_json::Value::Object(range) } }));
}
match (must.len(), must_not.is_empty()) {
(1, true) => Ok(must.remove(0)),
(0, false) => Ok(serde_json::json!({ "bool": { "must_not": must_not } })),
_ => {
let mut bool_query = serde_json::Map::new();
if !must.is_empty() {
bool_query.insert("must".to_string(), serde_json::Value::Array(must));
}
if !must_not.is_empty() {
bool_query.insert("must_not".to_string(), serde_json::Value::Array(must_not));
}
Ok(serde_json::json!({ "bool": bool_query }))
}
}
}
fn term_or_null_query(field: &str, value: &serde_json::Value) -> serde_json::Value {
if value.is_null() {
serde_json::json!({ "bool": { "must_not": [{ "exists": { "field": field } }] } })
} else {
serde_json::json!({ "term": { field: value.clone() } })
}
}
fn regex_like_query(
field: &str,
value: &serde_json::Value,
options: Option<&serde_json::Value>,
) -> Result<serde_json::Value, String> {
let pattern = value.as_str().ok_or_else(|| "$regex must be a string for Elasticsearch filters".to_string())?;
let case_insensitive = options
.and_then(serde_json::Value::as_str)
.is_some_and(|value| value.chars().any(|ch| ch.eq_ignore_ascii_case(&'i')));
Ok(serde_json::json!({
"wildcard": {
field: {
"value": wildcard_contains_pattern(pattern),
"case_insensitive": case_insensitive
}
}
}))
}
fn wildcard_contains_pattern(pattern: &str) -> String {
if pattern.starts_with('*') || pattern.ends_with('*') {
pattern.to_string()
} else {
format!("*{}*", pattern)
}
}
fn single_or_bool_filter(mut queries: Vec<serde_json::Value>) -> Option<serde_json::Value> {
match queries.len() {
0 => None,
1 => queries.pop(),
_ => Some(serde_json::json!({ "bool": { "filter": queries } })),
}
}
fn elasticsearch_sort_from_document_sort(sort: Option<&str>) -> Result<serde_json::Value, String> {
let Some(sort) = sort.map(str::trim).filter(|value| !value.is_empty()) else {
return Ok(serde_json::json!(["_doc"]));
};
let value: serde_json::Value = serde_json::from_str(sort).map_err(|e| format!("Invalid sort JSON: {e}"))?;
let object = value.as_object().ok_or_else(|| "Elasticsearch sort must be a JSON object".to_string())?;
if object.is_empty() {
return Ok(serde_json::json!(["_doc"]));
}
let items = object
.iter()
.map(|(field, direction)| {
let order = match direction {
serde_json::Value::Number(number) if number.as_i64().unwrap_or(1) < 0 => "desc",
serde_json::Value::String(value) if value.eq_ignore_ascii_case("desc") => "desc",
_ => "asc",
};
serde_json::json!({ field: { "order": order } })
})
.collect::<Vec<_>>();
Ok(serde_json::Value::Array(items))
}
pub async fn insert_document(
client: &EsClient,
index: &str,
doc_json: &str,
routing: Option<&str>,
) -> Result<String, String> {
// Prefer explicit routing arg; fall back to body metadata for backward compatibility.
let (doc, routing) = elasticsearch_document_body_and_routing_from_json(doc_json, routing)?;
let path = elasticsearch_auto_id_document_path(index, routing.as_deref());
let resp = client.post(&path).json(&doc).send().await.map_err(|e| format!("Elasticsearch request failed: {e}"))?;
if !client.response_status(&resp).is_success() {
let body = resp.text().await.unwrap_or_default();
return Err(format!("Elasticsearch error: {body}"));
}
let result: serde_json::Value = resp.json().await.map_err(|e| format!("Elasticsearch parse error: {e}"))?;
Ok(result["_id"].as_str().unwrap_or("").to_string())
}
pub async fn update_document(
client: &EsClient,
index: &str,
id: &str,
doc_json: &str,
routing: Option<&str>,
) -> Result<u64, String> {
let (doc, routing, document_type) = elasticsearch_update_document_body_and_metadata(doc_json, routing)?;
let path = elasticsearch_document_path(index, id, document_type.as_deref(), routing.as_deref());
let resp = client.put(&path).json(&doc).send().await.map_err(|e| format!("Elasticsearch request failed: {e}"))?;
if !client.response_status(&resp).is_success() {
let body = resp.text().await.unwrap_or_default();
return Err(format!("Elasticsearch error: {body}"));
}
Ok(1)
}
fn elasticsearch_update_document_body_and_metadata(
doc_json: &str,
routing: Option<&str>,
) -> Result<(serde_json::Value, Option<String>, Option<String>), String> {
let (mut doc, routing) = elasticsearch_document_body_and_routing_from_json(doc_json, routing)?;
let document_type = match &mut doc {
serde_json::Value::Object(map) => map.remove("_type").and_then(|value| match value {
serde_json::Value::String(value) => {
let trimmed = value.trim();
(!trimmed.is_empty()).then(|| trimmed.to_string())
}
_ => None,
}),
_ => None,
};
Ok((doc, routing, document_type))
}
fn elasticsearch_document_body_and_routing_from_json(
doc_json: &str,
routing: Option<&str>,
) -> Result<(serde_json::Value, Option<String>), String> {
let mut doc: serde_json::Value = serde_json::from_str(doc_json).map_err(|e| format!("Invalid JSON: {e}"))?;
let mut routing = routing.map(str::trim).filter(|value| !value.is_empty()).map(str::to_string);
if let serde_json::Value::Object(map) = &mut doc {
map.remove("_id");
if routing.is_none() {
routing = map.get("_routing").and_then(elasticsearch_routing_from_value);
}
map.remove("_routing");
}
Ok((doc, routing))
}
fn elasticsearch_routing_from_value(value: &serde_json::Value) -> Option<String> {
match value {
serde_json::Value::String(value) => {
let trimmed = value.trim();
(!trimmed.is_empty()).then(|| trimmed.to_string())
}
_ => None,
}
}
pub async fn delete_document(
client: &EsClient,
index: &str,
id: &str,
document_type: Option<&str>,
routing: Option<&str>,
) -> Result<u64, String> {
let path = elasticsearch_document_path(index, id, document_type, routing);
let resp = client.delete(&path).send().await.map_err(|e| format!("Elasticsearch request failed: {e}"))?;
if !client.response_status(&resp).is_success() {
let body = resp.text().await.unwrap_or_default();
return Err(format!("Elasticsearch error: {body}"));
}
Ok(1)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ElasticsearchRestBodyKind {
Json,
Ndjson,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct ElasticsearchRestRequest {
method: Method,
path: String,
body: Option<String>,
body_kind: ElasticsearchRestBodyKind,
}
fn strip_leading_elasticsearch_comments(input: &str) -> &str {
let mut rest = input;
loop {
rest = rest.trim_start();
if let Some(comment) = rest.strip_prefix('#').or_else(|| rest.strip_prefix("//")) {
rest = comment.split_once('\n').map_or("", |(_, remaining)| remaining);
continue;
}
if let Some(comment) = rest.strip_prefix("/*") {
rest = comment.split_once("*/").map_or("", |(_, remaining)| remaining);
continue;
}
return rest.trim();
}
}
fn is_elasticsearch_ndjson_path(path: &str) -> bool {
let path = path.split('?').next().unwrap_or(path).trim_end_matches('/');
path == "/_bulk"
|| path.ends_with("/_bulk")
|| path == "/_msearch"
|| path.ends_with("/_msearch")
|| path == "/_msearch/template"
|| path.ends_with("/_msearch/template")
}
fn is_elasticsearch_cat_path(path: &str) -> bool {
let path = path.split('?').next().unwrap_or(path).trim_end_matches('/');
path == "/_cat" || path.starts_with("/_cat/")
}
fn elasticsearch_query_parameter(path: &str, name: &str) -> Option<String> {
let query = path.split_once('?')?.1;
query.split('&').find_map(|part| {
let (key, value) = part.split_once('=').unwrap_or((part, ""));
let key = percent_decode_str(key).decode_utf8_lossy();
if key.eq_ignore_ascii_case(name) {
Some(percent_decode_str(value).decode_utf8_lossy().into_owned())
} else {
None
}
})
}
fn add_default_cat_json_format(mut request: ElasticsearchRestRequest) -> ElasticsearchRestRequest {
if is_elasticsearch_cat_path(&request.path) && elasticsearch_query_parameter(&request.path, "format").is_none() {
request.path.push(if request.path.contains('?') { '&' } else { '?' });
request.path.push_str("format=json");
}
request
}
fn normalize_elasticsearch_rest_path(path: &str) -> String {
let (path_part, query) = path.split_once('?').map_or((path, None), |(path, query)| (path, Some(query)));
let mut normalized = String::with_capacity(path.len());
let mut chars = path_part.chars().peekable();
let mut in_date_math = false;
while let Some(ch) = chars.next() {
if ch == '%' {
let mut lookahead = chars.clone();
if let (Some(first), Some(second)) = (lookahead.next(), lookahead.next()) {
if first.is_ascii_hexdigit() && second.is_ascii_hexdigit() {
normalized.push(ch);
normalized.push(chars.next().unwrap());
normalized.push(chars.next().unwrap());
continue;
}
}
}
if ch == '<' {
in_date_math = true;
}
let encoded = if in_date_math {
match ch {
'<' => Some("%3C"),
'>' => Some("%3E"),
'/' => Some("%2F"),
'{' => Some("%7B"),
'}' => Some("%7D"),
'|' => Some("%7C"),
'+' => Some("%2B"),
':' => Some("%3A"),
',' => Some("%2C"),
_ => None,
}
} else {
None
};
if let Some(encoded) = encoded {
normalized.push_str(encoded);
} else {
normalized.push(ch);
}
if ch == '>' {
in_date_math = false;
}
}
if let Some(query) = query {
normalized.push('?');
normalized.push_str(query);
}
normalized
}
fn parse_elasticsearch_rest_request(input: &str) -> Result<ElasticsearchRestRequest, String> {
let input = strip_leading_elasticsearch_comments(input);
if input.is_empty() {
return Err("Invalid query: expected METHOD /path".to_string());
}
let (request_line, body) = input.split_once('\n').map_or((input, None), |(line, body)| {
let body = body.trim();
(line, (!body.is_empty()).then(|| body.to_string()))
});
let (method, path) =
request_line.trim().split_once(char::is_whitespace).ok_or("Invalid query: expected METHOD /path")?;
let method = method.to_ascii_uppercase();
let method = Method::from_bytes(method.as_bytes())
.map_err(|_| format!("Unsupported HTTP method: {method}. Use GET, POST, PUT, DELETE, or HEAD."))?;
if !matches!(method, Method::GET | Method::POST | Method::PUT | Method::DELETE | Method::HEAD) {
return Err(format!("Unsupported HTTP method: {}. Use GET, POST, PUT, DELETE, or HEAD.", method.as_str()));
}
let path = path.trim();
if path.is_empty() {
return Err("Invalid query: expected METHOD /path".to_string());
}
let path = if path.starts_with('/') { path.to_string() } else { format!("/{path}") };
let path = normalize_elasticsearch_rest_path(&path);
let body_kind = if is_elasticsearch_ndjson_path(&path) {
ElasticsearchRestBodyKind::Ndjson
} else {
ElasticsearchRestBodyKind::Json
};
Ok(ElasticsearchRestRequest { method, path, body, body_kind })
}
fn validate_elasticsearch_ndjson(body: &str) -> Result<String, String> {
for (index, line) in body.lines().enumerate() {
let line = line.trim();
if line.is_empty() {
continue;
}
serde_json::from_str::<serde_json::Value>(line)
.map_err(|error| format!("Invalid NDJSON body at line {}: {error}", index + 1))?;
}
let mut normalized = body.trim_end().to_string();
normalized.push('\n');
Ok(normalized)
}
pub(crate) type SqlResponseParser = fn(&serde_json::Value, std::time::Instant) -> Option<QueryResult>;
pub async fn execute_rest_query(client: &EsClient, input: &str) -> Result<QueryResult, String> {
execute_rest_query_with_sql_parser(client, input, parse_sql_response).await
}
pub(crate) async fn execute_rest_query_with_sql_parser(
client: &EsClient,
input: &str,
sql_response_parser: SqlResponseParser,
) -> Result<QueryResult, String> {
let start = std::time::Instant::now();
let input = strip_leading_elasticsearch_comments(input);
if let Some(search_query) = parse_select_star_search_query(input) {
return execute_search_query(client, search_query, start, sql_response_parser).await;
}
if is_elasticsearch_sql_query(input) {
// `SELECT *` with clauses our simple parser doesn't cover (WHERE, IN,
// BETWEEN, LIKE, ...). Translate the SQL to an ES `_search` body
// ourselves rather than going through ES's `_sql` endpoint — `_sql`
// refuses several common shapes (LIKE on a `text` field with no
// `.keyword`, `SELECT *` returning an array field like `host.ip`, ...)
// that translate cleanly to raw DSL. Adapt first so that hyphenated
// index names (`aifanfan-python-bot-logs-*`) and `@timestamp`-style
// identifiers come out as double-quoted identifiers sqlparser will
// accept.
let adapted_for_translator = adapt_elasticsearch_sql_query(input);
match crate::db::elasticsearch_sql::translate_select_star(&adapted_for_translator) {
Ok(Some(translated)) => {
return execute_translated_select_star(client, translated, start, sql_response_parser).await;
}
Ok(None) => {}
Err(message) => return Err(format!("Elasticsearch SQL error: {message}")),
}
return execute_sql_query(client, input, start, sql_response_parser).await;
}
// CAT APIs default to text, so request JSON for an unformatted CAT call.
// The frontend renders the returned HTTP body in its JSON response panel.
let request = add_default_cat_json_format(parse_elasticsearch_rest_request(input)?);
let mut builder = client.request(request.method, &request.path);
if let Some(body) = request.body {
builder = match request.body_kind {
ElasticsearchRestBodyKind::Json => {
let json: serde_json::Value =
serde_json::from_str(&body).map_err(|e| format!("Invalid JSON body: {e}"))?;
builder.json(&json)
}
ElasticsearchRestBodyKind::Ndjson => builder
.header(reqwest::header::CONTENT_TYPE, "application/x-ndjson")
.body(validate_elasticsearch_ndjson(&body)?),
};
}
let resp = builder.send().await.map_err(|e| format!("Elasticsearch request failed: {e}"))?;
let status = client.response_status(&resp).as_u16();
let body = resp.text().await.map_err(|e| format!("Elasticsearch response read failed: {e}"))?;
parse_elasticsearch_rest_response_with_sql_parser(status, &body, start, sql_response_parser)
}
// Size to use when `SELECT *` is run without an explicit LIMIT — large enough
// to be useful, small enough that the user doesn't accidentally pull millions
// of documents. The result-grid surfaces the index's true total separately so
// the user can see how much was actually held back.
const AUTO_PAGED_SELECT_STAR_SIZE: usize = 100;
struct ElasticsearchSearchQuery {
index: String,
body: serde_json::Value,
// True when the SQL came through the pagination plan (it carries OFFSET).
// In that case the result-grid total must reflect the index's true match
// count so the front-end can compute the total page count. A bare
// user-written `LIMIT N` (no OFFSET) is the "give me exactly N rows" case
// and reports affected_rows = N for client-side paging.
from_plan_pagination: bool,
}
async fn execute_search_query(
client: &EsClient,
query: ElasticsearchSearchQuery,
start: std::time::Instant,
sql_response_parser: SqlResponseParser,
) -> Result<QueryResult, String> {
let report_index_total = query.from_plan_pagination;
let path = elasticsearch_index_path(&query.index, "_search");
let resp =
client.post(&path).json(&query.body).send().await.map_err(|e| format!("Elasticsearch request failed: {e}"))?;
let status = client.response_status(&resp).as_u16();
let body: serde_json::Value = resp.json().await.unwrap_or_else(|_| serde_json::Value::Null);
// Capture the index's true match total before the body is consumed by the
// parser — needed below when we report total instead of rows.len().
let index_total = body.pointer("/hits/total/value").and_then(|v| v.as_u64());
let mut result = parse_elasticsearch_response_with_sql_parser(status, body, start, sql_response_parser)?;
if report_index_total {
if let Some(total) = index_total {
result.affected_rows = total;
}
}
Ok(result)
}
#[cfg(test)]
fn parse_elasticsearch_response(
status: u16,
body: serde_json::Value,
start: std::time::Instant,
) -> Result<QueryResult, String> {
parse_elasticsearch_response_with_sql_parser(status, body, start, parse_sql_response)
}
fn parse_elasticsearch_response_with_sql_parser(
status: u16,
mut body: serde_json::Value,
start: std::time::Instant,
sql_response_parser: SqlResponseParser,
) -> Result<QueryResult, String> {
if let Some(result) = sql_response_parser(&body, start) {
Ok(result)
} else if let Some(aggs) = body.get("aggregations").or_else(|| body.get("aggs")).and_then(|v| v.as_object()) {
let (columns, rows) = parse_aggregations(aggs);
if !columns.is_empty() {
let row_count = rows.len() as u64;
Ok(crate::types::QueryResult {
columns,
column_types: Vec::new(),
column_sortables: vec![],
spatial_columns: vec![],
spatial_values: vec![],
rows,
affected_rows: row_count,
execution_time_ms: start.elapsed().as_millis(),
truncated: false,
session_id: None,
has_more: false,
elasticsearch_raw_body: None,
messages: Vec::new(),
})
} else {
Ok(json_response_result(status, &body, start))
}
} else if let Some(hits) = body.pointer_mut("/hits/hits").and_then(serde_json::Value::as_array_mut) {
// Treat any `_search`-shaped body as the hits result, even when empty —
// a 0-row match is a valid empty result, not a reason to fall back to
// the raw-JSON status/response view.
let hits = std::mem::take(hits);
let (columns, column_types, rows) = parse_elasticsearch_search_hits(hits);
let row_count = rows.len() as u64;
Ok(crate::types::QueryResult {
columns,
column_types,
column_sortables: vec![],
spatial_columns: vec![],
spatial_values: vec![],
rows,
affected_rows: row_count,
execution_time_ms: start.elapsed().as_millis(),
truncated: false,
session_id: None,
has_more: false,
elasticsearch_raw_body: None,
messages: Vec::new(),
})
} else {
Ok(json_response_result(status, &body, start))
}
}
fn parse_elasticsearch_search_hits(
hits: Vec<serde_json::Value>,
) -> (Vec<String>, Vec<String>, Vec<Vec<serde_json::Value>>) {
let mut columns = Vec::<String>::new();
let mut column_indexes = HashMap::<String, usize>::new();
let mut json_column_indexes = HashSet::<usize>::new();
let mut rows = Vec::<Vec<serde_json::Value>>::with_capacity(hits.len());
for mut hit in hits {
let mut row = vec![serde_json::Value::Null; columns.len()];
if let Some(source) = hit.get_mut("_source").and_then(serde_json::Value::as_object_mut) {
for (key, value) in std::mem::take(source) {
append_elasticsearch_json_cell(
&mut columns,
&mut column_indexes,
&mut json_column_indexes,
&mut rows,
&mut row,
key,
value,
);
}
}
let id = hit.get_mut("_id").map(serde_json::Value::take).unwrap_or(serde_json::Value::Null);
append_elasticsearch_json_cell(
&mut columns,
&mut column_indexes,
&mut json_column_indexes,
&mut rows,
&mut row,
"_id".to_string(),
id,
);
if let Some(routing) = hit.get_mut("_routing") {
append_elasticsearch_json_cell(
&mut columns,
&mut column_indexes,
&mut json_column_indexes,
&mut rows,
&mut row,
"_routing".to_string(),
routing.take(),
);
}
rows.push(row);
}
if columns.is_empty() {
columns.push("_id".to_string());
}
let column_types = infer_elasticsearch_json_column_types(&rows, columns.len(), &json_column_indexes);
(columns, column_types, rows)
}
fn append_elasticsearch_json_cell(
columns: &mut Vec<String>,
column_indexes: &mut HashMap<String, usize>,
json_column_indexes: &mut HashSet<usize>,
previous_rows: &mut [Vec<serde_json::Value>],
row: &mut Vec<serde_json::Value>,
column: String,
value: serde_json::Value,
) {
let is_json_cell = matches!(value, serde_json::Value::Array(_) | serde_json::Value::Object(_));
let value = if is_json_cell { serde_json::Value::String(value.to_string()) } else { value };
if let Some(index) = column_indexes.get(&column).copied() {
if is_json_cell {
json_column_indexes.insert(index);
}
row[index] = value;
return;
}
let index = columns.len();
if is_json_cell {
json_column_indexes.insert(index);
}
column_indexes.insert(column.clone(), index);
columns.push(column);
for previous_row in previous_rows {
previous_row.push(serde_json::Value::Null);
}
row.push(value);
}
fn infer_elasticsearch_json_column_types(
rows: &[Vec<serde_json::Value>],
column_count: usize,
json_column_indexes: &HashSet<usize>,
) -> Vec<String> {
(0..column_count)
.map(|column_index| {
if json_column_indexes.contains(&column_index) {
return "json".to_string();
}
let mut inferred = None;
for value in rows.iter().filter_map(|row| row.get(column_index)) {
let value_type = match value {
serde_json::Value::Null => continue,
serde_json::Value::Bool(_) => "boolean",
serde_json::Value::Number(_) => "number",
serde_json::Value::String(_) => "text",
serde_json::Value::Array(_) | serde_json::Value::Object(_) => "json",
};
inferred = match inferred {
None => Some(value_type),
Some(existing) if existing == value_type => Some(existing),
Some(_) => Some("json"),
};
if inferred == Some("json") {
break;
}
}
inferred.unwrap_or("unknown").to_string()
})
.collect()
}
fn elasticsearch_rest_search_exceeds_table_limits(body: &serde_json::Value) -> bool {
let Some(hits) = body.pointer("/hits/hits").and_then(serde_json::Value::as_array) else {
return false;
};
if hits.len() > ELASTICSEARCH_REST_TABLE_MAX_ROWS {
return true;
}
let mut columns = HashSet::<&str>::new();
columns.insert("_id");
for hit in hits {
if hit.get("_routing").is_some() {
columns.insert("_routing");
}
if let Some(source) = hit.get("_source").and_then(serde_json::Value::as_object) {
columns.extend(source.keys().map(String::as_str));
}
if hits.len().saturating_mul(columns.len()) > ELASTICSEARCH_REST_TABLE_MAX_CELLS {
return true;
}
}
false
}
fn json_response_result(status: u16, body: &serde_json::Value, start: std::time::Instant) -> crate::types::QueryResult {
let body_text = serde_json::to_string_pretty(body).unwrap_or_else(|_| body.to_string());
raw_json_response_result(status, body_text, start)
}
fn raw_json_response_result(
status: u16,
body_text: impl Into<String>,
start: std::time::Instant,
) -> crate::types::QueryResult {
crate::types::QueryResult {
columns: vec!["status".to_string(), "response".to_string()],
column_types: Vec::new(),
column_sortables: vec![],
spatial_columns: vec![],
spatial_values: vec![],
rows: vec![vec![serde_json::Value::Number(status.into()), serde_json::Value::String(body_text.into())]],
affected_rows: 0,
execution_time_ms: start.elapsed().as_millis(),
truncated: false,
session_id: None,
has_more: false,
elasticsearch_raw_body: None,
messages: Vec::new(),
}
}
#[cfg(test)]
fn parse_elasticsearch_rest_response(
status: u16,
body_text: &str,
start: std::time::Instant,
) -> Result<QueryResult, String> {
parse_elasticsearch_rest_response_with_sql_parser(status, body_text, start, parse_sql_response)
}
fn parse_elasticsearch_rest_response_with_sql_parser(
status: u16,
body_text: &str,
start: std::time::Instant,
sql_response_parser: SqlResponseParser,
) -> Result<QueryResult, String> {
if body_text.trim().is_empty() {
return Ok(json_response_result(status, &serde_json::Value::Null, start));
}
if status >= 400 {
return Ok(raw_json_response_result(status, body_text, start));
}
if body_text.len() > ELASTICSEARCH_REST_TABLE_MAX_BODY_BYTES {
return Ok(raw_json_response_result(status, body_text, start));
}
if let Ok(body) = serde_json::from_str::<serde_json::Value>(body_text) {
if elasticsearch_rest_search_exceeds_table_limits(&body) {
return Ok(raw_json_response_result(status, body_text, start));
}
// Prefer a tabular view for search hits (_source columns), SQL API
// responses, and aggregations so the desktop data grid can display and
// copy rows like relational results. Other JSON (mapping, cluster
// info, …) stays as a lossless status/response panel with the raw body.
let mut result = parse_elasticsearch_response_with_sql_parser(status, body, start, sql_response_parser)?;
if result.columns == ["status".to_string(), "response".to_string()] {
return Ok(raw_json_response_result(status, body_text, start));
}
// Attach the raw response body so the UI can toggle between the
// table and the original JSON for Elasticsearch REST results.
result.elasticsearch_raw_body = Some(body_text.to_string());
return Ok(result);
}
// CAT APIs default to text/plain for human-readable output. Keep those
// responses visible instead of dropping them when JSON parsing is not valid.
let rows: Vec<Vec<serde_json::Value>> =
body_text.lines().map(|line| vec![serde_json::Value::String(line.to_string())]).collect();
let affected_rows = rows.len() as u64;
Ok(crate::types::QueryResult {
columns: vec!["response".to_string()],
column_types: Vec::new(),
column_sortables: vec![],
spatial_columns: vec![],
spatial_values: vec![],
rows,
affected_rows,
execution_time_ms: start.elapsed().as_millis(),
truncated: false,
session_id: None,
has_more: false,
elasticsearch_raw_body: None,
messages: Vec::new(),
})
}
fn parse_select_star_search_query(input: &str) -> Option<ElasticsearchSearchQuery> {
let mut cursor = skip_sql_whitespace(input, 0);
cursor = consume_sql_keyword(input, cursor, "select")?;
cursor = skip_sql_whitespace(input, cursor);
if next_char_at(input, cursor)? != '*' {
return None;
}
cursor += '*'.len_utf8();
cursor = skip_sql_whitespace(input, cursor);
cursor = consume_sql_keyword(input, cursor, "from")?;
cursor = skip_sql_whitespace(input, cursor);
let (index, next_cursor) = read_sql_token(input, cursor)?;
cursor = next_cursor;
let mut sort_field = None;
let mut sort_order = "asc";
let mut limit = None;
let mut offset: Option<usize> = None;
loop {
cursor = skip_sql_whitespace(input, cursor);
if cursor >= input.len() {
break;
}
if next_char_at(input, cursor) == Some(';') {
cursor += ';'.len_utf8();
cursor = skip_sql_whitespace(input, cursor);
if cursor == input.len() {
break;
}
return None;
}
if is_keyword_at(input, cursor, "order") {
cursor = consume_sql_keyword(input, cursor, "order")?;
cursor = skip_sql_whitespace(input, cursor);
cursor = consume_sql_keyword(input, cursor, "by")?;
cursor = skip_sql_whitespace(input, cursor);
let (field, next_cursor) = read_sql_token(input, cursor)?;
sort_field = Some(field);
cursor = skip_sql_whitespace(input, next_cursor);
if is_keyword_at(input, cursor, "asc") {
sort_order = "asc";
cursor = consume_sql_keyword(input, cursor, "asc")?;
} else if is_keyword_at(input, cursor, "desc") {
sort_order = "desc";
cursor = consume_sql_keyword(input, cursor, "desc")?;
}
} else if is_keyword_at(input, cursor, "limit") {
cursor = consume_sql_keyword(input, cursor, "limit")?;
cursor = skip_sql_whitespace(input, cursor);
let (value, next_cursor) = read_while(input, cursor, |ch| ch.is_ascii_digit());
limit = value.parse::<usize>().ok();
cursor = next_cursor;
} else if is_keyword_at(input, cursor, "offset") {
cursor = consume_sql_keyword(input, cursor, "offset")?;
cursor = skip_sql_whitespace(input, cursor);
let (value, next_cursor) = read_while(input, cursor, |ch| ch.is_ascii_digit());
offset = value.parse::<usize>().ok();
cursor = next_cursor;
} else {
return None;
}
}
// The pagination plan emits `LIMIT N OFFSET M` (always with OFFSET, even
// when 0) for ES; a user-written SQL that only has `LIMIT N` leaves
// OFFSET absent. We use that as the signal for whether the front-end is
// driving server-side pagination — in that case affected_rows must reflect
// the index's true total so the grid can compute the total page count.
let from_plan_pagination = offset.is_some();
let effective_size = limit.unwrap_or(AUTO_PAGED_SELECT_STAR_SIZE);
let effective_from = offset.unwrap_or(0);
let mut body = serde_json::Map::new();
body.insert("size".to_string(), serde_json::json!(effective_size));
if effective_from > 0 {
body.insert("from".to_string(), serde_json::json!(effective_from));
}
if let Some(field) = sort_field {
let mut sort_item = serde_json::Map::new();
sort_item.insert(field, serde_json::json!({ "order": sort_order }));
body.insert("sort".to_string(), serde_json::Value::Array(vec![serde_json::Value::Object(sort_item)]));
}
Some(ElasticsearchSearchQuery { index, body: serde_json::Value::Object(body), from_plan_pagination })
}
fn is_elasticsearch_sql_query(input: &str) -> bool {
input
.trim_start()
.split_once(char::is_whitespace)
.map(|(keyword, _)| keyword.eq_ignore_ascii_case("select"))
.unwrap_or_else(|| input.trim_start().eq_ignore_ascii_case("select"))
}
async fn execute_translated_select_star(
client: &EsClient,
translated: crate::db::elasticsearch_sql::TranslatedSelectStar,
start: std::time::Instant,
sql_response_parser: SqlResponseParser,
) -> Result<QueryResult, String> {
let report_index_total = !translated.user_limited;
let path = elasticsearch_index_path(&translated.index, "_search");
let resp = client
.post(&path)
.json(&translated.body)
.send()
.await
.map_err(|e| format!("Elasticsearch request failed: {e}"))?;
let status = client.response_status(&resp).as_u16();
let body: serde_json::Value = resp.json().await.unwrap_or_else(|_| serde_json::Value::Null);
let index_total = body.pointer("/hits/total/value").and_then(|v| v.as_u64());
let mut result = parse_elasticsearch_response_with_sql_parser(status, body, start, sql_response_parser)?;
if report_index_total {
if let Some(total) = index_total {
result.affected_rows = total;
}
}
Ok(result)
}
async fn execute_sql_query(
client: &EsClient,
query: &str,
start: std::time::Instant,
sql_response_parser: SqlResponseParser,
) -> Result<QueryResult, String> {
let query = adapt_elasticsearch_sql_query(query);
let body = serde_json::json!({ "query": query });
let resp =
client.post("/_sql").json(&body).send().await.map_err(|e| format!("Elasticsearch request failed: {e}"))?;
let status = client.response_status(&resp);
let response_body: serde_json::Value = resp.json().await.unwrap_or_else(|_| serde_json::Value::Null);
if !status.is_success() {
return Err(format_sql_error(status, &response_body));
}
sql_response_parser(&response_body, start).ok_or_else(|| {
let pretty = serde_json::to_string_pretty(&response_body).unwrap_or_else(|_| response_body.to_string());
format!("Unexpected Elasticsearch SQL response: {pretty}")
})
}
fn adapt_elasticsearch_sql_query(query: &str) -> String {
let mut output = String::with_capacity(query.len());
let mut index = 0;
let mut state = SqlScanState::Normal;
while let Some(ch) = next_char_at(query, index) {
match state {
SqlScanState::Normal => match ch {
'\'' => {
output.push(ch);
index += ch.len_utf8();
state = SqlScanState::SingleQuoted;
}
'"' => {
output.push(ch);
index += ch.len_utf8();
state = SqlScanState::DoubleQuoted;
}
'`' => {
output.push(ch);
index += ch.len_utf8();
state = SqlScanState::BacktickQuoted;
}
'-' if query[index..].starts_with("--") => {
output.push_str("--");
index += 2;
state = SqlScanState::LineComment;
}
'/' if query[index..].starts_with("/*") => {
output.push_str("/*");
index += 2;
state = SqlScanState::BlockComment;
}
'@' if is_at_identifier_boundary(&output) => {
let (identifier, next_index) = read_while(query, index, is_elasticsearch_identifier_part);
output.push('"');
output.push_str(identifier);
output.push('"');
index = next_index;
}
_ => {
if let Some(keyword) = relation_keyword_at(query, index) {
index = quote_relation_after_keyword(query, index, keyword, &mut output);
} else {
output.push(ch);
index += ch.len_utf8();
}
}
},
SqlScanState::SingleQuoted => {
if copy_quoted_char(query, &mut index, ch, '\'', &mut output) {
state = SqlScanState::Normal;
}
}
SqlScanState::DoubleQuoted => {
if copy_quoted_char(query, &mut index, ch, '"', &mut output) {
state = SqlScanState::Normal;
}
}
SqlScanState::BacktickQuoted => {
if copy_quoted_char(query, &mut index, ch, '`', &mut output) {
state = SqlScanState::Normal;
}
}
SqlScanState::LineComment => {
output.push(ch);
index += ch.len_utf8();
if ch == '\n' {
state = SqlScanState::Normal;
}
}
SqlScanState::BlockComment => {
if query[index..].starts_with("*/") {
output.push_str("*/");
index += 2;
state = SqlScanState::Normal;
} else {
output.push(ch);
index += ch.len_utf8();
}
}
}
}
output
}
fn quote_relation_after_keyword(query: &str, index: usize, keyword: &str, output: &mut String) -> usize {
let mut cursor = index + keyword.len();
output.push_str(&query[index..cursor]);
while let Some(ch) = next_char_at(query, cursor) {
if !ch.is_whitespace() {
break;
}
output.push(ch);
cursor += ch.len_utf8();
}
if matches!(next_char_at(query, cursor), Some('"' | '`' | '\'' | '(')) {
return cursor;
}
let relation_start = cursor;
while let Some(ch) = next_char_at(query, cursor) {
if !is_relation_name_char(ch) {
break;
}
cursor += ch.len_utf8();
}
let relation = &query[relation_start..cursor];
if relation_name_needs_quotes(relation) {
output.push('"');
output.push_str(relation);
output.push('"');
} else {
output.push_str(relation);
}
cursor
}
fn copy_quoted_char(query: &str, index: &mut usize, ch: char, quote: char, output: &mut String) -> bool {
output.push(ch);
*index += ch.len_utf8();
if ch != quote {
return false;
}
if next_char_at(query, *index).is_some_and(|next| next == quote) {
output.push(quote);
*index += quote.len_utf8();
false
} else {
true
}
}
fn read_while(query: &str, start: usize, predicate: fn(char) -> bool) -> (&str, usize) {
let mut cursor = start;
while let Some(ch) = next_char_at(query, cursor) {
if !predicate(ch) {
break;
}
cursor += ch.len_utf8();
}
(&query[start..cursor], cursor)
}
fn skip_sql_whitespace(query: &str, mut cursor: usize) -> usize {
while let Some(ch) = next_char_at(query, cursor) {
if !ch.is_whitespace() {
break;
}
cursor += ch.len_utf8();
}
cursor
}
fn consume_sql_keyword(query: &str, cursor: usize, keyword: &str) -> Option<usize> {
is_keyword_at(query, cursor, keyword).then_some(cursor + keyword.len())
}
fn read_sql_token(query: &str, cursor: usize) -> Option<(String, usize)> {
let quote = match next_char_at(query, cursor)? {
'"' => Some('"'),
'`' => Some('`'),
_ => None,
};
if let Some(quote) = quote {
let mut output = String::new();
let mut next_cursor = cursor + quote.len_utf8();
while let Some(ch) = next_char_at(query, next_cursor) {
next_cursor += ch.len_utf8();
if ch == quote {
if next_char_at(query, next_cursor).is_some_and(|next| next == quote) {
output.push(quote);
next_cursor += quote.len_utf8();
} else {
return Some((output, next_cursor));
}
} else {
output.push(ch);
}
}
return None;
}
let (token, next_cursor) = read_while(query, cursor, is_relation_name_char);
(!token.is_empty()).then(|| (token.to_string(), next_cursor))
}
fn relation_keyword_at(query: &str, index: usize) -> Option<&'static str> {
["from", "join"].into_iter().find(|keyword| is_keyword_at(query, index, keyword))
}
#[derive(Clone, Copy)]
enum SqlScanState {
Normal,
SingleQuoted,
DoubleQuoted,
BacktickQuoted,
LineComment,
BlockComment,
}
fn is_at_identifier_boundary(output: &str) -> bool {
output.chars().next_back().is_none_or(|ch| !is_sql_identifier_part(ch))
}
fn is_sql_identifier_part(ch: char) -> bool {
ch.is_ascii_alphanumeric() || matches!(ch, '_' | '.')
}
fn is_elasticsearch_identifier_part(ch: char) -> bool {
ch.is_ascii_alphanumeric() || matches!(ch, '_' | '.' | '-' | '@')
}
fn is_relation_name_char(ch: char) -> bool {
!ch.is_whitespace() && !matches!(ch, ',' | ';' | '(' | ')')
}
fn relation_name_needs_quotes(relation: &str) -> bool {
relation.chars().any(|ch| matches!(ch, '-' | '*' | '@'))
}
fn is_keyword_at(query: &str, index: usize, keyword: &str) -> bool {
query.get(index..index + keyword.len()).is_some_and(|candidate| candidate.eq_ignore_ascii_case(keyword))
&& query[..index].chars().next_back().is_none_or(|ch| !is_keyword_boundary_char(ch))
&& query[index + keyword.len()..].chars().next().is_none_or(|ch| !is_keyword_boundary_char(ch))
}
fn is_keyword_boundary_char(ch: char) -> bool {
ch.is_ascii_alphanumeric() || ch == '_'
}
fn next_char_at(query: &str, index: usize) -> Option<char> {
query.get(index..)?.chars().next()
}
fn parse_sql_response(body: &serde_json::Value, start: std::time::Instant) -> Option<QueryResult> {
parse_tabular_sql_response(body, start, "columns", "rows", None)
}
pub(crate) fn parse_tabular_sql_response(
body: &serde_json::Value,
start: std::time::Instant,
columns_key: &str,
rows_key: &str,
total_key: Option<&str>,
) -> Option<QueryResult> {
let columns = body.get(columns_key)?.as_array()?;
let rows = body.get(rows_key)?.as_array()?;
let column_names: Vec<String> = columns
.iter()
.filter_map(|column| column.get("name").and_then(|name| name.as_str()).map(str::to_string))
.collect();
if column_names.is_empty() && !columns.is_empty() {
return None;
}
let result_rows: Vec<Vec<serde_json::Value>> =
rows.iter().filter_map(|row| row.as_array().map(|values| values.to_vec())).collect();
Some(QueryResult {
columns: column_names,
column_types: Vec::new(),
column_sortables: vec![],
spatial_columns: vec![],
spatial_values: vec![],
rows: result_rows,
affected_rows: total_key
.and_then(|key| body.get(key))
.and_then(serde_json::Value::as_u64)
.unwrap_or(rows.len() as u64),
execution_time_ms: start.elapsed().as_millis(),
truncated: false,
session_id: body.get("cursor").and_then(|cursor| cursor.as_str()).map(str::to_string),
has_more: body.get("cursor").and_then(|cursor| cursor.as_str()).is_some(),
elasticsearch_raw_body: None,
messages: Vec::new(),
})
}
fn format_sql_error(status: reqwest::StatusCode, body: &serde_json::Value) -> String {
let detail = body
.pointer("/error/reason")
.and_then(|reason| reason.as_str())
.map(str::to_string)
.unwrap_or_else(|| serde_json::to_string_pretty(body).unwrap_or_else(|_| body.to_string()));
if status == reqwest::StatusCode::NOT_FOUND {
format!("Elasticsearch SQL API is not available ({status}): {detail}")
} else {
format!("Elasticsearch SQL error ({status}): {detail}")
}
}
fn parse_aggregations(aggs: &serde_json::Map<String, serde_json::Value>) -> (Vec<String>, Vec<Vec<serde_json::Value>>) {
for (_name, agg_value) in aggs {
if let Some(buckets) = agg_value.get("buckets").and_then(|b| b.as_array()) {
if buckets.is_empty() {
continue;
}
let mut all_keys = Vec::<String>::new();
let mut bucket_rows = Vec::new();
for bucket in buckets {
if let Some(obj) = bucket.as_object() {
let mut row = serde_json::Map::new();
for (k, v) in obj {
if let Some(sub) = v.as_object() {
if let Some(val) = sub.get("value") {
row.insert(k.clone(), val.clone());
} else {
row.insert(k.clone(), serde_json::Value::String(v.to_string()));
}
} else {
row.insert(k.clone(), v.clone());
}
}
for key in row.keys() {
if !all_keys.contains(key) {
all_keys.push(key.clone());
}
}
bucket_rows.push(row);
}
}
let rows = bucket_rows
.iter()
.map(|br| {
all_keys
.iter()
.map(|k| {
br.get(k)
.map(|v| match v {
serde_json::Value::String(s) => serde_json::Value::String(s.clone()),
other => serde_json::Value::String(other.to_string()),
})
.unwrap_or(serde_json::Value::Null)
})
.collect()
})
.collect();
return (all_keys, rows);
}
}
let mut columns = Vec::new();
let mut values = Vec::new();
for (name, agg_value) in aggs {
if let Some(obj) = agg_value.as_object() {
if let Some(val) = obj.get("value") {
columns.push(name.clone());
values.push(match val {
serde_json::Value::String(s) => serde_json::Value::String(s.clone()),
other => serde_json::Value::String(other.to_string()),
});
}
}
}
if !columns.is_empty() {
return (columns, vec![values]);
}
(Vec::new(), Vec::new())
}
#[cfg(test)]
mod tests {
use super::{
build_count_documents_body, build_find_documents_body, elasticsearch_accept_invalid_certs,
elasticsearch_base_url_fallbacks, elasticsearch_index_grouping, group_index_names, normalize_index_names,
redact_elasticsearch_url, EsClient, SearchResponse,
};
use serde_json::json;
use std::time::Duration;
async fn read_http_request(socket: &mut tokio::net::TcpStream) -> String {
use tokio::io::AsyncReadExt;
let mut bytes = Vec::new();
let mut buffer = [0_u8; 1024];
loop {
let read = socket.read(&mut buffer).await.unwrap();
assert!(read > 0, "HTTP request ended before its body was received");
bytes.extend_from_slice(&buffer[..read]);
let Some(headers_end) = bytes.windows(4).position(|window| window == b"\r\n\r\n") else {
continue;
};
let content_length = std::str::from_utf8(&bytes[..headers_end])
.unwrap()
.lines()
.find_map(|line| {
let (name, value) = line.split_once(':')?;
name.eq_ignore_ascii_case("content-length").then(|| value.trim().parse::<usize>().unwrap())
})
.unwrap_or(0);
let request_end = headers_end + 4 + content_length;
if bytes.len() >= request_end {
bytes.truncate(request_end);
return String::from_utf8(bytes).unwrap();
}
}
}
#[test]
fn parses_rest_request_after_comments_and_preserves_query_parameters() {
let request = super::parse_elasticsearch_rest_request(
"# JVM statistics\n// available on Elasticsearch 7+\nGET /_nodes/stats/jvm?pretty",
)
.unwrap();
assert_eq!(request.method, reqwest::Method::GET);
assert_eq!(request.path, "/_nodes/stats/jvm?pretty");
assert_eq!(request.body, None);
assert_eq!(request.body_kind, super::ElasticsearchRestBodyKind::Json);
}
#[test]
fn normalize_index_names_filters_sorts_and_dedups() {
let names = normalize_index_names(
[".kibana", "ngx-log-2", "ngx-log-1", "ngx-log-1", ".security-7"].into_iter().map(String::from),
);
// 去掉点前缀内部索引、排序、去重。
assert_eq!(names, vec!["ngx-log-1".to_string(), "ngx-log-2".to_string()]);
}
#[test]
fn connectivity_check_disabled_parses_bool_and_string() {
use super::elasticsearch_connectivity_check_disabled as disabled;
assert!(!disabled(None));
assert!(!disabled(Some(&serde_json::json!({}))));
assert!(disabled(Some(&serde_json::json!({ "connectivityCheckDisabled": true }))));
assert!(disabled(Some(&serde_json::json!({ "connectivityCheckDisabled": "on" }))));
assert!(disabled(Some(&serde_json::json!({ "connectivityCheckDisabled": "TRUE" }))));
assert!(!disabled(Some(&serde_json::json!({ "connectivityCheckDisabled": false }))));
assert!(!disabled(Some(&serde_json::json!({ "connectivityCheckDisabled": "off" }))));
}
#[test]
fn group_index_names_off_by_default() {
// 缺省配置 → 关闭聚合,原样返回。
assert!(elasticsearch_index_grouping(None).is_none());
let raw = vec!["a-2026.08.04".to_string(), "a-2026.08.05".to_string()];
assert_eq!(group_index_names(raw.clone(), None), raw);
}
#[test]
fn group_index_names_tenant_level_with_capture_group() {
// 保留 `@<第一段>`(到第一个下划线),其后全部折叠成 `*`。用中性占位名。
let cfg = serde_json::json!({ "indexGroupingPattern": r"^([^@]*@[^_]+)_.*$" });
let re = elasticsearch_index_grouping(Some(&cfg));
let out = group_index_names(
vec![
"svc@alpha_r1-2026.08.06@0-000001".to_string(),
"svc@alpha_r1-2026.08.07@0-000001".to_string(),
"svc@beta_r1-2026.08.06@0-000001".to_string(),
"svc_err-2026.08.06@0-000001".to_string(), // 无 @ → 不匹配 → 原样
],
re.as_ref(),
);
assert_eq!(
out,
vec!["svc@alpha*".to_string(), "svc@beta*".to_string(), "svc_err-2026.08.06@0-000001".to_string(),]
);
}
#[test]
fn group_index_names_tail_strip_without_capture_group() {
// 无捕获组:按 ES 惯例剥掉“日期+滚动号”尾巴,`${1}` 为空即替换成 `*`。
let cfg = serde_json::json!({ "indexGroupingPattern": r"[-_.@]\d{4}[-_.]?\d{2}[-_.]?\d{2}.*$" });
let re = elasticsearch_index_grouping(Some(&cfg));
let out = group_index_names(
vec!["logs-2026.08.06".to_string(), "logs-2026.08.07".to_string(), "orders-2026.08.01".to_string()],
re.as_ref(),
);
assert_eq!(out, vec!["logs*".to_string(), "orders*".to_string()]);
}
#[test]
fn group_index_names_mixed_tenant_and_plain_scheme() {
// 同一条正则同时处理:真租户(@后跟字母)折到 @租户;无租户的按名字剥日期尾巴;
// 普通非时间序列索引保持原样。区分点:真租户 @ 后是字母,滚动号 @ 后是数字。用中性名。
let cfg = serde_json::json!({ "indexGroupingPattern": r"^([^@]*@[a-zA-Z][a-zA-Z0-9]*|[^-@]*)[-_.@].*$" });
let re = elasticsearch_index_grouping(Some(&cfg));
let out = group_index_names(
vec![
"svc_err-2026.08.10@0-000001".to_string(), // 无租户 → svc_err*
"svc_err-2026.08.11@0-000001".to_string(),
"svc@alpha_r1-2026.08.10@0-000001".to_string(), // 带区域租户 → svc@alpha*
"svc@beta-2026.08.10@0-000001".to_string(), // 无区域租户 → svc@beta*
"catalog".to_string(), // 普通索引无尾巴 → 原样
],
re.as_ref(),
);
assert_eq!(
out,
vec!["catalog".to_string(), "svc@alpha*".to_string(), "svc@beta*".to_string(), "svc_err*".to_string(),]
);
}
#[test]
fn parses_rest_request_after_multiline_block_comment() {
let request = super::parse_elasticsearch_rest_request(
"/* node statistics\n safe on supported clusters */\nGET /_nodes/stats/jvm?pretty",
)
.unwrap();
assert_eq!(request.method, reqwest::Method::GET);
assert_eq!(request.path, "/_nodes/stats/jvm?pretty");
}
#[test]
fn parses_lowercase_rest_method() {
let request = super::parse_elasticsearch_rest_request("get /_cluster/health").unwrap();
assert_eq!(request.method, reqwest::Method::GET);
assert_eq!(request.path, "/_cluster/health");
}
#[test]
fn encodes_raw_date_math_paths_without_double_encoding() {
let raw = super::parse_elasticsearch_rest_request("GET /<logs-{now/d}>/_search?pretty").unwrap();
assert_eq!(raw.path, "/%3Clogs-%7Bnow%2Fd%7D%3E/_search?pretty");
let encoded = super::parse_elasticsearch_rest_request("GET /%3Clogs-%7Bnow%2Fd%7D%3E/_search?pretty").unwrap();
assert_eq!(encoded.path, "/%3Clogs-%7Bnow%2Fd%7D%3E/_search?pretty");
}
#[test]
fn detects_ndjson_endpoints_with_index_and_query_parameters() {
let request = super::parse_elasticsearch_rest_request(
"POST /orders/_bulk?refresh=true\n{\"index\":{\"_id\":\"1\"}}\n{\"name\":\"Notebook\"}",
)
.unwrap();
assert_eq!(request.method, reqwest::Method::POST);
assert_eq!(request.body_kind, super::ElasticsearchRestBodyKind::Ndjson);
assert!(request.body.is_some());
}
#[test]
fn normalizes_ndjson_with_a_required_trailing_newline() {
let body = "{\"index\":{}}\n{\"name\":\"Notebook\"}";
assert_eq!(super::validate_elasticsearch_ndjson(body).unwrap(), format!("{body}\n"));
assert!(super::validate_elasticsearch_ndjson("{\"index\":{}}\nnot-json").is_err());
}
#[test]
fn url_params_can_disable_elasticsearch_tls_verification() {
assert!(elasticsearch_accept_invalid_certs(false, Some("sslmode=disable")));
assert!(elasticsearch_accept_invalid_certs(false, Some("?tlsVerify=false")));
assert!(elasticsearch_accept_invalid_certs(false, Some("verify=0")));
assert!(elasticsearch_accept_invalid_certs(false, Some("insecure=true")));
assert!(elasticsearch_accept_invalid_certs(false, Some("accept_invalid_certs=on")));
assert!(!elasticsearch_accept_invalid_certs(false, Some("sslmode=require&verify=true")));
}
#[test]
fn tls_checkbox_keeps_legacy_insecure_elasticsearch_behavior() {
assert!(elasticsearch_accept_invalid_certs(true, None));
}
#[test]
fn localhost_elasticsearch_url_falls_back_to_ipv4_loopback() {
assert_eq!(
elasticsearch_base_url_fallbacks("https://localhost:9200"),
vec!["https://127.0.0.1:9200".to_string()]
);
assert_eq!(elasticsearch_base_url_fallbacks("https://search.example.com:9200"), Vec::<String>::new());
}
#[test]
fn disabled_connectivity_check_keeps_localhost_request_fallback() {
assert_eq!(
super::elasticsearch_localhost_resolve_addrs("https://localhost:9200", true),
Some(["[::1]:0".parse().unwrap(), "127.0.0.1:0".parse().unwrap()])
);
assert_eq!(super::elasticsearch_localhost_resolve_addrs("https://localhost:9200", false), None);
assert_eq!(super::elasticsearch_localhost_resolve_addrs("https://search.example.com:9200", true), None);
}
#[tokio::test]
async fn disabled_connectivity_check_can_request_ipv4_localhost() {
use tokio::io::AsyncWriteExt;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let request = read_http_request(&mut socket).await;
assert!(request.starts_with("GET /_cluster/health "), "unexpected request: {request}");
let body = r#"{"status":"green"}"#;
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
body.len(),
body
);
socket.write_all(response.as_bytes()).await.unwrap();
});
let mut client = EsClient::from_config(
&format!("http://localhost:{}", addr.port()),
None,
None,
false,
None,
Some(&json!({ "connectivityCheckDisabled": "yes" })),
Duration::from_secs(2),
);
super::test_connection(&mut client, Duration::from_secs(2)).await.unwrap();
let response = client.get("/_cluster/health").send().await.unwrap();
assert!(response.status().is_success());
server.await.unwrap();
}
#[test]
fn elasticsearch_client_from_config_uses_url_params_for_tls_verification() {
let client = EsClient::from_config(
"https://localhost:9200/",
Some("elastic"),
Some("secret"),
false,
Some("sslmode=disable"),
None,
Duration::from_secs(1),
);
assert_eq!(client.base_url, "https://localhost:9200");
assert_eq!(client.fallback_base_urls, vec!["https://127.0.0.1:9200"]);
assert_eq!(client.connectivity_check_path, "/");
}
#[test]
fn connectivity_check_path_normalizes_get_path_and_defaults() {
assert_eq!(super::elasticsearch_connectivity_check_path(None), "/");
assert_eq!(super::elasticsearch_connectivity_check_path(Some(&json!({ "connectivityCheckPath": "" }))), "/");
assert_eq!(
super::elasticsearch_connectivity_check_path(Some(&json!({
"connectivityCheckPath": "GET pro-jmsau-nwm-applog-*/_search"
}))),
"/pro-jmsau-nwm-applog-*/_search"
);
assert_eq!(
super::elasticsearch_connectivity_check_path(Some(&json!({
"connectivityCheckPath": "my-index/_search\n{\"query\":{\"match_all\":{}}}"
}))),
"/my-index/_search"
);
let client = EsClient::from_config(
"https://localhost:5601/",
None,
None,
false,
None,
Some(&json!({
"mode": "kibana",
"connectivityCheckPath": "GET pro-logs-*/_search"
})),
Duration::from_secs(1),
);
assert_eq!(client.connectivity_check_path, "/pro-logs-*/_search");
}
#[tokio::test]
async fn test_connection_uses_configured_connectivity_check_path() {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let mut request = [0_u8; 2048];
let read = socket.read(&mut request).await.unwrap();
let request = String::from_utf8_lossy(&request[..read]);
assert!(request.starts_with("GET /pro-logs-*/_search "), "unexpected request: {request}");
let body = r#"{"hits":{"total":{"value":0,"relation":"eq"},"hits":[]}}"#;
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
body.len(),
body
);
socket.write_all(response.as_bytes()).await.unwrap();
});
let mut client = EsClient::from_config(
&format!("http://{addr}"),
None,
None,
false,
None,
Some(&json!({ "connectivityCheckPath": "/pro-logs-*/_search" })),
Duration::from_secs(2),
);
super::test_connection(&mut client, Duration::from_secs(2)).await.unwrap();
server.await.unwrap();
}
#[test]
fn elasticsearch_client_from_config_enables_kibana_proxy_with_base_path() {
let external_config = json!({ "mode": "kibana", "kibanaBasePath": "/kibana/s/analytics/" });
let client = EsClient::from_config(
"https://localhost:5601/",
Some("elastic"),
Some("secret"),
false,
None,
Some(&external_config),
Duration::from_secs(1),
);
assert_eq!(client.base_url, "https://localhost:5601/kibana/s/analytics");
assert_eq!(client.fallback_base_urls, vec!["https://127.0.0.1:5601/kibana/s/analytics"]);
assert_eq!(client.transport_mode, super::ElasticsearchTransportMode::KibanaProxy);
}
#[test]
fn redacts_elasticsearch_url_credentials_in_errors() {
assert_eq!(
redact_elasticsearch_url("https://elastic:secret@localhost:9200"),
"https://user:password@localhost:9200"
);
}
#[test]
fn encodes_elasticsearch_index_path_segments() {
assert_eq!(super::elasticsearch_index_path("%kuzzle.users", "_search"), "/%25kuzzle.users/_search");
assert_eq!(super::elasticsearch_index_path("logs-*", "_search"), "/logs-*/_search");
assert_eq!(super::elasticsearch_index_path("logs/2026", "_mapping"), "/logs%2F2026/_mapping");
}
#[test]
fn encodes_elasticsearch_document_id_path_segment() {
assert_eq!(super::elasticsearch_path_segment("a%b/c"), "a%25b%2Fc");
}
#[test]
fn builds_elasticsearch_document_path_with_type_and_routing() {
assert_eq!(
super::elasticsearch_document_path("orders/2026", "a%b/c", Some("legacy/order"), Some("tenant/a&b")),
"/orders%2F2026/legacy%2Forder/a%25b%2Fc?routing=tenant%2Fa%26b&refresh=true"
);
assert_eq!(
super::elasticsearch_document_path("orders", "1", Some("_doc"), None),
"/orders/_doc/1?refresh=true"
);
assert_eq!(super::elasticsearch_document_path("orders", "1", None, None), "/orders/_doc/1?refresh=true");
}
#[test]
fn builds_elasticsearch_auto_id_document_path_with_routing() {
assert_eq!(
super::elasticsearch_auto_id_document_path("orders/2026", Some("tenant/a&b")),
"/orders%2F2026/_doc?routing=tenant%2Fa%26b&refresh=true"
);
assert_eq!(super::elasticsearch_auto_id_document_path("orders", None), "/orders/_doc?refresh=true");
}
#[test]
fn insert_document_body_extracts_routing_without_embedding_it() {
let (doc, routing) =
super::elasticsearch_document_body_and_routing_from_json(r#"{"_routing":"tenant-1","name":"Alice"}"#, None)
.expect("parse insert body");
assert_eq!(routing.as_deref(), Some("tenant-1"));
assert_eq!(doc, serde_json::json!({"name":"Alice"}));
assert_eq!(
super::elasticsearch_auto_id_document_path("orders", routing.as_deref()),
"/orders/_doc?routing=tenant-1&refresh=true"
);
}
#[test]
fn elasticsearch_sql_detection_does_not_treat_rest_methods_as_sql() {
assert!(super::is_elasticsearch_sql_query("SELECT * FROM index_task_v1"));
assert!(super::is_elasticsearch_sql_query(" select count(*) from index_task_v1"));
assert!(!super::is_elasticsearch_sql_query("GET /index_task_v1/_mapping"));
assert!(!super::is_elasticsearch_sql_query("POST /index_task_v1/_search\n{}"));
assert!(!super::is_elasticsearch_sql_query("DELETE /index_task_v1/_doc/1"));
}
#[test]
fn builds_elasticsearch_find_body_with_filter_and_sort() {
let body = build_find_documents_body(20, 10, Some(r#"{"city":"长治"}"#), Some(r#"{"created_at":-1}"#)).unwrap();
assert_eq!(
body,
json!({
"from": 20,
"size": 10,
"query": { "term": { "city": "长治" } },
"sort": [{ "created_at": { "order": "desc" } }]
})
);
assert!(body.get("track_total_hits").is_none());
}
#[test]
fn builds_elasticsearch_find_body_with_native_query_builder_filter() {
let body = build_find_documents_body(
0,
25,
Some(
r#"{
"$esQuery": {
"bool": {
"must": [{"match": {"customer_name": "Customer"}}],
"filter": [{"range": {"amount": {"gte": 500}}}],
"must_not": [{"term": {"status": "cancelled"}}]
}
}
}"#,
),
None,
)
.unwrap();
assert_eq!(
body["query"],
json!({
"bool": {
"must": [{"match": {"customer_name": "Customer"}}],
"filter": [{"range": {"amount": {"gte": 500}}}],
"must_not": [{"term": {"status": "cancelled"}}]
}
})
);
}
#[test]
fn builds_elasticsearch_count_body_with_the_same_native_query_filter() {
let body = build_count_documents_body(Some(r#"{"$esQuery":{"term":{"status":"active"}}}"#)).unwrap();
assert_eq!(body, json!({ "query": { "term": { "status": "active" } } }));
assert!(body.get("from").is_none());
assert!(body.get("size").is_none());
assert!(body.get("sort").is_none());
}
#[test]
fn builds_elasticsearch_count_body_with_structured_filter_operators() {
let filter = r#"{"$and":[{"city":{"$ne":"上海"}},{"age":{"$gte":18}}]}"#;
assert_eq!(
build_count_documents_body(Some(filter)).unwrap(),
json!({
"query": {
"bool": {
"filter": [
{ "bool": { "must_not": [{ "term": { "city": "上海" } }] } },
{ "range": { "age": { "gte": 18 } } }
]
}
}
})
);
}
#[test]
fn accepts_only_complete_elasticsearch_count_shards() {
assert!(super::ElasticsearchShards { total: 3, successful: 2, skipped: 1, failed: 0 }.is_complete());
assert!(!super::ElasticsearchShards { total: 3, successful: 2, skipped: 0, failed: 0 }.is_complete());
assert!(!super::ElasticsearchShards { total: 3, successful: 2, skipped: 0, failed: 1 }.is_complete());
}
#[tokio::test]
async fn counts_documents_with_the_translated_filter() {
use tokio::io::AsyncWriteExt;
let response_body = r#"{"count":552033,"_shards":{"total":3,"successful":3,"skipped":0,"failed":0}}"#;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let request = read_http_request(&mut socket).await;
assert!(request.starts_with("POST /orders/_count "));
let body = request.split_once("\r\n\r\n").unwrap().1;
assert_eq!(
serde_json::from_str::<serde_json::Value>(body).unwrap(),
json!({ "query": { "term": { "status": "active" } } })
);
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
response_body.len(),
response_body,
);
socket.write_all(response.as_bytes()).await.unwrap();
});
let client = EsClient::new(&format!("http://{addr}"), None, None, false, Duration::from_secs(1));
assert_eq!(super::count_documents(&client, "orders", Some(r#"{"status":"active"}"#)).await.unwrap(), 552_033);
server.await.unwrap();
}
#[tokio::test]
async fn rejects_partial_elasticsearch_document_count() {
use tokio::io::AsyncWriteExt;
let response_body = r#"{"count":4,"_shards":{"total":3,"successful":2,"skipped":0,"failed":1}}"#;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let request = read_http_request(&mut socket).await;
assert!(request.starts_with("POST /orders/_count "));
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
response_body.len(),
response_body,
);
socket.write_all(response.as_bytes()).await.unwrap();
});
let client = EsClient::new(&format!("http://{addr}"), None, None, false, Duration::from_secs(1));
let error = super::count_documents(&client, "orders", None).await.unwrap_err();
assert!(error.contains("incomplete shard response: 2 successful, 0 skipped, 1 failed of 3 shards"));
server.await.unwrap();
}
#[test]
fn builds_elasticsearch_find_body_with_structured_filter_operators() {
let body = build_find_documents_body(
0,
100,
Some(
r#"{
"$and": [
{"city": {"$ne": "上海"}},
{"age": {"$gt": 18, "$lte": 60}},
{"name": {"$not": {"$regex": "test", "$options": "i"}}}
]
}"#,
),
None,
)
.unwrap();
assert_eq!(
body,
json!({
"from": 0,
"size": 100,
"query": {
"bool": {
"filter": [
{ "bool": { "must_not": [{ "term": { "city": "上海" } }] } },
{ "range": { "age": { "gt": 18, "lte": 60 } } },
{
"bool": {
"must_not": [
{
"wildcard": {
"name": {
"value": "*test*",
"case_insensitive": true
}
}
}
]
}
}
]
}
},
"sort": ["_doc"]
})
);
}
#[test]
fn builds_elasticsearch_find_body_with_or_filter() {
let body =
build_find_documents_body(0, 50, Some(r#"{"$or":[{"city":"长治"},{"city":"上海"}]}"#), None).unwrap();
assert_eq!(
body["query"],
json!({
"bool": {
"should": [
{ "term": { "city": "长治" } },
{ "term": { "city": "上海" } }
],
"minimum_should_match": 1
}
})
);
}
#[test]
fn parses_search_total_from_elasticsearch_6_number_shape() {
let response: SearchResponse = serde_json::from_value(json!({
"_shards": { "total": 1, "successful": 1, "skipped": 0, "failed": 0 },
"hits": {
"total": 5,
"hits": []
}
}))
.unwrap();
assert_eq!(response.hits.total.value(), 5);
assert!(response.hits.total.is_exact());
}
#[test]
fn parses_search_total_from_elasticsearch_7_object_shape() {
let response: SearchResponse = serde_json::from_value(json!({
"_shards": { "total": 1, "successful": 1, "skipped": 0, "failed": 0 },
"hits": {
"total": { "value": 5, "relation": "eq" },
"hits": []
}
}))
.unwrap();
assert_eq!(response.hits.total.value(), 5);
assert!(response.hits.total.is_exact());
let result = super::search_response_to_document_result(response).unwrap();
assert!(result.total_is_exact);
}
#[test]
fn preserves_elasticsearch_lower_bound_total_relation() {
let response: SearchResponse = serde_json::from_value(json!({
"hits": {
"total": { "value": 10_000, "relation": "gte" },
"hits": []
}
}))
.unwrap();
assert_eq!(response.hits.total.value(), 10_000);
assert!(!response.hits.total.is_exact());
let result = super::search_response_to_document_result(response).unwrap();
assert_eq!(result.total, 10_000);
assert!(!result.total_is_exact);
}
#[test]
fn treats_search_total_as_a_lower_bound_when_a_shard_failed() {
let response: SearchResponse = serde_json::from_value(json!({
"_shards": { "total": 3, "successful": 2, "skipped": 0, "failed": 1 },
"hits": {
"total": { "value": 5, "relation": "eq" },
"hits": []
}
}))
.unwrap();
let result = super::search_response_to_document_result(response).unwrap();
assert_eq!(result.total, 5);
assert!(!result.total_is_exact);
}
#[test]
fn treats_timed_out_or_terminated_searches_as_lower_bounds() {
for response in [
json!({
"timed_out": true,
"_shards": { "total": 1, "successful": 1, "skipped": 0, "failed": 0 },
"hits": { "total": { "value": 5, "relation": "eq" }, "hits": [] }
}),
json!({
"terminated_early": true,
"_shards": { "total": 1, "successful": 1, "skipped": 0, "failed": 0 },
"hits": { "total": { "value": 5, "relation": "eq" }, "hits": [] }
}),
] {
let response: SearchResponse = serde_json::from_value(response).unwrap();
let result = super::search_response_to_document_result(response).unwrap();
assert_eq!(result.total, 5);
assert!(!result.total_is_exact);
}
}
#[test]
fn treats_search_total_without_shard_metadata_as_a_lower_bound() {
let response: SearchResponse = serde_json::from_value(json!({
"hits": {
"total": { "value": 5, "relation": "eq" },
"hits": []
}
}))
.unwrap();
let result = super::search_response_to_document_result(response).unwrap();
assert_eq!(result.total, 5);
assert!(!result.total_is_exact);
}
#[test]
fn parses_elasticsearch_hit_routing_metadata() {
let response: SearchResponse = serde_json::from_value(json!({
"hits": {
"total": { "value": 1, "relation": "eq" },
"hits": [
{ "_id": "doc-1", "_routing": "tenant-1", "_source": { "name": "Alice" } }
]
}
}))
.unwrap();
assert_eq!(response.hits.hits[0].routing.as_deref(), Some("tenant-1"));
}
#[test]
fn preserves_legacy_elasticsearch_document_type_metadata() {
let response: SearchResponse = serde_json::from_value(json!({
"hits": {
"total": { "value": 2, "relation": "eq" },
"hits": [
{ "_id": "legacy-1", "_type": "order", "_source": { "name": "Legacy" } },
{ "_id": "modern-1", "_type": "_doc", "_source": { "name": "Modern" } }
]
}
}))
.unwrap();
let result = super::search_response_to_document_result(response).unwrap();
assert_eq!(result.documents[0]["_type"], json!("order"));
assert!(result.documents[1].get("_type").is_none());
}
#[test]
fn preserves_long_literals_in_document_transport_json() {
let response: SearchResponse = serde_json::from_str(
r#"{"hits":{"total":{"value":1,"relation":"eq"},"hits":[{"_id":"doc-1","_source":{"id":2018551659033767937,"string_id":"2018551659033767937","legacy":{"$numberLong":"2018551659033767937"}}}]}}"#,
)
.unwrap();
let result = super::search_response_to_document_result(response).unwrap();
assert_eq!(
result.raw_documents.as_deref(),
Some(&[r#"{"id":2018551659033767937,"string_id":"2018551659033767937","legacy":{"$numberLong":"2018551659033767937"},"_id":"doc-1"}"#.to_string()][..])
);
}
#[test]
fn parses_search_response_rows_with_routing_metadata() {
let result = super::parse_elasticsearch_response(
200,
json!({
"hits": {
"hits": [
{ "_id": "doc-1", "_routing": "tenant-1", "_source": { "name": "Alice" } }
]
}
}),
std::time::Instant::now(),
)
.unwrap();
assert_ne!(result.columns, vec!["status", "response"]);
let name_idx = result.columns.iter().position(|column| column == "name").unwrap();
assert_eq!(result.rows[0][name_idx], json!("Alice"));
let routing_idx = result.columns.iter().position(|column| column == "_routing").unwrap();
assert_eq!(result.rows[0][routing_idx], json!("tenant-1"));
}
#[test]
fn keeps_sql_api_response_tabular() {
let result = super::parse_elasticsearch_response(
200,
json!({
"columns": [{ "name": "name" }],
"rows": [["Alice"]]
}),
std::time::Instant::now(),
)
.unwrap();
assert_eq!(result.columns, vec!["name"]);
assert_eq!(result.rows, vec![vec![json!("Alice")]]);
}
#[test]
fn parses_aggregation_response_before_empty_hits() {
let result = super::parse_elasticsearch_response(
200,
json!({
"hits": {
"total": { "value": 5, "relation": "eq" },
"hits": []
},
"aggregations": {
"by_status": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{ "key": "paid", "doc_count": 3 },
{ "key": "cancelled", "doc_count": 1 },
{ "key": "pending", "doc_count": 1 }
]
}
}
}),
std::time::Instant::now(),
)
.unwrap();
let key_idx = result.columns.iter().position(|column| column == "key").unwrap();
let count_idx = result.columns.iter().position(|column| column == "doc_count").unwrap();
assert_eq!(result.rows.len(), 3);
assert_eq!(result.rows[0][key_idx], json!("paid"));
assert_eq!(result.rows[0][count_idx], json!("3"));
assert_eq!(result.affected_rows, 3);
}
#[test]
fn parses_plain_text_rest_response_without_dropping_body() {
let body =
"health status index docs.count store.size\ngreen open app-log-2026-07 42 10mb\n";
let result = super::parse_elasticsearch_rest_response(200, body, std::time::Instant::now()).unwrap();
assert_eq!(result.columns, vec!["response"]);
assert_eq!(result.rows.len(), 2);
assert_eq!(result.rows[0][0], json!("health status index docs.count store.size"));
assert_eq!(result.rows[1][0], json!("green open app-log-2026-07 42 10mb"));
assert_eq!(result.affected_rows, 2);
}
#[test]
fn adds_json_format_only_when_cat_format_is_not_explicit() {
let plain =
super::add_default_cat_json_format(super::parse_elasticsearch_rest_request("GET /_cat/indices").unwrap());
assert_eq!(plain.path, "/_cat/indices?format=json");
let defaulted =
super::add_default_cat_json_format(super::parse_elasticsearch_rest_request("GET /_cat/indices?v").unwrap());
assert_eq!(defaulted.path, "/_cat/indices?v&format=json");
let explicit_text = super::add_default_cat_json_format(
super::parse_elasticsearch_rest_request("GET /_cat/indices?format=txt").unwrap(),
);
assert_eq!(explicit_text.path, "/_cat/indices?format=txt");
}
#[test]
fn preserves_http_status_for_plain_text_rest_errors() {
let result =
super::parse_elasticsearch_rest_response(503, "service temporarily unavailable", std::time::Instant::now())
.unwrap();
assert_eq!(result.columns, vec!["status", "response"]);
assert_eq!(result.rows, vec![vec![json!(503), json!("service temporarily unavailable")]]);
}
#[test]
fn keeps_mapping_rest_response_numeric_literals_lossless() {
let body = r#"{
"products": {
"mappings": {
"_meta": {
"largest_id": 123456789012345678901234567890,
"ratio": 0.123456789012345678901234567890,
"estimate": 1e400
},
"properties": { "name": { "type": "keyword" } }
}
}
}"#;
let result = super::parse_elasticsearch_rest_response(200, body, std::time::Instant::now()).unwrap();
assert_eq!(result.columns, vec!["status", "response"]);
assert_eq!(result.rows[0][0], json!(200));
assert_eq!(result.rows[0][1].as_str(), Some(body));
}
#[tokio::test]
async fn execute_rest_query_preserves_index_specific_cat_json_response() {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let body = r#"[{"health":"green","index":"app-log-2026-07","docs.count":"42"}]"#;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let mut request = [0_u8; 1024];
let read = socket.read(&mut request).await.unwrap();
let request = String::from_utf8_lossy(&request[..read]);
assert!(request.starts_with("GET /_cat/indices/data_pack_and_box_index_v1?format=json "));
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
body.len(),
body
);
socket.write_all(response.as_bytes()).await.unwrap();
});
let client = EsClient::new(&format!("http://{addr}"), None, None, false, Duration::from_secs(1));
let result = super::execute_rest_query(&client, "GET /_cat/indices/data_pack_and_box_index_v1").await.unwrap();
server.await.unwrap();
assert_eq!(result.columns, vec!["status", "response"]);
assert_eq!(result.rows, vec![vec![json!(200), json!(body)]]);
}
#[tokio::test]
async fn execute_rest_query_preserves_explicit_cat_json_response() {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let body = r#"[{"index":"data_pack_and_box_index_v1","docs.count":"42"}]"#;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let mut request = [0_u8; 1024];
let read = socket.read(&mut request).await.unwrap();
let request = String::from_utf8_lossy(&request[..read]);
assert!(request.starts_with("GET /_cat/indices/data_pack_and_box_index_v1?format=json "));
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
body.len(),
body
);
socket.write_all(response.as_bytes()).await.unwrap();
});
let client = EsClient::new(&format!("http://{addr}"), None, None, false, Duration::from_secs(1));
let result = super::execute_rest_query(&client, "GET /_cat/indices/data_pack_and_box_index_v1?format=json")
.await
.unwrap();
server.await.unwrap();
assert_eq!(result.columns, vec!["status", "response"]);
assert_eq!(result.rows, vec![vec![json!(200), json!(body)]])
}
#[tokio::test]
async fn execute_rest_query_keeps_default_cat_text_when_server_does_not_return_json() {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let body = "health status\ngreen open\n";
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let mut request = [0_u8; 1024];
let read = socket.read(&mut request).await.unwrap();
let request = String::from_utf8_lossy(&request[..read]);
assert!(request.starts_with("GET /_cat/indices?format=json "));
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
body.len(),
body
);
socket.write_all(response.as_bytes()).await.unwrap();
});
let client = EsClient::new(&format!("http://{addr}"), None, None, false, Duration::from_secs(1));
let result = super::execute_rest_query(&client, "GET /_cat/indices").await.unwrap();
server.await.unwrap();
assert_eq!(result.columns, vec!["response"]);
assert_eq!(result.rows, vec![vec![json!("health status")], vec![json!("green open")]]);
}
#[tokio::test]
async fn execute_rest_query_keeps_explicit_text_cat_response() {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let body = "health status\ngreen open\n";
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let mut request = [0_u8; 1024];
let read = socket.read(&mut request).await.unwrap();
let request = String::from_utf8_lossy(&request[..read]);
assert!(request.starts_with("GET /_cat/indices?format=txt "));
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
body.len(),
body
);
socket.write_all(response.as_bytes()).await.unwrap();
});
let client = EsClient::new(&format!("http://{addr}"), None, None, false, Duration::from_secs(1));
let result = super::execute_rest_query(&client, "GET /_cat/indices?format=txt").await.unwrap();
server.await.unwrap();
assert_eq!(result.columns, vec!["response"]);
assert_eq!(result.rows, vec![vec![json!("health status")], vec![json!("green open")]]);
}
#[tokio::test]
async fn execute_rest_query_preserves_numeric_literals_from_http_body() {
use tokio::io::AsyncWriteExt;
let response_body = r#"{"largest_id":123456789012345678901234567890,"ratio":0.123456789012345678901234567890,"estimate":1e400}"#;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let request = read_http_request(&mut socket).await;
assert!(request.starts_with("GET /products/_mapping "));
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
response_body.len(),
response_body
);
socket.write_all(response.as_bytes()).await.unwrap();
});
let client = EsClient::new(&format!("http://{addr}"), None, None, false, Duration::from_secs(1));
let result = super::execute_rest_query(&client, "GET /products/_mapping").await.unwrap();
server.await.unwrap();
assert_eq!(result.columns, vec!["status", "response"]);
assert_eq!(result.rows[0][0], json!(200));
assert_eq!(result.rows[0][1].as_str(), Some(response_body));
}
#[tokio::test]
async fn execute_rest_head_request_returns_http_status() {
use tokio::io::AsyncWriteExt;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let request = read_http_request(&mut socket).await;
assert!(request.starts_with("HEAD /orders "));
socket.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").await.unwrap();
});
let client = EsClient::new(&format!("http://{addr}"), None, None, false, Duration::from_secs(1));
let result = super::execute_rest_query(&client, "HEAD /orders").await.unwrap();
server.await.unwrap();
assert_eq!(result.columns, vec!["status", "response"]);
assert_eq!(result.rows[0][0], json!(200));
assert_eq!(result.rows[0][1], json!("null"));
}
#[tokio::test]
async fn kibana_proxy_rewrites_method_path_and_preserves_elasticsearch_status() {
use std::collections::HashMap;
use tokio::io::AsyncWriteExt;
let response_body = r#"{"error":{"type":"index_not_found_exception"},"status":404}"#;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let request = read_http_request(&mut socket).await;
let (headers, body) = request.split_once("\r\n\r\n").unwrap();
let request_target = headers.lines().next().unwrap().split_whitespace().nth(1).unwrap();
let url = reqwest::Url::parse(&format!("http://localhost{request_target}")).unwrap();
let query = url.query_pairs().into_owned().collect::<HashMap<_, _>>();
assert_eq!(url.path(), "/kibana/s/analytics/api/console/proxy");
assert_eq!(query.get("path").map(String::as_str), Some("/missing/_doc/1?refresh=true"));
assert_eq!(query.get("method").map(String::as_str), Some("DELETE"));
assert!(headers.lines().any(|line| line.eq_ignore_ascii_case("kbn-xsrf: true")), "{headers}");
assert!(headers.lines().any(|line| line.eq_ignore_ascii_case("authorization: Basic ZWxhc3RpYzpzZWNyZXQ=")));
assert_eq!(serde_json::from_str::<serde_json::Value>(body).unwrap(), json!({ "reason": "cleanup" }));
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nx-console-proxy-status-code: 404\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
response_body.len(),
response_body
);
socket.write_all(response.as_bytes()).await.unwrap();
});
let external_config = json!({ "mode": "kibana", "kibanaBasePath": "/kibana/s/analytics" });
let client = EsClient::from_config(
&format!("http://{addr}"),
Some("elastic"),
Some("secret"),
false,
None,
Some(&external_config),
Duration::from_secs(1),
);
let result =
super::execute_rest_query(&client, "DELETE /missing/_doc/1?refresh=true\n{\"reason\":\"cleanup\"}")
.await
.unwrap();
server.await.unwrap();
assert_eq!(result.rows[0][0], json!(404));
assert_eq!(result.rows[0][1], json!(response_body));
}
#[tokio::test]
async fn execute_rest_query_sends_encoded_date_math_path() {
use tokio::io::AsyncWriteExt;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let request = read_http_request(&mut socket).await;
assert!(request.starts_with("GET /%3Clogs-%7Bnow%2Fd%7D%3E/_search?pretty "), "{request}");
socket.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}").await.unwrap();
});
let client = EsClient::new(&format!("http://{addr}"), None, None, false, Duration::from_secs(1));
super::execute_rest_query(&client, "GET /<logs-{now/d}>/_search?pretty").await.unwrap();
server.await.unwrap();
}
#[tokio::test]
async fn execute_rest_msearch_sends_ndjson_content_type_and_trailing_newline() {
use tokio::io::AsyncWriteExt;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let request = read_http_request(&mut socket).await;
let (headers, body) = request.split_once("\r\n\r\n").unwrap();
assert!(headers.starts_with("POST /_msearch "));
assert!(headers.lines().any(|line| line.eq_ignore_ascii_case("content-type: application/x-ndjson")));
assert_eq!(body, "{\"index\":\"orders\"}\n{\"size\":0}\n");
let response_body = r#"{"responses":[]}"#;
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
response_body.len(),
response_body
);
socket.write_all(response.as_bytes()).await.unwrap();
});
let client = EsClient::new(&format!("http://{addr}"), None, None, false, Duration::from_secs(1));
let result = super::execute_rest_query(
&client,
"// run two searches\nPOST /_msearch\n{\"index\":\"orders\"}\n{\"size\":0}",
)
.await
.unwrap();
server.await.unwrap();
assert_eq!(result.rows[0][0], json!(200));
}
#[tokio::test]
async fn execute_rest_delete_sends_json_body() {
use tokio::io::AsyncWriteExt;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let request = read_http_request(&mut socket).await;
let (headers, body) = request.split_once("\r\n\r\n").unwrap();
assert!(headers.starts_with("DELETE /_search/scroll "));
assert_eq!(serde_json::from_str::<serde_json::Value>(body).unwrap(), json!({ "scroll_id": ["scroll-1"] }));
let response_body = r#"{"succeeded":true,"num_freed":1}"#;
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
response_body.len(),
response_body
);
socket.write_all(response.as_bytes()).await.unwrap();
});
let client = EsClient::new(&format!("http://{addr}"), None, None, false, Duration::from_secs(1));
let result =
super::execute_rest_query(&client, "DELETE /_search/scroll\n{\"scroll_id\":[\"scroll-1\"]}").await.unwrap();
server.await.unwrap();
assert_eq!(result.columns, vec!["status", "response"]);
assert_eq!(result.rows[0][0], json!(200));
assert_eq!(
serde_json::from_str::<serde_json::Value>(result.rows[0][1].as_str().unwrap()).unwrap(),
json!({ "succeeded": true, "num_freed": 1 })
);
}
#[test]
fn parses_search_rest_response_as_source_table() {
let body = r#"{"took":1,"hits":{"total":{"value":2,"relation":"eq"},"hits":[{"_id":"1","_source":{"name":"Alice","age":30,"active":true,"deleted_at":null,"profile":{"team":"core"},"tags":["admin","reader"]}},{"_id":"2","_routing":"shard-a","_source":{"name":"Bob","city":"NYC"}}]}}"#;
let result = super::parse_elasticsearch_rest_response(200, body, std::time::Instant::now()).unwrap();
assert_ne!(result.columns, vec!["status", "response"]);
assert!(result.columns.contains(&"name".to_string()));
assert!(result.columns.contains(&"_id".to_string()));
assert_eq!(result.rows.len(), 2);
let name_idx = result.columns.iter().position(|c| c == "name").unwrap();
let id_idx = result.columns.iter().position(|c| c == "_id").unwrap();
assert_eq!(result.rows[0][name_idx], json!("Alice"));
assert_eq!(result.rows[0][id_idx], json!("1"));
let age_idx = result.columns.iter().position(|c| c == "age").unwrap();
let active_idx = result.columns.iter().position(|c| c == "active").unwrap();
let deleted_at_idx = result.columns.iter().position(|c| c == "deleted_at").unwrap();
let profile_idx = result.columns.iter().position(|c| c == "profile").unwrap();
let tags_idx = result.columns.iter().position(|c| c == "tags").unwrap();
assert_eq!(result.rows[0][age_idx], json!(30));
assert_eq!(result.rows[0][active_idx], json!(true));
assert_eq!(result.rows[0][deleted_at_idx], serde_json::Value::Null);
assert_eq!(result.rows[0][profile_idx], json!(r#"{"team":"core"}"#));
assert_eq!(result.rows[0][tags_idx], json!(r#"["admin","reader"]"#));
assert_eq!(result.column_types[age_idx], "number");
assert_eq!(result.column_types[active_idx], "boolean");
assert_eq!(result.column_types[profile_idx], "json");
assert_eq!(result.column_types[tags_idx], "json");
let city_idx = result.columns.iter().position(|c| c == "city").unwrap();
assert_eq!(result.rows[1][city_idx], json!("NYC"));
let routing_idx = result.columns.iter().position(|c| c == "_routing").unwrap();
assert_eq!(result.rows[1][routing_idx], json!("shard-a"));
assert_eq!(result.affected_rows, 2);
assert_eq!(result.elasticsearch_raw_body.as_deref(), Some(body));
}
#[test]
fn parses_empty_search_rest_response_as_empty_table() {
let body = r#"{"took":1,"hits":{"total":{"value":0,"relation":"eq"},"hits":[]}}"#;
let result = super::parse_elasticsearch_rest_response(200, body, std::time::Instant::now()).unwrap();
assert_eq!(result.columns, vec!["_id"]);
assert!(result.rows.is_empty());
assert_eq!(result.affected_rows, 0);
}
#[test]
fn sparse_search_response_over_table_cell_limit_falls_back_to_raw_json() {
let hits = (0..450)
.map(|index| {
let mut source = serde_json::Map::new();
source.insert(format!("field_{index}"), json!(index));
json!({ "_id": index.to_string(), "_source": source })
})
.collect::<Vec<_>>();
let body = json!({ "hits": { "hits": hits } }).to_string();
let result = super::parse_elasticsearch_rest_response(200, &body, std::time::Instant::now()).unwrap();
assert_eq!(result.columns, vec!["status", "response"]);
assert_eq!(result.rows[0][0], json!(200));
assert_eq!(result.rows[0][1], json!(body));
assert_eq!(result.elasticsearch_raw_body, None);
}
#[tokio::test]
async fn execute_rest_query_keeps_json_error_response() {
use tokio::io::AsyncWriteExt;
let response_body = r#"{"error":{"type":"index_not_found_exception","reason":"no such index"},"status":404}"#;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let request = read_http_request(&mut socket).await;
assert!(request.starts_with("GET /missing/_mapping "));
let response = format!(
"HTTP/1.1 404 Not Found\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
response_body.len(),
response_body
);
socket.write_all(response.as_bytes()).await.unwrap();
});
let client = EsClient::new(&format!("http://{addr}"), None, None, false, Duration::from_secs(1));
let result = super::execute_rest_query(&client, "GET /missing/_mapping").await.unwrap();
server.await.unwrap();
assert_eq!(result.columns, vec!["status", "response"]);
assert_eq!(result.rows[0][0], json!(404));
assert_eq!(
serde_json::from_str::<serde_json::Value>(result.rows[0][1].as_str().unwrap()).unwrap(),
json!({ "error": { "type": "index_not_found_exception", "reason": "no such index" }, "status": 404 })
);
}
#[tokio::test]
async fn execute_select_query_keeps_search_response_tabular() {
use tokio::io::AsyncWriteExt;
let response_body = r#"{"hits":{"hits":[{"_id":"product-1","_source":{"name":"Notebook"}}]}}"#;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let request = read_http_request(&mut socket).await;
assert!(request.starts_with("POST /products/_search "));
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
response_body.len(),
response_body
);
socket.write_all(response.as_bytes()).await.unwrap();
});
let client = EsClient::new(&format!("http://{addr}"), None, None, false, Duration::from_secs(1));
let result = super::execute_rest_query(&client, "SELECT * FROM products LIMIT 1").await.unwrap();
server.await.unwrap();
assert_ne!(result.columns, vec!["status", "response"]);
let name_idx = result.columns.iter().position(|column| column == "name").unwrap();
assert_eq!(result.rows[0][name_idx], json!("Notebook"));
}
#[tokio::test]
async fn update_document_uses_legacy_type_path_without_storing_metadata() {
use tokio::io::AsyncWriteExt;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let request = read_http_request(&mut socket).await;
assert!(request.starts_with("PUT /orders/order/abc?routing=tenant-1&refresh=true "));
assert!(request.ends_with(r#"{"name":"Alice"}"#));
assert!(!request.contains(r#""_type""#));
assert!(!request.contains(r#""_routing""#));
let response =
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}";
socket.write_all(response.as_bytes()).await.unwrap();
});
let client = EsClient::new(&format!("http://{addr}"), None, None, false, Duration::from_secs(1));
super::update_document(
&client,
"orders",
"abc",
r#"{"_id":"abc","_type":"order","_routing":"tenant-1","name":"Alice"}"#,
None,
)
.await
.unwrap();
server.await.unwrap();
}
#[tokio::test]
async fn delete_document_uses_legacy_type_path() {
use tokio::io::AsyncWriteExt;
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let request = read_http_request(&mut socket).await;
assert!(request.starts_with("DELETE /orders/order/abc?routing=tenant-1&refresh=true "));
let response =
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}";
socket.write_all(response.as_bytes()).await.unwrap();
});
let client = EsClient::new(&format!("http://{addr}"), None, None, false, Duration::from_secs(1));
super::delete_document(&client, "orders", "abc", Some("order"), Some("tenant-1")).await.unwrap();
server.await.unwrap();
}
#[tokio::test]
async fn execute_rest_search_preserves_full_json_response() {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let body = json!({
"took": 3,
"hits": {
"total": { "value": 1, "relation": "eq" },
"max_score": 1.0,
"hits": [{
"_index": "products",
"_id": "product-1",
"_score": 1.0,
"_source": { "name": "Notebook", "price": 1299 },
"highlight": { "name": ["<em>Note</em>book"] }
}]
},
"aggregations": {
"by_category": {
"buckets": [{ "key": "electronics", "doc_count": 1 }]
}
}
});
let response_body = body.to_string();
let server_response_body = response_body.clone();
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let mut request = [0_u8; 4096];
let read = socket.read(&mut request).await.unwrap();
let request = String::from_utf8_lossy(&request[..read]);
assert!(request.starts_with("POST /products/_search "));
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
server_response_body.len(),
server_response_body
);
socket.write_all(response.as_bytes()).await.unwrap();
});
let client = EsClient::new(&format!("http://{addr}"), None, None, false, Duration::from_secs(1));
let result =
super::execute_rest_query(&client, "POST /products/_search\n{\"query\":{\"match_all\":{}}}").await.unwrap();
server.await.unwrap();
// The response now parses hits+aggs into a tabular result (aggregation
// columns) instead of the raw status/response JSON panel.
assert!(result.columns.contains(&"key".to_string()));
assert!(result.columns.contains(&"doc_count".to_string()));
assert_ne!(result.columns, vec!["status", "response"]);
// Raw body is attached for JSON toggle in the UI.
assert!(result.elasticsearch_raw_body.is_some());
}
#[tokio::test]
#[ignore = "requires DBX_TEST_KIBANA_URL pointing to a reachable Kibana instance"]
async fn live_kibana_proxy_supports_metadata_queries_and_document_writes() {
let url = std::env::var("DBX_TEST_KIBANA_URL").expect("DBX_TEST_KIBANA_URL is required");
let username = std::env::var("DBX_TEST_KIBANA_USERNAME").ok();
let password = std::env::var("DBX_TEST_KIBANA_PASSWORD").ok();
let base_path = std::env::var("DBX_TEST_KIBANA_BASE_PATH").unwrap_or_default();
let external_config = json!({ "mode": "kibana", "kibanaBasePath": base_path });
let mut client = EsClient::from_config(
&url,
username.as_deref(),
password.as_deref(),
false,
None,
Some(&external_config),
Duration::from_secs(20),
);
super::test_connection(&mut client, Duration::from_secs(20)).await.unwrap();
let index = format!("dbx-kibana-proxy-{}", uuid::Uuid::new_v4().simple());
let create = super::execute_rest_query(
&client,
&format!(
"PUT /{index}\n{{\"mappings\":{{\"properties\":{{\"name\":{{\"type\":\"keyword\"}},\"price\":{{\"type\":\"double\"}}}}}}}}"
),
)
.await
.unwrap();
assert_eq!(create.rows[0][0], json!(200));
let id = super::insert_document(&client, &index, r#"{"name":"Notebook","price":12.5}"#, None).await.unwrap();
assert!(!id.is_empty());
assert!(super::list_indices(&client).await.unwrap().contains(&index));
let columns = super::get_columns(&client, &index).await.unwrap();
assert!(columns.iter().any(|column| column.name == "name" && column.data_type == "keyword"));
assert!(columns.iter().any(|column| column.name == "price" && column.data_type == "double"));
let documents = super::find_documents(&client, &index, 0, 10, None, None).await.unwrap();
assert_eq!(documents.total, 1);
assert_eq!(documents.documents[0]["name"], json!("Notebook"));
super::update_document(&client, &index, &id, r#"{"name":"Notebook Pro","price":15.0}"#, None).await.unwrap();
let sql_result = super::execute_rest_query(&client, &format!("SELECT * FROM {index} LIMIT 10")).await.unwrap();
let name_index = sql_result.columns.iter().position(|column| column == "name").unwrap();
assert_eq!(sql_result.rows[0][name_index], json!("Notebook Pro"));
super::delete_document(&client, &index, &id, None, None).await.unwrap();
let delete = super::execute_rest_query(&client, &format!("DELETE /{index}")).await.unwrap();
assert_eq!(delete.rows[0][0], json!(200));
}
#[test]
fn document_body_removes_elasticsearch_id_metadata() {
let (doc, _) = super::elasticsearch_document_body_and_routing_from_json(
r#"{"_id":"abc","_routing":"tenant-1","name":"Alice"}"#,
None,
)
.unwrap();
assert_eq!(doc, json!({ "name": "Alice" }));
}
#[test]
fn document_body_extracts_elasticsearch_routing_metadata() {
let (doc, routing) = super::elasticsearch_document_body_and_routing_from_json(
r#"{"_id":"abc","_routing":"tenant-1","name":"Alice"}"#,
None,
)
.unwrap();
assert_eq!(doc, json!({ "name": "Alice" }));
assert_eq!(routing.as_deref(), Some("tenant-1"));
}
#[test]
fn update_document_body_extracts_legacy_type_metadata() {
let (doc, routing, document_type) = super::elasticsearch_update_document_body_and_metadata(
r#"{"_id":"abc","_type":"order","_routing":"tenant-1","name":"Alice"}"#,
None,
)
.unwrap();
assert_eq!(doc, json!({ "name": "Alice" }));
assert_eq!(routing.as_deref(), Some("tenant-1"));
assert_eq!(document_type.as_deref(), Some("order"));
}
#[test]
fn explicit_elasticsearch_routing_overrides_document_metadata() {
let (doc, routing) = super::elasticsearch_document_body_and_routing_from_json(
r#"{"_id":"abc","_routing":"tenant-1","name":"Alice"}"#,
Some("tenant-2"),
)
.unwrap();
assert_eq!(doc, json!({ "name": "Alice" }));
assert_eq!(routing.as_deref(), Some("tenant-2"));
}
#[test]
fn document_body_preserves_user_field_order() {
let (doc, _) =
super::elasticsearch_document_body_and_routing_from_json(r#"{"z":1,"_id":"abc","a":2}"#, None).unwrap();
assert_eq!(serde_json::to_string(&doc).unwrap(), r#"{"z":1,"a":2}"#);
}
}