feat(elasticsearch): add index grouping pattern support
This commit is contained in:
1 parent
cd78c0ae15
commit
901f5ca3d4
11 files changed
+252
-11
No files matched your search
@@ -105,7 +105,7 @@ import { oceanbaseModeConnectionPatch, oceanbaseSubModeFromConfig } from "@/lib/
|
|||||||
import { translateBackendError } from "@/i18n/backend-errors";
|
import { translateBackendError } from "@/i18n/backend-errors";
|
||||||
import { applyHiveKerberosSubmitConfig, hiveKerberosFormConfig, type HiveKerberosAuthMode } from "@/lib/database/hiveKerberosOptions";
|
import { applyHiveKerberosSubmitConfig, hiveKerberosFormConfig, type HiveKerberosAuthMode } from "@/lib/database/hiveKerberosOptions";
|
||||||
import { hasCloudflareD1Credentials, isCloudflareD1Connection, normalizeCloudflareD1Connection } from "@/lib/connection/cloudflareD1";
|
import { hasCloudflareD1Credentials, isCloudflareD1Connection, normalizeCloudflareD1Connection } from "@/lib/connection/cloudflareD1";
|
||||||
import { buildElasticsearchExternalConfig, elasticsearchConnectionModeFromConfig, elasticsearchConnectivityCheckPathFromConfig, elasticsearchKibanaBasePathFromConfig, type ElasticsearchConnectionMode } from "@/lib/connection/elasticsearchKibanaProxy";
|
import { buildElasticsearchExternalConfig, elasticsearchConnectionModeFromConfig, elasticsearchConnectivityCheckPathFromConfig, elasticsearchIndexGroupingPatternFromConfig, elasticsearchKibanaBasePathFromConfig, type ElasticsearchConnectionMode } from "@/lib/connection/elasticsearchKibanaProxy";
|
||||||
import { GAUSSDB_M_JDBC_DRIVER_CLASS, gaussdbConnectionMode, gaussdbIdentifierQuoteStyle, setGaussdbConnectionMode, setGaussdbIdentifierQuoteStyle, supportsGaussdbIdentifierQuoteStyle, type GaussdbConnectionMode, type GaussdbIdentifierQuoteStyle } from "@/lib/database/jdbcDialect";
|
import { GAUSSDB_M_JDBC_DRIVER_CLASS, gaussdbConnectionMode, gaussdbIdentifierQuoteStyle, setGaussdbConnectionMode, setGaussdbIdentifierQuoteStyle, supportsGaussdbIdentifierQuoteStyle, type GaussdbConnectionMode, type GaussdbIdentifierQuoteStyle } from "@/lib/database/jdbcDialect";
|
||||||
import { normalizeStoredConnectionDatabase } from "@/lib/database/sqliteNamespace";
|
import { normalizeStoredConnectionDatabase } from "@/lib/database/sqliteNamespace";
|
||||||
import {
|
import {
|
||||||
@@ -318,6 +318,7 @@ const defaultForm = (): ConnectionForm => ({
|
|||||||
const elasticsearchConnectionMode = ref<ElasticsearchConnectionMode>("direct");
|
const elasticsearchConnectionMode = ref<ElasticsearchConnectionMode>("direct");
|
||||||
const elasticsearchKibanaBasePath = ref("");
|
const elasticsearchKibanaBasePath = ref("");
|
||||||
const elasticsearchConnectivityCheckPath = ref("");
|
const elasticsearchConnectivityCheckPath = ref("");
|
||||||
|
const elasticsearchIndexGroupingPattern = ref("");
|
||||||
const elasticsearchConnectionPorts = ref<Record<ElasticsearchConnectionMode, number>>({
|
const elasticsearchConnectionPorts = ref<Record<ElasticsearchConnectionMode, number>>({
|
||||||
direct: 9200,
|
direct: 9200,
|
||||||
kibana: 5601,
|
kibana: 5601,
|
||||||
@@ -328,6 +329,7 @@ function resetElasticsearchProxyFields(externalConfig?: unknown) {
|
|||||||
elasticsearchConnectionMode.value = mode;
|
elasticsearchConnectionMode.value = mode;
|
||||||
elasticsearchKibanaBasePath.value = elasticsearchKibanaBasePathFromConfig(externalConfig);
|
elasticsearchKibanaBasePath.value = elasticsearchKibanaBasePathFromConfig(externalConfig);
|
||||||
elasticsearchConnectivityCheckPath.value = elasticsearchConnectivityCheckPathFromConfig(externalConfig);
|
elasticsearchConnectivityCheckPath.value = elasticsearchConnectivityCheckPathFromConfig(externalConfig);
|
||||||
|
elasticsearchIndexGroupingPattern.value = elasticsearchIndexGroupingPatternFromConfig(externalConfig);
|
||||||
elasticsearchConnectionPorts.value = {
|
elasticsearchConnectionPorts.value = {
|
||||||
direct: mode === "direct" ? form.value.port : 9200,
|
direct: mode === "direct" ? form.value.port : 9200,
|
||||||
kibana: mode === "kibana" ? form.value.port : 5601,
|
kibana: mode === "kibana" ? form.value.port : 5601,
|
||||||
@@ -3821,7 +3823,7 @@ function connectionConfigForSubmit(id: string, generatedName = ""): ConnectionCo
|
|||||||
config.database = "metrics";
|
config.database = "metrics";
|
||||||
config.username = config.username.trim();
|
config.username = config.username.trim();
|
||||||
} else if (config.db_type === "elasticsearch") {
|
} else if (config.db_type === "elasticsearch") {
|
||||||
config.external_config = buildElasticsearchExternalConfig(elasticsearchConnectionMode.value, elasticsearchKibanaBasePath.value, elasticsearchConnectivityCheckPath.value);
|
config.external_config = buildElasticsearchExternalConfig(elasticsearchConnectionMode.value, elasticsearchKibanaBasePath.value, elasticsearchConnectivityCheckPath.value, elasticsearchIndexGroupingPattern.value);
|
||||||
} else if (config.db_type === "sqlserver") {
|
} else if (config.db_type === "sqlserver") {
|
||||||
config.external_config = sqlServerPortExplicitFromConfig(config) ? { portExplicit: true } : undefined;
|
config.external_config = sqlServerPortExplicitFromConfig(config) ? { portExplicit: true } : undefined;
|
||||||
} else if (supportsGaussdbIdentifierQuoteStyle(config)) {
|
} else if (supportsGaussdbIdentifierQuoteStyle(config)) {
|
||||||
@@ -6855,6 +6857,14 @@ function openExternalUrl(url: string) {
|
|||||||
<Input v-model="elasticsearchConnectivityCheckPath" class="col-span-3" :placeholder="t('connection.elasticsearchConnectivityCheckPathPlaceholder')" @input="resetTestState" />
|
<Input v-model="elasticsearchConnectivityCheckPath" class="col-span-3" :placeholder="t('connection.elasticsearchConnectivityCheckPathPlaceholder')" @input="resetTestState" />
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div v-if="form.db_type === 'elasticsearch'" class="grid grid-cols-4 items-center gap-4">
|
||||||
|
<Label :class="connectionLabelSmallClass">{{ t("connection.elasticsearchIndexGroupingPattern") }}</Label>
|
||||||
|
<div class="col-span-3 space-y-1">
|
||||||
|
<Input v-model="elasticsearchIndexGroupingPattern" :placeholder="t('connection.elasticsearchIndexGroupingPatternPlaceholder')" @input="resetTestState" />
|
||||||
|
<p class="text-xs text-muted-foreground">{{ t("connection.elasticsearchIndexGroupingPatternHint") }}</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<div v-if="form.driver_profile === 'gbase8s'" class="grid grid-cols-4 items-center gap-4">
|
<div v-if="form.driver_profile === 'gbase8s'" class="grid grid-cols-4 items-center gap-4">
|
||||||
<Label :class="connectionLabelSmallClass">{{ t("connection.gbaseServer") }}</Label>
|
<Label :class="connectionLabelSmallClass">{{ t("connection.gbaseServer") }}</Label>
|
||||||
<div class="col-span-3 space-y-1">
|
<div class="col-span-3 space-y-1">
|
||||||
|
|||||||
@@ -417,6 +417,9 @@ export default {
|
|||||||
elasticsearchKibanaBasePath: "Base Path",
|
elasticsearchKibanaBasePath: "Base Path",
|
||||||
elasticsearchConnectivityCheckPath: "Connectivity Path",
|
elasticsearchConnectivityCheckPath: "Connectivity Path",
|
||||||
elasticsearchConnectivityCheckPathPlaceholder: "/ or /my-index/_search",
|
elasticsearchConnectivityCheckPathPlaceholder: "/ or /my-index/_search",
|
||||||
|
elasticsearchIndexGroupingPattern: "Index grouping regex",
|
||||||
|
elasticsearchIndexGroupingPatternPlaceholder: "off by default; enter a regex to enable",
|
||||||
|
elasticsearchIndexGroupingPatternHint: "Off by default (naming schemes vary). Enter a regular expression to enable: matched rolling or time-series indices are aggregated into one queryable wildcard pattern, hiding per-day detail.",
|
||||||
version: "Version",
|
version: "Version",
|
||||||
victoriametricsApiPath: "API Path",
|
victoriametricsApiPath: "API Path",
|
||||||
victoriametricsApiPathHint: "Use /prometheus for single-node deployments, or /select/tenantID/prometheus for clusters.",
|
victoriametricsApiPathHint: "Use /prometheus for single-node deployments, or /select/tenantID/prometheus for clusters.",
|
||||||
|
|||||||
@@ -921,6 +921,9 @@ export default withEnglishFallback({
|
|||||||
elasticsearchKibanaBasePath: "ruta base",
|
elasticsearchKibanaBasePath: "ruta base",
|
||||||
elasticsearchConnectivityCheckPath: "Ruta de conectividad",
|
elasticsearchConnectivityCheckPath: "Ruta de conectividad",
|
||||||
elasticsearchConnectivityCheckPathPlaceholder: "/ o /my-index/_search",
|
elasticsearchConnectivityCheckPathPlaceholder: "/ o /my-index/_search",
|
||||||
|
elasticsearchIndexGroupingPattern: "Index grouping regex",
|
||||||
|
elasticsearchIndexGroupingPatternPlaceholder: "off by default; enter a regex to enable",
|
||||||
|
elasticsearchIndexGroupingPatternHint: "Off by default (naming schemes vary). Enter a regular expression to enable: matched rolling or time-series indices are aggregated into one queryable wildcard pattern, hiding per-day detail.",
|
||||||
mqSystemRocketMq: "Apache RocketMQ",
|
mqSystemRocketMq: "Apache RocketMQ",
|
||||||
rocketmqNamesrvAddr: "Dirección de NameServer",
|
rocketmqNamesrvAddr: "Dirección de NameServer",
|
||||||
rocketmqNamesrvAddrPlaceholder: "127.0.0.1:9876",
|
rocketmqNamesrvAddrPlaceholder: "127.0.0.1:9876",
|
||||||
|
|||||||
@@ -919,6 +919,9 @@ export default withEnglishFallback({
|
|||||||
elasticsearchKibanaBasePath: "Percorso base",
|
elasticsearchKibanaBasePath: "Percorso base",
|
||||||
elasticsearchConnectivityCheckPath: "Percorso di connettività",
|
elasticsearchConnectivityCheckPath: "Percorso di connettività",
|
||||||
elasticsearchConnectivityCheckPathPlaceholder: "/ o /my-index/_search",
|
elasticsearchConnectivityCheckPathPlaceholder: "/ o /my-index/_search",
|
||||||
|
elasticsearchIndexGroupingPattern: "Index grouping regex",
|
||||||
|
elasticsearchIndexGroupingPatternPlaceholder: "off by default; enter a regex to enable",
|
||||||
|
elasticsearchIndexGroupingPatternHint: "Off by default (naming schemes vary). Enter a regular expression to enable: matched rolling or time-series indices are aggregated into one queryable wildcard pattern, hiding per-day detail.",
|
||||||
mqSystemRocketMq: "Apache RocketMQ",
|
mqSystemRocketMq: "Apache RocketMQ",
|
||||||
rocketmqNamesrvAddr: "Indirizzo NameServer",
|
rocketmqNamesrvAddr: "Indirizzo NameServer",
|
||||||
rocketmqNamesrvAddrPlaceholder: "127.0.0.1:9876",
|
rocketmqNamesrvAddrPlaceholder: "127.0.0.1:9876",
|
||||||
|
|||||||
@@ -919,6 +919,9 @@ export default withEnglishFallback({
|
|||||||
elasticsearchKibanaBasePath: "ベースパス",
|
elasticsearchKibanaBasePath: "ベースパス",
|
||||||
elasticsearchConnectivityCheckPath: "接続確認パス",
|
elasticsearchConnectivityCheckPath: "接続確認パス",
|
||||||
elasticsearchConnectivityCheckPathPlaceholder: "/ または /my-index/_search",
|
elasticsearchConnectivityCheckPathPlaceholder: "/ または /my-index/_search",
|
||||||
|
elasticsearchIndexGroupingPattern: "Index grouping regex",
|
||||||
|
elasticsearchIndexGroupingPatternPlaceholder: "off by default; enter a regex to enable",
|
||||||
|
elasticsearchIndexGroupingPatternHint: "Off by default (naming schemes vary). Enter a regular expression to enable: matched rolling or time-series indices are aggregated into one queryable wildcard pattern, hiding per-day detail.",
|
||||||
mqSystemRocketMq: "Apache RocketMQ",
|
mqSystemRocketMq: "Apache RocketMQ",
|
||||||
rocketmqNamesrvAddr: "NameServer アドレス",
|
rocketmqNamesrvAddr: "NameServer アドレス",
|
||||||
rocketmqNamesrvAddrPlaceholder: "127.0.0.1:9876",
|
rocketmqNamesrvAddrPlaceholder: "127.0.0.1:9876",
|
||||||
|
|||||||
@@ -411,6 +411,9 @@ export default withEnglishFallback({
|
|||||||
elasticsearchKibanaBasePath: "기본 경로",
|
elasticsearchKibanaBasePath: "기본 경로",
|
||||||
elasticsearchConnectivityCheckPath: "연결 확인 경로",
|
elasticsearchConnectivityCheckPath: "연결 확인 경로",
|
||||||
elasticsearchConnectivityCheckPathPlaceholder: "/ 또는 /my-index/_search",
|
elasticsearchConnectivityCheckPathPlaceholder: "/ 또는 /my-index/_search",
|
||||||
|
elasticsearchIndexGroupingPattern: "Index grouping regex",
|
||||||
|
elasticsearchIndexGroupingPatternPlaceholder: "off by default; enter a regex to enable",
|
||||||
|
elasticsearchIndexGroupingPatternHint: "Off by default (naming schemes vary). Enter a regular expression to enable: matched rolling or time-series indices are aggregated into one queryable wildcard pattern, hiding per-day detail.",
|
||||||
version: "버전",
|
version: "버전",
|
||||||
victoriametricsApiPath: "API 경로",
|
victoriametricsApiPath: "API 경로",
|
||||||
victoriametricsApiPathHint: "단일 노드 배포에는 /prometheus를, 클러스터에는 /select/tenantID/prometheus를 사용하세요.",
|
victoriametricsApiPathHint: "단일 노드 배포에는 /prometheus를, 클러스터에는 /select/tenantID/prometheus를 사용하세요.",
|
||||||
|
|||||||
@@ -920,6 +920,9 @@ export default withEnglishFallback({
|
|||||||
elasticsearchKibanaBasePath: "Caminho Base",
|
elasticsearchKibanaBasePath: "Caminho Base",
|
||||||
elasticsearchConnectivityCheckPath: "Caminho de conectividade",
|
elasticsearchConnectivityCheckPath: "Caminho de conectividade",
|
||||||
elasticsearchConnectivityCheckPathPlaceholder: "/ ou /my-index/_search",
|
elasticsearchConnectivityCheckPathPlaceholder: "/ ou /my-index/_search",
|
||||||
|
elasticsearchIndexGroupingPattern: "Index grouping regex",
|
||||||
|
elasticsearchIndexGroupingPatternPlaceholder: "off by default; enter a regex to enable",
|
||||||
|
elasticsearchIndexGroupingPatternHint: "Off by default (naming schemes vary). Enter a regular expression to enable: matched rolling or time-series indices are aggregated into one queryable wildcard pattern, hiding per-day detail.",
|
||||||
mqSystemRocketMq: "Apache RocketMQ",
|
mqSystemRocketMq: "Apache RocketMQ",
|
||||||
rocketmqNamesrvAddr: "Endereço do NameServer",
|
rocketmqNamesrvAddr: "Endereço do NameServer",
|
||||||
rocketmqNamesrvAddrPlaceholder: "127.0.0.1:9876",
|
rocketmqNamesrvAddrPlaceholder: "127.0.0.1:9876",
|
||||||
|
|||||||
@@ -343,6 +343,9 @@ export default withEnglishFallback({
|
|||||||
elasticsearchKibanaBasePath: "基础路径",
|
elasticsearchKibanaBasePath: "基础路径",
|
||||||
elasticsearchConnectivityCheckPath: "连通性检查路径",
|
elasticsearchConnectivityCheckPath: "连通性检查路径",
|
||||||
elasticsearchConnectivityCheckPathPlaceholder: "/ 或 /my-index/_search",
|
elasticsearchConnectivityCheckPathPlaceholder: "/ 或 /my-index/_search",
|
||||||
|
elasticsearchIndexGroupingPattern: "索引聚合正则",
|
||||||
|
elasticsearchIndexGroupingPatternPlaceholder: "默认关闭,填正则表达式开启",
|
||||||
|
elasticsearchIndexGroupingPatternHint: "默认关闭(命名格式因环境而异)。填入正则表达式即可开启:将匹配到的滚动或时间序列索引聚合成一个可查询的通配 pattern,隐藏每日明细。",
|
||||||
version: "版本",
|
version: "版本",
|
||||||
victoriametricsApiPath: "API 路径",
|
victoriametricsApiPath: "API 路径",
|
||||||
victoriametricsApiPathHint: "单机版通常为 /prometheus;集群版可填写 /select/租户ID/prometheus。",
|
victoriametricsApiPathHint: "单机版通常为 /prometheus;集群版可填写 /select/租户ID/prometheus。",
|
||||||
|
|||||||
@@ -919,6 +919,9 @@ export default withEnglishFallback({
|
|||||||
elasticsearchKibanaBasePath: "基礎路徑",
|
elasticsearchKibanaBasePath: "基礎路徑",
|
||||||
elasticsearchConnectivityCheckPath: "連通性檢查路徑",
|
elasticsearchConnectivityCheckPath: "連通性檢查路徑",
|
||||||
elasticsearchConnectivityCheckPathPlaceholder: "/ 或 /my-index/_search",
|
elasticsearchConnectivityCheckPathPlaceholder: "/ 或 /my-index/_search",
|
||||||
|
elasticsearchIndexGroupingPattern: "索引聚合正則",
|
||||||
|
elasticsearchIndexGroupingPatternPlaceholder: "預設關閉,填正則表達式開啟",
|
||||||
|
elasticsearchIndexGroupingPatternHint: "預設關閉(命名格式因環境而異)。填入正則表達式即可開啟:將比對到的滾動或時間序列索引聚合成一個可查詢的通配 pattern,隱藏每日明細。",
|
||||||
mqSystemRocketMq: "Apache RocketMQ",
|
mqSystemRocketMq: "Apache RocketMQ",
|
||||||
rocketmqNamesrvAddr: "NameServer 地址",
|
rocketmqNamesrvAddr: "NameServer 地址",
|
||||||
rocketmqNamesrvAddrPlaceholder: "127.0.0.1:9876",
|
rocketmqNamesrvAddrPlaceholder: "127.0.0.1:9876",
|
||||||
|
|||||||
@@ -5,6 +5,8 @@ export interface ElasticsearchExternalConfig {
|
|||||||
kibanaBasePath?: string;
|
kibanaBasePath?: string;
|
||||||
/** GET path for connect/test/health. Empty means GET /. */
|
/** GET path for connect/test/health. Empty means GET /. */
|
||||||
connectivityCheckPath?: string;
|
connectivityCheckPath?: string;
|
||||||
|
/** Regex collapsing rolling/time-series index suffixes into a `*` pattern. Empty → default; "off" → no grouping. */
|
||||||
|
indexGroupingPattern?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
function externalConfigRecord(value: unknown): Record<string, unknown> {
|
function externalConfigRecord(value: unknown): Record<string, unknown> {
|
||||||
@@ -44,14 +46,26 @@ export function elasticsearchConnectivityCheckPathFromConfig(value: unknown): st
|
|||||||
return typeof path === "string" ? normalizeElasticsearchConnectivityCheckPath(path) : "";
|
return typeof path === "string" ? normalizeElasticsearchConnectivityCheckPath(path) : "";
|
||||||
}
|
}
|
||||||
|
|
||||||
export function buildElasticsearchExternalConfig(mode: ElasticsearchConnectionMode, kibanaBasePath: string, connectivityCheckPath = ""): ElasticsearchExternalConfig | undefined {
|
export function elasticsearchIndexGroupingPatternFromConfig(value: unknown): string {
|
||||||
|
const config = externalConfigRecord(value);
|
||||||
|
const pattern = config.indexGroupingPattern;
|
||||||
|
return typeof pattern === "string" ? pattern.trim() : "";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildElasticsearchExternalConfig(mode: ElasticsearchConnectionMode, kibanaBasePath: string, connectivityCheckPath = "", indexGroupingPattern = ""): ElasticsearchExternalConfig | undefined {
|
||||||
const checkPath = normalizeElasticsearchConnectivityCheckPath(connectivityCheckPath);
|
const checkPath = normalizeElasticsearchConnectivityCheckPath(connectivityCheckPath);
|
||||||
|
const grouping = indexGroupingPattern.trim();
|
||||||
if (mode !== "kibana") {
|
if (mode !== "kibana") {
|
||||||
return checkPath ? { connectivityCheckPath: checkPath } : undefined;
|
if (!checkPath && !grouping) return undefined;
|
||||||
|
const config: ElasticsearchExternalConfig = {};
|
||||||
|
if (checkPath) config.connectivityCheckPath = checkPath;
|
||||||
|
if (grouping) config.indexGroupingPattern = grouping;
|
||||||
|
return config;
|
||||||
}
|
}
|
||||||
const normalizedPath = normalizeKibanaBasePath(kibanaBasePath);
|
const normalizedPath = normalizeKibanaBasePath(kibanaBasePath);
|
||||||
const config: ElasticsearchExternalConfig = { mode: "kibana" };
|
const config: ElasticsearchExternalConfig = { mode: "kibana" };
|
||||||
if (normalizedPath) config.kibanaBasePath = normalizedPath;
|
if (normalizedPath) config.kibanaBasePath = normalizedPath;
|
||||||
if (checkPath) config.connectivityCheckPath = checkPath;
|
if (checkPath) config.connectivityCheckPath = checkPath;
|
||||||
|
if (grouping) config.indexGroupingPattern = grouping;
|
||||||
return config;
|
return config;
|
||||||
}
|
}
|
||||||
@@ -1,8 +1,9 @@
|
|||||||
use percent_encoding::{percent_decode_str, utf8_percent_encode, AsciiSet, CONTROLS};
|
use percent_encoding::{percent_decode_str, utf8_percent_encode, AsciiSet, CONTROLS};
|
||||||
|
use regex::Regex;
|
||||||
use reqwest::{Client as HttpClient, Method, StatusCode};
|
use reqwest::{Client as HttpClient, Method, StatusCode};
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use serde_json::Value;
|
use serde_json::Value;
|
||||||
use std::collections::{HashMap, HashSet};
|
use std::collections::{BTreeMap, HashMap, HashSet};
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
@@ -50,6 +51,9 @@ pub struct EsClient {
|
|||||||
transport_mode: ElasticsearchTransportMode,
|
transport_mode: ElasticsearchTransportMode,
|
||||||
/// GET path used for connect / health / test (default "/").
|
/// GET path used for connect / health / test (default "/").
|
||||||
connectivity_check_path: String,
|
connectivity_check_path: String,
|
||||||
|
/// 正则:把易变的时间/滚动后缀折叠成 `*`,将同一前缀的滚动索引聚合成一个
|
||||||
|
/// pattern 节点。`None` 表示关闭聚合,展示原始索引名。
|
||||||
|
index_grouping: Option<Regex>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl EsClient {
|
impl EsClient {
|
||||||
@@ -68,6 +72,7 @@ impl EsClient {
|
|||||||
timeout,
|
timeout,
|
||||||
ElasticsearchTransportMode::Direct,
|
ElasticsearchTransportMode::Direct,
|
||||||
"/".to_string(),
|
"/".to_string(),
|
||||||
|
None,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -79,6 +84,7 @@ impl EsClient {
|
|||||||
timeout: Duration,
|
timeout: Duration,
|
||||||
transport_mode: ElasticsearchTransportMode,
|
transport_mode: ElasticsearchTransportMode,
|
||||||
connectivity_check_path: String,
|
connectivity_check_path: String,
|
||||||
|
index_grouping: Option<Regex>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
let base_url = url.trim_end_matches('/').to_string();
|
let base_url = url.trim_end_matches('/').to_string();
|
||||||
let auth = match (username, password) {
|
let auth = match (username, password) {
|
||||||
@@ -88,7 +94,7 @@ impl EsClient {
|
|||||||
let builder = http_client_builder(timeout).danger_accept_invalid_certs(accept_invalid_certs);
|
let builder = http_client_builder(timeout).danger_accept_invalid_certs(accept_invalid_certs);
|
||||||
let http = builder.build().unwrap_or_else(|_| HttpClient::new());
|
let http = builder.build().unwrap_or_else(|_| HttpClient::new());
|
||||||
let fallback_base_urls = elasticsearch_base_url_fallbacks(&base_url);
|
let fallback_base_urls = elasticsearch_base_url_fallbacks(&base_url);
|
||||||
Self { http, base_url, fallback_base_urls, auth, transport_mode, connectivity_check_path }
|
Self { http, base_url, fallback_base_urls, auth, transport_mode, connectivity_check_path, index_grouping }
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn from_config(
|
pub fn from_config(
|
||||||
@@ -108,6 +114,7 @@ impl EsClient {
|
|||||||
};
|
};
|
||||||
let base_url = format!("{}{}", url.trim_end_matches('/'), kibana_base_path.as_deref().unwrap_or(""));
|
let base_url = format!("{}{}", url.trim_end_matches('/'), kibana_base_path.as_deref().unwrap_or(""));
|
||||||
let connectivity_check_path = elasticsearch_connectivity_check_path(external_config);
|
let connectivity_check_path = elasticsearch_connectivity_check_path(external_config);
|
||||||
|
let index_grouping = elasticsearch_index_grouping(external_config);
|
||||||
Self::new_with_mode(
|
Self::new_with_mode(
|
||||||
&base_url,
|
&base_url,
|
||||||
username,
|
username,
|
||||||
@@ -116,6 +123,7 @@ impl EsClient {
|
|||||||
timeout,
|
timeout,
|
||||||
transport_mode,
|
transport_mode,
|
||||||
connectivity_check_path,
|
connectivity_check_path,
|
||||||
|
index_grouping,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -180,6 +188,7 @@ impl Clone for EsClient {
|
|||||||
auth: self.auth.clone(),
|
auth: self.auth.clone(),
|
||||||
transport_mode: self.transport_mode,
|
transport_mode: self.transport_mode,
|
||||||
connectivity_check_path: self.connectivity_check_path.clone(),
|
connectivity_check_path: self.connectivity_check_path.clone(),
|
||||||
|
index_grouping: self.index_grouping.clone(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -227,6 +236,46 @@ pub fn elasticsearch_connectivity_check_path(external_config: Option<&Value>) ->
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// 解析连接配置里的索引聚合正则。**默认关闭**(命名格式因环境而异,不做全局假设)。
|
||||||
|
/// - 缺省/空/`off`/`none`/`false` → 关闭聚合,展示原始索引名。
|
||||||
|
/// - 其它 → 作为自定义正则;编译失败同样视为关闭,避免意外折叠。
|
||||||
|
///
|
||||||
|
/// 语义:用 `${1}*` 模板替换匹配区间——正则带捕获组 1 时保留其内容做前缀,
|
||||||
|
/// 不带捕获组时即把匹配到的“易变尾巴”替换成 `*`。
|
||||||
|
pub fn elasticsearch_index_grouping(external_config: Option<&Value>) -> Option<Regex> {
|
||||||
|
let raw = external_config
|
||||||
|
.and_then(Value::as_object)
|
||||||
|
.and_then(|config| config.get("indexGroupingPattern"))
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.unwrap_or("")
|
||||||
|
.trim();
|
||||||
|
if raw.is_empty()
|
||||||
|
|| raw.eq_ignore_ascii_case("off")
|
||||||
|
|| raw.eq_ignore_ascii_case("none")
|
||||||
|
|| raw.eq_ignore_ascii_case("false")
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Regex::new(raw).ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 用分组正则把索引名聚合成 pattern。`None` 表示关闭聚合,原样返回。
|
||||||
|
fn group_index_names(names: Vec<String>, pattern: Option<&Regex>) -> Vec<String> {
|
||||||
|
let Some(re) = pattern else {
|
||||||
|
return names;
|
||||||
|
};
|
||||||
|
let mut buckets: BTreeMap<String, Vec<String>> = BTreeMap::new();
|
||||||
|
for name in names {
|
||||||
|
// `${1}*`:有捕获组则保留组1做前缀,无组则把匹配尾巴替换成 `*`。
|
||||||
|
let key = re.replace(&name, "${1}*").into_owned();
|
||||||
|
buckets.entry(key).or_default().push(name);
|
||||||
|
}
|
||||||
|
let mut out: Vec<String> = buckets.into_keys().collect();
|
||||||
|
out.sort();
|
||||||
|
out.dedup();
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn test_connection(client: &mut EsClient, timeout: Duration) -> Result<(), String> {
|
pub async fn test_connection(client: &mut EsClient, timeout: Duration) -> Result<(), String> {
|
||||||
let mut errors = Vec::new();
|
let mut errors = Vec::new();
|
||||||
let urls = std::iter::once(client.base_url.clone()).chain(client.fallback_base_urls.clone());
|
let urls = std::iter::once(client.base_url.clone()).chain(client.fallback_base_urls.clone());
|
||||||
@@ -375,20 +424,92 @@ struct CatIndex {
|
|||||||
index: String,
|
index: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct ResolveIndexResponse {
|
||||||
|
#[serde(default)]
|
||||||
|
indices: Vec<ResolveNamed>,
|
||||||
|
#[serde(default)]
|
||||||
|
data_streams: Vec<ResolveNamed>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct ResolveNamed {
|
||||||
|
name: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 去掉 ES 内部索引(以 `.` 开头),排序并去重后返回可见索引名。
|
||||||
|
fn normalize_index_names(names: impl Iterator<Item = String>) -> Vec<String> {
|
||||||
|
let mut names: Vec<String> = names.filter(|name| !name.starts_with('.')).collect();
|
||||||
|
names.sort();
|
||||||
|
names.dedup();
|
||||||
|
names
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn list_indices(client: &EsClient) -> Result<Vec<String>, String> {
|
pub async fn list_indices(client: &EsClient) -> Result<Vec<String>, String> {
|
||||||
|
let names = list_raw_index_names(client).await?;
|
||||||
|
Ok(group_index_names(names, client.index_grouping.as_ref()))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list_raw_index_names(client: &EsClient) -> Result<Vec<String>, String> {
|
||||||
|
// 主路径 `_cat/indices` 需要集群级 `monitor` 权限。仅有索引级权限的账号
|
||||||
|
// (例如日志采集用户)会在这里拿到 401/403,此时降级到索引级元数据端点。
|
||||||
let resp = client
|
let resp = client
|
||||||
.get("/_cat/indices?format=json&h=index")
|
.get("/_cat/indices?format=json&h=index")
|
||||||
.send()
|
.send()
|
||||||
.await
|
.await
|
||||||
.map_err(|e| format!("Elasticsearch request failed: {e}"))?;
|
.map_err(|e| format!("Elasticsearch request failed: {e}"))?;
|
||||||
|
let status = client.response_status(&resp);
|
||||||
|
if status.is_success() {
|
||||||
|
let indices: Vec<CatIndex> = resp.json().await.map_err(|e| format!("Elasticsearch parse error: {e}"))?;
|
||||||
|
return Ok(normalize_index_names(indices.into_iter().map(|i| i.index)));
|
||||||
|
}
|
||||||
|
if status == StatusCode::FORBIDDEN || status == StatusCode::UNAUTHORIZED {
|
||||||
|
return list_indices_via_metadata(client).await;
|
||||||
|
}
|
||||||
|
let body = resp.text().await.unwrap_or_default();
|
||||||
|
Err(format!("Elasticsearch error: {body}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 集群 `monitor` 不可用时的降级:`_resolve/index` 与 `_alias` 属于
|
||||||
|
/// `indices:admin/*` 动作,`view_index_metadata`/`read` 索引权限即可访问,
|
||||||
|
/// 且 ES 安全层会把结果过滤为当前账号可见的索引。
|
||||||
|
async fn list_indices_via_metadata(client: &EsClient) -> Result<Vec<String>, String> {
|
||||||
|
// 优先 `_resolve/index`:同时覆盖普通索引与数据流(data stream)。
|
||||||
|
if let Some(names) = resolve_index_names(client).await? {
|
||||||
|
return Ok(names);
|
||||||
|
}
|
||||||
|
// 再退回 `_alias`:以对象 key 形式返回具体索引名。
|
||||||
|
alias_index_names(client).await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 通过 `GET /_resolve/index/*` 列举索引。该端点缺权限时返回 `Ok(None)`,
|
||||||
|
/// 以便继续尝试 `_alias`;其它错误如实上抛。
|
||||||
|
async fn resolve_index_names(client: &EsClient) -> Result<Option<Vec<String>>, String> {
|
||||||
|
let resp =
|
||||||
|
client.get("/_resolve/index/*").send().await.map_err(|e| format!("Elasticsearch request failed: {e}"))?;
|
||||||
|
let status = client.response_status(&resp);
|
||||||
|
if status == StatusCode::FORBIDDEN || status == StatusCode::UNAUTHORIZED {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
if !status.is_success() {
|
||||||
|
let body = resp.text().await.unwrap_or_default();
|
||||||
|
return Err(format!("Elasticsearch error: {body}"));
|
||||||
|
}
|
||||||
|
let body: ResolveIndexResponse = resp.json().await.map_err(|e| format!("Elasticsearch parse error: {e}"))?;
|
||||||
|
let names = body.indices.into_iter().chain(body.data_streams).map(|item| item.name);
|
||||||
|
Ok(Some(normalize_index_names(names)))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// 通过 `GET /_alias` 列举索引(对象 key 即索引名)。
|
||||||
|
async fn alias_index_names(client: &EsClient) -> Result<Vec<String>, String> {
|
||||||
|
let resp = client.get("/_alias").send().await.map_err(|e| format!("Elasticsearch request failed: {e}"))?;
|
||||||
if !client.response_status(&resp).is_success() {
|
if !client.response_status(&resp).is_success() {
|
||||||
let body = resp.text().await.unwrap_or_default();
|
let body = resp.text().await.unwrap_or_default();
|
||||||
return Err(format!("Elasticsearch error: {body}"));
|
return Err(format!("Elasticsearch error: {body}"));
|
||||||
}
|
}
|
||||||
let indices: Vec<CatIndex> = resp.json().await.map_err(|e| format!("Elasticsearch parse error: {e}"))?;
|
let body: serde_json::Map<String, Value> =
|
||||||
let mut names: Vec<String> = indices.into_iter().filter(|i| !i.index.starts_with('.')).map(|i| i.index).collect();
|
resp.json().await.map_err(|e| format!("Elasticsearch parse error: {e}"))?;
|
||||||
names.sort();
|
Ok(normalize_index_names(body.into_iter().map(|(name, _)| name)))
|
||||||
Ok(names)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn get_columns(client: &EsClient, index: &str) -> Result<Vec<crate::db::ColumnInfo>, String> {
|
pub async fn get_columns(client: &EsClient, index: &str) -> Result<Vec<crate::db::ColumnInfo>, String> {
|
||||||
@@ -2095,7 +2216,8 @@ fn parse_aggregations(aggs: &serde_json::Map<String, serde_json::Value>) -> (Vec
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::{
|
use super::{
|
||||||
build_count_documents_body, build_find_documents_body, elasticsearch_accept_invalid_certs,
|
build_count_documents_body, build_find_documents_body, elasticsearch_accept_invalid_certs,
|
||||||
elasticsearch_base_url_fallbacks, redact_elasticsearch_url, EsClient, SearchResponse,
|
elasticsearch_base_url_fallbacks, elasticsearch_index_grouping, group_index_names, normalize_index_names,
|
||||||
|
redact_elasticsearch_url, EsClient, SearchResponse,
|
||||||
};
|
};
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
@@ -2142,6 +2264,77 @@ mod tests {
|
|||||||
assert_eq!(request.body_kind, super::ElasticsearchRestBodyKind::Json);
|
assert_eq!(request.body_kind, super::ElasticsearchRestBodyKind::Json);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn normalize_index_names_filters_sorts_and_dedups() {
|
||||||
|
let names = normalize_index_names(
|
||||||
|
[".kibana", "ngx-log-2", "ngx-log-1", "ngx-log-1", ".security-7"].into_iter().map(String::from),
|
||||||
|
);
|
||||||
|
// 去掉点前缀内部索引、排序、去重。
|
||||||
|
assert_eq!(names, vec!["ngx-log-1".to_string(), "ngx-log-2".to_string()]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn group_index_names_off_by_default() {
|
||||||
|
// 缺省配置 → 关闭聚合,原样返回。
|
||||||
|
assert!(elasticsearch_index_grouping(None).is_none());
|
||||||
|
let raw = vec!["a-2026.08.04".to_string(), "a-2026.08.05".to_string()];
|
||||||
|
assert_eq!(group_index_names(raw.clone(), None), raw);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn group_index_names_tenant_level_with_capture_group() {
|
||||||
|
// 保留 `@<第一段>`(到第一个下划线),其后全部折叠成 `*`。用中性占位名。
|
||||||
|
let cfg = serde_json::json!({ "indexGroupingPattern": r"^([^@]*@[^_]+)_.*$" });
|
||||||
|
let re = elasticsearch_index_grouping(Some(&cfg));
|
||||||
|
let out = group_index_names(
|
||||||
|
vec![
|
||||||
|
"svc@alpha_r1-2026.08.06@0-000001".to_string(),
|
||||||
|
"svc@alpha_r1-2026.08.07@0-000001".to_string(),
|
||||||
|
"svc@beta_r1-2026.08.06@0-000001".to_string(),
|
||||||
|
"svc_err-2026.08.06@0-000001".to_string(), // 无 @ → 不匹配 → 原样
|
||||||
|
],
|
||||||
|
re.as_ref(),
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
out,
|
||||||
|
vec!["svc@alpha*".to_string(), "svc@beta*".to_string(), "svc_err-2026.08.06@0-000001".to_string(),]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn group_index_names_tail_strip_without_capture_group() {
|
||||||
|
// 无捕获组:按 ES 惯例剥掉“日期+滚动号”尾巴,`${1}` 为空即替换成 `*`。
|
||||||
|
let cfg = serde_json::json!({ "indexGroupingPattern": r"[-_.@]\d{4}[-_.]?\d{2}[-_.]?\d{2}.*$" });
|
||||||
|
let re = elasticsearch_index_grouping(Some(&cfg));
|
||||||
|
let out = group_index_names(
|
||||||
|
vec!["logs-2026.08.06".to_string(), "logs-2026.08.07".to_string(), "orders-2026.08.01".to_string()],
|
||||||
|
re.as_ref(),
|
||||||
|
);
|
||||||
|
assert_eq!(out, vec!["logs*".to_string(), "orders*".to_string()]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn group_index_names_mixed_tenant_and_plain_scheme() {
|
||||||
|
// 同一条正则同时处理:真租户(@后跟字母)折到 @租户;无租户的按名字剥日期尾巴;
|
||||||
|
// 普通非时间序列索引保持原样。区分点:真租户 @ 后是字母,滚动号 @ 后是数字。用中性名。
|
||||||
|
let cfg = serde_json::json!({ "indexGroupingPattern": r"^([^@]*@[a-zA-Z][a-zA-Z0-9]*|[^-@]*)[-_.@].*$" });
|
||||||
|
let re = elasticsearch_index_grouping(Some(&cfg));
|
||||||
|
let out = group_index_names(
|
||||||
|
vec![
|
||||||
|
"svc_err-2026.08.10@0-000001".to_string(), // 无租户 → svc_err*
|
||||||
|
"svc_err-2026.08.11@0-000001".to_string(),
|
||||||
|
"svc@alpha_r1-2026.08.10@0-000001".to_string(), // 带区域租户 → svc@alpha*
|
||||||
|
"svc@beta-2026.08.10@0-000001".to_string(), // 无区域租户 → svc@beta*
|
||||||
|
"catalog".to_string(), // 普通索引无尾巴 → 原样
|
||||||
|
],
|
||||||
|
re.as_ref(),
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
out,
|
||||||
|
vec!["catalog".to_string(), "svc@alpha*".to_string(), "svc@beta*".to_string(), "svc_err*".to_string(),]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn parses_rest_request_after_multiline_block_comment() {
|
fn parses_rest_request_after_multiline_block_comment() {
|
||||||
let request = super::parse_elasticsearch_rest_request(
|
let request = super::parse_elasticsearch_rest_request(
|
||||||
|
|||||||
Reference in new issue
Block a user