feat(nacos): add Nacos dashboard metrics
This commit is contained in:
41 files changed
+3268
-36
No files matched your search
@@ -626,6 +626,10 @@ mod tests {
|
||||
Err("unused".to_string())
|
||||
}
|
||||
|
||||
async fn get_dashboard(&self, _: NacosDashboardQuery) -> Result<NacosDashboardSnapshot, String> {
|
||||
Err("unused".to_string())
|
||||
}
|
||||
|
||||
async fn raw_request(&self, _: NacosRawRequest) -> Result<NacosRawResponse, String> {
|
||||
Err("unused".to_string())
|
||||
}
|
||||
|
||||
@@ -32,6 +32,15 @@ pub enum NacosVersionMode {
|
||||
V3,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum NacosMetricsMode {
|
||||
#[default]
|
||||
Auto,
|
||||
Disabled,
|
||||
Custom,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
|
||||
#[serde(tag = "kind", rename_all = "camelCase")]
|
||||
pub enum NacosRNacosConsoleAuth {
|
||||
@@ -73,6 +82,10 @@ pub struct NacosAdminConfig {
|
||||
pub auth: NacosAuthConfig,
|
||||
#[serde(default)]
|
||||
pub tls_skip_verify: bool,
|
||||
#[serde(default)]
|
||||
pub metrics_mode: NacosMetricsMode,
|
||||
#[serde(default, skip_serializing_if = "String::is_empty")]
|
||||
pub metrics_url: String,
|
||||
#[serde(default = "default_page_size")]
|
||||
pub page_size: u32,
|
||||
#[serde(skip)]
|
||||
@@ -113,6 +126,8 @@ impl NacosAdminConfig {
|
||||
NacosAuthConfig::UsernamePassword { username: cfg.username.clone(), password: cfg.password.clone() }
|
||||
},
|
||||
tls_skip_verify: false,
|
||||
metrics_mode: NacosMetricsMode::Auto,
|
||||
metrics_url: String::new(),
|
||||
page_size: default_page_size(),
|
||||
connect_override: None,
|
||||
}
|
||||
@@ -130,7 +145,19 @@ impl NacosAdminConfig {
|
||||
} else {
|
||||
self.display_server_addr = normalize_endpoint_url(&self.display_server_addr, "Nacos display address")?;
|
||||
}
|
||||
let context_path_is_explicit_root = self.context_path.trim() == "/";
|
||||
self.context_path = normalize_context_path(&self.context_path);
|
||||
// Nacos 3 separates the console (normally :8080) from the server-side
|
||||
// Admin API (normally :8848/nacos). Older DBX connection records did
|
||||
// not persist the default server context, so repair only explicit
|
||||
// Nacos 3 profiles here while preserving custom contexts.
|
||||
if self.context_path.is_empty()
|
||||
&& !context_path_is_explicit_root
|
||||
&& matches!(self.implementation, Some(NacosImplementation::Nacos))
|
||||
&& matches!(self.version_mode, Some(NacosVersionMode::V3))
|
||||
{
|
||||
self.context_path = "/nacos".to_string();
|
||||
}
|
||||
self.rnacos_console_addr = if self.rnacos_console_addr.trim().is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
@@ -144,6 +171,10 @@ impl NacosAdminConfig {
|
||||
return Err("r-nacos console username is empty".to_string());
|
||||
}
|
||||
}
|
||||
self.metrics_url = match self.metrics_mode {
|
||||
NacosMetricsMode::Custom => normalize_metrics_url(&self.metrics_url)?,
|
||||
NacosMetricsMode::Auto | NacosMetricsMode::Disabled => String::new(),
|
||||
};
|
||||
if self.page_size == 0 {
|
||||
self.page_size = default_page_size();
|
||||
}
|
||||
@@ -206,6 +237,24 @@ impl NacosAdminConfig {
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_metrics_url(value: &str) -> Result<String, String> {
|
||||
let value = value.trim();
|
||||
if value.is_empty() {
|
||||
return Err("Nacos Prometheus metrics URL is empty".to_string());
|
||||
}
|
||||
let url = reqwest::Url::parse(value).map_err(|e| format!("Nacos Prometheus metrics URL is invalid: {e}"))?;
|
||||
if !matches!(url.scheme(), "http" | "https") {
|
||||
return Err("Nacos Prometheus metrics URL must use http or https".to_string());
|
||||
}
|
||||
if !url.username().is_empty() || url.password().is_some() {
|
||||
return Err("Nacos Prometheus metrics URL must not contain embedded credentials".to_string());
|
||||
}
|
||||
if url.fragment().is_some() {
|
||||
return Err("Nacos Prometheus metrics URL must not contain a fragment".to_string());
|
||||
}
|
||||
Ok(url.to_string())
|
||||
}
|
||||
|
||||
fn normalize_endpoint_url(value: &str, label: &str) -> Result<String, String> {
|
||||
let mut url = reqwest::Url::parse(value.trim()).map_err(|e| format!("{label} is invalid: {e}"))?;
|
||||
if !url.username().is_empty() || url.password().is_some() {
|
||||
@@ -336,6 +385,41 @@ mod tests {
|
||||
assert!(err.contains("must not contain embedded credentials"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn defaults_prometheus_metrics_to_auto_and_validates_custom_urls() {
|
||||
let parsed = NacosAdminConfig::from_connection(&connection_with_external(serde_json::json!({
|
||||
"serverAddr": "http://127.0.0.1:8818"
|
||||
})))
|
||||
.unwrap();
|
||||
assert_eq!(parsed.metrics_mode, NacosMetricsMode::Auto);
|
||||
assert!(parsed.metrics_url.is_empty());
|
||||
|
||||
let parsed = NacosAdminConfig::from_connection(&connection_with_external(serde_json::json!({
|
||||
"serverAddr": "http://127.0.0.1:8848",
|
||||
"metricsMode": "custom",
|
||||
"metricsUrl": "http://127.0.0.1:8818/nacos/actuator/prometheus?node=a"
|
||||
})))
|
||||
.unwrap();
|
||||
assert_eq!(parsed.metrics_mode, NacosMetricsMode::Custom);
|
||||
assert_eq!(parsed.metrics_url, "http://127.0.0.1:8818/nacos/actuator/prometheus?node=a");
|
||||
|
||||
for metrics_url in [
|
||||
"",
|
||||
"file:///tmp/metrics",
|
||||
"http://user:secret@127.0.0.1:8818/metrics",
|
||||
"http://127.0.0.1:8818/metrics#private",
|
||||
"http://127.0.0.1:8818/metrics#",
|
||||
] {
|
||||
let error = NacosAdminConfig::from_connection(&connection_with_external(serde_json::json!({
|
||||
"serverAddr": "http://127.0.0.1:8848",
|
||||
"metricsMode": "custom",
|
||||
"metricsUrl": metrics_url
|
||||
})))
|
||||
.unwrap_err();
|
||||
assert!(error.contains("Prometheus metrics URL"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_external_context_path_defaults_to_root() {
|
||||
let cfg = connection_with_external(serde_json::json!({
|
||||
@@ -347,6 +431,33 @@ mod tests {
|
||||
assert_eq!(parsed.context_path, "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_nacos_v3_defaults_to_server_admin_context() {
|
||||
let cfg = connection_with_external(serde_json::json!({
|
||||
"implementation": "nacos",
|
||||
"versionMode": "v3",
|
||||
"serverAddr": "http://127.0.0.1:8848",
|
||||
"auth": { "kind": "none" }
|
||||
}));
|
||||
|
||||
let parsed = NacosAdminConfig::from_connection(&cfg).unwrap();
|
||||
assert_eq!(parsed.context_path, "/nacos");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_nacos_v3_root_context_is_preserved() {
|
||||
let cfg = connection_with_external(serde_json::json!({
|
||||
"implementation": "nacos",
|
||||
"versionMode": "v3",
|
||||
"serverAddr": "http://127.0.0.1:8848",
|
||||
"contextPath": "/",
|
||||
"auth": { "kind": "none" }
|
||||
}));
|
||||
|
||||
let parsed = NacosAdminConfig::from_connection(&cfg).unwrap();
|
||||
assert_eq!(parsed.context_path, "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn falls_back_to_connection_fields() {
|
||||
let mut cfg = connection_with_external(serde_json::Value::Null);
|
||||
|
||||
@@ -597,7 +597,7 @@ impl NacosOpenApiAdmin {
|
||||
fn api_path_allowed(&self, path: &str) -> bool {
|
||||
match self.cfg.version_mode.as_ref() {
|
||||
Some(NacosVersionMode::V2) => !path.starts_with("/v3/"),
|
||||
Some(NacosVersionMode::V3) => !path.starts_with("/v1/"),
|
||||
Some(NacosVersionMode::V3) => !path.starts_with("/v1/") && !path.starts_with("/v2/"),
|
||||
Some(NacosVersionMode::Auto) | None => true,
|
||||
}
|
||||
}
|
||||
@@ -834,6 +834,39 @@ impl NacosOpenApiAdmin {
|
||||
instances.retain(|instance| seen.insert((instance.ip.clone(), instance.port, instance.cluster_name.clone())));
|
||||
Ok(instances)
|
||||
}
|
||||
|
||||
async fn get_dashboard_nodes(&self) -> Result<Vec<NacosClusterNode>, String> {
|
||||
// r-nacos does not implement the official Nacos cluster-node Admin
|
||||
// APIs. An empty list represents this unsupported optional capability
|
||||
// without turning every dashboard refresh into a false warning.
|
||||
if self.is_explicit_rnacos() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
self.get_json_from_candidates(
|
||||
"load Nacos cluster nodes",
|
||||
vec![
|
||||
("/v3/admin/core/cluster/node/list", Vec::new()),
|
||||
("/v2/core/cluster/node/list", Vec::new()),
|
||||
("/v1/core/cluster/nodes", Vec::new()),
|
||||
("/v1/ns/operator/servers", Vec::new()),
|
||||
],
|
||||
)
|
||||
.await
|
||||
.map(parse_cluster_nodes)
|
||||
}
|
||||
|
||||
fn dashboard_warning(&self, error: String) -> String {
|
||||
if matches!(self.cfg.version_mode, Some(NacosVersionMode::V3))
|
||||
&& (error.contains("[contextPathMismatch]") || error.contains("[apiVersionMismatch]"))
|
||||
{
|
||||
return format!(
|
||||
"{error} Check the connection address: Nacos 3 dashboard APIs use the Server / Admin API endpoint \
|
||||
(normally http://host:8848/nacos), not the port 8080 Console."
|
||||
);
|
||||
}
|
||||
error
|
||||
}
|
||||
}
|
||||
|
||||
fn qualified_nacos_service_name(service_name: &str, group_name: Option<&str>) -> String {
|
||||
@@ -1570,6 +1603,102 @@ impl NacosAdmin for NacosOpenApiAdmin {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_dashboard(&self, query: NacosDashboardQuery) -> Result<NacosDashboardSnapshot, String> {
|
||||
let namespace = self.namespace(query.namespace.as_deref());
|
||||
let metrics_future = self.get_json_from_candidates(
|
||||
"load Nacos dashboard metrics",
|
||||
vec![
|
||||
("/v3/admin/ns/ops/metrics", vec![("onlyStatus".to_string(), "false".to_string())]),
|
||||
("/v2/ns/operator/metrics", vec![("onlyStatus".to_string(), "false".to_string())]),
|
||||
("/v1/ns/operator/metrics", Vec::new()),
|
||||
],
|
||||
);
|
||||
let nodes_future = self.get_dashboard_nodes();
|
||||
let namespaces_future = self.list_namespaces();
|
||||
let configs_future = self.list_configs(NacosConfigQuery {
|
||||
namespace: Some(namespace.clone()),
|
||||
group: None,
|
||||
data_id: None,
|
||||
app_name: None,
|
||||
search: None,
|
||||
page_no: Some(1),
|
||||
page_size: Some(1),
|
||||
});
|
||||
let services_future = self.list_services(NacosServiceQuery {
|
||||
namespace: Some(namespace.clone()),
|
||||
group_name: None,
|
||||
service_name: None,
|
||||
page_no: Some(1),
|
||||
page_size: Some(1),
|
||||
});
|
||||
let prometheus_future = crate::nacos::prometheus::scrape(&self.http, &self.cfg);
|
||||
|
||||
let (metrics_result, nodes_result, namespaces_result, configs_result, services_result, prometheus_result) = tokio::join!(
|
||||
metrics_future,
|
||||
nodes_future,
|
||||
namespaces_future,
|
||||
configs_future,
|
||||
services_future,
|
||||
prometheus_future
|
||||
);
|
||||
let mut warnings = Vec::new();
|
||||
|
||||
let mut metrics = match metrics_result {
|
||||
Ok(value) => Some(parse_dashboard_metrics(value)),
|
||||
Err(error) => {
|
||||
warnings.push(self.dashboard_warning(error));
|
||||
None
|
||||
}
|
||||
};
|
||||
let nodes = match nodes_result {
|
||||
Ok(value) => value,
|
||||
Err(error) => {
|
||||
warnings.push(self.dashboard_warning(error));
|
||||
Vec::new()
|
||||
}
|
||||
};
|
||||
let namespace_count = match namespaces_result {
|
||||
Ok(items) => Some(items.len() as u64),
|
||||
Err(error) => {
|
||||
warnings.push(error);
|
||||
None
|
||||
}
|
||||
};
|
||||
let config_count = match configs_result {
|
||||
Ok(result) => Some(result.total_count),
|
||||
Err(error) => {
|
||||
warnings.push(error);
|
||||
None
|
||||
}
|
||||
};
|
||||
let service_count = match services_result {
|
||||
Ok(result) => Some(result.total_count),
|
||||
Err(error) => {
|
||||
warnings.push(error);
|
||||
None
|
||||
}
|
||||
};
|
||||
let prometheus = match prometheus_result {
|
||||
Ok(value) => value,
|
||||
Err(error) => {
|
||||
warnings.push(error);
|
||||
None
|
||||
}
|
||||
};
|
||||
merge_prometheus_dashboard(&mut metrics, prometheus.as_ref());
|
||||
|
||||
Ok(NacosDashboardSnapshot {
|
||||
namespace,
|
||||
namespace_count,
|
||||
config_count,
|
||||
service_count,
|
||||
metrics,
|
||||
prometheus,
|
||||
nodes,
|
||||
warnings,
|
||||
})
|
||||
}
|
||||
|
||||
async fn raw_request(&self, req: NacosRawRequest) -> Result<NacosRawResponse, String> {
|
||||
validate_raw_api_path(&req.path)?;
|
||||
let method = reqwest::Method::from_bytes(req.method.to_ascii_uppercase().as_bytes())
|
||||
@@ -1660,6 +1789,107 @@ fn parse_namespaces(value: Value) -> Vec<NacosNamespaceInfo> {
|
||||
namespaces
|
||||
}
|
||||
|
||||
fn parse_dashboard_metrics(value: Value) -> NacosDashboardMetrics {
|
||||
let data = value.get("data").unwrap_or(&value);
|
||||
NacosDashboardMetrics {
|
||||
status: optional_string_field(data, &["status"])
|
||||
.or_else(|| data.as_str().map(str::to_string))
|
||||
.filter(|value| !value.trim().is_empty()),
|
||||
service_count: optional_u64_field(data, &["serviceCount"]),
|
||||
instance_count: optional_u64_field(data, &["instanceCount"]),
|
||||
subscribe_count: optional_u64_field(data, &["subscribeCount"]),
|
||||
raft_notify_task_count: optional_u64_field(data, &["raftNotifyTaskCount"]),
|
||||
responsible_service_count: optional_u64_field(data, &["responsibleServiceCount"]),
|
||||
responsible_instance_count: optional_u64_field(data, &["responsibleInstanceCount"]),
|
||||
client_count: optional_u64_field(data, &["clientCount"]),
|
||||
connection_based_client_count: optional_u64_field(data, &["connectionBasedClientCount"]),
|
||||
ephemeral_ip_port_client_count: optional_u64_field(data, &["ephemeralIpPortClientCount"]),
|
||||
persistent_ip_port_client_count: optional_u64_field(data, &["persistentIpPortClientCount"]),
|
||||
responsible_client_count: optional_u64_field(data, &["responsibleClientCount"]),
|
||||
cpu: optional_f64_field(data, &["cpu"]),
|
||||
load: optional_f64_field(data, &["load"]),
|
||||
mem: optional_f64_field(data, &["mem"]),
|
||||
}
|
||||
}
|
||||
|
||||
fn merge_prometheus_dashboard(
|
||||
metrics: &mut Option<NacosDashboardMetrics>,
|
||||
prometheus: Option<&NacosPrometheusSnapshot>,
|
||||
) {
|
||||
let Some(prometheus) = prometheus else {
|
||||
return;
|
||||
};
|
||||
let metrics = metrics.get_or_insert_with(NacosDashboardMetrics::default);
|
||||
if let Some(value) = finite_u64(prometheus.naming.instance_count) {
|
||||
metrics.instance_count = Some(value);
|
||||
}
|
||||
if let Some(value) = finite_u64(prometheus.naming.subscriber_count) {
|
||||
metrics.subscribe_count = Some(value);
|
||||
}
|
||||
if let Some(value) = finite_u64(prometheus.naming.connection_count) {
|
||||
metrics.client_count = Some(value);
|
||||
metrics.connection_based_client_count = Some(value);
|
||||
}
|
||||
if let Some(value) = prometheus.resource.cpu_ratio {
|
||||
metrics.cpu = Some(value);
|
||||
}
|
||||
if let Some(value) = prometheus.resource.memory_ratio {
|
||||
metrics.mem = Some(value);
|
||||
}
|
||||
if let Some(value) = prometheus.resource.load_1m {
|
||||
metrics.load = Some(value);
|
||||
}
|
||||
}
|
||||
|
||||
fn finite_u64(value: Option<f64>) -> Option<u64> {
|
||||
value.filter(|value| value.is_finite() && *value >= 0.0 && *value <= u64::MAX as f64).map(|value| value as u64)
|
||||
}
|
||||
|
||||
fn parse_cluster_nodes(value: Value) -> Vec<NacosClusterNode> {
|
||||
let data = value.get("data").unwrap_or(&value);
|
||||
let items = data
|
||||
.as_array()
|
||||
.cloned()
|
||||
.or_else(|| data.get("servers").and_then(Value::as_array).cloned())
|
||||
.or_else(|| data.get("members").and_then(Value::as_array).cloned())
|
||||
.or_else(|| data.get("nodes").and_then(Value::as_array).cloned())
|
||||
.or_else(|| data.get("pageItems").and_then(Value::as_array).cloned())
|
||||
.or_else(|| value.get("servers").and_then(Value::as_array).cloned())
|
||||
.unwrap_or_default();
|
||||
items
|
||||
.into_iter()
|
||||
.map(|item| {
|
||||
let ip = optional_string_field(&item, &["ip"]);
|
||||
let port = optional_u64_field(&item, &["port", "servePort"]).and_then(|port| u16::try_from(port).ok());
|
||||
let address = optional_string_field(&item, &["address", "key"])
|
||||
.filter(|value| !value.trim().is_empty())
|
||||
.unwrap_or_else(|| match (&ip, port) {
|
||||
(Some(ip), Some(port)) => format!("{ip}:{port}"),
|
||||
(Some(ip), None) => ip.clone(),
|
||||
_ => "-".to_string(),
|
||||
});
|
||||
let state = optional_string_field(&item, &["state", "status"]);
|
||||
let alive = optional_bool_field(&item, &["alive", "healthy"]).or_else(|| {
|
||||
state.as_ref().map(|state| matches!(state.to_ascii_uppercase().as_str(), "UP" | "ONLINE" | "HEALTHY"))
|
||||
});
|
||||
NacosClusterNode {
|
||||
address,
|
||||
ip,
|
||||
port,
|
||||
state,
|
||||
alive,
|
||||
site: optional_string_field(&item, &["site"]),
|
||||
weight: optional_f64_field(&item, &["weight", "adWeight"]),
|
||||
last_refresh_time: optional_string_field(&item, &["lastRefreshTime", "lastRefTimeStr", "lastRefTime"])
|
||||
.or_else(|| {
|
||||
item.get("extendInfo")
|
||||
.and_then(|extend_info| optional_string_field(extend_info, &["lastRefreshTime"]))
|
||||
}),
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn normalize_api_path(path: &str) -> String {
|
||||
let trimmed = path.trim();
|
||||
if trimmed.starts_with('/') {
|
||||
@@ -1759,10 +1989,34 @@ async fn error_for_status(resp: reqwest::Response, path: &str) -> Result<reqwest
|
||||
return Ok(resp);
|
||||
}
|
||||
let detail = resp.text().await.unwrap_or_default();
|
||||
let message = format!("Nacos admin {path} returned {status}: {}", detail.trim());
|
||||
let detail = compact_response_detail(&detail);
|
||||
let message = if detail.is_empty() {
|
||||
format!("Nacos admin {path} returned {status}")
|
||||
} else {
|
||||
format!("Nacos admin {path} returned {status}: {detail}")
|
||||
};
|
||||
Err(classified_error(classify_nacos_error(&message), &message))
|
||||
}
|
||||
|
||||
fn compact_response_detail(detail: &str) -> String {
|
||||
let detail = detail.trim();
|
||||
if detail.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
if detail.to_ascii_lowercase().contains("<!doctype html") || detail.to_ascii_lowercase().contains("<html") {
|
||||
return "HTML error page".to_string();
|
||||
}
|
||||
|
||||
const MAX_CHARS: usize = 512;
|
||||
let mut chars = detail.chars();
|
||||
let compact: String = chars.by_ref().take(MAX_CHARS).collect();
|
||||
if chars.next().is_some() {
|
||||
format!("{compact}…")
|
||||
} else {
|
||||
compact
|
||||
}
|
||||
}
|
||||
|
||||
fn classified_error(kind: &str, message: &str) -> String {
|
||||
format!("{NACOS_ERROR_PREFIX}[{kind}]: {message}")
|
||||
}
|
||||
@@ -2188,7 +2442,29 @@ fn normalize_config_format(value: String) -> String {
|
||||
}
|
||||
|
||||
fn optional_u64_field(value: &Value, keys: &[&str]) -> Option<u64> {
|
||||
keys.iter().find_map(|key| value.get(*key)).and_then(Value::as_u64)
|
||||
keys.iter().find_map(|key| value.get(*key)).and_then(|value| {
|
||||
value
|
||||
.as_u64()
|
||||
.or_else(|| value.as_i64().and_then(|value| u64::try_from(value).ok()))
|
||||
.or_else(|| value.as_str().and_then(|value| value.trim().parse().ok()))
|
||||
})
|
||||
}
|
||||
|
||||
fn optional_f64_field(value: &Value, keys: &[&str]) -> Option<f64> {
|
||||
keys.iter()
|
||||
.find_map(|key| value.get(*key))
|
||||
.and_then(|value| value.as_f64().or_else(|| value.as_str().and_then(|value| value.trim().parse().ok())))
|
||||
.filter(|value| value.is_finite())
|
||||
}
|
||||
|
||||
fn optional_bool_field(value: &Value, keys: &[&str]) -> Option<bool> {
|
||||
keys.iter().find_map(|key| value.get(*key)).and_then(|value| {
|
||||
value.as_bool().or_else(|| match value.as_str()?.trim().to_ascii_lowercase().as_str() {
|
||||
"true" | "up" | "online" | "healthy" => Some(true),
|
||||
"false" | "down" | "offline" | "unhealthy" => Some(false),
|
||||
_ => None,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn optional_i64_field(value: &Value, keys: &[&str]) -> Option<i64> {
|
||||
@@ -2311,6 +2587,8 @@ mod tests {
|
||||
rnacos_console_auth: Default::default(),
|
||||
auth: NacosAuthConfig::None,
|
||||
tls_skip_verify: false,
|
||||
metrics_mode: Default::default(),
|
||||
metrics_url: String::new(),
|
||||
page_size: 100,
|
||||
connect_override: None,
|
||||
}
|
||||
@@ -2334,6 +2612,124 @@ mod tests {
|
||||
server.await.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_v3_rejects_legacy_admin_api_paths() {
|
||||
let mut config = test_admin_config("http://127.0.0.1:8848".to_string());
|
||||
config.version_mode = Some(NacosVersionMode::V3);
|
||||
let admin = NacosOpenApiAdmin::new(config).unwrap();
|
||||
|
||||
assert!(admin.api_path_allowed("/v3/admin/ns/ops/metrics"));
|
||||
assert!(admin.api_path_allowed("/health"));
|
||||
assert!(!admin.api_path_allowed("/v2/ns/operator/metrics"));
|
||||
assert!(!admin.api_path_allowed("/v1/ns/operator/metrics"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compacts_html_error_pages_in_admin_warnings() {
|
||||
assert_eq!(
|
||||
compact_response_detail("<!doctype html><html><body><h1>HTTP Status 404</h1></body></html>"),
|
||||
"HTML error page"
|
||||
);
|
||||
assert!(compact_response_detail(&"x".repeat(600)).ends_with('…'));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gives_nacos_v3_dashboard_endpoint_guidance() {
|
||||
let mut config = test_admin_config("http://127.0.0.1:8080".to_string());
|
||||
config.version_mode = Some(NacosVersionMode::V3);
|
||||
let admin = NacosOpenApiAdmin::new(config).unwrap();
|
||||
|
||||
let warning =
|
||||
admin.dashboard_warning("NACOS_ERROR[contextPathMismatch]: No static resource v3/admin".to_string());
|
||||
assert!(warning.contains("http://host:8848/nacos"));
|
||||
assert!(warning.contains("not the port 8080 Console"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prometheus_metrics_preserve_namespace_dashboard_values() {
|
||||
let mut metrics = Some(NacosDashboardMetrics {
|
||||
service_count: Some(1),
|
||||
instance_count: Some(2),
|
||||
cpu: Some(0.1),
|
||||
..Default::default()
|
||||
});
|
||||
let config_count = Some(3);
|
||||
let service_count = Some(1);
|
||||
let prometheus = NacosPrometheusSnapshot {
|
||||
resource: NacosPrometheusResourceMetrics { cpu_ratio: Some(0.5), ..Default::default() },
|
||||
config: NacosPrometheusConfigMetrics { config_count: Some(7.0), ..Default::default() },
|
||||
naming: NacosPrometheusNamingMetrics {
|
||||
service_count: Some(8.0),
|
||||
instance_count: Some(9.0),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
merge_prometheus_dashboard(&mut metrics, Some(&prometheus));
|
||||
|
||||
let metrics = metrics.unwrap();
|
||||
assert_eq!(config_count, Some(3));
|
||||
assert_eq!(service_count, Some(1));
|
||||
assert_eq!(metrics.service_count, Some(1));
|
||||
assert_eq!(metrics.instance_count, Some(9));
|
||||
assert_eq!(metrics.cpu, Some(0.5));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn nacos_v2_dashboard_uses_core_cluster_node_api() {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut socket, _) = listener.accept().await.unwrap();
|
||||
assert_eq!(read_request_target(&mut socket).await, "/nacos/v2/core/cluster/node/list");
|
||||
write_json_response(&mut socket, r#"{"code":0,"data":[{"address":"127.0.0.1:8848","state":"UP"}]}"#).await;
|
||||
});
|
||||
let mut config = test_admin_config(format!("http://{address}"));
|
||||
config.context_path = "/nacos".to_string();
|
||||
config.version_mode = Some(NacosVersionMode::V2);
|
||||
let admin = NacosOpenApiAdmin::new(config).unwrap();
|
||||
|
||||
let nodes = admin.get_dashboard_nodes().await.unwrap();
|
||||
assert_eq!(nodes.len(), 1);
|
||||
assert_eq!(nodes[0].address, "127.0.0.1:8848");
|
||||
server.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn nacos_v2_dashboard_falls_back_to_legacy_core_cluster_nodes_api() {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut socket, _) = listener.accept().await.unwrap();
|
||||
assert_eq!(read_request_target(&mut socket).await, "/nacos/v2/core/cluster/node/list");
|
||||
write_not_found_response(&mut socket).await;
|
||||
|
||||
let (mut socket, _) = listener.accept().await.unwrap();
|
||||
assert_eq!(read_request_target(&mut socket).await, "/nacos/v1/core/cluster/nodes");
|
||||
write_json_response(&mut socket, r#"{"code":200,"data":[{"address":"127.0.0.1:8848","state":"UP"}]}"#)
|
||||
.await;
|
||||
});
|
||||
let mut config = test_admin_config(format!("http://{address}"));
|
||||
config.context_path = "/nacos".to_string();
|
||||
config.version_mode = Some(NacosVersionMode::V2);
|
||||
let admin = NacosOpenApiAdmin::new(config).unwrap();
|
||||
|
||||
let nodes = admin.get_dashboard_nodes().await.unwrap();
|
||||
assert_eq!(nodes.len(), 1);
|
||||
assert_eq!(nodes[0].address, "127.0.0.1:8848");
|
||||
server.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rnacos_dashboard_skips_unsupported_cluster_node_api() {
|
||||
let mut config = test_admin_config("http://127.0.0.1:1".to_string());
|
||||
config.implementation = Some(NacosImplementation::RNacos);
|
||||
let admin = NacosOpenApiAdmin::new(config).unwrap();
|
||||
|
||||
assert!(admin.get_dashboard_nodes().await.unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn version_mode_v3_uses_only_v3_config_paths() {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
@@ -3638,6 +4034,53 @@ mod tests {
|
||||
server.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dashboard_combines_metrics_nodes_and_namespace_totals() {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
let mut paths = HashSet::new();
|
||||
for _ in 0..6 {
|
||||
let (mut socket, _) = listener.accept().await.unwrap();
|
||||
let target = read_request_target(&mut socket).await;
|
||||
let url = reqwest::Url::parse(&format!("http://localhost{target}")).unwrap();
|
||||
paths.insert(url.path().to_string());
|
||||
let body = match url.path() {
|
||||
"/v3/admin/ns/ops/metrics" => {
|
||||
r#"{"code":0,"data":{"status":"UP","serviceCount":8,"instanceCount":13,"clientCount":5}}"#
|
||||
}
|
||||
"/v3/admin/core/cluster/node/list" => {
|
||||
r#"{"code":0,"data":[{"address":"127.0.0.1:8848","ip":"127.0.0.1","port":8848,"state":"UP"}]}"#
|
||||
}
|
||||
"/v3/console/core/namespace/list" => {
|
||||
r#"{"code":0,"data":{"pageItems":[{"namespaceId":"dev","namespaceName":"Development"}]}}"#
|
||||
}
|
||||
"/v3/console/cs/config/list" => r#"{"code":0,"data":{"totalCount":21,"pageItems":[]}}"#,
|
||||
"/v3/console/ns/service/list" => r#"{"code":0,"data":{"count":3,"serviceList":[]}}"#,
|
||||
"/actuator/prometheus" => {
|
||||
"# TYPE system_cpu_usage gauge\nsystem_cpu_usage 0.25\n# TYPE nacos_monitor gauge\nnacos_monitor{module=\"config\",name=\"configCount\"} 12\nnacos_monitor{module=\"naming\",name=\"serviceCount\"} 4\nnacos_monitor{module=\"naming\",name=\"ipCount\"} 14\n"
|
||||
}
|
||||
path => panic!("unexpected dashboard request path: {path}"),
|
||||
};
|
||||
write_json_response(&mut socket, body).await;
|
||||
}
|
||||
paths
|
||||
});
|
||||
|
||||
let admin = NacosOpenApiAdmin::new(test_admin_config(format!("http://{address}"))).unwrap();
|
||||
let snapshot = admin.get_dashboard(NacosDashboardQuery { namespace: Some("dev".to_string()) }).await.unwrap();
|
||||
|
||||
assert_eq!(snapshot.namespace, "dev");
|
||||
assert_eq!(snapshot.namespace_count, Some(2));
|
||||
assert_eq!(snapshot.config_count, Some(21));
|
||||
assert_eq!(snapshot.service_count, Some(3));
|
||||
assert_eq!(snapshot.metrics.as_ref().and_then(|metrics| metrics.instance_count), Some(14));
|
||||
assert_eq!(snapshot.prometheus.as_ref().and_then(|metrics| metrics.resource.cpu_ratio), Some(0.25));
|
||||
assert_eq!(snapshot.nodes.len(), 1);
|
||||
assert!(snapshot.warnings.is_empty());
|
||||
assert_eq!(server.await.unwrap().len(), 6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_namespace_list_shape() {
|
||||
let parsed = parse_namespaces(serde_json::json!({
|
||||
@@ -3667,6 +4110,61 @@ mod tests {
|
||||
assert_eq!(parsed[1].namespace_show_name, "Development");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_v3_dashboard_metrics_shape() {
|
||||
let parsed = parse_dashboard_metrics(serde_json::json!({
|
||||
"code": 0,
|
||||
"data": {
|
||||
"status": "UP",
|
||||
"serviceCount": 12,
|
||||
"instanceCount": 34,
|
||||
"clientCount": "5",
|
||||
"cpu": 0.25,
|
||||
"mem": "0.5"
|
||||
}
|
||||
}));
|
||||
|
||||
assert_eq!(parsed.status.as_deref(), Some("UP"));
|
||||
assert_eq!(parsed.service_count, Some(12));
|
||||
assert_eq!(parsed.instance_count, Some(34));
|
||||
assert_eq!(parsed.client_count, Some(5));
|
||||
assert_eq!(parsed.cpu, Some(0.25));
|
||||
assert_eq!(parsed.mem, Some(0.5));
|
||||
|
||||
let status_only = parse_dashboard_metrics(serde_json::json!({ "code": 0, "data": "UP" }));
|
||||
assert_eq!(status_only.status.as_deref(), Some("UP"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_v1_and_v3_cluster_node_shapes() {
|
||||
let v1 = parse_cluster_nodes(serde_json::json!({
|
||||
"servers": [{
|
||||
"ip": "192.0.2.1",
|
||||
"servePort": 8848,
|
||||
"alive": true,
|
||||
"site": "unknown",
|
||||
"lastRefTimeStr": "2026-07-26 10:00:00"
|
||||
}]
|
||||
}));
|
||||
assert_eq!(v1[0].address, "192.0.2.1:8848");
|
||||
assert_eq!(v1[0].alive, Some(true));
|
||||
assert_eq!(v1[0].last_refresh_time.as_deref(), Some("2026-07-26 10:00:00"));
|
||||
|
||||
let v3 = parse_cluster_nodes(serde_json::json!({
|
||||
"code": 0,
|
||||
"data": [{
|
||||
"address": "192.0.2.2:8848",
|
||||
"ip": "192.0.2.2",
|
||||
"port": 8848,
|
||||
"state": "UP",
|
||||
"extendInfo": { "lastRefreshTime": 1785031200000_u64 }
|
||||
}]
|
||||
}));
|
||||
assert_eq!(v3[0].address, "192.0.2.2:8848");
|
||||
assert_eq!(v3[0].alive, Some(true));
|
||||
assert_eq!(v3[0].last_refresh_time.as_deref(), Some("1785031200000"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validates_raw_api_paths() {
|
||||
for path in ["/v1/cs/configs", "/v2/console/example", "/v3/console/server/state"] {
|
||||
|
||||
@@ -11,6 +11,7 @@ pub mod batch;
|
||||
pub mod config;
|
||||
pub mod http;
|
||||
pub mod port;
|
||||
mod prometheus;
|
||||
pub mod search;
|
||||
pub mod service;
|
||||
pub mod types;
|
||||
@@ -183,6 +184,8 @@ mod tests {
|
||||
},
|
||||
auth: NacosAuthConfig::None,
|
||||
tls_skip_verify: false,
|
||||
metrics_mode: Default::default(),
|
||||
metrics_url: String::new(),
|
||||
page_size: 20,
|
||||
connect_override: None,
|
||||
}
|
||||
|
||||
@@ -31,5 +31,6 @@ pub trait NacosAdmin: Send + Sync {
|
||||
async fn list_services(&self, query: NacosServiceQuery) -> Result<NacosServiceList, String>;
|
||||
async fn list_instances(&self, query: NacosInstanceQuery) -> Result<Vec<NacosInstanceInfo>, String>;
|
||||
async fn update_instance(&self, req: NacosInstanceUpdate) -> Result<(), String>;
|
||||
async fn get_dashboard(&self, query: NacosDashboardQuery) -> Result<NacosDashboardSnapshot, String>;
|
||||
async fn raw_request(&self, req: NacosRawRequest) -> Result<NacosRawResponse, String>;
|
||||
}
|
||||
@@ -0,0 +1,820 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::io;
|
||||
use std::time::Duration;
|
||||
|
||||
use futures::StreamExt;
|
||||
use prometheus_parse::{Labels, Sample, Scrape, Value};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::nacos::config::{NacosAdminConfig, NacosImplementation, NacosMetricsMode};
|
||||
use crate::nacos::types::{
|
||||
NacosPrometheusConfigMetrics, NacosPrometheusNamingMetrics, NacosPrometheusResourceMetrics,
|
||||
NacosPrometheusSnapshot, NacosPrometheusSource, NacosPrometheusTrafficMetrics,
|
||||
};
|
||||
|
||||
const SCRAPE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const MAX_SCRAPE_BYTES: usize = 4 * 1024 * 1024;
|
||||
|
||||
pub(crate) fn endpoint_candidates(cfg: &NacosAdminConfig) -> Result<Vec<String>, String> {
|
||||
match cfg.metrics_mode {
|
||||
NacosMetricsMode::Disabled => return Ok(Vec::new()),
|
||||
NacosMetricsMode::Custom => return Ok(vec![cfg.metrics_url.clone()]),
|
||||
NacosMetricsMode::Auto => {}
|
||||
}
|
||||
|
||||
let base = cfg.server_addr.trim_end_matches('/');
|
||||
let context = cfg.context_path.trim_end_matches('/');
|
||||
let raw = match cfg.implementation.as_ref().unwrap_or(&NacosImplementation::Nacos) {
|
||||
NacosImplementation::Nacos => vec![
|
||||
format!("{base}{context}/actuator/prometheus"),
|
||||
format!("{base}/nacos/actuator/prometheus"),
|
||||
format!("{base}/actuator/prometheus"),
|
||||
],
|
||||
NacosImplementation::RNacos => {
|
||||
vec![format!("{base}/metrics"), format!("{base}{context}/metrics"), format!("{base}/rnacos/metrics")]
|
||||
}
|
||||
};
|
||||
|
||||
let mut seen = HashSet::new();
|
||||
raw.into_iter()
|
||||
.map(|candidate| {
|
||||
reqwest::Url::parse(&candidate)
|
||||
.map(|url| url.to_string())
|
||||
.map_err(|error| format!("Nacos Prometheus metrics URL is invalid: {error}"))
|
||||
})
|
||||
.filter(|candidate| match candidate {
|
||||
Ok(value) => seen.insert(value.clone()),
|
||||
Err(_) => true,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) async fn scrape(
|
||||
client: &reqwest::Client,
|
||||
cfg: &NacosAdminConfig,
|
||||
) -> Result<Option<NacosPrometheusSnapshot>, String> {
|
||||
scrape_with_timeout(client, cfg, SCRAPE_TIMEOUT).await
|
||||
}
|
||||
|
||||
async fn scrape_with_timeout(
|
||||
client: &reqwest::Client,
|
||||
cfg: &NacosAdminConfig,
|
||||
timeout: Duration,
|
||||
) -> Result<Option<NacosPrometheusSnapshot>, String> {
|
||||
let candidates = endpoint_candidates(cfg)?;
|
||||
if candidates.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let candidate_summary =
|
||||
candidates.iter().map(|candidate| redact_endpoint(candidate)).collect::<Vec<_>>().join(", ");
|
||||
|
||||
tokio::time::timeout(timeout, async {
|
||||
let mut errors = Vec::new();
|
||||
for endpoint in candidates {
|
||||
match scrape_endpoint(client, cfg, &endpoint).await {
|
||||
Ok(snapshot) => return Ok(Some(snapshot)),
|
||||
Err(error) => errors.push(format!("{}: {error}", redact_endpoint(&endpoint))),
|
||||
}
|
||||
}
|
||||
Err(format!("Prometheus metrics unavailable: {}", errors.join("; ")))
|
||||
})
|
||||
.await
|
||||
.map_err(|_| {
|
||||
format!("Prometheus metrics unavailable: scrape timed out after {timeout:?} while trying {candidate_summary}")
|
||||
})?
|
||||
}
|
||||
|
||||
async fn scrape_endpoint(
|
||||
client: &reqwest::Client,
|
||||
cfg: &NacosAdminConfig,
|
||||
endpoint: &str,
|
||||
) -> Result<NacosPrometheusSnapshot, String> {
|
||||
let response = client
|
||||
.get(endpoint)
|
||||
.timeout(SCRAPE_TIMEOUT)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|error| format!("request failed: {}", error.without_url()))?;
|
||||
if !response.status().is_success() {
|
||||
return Err(format!("returned HTTP {}", response.status()));
|
||||
}
|
||||
|
||||
let mut bytes = Vec::new();
|
||||
let mut stream = response.bytes_stream();
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.map_err(|error| format!("failed to read response: {}", error.without_url()))?;
|
||||
if bytes.len().saturating_add(chunk.len()) > MAX_SCRAPE_BYTES {
|
||||
return Err(format!("response exceeds {MAX_SCRAPE_BYTES} bytes"));
|
||||
}
|
||||
bytes.extend_from_slice(&chunk);
|
||||
}
|
||||
let body = String::from_utf8(bytes).map_err(|_| "response is not valid UTF-8".to_string())?;
|
||||
parse_scrape(&body, cfg.implementation.as_ref().unwrap_or(&NacosImplementation::Nacos), endpoint)
|
||||
}
|
||||
|
||||
pub(crate) fn parse_scrape(
|
||||
body: &str,
|
||||
implementation: &NacosImplementation,
|
||||
endpoint: &str,
|
||||
) -> Result<NacosPrometheusSnapshot, String> {
|
||||
let lines = body.lines().map(|line| Ok::<_, io::Error>(line.to_string()));
|
||||
let scrape = Scrape::parse(lines).map_err(|error| format!("failed to parse metrics: {error}"))?;
|
||||
if scrape.samples.is_empty() {
|
||||
return Err("response contains no Prometheus samples".to_string());
|
||||
}
|
||||
let source = NacosPrometheusSource {
|
||||
kind: match implementation {
|
||||
NacosImplementation::Nacos => "nacos",
|
||||
NacosImplementation::RNacos => "rnacos",
|
||||
}
|
||||
.to_string(),
|
||||
endpoint: redact_endpoint(endpoint),
|
||||
fingerprint: Some(endpoint_fingerprint(endpoint)),
|
||||
};
|
||||
Ok(match implementation {
|
||||
NacosImplementation::Nacos => normalize_nacos(source, &scrape.samples),
|
||||
NacosImplementation::RNacos => normalize_rnacos(source, &scrape.samples),
|
||||
})
|
||||
}
|
||||
|
||||
fn normalize_nacos(source: NacosPrometheusSource, samples: &[Sample]) -> NacosPrometheusSnapshot {
|
||||
let heap_used = scalar_sum_filtered(samples, "jvm_memory_used_bytes", |labels| label_eq(labels, "area", "heap"));
|
||||
let heap_max = scalar_sum_filtered(samples, "jvm_memory_max_bytes", |labels| label_eq(labels, "area", "heap"));
|
||||
let memory_ratio = ratio(heap_used, heap_max);
|
||||
let http_count = scalar_sum(samples, "http_server_requests_seconds_count");
|
||||
let grpc_count = scalar_sum_any(samples, &["grpc_server_requests_seconds_count", "grpc_server_requests_count"]);
|
||||
|
||||
NacosPrometheusSnapshot {
|
||||
source,
|
||||
resource: NacosPrometheusResourceMetrics {
|
||||
cpu_ratio: scalar_max_any(samples, &["system_cpu_usage", "process_cpu_usage"]).and_then(normalize_ratio),
|
||||
memory_ratio,
|
||||
memory_used_bytes: heap_used,
|
||||
memory_max_bytes: heap_max,
|
||||
rss_bytes: None,
|
||||
vms_bytes: None,
|
||||
system_total_memory_bytes: None,
|
||||
load_1m: scalar_max(samples, "system_load_average_1m"),
|
||||
jvm_daemon_threads: scalar_sum(samples, "jvm_threads_daemon"),
|
||||
gc_pause_count: scalar_sum_any(samples, &["jvm_gc_pause_seconds_count", "jvm_gc_pause_count"]),
|
||||
},
|
||||
traffic: NacosPrometheusTrafficMetrics {
|
||||
http_requests_total: http_count,
|
||||
grpc_requests_total: grpc_count,
|
||||
http_errors_total: scalar_sum_filtered(samples, "http_server_requests_seconds_count", is_error_labels),
|
||||
grpc_errors_total: scalar_sum_filtered_any(
|
||||
samples,
|
||||
&["grpc_server_requests_seconds_count", "grpc_server_requests_count"],
|
||||
is_grpc_error_labels,
|
||||
),
|
||||
http_duration_seconds_total: scalar_sum(samples, "http_server_requests_seconds_sum"),
|
||||
http_duration_count: http_count,
|
||||
grpc_duration_seconds_total: scalar_sum_any(
|
||||
samples,
|
||||
&["grpc_server_requests_seconds_sum", "grpc_server_requests_sum"],
|
||||
),
|
||||
grpc_duration_count: grpc_count,
|
||||
http_p50_ms: quantile_ms(samples, "http_server_requests_seconds", 0.5, 1_000.0),
|
||||
http_p95_ms: quantile_ms(samples, "http_server_requests_seconds", 0.95, 1_000.0),
|
||||
http_p99_ms: quantile_ms(samples, "http_server_requests_seconds", 0.99, 1_000.0),
|
||||
grpc_p50_ms: quantile_ms_any(
|
||||
samples,
|
||||
&["grpc_server_requests_seconds", "grpc_server_requests"],
|
||||
0.5,
|
||||
1_000.0,
|
||||
),
|
||||
grpc_p95_ms: quantile_ms_any(
|
||||
samples,
|
||||
&["grpc_server_requests_seconds", "grpc_server_requests"],
|
||||
0.95,
|
||||
1_000.0,
|
||||
),
|
||||
grpc_p99_ms: quantile_ms_any(
|
||||
samples,
|
||||
&["grpc_server_requests_seconds", "grpc_server_requests"],
|
||||
0.99,
|
||||
1_000.0,
|
||||
),
|
||||
executor_pool_size: scalar_max_filtered(samples, "grpc_server_executor", |labels| {
|
||||
label_in(labels, "name", &["poolSize", "pool_size"])
|
||||
}),
|
||||
executor_active_count: scalar_max_filtered(samples, "grpc_server_executor", |labels| {
|
||||
label_in(labels, "name", &["activeCount", "active_count"])
|
||||
}),
|
||||
executor_queued_tasks: scalar_max_filtered(samples, "grpc_server_executor", |labels| {
|
||||
label_in(labels, "name", &["queuedTasks", "queueSize", "queued_tasks"])
|
||||
}),
|
||||
},
|
||||
config: NacosPrometheusConfigMetrics {
|
||||
config_count: monitor(samples, "config", &["configCount"]),
|
||||
get_config_total: monitor(samples, "config", &["getConfig", "getConfigCount"]),
|
||||
publish_total: monitor(samples, "config", &["publish", "publishCount"]),
|
||||
long_polling: monitor(samples, "config", &["longPolling"]),
|
||||
listener_clients: None,
|
||||
listener_keys: None,
|
||||
notify_tasks: monitor(samples, "config", &["notifyTask"]),
|
||||
notify_client_tasks: monitor(samples, "config", &["notifyClientTask"]),
|
||||
dump_tasks: monitor(samples, "config", &["dumpTask"]),
|
||||
subscriber_count: scalar_sum(samples, "nacos_config_subscriber"),
|
||||
},
|
||||
naming: NacosPrometheusNamingMetrics {
|
||||
service_count: monitor(samples, "naming", &["serviceCount"]),
|
||||
instance_count: monitor(samples, "naming", &["ipCount", "instanceCount"]),
|
||||
subscriber_count: monitor(samples, "naming", &["subscriberCount"])
|
||||
.or_else(|| scalar_sum(samples, "nacos_naming_subscriber")),
|
||||
connection_count: monitor(samples, "core", &["longConnection"]),
|
||||
total_push: monitor(samples, "naming", &["totalPush"]),
|
||||
failed_push: monitor(samples, "naming", &["failedPush"]),
|
||||
empty_push: monitor(samples, "naming", &["emptyPush"]),
|
||||
push_pending_tasks: monitor(
|
||||
samples,
|
||||
"naming",
|
||||
&["pushPendingTaskCount", "pushPendingTask", "pushPendingTasks"],
|
||||
),
|
||||
avg_push_cost_ms: monitor(samples, "naming", &["avgPushCost"]),
|
||||
max_push_cost_ms: monitor(samples, "naming", &["maxPushCost"]),
|
||||
leader_status: monitor(samples, "naming", &["leaderStatus"]),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_rnacos(source: NacosPrometheusSource, samples: &[Sample]) -> NacosPrometheusSnapshot {
|
||||
let mib = 1024.0 * 1024.0;
|
||||
let http_count = scalar_sum(samples, "http_request_total_count");
|
||||
let grpc_count = scalar_sum(samples, "grpc_request_total_count");
|
||||
let rss_mib = scalar_sum(samples, "app_rss_memory");
|
||||
let system_memory_mib = scalar_sum(samples, "sys_total_memory");
|
||||
NacosPrometheusSnapshot {
|
||||
source,
|
||||
resource: NacosPrometheusResourceMetrics {
|
||||
cpu_ratio: scalar_max(samples, "app_cpu_usage").and_then(normalize_percentage),
|
||||
memory_ratio: scalar_max(samples, "app_memory_usage")
|
||||
.and_then(normalize_percentage)
|
||||
.or_else(|| ratio(rss_mib, system_memory_mib)),
|
||||
memory_used_bytes: rss_mib.map(|value| value * mib),
|
||||
memory_max_bytes: system_memory_mib.map(|value| value * mib),
|
||||
rss_bytes: rss_mib.map(|value| value * mib),
|
||||
vms_bytes: scalar_sum(samples, "app_vms_memory").map(|value| value * mib),
|
||||
system_total_memory_bytes: system_memory_mib.map(|value| value * mib),
|
||||
load_1m: None,
|
||||
jvm_daemon_threads: None,
|
||||
gc_pause_count: None,
|
||||
},
|
||||
traffic: NacosPrometheusTrafficMetrics {
|
||||
http_requests_total: http_count,
|
||||
grpc_requests_total: grpc_count,
|
||||
http_errors_total: None,
|
||||
grpc_errors_total: None,
|
||||
http_duration_seconds_total: scalar_sum(samples, "http_request_handle_rt_histogram_sum")
|
||||
.or_else(|| scalar_sum(samples, "http_request_handle_rt_summary_sum"))
|
||||
.map(|value| value / 1_000.0),
|
||||
http_duration_count: scalar_sum(samples, "http_request_handle_rt_histogram_count")
|
||||
.or_else(|| scalar_sum(samples, "http_request_handle_rt_summary_count"))
|
||||
.or(http_count),
|
||||
grpc_duration_seconds_total: scalar_sum(samples, "grpc_request_handle_rt_histogram_sum")
|
||||
.or_else(|| scalar_sum(samples, "grpc_request_handle_rt_summary_sum"))
|
||||
.map(|value| value / 1_000.0),
|
||||
grpc_duration_count: scalar_sum(samples, "grpc_request_handle_rt_histogram_count")
|
||||
.or_else(|| scalar_sum(samples, "grpc_request_handle_rt_summary_count"))
|
||||
.or(grpc_count),
|
||||
http_p50_ms: quantile_ms_any(
|
||||
samples,
|
||||
&["http_request_handle_rt_summary", "http_request_handle_rt_histogram"],
|
||||
0.5,
|
||||
1.0,
|
||||
),
|
||||
http_p95_ms: quantile_ms_any(
|
||||
samples,
|
||||
&["http_request_handle_rt_summary", "http_request_handle_rt_histogram"],
|
||||
0.95,
|
||||
1.0,
|
||||
),
|
||||
http_p99_ms: quantile_ms_any(
|
||||
samples,
|
||||
&["http_request_handle_rt_summary", "http_request_handle_rt_histogram"],
|
||||
0.99,
|
||||
1.0,
|
||||
),
|
||||
grpc_p50_ms: quantile_ms_any(
|
||||
samples,
|
||||
&["grpc_request_handle_rt_summary", "grpc_request_handle_rt_histogram"],
|
||||
0.5,
|
||||
1.0,
|
||||
),
|
||||
grpc_p95_ms: quantile_ms_any(
|
||||
samples,
|
||||
&["grpc_request_handle_rt_summary", "grpc_request_handle_rt_histogram"],
|
||||
0.95,
|
||||
1.0,
|
||||
),
|
||||
grpc_p99_ms: quantile_ms_any(
|
||||
samples,
|
||||
&["grpc_request_handle_rt_summary", "grpc_request_handle_rt_histogram"],
|
||||
0.99,
|
||||
1.0,
|
||||
),
|
||||
executor_pool_size: None,
|
||||
executor_active_count: None,
|
||||
executor_queued_tasks: None,
|
||||
},
|
||||
config: NacosPrometheusConfigMetrics {
|
||||
config_count: scalar_sum(samples, "config_data_size")
|
||||
.or_else(|| scalar_sum(samples, "config_index_config_size")),
|
||||
get_config_total: None,
|
||||
publish_total: None,
|
||||
long_polling: None,
|
||||
listener_clients: scalar_sum(samples, "config_listener_client_size"),
|
||||
listener_keys: scalar_sum(samples, "config_listener_key_size"),
|
||||
notify_tasks: None,
|
||||
notify_client_tasks: None,
|
||||
dump_tasks: None,
|
||||
subscriber_count: scalar_sum(samples, "config_subscriber_client_size")
|
||||
.or_else(|| scalar_sum(samples, "config_subscriber_client_value_size")),
|
||||
},
|
||||
naming: NacosPrometheusNamingMetrics {
|
||||
service_count: scalar_sum(samples, "naming_service_size")
|
||||
.or_else(|| scalar_sum(samples, "naming_index_service_size")),
|
||||
instance_count: scalar_sum(samples, "naming_instance_size"),
|
||||
subscriber_count: scalar_sum(samples, "naming_subscriber_client_size")
|
||||
.or_else(|| scalar_sum(samples, "naming_subscriber_client_value_size")),
|
||||
connection_count: scalar_sum(samples, "grpc_conn_size"),
|
||||
total_push: None,
|
||||
failed_push: None,
|
||||
empty_push: None,
|
||||
push_pending_tasks: None,
|
||||
avg_push_cost_ms: None,
|
||||
max_push_cost_ms: None,
|
||||
leader_status: None,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn scalar_value(sample: &Sample) -> Option<f64> {
|
||||
let value = match sample.value {
|
||||
Value::Counter(value) | Value::Gauge(value) | Value::Untyped(value) => value,
|
||||
Value::Histogram(_) | Value::Summary(_) => return None,
|
||||
};
|
||||
value.is_finite().then_some(value)
|
||||
}
|
||||
|
||||
fn scalar_sum(samples: &[Sample], metric: &str) -> Option<f64> {
|
||||
scalar_sum_filtered(samples, metric, |_| true)
|
||||
}
|
||||
|
||||
fn scalar_sum_any(samples: &[Sample], metrics: &[&str]) -> Option<f64> {
|
||||
metrics.iter().find_map(|metric| scalar_sum(samples, metric))
|
||||
}
|
||||
|
||||
fn scalar_sum_filtered(samples: &[Sample], metric: &str, predicate: impl Fn(&Labels) -> bool) -> Option<f64> {
|
||||
let values =
|
||||
samples.iter().filter(|sample| sample.metric == metric && predicate(&sample.labels)).filter_map(scalar_value);
|
||||
sum_non_empty(values)
|
||||
}
|
||||
|
||||
fn scalar_sum_filtered_any(
|
||||
samples: &[Sample],
|
||||
metrics: &[&str],
|
||||
predicate: impl Fn(&Labels) -> bool + Copy,
|
||||
) -> Option<f64> {
|
||||
metrics.iter().find_map(|metric| scalar_sum_filtered(samples, metric, predicate))
|
||||
}
|
||||
|
||||
fn scalar_max(samples: &[Sample], metric: &str) -> Option<f64> {
|
||||
scalar_max_filtered(samples, metric, |_| true)
|
||||
}
|
||||
|
||||
fn scalar_max_any(samples: &[Sample], metrics: &[&str]) -> Option<f64> {
|
||||
metrics.iter().find_map(|metric| scalar_max(samples, metric))
|
||||
}
|
||||
|
||||
fn scalar_max_filtered(samples: &[Sample], metric: &str, predicate: impl Fn(&Labels) -> bool) -> Option<f64> {
|
||||
samples
|
||||
.iter()
|
||||
.filter(|sample| sample.metric == metric && predicate(&sample.labels))
|
||||
.filter_map(scalar_value)
|
||||
.reduce(f64::max)
|
||||
}
|
||||
|
||||
fn sum_non_empty(values: impl Iterator<Item = f64>) -> Option<f64> {
|
||||
let mut found = false;
|
||||
let sum = values.fold(0.0, |sum, value| {
|
||||
found = true;
|
||||
sum + value
|
||||
});
|
||||
found.then_some(sum)
|
||||
}
|
||||
|
||||
fn monitor(samples: &[Sample], module: &str, names: &[&str]) -> Option<f64> {
|
||||
names.iter().find_map(|name| {
|
||||
scalar_sum_filtered(samples, "nacos_monitor", |labels| {
|
||||
label_eq(labels, "module", module) && label_eq(labels, "name", name)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn label_eq(labels: &Labels, key: &str, value: &str) -> bool {
|
||||
labels.get(key).is_some_and(|current| current.eq_ignore_ascii_case(value))
|
||||
}
|
||||
|
||||
fn label_in(labels: &Labels, key: &str, values: &[&str]) -> bool {
|
||||
values.iter().any(|value| label_eq(labels, key, value))
|
||||
}
|
||||
|
||||
fn is_error_labels(labels: &Labels) -> bool {
|
||||
if let Some(status) = labels.get("status").or_else(|| labels.get("code")) {
|
||||
return !status.starts_with('2') && status != "0";
|
||||
}
|
||||
labels.get("outcome").is_some_and(|outcome| !matches!(outcome.to_ascii_uppercase().as_str(), "SUCCESS" | "UNKNOWN"))
|
||||
}
|
||||
|
||||
fn is_grpc_error_labels(labels: &Labels) -> bool {
|
||||
if labels.get("success").is_some_and(|value| value.eq_ignore_ascii_case("false")) {
|
||||
return true;
|
||||
}
|
||||
labels
|
||||
.get("errorCode")
|
||||
.or_else(|| labels.get("error_code"))
|
||||
.or_else(|| labels.get("code"))
|
||||
.is_some_and(|value| value != "0" && !value.eq_ignore_ascii_case("OK"))
|
||||
|| labels.get("result").is_some_and(|value| !matches!(value.to_ascii_uppercase().as_str(), "SUCCESS" | "OK"))
|
||||
}
|
||||
|
||||
fn ratio(numerator: Option<f64>, denominator: Option<f64>) -> Option<f64> {
|
||||
match (numerator, denominator) {
|
||||
(Some(numerator), Some(denominator)) if denominator > 0.0 => normalize_ratio(numerator / denominator),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn normalize_ratio(value: f64) -> Option<f64> {
|
||||
if !value.is_finite() || value < 0.0 {
|
||||
return None;
|
||||
}
|
||||
Some((if value > 1.0 { value / 100.0 } else { value }).clamp(0.0, 1.0))
|
||||
}
|
||||
|
||||
fn normalize_percentage(value: f64) -> Option<f64> {
|
||||
if !value.is_finite() || value < 0.0 {
|
||||
return None;
|
||||
}
|
||||
Some((value / 100.0).clamp(0.0, 1.0))
|
||||
}
|
||||
|
||||
fn endpoint_fingerprint(endpoint: &str) -> String {
|
||||
format!("{:x}", Sha256::digest(endpoint.as_bytes()))
|
||||
}
|
||||
|
||||
fn redact_endpoint(endpoint: &str) -> String {
|
||||
let Ok(mut url) = reqwest::Url::parse(endpoint) else {
|
||||
return endpoint.split_once('?').map_or(endpoint, |(base, _)| base).to_string();
|
||||
};
|
||||
url.set_query(None);
|
||||
url.set_fragment(None);
|
||||
url.to_string()
|
||||
}
|
||||
|
||||
fn quantile_ms(samples: &[Sample], metric: &str, quantile: f64, multiplier: f64) -> Option<f64> {
|
||||
let mut histogram_buckets = HashMap::<u64, (f64, f64)>::new();
|
||||
let mut summary_quantiles = Vec::new();
|
||||
for sample in samples.iter().filter(|sample| sample.metric == metric) {
|
||||
match &sample.value {
|
||||
Value::Histogram(values) => {
|
||||
for value in values {
|
||||
if value.less_than.is_nan() || !value.count.is_finite() {
|
||||
continue;
|
||||
}
|
||||
histogram_buckets
|
||||
.entry(value.less_than.to_bits())
|
||||
.and_modify(|(_, count)| *count += value.count)
|
||||
.or_insert((value.less_than, value.count));
|
||||
}
|
||||
}
|
||||
Value::Summary(values) => summary_quantiles.push(values),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
if !histogram_buckets.is_empty() {
|
||||
return histogram_quantile(histogram_buckets.into_values(), quantile).map(|value| value * multiplier);
|
||||
}
|
||||
// Prometheus summaries are already-calculated client-side quantiles and
|
||||
// cannot be aggregated across label sets without changing their meaning.
|
||||
let [values] = summary_quantiles.as_slice() else {
|
||||
return None;
|
||||
};
|
||||
nearest_quantile(values.iter().map(|value| (value.quantile, value.count)), quantile).map(|value| value * multiplier)
|
||||
}
|
||||
|
||||
fn quantile_ms_any(samples: &[Sample], metrics: &[&str], quantile: f64, multiplier: f64) -> Option<f64> {
|
||||
metrics.iter().find_map(|metric| quantile_ms(samples, metric, quantile, multiplier))
|
||||
}
|
||||
|
||||
fn nearest_quantile(values: impl Iterator<Item = (f64, f64)>, target: f64) -> Option<f64> {
|
||||
values
|
||||
.filter(|(quantile, value)| quantile.is_finite() && value.is_finite())
|
||||
.min_by(|left, right| {
|
||||
(left.0 - target).abs().partial_cmp(&(right.0 - target).abs()).unwrap_or(std::cmp::Ordering::Equal)
|
||||
})
|
||||
.map(|(_, value)| value)
|
||||
}
|
||||
|
||||
fn histogram_quantile(values: impl Iterator<Item = (f64, f64)>, target: f64) -> Option<f64> {
|
||||
let mut buckets =
|
||||
values.filter(|(boundary, count)| !boundary.is_nan() && count.is_finite() && *count >= 0.0).collect::<Vec<_>>();
|
||||
buckets.sort_by(|left, right| left.0.partial_cmp(&right.0).unwrap_or(std::cmp::Ordering::Equal));
|
||||
let mut previous_count = 0.0;
|
||||
for (_, count) in &mut buckets {
|
||||
*count = count.max(previous_count);
|
||||
previous_count = *count;
|
||||
}
|
||||
let total = buckets.last()?.1;
|
||||
if total <= 0.0 {
|
||||
return None;
|
||||
}
|
||||
let threshold = total * target.clamp(0.0, 1.0);
|
||||
let index = buckets.iter().position(|(_, count)| *count >= threshold)?;
|
||||
let (upper_bound, upper_count) = buckets[index];
|
||||
if upper_bound.is_infinite() {
|
||||
return (index > 0).then_some(buckets[index - 1].0).filter(|boundary| boundary.is_finite());
|
||||
}
|
||||
let (lower_bound, lower_count) = if index == 0 { (0.0, 0.0) } else { buckets[index - 1] };
|
||||
if upper_count <= lower_count {
|
||||
return Some(upper_bound);
|
||||
}
|
||||
Some(lower_bound + (upper_bound - lower_bound) * ((threshold - lower_count) / (upper_count - lower_count)))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn derives_and_deduplicates_nacos_endpoints() {
|
||||
let cfg = test_config(NacosImplementation::Nacos, "http://127.0.0.1:8818", "/nacos");
|
||||
assert_eq!(
|
||||
endpoint_candidates(&cfg).unwrap(),
|
||||
vec!["http://127.0.0.1:8818/nacos/actuator/prometheus", "http://127.0.0.1:8818/actuator/prometheus",]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn derives_rnacos_endpoints() {
|
||||
let cfg = test_config(NacosImplementation::RNacos, "http://127.0.0.1:3848", "/nacos");
|
||||
assert_eq!(
|
||||
endpoint_candidates(&cfg).unwrap(),
|
||||
vec![
|
||||
"http://127.0.0.1:3848/metrics",
|
||||
"http://127.0.0.1:3848/nacos/metrics",
|
||||
"http://127.0.0.1:3848/rnacos/metrics",
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disabled_mode_skips_scraping() {
|
||||
let mut cfg = test_config(NacosImplementation::Nacos, "http://127.0.0.1:8818", "/nacos");
|
||||
cfg.metrics_mode = NacosMetricsMode::Disabled;
|
||||
assert!(endpoint_candidates(&cfg).unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_nacos_metrics_and_labels() {
|
||||
let body = r#"
|
||||
# TYPE system_cpu_usage gauge
|
||||
system_cpu_usage 0.25
|
||||
# TYPE jvm_memory_used_bytes gauge
|
||||
jvm_memory_used_bytes{area="heap",id="a"} 100
|
||||
jvm_memory_used_bytes{area="heap",id="b"} 50
|
||||
# TYPE jvm_memory_max_bytes gauge
|
||||
jvm_memory_max_bytes{area="heap",id="a"} 400
|
||||
# TYPE http_server_requests_seconds counter
|
||||
http_server_requests_seconds_count{status="200"} 90
|
||||
http_server_requests_seconds_count{status="500"} 10
|
||||
http_server_requests_seconds_sum 25
|
||||
# TYPE nacos_monitor gauge
|
||||
nacos_monitor{module="naming",name="serviceCount"} 12
|
||||
nacos_monitor{module="core",name="longConnection"} 8
|
||||
nacos_monitor{module="naming",name="pushPendingTaskCount"} 3
|
||||
# TYPE nacos_naming_subscriber gauge
|
||||
nacos_naming_subscriber{version="v1"} 2
|
||||
nacos_naming_subscriber{version="v2"} 4
|
||||
"#;
|
||||
let parsed = parse_scrape(body, &NacosImplementation::Nacos, "http://localhost/metrics").unwrap();
|
||||
assert_eq!(parsed.resource.cpu_ratio, Some(0.25));
|
||||
assert_eq!(parsed.resource.memory_used_bytes, Some(150.0));
|
||||
assert_eq!(parsed.resource.memory_ratio, Some(0.375));
|
||||
assert_eq!(parsed.traffic.http_requests_total, Some(100.0));
|
||||
assert_eq!(parsed.traffic.http_errors_total, Some(10.0));
|
||||
assert_eq!(parsed.naming.service_count, Some(12.0));
|
||||
assert_eq!(parsed.naming.connection_count, Some(8.0));
|
||||
assert_eq!(parsed.naming.subscriber_count, Some(6.0));
|
||||
assert_eq!(parsed.naming.push_pending_tasks, Some(3.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aggregates_histogram_buckets_across_label_sets() {
|
||||
let body = r#"
|
||||
# TYPE http_server_requests_seconds histogram
|
||||
http_server_requests_seconds_bucket{route="a",le="1"} 50
|
||||
http_server_requests_seconds_bucket{route="a",le="10"} 50
|
||||
http_server_requests_seconds_bucket{route="a",le="+Inf"} 50
|
||||
http_server_requests_seconds_sum{route="a"} 25
|
||||
http_server_requests_seconds_count{route="a"} 50
|
||||
http_server_requests_seconds_bucket{route="b",le="1"} 0
|
||||
http_server_requests_seconds_bucket{route="b",le="10"} 50
|
||||
http_server_requests_seconds_bucket{route="b",le="+Inf"} 50
|
||||
http_server_requests_seconds_sum{route="b"} 250
|
||||
http_server_requests_seconds_count{route="b"} 50
|
||||
"#;
|
||||
let parsed = parse_scrape(body, &NacosImplementation::Nacos, "http://localhost/metrics").unwrap();
|
||||
assert_eq!(parsed.traffic.http_p50_ms, Some(1_000.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redacts_endpoint_query_from_snapshot_source() {
|
||||
let parsed = parse_scrape(
|
||||
"# TYPE system_cpu_usage gauge\nsystem_cpu_usage 0.25\n",
|
||||
&NacosImplementation::Nacos,
|
||||
"http://localhost/metrics?token=secret&node=a",
|
||||
)
|
||||
.unwrap();
|
||||
let other_source = parse_scrape(
|
||||
"# TYPE system_cpu_usage gauge\nsystem_cpu_usage 0.25\n",
|
||||
&NacosImplementation::Nacos,
|
||||
"http://localhost/metrics?token=secret&node=b",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(parsed.source.endpoint, "http://localhost/metrics");
|
||||
assert_ne!(parsed.source.fingerprint, other_source.source.fingerprint);
|
||||
assert!(!parsed.source.fingerprint.as_deref().unwrap().contains("secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_rnacos_units_counts_and_summary() {
|
||||
let body = r#"
|
||||
# TYPE app_cpu_usage gauge
|
||||
app_cpu_usage 12.5
|
||||
# TYPE app_rss_memory gauge
|
||||
app_rss_memory 10
|
||||
# TYPE sys_total_memory gauge
|
||||
sys_total_memory 100
|
||||
# TYPE naming_service_size gauge
|
||||
naming_service_size 4
|
||||
# TYPE grpc_conn_size gauge
|
||||
grpc_conn_size 3
|
||||
# TYPE http_request_total_count counter
|
||||
http_request_total_count 20
|
||||
# TYPE http_request_handle_rt_summary summary
|
||||
http_request_handle_rt_summary{quantile="0.5"} 5
|
||||
http_request_handle_rt_summary{quantile="0.95"} 15
|
||||
http_request_handle_rt_summary_sum 100
|
||||
http_request_handle_rt_summary_count 20
|
||||
"#;
|
||||
let parsed = parse_scrape(body, &NacosImplementation::RNacos, "http://localhost/metrics").unwrap();
|
||||
assert_eq!(parsed.resource.cpu_ratio, Some(0.125));
|
||||
assert_eq!(parsed.resource.rss_bytes, Some(10.0 * 1024.0 * 1024.0));
|
||||
assert_eq!(parsed.resource.memory_ratio, Some(0.1));
|
||||
assert_eq!(parsed.naming.service_count, Some(4.0));
|
||||
assert_eq!(parsed.naming.connection_count, Some(3.0));
|
||||
assert_eq!(parsed.traffic.http_p95_ms, Some(15.0));
|
||||
assert_eq!(parsed.traffic.http_duration_seconds_total, Some(0.1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn converts_low_rnacos_percentages_to_ratios() {
|
||||
let body = r#"
|
||||
# TYPE app_cpu_usage gauge
|
||||
app_cpu_usage 0.5
|
||||
# TYPE app_memory_usage gauge
|
||||
app_memory_usage 1
|
||||
"#;
|
||||
let parsed = parse_scrape(body, &NacosImplementation::RNacos, "http://localhost/metrics").unwrap();
|
||||
assert_eq!(parsed.resource.cpu_ratio, Some(0.005));
|
||||
assert_eq!(parsed.resource.memory_ratio, Some(0.01));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_responses_without_samples() {
|
||||
assert!(parse_scrape("<html>not metrics</html>", &NacosImplementation::Nacos, "http://localhost/metrics")
|
||||
.unwrap_err()
|
||||
.contains("no Prometheus samples"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn falls_back_after_a_missing_auto_endpoint() {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
for index in 0..2 {
|
||||
let (mut socket, _) = listener.accept().await.unwrap();
|
||||
let mut request = vec![0_u8; 2048];
|
||||
let _ = tokio::io::AsyncReadExt::read(&mut socket, &mut request).await.unwrap();
|
||||
if index == 0 {
|
||||
let body = "not found";
|
||||
let response = format!(
|
||||
"HTTP/1.1 404 Not Found\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
);
|
||||
tokio::io::AsyncWriteExt::write_all(&mut socket, response.as_bytes()).await.unwrap();
|
||||
} else {
|
||||
let body = "# TYPE system_cpu_usage gauge\nsystem_cpu_usage 0.4\n";
|
||||
let response = format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
|
||||
body.len()
|
||||
);
|
||||
tokio::io::AsyncWriteExt::write_all(&mut socket, response.as_bytes()).await.unwrap();
|
||||
}
|
||||
}
|
||||
});
|
||||
let cfg = test_config(NacosImplementation::Nacos, &format!("http://{address}"), "");
|
||||
let client = reqwest::Client::new();
|
||||
let parsed = scrape(&client, &cfg).await.unwrap().unwrap();
|
||||
assert_eq!(parsed.resource.cpu_ratio, Some(0.4));
|
||||
assert!(parsed.source.endpoint.ends_with("/nacos/actuator/prometheus"));
|
||||
server.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejects_oversized_scrapes() {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut socket, _) = listener.accept().await.unwrap();
|
||||
let mut request = vec![0_u8; 2048];
|
||||
let _ = tokio::io::AsyncReadExt::read(&mut socket, &mut request).await.unwrap();
|
||||
let body = vec![b'x'; MAX_SCRAPE_BYTES + 1];
|
||||
let header = format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
|
||||
body.len()
|
||||
);
|
||||
let _ = tokio::io::AsyncWriteExt::write_all(&mut socket, header.as_bytes()).await;
|
||||
let _ = tokio::io::AsyncWriteExt::write_all(&mut socket, &body).await;
|
||||
});
|
||||
let mut cfg = test_config(NacosImplementation::Nacos, "http://127.0.0.1:8848", "");
|
||||
cfg.metrics_mode = NacosMetricsMode::Custom;
|
||||
cfg.metrics_url = format!("http://{address}/metrics");
|
||||
let error = scrape(&reqwest::Client::new(), &cfg).await.unwrap_err();
|
||||
assert!(error.contains("exceeds"));
|
||||
server.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn redacts_endpoint_query_from_scrape_errors() {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut socket, _) = listener.accept().await.unwrap();
|
||||
let mut request = vec![0_u8; 2048];
|
||||
let _ = tokio::io::AsyncReadExt::read(&mut socket, &mut request).await.unwrap();
|
||||
let response = "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n";
|
||||
tokio::io::AsyncWriteExt::write_all(&mut socket, response.as_bytes()).await.unwrap();
|
||||
});
|
||||
let mut cfg = test_config(NacosImplementation::Nacos, "http://127.0.0.1:8848", "");
|
||||
cfg.metrics_mode = NacosMetricsMode::Custom;
|
||||
cfg.metrics_url = format!("http://{address}/metrics?token=secret");
|
||||
let error = scrape(&reqwest::Client::new(), &cfg).await.unwrap_err();
|
||||
assert!(error.contains(&format!("http://{address}/metrics")));
|
||||
assert!(!error.contains("secret"));
|
||||
server.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn times_out_without_exposing_endpoint_query() {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut socket, _) = listener.accept().await.unwrap();
|
||||
let mut request = vec![0_u8; 2048];
|
||||
let _ = tokio::io::AsyncReadExt::read(&mut socket, &mut request).await.unwrap();
|
||||
std::future::pending::<()>().await;
|
||||
});
|
||||
let mut cfg = test_config(NacosImplementation::Nacos, "http://127.0.0.1:8848", "");
|
||||
cfg.metrics_mode = NacosMetricsMode::Custom;
|
||||
cfg.metrics_url = format!("http://{address}/metrics?token=secret");
|
||||
let error = scrape_with_timeout(&reqwest::Client::new(), &cfg, Duration::from_millis(25)).await.unwrap_err();
|
||||
assert!(error.contains("timed out"));
|
||||
assert!(error.contains(&format!("http://{address}/metrics")));
|
||||
assert!(!error.contains("secret"));
|
||||
server.abort();
|
||||
}
|
||||
|
||||
fn test_config(implementation: NacosImplementation, server_addr: &str, context_path: &str) -> NacosAdminConfig {
|
||||
NacosAdminConfig {
|
||||
implementation: Some(implementation),
|
||||
version_mode: None,
|
||||
server_addr: server_addr.to_string(),
|
||||
display_server_addr: server_addr.to_string(),
|
||||
namespace: String::new(),
|
||||
context_path: context_path.to_string(),
|
||||
rnacos_console_addr: String::new(),
|
||||
rnacos_history_enabled: None,
|
||||
rnacos_console_auth: Default::default(),
|
||||
auth: Default::default(),
|
||||
tls_skip_verify: false,
|
||||
metrics_mode: NacosMetricsMode::Auto,
|
||||
metrics_url: String::new(),
|
||||
page_size: 20,
|
||||
connect_override: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -699,6 +699,9 @@ mod tests {
|
||||
async fn update_instance(&self, _: NacosInstanceUpdate) -> Result<(), String> {
|
||||
Err("unused".to_string())
|
||||
}
|
||||
async fn get_dashboard(&self, _: NacosDashboardQuery) -> Result<NacosDashboardSnapshot, String> {
|
||||
Err("unused".to_string())
|
||||
}
|
||||
async fn raw_request(&self, _: NacosRawRequest) -> Result<NacosRawResponse, String> {
|
||||
Err("unused".to_string())
|
||||
}
|
||||
|
||||
@@ -162,6 +162,15 @@ pub async fn nacos_update_instance_core(
|
||||
admin.update_instance(req).await
|
||||
}
|
||||
|
||||
pub async fn nacos_get_dashboard_core(
|
||||
state: &AppState,
|
||||
conn_id: &str,
|
||||
query: NacosDashboardQuery,
|
||||
) -> Result<NacosDashboardSnapshot, String> {
|
||||
let admin = get_admin(state, conn_id).await?;
|
||||
admin.get_dashboard(query).await
|
||||
}
|
||||
|
||||
pub async fn nacos_raw_request_core(
|
||||
state: &AppState,
|
||||
conn_id: &str,
|
||||
|
||||
@@ -496,6 +496,175 @@ pub struct NacosInstanceUpdate {
|
||||
pub metadata: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct NacosDashboardQuery {
|
||||
#[serde(default)]
|
||||
pub namespace: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct NacosDashboardMetrics {
|
||||
#[serde(default)]
|
||||
pub status: Option<String>,
|
||||
#[serde(default)]
|
||||
pub service_count: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub instance_count: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub subscribe_count: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub raft_notify_task_count: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub responsible_service_count: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub responsible_instance_count: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub client_count: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub connection_based_client_count: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub ephemeral_ip_port_client_count: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub persistent_ip_port_client_count: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub responsible_client_count: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub cpu: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub load: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub mem: Option<f64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct NacosPrometheusSource {
|
||||
pub kind: String,
|
||||
pub endpoint: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub fingerprint: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct NacosPrometheusResourceMetrics {
|
||||
pub cpu_ratio: Option<f64>,
|
||||
pub memory_ratio: Option<f64>,
|
||||
pub memory_used_bytes: Option<f64>,
|
||||
pub memory_max_bytes: Option<f64>,
|
||||
pub rss_bytes: Option<f64>,
|
||||
pub vms_bytes: Option<f64>,
|
||||
pub system_total_memory_bytes: Option<f64>,
|
||||
pub load_1m: Option<f64>,
|
||||
pub jvm_daemon_threads: Option<f64>,
|
||||
pub gc_pause_count: Option<f64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct NacosPrometheusTrafficMetrics {
|
||||
pub http_requests_total: Option<f64>,
|
||||
pub grpc_requests_total: Option<f64>,
|
||||
pub http_errors_total: Option<f64>,
|
||||
pub grpc_errors_total: Option<f64>,
|
||||
pub http_duration_seconds_total: Option<f64>,
|
||||
pub http_duration_count: Option<f64>,
|
||||
pub grpc_duration_seconds_total: Option<f64>,
|
||||
pub grpc_duration_count: Option<f64>,
|
||||
pub http_p50_ms: Option<f64>,
|
||||
pub http_p95_ms: Option<f64>,
|
||||
pub http_p99_ms: Option<f64>,
|
||||
pub grpc_p50_ms: Option<f64>,
|
||||
pub grpc_p95_ms: Option<f64>,
|
||||
pub grpc_p99_ms: Option<f64>,
|
||||
pub executor_pool_size: Option<f64>,
|
||||
pub executor_active_count: Option<f64>,
|
||||
pub executor_queued_tasks: Option<f64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct NacosPrometheusConfigMetrics {
|
||||
pub config_count: Option<f64>,
|
||||
pub get_config_total: Option<f64>,
|
||||
pub publish_total: Option<f64>,
|
||||
pub long_polling: Option<f64>,
|
||||
pub listener_clients: Option<f64>,
|
||||
pub listener_keys: Option<f64>,
|
||||
pub notify_tasks: Option<f64>,
|
||||
pub notify_client_tasks: Option<f64>,
|
||||
pub dump_tasks: Option<f64>,
|
||||
pub subscriber_count: Option<f64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct NacosPrometheusNamingMetrics {
|
||||
pub service_count: Option<f64>,
|
||||
pub instance_count: Option<f64>,
|
||||
pub subscriber_count: Option<f64>,
|
||||
pub connection_count: Option<f64>,
|
||||
pub total_push: Option<f64>,
|
||||
pub failed_push: Option<f64>,
|
||||
pub empty_push: Option<f64>,
|
||||
pub push_pending_tasks: Option<f64>,
|
||||
pub avg_push_cost_ms: Option<f64>,
|
||||
pub max_push_cost_ms: Option<f64>,
|
||||
pub leader_status: Option<f64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct NacosPrometheusSnapshot {
|
||||
pub source: NacosPrometheusSource,
|
||||
pub resource: NacosPrometheusResourceMetrics,
|
||||
pub traffic: NacosPrometheusTrafficMetrics,
|
||||
pub config: NacosPrometheusConfigMetrics,
|
||||
pub naming: NacosPrometheusNamingMetrics,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct NacosClusterNode {
|
||||
pub address: String,
|
||||
#[serde(default)]
|
||||
pub ip: Option<String>,
|
||||
#[serde(default)]
|
||||
pub port: Option<u16>,
|
||||
#[serde(default)]
|
||||
pub state: Option<String>,
|
||||
#[serde(default)]
|
||||
pub alive: Option<bool>,
|
||||
#[serde(default)]
|
||||
pub site: Option<String>,
|
||||
#[serde(default)]
|
||||
pub weight: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub last_refresh_time: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct NacosDashboardSnapshot {
|
||||
pub namespace: String,
|
||||
#[serde(default)]
|
||||
pub namespace_count: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub config_count: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub service_count: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub metrics: Option<NacosDashboardMetrics>,
|
||||
#[serde(default)]
|
||||
pub prometheus: Option<NacosPrometheusSnapshot>,
|
||||
#[serde(default)]
|
||||
pub nodes: Vec<NacosClusterNode>,
|
||||
#[serde(default)]
|
||||
pub warnings: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct NacosRawRequest {
|
||||
|
||||
Reference in new issue
Block a user