feat(nacos): add Nacos dashboard metrics

This commit is contained in:
二丫讲梵 authored and GitHub committed 2026-07-27 12:16:39 +08:00
1 parent 98bf3d1dfd
commit f9c2fb27d3
41 files changed
+3268 -36

No files matched your search

+4
View File
@@ -626,6 +626,10 @@ mod tests {
Err("unused".to_string())
}
async fn get_dashboard(&self, _: NacosDashboardQuery) -> Result<NacosDashboardSnapshot, String> {
Err("unused".to_string())
}
async fn raw_request(&self, _: NacosRawRequest) -> Result<NacosRawResponse, String> {
Err("unused".to_string())
}
+111
View File
@@ -32,6 +32,15 @@ pub enum NacosVersionMode {
V3,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
#[serde(rename_all = "camelCase")]
pub enum NacosMetricsMode {
#[default]
Auto,
Disabled,
Custom,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
#[serde(tag = "kind", rename_all = "camelCase")]
pub enum NacosRNacosConsoleAuth {
@@ -73,6 +82,10 @@ pub struct NacosAdminConfig {
pub auth: NacosAuthConfig,
#[serde(default)]
pub tls_skip_verify: bool,
#[serde(default)]
pub metrics_mode: NacosMetricsMode,
#[serde(default, skip_serializing_if = "String::is_empty")]
pub metrics_url: String,
#[serde(default = "default_page_size")]
pub page_size: u32,
#[serde(skip)]
@@ -113,6 +126,8 @@ impl NacosAdminConfig {
NacosAuthConfig::UsernamePassword { username: cfg.username.clone(), password: cfg.password.clone() }
},
tls_skip_verify: false,
metrics_mode: NacosMetricsMode::Auto,
metrics_url: String::new(),
page_size: default_page_size(),
connect_override: None,
}
@@ -130,7 +145,19 @@ impl NacosAdminConfig {
} else {
self.display_server_addr = normalize_endpoint_url(&self.display_server_addr, "Nacos display address")?;
}
let context_path_is_explicit_root = self.context_path.trim() == "/";
self.context_path = normalize_context_path(&self.context_path);
// Nacos 3 separates the console (normally :8080) from the server-side
// Admin API (normally :8848/nacos). Older DBX connection records did
// not persist the default server context, so repair only explicit
// Nacos 3 profiles here while preserving custom contexts.
if self.context_path.is_empty()
&& !context_path_is_explicit_root
&& matches!(self.implementation, Some(NacosImplementation::Nacos))
&& matches!(self.version_mode, Some(NacosVersionMode::V3))
{
self.context_path = "/nacos".to_string();
}
self.rnacos_console_addr = if self.rnacos_console_addr.trim().is_empty() {
String::new()
} else {
@@ -144,6 +171,10 @@ impl NacosAdminConfig {
return Err("r-nacos console username is empty".to_string());
}
}
self.metrics_url = match self.metrics_mode {
NacosMetricsMode::Custom => normalize_metrics_url(&self.metrics_url)?,
NacosMetricsMode::Auto | NacosMetricsMode::Disabled => String::new(),
};
if self.page_size == 0 {
self.page_size = default_page_size();
}
@@ -206,6 +237,24 @@ impl NacosAdminConfig {
}
}
fn normalize_metrics_url(value: &str) -> Result<String, String> {
let value = value.trim();
if value.is_empty() {
return Err("Nacos Prometheus metrics URL is empty".to_string());
}
let url = reqwest::Url::parse(value).map_err(|e| format!("Nacos Prometheus metrics URL is invalid: {e}"))?;
if !matches!(url.scheme(), "http" | "https") {
return Err("Nacos Prometheus metrics URL must use http or https".to_string());
}
if !url.username().is_empty() || url.password().is_some() {
return Err("Nacos Prometheus metrics URL must not contain embedded credentials".to_string());
}
if url.fragment().is_some() {
return Err("Nacos Prometheus metrics URL must not contain a fragment".to_string());
}
Ok(url.to_string())
}
fn normalize_endpoint_url(value: &str, label: &str) -> Result<String, String> {
let mut url = reqwest::Url::parse(value.trim()).map_err(|e| format!("{label} is invalid: {e}"))?;
if !url.username().is_empty() || url.password().is_some() {
@@ -336,6 +385,41 @@ mod tests {
assert!(err.contains("must not contain embedded credentials"));
}
#[test]
fn defaults_prometheus_metrics_to_auto_and_validates_custom_urls() {
let parsed = NacosAdminConfig::from_connection(&connection_with_external(serde_json::json!({
"serverAddr": "http://127.0.0.1:8818"
})))
.unwrap();
assert_eq!(parsed.metrics_mode, NacosMetricsMode::Auto);
assert!(parsed.metrics_url.is_empty());
let parsed = NacosAdminConfig::from_connection(&connection_with_external(serde_json::json!({
"serverAddr": "http://127.0.0.1:8848",
"metricsMode": "custom",
"metricsUrl": "http://127.0.0.1:8818/nacos/actuator/prometheus?node=a"
})))
.unwrap();
assert_eq!(parsed.metrics_mode, NacosMetricsMode::Custom);
assert_eq!(parsed.metrics_url, "http://127.0.0.1:8818/nacos/actuator/prometheus?node=a");
for metrics_url in [
"",
"file:///tmp/metrics",
"http://user:secret@127.0.0.1:8818/metrics",
"http://127.0.0.1:8818/metrics#private",
"http://127.0.0.1:8818/metrics#",
] {
let error = NacosAdminConfig::from_connection(&connection_with_external(serde_json::json!({
"serverAddr": "http://127.0.0.1:8848",
"metricsMode": "custom",
"metricsUrl": metrics_url
})))
.unwrap_err();
assert!(error.contains("Prometheus metrics URL"));
}
}
#[test]
fn missing_external_context_path_defaults_to_root() {
let cfg = connection_with_external(serde_json::json!({
@@ -347,6 +431,33 @@ mod tests {
assert_eq!(parsed.context_path, "");
}
#[test]
fn explicit_nacos_v3_defaults_to_server_admin_context() {
let cfg = connection_with_external(serde_json::json!({
"implementation": "nacos",
"versionMode": "v3",
"serverAddr": "http://127.0.0.1:8848",
"auth": { "kind": "none" }
}));
let parsed = NacosAdminConfig::from_connection(&cfg).unwrap();
assert_eq!(parsed.context_path, "/nacos");
}
#[test]
fn explicit_nacos_v3_root_context_is_preserved() {
let cfg = connection_with_external(serde_json::json!({
"implementation": "nacos",
"versionMode": "v3",
"serverAddr": "http://127.0.0.1:8848",
"contextPath": "/",
"auth": { "kind": "none" }
}));
let parsed = NacosAdminConfig::from_connection(&cfg).unwrap();
assert_eq!(parsed.context_path, "");
}
#[test]
fn falls_back_to_connection_fields() {
let mut cfg = connection_with_external(serde_json::Value::Null);
+501 -3
View File
@@ -597,7 +597,7 @@ impl NacosOpenApiAdmin {
fn api_path_allowed(&self, path: &str) -> bool {
match self.cfg.version_mode.as_ref() {
Some(NacosVersionMode::V2) => !path.starts_with("/v3/"),
Some(NacosVersionMode::V3) => !path.starts_with("/v1/"),
Some(NacosVersionMode::V3) => !path.starts_with("/v1/") && !path.starts_with("/v2/"),
Some(NacosVersionMode::Auto) | None => true,
}
}
@@ -834,6 +834,39 @@ impl NacosOpenApiAdmin {
instances.retain(|instance| seen.insert((instance.ip.clone(), instance.port, instance.cluster_name.clone())));
Ok(instances)
}
async fn get_dashboard_nodes(&self) -> Result<Vec<NacosClusterNode>, String> {
// r-nacos does not implement the official Nacos cluster-node Admin
// APIs. An empty list represents this unsupported optional capability
// without turning every dashboard refresh into a false warning.
if self.is_explicit_rnacos() {
return Ok(Vec::new());
}
self.get_json_from_candidates(
"load Nacos cluster nodes",
vec![
("/v3/admin/core/cluster/node/list", Vec::new()),
("/v2/core/cluster/node/list", Vec::new()),
("/v1/core/cluster/nodes", Vec::new()),
("/v1/ns/operator/servers", Vec::new()),
],
)
.await
.map(parse_cluster_nodes)
}
fn dashboard_warning(&self, error: String) -> String {
if matches!(self.cfg.version_mode, Some(NacosVersionMode::V3))
&& (error.contains("[contextPathMismatch]") || error.contains("[apiVersionMismatch]"))
{
return format!(
"{error} Check the connection address: Nacos 3 dashboard APIs use the Server / Admin API endpoint \
(normally http://host:8848/nacos), not the port 8080 Console."
);
}
error
}
}
fn qualified_nacos_service_name(service_name: &str, group_name: Option<&str>) -> String {
@@ -1570,6 +1603,102 @@ impl NacosAdmin for NacosOpenApiAdmin {
.await
}
async fn get_dashboard(&self, query: NacosDashboardQuery) -> Result<NacosDashboardSnapshot, String> {
let namespace = self.namespace(query.namespace.as_deref());
let metrics_future = self.get_json_from_candidates(
"load Nacos dashboard metrics",
vec![
("/v3/admin/ns/ops/metrics", vec![("onlyStatus".to_string(), "false".to_string())]),
("/v2/ns/operator/metrics", vec![("onlyStatus".to_string(), "false".to_string())]),
("/v1/ns/operator/metrics", Vec::new()),
],
);
let nodes_future = self.get_dashboard_nodes();
let namespaces_future = self.list_namespaces();
let configs_future = self.list_configs(NacosConfigQuery {
namespace: Some(namespace.clone()),
group: None,
data_id: None,
app_name: None,
search: None,
page_no: Some(1),
page_size: Some(1),
});
let services_future = self.list_services(NacosServiceQuery {
namespace: Some(namespace.clone()),
group_name: None,
service_name: None,
page_no: Some(1),
page_size: Some(1),
});
let prometheus_future = crate::nacos::prometheus::scrape(&self.http, &self.cfg);
let (metrics_result, nodes_result, namespaces_result, configs_result, services_result, prometheus_result) = tokio::join!(
metrics_future,
nodes_future,
namespaces_future,
configs_future,
services_future,
prometheus_future
);
let mut warnings = Vec::new();
let mut metrics = match metrics_result {
Ok(value) => Some(parse_dashboard_metrics(value)),
Err(error) => {
warnings.push(self.dashboard_warning(error));
None
}
};
let nodes = match nodes_result {
Ok(value) => value,
Err(error) => {
warnings.push(self.dashboard_warning(error));
Vec::new()
}
};
let namespace_count = match namespaces_result {
Ok(items) => Some(items.len() as u64),
Err(error) => {
warnings.push(error);
None
}
};
let config_count = match configs_result {
Ok(result) => Some(result.total_count),
Err(error) => {
warnings.push(error);
None
}
};
let service_count = match services_result {
Ok(result) => Some(result.total_count),
Err(error) => {
warnings.push(error);
None
}
};
let prometheus = match prometheus_result {
Ok(value) => value,
Err(error) => {
warnings.push(error);
None
}
};
merge_prometheus_dashboard(&mut metrics, prometheus.as_ref());
Ok(NacosDashboardSnapshot {
namespace,
namespace_count,
config_count,
service_count,
metrics,
prometheus,
nodes,
warnings,
})
}
async fn raw_request(&self, req: NacosRawRequest) -> Result<NacosRawResponse, String> {
validate_raw_api_path(&req.path)?;
let method = reqwest::Method::from_bytes(req.method.to_ascii_uppercase().as_bytes())
@@ -1660,6 +1789,107 @@ fn parse_namespaces(value: Value) -> Vec<NacosNamespaceInfo> {
namespaces
}
fn parse_dashboard_metrics(value: Value) -> NacosDashboardMetrics {
let data = value.get("data").unwrap_or(&value);
NacosDashboardMetrics {
status: optional_string_field(data, &["status"])
.or_else(|| data.as_str().map(str::to_string))
.filter(|value| !value.trim().is_empty()),
service_count: optional_u64_field(data, &["serviceCount"]),
instance_count: optional_u64_field(data, &["instanceCount"]),
subscribe_count: optional_u64_field(data, &["subscribeCount"]),
raft_notify_task_count: optional_u64_field(data, &["raftNotifyTaskCount"]),
responsible_service_count: optional_u64_field(data, &["responsibleServiceCount"]),
responsible_instance_count: optional_u64_field(data, &["responsibleInstanceCount"]),
client_count: optional_u64_field(data, &["clientCount"]),
connection_based_client_count: optional_u64_field(data, &["connectionBasedClientCount"]),
ephemeral_ip_port_client_count: optional_u64_field(data, &["ephemeralIpPortClientCount"]),
persistent_ip_port_client_count: optional_u64_field(data, &["persistentIpPortClientCount"]),
responsible_client_count: optional_u64_field(data, &["responsibleClientCount"]),
cpu: optional_f64_field(data, &["cpu"]),
load: optional_f64_field(data, &["load"]),
mem: optional_f64_field(data, &["mem"]),
}
}
fn merge_prometheus_dashboard(
metrics: &mut Option<NacosDashboardMetrics>,
prometheus: Option<&NacosPrometheusSnapshot>,
) {
let Some(prometheus) = prometheus else {
return;
};
let metrics = metrics.get_or_insert_with(NacosDashboardMetrics::default);
if let Some(value) = finite_u64(prometheus.naming.instance_count) {
metrics.instance_count = Some(value);
}
if let Some(value) = finite_u64(prometheus.naming.subscriber_count) {
metrics.subscribe_count = Some(value);
}
if let Some(value) = finite_u64(prometheus.naming.connection_count) {
metrics.client_count = Some(value);
metrics.connection_based_client_count = Some(value);
}
if let Some(value) = prometheus.resource.cpu_ratio {
metrics.cpu = Some(value);
}
if let Some(value) = prometheus.resource.memory_ratio {
metrics.mem = Some(value);
}
if let Some(value) = prometheus.resource.load_1m {
metrics.load = Some(value);
}
}
fn finite_u64(value: Option<f64>) -> Option<u64> {
value.filter(|value| value.is_finite() && *value >= 0.0 && *value <= u64::MAX as f64).map(|value| value as u64)
}
fn parse_cluster_nodes(value: Value) -> Vec<NacosClusterNode> {
let data = value.get("data").unwrap_or(&value);
let items = data
.as_array()
.cloned()
.or_else(|| data.get("servers").and_then(Value::as_array).cloned())
.or_else(|| data.get("members").and_then(Value::as_array).cloned())
.or_else(|| data.get("nodes").and_then(Value::as_array).cloned())
.or_else(|| data.get("pageItems").and_then(Value::as_array).cloned())
.or_else(|| value.get("servers").and_then(Value::as_array).cloned())
.unwrap_or_default();
items
.into_iter()
.map(|item| {
let ip = optional_string_field(&item, &["ip"]);
let port = optional_u64_field(&item, &["port", "servePort"]).and_then(|port| u16::try_from(port).ok());
let address = optional_string_field(&item, &["address", "key"])
.filter(|value| !value.trim().is_empty())
.unwrap_or_else(|| match (&ip, port) {
(Some(ip), Some(port)) => format!("{ip}:{port}"),
(Some(ip), None) => ip.clone(),
_ => "-".to_string(),
});
let state = optional_string_field(&item, &["state", "status"]);
let alive = optional_bool_field(&item, &["alive", "healthy"]).or_else(|| {
state.as_ref().map(|state| matches!(state.to_ascii_uppercase().as_str(), "UP" | "ONLINE" | "HEALTHY"))
});
NacosClusterNode {
address,
ip,
port,
state,
alive,
site: optional_string_field(&item, &["site"]),
weight: optional_f64_field(&item, &["weight", "adWeight"]),
last_refresh_time: optional_string_field(&item, &["lastRefreshTime", "lastRefTimeStr", "lastRefTime"])
.or_else(|| {
item.get("extendInfo")
.and_then(|extend_info| optional_string_field(extend_info, &["lastRefreshTime"]))
}),
}
})
.collect()
}
fn normalize_api_path(path: &str) -> String {
let trimmed = path.trim();
if trimmed.starts_with('/') {
@@ -1759,10 +1989,34 @@ async fn error_for_status(resp: reqwest::Response, path: &str) -> Result<reqwest
return Ok(resp);
}
let detail = resp.text().await.unwrap_or_default();
let message = format!("Nacos admin {path} returned {status}: {}", detail.trim());
let detail = compact_response_detail(&detail);
let message = if detail.is_empty() {
format!("Nacos admin {path} returned {status}")
} else {
format!("Nacos admin {path} returned {status}: {detail}")
};
Err(classified_error(classify_nacos_error(&message), &message))
}
fn compact_response_detail(detail: &str) -> String {
let detail = detail.trim();
if detail.is_empty() {
return String::new();
}
if detail.to_ascii_lowercase().contains("<!doctype html") || detail.to_ascii_lowercase().contains("<html") {
return "HTML error page".to_string();
}
const MAX_CHARS: usize = 512;
let mut chars = detail.chars();
let compact: String = chars.by_ref().take(MAX_CHARS).collect();
if chars.next().is_some() {
format!("{compact}…")
} else {
compact
}
}
fn classified_error(kind: &str, message: &str) -> String {
format!("{NACOS_ERROR_PREFIX}[{kind}]: {message}")
}
@@ -2188,7 +2442,29 @@ fn normalize_config_format(value: String) -> String {
}
fn optional_u64_field(value: &Value, keys: &[&str]) -> Option<u64> {
keys.iter().find_map(|key| value.get(*key)).and_then(Value::as_u64)
keys.iter().find_map(|key| value.get(*key)).and_then(|value| {
value
.as_u64()
.or_else(|| value.as_i64().and_then(|value| u64::try_from(value).ok()))
.or_else(|| value.as_str().and_then(|value| value.trim().parse().ok()))
})
}
fn optional_f64_field(value: &Value, keys: &[&str]) -> Option<f64> {
keys.iter()
.find_map(|key| value.get(*key))
.and_then(|value| value.as_f64().or_else(|| value.as_str().and_then(|value| value.trim().parse().ok())))
.filter(|value| value.is_finite())
}
fn optional_bool_field(value: &Value, keys: &[&str]) -> Option<bool> {
keys.iter().find_map(|key| value.get(*key)).and_then(|value| {
value.as_bool().or_else(|| match value.as_str()?.trim().to_ascii_lowercase().as_str() {
"true" | "up" | "online" | "healthy" => Some(true),
"false" | "down" | "offline" | "unhealthy" => Some(false),
_ => None,
})
})
}
fn optional_i64_field(value: &Value, keys: &[&str]) -> Option<i64> {
@@ -2311,6 +2587,8 @@ mod tests {
rnacos_console_auth: Default::default(),
auth: NacosAuthConfig::None,
tls_skip_verify: false,
metrics_mode: Default::default(),
metrics_url: String::new(),
page_size: 100,
connect_override: None,
}
@@ -2334,6 +2612,124 @@ mod tests {
server.await.unwrap();
}
#[test]
fn explicit_v3_rejects_legacy_admin_api_paths() {
let mut config = test_admin_config("http://127.0.0.1:8848".to_string());
config.version_mode = Some(NacosVersionMode::V3);
let admin = NacosOpenApiAdmin::new(config).unwrap();
assert!(admin.api_path_allowed("/v3/admin/ns/ops/metrics"));
assert!(admin.api_path_allowed("/health"));
assert!(!admin.api_path_allowed("/v2/ns/operator/metrics"));
assert!(!admin.api_path_allowed("/v1/ns/operator/metrics"));
}
#[test]
fn compacts_html_error_pages_in_admin_warnings() {
assert_eq!(
compact_response_detail("<!doctype html><html><body><h1>HTTP Status 404</h1></body></html>"),
"HTML error page"
);
assert!(compact_response_detail(&"x".repeat(600)).ends_with('…'));
}
#[test]
fn gives_nacos_v3_dashboard_endpoint_guidance() {
let mut config = test_admin_config("http://127.0.0.1:8080".to_string());
config.version_mode = Some(NacosVersionMode::V3);
let admin = NacosOpenApiAdmin::new(config).unwrap();
let warning =
admin.dashboard_warning("NACOS_ERROR[contextPathMismatch]: No static resource v3/admin".to_string());
assert!(warning.contains("http://host:8848/nacos"));
assert!(warning.contains("not the port 8080 Console"));
}
#[test]
fn prometheus_metrics_preserve_namespace_dashboard_values() {
let mut metrics = Some(NacosDashboardMetrics {
service_count: Some(1),
instance_count: Some(2),
cpu: Some(0.1),
..Default::default()
});
let config_count = Some(3);
let service_count = Some(1);
let prometheus = NacosPrometheusSnapshot {
resource: NacosPrometheusResourceMetrics { cpu_ratio: Some(0.5), ..Default::default() },
config: NacosPrometheusConfigMetrics { config_count: Some(7.0), ..Default::default() },
naming: NacosPrometheusNamingMetrics {
service_count: Some(8.0),
instance_count: Some(9.0),
..Default::default()
},
..Default::default()
};
merge_prometheus_dashboard(&mut metrics, Some(&prometheus));
let metrics = metrics.unwrap();
assert_eq!(config_count, Some(3));
assert_eq!(service_count, Some(1));
assert_eq!(metrics.service_count, Some(1));
assert_eq!(metrics.instance_count, Some(9));
assert_eq!(metrics.cpu, Some(0.5));
}
#[tokio::test]
async fn nacos_v2_dashboard_uses_core_cluster_node_api() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
assert_eq!(read_request_target(&mut socket).await, "/nacos/v2/core/cluster/node/list");
write_json_response(&mut socket, r#"{"code":0,"data":[{"address":"127.0.0.1:8848","state":"UP"}]}"#).await;
});
let mut config = test_admin_config(format!("http://{address}"));
config.context_path = "/nacos".to_string();
config.version_mode = Some(NacosVersionMode::V2);
let admin = NacosOpenApiAdmin::new(config).unwrap();
let nodes = admin.get_dashboard_nodes().await.unwrap();
assert_eq!(nodes.len(), 1);
assert_eq!(nodes[0].address, "127.0.0.1:8848");
server.await.unwrap();
}
#[tokio::test]
async fn nacos_v2_dashboard_falls_back_to_legacy_core_cluster_nodes_api() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
assert_eq!(read_request_target(&mut socket).await, "/nacos/v2/core/cluster/node/list");
write_not_found_response(&mut socket).await;
let (mut socket, _) = listener.accept().await.unwrap();
assert_eq!(read_request_target(&mut socket).await, "/nacos/v1/core/cluster/nodes");
write_json_response(&mut socket, r#"{"code":200,"data":[{"address":"127.0.0.1:8848","state":"UP"}]}"#)
.await;
});
let mut config = test_admin_config(format!("http://{address}"));
config.context_path = "/nacos".to_string();
config.version_mode = Some(NacosVersionMode::V2);
let admin = NacosOpenApiAdmin::new(config).unwrap();
let nodes = admin.get_dashboard_nodes().await.unwrap();
assert_eq!(nodes.len(), 1);
assert_eq!(nodes[0].address, "127.0.0.1:8848");
server.await.unwrap();
}
#[tokio::test]
async fn rnacos_dashboard_skips_unsupported_cluster_node_api() {
let mut config = test_admin_config("http://127.0.0.1:1".to_string());
config.implementation = Some(NacosImplementation::RNacos);
let admin = NacosOpenApiAdmin::new(config).unwrap();
assert!(admin.get_dashboard_nodes().await.unwrap().is_empty());
}
#[tokio::test]
async fn version_mode_v3_uses_only_v3_config_paths() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
@@ -3638,6 +4034,53 @@ mod tests {
server.await.unwrap();
}
#[tokio::test]
async fn dashboard_combines_metrics_nodes_and_namespace_totals() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let mut paths = HashSet::new();
for _ in 0..6 {
let (mut socket, _) = listener.accept().await.unwrap();
let target = read_request_target(&mut socket).await;
let url = reqwest::Url::parse(&format!("http://localhost{target}")).unwrap();
paths.insert(url.path().to_string());
let body = match url.path() {
"/v3/admin/ns/ops/metrics" => {
r#"{"code":0,"data":{"status":"UP","serviceCount":8,"instanceCount":13,"clientCount":5}}"#
}
"/v3/admin/core/cluster/node/list" => {
r#"{"code":0,"data":[{"address":"127.0.0.1:8848","ip":"127.0.0.1","port":8848,"state":"UP"}]}"#
}
"/v3/console/core/namespace/list" => {
r#"{"code":0,"data":{"pageItems":[{"namespaceId":"dev","namespaceName":"Development"}]}}"#
}
"/v3/console/cs/config/list" => r#"{"code":0,"data":{"totalCount":21,"pageItems":[]}}"#,
"/v3/console/ns/service/list" => r#"{"code":0,"data":{"count":3,"serviceList":[]}}"#,
"/actuator/prometheus" => {
"# TYPE system_cpu_usage gauge\nsystem_cpu_usage 0.25\n# TYPE nacos_monitor gauge\nnacos_monitor{module=\"config\",name=\"configCount\"} 12\nnacos_monitor{module=\"naming\",name=\"serviceCount\"} 4\nnacos_monitor{module=\"naming\",name=\"ipCount\"} 14\n"
}
path => panic!("unexpected dashboard request path: {path}"),
};
write_json_response(&mut socket, body).await;
}
paths
});
let admin = NacosOpenApiAdmin::new(test_admin_config(format!("http://{address}"))).unwrap();
let snapshot = admin.get_dashboard(NacosDashboardQuery { namespace: Some("dev".to_string()) }).await.unwrap();
assert_eq!(snapshot.namespace, "dev");
assert_eq!(snapshot.namespace_count, Some(2));
assert_eq!(snapshot.config_count, Some(21));
assert_eq!(snapshot.service_count, Some(3));
assert_eq!(snapshot.metrics.as_ref().and_then(|metrics| metrics.instance_count), Some(14));
assert_eq!(snapshot.prometheus.as_ref().and_then(|metrics| metrics.resource.cpu_ratio), Some(0.25));
assert_eq!(snapshot.nodes.len(), 1);
assert!(snapshot.warnings.is_empty());
assert_eq!(server.await.unwrap().len(), 6);
}
#[test]
fn parses_namespace_list_shape() {
let parsed = parse_namespaces(serde_json::json!({
@@ -3667,6 +4110,61 @@ mod tests {
assert_eq!(parsed[1].namespace_show_name, "Development");
}
#[test]
fn parses_v3_dashboard_metrics_shape() {
let parsed = parse_dashboard_metrics(serde_json::json!({
"code": 0,
"data": {
"status": "UP",
"serviceCount": 12,
"instanceCount": 34,
"clientCount": "5",
"cpu": 0.25,
"mem": "0.5"
}
}));
assert_eq!(parsed.status.as_deref(), Some("UP"));
assert_eq!(parsed.service_count, Some(12));
assert_eq!(parsed.instance_count, Some(34));
assert_eq!(parsed.client_count, Some(5));
assert_eq!(parsed.cpu, Some(0.25));
assert_eq!(parsed.mem, Some(0.5));
let status_only = parse_dashboard_metrics(serde_json::json!({ "code": 0, "data": "UP" }));
assert_eq!(status_only.status.as_deref(), Some("UP"));
}
#[test]
fn parses_v1_and_v3_cluster_node_shapes() {
let v1 = parse_cluster_nodes(serde_json::json!({
"servers": [{
"ip": "192.0.2.1",
"servePort": 8848,
"alive": true,
"site": "unknown",
"lastRefTimeStr": "2026-07-26 10:00:00"
}]
}));
assert_eq!(v1[0].address, "192.0.2.1:8848");
assert_eq!(v1[0].alive, Some(true));
assert_eq!(v1[0].last_refresh_time.as_deref(), Some("2026-07-26 10:00:00"));
let v3 = parse_cluster_nodes(serde_json::json!({
"code": 0,
"data": [{
"address": "192.0.2.2:8848",
"ip": "192.0.2.2",
"port": 8848,
"state": "UP",
"extendInfo": { "lastRefreshTime": 1785031200000_u64 }
}]
}));
assert_eq!(v3[0].address, "192.0.2.2:8848");
assert_eq!(v3[0].alive, Some(true));
assert_eq!(v3[0].last_refresh_time.as_deref(), Some("1785031200000"));
}
#[test]
fn validates_raw_api_paths() {
for path in ["/v1/cs/configs", "/v2/console/example", "/v3/console/server/state"] {
+3
View File
@@ -11,6 +11,7 @@ pub mod batch;
pub mod config;
pub mod http;
pub mod port;
mod prometheus;
pub mod search;
pub mod service;
pub mod types;
@@ -183,6 +184,8 @@ mod tests {
},
auth: NacosAuthConfig::None,
tls_skip_verify: false,
metrics_mode: Default::default(),
metrics_url: String::new(),
page_size: 20,
connect_override: None,
}
+1
View File
@@ -31,5 +31,6 @@ pub trait NacosAdmin: Send + Sync {
async fn list_services(&self, query: NacosServiceQuery) -> Result<NacosServiceList, String>;
async fn list_instances(&self, query: NacosInstanceQuery) -> Result<Vec<NacosInstanceInfo>, String>;
async fn update_instance(&self, req: NacosInstanceUpdate) -> Result<(), String>;
async fn get_dashboard(&self, query: NacosDashboardQuery) -> Result<NacosDashboardSnapshot, String>;
async fn raw_request(&self, req: NacosRawRequest) -> Result<NacosRawResponse, String>;
}
+820
View File
@@ -0,0 +1,820 @@
use std::collections::{HashMap, HashSet};
use std::io;
use std::time::Duration;
use futures::StreamExt;
use prometheus_parse::{Labels, Sample, Scrape, Value};
use sha2::{Digest, Sha256};
use crate::nacos::config::{NacosAdminConfig, NacosImplementation, NacosMetricsMode};
use crate::nacos::types::{
NacosPrometheusConfigMetrics, NacosPrometheusNamingMetrics, NacosPrometheusResourceMetrics,
NacosPrometheusSnapshot, NacosPrometheusSource, NacosPrometheusTrafficMetrics,
};
const SCRAPE_TIMEOUT: Duration = Duration::from_secs(5);
const MAX_SCRAPE_BYTES: usize = 4 * 1024 * 1024;
pub(crate) fn endpoint_candidates(cfg: &NacosAdminConfig) -> Result<Vec<String>, String> {
match cfg.metrics_mode {
NacosMetricsMode::Disabled => return Ok(Vec::new()),
NacosMetricsMode::Custom => return Ok(vec![cfg.metrics_url.clone()]),
NacosMetricsMode::Auto => {}
}
let base = cfg.server_addr.trim_end_matches('/');
let context = cfg.context_path.trim_end_matches('/');
let raw = match cfg.implementation.as_ref().unwrap_or(&NacosImplementation::Nacos) {
NacosImplementation::Nacos => vec![
format!("{base}{context}/actuator/prometheus"),
format!("{base}/nacos/actuator/prometheus"),
format!("{base}/actuator/prometheus"),
],
NacosImplementation::RNacos => {
vec![format!("{base}/metrics"), format!("{base}{context}/metrics"), format!("{base}/rnacos/metrics")]
}
};
let mut seen = HashSet::new();
raw.into_iter()
.map(|candidate| {
reqwest::Url::parse(&candidate)
.map(|url| url.to_string())
.map_err(|error| format!("Nacos Prometheus metrics URL is invalid: {error}"))
})
.filter(|candidate| match candidate {
Ok(value) => seen.insert(value.clone()),
Err(_) => true,
})
.collect()
}
pub(crate) async fn scrape(
client: &reqwest::Client,
cfg: &NacosAdminConfig,
) -> Result<Option<NacosPrometheusSnapshot>, String> {
scrape_with_timeout(client, cfg, SCRAPE_TIMEOUT).await
}
async fn scrape_with_timeout(
client: &reqwest::Client,
cfg: &NacosAdminConfig,
timeout: Duration,
) -> Result<Option<NacosPrometheusSnapshot>, String> {
let candidates = endpoint_candidates(cfg)?;
if candidates.is_empty() {
return Ok(None);
}
let candidate_summary =
candidates.iter().map(|candidate| redact_endpoint(candidate)).collect::<Vec<_>>().join(", ");
tokio::time::timeout(timeout, async {
let mut errors = Vec::new();
for endpoint in candidates {
match scrape_endpoint(client, cfg, &endpoint).await {
Ok(snapshot) => return Ok(Some(snapshot)),
Err(error) => errors.push(format!("{}: {error}", redact_endpoint(&endpoint))),
}
}
Err(format!("Prometheus metrics unavailable: {}", errors.join("; ")))
})
.await
.map_err(|_| {
format!("Prometheus metrics unavailable: scrape timed out after {timeout:?} while trying {candidate_summary}")
})?
}
async fn scrape_endpoint(
client: &reqwest::Client,
cfg: &NacosAdminConfig,
endpoint: &str,
) -> Result<NacosPrometheusSnapshot, String> {
let response = client
.get(endpoint)
.timeout(SCRAPE_TIMEOUT)
.send()
.await
.map_err(|error| format!("request failed: {}", error.without_url()))?;
if !response.status().is_success() {
return Err(format!("returned HTTP {}", response.status()));
}
let mut bytes = Vec::new();
let mut stream = response.bytes_stream();
while let Some(chunk) = stream.next().await {
let chunk = chunk.map_err(|error| format!("failed to read response: {}", error.without_url()))?;
if bytes.len().saturating_add(chunk.len()) > MAX_SCRAPE_BYTES {
return Err(format!("response exceeds {MAX_SCRAPE_BYTES} bytes"));
}
bytes.extend_from_slice(&chunk);
}
let body = String::from_utf8(bytes).map_err(|_| "response is not valid UTF-8".to_string())?;
parse_scrape(&body, cfg.implementation.as_ref().unwrap_or(&NacosImplementation::Nacos), endpoint)
}
pub(crate) fn parse_scrape(
body: &str,
implementation: &NacosImplementation,
endpoint: &str,
) -> Result<NacosPrometheusSnapshot, String> {
let lines = body.lines().map(|line| Ok::<_, io::Error>(line.to_string()));
let scrape = Scrape::parse(lines).map_err(|error| format!("failed to parse metrics: {error}"))?;
if scrape.samples.is_empty() {
return Err("response contains no Prometheus samples".to_string());
}
let source = NacosPrometheusSource {
kind: match implementation {
NacosImplementation::Nacos => "nacos",
NacosImplementation::RNacos => "rnacos",
}
.to_string(),
endpoint: redact_endpoint(endpoint),
fingerprint: Some(endpoint_fingerprint(endpoint)),
};
Ok(match implementation {
NacosImplementation::Nacos => normalize_nacos(source, &scrape.samples),
NacosImplementation::RNacos => normalize_rnacos(source, &scrape.samples),
})
}
fn normalize_nacos(source: NacosPrometheusSource, samples: &[Sample]) -> NacosPrometheusSnapshot {
let heap_used = scalar_sum_filtered(samples, "jvm_memory_used_bytes", |labels| label_eq(labels, "area", "heap"));
let heap_max = scalar_sum_filtered(samples, "jvm_memory_max_bytes", |labels| label_eq(labels, "area", "heap"));
let memory_ratio = ratio(heap_used, heap_max);
let http_count = scalar_sum(samples, "http_server_requests_seconds_count");
let grpc_count = scalar_sum_any(samples, &["grpc_server_requests_seconds_count", "grpc_server_requests_count"]);
NacosPrometheusSnapshot {
source,
resource: NacosPrometheusResourceMetrics {
cpu_ratio: scalar_max_any(samples, &["system_cpu_usage", "process_cpu_usage"]).and_then(normalize_ratio),
memory_ratio,
memory_used_bytes: heap_used,
memory_max_bytes: heap_max,
rss_bytes: None,
vms_bytes: None,
system_total_memory_bytes: None,
load_1m: scalar_max(samples, "system_load_average_1m"),
jvm_daemon_threads: scalar_sum(samples, "jvm_threads_daemon"),
gc_pause_count: scalar_sum_any(samples, &["jvm_gc_pause_seconds_count", "jvm_gc_pause_count"]),
},
traffic: NacosPrometheusTrafficMetrics {
http_requests_total: http_count,
grpc_requests_total: grpc_count,
http_errors_total: scalar_sum_filtered(samples, "http_server_requests_seconds_count", is_error_labels),
grpc_errors_total: scalar_sum_filtered_any(
samples,
&["grpc_server_requests_seconds_count", "grpc_server_requests_count"],
is_grpc_error_labels,
),
http_duration_seconds_total: scalar_sum(samples, "http_server_requests_seconds_sum"),
http_duration_count: http_count,
grpc_duration_seconds_total: scalar_sum_any(
samples,
&["grpc_server_requests_seconds_sum", "grpc_server_requests_sum"],
),
grpc_duration_count: grpc_count,
http_p50_ms: quantile_ms(samples, "http_server_requests_seconds", 0.5, 1_000.0),
http_p95_ms: quantile_ms(samples, "http_server_requests_seconds", 0.95, 1_000.0),
http_p99_ms: quantile_ms(samples, "http_server_requests_seconds", 0.99, 1_000.0),
grpc_p50_ms: quantile_ms_any(
samples,
&["grpc_server_requests_seconds", "grpc_server_requests"],
0.5,
1_000.0,
),
grpc_p95_ms: quantile_ms_any(
samples,
&["grpc_server_requests_seconds", "grpc_server_requests"],
0.95,
1_000.0,
),
grpc_p99_ms: quantile_ms_any(
samples,
&["grpc_server_requests_seconds", "grpc_server_requests"],
0.99,
1_000.0,
),
executor_pool_size: scalar_max_filtered(samples, "grpc_server_executor", |labels| {
label_in(labels, "name", &["poolSize", "pool_size"])
}),
executor_active_count: scalar_max_filtered(samples, "grpc_server_executor", |labels| {
label_in(labels, "name", &["activeCount", "active_count"])
}),
executor_queued_tasks: scalar_max_filtered(samples, "grpc_server_executor", |labels| {
label_in(labels, "name", &["queuedTasks", "queueSize", "queued_tasks"])
}),
},
config: NacosPrometheusConfigMetrics {
config_count: monitor(samples, "config", &["configCount"]),
get_config_total: monitor(samples, "config", &["getConfig", "getConfigCount"]),
publish_total: monitor(samples, "config", &["publish", "publishCount"]),
long_polling: monitor(samples, "config", &["longPolling"]),
listener_clients: None,
listener_keys: None,
notify_tasks: monitor(samples, "config", &["notifyTask"]),
notify_client_tasks: monitor(samples, "config", &["notifyClientTask"]),
dump_tasks: monitor(samples, "config", &["dumpTask"]),
subscriber_count: scalar_sum(samples, "nacos_config_subscriber"),
},
naming: NacosPrometheusNamingMetrics {
service_count: monitor(samples, "naming", &["serviceCount"]),
instance_count: monitor(samples, "naming", &["ipCount", "instanceCount"]),
subscriber_count: monitor(samples, "naming", &["subscriberCount"])
.or_else(|| scalar_sum(samples, "nacos_naming_subscriber")),
connection_count: monitor(samples, "core", &["longConnection"]),
total_push: monitor(samples, "naming", &["totalPush"]),
failed_push: monitor(samples, "naming", &["failedPush"]),
empty_push: monitor(samples, "naming", &["emptyPush"]),
push_pending_tasks: monitor(
samples,
"naming",
&["pushPendingTaskCount", "pushPendingTask", "pushPendingTasks"],
),
avg_push_cost_ms: monitor(samples, "naming", &["avgPushCost"]),
max_push_cost_ms: monitor(samples, "naming", &["maxPushCost"]),
leader_status: monitor(samples, "naming", &["leaderStatus"]),
},
}
}
fn normalize_rnacos(source: NacosPrometheusSource, samples: &[Sample]) -> NacosPrometheusSnapshot {
let mib = 1024.0 * 1024.0;
let http_count = scalar_sum(samples, "http_request_total_count");
let grpc_count = scalar_sum(samples, "grpc_request_total_count");
let rss_mib = scalar_sum(samples, "app_rss_memory");
let system_memory_mib = scalar_sum(samples, "sys_total_memory");
NacosPrometheusSnapshot {
source,
resource: NacosPrometheusResourceMetrics {
cpu_ratio: scalar_max(samples, "app_cpu_usage").and_then(normalize_percentage),
memory_ratio: scalar_max(samples, "app_memory_usage")
.and_then(normalize_percentage)
.or_else(|| ratio(rss_mib, system_memory_mib)),
memory_used_bytes: rss_mib.map(|value| value * mib),
memory_max_bytes: system_memory_mib.map(|value| value * mib),
rss_bytes: rss_mib.map(|value| value * mib),
vms_bytes: scalar_sum(samples, "app_vms_memory").map(|value| value * mib),
system_total_memory_bytes: system_memory_mib.map(|value| value * mib),
load_1m: None,
jvm_daemon_threads: None,
gc_pause_count: None,
},
traffic: NacosPrometheusTrafficMetrics {
http_requests_total: http_count,
grpc_requests_total: grpc_count,
http_errors_total: None,
grpc_errors_total: None,
http_duration_seconds_total: scalar_sum(samples, "http_request_handle_rt_histogram_sum")
.or_else(|| scalar_sum(samples, "http_request_handle_rt_summary_sum"))
.map(|value| value / 1_000.0),
http_duration_count: scalar_sum(samples, "http_request_handle_rt_histogram_count")
.or_else(|| scalar_sum(samples, "http_request_handle_rt_summary_count"))
.or(http_count),
grpc_duration_seconds_total: scalar_sum(samples, "grpc_request_handle_rt_histogram_sum")
.or_else(|| scalar_sum(samples, "grpc_request_handle_rt_summary_sum"))
.map(|value| value / 1_000.0),
grpc_duration_count: scalar_sum(samples, "grpc_request_handle_rt_histogram_count")
.or_else(|| scalar_sum(samples, "grpc_request_handle_rt_summary_count"))
.or(grpc_count),
http_p50_ms: quantile_ms_any(
samples,
&["http_request_handle_rt_summary", "http_request_handle_rt_histogram"],
0.5,
1.0,
),
http_p95_ms: quantile_ms_any(
samples,
&["http_request_handle_rt_summary", "http_request_handle_rt_histogram"],
0.95,
1.0,
),
http_p99_ms: quantile_ms_any(
samples,
&["http_request_handle_rt_summary", "http_request_handle_rt_histogram"],
0.99,
1.0,
),
grpc_p50_ms: quantile_ms_any(
samples,
&["grpc_request_handle_rt_summary", "grpc_request_handle_rt_histogram"],
0.5,
1.0,
),
grpc_p95_ms: quantile_ms_any(
samples,
&["grpc_request_handle_rt_summary", "grpc_request_handle_rt_histogram"],
0.95,
1.0,
),
grpc_p99_ms: quantile_ms_any(
samples,
&["grpc_request_handle_rt_summary", "grpc_request_handle_rt_histogram"],
0.99,
1.0,
),
executor_pool_size: None,
executor_active_count: None,
executor_queued_tasks: None,
},
config: NacosPrometheusConfigMetrics {
config_count: scalar_sum(samples, "config_data_size")
.or_else(|| scalar_sum(samples, "config_index_config_size")),
get_config_total: None,
publish_total: None,
long_polling: None,
listener_clients: scalar_sum(samples, "config_listener_client_size"),
listener_keys: scalar_sum(samples, "config_listener_key_size"),
notify_tasks: None,
notify_client_tasks: None,
dump_tasks: None,
subscriber_count: scalar_sum(samples, "config_subscriber_client_size")
.or_else(|| scalar_sum(samples, "config_subscriber_client_value_size")),
},
naming: NacosPrometheusNamingMetrics {
service_count: scalar_sum(samples, "naming_service_size")
.or_else(|| scalar_sum(samples, "naming_index_service_size")),
instance_count: scalar_sum(samples, "naming_instance_size"),
subscriber_count: scalar_sum(samples, "naming_subscriber_client_size")
.or_else(|| scalar_sum(samples, "naming_subscriber_client_value_size")),
connection_count: scalar_sum(samples, "grpc_conn_size"),
total_push: None,
failed_push: None,
empty_push: None,
push_pending_tasks: None,
avg_push_cost_ms: None,
max_push_cost_ms: None,
leader_status: None,
},
}
}
fn scalar_value(sample: &Sample) -> Option<f64> {
let value = match sample.value {
Value::Counter(value) | Value::Gauge(value) | Value::Untyped(value) => value,
Value::Histogram(_) | Value::Summary(_) => return None,
};
value.is_finite().then_some(value)
}
fn scalar_sum(samples: &[Sample], metric: &str) -> Option<f64> {
scalar_sum_filtered(samples, metric, |_| true)
}
fn scalar_sum_any(samples: &[Sample], metrics: &[&str]) -> Option<f64> {
metrics.iter().find_map(|metric| scalar_sum(samples, metric))
}
fn scalar_sum_filtered(samples: &[Sample], metric: &str, predicate: impl Fn(&Labels) -> bool) -> Option<f64> {
let values =
samples.iter().filter(|sample| sample.metric == metric && predicate(&sample.labels)).filter_map(scalar_value);
sum_non_empty(values)
}
fn scalar_sum_filtered_any(
samples: &[Sample],
metrics: &[&str],
predicate: impl Fn(&Labels) -> bool + Copy,
) -> Option<f64> {
metrics.iter().find_map(|metric| scalar_sum_filtered(samples, metric, predicate))
}
fn scalar_max(samples: &[Sample], metric: &str) -> Option<f64> {
scalar_max_filtered(samples, metric, |_| true)
}
fn scalar_max_any(samples: &[Sample], metrics: &[&str]) -> Option<f64> {
metrics.iter().find_map(|metric| scalar_max(samples, metric))
}
fn scalar_max_filtered(samples: &[Sample], metric: &str, predicate: impl Fn(&Labels) -> bool) -> Option<f64> {
samples
.iter()
.filter(|sample| sample.metric == metric && predicate(&sample.labels))
.filter_map(scalar_value)
.reduce(f64::max)
}
fn sum_non_empty(values: impl Iterator<Item = f64>) -> Option<f64> {
let mut found = false;
let sum = values.fold(0.0, |sum, value| {
found = true;
sum + value
});
found.then_some(sum)
}
fn monitor(samples: &[Sample], module: &str, names: &[&str]) -> Option<f64> {
names.iter().find_map(|name| {
scalar_sum_filtered(samples, "nacos_monitor", |labels| {
label_eq(labels, "module", module) && label_eq(labels, "name", name)
})
})
}
fn label_eq(labels: &Labels, key: &str, value: &str) -> bool {
labels.get(key).is_some_and(|current| current.eq_ignore_ascii_case(value))
}
fn label_in(labels: &Labels, key: &str, values: &[&str]) -> bool {
values.iter().any(|value| label_eq(labels, key, value))
}
fn is_error_labels(labels: &Labels) -> bool {
if let Some(status) = labels.get("status").or_else(|| labels.get("code")) {
return !status.starts_with('2') && status != "0";
}
labels.get("outcome").is_some_and(|outcome| !matches!(outcome.to_ascii_uppercase().as_str(), "SUCCESS" | "UNKNOWN"))
}
fn is_grpc_error_labels(labels: &Labels) -> bool {
if labels.get("success").is_some_and(|value| value.eq_ignore_ascii_case("false")) {
return true;
}
labels
.get("errorCode")
.or_else(|| labels.get("error_code"))
.or_else(|| labels.get("code"))
.is_some_and(|value| value != "0" && !value.eq_ignore_ascii_case("OK"))
|| labels.get("result").is_some_and(|value| !matches!(value.to_ascii_uppercase().as_str(), "SUCCESS" | "OK"))
}
fn ratio(numerator: Option<f64>, denominator: Option<f64>) -> Option<f64> {
match (numerator, denominator) {
(Some(numerator), Some(denominator)) if denominator > 0.0 => normalize_ratio(numerator / denominator),
_ => None,
}
}
fn normalize_ratio(value: f64) -> Option<f64> {
if !value.is_finite() || value < 0.0 {
return None;
}
Some((if value > 1.0 { value / 100.0 } else { value }).clamp(0.0, 1.0))
}
fn normalize_percentage(value: f64) -> Option<f64> {
if !value.is_finite() || value < 0.0 {
return None;
}
Some((value / 100.0).clamp(0.0, 1.0))
}
fn endpoint_fingerprint(endpoint: &str) -> String {
format!("{:x}", Sha256::digest(endpoint.as_bytes()))
}
fn redact_endpoint(endpoint: &str) -> String {
let Ok(mut url) = reqwest::Url::parse(endpoint) else {
return endpoint.split_once('?').map_or(endpoint, |(base, _)| base).to_string();
};
url.set_query(None);
url.set_fragment(None);
url.to_string()
}
fn quantile_ms(samples: &[Sample], metric: &str, quantile: f64, multiplier: f64) -> Option<f64> {
let mut histogram_buckets = HashMap::<u64, (f64, f64)>::new();
let mut summary_quantiles = Vec::new();
for sample in samples.iter().filter(|sample| sample.metric == metric) {
match &sample.value {
Value::Histogram(values) => {
for value in values {
if value.less_than.is_nan() || !value.count.is_finite() {
continue;
}
histogram_buckets
.entry(value.less_than.to_bits())
.and_modify(|(_, count)| *count += value.count)
.or_insert((value.less_than, value.count));
}
}
Value::Summary(values) => summary_quantiles.push(values),
_ => {}
}
}
if !histogram_buckets.is_empty() {
return histogram_quantile(histogram_buckets.into_values(), quantile).map(|value| value * multiplier);
}
// Prometheus summaries are already-calculated client-side quantiles and
// cannot be aggregated across label sets without changing their meaning.
let [values] = summary_quantiles.as_slice() else {
return None;
};
nearest_quantile(values.iter().map(|value| (value.quantile, value.count)), quantile).map(|value| value * multiplier)
}
fn quantile_ms_any(samples: &[Sample], metrics: &[&str], quantile: f64, multiplier: f64) -> Option<f64> {
metrics.iter().find_map(|metric| quantile_ms(samples, metric, quantile, multiplier))
}
fn nearest_quantile(values: impl Iterator<Item = (f64, f64)>, target: f64) -> Option<f64> {
values
.filter(|(quantile, value)| quantile.is_finite() && value.is_finite())
.min_by(|left, right| {
(left.0 - target).abs().partial_cmp(&(right.0 - target).abs()).unwrap_or(std::cmp::Ordering::Equal)
})
.map(|(_, value)| value)
}
fn histogram_quantile(values: impl Iterator<Item = (f64, f64)>, target: f64) -> Option<f64> {
let mut buckets =
values.filter(|(boundary, count)| !boundary.is_nan() && count.is_finite() && *count >= 0.0).collect::<Vec<_>>();
buckets.sort_by(|left, right| left.0.partial_cmp(&right.0).unwrap_or(std::cmp::Ordering::Equal));
let mut previous_count = 0.0;
for (_, count) in &mut buckets {
*count = count.max(previous_count);
previous_count = *count;
}
let total = buckets.last()?.1;
if total <= 0.0 {
return None;
}
let threshold = total * target.clamp(0.0, 1.0);
let index = buckets.iter().position(|(_, count)| *count >= threshold)?;
let (upper_bound, upper_count) = buckets[index];
if upper_bound.is_infinite() {
return (index > 0).then_some(buckets[index - 1].0).filter(|boundary| boundary.is_finite());
}
let (lower_bound, lower_count) = if index == 0 { (0.0, 0.0) } else { buckets[index - 1] };
if upper_count <= lower_count {
return Some(upper_bound);
}
Some(lower_bound + (upper_bound - lower_bound) * ((threshold - lower_count) / (upper_count - lower_count)))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn derives_and_deduplicates_nacos_endpoints() {
let cfg = test_config(NacosImplementation::Nacos, "http://127.0.0.1:8818", "/nacos");
assert_eq!(
endpoint_candidates(&cfg).unwrap(),
vec!["http://127.0.0.1:8818/nacos/actuator/prometheus", "http://127.0.0.1:8818/actuator/prometheus",]
);
}
#[test]
fn derives_rnacos_endpoints() {
let cfg = test_config(NacosImplementation::RNacos, "http://127.0.0.1:3848", "/nacos");
assert_eq!(
endpoint_candidates(&cfg).unwrap(),
vec![
"http://127.0.0.1:3848/metrics",
"http://127.0.0.1:3848/nacos/metrics",
"http://127.0.0.1:3848/rnacos/metrics",
]
);
}
#[test]
fn disabled_mode_skips_scraping() {
let mut cfg = test_config(NacosImplementation::Nacos, "http://127.0.0.1:8818", "/nacos");
cfg.metrics_mode = NacosMetricsMode::Disabled;
assert!(endpoint_candidates(&cfg).unwrap().is_empty());
}
#[test]
fn parses_nacos_metrics_and_labels() {
let body = r#"
# TYPE system_cpu_usage gauge
system_cpu_usage 0.25
# TYPE jvm_memory_used_bytes gauge
jvm_memory_used_bytes{area="heap",id="a"} 100
jvm_memory_used_bytes{area="heap",id="b"} 50
# TYPE jvm_memory_max_bytes gauge
jvm_memory_max_bytes{area="heap",id="a"} 400
# TYPE http_server_requests_seconds counter
http_server_requests_seconds_count{status="200"} 90
http_server_requests_seconds_count{status="500"} 10
http_server_requests_seconds_sum 25
# TYPE nacos_monitor gauge
nacos_monitor{module="naming",name="serviceCount"} 12
nacos_monitor{module="core",name="longConnection"} 8
nacos_monitor{module="naming",name="pushPendingTaskCount"} 3
# TYPE nacos_naming_subscriber gauge
nacos_naming_subscriber{version="v1"} 2
nacos_naming_subscriber{version="v2"} 4
"#;
let parsed = parse_scrape(body, &NacosImplementation::Nacos, "http://localhost/metrics").unwrap();
assert_eq!(parsed.resource.cpu_ratio, Some(0.25));
assert_eq!(parsed.resource.memory_used_bytes, Some(150.0));
assert_eq!(parsed.resource.memory_ratio, Some(0.375));
assert_eq!(parsed.traffic.http_requests_total, Some(100.0));
assert_eq!(parsed.traffic.http_errors_total, Some(10.0));
assert_eq!(parsed.naming.service_count, Some(12.0));
assert_eq!(parsed.naming.connection_count, Some(8.0));
assert_eq!(parsed.naming.subscriber_count, Some(6.0));
assert_eq!(parsed.naming.push_pending_tasks, Some(3.0));
}
#[test]
fn aggregates_histogram_buckets_across_label_sets() {
let body = r#"
# TYPE http_server_requests_seconds histogram
http_server_requests_seconds_bucket{route="a",le="1"} 50
http_server_requests_seconds_bucket{route="a",le="10"} 50
http_server_requests_seconds_bucket{route="a",le="+Inf"} 50
http_server_requests_seconds_sum{route="a"} 25
http_server_requests_seconds_count{route="a"} 50
http_server_requests_seconds_bucket{route="b",le="1"} 0
http_server_requests_seconds_bucket{route="b",le="10"} 50
http_server_requests_seconds_bucket{route="b",le="+Inf"} 50
http_server_requests_seconds_sum{route="b"} 250
http_server_requests_seconds_count{route="b"} 50
"#;
let parsed = parse_scrape(body, &NacosImplementation::Nacos, "http://localhost/metrics").unwrap();
assert_eq!(parsed.traffic.http_p50_ms, Some(1_000.0));
}
#[test]
fn redacts_endpoint_query_from_snapshot_source() {
let parsed = parse_scrape(
"# TYPE system_cpu_usage gauge\nsystem_cpu_usage 0.25\n",
&NacosImplementation::Nacos,
"http://localhost/metrics?token=secret&node=a",
)
.unwrap();
let other_source = parse_scrape(
"# TYPE system_cpu_usage gauge\nsystem_cpu_usage 0.25\n",
&NacosImplementation::Nacos,
"http://localhost/metrics?token=secret&node=b",
)
.unwrap();
assert_eq!(parsed.source.endpoint, "http://localhost/metrics");
assert_ne!(parsed.source.fingerprint, other_source.source.fingerprint);
assert!(!parsed.source.fingerprint.as_deref().unwrap().contains("secret"));
}
#[test]
fn parses_rnacos_units_counts_and_summary() {
let body = r#"
# TYPE app_cpu_usage gauge
app_cpu_usage 12.5
# TYPE app_rss_memory gauge
app_rss_memory 10
# TYPE sys_total_memory gauge
sys_total_memory 100
# TYPE naming_service_size gauge
naming_service_size 4
# TYPE grpc_conn_size gauge
grpc_conn_size 3
# TYPE http_request_total_count counter
http_request_total_count 20
# TYPE http_request_handle_rt_summary summary
http_request_handle_rt_summary{quantile="0.5"} 5
http_request_handle_rt_summary{quantile="0.95"} 15
http_request_handle_rt_summary_sum 100
http_request_handle_rt_summary_count 20
"#;
let parsed = parse_scrape(body, &NacosImplementation::RNacos, "http://localhost/metrics").unwrap();
assert_eq!(parsed.resource.cpu_ratio, Some(0.125));
assert_eq!(parsed.resource.rss_bytes, Some(10.0 * 1024.0 * 1024.0));
assert_eq!(parsed.resource.memory_ratio, Some(0.1));
assert_eq!(parsed.naming.service_count, Some(4.0));
assert_eq!(parsed.naming.connection_count, Some(3.0));
assert_eq!(parsed.traffic.http_p95_ms, Some(15.0));
assert_eq!(parsed.traffic.http_duration_seconds_total, Some(0.1));
}
#[test]
fn converts_low_rnacos_percentages_to_ratios() {
let body = r#"
# TYPE app_cpu_usage gauge
app_cpu_usage 0.5
# TYPE app_memory_usage gauge
app_memory_usage 1
"#;
let parsed = parse_scrape(body, &NacosImplementation::RNacos, "http://localhost/metrics").unwrap();
assert_eq!(parsed.resource.cpu_ratio, Some(0.005));
assert_eq!(parsed.resource.memory_ratio, Some(0.01));
}
#[test]
fn rejects_responses_without_samples() {
assert!(parse_scrape("<html>not metrics</html>", &NacosImplementation::Nacos, "http://localhost/metrics")
.unwrap_err()
.contains("no Prometheus samples"));
}
#[tokio::test]
async fn falls_back_after_a_missing_auto_endpoint() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
for index in 0..2 {
let (mut socket, _) = listener.accept().await.unwrap();
let mut request = vec![0_u8; 2048];
let _ = tokio::io::AsyncReadExt::read(&mut socket, &mut request).await.unwrap();
if index == 0 {
let body = "not found";
let response = format!(
"HTTP/1.1 404 Not Found\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
body.len()
);
tokio::io::AsyncWriteExt::write_all(&mut socket, response.as_bytes()).await.unwrap();
} else {
let body = "# TYPE system_cpu_usage gauge\nsystem_cpu_usage 0.4\n";
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
body.len()
);
tokio::io::AsyncWriteExt::write_all(&mut socket, response.as_bytes()).await.unwrap();
}
}
});
let cfg = test_config(NacosImplementation::Nacos, &format!("http://{address}"), "");
let client = reqwest::Client::new();
let parsed = scrape(&client, &cfg).await.unwrap().unwrap();
assert_eq!(parsed.resource.cpu_ratio, Some(0.4));
assert!(parsed.source.endpoint.ends_with("/nacos/actuator/prometheus"));
server.await.unwrap();
}
#[tokio::test]
async fn rejects_oversized_scrapes() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let mut request = vec![0_u8; 2048];
let _ = tokio::io::AsyncReadExt::read(&mut socket, &mut request).await.unwrap();
let body = vec![b'x'; MAX_SCRAPE_BYTES + 1];
let header = format!(
"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body.len()
);
let _ = tokio::io::AsyncWriteExt::write_all(&mut socket, header.as_bytes()).await;
let _ = tokio::io::AsyncWriteExt::write_all(&mut socket, &body).await;
});
let mut cfg = test_config(NacosImplementation::Nacos, "http://127.0.0.1:8848", "");
cfg.metrics_mode = NacosMetricsMode::Custom;
cfg.metrics_url = format!("http://{address}/metrics");
let error = scrape(&reqwest::Client::new(), &cfg).await.unwrap_err();
assert!(error.contains("exceeds"));
server.await.unwrap();
}
#[tokio::test]
async fn redacts_endpoint_query_from_scrape_errors() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let mut request = vec![0_u8; 2048];
let _ = tokio::io::AsyncReadExt::read(&mut socket, &mut request).await.unwrap();
let response = "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n";
tokio::io::AsyncWriteExt::write_all(&mut socket, response.as_bytes()).await.unwrap();
});
let mut cfg = test_config(NacosImplementation::Nacos, "http://127.0.0.1:8848", "");
cfg.metrics_mode = NacosMetricsMode::Custom;
cfg.metrics_url = format!("http://{address}/metrics?token=secret");
let error = scrape(&reqwest::Client::new(), &cfg).await.unwrap_err();
assert!(error.contains(&format!("http://{address}/metrics")));
assert!(!error.contains("secret"));
server.await.unwrap();
}
#[tokio::test]
async fn times_out_without_exposing_endpoint_query() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let mut request = vec![0_u8; 2048];
let _ = tokio::io::AsyncReadExt::read(&mut socket, &mut request).await.unwrap();
std::future::pending::<()>().await;
});
let mut cfg = test_config(NacosImplementation::Nacos, "http://127.0.0.1:8848", "");
cfg.metrics_mode = NacosMetricsMode::Custom;
cfg.metrics_url = format!("http://{address}/metrics?token=secret");
let error = scrape_with_timeout(&reqwest::Client::new(), &cfg, Duration::from_millis(25)).await.unwrap_err();
assert!(error.contains("timed out"));
assert!(error.contains(&format!("http://{address}/metrics")));
assert!(!error.contains("secret"));
server.abort();
}
fn test_config(implementation: NacosImplementation, server_addr: &str, context_path: &str) -> NacosAdminConfig {
NacosAdminConfig {
implementation: Some(implementation),
version_mode: None,
server_addr: server_addr.to_string(),
display_server_addr: server_addr.to_string(),
namespace: String::new(),
context_path: context_path.to_string(),
rnacos_console_addr: String::new(),
rnacos_history_enabled: None,
rnacos_console_auth: Default::default(),
auth: Default::default(),
tls_skip_verify: false,
metrics_mode: NacosMetricsMode::Auto,
metrics_url: String::new(),
page_size: 20,
connect_override: None,
}
}
}
+3
View File
@@ -699,6 +699,9 @@ mod tests {
async fn update_instance(&self, _: NacosInstanceUpdate) -> Result<(), String> {
Err("unused".to_string())
}
async fn get_dashboard(&self, _: NacosDashboardQuery) -> Result<NacosDashboardSnapshot, String> {
Err("unused".to_string())
}
async fn raw_request(&self, _: NacosRawRequest) -> Result<NacosRawResponse, String> {
Err("unused".to_string())
}
+9
View File
@@ -162,6 +162,15 @@ pub async fn nacos_update_instance_core(
admin.update_instance(req).await
}
pub async fn nacos_get_dashboard_core(
state: &AppState,
conn_id: &str,
query: NacosDashboardQuery,
) -> Result<NacosDashboardSnapshot, String> {
let admin = get_admin(state, conn_id).await?;
admin.get_dashboard(query).await
}
pub async fn nacos_raw_request_core(
state: &AppState,
conn_id: &str,
+169
View File
@@ -496,6 +496,175 @@ pub struct NacosInstanceUpdate {
pub metadata: Option<serde_json::Value>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub struct NacosDashboardQuery {
#[serde(default)]
pub namespace: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
#[serde(rename_all = "camelCase")]
pub struct NacosDashboardMetrics {
#[serde(default)]
pub status: Option<String>,
#[serde(default)]
pub service_count: Option<u64>,
#[serde(default)]
pub instance_count: Option<u64>,
#[serde(default)]
pub subscribe_count: Option<u64>,
#[serde(default)]
pub raft_notify_task_count: Option<u64>,
#[serde(default)]
pub responsible_service_count: Option<u64>,
#[serde(default)]
pub responsible_instance_count: Option<u64>,
#[serde(default)]
pub client_count: Option<u64>,
#[serde(default)]
pub connection_based_client_count: Option<u64>,
#[serde(default)]
pub ephemeral_ip_port_client_count: Option<u64>,
#[serde(default)]
pub persistent_ip_port_client_count: Option<u64>,
#[serde(default)]
pub responsible_client_count: Option<u64>,
#[serde(default)]
pub cpu: Option<f64>,
#[serde(default)]
pub load: Option<f64>,
#[serde(default)]
pub mem: Option<f64>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
#[serde(rename_all = "camelCase")]
pub struct NacosPrometheusSource {
pub kind: String,
pub endpoint: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub fingerprint: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
#[serde(rename_all = "camelCase")]
pub struct NacosPrometheusResourceMetrics {
pub cpu_ratio: Option<f64>,
pub memory_ratio: Option<f64>,
pub memory_used_bytes: Option<f64>,
pub memory_max_bytes: Option<f64>,
pub rss_bytes: Option<f64>,
pub vms_bytes: Option<f64>,
pub system_total_memory_bytes: Option<f64>,
pub load_1m: Option<f64>,
pub jvm_daemon_threads: Option<f64>,
pub gc_pause_count: Option<f64>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
#[serde(rename_all = "camelCase")]
pub struct NacosPrometheusTrafficMetrics {
pub http_requests_total: Option<f64>,
pub grpc_requests_total: Option<f64>,
pub http_errors_total: Option<f64>,
pub grpc_errors_total: Option<f64>,
pub http_duration_seconds_total: Option<f64>,
pub http_duration_count: Option<f64>,
pub grpc_duration_seconds_total: Option<f64>,
pub grpc_duration_count: Option<f64>,
pub http_p50_ms: Option<f64>,
pub http_p95_ms: Option<f64>,
pub http_p99_ms: Option<f64>,
pub grpc_p50_ms: Option<f64>,
pub grpc_p95_ms: Option<f64>,
pub grpc_p99_ms: Option<f64>,
pub executor_pool_size: Option<f64>,
pub executor_active_count: Option<f64>,
pub executor_queued_tasks: Option<f64>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
#[serde(rename_all = "camelCase")]
pub struct NacosPrometheusConfigMetrics {
pub config_count: Option<f64>,
pub get_config_total: Option<f64>,
pub publish_total: Option<f64>,
pub long_polling: Option<f64>,
pub listener_clients: Option<f64>,
pub listener_keys: Option<f64>,
pub notify_tasks: Option<f64>,
pub notify_client_tasks: Option<f64>,
pub dump_tasks: Option<f64>,
pub subscriber_count: Option<f64>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
#[serde(rename_all = "camelCase")]
pub struct NacosPrometheusNamingMetrics {
pub service_count: Option<f64>,
pub instance_count: Option<f64>,
pub subscriber_count: Option<f64>,
pub connection_count: Option<f64>,
pub total_push: Option<f64>,
pub failed_push: Option<f64>,
pub empty_push: Option<f64>,
pub push_pending_tasks: Option<f64>,
pub avg_push_cost_ms: Option<f64>,
pub max_push_cost_ms: Option<f64>,
pub leader_status: Option<f64>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
#[serde(rename_all = "camelCase")]
pub struct NacosPrometheusSnapshot {
pub source: NacosPrometheusSource,
pub resource: NacosPrometheusResourceMetrics,
pub traffic: NacosPrometheusTrafficMetrics,
pub config: NacosPrometheusConfigMetrics,
pub naming: NacosPrometheusNamingMetrics,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct NacosClusterNode {
pub address: String,
#[serde(default)]
pub ip: Option<String>,
#[serde(default)]
pub port: Option<u16>,
#[serde(default)]
pub state: Option<String>,
#[serde(default)]
pub alive: Option<bool>,
#[serde(default)]
pub site: Option<String>,
#[serde(default)]
pub weight: Option<f64>,
#[serde(default)]
pub last_refresh_time: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct NacosDashboardSnapshot {
pub namespace: String,
#[serde(default)]
pub namespace_count: Option<u64>,
#[serde(default)]
pub config_count: Option<u64>,
#[serde(default)]
pub service_count: Option<u64>,
#[serde(default)]
pub metrics: Option<NacosDashboardMetrics>,
#[serde(default)]
pub prometheus: Option<NacosPrometheusSnapshot>,
#[serde(default)]
pub nodes: Vec<NacosClusterNode>,
#[serde(default)]
pub warnings: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase")]
pub struct NacosRawRequest {