Files
dbx/crates/dbx-core/src/nacos/prometheus.rs
T

821 lines
35 KiB
Rust

use std::collections::{HashMap, HashSet};
use std::io;
use std::time::Duration;
use futures::StreamExt;
use prometheus_parse::{Labels, Sample, Scrape, Value};
use sha2::{Digest, Sha256};
use crate::nacos::config::{NacosAdminConfig, NacosImplementation, NacosMetricsMode};
use crate::nacos::types::{
NacosPrometheusConfigMetrics, NacosPrometheusNamingMetrics, NacosPrometheusResourceMetrics,
NacosPrometheusSnapshot, NacosPrometheusSource, NacosPrometheusTrafficMetrics,
};
const SCRAPE_TIMEOUT: Duration = Duration::from_secs(5);
const MAX_SCRAPE_BYTES: usize = 4 * 1024 * 1024;
pub(crate) fn endpoint_candidates(cfg: &NacosAdminConfig) -> Result<Vec<String>, String> {
match cfg.metrics_mode {
NacosMetricsMode::Disabled => return Ok(Vec::new()),
NacosMetricsMode::Custom => return Ok(vec![cfg.metrics_url.clone()]),
NacosMetricsMode::Auto => {}
}
let base = cfg.server_addr.trim_end_matches('/');
let context = cfg.context_path.trim_end_matches('/');
let raw = match cfg.implementation.as_ref().unwrap_or(&NacosImplementation::Nacos) {
NacosImplementation::Nacos => vec![
format!("{base}{context}/actuator/prometheus"),
format!("{base}/nacos/actuator/prometheus"),
format!("{base}/actuator/prometheus"),
],
NacosImplementation::RNacos => {
vec![format!("{base}/metrics"), format!("{base}{context}/metrics"), format!("{base}/rnacos/metrics")]
}
};
let mut seen = HashSet::new();
raw.into_iter()
.map(|candidate| {
reqwest::Url::parse(&candidate)
.map(|url| url.to_string())
.map_err(|error| format!("Nacos Prometheus metrics URL is invalid: {error}"))
})
.filter(|candidate| match candidate {
Ok(value) => seen.insert(value.clone()),
Err(_) => true,
})
.collect()
}
pub(crate) async fn scrape(
client: &reqwest::Client,
cfg: &NacosAdminConfig,
) -> Result<Option<NacosPrometheusSnapshot>, String> {
scrape_with_timeout(client, cfg, SCRAPE_TIMEOUT).await
}
async fn scrape_with_timeout(
client: &reqwest::Client,
cfg: &NacosAdminConfig,
timeout: Duration,
) -> Result<Option<NacosPrometheusSnapshot>, String> {
let candidates = endpoint_candidates(cfg)?;
if candidates.is_empty() {
return Ok(None);
}
let candidate_summary =
candidates.iter().map(|candidate| redact_endpoint(candidate)).collect::<Vec<_>>().join(", ");
tokio::time::timeout(timeout, async {
let mut errors = Vec::new();
for endpoint in candidates {
match scrape_endpoint(client, cfg, &endpoint).await {
Ok(snapshot) => return Ok(Some(snapshot)),
Err(error) => errors.push(format!("{}: {error}", redact_endpoint(&endpoint))),
}
}
Err(format!("Prometheus metrics unavailable: {}", errors.join("; ")))
})
.await
.map_err(|_| {
format!("Prometheus metrics unavailable: scrape timed out after {timeout:?} while trying {candidate_summary}")
})?
}
async fn scrape_endpoint(
client: &reqwest::Client,
cfg: &NacosAdminConfig,
endpoint: &str,
) -> Result<NacosPrometheusSnapshot, String> {
let response = client
.get(endpoint)
.timeout(SCRAPE_TIMEOUT)
.send()
.await
.map_err(|error| format!("request failed: {}", error.without_url()))?;
if !response.status().is_success() {
return Err(format!("returned HTTP {}", response.status()));
}
let mut bytes = Vec::new();
let mut stream = response.bytes_stream();
while let Some(chunk) = stream.next().await {
let chunk = chunk.map_err(|error| format!("failed to read response: {}", error.without_url()))?;
if bytes.len().saturating_add(chunk.len()) > MAX_SCRAPE_BYTES {
return Err(format!("response exceeds {MAX_SCRAPE_BYTES} bytes"));
}
bytes.extend_from_slice(&chunk);
}
let body = String::from_utf8(bytes).map_err(|_| "response is not valid UTF-8".to_string())?;
parse_scrape(&body, cfg.implementation.as_ref().unwrap_or(&NacosImplementation::Nacos), endpoint)
}
pub(crate) fn parse_scrape(
body: &str,
implementation: &NacosImplementation,
endpoint: &str,
) -> Result<NacosPrometheusSnapshot, String> {
let lines = body.lines().map(|line| Ok::<_, io::Error>(line.to_string()));
let scrape = Scrape::parse(lines).map_err(|error| format!("failed to parse metrics: {error}"))?;
if scrape.samples.is_empty() {
return Err("response contains no Prometheus samples".to_string());
}
let source = NacosPrometheusSource {
kind: match implementation {
NacosImplementation::Nacos => "nacos",
NacosImplementation::RNacos => "rnacos",
}
.to_string(),
endpoint: redact_endpoint(endpoint),
fingerprint: Some(endpoint_fingerprint(endpoint)),
};
Ok(match implementation {
NacosImplementation::Nacos => normalize_nacos(source, &scrape.samples),
NacosImplementation::RNacos => normalize_rnacos(source, &scrape.samples),
})
}
fn normalize_nacos(source: NacosPrometheusSource, samples: &[Sample]) -> NacosPrometheusSnapshot {
let heap_used = scalar_sum_filtered(samples, "jvm_memory_used_bytes", |labels| label_eq(labels, "area", "heap"));
let heap_max = scalar_sum_filtered(samples, "jvm_memory_max_bytes", |labels| label_eq(labels, "area", "heap"));
let memory_ratio = ratio(heap_used, heap_max);
let http_count = scalar_sum(samples, "http_server_requests_seconds_count");
let grpc_count = scalar_sum_any(samples, &["grpc_server_requests_seconds_count", "grpc_server_requests_count"]);
NacosPrometheusSnapshot {
source,
resource: NacosPrometheusResourceMetrics {
cpu_ratio: scalar_max_any(samples, &["system_cpu_usage", "process_cpu_usage"]).and_then(normalize_ratio),
memory_ratio,
memory_used_bytes: heap_used,
memory_max_bytes: heap_max,
rss_bytes: None,
vms_bytes: None,
system_total_memory_bytes: None,
load_1m: scalar_max(samples, "system_load_average_1m"),
jvm_daemon_threads: scalar_sum(samples, "jvm_threads_daemon"),
gc_pause_count: scalar_sum_any(samples, &["jvm_gc_pause_seconds_count", "jvm_gc_pause_count"]),
},
traffic: NacosPrometheusTrafficMetrics {
http_requests_total: http_count,
grpc_requests_total: grpc_count,
http_errors_total: scalar_sum_filtered(samples, "http_server_requests_seconds_count", is_error_labels),
grpc_errors_total: scalar_sum_filtered_any(
samples,
&["grpc_server_requests_seconds_count", "grpc_server_requests_count"],
is_grpc_error_labels,
),
http_duration_seconds_total: scalar_sum(samples, "http_server_requests_seconds_sum"),
http_duration_count: http_count,
grpc_duration_seconds_total: scalar_sum_any(
samples,
&["grpc_server_requests_seconds_sum", "grpc_server_requests_sum"],
),
grpc_duration_count: grpc_count,
http_p50_ms: quantile_ms(samples, "http_server_requests_seconds", 0.5, 1_000.0),
http_p95_ms: quantile_ms(samples, "http_server_requests_seconds", 0.95, 1_000.0),
http_p99_ms: quantile_ms(samples, "http_server_requests_seconds", 0.99, 1_000.0),
grpc_p50_ms: quantile_ms_any(
samples,
&["grpc_server_requests_seconds", "grpc_server_requests"],
0.5,
1_000.0,
),
grpc_p95_ms: quantile_ms_any(
samples,
&["grpc_server_requests_seconds", "grpc_server_requests"],
0.95,
1_000.0,
),
grpc_p99_ms: quantile_ms_any(
samples,
&["grpc_server_requests_seconds", "grpc_server_requests"],
0.99,
1_000.0,
),
executor_pool_size: scalar_max_filtered(samples, "grpc_server_executor", |labels| {
label_in(labels, "name", &["poolSize", "pool_size"])
}),
executor_active_count: scalar_max_filtered(samples, "grpc_server_executor", |labels| {
label_in(labels, "name", &["activeCount", "active_count"])
}),
executor_queued_tasks: scalar_max_filtered(samples, "grpc_server_executor", |labels| {
label_in(labels, "name", &["queuedTasks", "queueSize", "queued_tasks"])
}),
},
config: NacosPrometheusConfigMetrics {
config_count: monitor(samples, "config", &["configCount"]),
get_config_total: monitor(samples, "config", &["getConfig", "getConfigCount"]),
publish_total: monitor(samples, "config", &["publish", "publishCount"]),
long_polling: monitor(samples, "config", &["longPolling"]),
listener_clients: None,
listener_keys: None,
notify_tasks: monitor(samples, "config", &["notifyTask"]),
notify_client_tasks: monitor(samples, "config", &["notifyClientTask"]),
dump_tasks: monitor(samples, "config", &["dumpTask"]),
subscriber_count: scalar_sum(samples, "nacos_config_subscriber"),
},
naming: NacosPrometheusNamingMetrics {
service_count: monitor(samples, "naming", &["serviceCount"]),
instance_count: monitor(samples, "naming", &["ipCount", "instanceCount"]),
subscriber_count: monitor(samples, "naming", &["subscriberCount"])
.or_else(|| scalar_sum(samples, "nacos_naming_subscriber")),
connection_count: monitor(samples, "core", &["longConnection"]),
total_push: monitor(samples, "naming", &["totalPush"]),
failed_push: monitor(samples, "naming", &["failedPush"]),
empty_push: monitor(samples, "naming", &["emptyPush"]),
push_pending_tasks: monitor(
samples,
"naming",
&["pushPendingTaskCount", "pushPendingTask", "pushPendingTasks"],
),
avg_push_cost_ms: monitor(samples, "naming", &["avgPushCost"]),
max_push_cost_ms: monitor(samples, "naming", &["maxPushCost"]),
leader_status: monitor(samples, "naming", &["leaderStatus"]),
},
}
}
fn normalize_rnacos(source: NacosPrometheusSource, samples: &[Sample]) -> NacosPrometheusSnapshot {
let mib = 1024.0 * 1024.0;
let http_count = scalar_sum(samples, "http_request_total_count");
let grpc_count = scalar_sum(samples, "grpc_request_total_count");
let rss_mib = scalar_sum(samples, "app_rss_memory");
let system_memory_mib = scalar_sum(samples, "sys_total_memory");
NacosPrometheusSnapshot {
source,
resource: NacosPrometheusResourceMetrics {
cpu_ratio: scalar_max(samples, "app_cpu_usage").and_then(normalize_percentage),
memory_ratio: scalar_max(samples, "app_memory_usage")
.and_then(normalize_percentage)
.or_else(|| ratio(rss_mib, system_memory_mib)),
memory_used_bytes: rss_mib.map(|value| value * mib),
memory_max_bytes: system_memory_mib.map(|value| value * mib),
rss_bytes: rss_mib.map(|value| value * mib),
vms_bytes: scalar_sum(samples, "app_vms_memory").map(|value| value * mib),
system_total_memory_bytes: system_memory_mib.map(|value| value * mib),
load_1m: None,
jvm_daemon_threads: None,
gc_pause_count: None,
},
traffic: NacosPrometheusTrafficMetrics {
http_requests_total: http_count,
grpc_requests_total: grpc_count,
http_errors_total: None,
grpc_errors_total: None,
http_duration_seconds_total: scalar_sum(samples, "http_request_handle_rt_histogram_sum")
.or_else(|| scalar_sum(samples, "http_request_handle_rt_summary_sum"))
.map(|value| value / 1_000.0),
http_duration_count: scalar_sum(samples, "http_request_handle_rt_histogram_count")
.or_else(|| scalar_sum(samples, "http_request_handle_rt_summary_count"))
.or(http_count),
grpc_duration_seconds_total: scalar_sum(samples, "grpc_request_handle_rt_histogram_sum")
.or_else(|| scalar_sum(samples, "grpc_request_handle_rt_summary_sum"))
.map(|value| value / 1_000.0),
grpc_duration_count: scalar_sum(samples, "grpc_request_handle_rt_histogram_count")
.or_else(|| scalar_sum(samples, "grpc_request_handle_rt_summary_count"))
.or(grpc_count),
http_p50_ms: quantile_ms_any(
samples,
&["http_request_handle_rt_summary", "http_request_handle_rt_histogram"],
0.5,
1.0,
),
http_p95_ms: quantile_ms_any(
samples,
&["http_request_handle_rt_summary", "http_request_handle_rt_histogram"],
0.95,
1.0,
),
http_p99_ms: quantile_ms_any(
samples,
&["http_request_handle_rt_summary", "http_request_handle_rt_histogram"],
0.99,
1.0,
),
grpc_p50_ms: quantile_ms_any(
samples,
&["grpc_request_handle_rt_summary", "grpc_request_handle_rt_histogram"],
0.5,
1.0,
),
grpc_p95_ms: quantile_ms_any(
samples,
&["grpc_request_handle_rt_summary", "grpc_request_handle_rt_histogram"],
0.95,
1.0,
),
grpc_p99_ms: quantile_ms_any(
samples,
&["grpc_request_handle_rt_summary", "grpc_request_handle_rt_histogram"],
0.99,
1.0,
),
executor_pool_size: None,
executor_active_count: None,
executor_queued_tasks: None,
},
config: NacosPrometheusConfigMetrics {
config_count: scalar_sum(samples, "config_data_size")
.or_else(|| scalar_sum(samples, "config_index_config_size")),
get_config_total: None,
publish_total: None,
long_polling: None,
listener_clients: scalar_sum(samples, "config_listener_client_size"),
listener_keys: scalar_sum(samples, "config_listener_key_size"),
notify_tasks: None,
notify_client_tasks: None,
dump_tasks: None,
subscriber_count: scalar_sum(samples, "config_subscriber_client_size")
.or_else(|| scalar_sum(samples, "config_subscriber_client_value_size")),
},
naming: NacosPrometheusNamingMetrics {
service_count: scalar_sum(samples, "naming_service_size")
.or_else(|| scalar_sum(samples, "naming_index_service_size")),
instance_count: scalar_sum(samples, "naming_instance_size"),
subscriber_count: scalar_sum(samples, "naming_subscriber_client_size")
.or_else(|| scalar_sum(samples, "naming_subscriber_client_value_size")),
connection_count: scalar_sum(samples, "grpc_conn_size"),
total_push: None,
failed_push: None,
empty_push: None,
push_pending_tasks: None,
avg_push_cost_ms: None,
max_push_cost_ms: None,
leader_status: None,
},
}
}
fn scalar_value(sample: &Sample) -> Option<f64> {
let value = match sample.value {
Value::Counter(value) | Value::Gauge(value) | Value::Untyped(value) => value,
Value::Histogram(_) | Value::Summary(_) => return None,
};
value.is_finite().then_some(value)
}
fn scalar_sum(samples: &[Sample], metric: &str) -> Option<f64> {
scalar_sum_filtered(samples, metric, |_| true)
}
fn scalar_sum_any(samples: &[Sample], metrics: &[&str]) -> Option<f64> {
metrics.iter().find_map(|metric| scalar_sum(samples, metric))
}
fn scalar_sum_filtered(samples: &[Sample], metric: &str, predicate: impl Fn(&Labels) -> bool) -> Option<f64> {
let values =
samples.iter().filter(|sample| sample.metric == metric && predicate(&sample.labels)).filter_map(scalar_value);
sum_non_empty(values)
}
fn scalar_sum_filtered_any(
samples: &[Sample],
metrics: &[&str],
predicate: impl Fn(&Labels) -> bool + Copy,
) -> Option<f64> {
metrics.iter().find_map(|metric| scalar_sum_filtered(samples, metric, predicate))
}
fn scalar_max(samples: &[Sample], metric: &str) -> Option<f64> {
scalar_max_filtered(samples, metric, |_| true)
}
fn scalar_max_any(samples: &[Sample], metrics: &[&str]) -> Option<f64> {
metrics.iter().find_map(|metric| scalar_max(samples, metric))
}
fn scalar_max_filtered(samples: &[Sample], metric: &str, predicate: impl Fn(&Labels) -> bool) -> Option<f64> {
samples
.iter()
.filter(|sample| sample.metric == metric && predicate(&sample.labels))
.filter_map(scalar_value)
.reduce(f64::max)
}
fn sum_non_empty(values: impl Iterator<Item = f64>) -> Option<f64> {
let mut found = false;
let sum = values.fold(0.0, |sum, value| {
found = true;
sum + value
});
found.then_some(sum)
}
fn monitor(samples: &[Sample], module: &str, names: &[&str]) -> Option<f64> {
names.iter().find_map(|name| {
scalar_sum_filtered(samples, "nacos_monitor", |labels| {
label_eq(labels, "module", module) && label_eq(labels, "name", name)
})
})
}
fn label_eq(labels: &Labels, key: &str, value: &str) -> bool {
labels.get(key).is_some_and(|current| current.eq_ignore_ascii_case(value))
}
fn label_in(labels: &Labels, key: &str, values: &[&str]) -> bool {
values.iter().any(|value| label_eq(labels, key, value))
}
fn is_error_labels(labels: &Labels) -> bool {
if let Some(status) = labels.get("status").or_else(|| labels.get("code")) {
return !status.starts_with('2') && status != "0";
}
labels.get("outcome").is_some_and(|outcome| !matches!(outcome.to_ascii_uppercase().as_str(), "SUCCESS" | "UNKNOWN"))
}
fn is_grpc_error_labels(labels: &Labels) -> bool {
if labels.get("success").is_some_and(|value| value.eq_ignore_ascii_case("false")) {
return true;
}
labels
.get("errorCode")
.or_else(|| labels.get("error_code"))
.or_else(|| labels.get("code"))
.is_some_and(|value| value != "0" && !value.eq_ignore_ascii_case("OK"))
|| labels.get("result").is_some_and(|value| !matches!(value.to_ascii_uppercase().as_str(), "SUCCESS" | "OK"))
}
fn ratio(numerator: Option<f64>, denominator: Option<f64>) -> Option<f64> {
match (numerator, denominator) {
(Some(numerator), Some(denominator)) if denominator > 0.0 => normalize_ratio(numerator / denominator),
_ => None,
}
}
fn normalize_ratio(value: f64) -> Option<f64> {
if !value.is_finite() || value < 0.0 {
return None;
}
Some((if value > 1.0 { value / 100.0 } else { value }).clamp(0.0, 1.0))
}
fn normalize_percentage(value: f64) -> Option<f64> {
if !value.is_finite() || value < 0.0 {
return None;
}
Some((value / 100.0).clamp(0.0, 1.0))
}
fn endpoint_fingerprint(endpoint: &str) -> String {
format!("{:x}", Sha256::digest(endpoint.as_bytes()))
}
fn redact_endpoint(endpoint: &str) -> String {
let Ok(mut url) = reqwest::Url::parse(endpoint) else {
return endpoint.split_once('?').map_or(endpoint, |(base, _)| base).to_string();
};
url.set_query(None);
url.set_fragment(None);
url.to_string()
}
fn quantile_ms(samples: &[Sample], metric: &str, quantile: f64, multiplier: f64) -> Option<f64> {
let mut histogram_buckets = HashMap::<u64, (f64, f64)>::new();
let mut summary_quantiles = Vec::new();
for sample in samples.iter().filter(|sample| sample.metric == metric) {
match &sample.value {
Value::Histogram(values) => {
for value in values {
if value.less_than.is_nan() || !value.count.is_finite() {
continue;
}
histogram_buckets
.entry(value.less_than.to_bits())
.and_modify(|(_, count)| *count += value.count)
.or_insert((value.less_than, value.count));
}
}
Value::Summary(values) => summary_quantiles.push(values),
_ => {}
}
}
if !histogram_buckets.is_empty() {
return histogram_quantile(histogram_buckets.into_values(), quantile).map(|value| value * multiplier);
}
// Prometheus summaries are already-calculated client-side quantiles and
// cannot be aggregated across label sets without changing their meaning.
let [values] = summary_quantiles.as_slice() else {
return None;
};
nearest_quantile(values.iter().map(|value| (value.quantile, value.count)), quantile).map(|value| value * multiplier)
}
fn quantile_ms_any(samples: &[Sample], metrics: &[&str], quantile: f64, multiplier: f64) -> Option<f64> {
metrics.iter().find_map(|metric| quantile_ms(samples, metric, quantile, multiplier))
}
fn nearest_quantile(values: impl Iterator<Item = (f64, f64)>, target: f64) -> Option<f64> {
values
.filter(|(quantile, value)| quantile.is_finite() && value.is_finite())
.min_by(|left, right| {
(left.0 - target).abs().partial_cmp(&(right.0 - target).abs()).unwrap_or(std::cmp::Ordering::Equal)
})
.map(|(_, value)| value)
}
fn histogram_quantile(values: impl Iterator<Item = (f64, f64)>, target: f64) -> Option<f64> {
let mut buckets =
values.filter(|(boundary, count)| !boundary.is_nan() && count.is_finite() && *count >= 0.0).collect::<Vec<_>>();
buckets.sort_by(|left, right| left.0.partial_cmp(&right.0).unwrap_or(std::cmp::Ordering::Equal));
let mut previous_count = 0.0;
for (_, count) in &mut buckets {
*count = count.max(previous_count);
previous_count = *count;
}
let total = buckets.last()?.1;
if total <= 0.0 {
return None;
}
let threshold = total * target.clamp(0.0, 1.0);
let index = buckets.iter().position(|(_, count)| *count >= threshold)?;
let (upper_bound, upper_count) = buckets[index];
if upper_bound.is_infinite() {
return (index > 0).then_some(buckets[index - 1].0).filter(|boundary| boundary.is_finite());
}
let (lower_bound, lower_count) = if index == 0 { (0.0, 0.0) } else { buckets[index - 1] };
if upper_count <= lower_count {
return Some(upper_bound);
}
Some(lower_bound + (upper_bound - lower_bound) * ((threshold - lower_count) / (upper_count - lower_count)))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn derives_and_deduplicates_nacos_endpoints() {
let cfg = test_config(NacosImplementation::Nacos, "http://127.0.0.1:8818", "/nacos");
assert_eq!(
endpoint_candidates(&cfg).unwrap(),
vec!["http://127.0.0.1:8818/nacos/actuator/prometheus", "http://127.0.0.1:8818/actuator/prometheus",]
);
}
#[test]
fn derives_rnacos_endpoints() {
let cfg = test_config(NacosImplementation::RNacos, "http://127.0.0.1:3848", "/nacos");
assert_eq!(
endpoint_candidates(&cfg).unwrap(),
vec![
"http://127.0.0.1:3848/metrics",
"http://127.0.0.1:3848/nacos/metrics",
"http://127.0.0.1:3848/rnacos/metrics",
]
);
}
#[test]
fn disabled_mode_skips_scraping() {
let mut cfg = test_config(NacosImplementation::Nacos, "http://127.0.0.1:8818", "/nacos");
cfg.metrics_mode = NacosMetricsMode::Disabled;
assert!(endpoint_candidates(&cfg).unwrap().is_empty());
}
#[test]
fn parses_nacos_metrics_and_labels() {
let body = r#"
# TYPE system_cpu_usage gauge
system_cpu_usage 0.25
# TYPE jvm_memory_used_bytes gauge
jvm_memory_used_bytes{area="heap",id="a"} 100
jvm_memory_used_bytes{area="heap",id="b"} 50
# TYPE jvm_memory_max_bytes gauge
jvm_memory_max_bytes{area="heap",id="a"} 400
# TYPE http_server_requests_seconds counter
http_server_requests_seconds_count{status="200"} 90
http_server_requests_seconds_count{status="500"} 10
http_server_requests_seconds_sum 25
# TYPE nacos_monitor gauge
nacos_monitor{module="naming",name="serviceCount"} 12
nacos_monitor{module="core",name="longConnection"} 8
nacos_monitor{module="naming",name="pushPendingTaskCount"} 3
# TYPE nacos_naming_subscriber gauge
nacos_naming_subscriber{version="v1"} 2
nacos_naming_subscriber{version="v2"} 4
"#;
let parsed = parse_scrape(body, &NacosImplementation::Nacos, "http://localhost/metrics").unwrap();
assert_eq!(parsed.resource.cpu_ratio, Some(0.25));
assert_eq!(parsed.resource.memory_used_bytes, Some(150.0));
assert_eq!(parsed.resource.memory_ratio, Some(0.375));
assert_eq!(parsed.traffic.http_requests_total, Some(100.0));
assert_eq!(parsed.traffic.http_errors_total, Some(10.0));
assert_eq!(parsed.naming.service_count, Some(12.0));
assert_eq!(parsed.naming.connection_count, Some(8.0));
assert_eq!(parsed.naming.subscriber_count, Some(6.0));
assert_eq!(parsed.naming.push_pending_tasks, Some(3.0));
}
#[test]
fn aggregates_histogram_buckets_across_label_sets() {
let body = r#"
# TYPE http_server_requests_seconds histogram
http_server_requests_seconds_bucket{route="a",le="1"} 50
http_server_requests_seconds_bucket{route="a",le="10"} 50
http_server_requests_seconds_bucket{route="a",le="+Inf"} 50
http_server_requests_seconds_sum{route="a"} 25
http_server_requests_seconds_count{route="a"} 50
http_server_requests_seconds_bucket{route="b",le="1"} 0
http_server_requests_seconds_bucket{route="b",le="10"} 50
http_server_requests_seconds_bucket{route="b",le="+Inf"} 50
http_server_requests_seconds_sum{route="b"} 250
http_server_requests_seconds_count{route="b"} 50
"#;
let parsed = parse_scrape(body, &NacosImplementation::Nacos, "http://localhost/metrics").unwrap();
assert_eq!(parsed.traffic.http_p50_ms, Some(1_000.0));
}
#[test]
fn redacts_endpoint_query_from_snapshot_source() {
let parsed = parse_scrape(
"# TYPE system_cpu_usage gauge\nsystem_cpu_usage 0.25\n",
&NacosImplementation::Nacos,
"http://localhost/metrics?token=secret&node=a",
)
.unwrap();
let other_source = parse_scrape(
"# TYPE system_cpu_usage gauge\nsystem_cpu_usage 0.25\n",
&NacosImplementation::Nacos,
"http://localhost/metrics?token=secret&node=b",
)
.unwrap();
assert_eq!(parsed.source.endpoint, "http://localhost/metrics");
assert_ne!(parsed.source.fingerprint, other_source.source.fingerprint);
assert!(!parsed.source.fingerprint.as_deref().unwrap().contains("secret"));
}
#[test]
fn parses_rnacos_units_counts_and_summary() {
let body = r#"
# TYPE app_cpu_usage gauge
app_cpu_usage 12.5
# TYPE app_rss_memory gauge
app_rss_memory 10
# TYPE sys_total_memory gauge
sys_total_memory 100
# TYPE naming_service_size gauge
naming_service_size 4
# TYPE grpc_conn_size gauge
grpc_conn_size 3
# TYPE http_request_total_count counter
http_request_total_count 20
# TYPE http_request_handle_rt_summary summary
http_request_handle_rt_summary{quantile="0.5"} 5
http_request_handle_rt_summary{quantile="0.95"} 15
http_request_handle_rt_summary_sum 100
http_request_handle_rt_summary_count 20
"#;
let parsed = parse_scrape(body, &NacosImplementation::RNacos, "http://localhost/metrics").unwrap();
assert_eq!(parsed.resource.cpu_ratio, Some(0.125));
assert_eq!(parsed.resource.rss_bytes, Some(10.0 * 1024.0 * 1024.0));
assert_eq!(parsed.resource.memory_ratio, Some(0.1));
assert_eq!(parsed.naming.service_count, Some(4.0));
assert_eq!(parsed.naming.connection_count, Some(3.0));
assert_eq!(parsed.traffic.http_p95_ms, Some(15.0));
assert_eq!(parsed.traffic.http_duration_seconds_total, Some(0.1));
}
#[test]
fn converts_low_rnacos_percentages_to_ratios() {
let body = r#"
# TYPE app_cpu_usage gauge
app_cpu_usage 0.5
# TYPE app_memory_usage gauge
app_memory_usage 1
"#;
let parsed = parse_scrape(body, &NacosImplementation::RNacos, "http://localhost/metrics").unwrap();
assert_eq!(parsed.resource.cpu_ratio, Some(0.005));
assert_eq!(parsed.resource.memory_ratio, Some(0.01));
}
#[test]
fn rejects_responses_without_samples() {
assert!(parse_scrape("<html>not metrics</html>", &NacosImplementation::Nacos, "http://localhost/metrics")
.unwrap_err()
.contains("no Prometheus samples"));
}
#[tokio::test]
async fn falls_back_after_a_missing_auto_endpoint() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
for index in 0..2 {
let (mut socket, _) = listener.accept().await.unwrap();
let mut request = vec![0_u8; 2048];
let _ = tokio::io::AsyncReadExt::read(&mut socket, &mut request).await.unwrap();
if index == 0 {
let body = "not found";
let response = format!(
"HTTP/1.1 404 Not Found\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
body.len()
);
tokio::io::AsyncWriteExt::write_all(&mut socket, response.as_bytes()).await.unwrap();
} else {
let body = "# TYPE system_cpu_usage gauge\nsystem_cpu_usage 0.4\n";
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
body.len()
);
tokio::io::AsyncWriteExt::write_all(&mut socket, response.as_bytes()).await.unwrap();
}
}
});
let cfg = test_config(NacosImplementation::Nacos, &format!("http://{address}"), "");
let client = reqwest::Client::new();
let parsed = scrape(&client, &cfg).await.unwrap().unwrap();
assert_eq!(parsed.resource.cpu_ratio, Some(0.4));
assert!(parsed.source.endpoint.ends_with("/nacos/actuator/prometheus"));
server.await.unwrap();
}
#[tokio::test]
async fn rejects_oversized_scrapes() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let mut request = vec![0_u8; 2048];
let _ = tokio::io::AsyncReadExt::read(&mut socket, &mut request).await.unwrap();
let body = vec![b'x'; MAX_SCRAPE_BYTES + 1];
let header = format!(
"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body.len()
);
let _ = tokio::io::AsyncWriteExt::write_all(&mut socket, header.as_bytes()).await;
let _ = tokio::io::AsyncWriteExt::write_all(&mut socket, &body).await;
});
let mut cfg = test_config(NacosImplementation::Nacos, "http://127.0.0.1:8848", "");
cfg.metrics_mode = NacosMetricsMode::Custom;
cfg.metrics_url = format!("http://{address}/metrics");
let error = scrape(&reqwest::Client::new(), &cfg).await.unwrap_err();
assert!(error.contains("exceeds"));
server.await.unwrap();
}
#[tokio::test]
async fn redacts_endpoint_query_from_scrape_errors() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let mut request = vec![0_u8; 2048];
let _ = tokio::io::AsyncReadExt::read(&mut socket, &mut request).await.unwrap();
let response = "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n";
tokio::io::AsyncWriteExt::write_all(&mut socket, response.as_bytes()).await.unwrap();
});
let mut cfg = test_config(NacosImplementation::Nacos, "http://127.0.0.1:8848", "");
cfg.metrics_mode = NacosMetricsMode::Custom;
cfg.metrics_url = format!("http://{address}/metrics?token=secret");
let error = scrape(&reqwest::Client::new(), &cfg).await.unwrap_err();
assert!(error.contains(&format!("http://{address}/metrics")));
assert!(!error.contains("secret"));
server.await.unwrap();
}
#[tokio::test]
async fn times_out_without_exposing_endpoint_query() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let mut request = vec![0_u8; 2048];
let _ = tokio::io::AsyncReadExt::read(&mut socket, &mut request).await.unwrap();
std::future::pending::<()>().await;
});
let mut cfg = test_config(NacosImplementation::Nacos, "http://127.0.0.1:8848", "");
cfg.metrics_mode = NacosMetricsMode::Custom;
cfg.metrics_url = format!("http://{address}/metrics?token=secret");
let error = scrape_with_timeout(&reqwest::Client::new(), &cfg, Duration::from_millis(25)).await.unwrap_err();
assert!(error.contains("timed out"));
assert!(error.contains(&format!("http://{address}/metrics")));
assert!(!error.contains("secret"));
server.abort();
}
fn test_config(implementation: NacosImplementation, server_addr: &str, context_path: &str) -> NacosAdminConfig {
NacosAdminConfig {
implementation: Some(implementation),
version_mode: None,
server_addr: server_addr.to_string(),
display_server_addr: server_addr.to_string(),
namespace: String::new(),
context_path: context_path.to_string(),
rnacos_console_addr: String::new(),
rnacos_history_enabled: None,
rnacos_console_auth: Default::default(),
auth: Default::default(),
tls_skip_verify: false,
metrics_mode: NacosMetricsMode::Auto,
metrics_url: String::new(),
page_size: 20,
connect_override: None,
}
}
}