fix(rabbitmq): add bounded paging for queue and exchange management lists
Closes #8879
This commit is contained in:
1 parent
33f212a452
commit
725b1ee252
22 files changed
+1445
-67
No files matched your search
@@ -186,6 +186,33 @@ impl MessageQueueAdmin for RabbitMqAdmin {
|
||||
Ok(topics.into_iter().map(|topic| topic_info_from_agent_value(&topic)).collect())
|
||||
}
|
||||
|
||||
async fn list_topics_page(
|
||||
&self,
|
||||
ns: &NamespaceRef,
|
||||
_opts: ListTopicsOpts,
|
||||
pagination: MqListPageRequest,
|
||||
) -> Result<MqListPage<TopicInfo>, String> {
|
||||
let params = with_virtual_host(rabbitmq_page_params(&pagination), &ns.namespace);
|
||||
let result: serde_json::Value = self.call("mq_list_topics", params).await?;
|
||||
let topics = result
|
||||
.get("topics")
|
||||
.and_then(|value| value.as_array())
|
||||
.cloned()
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.map(|topic| topic_info_from_agent_value(&topic))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
if let Some(metadata) = agent_page_metadata(&result, &pagination) {
|
||||
return Ok(metadata.with_items(topics));
|
||||
}
|
||||
|
||||
// Older installed agents ignore the optional page fields and return
|
||||
// the legacy full-list envelope. Keep that wire combination working
|
||||
// and bound what crosses the command boundary after it arrives.
|
||||
Ok(paginate_topic_infos(topics, &pagination))
|
||||
}
|
||||
|
||||
async fn create_topic(&self, topic: &TopicRef, _partitions: Option<u32>) -> Result<(), String> {
|
||||
require_specific_vhost(&topic.namespace)?;
|
||||
let params = with_virtual_host(
|
||||
@@ -237,6 +264,32 @@ impl MessageQueueAdmin for RabbitMqAdmin {
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn list_exchanges_page(
|
||||
&self,
|
||||
ns: &NamespaceRef,
|
||||
pagination: MqListPageRequest,
|
||||
) -> Result<MqListPage<MqExchangeInfo>, String> {
|
||||
let params = with_virtual_host(rabbitmq_page_params(&pagination), &ns.namespace);
|
||||
let result: serde_json::Value = self.call("mq_list_exchanges", params).await?;
|
||||
let exchanges = result
|
||||
.get("exchanges")
|
||||
.and_then(|value| value.as_array())
|
||||
.cloned()
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.map(|exchange| {
|
||||
serde_json::from_value(map_item_namespace(exchange))
|
||||
.map_err(|err| format!("Invalid exchange entry from agent: {err}"))
|
||||
})
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
|
||||
if let Some(metadata) = agent_page_metadata(&result, &pagination) {
|
||||
return Ok(metadata.with_items(exchanges));
|
||||
}
|
||||
|
||||
Ok(paginate_exchange_infos(exchanges, &pagination))
|
||||
}
|
||||
|
||||
async fn create_exchange(
|
||||
&self,
|
||||
ns: &NamespaceRef,
|
||||
@@ -679,6 +732,119 @@ impl MessageQueueAdmin for RabbitMqAdmin {
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
struct AgentPageMetadata {
|
||||
page: u32,
|
||||
page_size: u32,
|
||||
total_count: u64,
|
||||
has_more: bool,
|
||||
}
|
||||
|
||||
impl AgentPageMetadata {
|
||||
fn with_items<T>(self, items: Vec<T>) -> MqListPage<T> {
|
||||
MqListPage {
|
||||
items,
|
||||
page: self.page,
|
||||
page_size: self.page_size,
|
||||
total_count: self.total_count,
|
||||
has_more: self.has_more,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn rabbitmq_page_params(pagination: &MqListPageRequest) -> serde_json::Value {
|
||||
let sort = match pagination.sort {
|
||||
MqListSort::Name => "name",
|
||||
MqListSort::MessagesReady => "messages_ready",
|
||||
};
|
||||
let mut params = serde_json::json!({
|
||||
"page": pagination.page,
|
||||
"page_size": pagination.page_size,
|
||||
"sort": sort,
|
||||
"sort_reverse": pagination.sort_descending,
|
||||
});
|
||||
if let Some(search) = pagination.search.as_deref().map(str::trim).filter(|search| !search.is_empty()) {
|
||||
params["search"] = serde_json::json!(search);
|
||||
}
|
||||
params
|
||||
}
|
||||
|
||||
fn agent_page_metadata(result: &serde_json::Value, request: &MqListPageRequest) -> Option<AgentPageMetadata> {
|
||||
// `totalCount` distinguishes the new bounded agent envelope from the
|
||||
// legacy `{topics:[...]}` / `{exchanges:[...]}` response.
|
||||
let total_count = result.get("totalCount")?.as_u64()?;
|
||||
let page = result
|
||||
.get("page")
|
||||
.and_then(serde_json::Value::as_u64)
|
||||
.and_then(|value| u32::try_from(value).ok())
|
||||
.unwrap_or(request.page);
|
||||
let page_size = result
|
||||
.get("pageSize")
|
||||
.and_then(serde_json::Value::as_u64)
|
||||
.and_then(|value| u32::try_from(value).ok())
|
||||
.unwrap_or(request.page_size);
|
||||
let has_more = result
|
||||
.get("hasMore")
|
||||
.and_then(serde_json::Value::as_bool)
|
||||
.unwrap_or_else(|| u64::from(page).saturating_mul(u64::from(page_size)) < total_count);
|
||||
Some(AgentPageMetadata { page, page_size, total_count, has_more })
|
||||
}
|
||||
|
||||
fn paginate_topic_infos(mut topics: Vec<TopicInfo>, request: &MqListPageRequest) -> MqListPage<TopicInfo> {
|
||||
let search = request.search.as_deref().map(str::trim).unwrap_or_default().to_lowercase();
|
||||
if !search.is_empty() {
|
||||
topics.retain(|topic| {
|
||||
topic.name.to_lowercase().contains(&search) || topic.short_name.to_lowercase().contains(&search)
|
||||
});
|
||||
}
|
||||
topics.sort_by(|left, right| {
|
||||
let primary = match request.sort {
|
||||
MqListSort::Name => left.name.cmp(&right.name),
|
||||
MqListSort::MessagesReady => left
|
||||
.messages_ready
|
||||
.or(left.message_count)
|
||||
.unwrap_or_default()
|
||||
.cmp(&right.messages_ready.or(right.message_count).unwrap_or_default()),
|
||||
};
|
||||
let primary = if request.sort_descending { primary.reverse() } else { primary };
|
||||
primary.then_with(|| left.short_name.cmp(&right.short_name))
|
||||
});
|
||||
paginate_items(topics, request)
|
||||
}
|
||||
|
||||
fn paginate_exchange_infos(
|
||||
mut exchanges: Vec<MqExchangeInfo>,
|
||||
request: &MqListPageRequest,
|
||||
) -> MqListPage<MqExchangeInfo> {
|
||||
let search = request.search.as_deref().map(str::trim).unwrap_or_default().to_lowercase();
|
||||
if !search.is_empty() {
|
||||
exchanges.retain(|exchange| {
|
||||
let display_name = if exchange.name.is_empty() { "(AMQP default)" } else { &exchange.name };
|
||||
display_name.to_lowercase().contains(&search)
|
||||
});
|
||||
}
|
||||
exchanges.sort_by(|left, right| {
|
||||
let ordering = left.name.cmp(&right.name);
|
||||
if request.sort_descending {
|
||||
ordering.reverse()
|
||||
} else {
|
||||
ordering
|
||||
}
|
||||
});
|
||||
paginate_items(exchanges, request)
|
||||
}
|
||||
|
||||
fn paginate_items<T>(items: Vec<T>, request: &MqListPageRequest) -> MqListPage<T> {
|
||||
let total_count = items.len() as u64;
|
||||
let start = u64::from(request.page.saturating_sub(1)).saturating_mul(u64::from(request.page_size));
|
||||
let page_items = if start >= total_count {
|
||||
Vec::new()
|
||||
} else {
|
||||
items.into_iter().skip(start as usize).take(request.page_size as usize).collect()
|
||||
};
|
||||
let has_more = start.saturating_add(page_items.len() as u64) < total_count;
|
||||
MqListPage { items: page_items, page: request.page, page_size: request.page_size, total_count, has_more }
|
||||
}
|
||||
|
||||
/// Topic refs use flat queue names for RabbitMQ: the queue name is the short
|
||||
/// topic name, the tenant is ignored. The namespace maps to a virtual host
|
||||
/// and is passed through separately (see `with_virtual_host`).
|
||||
@@ -1689,6 +1855,61 @@ mod tests {
|
||||
assert!(params.get("virtual_host").is_none(), "all-vhosts listing must not scope to a vhost");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rabbitmq_page_params_map_search_and_message_sort_for_the_agent() {
|
||||
let params = rabbitmq_page_params(&MqListPageRequest {
|
||||
page: 3,
|
||||
page_size: 200,
|
||||
search: Some("orders".to_string()),
|
||||
sort: MqListSort::MessagesReady,
|
||||
sort_descending: true,
|
||||
});
|
||||
|
||||
assert_eq!(params.get("page").and_then(|value| value.as_u64()), Some(3));
|
||||
assert_eq!(params.get("page_size").and_then(|value| value.as_u64()), Some(200));
|
||||
assert_eq!(params.get("search").and_then(|value| value.as_str()), Some("orders"));
|
||||
assert_eq!(params.get("sort").and_then(|value| value.as_str()), Some("messages_ready"));
|
||||
assert_eq!(params.get("sort_reverse").and_then(|value| value.as_bool()), Some(true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_agent_topic_result_is_filtered_sorted_and_bounded_locally() {
|
||||
let topics = vec![
|
||||
TopicInfo {
|
||||
name: "orders-low".to_string(),
|
||||
short_name: "orders-low".to_string(),
|
||||
messages_ready: Some(2),
|
||||
..TopicInfo::default()
|
||||
},
|
||||
TopicInfo {
|
||||
name: "unrelated".to_string(),
|
||||
short_name: "unrelated".to_string(),
|
||||
messages_ready: Some(99),
|
||||
..TopicInfo::default()
|
||||
},
|
||||
TopicInfo {
|
||||
name: "orders-high".to_string(),
|
||||
short_name: "orders-high".to_string(),
|
||||
messages_ready: Some(8),
|
||||
..TopicInfo::default()
|
||||
},
|
||||
];
|
||||
let page = paginate_topic_infos(
|
||||
topics,
|
||||
&MqListPageRequest {
|
||||
page: 1,
|
||||
page_size: 1,
|
||||
search: Some("orders".to_string()),
|
||||
sort: MqListSort::MessagesReady,
|
||||
sort_descending: true,
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(page.total_count, 2);
|
||||
assert!(page.has_more);
|
||||
assert_eq!(page.items[0].name, "orders-high");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn map_item_namespace_moves_vhost_into_namespace_field() {
|
||||
let item = map_item_namespace(serde_json::json!({ "name": "dbx-q", "vhost": "orders" }));
|
||||
|
||||
@@ -40,6 +40,19 @@ pub trait MessageQueueAdmin: Send + Sync {
|
||||
|
||||
// ---- Topics ----
|
||||
async fn list_topics(&self, ns: &NamespaceRef, opts: ListTopicsOpts) -> Result<Vec<TopicInfo>, String>;
|
||||
|
||||
/// Bounded topic listing for providers that support server-side paging.
|
||||
/// The legacy unbounded method remains available for existing callers and
|
||||
/// providers whose native APIs already return small lists.
|
||||
async fn list_topics_page(
|
||||
&self,
|
||||
_ns: &NamespaceRef,
|
||||
_opts: ListTopicsOpts,
|
||||
_pagination: MqListPageRequest,
|
||||
) -> Result<MqListPage<TopicInfo>, String> {
|
||||
Err("Paged topic listing is not supported by this MQ system".to_string())
|
||||
}
|
||||
|
||||
async fn create_topic(&self, topic: &TopicRef, partitions: Option<u32>) -> Result<(), String>;
|
||||
async fn delete_topic(&self, topic: &TopicRef, force: bool) -> Result<(), String>;
|
||||
async fn update_partitions(&self, topic: &TopicRef, partitions: u32) -> Result<(), String>;
|
||||
@@ -56,6 +69,14 @@ pub trait MessageQueueAdmin: Send + Sync {
|
||||
Err("Exchanges are not supported by this MQ system".to_string())
|
||||
}
|
||||
|
||||
async fn list_exchanges_page(
|
||||
&self,
|
||||
_ns: &NamespaceRef,
|
||||
_pagination: MqListPageRequest,
|
||||
) -> Result<MqListPage<MqExchangeInfo>, String> {
|
||||
Err("Paged exchange listing is not supported by this MQ system".to_string())
|
||||
}
|
||||
|
||||
async fn create_exchange(
|
||||
&self,
|
||||
_ns: &NamespaceRef,
|
||||
|
||||
@@ -153,6 +153,18 @@ pub async fn mq_list_topics_core(
|
||||
adapter.list_topics(&ns, opts).await
|
||||
}
|
||||
|
||||
pub async fn mq_list_topics_page_core(
|
||||
state: &AppState,
|
||||
conn_id: &str,
|
||||
ns: NamespaceRef,
|
||||
opts: ListTopicsOpts,
|
||||
pagination: MqListPageRequest,
|
||||
) -> Result<MqListPage<TopicInfo>, String> {
|
||||
pagination.validate()?;
|
||||
let adapter = get_adapter(state, conn_id).await?;
|
||||
adapter.list_topics_page(&ns, opts, pagination).await
|
||||
}
|
||||
|
||||
pub async fn mq_create_topic_core(
|
||||
state: &AppState,
|
||||
conn_id: &str,
|
||||
@@ -206,6 +218,17 @@ pub async fn mq_list_exchanges_core(
|
||||
adapter.list_exchanges(&ns).await
|
||||
}
|
||||
|
||||
pub async fn mq_list_exchanges_page_core(
|
||||
state: &AppState,
|
||||
conn_id: &str,
|
||||
ns: NamespaceRef,
|
||||
pagination: MqListPageRequest,
|
||||
) -> Result<MqListPage<MqExchangeInfo>, String> {
|
||||
pagination.validate()?;
|
||||
let adapter = get_adapter(state, conn_id).await?;
|
||||
adapter.list_exchanges_page(&ns, pagination).await
|
||||
}
|
||||
|
||||
pub async fn mq_create_exchange_core(
|
||||
state: &AppState,
|
||||
conn_id: &str,
|
||||
|
||||
@@ -380,6 +380,79 @@ pub struct ListTopicsOpts {
|
||||
pub include_non_persistent: bool,
|
||||
}
|
||||
|
||||
pub const DEFAULT_MQ_LIST_PAGE_SIZE: u32 = 100;
|
||||
pub const MAX_MQ_LIST_PAGE_SIZE: u32 = 500;
|
||||
|
||||
/// Sort keys shared by the bounded MQ list endpoints. `MessagesReady` is used
|
||||
/// by RabbitMQ queues; exchange listings use the default name ordering.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum MqListSort {
|
||||
#[default]
|
||||
Name,
|
||||
MessagesReady,
|
||||
}
|
||||
|
||||
/// One-based request for a bounded MQ list page.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct MqListPageRequest {
|
||||
#[serde(default = "default_mq_list_page")]
|
||||
pub page: u32,
|
||||
#[serde(default = "default_mq_list_page_size")]
|
||||
pub page_size: u32,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub search: Option<String>,
|
||||
#[serde(default)]
|
||||
pub sort: MqListSort,
|
||||
#[serde(default)]
|
||||
pub sort_descending: bool,
|
||||
}
|
||||
|
||||
impl Default for MqListPageRequest {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
page: default_mq_list_page(),
|
||||
page_size: default_mq_list_page_size(),
|
||||
search: None,
|
||||
sort: MqListSort::default(),
|
||||
sort_descending: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl MqListPageRequest {
|
||||
pub fn validate(&self) -> Result<(), String> {
|
||||
if self.page == 0 {
|
||||
return Err("page must be at least 1".to_string());
|
||||
}
|
||||
if self.page_size == 0 || self.page_size > MAX_MQ_LIST_PAGE_SIZE {
|
||||
return Err(format!("pageSize must be between 1 and {MAX_MQ_LIST_PAGE_SIZE}"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
const fn default_mq_list_page() -> u32 {
|
||||
1
|
||||
}
|
||||
|
||||
const fn default_mq_list_page_size() -> u32 {
|
||||
DEFAULT_MQ_LIST_PAGE_SIZE
|
||||
}
|
||||
|
||||
/// A bounded list response. `total_count` is the number of rows matching the
|
||||
/// request's search filter, not merely the number loaded in this page.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct MqListPage<T> {
|
||||
pub items: Vec<T>,
|
||||
pub page: u32,
|
||||
pub page_size: u32,
|
||||
pub total_count: u64,
|
||||
pub has_more: bool,
|
||||
}
|
||||
|
||||
/// Aggregated, UI-friendly topic statistics. Parsed from the version-specific
|
||||
/// raw stats payload by the adapter's version profile.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
|
||||
Reference in new issue
Block a user