fix(rabbitmq): add bounded paging for queue and exchange management lists

Closes #8879
This commit is contained in:
t8y2 committed 2026-09-27 09:38:54 +00:00
1 parent 33f212a452
commit 725b1ee252
22 files changed
+1445 -67

No files matched your search

@@ -186,6 +186,33 @@ impl MessageQueueAdmin for RabbitMqAdmin {
Ok(topics.into_iter().map(|topic| topic_info_from_agent_value(&topic)).collect())
}
async fn list_topics_page(
&self,
ns: &NamespaceRef,
_opts: ListTopicsOpts,
pagination: MqListPageRequest,
) -> Result<MqListPage<TopicInfo>, String> {
let params = with_virtual_host(rabbitmq_page_params(&pagination), &ns.namespace);
let result: serde_json::Value = self.call("mq_list_topics", params).await?;
let topics = result
.get("topics")
.and_then(|value| value.as_array())
.cloned()
.unwrap_or_default()
.into_iter()
.map(|topic| topic_info_from_agent_value(&topic))
.collect::<Vec<_>>();
if let Some(metadata) = agent_page_metadata(&result, &pagination) {
return Ok(metadata.with_items(topics));
}
// Older installed agents ignore the optional page fields and return
// the legacy full-list envelope. Keep that wire combination working
// and bound what crosses the command boundary after it arrives.
Ok(paginate_topic_infos(topics, &pagination))
}
async fn create_topic(&self, topic: &TopicRef, _partitions: Option<u32>) -> Result<(), String> {
require_specific_vhost(&topic.namespace)?;
let params = with_virtual_host(
@@ -237,6 +264,32 @@ impl MessageQueueAdmin for RabbitMqAdmin {
.collect()
}
async fn list_exchanges_page(
&self,
ns: &NamespaceRef,
pagination: MqListPageRequest,
) -> Result<MqListPage<MqExchangeInfo>, String> {
let params = with_virtual_host(rabbitmq_page_params(&pagination), &ns.namespace);
let result: serde_json::Value = self.call("mq_list_exchanges", params).await?;
let exchanges = result
.get("exchanges")
.and_then(|value| value.as_array())
.cloned()
.unwrap_or_default()
.into_iter()
.map(|exchange| {
serde_json::from_value(map_item_namespace(exchange))
.map_err(|err| format!("Invalid exchange entry from agent: {err}"))
})
.collect::<Result<Vec<_>, _>>()?;
if let Some(metadata) = agent_page_metadata(&result, &pagination) {
return Ok(metadata.with_items(exchanges));
}
Ok(paginate_exchange_infos(exchanges, &pagination))
}
async fn create_exchange(
&self,
ns: &NamespaceRef,
@@ -679,6 +732,119 @@ impl MessageQueueAdmin for RabbitMqAdmin {
// Helpers
// ---------------------------------------------------------------------------
struct AgentPageMetadata {
page: u32,
page_size: u32,
total_count: u64,
has_more: bool,
}
impl AgentPageMetadata {
fn with_items<T>(self, items: Vec<T>) -> MqListPage<T> {
MqListPage {
items,
page: self.page,
page_size: self.page_size,
total_count: self.total_count,
has_more: self.has_more,
}
}
}
fn rabbitmq_page_params(pagination: &MqListPageRequest) -> serde_json::Value {
let sort = match pagination.sort {
MqListSort::Name => "name",
MqListSort::MessagesReady => "messages_ready",
};
let mut params = serde_json::json!({
"page": pagination.page,
"page_size": pagination.page_size,
"sort": sort,
"sort_reverse": pagination.sort_descending,
});
if let Some(search) = pagination.search.as_deref().map(str::trim).filter(|search| !search.is_empty()) {
params["search"] = serde_json::json!(search);
}
params
}
fn agent_page_metadata(result: &serde_json::Value, request: &MqListPageRequest) -> Option<AgentPageMetadata> {
// `totalCount` distinguishes the new bounded agent envelope from the
// legacy `{topics:[...]}` / `{exchanges:[...]}` response.
let total_count = result.get("totalCount")?.as_u64()?;
let page = result
.get("page")
.and_then(serde_json::Value::as_u64)
.and_then(|value| u32::try_from(value).ok())
.unwrap_or(request.page);
let page_size = result
.get("pageSize")
.and_then(serde_json::Value::as_u64)
.and_then(|value| u32::try_from(value).ok())
.unwrap_or(request.page_size);
let has_more = result
.get("hasMore")
.and_then(serde_json::Value::as_bool)
.unwrap_or_else(|| u64::from(page).saturating_mul(u64::from(page_size)) < total_count);
Some(AgentPageMetadata { page, page_size, total_count, has_more })
}
fn paginate_topic_infos(mut topics: Vec<TopicInfo>, request: &MqListPageRequest) -> MqListPage<TopicInfo> {
let search = request.search.as_deref().map(str::trim).unwrap_or_default().to_lowercase();
if !search.is_empty() {
topics.retain(|topic| {
topic.name.to_lowercase().contains(&search) || topic.short_name.to_lowercase().contains(&search)
});
}
topics.sort_by(|left, right| {
let primary = match request.sort {
MqListSort::Name => left.name.cmp(&right.name),
MqListSort::MessagesReady => left
.messages_ready
.or(left.message_count)
.unwrap_or_default()
.cmp(&right.messages_ready.or(right.message_count).unwrap_or_default()),
};
let primary = if request.sort_descending { primary.reverse() } else { primary };
primary.then_with(|| left.short_name.cmp(&right.short_name))
});
paginate_items(topics, request)
}
fn paginate_exchange_infos(
mut exchanges: Vec<MqExchangeInfo>,
request: &MqListPageRequest,
) -> MqListPage<MqExchangeInfo> {
let search = request.search.as_deref().map(str::trim).unwrap_or_default().to_lowercase();
if !search.is_empty() {
exchanges.retain(|exchange| {
let display_name = if exchange.name.is_empty() { "(AMQP default)" } else { &exchange.name };
display_name.to_lowercase().contains(&search)
});
}
exchanges.sort_by(|left, right| {
let ordering = left.name.cmp(&right.name);
if request.sort_descending {
ordering.reverse()
} else {
ordering
}
});
paginate_items(exchanges, request)
}
fn paginate_items<T>(items: Vec<T>, request: &MqListPageRequest) -> MqListPage<T> {
let total_count = items.len() as u64;
let start = u64::from(request.page.saturating_sub(1)).saturating_mul(u64::from(request.page_size));
let page_items = if start >= total_count {
Vec::new()
} else {
items.into_iter().skip(start as usize).take(request.page_size as usize).collect()
};
let has_more = start.saturating_add(page_items.len() as u64) < total_count;
MqListPage { items: page_items, page: request.page, page_size: request.page_size, total_count, has_more }
}
/// Topic refs use flat queue names for RabbitMQ: the queue name is the short
/// topic name, the tenant is ignored. The namespace maps to a virtual host
/// and is passed through separately (see `with_virtual_host`).
@@ -1689,6 +1855,61 @@ mod tests {
assert!(params.get("virtual_host").is_none(), "all-vhosts listing must not scope to a vhost");
}
#[test]
fn rabbitmq_page_params_map_search_and_message_sort_for_the_agent() {
let params = rabbitmq_page_params(&MqListPageRequest {
page: 3,
page_size: 200,
search: Some("orders".to_string()),
sort: MqListSort::MessagesReady,
sort_descending: true,
});
assert_eq!(params.get("page").and_then(|value| value.as_u64()), Some(3));
assert_eq!(params.get("page_size").and_then(|value| value.as_u64()), Some(200));
assert_eq!(params.get("search").and_then(|value| value.as_str()), Some("orders"));
assert_eq!(params.get("sort").and_then(|value| value.as_str()), Some("messages_ready"));
assert_eq!(params.get("sort_reverse").and_then(|value| value.as_bool()), Some(true));
}
#[test]
fn legacy_agent_topic_result_is_filtered_sorted_and_bounded_locally() {
let topics = vec![
TopicInfo {
name: "orders-low".to_string(),
short_name: "orders-low".to_string(),
messages_ready: Some(2),
..TopicInfo::default()
},
TopicInfo {
name: "unrelated".to_string(),
short_name: "unrelated".to_string(),
messages_ready: Some(99),
..TopicInfo::default()
},
TopicInfo {
name: "orders-high".to_string(),
short_name: "orders-high".to_string(),
messages_ready: Some(8),
..TopicInfo::default()
},
];
let page = paginate_topic_infos(
topics,
&MqListPageRequest {
page: 1,
page_size: 1,
search: Some("orders".to_string()),
sort: MqListSort::MessagesReady,
sort_descending: true,
},
);
assert_eq!(page.total_count, 2);
assert!(page.has_more);
assert_eq!(page.items[0].name, "orders-high");
}
#[test]
fn map_item_namespace_moves_vhost_into_namespace_field() {
let item = map_item_namespace(serde_json::json!({ "name": "dbx-q", "vhost": "orders" }));
+21
View File
@@ -40,6 +40,19 @@ pub trait MessageQueueAdmin: Send + Sync {
// ---- Topics ----
async fn list_topics(&self, ns: &NamespaceRef, opts: ListTopicsOpts) -> Result<Vec<TopicInfo>, String>;
/// Bounded topic listing for providers that support server-side paging.
/// The legacy unbounded method remains available for existing callers and
/// providers whose native APIs already return small lists.
async fn list_topics_page(
&self,
_ns: &NamespaceRef,
_opts: ListTopicsOpts,
_pagination: MqListPageRequest,
) -> Result<MqListPage<TopicInfo>, String> {
Err("Paged topic listing is not supported by this MQ system".to_string())
}
async fn create_topic(&self, topic: &TopicRef, partitions: Option<u32>) -> Result<(), String>;
async fn delete_topic(&self, topic: &TopicRef, force: bool) -> Result<(), String>;
async fn update_partitions(&self, topic: &TopicRef, partitions: u32) -> Result<(), String>;
@@ -56,6 +69,14 @@ pub trait MessageQueueAdmin: Send + Sync {
Err("Exchanges are not supported by this MQ system".to_string())
}
async fn list_exchanges_page(
&self,
_ns: &NamespaceRef,
_pagination: MqListPageRequest,
) -> Result<MqListPage<MqExchangeInfo>, String> {
Err("Paged exchange listing is not supported by this MQ system".to_string())
}
async fn create_exchange(
&self,
_ns: &NamespaceRef,
+23
View File
@@ -153,6 +153,18 @@ pub async fn mq_list_topics_core(
adapter.list_topics(&ns, opts).await
}
pub async fn mq_list_topics_page_core(
state: &AppState,
conn_id: &str,
ns: NamespaceRef,
opts: ListTopicsOpts,
pagination: MqListPageRequest,
) -> Result<MqListPage<TopicInfo>, String> {
pagination.validate()?;
let adapter = get_adapter(state, conn_id).await?;
adapter.list_topics_page(&ns, opts, pagination).await
}
pub async fn mq_create_topic_core(
state: &AppState,
conn_id: &str,
@@ -206,6 +218,17 @@ pub async fn mq_list_exchanges_core(
adapter.list_exchanges(&ns).await
}
pub async fn mq_list_exchanges_page_core(
state: &AppState,
conn_id: &str,
ns: NamespaceRef,
pagination: MqListPageRequest,
) -> Result<MqListPage<MqExchangeInfo>, String> {
pagination.validate()?;
let adapter = get_adapter(state, conn_id).await?;
adapter.list_exchanges_page(&ns, pagination).await
}
pub async fn mq_create_exchange_core(
state: &AppState,
conn_id: &str,
+73
View File
@@ -380,6 +380,79 @@ pub struct ListTopicsOpts {
pub include_non_persistent: bool,
}
pub const DEFAULT_MQ_LIST_PAGE_SIZE: u32 = 100;
pub const MAX_MQ_LIST_PAGE_SIZE: u32 = 500;
/// Sort keys shared by the bounded MQ list endpoints. `MessagesReady` is used
/// by RabbitMQ queues; exchange listings use the default name ordering.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
#[serde(rename_all = "camelCase")]
pub enum MqListSort {
#[default]
Name,
MessagesReady,
}
/// One-based request for a bounded MQ list page.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MqListPageRequest {
#[serde(default = "default_mq_list_page")]
pub page: u32,
#[serde(default = "default_mq_list_page_size")]
pub page_size: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub search: Option<String>,
#[serde(default)]
pub sort: MqListSort,
#[serde(default)]
pub sort_descending: bool,
}
impl Default for MqListPageRequest {
fn default() -> Self {
Self {
page: default_mq_list_page(),
page_size: default_mq_list_page_size(),
search: None,
sort: MqListSort::default(),
sort_descending: false,
}
}
}
impl MqListPageRequest {
pub fn validate(&self) -> Result<(), String> {
if self.page == 0 {
return Err("page must be at least 1".to_string());
}
if self.page_size == 0 || self.page_size > MAX_MQ_LIST_PAGE_SIZE {
return Err(format!("pageSize must be between 1 and {MAX_MQ_LIST_PAGE_SIZE}"));
}
Ok(())
}
}
const fn default_mq_list_page() -> u32 {
1
}
const fn default_mq_list_page_size() -> u32 {
DEFAULT_MQ_LIST_PAGE_SIZE
}
/// A bounded list response. `total_count` is the number of rows matching the
/// request's search filter, not merely the number loaded in this page.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MqListPage<T> {
pub items: Vec<T>,
pub page: u32,
pub page_size: u32,
pub total_count: u64,
pub has_more: bool,
}
/// Aggregated, UI-friendly topic statistics. Parsed from the version-specific
/// raw stats payload by the adapter's version profile.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]