fix(rocketmq): paginate topic consumer groups past 200

This commit is contained in:
jearhe authored and GitHub committed 2026-09-03 10:27:52 +08:00
1 parent e2eb481b8a
commit 35946427a6
1 file changed
+50 -27
+50 -27
View File
@@ -144,6 +144,13 @@ impl RocketMqAdmin {
})
.await
}
async fn list_all_topic_consumer_groups(&self, topic: &str, enrich: bool) -> Result<Vec<SubscriptionInfo>, String> {
collect_paginated_consumer_group_pages(|offset| {
self.call("mq_list_consumer_groups", rocketmq_topic_consumer_group_page_params(topic, offset, enrich))
})
.await
}
}
#[async_trait]
@@ -380,20 +387,7 @@ impl MessageQueueAdmin for RocketMqAdmin {
}
// Topic path: skip list enrich; batch lag in one agent RPC (no N+1).
let result: serde_json::Value = self
.call(
"mq_list_consumer_groups",
serde_json::json!({
"topic": topic.topic,
"limit": 200,
"offset": 0,
"enrich": false,
"includeLag": true,
}),
)
.await?;
let groups = result.get("groups").and_then(|v| v.as_array()).cloned().unwrap_or_default();
Ok(groups.iter().map(rocketmq_subscription_from_group).collect())
self.list_all_topic_consumer_groups(&topic.topic, false).await
}
async fn enrich_subscriptions(&self, topic: &TopicRef) -> Result<Vec<SubscriptionInfo>, String> {
@@ -403,19 +397,7 @@ impl MessageQueueAdmin for RocketMqAdmin {
}
// Topic path: skip list enrich; batch lag in one agent RPC (no N+1).
let mut params = serde_json::json!({
"limit": 500,
"offset": 0,
"enrich": true,
});
if !topic.topic.is_empty() {
params["topic"] = serde_json::json!(topic.topic);
params["limit"] = serde_json::json!(200);
params["includeLag"] = serde_json::json!(true);
}
let result: serde_json::Value = self.call("mq_list_consumer_groups", params).await?;
let groups = result.get("groups").and_then(|v| v.as_array()).cloned().unwrap_or_default();
Ok(groups.iter().map(rocketmq_subscription_from_group).collect())
self.list_all_topic_consumer_groups(&topic.topic, true).await
}
async fn create_subscription(&self, _topic: &TopicRef, _sub: &str, _pos: ResetPosition) -> Result<(), String> {
@@ -793,6 +775,16 @@ fn rocketmq_cluster_consumer_group_page_params(offset: usize, enrich: bool) -> s
})
}
fn rocketmq_topic_consumer_group_page_params(topic: &str, offset: usize, enrich: bool) -> serde_json::Value {
serde_json::json!({
"topic": topic,
"limit": CONSUMER_GROUP_LIST_PAGE_SIZE,
"offset": offset,
"enrich": enrich,
"includeLag": true,
})
}
/// Collect all pages from the agent while preserving its sorted order.
///
/// The offset advances by the number of rows actually returned, rather than the
@@ -1465,6 +1457,37 @@ mod tests {
assert_eq!(enriched.get("enrich").and_then(|value| value.as_bool()), Some(true));
}
#[tokio::test]
async fn topic_consumer_group_pagination_loads_list_and_enrich_pages() {
let groups: Vec<serde_json::Value> = (1..=201)
.map(|index| serde_json::json!({ "groupId": format!("group-{index:04}"), "groupType": "NORMAL" }))
.collect();
let total = groups.len();
for enrich in [false, true] {
let mut offsets = Vec::new();
let subscriptions = collect_paginated_consumer_group_pages(|offset| {
offsets.push(offset);
let params = rocketmq_topic_consumer_group_page_params("orders", offset, enrich);
assert_eq!(params.get("topic").and_then(|value| value.as_str()), Some("orders"));
assert_eq!(params.get("limit").and_then(|value| value.as_i64()), Some(500));
assert_eq!(params.get("offset").and_then(|value| value.as_u64()), Some(offset as u64));
assert_eq!(params.get("enrich").and_then(|value| value.as_bool()), Some(enrich));
assert_eq!(params.get("includeLag").and_then(|value| value.as_bool()), Some(true));
let end = offset.saturating_add(200).min(total);
let page = groups[offset..end].to_vec();
async move { Ok(serde_json::json!({ "groups": page, "total": total })) }
})
.await
.expect("topic consumer group pages should be collected");
assert_eq!(offsets, vec![0, 200]);
assert_eq!(subscriptions.len(), 201);
assert_eq!(subscriptions[200].name, "group-0201");
}
}
#[tokio::test]
async fn consumer_group_pagination_loads_groups_after_first_page() {
let groups: Vec<serde_json::Value> = (1..=1000)