perf(kafka): batch consumer group subscriptions into one agent round trip

This commit is contained in:
jearhe authored and GitHub committed 2026-08-26 13:51:26 +08:00
1 parent a4bf8442d7
commit ef22b84fbe
3 files changed
+364 -63

No files matched your search

@@ -963,22 +963,184 @@ public final class KafkaAgent {
private static Object listConsumerGroups(JsonObject params) throws Exception { private static Object listConsumerGroups(JsonObject params) throws Exception {
AdminClient admin = requireAdmin(); AdminClient admin = requireAdmin();
int timeout = requestTimeout(params); int timeout = requestTimeout(params);
String filterTopic = stringOrEmpty(params, "topic");
Collection<ConsumerGroupListing> groups = admin.listConsumerGroups( Collection<ConsumerGroupListing> groups = admin.listConsumerGroups(
new ListConsumerGroupsOptions().timeoutMs(timeout)) new ListConsumerGroupsOptions().timeoutMs(timeout))
.all().get(timeout, TimeUnit.MILLISECONDS); .all().get(timeout, TimeUnit.MILLISECONDS);
List<String> groupIds = groups.stream()
.map(ConsumerGroupListing::groupId)
.sorted()
.toList();
if (groupIds.isEmpty()) {
return Collections.singletonMap("groups", Collections.emptyList());
}
// Batch-describe every group in one Kafka Admin request. Per-group
// failures degrade to an empty member list instead of failing the batch.
Map<String, ConsumerGroupDescription> descriptions = new HashMap<>();
DescribeConsumerGroupsResult described = admin.describeConsumerGroups(
groupIds,
new DescribeConsumerGroupsOptions().timeoutMs(timeout)
);
for (String groupId : groupIds) {
try {
descriptions.put(
groupId,
described.describedGroups().get(groupId).get(timeout, TimeUnit.MILLISECONDS)
);
} catch (Exception error) {
logger().debug("Consumer group details unavailable for {}: {}", groupId, normalizeErrorMessage(error));
}
}
// Batch-resolve committed offsets for every group in one Admin request.
Map<String, ListConsumerGroupOffsetsSpec> offsetSpecs = new LinkedHashMap<>();
for (String groupId : groupIds) {
offsetSpecs.put(groupId, new ListConsumerGroupOffsetsSpec());
}
Map<String, Map<TopicPartition, OffsetAndMetadata>> committedOffsets = new HashMap<>();
ListConsumerGroupOffsetsResult offsetsResult = admin.listConsumerGroupOffsets(
offsetSpecs,
new ListConsumerGroupOffsetsOptions().timeoutMs(timeout)
);
for (String groupId : groupIds) {
try {
Map<TopicPartition, OffsetAndMetadata> rows = offsetsResult
.partitionsToOffsetAndMetadata(groupId)
.get(timeout, TimeUnit.MILLISECONDS);
committedOffsets.put(groupId, rows == null ? Collections.emptyMap() : rows);
} catch (Exception error) {
committedOffsets.put(groupId, Collections.emptyMap());
logger().debug("Committed offsets unavailable for {}: {}", groupId, normalizeErrorMessage(error));
}
}
// End offsets: one batched Admin request limited to the requested
// topic's partitions (deduplicated across groups). Without a topic
// filter no end offsets are resolved, keeping the listing cheap.
Map<TopicPartition, Long> endOffsets = new HashMap<>();
if (!filterTopic.isEmpty()) {
LinkedHashSet<TopicPartition> topicPartitions = new LinkedHashSet<>();
for (Map<TopicPartition, OffsetAndMetadata> rows : committedOffsets.values()) {
for (TopicPartition topicPartition : rows.keySet()) {
if (topicPartition.topic().equals(filterTopic)) {
topicPartitions.add(topicPartition);
}
}
}
if (!topicPartitions.isEmpty()) {
Map<TopicPartition, OffsetSpec> endOffsetSpecs = new LinkedHashMap<>();
for (TopicPartition topicPartition : topicPartitions) {
endOffsetSpecs.put(topicPartition, OffsetSpec.latest());
}
try {
ListOffsetsResult latestOffsets = admin.listOffsets(
endOffsetSpecs,
new ListOffsetsOptions().timeoutMs(timeout)
);
for (TopicPartition topicPartition : topicPartitions) {
try {
ListOffsetsResult.ListOffsetsResultInfo info = latestOffsets
.partitionResult(topicPartition)
.get(timeout, TimeUnit.MILLISECONDS);
endOffsets.put(topicPartition, info.offset());
} catch (Exception error) {
// Leave the partition absent so callers can tell an
// unknown end offset (no lag contribution) apart from zero lag.
logger().debug("End offset unavailable for {}: {}", topicPartition, normalizeErrorMessage(error));
}
}
} catch (Exception error) {
logger().debug("End offset batch unavailable: {}", normalizeErrorMessage(error));
}
}
}
Map<String, ConsumerGroupListing> listingByGroup = groups.stream()
.collect(Collectors.toMap(ConsumerGroupListing::groupId, listing -> listing));
List<Map<String, Object>> result = new ArrayList<>(); List<Map<String, Object>> result = new ArrayList<>();
for (ConsumerGroupListing group : groups) { for (String groupId : groupIds) {
ConsumerGroupListing listing = listingByGroup.get(groupId);
ConsumerGroupDescription description = descriptions.get(groupId);
Map<TopicPartition, OffsetAndMetadata> committed = committedOffsets.getOrDefault(groupId, Collections.emptyMap());
Map<String, Object> g = new LinkedHashMap<>(); Map<String, Object> g = new LinkedHashMap<>();
g.put("groupId", group.groupId()); g.put("groupId", groupId);
g.put("state", group.state().map(Enum::name).orElse("UNKNOWN")); g.put("state", listing == null ? "UNKNOWN" : listing.state().map(Enum::name).orElse("UNKNOWN"));
g.put("simpleGroup", group.isSimpleConsumerGroup()); g.put("simpleGroup", listing != null && listing.isSimpleConsumerGroup());
g.put("members", memberMaps(description == null ? Collections.emptyList() : description.members()));
g.put("committedOffsets", offsetRows(committed, filterTopic));
g.put("endOffsets", endOffsetRows(committed, endOffsets, filterTopic));
result.add(g); result.add(g);
} }
result.sort(Comparator.comparing(m -> (String) m.get("groupId")));
return Collections.singletonMap("groups", result); return Collections.singletonMap("groups", result);
} }
static List<Map<String, Object>> memberMaps(Collection<MemberDescription> members) {
List<Map<String, Object>> result = new ArrayList<>();
for (MemberDescription member : members) {
Map<String, Object> m = new LinkedHashMap<>();
m.put("memberId", member.consumerId());
m.put("clientId", member.clientId());
m.put("host", member.host());
List<Map<String, Object>> assignments = new ArrayList<>();
for (TopicPartition topicPartition : member.assignment().topicPartitions()) {
Map<String, Object> a = new LinkedHashMap<>();
a.put("topic", topicPartition.topic());
a.put("partition", topicPartition.partition());
assignments.add(a);
}
m.put("assignments", assignments);
result.add(m);
}
return result;
}
/** Committed offsets as rows, optionally restricted to one topic. */
static List<Map<String, Object>> offsetRows(Map<TopicPartition, OffsetAndMetadata> offsets, String topic) {
List<Map<String, Object>> rows = new ArrayList<>();
for (Map.Entry<TopicPartition, OffsetAndMetadata> entry : offsets.entrySet()) {
if (!topic.isEmpty() && !entry.getKey().topic().equals(topic)) {
continue;
}
Map<String, Object> row = new LinkedHashMap<>();
row.put("topic", entry.getKey().topic());
row.put("partition", entry.getKey().partition());
row.put("offset", entry.getValue().offset());
rows.add(row);
}
rows.sort(Comparator.comparingInt(row -> (int) row.get("partition")));
return rows;
}
/** End offsets for a group's committed partitions, restricted to one topic. */
static List<Map<String, Object>> endOffsetRows(
Map<TopicPartition, OffsetAndMetadata> committed,
Map<TopicPartition, Long> endOffsets,
String topic
) {
List<Map<String, Object>> rows = new ArrayList<>();
for (TopicPartition topicPartition : committed.keySet()) {
if (!topic.isEmpty() && !topicPartition.topic().equals(topic)) {
continue;
}
Long endOffset = endOffsets.get(topicPartition);
if (endOffset == null) {
continue;
}
Map<String, Object> row = new LinkedHashMap<>();
row.put("topic", topicPartition.topic());
row.put("partition", topicPartition.partition());
row.put("offset", endOffset);
rows.add(row);
}
rows.sort(Comparator.comparingInt(row -> (int) row.get("partition")));
return rows;
}
private static Object getConsumerGroupSnapshot(JsonObject params) throws Exception { private static Object getConsumerGroupSnapshot(JsonObject params) throws Exception {
AdminClient admin = requireAdmin(); AdminClient admin = requireAdmin();
int timeout = requestTimeout(params); int timeout = requestTimeout(params);
@@ -142,6 +142,65 @@ class KafkaAgentTest {
assertEquals("End offsets unavailable for 1 partition(s)", group.get("error")); assertEquals("End offsets unavailable for 1 partition(s)", group.get("error"));
} }
@Test
void consumerGroupListRowsFilterToTheRequestedTopicAndSortByPartition() {
Map<TopicPartition, OffsetAndMetadata> offsets = new HashMap<>();
offsets.put(new TopicPartition("orders", 2), new OffsetAndMetadata(20));
offsets.put(new TopicPartition("payments", 0), new OffsetAndMetadata(5));
offsets.put(new TopicPartition("orders", 0), new OffsetAndMetadata(8));
List<Map<String, Object>> rows = KafkaAgent.offsetRows(offsets, "orders");
assertEquals(List.of("orders:0", "orders:2"), rows.stream()
.map(row -> row.get("topic") + ":" + row.get("partition"))
.toList());
assertEquals(8L, rows.get(0).get("offset"));
assertEquals(20L, rows.get(1).get("offset"));
}
@Test
void consumerGroupListRowsKeepEveryPartitionWithoutATopicFilter() {
Map<TopicPartition, OffsetAndMetadata> offsets = new HashMap<>();
offsets.put(new TopicPartition("orders", 0), new OffsetAndMetadata(8));
offsets.put(new TopicPartition("payments", 1), new OffsetAndMetadata(3));
List<Map<String, Object>> rows = KafkaAgent.offsetRows(offsets, "");
assertEquals(2, rows.size());
}
@Test
void consumerGroupListEndOffsetRowsSkipPartitionsWithoutResolvedEndOffsets() {
Map<TopicPartition, OffsetAndMetadata> committed = new HashMap<>();
committed.put(new TopicPartition("orders", 0), new OffsetAndMetadata(8));
committed.put(new TopicPartition("orders", 1), new OffsetAndMetadata(20));
Map<TopicPartition, Long> endOffsets = new HashMap<>();
endOffsets.put(new TopicPartition("orders", 1), 24L);
List<Map<String, Object>> rows = KafkaAgent.endOffsetRows(committed, endOffsets, "orders");
assertEquals(List.of("orders:1"), rows.stream()
.map(row -> row.get("topic") + ":" + row.get("partition"))
.toList());
assertEquals(24L, rows.get(0).get("offset"));
}
@Test
void consumerGroupListEndOffsetRowsFilterToTheRequestedTopic() {
Map<TopicPartition, OffsetAndMetadata> committed = new HashMap<>();
committed.put(new TopicPartition("orders", 0), new OffsetAndMetadata(8));
committed.put(new TopicPartition("payments", 0), new OffsetAndMetadata(4));
Map<TopicPartition, Long> endOffsets = new HashMap<>();
endOffsets.put(new TopicPartition("orders", 0), 10L);
endOffsets.put(new TopicPartition("payments", 0), 9L);
List<Map<String, Object>> rows = KafkaAgent.endOffsetRows(committed, endOffsets, "orders");
assertEquals(List.of("orders:0"), rows.stream()
.map(row -> row.get("topic") + ":" + row.get("partition"))
.toList());
}
@Test @Test
void explicitConsumerGroupOffsetsAcceptOneBoundedPartitionOffset() { void explicitConsumerGroupOffsetsAcceptOneBoundedPartitionOffset() {
JsonObject params = JsonParser.parseString(""" JsonObject params = JsonParser.parseString("""
+138 -58
View File
@@ -269,32 +269,20 @@ impl MessageQueueAdmin for KafkaAdmin {
// ---- Subscriptions (mapped to consumer groups) ---- // ---- Subscriptions (mapped to consumer groups) ----
async fn list_subscriptions(&self, topic: &TopicRef) -> Result<Vec<SubscriptionInfo>, String> { async fn list_subscriptions(&self, topic: &TopicRef) -> Result<Vec<SubscriptionInfo>, String> {
// List consumer groups and filter those subscribed to this topic // One batched agent RPC: the agent lists every consumer group, batch-describes
let result: serde_json::Value = self.call("mq_list_consumer_groups", serde_json::json!({})).await?; // them, and resolves committed + end offsets for the requested topic with a
// handful of Kafka Admin calls. Subscriptions are filtered in memory below.
// (The previous 1 + 2N serial RPC pattern — describe + lag per group — took
// tens of seconds on remote clusters with many groups, see #7163.)
let result: serde_json::Value =
self.call("mq_list_consumer_groups", serde_json::json!({ "topic": topic.topic })).await?;
let groups = result.get("groups").and_then(|v| v.as_array()).cloned().unwrap_or_default(); let groups = result.get("groups").and_then(|v| v.as_array()).cloned().unwrap_or_default();
// For each group, check both active assignments and committed offsets. // For each group, check both active assignments and committed offsets.
let mut subs = Vec::new(); let mut subs = Vec::new();
for group in groups { for group in groups {
let group_id = group.get("groupId").and_then(|v| v.as_str()).unwrap_or_default(); let group_id = group.get("groupId").and_then(|v| v.as_str()).unwrap_or_default();
let desc = match self if let Some(sub) = kafka_subscription_from_group_row(group_id, &topic.topic, &group) {
.call::<serde_json::Value>("mq_describe_consumer_group", serde_json::json!({ "groupId": group_id }))
.await
{
Ok(desc) => desc,
Err(_) => continue, // Skip groups we can't describe
};
let lag = self
.call::<serde_json::Value>(
"mq_get_consumer_lag",
serde_json::json!({
"groupId": group_id,
"topic": topic.topic,
}),
)
.await
.ok();
if let Some(sub) = kafka_subscription_for_topic(group_id, &topic.topic, &desc, lag.as_ref()) {
subs.push(sub); subs.push(sub);
} }
} }
@@ -763,13 +751,12 @@ fn reset_cursor_params(topic: &TopicRef, sub: &str, pos: ResetPosition) -> Resul
} }
} }
fn kafka_subscription_for_topic( /// Build a `SubscriptionInfo` for `topic` from one row of the batched
group_id: &str, /// `mq_list_consumer_groups` response. The agent already resolved committed
topic: &str, /// offsets (and end offsets for the requested topic) for every group, so no
desc: &serde_json::Value, /// per-group RPC happens here.
lag: Option<&serde_json::Value>, fn kafka_subscription_from_group_row(group_id: &str, topic: &str, row: &serde_json::Value) -> Option<SubscriptionInfo> {
) -> Option<SubscriptionInfo> { let consumers = row
let consumers = desc
.get("members") .get("members")
.and_then(|v| v.as_array()) .and_then(|v| v.as_array())
.into_iter() .into_iter()
@@ -778,20 +765,58 @@ fn kafka_subscription_for_topic(
.map(kafka_consumer_from_member) .map(kafka_consumer_from_member)
.collect::<Vec<_>>(); .collect::<Vec<_>>();
let has_active_assignment = !consumers.is_empty(); let has_active_assignment = !consumers.is_empty();
let has_committed_offsets = lag
.and_then(|v| v.get("partitions")) let committed = row
.get("committedOffsets")
.and_then(|v| v.as_array()) .and_then(|v| v.as_array())
.map(|partitions| !partitions.is_empty()) .into_iter()
.unwrap_or(false); .flatten()
.filter(|partition| partition.get("topic").and_then(|v| v.as_str()) == Some(topic))
.collect::<Vec<_>>();
let has_committed_offsets = !committed.is_empty();
if !has_active_assignment && !has_committed_offsets { if !has_active_assignment && !has_committed_offsets {
return None; return None;
} }
let end_offsets = row
.get("endOffsets")
.and_then(|v| v.as_array())
.map(|partitions| {
partitions
.iter()
.filter_map(|partition| {
// Defensive: the agent already filters end offsets to the
// requested topic, but keying by partition id alone would
// let a same-numbered partition of another topic collide.
if partition.get("topic").and_then(|v| v.as_str()) != Some(topic) {
return None;
}
let partition_id = partition.get("partition").and_then(|v| v.as_i64())?;
let offset = partition.get("offset").and_then(|v| v.as_i64())?;
Some((partition_id, offset))
})
.collect::<HashMap<_, _>>()
})
.unwrap_or_default();
// Sum lag over the topic's committed partitions. A partition without a
// resolved end offset contributes nothing (matches the agent's lag probe:
// unknown end offsets are not reported as zero lag).
let msg_backlog = committed
.iter()
.filter_map(|partition| {
let partition_id = partition.get("partition").and_then(|v| v.as_i64())?;
let committed_offset = partition.get("offset").and_then(|v| v.as_i64())?;
let end_offset = end_offsets.get(&partition_id).copied()?;
Some((end_offset - committed_offset).max(0))
})
.sum();
Some(SubscriptionInfo { Some(SubscriptionInfo {
name: group_id.to_string(), name: group_id.to_string(),
sub_type: "consumer-group".to_string(), sub_type: "consumer-group".to_string(),
msg_backlog: lag.and_then(|v| v.get("totalLag")).and_then(|v| v.as_i64()).unwrap_or(0), msg_backlog,
msg_rate_out: 0.0, msg_rate_out: 0.0,
msg_throughput_out: 0.0, msg_throughput_out: 0.0,
consumers, consumers,
@@ -1163,31 +1188,37 @@ mod tests {
} }
#[test] #[test]
fn kafka_subscription_for_topic_includes_offline_group_with_committed_offsets() { fn kafka_subscription_from_group_row_includes_offline_group_with_committed_offsets() {
let desc = serde_json::json!({ let row = serde_json::json!({
"groupId": "orders-service", "groupId": "orders-service",
"members": [] "state": "EMPTY",
}); "simpleGroup": false,
let lag = serde_json::json!({ "members": [],
"totalLag": 7, "committedOffsets": [
"partitions": [ { "topic": "orders", "partition": 0, "offset": 3 },
{ "partition": 0, "currentOffset": 3, "endOffset": 10, "lag": 7 } { "topic": "orders", "partition": 1, "offset": 8 }
],
"endOffsets": [
{ "topic": "orders", "partition": 0, "offset": 10 },
{ "topic": "orders", "partition": 1, "offset": 12 }
] ]
}); });
let sub = kafka_subscription_for_topic("orders-service", "orders", &desc, Some(&lag)) let sub = kafka_subscription_from_group_row("orders-service", "orders", &row)
.expect("committed offsets should make an inactive group visible"); .expect("committed offsets should make an inactive group visible");
assert_eq!(sub.name, "orders-service"); assert_eq!(sub.name, "orders-service");
assert_eq!(sub.sub_type, "consumer-group"); assert_eq!(sub.sub_type, "consumer-group");
assert_eq!(sub.msg_backlog, 7); assert_eq!(sub.msg_backlog, 11); // (10-3) + (12-8)
assert!(sub.consumers.is_empty()); assert!(sub.consumers.is_empty());
} }
#[test] #[test]
fn kafka_subscription_for_topic_includes_active_assignment_without_committed_offsets() { fn kafka_subscription_from_group_row_includes_active_assignment_without_committed_offsets() {
let desc = serde_json::json!({ let row = serde_json::json!({
"groupId": "live-service", "groupId": "live-service",
"state": "STABLE",
"simpleGroup": false,
"members": [ "members": [
{ {
"memberId": "consumer-events", "memberId": "consumer-events",
@@ -1203,14 +1234,12 @@ mod tests {
{ "topic": "audit", "partition": 0 } { "topic": "audit", "partition": 0 }
] ]
} }
] ],
}); "committedOffsets": [],
let lag = serde_json::json!({ "endOffsets": []
"totalLag": 0,
"partitions": []
}); });
let sub = kafka_subscription_for_topic("live-service", "events", &desc, Some(&lag)) let sub = kafka_subscription_from_group_row("live-service", "events", &row)
.expect("active assignments should make the group visible"); .expect("active assignments should make the group visible");
assert_eq!(sub.name, "live-service"); assert_eq!(sub.name, "live-service");
@@ -1221,21 +1250,72 @@ mod tests {
} }
#[test] #[test]
fn kafka_subscription_for_topic_ignores_unrelated_group() { fn kafka_subscription_from_group_row_ignores_unrelated_group() {
let desc = serde_json::json!({ let row = serde_json::json!({
"groupId": "billing-service", "groupId": "billing-service",
"state": "EMPTY",
"simpleGroup": false,
"members": [{ "members": [{
"assignments": [ "assignments": [
{ "topic": "billing", "partition": 0 } { "topic": "billing", "partition": 0 }
] ]
}] }],
}); "committedOffsets": [
let lag = serde_json::json!({ { "topic": "billing", "partition": 0, "offset": 2 }
"totalLag": 0, ],
"partitions": [] "endOffsets": [
{ "topic": "billing", "partition": 0, "offset": 5 }
]
}); });
assert!(kafka_subscription_for_topic("billing-service", "orders", &desc, Some(&lag)).is_none()); assert!(kafka_subscription_from_group_row("billing-service", "orders", &row).is_none());
}
#[test]
fn kafka_subscription_from_group_row_skips_partitions_without_end_offsets() {
let row = serde_json::json!({
"groupId": "orders-service",
"state": "EMPTY",
"simpleGroup": false,
"members": [],
"committedOffsets": [
{ "topic": "orders", "partition": 0, "offset": 3 },
{ "topic": "orders", "partition": 1, "offset": 8 }
],
"endOffsets": [
// partition 1's end offset is unavailable (agent-side failure)
{ "topic": "orders", "partition": 0, "offset": 10 }
]
});
let sub = kafka_subscription_from_group_row("orders-service", "orders", &row)
.expect("the group still has visible committed offsets");
assert_eq!(sub.msg_backlog, 7); // only partition 0 contributes
}
#[test]
fn kafka_subscription_from_group_row_filters_committed_offsets_to_the_requested_topic() {
let row = serde_json::json!({
"groupId": "multi-topic-service",
"state": "EMPTY",
"simpleGroup": false,
"members": [],
"committedOffsets": [
{ "topic": "orders", "partition": 0, "offset": 3 },
{ "topic": "payments", "partition": 0, "offset": 7 }
],
"endOffsets": [
{ "topic": "orders", "partition": 0, "offset": 8 },
{ "topic": "payments", "partition": 0, "offset": 9 }
]
});
let sub = kafka_subscription_from_group_row("multi-topic-service", "orders", &row)
.expect("committed offsets on the requested topic make the group visible");
// Lag totals only the requested topic's partitions.
assert_eq!(sub.msg_backlog, 5);
} }
#[test] #[test]