perf(kafka): batch consumer group subscriptions into one agent round trip

This commit is contained in:
jearhe authored and GitHub committed 2026-08-26 13:51:26 +08:00
1 parent a4bf8442d7
commit ef22b84fbe
3 files changed
+364 -63

No files matched your search

@@ -963,22 +963,184 @@ public final class KafkaAgent {
private static Object listConsumerGroups(JsonObject params) throws Exception {
AdminClient admin = requireAdmin();
int timeout = requestTimeout(params);
String filterTopic = stringOrEmpty(params, "topic");
Collection<ConsumerGroupListing> groups = admin.listConsumerGroups(
new ListConsumerGroupsOptions().timeoutMs(timeout))
.all().get(timeout, TimeUnit.MILLISECONDS);
List<String> groupIds = groups.stream()
.map(ConsumerGroupListing::groupId)
.sorted()
.toList();
if (groupIds.isEmpty()) {
return Collections.singletonMap("groups", Collections.emptyList());
}
// Batch-describe every group in one Kafka Admin request. Per-group
// failures degrade to an empty member list instead of failing the batch.
Map<String, ConsumerGroupDescription> descriptions = new HashMap<>();
DescribeConsumerGroupsResult described = admin.describeConsumerGroups(
groupIds,
new DescribeConsumerGroupsOptions().timeoutMs(timeout)
);
for (String groupId : groupIds) {
try {
descriptions.put(
groupId,
described.describedGroups().get(groupId).get(timeout, TimeUnit.MILLISECONDS)
);
} catch (Exception error) {
logger().debug("Consumer group details unavailable for {}: {}", groupId, normalizeErrorMessage(error));
}
}
// Batch-resolve committed offsets for every group in one Admin request.
Map<String, ListConsumerGroupOffsetsSpec> offsetSpecs = new LinkedHashMap<>();
for (String groupId : groupIds) {
offsetSpecs.put(groupId, new ListConsumerGroupOffsetsSpec());
}
Map<String, Map<TopicPartition, OffsetAndMetadata>> committedOffsets = new HashMap<>();
ListConsumerGroupOffsetsResult offsetsResult = admin.listConsumerGroupOffsets(
offsetSpecs,
new ListConsumerGroupOffsetsOptions().timeoutMs(timeout)
);
for (String groupId : groupIds) {
try {
Map<TopicPartition, OffsetAndMetadata> rows = offsetsResult
.partitionsToOffsetAndMetadata(groupId)
.get(timeout, TimeUnit.MILLISECONDS);
committedOffsets.put(groupId, rows == null ? Collections.emptyMap() : rows);
} catch (Exception error) {
committedOffsets.put(groupId, Collections.emptyMap());
logger().debug("Committed offsets unavailable for {}: {}", groupId, normalizeErrorMessage(error));
}
}
// End offsets: one batched Admin request limited to the requested
// topic's partitions (deduplicated across groups). Without a topic
// filter no end offsets are resolved, keeping the listing cheap.
Map<TopicPartition, Long> endOffsets = new HashMap<>();
if (!filterTopic.isEmpty()) {
LinkedHashSet<TopicPartition> topicPartitions = new LinkedHashSet<>();
for (Map<TopicPartition, OffsetAndMetadata> rows : committedOffsets.values()) {
for (TopicPartition topicPartition : rows.keySet()) {
if (topicPartition.topic().equals(filterTopic)) {
topicPartitions.add(topicPartition);
}
}
}
if (!topicPartitions.isEmpty()) {
Map<TopicPartition, OffsetSpec> endOffsetSpecs = new LinkedHashMap<>();
for (TopicPartition topicPartition : topicPartitions) {
endOffsetSpecs.put(topicPartition, OffsetSpec.latest());
}
try {
ListOffsetsResult latestOffsets = admin.listOffsets(
endOffsetSpecs,
new ListOffsetsOptions().timeoutMs(timeout)
);
for (TopicPartition topicPartition : topicPartitions) {
try {
ListOffsetsResult.ListOffsetsResultInfo info = latestOffsets
.partitionResult(topicPartition)
.get(timeout, TimeUnit.MILLISECONDS);
endOffsets.put(topicPartition, info.offset());
} catch (Exception error) {
// Leave the partition absent so callers can tell an
// unknown end offset (no lag contribution) apart from zero lag.
logger().debug("End offset unavailable for {}: {}", topicPartition, normalizeErrorMessage(error));
}
}
} catch (Exception error) {
logger().debug("End offset batch unavailable: {}", normalizeErrorMessage(error));
}
}
}
Map<String, ConsumerGroupListing> listingByGroup = groups.stream()
.collect(Collectors.toMap(ConsumerGroupListing::groupId, listing -> listing));
List<Map<String, Object>> result = new ArrayList<>();
for (ConsumerGroupListing group : groups) {
for (String groupId : groupIds) {
ConsumerGroupListing listing = listingByGroup.get(groupId);
ConsumerGroupDescription description = descriptions.get(groupId);
Map<TopicPartition, OffsetAndMetadata> committed = committedOffsets.getOrDefault(groupId, Collections.emptyMap());
Map<String, Object> g = new LinkedHashMap<>();
g.put("groupId", group.groupId());
g.put("state", group.state().map(Enum::name).orElse("UNKNOWN"));
g.put("simpleGroup", group.isSimpleConsumerGroup());
g.put("groupId", groupId);
g.put("state", listing == null ? "UNKNOWN" : listing.state().map(Enum::name).orElse("UNKNOWN"));
g.put("simpleGroup", listing != null && listing.isSimpleConsumerGroup());
g.put("members", memberMaps(description == null ? Collections.emptyList() : description.members()));
g.put("committedOffsets", offsetRows(committed, filterTopic));
g.put("endOffsets", endOffsetRows(committed, endOffsets, filterTopic));
result.add(g);
}
result.sort(Comparator.comparing(m -> (String) m.get("groupId")));
return Collections.singletonMap("groups", result);
}
static List<Map<String, Object>> memberMaps(Collection<MemberDescription> members) {
List<Map<String, Object>> result = new ArrayList<>();
for (MemberDescription member : members) {
Map<String, Object> m = new LinkedHashMap<>();
m.put("memberId", member.consumerId());
m.put("clientId", member.clientId());
m.put("host", member.host());
List<Map<String, Object>> assignments = new ArrayList<>();
for (TopicPartition topicPartition : member.assignment().topicPartitions()) {
Map<String, Object> a = new LinkedHashMap<>();
a.put("topic", topicPartition.topic());
a.put("partition", topicPartition.partition());
assignments.add(a);
}
m.put("assignments", assignments);
result.add(m);
}
return result;
}
/** Committed offsets as rows, optionally restricted to one topic. */
static List<Map<String, Object>> offsetRows(Map<TopicPartition, OffsetAndMetadata> offsets, String topic) {
List<Map<String, Object>> rows = new ArrayList<>();
for (Map.Entry<TopicPartition, OffsetAndMetadata> entry : offsets.entrySet()) {
if (!topic.isEmpty() && !entry.getKey().topic().equals(topic)) {
continue;
}
Map<String, Object> row = new LinkedHashMap<>();
row.put("topic", entry.getKey().topic());
row.put("partition", entry.getKey().partition());
row.put("offset", entry.getValue().offset());
rows.add(row);
}
rows.sort(Comparator.comparingInt(row -> (int) row.get("partition")));
return rows;
}
/** End offsets for a group's committed partitions, restricted to one topic. */
static List<Map<String, Object>> endOffsetRows(
Map<TopicPartition, OffsetAndMetadata> committed,
Map<TopicPartition, Long> endOffsets,
String topic
) {
List<Map<String, Object>> rows = new ArrayList<>();
for (TopicPartition topicPartition : committed.keySet()) {
if (!topic.isEmpty() && !topicPartition.topic().equals(topic)) {
continue;
}
Long endOffset = endOffsets.get(topicPartition);
if (endOffset == null) {
continue;
}
Map<String, Object> row = new LinkedHashMap<>();
row.put("topic", topicPartition.topic());
row.put("partition", topicPartition.partition());
row.put("offset", endOffset);
rows.add(row);
}
rows.sort(Comparator.comparingInt(row -> (int) row.get("partition")));
return rows;
}
private static Object getConsumerGroupSnapshot(JsonObject params) throws Exception {
AdminClient admin = requireAdmin();
int timeout = requestTimeout(params);
@@ -142,6 +142,65 @@ class KafkaAgentTest {
assertEquals("End offsets unavailable for 1 partition(s)", group.get("error"));
}
@Test
void consumerGroupListRowsFilterToTheRequestedTopicAndSortByPartition() {
Map<TopicPartition, OffsetAndMetadata> offsets = new HashMap<>();
offsets.put(new TopicPartition("orders", 2), new OffsetAndMetadata(20));
offsets.put(new TopicPartition("payments", 0), new OffsetAndMetadata(5));
offsets.put(new TopicPartition("orders", 0), new OffsetAndMetadata(8));
List<Map<String, Object>> rows = KafkaAgent.offsetRows(offsets, "orders");
assertEquals(List.of("orders:0", "orders:2"), rows.stream()
.map(row -> row.get("topic") + ":" + row.get("partition"))
.toList());
assertEquals(8L, rows.get(0).get("offset"));
assertEquals(20L, rows.get(1).get("offset"));
}
@Test
void consumerGroupListRowsKeepEveryPartitionWithoutATopicFilter() {
Map<TopicPartition, OffsetAndMetadata> offsets = new HashMap<>();
offsets.put(new TopicPartition("orders", 0), new OffsetAndMetadata(8));
offsets.put(new TopicPartition("payments", 1), new OffsetAndMetadata(3));
List<Map<String, Object>> rows = KafkaAgent.offsetRows(offsets, "");
assertEquals(2, rows.size());
}
@Test
void consumerGroupListEndOffsetRowsSkipPartitionsWithoutResolvedEndOffsets() {
Map<TopicPartition, OffsetAndMetadata> committed = new HashMap<>();
committed.put(new TopicPartition("orders", 0), new OffsetAndMetadata(8));
committed.put(new TopicPartition("orders", 1), new OffsetAndMetadata(20));
Map<TopicPartition, Long> endOffsets = new HashMap<>();
endOffsets.put(new TopicPartition("orders", 1), 24L);
List<Map<String, Object>> rows = KafkaAgent.endOffsetRows(committed, endOffsets, "orders");
assertEquals(List.of("orders:1"), rows.stream()
.map(row -> row.get("topic") + ":" + row.get("partition"))
.toList());
assertEquals(24L, rows.get(0).get("offset"));
}
@Test
void consumerGroupListEndOffsetRowsFilterToTheRequestedTopic() {
Map<TopicPartition, OffsetAndMetadata> committed = new HashMap<>();
committed.put(new TopicPartition("orders", 0), new OffsetAndMetadata(8));
committed.put(new TopicPartition("payments", 0), new OffsetAndMetadata(4));
Map<TopicPartition, Long> endOffsets = new HashMap<>();
endOffsets.put(new TopicPartition("orders", 0), 10L);
endOffsets.put(new TopicPartition("payments", 0), 9L);
List<Map<String, Object>> rows = KafkaAgent.endOffsetRows(committed, endOffsets, "orders");
assertEquals(List.of("orders:0"), rows.stream()
.map(row -> row.get("topic") + ":" + row.get("partition"))
.toList());
}
@Test
void explicitConsumerGroupOffsetsAcceptOneBoundedPartitionOffset() {
JsonObject params = JsonParser.parseString("""