feat(kafka): add consumer group backlog view

This commit is contained in:
Guoyu Su authored and GitHub committed 2026-08-20 08:47:05 +08:00
1 parent dc8b083c63
commit f88167c3d0
28 files changed
+1274 -2

No files matched your search

@@ -146,6 +146,7 @@ public final class KafkaAgent {
case "mq_alter_topic_config" -> alterTopicConfig(params);
// Consumer groups
case "mq_list_consumer_groups" -> listConsumerGroups(params);
case "mq_get_consumer_group_snapshot" -> getConsumerGroupSnapshot(params);
case "mq_describe_consumer_group" -> describeConsumerGroup(params);
case "mq_delete_consumer_group" -> deleteConsumerGroup(params);
case "mq_reset_consumer_group_offsets" -> resetConsumerGroupOffsets(params);
@@ -971,6 +972,208 @@ public final class KafkaAgent {
return Collections.singletonMap("groups", result);
}
private static Object getConsumerGroupSnapshot(JsonObject params) throws Exception {
AdminClient admin = requireAdmin();
int timeout = requestTimeout(params);
Collection<ConsumerGroupListing> listings = admin.listConsumerGroups(
new ListConsumerGroupsOptions().timeoutMs(timeout))
.all().get(timeout, TimeUnit.MILLISECONDS);
List<ConsumerGroupListing> sortedListings = new ArrayList<>(listings);
sortedListings.sort(Comparator.comparing(ConsumerGroupListing::groupId));
if (sortedListings.isEmpty()) {
return Collections.singletonMap("groups", Collections.emptyList());
}
List<String> groupIds = sortedListings.stream()
.map(ConsumerGroupListing::groupId)
.toList();
Map<String, ConsumerGroupDescription> descriptions = new HashMap<>();
Map<String, Map<TopicPartition, OffsetAndMetadata>> committedOffsets = new HashMap<>();
Map<String, Boolean> committedOffsetsAvailable = new HashMap<>();
Map<String, LinkedHashSet<String>> errors = new HashMap<>();
DescribeConsumerGroupsResult described = admin.describeConsumerGroups(
groupIds,
new DescribeConsumerGroupsOptions().timeoutMs(timeout)
);
for (String groupId : groupIds) {
try {
descriptions.put(
groupId,
described.describedGroups().get(groupId).get(timeout, TimeUnit.MILLISECONDS)
);
} catch (Exception error) {
snapshotErrors(errors, groupId).add("Consumer group details unavailable: " + normalizeErrorMessage(error));
}
}
Map<String, ListConsumerGroupOffsetsSpec> offsetSpecs = new LinkedHashMap<>();
for (String groupId : groupIds) {
offsetSpecs.put(groupId, new ListConsumerGroupOffsetsSpec());
}
ListConsumerGroupOffsetsResult offsetsResult = admin.listConsumerGroupOffsets(
offsetSpecs,
new ListConsumerGroupOffsetsOptions().timeoutMs(timeout)
);
for (String groupId : groupIds) {
try {
Map<TopicPartition, OffsetAndMetadata> groupOffsets = offsetsResult
.partitionsToOffsetAndMetadata(groupId)
.get(timeout, TimeUnit.MILLISECONDS);
committedOffsets.put(groupId, groupOffsets == null ? Collections.emptyMap() : groupOffsets);
committedOffsetsAvailable.put(groupId, true);
} catch (Exception error) {
committedOffsets.put(groupId, Collections.emptyMap());
committedOffsetsAvailable.put(groupId, false);
snapshotErrors(errors, groupId).add("Committed offsets unavailable: " + normalizeErrorMessage(error));
}
}
Map<String, Set<TopicPartition>> assignedPartitions = new HashMap<>();
Set<TopicPartition> allPartitions = new LinkedHashSet<>();
for (String groupId : groupIds) {
Set<TopicPartition> assigned = new LinkedHashSet<>();
ConsumerGroupDescription description = descriptions.get(groupId);
if (description != null) {
for (MemberDescription member : description.members()) {
assigned.addAll(member.assignment().topicPartitions());
}
}
assignedPartitions.put(groupId, assigned);
allPartitions.addAll(assigned);
allPartitions.addAll(committedOffsets.getOrDefault(groupId, Collections.emptyMap()).keySet());
}
Map<TopicPartition, Long> endOffsets = new HashMap<>();
Map<TopicPartition, String> endOffsetErrors = new HashMap<>();
if (!allPartitions.isEmpty()) {
Map<TopicPartition, OffsetSpec> endOffsetSpecs = new LinkedHashMap<>();
for (TopicPartition topicPartition : allPartitions) {
endOffsetSpecs.put(topicPartition, OffsetSpec.latest());
}
try {
ListOffsetsResult latestOffsets = admin.listOffsets(
endOffsetSpecs,
new ListOffsetsOptions().timeoutMs(timeout)
);
for (TopicPartition topicPartition : allPartitions) {
try {
ListOffsetsResult.ListOffsetsResultInfo info = latestOffsets
.partitionResult(topicPartition)
.get(timeout, TimeUnit.MILLISECONDS);
endOffsets.put(topicPartition, info.offset());
} catch (Exception error) {
endOffsetErrors.put(topicPartition, normalizeErrorMessage(error));
}
}
} catch (Exception error) {
String message = normalizeErrorMessage(error);
for (TopicPartition topicPartition : allPartitions) {
endOffsetErrors.put(topicPartition, message);
}
}
}
List<Map<String, Object>> groups = new ArrayList<>();
for (ConsumerGroupListing listing : sortedListings) {
String groupId = listing.groupId();
ConsumerGroupDescription description = descriptions.get(groupId);
Set<TopicPartition> groupPartitions = new LinkedHashSet<>(
assignedPartitions.getOrDefault(groupId, Collections.emptySet())
);
groupPartitions.addAll(committedOffsets.getOrDefault(groupId, Collections.emptyMap()).keySet());
long unavailableEndOffsetCount = groupPartitions.stream()
.filter(endOffsetErrors::containsKey)
.count();
if (unavailableEndOffsetCount > 0) {
snapshotErrors(errors, groupId).add(
"End offsets unavailable for " + unavailableEndOffsetCount + " partition(s)"
);
}
groups.add(consumerGroupSnapshotRow(
groupId,
description != null
? description.state().name()
: listing.state().map(Enum::name).orElse("UNKNOWN"),
listing.isSimpleConsumerGroup(),
description == null ? null : description.members().size(),
assignedPartitions.getOrDefault(groupId, Collections.emptySet()),
committedOffsets.getOrDefault(groupId, Collections.emptyMap()),
committedOffsetsAvailable.getOrDefault(groupId, false),
endOffsets,
errors.getOrDefault(groupId, new LinkedHashSet<>())
));
}
return Collections.singletonMap("groups", groups);
}
private static LinkedHashSet<String> snapshotErrors(
Map<String, LinkedHashSet<String>> errors,
String groupId
) {
return errors.computeIfAbsent(groupId, ignored -> new LinkedHashSet<>());
}
static Map<String, Object> consumerGroupSnapshotRow(
String groupId,
String state,
boolean simpleGroup,
Integer memberCount,
Collection<TopicPartition> assignedPartitions,
Map<TopicPartition, OffsetAndMetadata> committedOffsets,
boolean committedOffsetsAvailable,
Map<TopicPartition, Long> endOffsets,
Collection<String> errors
) {
Set<TopicPartition> topicPartitions = new LinkedHashSet<>(assignedPartitions);
topicPartitions.addAll(committedOffsets.keySet());
List<TopicPartition> sortedPartitions = new ArrayList<>(topicPartitions);
sortedPartitions.sort(Comparator
.comparing(TopicPartition::topic)
.thenComparingInt(TopicPartition::partition));
Set<String> topics = new TreeSet<>();
List<Map<String, Object>> partitions = new ArrayList<>();
long totalLag = 0;
boolean lagAvailable = committedOffsetsAvailable && !sortedPartitions.isEmpty();
for (TopicPartition topicPartition : sortedPartitions) {
topics.add(topicPartition.topic());
OffsetAndMetadata committed = committedOffsets.get(topicPartition);
Long currentOffset = committed == null ? null : committed.offset();
Long endOffset = endOffsets.get(topicPartition);
Long lag = currentOffset == null || endOffset == null
? null
: Math.max(0, endOffset - currentOffset);
if (lag == null) {
lagAvailable = false;
} else {
totalLag += lag;
}
Map<String, Object> partition = new LinkedHashMap<>();
partition.put("topic", topicPartition.topic());
partition.put("partition", topicPartition.partition());
partition.put("currentOffset", currentOffset);
partition.put("endOffset", endOffset);
partition.put("lag", lag);
partitions.add(partition);
}
Map<String, Object> group = new LinkedHashMap<>();
group.put("groupId", groupId);
group.put("state", state);
group.put("simpleGroup", simpleGroup);
group.put("memberCount", memberCount);
group.put("topics", new ArrayList<>(topics));
group.put("totalLag", lagAvailable ? totalLag : null);
group.put("lagAvailable", lagAvailable);
group.put("partitions", partitions);
group.put("error", errors.isEmpty() ? null : String.join("; ", errors));
return group;
}
private static Object describeConsumerGroup(JsonObject params) throws Exception {
AdminClient admin = requireAdmin();
int timeout = requestTimeout(params);
@@ -29,6 +29,7 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.admin.AlterConfigOp;
import org.apache.kafka.clients.admin.Config;
@@ -56,6 +57,87 @@ class KafkaAgentTest {
@TempDir
Path tempDir;
@Test
void consumerGroupSnapshotCalculatesLagAndSortsTopicsAndPartitions() {
TopicPartition alphaOne = new TopicPartition("alpha", 1);
TopicPartition alphaZero = new TopicPartition("alpha", 0);
TopicPartition betaZero = new TopicPartition("beta", 0);
Map<String, Object> group = KafkaAgent.consumerGroupSnapshotRow(
"orders-service",
"STABLE",
false,
2,
Arrays.asList(betaZero, alphaOne, alphaZero),
Map.of(
alphaZero, new OffsetAndMetadata(8),
alphaOne, new OffsetAndMetadata(20),
betaZero, new OffsetAndMetadata(5)
),
true,
Map.of(alphaZero, 10L, alphaOne, 24L, betaZero, 6L),
Collections.emptyList()
);
assertEquals(List.of("alpha", "beta"), group.get("topics"));
assertEquals(7L, group.get("totalLag"));
assertEquals(true, group.get("lagAvailable"));
assertEquals(2, group.get("memberCount"));
@SuppressWarnings("unchecked")
List<Map<String, Object>> partitions = (List<Map<String, Object>>) group.get("partitions");
assertEquals(
List.of("alpha:0", "alpha:1", "beta:0"),
partitions.stream()
.map(partition -> partition.get("topic") + ":" + partition.get("partition"))
.toList()
);
}
@Test
void consumerGroupSnapshotDoesNotReportUnknownLagAsZero() {
TopicPartition assignedOnly = new TopicPartition("orders", 0);
Map<String, Object> group = KafkaAgent.consumerGroupSnapshotRow(
"new-consumer",
"EMPTY",
false,
0,
Collections.singletonList(assignedOnly),
Collections.emptyMap(),
true,
Collections.singletonMap(assignedOnly, 42L),
Collections.emptyList()
);
assertEquals(false, group.get("lagAvailable"));
assertNull(group.get("totalLag"));
@SuppressWarnings("unchecked")
List<Map<String, Object>> partitions = (List<Map<String, Object>>) group.get("partitions");
assertNull(partitions.get(0).get("currentOffset"));
assertNull(partitions.get(0).get("lag"));
}
@Test
void consumerGroupSnapshotPreservesPartialFailuresWithoutInventingLag() {
TopicPartition partition = new TopicPartition("orders", 0);
Map<String, Object> group = KafkaAgent.consumerGroupSnapshotRow(
"orders-service",
"UNKNOWN",
false,
null,
Collections.emptyList(),
Collections.singletonMap(partition, new OffsetAndMetadata(9)),
true,
Collections.emptyMap(),
Collections.singletonList("End offsets unavailable for 1 partition(s)")
);
assertEquals(false, group.get("lagAvailable"));
assertNull(group.get("totalLag"));
assertNull(group.get("memberCount"));
assertEquals("End offsets unavailable for 1 partition(s)", group.get("error"));
}
@Test
void explicitConsumerGroupOffsetsAcceptOneBoundedPartitionOffset() {
JsonObject params = JsonParser.parseString("""