perf(kafka): batch consumer group subscriptions into one agent round trip
This commit is contained in:
1 parent
a4bf8442d7
commit
ef22b84fbe
3 files changed
+364
-63
No files matched your search
@@ -963,22 +963,184 @@ public final class KafkaAgent {
|
||||
private static Object listConsumerGroups(JsonObject params) throws Exception {
|
||||
AdminClient admin = requireAdmin();
|
||||
int timeout = requestTimeout(params);
|
||||
String filterTopic = stringOrEmpty(params, "topic");
|
||||
|
||||
Collection<ConsumerGroupListing> groups = admin.listConsumerGroups(
|
||||
new ListConsumerGroupsOptions().timeoutMs(timeout))
|
||||
.all().get(timeout, TimeUnit.MILLISECONDS);
|
||||
|
||||
List<String> groupIds = groups.stream()
|
||||
.map(ConsumerGroupListing::groupId)
|
||||
.sorted()
|
||||
.toList();
|
||||
if (groupIds.isEmpty()) {
|
||||
return Collections.singletonMap("groups", Collections.emptyList());
|
||||
}
|
||||
|
||||
// Batch-describe every group in one Kafka Admin request. Per-group
|
||||
// failures degrade to an empty member list instead of failing the batch.
|
||||
Map<String, ConsumerGroupDescription> descriptions = new HashMap<>();
|
||||
DescribeConsumerGroupsResult described = admin.describeConsumerGroups(
|
||||
groupIds,
|
||||
new DescribeConsumerGroupsOptions().timeoutMs(timeout)
|
||||
);
|
||||
for (String groupId : groupIds) {
|
||||
try {
|
||||
descriptions.put(
|
||||
groupId,
|
||||
described.describedGroups().get(groupId).get(timeout, TimeUnit.MILLISECONDS)
|
||||
);
|
||||
} catch (Exception error) {
|
||||
logger().debug("Consumer group details unavailable for {}: {}", groupId, normalizeErrorMessage(error));
|
||||
}
|
||||
}
|
||||
|
||||
// Batch-resolve committed offsets for every group in one Admin request.
|
||||
Map<String, ListConsumerGroupOffsetsSpec> offsetSpecs = new LinkedHashMap<>();
|
||||
for (String groupId : groupIds) {
|
||||
offsetSpecs.put(groupId, new ListConsumerGroupOffsetsSpec());
|
||||
}
|
||||
Map<String, Map<TopicPartition, OffsetAndMetadata>> committedOffsets = new HashMap<>();
|
||||
ListConsumerGroupOffsetsResult offsetsResult = admin.listConsumerGroupOffsets(
|
||||
offsetSpecs,
|
||||
new ListConsumerGroupOffsetsOptions().timeoutMs(timeout)
|
||||
);
|
||||
for (String groupId : groupIds) {
|
||||
try {
|
||||
Map<TopicPartition, OffsetAndMetadata> rows = offsetsResult
|
||||
.partitionsToOffsetAndMetadata(groupId)
|
||||
.get(timeout, TimeUnit.MILLISECONDS);
|
||||
committedOffsets.put(groupId, rows == null ? Collections.emptyMap() : rows);
|
||||
} catch (Exception error) {
|
||||
committedOffsets.put(groupId, Collections.emptyMap());
|
||||
logger().debug("Committed offsets unavailable for {}: {}", groupId, normalizeErrorMessage(error));
|
||||
}
|
||||
}
|
||||
|
||||
// End offsets: one batched Admin request limited to the requested
|
||||
// topic's partitions (deduplicated across groups). Without a topic
|
||||
// filter no end offsets are resolved, keeping the listing cheap.
|
||||
Map<TopicPartition, Long> endOffsets = new HashMap<>();
|
||||
if (!filterTopic.isEmpty()) {
|
||||
LinkedHashSet<TopicPartition> topicPartitions = new LinkedHashSet<>();
|
||||
for (Map<TopicPartition, OffsetAndMetadata> rows : committedOffsets.values()) {
|
||||
for (TopicPartition topicPartition : rows.keySet()) {
|
||||
if (topicPartition.topic().equals(filterTopic)) {
|
||||
topicPartitions.add(topicPartition);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!topicPartitions.isEmpty()) {
|
||||
Map<TopicPartition, OffsetSpec> endOffsetSpecs = new LinkedHashMap<>();
|
||||
for (TopicPartition topicPartition : topicPartitions) {
|
||||
endOffsetSpecs.put(topicPartition, OffsetSpec.latest());
|
||||
}
|
||||
try {
|
||||
ListOffsetsResult latestOffsets = admin.listOffsets(
|
||||
endOffsetSpecs,
|
||||
new ListOffsetsOptions().timeoutMs(timeout)
|
||||
);
|
||||
for (TopicPartition topicPartition : topicPartitions) {
|
||||
try {
|
||||
ListOffsetsResult.ListOffsetsResultInfo info = latestOffsets
|
||||
.partitionResult(topicPartition)
|
||||
.get(timeout, TimeUnit.MILLISECONDS);
|
||||
endOffsets.put(topicPartition, info.offset());
|
||||
} catch (Exception error) {
|
||||
// Leave the partition absent so callers can tell an
|
||||
// unknown end offset (no lag contribution) apart from zero lag.
|
||||
logger().debug("End offset unavailable for {}: {}", topicPartition, normalizeErrorMessage(error));
|
||||
}
|
||||
}
|
||||
} catch (Exception error) {
|
||||
logger().debug("End offset batch unavailable: {}", normalizeErrorMessage(error));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Map<String, ConsumerGroupListing> listingByGroup = groups.stream()
|
||||
.collect(Collectors.toMap(ConsumerGroupListing::groupId, listing -> listing));
|
||||
|
||||
List<Map<String, Object>> result = new ArrayList<>();
|
||||
for (ConsumerGroupListing group : groups) {
|
||||
for (String groupId : groupIds) {
|
||||
ConsumerGroupListing listing = listingByGroup.get(groupId);
|
||||
ConsumerGroupDescription description = descriptions.get(groupId);
|
||||
Map<TopicPartition, OffsetAndMetadata> committed = committedOffsets.getOrDefault(groupId, Collections.emptyMap());
|
||||
|
||||
Map<String, Object> g = new LinkedHashMap<>();
|
||||
g.put("groupId", group.groupId());
|
||||
g.put("state", group.state().map(Enum::name).orElse("UNKNOWN"));
|
||||
g.put("simpleGroup", group.isSimpleConsumerGroup());
|
||||
g.put("groupId", groupId);
|
||||
g.put("state", listing == null ? "UNKNOWN" : listing.state().map(Enum::name).orElse("UNKNOWN"));
|
||||
g.put("simpleGroup", listing != null && listing.isSimpleConsumerGroup());
|
||||
g.put("members", memberMaps(description == null ? Collections.emptyList() : description.members()));
|
||||
g.put("committedOffsets", offsetRows(committed, filterTopic));
|
||||
g.put("endOffsets", endOffsetRows(committed, endOffsets, filterTopic));
|
||||
result.add(g);
|
||||
}
|
||||
result.sort(Comparator.comparing(m -> (String) m.get("groupId")));
|
||||
return Collections.singletonMap("groups", result);
|
||||
}
|
||||
|
||||
static List<Map<String, Object>> memberMaps(Collection<MemberDescription> members) {
|
||||
List<Map<String, Object>> result = new ArrayList<>();
|
||||
for (MemberDescription member : members) {
|
||||
Map<String, Object> m = new LinkedHashMap<>();
|
||||
m.put("memberId", member.consumerId());
|
||||
m.put("clientId", member.clientId());
|
||||
m.put("host", member.host());
|
||||
List<Map<String, Object>> assignments = new ArrayList<>();
|
||||
for (TopicPartition topicPartition : member.assignment().topicPartitions()) {
|
||||
Map<String, Object> a = new LinkedHashMap<>();
|
||||
a.put("topic", topicPartition.topic());
|
||||
a.put("partition", topicPartition.partition());
|
||||
assignments.add(a);
|
||||
}
|
||||
m.put("assignments", assignments);
|
||||
result.add(m);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Committed offsets as rows, optionally restricted to one topic. */
|
||||
static List<Map<String, Object>> offsetRows(Map<TopicPartition, OffsetAndMetadata> offsets, String topic) {
|
||||
List<Map<String, Object>> rows = new ArrayList<>();
|
||||
for (Map.Entry<TopicPartition, OffsetAndMetadata> entry : offsets.entrySet()) {
|
||||
if (!topic.isEmpty() && !entry.getKey().topic().equals(topic)) {
|
||||
continue;
|
||||
}
|
||||
Map<String, Object> row = new LinkedHashMap<>();
|
||||
row.put("topic", entry.getKey().topic());
|
||||
row.put("partition", entry.getKey().partition());
|
||||
row.put("offset", entry.getValue().offset());
|
||||
rows.add(row);
|
||||
}
|
||||
rows.sort(Comparator.comparingInt(row -> (int) row.get("partition")));
|
||||
return rows;
|
||||
}
|
||||
|
||||
/** End offsets for a group's committed partitions, restricted to one topic. */
|
||||
static List<Map<String, Object>> endOffsetRows(
|
||||
Map<TopicPartition, OffsetAndMetadata> committed,
|
||||
Map<TopicPartition, Long> endOffsets,
|
||||
String topic
|
||||
) {
|
||||
List<Map<String, Object>> rows = new ArrayList<>();
|
||||
for (TopicPartition topicPartition : committed.keySet()) {
|
||||
if (!topic.isEmpty() && !topicPartition.topic().equals(topic)) {
|
||||
continue;
|
||||
}
|
||||
Long endOffset = endOffsets.get(topicPartition);
|
||||
if (endOffset == null) {
|
||||
continue;
|
||||
}
|
||||
Map<String, Object> row = new LinkedHashMap<>();
|
||||
row.put("topic", topicPartition.topic());
|
||||
row.put("partition", topicPartition.partition());
|
||||
row.put("offset", endOffset);
|
||||
rows.add(row);
|
||||
}
|
||||
rows.sort(Comparator.comparingInt(row -> (int) row.get("partition")));
|
||||
return rows;
|
||||
}
|
||||
|
||||
private static Object getConsumerGroupSnapshot(JsonObject params) throws Exception {
|
||||
AdminClient admin = requireAdmin();
|
||||
int timeout = requestTimeout(params);
|
||||
|
||||
@@ -142,6 +142,65 @@ class KafkaAgentTest {
|
||||
assertEquals("End offsets unavailable for 1 partition(s)", group.get("error"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void consumerGroupListRowsFilterToTheRequestedTopicAndSortByPartition() {
|
||||
Map<TopicPartition, OffsetAndMetadata> offsets = new HashMap<>();
|
||||
offsets.put(new TopicPartition("orders", 2), new OffsetAndMetadata(20));
|
||||
offsets.put(new TopicPartition("payments", 0), new OffsetAndMetadata(5));
|
||||
offsets.put(new TopicPartition("orders", 0), new OffsetAndMetadata(8));
|
||||
|
||||
List<Map<String, Object>> rows = KafkaAgent.offsetRows(offsets, "orders");
|
||||
|
||||
assertEquals(List.of("orders:0", "orders:2"), rows.stream()
|
||||
.map(row -> row.get("topic") + ":" + row.get("partition"))
|
||||
.toList());
|
||||
assertEquals(8L, rows.get(0).get("offset"));
|
||||
assertEquals(20L, rows.get(1).get("offset"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void consumerGroupListRowsKeepEveryPartitionWithoutATopicFilter() {
|
||||
Map<TopicPartition, OffsetAndMetadata> offsets = new HashMap<>();
|
||||
offsets.put(new TopicPartition("orders", 0), new OffsetAndMetadata(8));
|
||||
offsets.put(new TopicPartition("payments", 1), new OffsetAndMetadata(3));
|
||||
|
||||
List<Map<String, Object>> rows = KafkaAgent.offsetRows(offsets, "");
|
||||
|
||||
assertEquals(2, rows.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void consumerGroupListEndOffsetRowsSkipPartitionsWithoutResolvedEndOffsets() {
|
||||
Map<TopicPartition, OffsetAndMetadata> committed = new HashMap<>();
|
||||
committed.put(new TopicPartition("orders", 0), new OffsetAndMetadata(8));
|
||||
committed.put(new TopicPartition("orders", 1), new OffsetAndMetadata(20));
|
||||
Map<TopicPartition, Long> endOffsets = new HashMap<>();
|
||||
endOffsets.put(new TopicPartition("orders", 1), 24L);
|
||||
|
||||
List<Map<String, Object>> rows = KafkaAgent.endOffsetRows(committed, endOffsets, "orders");
|
||||
|
||||
assertEquals(List.of("orders:1"), rows.stream()
|
||||
.map(row -> row.get("topic") + ":" + row.get("partition"))
|
||||
.toList());
|
||||
assertEquals(24L, rows.get(0).get("offset"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void consumerGroupListEndOffsetRowsFilterToTheRequestedTopic() {
|
||||
Map<TopicPartition, OffsetAndMetadata> committed = new HashMap<>();
|
||||
committed.put(new TopicPartition("orders", 0), new OffsetAndMetadata(8));
|
||||
committed.put(new TopicPartition("payments", 0), new OffsetAndMetadata(4));
|
||||
Map<TopicPartition, Long> endOffsets = new HashMap<>();
|
||||
endOffsets.put(new TopicPartition("orders", 0), 10L);
|
||||
endOffsets.put(new TopicPartition("payments", 0), 9L);
|
||||
|
||||
List<Map<String, Object>> rows = KafkaAgent.endOffsetRows(committed, endOffsets, "orders");
|
||||
|
||||
assertEquals(List.of("orders:0"), rows.stream()
|
||||
.map(row -> row.get("topic") + ":" + row.get("partition"))
|
||||
.toList());
|
||||
}
|
||||
|
||||
@Test
|
||||
void explicitConsumerGroupOffsetsAcceptOneBoundedPartitionOffset() {
|
||||
JsonObject params = JsonParser.parseString("""
|
||||
|
||||
Reference in new issue
Block a user