perf(kafka): batch consumer group subscriptions into one agent round trip
This commit is contained in:
1 parent
a4bf8442d7
commit
ef22b84fbe
3 files changed
+364
-63
No files matched your search
@@ -963,22 +963,184 @@ public final class KafkaAgent {
|
|||||||
private static Object listConsumerGroups(JsonObject params) throws Exception {
|
private static Object listConsumerGroups(JsonObject params) throws Exception {
|
||||||
AdminClient admin = requireAdmin();
|
AdminClient admin = requireAdmin();
|
||||||
int timeout = requestTimeout(params);
|
int timeout = requestTimeout(params);
|
||||||
|
String filterTopic = stringOrEmpty(params, "topic");
|
||||||
|
|
||||||
Collection<ConsumerGroupListing> groups = admin.listConsumerGroups(
|
Collection<ConsumerGroupListing> groups = admin.listConsumerGroups(
|
||||||
new ListConsumerGroupsOptions().timeoutMs(timeout))
|
new ListConsumerGroupsOptions().timeoutMs(timeout))
|
||||||
.all().get(timeout, TimeUnit.MILLISECONDS);
|
.all().get(timeout, TimeUnit.MILLISECONDS);
|
||||||
|
|
||||||
|
List<String> groupIds = groups.stream()
|
||||||
|
.map(ConsumerGroupListing::groupId)
|
||||||
|
.sorted()
|
||||||
|
.toList();
|
||||||
|
if (groupIds.isEmpty()) {
|
||||||
|
return Collections.singletonMap("groups", Collections.emptyList());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Batch-describe every group in one Kafka Admin request. Per-group
|
||||||
|
// failures degrade to an empty member list instead of failing the batch.
|
||||||
|
Map<String, ConsumerGroupDescription> descriptions = new HashMap<>();
|
||||||
|
DescribeConsumerGroupsResult described = admin.describeConsumerGroups(
|
||||||
|
groupIds,
|
||||||
|
new DescribeConsumerGroupsOptions().timeoutMs(timeout)
|
||||||
|
);
|
||||||
|
for (String groupId : groupIds) {
|
||||||
|
try {
|
||||||
|
descriptions.put(
|
||||||
|
groupId,
|
||||||
|
described.describedGroups().get(groupId).get(timeout, TimeUnit.MILLISECONDS)
|
||||||
|
);
|
||||||
|
} catch (Exception error) {
|
||||||
|
logger().debug("Consumer group details unavailable for {}: {}", groupId, normalizeErrorMessage(error));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Batch-resolve committed offsets for every group in one Admin request.
|
||||||
|
Map<String, ListConsumerGroupOffsetsSpec> offsetSpecs = new LinkedHashMap<>();
|
||||||
|
for (String groupId : groupIds) {
|
||||||
|
offsetSpecs.put(groupId, new ListConsumerGroupOffsetsSpec());
|
||||||
|
}
|
||||||
|
Map<String, Map<TopicPartition, OffsetAndMetadata>> committedOffsets = new HashMap<>();
|
||||||
|
ListConsumerGroupOffsetsResult offsetsResult = admin.listConsumerGroupOffsets(
|
||||||
|
offsetSpecs,
|
||||||
|
new ListConsumerGroupOffsetsOptions().timeoutMs(timeout)
|
||||||
|
);
|
||||||
|
for (String groupId : groupIds) {
|
||||||
|
try {
|
||||||
|
Map<TopicPartition, OffsetAndMetadata> rows = offsetsResult
|
||||||
|
.partitionsToOffsetAndMetadata(groupId)
|
||||||
|
.get(timeout, TimeUnit.MILLISECONDS);
|
||||||
|
committedOffsets.put(groupId, rows == null ? Collections.emptyMap() : rows);
|
||||||
|
} catch (Exception error) {
|
||||||
|
committedOffsets.put(groupId, Collections.emptyMap());
|
||||||
|
logger().debug("Committed offsets unavailable for {}: {}", groupId, normalizeErrorMessage(error));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// End offsets: one batched Admin request limited to the requested
|
||||||
|
// topic's partitions (deduplicated across groups). Without a topic
|
||||||
|
// filter no end offsets are resolved, keeping the listing cheap.
|
||||||
|
Map<TopicPartition, Long> endOffsets = new HashMap<>();
|
||||||
|
if (!filterTopic.isEmpty()) {
|
||||||
|
LinkedHashSet<TopicPartition> topicPartitions = new LinkedHashSet<>();
|
||||||
|
for (Map<TopicPartition, OffsetAndMetadata> rows : committedOffsets.values()) {
|
||||||
|
for (TopicPartition topicPartition : rows.keySet()) {
|
||||||
|
if (topicPartition.topic().equals(filterTopic)) {
|
||||||
|
topicPartitions.add(topicPartition);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!topicPartitions.isEmpty()) {
|
||||||
|
Map<TopicPartition, OffsetSpec> endOffsetSpecs = new LinkedHashMap<>();
|
||||||
|
for (TopicPartition topicPartition : topicPartitions) {
|
||||||
|
endOffsetSpecs.put(topicPartition, OffsetSpec.latest());
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
ListOffsetsResult latestOffsets = admin.listOffsets(
|
||||||
|
endOffsetSpecs,
|
||||||
|
new ListOffsetsOptions().timeoutMs(timeout)
|
||||||
|
);
|
||||||
|
for (TopicPartition topicPartition : topicPartitions) {
|
||||||
|
try {
|
||||||
|
ListOffsetsResult.ListOffsetsResultInfo info = latestOffsets
|
||||||
|
.partitionResult(topicPartition)
|
||||||
|
.get(timeout, TimeUnit.MILLISECONDS);
|
||||||
|
endOffsets.put(topicPartition, info.offset());
|
||||||
|
} catch (Exception error) {
|
||||||
|
// Leave the partition absent so callers can tell an
|
||||||
|
// unknown end offset (no lag contribution) apart from zero lag.
|
||||||
|
logger().debug("End offset unavailable for {}: {}", topicPartition, normalizeErrorMessage(error));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (Exception error) {
|
||||||
|
logger().debug("End offset batch unavailable: {}", normalizeErrorMessage(error));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Map<String, ConsumerGroupListing> listingByGroup = groups.stream()
|
||||||
|
.collect(Collectors.toMap(ConsumerGroupListing::groupId, listing -> listing));
|
||||||
|
|
||||||
List<Map<String, Object>> result = new ArrayList<>();
|
List<Map<String, Object>> result = new ArrayList<>();
|
||||||
for (ConsumerGroupListing group : groups) {
|
for (String groupId : groupIds) {
|
||||||
|
ConsumerGroupListing listing = listingByGroup.get(groupId);
|
||||||
|
ConsumerGroupDescription description = descriptions.get(groupId);
|
||||||
|
Map<TopicPartition, OffsetAndMetadata> committed = committedOffsets.getOrDefault(groupId, Collections.emptyMap());
|
||||||
|
|
||||||
Map<String, Object> g = new LinkedHashMap<>();
|
Map<String, Object> g = new LinkedHashMap<>();
|
||||||
g.put("groupId", group.groupId());
|
g.put("groupId", groupId);
|
||||||
g.put("state", group.state().map(Enum::name).orElse("UNKNOWN"));
|
g.put("state", listing == null ? "UNKNOWN" : listing.state().map(Enum::name).orElse("UNKNOWN"));
|
||||||
g.put("simpleGroup", group.isSimpleConsumerGroup());
|
g.put("simpleGroup", listing != null && listing.isSimpleConsumerGroup());
|
||||||
|
g.put("members", memberMaps(description == null ? Collections.emptyList() : description.members()));
|
||||||
|
g.put("committedOffsets", offsetRows(committed, filterTopic));
|
||||||
|
g.put("endOffsets", endOffsetRows(committed, endOffsets, filterTopic));
|
||||||
result.add(g);
|
result.add(g);
|
||||||
}
|
}
|
||||||
result.sort(Comparator.comparing(m -> (String) m.get("groupId")));
|
|
||||||
return Collections.singletonMap("groups", result);
|
return Collections.singletonMap("groups", result);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static List<Map<String, Object>> memberMaps(Collection<MemberDescription> members) {
|
||||||
|
List<Map<String, Object>> result = new ArrayList<>();
|
||||||
|
for (MemberDescription member : members) {
|
||||||
|
Map<String, Object> m = new LinkedHashMap<>();
|
||||||
|
m.put("memberId", member.consumerId());
|
||||||
|
m.put("clientId", member.clientId());
|
||||||
|
m.put("host", member.host());
|
||||||
|
List<Map<String, Object>> assignments = new ArrayList<>();
|
||||||
|
for (TopicPartition topicPartition : member.assignment().topicPartitions()) {
|
||||||
|
Map<String, Object> a = new LinkedHashMap<>();
|
||||||
|
a.put("topic", topicPartition.topic());
|
||||||
|
a.put("partition", topicPartition.partition());
|
||||||
|
assignments.add(a);
|
||||||
|
}
|
||||||
|
m.put("assignments", assignments);
|
||||||
|
result.add(m);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Committed offsets as rows, optionally restricted to one topic. */
|
||||||
|
static List<Map<String, Object>> offsetRows(Map<TopicPartition, OffsetAndMetadata> offsets, String topic) {
|
||||||
|
List<Map<String, Object>> rows = new ArrayList<>();
|
||||||
|
for (Map.Entry<TopicPartition, OffsetAndMetadata> entry : offsets.entrySet()) {
|
||||||
|
if (!topic.isEmpty() && !entry.getKey().topic().equals(topic)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
Map<String, Object> row = new LinkedHashMap<>();
|
||||||
|
row.put("topic", entry.getKey().topic());
|
||||||
|
row.put("partition", entry.getKey().partition());
|
||||||
|
row.put("offset", entry.getValue().offset());
|
||||||
|
rows.add(row);
|
||||||
|
}
|
||||||
|
rows.sort(Comparator.comparingInt(row -> (int) row.get("partition")));
|
||||||
|
return rows;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** End offsets for a group's committed partitions, restricted to one topic. */
|
||||||
|
static List<Map<String, Object>> endOffsetRows(
|
||||||
|
Map<TopicPartition, OffsetAndMetadata> committed,
|
||||||
|
Map<TopicPartition, Long> endOffsets,
|
||||||
|
String topic
|
||||||
|
) {
|
||||||
|
List<Map<String, Object>> rows = new ArrayList<>();
|
||||||
|
for (TopicPartition topicPartition : committed.keySet()) {
|
||||||
|
if (!topic.isEmpty() && !topicPartition.topic().equals(topic)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
Long endOffset = endOffsets.get(topicPartition);
|
||||||
|
if (endOffset == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
Map<String, Object> row = new LinkedHashMap<>();
|
||||||
|
row.put("topic", topicPartition.topic());
|
||||||
|
row.put("partition", topicPartition.partition());
|
||||||
|
row.put("offset", endOffset);
|
||||||
|
rows.add(row);
|
||||||
|
}
|
||||||
|
rows.sort(Comparator.comparingInt(row -> (int) row.get("partition")));
|
||||||
|
return rows;
|
||||||
|
}
|
||||||
|
|
||||||
private static Object getConsumerGroupSnapshot(JsonObject params) throws Exception {
|
private static Object getConsumerGroupSnapshot(JsonObject params) throws Exception {
|
||||||
AdminClient admin = requireAdmin();
|
AdminClient admin = requireAdmin();
|
||||||
int timeout = requestTimeout(params);
|
int timeout = requestTimeout(params);
|
||||||
|
|||||||
@@ -142,6 +142,65 @@ class KafkaAgentTest {
|
|||||||
assertEquals("End offsets unavailable for 1 partition(s)", group.get("error"));
|
assertEquals("End offsets unavailable for 1 partition(s)", group.get("error"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void consumerGroupListRowsFilterToTheRequestedTopicAndSortByPartition() {
|
||||||
|
Map<TopicPartition, OffsetAndMetadata> offsets = new HashMap<>();
|
||||||
|
offsets.put(new TopicPartition("orders", 2), new OffsetAndMetadata(20));
|
||||||
|
offsets.put(new TopicPartition("payments", 0), new OffsetAndMetadata(5));
|
||||||
|
offsets.put(new TopicPartition("orders", 0), new OffsetAndMetadata(8));
|
||||||
|
|
||||||
|
List<Map<String, Object>> rows = KafkaAgent.offsetRows(offsets, "orders");
|
||||||
|
|
||||||
|
assertEquals(List.of("orders:0", "orders:2"), rows.stream()
|
||||||
|
.map(row -> row.get("topic") + ":" + row.get("partition"))
|
||||||
|
.toList());
|
||||||
|
assertEquals(8L, rows.get(0).get("offset"));
|
||||||
|
assertEquals(20L, rows.get(1).get("offset"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void consumerGroupListRowsKeepEveryPartitionWithoutATopicFilter() {
|
||||||
|
Map<TopicPartition, OffsetAndMetadata> offsets = new HashMap<>();
|
||||||
|
offsets.put(new TopicPartition("orders", 0), new OffsetAndMetadata(8));
|
||||||
|
offsets.put(new TopicPartition("payments", 1), new OffsetAndMetadata(3));
|
||||||
|
|
||||||
|
List<Map<String, Object>> rows = KafkaAgent.offsetRows(offsets, "");
|
||||||
|
|
||||||
|
assertEquals(2, rows.size());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void consumerGroupListEndOffsetRowsSkipPartitionsWithoutResolvedEndOffsets() {
|
||||||
|
Map<TopicPartition, OffsetAndMetadata> committed = new HashMap<>();
|
||||||
|
committed.put(new TopicPartition("orders", 0), new OffsetAndMetadata(8));
|
||||||
|
committed.put(new TopicPartition("orders", 1), new OffsetAndMetadata(20));
|
||||||
|
Map<TopicPartition, Long> endOffsets = new HashMap<>();
|
||||||
|
endOffsets.put(new TopicPartition("orders", 1), 24L);
|
||||||
|
|
||||||
|
List<Map<String, Object>> rows = KafkaAgent.endOffsetRows(committed, endOffsets, "orders");
|
||||||
|
|
||||||
|
assertEquals(List.of("orders:1"), rows.stream()
|
||||||
|
.map(row -> row.get("topic") + ":" + row.get("partition"))
|
||||||
|
.toList());
|
||||||
|
assertEquals(24L, rows.get(0).get("offset"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void consumerGroupListEndOffsetRowsFilterToTheRequestedTopic() {
|
||||||
|
Map<TopicPartition, OffsetAndMetadata> committed = new HashMap<>();
|
||||||
|
committed.put(new TopicPartition("orders", 0), new OffsetAndMetadata(8));
|
||||||
|
committed.put(new TopicPartition("payments", 0), new OffsetAndMetadata(4));
|
||||||
|
Map<TopicPartition, Long> endOffsets = new HashMap<>();
|
||||||
|
endOffsets.put(new TopicPartition("orders", 0), 10L);
|
||||||
|
endOffsets.put(new TopicPartition("payments", 0), 9L);
|
||||||
|
|
||||||
|
List<Map<String, Object>> rows = KafkaAgent.endOffsetRows(committed, endOffsets, "orders");
|
||||||
|
|
||||||
|
assertEquals(List.of("orders:0"), rows.stream()
|
||||||
|
.map(row -> row.get("topic") + ":" + row.get("partition"))
|
||||||
|
.toList());
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void explicitConsumerGroupOffsetsAcceptOneBoundedPartitionOffset() {
|
void explicitConsumerGroupOffsetsAcceptOneBoundedPartitionOffset() {
|
||||||
JsonObject params = JsonParser.parseString("""
|
JsonObject params = JsonParser.parseString("""
|
||||||
|
|||||||
@@ -269,32 +269,20 @@ impl MessageQueueAdmin for KafkaAdmin {
|
|||||||
// ---- Subscriptions (mapped to consumer groups) ----
|
// ---- Subscriptions (mapped to consumer groups) ----
|
||||||
|
|
||||||
async fn list_subscriptions(&self, topic: &TopicRef) -> Result<Vec<SubscriptionInfo>, String> {
|
async fn list_subscriptions(&self, topic: &TopicRef) -> Result<Vec<SubscriptionInfo>, String> {
|
||||||
// List consumer groups and filter those subscribed to this topic
|
// One batched agent RPC: the agent lists every consumer group, batch-describes
|
||||||
let result: serde_json::Value = self.call("mq_list_consumer_groups", serde_json::json!({})).await?;
|
// them, and resolves committed + end offsets for the requested topic with a
|
||||||
|
// handful of Kafka Admin calls. Subscriptions are filtered in memory below.
|
||||||
|
// (The previous 1 + 2N serial RPC pattern — describe + lag per group — took
|
||||||
|
// tens of seconds on remote clusters with many groups, see #7163.)
|
||||||
|
let result: serde_json::Value =
|
||||||
|
self.call("mq_list_consumer_groups", serde_json::json!({ "topic": topic.topic })).await?;
|
||||||
let groups = result.get("groups").and_then(|v| v.as_array()).cloned().unwrap_or_default();
|
let groups = result.get("groups").and_then(|v| v.as_array()).cloned().unwrap_or_default();
|
||||||
|
|
||||||
// For each group, check both active assignments and committed offsets.
|
// For each group, check both active assignments and committed offsets.
|
||||||
let mut subs = Vec::new();
|
let mut subs = Vec::new();
|
||||||
for group in groups {
|
for group in groups {
|
||||||
let group_id = group.get("groupId").and_then(|v| v.as_str()).unwrap_or_default();
|
let group_id = group.get("groupId").and_then(|v| v.as_str()).unwrap_or_default();
|
||||||
let desc = match self
|
if let Some(sub) = kafka_subscription_from_group_row(group_id, &topic.topic, &group) {
|
||||||
.call::<serde_json::Value>("mq_describe_consumer_group", serde_json::json!({ "groupId": group_id }))
|
|
||||||
.await
|
|
||||||
{
|
|
||||||
Ok(desc) => desc,
|
|
||||||
Err(_) => continue, // Skip groups we can't describe
|
|
||||||
};
|
|
||||||
let lag = self
|
|
||||||
.call::<serde_json::Value>(
|
|
||||||
"mq_get_consumer_lag",
|
|
||||||
serde_json::json!({
|
|
||||||
"groupId": group_id,
|
|
||||||
"topic": topic.topic,
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
.ok();
|
|
||||||
if let Some(sub) = kafka_subscription_for_topic(group_id, &topic.topic, &desc, lag.as_ref()) {
|
|
||||||
subs.push(sub);
|
subs.push(sub);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -763,13 +751,12 @@ fn reset_cursor_params(topic: &TopicRef, sub: &str, pos: ResetPosition) -> Resul
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn kafka_subscription_for_topic(
|
/// Build a `SubscriptionInfo` for `topic` from one row of the batched
|
||||||
group_id: &str,
|
/// `mq_list_consumer_groups` response. The agent already resolved committed
|
||||||
topic: &str,
|
/// offsets (and end offsets for the requested topic) for every group, so no
|
||||||
desc: &serde_json::Value,
|
/// per-group RPC happens here.
|
||||||
lag: Option<&serde_json::Value>,
|
fn kafka_subscription_from_group_row(group_id: &str, topic: &str, row: &serde_json::Value) -> Option<SubscriptionInfo> {
|
||||||
) -> Option<SubscriptionInfo> {
|
let consumers = row
|
||||||
let consumers = desc
|
|
||||||
.get("members")
|
.get("members")
|
||||||
.and_then(|v| v.as_array())
|
.and_then(|v| v.as_array())
|
||||||
.into_iter()
|
.into_iter()
|
||||||
@@ -778,20 +765,58 @@ fn kafka_subscription_for_topic(
|
|||||||
.map(kafka_consumer_from_member)
|
.map(kafka_consumer_from_member)
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
let has_active_assignment = !consumers.is_empty();
|
let has_active_assignment = !consumers.is_empty();
|
||||||
let has_committed_offsets = lag
|
|
||||||
.and_then(|v| v.get("partitions"))
|
let committed = row
|
||||||
|
.get("committedOffsets")
|
||||||
.and_then(|v| v.as_array())
|
.and_then(|v| v.as_array())
|
||||||
.map(|partitions| !partitions.is_empty())
|
.into_iter()
|
||||||
.unwrap_or(false);
|
.flatten()
|
||||||
|
.filter(|partition| partition.get("topic").and_then(|v| v.as_str()) == Some(topic))
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let has_committed_offsets = !committed.is_empty();
|
||||||
|
|
||||||
if !has_active_assignment && !has_committed_offsets {
|
if !has_active_assignment && !has_committed_offsets {
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let end_offsets = row
|
||||||
|
.get("endOffsets")
|
||||||
|
.and_then(|v| v.as_array())
|
||||||
|
.map(|partitions| {
|
||||||
|
partitions
|
||||||
|
.iter()
|
||||||
|
.filter_map(|partition| {
|
||||||
|
// Defensive: the agent already filters end offsets to the
|
||||||
|
// requested topic, but keying by partition id alone would
|
||||||
|
// let a same-numbered partition of another topic collide.
|
||||||
|
if partition.get("topic").and_then(|v| v.as_str()) != Some(topic) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let partition_id = partition.get("partition").and_then(|v| v.as_i64())?;
|
||||||
|
let offset = partition.get("offset").and_then(|v| v.as_i64())?;
|
||||||
|
Some((partition_id, offset))
|
||||||
|
})
|
||||||
|
.collect::<HashMap<_, _>>()
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
|
||||||
|
// Sum lag over the topic's committed partitions. A partition without a
|
||||||
|
// resolved end offset contributes nothing (matches the agent's lag probe:
|
||||||
|
// unknown end offsets are not reported as zero lag).
|
||||||
|
let msg_backlog = committed
|
||||||
|
.iter()
|
||||||
|
.filter_map(|partition| {
|
||||||
|
let partition_id = partition.get("partition").and_then(|v| v.as_i64())?;
|
||||||
|
let committed_offset = partition.get("offset").and_then(|v| v.as_i64())?;
|
||||||
|
let end_offset = end_offsets.get(&partition_id).copied()?;
|
||||||
|
Some((end_offset - committed_offset).max(0))
|
||||||
|
})
|
||||||
|
.sum();
|
||||||
|
|
||||||
Some(SubscriptionInfo {
|
Some(SubscriptionInfo {
|
||||||
name: group_id.to_string(),
|
name: group_id.to_string(),
|
||||||
sub_type: "consumer-group".to_string(),
|
sub_type: "consumer-group".to_string(),
|
||||||
msg_backlog: lag.and_then(|v| v.get("totalLag")).and_then(|v| v.as_i64()).unwrap_or(0),
|
msg_backlog,
|
||||||
msg_rate_out: 0.0,
|
msg_rate_out: 0.0,
|
||||||
msg_throughput_out: 0.0,
|
msg_throughput_out: 0.0,
|
||||||
consumers,
|
consumers,
|
||||||
@@ -1163,31 +1188,37 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn kafka_subscription_for_topic_includes_offline_group_with_committed_offsets() {
|
fn kafka_subscription_from_group_row_includes_offline_group_with_committed_offsets() {
|
||||||
let desc = serde_json::json!({
|
let row = serde_json::json!({
|
||||||
"groupId": "orders-service",
|
"groupId": "orders-service",
|
||||||
"members": []
|
"state": "EMPTY",
|
||||||
});
|
"simpleGroup": false,
|
||||||
let lag = serde_json::json!({
|
"members": [],
|
||||||
"totalLag": 7,
|
"committedOffsets": [
|
||||||
"partitions": [
|
{ "topic": "orders", "partition": 0, "offset": 3 },
|
||||||
{ "partition": 0, "currentOffset": 3, "endOffset": 10, "lag": 7 }
|
{ "topic": "orders", "partition": 1, "offset": 8 }
|
||||||
|
],
|
||||||
|
"endOffsets": [
|
||||||
|
{ "topic": "orders", "partition": 0, "offset": 10 },
|
||||||
|
{ "topic": "orders", "partition": 1, "offset": 12 }
|
||||||
]
|
]
|
||||||
});
|
});
|
||||||
|
|
||||||
let sub = kafka_subscription_for_topic("orders-service", "orders", &desc, Some(&lag))
|
let sub = kafka_subscription_from_group_row("orders-service", "orders", &row)
|
||||||
.expect("committed offsets should make an inactive group visible");
|
.expect("committed offsets should make an inactive group visible");
|
||||||
|
|
||||||
assert_eq!(sub.name, "orders-service");
|
assert_eq!(sub.name, "orders-service");
|
||||||
assert_eq!(sub.sub_type, "consumer-group");
|
assert_eq!(sub.sub_type, "consumer-group");
|
||||||
assert_eq!(sub.msg_backlog, 7);
|
assert_eq!(sub.msg_backlog, 11); // (10-3) + (12-8)
|
||||||
assert!(sub.consumers.is_empty());
|
assert!(sub.consumers.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn kafka_subscription_for_topic_includes_active_assignment_without_committed_offsets() {
|
fn kafka_subscription_from_group_row_includes_active_assignment_without_committed_offsets() {
|
||||||
let desc = serde_json::json!({
|
let row = serde_json::json!({
|
||||||
"groupId": "live-service",
|
"groupId": "live-service",
|
||||||
|
"state": "STABLE",
|
||||||
|
"simpleGroup": false,
|
||||||
"members": [
|
"members": [
|
||||||
{
|
{
|
||||||
"memberId": "consumer-events",
|
"memberId": "consumer-events",
|
||||||
@@ -1203,14 +1234,12 @@ mod tests {
|
|||||||
{ "topic": "audit", "partition": 0 }
|
{ "topic": "audit", "partition": 0 }
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
]
|
],
|
||||||
});
|
"committedOffsets": [],
|
||||||
let lag = serde_json::json!({
|
"endOffsets": []
|
||||||
"totalLag": 0,
|
|
||||||
"partitions": []
|
|
||||||
});
|
});
|
||||||
|
|
||||||
let sub = kafka_subscription_for_topic("live-service", "events", &desc, Some(&lag))
|
let sub = kafka_subscription_from_group_row("live-service", "events", &row)
|
||||||
.expect("active assignments should make the group visible");
|
.expect("active assignments should make the group visible");
|
||||||
|
|
||||||
assert_eq!(sub.name, "live-service");
|
assert_eq!(sub.name, "live-service");
|
||||||
@@ -1221,21 +1250,72 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn kafka_subscription_for_topic_ignores_unrelated_group() {
|
fn kafka_subscription_from_group_row_ignores_unrelated_group() {
|
||||||
let desc = serde_json::json!({
|
let row = serde_json::json!({
|
||||||
"groupId": "billing-service",
|
"groupId": "billing-service",
|
||||||
|
"state": "EMPTY",
|
||||||
|
"simpleGroup": false,
|
||||||
"members": [{
|
"members": [{
|
||||||
"assignments": [
|
"assignments": [
|
||||||
{ "topic": "billing", "partition": 0 }
|
{ "topic": "billing", "partition": 0 }
|
||||||
]
|
]
|
||||||
}]
|
}],
|
||||||
});
|
"committedOffsets": [
|
||||||
let lag = serde_json::json!({
|
{ "topic": "billing", "partition": 0, "offset": 2 }
|
||||||
"totalLag": 0,
|
],
|
||||||
"partitions": []
|
"endOffsets": [
|
||||||
|
{ "topic": "billing", "partition": 0, "offset": 5 }
|
||||||
|
]
|
||||||
});
|
});
|
||||||
|
|
||||||
assert!(kafka_subscription_for_topic("billing-service", "orders", &desc, Some(&lag)).is_none());
|
assert!(kafka_subscription_from_group_row("billing-service", "orders", &row).is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn kafka_subscription_from_group_row_skips_partitions_without_end_offsets() {
|
||||||
|
let row = serde_json::json!({
|
||||||
|
"groupId": "orders-service",
|
||||||
|
"state": "EMPTY",
|
||||||
|
"simpleGroup": false,
|
||||||
|
"members": [],
|
||||||
|
"committedOffsets": [
|
||||||
|
{ "topic": "orders", "partition": 0, "offset": 3 },
|
||||||
|
{ "topic": "orders", "partition": 1, "offset": 8 }
|
||||||
|
],
|
||||||
|
"endOffsets": [
|
||||||
|
// partition 1's end offset is unavailable (agent-side failure)
|
||||||
|
{ "topic": "orders", "partition": 0, "offset": 10 }
|
||||||
|
]
|
||||||
|
});
|
||||||
|
|
||||||
|
let sub = kafka_subscription_from_group_row("orders-service", "orders", &row)
|
||||||
|
.expect("the group still has visible committed offsets");
|
||||||
|
|
||||||
|
assert_eq!(sub.msg_backlog, 7); // only partition 0 contributes
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn kafka_subscription_from_group_row_filters_committed_offsets_to_the_requested_topic() {
|
||||||
|
let row = serde_json::json!({
|
||||||
|
"groupId": "multi-topic-service",
|
||||||
|
"state": "EMPTY",
|
||||||
|
"simpleGroup": false,
|
||||||
|
"members": [],
|
||||||
|
"committedOffsets": [
|
||||||
|
{ "topic": "orders", "partition": 0, "offset": 3 },
|
||||||
|
{ "topic": "payments", "partition": 0, "offset": 7 }
|
||||||
|
],
|
||||||
|
"endOffsets": [
|
||||||
|
{ "topic": "orders", "partition": 0, "offset": 8 },
|
||||||
|
{ "topic": "payments", "partition": 0, "offset": 9 }
|
||||||
|
]
|
||||||
|
});
|
||||||
|
|
||||||
|
let sub = kafka_subscription_from_group_row("multi-topic-service", "orders", &row)
|
||||||
|
.expect("committed offsets on the requested topic make the group visible");
|
||||||
|
|
||||||
|
// Lag totals only the requested topic's partitions.
|
||||||
|
assert_eq!(sub.msg_backlog, 5);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
Reference in new issue
Block a user