fix(kafka): improve message browsing and topic navigation

This commit is contained in:
miracle authored and GitHub committed 2026-08-14 00:19:09 +08:00
1 parent f6518ff3a0
commit e77b547cea
24 files changed
+751 -114

No files matched your search

@@ -1140,8 +1140,12 @@ public final class KafkaAgent {
return peekMessagesResult(Collections.emptyList(), false);
}
int messagesPerPartition = peekMessagesPerPartition(count, readablePartitions.size());
int scanLimit = peekScanLimit(count, readablePartitions.size());
int messagesPerPartition = startPosition == PeekStartPosition.LATEST
? latestPeekMessagesPerPartition(count, readablePartitions.size())
: peekMessagesPerPartition(count, readablePartitions.size());
int scanLimit = peekScanLimit(count, readablePartitions.size(), startPosition);
boolean latestBudgetLimited = startPosition == PeekStartPosition.LATEST
&& latestPeekBudgetLimited(count, readablePartitions.size());
Map<TopicPartition, Long> snapshotEndOffsets = new LinkedHashMap<>();
if (startPosition == PeekStartPosition.LATEST) {
for (TopicPartition tp : readablePartitions) {
@@ -1176,6 +1180,7 @@ public final class KafkaAgent {
deadlineNs,
pollTimeout
);
collection.incomplete = collection.incomplete || latestBudgetLimited;
} else {
PeekCollectionCompletionChecker snapshotComplete = state -> allPeekPartitionsComplete(
readablePartitions,
@@ -1194,11 +1199,9 @@ public final class KafkaAgent {
pollTimeout
);
}
List<Map<String, Object>> messages = collection.messages;
sortPeekedMessages(messages, startPosition);
if (messages.size() > count) {
messages = new ArrayList<>(messages.subList(0, count));
}
List<Map<String, Object>> messages = sortAndLimitPeekedMessages(
collection.messages, count, startPosition
);
return peekMessagesResult(messages, collection.incomplete);
}
}
@@ -1311,9 +1314,10 @@ public final class KafkaAgent {
continue;
}
for (ConsumerRecord<String, byte[]> record : records) {
if (++collection.scannedRecords > maxScanRecords) {
if (collection.scannedRecords >= maxScanRecords) {
return false;
}
collection.scannedRecords++;
TopicPartition partition = new TopicPartition(record.topic(), record.partition());
int remaining = collection.remainingByPartition.getOrDefault(partition, 0);
if (remaining <= 0 || !recordFilter.include(record)) {
@@ -1429,10 +1433,11 @@ public final class KafkaAgent {
: pollTimeout;
ConsumerRecords<String, byte[]> records = poller.poll(timeout);
for (ConsumerRecord<String, byte[]> record : records) {
if (++collection.scannedRecords > maxScanRecords) {
if (collection.scannedRecords >= maxScanRecords) {
commitLatestPeekRange(rangeMessages, collection);
return false;
}
collection.scannedRecords++;
TopicPartition partition = new TopicPartition(record.topic(), record.partition());
int remaining = collection.remainingByPartition.getOrDefault(partition, 0);
if (remaining <= 0 || !recordFilter.include(record)) {
@@ -1568,19 +1573,25 @@ public final class KafkaAgent {
Integer partition,
Duration timeout
) {
if (partition != null) {
return resolvePeekPartitions(topic, partition, Collections.emptyList());
}
List<PartitionInfo> infos = consumer.partitionsFor(topic, timeout);
if (infos == null || infos.isEmpty()) {
return Collections.emptyList();
}
List<Integer> available = infos.stream().map(PartitionInfo::partition).collect(Collectors.toList());
return resolvePeekPartitions(topic, null, available);
return resolvePeekPartitions(topic, partition, available);
}
static List<TopicPartition> resolvePeekPartitions(String topic, Integer partition, List<Integer> availablePartitions) {
if (partition != null) {
if (availablePartitions == null || !availablePartitions.contains(partition)) {
String available = availablePartitions == null || availablePartitions.isEmpty()
? "none"
: availablePartitions.stream().sorted().map(String::valueOf).collect(Collectors.joining(", "));
throw new IllegalArgumentException(
"Kafka partition " + partition + " does not exist for topic '" + topic
+ "'. Available partitions: " + available
);
}
return Collections.singletonList(new TopicPartition(topic, partition));
}
if (availablePartitions == null || availablePartitions.isEmpty()) {
@@ -1657,17 +1668,52 @@ public final class KafkaAgent {
}
static int peekScanLimit(int count, int readablePartitionCount) {
int fetchCount = recentPeekFetchCount(
peekMessagesPerPartition(count, readablePartitionCount), readablePartitionCount
);
if (fetchCount > MAX_PEEK_SCAN_RECORDS) {
return peekScanLimit(count, readablePartitionCount, PeekStartPosition.EARLIEST);
}
static int peekScanLimit(int count, int readablePartitionCount, PeekStartPosition startPosition) {
if (startPosition == PeekStartPosition.LATEST
&& readablePartitionCount > MAX_PEEK_SCAN_RECORDS) {
throw new IllegalArgumentException(
"Kafka message browse would scan more than " + MAX_PEEK_SCAN_RECORDS + " records"
"Kafka topic has more than " + MAX_PEEK_SCAN_RECORDS
+ " readable partitions; select a partition to browse latest messages"
);
}
return MAX_PEEK_SCAN_RECORDS;
}
/**
* Latest is a topic-level query. Below the scan budget, every partition contributes the
* requested count so the global merge is exact. Above it, the fixed budget is shared across
* partitions and the response is marked incomplete.
*/
static int latestPeekMessagesPerPartition(int count, int readablePartitionCount) {
int safePartitionCount = Math.max(1, readablePartitionCount);
if (safePartitionCount > MAX_PEEK_SCAN_RECORDS) {
throw new IllegalArgumentException(
"Kafka topic has more than " + MAX_PEEK_SCAN_RECORDS
+ " readable partitions; select a partition to browse latest messages"
);
}
return Math.min(count, MAX_PEEK_SCAN_RECORDS / safePartitionCount);
}
static boolean latestPeekBudgetLimited(int count, int readablePartitionCount) {
return latestPeekMessagesPerPartition(count, readablePartitionCount) < count;
}
static List<Map<String, Object>> sortAndLimitPeekedMessages(
List<Map<String, Object>> messages,
int count,
PeekStartPosition startPosition
) {
sortPeekedMessages(messages, startPosition);
if (messages.size() <= count) {
return messages;
}
return new ArrayList<>(messages.subList(0, count));
}
static void sortPeekedMessages(List<Map<String, Object>> messages) {
sortPeekedMessages(messages, PeekStartPosition.EARLIEST);
}
@@ -1698,7 +1744,9 @@ public final class KafkaAgent {
}
long leftOffset = ((Number) left.getOrDefault("offset", 0L)).longValue();
long rightOffset = ((Number) right.getOrDefault("offset", 0L)).longValue();
return Long.compare(leftOffset, rightOffset);
return startPosition == PeekStartPosition.LATEST
? Long.compare(rightOffset, leftOffset)
: Long.compare(leftOffset, rightOffset);
};
messages.sort(comparator);
}
@@ -2142,11 +2190,6 @@ public final class KafkaAgent {
return currentStartOffset - Math.min(distanceToBeginning, expandedWindowWidth);
}
static int recentPeekFetchCount(int messagesPerPartition, int partitionCount) {
long total = (long) messagesPerPartition * Math.max(1, partitionCount);
return (int) Math.min(Integer.MAX_VALUE, total);
}
private static String stringOrNull(JsonObject object, String key) {
JsonElement element = object.get(key);
return element == null || element.isJsonNull() ? null : element.getAsString();
@@ -568,15 +568,23 @@ class KafkaAgentTest {
}
@Test
void boundsThePerPartitionMessageQuotaBeforeTrimmingTheResult() {
assertEquals(12, KafkaAgent.recentPeekFetchCount(4, 3));
void latestPeekReportsWhetherTheGlobalBudgetCanCoverEveryPartitionQuota() {
assertFalse(KafkaAgent.latestPeekBudgetLimited(20, 50));
assertTrue(KafkaAgent.latestPeekBudgetLimited(20, 51));
assertTrue(KafkaAgent.latestPeekBudgetLimited(100, 11));
assertEquals(4, KafkaAgent.peekMessagesPerPartition(10, 3));
}
@Test
void peekRejectsAWindowThatExceedsTheScanLimit() {
assertThrows(IllegalArgumentException.class, () ->
KafkaAgent.peekScanLimit(100, 1_001));
void latestPeekSharesTheGlobalScanBudgetAcrossManyPartitions() {
assertEquals(20, KafkaAgent.latestPeekMessagesPerPartition(20, 3));
assertEquals(19, KafkaAgent.latestPeekMessagesPerPartition(20, 51));
assertEquals(16, KafkaAgent.latestPeekMessagesPerPartition(20, 60));
assertEquals(10, KafkaAgent.latestPeekMessagesPerPartition(20, 100));
assertEquals(90, KafkaAgent.latestPeekMessagesPerPartition(100, 11));
assertEquals(1_000, KafkaAgent.peekScanLimit(
100, 11, KafkaAgent.PeekStartPosition.LATEST
));
}
@Test
@@ -648,6 +656,25 @@ class KafkaAgentTest {
assertEquals("events", partitions.get(0).topic());
}
@Test
void resolvePeekPartitionsRejectsMissingPartitionBeforeOffsetLookup() {
IllegalArgumentException error = assertThrows(IllegalArgumentException.class, () ->
KafkaAgent.resolvePeekPartitions("events", 5, List.of(0, 1, 2))
);
assertEquals(
"Kafka partition 5 does not exist for topic 'events'. Available partitions: 0, 1, 2",
error.getMessage()
);
}
@Test
void resolvePeekPartitionsKeepsRequestedPartitionAfterMetadataLookup() {
var partitions = KafkaAgent.resolvePeekPartitions("events", 1, List.of(0, 1, 2));
assertEquals(List.of(1), partitions.stream().map(TopicPartition::partition).toList());
}
@Test
void resolvePeekPartitionsUsesAllPartitionsWhenUnspecified() {
var partitions = KafkaAgent.resolvePeekPartitions("events", null, List.of(2, 0, 1));
@@ -680,11 +707,47 @@ class KafkaAgentTest {
assertEquals(20L, messages.get(0).get("timestamp"));
assertEquals(0, messages.get(1).get("partition"));
assertEquals(2L, messages.get(1).get("offset"));
assertEquals(5L, messages.get(2).get("offset"));
assertEquals(5L, messages.get(1).get("offset"));
assertEquals(2L, messages.get(2).get("offset"));
assertEquals(1, messages.get(3).get("partition"));
}
@Test
void latestPeekKeepsTheFullQuotaWhenItFitsTheGlobalScanBudget() {
assertEquals(1_000, KafkaAgent.peekScanLimit(
100, 10, KafkaAgent.PeekStartPosition.LATEST
));
assertEquals(100, KafkaAgent.latestPeekMessagesPerPartition(100, 10));
assertEquals(20, KafkaAgent.latestPeekMessagesPerPartition(20, 50));
}
@Test
void latestPeekRejectsTopicsWhosePartitionCountAloneExceedsTheScanBudget() {
IllegalArgumentException error = assertThrows(IllegalArgumentException.class, () ->
KafkaAgent.latestPeekMessagesPerPartition(20, 1_001)
);
assertTrue(error.getMessage().contains("select a partition"));
}
@Test
void latestPeekGloballyMergesInterleavedPartitionTimelinesBeforeLimiting() {
var messages = new java.util.ArrayList<Map<String, Object>>();
messages.add(Map.of("timestamp", 101L, "partition", 0, "offset", 8L));
messages.add(Map.of("timestamp", 105L, "partition", 1, "offset", 3L));
messages.add(Map.of("timestamp", 103L, "partition", 2, "offset", 9L));
messages.add(Map.of("timestamp", 104L, "partition", 0, "offset", 9L));
messages.add(Map.of("timestamp", 102L, "partition", 1, "offset", 2L));
List<Map<String, Object>> latest = KafkaAgent.sortAndLimitPeekedMessages(
messages, 3, KafkaAgent.PeekStartPosition.LATEST
);
assertEquals(List.of(105L, 104L, 103L), latest.stream()
.map(message -> ((Number) message.get("timestamp")).longValue())
.toList());
}
@Test
void sortPeekedMessagesOrdersOffsetModeByOffsetAscending() {
var messages = new java.util.ArrayList<Map<String, Object>>();