fix(kafka): improve message browsing and topic navigation
This commit is contained in:
1 parent
f6518ff3a0
commit
e77b547cea
24 files changed
+751
-114
No files matched your search
@@ -1140,8 +1140,12 @@ public final class KafkaAgent {
|
||||
return peekMessagesResult(Collections.emptyList(), false);
|
||||
}
|
||||
|
||||
int messagesPerPartition = peekMessagesPerPartition(count, readablePartitions.size());
|
||||
int scanLimit = peekScanLimit(count, readablePartitions.size());
|
||||
int messagesPerPartition = startPosition == PeekStartPosition.LATEST
|
||||
? latestPeekMessagesPerPartition(count, readablePartitions.size())
|
||||
: peekMessagesPerPartition(count, readablePartitions.size());
|
||||
int scanLimit = peekScanLimit(count, readablePartitions.size(), startPosition);
|
||||
boolean latestBudgetLimited = startPosition == PeekStartPosition.LATEST
|
||||
&& latestPeekBudgetLimited(count, readablePartitions.size());
|
||||
Map<TopicPartition, Long> snapshotEndOffsets = new LinkedHashMap<>();
|
||||
if (startPosition == PeekStartPosition.LATEST) {
|
||||
for (TopicPartition tp : readablePartitions) {
|
||||
@@ -1176,6 +1180,7 @@ public final class KafkaAgent {
|
||||
deadlineNs,
|
||||
pollTimeout
|
||||
);
|
||||
collection.incomplete = collection.incomplete || latestBudgetLimited;
|
||||
} else {
|
||||
PeekCollectionCompletionChecker snapshotComplete = state -> allPeekPartitionsComplete(
|
||||
readablePartitions,
|
||||
@@ -1194,11 +1199,9 @@ public final class KafkaAgent {
|
||||
pollTimeout
|
||||
);
|
||||
}
|
||||
List<Map<String, Object>> messages = collection.messages;
|
||||
sortPeekedMessages(messages, startPosition);
|
||||
if (messages.size() > count) {
|
||||
messages = new ArrayList<>(messages.subList(0, count));
|
||||
}
|
||||
List<Map<String, Object>> messages = sortAndLimitPeekedMessages(
|
||||
collection.messages, count, startPosition
|
||||
);
|
||||
return peekMessagesResult(messages, collection.incomplete);
|
||||
}
|
||||
}
|
||||
@@ -1311,9 +1314,10 @@ public final class KafkaAgent {
|
||||
continue;
|
||||
}
|
||||
for (ConsumerRecord<String, byte[]> record : records) {
|
||||
if (++collection.scannedRecords > maxScanRecords) {
|
||||
if (collection.scannedRecords >= maxScanRecords) {
|
||||
return false;
|
||||
}
|
||||
collection.scannedRecords++;
|
||||
TopicPartition partition = new TopicPartition(record.topic(), record.partition());
|
||||
int remaining = collection.remainingByPartition.getOrDefault(partition, 0);
|
||||
if (remaining <= 0 || !recordFilter.include(record)) {
|
||||
@@ -1429,10 +1433,11 @@ public final class KafkaAgent {
|
||||
: pollTimeout;
|
||||
ConsumerRecords<String, byte[]> records = poller.poll(timeout);
|
||||
for (ConsumerRecord<String, byte[]> record : records) {
|
||||
if (++collection.scannedRecords > maxScanRecords) {
|
||||
if (collection.scannedRecords >= maxScanRecords) {
|
||||
commitLatestPeekRange(rangeMessages, collection);
|
||||
return false;
|
||||
}
|
||||
collection.scannedRecords++;
|
||||
TopicPartition partition = new TopicPartition(record.topic(), record.partition());
|
||||
int remaining = collection.remainingByPartition.getOrDefault(partition, 0);
|
||||
if (remaining <= 0 || !recordFilter.include(record)) {
|
||||
@@ -1568,19 +1573,25 @@ public final class KafkaAgent {
|
||||
Integer partition,
|
||||
Duration timeout
|
||||
) {
|
||||
if (partition != null) {
|
||||
return resolvePeekPartitions(topic, partition, Collections.emptyList());
|
||||
}
|
||||
List<PartitionInfo> infos = consumer.partitionsFor(topic, timeout);
|
||||
if (infos == null || infos.isEmpty()) {
|
||||
return Collections.emptyList();
|
||||
}
|
||||
List<Integer> available = infos.stream().map(PartitionInfo::partition).collect(Collectors.toList());
|
||||
return resolvePeekPartitions(topic, null, available);
|
||||
return resolvePeekPartitions(topic, partition, available);
|
||||
}
|
||||
|
||||
static List<TopicPartition> resolvePeekPartitions(String topic, Integer partition, List<Integer> availablePartitions) {
|
||||
if (partition != null) {
|
||||
if (availablePartitions == null || !availablePartitions.contains(partition)) {
|
||||
String available = availablePartitions == null || availablePartitions.isEmpty()
|
||||
? "none"
|
||||
: availablePartitions.stream().sorted().map(String::valueOf).collect(Collectors.joining(", "));
|
||||
throw new IllegalArgumentException(
|
||||
"Kafka partition " + partition + " does not exist for topic '" + topic
|
||||
+ "'. Available partitions: " + available
|
||||
);
|
||||
}
|
||||
return Collections.singletonList(new TopicPartition(topic, partition));
|
||||
}
|
||||
if (availablePartitions == null || availablePartitions.isEmpty()) {
|
||||
@@ -1657,17 +1668,52 @@ public final class KafkaAgent {
|
||||
}
|
||||
|
||||
static int peekScanLimit(int count, int readablePartitionCount) {
|
||||
int fetchCount = recentPeekFetchCount(
|
||||
peekMessagesPerPartition(count, readablePartitionCount), readablePartitionCount
|
||||
);
|
||||
if (fetchCount > MAX_PEEK_SCAN_RECORDS) {
|
||||
return peekScanLimit(count, readablePartitionCount, PeekStartPosition.EARLIEST);
|
||||
}
|
||||
|
||||
static int peekScanLimit(int count, int readablePartitionCount, PeekStartPosition startPosition) {
|
||||
if (startPosition == PeekStartPosition.LATEST
|
||||
&& readablePartitionCount > MAX_PEEK_SCAN_RECORDS) {
|
||||
throw new IllegalArgumentException(
|
||||
"Kafka message browse would scan more than " + MAX_PEEK_SCAN_RECORDS + " records"
|
||||
"Kafka topic has more than " + MAX_PEEK_SCAN_RECORDS
|
||||
+ " readable partitions; select a partition to browse latest messages"
|
||||
);
|
||||
}
|
||||
return MAX_PEEK_SCAN_RECORDS;
|
||||
}
|
||||
|
||||
/**
|
||||
* Latest is a topic-level query. Below the scan budget, every partition contributes the
|
||||
* requested count so the global merge is exact. Above it, the fixed budget is shared across
|
||||
* partitions and the response is marked incomplete.
|
||||
*/
|
||||
static int latestPeekMessagesPerPartition(int count, int readablePartitionCount) {
|
||||
int safePartitionCount = Math.max(1, readablePartitionCount);
|
||||
if (safePartitionCount > MAX_PEEK_SCAN_RECORDS) {
|
||||
throw new IllegalArgumentException(
|
||||
"Kafka topic has more than " + MAX_PEEK_SCAN_RECORDS
|
||||
+ " readable partitions; select a partition to browse latest messages"
|
||||
);
|
||||
}
|
||||
return Math.min(count, MAX_PEEK_SCAN_RECORDS / safePartitionCount);
|
||||
}
|
||||
|
||||
static boolean latestPeekBudgetLimited(int count, int readablePartitionCount) {
|
||||
return latestPeekMessagesPerPartition(count, readablePartitionCount) < count;
|
||||
}
|
||||
|
||||
static List<Map<String, Object>> sortAndLimitPeekedMessages(
|
||||
List<Map<String, Object>> messages,
|
||||
int count,
|
||||
PeekStartPosition startPosition
|
||||
) {
|
||||
sortPeekedMessages(messages, startPosition);
|
||||
if (messages.size() <= count) {
|
||||
return messages;
|
||||
}
|
||||
return new ArrayList<>(messages.subList(0, count));
|
||||
}
|
||||
|
||||
static void sortPeekedMessages(List<Map<String, Object>> messages) {
|
||||
sortPeekedMessages(messages, PeekStartPosition.EARLIEST);
|
||||
}
|
||||
@@ -1698,7 +1744,9 @@ public final class KafkaAgent {
|
||||
}
|
||||
long leftOffset = ((Number) left.getOrDefault("offset", 0L)).longValue();
|
||||
long rightOffset = ((Number) right.getOrDefault("offset", 0L)).longValue();
|
||||
return Long.compare(leftOffset, rightOffset);
|
||||
return startPosition == PeekStartPosition.LATEST
|
||||
? Long.compare(rightOffset, leftOffset)
|
||||
: Long.compare(leftOffset, rightOffset);
|
||||
};
|
||||
messages.sort(comparator);
|
||||
}
|
||||
@@ -2142,11 +2190,6 @@ public final class KafkaAgent {
|
||||
return currentStartOffset - Math.min(distanceToBeginning, expandedWindowWidth);
|
||||
}
|
||||
|
||||
static int recentPeekFetchCount(int messagesPerPartition, int partitionCount) {
|
||||
long total = (long) messagesPerPartition * Math.max(1, partitionCount);
|
||||
return (int) Math.min(Integer.MAX_VALUE, total);
|
||||
}
|
||||
|
||||
private static String stringOrNull(JsonObject object, String key) {
|
||||
JsonElement element = object.get(key);
|
||||
return element == null || element.isJsonNull() ? null : element.getAsString();
|
||||
|
||||
@@ -568,15 +568,23 @@ class KafkaAgentTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void boundsThePerPartitionMessageQuotaBeforeTrimmingTheResult() {
|
||||
assertEquals(12, KafkaAgent.recentPeekFetchCount(4, 3));
|
||||
void latestPeekReportsWhetherTheGlobalBudgetCanCoverEveryPartitionQuota() {
|
||||
assertFalse(KafkaAgent.latestPeekBudgetLimited(20, 50));
|
||||
assertTrue(KafkaAgent.latestPeekBudgetLimited(20, 51));
|
||||
assertTrue(KafkaAgent.latestPeekBudgetLimited(100, 11));
|
||||
assertEquals(4, KafkaAgent.peekMessagesPerPartition(10, 3));
|
||||
}
|
||||
|
||||
@Test
|
||||
void peekRejectsAWindowThatExceedsTheScanLimit() {
|
||||
assertThrows(IllegalArgumentException.class, () ->
|
||||
KafkaAgent.peekScanLimit(100, 1_001));
|
||||
void latestPeekSharesTheGlobalScanBudgetAcrossManyPartitions() {
|
||||
assertEquals(20, KafkaAgent.latestPeekMessagesPerPartition(20, 3));
|
||||
assertEquals(19, KafkaAgent.latestPeekMessagesPerPartition(20, 51));
|
||||
assertEquals(16, KafkaAgent.latestPeekMessagesPerPartition(20, 60));
|
||||
assertEquals(10, KafkaAgent.latestPeekMessagesPerPartition(20, 100));
|
||||
assertEquals(90, KafkaAgent.latestPeekMessagesPerPartition(100, 11));
|
||||
assertEquals(1_000, KafkaAgent.peekScanLimit(
|
||||
100, 11, KafkaAgent.PeekStartPosition.LATEST
|
||||
));
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -648,6 +656,25 @@ class KafkaAgentTest {
|
||||
assertEquals("events", partitions.get(0).topic());
|
||||
}
|
||||
|
||||
@Test
|
||||
void resolvePeekPartitionsRejectsMissingPartitionBeforeOffsetLookup() {
|
||||
IllegalArgumentException error = assertThrows(IllegalArgumentException.class, () ->
|
||||
KafkaAgent.resolvePeekPartitions("events", 5, List.of(0, 1, 2))
|
||||
);
|
||||
|
||||
assertEquals(
|
||||
"Kafka partition 5 does not exist for topic 'events'. Available partitions: 0, 1, 2",
|
||||
error.getMessage()
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void resolvePeekPartitionsKeepsRequestedPartitionAfterMetadataLookup() {
|
||||
var partitions = KafkaAgent.resolvePeekPartitions("events", 1, List.of(0, 1, 2));
|
||||
|
||||
assertEquals(List.of(1), partitions.stream().map(TopicPartition::partition).toList());
|
||||
}
|
||||
|
||||
@Test
|
||||
void resolvePeekPartitionsUsesAllPartitionsWhenUnspecified() {
|
||||
var partitions = KafkaAgent.resolvePeekPartitions("events", null, List.of(2, 0, 1));
|
||||
@@ -680,11 +707,47 @@ class KafkaAgentTest {
|
||||
|
||||
assertEquals(20L, messages.get(0).get("timestamp"));
|
||||
assertEquals(0, messages.get(1).get("partition"));
|
||||
assertEquals(2L, messages.get(1).get("offset"));
|
||||
assertEquals(5L, messages.get(2).get("offset"));
|
||||
assertEquals(5L, messages.get(1).get("offset"));
|
||||
assertEquals(2L, messages.get(2).get("offset"));
|
||||
assertEquals(1, messages.get(3).get("partition"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void latestPeekKeepsTheFullQuotaWhenItFitsTheGlobalScanBudget() {
|
||||
assertEquals(1_000, KafkaAgent.peekScanLimit(
|
||||
100, 10, KafkaAgent.PeekStartPosition.LATEST
|
||||
));
|
||||
assertEquals(100, KafkaAgent.latestPeekMessagesPerPartition(100, 10));
|
||||
assertEquals(20, KafkaAgent.latestPeekMessagesPerPartition(20, 50));
|
||||
}
|
||||
|
||||
@Test
|
||||
void latestPeekRejectsTopicsWhosePartitionCountAloneExceedsTheScanBudget() {
|
||||
IllegalArgumentException error = assertThrows(IllegalArgumentException.class, () ->
|
||||
KafkaAgent.latestPeekMessagesPerPartition(20, 1_001)
|
||||
);
|
||||
|
||||
assertTrue(error.getMessage().contains("select a partition"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void latestPeekGloballyMergesInterleavedPartitionTimelinesBeforeLimiting() {
|
||||
var messages = new java.util.ArrayList<Map<String, Object>>();
|
||||
messages.add(Map.of("timestamp", 101L, "partition", 0, "offset", 8L));
|
||||
messages.add(Map.of("timestamp", 105L, "partition", 1, "offset", 3L));
|
||||
messages.add(Map.of("timestamp", 103L, "partition", 2, "offset", 9L));
|
||||
messages.add(Map.of("timestamp", 104L, "partition", 0, "offset", 9L));
|
||||
messages.add(Map.of("timestamp", 102L, "partition", 1, "offset", 2L));
|
||||
|
||||
List<Map<String, Object>> latest = KafkaAgent.sortAndLimitPeekedMessages(
|
||||
messages, 3, KafkaAgent.PeekStartPosition.LATEST
|
||||
);
|
||||
|
||||
assertEquals(List.of(105L, 104L, 103L), latest.stream()
|
||||
.map(message -> ((Number) message.get("timestamp")).longValue())
|
||||
.toList());
|
||||
}
|
||||
|
||||
@Test
|
||||
void sortPeekedMessagesOrdersOffsetModeByOffsetAscending() {
|
||||
var messages = new java.util.ArrayList<Map<String, Object>>();
|
||||
|
||||
Reference in new issue
Block a user