feat(mq): improve Kafka peek and admin localization

This commit is contained in:
Freedom authored and GitHub committed 2026-07-16 18:20:27 +08:00
1 parent a9e76dc130
commit 68b1777aa4
29 files changed
+4561 -636

No files matched your search

@@ -793,9 +793,9 @@ public final class KafkaAgent {
private static Object peekMessages(JsonObject params) throws Exception {
String topic = stringOrEmpty(params, "topic");
int partition = intOrDefault(params, "partition", 0);
long offset = longOrDefault(params, "offset", 0);
int count = intOrDefault(params, "count", 10);
Integer partition = integerOrNull(params, "partition");
Long offset = longOrNull(params, "offset");
int count = Math.max(1, intOrDefault(params, "count", 10));
// Build a temporary consumer for peeking (no commit)
Properties props = new Properties();
@@ -818,52 +818,196 @@ public final class KafkaAgent {
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "none");
props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, count);
TopicPartition tp = new TopicPartition(topic, partition);
try (KafkaConsumer<String, byte[]> consumer = new KafkaConsumer<>(props)) {
consumer.assign(Collections.singletonList(tp));
Map<TopicPartition, Long> beginningOffsets =
consumer.beginningOffsets(Collections.singletonList(tp), Duration.ofSeconds(5));
Map<TopicPartition, Long> endOffsets =
consumer.endOffsets(Collections.singletonList(tp), Duration.ofSeconds(5));
long beginningOffset = beginningOffsets.getOrDefault(tp, 0L);
long endOffset = endOffsets.getOrDefault(tp, beginningOffset);
Long seekOffset = normalizePeekOffset(offset, beginningOffset, endOffset);
if (seekOffset == null) {
List<TopicPartition> candidatePartitions = resolvePeekPartitions(consumer, topic, partition);
if (candidatePartitions.isEmpty()) {
return Collections.singletonMap("messages", Collections.emptyList());
}
consumer.seek(tp, seekOffset);
List<Map<String, Object>> messages = new ArrayList<>();
ConsumerRecords<String, byte[]> records = consumer.poll(Duration.ofSeconds(5));
for (ConsumerRecord<String, byte[]> record : records) {
if (messages.size() >= count) break;
Map<String, Object> msg = new LinkedHashMap<>();
msg.put("topic", record.topic());
msg.put("partition", record.partition());
msg.put("offset", record.offset());
msg.put("timestamp", record.timestamp());
msg.put("key", record.key());
// Headers
Map<String, String> headers = new LinkedHashMap<>();
record.headers().forEach(h ->
headers.put(h.key(), new String(h.value(), StandardCharsets.UTF_8)));
msg.put("headers", headers);
// Payload
if (record.value() != null) {
msg.put("payloadBase64", Base64.getEncoder().encodeToString(record.value()));
String text = tryDecodeUtf8(record.value());
if (text != null) {
msg.put("payloadText", text);
}
} else {
msg.put("payloadBase64", "");
Map<TopicPartition, Long> beginningOffsets =
consumer.beginningOffsets(candidatePartitions, Duration.ofSeconds(5));
Map<TopicPartition, Long> endOffsets =
consumer.endOffsets(candidatePartitions, Duration.ofSeconds(5));
List<TopicPartition> readablePartitions = new ArrayList<>();
Map<TopicPartition, Long> seekOffsets = new LinkedHashMap<>();
for (TopicPartition tp : candidatePartitions) {
long beginningOffset = beginningOffsets.getOrDefault(tp, 0L);
long endOffset = endOffsets.getOrDefault(tp, beginningOffset);
long requestedOffset = offset != null ? offset : beginningOffset;
Long seekOffset = normalizePeekOffset(requestedOffset, beginningOffset, endOffset);
if (seekOffset == null) {
continue;
}
messages.add(msg);
readablePartitions.add(tp);
seekOffsets.put(tp, seekOffset);
}
if (readablePartitions.isEmpty()) {
return Collections.singletonMap("messages", Collections.emptyList());
}
consumer.assign(readablePartitions);
for (Map.Entry<TopicPartition, Long> entry : seekOffsets.entrySet()) {
consumer.seek(entry.getKey(), entry.getValue());
}
List<Map<String, Object>> messages = collectPeekedMessages(
timeout -> consumer.poll(timeout),
() -> {
Map<TopicPartition, Long> positions = new LinkedHashMap<>();
for (TopicPartition tp : readablePartitions) {
positions.put(tp, consumer.position(tp));
}
return allPeekPartitionsCaughtUp(readablePartitions, positions, endOffsets);
},
count,
System.nanoTime() + Duration.ofSeconds(5).toNanos(),
Duration.ofMillis(500)
);
sortPeekedMessages(messages);
if (messages.size() > count) {
messages = new ArrayList<>(messages.subList(0, count));
}
return Collections.singletonMap("messages", messages);
}
}
/**
* Poll until {@code count} messages are collected, every assigned partition has reached its
* end offset, or {@code deadlineNs} expires. Empty polls retry until caught-up or deadline —
* they must not abort early (broker / network / first-fetch latency can exceed one poll).
*/
static List<Map<String, Object>> collectPeekedMessages(
PeekRecordPoller poller,
PeekCaughtUpChecker caughtUpChecker,
int count,
long deadlineNs,
Duration pollTimeout
) {
List<Map<String, Object>> messages = new ArrayList<>();
while (messages.size() < count && System.nanoTime() < deadlineNs) {
long remainingNs = deadlineNs - System.nanoTime();
if (remainingNs <= 0) {
break;
}
Duration timeout = pollTimeout.toNanos() > remainingNs
? Duration.ofNanos(remainingNs)
: pollTimeout;
ConsumerRecords<String, byte[]> records = poller.poll(timeout);
if (records.isEmpty()) {
if (caughtUpChecker.allPartitionsCaughtUp()) {
break;
}
continue;
}
for (ConsumerRecord<String, byte[]> record : records) {
messages.add(peekedMessageFromRecord(record));
if (messages.size() >= count) {
break;
}
}
}
return messages;
}
static boolean allPeekPartitionsCaughtUp(
List<TopicPartition> partitions,
Map<TopicPartition, Long> positions,
Map<TopicPartition, Long> endOffsets
) {
for (TopicPartition tp : partitions) {
long endOffset = endOffsets.getOrDefault(tp, 0L);
long position = positions.getOrDefault(tp, 0L);
if (position < endOffset) {
return false;
}
}
return true;
}
@FunctionalInterface
interface PeekRecordPoller {
ConsumerRecords<String, byte[]> poll(Duration timeout);
}
@FunctionalInterface
interface PeekCaughtUpChecker {
boolean allPartitionsCaughtUp();
}
/** When partition is null, peek across every partition of the topic. */
static List<TopicPartition> resolvePeekPartitions(
KafkaConsumer<String, byte[]> consumer,
String topic,
Integer partition
) {
if (partition != null) {
return resolvePeekPartitions(topic, partition, Collections.emptyList());
}
List<PartitionInfo> infos = consumer.partitionsFor(topic, Duration.ofSeconds(5));
if (infos == null || infos.isEmpty()) {
return Collections.emptyList();
}
List<Integer> available = infos.stream().map(PartitionInfo::partition).collect(Collectors.toList());
return resolvePeekPartitions(topic, null, available);
}
static List<TopicPartition> resolvePeekPartitions(String topic, Integer partition, List<Integer> availablePartitions) {
if (partition != null) {
return Collections.singletonList(new TopicPartition(topic, partition));
}
if (availablePartitions == null || availablePartitions.isEmpty()) {
return Collections.emptyList();
}
return availablePartitions.stream()
.sorted()
.map(id -> new TopicPartition(topic, id))
.collect(Collectors.toList());
}
static void sortPeekedMessages(List<Map<String, Object>> messages) {
messages.sort((left, right) -> {
long leftTs = ((Number) left.getOrDefault("timestamp", 0L)).longValue();
long rightTs = ((Number) right.getOrDefault("timestamp", 0L)).longValue();
int byTs = Long.compare(leftTs, rightTs);
if (byTs != 0) {
return byTs;
}
int leftPartition = ((Number) left.getOrDefault("partition", 0)).intValue();
int rightPartition = ((Number) right.getOrDefault("partition", 0)).intValue();
int byPartition = Integer.compare(leftPartition, rightPartition);
if (byPartition != 0) {
return byPartition;
}
long leftOffset = ((Number) left.getOrDefault("offset", 0L)).longValue();
long rightOffset = ((Number) right.getOrDefault("offset", 0L)).longValue();
return Long.compare(leftOffset, rightOffset);
});
}
private static Map<String, Object> peekedMessageFromRecord(ConsumerRecord<String, byte[]> record) {
Map<String, Object> msg = new LinkedHashMap<>();
msg.put("topic", record.topic());
msg.put("partition", record.partition());
msg.put("offset", record.offset());
msg.put("timestamp", record.timestamp());
msg.put("key", record.key());
Map<String, String> headers = new LinkedHashMap<>();
record.headers().forEach(h ->
headers.put(h.key(), new String(h.value(), StandardCharsets.UTF_8)));
msg.put("headers", headers);
if (record.value() != null) {
msg.put("payloadBase64", Base64.getEncoder().encodeToString(record.value()));
String text = tryDecodeUtf8(record.value());
if (text != null) {
msg.put("payloadText", text);
}
} else {
msg.put("payloadBase64", "");
}
return msg;
}
private static Object sendMessage(JsonObject params) throws Exception {
if (producer == null) {
throw new IllegalStateException("Producer is not initialized. Call connect first.");
@@ -1229,14 +1373,24 @@ public final class KafkaAgent {
return value == null ? fallback : value;
}
private static int intOrDefault(JsonObject object, String key, int fallback) {
private static Integer integerOrNull(JsonObject object, String key) {
JsonElement element = object.get(key);
return element == null || element.isJsonNull() ? fallback : element.getAsInt();
return element == null || element.isJsonNull() ? null : element.getAsInt();
}
private static Long longOrNull(JsonObject object, String key) {
JsonElement element = object.get(key);
return element == null || element.isJsonNull() ? null : element.getAsLong();
}
private static int intOrDefault(JsonObject object, String key, int fallback) {
Integer value = integerOrNull(object, key);
return value == null ? fallback : value;
}
private static long longOrDefault(JsonObject object, String key, long fallback) {
JsonElement element = object.get(key);
return element == null || element.isJsonNull() ? fallback : element.getAsLong();
Long value = longOrNull(object, key);
return value == null ? fallback : value;
}
private static boolean boolOrDefault(JsonObject object, String key, boolean fallback) {
@@ -1,11 +1,21 @@
package com.dbx.agent.kafka;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import com.google.gson.JsonParser;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.common.TopicPartition;
import org.junit.jupiter.api.Test;
class KafkaAgentTest {
@@ -34,6 +44,99 @@ class KafkaAgentTest {
assertNull(KafkaAgent.normalizePeekOffset(0, 5, 5));
}
@Test
void resolvePeekPartitionsUsesSinglePartitionWhenSpecified() {
var partitions = KafkaAgent.resolvePeekPartitions("events", 2, List.of(0, 1, 2));
assertEquals(1, partitions.size());
assertEquals(2, partitions.get(0).partition());
assertEquals("events", partitions.get(0).topic());
}
@Test
void resolvePeekPartitionsUsesAllPartitionsWhenUnspecified() {
var partitions = KafkaAgent.resolvePeekPartitions("events", null, List.of(2, 0, 1));
assertEquals(List.of(0, 1, 2), partitions.stream().map(org.apache.kafka.common.TopicPartition::partition).toList());
}
@Test
void sortPeekedMessagesOrdersByTimestampThenPartitionThenOffset() {
var messages = new java.util.ArrayList<Map<String, Object>>();
messages.add(Map.of("timestamp", 20L, "partition", 1, "offset", 1L));
messages.add(Map.of("timestamp", 10L, "partition", 0, "offset", 5L));
messages.add(Map.of("timestamp", 10L, "partition", 0, "offset", 2L));
messages.add(Map.of("timestamp", 10L, "partition", 1, "offset", 0L));
KafkaAgent.sortPeekedMessages(messages);
assertEquals(2L, messages.get(0).get("offset"));
assertEquals(5L, messages.get(1).get("offset"));
assertEquals(1, messages.get(2).get("partition"));
assertEquals(20L, messages.get(3).get("timestamp"));
}
@Test
void allPeekPartitionsCaughtUpRequiresEveryPartitionAtEndOffset() {
TopicPartition p0 = new TopicPartition("events", 0);
TopicPartition p1 = new TopicPartition("events", 1);
Map<TopicPartition, Long> endOffsets = Map.of(p0, 10L, p1, 5L);
assertFalse(KafkaAgent.allPeekPartitionsCaughtUp(
List.of(p0, p1),
Map.of(p0, 10L, p1, 4L),
endOffsets
));
assertTrue(KafkaAgent.allPeekPartitionsCaughtUp(
List.of(p0, p1),
Map.of(p0, 10L, p1, 5L),
endOffsets
));
}
@Test
void collectPeekedMessagesRetriesAfterEmptyFirstPoll() {
TopicPartition tp = new TopicPartition("events", 0);
ConsumerRecord<String, byte[]> record = new ConsumerRecord<>(
"events",
0,
7L,
"k",
"hello".getBytes(StandardCharsets.UTF_8)
);
Map<TopicPartition, List<ConsumerRecord<String, byte[]>>> batch = new HashMap<>();
batch.put(tp, List.of(record));
ConsumerRecords<String, byte[]> withData = new ConsumerRecords<>(batch);
AtomicInteger polls = new AtomicInteger();
List<Map<String, Object>> messages = KafkaAgent.collectPeekedMessages(
timeout -> polls.getAndIncrement() == 0 ? ConsumerRecords.empty() : withData,
() -> false,
1,
System.nanoTime() + Duration.ofSeconds(5).toNanos(),
Duration.ofMillis(1)
);
assertEquals(2, polls.get());
assertEquals(1, messages.size());
assertEquals(7L, messages.get(0).get("offset"));
assertEquals("hello", messages.get(0).get("payloadText"));
}
@Test
void collectPeekedMessagesStopsOnEmptyPollWhenCaughtUp() {
AtomicInteger polls = new AtomicInteger();
List<Map<String, Object>> messages = KafkaAgent.collectPeekedMessages(
timeout -> {
polls.incrementAndGet();
return ConsumerRecords.empty();
},
() -> true,
10,
System.nanoTime() + Duration.ofSeconds(5).toNanos(),
Duration.ofMillis(1)
);
assertEquals(1, polls.get());
assertTrue(messages.isEmpty());
}
@Test
void appliesKerberosKafkaProperties() {
Properties props = new Properties();