feat(kafka): redesign message browsing

This commit is contained in:
onenewcode authored and GitHub committed 2026-08-04 13:13:49 +08:00
1 parent 2e57ac8bc0
commit ae87353a28
32 files changed
+2776 -624

No files matched your search

@@ -41,6 +41,8 @@ public final class KafkaAgent {
private static final PrintStream JSON_RPC_OUT = System.out; private static final PrintStream JSON_RPC_OUT = System.out;
private static final Gson GSON = new GsonBuilder().serializeNulls().create(); private static final Gson GSON = new GsonBuilder().serializeNulls().create();
private static final int DEFAULT_REQUEST_TIMEOUT_MS = 30_000; private static final int DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
private static final int MAX_PEEK_MESSAGE_COUNT = 100;
private static final int MAX_PEEK_SCAN_RECORDS = 1_000;
private static final int DEFAULT_SESSION_TIMEOUT_MS = 30_000; private static final int DEFAULT_SESSION_TIMEOUT_MS = 30_000;
private static final int DEFAULT_ZOOKEEPER_CONNECTION_TIMEOUT_MS = 10_000; private static final int DEFAULT_ZOOKEEPER_CONNECTION_TIMEOUT_MS = 10_000;
private static final String ZOOKEEPER_PROPERTY_PREFIX = "zookeeper."; private static final String ZOOKEEPER_PROPERTY_PREFIX = "zookeeper.";
@@ -1025,31 +1027,43 @@ public final class KafkaAgent {
String topic = stringOrEmpty(params, "topic"); String topic = stringOrEmpty(params, "topic");
Integer partition = integerOrNull(params, "partition"); Integer partition = integerOrNull(params, "partition");
Long offset = longOrNull(params, "offset"); Long offset = longOrNull(params, "offset");
int count = Math.max(1, intOrDefault(params, "count", 10)); int count = validatedPeekCount(intOrDefault(params, "count", 10));
PeekStartPosition startPosition = peekStartPosition(params);
boolean explicitStartPosition = stringOrNull(params, "startPosition") != null;
validatePeekRequest(startPosition, explicitStartPosition, partition, offset);
boolean legacyOffsetRequest = !explicitStartPosition && offset != null;
JsonObject conn = activeConnection; JsonObject conn = activeConnection;
if (conn == null) { if (conn == null) {
throw new IllegalStateException("Kafka Agent is not connected"); throw new IllegalStateException("Kafka Agent is not connected");
} }
Properties props = peekConsumerProperties(conn, count); Properties props = peekConsumerProperties(conn, count);
Duration requestTimeout = Duration.ofMillis(peekRequestTimeoutMs(conn, props));
try (KafkaConsumer<String, byte[]> consumer = new KafkaConsumer<>(props)) { try (KafkaConsumer<String, byte[]> consumer = new KafkaConsumer<>(props)) {
List<TopicPartition> candidatePartitions = resolvePeekPartitions(consumer, topic, partition); List<TopicPartition> candidatePartitions = resolvePeekPartitions(
consumer, topic, partition, requestTimeout
);
if (candidatePartitions.isEmpty()) { if (candidatePartitions.isEmpty()) {
return Collections.singletonMap("messages", Collections.emptyList()); return peekMessagesResult(Collections.emptyList(), false);
} }
Map<TopicPartition, Long> beginningOffsets = Map<TopicPartition, Long> beginningOffsets =
consumer.beginningOffsets(candidatePartitions, Duration.ofSeconds(5)); consumer.beginningOffsets(candidatePartitions, requestTimeout);
Map<TopicPartition, Long> endOffsets = Map<TopicPartition, Long> endOffsets =
consumer.endOffsets(candidatePartitions, Duration.ofSeconds(5)); consumer.endOffsets(candidatePartitions, requestTimeout);
List<TopicPartition> readablePartitions = new ArrayList<>(); List<TopicPartition> readablePartitions = new ArrayList<>();
Map<TopicPartition, Long> seekOffsets = new LinkedHashMap<>(); Map<TopicPartition, Long> seekOffsets = new LinkedHashMap<>();
for (TopicPartition tp : candidatePartitions) { for (TopicPartition tp : candidatePartitions) {
long beginningOffset = beginningOffsets.getOrDefault(tp, 0L); long beginningOffset = beginningOffsets.getOrDefault(tp, 0L);
long endOffset = endOffsets.getOrDefault(tp, beginningOffset); long endOffset = endOffsets.getOrDefault(tp, beginningOffset);
long requestedOffset = offset != null ? offset : beginningOffset; Long requestedOffset = requestedPeekOffset(
startPosition, offset, legacyOffsetRequest, beginningOffset, endOffset
);
if (requestedOffset == null) {
continue;
}
Long seekOffset = normalizePeekOffset(requestedOffset, beginningOffset, endOffset); Long seekOffset = normalizePeekOffset(requestedOffset, beginningOffset, endOffset);
if (seekOffset == null) { if (seekOffset == null) {
continue; continue;
@@ -1058,7 +1072,23 @@ public final class KafkaAgent {
seekOffsets.put(tp, seekOffset); seekOffsets.put(tp, seekOffset);
} }
if (readablePartitions.isEmpty()) { if (readablePartitions.isEmpty()) {
return Collections.singletonMap("messages", Collections.emptyList()); return peekMessagesResult(Collections.emptyList(), false);
}
int messagesPerPartition = peekMessagesPerPartition(count, readablePartitions.size());
int scanLimit = peekScanLimit(count, readablePartitions.size());
Map<TopicPartition, Long> snapshotEndOffsets = new LinkedHashMap<>();
if (startPosition == PeekStartPosition.LATEST) {
for (TopicPartition tp : readablePartitions) {
long beginningOffset = beginningOffsets.getOrDefault(tp, 0L);
long endOffset = endOffsets.getOrDefault(tp, beginningOffset);
seekOffsets.put(tp, recentPeekStartOffset(
beginningOffset, endOffset, messagesPerPartition
));
}
}
for (TopicPartition tp : readablePartitions) {
snapshotEndOffsets.put(tp, endOffsets.getOrDefault(tp, 0L));
} }
consumer.assign(readablePartitions); consumer.assign(readablePartitions);
@@ -1066,27 +1096,89 @@ public final class KafkaAgent {
consumer.seek(entry.getKey(), entry.getValue()); consumer.seek(entry.getKey(), entry.getValue());
} }
List<Map<String, Object>> messages = collectPeekedMessages( long deadlineNs = System.nanoTime() + requestTimeout.toNanos();
timeout -> consumer.poll(timeout), Duration pollTimeout = Duration.ofMillis(Math.min(500L, requestTimeout.toMillis()));
() -> { PeekCollectionState collection;
Map<TopicPartition, Long> positions = new LinkedHashMap<>(); if (startPosition == PeekStartPosition.LATEST) {
for (TopicPartition tp : readablePartitions) { collection = collectLatestPeekedMessages(
positions.put(tp, consumer.position(tp)); consumer,
} readablePartitions,
return allPeekPartitionsCaughtUp(readablePartitions, positions, endOffsets); beginningOffsets,
}, seekOffsets,
count, snapshotEndOffsets,
System.nanoTime() + Duration.ofSeconds(5).toNanos(), messagesPerPartition,
Duration.ofMillis(500) scanLimit,
); deadlineNs,
sortPeekedMessages(messages); pollTimeout
);
} else {
PeekCollectionCompletionChecker snapshotComplete = state -> allPeekPartitionsComplete(
readablePartitions,
state.remainingByPartition,
currentPeekPositions(consumer, readablePartitions),
snapshotEndOffsets
);
collection = new PeekCollectionState(readablePartitions, messagesPerPartition);
collection.incomplete = !collectPeekedMessages(
timeout -> consumer.poll(timeout),
snapshotComplete,
record -> recordIsBeforeEndOffset(record, snapshotEndOffsets),
collection,
scanLimit,
deadlineNs,
pollTimeout
);
}
List<Map<String, Object>> messages = collection.messages;
sortPeekedMessages(messages, startPosition);
if (messages.size() > count) { if (messages.size() > count) {
messages = new ArrayList<>(messages.subList(0, count)); messages = new ArrayList<>(messages.subList(0, count));
} }
return Collections.singletonMap("messages", messages); return peekMessagesResult(messages, collection.incomplete);
} }
} }
static Map<String, Object> peekMessagesResult(List<Map<String, Object>> messages, boolean incomplete) {
Map<String, Object> result = new LinkedHashMap<>();
result.put("messages", messages);
result.put("incomplete", incomplete);
return result;
}
static int validatedPeekCount(int count) {
if (count < 1 || count > MAX_PEEK_MESSAGE_COUNT) {
throw new IllegalArgumentException(
"Peek message count must be between 1 and " + MAX_PEEK_MESSAGE_COUNT
);
}
return count;
}
static int peekRequestTimeoutMs(JsonObject conn, Properties props) {
Integer connectionTimeout = integerOrNull(conn, "request_timeout_ms");
if (connectionTimeout != null) {
return positiveTimeoutMs("request_timeout_ms", connectionTimeout);
}
String configuredTimeout = props.getProperty(
ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG,
String.valueOf(DEFAULT_REQUEST_TIMEOUT_MS)
);
try {
return positiveTimeoutMs(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG, Integer.parseInt(configuredTimeout));
} catch (NumberFormatException error) {
throw new IllegalArgumentException(
ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG + " must be a positive integer", error
);
}
}
private static int positiveTimeoutMs(String name, int timeoutMs) {
if (timeoutMs <= 0) {
throw new IllegalArgumentException(name + " must be a positive integer");
}
return timeoutMs;
}
static Properties peekConsumerProperties(JsonObject conn, int count) { static Properties peekConsumerProperties(JsonObject conn, int count) {
Properties props = new Properties(); Properties props = new Properties();
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers(conn)); props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers(conn));
@@ -1103,19 +1195,45 @@ public final class KafkaAgent {
} }
/** /**
* Poll until {@code count} messages are collected, every assigned partition has reached its * Reads until each partition supplies its share of the page or reaches the snapshot boundary.
* end offset, or {@code deadlineNs} expires. Empty polls retry until caught-up or deadline — * This counts retained records, rather than treating an offset range as a record count.
* they must not abort early (broker / network / first-fetch latency can exceed one poll).
*/ */
static List<Map<String, Object>> collectPeekedMessages( static List<Map<String, Object>> collectPeekedMessages(
PeekRecordPoller poller, PeekRecordPoller poller,
PeekCaughtUpChecker caughtUpChecker, PeekCaughtUpChecker caughtUpChecker,
int count, PeekRecordFilter recordFilter,
List<TopicPartition> partitions,
int messagesPerPartition,
int maxScanRecords,
long deadlineNs, long deadlineNs,
Duration pollTimeout Duration pollTimeout
) { ) {
List<Map<String, Object>> messages = new ArrayList<>(); PeekCollectionState collection = new PeekCollectionState(partitions, messagesPerPartition);
while (messages.size() < count && System.nanoTime() < deadlineNs) { collectPeekedMessages(
poller,
state -> state.allPartitionQuotasSatisfied() || caughtUpChecker.allPartitionsCaughtUp(),
recordFilter,
collection,
maxScanRecords,
deadlineNs,
pollTimeout
);
return collection.messages;
}
private static boolean collectPeekedMessages(
PeekRecordPoller poller,
PeekCollectionCompletionChecker completionChecker,
PeekRecordFilter recordFilter,
PeekCollectionState collection,
int maxScanRecords,
long deadlineNs,
Duration pollTimeout
) {
while (System.nanoTime() < deadlineNs) {
if (completionChecker.isComplete(collection)) {
return true;
}
long remainingNs = deadlineNs - System.nanoTime(); long remainingNs = deadlineNs - System.nanoTime();
if (remainingNs <= 0) { if (remainingNs <= 0) {
break; break;
@@ -1125,19 +1243,205 @@ public final class KafkaAgent {
: pollTimeout; : pollTimeout;
ConsumerRecords<String, byte[]> records = poller.poll(timeout); ConsumerRecords<String, byte[]> records = poller.poll(timeout);
if (records.isEmpty()) { if (records.isEmpty()) {
if (caughtUpChecker.allPartitionsCaughtUp()) {
break;
}
continue; continue;
} }
for (ConsumerRecord<String, byte[]> record : records) { for (ConsumerRecord<String, byte[]> record : records) {
messages.add(peekedMessageFromRecord(record)); if (++collection.scannedRecords > maxScanRecords) {
if (messages.size() >= count) { return false;
break;
} }
TopicPartition partition = new TopicPartition(record.topic(), record.partition());
int remaining = collection.remainingByPartition.getOrDefault(partition, 0);
if (remaining <= 0 || !recordFilter.include(record)) {
continue;
}
collection.messages.add(peekedMessageFromRecord(record));
collection.remainingByPartition.put(partition, remaining - 1);
} }
} }
return messages; return completionChecker.isComplete(collection);
}
/**
* Starts at the snapshot tail and widens backward when compacted or retained-offset gaps
* leave a partition short of its record quota.
*/
private static PeekCollectionState collectLatestPeekedMessages(
KafkaConsumer<String, byte[]> consumer,
List<TopicPartition> partitions,
Map<TopicPartition, Long> beginningOffsets,
Map<TopicPartition, Long> initialSeekOffsets,
Map<TopicPartition, Long> snapshotEndOffsets,
int messagesPerPartition,
int maxScanRecords,
long deadlineNs,
Duration pollTimeout
) {
Map<TopicPartition, Long> rangeStartOffsets = new LinkedHashMap<>(initialSeekOffsets);
Map<TopicPartition, Long> rangeEndOffsets = new LinkedHashMap<>(snapshotEndOffsets);
Map<TopicPartition, Long> rangeWidths = new LinkedHashMap<>();
for (TopicPartition partition : partitions) {
long rangeStart = rangeStartOffsets.getOrDefault(partition, 0L);
long rangeEnd = rangeEndOffsets.getOrDefault(partition, rangeStart);
rangeWidths.put(partition, Math.max(1L, rangeEnd - rangeStart));
}
PeekCollectionState collection = new PeekCollectionState(partitions, messagesPerPartition);
while (!collection.allPartitionQuotasSatisfied()) {
PeekCollectionCompletionChecker rangeComplete = state -> allPeekPartitionsComplete(
partitions,
state.remainingByPartition,
currentPeekPositions(consumer, partitions),
rangeEndOffsets
);
if (!collectLatestPeekRange(
timeout -> consumer.poll(timeout),
rangeComplete,
record -> recordIsBeforeEndOffset(record, rangeEndOffsets),
collection,
maxScanRecords,
deadlineNs,
pollTimeout
)) {
collection.incomplete = true;
break;
}
if (collection.allPartitionQuotasSatisfied()) {
break;
}
boolean expanded = false;
for (TopicPartition partition : partitions) {
if (collection.remainingByPartition.getOrDefault(partition, 0) <= 0) {
continue;
}
long beginningOffset = beginningOffsets.getOrDefault(partition, 0L);
long currentStart = rangeStartOffsets.getOrDefault(partition, beginningOffset);
if (currentStart <= beginningOffset) {
continue;
}
long nextStart = previousLatestPeekStartOffset(
beginningOffset,
currentStart,
rangeWidths.getOrDefault(partition, 1L)
);
rangeEndOffsets.put(partition, currentStart);
rangeStartOffsets.put(partition, nextStart);
rangeWidths.put(partition, Math.max(1L, currentStart - nextStart));
consumer.seek(partition, nextStart);
expanded = true;
}
if (!expanded) {
break;
}
}
return collection;
}
/**
* Scans one backward-expanded range to its end, retaining only each partition's newest
* remaining records. Stopping early would select older records from a widened range.
*/
private static boolean collectLatestPeekRange(
PeekRecordPoller poller,
PeekCollectionCompletionChecker rangeComplete,
PeekRecordFilter recordFilter,
PeekCollectionState collection,
int maxScanRecords,
long deadlineNs,
Duration pollTimeout
) {
Map<TopicPartition, Deque<Map<String, Object>>> rangeMessages = new HashMap<>();
while (System.nanoTime() < deadlineNs) {
if (rangeComplete.isComplete(collection)) {
break;
}
long remainingNs = deadlineNs - System.nanoTime();
if (remainingNs <= 0) {
break;
}
Duration timeout = pollTimeout.toNanos() > remainingNs
? Duration.ofNanos(remainingNs)
: pollTimeout;
ConsumerRecords<String, byte[]> records = poller.poll(timeout);
for (ConsumerRecord<String, byte[]> record : records) {
if (++collection.scannedRecords > maxScanRecords) {
commitLatestPeekRange(rangeMessages, collection);
return false;
}
TopicPartition partition = new TopicPartition(record.topic(), record.partition());
int remaining = collection.remainingByPartition.getOrDefault(partition, 0);
if (remaining <= 0 || !recordFilter.include(record)) {
continue;
}
Deque<Map<String, Object>> latestRecords = rangeMessages.computeIfAbsent(
partition,
ignored -> new ArrayDeque<>()
);
retainLatestPeekRecord(latestRecords, peekedMessageFromRecord(record), remaining);
}
}
boolean complete = rangeComplete.isComplete(collection);
commitLatestPeekRange(rangeMessages, collection);
return complete;
}
private static void commitLatestPeekRange(
Map<TopicPartition, Deque<Map<String, Object>>> rangeMessages,
PeekCollectionState collection
) {
for (Map.Entry<TopicPartition, Deque<Map<String, Object>>> entry : rangeMessages.entrySet()) {
int retainedCount = entry.getValue().size();
collection.messages.addAll(entry.getValue());
collection.remainingByPartition.computeIfPresent(
entry.getKey(),
(ignored, remaining) -> remaining - retainedCount
);
}
}
static <T> void retainLatestPeekRecord(Deque<T> records, T record, int maxRecords) {
records.addLast(record);
if (records.size() > maxRecords) {
records.removeFirst();
}
}
private static boolean recordIsBeforeEndOffset(
ConsumerRecord<String, byte[]> record,
Map<TopicPartition, Long> endOffsets
) {
Long endOffset = endOffsets.get(new TopicPartition(record.topic(), record.partition()));
return endOffset != null && record.offset() < endOffset;
}
private static Map<TopicPartition, Long> currentPeekPositions(
KafkaConsumer<String, byte[]> consumer,
List<TopicPartition> partitions
) {
Map<TopicPartition, Long> positions = new LinkedHashMap<>();
for (TopicPartition partition : partitions) {
positions.put(partition, consumer.position(partition));
}
return positions;
}
static boolean allPeekPartitionsComplete(
List<TopicPartition> partitions,
Map<TopicPartition, Integer> remainingByPartition,
Map<TopicPartition, Long> positions,
Map<TopicPartition, Long> endOffsets
) {
for (TopicPartition partition : partitions) {
if (remainingByPartition.getOrDefault(partition, 0) <= 0) {
continue;
}
long endOffset = endOffsets.getOrDefault(partition, 0L);
long position = positions.getOrDefault(partition, 0L);
if (position < endOffset) {
return false;
}
}
return true;
} }
static boolean allPeekPartitionsCaughtUp( static boolean allPeekPartitionsCaughtUp(
@@ -1165,16 +1469,44 @@ public final class KafkaAgent {
boolean allPartitionsCaughtUp(); boolean allPartitionsCaughtUp();
} }
@FunctionalInterface
private interface PeekCollectionCompletionChecker {
boolean isComplete(PeekCollectionState collection);
}
@FunctionalInterface
interface PeekRecordFilter {
boolean include(ConsumerRecord<String, byte[]> record);
}
private static final class PeekCollectionState {
private final List<Map<String, Object>> messages = new ArrayList<>();
private final Map<TopicPartition, Integer> remainingByPartition = new HashMap<>();
private int scannedRecords;
private boolean incomplete;
private PeekCollectionState(List<TopicPartition> partitions, int messagesPerPartition) {
for (TopicPartition partition : partitions) {
remainingByPartition.put(partition, messagesPerPartition);
}
}
private boolean allPartitionQuotasSatisfied() {
return remainingByPartition.values().stream().allMatch(remaining -> remaining <= 0);
}
}
/** When partition is null, peek across every partition of the topic. */ /** When partition is null, peek across every partition of the topic. */
static List<TopicPartition> resolvePeekPartitions( static List<TopicPartition> resolvePeekPartitions(
KafkaConsumer<String, byte[]> consumer, KafkaConsumer<String, byte[]> consumer,
String topic, String topic,
Integer partition Integer partition,
Duration timeout
) { ) {
if (partition != null) { if (partition != null) {
return resolvePeekPartitions(topic, partition, Collections.emptyList()); return resolvePeekPartitions(topic, partition, Collections.emptyList());
} }
List<PartitionInfo> infos = consumer.partitionsFor(topic, Duration.ofSeconds(5)); List<PartitionInfo> infos = consumer.partitionsFor(topic, timeout);
if (infos == null || infos.isEmpty()) { if (infos == null || infos.isEmpty()) {
return Collections.emptyList(); return Collections.emptyList();
} }
@@ -1195,13 +1527,103 @@ public final class KafkaAgent {
.collect(Collectors.toList()); .collect(Collectors.toList());
} }
enum PeekStartPosition {
EARLIEST,
LATEST,
OFFSET,
}
/** Omitting startPosition preserves the old earliest default (or explicit legacy offset) behavior. */
static PeekStartPosition peekStartPosition(JsonObject params) {
String value = stringOrNull(params, "startPosition");
if (value == null) {
return PeekStartPosition.EARLIEST;
}
return switch (value.trim().toLowerCase(Locale.ROOT)) {
case "earliest" -> PeekStartPosition.EARLIEST;
case "latest" -> PeekStartPosition.LATEST;
case "offset" -> PeekStartPosition.OFFSET;
default -> throw new IllegalArgumentException("Unsupported peek startPosition: " + value);
};
}
static void validatePeekRequest(
PeekStartPosition startPosition,
boolean explicitStartPosition,
Integer partition,
Long offset
) {
if (partition != null && partition < 0) {
throw new IllegalArgumentException("partition must be non-negative");
}
if (!explicitStartPosition) {
// Older clients used offset directly without a startPosition field.
if (offset != null && offset < 0) {
throw new IllegalArgumentException("offset must be non-negative");
}
return;
}
if (startPosition != PeekStartPosition.OFFSET) {
if (offset != null) {
throw new IllegalArgumentException("offset is only supported when startPosition is offset");
}
return;
}
if (offset == null) {
throw new IllegalArgumentException("offset is required when startPosition is offset");
}
if (offset < 0) {
throw new IllegalArgumentException("offset must be non-negative when startPosition is offset");
}
}
static Long requestedPeekOffset(
PeekStartPosition startPosition,
Long offset,
boolean legacyOffsetRequest,
long beginningOffset,
long endOffset
) {
return switch (startPosition) {
case LATEST -> endOffset > beginningOffset ? beginningOffset : null;
case OFFSET -> offset;
case EARLIEST -> legacyOffsetRequest ? offset : beginningOffset;
};
}
static int peekScanLimit(int count, int readablePartitionCount) {
int fetchCount = recentPeekFetchCount(
peekMessagesPerPartition(count, readablePartitionCount), readablePartitionCount
);
if (fetchCount > MAX_PEEK_SCAN_RECORDS) {
throw new IllegalArgumentException(
"Kafka message browse would scan more than " + MAX_PEEK_SCAN_RECORDS + " records"
);
}
return MAX_PEEK_SCAN_RECORDS;
}
static void sortPeekedMessages(List<Map<String, Object>> messages) { static void sortPeekedMessages(List<Map<String, Object>> messages) {
messages.sort((left, right) -> { sortPeekedMessages(messages, PeekStartPosition.EARLIEST);
}
static void sortPeekedMessages(List<Map<String, Object>> messages, PeekStartPosition startPosition) {
if (startPosition == PeekStartPosition.OFFSET) {
messages.sort(Comparator
.comparingLong((Map<String, Object> message) ->
((Number) message.getOrDefault("offset", 0L)).longValue()
)
.thenComparingInt(message -> ((Number) message.getOrDefault("partition", 0)).intValue())
);
return;
}
Comparator<Map<String, Object>> comparator = (left, right) -> {
long leftTs = ((Number) left.getOrDefault("timestamp", 0L)).longValue(); long leftTs = ((Number) left.getOrDefault("timestamp", 0L)).longValue();
long rightTs = ((Number) right.getOrDefault("timestamp", 0L)).longValue(); long rightTs = ((Number) right.getOrDefault("timestamp", 0L)).longValue();
int byTs = Long.compare(leftTs, rightTs); int byTs = Long.compare(leftTs, rightTs);
if (byTs != 0) { if (byTs != 0) {
return byTs; return startPosition == PeekStartPosition.LATEST ? -byTs : byTs;
} }
int leftPartition = ((Number) left.getOrDefault("partition", 0)).intValue(); int leftPartition = ((Number) left.getOrDefault("partition", 0)).intValue();
int rightPartition = ((Number) right.getOrDefault("partition", 0)).intValue(); int rightPartition = ((Number) right.getOrDefault("partition", 0)).intValue();
@@ -1212,7 +1634,8 @@ public final class KafkaAgent {
long leftOffset = ((Number) left.getOrDefault("offset", 0L)).longValue(); long leftOffset = ((Number) left.getOrDefault("offset", 0L)).longValue();
long rightOffset = ((Number) right.getOrDefault("offset", 0L)).longValue(); long rightOffset = ((Number) right.getOrDefault("offset", 0L)).longValue();
return Long.compare(leftOffset, rightOffset); return Long.compare(leftOffset, rightOffset);
}); };
messages.sort(comparator);
} }
private static Map<String, Object> peekedMessageFromRecord(ConsumerRecord<String, byte[]> record) { private static Map<String, Object> peekedMessageFromRecord(ConsumerRecord<String, byte[]> record) {
@@ -1224,7 +1647,7 @@ public final class KafkaAgent {
msg.put("key", record.key()); msg.put("key", record.key());
Map<String, String> headers = new LinkedHashMap<>(); Map<String, String> headers = new LinkedHashMap<>();
record.headers().forEach(h -> record.headers().forEach(h ->
headers.put(h.key(), new String(h.value(), StandardCharsets.UTF_8))); headers.put(h.key(), h.value() == null ? "" : new String(h.value(), StandardCharsets.UTF_8)));
msg.put("headers", headers); msg.put("headers", headers);
if (record.value() != null) { if (record.value() != null) {
msg.put("payloadBase64", Base64.getEncoder().encodeToString(record.value())); msg.put("payloadBase64", Base64.getEncoder().encodeToString(record.value()));
@@ -1584,6 +2007,34 @@ public final class KafkaAgent {
return requestedOffset; return requestedOffset;
} }
/** Splits the requested page quota across partitions. */
static int peekMessagesPerPartition(int count, int partitionCount) {
int safePartitionCount = Math.max(1, partitionCount);
return (int) (((long) count + safePartitionCount - 1) / safePartitionCount);
}
static long recentPeekStartOffset(long beginningOffset, long endOffset, int messagesPerPartition) {
return Math.max(beginningOffset, endOffset - messagesPerPartition);
}
static long previousLatestPeekStartOffset(
long beginningOffset,
long currentStartOffset,
long currentWindowWidth
) {
long safeWindowWidth = Math.max(1L, currentWindowWidth);
long expandedWindowWidth = safeWindowWidth > Long.MAX_VALUE / 2
? Long.MAX_VALUE
: safeWindowWidth * 2;
long distanceToBeginning = currentStartOffset - beginningOffset;
return currentStartOffset - Math.min(distanceToBeginning, expandedWindowWidth);
}
static int recentPeekFetchCount(int messagesPerPartition, int partitionCount) {
long total = (long) messagesPerPartition * Math.max(1, partitionCount);
return (int) Math.min(Integer.MAX_VALUE, total);
}
private static String stringOrNull(JsonObject object, String key) { private static String stringOrNull(JsonObject object, String key) {
JsonElement element = object.get(key); JsonElement element = object.get(key);
return element == null || element.isJsonNull() ? null : element.getAsString(); return element == null || element.isJsonNull() ? null : element.getAsString();
@@ -1,9 +1,11 @@
package com.dbx.agent.kafka; package com.dbx.agent.kafka;
import com.google.gson.JsonObject; import com.google.gson.JsonObject;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue;
import com.google.gson.JsonParser; import com.google.gson.JsonParser;
@@ -12,8 +14,11 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.time.Duration; import java.time.Duration;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.Deque;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
@@ -302,6 +307,183 @@ class KafkaAgentTest {
assertNull(KafkaAgent.normalizePeekOffset(0, 5, 5)); assertNull(KafkaAgent.normalizePeekOffset(0, 5, 5));
} }
@Test
void peekStartPositionDefaultsToEarliestForOlderClients() {
assertEquals(KafkaAgent.PeekStartPosition.EARLIEST,
KafkaAgent.peekStartPosition(new JsonObject()));
}
@Test
void peekStartPositionRecognizesEveryExplicitMode() {
JsonObject latest = new JsonObject();
latest.addProperty("startPosition", "latest");
JsonObject earliest = new JsonObject();
earliest.addProperty("startPosition", "earliest");
JsonObject offset = new JsonObject();
offset.addProperty("startPosition", "offset");
assertEquals(KafkaAgent.PeekStartPosition.LATEST, KafkaAgent.peekStartPosition(latest));
assertEquals(KafkaAgent.PeekStartPosition.EARLIEST, KafkaAgent.peekStartPosition(earliest));
assertEquals(KafkaAgent.PeekStartPosition.OFFSET, KafkaAgent.peekStartPosition(offset));
}
@Test
void peekStartPositionRejectsUnknownValues() {
JsonObject params = new JsonObject();
params.addProperty("startPosition", "middle");
assertThrows(IllegalArgumentException.class, () -> KafkaAgent.peekStartPosition(params));
}
@Test
void offsetStartPositionAllowsAllPartitionsButRequiresANonNegativeOffset() {
assertDoesNotThrow(() ->
KafkaAgent.validatePeekRequest(KafkaAgent.PeekStartPosition.OFFSET, true, null, 0L));
assertThrows(IllegalArgumentException.class, () ->
KafkaAgent.validatePeekRequest(KafkaAgent.PeekStartPosition.OFFSET, true, 0, null));
assertThrows(IllegalArgumentException.class, () ->
KafkaAgent.validatePeekRequest(KafkaAgent.PeekStartPosition.OFFSET, true, -1, 0L));
assertThrows(IllegalArgumentException.class, () ->
KafkaAgent.validatePeekRequest(KafkaAgent.PeekStartPosition.OFFSET, true, 0, -1L));
}
@Test
void nonOffsetStartPositionsRejectAnOffset() {
assertThrows(IllegalArgumentException.class, () ->
KafkaAgent.validatePeekRequest(KafkaAgent.PeekStartPosition.LATEST, true, 0, 7L));
assertThrows(IllegalArgumentException.class, () ->
KafkaAgent.validatePeekRequest(KafkaAgent.PeekStartPosition.EARLIEST, true, 0, 7L));
}
@Test
void latestSkipsEmptyPartitions() {
assertNull(KafkaAgent.requestedPeekOffset(
KafkaAgent.PeekStartPosition.LATEST, null, false, 5L, 5L
));
}
@Test
void everyStartPositionRejectsNegativePartitions() {
assertThrows(IllegalArgumentException.class, () ->
KafkaAgent.validatePeekRequest(KafkaAgent.PeekStartPosition.LATEST, true, -1, null));
assertThrows(IllegalArgumentException.class, () ->
KafkaAgent.validatePeekRequest(KafkaAgent.PeekStartPosition.EARLIEST, true, -1, null));
}
@Test
void legacyOffsetWithoutStartPositionKeepsTheExistingReadBehavior() {
KafkaAgent.validatePeekRequest(KafkaAgent.PeekStartPosition.EARLIEST, false, null, 7L);
assertEquals(7L, KafkaAgent.requestedPeekOffset(
KafkaAgent.PeekStartPosition.EARLIEST, 7L, true, 0L, 10L
));
}
@Test
void explicitEarliestDoesNotReuseAnOffsetFromAnOlderRequest() {
assertEquals(0L, KafkaAgent.requestedPeekOffset(
KafkaAgent.PeekStartPosition.EARLIEST, 7L, false, 0L, 10L
));
}
@Test
void offsetSortUsesPartitionAsADeterministicTieBreaker() {
var messages = new java.util.ArrayList<Map<String, Object>>();
messages.add(Map.of("partition", 2, "offset", 7L));
messages.add(Map.of("partition", 1, "offset", 7L));
messages.add(Map.of("partition", 0, "offset", 8L));
KafkaAgent.sortPeekedMessages(messages, KafkaAgent.PeekStartPosition.OFFSET);
assertEquals(1, messages.get(0).get("partition"));
assertEquals(2, messages.get(1).get("partition"));
assertEquals(8L, messages.get(2).get("offset"));
}
@Test
void splitsMessageWindowAcrossPartitions() {
assertEquals(4, KafkaAgent.peekMessagesPerPartition(10, 3));
assertEquals(10, KafkaAgent.peekMessagesPerPartition(10, 1));
}
@Test
void startsLatestMessageWindowNearThePartitionEnd() {
assertEquals(90L, KafkaAgent.recentPeekStartOffset(0, 100, 10));
assertEquals(5L, KafkaAgent.recentPeekStartOffset(5, 8, 10));
}
@Test
void boundsThePerPartitionMessageQuotaBeforeTrimmingTheResult() {
assertEquals(12, KafkaAgent.recentPeekFetchCount(4, 3));
assertEquals(4, KafkaAgent.peekMessagesPerPartition(10, 3));
}
@Test
void peekRejectsAWindowThatExceedsTheScanLimit() {
assertThrows(IllegalArgumentException.class, () ->
KafkaAgent.peekScanLimit(100, 1_001));
}
@Test
void peekWindowCalculationDoesNotOverflow() {
assertEquals(1, KafkaAgent.peekMessagesPerPartition(
Integer.MAX_VALUE, Integer.MAX_VALUE
));
}
@Test
void latestPeekExpandsBackwardAcrossSparseOffsetGaps() {
assertEquals(6L, KafkaAgent.recentPeekStartOffset(0L, 11L, 5));
assertEquals(0L, KafkaAgent.previousLatestPeekStartOffset(0L, 6L, 5L));
assertEquals(12L, KafkaAgent.previousLatestPeekStartOffset(0L, 32L, 10L));
}
@Test
void latestPeekRetainsTheNewestRecordsFromAnExpandedRange() {
Deque<Long> latestOffsets = new ArrayDeque<>();
for (long offset = 86L; offset <= 95L; offset++) {
KafkaAgent.retainLatestPeekRecord(latestOffsets, offset, 4);
}
assertEquals(List.of(92L, 93L, 94L, 95L), new ArrayList<>(latestOffsets));
}
@Test
void peekCountMustStayWithinTheServiceLimit() {
assertEquals(100, KafkaAgent.validatedPeekCount(100));
assertThrows(IllegalArgumentException.class, () -> KafkaAgent.validatedPeekCount(0));
assertThrows(IllegalArgumentException.class, () -> KafkaAgent.validatedPeekCount(101));
}
@Test
void peekUsesTheConfiguredConsumerRequestTimeout() {
Properties properties = new Properties();
properties.put("request.timeout.ms", "1500");
assertEquals(1_500, KafkaAgent.peekRequestTimeoutMs(new JsonObject(), properties));
}
@Test
void peekRequestTimeoutPrefersTheConnectionOverrideAndRejectsInvalidValues() {
Properties properties = new Properties();
properties.put("request.timeout.ms", "1500");
JsonObject connection = new JsonObject();
connection.addProperty("request_timeout_ms", 2_500);
assertEquals(2_500, KafkaAgent.peekRequestTimeoutMs(connection, properties));
properties.put("request.timeout.ms", "0");
assertThrows(IllegalArgumentException.class, () -> KafkaAgent.peekRequestTimeoutMs(new JsonObject(), properties));
}
@Test
void incompletePeekResultsAreExplicitlyMarked() {
Map<String, Object> partial = KafkaAgent.peekMessagesResult(List.of(), true);
Map<String, Object> complete = KafkaAgent.peekMessagesResult(List.of(), false);
assertEquals(true, partial.get("incomplete"));
assertEquals(false, complete.get("incomplete"));
}
@Test @Test
void resolvePeekPartitionsUsesSinglePartitionWhenSpecified() { void resolvePeekPartitionsUsesSinglePartitionWhenSpecified() {
var partitions = KafkaAgent.resolvePeekPartitions("events", 2, List.of(0, 1, 2)); var partitions = KafkaAgent.resolvePeekPartitions("events", 2, List.of(0, 1, 2));
@@ -330,6 +512,37 @@ class KafkaAgentTest {
assertEquals(20L, messages.get(3).get("timestamp")); assertEquals(20L, messages.get(3).get("timestamp"));
} }
@Test
void sortPeekedMessagesCanOrderNewestFirst() {
var messages = new java.util.ArrayList<Map<String, Object>>();
messages.add(Map.of("timestamp", 20L, "partition", 1, "offset", 1L));
messages.add(Map.of("timestamp", 10L, "partition", 0, "offset", 5L));
messages.add(Map.of("timestamp", 10L, "partition", 0, "offset", 2L));
messages.add(Map.of("timestamp", 10L, "partition", 1, "offset", 0L));
KafkaAgent.sortPeekedMessages(messages, KafkaAgent.PeekStartPosition.LATEST);
assertEquals(20L, messages.get(0).get("timestamp"));
assertEquals(0, messages.get(1).get("partition"));
assertEquals(2L, messages.get(1).get("offset"));
assertEquals(5L, messages.get(2).get("offset"));
assertEquals(1, messages.get(3).get("partition"));
}
@Test
void sortPeekedMessagesOrdersOffsetModeByOffsetAscending() {
var messages = new java.util.ArrayList<Map<String, Object>>();
messages.add(Map.of("timestamp", 10L, "partition", 0, "offset", 5L));
messages.add(Map.of("timestamp", 30L, "partition", 0, "offset", 2L));
messages.add(Map.of("timestamp", 20L, "partition", 0, "offset", 3L));
KafkaAgent.sortPeekedMessages(messages, KafkaAgent.PeekStartPosition.OFFSET);
assertEquals(2L, messages.get(0).get("offset"));
assertEquals(3L, messages.get(1).get("offset"));
assertEquals(5L, messages.get(2).get("offset"));
}
@Test @Test
void allPeekPartitionsCaughtUpRequiresEveryPartitionAtEndOffset() { void allPeekPartitionsCaughtUpRequiresEveryPartitionAtEndOffset() {
TopicPartition p0 = new TopicPartition("events", 0); TopicPartition p0 = new TopicPartition("events", 0);
@@ -348,6 +561,33 @@ class KafkaAgentTest {
)); ));
} }
@Test
void peekCompletionStopsAfterEachPartitionSuppliesItsQuota() {
TopicPartition p0 = new TopicPartition("events", 0);
TopicPartition p1 = new TopicPartition("events", 1);
List<TopicPartition> partitions = List.of(p0, p1);
Map<TopicPartition, Long> endOffsets = Map.of(p0, 100L, p1, 100L);
assertTrue(KafkaAgent.allPeekPartitionsComplete(
partitions,
Map.of(p0, 0, p1, 0),
Map.of(p0, 1L, p1, 1L),
endOffsets
));
assertFalse(KafkaAgent.allPeekPartitionsComplete(
partitions,
Map.of(p0, 0, p1, 1),
Map.of(p0, 1L, p1, 1L),
endOffsets
));
assertTrue(KafkaAgent.allPeekPartitionsComplete(
partitions,
Map.of(p0, 0, p1, 1),
Map.of(p0, 1L, p1, 100L),
endOffsets
));
}
@Test @Test
void collectPeekedMessagesRetriesAfterEmptyFirstPoll() { void collectPeekedMessagesRetriesAfterEmptyFirstPoll() {
TopicPartition tp = new TopicPartition("events", 0); TopicPartition tp = new TopicPartition("events", 0);
@@ -365,8 +605,11 @@ class KafkaAgentTest {
AtomicInteger polls = new AtomicInteger(); AtomicInteger polls = new AtomicInteger();
List<Map<String, Object>> messages = KafkaAgent.collectPeekedMessages( List<Map<String, Object>> messages = KafkaAgent.collectPeekedMessages(
timeout -> polls.getAndIncrement() == 0 ? ConsumerRecords.empty() : withData, timeout -> polls.getAndIncrement() == 0 ? ConsumerRecords.empty() : withData,
() -> false, () -> polls.get() >= 2,
ignored -> true,
List.of(tp),
1, 1,
1_000,
System.nanoTime() + Duration.ofSeconds(5).toNanos(), System.nanoTime() + Duration.ofSeconds(5).toNanos(),
Duration.ofMillis(1) Duration.ofMillis(1)
); );
@@ -378,7 +621,8 @@ class KafkaAgentTest {
} }
@Test @Test
void collectPeekedMessagesStopsOnEmptyPollWhenCaughtUp() { void collectPeekedMessagesDoesNotPollWhenAlreadyCaughtUp() {
TopicPartition tp = new TopicPartition("events", 0);
AtomicInteger polls = new AtomicInteger(); AtomicInteger polls = new AtomicInteger();
List<Map<String, Object>> messages = KafkaAgent.collectPeekedMessages( List<Map<String, Object>> messages = KafkaAgent.collectPeekedMessages(
timeout -> { timeout -> {
@@ -386,13 +630,208 @@ class KafkaAgentTest {
return ConsumerRecords.empty(); return ConsumerRecords.empty();
}, },
() -> true, () -> true,
record -> true,
List.of(tp),
10, 10,
1_000,
System.nanoTime() + Duration.ofSeconds(5).toNanos(),
Duration.ofMillis(1)
);
assertEquals(0, polls.get());
assertTrue(messages.isEmpty());
}
@Test
void collectPeekedMessagesExcludesRecordsPastTheSnapshotEndOffset() {
TopicPartition tp = new TopicPartition("events", 0);
ConsumerRecord<String, byte[]> included = new ConsumerRecord<>(
"events", 0, 9L, "before", "before".getBytes(StandardCharsets.UTF_8)
);
ConsumerRecord<String, byte[]> excluded = new ConsumerRecord<>(
"events", 0, 10L, "after", "after".getBytes(StandardCharsets.UTF_8)
);
ConsumerRecords<String, byte[]> batch = new ConsumerRecords<>(Map.of(tp, List.of(included, excluded)));
AtomicInteger polls = new AtomicInteger();
AtomicInteger caughtUpChecks = new AtomicInteger();
List<Map<String, Object>> messages = KafkaAgent.collectPeekedMessages(
timeout -> {
polls.incrementAndGet();
return batch;
},
() -> caughtUpChecks.getAndIncrement() > 0,
record -> record.offset() < 10L,
List.of(tp),
2,
1_000,
System.nanoTime() + Duration.ofSeconds(5).toNanos(), System.nanoTime() + Duration.ofSeconds(5).toNanos(),
Duration.ofMillis(1) Duration.ofMillis(1)
); );
assertEquals(1, polls.get()); assertEquals(1, polls.get());
assertTrue(messages.isEmpty()); assertEquals(1, messages.size());
assertEquals(9L, messages.get(0).get("offset"));
}
@Test
void peekCollectsFromEveryPartitionBeforeTrimming() {
TopicPartition p0 = new TopicPartition("events", 0);
TopicPartition p1 = new TopicPartition("events", 1);
ConsumerRecords<String, byte[]> batch = new ConsumerRecords<>(Map.of(
p0, List.of(
new ConsumerRecord<>("events", 0, 9L, "p0-first", "one".getBytes(StandardCharsets.UTF_8)),
new ConsumerRecord<>("events", 0, 10L, "p0-second", "two".getBytes(StandardCharsets.UTF_8))
),
p1, List.of(
new ConsumerRecord<>("events", 1, 7L, "p1-first", "three".getBytes(StandardCharsets.UTF_8))
)
));
AtomicInteger polls = new AtomicInteger();
List<Map<String, Object>> messages = KafkaAgent.collectPeekedMessages(
timeout -> polls.getAndIncrement() == 0 ? batch : ConsumerRecords.empty(),
() -> polls.get() > 0,
record -> true,
List.of(p0, p1),
1,
1_000,
System.nanoTime() + Duration.ofSeconds(5).toNanos(),
Duration.ofMillis(1)
);
assertEquals(2, messages.size());
assertTrue(messages.stream().anyMatch(message -> message.get("partition").equals(0)));
assertTrue(messages.stream().anyMatch(message -> message.get("partition").equals(1)));
}
@Test
void peekWaitsForEveryPartitionWindowWhenOnePartitionRespondsFirst() {
TopicPartition p0 = new TopicPartition("events", 0);
TopicPartition p1 = new TopicPartition("events", 1);
ConsumerRecords<String, byte[]> firstPartition = new ConsumerRecords<>(Map.of(p0, List.of(
new ConsumerRecord<>("events", 0, 0L, "p0-first", "one".getBytes(StandardCharsets.UTF_8)),
new ConsumerRecord<>("events", 0, 1L, "p0-second", "two".getBytes(StandardCharsets.UTF_8))
)));
ConsumerRecords<String, byte[]> secondPartition = new ConsumerRecords<>(Map.of(p1, List.of(
new ConsumerRecord<>("events", 1, 0L, "p1-first", "three".getBytes(StandardCharsets.UTF_8))
)));
AtomicInteger polls = new AtomicInteger();
List<Map<String, Object>> messages = KafkaAgent.collectPeekedMessages(
timeout -> polls.getAndIncrement() == 0 ? firstPartition : secondPartition,
() -> polls.get() >= 2,
ignored -> true,
List.of(p0, p1),
1,
1_000,
System.nanoTime() + Duration.ofSeconds(5).toNanos(),
Duration.ofMillis(1)
);
assertEquals(2, polls.get());
assertEquals(2, messages.size());
assertTrue(messages.stream().anyMatch(message -> message.get("partition").equals(0)));
assertTrue(messages.stream().anyMatch(message -> message.get("partition").equals(1)));
}
@Test
void peekRetainsRecordsReadBeforeTheScanLimit() {
TopicPartition partition = new TopicPartition("events", 0);
ConsumerRecords<String, byte[]> batch = new ConsumerRecords<>(Map.of(partition, List.of(
new ConsumerRecord<>("events", 0, 9L, "first", "one".getBytes(StandardCharsets.UTF_8)),
new ConsumerRecord<>("events", 0, 10L, "second", "two".getBytes(StandardCharsets.UTF_8))
)));
List<Map<String, Object>> messages = KafkaAgent.collectPeekedMessages(
timeout -> batch,
() -> false,
record -> true,
List.of(partition),
2,
1,
System.nanoTime() + Duration.ofSeconds(5).toNanos(),
Duration.ofMillis(1)
);
assertEquals(1, messages.size());
assertEquals(9L, messages.get(0).get("offset"));
}
@Test
void peekCountsSparseOffsetsAsRecordsInsteadOfOffsetWindowWidth() {
TopicPartition partition = new TopicPartition("events", 0);
// A compacted topic can retain these two records while offsets 1..9 are absent.
ConsumerRecords<String, byte[]> batch = new ConsumerRecords<>(Map.of(partition, List.of(
new ConsumerRecord<>("events", 0, 0L, "first", "one".getBytes(StandardCharsets.UTF_8)),
new ConsumerRecord<>("events", 0, 10L, "second", "two".getBytes(StandardCharsets.UTF_8))
)));
AtomicInteger polls = new AtomicInteger();
List<Map<String, Object>> messages = KafkaAgent.collectPeekedMessages(
timeout -> polls.getAndIncrement() == 0 ? batch : ConsumerRecords.empty(),
() -> polls.get() > 0,
record -> true,
List.of(partition),
5,
1_000,
System.nanoTime() + Duration.ofSeconds(5).toNanos(),
Duration.ofMillis(1)
);
assertEquals(2, messages.size());
assertEquals(0L, messages.get(0).get("offset"));
assertEquals(10L, messages.get(1).get("offset"));
}
@Test
void peekHandlesKafkaHeadersWithNullValues() {
TopicPartition partition = new TopicPartition("events", 0);
ConsumerRecord<String, byte[]> record = new ConsumerRecord<>(
"events", 0, 0L, "key", "value".getBytes(StandardCharsets.UTF_8)
);
record.headers().add("tombstone", null);
ConsumerRecords<String, byte[]> batch = new ConsumerRecords<>(Map.of(partition, List.of(record)));
AtomicInteger polls = new AtomicInteger();
List<Map<String, Object>> messages = KafkaAgent.collectPeekedMessages(
timeout -> {
polls.incrementAndGet();
return batch;
},
() -> polls.get() > 0,
ignored -> true,
List.of(partition),
1,
1_000,
System.nanoTime() + Duration.ofSeconds(5).toNanos(),
Duration.ofMillis(1)
);
assertEquals("", ((Map<?, ?>) messages.get(0).get("headers")).get("tombstone"));
}
@Test
void incompleteCollectCanBeReturnedWithAnExplicitStatus() {
TopicPartition partition = new TopicPartition("events", 0);
ConsumerRecords<String, byte[]> batch = new ConsumerRecords<>(Map.of(partition, List.of(
new ConsumerRecord<>("events", 0, 0L, "only", "one".getBytes(StandardCharsets.UTF_8))
)));
List<Map<String, Object>> partial = KafkaAgent.collectPeekedMessages(
timeout -> batch,
() -> false,
record -> true,
List.of(partition),
5,
1_000,
System.nanoTime() - 1,
Duration.ofMillis(1)
);
assertEquals(0, partial.size());
Map<String, Object> result = KafkaAgent.peekMessagesResult(partial, true);
assertEquals(true, result.get("incomplete"));
} }
@Test @Test
@@ -0,0 +1,462 @@
<script setup lang="ts">
import { computed, ref, watch } from "vue";
import { useI18n } from "vue-i18n";
import type { MqSystemKind, PeekedMessage, PeekMessagesOptions, TopicRef } from "@/types/mq";
import { mqPeekMessages } from "@/lib/backend/api";
import { formatError } from "@/lib/backend/errorUtils";
import { parseNonNegativeSafeInteger } from "@/lib/mq/mqPeekFilters";
import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
type MessageBrowserAppearance = "form" | "monitoring";
interface Props {
connectionId: string;
topic?: TopicRef | null;
mqSystemKind?: MqSystemKind;
/** Flatten chrome when embedded in MonitoringPanel so it is not a second nested card. */
appearance?: MessageBrowserAppearance;
}
const props = withDefaults(defineProps<Props>(), {
appearance: "form",
});
const { t } = useI18n();
const loading = ref(false);
const error = ref<string>();
const messages = ref<PeekedMessage[]>([]);
const incomplete = ref(false);
const partition = ref<string | number>("");
const offset = ref<string | number>("");
const count = ref(20);
const advancedExpanded = ref(false);
type KafkaPeekStartPosition = NonNullable<PeekMessagesOptions["startPosition"]>;
const kafkaStartPosition = ref<KafkaPeekStartPosition>("latest");
let messageRequestVersion = 0;
const isKafka = computed(() => props.mqSystemKind === "kafka");
const isKafkaOffsetMode = computed(() => kafkaStartPosition.value === "offset");
const isMonitoring = computed(() => props.appearance === "monitoring");
function peekGroupName(): string {
if (props.mqSystemKind === "rocketmq") return "__dbx_rocketmq_viewer__";
return "__dbx_kafka_viewer__";
}
async function loadMessages() {
const topic = props.topic;
if (!topic || loading.value) return;
const requestVersion = ++messageRequestVersion;
loading.value = true;
error.value = undefined;
incomplete.value = false;
try {
const resultLimit = Math.max(1, Math.min(100, Math.trunc(Number(count.value) || 20)));
count.value = resultLimit;
const options: PeekMessagesOptions = {};
const partitionText = String(partition.value).trim();
const offsetText = String(offset.value).trim();
if (isKafka.value) {
options.startPosition = kafkaStartPosition.value;
if (partitionText !== "") {
const parsedPartition = parseNonNegativeSafeInteger(partitionText);
if (parsedPartition == null) throw new Error(t("mqMessages.partitionMustBeNonNegativeInt"));
options.partition = parsedPartition;
partition.value = String(parsedPartition);
}
if (isKafkaOffsetMode.value) {
if (offsetText === "") throw new Error(t("mqMessages.offsetRequiredForOffset"));
const parsedOffset = parseNonNegativeSafeInteger(offsetText);
if (parsedOffset == null) throw new Error(t("mqMessages.offsetMustBeNonNegativeIntRequired"));
options.offset = parsedOffset;
offset.value = String(parsedOffset);
}
} else {
if (partitionText !== "") {
const parsedPartition = parseNonNegativeSafeInteger(partitionText);
if (parsedPartition == null) throw new Error(t("mqMessages.partitionMustBeNonNegativeInt"));
options.partition = parsedPartition;
partition.value = String(parsedPartition);
}
if (offsetText !== "") {
const parsedOffset = parseNonNegativeSafeInteger(offsetText);
if (parsedOffset == null) throw new Error(t("mqMessages.offsetMustBeNonNegativeInt"));
options.offset = parsedOffset;
offset.value = String(parsedOffset);
}
}
const result = await mqPeekMessages(props.connectionId, topic, peekGroupName(), resultLimit, options);
if (requestVersion === messageRequestVersion) {
const browseResult = Array.isArray(result) ? { messages: result, incomplete: false } : result;
messages.value = browseResult.messages;
incomplete.value = browseResult.incomplete;
}
} catch (cause: unknown) {
if (requestVersion === messageRequestVersion) {
error.value = formatError(cause);
}
} finally {
if (requestVersion === messageRequestVersion) {
loading.value = false;
}
}
}
function invalidateMessageRequest() {
messageRequestVersion += 1;
loading.value = false;
error.value = undefined;
messages.value = [];
incomplete.value = false;
}
function messagePayload(message: PeekedMessage): string {
return message.payloadText ?? message.payloadBase64;
}
function formatMessageTimestamp(value?: string): string {
if (!value) return "-";
const numeric = Number(value);
if (!Number.isFinite(numeric)) return value;
return new Date(numeric).toLocaleString();
}
watch([() => props.connectionId, () => props.mqSystemKind, () => JSON.stringify(props.topic ?? null)], () => {
invalidateMessageRequest();
});
watch(kafkaStartPosition, () => {
// Keep offset values for switching back, but never retain results from another start mode.
invalidateMessageRequest();
});
</script>
<template>
<section v-if="topic" class="message-browser" :class="{ 'is-monitoring': isMonitoring }" data-testid="message-browser">
<div class="message-browser-header">
<h4>{{ t("mqMessages.messageList") }}</h4>
<button type="button" class="btn-sm" :disabled="loading" @click="loadMessages">
{{ loading ? t("mqMessages.loading") : t("mqMessages.loadMessages") }}
</button>
</div>
<p v-if="isKafka" class="peek-default-hint">
<template v-if="kafkaStartPosition === 'latest'">
{{ t("mqMessages.kafkaLatestHint") }}
</template>
<template v-else-if="kafkaStartPosition === 'earliest'">
{{ t("mqMessages.kafkaEarliestHint") }}
</template>
<template v-else>
{{ t("mqMessages.kafkaOffsetHint") }}
</template>
</p>
<p v-else class="peek-default-hint">{{ t("mqMessages.peekDefaultHint") }}</p>
<p v-if="incomplete" class="peek-incomplete" role="status" data-testid="peek-incomplete">
{{ t("mqMessages.peekIncomplete") }}
</p>
<div class="peek-controls">
<label>
<span>{{ t("mqMessages.count") }}</span>
<input v-model.number="count" data-testid="peek-count" type="number" min="1" max="100" :disabled="loading" />
</label>
<label v-if="isKafka">
<span>{{ t("mqMessages.startPosition") }}</span>
<Select v-model="kafkaStartPosition" :disabled="loading">
<SelectTrigger data-testid="kafka-peek-start-position" class="message-browser-start-position">
<SelectValue />
</SelectTrigger>
<SelectContent position="popper" class="message-browser-start-position-content">
<SelectItem value="latest">{{ t("mqMessages.kafkaLatest") }}</SelectItem>
<SelectItem value="earliest">{{ t("mqMessages.kafkaEarliest") }}</SelectItem>
<SelectItem value="offset">{{ t("mqMessages.kafkaOffset") }}</SelectItem>
</SelectContent>
</Select>
</label>
<label v-if="isKafka">
<span>{{ t("mqMessages.partition") }}</span>
<input v-model="partition" data-testid="kafka-peek-partition" type="number" min="0" :placeholder="t('mqMessages.partitionPlaceholderAll')" :disabled="loading" />
</label>
<label v-if="isKafkaOffsetMode">
<span>{{ t("mqMessages.offset") }}</span>
<input v-model="offset" data-testid="kafka-peek-offset" type="number" min="0" :placeholder="t('mqMessages.offsetPlaceholderRequired')" :disabled="loading" />
</label>
</div>
<template v-if="!isKafka">
<button type="button" class="collapse-toggle peek-advanced-toggle" @click="advancedExpanded = !advancedExpanded">
<span class="collapse-arrow" :class="{ expanded: advancedExpanded }">&#9654;</span>
<span>{{ t("mqMessages.advancedFilter") }}</span>
<span v-if="(partition || offset) && !advancedExpanded" class="collapse-badge">&middot;</span>
</button>
<div v-if="advancedExpanded" class="peek-controls non-kafka-controls">
<label>
<span>{{ t("mqMessages.partition") }}</span>
<input v-model="partition" type="number" min="0" :placeholder="t('mqMessages.partitionPlaceholderAll')" :disabled="loading" />
</label>
<label>
<span>{{ t("mqMessages.offset") }}</span>
<input v-model="offset" type="number" min="0" :placeholder="t('mqMessages.offsetPlaceholderEarliest')" :disabled="loading" />
</label>
</div>
</template>
<div v-if="error" class="panel-error">{{ error }}</div>
<div v-else-if="loading" class="message-empty">{{ t("mqMessages.messagesLoading") }}</div>
<div v-else-if="!messages.length" class="message-empty">{{ t("mqMessages.noMessages") }}</div>
<div v-else class="message-list">
<article v-for="message in messages" :key="`${message.properties?.partition ?? 'p'}-${message.messageId || message.position}`" class="message-row">
<div class="message-meta">
<span>#{{ message.position }}</span>
<span v-if="message.properties?.partition != null">{{ t("mqMessages.metaPartition", { partition: message.properties.partition }) }}</span>
<span>{{ t("mqMessages.metaOffset", { offset: message.messageId || "-" }) }}</span>
<span v-if="message.key">{{ t("mqMessages.metaKey", { key: message.key }) }}</span>
<span>{{ formatMessageTimestamp(message.publishTime) }}</span>
</div>
<pre class="message-payload">{{ messagePayload(message) }}</pre>
<div v-if="Object.keys(message.headers || {}).length" class="message-headers">
<span v-for="(value, key) in message.headers" :key="key">{{ key }}: {{ value }}</span>
</div>
</article>
</div>
</section>
</template>
<style scoped>
.message-browser {
margin-top: 4px;
padding: 14px;
border: 1px solid var(--color-border);
border-radius: var(--dbx-radius-fixed-6);
background: var(--color-background-secondary);
}
/* Only flatten outer chrome when embedded in MonitoringPanel — do not restyle list rows. */
.message-browser.is-monitoring {
margin: 0;
padding: 0;
border: 0;
border-radius: 0;
background: transparent;
}
.message-browser-header {
display: flex;
align-items: center;
justify-content: space-between;
gap: 12px;
margin-bottom: 12px;
}
.message-browser-header h4 {
margin: 0;
color: var(--color-text);
font-size: 14px;
font-weight: 600;
}
.btn-sm {
padding: 4px 10px;
border: 1px solid var(--color-border);
border-radius: var(--dbx-radius-fixed-6);
background: var(--color-background);
color: var(--color-text);
cursor: pointer;
font-size: 12px;
}
.btn-sm:hover:not(:disabled) {
background: var(--color-background-secondary);
}
.btn-sm:disabled,
.peek-controls input:disabled,
.peek-controls :deep(.message-browser-start-position[data-disabled]) {
opacity: 0.5;
cursor: not-allowed;
}
.peek-default-hint {
margin: 0 0 12px;
padding: 8px 10px;
border-radius: var(--dbx-radius-fixed-6);
background: color-mix(in srgb, var(--color-primary) 8%, transparent);
color: var(--color-text-secondary);
font-size: 12px;
line-height: 1.5;
}
.peek-incomplete {
margin: 0 0 12px;
padding: 8px 10px;
border: 1px solid var(--color-warning-border, #d99a22);
border-radius: var(--dbx-radius-fixed-6);
background: var(--color-warning-background, #fff6df);
color: var(--color-warning-text, #7a4a00);
font-size: 12px;
}
.peek-controls {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(120px, 1fr));
gap: 10px;
margin-bottom: 12px;
}
.peek-controls label {
display: flex;
flex-direction: column;
gap: 5px;
color: var(--color-text-secondary);
font-size: 12px;
font-weight: 500;
}
.peek-controls input,
.peek-controls :deep(.message-browser-start-position) {
height: 32px;
width: 100%;
padding: 7px 10px;
box-sizing: border-box;
border: 1px solid var(--color-border);
border-radius: var(--dbx-radius-fixed-6);
background: var(--color-background);
color: var(--color-text);
font-size: 13px;
}
.peek-controls input:focus,
.peek-controls :deep(.message-browser-start-position:focus-visible) {
outline: none;
border-color: var(--color-primary);
box-shadow: 0 0 0 2px var(--color-primary-alpha);
}
.non-kafka-controls {
margin-top: 6px;
}
.collapse-toggle {
display: flex;
align-items: center;
gap: 6px;
padding: 0;
border: none;
background: none;
color: var(--color-text-secondary);
font-size: 13px;
font-weight: 500;
cursor: pointer;
user-select: none;
}
.peek-advanced-toggle {
margin-bottom: 10px;
}
.collapse-arrow {
display: inline-block;
font-size: 10px;
transition: transform 0.15s;
}
.collapse-arrow.expanded {
transform: rotate(90deg);
}
.collapse-badge {
color: var(--color-primary);
font-weight: 700;
}
.panel-error {
padding: 10px 14px;
border-radius: var(--dbx-radius-fixed-6);
background: var(--color-error-bg);
color: var(--color-error);
font-size: 13px;
}
.message-empty {
padding: 18px;
border: 1px dashed var(--color-border);
border-radius: var(--dbx-radius-fixed-6);
color: var(--color-text-tertiary);
text-align: center;
font-size: 13px;
}
.message-list {
display: flex;
flex-direction: column;
gap: 10px;
max-height: 360px;
overflow: auto;
}
/* Monitoring page already scrolls via .stats-container — avoid a second vertical bar. */
.message-browser.is-monitoring .message-list {
max-height: none;
overflow: visible;
}
.message-row {
padding: 10px 12px;
border: 1px solid var(--color-border);
border-radius: var(--dbx-radius-fixed-6);
background: var(--color-background);
}
.message-meta {
display: flex;
align-items: center;
gap: 10px;
flex-wrap: wrap;
color: var(--color-text-tertiary);
font-size: 12px;
}
.message-meta span:first-child {
color: var(--color-primary);
font-weight: 700;
}
.message-payload {
margin: 8px 0 0;
padding: 10px;
max-height: 160px;
overflow: auto;
border-radius: var(--dbx-radius-fixed-6);
background: var(--color-background-tertiary, var(--color-background-secondary));
color: var(--color-text);
font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace;
font-size: 12px;
line-height: 1.5;
white-space: pre-wrap;
word-break: break-word;
}
.message-headers {
display: flex;
flex-wrap: wrap;
gap: 6px;
margin-top: 8px;
}
.message-headers span {
padding: 2px 6px;
border: 1px solid var(--color-border);
border-radius: var(--dbx-radius-fixed-4);
color: var(--color-text-secondary);
background: var(--color-background-secondary);
font-size: 12px;
}
@media (max-width: 720px) {
.peek-controls {
grid-template-columns: 1fr;
}
}
</style>
@@ -151,7 +151,8 @@ async function runQuery() {
} else { } else {
const group = consumerGroup.value.trim() || "__dbx_rocketmq_dlq__"; const group = consumerGroup.value.trim() || "__dbx_rocketmq_dlq__";
const peeked = await mqPeekMessages(props.connectionId, topic, group, 64); const peeked = await mqPeekMessages(props.connectionId, topic, group, 64);
queryMessages.value = peeked.map((msg) => rocketMqDisplayFromPeeked(msg, topic.topic)); const messages = Array.isArray(peeked) ? peeked : peeked.messages;
queryMessages.value = messages.map((msg) => rocketMqDisplayFromPeeked(msg, topic.topic));
} }
} else if (activeQueryMode.value === "msgId") { } else if (activeQueryMode.value === "msgId") {
const id = queryMsgId.value.trim(); const id = queryMsgId.value.trim();
@@ -8,8 +8,9 @@ import { LineChart } from "echarts/charts";
import { GridComponent, LegendComponent, TooltipComponent } from "echarts/components"; import { GridComponent, LegendComponent, TooltipComponent } from "echarts/components";
import VChart from "vue-echarts"; import VChart from "vue-echarts";
import { Activity, AlertTriangle, BarChart3, Boxes, CheckCircle2, Database, Download, Gauge, Hash, HardDrive, Layers3, Loader2, Package, RadioTower, RefreshCw, Send, ShieldCheck, Table2, Upload, Users } from "@lucide/vue"; import { Activity, AlertTriangle, BarChart3, Boxes, CheckCircle2, Database, Download, Gauge, Hash, HardDrive, Layers3, Loader2, Package, RadioTower, RefreshCw, Send, ShieldCheck, Table2, Upload, Users } from "@lucide/vue";
import type { MqSystemKind, TopicRef, TopicInfo, TopicStats, BacklogStats, PeekedMessage } from "@/types/mq"; import type { MqSystemKind, TopicRef, TopicInfo, TopicStats, BacklogStats } from "@/types/mq";
import { mqGetTopicStats, mqGetBacklog, mqPeekMessages } from "@/lib/backend/api"; import { mqGetTopicStats, mqGetBacklog } from "@/lib/backend/api";
import MessageBrowser from "./MessageBrowser.vue";
use([CanvasRenderer, LineChart, GridComponent, LegendComponent, TooltipComponent]); use([CanvasRenderer, LineChart, GridComponent, LegendComponent, TooltipComponent]);
@@ -73,10 +74,6 @@ const error = ref<string>();
const autoRefresh = ref(true); const autoRefresh = ref(true);
const refreshInterval = ref(5); // seconds const refreshInterval = ref(5); // seconds
const selectedPartitionName = ref<string>(); const selectedPartitionName = ref<string>();
const kafkaMessageSql = ref("");
const kafkaMessageLoading = ref(false);
const kafkaMessageError = ref<string>();
const kafkaMessages = ref<PeekedMessage[]>([]);
let refreshTimer: number | undefined; let refreshTimer: number | undefined;
const history = ref<MetricPoint[]>([]); const history = ref<MetricPoint[]>([]);
@@ -221,69 +218,6 @@ function refreshNow() {
void loadStats(); void loadStats();
} }
function defaultKafkaMessageSql(): string {
const topic = props.topic?.shortName;
return topic ? `SELECT * FROM "${topic}" LIMIT 20` : "";
}
function parseKafkaMessageSql(sql: string): { topic: string; partition?: number; offset?: number; limit: number } {
const match = sql.trim().match(/^\s*select\s+\*\s+from\s+(?:"([^"]+)"|`([^`]+)`|'([^']+)'|([^\s;]+))(?:\s+partition\s+(\d+))?(?:\s+offset\s+(\d+))?(?:\s+limit\s+(\d+))?\s*;?\s*$/i);
if (!match) {
throw new Error(t("mqMonitoring.sqlSyntaxError"));
}
const topic = match[1] || match[2] || match[3] || match[4] || "";
const partition = match[5] != null ? Math.max(0, Number(match[5])) : undefined;
const offset = match[6] != null ? Math.max(0, Number(match[6])) : undefined;
const limit = Math.max(1, Math.min(100, Number(match[7] ?? 20)));
return { topic, partition, offset, limit };
}
function flatMqMonitorGroupName(): string {
if (props.mqSystemKind === "rocketmq") return "__dbx_rocketmq_monitor__";
return "__dbx_kafka_monitor__";
}
async function runKafkaMessageSql() {
if (!props.tenant || !props.namespace) return;
kafkaMessageLoading.value = true;
kafkaMessageError.value = undefined;
try {
const parsed = parseKafkaMessageSql(kafkaMessageSql.value);
const selected = props.topic && parsed.topic === props.topic.shortName ? props.topic : undefined;
const options: { partition?: number; offset?: number } = {};
if (parsed.partition != null) options.partition = parsed.partition;
if (parsed.offset != null) options.offset = parsed.offset;
kafkaMessages.value = await mqPeekMessages(
props.connectionId,
{
tenant: props.tenant,
namespace: props.namespace,
topic: parsed.topic,
persistent: selected?.persistent ?? true,
partitioned: selected?.partitioned,
},
flatMqMonitorGroupName(),
parsed.limit,
options,
);
} catch (e: unknown) {
kafkaMessageError.value = formatError(e);
} finally {
kafkaMessageLoading.value = false;
}
}
function kafkaMessagePayload(message: PeekedMessage): string {
return message.payloadText ?? message.payloadBase64;
}
function formatKafkaMessageTimestamp(value?: string): string {
if (!value) return "-";
const numeric = Number(value);
if (!Number.isFinite(numeric)) return value;
return new Date(numeric).toLocaleString();
}
function appendHistoryPoint(statsData: TopicStats, backlogData?: BacklogStats) { function appendHistoryPoint(statsData: TopicStats, backlogData?: BacklogStats) {
const point: MetricPoint = { const point: MetricPoint = {
time: new Date().toLocaleTimeString(), time: new Date().toLocaleTimeString(),
@@ -452,9 +386,6 @@ watch(
() => { () => {
history.value = []; history.value = [];
selectedPartitionName.value = undefined; selectedPartitionName.value = undefined;
kafkaMessageSql.value = defaultKafkaMessageSql();
kafkaMessageError.value = undefined;
kafkaMessages.value = [];
void loadStats(); void loadStats();
startAutoRefresh(); startAutoRefresh();
}, },
@@ -515,16 +446,16 @@ onUnmounted(() => {
</div> </div>
</div> </div>
<div v-if="error && topic" class="panel-error">
<AlertTriangle :size="18" />
<span>{{ error }}</span>
</div>
<div v-if="!topic" class="panel-placeholder"> <div v-if="!topic" class="panel-placeholder">
<Table2 :size="24" /> <Table2 :size="24" />
<span>{{ t("mqMonitoring.selectTopicFirst") }}</span> <span>{{ t("mqMonitoring.selectTopicFirst") }}</span>
</div> </div>
<div v-else-if="error" class="panel-error">
<AlertTriangle :size="18" />
<span>{{ error }}</span>
</div>
<div v-else-if="loading && !stats" class="panel-loading"> <div v-else-if="loading && !stats" class="panel-loading">
<Loader2 class="loading-icon spinning" :size="22" /> <Loader2 class="loading-icon spinning" :size="22" />
<span>{{ t("mqMonitoring.loadingStats") }}</span> <span>{{ t("mqMonitoring.loadingStats") }}</span>
@@ -698,36 +629,7 @@ onUnmounted(() => {
</div> </div>
<div class="stats-section"> <div class="stats-section">
<div class="section-title-row"> <MessageBrowser :connection-id="connectionId" :topic="getTopicRef()" mq-system-kind="kafka" appearance="monitoring" />
<h4>{{ t("mqMonitoring.kafkaMessageQuery") }}</h4>
<button type="button" class="btn-sm" :disabled="kafkaMessageLoading || !kafkaMessageSql.trim()" @click="runKafkaMessageSql">
<Loader2 v-if="kafkaMessageLoading" class="btn-icon spinning" :size="14" />
<span>{{ kafkaMessageLoading ? t("mqMonitoring.querying") : t("mqMonitoring.queryMessages") }}</span>
</button>
</div>
<textarea v-model="kafkaMessageSql" class="kafka-sql-input" rows="2" spellcheck="false" />
<div class="query-hint">{{ t("mqMonitoring.queryHint") }}</div>
<div v-if="kafkaMessageError" class="panel-error inline-error">
<AlertTriangle :size="16" />
<span>{{ kafkaMessageError }}</span>
</div>
<div v-else-if="kafkaMessageLoading" class="empty-state compact">{{ t("mqMonitoring.messagesLoading") }}</div>
<div v-else-if="!kafkaMessages.length" class="empty-state compact">{{ t("mqMonitoring.noMessages") }}</div>
<div v-else class="kafka-message-list">
<article v-for="message in kafkaMessages" :key="`${message.properties?.partition ?? 'p'}-${message.messageId || message.position}`" class="kafka-message-row">
<div class="kafka-message-meta">
<span>#{{ message.position }}</span>
<span v-if="message.properties?.partition != null">{{ t("mqMonitoring.metaPartition", { partition: message.properties.partition }) }}</span>
<span>{{ t("mqMonitoring.metaOffset", { offset: message.messageId || "-" }) }}</span>
<span v-if="message.key">{{ t("mqMonitoring.metaKey", { key: message.key }) }}</span>
<span>{{ formatKafkaMessageTimestamp(message.publishTime) }}</span>
</div>
<pre class="kafka-message-payload">{{ kafkaMessagePayload(message) }}</pre>
<div v-if="Object.keys(message.headers || {}).length" class="kafka-message-headers">
<span v-for="(value, key) in message.headers" :key="key">{{ key }}: {{ value }}</span>
</div>
</article>
</div>
</div> </div>
</div> </div>
@@ -1266,87 +1168,6 @@ onUnmounted(() => {
margin-bottom: 0; margin-bottom: 0;
} }
.kafka-sql-input {
width: 100%;
min-height: 54px;
padding: 10px 12px;
border: 1px solid var(--monitor-border);
border-radius: var(--dbx-radius-fixed-6);
background: var(--monitor-surface);
color: var(--monitor-text);
font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace;
font-size: 12px;
line-height: 1.5;
resize: vertical;
box-sizing: border-box;
}
.kafka-sql-input:focus {
outline: none;
border-color: var(--monitor-accent);
box-shadow: 0 0 0 3px var(--monitor-accent-soft);
}
.query-hint {
margin-top: 6px;
color: var(--monitor-faint);
font-size: 12px;
}
.inline-error {
justify-content: flex-start;
margin-top: 10px;
border: 1px solid color-mix(in srgb, var(--monitor-danger) 22%, transparent);
border-radius: var(--dbx-radius-fixed-6);
padding: 10px 12px;
}
.kafka-message-list {
display: grid;
gap: 10px;
margin-top: 12px;
}
.kafka-message-row {
border: 1px solid var(--monitor-border);
border-radius: var(--dbx-radius-fixed-6);
background: var(--monitor-surface);
overflow: hidden;
}
.kafka-message-meta {
display: flex;
flex-wrap: wrap;
gap: 10px;
padding: 8px 10px;
border-bottom: 1px solid var(--monitor-border);
color: var(--monitor-muted);
font-size: 12px;
}
.kafka-message-payload {
max-height: 220px;
margin: 0;
overflow: auto;
padding: 10px;
color: var(--monitor-text);
font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace;
font-size: 12px;
line-height: 1.5;
white-space: pre-wrap;
word-break: break-word;
}
.kafka-message-headers {
display: flex;
flex-wrap: wrap;
gap: 6px;
padding: 8px 10px;
border-top: 1px solid var(--monitor-border);
color: var(--monitor-muted);
font-size: 11px;
}
.charts-grid { .charts-grid {
display: grid; display: grid;
grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); grid-template-columns: repeat(auto-fit, minmax(280px, 1fr));
@@ -1,11 +1,11 @@
<script setup lang="ts"> <script setup lang="ts">
import { computed, onUnmounted, ref, watch } from "vue"; import { computed, onUnmounted, ref, watch } from "vue";
import { useI18n } from "vue-i18n"; import { useI18n } from "vue-i18n";
import type { MqSystemKind, PeekedMessage, TopicInfo, TopicRef, SendMessageRequest, SendMessageResponse } from "@/types/mq"; import type { MqSystemKind, TopicInfo, TopicRef, SendMessageRequest, SendMessageResponse } from "@/types/mq";
import { mqSendMessage, mqListTopics, mqPeekMessages } from "@/lib/backend/api"; import { mqSendMessage, mqListTopics } from "@/lib/backend/api";
import { formatError } from "@/lib/backend/errorUtils"; import { formatError } from "@/lib/backend/errorUtils";
import { parseNonNegativeSafeInteger } from "@/lib/mq/mqPeekFilters";
import { resolveRabbitMqSendNamespace } from "@/lib/mq/mqConsoleDefaults"; import { resolveRabbitMqSendNamespace } from "@/lib/mq/mqConsoleDefaults";
import MessageBrowser from "./MessageBrowser.vue";
import RocketMqTopicSelect from "./shared/RocketMqTopicSelect.vue"; import RocketMqTopicSelect from "./shared/RocketMqTopicSelect.vue";
interface Props { interface Props {
@@ -38,13 +38,6 @@ const availableTopics = ref<TopicInfo[]>([]);
const topicsLoading = ref(false); const topicsLoading = ref(false);
const rocketMqTopicSelectRef = ref<InstanceType<typeof RocketMqTopicSelect>>(); const rocketMqTopicSelectRef = ref<InstanceType<typeof RocketMqTopicSelect>>();
const headersExpanded = ref(false); const headersExpanded = ref(false);
const peekAdvancedExpanded = ref(false);
const peekLoading = ref(false);
const peekError = ref<string>();
const peekMessages = ref<PeekedMessage[]>([]);
const peekPartition = ref<string | number>("");
const peekOffset = ref<string | number>("");
const peekCount = ref(20);
let successTimer: ReturnType<typeof setTimeout> | undefined; let successTimer: ReturnType<typeof setTimeout> | undefined;
@@ -76,11 +69,6 @@ const selectedTopicRef = computed<TopicRef | null>(() => {
}); });
const canBrowseMessages = computed(() => props.isFlatMqCluster === true && props.supportsPeekMessages !== false && !isRocketMqCluster.value); const canBrowseMessages = computed(() => props.isFlatMqCluster === true && props.supportsPeekMessages !== false && !isRocketMqCluster.value);
function flatMqPeekGroupName(): string {
if (props.mqSystemKind === "rocketmq") return "__dbx_rocketmq_viewer__";
return "__dbx_kafka_viewer__";
}
const topicListId = computed(() => `mq-topic-options-${props.connectionId}`); const topicListId = computed(() => `mq-topic-options-${props.connectionId}`);
function clearSuccessLater() { function clearSuccessLater() {
@@ -206,55 +194,6 @@ async function sendMessage() {
} }
} }
async function loadMessages() {
const topic = selectedTopicRef.value;
if (!topic) {
peekError.value = t("mqMessages.selectTopicBeforeLoad");
return;
}
peekLoading.value = true;
peekError.value = undefined;
try {
const count = Math.max(1, Math.min(100, Number(peekCount.value) || 20));
peekCount.value = count;
const options: { partition?: number; offset?: number } = {};
const partitionText = String(peekPartition.value).trim();
const offsetText = String(peekOffset.value).trim();
if (partitionText !== "") {
const partition = parseNonNegativeSafeInteger(partitionText);
if (partition == null) {
throw new Error(t("mqMessages.partitionMustBeNonNegativeInt"));
}
options.partition = partition;
peekPartition.value = String(partition);
}
if (offsetText !== "") {
const offset = parseNonNegativeSafeInteger(offsetText);
if (offset == null) {
throw new Error(t("mqMessages.offsetMustBeNonNegativeInt"));
}
options.offset = offset;
peekOffset.value = String(offset);
}
peekMessages.value = await mqPeekMessages(props.connectionId, topic, flatMqPeekGroupName(), count, options);
} catch (e: unknown) {
peekError.value = formatError(e);
} finally {
peekLoading.value = false;
}
}
function messagePayload(message: PeekedMessage): string {
return message.payloadText ?? message.payloadBase64;
}
function formatMessageTimestamp(value?: string): string {
if (!value) return "-";
const numeric = Number(value);
if (!Number.isFinite(numeric)) return value;
return new Date(numeric).toLocaleString();
}
function formatJson() { function formatJson() {
try { try {
const parsed = JSON.parse(messageValue.value); const parsed = JSON.parse(messageValue.value);
@@ -283,11 +222,6 @@ watch(
{ immediate: true }, { immediate: true },
); );
watch(topicName, () => {
peekError.value = undefined;
peekMessages.value = [];
});
watch( watch(
() => props.topic, () => props.topic,
(newTopic) => { (newTopic) => {
@@ -387,58 +321,7 @@ watch(
</button> </button>
</div> </div>
<section v-if="canBrowseMessages" class="message-browser"> <MessageBrowser v-if="canBrowseMessages" :connection-id="connectionId" :topic="selectedTopicRef" :mq-system-kind="mqSystemKind" />
<div class="message-browser-header">
<h4>{{ t("mqMessages.messageList") }}</h4>
<button type="button" class="btn-sm" :disabled="peekLoading || !selectedTopicRef" @click="loadMessages">
{{ peekLoading ? t("mqMessages.loading") : t("mqMessages.loadMessages") }}
</button>
</div>
<p class="peek-default-hint">{{ t("mqMessages.peekDefaultHint", { count: peekCount }) }}</p>
<div class="peek-controls">
<label>
<span>{{ t("mqMessages.count") }}</span>
<input v-model.number="peekCount" type="number" min="1" max="100" :disabled="peekLoading" />
</label>
</div>
<button type="button" class="collapse-toggle peek-advanced-toggle" @click="peekAdvancedExpanded = !peekAdvancedExpanded">
<span class="collapse-arrow" :class="{ expanded: peekAdvancedExpanded }">▶</span>
<span>{{ t("mqMessages.advancedFilter") }}</span>
<span v-if="(peekPartition || peekOffset) && !peekAdvancedExpanded" class="collapse-badge">·</span>
</button>
<div v-if="peekAdvancedExpanded" class="peek-controls collapse-body">
<label>
<span>{{ t("mqMessages.partition") }}</span>
<input v-model="peekPartition" type="number" min="0" :placeholder="t('mqMessages.partitionPlaceholderAll')" :disabled="peekLoading" />
</label>
<label>
<span>{{ t("mqMessages.offset") }}</span>
<input v-model="peekOffset" type="number" min="0" :placeholder="t('mqMessages.offsetPlaceholderEarliest')" :disabled="peekLoading" />
</label>
</div>
<div v-if="peekError" class="panel-error">{{ peekError }}</div>
<div v-else-if="peekLoading" class="message-empty">{{ t("mqMessages.messagesLoading") }}</div>
<div v-else-if="!peekMessages.length" class="message-empty">{{ t("mqMessages.noMessages") }}</div>
<div v-else class="message-list">
<article v-for="message in peekMessages" :key="`${message.properties?.partition ?? 'p'}-${message.messageId || message.position}`" class="message-row">
<div class="message-meta">
<span>#{{ message.position }}</span>
<span v-if="message.properties?.partition != null">{{ t("mqMessages.metaPartition", { partition: message.properties.partition }) }}</span>
<span>{{ t("mqMessages.metaOffset", { offset: message.messageId || "-" }) }}</span>
<span v-if="message.key">{{ t("mqMessages.metaKey", { key: message.key }) }}</span>
<span>{{ formatMessageTimestamp(message.publishTime) }}</span>
</div>
<pre class="message-payload">{{ messagePayload(message) }}</pre>
<div v-if="Object.keys(message.headers || {}).length" class="message-headers">
<span v-for="(value, key) in message.headers" :key="key">{{ key }}: {{ value }}</span>
</div>
</article>
</div>
</section>
</div> </div>
</div> </div>
</template> </template>
@@ -733,151 +616,6 @@ input[type="number"]:focus {
opacity: 0.9; opacity: 0.9;
} }
.query-mode-tabs {
display: flex;
flex-wrap: wrap;
gap: 8px;
margin-bottom: 12px;
}
.query-mode-tabs .btn-sm.active {
border-color: var(--color-primary);
color: var(--color-primary);
background: var(--color-primary-alpha);
}
.message-browser {
margin-top: 4px;
padding: 14px;
border: 1px solid var(--color-border);
border-radius: var(--dbx-radius-fixed-6);
background: var(--color-background-secondary);
}
.message-browser-header {
display: flex;
align-items: center;
justify-content: space-between;
gap: 12px;
margin-bottom: 12px;
}
.message-browser-header h4 {
margin: 0;
color: var(--color-text);
font-size: 14px;
font-weight: 600;
}
.peek-default-hint {
margin: 0 0 12px;
padding: 8px 10px;
border-radius: var(--dbx-radius-fixed-6);
background: color-mix(in srgb, var(--color-primary) 8%, transparent);
color: var(--color-text-secondary);
font-size: 12px;
line-height: 1.5;
}
.peek-advanced-toggle {
margin-bottom: 10px;
}
.peek-controls {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(120px, 1fr));
gap: 10px;
margin-bottom: 12px;
}
.peek-controls label {
display: flex;
flex-direction: column;
gap: 5px;
color: var(--color-text-secondary);
font-size: 12px;
font-weight: 500;
}
.peek-controls input {
width: 100%;
}
.message-empty {
padding: 18px;
border: 1px dashed var(--color-border);
border-radius: var(--dbx-radius-fixed-6);
color: var(--color-text-tertiary);
text-align: center;
font-size: 13px;
}
.message-list {
display: flex;
flex-direction: column;
gap: 10px;
max-height: 360px;
overflow: auto;
}
.message-row {
padding: 10px 12px;
border: 1px solid var(--color-border);
border-radius: var(--dbx-radius-fixed-6);
background: var(--color-background);
}
.message-meta {
display: flex;
align-items: center;
gap: 10px;
flex-wrap: wrap;
color: var(--color-text-tertiary);
font-size: 12px;
}
.message-meta span:first-child {
color: var(--color-primary);
font-weight: 700;
}
.message-payload {
margin: 8px 0 0;
padding: 10px;
max-height: 160px;
overflow: auto;
border-radius: var(--dbx-radius-fixed-6);
background: var(--color-background-tertiary, var(--color-background-secondary));
color: var(--color-text);
font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace;
font-size: 12px;
line-height: 1.5;
white-space: pre-wrap;
word-break: break-word;
}
.message-headers {
display: flex;
gap: 6px;
flex-wrap: wrap;
margin-top: 8px;
}
.message-headers span {
padding: 2px 6px;
border: 1px solid var(--color-border);
border-radius: var(--dbx-radius-fixed-4);
color: var(--color-text-secondary);
background: var(--color-background-secondary);
font-size: 12px;
}
@media (max-width: 720px) {
.peek-controls {
grid-template-columns: 1fr;
}
}
button:disabled, button:disabled,
input:disabled, input:disabled,
textarea:disabled, textarea:disabled,
@@ -43,7 +43,9 @@ const showExpireDialog = ref(false);
const selectedSub = ref<SubscriptionInfo>(); const selectedSub = ref<SubscriptionInfo>();
const peekedMessages = ref<PeekedMessage[]>([]); const peekedMessages = ref<PeekedMessage[]>([]);
const peekLoading = ref(false); const peekLoading = ref(false);
const peekIncomplete = ref(false);
const peekCount = ref(5); const peekCount = ref(5);
let peekRequestVersion = 0;
const deleteTarget = ref<SubscriptionInfo>(); const deleteTarget = ref<SubscriptionInfo>();
const showDeleteDialog = ref(false); const showDeleteDialog = ref(false);
const deleting = ref(false); const deleting = ref(false);
@@ -212,6 +214,7 @@ function openPeekDialog(sub: SubscriptionInfo) {
selectedSub.value = sub; selectedSub.value = sub;
peekCount.value = 5; peekCount.value = 5;
peekedMessages.value = []; peekedMessages.value = [];
peekIncomplete.value = false;
showPeekDialog.value = true; showPeekDialog.value = true;
void handlePeekMessages(); void handlePeekMessages();
} }
@@ -344,20 +347,39 @@ async function confirmClearBacklog() {
async function handlePeekMessages() { async function handlePeekMessages() {
const topicRef = getPulsarTopicRef(); const topicRef = getPulsarTopicRef();
if (!selectedSub.value || !topicRef) return; const sub = selectedSub.value;
if (!sub || !topicRef) return;
const requestVersion = ++peekRequestVersion;
const count = Math.max(1, Math.min(100, Number(peekCount.value) || 1)); const count = Math.max(1, Math.min(100, Number(peekCount.value) || 1));
peekCount.value = count; peekCount.value = count;
peekLoading.value = true; peekLoading.value = true;
peekIncomplete.value = false;
error.value = undefined; error.value = undefined;
try { try {
peekedMessages.value = await mqPeekMessages(props.connectionId, topicRef, selectedSub.value.name, count); const result = await mqPeekMessages(props.connectionId, topicRef, sub.name, count);
if (requestVersion === peekRequestVersion) {
if (Array.isArray(result)) {
peekedMessages.value = result;
peekIncomplete.value = false;
} else {
peekedMessages.value = result.messages;
peekIncomplete.value = result.incomplete;
}
}
} catch (e: unknown) { } catch (e: unknown) {
error.value = formatError(e); if (requestVersion === peekRequestVersion) error.value = formatError(e);
} finally { } finally {
peekLoading.value = false; if (requestVersion === peekRequestVersion) peekLoading.value = false;
} }
} }
function invalidatePeekRequest() {
peekRequestVersion += 1;
peekLoading.value = false;
peekedMessages.value = [];
peekIncomplete.value = false;
}
async function handleExpireMessages() { async function handleExpireMessages() {
if (!guardWritable()) return; if (!guardWritable()) return;
const topicRef = getPulsarTopicRef(); const topicRef = getPulsarTopicRef();
@@ -379,6 +401,7 @@ watch(
() => [props.topic, props.tenant, props.namespace, props.mqSystemKind], () => [props.topic, props.tenant, props.namespace, props.mqSystemKind],
() => { () => {
selectedSub.value = undefined; selectedSub.value = undefined;
invalidatePeekRequest();
loadSubscriptions(); loadSubscriptions();
}, },
{ immediate: true }, { immediate: true },
@@ -605,6 +628,9 @@ watch(
{{ peekLoading ? t("mqSubscriptions.loading") : t("mqSubscriptions.refresh") }} {{ peekLoading ? t("mqSubscriptions.loading") : t("mqSubscriptions.refresh") }}
</button> </button>
</div> </div>
<div v-if="peekIncomplete" class="form-warning" role="status" data-testid="peek-incomplete">
{{ t("mqMessages.peekIncomplete") }}
</div>
<div v-if="error" class="form-error">{{ error }}</div> <div v-if="error" class="form-error">{{ error }}</div>
<div v-else-if="peekLoading && !peekedMessages.length" class="panel-loading">{{ t("mqSubscriptions.loading") }}</div> <div v-else-if="peekLoading && !peekedMessages.length" class="panel-loading">{{ t("mqSubscriptions.loading") }}</div>
<div v-else-if="!peekedMessages.length" class="panel-placeholder">{{ t("mqSubscriptions.noPeekMessages") }}</div> <div v-else-if="!peekedMessages.length" class="panel-placeholder">{{ t("mqSubscriptions.noPeekMessages") }}</div>
@@ -1050,6 +1076,16 @@ button:disabled {
font-size: 13px; font-size: 13px;
} }
.form-warning {
margin-top: 12px;
padding: 8px 12px;
border: 1px solid var(--color-warning-border, #d99a22);
border-radius: var(--dbx-radius-fixed-4);
background: var(--color-warning-background, #fff6df);
color: var(--color-warning-text, #7a4a00);
font-size: 13px;
}
.peek-toolbar { .peek-toolbar {
display: flex; display: flex;
align-items: end; align-items: end;
@@ -0,0 +1,458 @@
// @vitest-environment happy-dom
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { createApp, h, nextTick, ref, type App } from "vue";
import type { MqSystemKind, PeekedMessage } from "@/types/mq";
const backend = vi.hoisted(() => ({
mqPeekMessages: vi.fn(),
}));
vi.mock("vue-i18n", () => ({
useI18n: () => ({ t: (key: string) => key }),
}));
vi.mock("@/lib/backend/api", () => ({
mqPeekMessages: backend.mqPeekMessages,
}));
vi.mock("@/components/ui/select", async () => (await import("./selectStub")).createSelectStub());
import MessageBrowser from "@/components/mq/MessageBrowser.vue";
const TOPIC = {
tenant: "_kafka",
namespace: "default",
topic: "events",
persistent: true,
partitioned: false,
};
let app: App<Element> | null = null;
let root: HTMLDivElement | null = null;
async function flushUi() {
await Promise.resolve();
await Promise.resolve();
await nextTick();
}
function buttonByText(container: ParentNode, text: string): HTMLButtonElement {
const button = [...container.querySelectorAll<HTMLButtonElement>("button")].find((item) => item.textContent?.includes(text));
if (!button) throw new Error(`Button not found: ${text}`);
return button;
}
async function setInputValue(input: HTMLInputElement, value: string) {
input.value = value;
input.dispatchEvent(new Event("input", { bubbles: true }));
await nextTick();
}
async function setSelectValue(trigger: HTMLElement, value: string) {
trigger.click();
await flushUi();
const item = document.querySelector<HTMLElement>(`[data-slot="select-item"][data-value="${value}"]`);
if (!item) throw new Error(`Select item not found: ${value}`);
item.click();
await flushUi();
}
async function mountBrowser(mqSystemKind: "kafka" | "rabbitmq" = "kafka") {
root = document.createElement("div");
document.body.appendChild(root);
app = createApp(MessageBrowser, {
connectionId: "mq-1",
topic: { ...TOPIC, tenant: mqSystemKind === "rabbitmq" ? "_rabbitmq" : "_kafka" },
mqSystemKind,
});
app.mount(root);
await flushUi();
return root;
}
async function mountBrowserWithMutableTopic() {
const topic = ref({ ...TOPIC });
const connectionId = ref("mq-1");
const mqSystemKind = ref<MqSystemKind>("kafka");
root = document.createElement("div");
document.body.appendChild(root);
app = createApp({
setup: () => () =>
h(MessageBrowser, {
connectionId: connectionId.value,
topic: topic.value,
mqSystemKind: mqSystemKind.value,
}),
});
app.mount(root);
await flushUi();
return { browser: root, topic, connectionId, mqSystemKind };
}
function deferred<T>() {
let resolve!: (value: T) => void;
let reject!: (reason?: unknown) => void;
const promise = new Promise<T>((resolvePromise, rejectPromise) => {
resolve = resolvePromise;
reject = rejectPromise;
});
return { promise, resolve, reject };
}
async function loadMessages(container: ParentNode) {
buttonByText(container, "mqMessages.loadMessages").click();
await flushUi();
}
beforeEach(() => {
backend.mqPeekMessages.mockReset();
backend.mqPeekMessages.mockResolvedValue([
{
position: 1,
messageId: "17",
payloadBase64: "",
payloadText: "existing message",
properties: { partition: "0" },
headers: {},
},
]);
});
afterEach(() => {
app?.unmount();
app = null;
root?.remove();
root = null;
});
describe("MessageBrowser", () => {
it("loads Kafka's latest messages by default", async () => {
const browser = await mountBrowser();
await loadMessages(browser);
expect(backend.mqPeekMessages).toHaveBeenCalledWith("mq-1", expect.objectContaining({ topic: "events" }), "__dbx_kafka_viewer__", 20, { startPosition: "latest" });
});
it("normalizes a decimal count before sending the request", async () => {
const browser = await mountBrowser();
const countInput = browser.querySelector<HTMLInputElement>('[data-testid="peek-count"]');
if (!countInput) throw new Error("Peek count input not found");
await setInputValue(countInput, "2.9");
await loadMessages(browser);
expect(countInput.value).toBe("2");
expect(backend.mqPeekMessages).toHaveBeenCalledWith("mq-1", expect.objectContaining({ topic: "events" }), "__dbx_kafka_viewer__", 2, { startPosition: "latest" });
});
it("shows an explicit warning when the broker returns an incomplete snapshot", async () => {
backend.mqPeekMessages.mockResolvedValueOnce({
messages: [
{
position: 1,
messageId: "partial",
payloadBase64: "",
payloadText: "partial message",
properties: {},
headers: {},
},
],
incomplete: true,
});
const browser = await mountBrowser();
await loadMessages(browser);
expect(browser.querySelector('[data-testid="peek-incomplete"]')?.textContent).toContain("mqMessages.peekIncomplete");
expect(browser.textContent).toContain("partial message");
});
it("sends explicit earliest and offset read positions", async () => {
const browser = await mountBrowser();
const startPosition = browser.querySelector<HTMLElement>('[data-testid="kafka-peek-start-position"]');
if (!startPosition) throw new Error("Kafka start position select not found");
await setSelectValue(startPosition, "earliest");
await loadMessages(browser);
expect(backend.mqPeekMessages).toHaveBeenLastCalledWith("mq-1", expect.objectContaining({ topic: "events" }), "__dbx_kafka_viewer__", 20, { startPosition: "earliest" });
await setSelectValue(startPosition, "offset");
const partition = browser.querySelector<HTMLInputElement>('[data-testid="kafka-peek-partition"]');
const offset = browser.querySelector<HTMLInputElement>('[data-testid="kafka-peek-offset"]');
if (!partition || !offset) throw new Error("Kafka offset inputs not found");
await setInputValue(partition, "2");
await setInputValue(offset, "17");
await loadMessages(browser);
expect(backend.mqPeekMessages).toHaveBeenLastCalledWith("mq-1", expect.objectContaining({ topic: "events" }), "__dbx_kafka_viewer__", 20, { startPosition: "offset", partition: 2, offset: 17 });
});
it("allows an all-partition offset read but requires an offset", async () => {
const browser = await mountBrowser();
const startPosition = browser.querySelector<HTMLElement>('[data-testid="kafka-peek-start-position"]');
if (!startPosition) throw new Error("Kafka start position control not found");
await setSelectValue(startPosition, "offset");
await loadMessages(browser);
expect(backend.mqPeekMessages).not.toHaveBeenCalled();
expect(browser.textContent).toContain("mqMessages.offsetRequiredForOffset");
const offset = browser.querySelector<HTMLInputElement>('[data-testid="kafka-peek-offset"]');
if (!offset) throw new Error("Kafka offset input not found");
await setInputValue(offset, "-1");
await loadMessages(browser);
expect(backend.mqPeekMessages).not.toHaveBeenCalled();
expect(browser.textContent).toContain("mqMessages.offsetMustBeNonNegativeIntRequired");
await setInputValue(offset, "17");
await loadMessages(browser);
expect(backend.mqPeekMessages).toHaveBeenLastCalledWith("mq-1", expect.objectContaining({ topic: "events" }), "__dbx_kafka_viewer__", 20, { startPosition: "offset", offset: 17 });
});
it("clears results and does not leak an offset into a different read mode", async () => {
const browser = await mountBrowser();
const startPosition = browser.querySelector<HTMLElement>('[data-testid="kafka-peek-start-position"]');
const partition = browser.querySelector<HTMLInputElement>('[data-testid="kafka-peek-partition"]');
if (!startPosition || !partition) throw new Error("Kafka start position controls not found");
await loadMessages(browser);
expect(browser.textContent).toContain("existing message");
await setSelectValue(startPosition, "offset");
expect(browser.textContent).not.toContain("existing message");
await setInputValue(partition, "2");
const offset = browser.querySelector<HTMLInputElement>('[data-testid="kafka-peek-offset"]');
if (!offset) throw new Error("Kafka offset input not found");
await setInputValue(offset, "17");
await setSelectValue(startPosition, "latest");
await loadMessages(browser);
expect(backend.mqPeekMessages).toHaveBeenLastCalledWith("mq-1", expect.objectContaining({ topic: "events" }), "__dbx_kafka_viewer__", 20, { startPosition: "latest", partition: 2 });
});
it("ignores an older topic request that resolves after the current request", async () => {
const first = deferred<PeekedMessage[]>();
const second = deferred<PeekedMessage[]>();
backend.mqPeekMessages.mockReturnValueOnce(first.promise).mockReturnValueOnce(second.promise);
const { browser, topic } = await mountBrowserWithMutableTopic();
await loadMessages(browser);
topic.value = { ...TOPIC, topic: "payments" };
await flushUi();
await loadMessages(browser);
second.resolve([
{
position: 1,
messageId: "new",
payloadBase64: "",
payloadText: "new topic message",
properties: {},
headers: {},
},
]);
await flushUi();
expect(browser.textContent).toContain("new topic message");
first.resolve([
{
position: 1,
messageId: "old",
payloadBase64: "",
payloadText: "old topic message",
properties: {},
headers: {},
},
]);
await flushUi();
expect(browser.textContent).toContain("new topic message");
expect(browser.textContent).not.toContain("old topic message");
});
it("invalidates a request when the topic persistence changes", async () => {
const first = deferred<PeekedMessage[]>();
const second = deferred<PeekedMessage[]>();
backend.mqPeekMessages.mockReturnValueOnce(first.promise).mockReturnValueOnce(second.promise);
const { browser, topic } = await mountBrowserWithMutableTopic();
await loadMessages(browser);
topic.value = { ...TOPIC, persistent: false };
await flushUi();
await loadMessages(browser);
second.resolve([
{
position: 1,
messageId: "non-persistent",
payloadBase64: "",
payloadText: "new persistence message",
properties: {},
headers: {},
},
]);
await flushUi();
first.resolve([
{
position: 1,
messageId: "persistent",
payloadBase64: "",
payloadText: "stale persistence message",
properties: {},
headers: {},
},
]);
await flushUi();
expect(browser.textContent).toContain("new persistence message");
expect(browser.textContent).not.toContain("stale persistence message");
});
it("ignores an older topic request failure after the current request succeeds", async () => {
const first = deferred<PeekedMessage[]>();
const second = deferred<PeekedMessage[]>();
backend.mqPeekMessages.mockReturnValueOnce(first.promise).mockReturnValueOnce(second.promise);
const { browser, topic } = await mountBrowserWithMutableTopic();
await loadMessages(browser);
topic.value = { ...TOPIC, topic: "payments" };
await flushUi();
await loadMessages(browser);
second.resolve([
{
position: 1,
messageId: "new",
payloadBase64: "",
payloadText: "new topic message",
properties: {},
headers: {},
},
]);
await flushUi();
first.reject(new Error("old topic request failed"));
await flushUi();
expect(browser.textContent).toContain("new topic message");
expect(browser.textContent).not.toContain("old topic request failed");
});
it("ignores an older connection request after the connection changes", async () => {
const first = deferred<PeekedMessage[]>();
const second = deferred<PeekedMessage[]>();
backend.mqPeekMessages.mockReturnValueOnce(first.promise).mockReturnValueOnce(second.promise);
const { browser, connectionId } = await mountBrowserWithMutableTopic();
await loadMessages(browser);
connectionId.value = "mq-2";
await flushUi();
await loadMessages(browser);
second.resolve([
{
position: 1,
messageId: "new-connection",
payloadBase64: "",
payloadText: "new connection message",
properties: {},
headers: {},
},
]);
await flushUi();
expect(browser.textContent).toContain("new connection message");
first.resolve([
{
position: 1,
messageId: "old-connection",
payloadBase64: "",
payloadText: "old connection message",
properties: {},
headers: {},
},
]);
await flushUi();
expect(browser.textContent).toContain("new connection message");
expect(browser.textContent).not.toContain("old connection message");
});
it("ignores an older start-mode request that resolves after the mode switches", async () => {
const first = deferred<PeekedMessage[]>();
backend.mqPeekMessages.mockReturnValueOnce(first.promise);
const browser = await mountBrowser();
const startPosition = browser.querySelector<HTMLElement>('[data-testid="kafka-peek-start-position"]');
if (!startPosition) throw new Error("Kafka start position select not found");
await loadMessages(browser);
await setSelectValue(startPosition, "earliest");
expect(browser.textContent).not.toContain("latest-mode message");
first.resolve([
{
position: 1,
messageId: "stale",
payloadBase64: "",
payloadText: "latest-mode message",
properties: {},
headers: {},
},
]);
await flushUi();
expect(browser.textContent).not.toContain("latest-mode message");
expect(browser.querySelector(".panel-error")).toBeNull();
expect(browser.textContent).toContain("mqMessages.noMessages");
});
it("ignores an older start-mode failure after the mode switches and a new load succeeds", async () => {
const first = deferred<PeekedMessage[]>();
const second = deferred<PeekedMessage[]>();
backend.mqPeekMessages.mockReturnValueOnce(first.promise).mockReturnValueOnce(second.promise);
const browser = await mountBrowser();
const startPosition = browser.querySelector<HTMLElement>('[data-testid="kafka-peek-start-position"]');
if (!startPosition) throw new Error("Kafka start position select not found");
await loadMessages(browser);
await setSelectValue(startPosition, "earliest");
await loadMessages(browser);
second.resolve([
{
position: 1,
messageId: "fresh",
payloadBase64: "",
payloadText: "earliest-mode message",
properties: {},
headers: {},
},
]);
await flushUi();
expect(browser.textContent).toContain("earliest-mode message");
first.reject(new Error("latest-mode request failed"));
await flushUi();
expect(browser.textContent).toContain("earliest-mode message");
expect(browser.textContent).not.toContain("latest-mode request failed");
});
it("keeps RabbitMQ's advanced filters collapsed and sends its existing request shape", async () => {
const browser = await mountBrowser("rabbitmq");
expect(browser.querySelector('input[placeholder="mqMessages.partitionPlaceholderAll"]')).toBeNull();
buttonByText(browser, "mqMessages.advancedFilter").click();
await nextTick();
const partition = browser.querySelector<HTMLInputElement>('input[placeholder="mqMessages.partitionPlaceholderAll"]');
const offset = browser.querySelector<HTMLInputElement>('input[placeholder="mqMessages.offsetPlaceholderEarliest"]');
if (!partition || !offset) throw new Error("RabbitMQ advanced filter inputs not found");
await setInputValue(partition, "2");
await setInputValue(offset, "17");
await loadMessages(browser);
expect(backend.mqPeekMessages).toHaveBeenCalledWith("mq-1", expect.objectContaining({ tenant: "_rabbitmq", topic: "events" }), "__dbx_kafka_viewer__", 20, { partition: 2, offset: 17 });
});
});
@@ -0,0 +1,181 @@
// @vitest-environment happy-dom
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { createApp, nextTick, type App } from "vue";
const backend = vi.hoisted(() => ({
mqGetTopicStats: vi.fn(),
mqGetBacklog: vi.fn(),
mqPeekMessages: vi.fn(),
}));
vi.mock("vue-i18n", () => ({
useI18n: () => ({ t: (key: string) => key }),
}));
vi.mock("@/lib/backend/api", () => backend);
vi.mock("@/components/ui/select", async () => (await import("./selectStub")).createSelectStub());
vi.mock("echarts/core", () => ({ use: vi.fn() }));
vi.mock("echarts/renderers", () => ({ CanvasRenderer: {} }));
vi.mock("echarts/charts", () => ({ LineChart: {} }));
vi.mock("echarts/components", () => ({ GridComponent: {}, LegendComponent: {}, TooltipComponent: {} }));
vi.mock("vue-echarts", async () => {
const { defineComponent, h } = await import("vue");
return { default: defineComponent({ name: "VChart", setup: () => () => h("div") }) };
});
vi.mock("@lucide/vue", async () => {
const { defineComponent, h } = await import("vue");
const Icon = defineComponent({ name: "Icon", setup: () => () => h("i") });
return {
Activity: Icon,
AlertTriangle: Icon,
BarChart3: Icon,
Boxes: Icon,
CheckCircle2: Icon,
Database: Icon,
Download: Icon,
Gauge: Icon,
Hash: Icon,
HardDrive: Icon,
Layers3: Icon,
Loader2: Icon,
Package: Icon,
RadioTower: Icon,
RefreshCw: Icon,
Send: Icon,
ShieldCheck: Icon,
Table2: Icon,
Upload: Icon,
Users: Icon,
};
});
import MonitoringPanel from "@/components/mq/MonitoringPanel.vue";
const TOPIC = {
name: "persistent://public/default/test",
shortName: "test",
persistent: true,
partitioned: false,
};
const KAFKA_STATS = {
msgRateIn: 0,
msgRateOut: 0,
msgThroughputIn: 0,
msgThroughputOut: 0,
storageSize: 0,
backlogSize: 0,
msgInCounter: 19,
msgOutCounter: 0,
subscriptionCount: 0,
producerCount: 0,
raw: {
partitions: 1,
replicationFactor: 1,
totalMessages: 19,
partitionStats: [{ partition: 0, beginOffset: 0, endOffset: 19, messageCount: 19, leader: 1, replicas: [1], isr: [1] }],
},
};
let app: App<Element> | null = null;
let root: HTMLDivElement | null = null;
async function flushUi() {
await Promise.resolve();
await Promise.resolve();
await nextTick();
}
async function mountPanel() {
root = document.createElement("div");
document.body.appendChild(root);
app = createApp(MonitoringPanel, {
connectionId: "mq-1",
tenant: "_kafka",
namespace: "default",
topic: TOPIC,
mqSystemKind: "kafka",
});
app.mount(root);
await flushUi();
return root;
}
beforeEach(() => {
backend.mqGetTopicStats.mockReset();
backend.mqGetBacklog.mockReset();
backend.mqPeekMessages.mockReset();
backend.mqGetTopicStats.mockResolvedValue(KAFKA_STATS);
backend.mqPeekMessages.mockResolvedValue([]);
});
afterEach(() => {
app?.unmount();
app = null;
root?.remove();
root = null;
});
describe("MonitoringPanel Kafka message browser", () => {
it("reuses the explicit Kafka browser instead of rendering a SQL query field", async () => {
const panel = await mountPanel();
const browser = panel.querySelector<HTMLElement>('[data-testid="message-browser"]');
const loadButton = [...panel.querySelectorAll<HTMLButtonElement>("button")].find((button) => button.textContent?.includes("mqMessages.loadMessages"));
if (!browser || !loadButton) throw new Error("Kafka message browser not found");
expect(panel.querySelector("textarea")).toBeNull();
expect(browser.classList.contains("message-browser")).toBe(true);
expect(browser.classList.contains("is-monitoring")).toBe(true);
expect(browser.querySelector('[data-testid="kafka-peek-start-position"]')).not.toBeNull();
loadButton.click();
await flushUi();
expect(backend.mqPeekMessages).toHaveBeenCalledWith("mq-1", expect.objectContaining({ topic: "test" }), "__dbx_kafka_viewer__", 20, { startPosition: "latest" });
});
it("keeps automatic monitoring refresh separate from message browsing", async () => {
vi.useFakeTimers();
try {
const panel = await mountPanel();
const browser = panel.querySelector<HTMLElement>('[data-testid="message-browser"]');
const loadButton = [...panel.querySelectorAll<HTMLButtonElement>("button")].find((button) => button.textContent?.includes("mqMessages.loadMessages"));
if (!browser || !loadButton) throw new Error("Kafka message browser not found");
backend.mqPeekMessages.mockResolvedValueOnce([
{
position: 1,
messageId: "18",
payloadBase64: "",
payloadText: "loaded message",
properties: { partition: "0" },
headers: {},
},
]);
loadButton.click();
await flushUi();
expect(browser.textContent).toContain("loaded message");
expect(backend.mqPeekMessages).toHaveBeenCalledTimes(1);
await vi.advanceTimersByTimeAsync(5_000);
await flushUi();
expect(backend.mqGetTopicStats).toHaveBeenCalledTimes(2);
expect(backend.mqPeekMessages).toHaveBeenCalledTimes(1);
expect(browser.textContent).toContain("loaded message");
backend.mqGetTopicStats.mockRejectedValueOnce(new Error("monitoring refresh failed"));
await vi.advanceTimersByTimeAsync(5_000);
await flushUi();
expect(backend.mqGetTopicStats).toHaveBeenCalledTimes(3);
expect(backend.mqPeekMessages).toHaveBeenCalledTimes(1);
expect(browser.isConnected).toBe(true);
expect(browser.textContent).toContain("loaded message");
} finally {
vi.useRealTimers();
}
});
});
@@ -19,6 +19,8 @@ vi.mock("@/lib/backend/api", () => ({
mqSendMessage: backend.mqSendMessage, mqSendMessage: backend.mqSendMessage,
})); }));
vi.mock("@/components/ui/select", async () => (await import("./selectStub")).createSelectStub());
import SendMessagePanel from "@/components/mq/SendMessagePanel.vue"; import SendMessagePanel from "@/components/mq/SendMessagePanel.vue";
const TOPIC = { const TOPIC = {
@@ -51,6 +53,15 @@ async function setInputValue(input: HTMLInputElement | HTMLTextAreaElement, valu
await nextTick(); await nextTick();
} }
async function setSelectValue(trigger: HTMLElement, value: string) {
trigger.click();
await flushUi();
const item = document.querySelector<HTMLElement>(`[data-slot="select-item"][data-value="${value}"]`);
if (!item) throw new Error(`Select item not found: ${value}`);
item.click();
await flushUi();
}
function expectedTopic(system: BrowseableSystem) { function expectedTopic(system: BrowseableSystem) {
return { return {
tenant: system === "rabbitmq" ? "_rabbitmq" : "_kafka", tenant: system === "rabbitmq" ? "_rabbitmq" : "_kafka",
@@ -124,8 +135,10 @@ describe("SendMessagePanel post-send browsing", () => {
it(`keeps ${system} results unchanged after sending`, async () => { it(`keeps ${system} results unchanged after sending`, async () => {
const panel = await mountPanel(system); const panel = await mountPanel(system);
expect(panel.querySelector('[data-testid="message-browser"]')?.classList.contains("message-browser")).toBe(true);
expect(panel.querySelector('[data-testid="message-browser"]')?.classList.contains("is-monitoring")).toBe(false);
await loadMessages(panel); await loadMessages(panel);
expect(backend.mqPeekMessages).toHaveBeenCalledWith("mq-1", expectedTopic(system), "__dbx_kafka_viewer__", 20, {}); expect(backend.mqPeekMessages).toHaveBeenCalledWith("mq-1", expectedTopic(system), "__dbx_kafka_viewer__", 20, system === "kafka" ? { startPosition: "latest" } : {});
expect(panel.textContent).toContain("existing message"); expect(panel.textContent).toContain("existing message");
await sendMessage(panel, "new message"); await sendMessage(panel, "new message");
@@ -135,14 +148,19 @@ describe("SendMessagePanel post-send browsing", () => {
}); });
} }
it("preserves Kafka advanced filters until the user explicitly loads messages", async () => { it("sends Kafka's selected start position and specific offset", async () => {
const panel = await mountPanel("kafka"); const panel = await mountPanel("kafka");
buttonByText(panel, "mqMessages.advancedFilter").click(); const startPosition = panel.querySelector<HTMLElement>('[data-testid="kafka-peek-start-position"]');
await flushUi(); if (!startPosition) throw new Error("Kafka start position select not found");
const partitionInput = panel.querySelector<HTMLInputElement>('input[placeholder="mqMessages.partitionPlaceholderAll"]'); await setSelectValue(startPosition, "earliest");
const offsetInput = panel.querySelector<HTMLInputElement>('input[placeholder="mqMessages.offsetPlaceholderEarliest"]'); await loadMessages(panel);
if (!partitionInput || !offsetInput) throw new Error("Advanced filter inputs not found"); expect(backend.mqPeekMessages).toHaveBeenLastCalledWith("mq-1", expectedTopic("kafka"), "__dbx_kafka_viewer__", 20, { startPosition: "earliest" });
await setSelectValue(startPosition, "offset");
const partitionInput = panel.querySelector<HTMLInputElement>('[data-testid="kafka-peek-partition"]');
const offsetInput = panel.querySelector<HTMLInputElement>('[data-testid="kafka-peek-offset"]');
if (!partitionInput || !offsetInput) throw new Error("Kafka offset inputs not found");
await setInputValue(partitionInput, "2"); await setInputValue(partitionInput, "2");
await setInputValue(offsetInput, "17"); await setInputValue(offsetInput, "17");
@@ -150,9 +168,99 @@ describe("SendMessagePanel post-send browsing", () => {
expect(partitionInput.value).toBe("2"); expect(partitionInput.value).toBe("2");
expect(offsetInput.value).toBe("17"); expect(offsetInput.value).toBe("17");
expect(backend.mqPeekMessages).not.toHaveBeenCalled(); expect(backend.mqPeekMessages).toHaveBeenCalledTimes(1);
await loadMessages(panel); await loadMessages(panel);
expect(backend.mqPeekMessages).toHaveBeenCalledWith("mq-1", expectedTopic("kafka"), "__dbx_kafka_viewer__", 20, { partition: 2, offset: 17 }); expect(backend.mqPeekMessages).toHaveBeenLastCalledWith("mq-1", expectedTopic("kafka"), "__dbx_kafka_viewer__", 20, { startPosition: "offset", partition: 2, offset: 17 });
});
it("keeps Kafka's result limit within the supported range", async () => {
const panel = await mountPanel("kafka");
const countInput = panel.querySelector<HTMLInputElement>('[data-testid="peek-count"]');
if (!countInput) throw new Error("Peek count input not found");
await setInputValue(countInput, "999");
await loadMessages(panel);
expect(countInput.value).toBe("100");
expect(backend.mqPeekMessages).toHaveBeenCalledWith("mq-1", expectedTopic("kafka"), "__dbx_kafka_viewer__", 100, { startPosition: "latest" });
await setInputValue(countInput, "");
await loadMessages(panel);
expect(countInput.value).toBe("20");
expect(backend.mqPeekMessages).toHaveBeenLastCalledWith("mq-1", expectedTopic("kafka"), "__dbx_kafka_viewer__", 20, { startPosition: "latest" });
});
it("does not request Kafka offset mode without an offset", async () => {
const panel = await mountPanel("kafka");
const startPosition = panel.querySelector<HTMLElement>('[data-testid="kafka-peek-start-position"]');
if (!startPosition) throw new Error("Kafka start position control not found");
await setSelectValue(startPosition, "offset");
await loadMessages(panel);
expect(backend.mqPeekMessages).not.toHaveBeenCalled();
expect(panel.textContent).toContain("mqMessages.offsetRequiredForOffset");
const offsetInput = panel.querySelector<HTMLInputElement>('[data-testid="kafka-peek-offset"]');
if (!offsetInput) throw new Error("Kafka offset input not found");
await setInputValue(offsetInput, "17");
await loadMessages(panel);
expect(backend.mqPeekMessages).toHaveBeenCalledWith("mq-1", expectedTopic("kafka"), "__dbx_kafka_viewer__", 20, { startPosition: "offset", offset: 17 });
});
it("clears Kafka results and validation errors when the start position changes", async () => {
const panel = await mountPanel("kafka");
const startPosition = panel.querySelector<HTMLElement>('[data-testid="kafka-peek-start-position"]');
if (!startPosition) throw new Error("Kafka start position select not found");
await loadMessages(panel);
expect(panel.textContent).toContain("existing message");
await setSelectValue(startPosition, "offset");
expect(panel.textContent).not.toContain("existing message");
await loadMessages(panel);
expect(panel.textContent).toContain("mqMessages.offsetRequiredForOffset");
await setSelectValue(startPosition, "earliest");
expect(panel.textContent).not.toContain("existing message");
expect(panel.textContent).not.toContain("mqMessages.offsetRequiredForOffset");
});
it("does not leak a saved Kafka offset after switching start positions", async () => {
const panel = await mountPanel("kafka");
const startPosition = panel.querySelector<HTMLElement>('[data-testid="kafka-peek-start-position"]');
const partitionInput = panel.querySelector<HTMLInputElement>('[data-testid="kafka-peek-partition"]');
if (!startPosition || !partitionInput) throw new Error("Kafka start position controls not found");
await setSelectValue(startPosition, "offset");
const offsetInput = panel.querySelector<HTMLInputElement>('[data-testid="kafka-peek-offset"]');
if (!offsetInput) throw new Error("Kafka offset input not found");
await setInputValue(partitionInput, "2");
await setInputValue(offsetInput, "17");
await setSelectValue(startPosition, "latest");
await loadMessages(panel);
expect(backend.mqPeekMessages).toHaveBeenCalledWith("mq-1", expectedTopic("kafka"), "__dbx_kafka_viewer__", 20, { startPosition: "latest", partition: 2 });
await setSelectValue(startPosition, "offset");
const restoredOffsetInput = panel.querySelector<HTMLInputElement>('[data-testid="kafka-peek-offset"]');
expect(restoredOffsetInput?.value).toBe("17");
});
it("keeps RabbitMQ advanced-filter requests unchanged", async () => {
const panel = await mountPanel("rabbitmq");
buttonByText(panel, "mqMessages.advancedFilter").click();
await flushUi();
const partitionInput = panel.querySelector<HTMLInputElement>('input[placeholder="mqMessages.partitionPlaceholderAll"]');
const offsetInput = panel.querySelector<HTMLInputElement>('input[placeholder="mqMessages.offsetPlaceholderEarliest"]');
if (!partitionInput || !offsetInput) throw new Error("RabbitMQ advanced filter inputs not found");
await setInputValue(partitionInput, "2");
await setInputValue(offsetInput, "17");
await loadMessages(panel);
expect(backend.mqPeekMessages).toHaveBeenCalledWith("mq-1", expectedTopic("rabbitmq"), "__dbx_kafka_viewer__", 20, { partition: 2, offset: 17 });
}); });
}); });
@@ -0,0 +1,178 @@
// @vitest-environment happy-dom
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { createApp, nextTick, type App } from "vue";
import type { PeekedMessage } from "@/types/mq";
const backend = vi.hoisted(() => ({
mqListSubscriptions: vi.fn(),
mqCreateSubscription: vi.fn(),
mqDeleteSubscription: vi.fn(),
mqResetCursor: vi.fn(),
mqSkipMessages: vi.fn(),
mqClearBacklog: vi.fn(),
mqPeekMessages: vi.fn(),
mqExpireMessages: vi.fn(),
}));
vi.mock("vue-i18n", () => ({
useI18n: () => ({ t: (key: string) => key }),
}));
vi.mock("@/lib/backend/api", () => backend);
vi.mock("@/components/editor/DangerConfirmDialog.vue", () => ({
default: { template: "<div />" },
}));
vi.mock("@/components/mq/rocketmq/RocketMqConsumerGroupDialogs.vue", () => ({
default: { template: "<div />" },
}));
import SubscriptionsPanel from "@/components/mq/SubscriptionsPanel.vue";
const TOPIC = {
name: "persistent://public/default/events",
shortName: "events",
partitioned: false,
persistent: true,
};
let app: App<Element> | null = null;
let root: HTMLDivElement | null = null;
async function flushUi() {
await Promise.resolve();
await Promise.resolve();
await nextTick();
}
function buttonByText(container: ParentNode, text: string): HTMLButtonElement {
const button = [...container.querySelectorAll<HTMLButtonElement>("button")].find((item) => item.textContent?.includes(text));
if (!button) throw new Error(`Button not found: ${text}`);
return button;
}
function deferred<T>() {
let resolve!: (value: T) => void;
const promise = new Promise<T>((resolvePromise) => {
resolve = resolvePromise;
});
return { promise, resolve };
}
async function mountPanel() {
root = document.createElement("div");
document.body.appendChild(root);
app = createApp(SubscriptionsPanel, {
connectionId: "mq-1",
topic: TOPIC,
tenant: "_kafka",
namespace: "default",
mqSystemKind: "kafka",
supportsPeekMessages: true,
});
app.mount(root);
await flushUi();
return root;
}
beforeEach(() => {
Object.values(backend).forEach((mock) => mock.mockReset());
backend.mqListSubscriptions.mockResolvedValue([
{
name: "orders-consumer",
subType: "shared",
msgBacklog: 0,
msgRateOut: 0,
msgThroughputOut: 0,
consumers: [],
},
]);
});
afterEach(() => {
app?.unmount();
app = null;
root?.remove();
root = null;
});
describe("SubscriptionsPanel message peek", () => {
it("warns when a Kafka peek is incomplete and clears the warning after a complete refresh", async () => {
backend.mqPeekMessages
.mockResolvedValueOnce({
messages: [
{
position: 1,
messageId: "17",
payloadBase64: "",
payloadText: "partial message",
properties: { partition: "0" },
headers: {},
},
],
incomplete: true,
})
.mockResolvedValueOnce({ messages: [], incomplete: false });
const panel = await mountPanel();
buttonByText(panel, "mqSubscriptions.peek").click();
await flushUi();
expect(backend.mqPeekMessages).toHaveBeenCalledWith("mq-1", expect.objectContaining({ topic: "events" }), "orders-consumer", 5);
expect(panel.querySelector('[data-testid="peek-incomplete"]')?.textContent).toContain("mqMessages.peekIncomplete");
expect(panel.textContent).toContain("partial message");
buttonByText(panel, "mqSubscriptions.refresh").click();
await flushUi();
expect(panel.querySelector('[data-testid="peek-incomplete"]')).toBeNull();
});
it("ignores a stale peek when another subscription is opened", async () => {
const first = deferred<{ messages: PeekedMessage[]; incomplete: boolean }>();
const second = deferred<{ messages: PeekedMessage[]; incomplete: boolean }>();
backend.mqListSubscriptions.mockResolvedValue([
{
name: "orders-consumer",
subType: "shared",
msgBacklog: 0,
msgRateOut: 0,
msgThroughputOut: 0,
consumers: [],
},
{
name: "payments-consumer",
subType: "shared",
msgBacklog: 0,
msgRateOut: 0,
msgThroughputOut: 0,
consumers: [],
},
]);
backend.mqPeekMessages.mockReturnValueOnce(first.promise).mockReturnValueOnce(second.promise);
const panel = await mountPanel();
const peekButtons = [...panel.querySelectorAll<HTMLButtonElement>("button")].filter((button) => button.textContent?.includes("mqSubscriptions.peek"));
expect(peekButtons).toHaveLength(2);
peekButtons[0].click();
await flushUi();
peekButtons[1].click();
await flushUi();
second.resolve({
messages: [{ position: 1, messageId: "new", payloadBase64: "", payloadText: "payments message", properties: {}, headers: {} }],
incomplete: false,
});
await flushUi();
first.resolve({
messages: [{ position: 1, messageId: "old", payloadBase64: "", payloadText: "orders message", properties: {}, headers: {} }],
incomplete: false,
});
await flushUi();
expect(panel.textContent).toContain("payments message");
expect(panel.textContent).not.toContain("orders message");
});
});
+36
View File
@@ -0,0 +1,36 @@
import { defineComponent, h, inject, provide } from "vue";
export function createSelectStub() {
const updateKey = Symbol("select-update");
const Select = defineComponent({
emits: ["update:modelValue"],
setup(_, { emit, slots }) {
provide(updateKey, (value: string) => emit("update:modelValue", value));
return () => h("div", { "data-slot": "select" }, slots.default?.());
},
});
const SelectTrigger = defineComponent({
setup(_, { attrs, slots }) {
return () => h("button", { ...attrs, type: "button" }, slots.default?.());
},
});
const SelectContent = defineComponent({
setup(_, { attrs, slots }) {
return () => h("div", attrs, slots.default?.());
},
});
const SelectItem = defineComponent({
props: { value: { type: String, required: true } },
setup(props, { attrs, slots }) {
const update = inject<(value: string) => void>(updateKey);
return () => h("button", { ...attrs, "data-slot": "select-item", "data-value": props.value, type: "button", onClick: () => update?.(props.value) }, slots.default?.());
},
});
const SelectValue = defineComponent({
setup(_, { attrs, slots }) {
return () => h("span", attrs, slots.default?.());
},
});
return { Select, SelectContent, SelectItem, SelectTrigger, SelectValue };
}
+12 -6
View File
@@ -6011,18 +6011,29 @@ export default {
messageList: "Messages", messageList: "Messages",
loadMessages: "Load messages", loadMessages: "Load messages",
loading: "Loading...", loading: "Loading...",
peekDefaultHint: "Without a partition, reads all partitions from earliest, up to {count} messages.", peekDefaultHint: "Without a partition, reads all partitions from earliest.",
count: "Count", count: "Count",
advancedFilter: "Advanced (partition / offset)", advancedFilter: "Advanced (partition / offset)",
partition: "Partition", partition: "Partition",
partitionPlaceholderAll: "Empty = all", partitionPlaceholderAll: "Empty = all",
offset: "Offset", offset: "Offset",
offsetPlaceholderEarliest: "Empty = earliest", offsetPlaceholderEarliest: "Empty = earliest",
offsetPlaceholderRequired: "Required",
startPosition: "Start position",
kafkaLatest: "Recent N messages (default)",
kafkaEarliest: "Earliest messages",
kafkaOffset: "From offset",
kafkaLatestHint: "Reads a balanced window near the end of each partition and merges by timestamp newest first.",
kafkaEarliestHint: "Reads history forward from each selected partition.",
kafkaOffsetHint: "Reads forward from the specified offset in the chosen partition, or every partition when left empty.",
messagesLoading: "Loading messages...", messagesLoading: "Loading messages...",
noMessages: "No messages", noMessages: "No messages",
peekIncomplete: "The message results are incomplete because the broker did not finish the snapshot.",
selectTopicBeforeLoad: "Select a topic before loading messages", selectTopicBeforeLoad: "Select a topic before loading messages",
partitionMustBeNonNegativeInt: "Partition must be an integer ≥ 0; leave empty for all partitions", partitionMustBeNonNegativeInt: "Partition must be an integer ≥ 0; leave empty for all partitions",
offsetMustBeNonNegativeInt: "Offset must be an integer ≥ 0; leave empty for earliest", offsetMustBeNonNegativeInt: "Offset must be an integer ≥ 0; leave empty for earliest",
offsetMustBeNonNegativeIntRequired: "Offset must be an integer ≥ 0",
offsetRequiredForOffset: "An offset is required when reading from a specific offset",
metaPartition: "partition {partition}", metaPartition: "partition {partition}",
metaOffset: "offset {offset}", metaOffset: "offset {offset}",
metaKey: "key {key}", metaKey: "key {key}",
@@ -6099,11 +6110,6 @@ export default {
statusNoLeader: "No leader", statusNoLeader: "No leader",
statusIsrIncomplete: "ISR incomplete", statusIsrIncomplete: "ISR incomplete",
statusHealthy: "Healthy", statusHealthy: "Healthy",
kafkaMessageQuery: "Kafka message query",
querying: "Querying...",
queryMessages: "Query",
queryHint: "Omit PARTITION / OFFSET to read all partitions (max 100). Add them to narrow the range.",
sqlSyntaxError: 'Only SELECT * FROM "topic" [PARTITION n] [OFFSET n] [LIMIT n] is supported',
messagesLoading: "Loading messages...", messagesLoading: "Loading messages...",
noMessages: "No messages", noMessages: "No messages",
messageRate: "Message rate", messageRate: "Message rate",
+12 -6
View File
@@ -5864,18 +5864,29 @@ export default withEnglishFallback({
messageList: "Mensajes", messageList: "Mensajes",
loadMessages: "Cargar mensajes", loadMessages: "Cargar mensajes",
loading: "Cargando...", loading: "Cargando...",
peekDefaultHint: "Sin partición, lee todas las particiones desde earliest, hasta {count} mensajes.", peekDefaultHint: "Sin partición, lee todas las particiones desde earliest.",
count: "Cantidad", count: "Cantidad",
advancedFilter: "Avanzado (partición / offset)", advancedFilter: "Avanzado (partición / offset)",
partition: "Partición", partition: "Partición",
partitionPlaceholderAll: "Vacío = todas", partitionPlaceholderAll: "Vacío = todas",
offset: "Offset", offset: "Offset",
offsetPlaceholderEarliest: "Vacío = earliest", offsetPlaceholderEarliest: "Vacío = earliest",
offsetPlaceholderRequired: "Obligatorio",
startPosition: "Punto de inicio",
kafkaLatest: "Últimos N mensajes (predeterminado)",
kafkaEarliest: "Mensajes más antiguos",
kafkaOffset: "Desde offset",
kafkaLatestHint: "Lee una ventana equilibrada cerca del final de cada partición y combina por marca de tiempo con los más nuevos primero.",
kafkaEarliestHint: "Lee el historial hacia adelante desde cada partición seleccionada.",
kafkaOffsetHint: "Lee hacia adelante desde el offset indicado en la partición elegida, o en todas si se deja vacía.",
messagesLoading: "Cargando mensajes...", messagesLoading: "Cargando mensajes...",
noMessages: "No hay mensajes", noMessages: "No hay mensajes",
peekIncomplete: "Los resultados de mensajes están incompletos porque el broker no terminó la instantánea.",
selectTopicBeforeLoad: "Seleccione un tema antes de cargar mensajes", selectTopicBeforeLoad: "Seleccione un tema antes de cargar mensajes",
partitionMustBeNonNegativeInt: "La partición debe ser un entero ≥ 0; déjelo vacío para todas las particiones", partitionMustBeNonNegativeInt: "La partición debe ser un entero ≥ 0; déjelo vacío para todas las particiones",
offsetMustBeNonNegativeInt: "El offset debe ser un entero ≥ 0; déjelo vacío para earliest", offsetMustBeNonNegativeInt: "El offset debe ser un entero ≥ 0; déjelo vacío para earliest",
offsetMustBeNonNegativeIntRequired: "El offset debe ser un entero ≥ 0",
offsetRequiredForOffset: "Se requiere un offset para leer desde un offset específico",
metaPartition: "partición {partition}", metaPartition: "partición {partition}",
metaOffset: "offset {offset}", metaOffset: "offset {offset}",
metaKey: "clave {key}", metaKey: "clave {key}",
@@ -5957,11 +5968,6 @@ export default withEnglishFallback({
statusNoLeader: "Sin leader", statusNoLeader: "Sin leader",
statusIsrIncomplete: "ISR incompleto", statusIsrIncomplete: "ISR incompleto",
statusHealthy: "Saludable", statusHealthy: "Saludable",
kafkaMessageQuery: "Consulta de mensajes de Kafka",
querying: "Consultando...",
queryMessages: "Consultar",
queryHint: "Omita PARTITION / OFFSET para leer todas las particiones (máx. 100). Agréguelos para acotar el rango.",
sqlSyntaxError: 'Solo se admite SELECT * FROM "topic" [PARTITION n] [OFFSET n] [LIMIT n]',
messagesLoading: "Cargando mensajes...", messagesLoading: "Cargando mensajes...",
noMessages: "No hay mensajes", noMessages: "No hay mensajes",
messageRate: "Tasa de mensajes", messageRate: "Tasa de mensajes",
+12 -6
View File
@@ -5864,18 +5864,29 @@ export default withEnglishFallback({
messageList: "Messaggi", messageList: "Messaggi",
loadMessages: "Carica messaggi", loadMessages: "Carica messaggi",
loading: "Caricamento in corso...", loading: "Caricamento in corso...",
peekDefaultHint: "Senza partizione, legge tutte le partizioni dall'inizio (earliest), fino a {count} messaggi.", peekDefaultHint: "Senza partizione, legge tutte le partizioni dall'inizio (earliest).",
count: "Quantità", count: "Quantità",
advancedFilter: "Avanzate (partizione / offset)", advancedFilter: "Avanzate (partizione / offset)",
partition: "Partizione", partition: "Partizione",
partitionPlaceholderAll: "Vuoto = tutte", partitionPlaceholderAll: "Vuoto = tutte",
offset: "Offset", offset: "Offset",
offsetPlaceholderEarliest: "Vuoto = inizio (earliest)", offsetPlaceholderEarliest: "Vuoto = inizio (earliest)",
offsetPlaceholderRequired: "Obbligatorio",
startPosition: "Posizione iniziale",
kafkaLatest: "Ultimi N messaggi (predefinito)",
kafkaEarliest: "Primi messaggi",
kafkaOffset: "Da offset",
kafkaLatestHint: "Legge una finestra bilanciata vicino alla fine di ogni partizione e unisce per timestamp con i più recenti prima.",
kafkaEarliestHint: "Legge la cronologia in avanti da ogni partizione selezionata.",
kafkaOffsetHint: "Legge in avanti dall'offset indicato nella partizione scelta, o in tutte se lasciata vuota.",
messagesLoading: "Caricamento messaggi...", messagesLoading: "Caricamento messaggi...",
noMessages: "Nessun messaggio", noMessages: "Nessun messaggio",
peekIncomplete: "I risultati dei messaggi sono incompleti perché il broker non ha completato lo snapshot.",
selectTopicBeforeLoad: "Seleziona un topic prima di caricare i messaggi", selectTopicBeforeLoad: "Seleziona un topic prima di caricare i messaggi",
partitionMustBeNonNegativeInt: "La partizione deve essere un intero ≥ 0; lascia vuoto per tutte le partizioni", partitionMustBeNonNegativeInt: "La partizione deve essere un intero ≥ 0; lascia vuoto per tutte le partizioni",
offsetMustBeNonNegativeInt: "L'offset deve essere un intero ≥ 0; lascia vuoto per l'inizio (earliest)", offsetMustBeNonNegativeInt: "L'offset deve essere un intero ≥ 0; lascia vuoto per l'inizio (earliest)",
offsetMustBeNonNegativeIntRequired: "L'offset deve essere un intero ≥ 0",
offsetRequiredForOffset: "Per leggere da un offset specifico è richiesto un offset",
metaPartition: "partizione {partition}", metaPartition: "partizione {partition}",
metaOffset: "offset {offset}", metaOffset: "offset {offset}",
metaKey: "chiave {key}", metaKey: "chiave {key}",
@@ -5957,11 +5968,6 @@ export default withEnglishFallback({
statusNoLeader: "Senza leader", statusNoLeader: "Senza leader",
statusIsrIncomplete: "ISR incompleto", statusIsrIncomplete: "ISR incompleto",
statusHealthy: "Integro", statusHealthy: "Integro",
kafkaMessageQuery: "Interrogazione messaggi Kafka",
querying: "Interrogazione in corso...",
queryMessages: "Interroga",
queryHint: "Ometti PARTITION / OFFSET per leggere tutte le partizioni (max 100). Aggiungili per restringere l'intervallo.",
sqlSyntaxError: 'È supportato solo SELECT * FROM "topic" [PARTITION n] [OFFSET n] [LIMIT n]',
messagesLoading: "Caricamento messaggi...", messagesLoading: "Caricamento messaggi...",
noMessages: "Nessun messaggio", noMessages: "Nessun messaggio",
messageRate: "Velocità messaggi", messageRate: "Velocità messaggi",
+12 -6
View File
@@ -5919,18 +5919,29 @@ export default withEnglishFallback({
messageList: "メッセージ一覧", messageList: "メッセージ一覧",
loadMessages: "メッセージを読み込む", loadMessages: "メッセージを読み込む",
loading: "読み込み中...", loading: "読み込み中...",
peekDefaultHint: "パーティション未指定時は、全パーティションから Earliest 位置で最大 {count} 件まで読み取ります。", peekDefaultHint: "パーティション未指定時は、全パーティションから Earliest 位置で読み取ります。",
count: "件数", count: "件数",
advancedFilter: "詳細 (パーティション / offset)", advancedFilter: "詳細 (パーティション / offset)",
partition: "パーティション", partition: "パーティション",
partitionPlaceholderAll: "空欄 = すべて", partitionPlaceholderAll: "空欄 = すべて",
offset: "Offset", offset: "Offset",
offsetPlaceholderEarliest: "空欄 = Earliest", offsetPlaceholderEarliest: "空欄 = Earliest",
offsetPlaceholderRequired: "必須",
startPosition: "読み取り開始位置",
kafkaLatest: "最新 N 件(デフォルト)",
kafkaEarliest: "最も古いメッセージ",
kafkaOffset: "Offset から読み取る",
kafkaLatestHint: "各パーティションの末尾付近から均等に読み取り、タイムスタンプの新しい順に統合します。",
kafkaEarliestHint: "選択した各パーティションの先頭から昇順に読み取ります。",
kafkaOffsetHint: "指定 Offset から、選択したパーティション(空欄ならすべて)を昇順に読み取ります。",
messagesLoading: "メッセージを読み込み中...", messagesLoading: "メッセージを読み込み中...",
noMessages: "メッセージがありません", noMessages: "メッセージがありません",
peekIncomplete: "ブローカーがスナップショットの読み取りを完了できなかったため、メッセージ結果は不完全です。",
selectTopicBeforeLoad: "メッセージを読み込む前にトピックを選択してください", selectTopicBeforeLoad: "メッセージを読み込む前にトピックを選択してください",
partitionMustBeNonNegativeInt: "パーティションは 0 以上の整数である必要があります。空欄の場合はすべてのパーティション", partitionMustBeNonNegativeInt: "パーティションは 0 以上の整数である必要があります。空欄の場合はすべてのパーティション",
offsetMustBeNonNegativeInt: "Offset は 0 以上の整数である必要があります。空欄の場合は Earliest", offsetMustBeNonNegativeInt: "Offset は 0 以上の整数である必要があります。空欄の場合は Earliest",
offsetMustBeNonNegativeIntRequired: "Offset は 0 以上の整数である必要があります",
offsetRequiredForOffset: "指定 Offset から読み取るには Offset が必要です",
metaPartition: "パーティション {partition}", metaPartition: "パーティション {partition}",
metaOffset: "オフセット {offset}", metaOffset: "オフセット {offset}",
metaKey: "キー {key}", metaKey: "キー {key}",
@@ -6012,11 +6023,6 @@ export default withEnglishFallback({
statusNoLeader: "Leader なし", statusNoLeader: "Leader なし",
statusIsrIncomplete: "ISR 不完全", statusIsrIncomplete: "ISR 不完全",
statusHealthy: "正常", statusHealthy: "正常",
kafkaMessageQuery: "Kafka メッセージクエリ",
querying: "クエリ中...",
queryMessages: "クエリ",
queryHint: "PARTITION / OFFSET を省略すると全パーティションを読み取ります (最大 100 件)。指定すると範囲を絞り込めます。",
sqlSyntaxError: 'SELECT * FROM "topic" [PARTITION n] [OFFSET n] [LIMIT n] のみサポートしています',
messagesLoading: "メッセージを読み込み中...", messagesLoading: "メッセージを読み込み中...",
noMessages: "メッセージがありません", noMessages: "メッセージがありません",
messageRate: "メッセージレート", messageRate: "メッセージレート",
+12 -6
View File
@@ -5603,18 +5603,29 @@ export default withEnglishFallback({
messageList: "메시지", messageList: "메시지",
loadMessages: "메시지 불러오기", loadMessages: "메시지 불러오기",
loading: "불러오는 중...", loading: "불러오는 중...",
peekDefaultHint: "파티션 없이 처음부터 모든 파티션을 최대 {count}개의 메시지까지 읽습니다.", peekDefaultHint: "파티션 없이 처음부터 모든 파티션을 읽습니다.",
count: "개수", count: "개수",
advancedFilter: "고급 (파티션 / 오프셋)", advancedFilter: "고급 (파티션 / 오프셋)",
partition: "파티션", partition: "파티션",
partitionPlaceholderAll: "비움 = 전체", partitionPlaceholderAll: "비움 = 전체",
offset: "오프셋", offset: "오프셋",
offsetPlaceholderEarliest: "비움 = 처음", offsetPlaceholderEarliest: "비움 = 처음",
offsetPlaceholderRequired: "필수",
startPosition: "읽기 시작 위치",
kafkaLatest: "최근 N개 메시지(기본값)",
kafkaEarliest: "가장 이른 메시지",
kafkaOffset: "오프셋부터 읽기",
kafkaLatestHint: "각 파티션 끝 부근에서 균등하게 읽고 타임스탬프 내림차순으로 병합합니다.",
kafkaEarliestHint: "선택한 각 파티션의 처음부터 오름차순으로 읽습니다.",
kafkaOffsetHint: "지정한 오프셋부터 선택한 파티션(비워 두면 전체)을 오름차순으로 읽습니다.",
messagesLoading: "메시지를 불러오는 중...", messagesLoading: "메시지를 불러오는 중...",
noMessages: "메시지가 없습니다", noMessages: "메시지가 없습니다",
peekIncomplete: "브로커가 스냅샷 읽기를 완료하지 못해 메시지 결과가 불완전합니다.",
selectTopicBeforeLoad: "메시지를 불러오기 전에 토픽을 선택하세요", selectTopicBeforeLoad: "메시지를 불러오기 전에 토픽을 선택하세요",
partitionMustBeNonNegativeInt: "파티션은 0 이상의 정수여야 하며, 모든 파티션은 비워 두세요", partitionMustBeNonNegativeInt: "파티션은 0 이상의 정수여야 하며, 모든 파티션은 비워 두세요",
offsetMustBeNonNegativeInt: "오프셋은 0 이상의 정수여야 하며, 처음은 비워 두세요", offsetMustBeNonNegativeInt: "오프셋은 0 이상의 정수여야 하며, 처음은 비워 두세요",
offsetMustBeNonNegativeIntRequired: "오프셋은 0 이상의 정수여야 합니다",
offsetRequiredForOffset: "특정 오프셋부터 읽으려면 오프셋이 필요합니다",
metaPartition: "파티션 {partition}", metaPartition: "파티션 {partition}",
metaOffset: "오프셋 {offset}", metaOffset: "오프셋 {offset}",
metaKey: "키 {key}", metaKey: "키 {key}",
@@ -5691,11 +5702,6 @@ export default withEnglishFallback({
statusNoLeader: "리더 없음", statusNoLeader: "리더 없음",
statusIsrIncomplete: "ISR 불완전", statusIsrIncomplete: "ISR 불완전",
statusHealthy: "정상", statusHealthy: "정상",
kafkaMessageQuery: "Kafka 메시지 쿼리",
querying: "쿼리 중...",
queryMessages: "쿼리",
queryHint: "PARTITION / OFFSET를 생략하면 모든 파티션을 읽습니다 (최대 100). 범위를 좁히려면 추가하세요.",
sqlSyntaxError: 'SELECT * FROM "topic" [PARTITION n] [OFFSET n] [LIMIT n] 만 지원됩니다',
messagesLoading: "메시지를 불러오는 중...", messagesLoading: "메시지를 불러오는 중...",
noMessages: "메시지가 없습니다", noMessages: "메시지가 없습니다",
messageRate: "메시지 비율", messageRate: "메시지 비율",
+12 -6
View File
@@ -5866,18 +5866,29 @@ export default withEnglishFallback({
messageList: "Mensagens", messageList: "Mensagens",
loadMessages: "Carregar mensagens", loadMessages: "Carregar mensagens",
loading: "Carregando...", loading: "Carregando...",
peekDefaultHint: "Sem partição definida, lê todas as partições desde Earliest, até {count} mensagens.", peekDefaultHint: "Sem partição definida, lê todas as partições desde Earliest.",
count: "Quantidade", count: "Quantidade",
advancedFilter: "Avançado (partição / offset)", advancedFilter: "Avançado (partição / offset)",
partition: "Partição", partition: "Partição",
partitionPlaceholderAll: "Vazio = todas", partitionPlaceholderAll: "Vazio = todas",
offset: "Offset", offset: "Offset",
offsetPlaceholderEarliest: "Vazio = Earliest", offsetPlaceholderEarliest: "Vazio = Earliest",
offsetPlaceholderRequired: "Obrigatório",
startPosition: "Posição inicial",
kafkaLatest: "Últimas N mensagens (padrão)",
kafkaEarliest: "Mensagens mais antigas",
kafkaOffset: "A partir do offset",
kafkaLatestHint: "Lê uma janela equilibrada perto do fim de cada partição e mescla por data e hora com as mais novas primeiro.",
kafkaEarliestHint: "Lê o histórico em ordem crescente de cada partição selecionada.",
kafkaOffsetHint: "Lê em ordem crescente a partir do offset informado na partição escolhida, ou em todas se deixada vazia.",
messagesLoading: "Carregando mensagens...", messagesLoading: "Carregando mensagens...",
noMessages: "Nenhuma mensagem", noMessages: "Nenhuma mensagem",
peekIncomplete: "Os resultados das mensagens estão incompletos porque o broker não concluiu o snapshot.",
selectTopicBeforeLoad: "Selecione um tópico antes de carregar mensagens", selectTopicBeforeLoad: "Selecione um tópico antes de carregar mensagens",
partitionMustBeNonNegativeInt: "A partição deve ser um inteiro ≥ 0; deixe vazio para todas as partições", partitionMustBeNonNegativeInt: "A partição deve ser um inteiro ≥ 0; deixe vazio para todas as partições",
offsetMustBeNonNegativeInt: "O Offset deve ser um inteiro ≥ 0; deixe vazio para Earliest", offsetMustBeNonNegativeInt: "O Offset deve ser um inteiro ≥ 0; deixe vazio para Earliest",
offsetMustBeNonNegativeIntRequired: "O Offset deve ser um inteiro ≥ 0",
offsetRequiredForOffset: "Um offset é obrigatório para ler de um offset específico",
metaPartition: "partição {partition}", metaPartition: "partição {partition}",
metaOffset: "offset {offset}", metaOffset: "offset {offset}",
metaKey: "chave {key}", metaKey: "chave {key}",
@@ -5959,11 +5970,6 @@ export default withEnglishFallback({
statusNoLeader: "Sem leader", statusNoLeader: "Sem leader",
statusIsrIncomplete: "ISR incompleto", statusIsrIncomplete: "ISR incompleto",
statusHealthy: "Saudável", statusHealthy: "Saudável",
kafkaMessageQuery: "Consulta de mensagens Kafka",
querying: "Consultando...",
queryMessages: "Consultar",
queryHint: "Omita PARTITION / OFFSET para ler todas as partições (máx. 100). Adicione-os para restringir o intervalo.",
sqlSyntaxError: 'Apenas SELECT * FROM "topic" [PARTITION n] [OFFSET n] [LIMIT n] é suportado',
messagesLoading: "Carregando mensagens...", messagesLoading: "Carregando mensagens...",
noMessages: "Nenhuma mensagem", noMessages: "Nenhuma mensagem",
messageRate: "Taxa de mensagens", messageRate: "Taxa de mensagens",
+12 -6
View File
@@ -6011,18 +6011,29 @@ export default withEnglishFallback({
messageList: "消息列表", messageList: "消息列表",
loadMessages: "加载消息", loadMessages: "加载消息",
loading: "加载中...", loading: "加载中...",
peekDefaultHint: "未指定分区时读取全部区,从各区 earliest 起,最多 {count} 条。", peekDefaultHint: "未指定分区时读取全部区,从各区 earliest 起。",
count: "数量", count: "数量",
advancedFilter: "高级筛选(分区 / Offset)", advancedFilter: "高级筛选(分区 / Offset)",
partition: "分区", partition: "分区",
partitionPlaceholderAll: "留空=全部", partitionPlaceholderAll: "留空=全部",
offset: "Offset", offset: "Offset",
offsetPlaceholderEarliest: "留空=最早", offsetPlaceholderEarliest: "留空=最早",
offsetPlaceholderRequired: "必填",
startPosition: "读取起点",
kafkaLatest: "最近 N 条(默认)",
kafkaEarliest: "最早消息",
kafkaOffset: "从指定 Offset 读取",
kafkaLatestHint: "从各分区末尾附近均衡读取,按时间倒序合并。",
kafkaEarliestHint: "从每个选定分区的最早可读消息开始正序读取。",
kafkaOffsetHint: "从指定 Offset 开始正序读取;填写分区时仅读该分区,留空则对全部分区分别读取。",
messagesLoading: "消息加载中...", messagesLoading: "消息加载中...",
noMessages: "暂无消息", noMessages: "暂无消息",
peekIncomplete: "消息结果不完整:Broker 未能在限制内完成快照读取。",
selectTopicBeforeLoad: "请先选择主题再加载消息", selectTopicBeforeLoad: "请先选择主题再加载消息",
partitionMustBeNonNegativeInt: "分区必须是大于等于 0 的整数;留空表示查询全部分区", partitionMustBeNonNegativeInt: "分区必须是大于等于 0 的整数;留空表示查询全部分区",
offsetMustBeNonNegativeInt: "Offset 必须是大于等于 0 的整数;留空表示从各分区最早可读位置开始", offsetMustBeNonNegativeInt: "Offset 必须是大于等于 0 的整数;留空表示从各分区最早可读位置开始",
offsetMustBeNonNegativeIntRequired: "Offset 必须是大于等于 0 的整数",
offsetRequiredForOffset: "按指定 Offset 读取时必须填写 Offset",
metaPartition: "分区 {partition}", metaPartition: "分区 {partition}",
metaOffset: "偏移 {offset}", metaOffset: "偏移 {offset}",
metaKey: "键 {key}", metaKey: "键 {key}",
@@ -6098,11 +6109,6 @@ export default withEnglishFallback({
statusNoLeader: "无 leader", statusNoLeader: "无 leader",
statusIsrIncomplete: "ISR 不完整", statusIsrIncomplete: "ISR 不完整",
statusHealthy: "正常", statusHealthy: "正常",
kafkaMessageQuery: "Kafka 消息查询",
querying: "查询中...",
queryMessages: "查询消息",
queryHint: "省略 PARTITION / OFFSET 时读取全部区,最多 100 条;指定分区或 offset 时仅读对应范围。",
sqlSyntaxError: '仅支持 SELECT * FROM "topic" [PARTITION n] [OFFSET n] [LIMIT n]',
messagesLoading: "消息加载中...", messagesLoading: "消息加载中...",
noMessages: "暂无消息", noMessages: "暂无消息",
messageRate: "消息速率", messageRate: "消息速率",
+12 -6
View File
@@ -5859,18 +5859,29 @@ export default withEnglishFallback({
messageList: "訊息清單", messageList: "訊息清單",
loadMessages: "載入訊息", loadMessages: "載入訊息",
loading: "載入中...", loading: "載入中...",
peekDefaultHint: "未指定分區時讀取全部分區,從各分區 earliest 起,最多 {count} 則。", peekDefaultHint: "未指定分區時讀取全部分區,從各分區 earliest 起。",
count: "數量", count: "數量",
advancedFilter: "進階篩選(分區 / Offset)", advancedFilter: "進階篩選(分區 / Offset)",
partition: "分區", partition: "分區",
partitionPlaceholderAll: "留空=全部", partitionPlaceholderAll: "留空=全部",
offset: "Offset", offset: "Offset",
offsetPlaceholderEarliest: "留空=最早", offsetPlaceholderEarliest: "留空=最早",
offsetPlaceholderRequired: "必填",
startPosition: "讀取起點",
kafkaLatest: "最近 N 則(預設)",
kafkaEarliest: "最早訊息",
kafkaOffset: "從指定 Offset 讀取",
kafkaLatestHint: "從各分區末尾附近平均讀取,依時間倒序合併。",
kafkaEarliestHint: "從每個選定分區最早可讀的訊息開始正序讀取。",
kafkaOffsetHint: "從指定 Offset 開始正序讀取;填寫分區時只讀該分區,留空則對全部分區分別讀取。",
messagesLoading: "訊息載入中...", messagesLoading: "訊息載入中...",
noMessages: "暫無訊息", noMessages: "暫無訊息",
peekIncomplete: "訊息結果不完整:Broker 未能在限制內完成快照讀取。",
selectTopicBeforeLoad: "請先選取主題再載入訊息", selectTopicBeforeLoad: "請先選取主題再載入訊息",
partitionMustBeNonNegativeInt: "分區必須是大於等於 0 的整數;留空表示查詢全部分區", partitionMustBeNonNegativeInt: "分區必須是大於等於 0 的整數;留空表示查詢全部分區",
offsetMustBeNonNegativeInt: "Offset 必須是大於等於 0 的整數;留空表示從各分區最早可讀位置開始", offsetMustBeNonNegativeInt: "Offset 必須是大於等於 0 的整數;留空表示從各分區最早可讀位置開始",
offsetMustBeNonNegativeIntRequired: "Offset 必須是大於等於 0 的整數",
offsetRequiredForOffset: "以指定 Offset 讀取時必須填寫 Offset",
metaPartition: "分區 {partition}", metaPartition: "分區 {partition}",
metaOffset: "偏移 {offset}", metaOffset: "偏移 {offset}",
metaKey: "鍵 {key}", metaKey: "鍵 {key}",
@@ -5952,11 +5963,6 @@ export default withEnglishFallback({
statusNoLeader: "無 leader", statusNoLeader: "無 leader",
statusIsrIncomplete: "ISR 不完整", statusIsrIncomplete: "ISR 不完整",
statusHealthy: "正常", statusHealthy: "正常",
kafkaMessageQuery: "Kafka 訊息查詢",
querying: "查詢中...",
queryMessages: "查詢訊息",
queryHint: "省略 PARTITION / OFFSET 時讀取全部分區,最多 100 則;指定分區或 offset 時僅讀取對應範圍。",
sqlSyntaxError: '僅支援 SELECT * FROM "topic" [PARTITION n] [OFFSET n] [LIMIT n]',
messagesLoading: "訊息載入中...", messagesLoading: "訊息載入中...",
noMessages: "暫無訊息", noMessages: "暫無訊息",
messageRate: "訊息速率", messageRate: "訊息速率",
+2 -2
View File
@@ -30,7 +30,7 @@ import type {
MqIssuedToken, MqIssuedToken,
BacklogStats, BacklogStats,
RocketMqConsumerGroupConfig, RocketMqConsumerGroupConfig,
PeekedMessage, PeekMessagesResult,
PeekMessagesOptions, PeekMessagesOptions,
MqRawRequest, MqRawRequest,
MqRawResponse, MqRawResponse,
@@ -237,7 +237,7 @@ export async function mqClearBacklog(connectionId: string, topic: TopicRef, sub:
return post("/api/mq/subscriptions/clear-backlog", { connectionId, topic, sub }); return post("/api/mq/subscriptions/clear-backlog", { connectionId, topic, sub });
} }
export async function mqPeekMessages(connectionId: string, topic: TopicRef, sub: string, count: number, options?: PeekMessagesOptions): Promise<PeekedMessage[]> { export async function mqPeekMessages(connectionId: string, topic: TopicRef, sub: string, count: number, options?: PeekMessagesOptions): Promise<PeekMessagesResult> {
return post("/api/mq/subscriptions/peek-messages", { connectionId, topic, sub, count, options }); return post("/api/mq/subscriptions/peek-messages", { connectionId, topic, sub, count, options });
} }
+2 -2
View File
@@ -30,7 +30,7 @@ import type {
MqIssuedToken, MqIssuedToken,
BacklogStats, BacklogStats,
RocketMqConsumerGroupConfig, RocketMqConsumerGroupConfig,
PeekedMessage, PeekMessagesResult,
PeekMessagesOptions, PeekMessagesOptions,
MqRawRequest, MqRawRequest,
MqRawResponse, MqRawResponse,
@@ -231,7 +231,7 @@ export async function mqClearBacklog(connectionId: string, topic: TopicRef, sub:
return invoke("mq_clear_backlog", { connectionId, topic, sub }); return invoke("mq_clear_backlog", { connectionId, topic, sub });
} }
export async function mqPeekMessages(connectionId: string, topic: TopicRef, sub: string, count: number, options?: PeekMessagesOptions): Promise<PeekedMessage[]> { export async function mqPeekMessages(connectionId: string, topic: TopicRef, sub: string, count: number, options?: PeekMessagesOptions): Promise<PeekMessagesResult> {
return invoke("mq_peek_messages", { connectionId, topic, sub, count, options }); return invoke("mq_peek_messages", { connectionId, topic, sub, count, options });
} }
+8
View File
@@ -263,7 +263,15 @@ export interface PeekedMessage {
payloadText?: string; payloadText?: string;
} }
export interface PeekMessagesResult {
messages: PeekedMessage[];
/** True when the broker could not finish the requested snapshot before its limit. */
incomplete: boolean;
}
export interface PeekMessagesOptions { export interface PeekMessagesOptions {
/** Kafka only. Omitted starts at earliest unless a legacy caller supplies offset. */
startPosition?: "latest" | "earliest" | "offset";
partition?: number; partition?: number;
offset?: number; offset?: number;
} }
+168 -44
View File
@@ -85,6 +85,17 @@ impl KafkaAdmin {
client.call(method, params).await client.call(method, params).await
} }
/// The Kafka agent bounds message browsing with its configured request timeout.
/// Do not preempt that with the driver's generic 30-second RPC timeout.
async fn call_with_agent_timeout<T: DeserializeOwned + Send + 'static>(
&self,
method: &str,
params: serde_json::Value,
) -> Result<T, String> {
let mut client = self.client.lock().await;
client.call_with_timeout(method, params, None).await
}
/// Send a JSON-RPC call that returns `{ok: true}` on success. /// Send a JSON-RPC call that returns `{ok: true}` on success.
async fn call_ok(&self, method: &str, params: serde_json::Value) -> Result<(), String> { async fn call_ok(&self, method: &str, params: serde_json::Value) -> Result<(), String> {
let _: serde_json::Value = self.call(method, params).await?; let _: serde_json::Value = self.call(method, params).await?;
@@ -317,51 +328,11 @@ impl MessageQueueAdmin for KafkaAdmin {
_sub: &str, _sub: &str,
count: u32, count: u32,
options: PeekMessagesOptions, options: PeekMessagesOptions,
) -> Result<Vec<PeekedMessage>, String> { ) -> Result<PeekMessagesResult, String> {
let conn_params = build_connection_params(&self.config); let params = peek_messages_params(&self.config, topic, count, options);
let mut params = serde_json::json!({ let result: serde_json::Value = self.call_with_agent_timeout("mq_peek_messages", params).await?;
"topic": topic.topic,
"count": count,
"connection": conn_params,
});
// Omit partition/offset so the agent defaults to all partitions + earliest.
// Do not coerce missing values to 0 — that forced PARTITION 0 OFFSET 0 UX.
if let Some(partition) = options.partition {
params["partition"] = serde_json::json!(partition);
}
if let Some(offset) = options.offset {
params["offset"] = serde_json::json!(offset);
}
let result: serde_json::Value = self.call("mq_peek_messages", params).await?;
let messages = result.get("messages").and_then(|v| v.as_array()).cloned().unwrap_or_default(); Ok(peek_messages_result_from_agent(&result))
Ok(messages
.into_iter()
.enumerate()
.map(|(idx, m)| {
let mut properties = HashMap::new();
if let Some(partition) = m.get("partition").and_then(|v| v.as_i64()) {
properties.insert("partition".to_string(), partition.to_string());
}
PeekedMessage {
position: (idx + 1) as u32,
message_id: m.get("offset").and_then(|v| v.as_i64()).map(|v| v.to_string()),
key: m.get("key").and_then(|v| v.as_str()).map(String::from),
publish_time: m.get("timestamp").and_then(|v| v.as_i64()).map(|v| v.to_string()),
event_time: None,
properties,
headers: m
.get("headers")
.and_then(|v| v.as_object())
.map(|obj| {
obj.iter().map(|(k, v)| (k.clone(), v.as_str().unwrap_or_default().to_string())).collect()
})
.unwrap_or_default(),
payload_base64: m.get("payloadBase64").and_then(|v| v.as_str()).unwrap_or_default().to_string(),
payload_text: m.get("payloadText").and_then(|v| v.as_str()).map(String::from),
}
})
.collect())
} }
async fn expire_messages(&self, _topic: &TopicRef, _sub: &str, _expire_seconds: i64) -> Result<(), String> { async fn expire_messages(&self, _topic: &TopicRef, _sub: &str, _expire_seconds: i64) -> Result<(), String> {
@@ -687,6 +658,70 @@ fn build_connection_params(cfg: &MqAdminConfig) -> serde_json::Value {
}) })
} }
fn peek_messages_params(
cfg: &MqAdminConfig,
topic: &TopicRef,
count: u32,
options: PeekMessagesOptions,
) -> serde_json::Value {
let mut params = serde_json::json!({
"topic": topic.topic,
"count": count,
"connection": build_connection_params(cfg),
});
// Keep absent fields absent so older earliest and offset requests retain
// their established agent semantics, including offset 0.
if let Some(start_position) = options.start_position {
params["startPosition"] = serde_json::json!(start_position);
}
if let Some(partition) = options.partition {
params["partition"] = serde_json::json!(partition);
}
if let Some(offset) = options.offset {
params["offset"] = serde_json::json!(offset);
}
params
}
fn peek_messages_result_from_agent(result: &serde_json::Value) -> PeekMessagesResult {
let (messages, incomplete) = if let Some(messages) = result.as_array() {
(messages.clone(), false)
} else {
(
result.get("messages").and_then(|v| v.as_array()).cloned().unwrap_or_default(),
result.get("incomplete").and_then(|v| v.as_bool()).unwrap_or(false),
)
};
let messages = messages
.into_iter()
.enumerate()
.map(|(idx, m)| {
let mut properties = HashMap::new();
if let Some(partition) = m.get("partition").and_then(|v| v.as_i64()) {
properties.insert("partition".to_string(), partition.to_string());
}
PeekedMessage {
position: (idx + 1) as u32,
message_id: m.get("offset").and_then(|v| v.as_i64()).map(|v| v.to_string()),
key: m.get("key").and_then(|v| v.as_str()).map(String::from),
publish_time: m.get("timestamp").and_then(|v| v.as_i64()).map(|v| v.to_string()),
event_time: None,
properties,
headers: m
.get("headers")
.and_then(|v| v.as_object())
.map(|obj| {
obj.iter().map(|(k, v)| (k.clone(), v.as_str().unwrap_or_default().to_string())).collect()
})
.unwrap_or_default(),
payload_base64: m.get("payloadBase64").and_then(|v| v.as_str()).unwrap_or_default().to_string(),
payload_text: m.get("payloadText").and_then(|v| v.as_str()).map(String::from),
}
})
.collect();
PeekMessagesResult { messages, incomplete }
}
fn reset_cursor_params(topic: &TopicRef, sub: &str, pos: ResetPosition) -> Result<serde_json::Value, String> { fn reset_cursor_params(topic: &TopicRef, sub: &str, pos: ResetPosition) -> Result<serde_json::Value, String> {
match pos { match pos {
ResetPosition::Earliest => Ok(serde_json::json!({ ResetPosition::Earliest => Ok(serde_json::json!({
@@ -771,6 +806,95 @@ mod tests {
} }
} }
fn topic_ref() -> TopicRef {
TopicRef {
tenant: "_kafka".to_string(),
namespace: "_kafka".to_string(),
topic: "events".to_string(),
persistent: true,
partitioned: None,
message_type: None,
..TopicRef::default()
}
}
#[test]
fn peek_message_params_forward_explicit_start_position_and_offset_filters() {
let cfg = kafka_config(serde_json::json!({ "bootstrapServers": "broker:9092" }), MqAuth::None, false);
let params = peek_messages_params(
&cfg,
&topic_ref(),
20,
PeekMessagesOptions {
start_position: Some(PeekStartPosition::Offset),
partition: Some(2),
offset: Some(17),
},
);
assert_eq!(params.get("topic").and_then(|value| value.as_str()), Some("events"));
assert_eq!(params.get("count").and_then(|value| value.as_u64()), Some(20));
assert_eq!(params.get("startPosition").and_then(|value| value.as_str()), Some("offset"));
assert_eq!(params.get("partition").and_then(|value| value.as_i64()), Some(2));
assert_eq!(params.get("offset").and_then(|value| value.as_i64()), Some(17));
}
#[test]
fn peek_message_params_omit_legacy_optional_fields() {
let cfg = kafka_config(serde_json::json!({ "bootstrapServers": "broker:9092" }), MqAuth::None, false);
let params = peek_messages_params(&cfg, &topic_ref(), 20, PeekMessagesOptions::default());
assert!(params.get("startPosition").is_none());
assert!(params.get("partition").is_none());
assert!(params.get("offset").is_none());
}
#[test]
fn peek_result_preserves_the_agent_incomplete_status() {
let result = peek_messages_result_from_agent(&serde_json::json!({
"messages": [{
"partition": 2,
"offset": 17,
"payloadBase64": "aGVsbG8=",
}],
"incomplete": true,
}));
assert!(result.incomplete);
assert_eq!(result.messages.len(), 1);
assert_eq!(result.messages[0].properties.get("partition").map(String::as_str), Some("2"));
assert_eq!(result.messages[0].message_id.as_deref(), Some("17"));
}
#[test]
fn peek_result_accepts_legacy_agent_array_responses() {
let result = peek_messages_result_from_agent(&serde_json::json!([{
"partition": 1,
"offset": 9,
"payloadBase64": "bGVnYWN5",
}]));
assert!(!result.incomplete);
assert_eq!(result.messages.len(), 1);
assert_eq!(result.messages[0].properties.get("partition").map(String::as_str), Some("1"));
assert_eq!(result.messages[0].message_id.as_deref(), Some("9"));
assert_eq!(result.messages[0].payload_base64, "bGVnYWN5");
}
#[test]
fn peek_message_params_preserve_legacy_offset_requests() {
let cfg = kafka_config(serde_json::json!({ "bootstrapServers": "broker:9092" }), MqAuth::None, false);
let params = peek_messages_params(
&cfg,
&topic_ref(),
20,
PeekMessagesOptions { start_position: None, partition: None, offset: Some(17) },
);
assert!(params.get("startPosition").is_none());
assert_eq!(params.get("offset").and_then(|value| value.as_i64()), Some(17));
}
#[test] #[test]
fn connection_params_map_basic_auth_to_kafka_sasl_without_plaintext_password_extra() { fn connection_params_map_basic_auth_to_kafka_sasl_without_plaintext_password_extra() {
let cfg = kafka_config( let cfg = kafka_config(
+3 -3
View File
@@ -451,9 +451,9 @@ impl MessageQueueAdmin for PulsarAdmin {
sub: &str, sub: &str,
count: u32, count: u32,
_options: PeekMessagesOptions, _options: PeekMessagesOptions,
) -> Result<Vec<PeekedMessage>, String> { ) -> Result<PeekMessagesResult, String> {
if count == 0 { if count == 0 {
return Ok(Vec::new()); return Ok(PeekMessagesResult::default());
} }
let messages: Vec<PeekedMessage> = stream::iter(1..=count) let messages: Vec<PeekedMessage> = stream::iter(1..=count)
@@ -477,7 +477,7 @@ impl MessageQueueAdmin for PulsarAdmin {
.flatten() .flatten()
.collect(); .collect();
Ok(messages) Ok(PeekMessagesResult::complete(messages))
} }
async fn expire_messages(&self, topic: &TopicRef, sub: &str, expire_seconds: i64) -> Result<(), String> { async fn expire_messages(&self, topic: &TopicRef, sub: &str, expire_seconds: i64) -> Result<(), String> {
+4 -2
View File
@@ -517,7 +517,7 @@ impl MessageQueueAdmin for RabbitMqAdmin {
_sub: &str, _sub: &str,
count: u32, count: u32,
_options: PeekMessagesOptions, _options: PeekMessagesOptions,
) -> Result<Vec<PeekedMessage>, String> { ) -> Result<PeekMessagesResult, String> {
let conn_params = build_connection_params(&self.config)?; let conn_params = build_connection_params(&self.config)?;
require_specific_vhost(&topic.namespace)?; require_specific_vhost(&topic.namespace)?;
let params = with_virtual_host( let params = with_virtual_host(
@@ -531,7 +531,9 @@ impl MessageQueueAdmin for RabbitMqAdmin {
let result: serde_json::Value = self.call("mq_peek_messages", params).await?; let result: serde_json::Value = self.call("mq_peek_messages", params).await?;
let messages = result.get("messages").and_then(|v| v.as_array()).cloned().unwrap_or_default(); let messages = result.get("messages").and_then(|v| v.as_array()).cloned().unwrap_or_default();
Ok(messages.into_iter().enumerate().map(|(idx, m)| peeked_message_from_json(idx, &m)).collect()) Ok(PeekMessagesResult::complete(
messages.into_iter().enumerate().map(|(idx, m)| peeked_message_from_json(idx, &m)).collect(),
))
} }
async fn expire_messages(&self, _topic: &TopicRef, _sub: &str, _expire_seconds: i64) -> Result<(), String> { async fn expire_messages(&self, _topic: &TopicRef, _sub: &str, _expire_seconds: i64) -> Result<(), String> {
+4 -2
View File
@@ -444,7 +444,7 @@ impl MessageQueueAdmin for RocketMqAdmin {
_sub: &str, _sub: &str,
count: u32, count: u32,
options: PeekMessagesOptions, options: PeekMessagesOptions,
) -> Result<Vec<PeekedMessage>, String> { ) -> Result<PeekMessagesResult, String> {
let conn_params = build_connection_params(&self.config); let conn_params = build_connection_params(&self.config);
let mut params = serde_json::json!({ let mut params = serde_json::json!({
"topic": topic.topic, "topic": topic.topic,
@@ -462,7 +462,9 @@ impl MessageQueueAdmin for RocketMqAdmin {
let result: serde_json::Value = self.call("mq_peek_messages", params).await?; let result: serde_json::Value = self.call("mq_peek_messages", params).await?;
let messages = result.get("messages").and_then(|v| v.as_array()).cloned().unwrap_or_default(); let messages = result.get("messages").and_then(|v| v.as_array()).cloned().unwrap_or_default();
Ok(messages.into_iter().enumerate().map(|(idx, m)| peeked_message_from_agent_json(idx, &m)).collect()) Ok(PeekMessagesResult::complete(
messages.into_iter().enumerate().map(|(idx, m)| peeked_message_from_agent_json(idx, &m)).collect(),
))
} }
async fn expire_messages(&self, _topic: &TopicRef, _sub: &str, _expire_seconds: i64) -> Result<(), String> { async fn expire_messages(&self, _topic: &TopicRef, _sub: &str, _expire_seconds: i64) -> Result<(), String> {
+1 -1
View File
@@ -160,7 +160,7 @@ pub trait MessageQueueAdmin: Send + Sync {
sub: &str, sub: &str,
count: u32, count: u32,
options: PeekMessagesOptions, options: PeekMessagesOptions,
) -> Result<Vec<PeekedMessage>, String>; ) -> Result<PeekMessagesResult, String>;
async fn expire_messages(&self, topic: &TopicRef, sub: &str, expire_seconds: i64) -> Result<(), String>; async fn expire_messages(&self, topic: &TopicRef, sub: &str, expire_seconds: i64) -> Result<(), String>;
/// RocketMQ: read subscription group config from broker metadata. /// RocketMQ: read subscription group config from broker metadata.
+2 -2
View File
@@ -476,9 +476,9 @@ pub async fn mq_peek_messages_core(
sub: String, sub: String,
count: u32, count: u32,
options: Option<PeekMessagesOptions>, options: Option<PeekMessagesOptions>,
) -> Result<Vec<PeekedMessage>, String> { ) -> Result<PeekMessagesResult, String> {
if count == 0 { if count == 0 {
return Ok(Vec::new()); return Ok(PeekMessagesResult::default());
} }
if count > MAX_PEEK_MESSAGES { if count > MAX_PEEK_MESSAGES {
return Err(format!("Peek message count must be between 1 and {MAX_PEEK_MESSAGES}")); return Err(format!("Peek message count must be between 1 and {MAX_PEEK_MESSAGES}"));
+63 -4
View File
@@ -496,13 +496,45 @@ pub struct PeekedMessage {
pub payload_text: Option<String>, pub payload_text: Option<String>,
} }
/// Optional hints for reading messages. Pulsar ignores these today; Kafka uses /// A message browse response. `incomplete` is set when the adapter could only
/// them to optionally narrow a non-committing peek to one partition / offset. /// return a partial snapshot before its configured deadline or scan limit.
/// When omitted, Kafka peeks across all partitions from each partition's earliest #[derive(Debug, Clone, Default, Serialize, Deserialize)]
/// readable offset (still capped by `count`). #[serde(rename_all = "camelCase")]
pub struct PeekMessagesResult {
#[serde(default)]
pub messages: Vec<PeekedMessage>,
#[serde(default)]
pub incomplete: bool,
}
impl PeekMessagesResult {
pub fn complete(messages: Vec<PeekedMessage>) -> Self {
Self { messages, incomplete: false }
}
}
/// Kafka's explicit starting position for a non-committing message peek.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum PeekStartPosition {
Latest,
Earliest,
Offset,
}
/// Optional hints for reading messages. Non-Kafka adapters ignore
/// `start_position` and retain their existing partition/offset handling.
/// For Kafka, an omitted position keeps legacy behavior: it starts at the
/// earliest available message unless an older caller supplies `offset`.
/// `start_position: Offset` requires a non-negative offset. When no partition
/// is supplied, Kafka reads forward from that offset in every topic partition.
#[derive(Debug, Clone, Default, Serialize, Deserialize)] #[derive(Debug, Clone, Default, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub struct PeekMessagesOptions { pub struct PeekMessagesOptions {
/// Kafka's explicit read starting point. This remains optional so callers
/// that predate it retain their original earliest/offset behavior.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub start_position: Option<PeekStartPosition>,
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
pub partition: Option<i32>, pub partition: Option<i32>,
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
@@ -1191,4 +1223,31 @@ mod tests {
assert!(!caps.supports_cluster_monitoring); assert!(!caps.supports_cluster_monitoring);
assert!(caps.supports_tenants); assert!(caps.supports_tenants);
} }
#[test]
fn peek_options_omit_optional_fields_and_serialize_start_position() {
let legacy: super::PeekMessagesOptions =
serde_json::from_str(r#"{"partition":2}"#).expect("legacy peek options");
assert_eq!(legacy.start_position, None);
assert_eq!(legacy.partition, Some(2));
assert_eq!(legacy.offset, None);
let latest: super::PeekMessagesOptions =
serde_json::from_str(r#"{"startPosition":"latest"}"#).expect("latest peek options");
assert_eq!(latest.start_position, Some(super::PeekStartPosition::Latest));
let json = serde_json::to_value(latest).expect("serialize latest peek options");
assert_eq!(json.get("startPosition").and_then(|value| value.as_str()), Some("latest"));
}
#[test]
fn peek_result_serializes_completion_status_and_accepts_legacy_responses() {
let complete = super::PeekMessagesResult::complete(Vec::new());
let json = serde_json::to_value(&complete).expect("serialize complete peek result");
assert_eq!(json, serde_json::json!({ "messages": [], "incomplete": false }));
let legacy: super::PeekMessagesResult =
serde_json::from_value(serde_json::json!({ "messages": [] })).expect("deserialize legacy peek result");
assert!(!legacy.incomplete);
assert!(legacy.messages.is_empty());
}
} }
+1 -1
View File
@@ -850,7 +850,7 @@ pub async fn peek_messages(
State(state): State<Arc<WebState>>, State(state): State<Arc<WebState>>,
headers: HeaderMap, headers: HeaderMap,
Json(req): Json<PeekMessagesReq>, Json(req): Json<PeekMessagesReq>,
) -> Result<Json<Vec<dbx_core::mq::PeekedMessage>>, AppError> { ) -> Result<Json<dbx_core::mq::PeekMessagesResult>, AppError> {
super::mcp_policy::ensure_scope(&state, &headers, &req.connection_id).await?; super::mcp_policy::ensure_scope(&state, &headers, &req.connection_id).await?;
let result = dbx_core::mq::service::mq_peek_messages_core( let result = dbx_core::mq::service::mq_peek_messages_core(
&state.app, &state.app,
+1 -1
View File
@@ -354,7 +354,7 @@ pub async fn mq_peek_messages(
sub: String, sub: String,
count: u32, count: u32,
options: Option<dbx_core::mq::PeekMessagesOptions>, options: Option<dbx_core::mq::PeekMessagesOptions>,
) -> Result<Vec<dbx_core::mq::PeekedMessage>, String> { ) -> Result<dbx_core::mq::PeekMessagesResult, String> {
dbx_core::mq::service::mq_peek_messages_core(&state, &connection_id, topic, sub, count, options).await dbx_core::mq::service::mq_peek_messages_core(&state, &connection_id, topic, sub, count, options).await
} }