fix(kafka): clamp peek offset to available range
This commit is contained in:
1 parent
ac6b2f6cb5
commit
fd80464cb2
2 files changed
+57
-1
No files matched your search
@@ -753,7 +753,17 @@ public final class KafkaAgent {
|
|||||||
TopicPartition tp = new TopicPartition(topic, partition);
|
TopicPartition tp = new TopicPartition(topic, partition);
|
||||||
try (KafkaConsumer<String, byte[]> consumer = new KafkaConsumer<>(props)) {
|
try (KafkaConsumer<String, byte[]> consumer = new KafkaConsumer<>(props)) {
|
||||||
consumer.assign(Collections.singletonList(tp));
|
consumer.assign(Collections.singletonList(tp));
|
||||||
consumer.seek(tp, offset);
|
Map<TopicPartition, Long> beginningOffsets =
|
||||||
|
consumer.beginningOffsets(Collections.singletonList(tp), Duration.ofSeconds(5));
|
||||||
|
Map<TopicPartition, Long> endOffsets =
|
||||||
|
consumer.endOffsets(Collections.singletonList(tp), Duration.ofSeconds(5));
|
||||||
|
long beginningOffset = beginningOffsets.getOrDefault(tp, 0L);
|
||||||
|
long endOffset = endOffsets.getOrDefault(tp, beginningOffset);
|
||||||
|
Long seekOffset = normalizePeekOffset(offset, beginningOffset, endOffset);
|
||||||
|
if (seekOffset == null) {
|
||||||
|
return Collections.singletonMap("messages", Collections.emptyList());
|
||||||
|
}
|
||||||
|
consumer.seek(tp, seekOffset);
|
||||||
|
|
||||||
List<Map<String, Object>> messages = new ArrayList<>();
|
List<Map<String, Object>> messages = new ArrayList<>();
|
||||||
ConsumerRecords<String, byte[]> records = consumer.poll(Duration.ofSeconds(5));
|
ConsumerRecords<String, byte[]> records = consumer.poll(Duration.ofSeconds(5));
|
||||||
@@ -1124,6 +1134,19 @@ public final class KafkaAgent {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static Long normalizePeekOffset(long requestedOffset, long beginningOffset, long endOffset) {
|
||||||
|
if (endOffset <= beginningOffset) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (requestedOffset < beginningOffset) {
|
||||||
|
return beginningOffset;
|
||||||
|
}
|
||||||
|
if (requestedOffset >= endOffset) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return requestedOffset;
|
||||||
|
}
|
||||||
|
|
||||||
private static String stringOrNull(JsonObject object, String key) {
|
private static String stringOrNull(JsonObject object, String key) {
|
||||||
JsonElement element = object.get(key);
|
JsonElement element = object.get(key);
|
||||||
return element == null || element.isJsonNull() ? null : element.getAsString();
|
return element == null || element.isJsonNull() ? null : element.getAsString();
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
package com.dbx.agent.kafka;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
class KafkaAgentTest {
|
||||||
|
@Test
|
||||||
|
void normalizesPeekOffsetToEarliestAvailableOffset() {
|
||||||
|
assertEquals(5L, KafkaAgent.normalizePeekOffset(0, 5, 10));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void normalizesNegativePeekOffsetToEarliestAvailableOffset() {
|
||||||
|
assertEquals(0L, KafkaAgent.normalizePeekOffset(-1, 0, 10));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void keepsPeekOffsetWhenItIsWithinAvailableRange() {
|
||||||
|
assertEquals(7L, KafkaAgent.normalizePeekOffset(7, 5, 10));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void returnsNoSeekOffsetWhenRequestedOffsetIsAtOrAfterEnd() {
|
||||||
|
assertNull(KafkaAgent.normalizePeekOffset(10, 5, 10));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void returnsNoSeekOffsetWhenTopicHasNoReadableMessages() {
|
||||||
|
assertNull(KafkaAgent.normalizePeekOffset(0, 5, 5));
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user