fix(kafka): support ZooKeeper broker discovery
This commit is contained in:
1 parent
34448cba9d
commit
8adf802444
18 files changed
+977
-77
No files matched your search
@@ -13,12 +13,21 @@ import org.apache.kafka.common.resource.PatternType;
|
||||
import org.apache.kafka.common.resource.ResourcePattern;
|
||||
import org.apache.kafka.common.resource.ResourcePatternFilter;
|
||||
import org.apache.kafka.common.resource.ResourceType;
|
||||
import org.apache.zookeeper.KeeperException;
|
||||
import org.apache.zookeeper.Watcher;
|
||||
import org.apache.zookeeper.ZooKeeper;
|
||||
import org.apache.zookeeper.client.ZKClientConfig;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.PrintStream;
|
||||
import java.net.URI;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@@ -29,9 +38,12 @@ import java.util.stream.Collectors;
|
||||
*/
|
||||
public final class KafkaAgent {
|
||||
|
||||
private static final PrintStream JSON_RPC_OUT = System.out;
|
||||
private static final Gson GSON = new GsonBuilder().serializeNulls().create();
|
||||
private static final int DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
|
||||
private static final int DEFAULT_SESSION_TIMEOUT_MS = 30_000;
|
||||
private static final int DEFAULT_ZOOKEEPER_CONNECTION_TIMEOUT_MS = 10_000;
|
||||
private static final String ZOOKEEPER_PROPERTY_PREFIX = "zookeeper.";
|
||||
private static final Set<String> KERBEROS_SYSTEM_PROPERTY_KEYS = Set.of(
|
||||
"java.security.krb5.conf",
|
||||
"sun.security.krb5.debug",
|
||||
@@ -47,26 +59,40 @@ public final class KafkaAgent {
|
||||
|
||||
private static AdminClient adminClient;
|
||||
private static KafkaProducer<String, byte[]> producer;
|
||||
private static JsonObject activeConnection;
|
||||
private static volatile boolean shutdownRequested;
|
||||
|
||||
private KafkaAgent() {}
|
||||
|
||||
private static Logger logger() {
|
||||
// Initialize only after main redirects System.out, so any logging backend
|
||||
// that defaults to stdout still cannot write into the JSON-RPC channel.
|
||||
return LoggerHolder.INSTANCE;
|
||||
}
|
||||
|
||||
private static final class LoggerHolder {
|
||||
private static final Logger INSTANCE = LoggerFactory.getLogger(KafkaAgent.class);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Entry point
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
// Keep the original stdout exclusively for JSON-RPC. Redirect accidental
|
||||
// System.out writes from dependencies to stderr so they cannot corrupt the protocol.
|
||||
System.setOut(System.err);
|
||||
System.setProperty("org.slf4j.simpleLogger.logFile", "System.err");
|
||||
System.out.println("{\"ready\":true}");
|
||||
System.out.flush();
|
||||
JSON_RPC_OUT.println("{\"ready\":true}");
|
||||
JSON_RPC_OUT.flush();
|
||||
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(System.in));
|
||||
while (true) {
|
||||
String line = reader.readLine();
|
||||
if (line == null) break;
|
||||
String response = handleRequest(line);
|
||||
System.out.println(response);
|
||||
System.out.flush();
|
||||
JSON_RPC_OUT.println(response);
|
||||
JSON_RPC_OUT.flush();
|
||||
if (shutdownRequested) {
|
||||
System.exit(0);
|
||||
}
|
||||
@@ -92,6 +118,7 @@ public final class KafkaAgent {
|
||||
Object result = dispatch(method, params);
|
||||
response.add("result", GSON.toJsonTree(result));
|
||||
} catch (Exception e) {
|
||||
logger().warn("Kafka Agent request failed: method={}, id={}", method, id, e);
|
||||
JsonObject error = new JsonObject();
|
||||
error.addProperty("code", -1);
|
||||
error.addProperty("message", normalizeErrorMessage(e));
|
||||
@@ -144,7 +171,7 @@ public final class KafkaAgent {
|
||||
}
|
||||
|
||||
private static Object connect(JsonObject params) throws Exception {
|
||||
JsonObject conn = connectionObject(params);
|
||||
JsonObject conn = resolveBrokerConnection(connectionObject(params));
|
||||
Map<String, String> previousKerberosSystemProperties = applyKerberosSystemProperties(conn);
|
||||
AdminClient nextAdmin = null;
|
||||
KafkaProducer<String, byte[]> nextProducer = null;
|
||||
@@ -158,6 +185,7 @@ public final class KafkaAgent {
|
||||
applyKerberosSystemProperties(conn);
|
||||
adminClient = nextAdmin;
|
||||
producer = nextProducer;
|
||||
activeConnection = conn.deepCopy();
|
||||
return Collections.singletonMap("ok", true);
|
||||
} catch (Exception e) {
|
||||
if (nextAdmin != null) {
|
||||
@@ -172,7 +200,7 @@ public final class KafkaAgent {
|
||||
}
|
||||
|
||||
private static Object testConnection(JsonObject params) throws Exception {
|
||||
JsonObject conn = connectionObject(params);
|
||||
JsonObject conn = resolveBrokerConnection(connectionObject(params));
|
||||
Map<String, String> previousKerberosSystemProperties = applyKerberosSystemProperties(conn);
|
||||
AdminClient probe = null;
|
||||
try {
|
||||
@@ -189,14 +217,11 @@ public final class KafkaAgent {
|
||||
probe.describeAcls(AclBindingFilter.ANY)
|
||||
.values().get(timeout, TimeUnit.MILLISECONDS);
|
||||
} catch (Exception aclEx) {
|
||||
Throwable cause = aclEx;
|
||||
while (cause != null) {
|
||||
if (cause.getClass().getSimpleName().contains("SecurityDisabled")
|
||||
|| (cause.getMessage() != null && cause.getMessage().contains("No Authorizer"))) {
|
||||
aclEnabled = false;
|
||||
break;
|
||||
}
|
||||
cause = cause.getCause();
|
||||
if (isAclDisabledError(aclEx)) {
|
||||
aclEnabled = false;
|
||||
logger().debug("Kafka ACL support is disabled by the broker");
|
||||
} else {
|
||||
logger().warn("Kafka ACL capability probe failed; leaving the capability enabled", aclEx);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -219,6 +244,18 @@ public final class KafkaAgent {
|
||||
}
|
||||
}
|
||||
|
||||
static boolean isAclDisabledError(Throwable error) {
|
||||
Throwable cause = error;
|
||||
while (cause != null) {
|
||||
if (cause.getClass().getSimpleName().contains("SecurityDisabled")
|
||||
|| (cause.getMessage() != null && cause.getMessage().contains("No Authorizer"))) {
|
||||
return true;
|
||||
}
|
||||
cause = cause.getCause();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static void closeClients() {
|
||||
if (adminClient != null) {
|
||||
adminClient.close(Duration.ofSeconds(5));
|
||||
@@ -228,6 +265,7 @@ public final class KafkaAgent {
|
||||
producer.close(Duration.ofSeconds(5));
|
||||
producer = null;
|
||||
}
|
||||
activeConnection = null;
|
||||
restoreKerberosSystemProperties(BASELINE_KERBEROS_SYSTEM_PROPERTIES);
|
||||
}
|
||||
|
||||
@@ -269,6 +307,160 @@ public final class KafkaAgent {
|
||||
return servers;
|
||||
}
|
||||
|
||||
static JsonObject resolveBrokerConnection(JsonObject conn) throws Exception {
|
||||
String configured = stringOrEmpty(conn, "bootstrap_servers");
|
||||
if (configured.isBlank()) configured = stringOrEmpty(conn, "bootstrapServers");
|
||||
if (!configured.isBlank()) return conn;
|
||||
|
||||
String connectString = stringOrEmpty(conn, "zookeeper_connect_string");
|
||||
if (connectString.isBlank()) connectString = stringOrEmpty(conn, "zookeeperServers");
|
||||
if (connectString.isBlank()) {
|
||||
throw new IllegalArgumentException("bootstrap_servers or zookeeper_connect_string is required");
|
||||
}
|
||||
|
||||
JsonObject resolved = conn.deepCopy();
|
||||
resolved.addProperty("bootstrap_servers", discoverBootstrapServers(connectString, securityProtocol(conn), conn));
|
||||
return resolved;
|
||||
}
|
||||
|
||||
private static String discoverBootstrapServers(String connectString, String securityProtocol, JsonObject conn)
|
||||
throws Exception {
|
||||
int sessionTimeout = intOrDefault(conn, "zookeeper_session_timeout_ms", DEFAULT_SESSION_TIMEOUT_MS);
|
||||
int connectionTimeout = intOrDefault(
|
||||
conn,
|
||||
"zookeeper_connection_timeout_ms",
|
||||
DEFAULT_ZOOKEEPER_CONNECTION_TIMEOUT_MS
|
||||
);
|
||||
CountDownLatch connected = new CountDownLatch(1);
|
||||
ZooKeeper zooKeeper = new ZooKeeper(connectString, sessionTimeout, event -> {
|
||||
if (event.getState() == Watcher.Event.KeeperState.SyncConnected) connected.countDown();
|
||||
}, zooKeeperClientConfig(conn));
|
||||
try {
|
||||
if (!connected.await(connectionTimeout, TimeUnit.MILLISECONDS)) {
|
||||
throw new IllegalStateException("Timed out connecting to ZooKeeper for Kafka broker discovery");
|
||||
}
|
||||
|
||||
List<String> brokerIds;
|
||||
try {
|
||||
brokerIds = new ArrayList<>(zooKeeper.getChildren("/brokers/ids", false));
|
||||
} catch (KeeperException.NoNodeException e) {
|
||||
throw new IllegalStateException("ZooKeeper path /brokers/ids does not exist", e);
|
||||
}
|
||||
brokerIds.sort(KafkaAgent::compareBrokerIds);
|
||||
|
||||
List<JsonObject> registrations = new ArrayList<>();
|
||||
for (String brokerId : brokerIds) {
|
||||
try {
|
||||
byte[] data = zooKeeper.getData("/brokers/ids/" + brokerId, false, null);
|
||||
registrations.add(JsonParser.parseString(new String(data, StandardCharsets.UTF_8)).getAsJsonObject());
|
||||
} catch (KeeperException.NoNodeException e) {
|
||||
// Expected race: a broker may refresh its ephemeral node between list and read.
|
||||
logger().debug("Kafka broker {} disappeared during ZooKeeper discovery", brokerId);
|
||||
} catch (RuntimeException e) {
|
||||
logger().warn("Skipping malformed ZooKeeper registration for Kafka broker {}", brokerId, e);
|
||||
}
|
||||
}
|
||||
return brokerEndpoints(registrations, securityProtocol);
|
||||
} finally {
|
||||
zooKeeper.close();
|
||||
}
|
||||
}
|
||||
|
||||
static ZKClientConfig zooKeeperClientConfig(JsonObject conn) {
|
||||
ZKClientConfig clientConfig = new ZKClientConfig();
|
||||
JsonObject properties = connectionProperties(conn);
|
||||
if (properties == null) return clientConfig;
|
||||
|
||||
for (Map.Entry<String, JsonElement> entry : properties.entrySet()) {
|
||||
if (entry.getKey().startsWith(ZOOKEEPER_PROPERTY_PREFIX)
|
||||
&& entry.getValue().isJsonPrimitive()) {
|
||||
clientConfig.setProperty(entry.getKey(), entry.getValue().getAsString());
|
||||
}
|
||||
}
|
||||
return clientConfig;
|
||||
}
|
||||
|
||||
private static int compareBrokerIds(String left, String right) {
|
||||
try {
|
||||
return Integer.compare(Integer.parseInt(left), Integer.parseInt(right));
|
||||
} catch (NumberFormatException e) {
|
||||
// Third-party registries may use non-numeric IDs; lexical ordering remains deterministic.
|
||||
logger().debug("Sorting non-numeric Kafka broker IDs lexically: left={}, right={}", left, right);
|
||||
return left.compareTo(right);
|
||||
}
|
||||
}
|
||||
|
||||
static String brokerEndpoints(List<JsonObject> registrations, String securityProtocol) {
|
||||
String targetProtocol = securityProtocol == null || securityProtocol.isBlank()
|
||||
? "PLAINTEXT"
|
||||
: securityProtocol.toUpperCase(Locale.ROOT);
|
||||
Set<String> addresses = new LinkedHashSet<>();
|
||||
|
||||
for (JsonObject registration : registrations) {
|
||||
int addressCount = addresses.size();
|
||||
JsonObject protocolMap = registration.has("listener_security_protocol_map")
|
||||
&& registration.get("listener_security_protocol_map").isJsonObject()
|
||||
? registration.getAsJsonObject("listener_security_protocol_map")
|
||||
: new JsonObject();
|
||||
JsonArray endpoints = registration.has("endpoints") && registration.get("endpoints").isJsonArray()
|
||||
? registration.getAsJsonArray("endpoints")
|
||||
: new JsonArray();
|
||||
|
||||
for (JsonElement element : endpoints) {
|
||||
if (!element.isJsonPrimitive() || !element.getAsJsonPrimitive().isString()) continue;
|
||||
String endpoint = element.getAsString();
|
||||
int separator = endpoint.indexOf("://");
|
||||
if (separator <= 0) continue;
|
||||
String listener = endpoint.substring(0, separator).toUpperCase(Locale.ROOT);
|
||||
JsonElement mapped = protocolMap.get(listener);
|
||||
String mappedProtocol = mapped != null && mapped.isJsonPrimitive()
|
||||
? mapped.getAsString().toUpperCase(Locale.ROOT)
|
||||
: listener;
|
||||
if (!targetProtocol.equals(mappedProtocol)) continue;
|
||||
String address = endpointAddress(endpoint);
|
||||
if (address != null) addresses.add(address);
|
||||
}
|
||||
|
||||
if (addresses.size() == addressCount && endpoints.size() == 0 && registration.has("host") && registration.has("port")) {
|
||||
try {
|
||||
String host = registration.get("host").getAsString().trim();
|
||||
int port = registration.get("port").getAsInt();
|
||||
if (!host.isEmpty() && port > 0 && port <= 65535) addresses.add(formatHostPort(host, port));
|
||||
} catch (RuntimeException e) {
|
||||
logger().warn("Skipping malformed legacy Kafka broker registration", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (addresses.isEmpty()) {
|
||||
throw new IllegalArgumentException("ZooKeeper did not return any usable Kafka broker endpoints");
|
||||
}
|
||||
return String.join(",", addresses);
|
||||
}
|
||||
|
||||
private static String endpointAddress(String endpoint) {
|
||||
try {
|
||||
URI uri = URI.create(endpoint);
|
||||
String host = uri.getHost();
|
||||
int port = uri.getPort();
|
||||
if (host == null || host.isBlank() || port <= 0 || port > 65535) return null;
|
||||
return formatHostPort(host, port);
|
||||
} catch (IllegalArgumentException e) {
|
||||
logger().debug("Skipping malformed Kafka broker endpoint", e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static String formatHostPort(String host, int port) {
|
||||
return host.contains(":") && !host.startsWith("[") ? "[" + host + "]:" + port : host + ":" + port;
|
||||
}
|
||||
|
||||
private static String securityProtocol(JsonObject conn) {
|
||||
String protocol = stringOrEmpty(conn, "security_protocol");
|
||||
if (protocol.isBlank()) protocol = stringOrEmpty(conn, "securityProtocol");
|
||||
return protocol.isBlank() ? "PLAINTEXT" : protocol;
|
||||
}
|
||||
|
||||
static void applySecurityProperties(JsonObject conn, Properties props) {
|
||||
String securityProtocol = stringOrEmpty(conn, "security_protocol");
|
||||
if (securityProtocol.isBlank()) {
|
||||
@@ -353,6 +545,7 @@ public final class KafkaAgent {
|
||||
for (Map.Entry<String, JsonElement> entry : properties.entrySet()) {
|
||||
if (entry.getValue().isJsonPrimitive()) {
|
||||
String key = entry.getKey();
|
||||
if (key.startsWith(ZOOKEEPER_PROPERTY_PREFIX)) continue;
|
||||
String value = entry.getValue().getAsString();
|
||||
props.put(key, value);
|
||||
}
|
||||
@@ -578,11 +771,47 @@ public final class KafkaAgent {
|
||||
}
|
||||
|
||||
ConfigResource resource = new ConfigResource(ConfigResource.Type.TOPIC, name);
|
||||
admin.incrementalAlterConfigs(Collections.singletonMap(resource, ops))
|
||||
.all().get(timeout, TimeUnit.MILLISECONDS);
|
||||
try {
|
||||
admin.incrementalAlterConfigs(Collections.singletonMap(resource, ops))
|
||||
.all().get(timeout, TimeUnit.MILLISECONDS);
|
||||
} catch (Exception e) {
|
||||
if (!isUnsupportedVersionError(e)) throw e;
|
||||
logger().info("Kafka broker does not support incrementalAlterConfigs; using legacy alterConfigs for topic {}", name);
|
||||
Config current = admin.describeConfigs(Collections.singletonList(resource))
|
||||
.all().get(timeout, TimeUnit.MILLISECONDS).get(resource);
|
||||
Map<String, String> values = legacyTopicConfig(current, ops);
|
||||
Config replacement = new Config(values.entrySet().stream()
|
||||
.map(entry -> new ConfigEntry(entry.getKey(), entry.getValue()))
|
||||
.collect(Collectors.toList()));
|
||||
admin.alterConfigs(Collections.singletonMap(resource, replacement))
|
||||
.all().get(timeout, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
return Collections.singletonMap("ok", true);
|
||||
}
|
||||
|
||||
static Map<String, String> legacyTopicConfig(Config current, List<AlterConfigOp> ops) {
|
||||
Map<String, String> values = new LinkedHashMap<>();
|
||||
for (ConfigEntry entry : current.entries()) {
|
||||
boolean topicOverride = entry.source() == ConfigEntry.ConfigSource.DYNAMIC_TOPIC_CONFIG
|
||||
|| entry.source() == ConfigEntry.ConfigSource.UNKNOWN;
|
||||
if (topicOverride && !entry.isReadOnly() && !entry.isSensitive() && entry.value() != null) {
|
||||
values.put(entry.name(), entry.value());
|
||||
}
|
||||
}
|
||||
|
||||
for (AlterConfigOp op : ops) {
|
||||
String key = op.configEntry().name();
|
||||
switch (op.opType()) {
|
||||
case SET -> values.put(key, op.configEntry().value());
|
||||
case DELETE -> values.remove(key);
|
||||
case APPEND, SUBTRACT -> throw new IllegalArgumentException(
|
||||
"Kafka broker does not support " + op.opType() + " config operations through the legacy alterConfigs API"
|
||||
);
|
||||
}
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Consumer groups
|
||||
// -----------------------------------------------------------------------
|
||||
@@ -698,6 +927,7 @@ public final class KafkaAgent {
|
||||
return Collections.singletonMap("producers", new ArrayList<>(byProducer.values()));
|
||||
} catch (Exception e) {
|
||||
if (isUnsupportedVersionError(e)) {
|
||||
logger().info("Kafka broker does not support describeProducers; returning an empty producer list");
|
||||
return Collections.singletonMap("producers", Collections.emptyList());
|
||||
}
|
||||
throw e;
|
||||
@@ -797,26 +1027,11 @@ public final class KafkaAgent {
|
||||
Long offset = longOrNull(params, "offset");
|
||||
int count = Math.max(1, intOrDefault(params, "count", 10));
|
||||
|
||||
// Build a temporary consumer for peeking (no commit)
|
||||
Properties props = new Properties();
|
||||
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,
|
||||
adminClient != null ? adminClient.describeCluster().clusterId()
|
||||
.get(5, TimeUnit.SECONDS) : "localhost:9092");
|
||||
// Reuse the admin's bootstrap servers
|
||||
JsonObject conn = params.has("connection") && params.get("connection").isJsonObject()
|
||||
? params.getAsJsonObject("connection") : null;
|
||||
if (conn != null) {
|
||||
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers(conn));
|
||||
applyConnectionProperties(conn, props);
|
||||
JsonObject conn = activeConnection;
|
||||
if (conn == null) {
|
||||
throw new IllegalStateException("Kafka Agent is not connected");
|
||||
}
|
||||
props.put(ConsumerConfig.GROUP_ID_CONFIG, "dbx-peek-" + UUID.randomUUID());
|
||||
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,
|
||||
"org.apache.kafka.common.serialization.StringDeserializer");
|
||||
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,
|
||||
"org.apache.kafka.common.serialization.ByteArrayDeserializer");
|
||||
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
|
||||
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "none");
|
||||
props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, count);
|
||||
Properties props = peekConsumerProperties(conn, count);
|
||||
|
||||
try (KafkaConsumer<String, byte[]> consumer = new KafkaConsumer<>(props)) {
|
||||
List<TopicPartition> candidatePartitions = resolvePeekPartitions(consumer, topic, partition);
|
||||
@@ -872,6 +1087,21 @@ public final class KafkaAgent {
|
||||
}
|
||||
}
|
||||
|
||||
static Properties peekConsumerProperties(JsonObject conn, int count) {
|
||||
Properties props = new Properties();
|
||||
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers(conn));
|
||||
applyConnectionProperties(conn, props);
|
||||
props.put(ConsumerConfig.GROUP_ID_CONFIG, "dbx-peek-" + UUID.randomUUID());
|
||||
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,
|
||||
"org.apache.kafka.common.serialization.StringDeserializer");
|
||||
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,
|
||||
"org.apache.kafka.common.serialization.ByteArrayDeserializer");
|
||||
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
|
||||
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "none");
|
||||
props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, count);
|
||||
return props;
|
||||
}
|
||||
|
||||
/**
|
||||
* Poll until {@code count} messages are collected, every assigned partition has reached its
|
||||
* end offset, or {@code deadlineNs} expires. Empty polls retry until caught-up or deadline —
|
||||
@@ -1335,15 +1565,10 @@ public final class KafkaAgent {
|
||||
}
|
||||
|
||||
private static String tryDecodeUtf8(byte[] bytes) {
|
||||
try {
|
||||
String text = new String(bytes, StandardCharsets.UTF_8);
|
||||
// Verify round-trip
|
||||
byte[] reEncoded = text.getBytes(StandardCharsets.UTF_8);
|
||||
if (Arrays.equals(bytes, reEncoded)) {
|
||||
return text;
|
||||
}
|
||||
} catch (Exception ignored) {}
|
||||
return null;
|
||||
String text = new String(bytes, StandardCharsets.UTF_8);
|
||||
// Replacement characters change the bytes on round-trip, identifying invalid UTF-8 without exceptions.
|
||||
byte[] reEncoded = text.getBytes(StandardCharsets.UTF_8);
|
||||
return Arrays.equals(bytes, reEncoded) ? text : null;
|
||||
}
|
||||
|
||||
static Long normalizePeekOffset(long requestedOffset, long beginningOffset, long endOffset) {
|
||||
|
||||
@@ -1,24 +1,282 @@
|
||||
package com.dbx.agent.kafka;
|
||||
|
||||
import com.google.gson.JsonObject;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import com.google.gson.JsonParser;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.time.Duration;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import org.apache.kafka.clients.consumer.ConsumerRecord;
|
||||
import org.apache.kafka.clients.consumer.ConsumerRecords;
|
||||
import org.apache.kafka.clients.admin.AlterConfigOp;
|
||||
import org.apache.kafka.clients.admin.Config;
|
||||
import org.apache.kafka.clients.admin.ConfigEntry;
|
||||
import org.apache.kafka.common.TopicPartition;
|
||||
import org.apache.zookeeper.CreateMode;
|
||||
import org.apache.zookeeper.Watcher;
|
||||
import org.apache.zookeeper.ZooDefs;
|
||||
import org.apache.zookeeper.ZooKeeper;
|
||||
import org.apache.zookeeper.client.ZKClientConfig;
|
||||
import org.apache.zookeeper.server.NIOServerCnxnFactory;
|
||||
import org.apache.zookeeper.server.ZooKeeperServer;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
class KafkaAgentTest {
|
||||
@TempDir
|
||||
Path tempDir;
|
||||
|
||||
@Test
|
||||
void resolvesBootstrapServersFromKafka11ZooKeeperRegistrationWithChroot() throws Exception {
|
||||
Path snapshots = Files.createDirectory(tempDir.resolve("snapshots"));
|
||||
Path logs = Files.createDirectory(tempDir.resolve("logs"));
|
||||
ZooKeeperServer server = new ZooKeeperServer(snapshots.toFile(), logs.toFile(), 2_000);
|
||||
NIOServerCnxnFactory factory = new NIOServerCnxnFactory();
|
||||
factory.configure(new InetSocketAddress("127.0.0.1", 0), 10);
|
||||
factory.startup(server);
|
||||
|
||||
ZooKeeper client = null;
|
||||
String previousSaslSetting = System.getProperty("zookeeper.sasl.client");
|
||||
try {
|
||||
CountDownLatch connected = new CountDownLatch(1);
|
||||
System.setProperty("zookeeper.sasl.client", "false");
|
||||
client = new ZooKeeper("127.0.0.1:" + factory.getLocalPort(), 5_000, event -> {
|
||||
if (event.getState() == Watcher.Event.KeeperState.SyncConnected) connected.countDown();
|
||||
});
|
||||
assertTrue(connected.await(5, TimeUnit.SECONDS));
|
||||
client.create("/kafka", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
|
||||
client.create("/kafka/brokers", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
|
||||
client.create("/kafka/brokers/ids", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
|
||||
client.create(
|
||||
"/kafka/brokers/ids/0",
|
||||
"{\"listener_security_protocol_map\":{\"PLAINTEXT\":\"PLAINTEXT\"},\"endpoints\":[\"PLAINTEXT://legacy-broker:9092\"]}".getBytes(StandardCharsets.UTF_8),
|
||||
ZooDefs.Ids.OPEN_ACL_UNSAFE,
|
||||
CreateMode.EPHEMERAL
|
||||
);
|
||||
|
||||
JsonObject connection = new JsonObject();
|
||||
connection.addProperty("zookeeper_connect_string", "127.0.0.1:" + factory.getLocalPort() + "/kafka");
|
||||
connection.addProperty("security_protocol", "PLAINTEXT");
|
||||
connection.addProperty("zookeeper_connection_timeout_ms", 5_000);
|
||||
|
||||
JsonObject resolved = KafkaAgent.resolveBrokerConnection(connection);
|
||||
|
||||
assertEquals("legacy-broker:9092", resolved.get("bootstrap_servers").getAsString());
|
||||
} finally {
|
||||
if (client != null) client.close();
|
||||
factory.shutdown();
|
||||
server.shutdown();
|
||||
server.getTxnLogFactory().close();
|
||||
if (previousSaslSetting == null) {
|
||||
System.clearProperty("zookeeper.sasl.client");
|
||||
} else {
|
||||
System.setProperty("zookeeper.sasl.client", previousSaslSetting);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void zooKeeperClientConfigPreservesSaslAndTlsSystemDefaults() {
|
||||
Map<String, String> previous = preserveSystemProperties(
|
||||
"zookeeper.sasl.client",
|
||||
"zookeeper.sasl.clientconfig",
|
||||
"zookeeper.client.secure",
|
||||
"zookeeper.clientCnxnSocket",
|
||||
"zookeeper.ssl.trustStore.location",
|
||||
"java.security.auth.login.config"
|
||||
);
|
||||
try {
|
||||
System.setProperty("zookeeper.sasl.client", "true");
|
||||
System.setProperty("zookeeper.sasl.clientconfig", "DbxZooKeeperClient");
|
||||
System.setProperty("zookeeper.client.secure", "true");
|
||||
System.setProperty("zookeeper.clientCnxnSocket", "org.apache.zookeeper.ClientCnxnSocketNetty");
|
||||
System.setProperty("zookeeper.ssl.trustStore.location", "/etc/dbx/zookeeper-truststore.p12");
|
||||
System.setProperty("java.security.auth.login.config", "/etc/dbx/zookeeper-jaas.conf");
|
||||
|
||||
ZKClientConfig config = KafkaAgent.zooKeeperClientConfig(new JsonObject());
|
||||
|
||||
assertTrue(config.isSaslClientEnabled());
|
||||
assertEquals("DbxZooKeeperClient", config.getProperty("zookeeper.sasl.clientconfig"));
|
||||
assertEquals("true", config.getProperty("zookeeper.client.secure"));
|
||||
assertEquals(
|
||||
"org.apache.zookeeper.ClientCnxnSocketNetty",
|
||||
config.getProperty("zookeeper.clientCnxnSocket")
|
||||
);
|
||||
assertEquals(
|
||||
"/etc/dbx/zookeeper-truststore.p12",
|
||||
config.getProperty("zookeeper.ssl.trustStore.location")
|
||||
);
|
||||
assertEquals("/etc/dbx/zookeeper-jaas.conf", config.getJaasConfKey());
|
||||
} finally {
|
||||
restoreSystemProperties(previous);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void zooKeeperClientConfigAppliesPerConnectionSaslAndTlsOverridesWithoutChangingJvmState() {
|
||||
Map<String, String> previous = preserveSystemProperties(
|
||||
"zookeeper.sasl.client",
|
||||
"zookeeper.sasl.clientconfig",
|
||||
"zookeeper.client.secure",
|
||||
"zookeeper.clientCnxnSocket",
|
||||
"zookeeper.ssl.keyStore.location"
|
||||
);
|
||||
try {
|
||||
System.setProperty("zookeeper.sasl.client", "false");
|
||||
System.setProperty("zookeeper.client.secure", "false");
|
||||
|
||||
JsonObject properties = new JsonObject();
|
||||
properties.addProperty("zookeeper.sasl.client", "true");
|
||||
properties.addProperty("zookeeper.sasl.clientconfig", "DbxZooKeeperClient");
|
||||
properties.addProperty("zookeeper.client.secure", "true");
|
||||
properties.addProperty("zookeeper.clientCnxnSocket", "org.apache.zookeeper.ClientCnxnSocketNetty");
|
||||
properties.addProperty("zookeeper.ssl.keyStore.location", "/etc/dbx/zookeeper-keystore.p12");
|
||||
properties.addProperty("security.protocol", "SASL_SSL");
|
||||
JsonObject connection = new JsonObject();
|
||||
connection.add("properties", properties);
|
||||
|
||||
ZKClientConfig config = KafkaAgent.zooKeeperClientConfig(connection);
|
||||
|
||||
assertTrue(config.isSaslClientEnabled());
|
||||
assertEquals("DbxZooKeeperClient", config.getProperty("zookeeper.sasl.clientconfig"));
|
||||
assertEquals("true", config.getProperty("zookeeper.client.secure"));
|
||||
assertEquals(
|
||||
"org.apache.zookeeper.ClientCnxnSocketNetty",
|
||||
config.getProperty("zookeeper.clientCnxnSocket")
|
||||
);
|
||||
assertEquals(
|
||||
"/etc/dbx/zookeeper-keystore.p12",
|
||||
config.getProperty("zookeeper.ssl.keyStore.location")
|
||||
);
|
||||
assertNull(config.getProperty("security.protocol"));
|
||||
assertEquals("false", System.getProperty("zookeeper.sasl.client"));
|
||||
assertEquals("false", System.getProperty("zookeeper.client.secure"));
|
||||
} finally {
|
||||
restoreSystemProperties(previous);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void brokerEndpointsUseListenerSecurityProtocolMapForNamedListenersAndKeepBrokerOrder() {
|
||||
List<JsonObject> registrations = Arrays.asList(
|
||||
broker("{\"listener_security_protocol_map\":{\"INTERNAL\":\"PLAINTEXT\",\"CLIENT\":\"SASL_SSL\"},\"endpoints\":[\"INTERNAL://broker-2:9092\",\"CLIENT://public-2:9093\"]}"),
|
||||
broker("{\"listener_security_protocol_map\":{\"INTERNAL\":\"PLAINTEXT\",\"CLIENT\":\"SASL_SSL\"},\"endpoints\":[\"CLIENT://public-1:9093\",\"INTERNAL://broker-1:9092\"]}")
|
||||
);
|
||||
|
||||
assertEquals("public-2:9093,public-1:9093", KafkaAgent.brokerEndpoints(registrations, "SASL_SSL"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void kafkaClientPropertiesExcludeZooKeeperSecuritySettings() {
|
||||
JsonObject properties = new JsonObject();
|
||||
properties.addProperty("client.id", "dbx");
|
||||
properties.addProperty("zookeeper.sasl.client", "true");
|
||||
properties.addProperty("zookeeper.ssl.trustStore.password", "secret");
|
||||
JsonObject connection = new JsonObject();
|
||||
connection.add("properties", properties);
|
||||
|
||||
Properties kafkaProperties = new Properties();
|
||||
KafkaAgent.applyConnectionProperties(connection, kafkaProperties);
|
||||
|
||||
assertEquals("dbx", kafkaProperties.getProperty("client.id"));
|
||||
assertNull(kafkaProperties.getProperty("zookeeper.sasl.client"));
|
||||
assertNull(kafkaProperties.getProperty("zookeeper.ssl.trustStore.password"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void brokerEndpointsFallBackToLegacyHostAndPort() {
|
||||
assertEquals("legacy-broker:9092", KafkaAgent.brokerEndpoints(
|
||||
Collections.singletonList(broker("{\"host\":\"legacy-broker\",\"port\":9092}")), "PLAINTEXT"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void brokerEndpointsSkipMalformedRegistrationWhenAnotherBrokerIsUsable() {
|
||||
assertEquals("healthy-broker:9092", KafkaAgent.brokerEndpoints(Arrays.asList(
|
||||
broker("{\"host\":\"broken\",\"port\":\"not-a-port\"}"),
|
||||
broker("{\"host\":\"healthy-broker\",\"port\":9092}")
|
||||
), "PLAINTEXT"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void brokerEndpointsRejectRegistrationsWithoutUsableAddresses() {
|
||||
var error = org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class,
|
||||
() -> KafkaAgent.brokerEndpoints(Collections.singletonList(broker("{\"endpoints\":[]}")), "PLAINTEXT"));
|
||||
assertTrue(error.getMessage().contains("usable Kafka broker endpoints"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void peekConsumerPropertiesReuseResolvedConnection() {
|
||||
JsonObject resolved = new JsonObject();
|
||||
resolved.addProperty("bootstrap_servers", "legacy-broker:9092");
|
||||
resolved.addProperty("security_protocol", "PLAINTEXT");
|
||||
|
||||
Properties properties = KafkaAgent.peekConsumerProperties(resolved, 25);
|
||||
|
||||
assertEquals("legacy-broker:9092", properties.getProperty("bootstrap.servers"));
|
||||
assertEquals(25, properties.get("max.poll.records"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void aclDisabledDetectionOnlyAcceptsKnownAuthorizerErrors() {
|
||||
Exception disabled = new RuntimeException(
|
||||
"ACL probe failed",
|
||||
new IllegalStateException("No Authorizer is configured on the broker")
|
||||
);
|
||||
|
||||
assertTrue(KafkaAgent.isAclDisabledError(disabled));
|
||||
assertFalse(KafkaAgent.isAclDisabledError(new RuntimeException("Timed out waiting for broker response")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void legacyTopicConfigAppliesSetAndDeleteWithoutLosingExistingOverrides() {
|
||||
Config current = new Config(Arrays.asList(
|
||||
new ConfigEntry("cleanup.policy", "delete"),
|
||||
new ConfigEntry("retention.ms", "60000"),
|
||||
new ConfigEntry(
|
||||
"segment.bytes",
|
||||
"1073741824",
|
||||
ConfigEntry.ConfigSource.DYNAMIC_BROKER_CONFIG,
|
||||
false,
|
||||
false,
|
||||
Collections.emptyList(),
|
||||
ConfigEntry.ConfigType.LONG,
|
||||
null
|
||||
)
|
||||
));
|
||||
List<AlterConfigOp> ops = Arrays.asList(
|
||||
new AlterConfigOp(new ConfigEntry("retention.ms", "120000"), AlterConfigOp.OpType.SET),
|
||||
new AlterConfigOp(new ConfigEntry("cleanup.policy", null), AlterConfigOp.OpType.DELETE)
|
||||
);
|
||||
|
||||
Map<String, String> merged = KafkaAgent.legacyTopicConfig(current, ops);
|
||||
|
||||
assertEquals(Collections.singletonMap("retention.ms", "120000"), merged);
|
||||
}
|
||||
|
||||
@Test
|
||||
void legacyTopicConfigRejectsAppendAndSubtractOperations() {
|
||||
Config current = new Config(Collections.singletonList(new ConfigEntry("cleanup.policy", "delete")));
|
||||
AlterConfigOp append = new AlterConfigOp(new ConfigEntry("cleanup.policy", "compact"), AlterConfigOp.OpType.APPEND);
|
||||
|
||||
var error = org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class,
|
||||
() -> KafkaAgent.legacyTopicConfig(current, Collections.singletonList(append)));
|
||||
assertTrue(error.getMessage().contains("APPEND"));
|
||||
}
|
||||
@Test
|
||||
void normalizesPeekOffsetToEarliestAvailableOffset() {
|
||||
assertEquals(5L, KafkaAgent.normalizePeekOffset(0, 5, 10));
|
||||
@@ -241,4 +499,24 @@ class KafkaAgentTest {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static JsonObject broker(String json) {
|
||||
return JsonParser.parseString(json).getAsJsonObject();
|
||||
}
|
||||
|
||||
private static Map<String, String> preserveSystemProperties(String... keys) {
|
||||
Map<String, String> previous = new HashMap<>();
|
||||
for (String key : keys) previous.put(key, System.getProperty(key));
|
||||
return previous;
|
||||
}
|
||||
|
||||
private static void restoreSystemProperties(Map<String, String> properties) {
|
||||
for (Map.Entry<String, String> entry : properties.entrySet()) {
|
||||
if (entry.getValue() == null) {
|
||||
System.clearProperty(entry.getKey());
|
||||
} else {
|
||||
System.setProperty(entry.getKey(), entry.getValue());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user