fix(kafka): support ZooKeeper broker discovery

This commit is contained in:
miracle authored and GitHub committed 2026-07-23 00:20:47 +08:00
1 parent 34448cba9d
commit 8adf802444
18 files changed
+977 -77

No files matched your search

@@ -13,12 +13,21 @@ import org.apache.kafka.common.resource.PatternType;
import org.apache.kafka.common.resource.ResourcePattern;
import org.apache.kafka.common.resource.ResourcePatternFilter;
import org.apache.kafka.common.resource.ResourceType;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.client.ZKClientConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.PrintStream;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.*;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@@ -29,9 +38,12 @@ import java.util.stream.Collectors;
*/
public final class KafkaAgent {
private static final PrintStream JSON_RPC_OUT = System.out;
private static final Gson GSON = new GsonBuilder().serializeNulls().create();
private static final int DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
private static final int DEFAULT_SESSION_TIMEOUT_MS = 30_000;
private static final int DEFAULT_ZOOKEEPER_CONNECTION_TIMEOUT_MS = 10_000;
private static final String ZOOKEEPER_PROPERTY_PREFIX = "zookeeper.";
private static final Set<String> KERBEROS_SYSTEM_PROPERTY_KEYS = Set.of(
"java.security.krb5.conf",
"sun.security.krb5.debug",
@@ -47,26 +59,40 @@ public final class KafkaAgent {
private static AdminClient adminClient;
private static KafkaProducer<String, byte[]> producer;
private static JsonObject activeConnection;
private static volatile boolean shutdownRequested;
private KafkaAgent() {}
private static Logger logger() {
// Initialize only after main redirects System.out, so any logging backend
// that defaults to stdout still cannot write into the JSON-RPC channel.
return LoggerHolder.INSTANCE;
}
private static final class LoggerHolder {
private static final Logger INSTANCE = LoggerFactory.getLogger(KafkaAgent.class);
}
// -----------------------------------------------------------------------
// Entry point
// -----------------------------------------------------------------------
public static void main(String[] args) throws Exception {
// Keep the original stdout exclusively for JSON-RPC. Redirect accidental
// System.out writes from dependencies to stderr so they cannot corrupt the protocol.
System.setOut(System.err);
System.setProperty("org.slf4j.simpleLogger.logFile", "System.err");
System.out.println("{\"ready\":true}");
System.out.flush();
JSON_RPC_OUT.println("{\"ready\":true}");
JSON_RPC_OUT.flush();
BufferedReader reader = new BufferedReader(new InputStreamReader(System.in));
while (true) {
String line = reader.readLine();
if (line == null) break;
String response = handleRequest(line);
System.out.println(response);
System.out.flush();
JSON_RPC_OUT.println(response);
JSON_RPC_OUT.flush();
if (shutdownRequested) {
System.exit(0);
}
@@ -92,6 +118,7 @@ public final class KafkaAgent {
Object result = dispatch(method, params);
response.add("result", GSON.toJsonTree(result));
} catch (Exception e) {
logger().warn("Kafka Agent request failed: method={}, id={}", method, id, e);
JsonObject error = new JsonObject();
error.addProperty("code", -1);
error.addProperty("message", normalizeErrorMessage(e));
@@ -144,7 +171,7 @@ public final class KafkaAgent {
}
private static Object connect(JsonObject params) throws Exception {
JsonObject conn = connectionObject(params);
JsonObject conn = resolveBrokerConnection(connectionObject(params));
Map<String, String> previousKerberosSystemProperties = applyKerberosSystemProperties(conn);
AdminClient nextAdmin = null;
KafkaProducer<String, byte[]> nextProducer = null;
@@ -158,6 +185,7 @@ public final class KafkaAgent {
applyKerberosSystemProperties(conn);
adminClient = nextAdmin;
producer = nextProducer;
activeConnection = conn.deepCopy();
return Collections.singletonMap("ok", true);
} catch (Exception e) {
if (nextAdmin != null) {
@@ -172,7 +200,7 @@ public final class KafkaAgent {
}
private static Object testConnection(JsonObject params) throws Exception {
JsonObject conn = connectionObject(params);
JsonObject conn = resolveBrokerConnection(connectionObject(params));
Map<String, String> previousKerberosSystemProperties = applyKerberosSystemProperties(conn);
AdminClient probe = null;
try {
@@ -189,14 +217,11 @@ public final class KafkaAgent {
probe.describeAcls(AclBindingFilter.ANY)
.values().get(timeout, TimeUnit.MILLISECONDS);
} catch (Exception aclEx) {
Throwable cause = aclEx;
while (cause != null) {
if (cause.getClass().getSimpleName().contains("SecurityDisabled")
|| (cause.getMessage() != null && cause.getMessage().contains("No Authorizer"))) {
aclEnabled = false;
break;
}
cause = cause.getCause();
if (isAclDisabledError(aclEx)) {
aclEnabled = false;
logger().debug("Kafka ACL support is disabled by the broker");
} else {
logger().warn("Kafka ACL capability probe failed; leaving the capability enabled", aclEx);
}
}
@@ -219,6 +244,18 @@ public final class KafkaAgent {
}
}
static boolean isAclDisabledError(Throwable error) {
Throwable cause = error;
while (cause != null) {
if (cause.getClass().getSimpleName().contains("SecurityDisabled")
|| (cause.getMessage() != null && cause.getMessage().contains("No Authorizer"))) {
return true;
}
cause = cause.getCause();
}
return false;
}
private static void closeClients() {
if (adminClient != null) {
adminClient.close(Duration.ofSeconds(5));
@@ -228,6 +265,7 @@ public final class KafkaAgent {
producer.close(Duration.ofSeconds(5));
producer = null;
}
activeConnection = null;
restoreKerberosSystemProperties(BASELINE_KERBEROS_SYSTEM_PROPERTIES);
}
@@ -269,6 +307,160 @@ public final class KafkaAgent {
return servers;
}
static JsonObject resolveBrokerConnection(JsonObject conn) throws Exception {
String configured = stringOrEmpty(conn, "bootstrap_servers");
if (configured.isBlank()) configured = stringOrEmpty(conn, "bootstrapServers");
if (!configured.isBlank()) return conn;
String connectString = stringOrEmpty(conn, "zookeeper_connect_string");
if (connectString.isBlank()) connectString = stringOrEmpty(conn, "zookeeperServers");
if (connectString.isBlank()) {
throw new IllegalArgumentException("bootstrap_servers or zookeeper_connect_string is required");
}
JsonObject resolved = conn.deepCopy();
resolved.addProperty("bootstrap_servers", discoverBootstrapServers(connectString, securityProtocol(conn), conn));
return resolved;
}
private static String discoverBootstrapServers(String connectString, String securityProtocol, JsonObject conn)
throws Exception {
int sessionTimeout = intOrDefault(conn, "zookeeper_session_timeout_ms", DEFAULT_SESSION_TIMEOUT_MS);
int connectionTimeout = intOrDefault(
conn,
"zookeeper_connection_timeout_ms",
DEFAULT_ZOOKEEPER_CONNECTION_TIMEOUT_MS
);
CountDownLatch connected = new CountDownLatch(1);
ZooKeeper zooKeeper = new ZooKeeper(connectString, sessionTimeout, event -> {
if (event.getState() == Watcher.Event.KeeperState.SyncConnected) connected.countDown();
}, zooKeeperClientConfig(conn));
try {
if (!connected.await(connectionTimeout, TimeUnit.MILLISECONDS)) {
throw new IllegalStateException("Timed out connecting to ZooKeeper for Kafka broker discovery");
}
List<String> brokerIds;
try {
brokerIds = new ArrayList<>(zooKeeper.getChildren("/brokers/ids", false));
} catch (KeeperException.NoNodeException e) {
throw new IllegalStateException("ZooKeeper path /brokers/ids does not exist", e);
}
brokerIds.sort(KafkaAgent::compareBrokerIds);
List<JsonObject> registrations = new ArrayList<>();
for (String brokerId : brokerIds) {
try {
byte[] data = zooKeeper.getData("/brokers/ids/" + brokerId, false, null);
registrations.add(JsonParser.parseString(new String(data, StandardCharsets.UTF_8)).getAsJsonObject());
} catch (KeeperException.NoNodeException e) {
// Expected race: a broker may refresh its ephemeral node between list and read.
logger().debug("Kafka broker {} disappeared during ZooKeeper discovery", brokerId);
} catch (RuntimeException e) {
logger().warn("Skipping malformed ZooKeeper registration for Kafka broker {}", brokerId, e);
}
}
return brokerEndpoints(registrations, securityProtocol);
} finally {
zooKeeper.close();
}
}
static ZKClientConfig zooKeeperClientConfig(JsonObject conn) {
ZKClientConfig clientConfig = new ZKClientConfig();
JsonObject properties = connectionProperties(conn);
if (properties == null) return clientConfig;
for (Map.Entry<String, JsonElement> entry : properties.entrySet()) {
if (entry.getKey().startsWith(ZOOKEEPER_PROPERTY_PREFIX)
&& entry.getValue().isJsonPrimitive()) {
clientConfig.setProperty(entry.getKey(), entry.getValue().getAsString());
}
}
return clientConfig;
}
private static int compareBrokerIds(String left, String right) {
try {
return Integer.compare(Integer.parseInt(left), Integer.parseInt(right));
} catch (NumberFormatException e) {
// Third-party registries may use non-numeric IDs; lexical ordering remains deterministic.
logger().debug("Sorting non-numeric Kafka broker IDs lexically: left={}, right={}", left, right);
return left.compareTo(right);
}
}
static String brokerEndpoints(List<JsonObject> registrations, String securityProtocol) {
String targetProtocol = securityProtocol == null || securityProtocol.isBlank()
? "PLAINTEXT"
: securityProtocol.toUpperCase(Locale.ROOT);
Set<String> addresses = new LinkedHashSet<>();
for (JsonObject registration : registrations) {
int addressCount = addresses.size();
JsonObject protocolMap = registration.has("listener_security_protocol_map")
&& registration.get("listener_security_protocol_map").isJsonObject()
? registration.getAsJsonObject("listener_security_protocol_map")
: new JsonObject();
JsonArray endpoints = registration.has("endpoints") && registration.get("endpoints").isJsonArray()
? registration.getAsJsonArray("endpoints")
: new JsonArray();
for (JsonElement element : endpoints) {
if (!element.isJsonPrimitive() || !element.getAsJsonPrimitive().isString()) continue;
String endpoint = element.getAsString();
int separator = endpoint.indexOf("://");
if (separator <= 0) continue;
String listener = endpoint.substring(0, separator).toUpperCase(Locale.ROOT);
JsonElement mapped = protocolMap.get(listener);
String mappedProtocol = mapped != null && mapped.isJsonPrimitive()
? mapped.getAsString().toUpperCase(Locale.ROOT)
: listener;
if (!targetProtocol.equals(mappedProtocol)) continue;
String address = endpointAddress(endpoint);
if (address != null) addresses.add(address);
}
if (addresses.size() == addressCount && endpoints.size() == 0 && registration.has("host") && registration.has("port")) {
try {
String host = registration.get("host").getAsString().trim();
int port = registration.get("port").getAsInt();
if (!host.isEmpty() && port > 0 && port <= 65535) addresses.add(formatHostPort(host, port));
} catch (RuntimeException e) {
logger().warn("Skipping malformed legacy Kafka broker registration", e);
}
}
}
if (addresses.isEmpty()) {
throw new IllegalArgumentException("ZooKeeper did not return any usable Kafka broker endpoints");
}
return String.join(",", addresses);
}
private static String endpointAddress(String endpoint) {
try {
URI uri = URI.create(endpoint);
String host = uri.getHost();
int port = uri.getPort();
if (host == null || host.isBlank() || port <= 0 || port > 65535) return null;
return formatHostPort(host, port);
} catch (IllegalArgumentException e) {
logger().debug("Skipping malformed Kafka broker endpoint", e);
return null;
}
}
private static String formatHostPort(String host, int port) {
return host.contains(":") && !host.startsWith("[") ? "[" + host + "]:" + port : host + ":" + port;
}
private static String securityProtocol(JsonObject conn) {
String protocol = stringOrEmpty(conn, "security_protocol");
if (protocol.isBlank()) protocol = stringOrEmpty(conn, "securityProtocol");
return protocol.isBlank() ? "PLAINTEXT" : protocol;
}
static void applySecurityProperties(JsonObject conn, Properties props) {
String securityProtocol = stringOrEmpty(conn, "security_protocol");
if (securityProtocol.isBlank()) {
@@ -353,6 +545,7 @@ public final class KafkaAgent {
for (Map.Entry<String, JsonElement> entry : properties.entrySet()) {
if (entry.getValue().isJsonPrimitive()) {
String key = entry.getKey();
if (key.startsWith(ZOOKEEPER_PROPERTY_PREFIX)) continue;
String value = entry.getValue().getAsString();
props.put(key, value);
}
@@ -578,11 +771,47 @@ public final class KafkaAgent {
}
ConfigResource resource = new ConfigResource(ConfigResource.Type.TOPIC, name);
admin.incrementalAlterConfigs(Collections.singletonMap(resource, ops))
.all().get(timeout, TimeUnit.MILLISECONDS);
try {
admin.incrementalAlterConfigs(Collections.singletonMap(resource, ops))
.all().get(timeout, TimeUnit.MILLISECONDS);
} catch (Exception e) {
if (!isUnsupportedVersionError(e)) throw e;
logger().info("Kafka broker does not support incrementalAlterConfigs; using legacy alterConfigs for topic {}", name);
Config current = admin.describeConfigs(Collections.singletonList(resource))
.all().get(timeout, TimeUnit.MILLISECONDS).get(resource);
Map<String, String> values = legacyTopicConfig(current, ops);
Config replacement = new Config(values.entrySet().stream()
.map(entry -> new ConfigEntry(entry.getKey(), entry.getValue()))
.collect(Collectors.toList()));
admin.alterConfigs(Collections.singletonMap(resource, replacement))
.all().get(timeout, TimeUnit.MILLISECONDS);
}
return Collections.singletonMap("ok", true);
}
static Map<String, String> legacyTopicConfig(Config current, List<AlterConfigOp> ops) {
Map<String, String> values = new LinkedHashMap<>();
for (ConfigEntry entry : current.entries()) {
boolean topicOverride = entry.source() == ConfigEntry.ConfigSource.DYNAMIC_TOPIC_CONFIG
|| entry.source() == ConfigEntry.ConfigSource.UNKNOWN;
if (topicOverride && !entry.isReadOnly() && !entry.isSensitive() && entry.value() != null) {
values.put(entry.name(), entry.value());
}
}
for (AlterConfigOp op : ops) {
String key = op.configEntry().name();
switch (op.opType()) {
case SET -> values.put(key, op.configEntry().value());
case DELETE -> values.remove(key);
case APPEND, SUBTRACT -> throw new IllegalArgumentException(
"Kafka broker does not support " + op.opType() + " config operations through the legacy alterConfigs API"
);
}
}
return values;
}
// -----------------------------------------------------------------------
// Consumer groups
// -----------------------------------------------------------------------
@@ -698,6 +927,7 @@ public final class KafkaAgent {
return Collections.singletonMap("producers", new ArrayList<>(byProducer.values()));
} catch (Exception e) {
if (isUnsupportedVersionError(e)) {
logger().info("Kafka broker does not support describeProducers; returning an empty producer list");
return Collections.singletonMap("producers", Collections.emptyList());
}
throw e;
@@ -797,26 +1027,11 @@ public final class KafkaAgent {
Long offset = longOrNull(params, "offset");
int count = Math.max(1, intOrDefault(params, "count", 10));
// Build a temporary consumer for peeking (no commit)
Properties props = new Properties();
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,
adminClient != null ? adminClient.describeCluster().clusterId()
.get(5, TimeUnit.SECONDS) : "localhost:9092");
// Reuse the admin's bootstrap servers
JsonObject conn = params.has("connection") && params.get("connection").isJsonObject()
? params.getAsJsonObject("connection") : null;
if (conn != null) {
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers(conn));
applyConnectionProperties(conn, props);
JsonObject conn = activeConnection;
if (conn == null) {
throw new IllegalStateException("Kafka Agent is not connected");
}
props.put(ConsumerConfig.GROUP_ID_CONFIG, "dbx-peek-" + UUID.randomUUID());
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,
"org.apache.kafka.common.serialization.StringDeserializer");
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,
"org.apache.kafka.common.serialization.ByteArrayDeserializer");
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "none");
props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, count);
Properties props = peekConsumerProperties(conn, count);
try (KafkaConsumer<String, byte[]> consumer = new KafkaConsumer<>(props)) {
List<TopicPartition> candidatePartitions = resolvePeekPartitions(consumer, topic, partition);
@@ -872,6 +1087,21 @@ public final class KafkaAgent {
}
}
static Properties peekConsumerProperties(JsonObject conn, int count) {
Properties props = new Properties();
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers(conn));
applyConnectionProperties(conn, props);
props.put(ConsumerConfig.GROUP_ID_CONFIG, "dbx-peek-" + UUID.randomUUID());
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,
"org.apache.kafka.common.serialization.StringDeserializer");
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,
"org.apache.kafka.common.serialization.ByteArrayDeserializer");
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "none");
props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, count);
return props;
}
/**
* Poll until {@code count} messages are collected, every assigned partition has reached its
* end offset, or {@code deadlineNs} expires. Empty polls retry until caught-up or deadline —
@@ -1335,15 +1565,10 @@ public final class KafkaAgent {
}
private static String tryDecodeUtf8(byte[] bytes) {
try {
String text = new String(bytes, StandardCharsets.UTF_8);
// Verify round-trip
byte[] reEncoded = text.getBytes(StandardCharsets.UTF_8);
if (Arrays.equals(bytes, reEncoded)) {
return text;
}
} catch (Exception ignored) {}
return null;
String text = new String(bytes, StandardCharsets.UTF_8);
// Replacement characters change the bytes on round-trip, identifying invalid UTF-8 without exceptions.
byte[] reEncoded = text.getBytes(StandardCharsets.UTF_8);
return Arrays.equals(bytes, reEncoded) ? text : null;
}
static Long normalizePeekOffset(long requestedOffset, long beginningOffset, long endOffset) {
@@ -1,24 +1,282 @@
package com.dbx.agent.kafka;
import com.google.gson.JsonObject;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import com.google.gson.JsonParser;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.admin.AlterConfigOp;
import org.apache.kafka.clients.admin.Config;
import org.apache.kafka.clients.admin.ConfigEntry;
import org.apache.kafka.common.TopicPartition;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.client.ZKClientConfig;
import org.apache.zookeeper.server.NIOServerCnxnFactory;
import org.apache.zookeeper.server.ZooKeeperServer;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
class KafkaAgentTest {
@TempDir
Path tempDir;
@Test
void resolvesBootstrapServersFromKafka11ZooKeeperRegistrationWithChroot() throws Exception {
Path snapshots = Files.createDirectory(tempDir.resolve("snapshots"));
Path logs = Files.createDirectory(tempDir.resolve("logs"));
ZooKeeperServer server = new ZooKeeperServer(snapshots.toFile(), logs.toFile(), 2_000);
NIOServerCnxnFactory factory = new NIOServerCnxnFactory();
factory.configure(new InetSocketAddress("127.0.0.1", 0), 10);
factory.startup(server);
ZooKeeper client = null;
String previousSaslSetting = System.getProperty("zookeeper.sasl.client");
try {
CountDownLatch connected = new CountDownLatch(1);
System.setProperty("zookeeper.sasl.client", "false");
client = new ZooKeeper("127.0.0.1:" + factory.getLocalPort(), 5_000, event -> {
if (event.getState() == Watcher.Event.KeeperState.SyncConnected) connected.countDown();
});
assertTrue(connected.await(5, TimeUnit.SECONDS));
client.create("/kafka", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
client.create("/kafka/brokers", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
client.create("/kafka/brokers/ids", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
client.create(
"/kafka/brokers/ids/0",
"{\"listener_security_protocol_map\":{\"PLAINTEXT\":\"PLAINTEXT\"},\"endpoints\":[\"PLAINTEXT://legacy-broker:9092\"]}".getBytes(StandardCharsets.UTF_8),
ZooDefs.Ids.OPEN_ACL_UNSAFE,
CreateMode.EPHEMERAL
);
JsonObject connection = new JsonObject();
connection.addProperty("zookeeper_connect_string", "127.0.0.1:" + factory.getLocalPort() + "/kafka");
connection.addProperty("security_protocol", "PLAINTEXT");
connection.addProperty("zookeeper_connection_timeout_ms", 5_000);
JsonObject resolved = KafkaAgent.resolveBrokerConnection(connection);
assertEquals("legacy-broker:9092", resolved.get("bootstrap_servers").getAsString());
} finally {
if (client != null) client.close();
factory.shutdown();
server.shutdown();
server.getTxnLogFactory().close();
if (previousSaslSetting == null) {
System.clearProperty("zookeeper.sasl.client");
} else {
System.setProperty("zookeeper.sasl.client", previousSaslSetting);
}
}
}
@Test
void zooKeeperClientConfigPreservesSaslAndTlsSystemDefaults() {
Map<String, String> previous = preserveSystemProperties(
"zookeeper.sasl.client",
"zookeeper.sasl.clientconfig",
"zookeeper.client.secure",
"zookeeper.clientCnxnSocket",
"zookeeper.ssl.trustStore.location",
"java.security.auth.login.config"
);
try {
System.setProperty("zookeeper.sasl.client", "true");
System.setProperty("zookeeper.sasl.clientconfig", "DbxZooKeeperClient");
System.setProperty("zookeeper.client.secure", "true");
System.setProperty("zookeeper.clientCnxnSocket", "org.apache.zookeeper.ClientCnxnSocketNetty");
System.setProperty("zookeeper.ssl.trustStore.location", "/etc/dbx/zookeeper-truststore.p12");
System.setProperty("java.security.auth.login.config", "/etc/dbx/zookeeper-jaas.conf");
ZKClientConfig config = KafkaAgent.zooKeeperClientConfig(new JsonObject());
assertTrue(config.isSaslClientEnabled());
assertEquals("DbxZooKeeperClient", config.getProperty("zookeeper.sasl.clientconfig"));
assertEquals("true", config.getProperty("zookeeper.client.secure"));
assertEquals(
"org.apache.zookeeper.ClientCnxnSocketNetty",
config.getProperty("zookeeper.clientCnxnSocket")
);
assertEquals(
"/etc/dbx/zookeeper-truststore.p12",
config.getProperty("zookeeper.ssl.trustStore.location")
);
assertEquals("/etc/dbx/zookeeper-jaas.conf", config.getJaasConfKey());
} finally {
restoreSystemProperties(previous);
}
}
@Test
void zooKeeperClientConfigAppliesPerConnectionSaslAndTlsOverridesWithoutChangingJvmState() {
Map<String, String> previous = preserveSystemProperties(
"zookeeper.sasl.client",
"zookeeper.sasl.clientconfig",
"zookeeper.client.secure",
"zookeeper.clientCnxnSocket",
"zookeeper.ssl.keyStore.location"
);
try {
System.setProperty("zookeeper.sasl.client", "false");
System.setProperty("zookeeper.client.secure", "false");
JsonObject properties = new JsonObject();
properties.addProperty("zookeeper.sasl.client", "true");
properties.addProperty("zookeeper.sasl.clientconfig", "DbxZooKeeperClient");
properties.addProperty("zookeeper.client.secure", "true");
properties.addProperty("zookeeper.clientCnxnSocket", "org.apache.zookeeper.ClientCnxnSocketNetty");
properties.addProperty("zookeeper.ssl.keyStore.location", "/etc/dbx/zookeeper-keystore.p12");
properties.addProperty("security.protocol", "SASL_SSL");
JsonObject connection = new JsonObject();
connection.add("properties", properties);
ZKClientConfig config = KafkaAgent.zooKeeperClientConfig(connection);
assertTrue(config.isSaslClientEnabled());
assertEquals("DbxZooKeeperClient", config.getProperty("zookeeper.sasl.clientconfig"));
assertEquals("true", config.getProperty("zookeeper.client.secure"));
assertEquals(
"org.apache.zookeeper.ClientCnxnSocketNetty",
config.getProperty("zookeeper.clientCnxnSocket")
);
assertEquals(
"/etc/dbx/zookeeper-keystore.p12",
config.getProperty("zookeeper.ssl.keyStore.location")
);
assertNull(config.getProperty("security.protocol"));
assertEquals("false", System.getProperty("zookeeper.sasl.client"));
assertEquals("false", System.getProperty("zookeeper.client.secure"));
} finally {
restoreSystemProperties(previous);
}
}
@Test
void brokerEndpointsUseListenerSecurityProtocolMapForNamedListenersAndKeepBrokerOrder() {
List<JsonObject> registrations = Arrays.asList(
broker("{\"listener_security_protocol_map\":{\"INTERNAL\":\"PLAINTEXT\",\"CLIENT\":\"SASL_SSL\"},\"endpoints\":[\"INTERNAL://broker-2:9092\",\"CLIENT://public-2:9093\"]}"),
broker("{\"listener_security_protocol_map\":{\"INTERNAL\":\"PLAINTEXT\",\"CLIENT\":\"SASL_SSL\"},\"endpoints\":[\"CLIENT://public-1:9093\",\"INTERNAL://broker-1:9092\"]}")
);
assertEquals("public-2:9093,public-1:9093", KafkaAgent.brokerEndpoints(registrations, "SASL_SSL"));
}
@Test
void kafkaClientPropertiesExcludeZooKeeperSecuritySettings() {
JsonObject properties = new JsonObject();
properties.addProperty("client.id", "dbx");
properties.addProperty("zookeeper.sasl.client", "true");
properties.addProperty("zookeeper.ssl.trustStore.password", "secret");
JsonObject connection = new JsonObject();
connection.add("properties", properties);
Properties kafkaProperties = new Properties();
KafkaAgent.applyConnectionProperties(connection, kafkaProperties);
assertEquals("dbx", kafkaProperties.getProperty("client.id"));
assertNull(kafkaProperties.getProperty("zookeeper.sasl.client"));
assertNull(kafkaProperties.getProperty("zookeeper.ssl.trustStore.password"));
}
@Test
void brokerEndpointsFallBackToLegacyHostAndPort() {
assertEquals("legacy-broker:9092", KafkaAgent.brokerEndpoints(
Collections.singletonList(broker("{\"host\":\"legacy-broker\",\"port\":9092}")), "PLAINTEXT"));
}
@Test
void brokerEndpointsSkipMalformedRegistrationWhenAnotherBrokerIsUsable() {
assertEquals("healthy-broker:9092", KafkaAgent.brokerEndpoints(Arrays.asList(
broker("{\"host\":\"broken\",\"port\":\"not-a-port\"}"),
broker("{\"host\":\"healthy-broker\",\"port\":9092}")
), "PLAINTEXT"));
}
@Test
void brokerEndpointsRejectRegistrationsWithoutUsableAddresses() {
var error = org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class,
() -> KafkaAgent.brokerEndpoints(Collections.singletonList(broker("{\"endpoints\":[]}")), "PLAINTEXT"));
assertTrue(error.getMessage().contains("usable Kafka broker endpoints"));
}
@Test
void peekConsumerPropertiesReuseResolvedConnection() {
JsonObject resolved = new JsonObject();
resolved.addProperty("bootstrap_servers", "legacy-broker:9092");
resolved.addProperty("security_protocol", "PLAINTEXT");
Properties properties = KafkaAgent.peekConsumerProperties(resolved, 25);
assertEquals("legacy-broker:9092", properties.getProperty("bootstrap.servers"));
assertEquals(25, properties.get("max.poll.records"));
}
@Test
void aclDisabledDetectionOnlyAcceptsKnownAuthorizerErrors() {
Exception disabled = new RuntimeException(
"ACL probe failed",
new IllegalStateException("No Authorizer is configured on the broker")
);
assertTrue(KafkaAgent.isAclDisabledError(disabled));
assertFalse(KafkaAgent.isAclDisabledError(new RuntimeException("Timed out waiting for broker response")));
}
@Test
void legacyTopicConfigAppliesSetAndDeleteWithoutLosingExistingOverrides() {
Config current = new Config(Arrays.asList(
new ConfigEntry("cleanup.policy", "delete"),
new ConfigEntry("retention.ms", "60000"),
new ConfigEntry(
"segment.bytes",
"1073741824",
ConfigEntry.ConfigSource.DYNAMIC_BROKER_CONFIG,
false,
false,
Collections.emptyList(),
ConfigEntry.ConfigType.LONG,
null
)
));
List<AlterConfigOp> ops = Arrays.asList(
new AlterConfigOp(new ConfigEntry("retention.ms", "120000"), AlterConfigOp.OpType.SET),
new AlterConfigOp(new ConfigEntry("cleanup.policy", null), AlterConfigOp.OpType.DELETE)
);
Map<String, String> merged = KafkaAgent.legacyTopicConfig(current, ops);
assertEquals(Collections.singletonMap("retention.ms", "120000"), merged);
}
@Test
void legacyTopicConfigRejectsAppendAndSubtractOperations() {
Config current = new Config(Collections.singletonList(new ConfigEntry("cleanup.policy", "delete")));
AlterConfigOp append = new AlterConfigOp(new ConfigEntry("cleanup.policy", "compact"), AlterConfigOp.OpType.APPEND);
var error = org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class,
() -> KafkaAgent.legacyTopicConfig(current, Collections.singletonList(append)));
assertTrue(error.getMessage().contains("APPEND"));
}
@Test
void normalizesPeekOffsetToEarliestAvailableOffset() {
assertEquals(5L, KafkaAgent.normalizePeekOffset(0, 5, 10));
@@ -241,4 +499,24 @@ class KafkaAgentTest {
}
}
}
private static JsonObject broker(String json) {
return JsonParser.parseString(json).getAsJsonObject();
}
private static Map<String, String> preserveSystemProperties(String... keys) {
Map<String, String> previous = new HashMap<>();
for (String key : keys) previous.put(key, System.getProperty(key));
return previous;
}
private static void restoreSystemProperties(Map<String, String> properties) {
for (Map.Entry<String, String> entry : properties.entrySet()) {
if (entry.getValue() == null) {
System.clearProperty(entry.getKey());
} else {
System.setProperty(entry.getKey(), entry.getValue());
}
}
}
}