fix(kafka): support ZooKeeper broker discovery
This commit is contained in:
1 parent
34448cba9d
commit
8adf802444
18 files changed
+977
-77
No files matched your search
@@ -1,6 +1,15 @@
|
|||||||
dependencies {
|
dependencies {
|
||||||
implementation 'com.google.code.gson:gson:2.12.1'
|
implementation 'com.google.code.gson:gson:2.12.1'
|
||||||
implementation 'org.apache.kafka:kafka-clients:3.9.0'
|
implementation 'org.apache.kafka:kafka-clients:3.9.0'
|
||||||
|
implementation('org.apache.zookeeper:zookeeper:3.8.4') {
|
||||||
|
exclude group: 'log4j', module: 'log4j'
|
||||||
|
exclude group: 'org.slf4j', module: 'slf4j-log4j12'
|
||||||
|
// The agent stdout is a JSON-RPC transport; Logback's default appender
|
||||||
|
// writes Kafka/ZooKeeper logs to stdout and corrupts that protocol.
|
||||||
|
exclude group: 'ch.qos.logback', module: 'logback-classic'
|
||||||
|
exclude group: 'ch.qos.logback', module: 'logback-core'
|
||||||
|
}
|
||||||
|
implementation 'io.dropwizard.metrics:metrics-core:4.1.12.1'
|
||||||
runtimeOnly 'org.slf4j:slf4j-simple:1.7.36'
|
runtimeOnly 'org.slf4j:slf4j-simple:1.7.36'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -13,12 +13,21 @@ import org.apache.kafka.common.resource.PatternType;
|
|||||||
import org.apache.kafka.common.resource.ResourcePattern;
|
import org.apache.kafka.common.resource.ResourcePattern;
|
||||||
import org.apache.kafka.common.resource.ResourcePatternFilter;
|
import org.apache.kafka.common.resource.ResourcePatternFilter;
|
||||||
import org.apache.kafka.common.resource.ResourceType;
|
import org.apache.kafka.common.resource.ResourceType;
|
||||||
|
import org.apache.zookeeper.KeeperException;
|
||||||
|
import org.apache.zookeeper.Watcher;
|
||||||
|
import org.apache.zookeeper.ZooKeeper;
|
||||||
|
import org.apache.zookeeper.client.ZKClientConfig;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import java.io.BufferedReader;
|
import java.io.BufferedReader;
|
||||||
import java.io.InputStreamReader;
|
import java.io.InputStreamReader;
|
||||||
|
import java.io.PrintStream;
|
||||||
|
import java.net.URI;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.time.Duration;
|
import java.time.Duration;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
|
import java.util.concurrent.CountDownLatch;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
@@ -29,9 +38,12 @@ import java.util.stream.Collectors;
|
|||||||
*/
|
*/
|
||||||
public final class KafkaAgent {
|
public final class KafkaAgent {
|
||||||
|
|
||||||
|
private static final PrintStream JSON_RPC_OUT = System.out;
|
||||||
private static final Gson GSON = new GsonBuilder().serializeNulls().create();
|
private static final Gson GSON = new GsonBuilder().serializeNulls().create();
|
||||||
private static final int DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
|
private static final int DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
|
||||||
private static final int DEFAULT_SESSION_TIMEOUT_MS = 30_000;
|
private static final int DEFAULT_SESSION_TIMEOUT_MS = 30_000;
|
||||||
|
private static final int DEFAULT_ZOOKEEPER_CONNECTION_TIMEOUT_MS = 10_000;
|
||||||
|
private static final String ZOOKEEPER_PROPERTY_PREFIX = "zookeeper.";
|
||||||
private static final Set<String> KERBEROS_SYSTEM_PROPERTY_KEYS = Set.of(
|
private static final Set<String> KERBEROS_SYSTEM_PROPERTY_KEYS = Set.of(
|
||||||
"java.security.krb5.conf",
|
"java.security.krb5.conf",
|
||||||
"sun.security.krb5.debug",
|
"sun.security.krb5.debug",
|
||||||
@@ -47,26 +59,40 @@ public final class KafkaAgent {
|
|||||||
|
|
||||||
private static AdminClient adminClient;
|
private static AdminClient adminClient;
|
||||||
private static KafkaProducer<String, byte[]> producer;
|
private static KafkaProducer<String, byte[]> producer;
|
||||||
|
private static JsonObject activeConnection;
|
||||||
private static volatile boolean shutdownRequested;
|
private static volatile boolean shutdownRequested;
|
||||||
|
|
||||||
private KafkaAgent() {}
|
private KafkaAgent() {}
|
||||||
|
|
||||||
|
private static Logger logger() {
|
||||||
|
// Initialize only after main redirects System.out, so any logging backend
|
||||||
|
// that defaults to stdout still cannot write into the JSON-RPC channel.
|
||||||
|
return LoggerHolder.INSTANCE;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static final class LoggerHolder {
|
||||||
|
private static final Logger INSTANCE = LoggerFactory.getLogger(KafkaAgent.class);
|
||||||
|
}
|
||||||
|
|
||||||
// -----------------------------------------------------------------------
|
// -----------------------------------------------------------------------
|
||||||
// Entry point
|
// Entry point
|
||||||
// -----------------------------------------------------------------------
|
// -----------------------------------------------------------------------
|
||||||
|
|
||||||
public static void main(String[] args) throws Exception {
|
public static void main(String[] args) throws Exception {
|
||||||
|
// Keep the original stdout exclusively for JSON-RPC. Redirect accidental
|
||||||
|
// System.out writes from dependencies to stderr so they cannot corrupt the protocol.
|
||||||
|
System.setOut(System.err);
|
||||||
System.setProperty("org.slf4j.simpleLogger.logFile", "System.err");
|
System.setProperty("org.slf4j.simpleLogger.logFile", "System.err");
|
||||||
System.out.println("{\"ready\":true}");
|
JSON_RPC_OUT.println("{\"ready\":true}");
|
||||||
System.out.flush();
|
JSON_RPC_OUT.flush();
|
||||||
|
|
||||||
BufferedReader reader = new BufferedReader(new InputStreamReader(System.in));
|
BufferedReader reader = new BufferedReader(new InputStreamReader(System.in));
|
||||||
while (true) {
|
while (true) {
|
||||||
String line = reader.readLine();
|
String line = reader.readLine();
|
||||||
if (line == null) break;
|
if (line == null) break;
|
||||||
String response = handleRequest(line);
|
String response = handleRequest(line);
|
||||||
System.out.println(response);
|
JSON_RPC_OUT.println(response);
|
||||||
System.out.flush();
|
JSON_RPC_OUT.flush();
|
||||||
if (shutdownRequested) {
|
if (shutdownRequested) {
|
||||||
System.exit(0);
|
System.exit(0);
|
||||||
}
|
}
|
||||||
@@ -92,6 +118,7 @@ public final class KafkaAgent {
|
|||||||
Object result = dispatch(method, params);
|
Object result = dispatch(method, params);
|
||||||
response.add("result", GSON.toJsonTree(result));
|
response.add("result", GSON.toJsonTree(result));
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
|
logger().warn("Kafka Agent request failed: method={}, id={}", method, id, e);
|
||||||
JsonObject error = new JsonObject();
|
JsonObject error = new JsonObject();
|
||||||
error.addProperty("code", -1);
|
error.addProperty("code", -1);
|
||||||
error.addProperty("message", normalizeErrorMessage(e));
|
error.addProperty("message", normalizeErrorMessage(e));
|
||||||
@@ -144,7 +171,7 @@ public final class KafkaAgent {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private static Object connect(JsonObject params) throws Exception {
|
private static Object connect(JsonObject params) throws Exception {
|
||||||
JsonObject conn = connectionObject(params);
|
JsonObject conn = resolveBrokerConnection(connectionObject(params));
|
||||||
Map<String, String> previousKerberosSystemProperties = applyKerberosSystemProperties(conn);
|
Map<String, String> previousKerberosSystemProperties = applyKerberosSystemProperties(conn);
|
||||||
AdminClient nextAdmin = null;
|
AdminClient nextAdmin = null;
|
||||||
KafkaProducer<String, byte[]> nextProducer = null;
|
KafkaProducer<String, byte[]> nextProducer = null;
|
||||||
@@ -158,6 +185,7 @@ public final class KafkaAgent {
|
|||||||
applyKerberosSystemProperties(conn);
|
applyKerberosSystemProperties(conn);
|
||||||
adminClient = nextAdmin;
|
adminClient = nextAdmin;
|
||||||
producer = nextProducer;
|
producer = nextProducer;
|
||||||
|
activeConnection = conn.deepCopy();
|
||||||
return Collections.singletonMap("ok", true);
|
return Collections.singletonMap("ok", true);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
if (nextAdmin != null) {
|
if (nextAdmin != null) {
|
||||||
@@ -172,7 +200,7 @@ public final class KafkaAgent {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private static Object testConnection(JsonObject params) throws Exception {
|
private static Object testConnection(JsonObject params) throws Exception {
|
||||||
JsonObject conn = connectionObject(params);
|
JsonObject conn = resolveBrokerConnection(connectionObject(params));
|
||||||
Map<String, String> previousKerberosSystemProperties = applyKerberosSystemProperties(conn);
|
Map<String, String> previousKerberosSystemProperties = applyKerberosSystemProperties(conn);
|
||||||
AdminClient probe = null;
|
AdminClient probe = null;
|
||||||
try {
|
try {
|
||||||
@@ -189,14 +217,11 @@ public final class KafkaAgent {
|
|||||||
probe.describeAcls(AclBindingFilter.ANY)
|
probe.describeAcls(AclBindingFilter.ANY)
|
||||||
.values().get(timeout, TimeUnit.MILLISECONDS);
|
.values().get(timeout, TimeUnit.MILLISECONDS);
|
||||||
} catch (Exception aclEx) {
|
} catch (Exception aclEx) {
|
||||||
Throwable cause = aclEx;
|
if (isAclDisabledError(aclEx)) {
|
||||||
while (cause != null) {
|
aclEnabled = false;
|
||||||
if (cause.getClass().getSimpleName().contains("SecurityDisabled")
|
logger().debug("Kafka ACL support is disabled by the broker");
|
||||||
|| (cause.getMessage() != null && cause.getMessage().contains("No Authorizer"))) {
|
} else {
|
||||||
aclEnabled = false;
|
logger().warn("Kafka ACL capability probe failed; leaving the capability enabled", aclEx);
|
||||||
break;
|
|
||||||
}
|
|
||||||
cause = cause.getCause();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -219,6 +244,18 @@ public final class KafkaAgent {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static boolean isAclDisabledError(Throwable error) {
|
||||||
|
Throwable cause = error;
|
||||||
|
while (cause != null) {
|
||||||
|
if (cause.getClass().getSimpleName().contains("SecurityDisabled")
|
||||||
|
|| (cause.getMessage() != null && cause.getMessage().contains("No Authorizer"))) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
cause = cause.getCause();
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
private static void closeClients() {
|
private static void closeClients() {
|
||||||
if (adminClient != null) {
|
if (adminClient != null) {
|
||||||
adminClient.close(Duration.ofSeconds(5));
|
adminClient.close(Duration.ofSeconds(5));
|
||||||
@@ -228,6 +265,7 @@ public final class KafkaAgent {
|
|||||||
producer.close(Duration.ofSeconds(5));
|
producer.close(Duration.ofSeconds(5));
|
||||||
producer = null;
|
producer = null;
|
||||||
}
|
}
|
||||||
|
activeConnection = null;
|
||||||
restoreKerberosSystemProperties(BASELINE_KERBEROS_SYSTEM_PROPERTIES);
|
restoreKerberosSystemProperties(BASELINE_KERBEROS_SYSTEM_PROPERTIES);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -269,6 +307,160 @@ public final class KafkaAgent {
|
|||||||
return servers;
|
return servers;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static JsonObject resolveBrokerConnection(JsonObject conn) throws Exception {
|
||||||
|
String configured = stringOrEmpty(conn, "bootstrap_servers");
|
||||||
|
if (configured.isBlank()) configured = stringOrEmpty(conn, "bootstrapServers");
|
||||||
|
if (!configured.isBlank()) return conn;
|
||||||
|
|
||||||
|
String connectString = stringOrEmpty(conn, "zookeeper_connect_string");
|
||||||
|
if (connectString.isBlank()) connectString = stringOrEmpty(conn, "zookeeperServers");
|
||||||
|
if (connectString.isBlank()) {
|
||||||
|
throw new IllegalArgumentException("bootstrap_servers or zookeeper_connect_string is required");
|
||||||
|
}
|
||||||
|
|
||||||
|
JsonObject resolved = conn.deepCopy();
|
||||||
|
resolved.addProperty("bootstrap_servers", discoverBootstrapServers(connectString, securityProtocol(conn), conn));
|
||||||
|
return resolved;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String discoverBootstrapServers(String connectString, String securityProtocol, JsonObject conn)
|
||||||
|
throws Exception {
|
||||||
|
int sessionTimeout = intOrDefault(conn, "zookeeper_session_timeout_ms", DEFAULT_SESSION_TIMEOUT_MS);
|
||||||
|
int connectionTimeout = intOrDefault(
|
||||||
|
conn,
|
||||||
|
"zookeeper_connection_timeout_ms",
|
||||||
|
DEFAULT_ZOOKEEPER_CONNECTION_TIMEOUT_MS
|
||||||
|
);
|
||||||
|
CountDownLatch connected = new CountDownLatch(1);
|
||||||
|
ZooKeeper zooKeeper = new ZooKeeper(connectString, sessionTimeout, event -> {
|
||||||
|
if (event.getState() == Watcher.Event.KeeperState.SyncConnected) connected.countDown();
|
||||||
|
}, zooKeeperClientConfig(conn));
|
||||||
|
try {
|
||||||
|
if (!connected.await(connectionTimeout, TimeUnit.MILLISECONDS)) {
|
||||||
|
throw new IllegalStateException("Timed out connecting to ZooKeeper for Kafka broker discovery");
|
||||||
|
}
|
||||||
|
|
||||||
|
List<String> brokerIds;
|
||||||
|
try {
|
||||||
|
brokerIds = new ArrayList<>(zooKeeper.getChildren("/brokers/ids", false));
|
||||||
|
} catch (KeeperException.NoNodeException e) {
|
||||||
|
throw new IllegalStateException("ZooKeeper path /brokers/ids does not exist", e);
|
||||||
|
}
|
||||||
|
brokerIds.sort(KafkaAgent::compareBrokerIds);
|
||||||
|
|
||||||
|
List<JsonObject> registrations = new ArrayList<>();
|
||||||
|
for (String brokerId : brokerIds) {
|
||||||
|
try {
|
||||||
|
byte[] data = zooKeeper.getData("/brokers/ids/" + brokerId, false, null);
|
||||||
|
registrations.add(JsonParser.parseString(new String(data, StandardCharsets.UTF_8)).getAsJsonObject());
|
||||||
|
} catch (KeeperException.NoNodeException e) {
|
||||||
|
// Expected race: a broker may refresh its ephemeral node between list and read.
|
||||||
|
logger().debug("Kafka broker {} disappeared during ZooKeeper discovery", brokerId);
|
||||||
|
} catch (RuntimeException e) {
|
||||||
|
logger().warn("Skipping malformed ZooKeeper registration for Kafka broker {}", brokerId, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return brokerEndpoints(registrations, securityProtocol);
|
||||||
|
} finally {
|
||||||
|
zooKeeper.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static ZKClientConfig zooKeeperClientConfig(JsonObject conn) {
|
||||||
|
ZKClientConfig clientConfig = new ZKClientConfig();
|
||||||
|
JsonObject properties = connectionProperties(conn);
|
||||||
|
if (properties == null) return clientConfig;
|
||||||
|
|
||||||
|
for (Map.Entry<String, JsonElement> entry : properties.entrySet()) {
|
||||||
|
if (entry.getKey().startsWith(ZOOKEEPER_PROPERTY_PREFIX)
|
||||||
|
&& entry.getValue().isJsonPrimitive()) {
|
||||||
|
clientConfig.setProperty(entry.getKey(), entry.getValue().getAsString());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return clientConfig;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static int compareBrokerIds(String left, String right) {
|
||||||
|
try {
|
||||||
|
return Integer.compare(Integer.parseInt(left), Integer.parseInt(right));
|
||||||
|
} catch (NumberFormatException e) {
|
||||||
|
// Third-party registries may use non-numeric IDs; lexical ordering remains deterministic.
|
||||||
|
logger().debug("Sorting non-numeric Kafka broker IDs lexically: left={}, right={}", left, right);
|
||||||
|
return left.compareTo(right);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static String brokerEndpoints(List<JsonObject> registrations, String securityProtocol) {
|
||||||
|
String targetProtocol = securityProtocol == null || securityProtocol.isBlank()
|
||||||
|
? "PLAINTEXT"
|
||||||
|
: securityProtocol.toUpperCase(Locale.ROOT);
|
||||||
|
Set<String> addresses = new LinkedHashSet<>();
|
||||||
|
|
||||||
|
for (JsonObject registration : registrations) {
|
||||||
|
int addressCount = addresses.size();
|
||||||
|
JsonObject protocolMap = registration.has("listener_security_protocol_map")
|
||||||
|
&& registration.get("listener_security_protocol_map").isJsonObject()
|
||||||
|
? registration.getAsJsonObject("listener_security_protocol_map")
|
||||||
|
: new JsonObject();
|
||||||
|
JsonArray endpoints = registration.has("endpoints") && registration.get("endpoints").isJsonArray()
|
||||||
|
? registration.getAsJsonArray("endpoints")
|
||||||
|
: new JsonArray();
|
||||||
|
|
||||||
|
for (JsonElement element : endpoints) {
|
||||||
|
if (!element.isJsonPrimitive() || !element.getAsJsonPrimitive().isString()) continue;
|
||||||
|
String endpoint = element.getAsString();
|
||||||
|
int separator = endpoint.indexOf("://");
|
||||||
|
if (separator <= 0) continue;
|
||||||
|
String listener = endpoint.substring(0, separator).toUpperCase(Locale.ROOT);
|
||||||
|
JsonElement mapped = protocolMap.get(listener);
|
||||||
|
String mappedProtocol = mapped != null && mapped.isJsonPrimitive()
|
||||||
|
? mapped.getAsString().toUpperCase(Locale.ROOT)
|
||||||
|
: listener;
|
||||||
|
if (!targetProtocol.equals(mappedProtocol)) continue;
|
||||||
|
String address = endpointAddress(endpoint);
|
||||||
|
if (address != null) addresses.add(address);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (addresses.size() == addressCount && endpoints.size() == 0 && registration.has("host") && registration.has("port")) {
|
||||||
|
try {
|
||||||
|
String host = registration.get("host").getAsString().trim();
|
||||||
|
int port = registration.get("port").getAsInt();
|
||||||
|
if (!host.isEmpty() && port > 0 && port <= 65535) addresses.add(formatHostPort(host, port));
|
||||||
|
} catch (RuntimeException e) {
|
||||||
|
logger().warn("Skipping malformed legacy Kafka broker registration", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (addresses.isEmpty()) {
|
||||||
|
throw new IllegalArgumentException("ZooKeeper did not return any usable Kafka broker endpoints");
|
||||||
|
}
|
||||||
|
return String.join(",", addresses);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String endpointAddress(String endpoint) {
|
||||||
|
try {
|
||||||
|
URI uri = URI.create(endpoint);
|
||||||
|
String host = uri.getHost();
|
||||||
|
int port = uri.getPort();
|
||||||
|
if (host == null || host.isBlank() || port <= 0 || port > 65535) return null;
|
||||||
|
return formatHostPort(host, port);
|
||||||
|
} catch (IllegalArgumentException e) {
|
||||||
|
logger().debug("Skipping malformed Kafka broker endpoint", e);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String formatHostPort(String host, int port) {
|
||||||
|
return host.contains(":") && !host.startsWith("[") ? "[" + host + "]:" + port : host + ":" + port;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String securityProtocol(JsonObject conn) {
|
||||||
|
String protocol = stringOrEmpty(conn, "security_protocol");
|
||||||
|
if (protocol.isBlank()) protocol = stringOrEmpty(conn, "securityProtocol");
|
||||||
|
return protocol.isBlank() ? "PLAINTEXT" : protocol;
|
||||||
|
}
|
||||||
|
|
||||||
static void applySecurityProperties(JsonObject conn, Properties props) {
|
static void applySecurityProperties(JsonObject conn, Properties props) {
|
||||||
String securityProtocol = stringOrEmpty(conn, "security_protocol");
|
String securityProtocol = stringOrEmpty(conn, "security_protocol");
|
||||||
if (securityProtocol.isBlank()) {
|
if (securityProtocol.isBlank()) {
|
||||||
@@ -353,6 +545,7 @@ public final class KafkaAgent {
|
|||||||
for (Map.Entry<String, JsonElement> entry : properties.entrySet()) {
|
for (Map.Entry<String, JsonElement> entry : properties.entrySet()) {
|
||||||
if (entry.getValue().isJsonPrimitive()) {
|
if (entry.getValue().isJsonPrimitive()) {
|
||||||
String key = entry.getKey();
|
String key = entry.getKey();
|
||||||
|
if (key.startsWith(ZOOKEEPER_PROPERTY_PREFIX)) continue;
|
||||||
String value = entry.getValue().getAsString();
|
String value = entry.getValue().getAsString();
|
||||||
props.put(key, value);
|
props.put(key, value);
|
||||||
}
|
}
|
||||||
@@ -578,11 +771,47 @@ public final class KafkaAgent {
|
|||||||
}
|
}
|
||||||
|
|
||||||
ConfigResource resource = new ConfigResource(ConfigResource.Type.TOPIC, name);
|
ConfigResource resource = new ConfigResource(ConfigResource.Type.TOPIC, name);
|
||||||
admin.incrementalAlterConfigs(Collections.singletonMap(resource, ops))
|
try {
|
||||||
.all().get(timeout, TimeUnit.MILLISECONDS);
|
admin.incrementalAlterConfigs(Collections.singletonMap(resource, ops))
|
||||||
|
.all().get(timeout, TimeUnit.MILLISECONDS);
|
||||||
|
} catch (Exception e) {
|
||||||
|
if (!isUnsupportedVersionError(e)) throw e;
|
||||||
|
logger().info("Kafka broker does not support incrementalAlterConfigs; using legacy alterConfigs for topic {}", name);
|
||||||
|
Config current = admin.describeConfigs(Collections.singletonList(resource))
|
||||||
|
.all().get(timeout, TimeUnit.MILLISECONDS).get(resource);
|
||||||
|
Map<String, String> values = legacyTopicConfig(current, ops);
|
||||||
|
Config replacement = new Config(values.entrySet().stream()
|
||||||
|
.map(entry -> new ConfigEntry(entry.getKey(), entry.getValue()))
|
||||||
|
.collect(Collectors.toList()));
|
||||||
|
admin.alterConfigs(Collections.singletonMap(resource, replacement))
|
||||||
|
.all().get(timeout, TimeUnit.MILLISECONDS);
|
||||||
|
}
|
||||||
return Collections.singletonMap("ok", true);
|
return Collections.singletonMap("ok", true);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static Map<String, String> legacyTopicConfig(Config current, List<AlterConfigOp> ops) {
|
||||||
|
Map<String, String> values = new LinkedHashMap<>();
|
||||||
|
for (ConfigEntry entry : current.entries()) {
|
||||||
|
boolean topicOverride = entry.source() == ConfigEntry.ConfigSource.DYNAMIC_TOPIC_CONFIG
|
||||||
|
|| entry.source() == ConfigEntry.ConfigSource.UNKNOWN;
|
||||||
|
if (topicOverride && !entry.isReadOnly() && !entry.isSensitive() && entry.value() != null) {
|
||||||
|
values.put(entry.name(), entry.value());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (AlterConfigOp op : ops) {
|
||||||
|
String key = op.configEntry().name();
|
||||||
|
switch (op.opType()) {
|
||||||
|
case SET -> values.put(key, op.configEntry().value());
|
||||||
|
case DELETE -> values.remove(key);
|
||||||
|
case APPEND, SUBTRACT -> throw new IllegalArgumentException(
|
||||||
|
"Kafka broker does not support " + op.opType() + " config operations through the legacy alterConfigs API"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return values;
|
||||||
|
}
|
||||||
|
|
||||||
// -----------------------------------------------------------------------
|
// -----------------------------------------------------------------------
|
||||||
// Consumer groups
|
// Consumer groups
|
||||||
// -----------------------------------------------------------------------
|
// -----------------------------------------------------------------------
|
||||||
@@ -698,6 +927,7 @@ public final class KafkaAgent {
|
|||||||
return Collections.singletonMap("producers", new ArrayList<>(byProducer.values()));
|
return Collections.singletonMap("producers", new ArrayList<>(byProducer.values()));
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
if (isUnsupportedVersionError(e)) {
|
if (isUnsupportedVersionError(e)) {
|
||||||
|
logger().info("Kafka broker does not support describeProducers; returning an empty producer list");
|
||||||
return Collections.singletonMap("producers", Collections.emptyList());
|
return Collections.singletonMap("producers", Collections.emptyList());
|
||||||
}
|
}
|
||||||
throw e;
|
throw e;
|
||||||
@@ -797,26 +1027,11 @@ public final class KafkaAgent {
|
|||||||
Long offset = longOrNull(params, "offset");
|
Long offset = longOrNull(params, "offset");
|
||||||
int count = Math.max(1, intOrDefault(params, "count", 10));
|
int count = Math.max(1, intOrDefault(params, "count", 10));
|
||||||
|
|
||||||
// Build a temporary consumer for peeking (no commit)
|
JsonObject conn = activeConnection;
|
||||||
Properties props = new Properties();
|
if (conn == null) {
|
||||||
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,
|
throw new IllegalStateException("Kafka Agent is not connected");
|
||||||
adminClient != null ? adminClient.describeCluster().clusterId()
|
|
||||||
.get(5, TimeUnit.SECONDS) : "localhost:9092");
|
|
||||||
// Reuse the admin's bootstrap servers
|
|
||||||
JsonObject conn = params.has("connection") && params.get("connection").isJsonObject()
|
|
||||||
? params.getAsJsonObject("connection") : null;
|
|
||||||
if (conn != null) {
|
|
||||||
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers(conn));
|
|
||||||
applyConnectionProperties(conn, props);
|
|
||||||
}
|
}
|
||||||
props.put(ConsumerConfig.GROUP_ID_CONFIG, "dbx-peek-" + UUID.randomUUID());
|
Properties props = peekConsumerProperties(conn, count);
|
||||||
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,
|
|
||||||
"org.apache.kafka.common.serialization.StringDeserializer");
|
|
||||||
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,
|
|
||||||
"org.apache.kafka.common.serialization.ByteArrayDeserializer");
|
|
||||||
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
|
|
||||||
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "none");
|
|
||||||
props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, count);
|
|
||||||
|
|
||||||
try (KafkaConsumer<String, byte[]> consumer = new KafkaConsumer<>(props)) {
|
try (KafkaConsumer<String, byte[]> consumer = new KafkaConsumer<>(props)) {
|
||||||
List<TopicPartition> candidatePartitions = resolvePeekPartitions(consumer, topic, partition);
|
List<TopicPartition> candidatePartitions = resolvePeekPartitions(consumer, topic, partition);
|
||||||
@@ -872,6 +1087,21 @@ public final class KafkaAgent {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static Properties peekConsumerProperties(JsonObject conn, int count) {
|
||||||
|
Properties props = new Properties();
|
||||||
|
props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers(conn));
|
||||||
|
applyConnectionProperties(conn, props);
|
||||||
|
props.put(ConsumerConfig.GROUP_ID_CONFIG, "dbx-peek-" + UUID.randomUUID());
|
||||||
|
props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,
|
||||||
|
"org.apache.kafka.common.serialization.StringDeserializer");
|
||||||
|
props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,
|
||||||
|
"org.apache.kafka.common.serialization.ByteArrayDeserializer");
|
||||||
|
props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
|
||||||
|
props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "none");
|
||||||
|
props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, count);
|
||||||
|
return props;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Poll until {@code count} messages are collected, every assigned partition has reached its
|
* Poll until {@code count} messages are collected, every assigned partition has reached its
|
||||||
* end offset, or {@code deadlineNs} expires. Empty polls retry until caught-up or deadline —
|
* end offset, or {@code deadlineNs} expires. Empty polls retry until caught-up or deadline —
|
||||||
@@ -1335,15 +1565,10 @@ public final class KafkaAgent {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private static String tryDecodeUtf8(byte[] bytes) {
|
private static String tryDecodeUtf8(byte[] bytes) {
|
||||||
try {
|
String text = new String(bytes, StandardCharsets.UTF_8);
|
||||||
String text = new String(bytes, StandardCharsets.UTF_8);
|
// Replacement characters change the bytes on round-trip, identifying invalid UTF-8 without exceptions.
|
||||||
// Verify round-trip
|
byte[] reEncoded = text.getBytes(StandardCharsets.UTF_8);
|
||||||
byte[] reEncoded = text.getBytes(StandardCharsets.UTF_8);
|
return Arrays.equals(bytes, reEncoded) ? text : null;
|
||||||
if (Arrays.equals(bytes, reEncoded)) {
|
|
||||||
return text;
|
|
||||||
}
|
|
||||||
} catch (Exception ignored) {}
|
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static Long normalizePeekOffset(long requestedOffset, long beginningOffset, long endOffset) {
|
static Long normalizePeekOffset(long requestedOffset, long beginningOffset, long endOffset) {
|
||||||
|
|||||||
@@ -1,24 +1,282 @@
|
|||||||
package com.dbx.agent.kafka;
|
package com.dbx.agent.kafka;
|
||||||
|
|
||||||
|
import com.google.gson.JsonObject;
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
import com.google.gson.JsonParser;
|
import com.google.gson.JsonParser;
|
||||||
|
import java.net.InetSocketAddress;
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
import java.time.Duration;
|
import java.time.Duration;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Collections;
|
||||||
import java.util.HashMap;
|
import java.util.HashMap;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import java.util.Properties;
|
import java.util.Properties;
|
||||||
import java.util.concurrent.atomic.AtomicInteger;
|
import java.util.concurrent.atomic.AtomicInteger;
|
||||||
|
import java.util.concurrent.CountDownLatch;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
import org.apache.kafka.clients.consumer.ConsumerRecord;
|
import org.apache.kafka.clients.consumer.ConsumerRecord;
|
||||||
import org.apache.kafka.clients.consumer.ConsumerRecords;
|
import org.apache.kafka.clients.consumer.ConsumerRecords;
|
||||||
|
import org.apache.kafka.clients.admin.AlterConfigOp;
|
||||||
|
import org.apache.kafka.clients.admin.Config;
|
||||||
|
import org.apache.kafka.clients.admin.ConfigEntry;
|
||||||
import org.apache.kafka.common.TopicPartition;
|
import org.apache.kafka.common.TopicPartition;
|
||||||
|
import org.apache.zookeeper.CreateMode;
|
||||||
|
import org.apache.zookeeper.Watcher;
|
||||||
|
import org.apache.zookeeper.ZooDefs;
|
||||||
|
import org.apache.zookeeper.ZooKeeper;
|
||||||
|
import org.apache.zookeeper.client.ZKClientConfig;
|
||||||
|
import org.apache.zookeeper.server.NIOServerCnxnFactory;
|
||||||
|
import org.apache.zookeeper.server.ZooKeeperServer;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.junit.jupiter.api.io.TempDir;
|
||||||
|
|
||||||
class KafkaAgentTest {
|
class KafkaAgentTest {
|
||||||
|
@TempDir
|
||||||
|
Path tempDir;
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void resolvesBootstrapServersFromKafka11ZooKeeperRegistrationWithChroot() throws Exception {
|
||||||
|
Path snapshots = Files.createDirectory(tempDir.resolve("snapshots"));
|
||||||
|
Path logs = Files.createDirectory(tempDir.resolve("logs"));
|
||||||
|
ZooKeeperServer server = new ZooKeeperServer(snapshots.toFile(), logs.toFile(), 2_000);
|
||||||
|
NIOServerCnxnFactory factory = new NIOServerCnxnFactory();
|
||||||
|
factory.configure(new InetSocketAddress("127.0.0.1", 0), 10);
|
||||||
|
factory.startup(server);
|
||||||
|
|
||||||
|
ZooKeeper client = null;
|
||||||
|
String previousSaslSetting = System.getProperty("zookeeper.sasl.client");
|
||||||
|
try {
|
||||||
|
CountDownLatch connected = new CountDownLatch(1);
|
||||||
|
System.setProperty("zookeeper.sasl.client", "false");
|
||||||
|
client = new ZooKeeper("127.0.0.1:" + factory.getLocalPort(), 5_000, event -> {
|
||||||
|
if (event.getState() == Watcher.Event.KeeperState.SyncConnected) connected.countDown();
|
||||||
|
});
|
||||||
|
assertTrue(connected.await(5, TimeUnit.SECONDS));
|
||||||
|
client.create("/kafka", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
|
||||||
|
client.create("/kafka/brokers", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
|
||||||
|
client.create("/kafka/brokers/ids", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
|
||||||
|
client.create(
|
||||||
|
"/kafka/brokers/ids/0",
|
||||||
|
"{\"listener_security_protocol_map\":{\"PLAINTEXT\":\"PLAINTEXT\"},\"endpoints\":[\"PLAINTEXT://legacy-broker:9092\"]}".getBytes(StandardCharsets.UTF_8),
|
||||||
|
ZooDefs.Ids.OPEN_ACL_UNSAFE,
|
||||||
|
CreateMode.EPHEMERAL
|
||||||
|
);
|
||||||
|
|
||||||
|
JsonObject connection = new JsonObject();
|
||||||
|
connection.addProperty("zookeeper_connect_string", "127.0.0.1:" + factory.getLocalPort() + "/kafka");
|
||||||
|
connection.addProperty("security_protocol", "PLAINTEXT");
|
||||||
|
connection.addProperty("zookeeper_connection_timeout_ms", 5_000);
|
||||||
|
|
||||||
|
JsonObject resolved = KafkaAgent.resolveBrokerConnection(connection);
|
||||||
|
|
||||||
|
assertEquals("legacy-broker:9092", resolved.get("bootstrap_servers").getAsString());
|
||||||
|
} finally {
|
||||||
|
if (client != null) client.close();
|
||||||
|
factory.shutdown();
|
||||||
|
server.shutdown();
|
||||||
|
server.getTxnLogFactory().close();
|
||||||
|
if (previousSaslSetting == null) {
|
||||||
|
System.clearProperty("zookeeper.sasl.client");
|
||||||
|
} else {
|
||||||
|
System.setProperty("zookeeper.sasl.client", previousSaslSetting);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void zooKeeperClientConfigPreservesSaslAndTlsSystemDefaults() {
|
||||||
|
Map<String, String> previous = preserveSystemProperties(
|
||||||
|
"zookeeper.sasl.client",
|
||||||
|
"zookeeper.sasl.clientconfig",
|
||||||
|
"zookeeper.client.secure",
|
||||||
|
"zookeeper.clientCnxnSocket",
|
||||||
|
"zookeeper.ssl.trustStore.location",
|
||||||
|
"java.security.auth.login.config"
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
System.setProperty("zookeeper.sasl.client", "true");
|
||||||
|
System.setProperty("zookeeper.sasl.clientconfig", "DbxZooKeeperClient");
|
||||||
|
System.setProperty("zookeeper.client.secure", "true");
|
||||||
|
System.setProperty("zookeeper.clientCnxnSocket", "org.apache.zookeeper.ClientCnxnSocketNetty");
|
||||||
|
System.setProperty("zookeeper.ssl.trustStore.location", "/etc/dbx/zookeeper-truststore.p12");
|
||||||
|
System.setProperty("java.security.auth.login.config", "/etc/dbx/zookeeper-jaas.conf");
|
||||||
|
|
||||||
|
ZKClientConfig config = KafkaAgent.zooKeeperClientConfig(new JsonObject());
|
||||||
|
|
||||||
|
assertTrue(config.isSaslClientEnabled());
|
||||||
|
assertEquals("DbxZooKeeperClient", config.getProperty("zookeeper.sasl.clientconfig"));
|
||||||
|
assertEquals("true", config.getProperty("zookeeper.client.secure"));
|
||||||
|
assertEquals(
|
||||||
|
"org.apache.zookeeper.ClientCnxnSocketNetty",
|
||||||
|
config.getProperty("zookeeper.clientCnxnSocket")
|
||||||
|
);
|
||||||
|
assertEquals(
|
||||||
|
"/etc/dbx/zookeeper-truststore.p12",
|
||||||
|
config.getProperty("zookeeper.ssl.trustStore.location")
|
||||||
|
);
|
||||||
|
assertEquals("/etc/dbx/zookeeper-jaas.conf", config.getJaasConfKey());
|
||||||
|
} finally {
|
||||||
|
restoreSystemProperties(previous);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void zooKeeperClientConfigAppliesPerConnectionSaslAndTlsOverridesWithoutChangingJvmState() {
|
||||||
|
Map<String, String> previous = preserveSystemProperties(
|
||||||
|
"zookeeper.sasl.client",
|
||||||
|
"zookeeper.sasl.clientconfig",
|
||||||
|
"zookeeper.client.secure",
|
||||||
|
"zookeeper.clientCnxnSocket",
|
||||||
|
"zookeeper.ssl.keyStore.location"
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
System.setProperty("zookeeper.sasl.client", "false");
|
||||||
|
System.setProperty("zookeeper.client.secure", "false");
|
||||||
|
|
||||||
|
JsonObject properties = new JsonObject();
|
||||||
|
properties.addProperty("zookeeper.sasl.client", "true");
|
||||||
|
properties.addProperty("zookeeper.sasl.clientconfig", "DbxZooKeeperClient");
|
||||||
|
properties.addProperty("zookeeper.client.secure", "true");
|
||||||
|
properties.addProperty("zookeeper.clientCnxnSocket", "org.apache.zookeeper.ClientCnxnSocketNetty");
|
||||||
|
properties.addProperty("zookeeper.ssl.keyStore.location", "/etc/dbx/zookeeper-keystore.p12");
|
||||||
|
properties.addProperty("security.protocol", "SASL_SSL");
|
||||||
|
JsonObject connection = new JsonObject();
|
||||||
|
connection.add("properties", properties);
|
||||||
|
|
||||||
|
ZKClientConfig config = KafkaAgent.zooKeeperClientConfig(connection);
|
||||||
|
|
||||||
|
assertTrue(config.isSaslClientEnabled());
|
||||||
|
assertEquals("DbxZooKeeperClient", config.getProperty("zookeeper.sasl.clientconfig"));
|
||||||
|
assertEquals("true", config.getProperty("zookeeper.client.secure"));
|
||||||
|
assertEquals(
|
||||||
|
"org.apache.zookeeper.ClientCnxnSocketNetty",
|
||||||
|
config.getProperty("zookeeper.clientCnxnSocket")
|
||||||
|
);
|
||||||
|
assertEquals(
|
||||||
|
"/etc/dbx/zookeeper-keystore.p12",
|
||||||
|
config.getProperty("zookeeper.ssl.keyStore.location")
|
||||||
|
);
|
||||||
|
assertNull(config.getProperty("security.protocol"));
|
||||||
|
assertEquals("false", System.getProperty("zookeeper.sasl.client"));
|
||||||
|
assertEquals("false", System.getProperty("zookeeper.client.secure"));
|
||||||
|
} finally {
|
||||||
|
restoreSystemProperties(previous);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void brokerEndpointsUseListenerSecurityProtocolMapForNamedListenersAndKeepBrokerOrder() {
|
||||||
|
List<JsonObject> registrations = Arrays.asList(
|
||||||
|
broker("{\"listener_security_protocol_map\":{\"INTERNAL\":\"PLAINTEXT\",\"CLIENT\":\"SASL_SSL\"},\"endpoints\":[\"INTERNAL://broker-2:9092\",\"CLIENT://public-2:9093\"]}"),
|
||||||
|
broker("{\"listener_security_protocol_map\":{\"INTERNAL\":\"PLAINTEXT\",\"CLIENT\":\"SASL_SSL\"},\"endpoints\":[\"CLIENT://public-1:9093\",\"INTERNAL://broker-1:9092\"]}")
|
||||||
|
);
|
||||||
|
|
||||||
|
assertEquals("public-2:9093,public-1:9093", KafkaAgent.brokerEndpoints(registrations, "SASL_SSL"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void kafkaClientPropertiesExcludeZooKeeperSecuritySettings() {
|
||||||
|
JsonObject properties = new JsonObject();
|
||||||
|
properties.addProperty("client.id", "dbx");
|
||||||
|
properties.addProperty("zookeeper.sasl.client", "true");
|
||||||
|
properties.addProperty("zookeeper.ssl.trustStore.password", "secret");
|
||||||
|
JsonObject connection = new JsonObject();
|
||||||
|
connection.add("properties", properties);
|
||||||
|
|
||||||
|
Properties kafkaProperties = new Properties();
|
||||||
|
KafkaAgent.applyConnectionProperties(connection, kafkaProperties);
|
||||||
|
|
||||||
|
assertEquals("dbx", kafkaProperties.getProperty("client.id"));
|
||||||
|
assertNull(kafkaProperties.getProperty("zookeeper.sasl.client"));
|
||||||
|
assertNull(kafkaProperties.getProperty("zookeeper.ssl.trustStore.password"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void brokerEndpointsFallBackToLegacyHostAndPort() {
|
||||||
|
assertEquals("legacy-broker:9092", KafkaAgent.brokerEndpoints(
|
||||||
|
Collections.singletonList(broker("{\"host\":\"legacy-broker\",\"port\":9092}")), "PLAINTEXT"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void brokerEndpointsSkipMalformedRegistrationWhenAnotherBrokerIsUsable() {
|
||||||
|
assertEquals("healthy-broker:9092", KafkaAgent.brokerEndpoints(Arrays.asList(
|
||||||
|
broker("{\"host\":\"broken\",\"port\":\"not-a-port\"}"),
|
||||||
|
broker("{\"host\":\"healthy-broker\",\"port\":9092}")
|
||||||
|
), "PLAINTEXT"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void brokerEndpointsRejectRegistrationsWithoutUsableAddresses() {
|
||||||
|
var error = org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class,
|
||||||
|
() -> KafkaAgent.brokerEndpoints(Collections.singletonList(broker("{\"endpoints\":[]}")), "PLAINTEXT"));
|
||||||
|
assertTrue(error.getMessage().contains("usable Kafka broker endpoints"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void peekConsumerPropertiesReuseResolvedConnection() {
|
||||||
|
JsonObject resolved = new JsonObject();
|
||||||
|
resolved.addProperty("bootstrap_servers", "legacy-broker:9092");
|
||||||
|
resolved.addProperty("security_protocol", "PLAINTEXT");
|
||||||
|
|
||||||
|
Properties properties = KafkaAgent.peekConsumerProperties(resolved, 25);
|
||||||
|
|
||||||
|
assertEquals("legacy-broker:9092", properties.getProperty("bootstrap.servers"));
|
||||||
|
assertEquals(25, properties.get("max.poll.records"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void aclDisabledDetectionOnlyAcceptsKnownAuthorizerErrors() {
|
||||||
|
Exception disabled = new RuntimeException(
|
||||||
|
"ACL probe failed",
|
||||||
|
new IllegalStateException("No Authorizer is configured on the broker")
|
||||||
|
);
|
||||||
|
|
||||||
|
assertTrue(KafkaAgent.isAclDisabledError(disabled));
|
||||||
|
assertFalse(KafkaAgent.isAclDisabledError(new RuntimeException("Timed out waiting for broker response")));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void legacyTopicConfigAppliesSetAndDeleteWithoutLosingExistingOverrides() {
|
||||||
|
Config current = new Config(Arrays.asList(
|
||||||
|
new ConfigEntry("cleanup.policy", "delete"),
|
||||||
|
new ConfigEntry("retention.ms", "60000"),
|
||||||
|
new ConfigEntry(
|
||||||
|
"segment.bytes",
|
||||||
|
"1073741824",
|
||||||
|
ConfigEntry.ConfigSource.DYNAMIC_BROKER_CONFIG,
|
||||||
|
false,
|
||||||
|
false,
|
||||||
|
Collections.emptyList(),
|
||||||
|
ConfigEntry.ConfigType.LONG,
|
||||||
|
null
|
||||||
|
)
|
||||||
|
));
|
||||||
|
List<AlterConfigOp> ops = Arrays.asList(
|
||||||
|
new AlterConfigOp(new ConfigEntry("retention.ms", "120000"), AlterConfigOp.OpType.SET),
|
||||||
|
new AlterConfigOp(new ConfigEntry("cleanup.policy", null), AlterConfigOp.OpType.DELETE)
|
||||||
|
);
|
||||||
|
|
||||||
|
Map<String, String> merged = KafkaAgent.legacyTopicConfig(current, ops);
|
||||||
|
|
||||||
|
assertEquals(Collections.singletonMap("retention.ms", "120000"), merged);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void legacyTopicConfigRejectsAppendAndSubtractOperations() {
|
||||||
|
Config current = new Config(Collections.singletonList(new ConfigEntry("cleanup.policy", "delete")));
|
||||||
|
AlterConfigOp append = new AlterConfigOp(new ConfigEntry("cleanup.policy", "compact"), AlterConfigOp.OpType.APPEND);
|
||||||
|
|
||||||
|
var error = org.junit.jupiter.api.Assertions.assertThrows(IllegalArgumentException.class,
|
||||||
|
() -> KafkaAgent.legacyTopicConfig(current, Collections.singletonList(append)));
|
||||||
|
assertTrue(error.getMessage().contains("APPEND"));
|
||||||
|
}
|
||||||
@Test
|
@Test
|
||||||
void normalizesPeekOffsetToEarliestAvailableOffset() {
|
void normalizesPeekOffsetToEarliestAvailableOffset() {
|
||||||
assertEquals(5L, KafkaAgent.normalizePeekOffset(0, 5, 10));
|
assertEquals(5L, KafkaAgent.normalizePeekOffset(0, 5, 10));
|
||||||
@@ -241,4 +499,24 @@ class KafkaAgentTest {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static JsonObject broker(String json) {
|
||||||
|
return JsonParser.parseString(json).getAsJsonObject();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Map<String, String> preserveSystemProperties(String... keys) {
|
||||||
|
Map<String, String> previous = new HashMap<>();
|
||||||
|
for (String key : keys) previous.put(key, System.getProperty(key));
|
||||||
|
return previous;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void restoreSystemProperties(Map<String, String> properties) {
|
||||||
|
for (Map.Entry<String, String> entry : properties.entrySet()) {
|
||||||
|
if (entry.getValue() == null) {
|
||||||
|
System.clearProperty(entry.getKey());
|
||||||
|
} else {
|
||||||
|
System.setProperty(entry.getKey(), entry.getValue());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -46,7 +46,7 @@ import { SQLITE_DATABASE_FILE_EXTENSIONS } from "@/lib/database/databaseFileDete
|
|||||||
import { connectionAttemptOriginalErrorMessage, connectionAttemptTimeoutMessage, connectionAttemptTimeoutMs } from "@/lib/connection/connectionAttemptTimeout";
|
import { connectionAttemptOriginalErrorMessage, connectionAttemptTimeoutMessage, connectionAttemptTimeoutMs } from "@/lib/connection/connectionAttemptTimeout";
|
||||||
import { appendConnectionErrorHints, isJdbcMissingRuntimeDependencyError } from "@/lib/connection/connectionErrorHints";
|
import { appendConnectionErrorHints, isJdbcMissingRuntimeDependencyError } from "@/lib/connection/connectionErrorHints";
|
||||||
import { postgresTlsModeForForm } from "@/lib/connection/postgresTlsMode";
|
import { postgresTlsModeForForm } from "@/lib/connection/postgresTlsMode";
|
||||||
import { normalizeKafkaBootstrapServers } from "@/lib/connection/kafkaBootstrapServers";
|
import { buildMqKafkaConnectionExtra, mqKafkaConnectionTarget, resolveMqKafkaConnectionSource, type MqKafkaConnectionSource } from "@/lib/connection/mqKafkaConnection";
|
||||||
import { assertCompleteDatabaseCategories, databaseSelectionForCategory } from "@/lib/connection/databaseCategoryOptions";
|
import { assertCompleteDatabaseCategories, databaseSelectionForCategory } from "@/lib/connection/databaseCategoryOptions";
|
||||||
import { normalizeRocketmqNamesrvAddr } from "@/lib/connection/rocketmqNamesrv";
|
import { normalizeRocketmqNamesrvAddr } from "@/lib/connection/rocketmqNamesrv";
|
||||||
import { normalizeRabbitmqAddresses } from "@/lib/connection/rabbitmqAddresses";
|
import { normalizeRabbitmqAddresses } from "@/lib/connection/rabbitmqAddresses";
|
||||||
@@ -517,11 +517,13 @@ const configTab = ref<ConfigTab>("connection");
|
|||||||
const MQ_KAFKA_SECURITY_PROTOCOL_AUTO = "__auto";
|
const MQ_KAFKA_SECURITY_PROTOCOL_AUTO = "__auto";
|
||||||
const mqAdminUrl = ref("http://127.0.0.1:8080");
|
const mqAdminUrl = ref("http://127.0.0.1:8080");
|
||||||
const mqSystemKind = ref<MqSystemKind>("pulsar");
|
const mqSystemKind = ref<MqSystemKind>("pulsar");
|
||||||
|
const mqKafkaConnectionSource = ref<MqKafkaConnectionSource>("bootstrap");
|
||||||
const mqRocketmqNamesrvAddr = ref("127.0.0.1:9876");
|
const mqRocketmqNamesrvAddr = ref("127.0.0.1:9876");
|
||||||
const mqRocketmqClusterName = ref("");
|
const mqRocketmqClusterName = ref("");
|
||||||
const mqRabbitmqAddresses = ref("127.0.0.1:5672");
|
const mqRabbitmqAddresses = ref("127.0.0.1:5672");
|
||||||
const mqRabbitmqVirtualHost = ref("/");
|
const mqRabbitmqVirtualHost = ref("/");
|
||||||
const mqKafkaBootstrapServers = ref("127.0.0.1:9092");
|
const mqKafkaBootstrapServers = ref("127.0.0.1:9092");
|
||||||
|
const mqKafkaZooKeeperServers = ref("");
|
||||||
const mqKafkaSecurityProtocol = ref(MQ_KAFKA_SECURITY_PROTOCOL_AUTO);
|
const mqKafkaSecurityProtocol = ref(MQ_KAFKA_SECURITY_PROTOCOL_AUTO);
|
||||||
const mqKafkaSaslMechanism = ref("PLAIN");
|
const mqKafkaSaslMechanism = ref("PLAIN");
|
||||||
const mqKafkaKerberosPrincipal = ref("");
|
const mqKafkaKerberosPrincipal = ref("");
|
||||||
@@ -575,6 +577,10 @@ const mqKafkaSecurityProtocolOptions = computed(() => [
|
|||||||
{ value: "SASL_PLAINTEXT", label: "SASL_PLAINTEXT" },
|
{ value: "SASL_PLAINTEXT", label: "SASL_PLAINTEXT" },
|
||||||
{ value: "SASL_SSL", label: "SASL_SSL" },
|
{ value: "SASL_SSL", label: "SASL_SSL" },
|
||||||
]);
|
]);
|
||||||
|
const mqKafkaConnectionSourceOptions = computed(() => [
|
||||||
|
{ value: "bootstrap" as const, label: t("connection.mqKafkaConnectionSourceBootstrap") },
|
||||||
|
{ value: "zookeeper" as const, label: t("connection.mqKafkaConnectionSourceZooKeeper") },
|
||||||
|
]);
|
||||||
const mqKafkaSaslMechanismOptions = [
|
const mqKafkaSaslMechanismOptions = [
|
||||||
{ value: "PLAIN", label: "PLAIN" },
|
{ value: "PLAIN", label: "PLAIN" },
|
||||||
{ value: "SCRAM-SHA-256", label: "SCRAM-SHA-256" },
|
{ value: "SCRAM-SHA-256", label: "SCRAM-SHA-256" },
|
||||||
@@ -892,7 +898,9 @@ function resetMqFields(config?: Partial<MqAdminConfig>) {
|
|||||||
mqSystemKind.value = systemKind;
|
mqSystemKind.value = systemKind;
|
||||||
const storedAdminUrl = config?.adminUrl?.trim() || (config ? mqExtraString(config as Record<string, unknown>, "admin_url").trim() : "");
|
const storedAdminUrl = config?.adminUrl?.trim() || (config ? mqExtraString(config as Record<string, unknown>, "admin_url").trim() : "");
|
||||||
mqAdminUrl.value = storedAdminUrl || (systemKind === "kafka" || systemKind === "rocketmq" || systemKind === "rabbitmq" ? "" : "http://127.0.0.1:8080");
|
mqAdminUrl.value = storedAdminUrl || (systemKind === "kafka" || systemKind === "rocketmq" || systemKind === "rabbitmq" ? "" : "http://127.0.0.1:8080");
|
||||||
|
mqKafkaConnectionSource.value = resolveMqKafkaConnectionSource(extra);
|
||||||
mqKafkaBootstrapServers.value = mqExtraString(extra, "bootstrapServers") || "127.0.0.1:9092";
|
mqKafkaBootstrapServers.value = mqExtraString(extra, "bootstrapServers") || "127.0.0.1:9092";
|
||||||
|
mqKafkaZooKeeperServers.value = mqExtraString(extra, "zookeeperServers");
|
||||||
mqRocketmqNamesrvAddr.value = mqExtraString(extra, "namesrvAddr") || mqExtraString(extra, "namesrv_addr") || "127.0.0.1:9876";
|
mqRocketmqNamesrvAddr.value = mqExtraString(extra, "namesrvAddr") || mqExtraString(extra, "namesrv_addr") || "127.0.0.1:9876";
|
||||||
mqRocketmqClusterName.value = mqExtraString(extra, "clusterName") || mqExtraString(extra, "cluster_name");
|
mqRocketmqClusterName.value = mqExtraString(extra, "clusterName") || mqExtraString(extra, "cluster_name");
|
||||||
mqRabbitmqAddresses.value = mqExtraString(extra, "addresses") || "127.0.0.1:5672";
|
mqRabbitmqAddresses.value = mqExtraString(extra, "addresses") || "127.0.0.1:5672";
|
||||||
@@ -969,7 +977,7 @@ watch(selectedType, () => {
|
|||||||
|
|
||||||
watch(mqSystemKind, (kind) => {
|
watch(mqSystemKind, (kind) => {
|
||||||
if (kind === "kafka") {
|
if (kind === "kafka") {
|
||||||
if (!mqKafkaBootstrapServers.value.trim()) mqKafkaBootstrapServers.value = "127.0.0.1:9092";
|
if (mqKafkaConnectionSource.value === "bootstrap" && !mqKafkaBootstrapServers.value.trim()) mqKafkaBootstrapServers.value = "127.0.0.1:9092";
|
||||||
if (!isMqAuthKindAllowedForSystem(kind, mqAuthKind.value)) mqAuthKind.value = "none";
|
if (!isMqAuthKindAllowedForSystem(kind, mqAuthKind.value)) mqAuthKind.value = "none";
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -1108,9 +1116,14 @@ function buildMqTokenSigning() {
|
|||||||
function buildMqAdminConfig(): MqAdminConfig {
|
function buildMqAdminConfig(): MqAdminConfig {
|
||||||
const systemKind = mqSystemKind.value;
|
const systemKind = mqSystemKind.value;
|
||||||
if (systemKind === "kafka") {
|
if (systemKind === "kafka") {
|
||||||
const bootstrapServers = normalizeKafkaBootstrapServers(mqKafkaBootstrapServers.value);
|
const configuredSecurityProtocol = mqKafkaSecurityProtocol.value === MQ_KAFKA_SECURITY_PROTOCOL_AUTO ? "" : mqKafkaSecurityProtocol.value;
|
||||||
const extra: Record<string, unknown> = { bootstrapServers };
|
const extra: Record<string, unknown> = buildMqKafkaConnectionExtra({
|
||||||
const securityProtocol = mqKafkaSecurityProtocol.value === MQ_KAFKA_SECURITY_PROTOCOL_AUTO ? "" : mqKafkaSecurityProtocol.value.trim();
|
connectionSource: mqKafkaConnectionSource.value,
|
||||||
|
bootstrapServers: mqKafkaBootstrapServers.value,
|
||||||
|
zookeeperServers: mqKafkaZooKeeperServers.value,
|
||||||
|
securityProtocol: configuredSecurityProtocol,
|
||||||
|
});
|
||||||
|
const securityProtocol = mqExtraString(extra, "securityProtocol");
|
||||||
const saslMechanism = mqAuthKind.value === "kerberos" ? "GSSAPI" : mqKafkaSaslMechanism.value.trim();
|
const saslMechanism = mqAuthKind.value === "kerberos" ? "GSSAPI" : mqKafkaSaslMechanism.value.trim();
|
||||||
const properties: Record<string, string> = {};
|
const properties: Record<string, string> = {};
|
||||||
if (securityProtocol) extra.securityProtocol = securityProtocol;
|
if (securityProtocol) extra.securityProtocol = securityProtocol;
|
||||||
@@ -1518,18 +1531,17 @@ function applyMqAdminUrl(config: LegacyConnectionConfig, adminUrl: string) {
|
|||||||
config.ssl = parsed.protocol === "https:";
|
config.ssl = parsed.protocol === "https:";
|
||||||
}
|
}
|
||||||
|
|
||||||
function applyMqKafkaBootstrapServers(config: LegacyConnectionConfig, bootstrapServers: string, securityProtocol?: string) {
|
function applyMqKafkaConnectionTarget(config: LegacyConnectionConfig, extra: Record<string, unknown>) {
|
||||||
const first = normalizeKafkaBootstrapServers(bootstrapServers).split(",")[0];
|
const source = resolveMqKafkaConnectionSource(extra);
|
||||||
if (!first) throw new Error(t("connection.mqBootstrapServersRequired"));
|
const target = mqKafkaConnectionTarget({
|
||||||
let parsed: URL;
|
connectionSource: source,
|
||||||
try {
|
bootstrapServers: mqExtraString(extra, "bootstrapServers"),
|
||||||
parsed = new URL(`kafka://${first}`);
|
zookeeperServers: mqExtraString(extra, "zookeeperServers"),
|
||||||
} catch {
|
securityProtocol: mqExtraString(extra, "securityProtocol"),
|
||||||
throw new Error(t("connection.mqBootstrapServersInvalid"));
|
});
|
||||||
}
|
config.host = target.host;
|
||||||
config.host = parsed.hostname;
|
config.port = target.port;
|
||||||
config.port = Number(parsed.port) || 9092;
|
config.ssl = target.ssl;
|
||||||
config.ssl = securityProtocol === "SSL" || securityProtocol === "SASL_SSL";
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function applyMqRabbitmqAddresses(config: LegacyConnectionConfig, addresses: string) {
|
function applyMqRabbitmqAddresses(config: LegacyConnectionConfig, addresses: string) {
|
||||||
@@ -2548,7 +2560,7 @@ const connectionLabelTopClass = `${connectionLabelClass} mt-2`;
|
|||||||
const connectionLabelSmallPaddedClass = `${connectionLabelClass} pt-2 text-xs`;
|
const connectionLabelSmallPaddedClass = `${connectionLabelClass} pt-2 text-xs`;
|
||||||
const hasRequiredConnectionTarget = computed(() => {
|
const hasRequiredConnectionTarget = computed(() => {
|
||||||
if (form.value.db_type === "mq") {
|
if (form.value.db_type === "mq") {
|
||||||
if (mqSystemKind.value === "kafka") return !!mqKafkaBootstrapServers.value.trim();
|
if (mqSystemKind.value === "kafka") return mqKafkaConnectionSource.value === "zookeeper" ? !!mqKafkaZooKeeperServers.value.trim() : !!mqKafkaBootstrapServers.value.trim();
|
||||||
if (mqSystemKind.value === "rocketmq") return !!mqRocketmqNamesrvAddr.value.trim();
|
if (mqSystemKind.value === "rocketmq") return !!mqRocketmqNamesrvAddr.value.trim();
|
||||||
if (mqSystemKind.value === "rabbitmq") return !!mqRabbitmqAddresses.value.trim();
|
if (mqSystemKind.value === "rabbitmq") return !!mqRabbitmqAddresses.value.trim();
|
||||||
return !!mqAdminUrl.value.trim();
|
return !!mqAdminUrl.value.trim();
|
||||||
@@ -2881,7 +2893,7 @@ function connectionConfigForSubmit(id: string, generatedName = ""): ConnectionCo
|
|||||||
config.driver_label = MQ_DRIVER_LABELS[mqConfig.systemKind];
|
config.driver_label = MQ_DRIVER_LABELS[mqConfig.systemKind];
|
||||||
if (mqConfig.systemKind === "kafka") {
|
if (mqConfig.systemKind === "kafka") {
|
||||||
const extra = mqExtraRecord(mqConfig);
|
const extra = mqExtraRecord(mqConfig);
|
||||||
applyMqKafkaBootstrapServers(config, mqExtraString(extra, "bootstrapServers"), mqExtraString(extra, "securityProtocol"));
|
applyMqKafkaConnectionTarget(config, extra);
|
||||||
} else if (mqConfig.systemKind === "rocketmq") {
|
} else if (mqConfig.systemKind === "rocketmq") {
|
||||||
const extra = mqExtraRecord(mqConfig);
|
const extra = mqExtraRecord(mqConfig);
|
||||||
applyMqRocketmqNamesrv(config, mqExtraString(extra, "namesrvAddr") || mqExtraString(extra, "namesrv_addr"));
|
applyMqRocketmqNamesrv(config, mqExtraString(extra, "namesrvAddr") || mqExtraString(extra, "namesrv_addr"));
|
||||||
@@ -4769,9 +4781,26 @@ function openExternalUrl(url: string) {
|
|||||||
<template v-else-if="form.db_type === 'mq'">
|
<template v-else-if="form.db_type === 'mq'">
|
||||||
<template v-if="mqSystemKind === 'kafka'">
|
<template v-if="mqSystemKind === 'kafka'">
|
||||||
<div class="grid grid-cols-4 items-center gap-4">
|
<div class="grid grid-cols-4 items-center gap-4">
|
||||||
|
<Label :class="connectionLabelClass">{{ t("connection.mqKafkaConnectionSource") }}</Label>
|
||||||
|
<Select v-model="mqKafkaConnectionSource">
|
||||||
|
<SelectTrigger class="col-span-3 h-9">
|
||||||
|
<SelectValue />
|
||||||
|
</SelectTrigger>
|
||||||
|
<SelectContent>
|
||||||
|
<SelectItem v-for="option in mqKafkaConnectionSourceOptions" :key="option.value" :value="option.value">
|
||||||
|
{{ option.label }}
|
||||||
|
</SelectItem>
|
||||||
|
</SelectContent>
|
||||||
|
</Select>
|
||||||
|
</div>
|
||||||
|
<div v-if="mqKafkaConnectionSource === 'bootstrap'" class="grid grid-cols-4 items-center gap-4">
|
||||||
<Label :class="connectionLabelClass">{{ t("connection.mqBootstrapServers") }}</Label>
|
<Label :class="connectionLabelClass">{{ t("connection.mqBootstrapServers") }}</Label>
|
||||||
<Input v-model="mqKafkaBootstrapServers" class="col-span-3" :placeholder="t('connection.mqBootstrapServersPlaceholder')" />
|
<Input v-model="mqKafkaBootstrapServers" class="col-span-3" :placeholder="t('connection.mqBootstrapServersPlaceholder')" />
|
||||||
</div>
|
</div>
|
||||||
|
<div v-else class="grid grid-cols-4 items-center gap-4">
|
||||||
|
<Label :class="connectionLabelClass">{{ t("connection.mqKafkaZooKeeperServers") }}</Label>
|
||||||
|
<Input v-model="mqKafkaZooKeeperServers" class="col-span-3" :placeholder="t('connection.mqKafkaZooKeeperServersPlaceholder')" />
|
||||||
|
</div>
|
||||||
<div class="grid grid-cols-4 items-center gap-4">
|
<div class="grid grid-cols-4 items-center gap-4">
|
||||||
<Label :class="connectionLabelClass">{{ t("connection.mqSecurity") }}</Label>
|
<Label :class="connectionLabelClass">{{ t("connection.mqSecurity") }}</Label>
|
||||||
<Select v-model="mqKafkaSecurityProtocol">
|
<Select v-model="mqKafkaSecurityProtocol">
|
||||||
|
|||||||
@@ -356,6 +356,11 @@ export default {
|
|||||||
mqBootstrapServersPlaceholder: "127.0.0.1:9092",
|
mqBootstrapServersPlaceholder: "127.0.0.1:9092",
|
||||||
mqBootstrapServersRequired: "Kafka bootstrap servers are required",
|
mqBootstrapServersRequired: "Kafka bootstrap servers are required",
|
||||||
mqBootstrapServersInvalid: "Kafka bootstrap servers are invalid",
|
mqBootstrapServersInvalid: "Kafka bootstrap servers are invalid",
|
||||||
|
mqKafkaConnectionSource: "Connection Source",
|
||||||
|
mqKafkaConnectionSourceBootstrap: "Bootstrap Servers",
|
||||||
|
mqKafkaConnectionSourceZooKeeper: "ZooKeeper (Kafka 1.x)",
|
||||||
|
mqKafkaZooKeeperServers: "ZooKeeper Servers",
|
||||||
|
mqKafkaZooKeeperServersPlaceholder: "zk1:2181,zk2:2181/kafka",
|
||||||
mqSystemRabbitMq: "RabbitMQ",
|
mqSystemRabbitMq: "RabbitMQ",
|
||||||
mqRabbitmqAddresses: "Addresses",
|
mqRabbitmqAddresses: "Addresses",
|
||||||
mqRabbitmqAddressesPlaceholder: "127.0.0.1:5672",
|
mqRabbitmqAddressesPlaceholder: "127.0.0.1:5672",
|
||||||
|
|||||||
@@ -487,6 +487,11 @@ export default withEnglishFallback({
|
|||||||
mqBootstrapServersPlaceholder: "127.0.0.1:9092",
|
mqBootstrapServersPlaceholder: "127.0.0.1:9092",
|
||||||
mqBootstrapServersRequired: "Los bootstrap servers de Kafka son obligatorios",
|
mqBootstrapServersRequired: "Los bootstrap servers de Kafka son obligatorios",
|
||||||
mqBootstrapServersInvalid: "Los bootstrap servers de Kafka no son válidos",
|
mqBootstrapServersInvalid: "Los bootstrap servers de Kafka no son válidos",
|
||||||
|
mqKafkaConnectionSource: "Origen de conexión",
|
||||||
|
mqKafkaConnectionSourceBootstrap: "Bootstrap Servers",
|
||||||
|
mqKafkaConnectionSourceZooKeeper: "ZooKeeper (Kafka 1.x)",
|
||||||
|
mqKafkaZooKeeperServers: "Servidores ZooKeeper",
|
||||||
|
mqKafkaZooKeeperServersPlaceholder: "zk1:2181,zk2:2181/kafka",
|
||||||
mqSystemRabbitMq: "RabbitMQ",
|
mqSystemRabbitMq: "RabbitMQ",
|
||||||
mqRabbitmqAddresses: "Direcciones",
|
mqRabbitmqAddresses: "Direcciones",
|
||||||
mqRabbitmqAddressesPlaceholder: "127.0.0.1:5672",
|
mqRabbitmqAddressesPlaceholder: "127.0.0.1:5672",
|
||||||
|
|||||||
@@ -485,6 +485,11 @@ export default withEnglishFallback({
|
|||||||
mqBootstrapServersPlaceholder: "127.0.0.1:9092",
|
mqBootstrapServersPlaceholder: "127.0.0.1:9092",
|
||||||
mqBootstrapServersRequired: "I bootstrap server Kafka sono obbligatori",
|
mqBootstrapServersRequired: "I bootstrap server Kafka sono obbligatori",
|
||||||
mqBootstrapServersInvalid: "I bootstrap server Kafka non sono validi",
|
mqBootstrapServersInvalid: "I bootstrap server Kafka non sono validi",
|
||||||
|
mqKafkaConnectionSource: "Origine connessione",
|
||||||
|
mqKafkaConnectionSourceBootstrap: "Bootstrap Servers",
|
||||||
|
mqKafkaConnectionSourceZooKeeper: "ZooKeeper (Kafka 1.x)",
|
||||||
|
mqKafkaZooKeeperServers: "Server ZooKeeper",
|
||||||
|
mqKafkaZooKeeperServersPlaceholder: "zk1:2181,zk2:2181/kafka",
|
||||||
mqSystemRabbitMq: "RabbitMQ",
|
mqSystemRabbitMq: "RabbitMQ",
|
||||||
mqRabbitmqAddresses: "Indirizzi",
|
mqRabbitmqAddresses: "Indirizzi",
|
||||||
mqRabbitmqAddressesPlaceholder: "127.0.0.1:5672",
|
mqRabbitmqAddressesPlaceholder: "127.0.0.1:5672",
|
||||||
|
|||||||
@@ -485,6 +485,11 @@ export default withEnglishFallback({
|
|||||||
mqBootstrapServersPlaceholder: "127.0.0.1:9092",
|
mqBootstrapServersPlaceholder: "127.0.0.1:9092",
|
||||||
mqBootstrapServersRequired: "Kafka Bootstrap Servers は必須です",
|
mqBootstrapServersRequired: "Kafka Bootstrap Servers は必須です",
|
||||||
mqBootstrapServersInvalid: "Kafka Bootstrap Servers が無効です",
|
mqBootstrapServersInvalid: "Kafka Bootstrap Servers が無効です",
|
||||||
|
mqKafkaConnectionSource: "接続元",
|
||||||
|
mqKafkaConnectionSourceBootstrap: "Bootstrap Servers",
|
||||||
|
mqKafkaConnectionSourceZooKeeper: "ZooKeeper (Kafka 1.x)",
|
||||||
|
mqKafkaZooKeeperServers: "ZooKeeper サーバー",
|
||||||
|
mqKafkaZooKeeperServersPlaceholder: "zk1:2181,zk2:2181/kafka",
|
||||||
mqSystemRabbitMq: "RabbitMQ",
|
mqSystemRabbitMq: "RabbitMQ",
|
||||||
mqRabbitmqAddresses: "アドレス",
|
mqRabbitmqAddresses: "アドレス",
|
||||||
mqRabbitmqAddressesPlaceholder: "127.0.0.1:5672",
|
mqRabbitmqAddressesPlaceholder: "127.0.0.1:5672",
|
||||||
|
|||||||
@@ -486,6 +486,11 @@ export default withEnglishFallback({
|
|||||||
mqBootstrapServersPlaceholder: "127.0.0.1:9092",
|
mqBootstrapServersPlaceholder: "127.0.0.1:9092",
|
||||||
mqBootstrapServersRequired: "Bootstrap Servers do Kafka são obrigatórios",
|
mqBootstrapServersRequired: "Bootstrap Servers do Kafka são obrigatórios",
|
||||||
mqBootstrapServersInvalid: "Bootstrap Servers do Kafka são inválidos",
|
mqBootstrapServersInvalid: "Bootstrap Servers do Kafka são inválidos",
|
||||||
|
mqKafkaConnectionSource: "Origem da conexão",
|
||||||
|
mqKafkaConnectionSourceBootstrap: "Bootstrap Servers",
|
||||||
|
mqKafkaConnectionSourceZooKeeper: "ZooKeeper (Kafka 1.x)",
|
||||||
|
mqKafkaZooKeeperServers: "Servidores ZooKeeper",
|
||||||
|
mqKafkaZooKeeperServersPlaceholder: "zk1:2181,zk2:2181/kafka",
|
||||||
mqSystemRabbitMq: "RabbitMQ",
|
mqSystemRabbitMq: "RabbitMQ",
|
||||||
mqRabbitmqAddresses: "Endereços",
|
mqRabbitmqAddresses: "Endereços",
|
||||||
mqRabbitmqAddressesPlaceholder: "127.0.0.1:5672",
|
mqRabbitmqAddressesPlaceholder: "127.0.0.1:5672",
|
||||||
|
|||||||
@@ -358,6 +358,11 @@ export default withEnglishFallback({
|
|||||||
mqBootstrapServersPlaceholder: "127.0.0.1:9092",
|
mqBootstrapServersPlaceholder: "127.0.0.1:9092",
|
||||||
mqBootstrapServersRequired: "Kafka Bootstrap Servers 不能为空",
|
mqBootstrapServersRequired: "Kafka Bootstrap Servers 不能为空",
|
||||||
mqBootstrapServersInvalid: "Kafka Bootstrap Servers 无效",
|
mqBootstrapServersInvalid: "Kafka Bootstrap Servers 无效",
|
||||||
|
mqKafkaConnectionSource: "连接来源",
|
||||||
|
mqKafkaConnectionSourceBootstrap: "Bootstrap Servers",
|
||||||
|
mqKafkaConnectionSourceZooKeeper: "ZooKeeper(Kafka 1.x)",
|
||||||
|
mqKafkaZooKeeperServers: "ZooKeeper Servers",
|
||||||
|
mqKafkaZooKeeperServersPlaceholder: "zk1:2181,zk2:2181/kafka",
|
||||||
mqSystemRabbitMq: "RabbitMQ",
|
mqSystemRabbitMq: "RabbitMQ",
|
||||||
mqRabbitmqAddresses: "地址",
|
mqRabbitmqAddresses: "地址",
|
||||||
mqRabbitmqAddressesPlaceholder: "127.0.0.1:5672",
|
mqRabbitmqAddressesPlaceholder: "127.0.0.1:5672",
|
||||||
|
|||||||
@@ -486,6 +486,11 @@ export default withEnglishFallback({
|
|||||||
mqBootstrapServersPlaceholder: "127.0.0.1:9092",
|
mqBootstrapServersPlaceholder: "127.0.0.1:9092",
|
||||||
mqBootstrapServersRequired: "Kafka Bootstrap Servers 不能為空",
|
mqBootstrapServersRequired: "Kafka Bootstrap Servers 不能為空",
|
||||||
mqBootstrapServersInvalid: "Kafka Bootstrap Servers 無效",
|
mqBootstrapServersInvalid: "Kafka Bootstrap Servers 無效",
|
||||||
|
mqKafkaConnectionSource: "連線來源",
|
||||||
|
mqKafkaConnectionSourceBootstrap: "Bootstrap Servers",
|
||||||
|
mqKafkaConnectionSourceZooKeeper: "ZooKeeper(Kafka 1.x)",
|
||||||
|
mqKafkaZooKeeperServers: "ZooKeeper Servers",
|
||||||
|
mqKafkaZooKeeperServersPlaceholder: "zk1:2181,zk2:2181/kafka",
|
||||||
mqSystemRabbitMq: "RabbitMQ",
|
mqSystemRabbitMq: "RabbitMQ",
|
||||||
mqRabbitmqAddresses: "位址",
|
mqRabbitmqAddresses: "位址",
|
||||||
mqRabbitmqAddressesPlaceholder: "127.0.0.1:5672",
|
mqRabbitmqAddressesPlaceholder: "127.0.0.1:5672",
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import { normalizeKafkaBootstrapServers } from "@/lib/connection/kafkaBootstrapServers";
|
import { normalizeKafkaBootstrapServers, parseKafkaBootstrapServers, resolveKafkaSecurityProtocol } from "@/lib/connection/kafkaBootstrapServers";
|
||||||
|
|
||||||
describe("Kafka bootstrap servers", () => {
|
describe("Kafka bootstrap servers", () => {
|
||||||
it("keeps comma-separated bootstrap servers", () => {
|
it("keeps comma-separated bootstrap servers", () => {
|
||||||
@@ -14,11 +14,34 @@ describe("Kafka bootstrap servers", () => {
|
|||||||
expect(normalizeKafkaBootstrapServers("[::1]:9092;[2001:db8::1]:9092")).toBe("[::1]:9092,[2001:db8::1]:9092");
|
expect(normalizeKafkaBootstrapServers("[::1]:9092;[2001:db8::1]:9092")).toBe("[::1]:9092,[2001:db8::1]:9092");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("rejects bootstrap servers with URL schemes", () => {
|
it("normalizes Kafka listener URIs and exposes the inferred security protocol", () => {
|
||||||
expect(() => normalizeKafkaBootstrapServers("PLAINTEXT://broker1:9092,broker2:9092")).toThrow("Kafka bootstrap servers must be host:port values without a URL scheme");
|
expect(parseKafkaBootstrapServers("PLAINTEXT://broker1:9092, broker2:9092")).toEqual({
|
||||||
|
bootstrapServers: "broker1:9092,broker2:9092",
|
||||||
|
inferredSecurityProtocol: "PLAINTEXT",
|
||||||
|
});
|
||||||
|
expect(parseKafkaBootstrapServers("SASL_SSL://secure-broker:9093")).toEqual({
|
||||||
|
bootstrapServers: "secure-broker:9093",
|
||||||
|
inferredSecurityProtocol: "SASL_SSL",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects bootstrap servers that declare conflicting security protocols", () => {
|
||||||
|
expect(() => normalizeKafkaBootstrapServers("SSL://broker1:9093,PLAINTEXT://broker2:9092")).toThrow("Kafka bootstrap servers must use one security protocol");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects unknown listener URI schemes", () => {
|
||||||
|
expect(() => normalizeKafkaBootstrapServers("INTERNAL://broker1:9092")).toThrow("Kafka bootstrap server protocol is invalid");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses an inferred protocol only when security remains automatic", () => {
|
||||||
|
expect(resolveKafkaSecurityProtocol("", "SASL_SSL")).toBe("SASL_SSL");
|
||||||
|
expect(resolveKafkaSecurityProtocol("SSL", "SASL_SSL")).toBe("SSL");
|
||||||
|
expect(resolveKafkaSecurityProtocol("", undefined)).toBe("");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("rejects invalid bootstrap server values", () => {
|
it("rejects invalid bootstrap server values", () => {
|
||||||
expect(() => normalizeKafkaBootstrapServers("broker1:9092/path,broker2:9092")).toThrow("Kafka bootstrap servers are invalid");
|
expect(() => normalizeKafkaBootstrapServers("broker1:9092/path,broker2:9092")).toThrow("Kafka bootstrap servers are invalid");
|
||||||
|
expect(() => normalizeKafkaBootstrapServers("broker1")).toThrow("Kafka bootstrap servers must be host:port values");
|
||||||
|
expect(() => normalizeKafkaBootstrapServers("broker1:70000")).toThrow("Kafka bootstrap servers are invalid");
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@@ -0,0 +1,76 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { buildMqKafkaConnectionExtra, mqKafkaConnectionTarget, normalizeMqKafkaZooKeeperServers, resolveMqKafkaConnectionSource } from "@/lib/connection/mqKafkaConnection";
|
||||||
|
|
||||||
|
describe("MQ Kafka connection", () => {
|
||||||
|
it("keeps existing configurations on the Bootstrap source", () => {
|
||||||
|
expect(resolveMqKafkaConnectionSource({ bootstrapServers: "broker-1:9092" })).toBe("bootstrap");
|
||||||
|
expect(resolveMqKafkaConnectionSource({})).toBe("bootstrap");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("recognizes explicit and legacy ZooKeeper discovery configurations", () => {
|
||||||
|
expect(resolveMqKafkaConnectionSource({ connectionSource: "zookeeper", zookeeperServers: "zk-1:2181" })).toBe("zookeeper");
|
||||||
|
expect(resolveMqKafkaConnectionSource({ zookeeperServers: "zk-legacy:2181" })).toBe("zookeeper");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("builds Bootstrap extra fields from listener URIs without persisting the scheme", () => {
|
||||||
|
expect(
|
||||||
|
buildMqKafkaConnectionExtra({
|
||||||
|
connectionSource: "bootstrap",
|
||||||
|
bootstrapServers: "SASL_SSL://broker-1:9093, broker-2:9093",
|
||||||
|
zookeeperServers: "ignored:2181",
|
||||||
|
securityProtocol: "",
|
||||||
|
}),
|
||||||
|
).toEqual({ bootstrapServers: "broker-1:9093,broker-2:9093", securityProtocol: "SASL_SSL" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps an explicit security protocol ahead of the listener URI hint", () => {
|
||||||
|
expect(
|
||||||
|
buildMqKafkaConnectionExtra({
|
||||||
|
connectionSource: "bootstrap",
|
||||||
|
bootstrapServers: "PLAINTEXT://broker-1:9092",
|
||||||
|
zookeeperServers: "",
|
||||||
|
securityProtocol: "SSL",
|
||||||
|
}),
|
||||||
|
).toEqual({ bootstrapServers: "broker-1:9092", securityProtocol: "SSL" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("builds ZooKeeper discovery fields without a fake Bootstrap address", () => {
|
||||||
|
expect(
|
||||||
|
buildMqKafkaConnectionExtra({
|
||||||
|
connectionSource: "zookeeper",
|
||||||
|
bootstrapServers: "ignored:9092",
|
||||||
|
zookeeperServers: "zookeeper://zk-1:2181; zk-2:2181/kafka",
|
||||||
|
securityProtocol: "PLAINTEXT",
|
||||||
|
}),
|
||||||
|
).toEqual({
|
||||||
|
connectionSource: "zookeeper",
|
||||||
|
zookeeperServers: "zk-1:2181,zk-2:2181/kafka",
|
||||||
|
securityProtocol: "PLAINTEXT",
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("rejects malformed ZooKeeper discovery addresses", () => {
|
||||||
|
expect(() => normalizeMqKafkaZooKeeperServers("zk-1:not-a-port/kafka")).toThrow("Kafka ZooKeeper servers are invalid");
|
||||||
|
expect(() => normalizeMqKafkaZooKeeperServers("zk-1:70000/kafka")).toThrow("Kafka ZooKeeper servers are invalid");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("maps the active source to the generic connection target", () => {
|
||||||
|
expect(
|
||||||
|
mqKafkaConnectionTarget({
|
||||||
|
connectionSource: "bootstrap",
|
||||||
|
bootstrapServers: "SSL://broker-1:9093,broker-2:9093",
|
||||||
|
zookeeperServers: "",
|
||||||
|
securityProtocol: "",
|
||||||
|
}),
|
||||||
|
).toEqual({ host: "broker-1", port: 9093, ssl: true });
|
||||||
|
|
||||||
|
expect(
|
||||||
|
mqKafkaConnectionTarget({
|
||||||
|
connectionSource: "zookeeper",
|
||||||
|
bootstrapServers: "",
|
||||||
|
zookeeperServers: "zk-1:2281,zk-2:2181/kafka",
|
||||||
|
securityProtocol: "SASL_SSL",
|
||||||
|
}),
|
||||||
|
).toEqual({ host: "zk-1", port: 2281, ssl: false });
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1,4 +1,13 @@
|
|||||||
const KAFKA_BOOTSTRAP_SERVER_SEPARATOR = /[\s,;,;]+/u;
|
const KAFKA_BOOTSTRAP_SERVER_SEPARATOR = /[\s,;,;]+/u;
|
||||||
|
const KAFKA_BOOTSTRAP_SERVER_SCHEME = /^([a-z][a-z0-9_-]*):\/\/(.+)$/iu;
|
||||||
|
const KAFKA_SECURITY_PROTOCOLS = new Set(["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"] as const);
|
||||||
|
|
||||||
|
export type KafkaSecurityProtocol = "PLAINTEXT" | "SSL" | "SASL_PLAINTEXT" | "SASL_SSL";
|
||||||
|
|
||||||
|
export interface ParsedKafkaBootstrapServers {
|
||||||
|
bootstrapServers: string;
|
||||||
|
inferredSecurityProtocol?: KafkaSecurityProtocol;
|
||||||
|
}
|
||||||
|
|
||||||
function requireKafkaBootstrapServers(value: string): string {
|
function requireKafkaBootstrapServers(value: string): string {
|
||||||
const trimmed = value.trim();
|
const trimmed = value.trim();
|
||||||
@@ -6,28 +15,60 @@ function requireKafkaBootstrapServers(value: string): string {
|
|||||||
return trimmed;
|
return trimmed;
|
||||||
}
|
}
|
||||||
|
|
||||||
function normalizeKafkaBootstrapServer(server: string): string {
|
function normalizeKafkaBootstrapServer(server: string): { address: string; securityProtocol?: KafkaSecurityProtocol } {
|
||||||
if (server.includes("://")) {
|
const schemeMatch = server.match(KAFKA_BOOTSTRAP_SERVER_SCHEME);
|
||||||
throw new Error("Kafka bootstrap servers must be host:port values without a URL scheme");
|
let address = server;
|
||||||
|
let securityProtocol: KafkaSecurityProtocol | undefined;
|
||||||
|
if (schemeMatch) {
|
||||||
|
const protocol = schemeMatch[1].toUpperCase();
|
||||||
|
if (!KAFKA_SECURITY_PROTOCOLS.has(protocol as KafkaSecurityProtocol)) {
|
||||||
|
throw new Error("Kafka bootstrap server protocol is invalid");
|
||||||
|
}
|
||||||
|
securityProtocol = protocol as KafkaSecurityProtocol;
|
||||||
|
address = schemeMatch[2];
|
||||||
|
} else if (server.includes("://")) {
|
||||||
|
throw new Error("Kafka bootstrap server protocol is invalid");
|
||||||
}
|
}
|
||||||
|
|
||||||
let parsed: URL;
|
let parsed: URL;
|
||||||
try {
|
try {
|
||||||
parsed = new URL(`kafka://${server}`);
|
parsed = new URL(`kafka://${address}`);
|
||||||
} catch {
|
} catch {
|
||||||
throw new Error("Kafka bootstrap servers are invalid");
|
throw new Error("Kafka bootstrap servers are invalid");
|
||||||
}
|
}
|
||||||
if (!parsed.hostname || parsed.username || parsed.password || parsed.search || parsed.hash || (parsed.pathname && parsed.pathname !== "/")) {
|
if (!parsed.hostname || parsed.username || parsed.password || parsed.search || parsed.hash || (parsed.pathname && parsed.pathname !== "/")) {
|
||||||
throw new Error("Kafka bootstrap servers are invalid");
|
throw new Error("Kafka bootstrap servers are invalid");
|
||||||
}
|
}
|
||||||
return server;
|
if (!parsed.port) {
|
||||||
|
throw new Error("Kafka bootstrap servers must be host:port values");
|
||||||
|
}
|
||||||
|
return { address, securityProtocol };
|
||||||
}
|
}
|
||||||
|
|
||||||
export function normalizeKafkaBootstrapServers(value: string): string {
|
export function parseKafkaBootstrapServers(value: string): ParsedKafkaBootstrapServers {
|
||||||
const servers = requireKafkaBootstrapServers(value)
|
const parsedServers = requireKafkaBootstrapServers(value)
|
||||||
.split(KAFKA_BOOTSTRAP_SERVER_SEPARATOR)
|
.split(KAFKA_BOOTSTRAP_SERVER_SEPARATOR)
|
||||||
.map((server) => server.trim())
|
.map((server) => server.trim())
|
||||||
.filter(Boolean)
|
.filter(Boolean)
|
||||||
.map(normalizeKafkaBootstrapServer);
|
.map(normalizeKafkaBootstrapServer);
|
||||||
if (!servers.length) throw new Error("Kafka bootstrap servers are required");
|
if (!parsedServers.length) throw new Error("Kafka bootstrap servers are required");
|
||||||
return servers.join(",");
|
|
||||||
|
const protocols = new Set(parsedServers.map((server) => server.securityProtocol).filter((protocol): protocol is KafkaSecurityProtocol => !!protocol));
|
||||||
|
if (protocols.size > 1) {
|
||||||
|
throw new Error("Kafka bootstrap servers must use one security protocol");
|
||||||
|
}
|
||||||
|
|
||||||
|
const inferredSecurityProtocol = protocols.values().next().value;
|
||||||
|
return {
|
||||||
|
bootstrapServers: parsedServers.map((server) => server.address).join(","),
|
||||||
|
...(inferredSecurityProtocol ? { inferredSecurityProtocol } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeKafkaBootstrapServers(value: string): string {
|
||||||
|
return parseKafkaBootstrapServers(value).bootstrapServers;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveKafkaSecurityProtocol(configured: string, inferred?: KafkaSecurityProtocol): string {
|
||||||
|
return configured.trim() || inferred || "";
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
import { parseKafkaBootstrapServers, resolveKafkaSecurityProtocol } from "@/lib/connection/kafkaBootstrapServers";
|
||||||
|
import { firstZooKeeperEndpoint, normalizeZooKeeperConnectString } from "@/lib/zookeeper/zookeeperConnection";
|
||||||
|
|
||||||
|
export type MqKafkaConnectionSource = "bootstrap" | "zookeeper";
|
||||||
|
|
||||||
|
export interface MqKafkaConnectionInput {
|
||||||
|
connectionSource: MqKafkaConnectionSource;
|
||||||
|
bootstrapServers: string;
|
||||||
|
zookeeperServers: string;
|
||||||
|
securityProtocol?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MqKafkaConnectionTarget {
|
||||||
|
host: string;
|
||||||
|
port: number;
|
||||||
|
ssl: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveMqKafkaConnectionSource(extra: Record<string, unknown>): MqKafkaConnectionSource {
|
||||||
|
if (extra.connectionSource === "zookeeper") return "zookeeper";
|
||||||
|
if (typeof extra.zookeeperServers === "string" && extra.zookeeperServers.trim() && !(typeof extra.bootstrapServers === "string" && extra.bootstrapServers.trim())) {
|
||||||
|
return "zookeeper";
|
||||||
|
}
|
||||||
|
return "bootstrap";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeMqKafkaZooKeeperServers(value: string): string {
|
||||||
|
const normalized = normalizeZooKeeperConnectString(value.trim());
|
||||||
|
if (!normalized) throw new Error("Kafka ZooKeeper servers are required");
|
||||||
|
|
||||||
|
const chrootIndex = normalized.indexOf("/");
|
||||||
|
const ensemble = chrootIndex >= 0 ? normalized.slice(0, chrootIndex) : normalized;
|
||||||
|
const chroot = chrootIndex >= 0 ? normalized.slice(chrootIndex) : "";
|
||||||
|
if (chroot && (!chroot.startsWith("/") || chroot.includes("//"))) {
|
||||||
|
throw new Error("Kafka ZooKeeper servers are invalid");
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const endpoint of ensemble.split(",")) {
|
||||||
|
const match = endpoint.match(/^(?:\[[^\]]+\]|[^:\s/?#]+):(\d+)$/u);
|
||||||
|
const port = match ? Number(match[1]) : Number.NaN;
|
||||||
|
if (!match || !Number.isInteger(port) || port < 1 || port > 65535) {
|
||||||
|
throw new Error("Kafka ZooKeeper servers are invalid");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildMqKafkaConnectionExtra(input: MqKafkaConnectionInput): Record<string, string> {
|
||||||
|
let extra: Record<string, string>;
|
||||||
|
let securityProtocol = input.securityProtocol?.trim() || "";
|
||||||
|
|
||||||
|
if (input.connectionSource === "zookeeper") {
|
||||||
|
extra = {
|
||||||
|
connectionSource: "zookeeper",
|
||||||
|
zookeeperServers: normalizeMqKafkaZooKeeperServers(input.zookeeperServers),
|
||||||
|
};
|
||||||
|
} else {
|
||||||
|
const parsed = parseKafkaBootstrapServers(input.bootstrapServers);
|
||||||
|
securityProtocol = resolveKafkaSecurityProtocol(securityProtocol, parsed.inferredSecurityProtocol);
|
||||||
|
extra = { bootstrapServers: parsed.bootstrapServers };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (securityProtocol) extra.securityProtocol = securityProtocol;
|
||||||
|
return extra;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function mqKafkaConnectionTarget(input: MqKafkaConnectionInput): MqKafkaConnectionTarget {
|
||||||
|
if (input.connectionSource === "zookeeper") {
|
||||||
|
const endpoint = firstZooKeeperEndpoint(normalizeMqKafkaZooKeeperServers(input.zookeeperServers));
|
||||||
|
if (!endpoint) throw new Error("Kafka ZooKeeper servers are required");
|
||||||
|
return { ...endpoint, ssl: false };
|
||||||
|
}
|
||||||
|
|
||||||
|
const parsed = parseKafkaBootstrapServers(input.bootstrapServers);
|
||||||
|
const first = parsed.bootstrapServers.split(",")[0];
|
||||||
|
const endpoint = new URL(`kafka://${first}`);
|
||||||
|
const securityProtocol = resolveKafkaSecurityProtocol(input.securityProtocol || "", parsed.inferredSecurityProtocol);
|
||||||
|
return {
|
||||||
|
host: endpoint.hostname,
|
||||||
|
port: Number(endpoint.port),
|
||||||
|
ssl: securityProtocol === "SSL" || securityProtocol === "SASL_SSL",
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -17,6 +17,10 @@ fn default_jre_key() -> String {
|
|||||||
DEFAULT_JRE_KEY.to_string()
|
DEFAULT_JRE_KEY.to_string()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn strip_utf8_bom(value: &str) -> &str {
|
||||||
|
value.strip_prefix('\u{feff}').unwrap_or(value)
|
||||||
|
}
|
||||||
|
|
||||||
fn is_valid_jar_file(path: &Path) -> bool {
|
fn is_valid_jar_file(path: &Path) -> bool {
|
||||||
if !path.is_file() {
|
if !path.is_file() {
|
||||||
return false;
|
return false;
|
||||||
@@ -138,6 +142,21 @@ mod tests {
|
|||||||
assert!(state.pending_jre_cleanup.is_empty());
|
assert!(state.pending_jre_cleanup.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn loads_agent_state_with_utf8_bom() {
|
||||||
|
let manager = test_manager("state-utf8-bom");
|
||||||
|
fs::create_dir_all(manager.base_dir()).unwrap();
|
||||||
|
fs::write(
|
||||||
|
manager.state_path(),
|
||||||
|
b"\xEF\xBB\xBF{\"installed_drivers\":{\"kafka\":{\"version\":\"0.1.4\",\"installed_at\":\"now\",\"jre\":\"21\"}}}",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let state = manager.load_state();
|
||||||
|
|
||||||
|
assert_eq!(state.installed_drivers.get("kafka").map(|driver| driver.version.as_str()), Some("0.1.4"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn resolves_managed_java_runtime_by_default() {
|
fn resolves_managed_java_runtime_by_default() {
|
||||||
let manager = test_manager("managed");
|
let manager = test_manager("managed");
|
||||||
@@ -279,6 +298,25 @@ mod tests {
|
|||||||
);
|
);
|
||||||
assert_eq!(launch.working_dir.as_deref(), Some(driver_dir.as_path()));
|
assert_eq!(launch.working_dir.as_deref(), Some(driver_dir.as_path()));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn resolves_manifest_agent_launch_with_utf8_bom() {
|
||||||
|
let manager = test_manager("manifest-agent-utf8-bom");
|
||||||
|
let driver_dir = manager.driver_dir("kafka");
|
||||||
|
fs::create_dir_all(&driver_dir).unwrap();
|
||||||
|
fs::write(
|
||||||
|
manager.driver_launch_config_path("kafka"),
|
||||||
|
b"\xEF\xBB\xBF{\"command\":\"java\",\"args\":[\"-jar\",\"{driver_dir}/agent.jar\"]}",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let launch = manager
|
||||||
|
.resolve_agent_launch_spec(&AgentState::default(), "kafka", DEFAULT_JRE_KEY)
|
||||||
|
.expect("manifest launch with UTF-8 BOM should resolve");
|
||||||
|
|
||||||
|
assert_eq!(launch.program, PathBuf::from("java"));
|
||||||
|
assert_eq!(launch.args, vec!["-jar".to_string(), format!("{}/agent.jar", driver_dir.to_string_lossy())]);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
@@ -550,7 +588,10 @@ impl AgentManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn load_state(&self) -> AgentState {
|
pub fn load_state(&self) -> AgentState {
|
||||||
std::fs::read_to_string(self.state_path()).ok().and_then(|s| serde_json::from_str(&s).ok()).unwrap_or_default()
|
std::fs::read_to_string(self.state_path())
|
||||||
|
.ok()
|
||||||
|
.and_then(|s| serde_json::from_str(strip_utf8_bom(&s)).ok())
|
||||||
|
.unwrap_or_default()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn save_state(&self, state: &AgentState) -> Result<(), String> {
|
pub fn save_state(&self, state: &AgentState) -> Result<(), String> {
|
||||||
@@ -629,7 +670,7 @@ impl AgentManager {
|
|||||||
) -> Result<AgentLaunchSpec, String> {
|
) -> Result<AgentLaunchSpec, String> {
|
||||||
let json = std::fs::read_to_string(config_path)
|
let json = std::fs::read_to_string(config_path)
|
||||||
.map_err(|e| format!("Failed to read {driver_key} agent launch config: {e}"))?;
|
.map_err(|e| format!("Failed to read {driver_key} agent launch config: {e}"))?;
|
||||||
let config: AgentLaunchConfig = serde_json::from_str(&json)
|
let config: AgentLaunchConfig = serde_json::from_str(strip_utf8_bom(&json))
|
||||||
.map_err(|e| format!("Failed to parse {driver_key} agent launch config: {e}"))?;
|
.map_err(|e| format!("Failed to parse {driver_key} agent launch config: {e}"))?;
|
||||||
let command = config.command.trim();
|
let command = config.command.trim();
|
||||||
if command.is_empty() {
|
if command.is_empty() {
|
||||||
|
|||||||
@@ -668,11 +668,13 @@ fn build_connection_params(cfg: &MqAdminConfig) -> serde_json::Value {
|
|||||||
} else {
|
} else {
|
||||||
"PLAINTEXT"
|
"PLAINTEXT"
|
||||||
});
|
});
|
||||||
|
let zookeeper_connect_string = extra_str(extra, "zookeeperServers").unwrap_or("");
|
||||||
let properties =
|
let properties =
|
||||||
extra.get("properties").filter(|value| value.is_object()).cloned().unwrap_or_else(|| serde_json::json!({}));
|
extra.get("properties").filter(|value| value.is_object()).cloned().unwrap_or_else(|| serde_json::json!({}));
|
||||||
|
|
||||||
serde_json::json!({
|
serde_json::json!({
|
||||||
"bootstrap_servers": bootstrap_servers(cfg),
|
"bootstrap_servers": bootstrap_servers(cfg),
|
||||||
|
"zookeeper_connect_string": zookeeper_connect_string,
|
||||||
"security_protocol": security_protocol,
|
"security_protocol": security_protocol,
|
||||||
"sasl_mechanism": sasl_mechanism,
|
"sasl_mechanism": sasl_mechanism,
|
||||||
"sasl_username": sasl_username,
|
"sasl_username": sasl_username,
|
||||||
@@ -838,6 +840,63 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn connection_params_pass_zookeeper_discovery_without_fake_bootstrap_servers() {
|
||||||
|
let cfg = kafka_config(
|
||||||
|
serde_json::json!({
|
||||||
|
"connectionSource": "zookeeper",
|
||||||
|
"zookeeperServers": "zk-1:2181,zk-2:2181/kafka",
|
||||||
|
"securityProtocol": "PLAINTEXT"
|
||||||
|
}),
|
||||||
|
MqAuth::None,
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
|
||||||
|
let params = build_connection_params(&cfg);
|
||||||
|
|
||||||
|
assert_eq!(params.get("bootstrap_servers").and_then(|v| v.as_str()), Some(""));
|
||||||
|
assert_eq!(params.get("zookeeper_connect_string").and_then(|v| v.as_str()), Some("zk-1:2181,zk-2:2181/kafka"));
|
||||||
|
assert_eq!(params.get("security_protocol").and_then(|v| v.as_str()), Some("PLAINTEXT"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn connection_params_preserve_zookeeper_sasl_and_tls_properties() {
|
||||||
|
let cfg = kafka_config(
|
||||||
|
serde_json::json!({
|
||||||
|
"connectionSource": "zookeeper",
|
||||||
|
"zookeeperServers": "zk-secure:2281/kafka",
|
||||||
|
"securityProtocol": "SASL_SSL",
|
||||||
|
"properties": {
|
||||||
|
"zookeeper.sasl.client": "true",
|
||||||
|
"zookeeper.sasl.clientconfig": "DbxZooKeeperClient",
|
||||||
|
"zookeeper.client.secure": "true",
|
||||||
|
"zookeeper.clientCnxnSocket": "org.apache.zookeeper.ClientCnxnSocketNetty",
|
||||||
|
"zookeeper.ssl.trustStore.location": "/etc/dbx/zookeeper-truststore.p12"
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
MqAuth::None,
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
|
||||||
|
let params = build_connection_params(&cfg);
|
||||||
|
|
||||||
|
assert_eq!(params.get("zookeeper_connect_string").and_then(|v| v.as_str()), Some("zk-secure:2281/kafka"));
|
||||||
|
assert_eq!(params.pointer("/properties/zookeeper.sasl.client").and_then(|v| v.as_str()), Some("true"));
|
||||||
|
assert_eq!(
|
||||||
|
params.pointer("/properties/zookeeper.sasl.clientconfig").and_then(|v| v.as_str()),
|
||||||
|
Some("DbxZooKeeperClient")
|
||||||
|
);
|
||||||
|
assert_eq!(params.pointer("/properties/zookeeper.client.secure").and_then(|v| v.as_str()), Some("true"));
|
||||||
|
assert_eq!(
|
||||||
|
params.pointer("/properties/zookeeper.clientCnxnSocket").and_then(|v| v.as_str()),
|
||||||
|
Some("org.apache.zookeeper.ClientCnxnSocketNetty")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
params.pointer("/properties/zookeeper.ssl.trustStore.location").and_then(|v| v.as_str()),
|
||||||
|
Some("/etc/dbx/zookeeper-truststore.p12")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn reset_cursor_params_preserve_timestamp_position() {
|
fn reset_cursor_params_preserve_timestamp_position() {
|
||||||
let topic = TopicRef {
|
let topic = TopicRef {
|
||||||
|
|||||||
@@ -1541,6 +1541,7 @@ mod tests {
|
|||||||
sessions: RwLock::new(HashSet::new()),
|
sessions: RwLock::new(HashSet::new()),
|
||||||
sse_channels: RwLock::new(HashMap::new()),
|
sse_channels: RwLock::new(HashMap::new()),
|
||||||
sql_file_executions: RwLock::new(HashMap::new()),
|
sql_file_executions: RwLock::new(HashMap::new()),
|
||||||
|
table_import_channels: RwLock::new(HashMap::new()),
|
||||||
login_rate_limit: Mutex::new(LoginRateLimit { fail_count: 0, locked_until: None }),
|
login_rate_limit: Mutex::new(LoginRateLimit { fail_count: 0, locked_until: None }),
|
||||||
export_files: RwLock::new(HashMap::new()),
|
export_files: RwLock::new(HashMap::new()),
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in new issue
Block a user