feat(kafka): improve topic management and Kerberos auth

This commit is contained in:
zipg authored and GitHub committed 2026-07-09 11:41:22 +08:00
1 parent b31d52c00c
commit 1317301bcd
18 files changed
+1096 -81

No files matched your search

@@ -32,6 +32,13 @@ public final class KafkaAgent {
private static final Gson GSON = new GsonBuilder().serializeNulls().create();
private static final int DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
private static final int DEFAULT_SESSION_TIMEOUT_MS = 30_000;
private static final Set<String> KERBEROS_SYSTEM_PROPERTY_KEYS = Set.of(
"java.security.krb5.conf",
"sun.security.krb5.debug",
"javax.security.auth.useSubjectCredsOnly"
);
private static final Map<String, String> BASELINE_KERBEROS_SYSTEM_PROPERTIES =
snapshotKerberosSystemProperties();
private static final List<String> CAPABILITIES = Collections.unmodifiableList(Arrays.asList(
"mq_connect", "mq_test_connection", "mq_topics", "mq_consumer_groups",
@@ -138,30 +145,38 @@ public final class KafkaAgent {
private static Object connect(JsonObject params) throws Exception {
JsonObject conn = connectionObject(params);
AdminClient nextAdmin = buildAdminClient(conn);
Map<String, String> previousKerberosSystemProperties = applyKerberosSystemProperties(conn);
AdminClient nextAdmin = null;
KafkaProducer<String, byte[]> nextProducer = null;
try {
nextAdmin = buildAdminClient(conn);
// Verify connectivity
nextAdmin.describeCluster().clusterId().get(
intOrDefault(conn, "request_timeout_ms", DEFAULT_REQUEST_TIMEOUT_MS), TimeUnit.MILLISECONDS);
nextProducer = buildProducer(conn);
closeClients();
applyKerberosSystemProperties(conn);
adminClient = nextAdmin;
producer = nextProducer;
return Collections.singletonMap("ok", true);
} catch (Exception e) {
nextAdmin.close(Duration.ofSeconds(5));
if (nextAdmin != null) {
nextAdmin.close(Duration.ofSeconds(5));
}
if (nextProducer != null) {
nextProducer.close(Duration.ofSeconds(5));
}
restoreKerberosSystemProperties(previousKerberosSystemProperties);
throw e;
}
}
private static Object testConnection(JsonObject params) throws Exception {
JsonObject conn = connectionObject(params);
AdminClient probe = buildAdminClient(conn);
Map<String, String> previousKerberosSystemProperties = applyKerberosSystemProperties(conn);
AdminClient probe = null;
try {
probe = buildAdminClient(conn);
int timeout = intOrDefault(conn, "request_timeout_ms", DEFAULT_REQUEST_TIMEOUT_MS);
DescribeClusterResult cluster = probe.describeCluster();
String clusterId = cluster.clusterId().get(timeout, TimeUnit.MILLISECONDS);
@@ -197,7 +212,10 @@ public final class KafkaAgent {
result.put("brokers", brokerList);
return result;
} finally {
probe.close(Duration.ofSeconds(5));
if (probe != null) {
probe.close(Duration.ofSeconds(5));
}
restoreKerberosSystemProperties(previousKerberosSystemProperties);
}
}
@@ -210,6 +228,7 @@ public final class KafkaAgent {
producer.close(Duration.ofSeconds(5));
producer = null;
}
restoreKerberosSystemProperties(BASELINE_KERBEROS_SYSTEM_PROPERTIES);
}
// -----------------------------------------------------------------------
@@ -333,12 +352,61 @@ public final class KafkaAgent {
if (properties != null) {
for (Map.Entry<String, JsonElement> entry : properties.entrySet()) {
if (entry.getValue().isJsonPrimitive()) {
props.put(entry.getKey(), entry.getValue().getAsString());
String key = entry.getKey();
String value = entry.getValue().getAsString();
props.put(key, value);
}
}
}
}
static Map<String, String> applyKerberosSystemProperties(JsonObject conn) {
Map<String, String> previous = snapshotKerberosSystemProperties();
JsonObject properties = connectionProperties(conn);
for (String key : KERBEROS_SYSTEM_PROPERTY_KEYS) {
String value = stringProperty(properties, key);
if (value == null || value.isBlank()) {
value = BASELINE_KERBEROS_SYSTEM_PROPERTIES.get(key);
}
setOrClearSystemProperty(key, value);
}
return previous;
}
static void restoreKerberosSystemProperties(Map<String, String> values) {
for (String key : KERBEROS_SYSTEM_PROPERTY_KEYS) {
setOrClearSystemProperty(key, values.get(key));
}
}
private static Map<String, String> snapshotKerberosSystemProperties() {
Map<String, String> values = new LinkedHashMap<>();
for (String key : KERBEROS_SYSTEM_PROPERTY_KEYS) {
values.put(key, System.getProperty(key));
}
return values;
}
private static JsonObject connectionProperties(JsonObject conn) {
return conn.has("properties") && conn.get("properties").isJsonObject()
? conn.getAsJsonObject("properties") : null;
}
private static String stringProperty(JsonObject properties, String key) {
if (properties == null || !properties.has(key) || !properties.get(key).isJsonPrimitive()) {
return null;
}
return properties.get(key).getAsString();
}
private static void setOrClearSystemProperty(String key, String value) {
if (value == null || value.isBlank()) {
System.clearProperty(key);
} else {
System.setProperty(key, value);
}
}
// -----------------------------------------------------------------------
// Topic management
// -----------------------------------------------------------------------
@@ -3,6 +3,9 @@ package com.dbx.agent.kafka;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import com.google.gson.JsonParser;
import java.util.Map;
import java.util.Properties;
import org.junit.jupiter.api.Test;
class KafkaAgentTest {
@@ -30,4 +33,109 @@ class KafkaAgentTest {
void returnsNoSeekOffsetWhenTopicHasNoReadableMessages() {
assertNull(KafkaAgent.normalizePeekOffset(0, 5, 5));
}
@Test
void appliesKerberosKafkaProperties() {
Properties props = new Properties();
KafkaAgent.applyConnectionProperties(JsonParser.parseString("""
{
"security_protocol": "SASL_SSL",
"sasl_mechanism": "GSSAPI",
"properties": {
"sasl.jaas.config": "com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true keyTab=\\"/tmp/user.keytab\\" principal=\\"user@EXAMPLE.COM\\";",
"sasl.kerberos.service.name": "kafka"
}
}
""").getAsJsonObject(), props);
assertEquals("SASL_SSL", props.getProperty("security.protocol"));
assertEquals("GSSAPI", props.getProperty("sasl.mechanism"));
assertEquals("kafka", props.getProperty("sasl.kerberos.service.name"));
assertEquals(
"com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true keyTab=\"/tmp/user.keytab\" principal=\"user@EXAMPLE.COM\";",
props.getProperty("sasl.jaas.config")
);
}
@Test
void appliesAllowedKerberosSystemPropertiesFromConnectionProperties() {
Map<String, String> previous = KafkaAgent.applyKerberosSystemProperties(JsonParser.parseString("""
{
"properties": {
"java.security.krb5.conf": "/tmp/krb5.conf",
"sun.security.krb5.debug": "true",
"custom.system.property": "should-not-leak"
}
}
""").getAsJsonObject());
try {
assertEquals("/tmp/krb5.conf", System.getProperty("java.security.krb5.conf"));
assertEquals("true", System.getProperty("sun.security.krb5.debug"));
assertNull(System.getProperty("custom.system.property"));
} finally {
KafkaAgent.restoreKerberosSystemProperties(previous);
}
}
@Test
void clearsPreviousKerberosSystemPropertiesForNextConnection() {
String baseline = System.getProperty("java.security.krb5.conf");
Map<String, String> previous = KafkaAgent.applyKerberosSystemProperties(JsonParser.parseString("""
{
"properties": {
"java.security.krb5.conf": "/tmp/cluster-a.krb5.conf"
}
}
""").getAsJsonObject());
try {
assertEquals("/tmp/cluster-a.krb5.conf", System.getProperty("java.security.krb5.conf"));
Map<String, String> beforeSecondConnection = KafkaAgent.applyKerberosSystemProperties(JsonParser.parseString("""
{
"properties": {
"sasl.kerberos.service.name": "kafka"
}
}
""").getAsJsonObject());
try {
assertEquals(baseline, System.getProperty("java.security.krb5.conf"));
} finally {
KafkaAgent.restoreKerberosSystemProperties(beforeSecondConnection);
}
} finally {
KafkaAgent.restoreKerberosSystemProperties(previous);
}
}
@Test
void restoresKerberosSystemPropertiesWhenTestConnectionClientConstructionFails() {
String previous = System.getProperty("java.security.krb5.conf");
try {
String response = KafkaAgent.handleRequest("""
{
"jsonrpc": "2.0",
"id": 42,
"method": "test_connection",
"params": {
"connection": {
"bootstrap_servers": "",
"properties": {
"java.security.krb5.conf": "/tmp/leaked-test-connection.krb5.conf"
}
}
}
}
""");
assertEquals(-1, JsonParser.parseString(response).getAsJsonObject()
.getAsJsonObject("error").get("code").getAsInt());
assertEquals(previous, System.getProperty("java.security.krb5.conf"));
} finally {
if (previous == null) {
System.clearProperty("java.security.krb5.conf");
} else {
System.setProperty("java.security.krb5.conf", previous);
}
}
}
}