feat(kafka): improve topic management and Kerberos auth
This commit is contained in:
18 files changed
+1096
-81
No files matched your search
@@ -32,6 +32,13 @@ public final class KafkaAgent {
|
||||
private static final Gson GSON = new GsonBuilder().serializeNulls().create();
|
||||
private static final int DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
|
||||
private static final int DEFAULT_SESSION_TIMEOUT_MS = 30_000;
|
||||
private static final Set<String> KERBEROS_SYSTEM_PROPERTY_KEYS = Set.of(
|
||||
"java.security.krb5.conf",
|
||||
"sun.security.krb5.debug",
|
||||
"javax.security.auth.useSubjectCredsOnly"
|
||||
);
|
||||
private static final Map<String, String> BASELINE_KERBEROS_SYSTEM_PROPERTIES =
|
||||
snapshotKerberosSystemProperties();
|
||||
|
||||
private static final List<String> CAPABILITIES = Collections.unmodifiableList(Arrays.asList(
|
||||
"mq_connect", "mq_test_connection", "mq_topics", "mq_consumer_groups",
|
||||
@@ -138,30 +145,38 @@ public final class KafkaAgent {
|
||||
|
||||
private static Object connect(JsonObject params) throws Exception {
|
||||
JsonObject conn = connectionObject(params);
|
||||
AdminClient nextAdmin = buildAdminClient(conn);
|
||||
Map<String, String> previousKerberosSystemProperties = applyKerberosSystemProperties(conn);
|
||||
AdminClient nextAdmin = null;
|
||||
KafkaProducer<String, byte[]> nextProducer = null;
|
||||
try {
|
||||
nextAdmin = buildAdminClient(conn);
|
||||
// Verify connectivity
|
||||
nextAdmin.describeCluster().clusterId().get(
|
||||
intOrDefault(conn, "request_timeout_ms", DEFAULT_REQUEST_TIMEOUT_MS), TimeUnit.MILLISECONDS);
|
||||
nextProducer = buildProducer(conn);
|
||||
closeClients();
|
||||
applyKerberosSystemProperties(conn);
|
||||
adminClient = nextAdmin;
|
||||
producer = nextProducer;
|
||||
return Collections.singletonMap("ok", true);
|
||||
} catch (Exception e) {
|
||||
nextAdmin.close(Duration.ofSeconds(5));
|
||||
if (nextAdmin != null) {
|
||||
nextAdmin.close(Duration.ofSeconds(5));
|
||||
}
|
||||
if (nextProducer != null) {
|
||||
nextProducer.close(Duration.ofSeconds(5));
|
||||
}
|
||||
restoreKerberosSystemProperties(previousKerberosSystemProperties);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
private static Object testConnection(JsonObject params) throws Exception {
|
||||
JsonObject conn = connectionObject(params);
|
||||
AdminClient probe = buildAdminClient(conn);
|
||||
Map<String, String> previousKerberosSystemProperties = applyKerberosSystemProperties(conn);
|
||||
AdminClient probe = null;
|
||||
try {
|
||||
probe = buildAdminClient(conn);
|
||||
int timeout = intOrDefault(conn, "request_timeout_ms", DEFAULT_REQUEST_TIMEOUT_MS);
|
||||
DescribeClusterResult cluster = probe.describeCluster();
|
||||
String clusterId = cluster.clusterId().get(timeout, TimeUnit.MILLISECONDS);
|
||||
@@ -197,7 +212,10 @@ public final class KafkaAgent {
|
||||
result.put("brokers", brokerList);
|
||||
return result;
|
||||
} finally {
|
||||
probe.close(Duration.ofSeconds(5));
|
||||
if (probe != null) {
|
||||
probe.close(Duration.ofSeconds(5));
|
||||
}
|
||||
restoreKerberosSystemProperties(previousKerberosSystemProperties);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -210,6 +228,7 @@ public final class KafkaAgent {
|
||||
producer.close(Duration.ofSeconds(5));
|
||||
producer = null;
|
||||
}
|
||||
restoreKerberosSystemProperties(BASELINE_KERBEROS_SYSTEM_PROPERTIES);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
@@ -333,12 +352,61 @@ public final class KafkaAgent {
|
||||
if (properties != null) {
|
||||
for (Map.Entry<String, JsonElement> entry : properties.entrySet()) {
|
||||
if (entry.getValue().isJsonPrimitive()) {
|
||||
props.put(entry.getKey(), entry.getValue().getAsString());
|
||||
String key = entry.getKey();
|
||||
String value = entry.getValue().getAsString();
|
||||
props.put(key, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static Map<String, String> applyKerberosSystemProperties(JsonObject conn) {
|
||||
Map<String, String> previous = snapshotKerberosSystemProperties();
|
||||
JsonObject properties = connectionProperties(conn);
|
||||
for (String key : KERBEROS_SYSTEM_PROPERTY_KEYS) {
|
||||
String value = stringProperty(properties, key);
|
||||
if (value == null || value.isBlank()) {
|
||||
value = BASELINE_KERBEROS_SYSTEM_PROPERTIES.get(key);
|
||||
}
|
||||
setOrClearSystemProperty(key, value);
|
||||
}
|
||||
return previous;
|
||||
}
|
||||
|
||||
static void restoreKerberosSystemProperties(Map<String, String> values) {
|
||||
for (String key : KERBEROS_SYSTEM_PROPERTY_KEYS) {
|
||||
setOrClearSystemProperty(key, values.get(key));
|
||||
}
|
||||
}
|
||||
|
||||
private static Map<String, String> snapshotKerberosSystemProperties() {
|
||||
Map<String, String> values = new LinkedHashMap<>();
|
||||
for (String key : KERBEROS_SYSTEM_PROPERTY_KEYS) {
|
||||
values.put(key, System.getProperty(key));
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
private static JsonObject connectionProperties(JsonObject conn) {
|
||||
return conn.has("properties") && conn.get("properties").isJsonObject()
|
||||
? conn.getAsJsonObject("properties") : null;
|
||||
}
|
||||
|
||||
private static String stringProperty(JsonObject properties, String key) {
|
||||
if (properties == null || !properties.has(key) || !properties.get(key).isJsonPrimitive()) {
|
||||
return null;
|
||||
}
|
||||
return properties.get(key).getAsString();
|
||||
}
|
||||
|
||||
private static void setOrClearSystemProperty(String key, String value) {
|
||||
if (value == null || value.isBlank()) {
|
||||
System.clearProperty(key);
|
||||
} else {
|
||||
System.setProperty(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Topic management
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
@@ -3,6 +3,9 @@ package com.dbx.agent.kafka;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
|
||||
import com.google.gson.JsonParser;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
class KafkaAgentTest {
|
||||
@@ -30,4 +33,109 @@ class KafkaAgentTest {
|
||||
void returnsNoSeekOffsetWhenTopicHasNoReadableMessages() {
|
||||
assertNull(KafkaAgent.normalizePeekOffset(0, 5, 5));
|
||||
}
|
||||
|
||||
@Test
|
||||
void appliesKerberosKafkaProperties() {
|
||||
Properties props = new Properties();
|
||||
KafkaAgent.applyConnectionProperties(JsonParser.parseString("""
|
||||
{
|
||||
"security_protocol": "SASL_SSL",
|
||||
"sasl_mechanism": "GSSAPI",
|
||||
"properties": {
|
||||
"sasl.jaas.config": "com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true keyTab=\\"/tmp/user.keytab\\" principal=\\"user@EXAMPLE.COM\\";",
|
||||
"sasl.kerberos.service.name": "kafka"
|
||||
}
|
||||
}
|
||||
""").getAsJsonObject(), props);
|
||||
|
||||
assertEquals("SASL_SSL", props.getProperty("security.protocol"));
|
||||
assertEquals("GSSAPI", props.getProperty("sasl.mechanism"));
|
||||
assertEquals("kafka", props.getProperty("sasl.kerberos.service.name"));
|
||||
assertEquals(
|
||||
"com.sun.security.auth.module.Krb5LoginModule required useKeyTab=true keyTab=\"/tmp/user.keytab\" principal=\"user@EXAMPLE.COM\";",
|
||||
props.getProperty("sasl.jaas.config")
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void appliesAllowedKerberosSystemPropertiesFromConnectionProperties() {
|
||||
Map<String, String> previous = KafkaAgent.applyKerberosSystemProperties(JsonParser.parseString("""
|
||||
{
|
||||
"properties": {
|
||||
"java.security.krb5.conf": "/tmp/krb5.conf",
|
||||
"sun.security.krb5.debug": "true",
|
||||
"custom.system.property": "should-not-leak"
|
||||
}
|
||||
}
|
||||
""").getAsJsonObject());
|
||||
try {
|
||||
assertEquals("/tmp/krb5.conf", System.getProperty("java.security.krb5.conf"));
|
||||
assertEquals("true", System.getProperty("sun.security.krb5.debug"));
|
||||
assertNull(System.getProperty("custom.system.property"));
|
||||
} finally {
|
||||
KafkaAgent.restoreKerberosSystemProperties(previous);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void clearsPreviousKerberosSystemPropertiesForNextConnection() {
|
||||
String baseline = System.getProperty("java.security.krb5.conf");
|
||||
Map<String, String> previous = KafkaAgent.applyKerberosSystemProperties(JsonParser.parseString("""
|
||||
{
|
||||
"properties": {
|
||||
"java.security.krb5.conf": "/tmp/cluster-a.krb5.conf"
|
||||
}
|
||||
}
|
||||
""").getAsJsonObject());
|
||||
try {
|
||||
assertEquals("/tmp/cluster-a.krb5.conf", System.getProperty("java.security.krb5.conf"));
|
||||
|
||||
Map<String, String> beforeSecondConnection = KafkaAgent.applyKerberosSystemProperties(JsonParser.parseString("""
|
||||
{
|
||||
"properties": {
|
||||
"sasl.kerberos.service.name": "kafka"
|
||||
}
|
||||
}
|
||||
""").getAsJsonObject());
|
||||
try {
|
||||
assertEquals(baseline, System.getProperty("java.security.krb5.conf"));
|
||||
} finally {
|
||||
KafkaAgent.restoreKerberosSystemProperties(beforeSecondConnection);
|
||||
}
|
||||
} finally {
|
||||
KafkaAgent.restoreKerberosSystemProperties(previous);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void restoresKerberosSystemPropertiesWhenTestConnectionClientConstructionFails() {
|
||||
String previous = System.getProperty("java.security.krb5.conf");
|
||||
try {
|
||||
String response = KafkaAgent.handleRequest("""
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"id": 42,
|
||||
"method": "test_connection",
|
||||
"params": {
|
||||
"connection": {
|
||||
"bootstrap_servers": "",
|
||||
"properties": {
|
||||
"java.security.krb5.conf": "/tmp/leaked-test-connection.krb5.conf"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
""");
|
||||
|
||||
assertEquals(-1, JsonParser.parseString(response).getAsJsonObject()
|
||||
.getAsJsonObject("error").get("code").getAsInt());
|
||||
assertEquals(previous, System.getProperty("java.security.krb5.conf"));
|
||||
} finally {
|
||||
if (previous == null) {
|
||||
System.clearProperty("java.security.krb5.conf");
|
||||
} else {
|
||||
System.setProperty("java.security.krb5.conf", previous);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user