From 520de302dd1136e774e7cff0657cd63f85cca58a Mon Sep 17 00:00:00 2001 From: t8y2 <1156263951@qq.com> Date: Mon, 17 Aug 2026 03:52:27 +0000 Subject: [PATCH] fix(kafka): support message sends to legacy brokers Closes #6408 --- .../java/com/dbx/agent/kafka/KafkaAgent.java | 8 +++++- .../com/dbx/agent/kafka/KafkaAgentTest.java | 25 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/agents/drivers/kafka/src/main/java/com/dbx/agent/kafka/KafkaAgent.java b/agents/drivers/kafka/src/main/java/com/dbx/agent/kafka/KafkaAgent.java index 749f52bf61..649fdc83b2 100644 --- a/agents/drivers/kafka/src/main/java/com/dbx/agent/kafka/KafkaAgent.java +++ b/agents/drivers/kafka/src/main/java/com/dbx/agent/kafka/KafkaAgent.java @@ -287,6 +287,10 @@ public final class KafkaAgent { } private static KafkaProducer buildProducer(JsonObject conn) { + return new KafkaProducer<>(producerProperties(conn)); + } + + static Properties producerProperties(JsonObject conn) { Properties props = new Properties(); props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers(conn)); props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, @@ -294,8 +298,10 @@ public final class KafkaAgent { props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.ByteArraySerializer"); props.put(ProducerConfig.ACKS_CONFIG, "all"); + // Kafka 3.x enables idempotence by default, which rejects pre-0.11 brokers. + props.put(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, "false"); applyConnectionProperties(conn, props); - return new KafkaProducer<>(props); + return props; } private static String bootstrapServers(JsonObject conn) { diff --git a/agents/drivers/kafka/src/test/java/com/dbx/agent/kafka/KafkaAgentTest.java b/agents/drivers/kafka/src/test/java/com/dbx/agent/kafka/KafkaAgentTest.java index 8af5abbd7a..1d21b3e54a 100644 --- a/agents/drivers/kafka/src/test/java/com/dbx/agent/kafka/KafkaAgentTest.java +++ b/agents/drivers/kafka/src/test/java/com/dbx/agent/kafka/KafkaAgentTest.java @@ -29,6 +29,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.admin.AlterConfigOp; import org.apache.kafka.clients.admin.Config; import org.apache.kafka.clients.admin.ConfigEntry; @@ -416,6 +417,30 @@ class KafkaAgentTest { assertNull(kafkaProperties.getProperty("zookeeper.ssl.trustStore.password")); } + @Test + void producerDefaultsToLegacyCompatibleNonIdempotentDelivery() { + JsonObject connection = new JsonObject(); + connection.addProperty("bootstrap_servers", "legacy-broker:9092"); + + Properties properties = KafkaAgent.producerProperties(connection); + + assertEquals("all", properties.getProperty(ProducerConfig.ACKS_CONFIG)); + assertEquals("false", properties.getProperty(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG)); + } + + @Test + void producerAllowsExplicitIdempotenceForModernBrokers() { + JsonObject extraProperties = new JsonObject(); + extraProperties.addProperty(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG, "true"); + JsonObject connection = new JsonObject(); + connection.addProperty("bootstrap_servers", "modern-broker:9092"); + connection.add("properties", extraProperties); + + Properties properties = KafkaAgent.producerProperties(connection); + + assertEquals("true", properties.getProperty(ProducerConfig.ENABLE_IDEMPOTENCE_CONFIG)); + } + @Test void brokerEndpointsFallBackToLegacyHostAndPort() { assertEquals("legacy-broker:9092", KafkaAgent.brokerEndpoints(