fix(kafka): support legacy topic metadata and config gaps

Closes #6217
This commit is contained in:
t8y2 committed 2026-08-14 07:01:07 +00:00
1 parent df95288aab
commit 9e730fac56
5 files changed
+262 -5

No files matched your search

@@ -704,8 +704,22 @@ public final class KafkaAgent {
int timeout = requestTimeout(params); int timeout = requestTimeout(params);
String name = stringOrEmpty(params, "name"); String name = stringOrEmpty(params, "name");
try {
return modernTopicStats(admin, name, timeout);
} catch (Exception error) {
if (!hasUnsupportedVersionException(error)) {
throw error;
}
return legacyTopicStats(name, timeout);
}
}
private static Object modernTopicStats(AdminClient admin, String name, int timeout) throws Exception {
TopicDescription desc = admin.describeTopics(Collections.singletonList(name)) TopicDescription desc = admin.describeTopics(Collections.singletonList(name))
.allTopicNames().get(timeout, TimeUnit.MILLISECONDS).get(name); .allTopicNames().get(timeout, TimeUnit.MILLISECONDS).get(name);
if (desc == null) {
throw new UnknownTopicOrPartitionException("Kafka topic does not exist: " + name);
}
// Collect offsets for size estimation // Collect offsets for size estimation
Map<TopicPartition, ListOffsetsResult.ListOffsetsResultInfo> endOffsets = new LinkedHashMap<>(); Map<TopicPartition, ListOffsetsResult.ListOffsetsResultInfo> endOffsets = new LinkedHashMap<>();
@@ -748,14 +762,101 @@ public final class KafkaAgent {
return result; return result;
} }
private static Object legacyTopicStats(String name, int timeout) {
Properties props = topicStatsConsumerProperties(activeConnection);
Duration requestTimeout = Duration.ofMillis(timeout);
try (KafkaConsumer<String, byte[]> consumer = new KafkaConsumer<>(props)) {
Map<String, List<PartitionInfo>> topics = consumer.listTopics(requestTimeout);
requireExistingTopic(topics.keySet(), name);
List<PartitionInfo> partitions = topics.get(name);
List<TopicPartition> topicPartitions = partitions.stream()
.map(partition -> new TopicPartition(name, partition.partition()))
.collect(Collectors.toList());
Map<TopicPartition, Long> beginningOffsets = consumer.beginningOffsets(topicPartitions, requestTimeout);
Map<TopicPartition, Long> endOffsets = consumer.endOffsets(topicPartitions, requestTimeout);
return legacyTopicStatsResult(name, partitions, beginningOffsets, endOffsets);
}
}
static void requireExistingTopic(Collection<String> topicNames, String name) {
if (!topicNames.contains(name)) {
throw new UnknownTopicOrPartitionException("Kafka topic does not exist: " + name);
}
}
static Properties topicStatsConsumerProperties(JsonObject conn) {
if (conn == null) {
throw new IllegalStateException("Kafka Agent is not connected");
}
Properties props = peekConsumerProperties(conn, 1);
// The fallback discovers the topic through all-topics metadata, which cannot create
// a topic, before issuing requests for its concrete partitions.
props.put(ConsumerConfig.ALLOW_AUTO_CREATE_TOPICS_CONFIG, "true");
return props;
}
static Object legacyTopicStatsResult(
String name,
List<PartitionInfo> partitions,
Map<TopicPartition, Long> beginningOffsets,
Map<TopicPartition, Long> endOffsets
) {
long totalMessages = 0;
List<Map<String, Object>> partitionStats = new ArrayList<>();
for (PartitionInfo partition : partitions) {
TopicPartition topicPartition = new TopicPartition(name, partition.partition());
long begin = beginningOffsets.get(topicPartition);
long end = endOffsets.get(topicPartition);
long count = end - begin;
totalMessages += count;
Map<String, Object> stats = new LinkedHashMap<>();
stats.put("partition", partition.partition());
stats.put("leader", partition.leader() != null ? partition.leader().id() : -1);
stats.put("replicas", Arrays.stream(partition.replicas()).map(Node::id).collect(Collectors.toList()));
stats.put("isr", Arrays.stream(partition.inSyncReplicas()).map(Node::id).collect(Collectors.toList()));
stats.put("beginOffset", begin);
stats.put("endOffset", end);
stats.put("messageCount", count);
partitionStats.add(stats);
}
Map<String, Object> result = new LinkedHashMap<>();
result.put("name", name);
result.put("partitions", partitions.size());
result.put("replicationFactor", partitions.isEmpty() ? 0 : partitions.get(0).replicas().length);
result.put("totalMessages", totalMessages);
result.put("partitionStats", partitionStats);
return result;
}
private static Object getTopicConfig(JsonObject params) throws Exception { private static Object getTopicConfig(JsonObject params) throws Exception {
AdminClient admin = requireAdmin(); AdminClient admin = requireAdmin();
int timeout = requestTimeout(params); int timeout = requestTimeout(params);
String name = stringOrEmpty(params, "name"); String name = stringOrEmpty(params, "name");
ConfigResource resource = new ConfigResource(ConfigResource.Type.TOPIC, name); ConfigResource resource = new ConfigResource(ConfigResource.Type.TOPIC, name);
Config config = admin.describeConfigs(Collections.singletonList(resource)) return topicConfigResult(() -> admin.describeConfigs(Collections.singletonList(resource))
.all().get(timeout, TimeUnit.MILLISECONDS).get(resource); .all().get(timeout, TimeUnit.MILLISECONDS).get(resource));
}
static Object topicConfigResult(TopicConfigLoader configLoader) throws Exception {
Config config;
try {
config = configLoader.load();
} catch (Exception error) {
if (!hasUnsupportedVersionException(error)) {
throw error;
}
Map<String, Object> unsupported = new LinkedHashMap<>();
unsupported.put("configs", Collections.emptyMap());
unsupported.put("configSupported", false);
unsupported.put(
"unsupportedReason",
"Topic configuration is unavailable because this Kafka broker does not support DescribeConfigs."
);
return unsupported;
}
Map<String, Object> configs = new LinkedHashMap<>(); Map<String, Object> configs = new LinkedHashMap<>();
for (ConfigEntry entry : config.entries()) { for (ConfigEntry entry : config.entries()) {
@@ -770,6 +871,11 @@ public final class KafkaAgent {
return Collections.singletonMap("configs", configs); return Collections.singletonMap("configs", configs);
} }
@FunctionalInterface
interface TopicConfigLoader {
Config load() throws Exception;
}
private static Object alterTopicConfig(JsonObject params) throws Exception { private static Object alterTopicConfig(JsonObject params) throws Exception {
AdminClient admin = requireAdmin(); AdminClient admin = requireAdmin();
int timeout = requestTimeout(params); int timeout = requestTimeout(params);
@@ -2098,6 +2204,10 @@ public final class KafkaAgent {
return false; return false;
} }
static boolean hasUnsupportedVersionException(Throwable error) {
return causeChain(error).stream().anyMatch(UnsupportedVersionException.class::isInstance);
}
private static Throwable rootCause(Throwable error) { private static Throwable rootCause(Throwable error) {
Throwable current = null; Throwable current = null;
for (Throwable cause : causeChain(error)) { for (Throwable cause : causeChain(error)) {
@@ -36,6 +36,7 @@ import org.apache.kafka.clients.admin.RaftVoterEndpoint;
import org.apache.kafka.clients.admin.TopicDescription; import org.apache.kafka.clients.admin.TopicDescription;
import org.apache.kafka.clients.admin.TopicListing; import org.apache.kafka.clients.admin.TopicListing;
import org.apache.kafka.common.Node; import org.apache.kafka.common.Node;
import org.apache.kafka.common.PartitionInfo;
import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.TopicPartitionInfo; import org.apache.kafka.common.TopicPartitionInfo;
import org.apache.kafka.common.Uuid; import org.apache.kafka.common.Uuid;
@@ -150,6 +151,114 @@ class KafkaAgentTest {
assertEquals(false, topics.get(0).get("internal")); assertEquals(false, topics.get(0).get("internal"));
} }
@Test
void legacyStatsFallbackRequiresAnUnsupportedVersionException() {
assertTrue(KafkaAgent.hasUnsupportedVersionException(new ExecutionException(
new UnsupportedVersionException(
"MetadataRequest versions older than 4 don't support the allowAutoTopicCreation field"
)
)));
assertFalse(KafkaAgent.hasUnsupportedVersionException(
new IllegalStateException("broker reports unsupported version text")
));
}
@Test
void legacyTopicStatsRequireAnExistingTopicFromAllTopicMetadata() {
var error = assertThrows(
org.apache.kafka.common.errors.UnknownTopicOrPartitionException.class,
() -> KafkaAgent.requireExistingTopic(Collections.singleton("payments"), "orders")
);
assertTrue(error.getMessage().contains("orders"));
}
@Test
void legacyTopicStatsConsumerUsesCompatibleMetadataOnlyForConnectedSessions() {
assertThrows(IllegalStateException.class, () -> KafkaAgent.topicStatsConsumerProperties(null));
JsonObject connection = new JsonObject();
connection.addProperty("bootstrap_servers", "legacy-broker:9092");
Properties properties = KafkaAgent.topicStatsConsumerProperties(connection);
assertEquals("true", properties.getProperty("allow.auto.create.topics"));
assertEquals("false", properties.getProperty("enable.auto.commit"));
}
@Test
void legacyTopicStatsPreserveTheExistingStatsShape() {
Node leader = new Node(1, "broker-1", 9092);
Node replica = new Node(2, "broker-2", 9092);
PartitionInfo partition = new PartitionInfo(
"orders",
0,
leader,
new Node[] { leader, replica },
new Node[] { leader }
);
TopicPartition topicPartition = new TopicPartition("orders", 0);
@SuppressWarnings("unchecked")
Map<String, Object> result = (Map<String, Object>) KafkaAgent.legacyTopicStatsResult(
"orders",
Collections.singletonList(partition),
Collections.singletonMap(topicPartition, 4L),
Collections.singletonMap(topicPartition, 10L)
);
assertEquals("orders", result.get("name"));
assertEquals(1, result.get("partitions"));
assertEquals(2, result.get("replicationFactor"));
assertEquals(6L, result.get("totalMessages"));
@SuppressWarnings("unchecked")
List<Map<String, Object>> partitionStats = (List<Map<String, Object>>) result.get("partitionStats");
assertEquals(List.of(1, 2), partitionStats.get(0).get("replicas"));
assertEquals(Collections.singletonList(1), partitionStats.get(0).get("isr"));
assertEquals(4L, partitionStats.get(0).get("beginOffset"));
assertEquals(10L, partitionStats.get(0).get("endOffset"));
}
@Test
void topicConfigReturnsAnExplicitUnsupportedMarkerForLegacyBrokers() throws Exception {
@SuppressWarnings("unchecked")
Map<String, Object> result = (Map<String, Object>) KafkaAgent.topicConfigResult(() -> {
throw new ExecutionException(new UnsupportedVersionException(
"The node does not support DESCRIBE_CONFIGS"
));
});
assertEquals(Collections.emptyMap(), result.get("configs"));
assertEquals(false, result.get("configSupported"));
assertTrue(((String) result.get("unsupportedReason")).contains("DescribeConfigs"));
}
@Test
void topicConfigPreservesModernConfigEntries() throws Exception {
Config config = new Config(Collections.singletonList(new ConfigEntry("retention.ms", "60000")));
@SuppressWarnings("unchecked")
Map<String, Object> result = (Map<String, Object>) KafkaAgent.topicConfigResult(() -> config);
@SuppressWarnings("unchecked")
Map<String, Map<String, Object>> configs = (Map<String, Map<String, Object>>) result.get("configs");
assertEquals("60000", configs.get("retention.ms").get("value"));
assertFalse(result.containsKey("configSupported"));
}
@Test
void topicConfigPreservesNonVersionFailures() {
IllegalStateException failure = new IllegalStateException("broker reports unsupported version text");
IllegalStateException thrown = assertThrows(
IllegalStateException.class,
() -> KafkaAgent.topicConfigResult(() -> {
throw failure;
})
);
assertEquals(failure, thrown);
}
@Test @Test
void resolvesBootstrapServersFromKafka11ZooKeeperRegistrationWithChroot() throws Exception { void resolvesBootstrapServersFromKafka11ZooKeeperRegistrationWithChroot() throws Exception {
Path snapshots = Files.createDirectory(tempDir.resolve("snapshots")); Path snapshots = Files.createDirectory(tempDir.resolve("snapshots"));
@@ -4,7 +4,7 @@ import { computed, ref, watch } from "vue";
import { useI18n } from "vue-i18n"; import { useI18n } from "vue-i18n";
import type { BacklogQuota, DispatchRate, PolicyScope, PublishRate, RetentionPolicy, SubscribeRate, TopicInfo } from "@/types/mq"; import type { BacklogQuota, DispatchRate, PolicyScope, PublishRate, RetentionPolicy, SubscribeRate, TopicInfo } from "@/types/mq";
import { mqGetEffectivePolicies, mqSetBacklogQuota, mqSetDispatchRate, mqSetPublishRate, mqSetRetention, mqSetSubscribeRate } from "@/lib/backend/api"; import { mqGetEffectivePolicies, mqSetBacklogQuota, mqSetDispatchRate, mqSetPublishRate, mqSetRetention, mqSetSubscribeRate } from "@/lib/backend/api";
import { defaultMqPolicyForms, policyFormsFromEffectivePolicies } from "@/lib/mq/mqPolicyForms"; import { defaultMqPolicyForms, policyAccessFromEffectivePolicies, policyFormsFromEffectivePolicies } from "@/lib/mq/mqPolicyForms";
import { useMqMutationGuard } from "@/composables/useMqMutationGuard"; import { useMqMutationGuard } from "@/composables/useMqMutationGuard";
interface Props { interface Props {
@@ -29,6 +29,7 @@ const error = ref<string>();
const notice = ref<string>(); const notice = ref<string>();
const readOnlyMessage = computed(() => t("mqPolicies.readOnly")); const readOnlyMessage = computed(() => t("mqPolicies.readOnly"));
const defaultForms = defaultMqPolicyForms(); const defaultForms = defaultMqPolicyForms();
const policyAccess = ref(policyAccessFromEffectivePolicies(undefined));
const publishForm = ref<PublishRate>({ ...defaultForms.publishForm }); const publishForm = ref<PublishRate>({ ...defaultForms.publishForm });
@@ -84,6 +85,7 @@ async function guardWritable(operation: string): Promise<boolean> {
async function loadPolicies() { async function loadPolicies() {
const current = scope.value; const current = scope.value;
policies.value = undefined; policies.value = undefined;
policyAccess.value = policyAccessFromEffectivePolicies(undefined);
notice.value = undefined; notice.value = undefined;
error.value = undefined; error.value = undefined;
if (!current) return; if (!current) return;
@@ -92,6 +94,7 @@ async function loadPolicies() {
try { try {
const loaded = await mqGetEffectivePolicies(props.connectionId, current); const loaded = await mqGetEffectivePolicies(props.connectionId, current);
policies.value = loaded; policies.value = loaded;
policyAccess.value = policyAccessFromEffectivePolicies(loaded);
const hydrated = policyFormsFromEffectivePolicies(loaded, defaultMqPolicyForms()); const hydrated = policyFormsFromEffectivePolicies(loaded, defaultMqPolicyForms());
applyPolicyForms(hydrated); applyPolicyForms(hydrated);
} catch (e: unknown) { } catch (e: unknown) {
@@ -189,7 +192,8 @@ watch(
<div v-if="error" class="panel-error">{{ error }}</div> <div v-if="error" class="panel-error">{{ error }}</div>
<div v-if="notice" class="panel-notice">{{ notice }}</div> <div v-if="notice" class="panel-notice">{{ notice }}</div>
<div class="policy-grid"> <div v-if="!policyAccess.readable" class="readonly-hint">{{ policyAccess.unsupportedReason }}</div>
<div v-else class="policy-grid">
<section v-if="supportsRateLimits !== false" class="policy-section"> <section v-if="supportsRateLimits !== false" class="policy-section">
<h4>{{ t("mqPolicies.publishRate") }}</h4> <h4>{{ t("mqPolicies.publishRate") }}</h4>
<label> <label>
+21 -1
View File
@@ -1,7 +1,27 @@
import { describe, expect, it } from "vitest"; import { describe, expect, it } from "vitest";
import { defaultMqPolicyForms, policyFormsFromEffectivePolicies } from "@/lib/mq/mqPolicyForms"; import { defaultMqPolicyForms, policyAccessFromEffectivePolicies, policyFormsFromEffectivePolicies } from "@/lib/mq/mqPolicyForms";
describe("mqPolicyForms", () => { describe("mqPolicyForms", () => {
it("keeps ordinary policy responses available", () => {
expect(policyAccessFromEffectivePolicies({ configs: {} })).toEqual({
readable: true,
unsupportedReason: undefined,
});
});
it("marks legacy Kafka configuration responses as unavailable with a reason", () => {
expect(
policyAccessFromEffectivePolicies({
configs: {},
configSupported: false,
unsupportedReason: "The broker does not support DescribeConfigs.",
}),
).toEqual({
readable: false,
unsupportedReason: "The broker does not support DescribeConfigs.",
});
});
it("hydrates forms from topic effective policies", () => { it("hydrates forms from topic effective policies", () => {
const forms = policyFormsFromEffectivePolicies( const forms = policyFormsFromEffectivePolicies(
{ {
+14
View File
@@ -8,6 +8,11 @@ export interface MqPolicyForms {
retentionForm: RetentionPolicy; retentionForm: RetentionPolicy;
} }
export interface MqPolicyAccess {
readable: boolean;
unsupportedReason?: string;
}
export function defaultMqPolicyForms(): MqPolicyForms { export function defaultMqPolicyForms(): MqPolicyForms {
return { return {
publishForm: { publishForm: {
@@ -36,6 +41,15 @@ export function defaultMqPolicyForms(): MqPolicyForms {
}; };
} }
export function policyAccessFromEffectivePolicies(raw: unknown): MqPolicyAccess {
const root = objectRecord(raw);
const readable = root.configSupported !== false;
return {
readable,
unsupportedReason: readable ? undefined : stringField(root.unsupportedReason),
};
}
export function policyFormsFromEffectivePolicies(raw: unknown, fallback: MqPolicyForms): MqPolicyForms { export function policyFormsFromEffectivePolicies(raw: unknown, fallback: MqPolicyForms): MqPolicyForms {
const root = objectRecord(raw); const root = objectRecord(raw);
const topicPolicies = objectRecord(root.topicPolicies); const topicPolicies = objectRecord(root.topicPolicies);