fix(kafka): support ZooKeeper broker discovery
This commit is contained in:
1 parent
34448cba9d
commit
8adf802444
18 files changed
+977
-77
No files matched your search
@@ -1,4 +1,13 @@
|
||||
const KAFKA_BOOTSTRAP_SERVER_SEPARATOR = /[\s,;,;]+/u;
|
||||
const KAFKA_BOOTSTRAP_SERVER_SCHEME = /^([a-z][a-z0-9_-]*):\/\/(.+)$/iu;
|
||||
const KAFKA_SECURITY_PROTOCOLS = new Set(["PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL"] as const);
|
||||
|
||||
export type KafkaSecurityProtocol = "PLAINTEXT" | "SSL" | "SASL_PLAINTEXT" | "SASL_SSL";
|
||||
|
||||
export interface ParsedKafkaBootstrapServers {
|
||||
bootstrapServers: string;
|
||||
inferredSecurityProtocol?: KafkaSecurityProtocol;
|
||||
}
|
||||
|
||||
function requireKafkaBootstrapServers(value: string): string {
|
||||
const trimmed = value.trim();
|
||||
@@ -6,28 +15,60 @@ function requireKafkaBootstrapServers(value: string): string {
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
function normalizeKafkaBootstrapServer(server: string): string {
|
||||
if (server.includes("://")) {
|
||||
throw new Error("Kafka bootstrap servers must be host:port values without a URL scheme");
|
||||
function normalizeKafkaBootstrapServer(server: string): { address: string; securityProtocol?: KafkaSecurityProtocol } {
|
||||
const schemeMatch = server.match(KAFKA_BOOTSTRAP_SERVER_SCHEME);
|
||||
let address = server;
|
||||
let securityProtocol: KafkaSecurityProtocol | undefined;
|
||||
if (schemeMatch) {
|
||||
const protocol = schemeMatch[1].toUpperCase();
|
||||
if (!KAFKA_SECURITY_PROTOCOLS.has(protocol as KafkaSecurityProtocol)) {
|
||||
throw new Error("Kafka bootstrap server protocol is invalid");
|
||||
}
|
||||
securityProtocol = protocol as KafkaSecurityProtocol;
|
||||
address = schemeMatch[2];
|
||||
} else if (server.includes("://")) {
|
||||
throw new Error("Kafka bootstrap server protocol is invalid");
|
||||
}
|
||||
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(`kafka://${server}`);
|
||||
parsed = new URL(`kafka://${address}`);
|
||||
} catch {
|
||||
throw new Error("Kafka bootstrap servers are invalid");
|
||||
}
|
||||
if (!parsed.hostname || parsed.username || parsed.password || parsed.search || parsed.hash || (parsed.pathname && parsed.pathname !== "/")) {
|
||||
throw new Error("Kafka bootstrap servers are invalid");
|
||||
}
|
||||
return server;
|
||||
if (!parsed.port) {
|
||||
throw new Error("Kafka bootstrap servers must be host:port values");
|
||||
}
|
||||
return { address, securityProtocol };
|
||||
}
|
||||
|
||||
export function normalizeKafkaBootstrapServers(value: string): string {
|
||||
const servers = requireKafkaBootstrapServers(value)
|
||||
export function parseKafkaBootstrapServers(value: string): ParsedKafkaBootstrapServers {
|
||||
const parsedServers = requireKafkaBootstrapServers(value)
|
||||
.split(KAFKA_BOOTSTRAP_SERVER_SEPARATOR)
|
||||
.map((server) => server.trim())
|
||||
.filter(Boolean)
|
||||
.map(normalizeKafkaBootstrapServer);
|
||||
if (!servers.length) throw new Error("Kafka bootstrap servers are required");
|
||||
return servers.join(",");
|
||||
if (!parsedServers.length) throw new Error("Kafka bootstrap servers are required");
|
||||
|
||||
const protocols = new Set(parsedServers.map((server) => server.securityProtocol).filter((protocol): protocol is KafkaSecurityProtocol => !!protocol));
|
||||
if (protocols.size > 1) {
|
||||
throw new Error("Kafka bootstrap servers must use one security protocol");
|
||||
}
|
||||
|
||||
const inferredSecurityProtocol = protocols.values().next().value;
|
||||
return {
|
||||
bootstrapServers: parsedServers.map((server) => server.address).join(","),
|
||||
...(inferredSecurityProtocol ? { inferredSecurityProtocol } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizeKafkaBootstrapServers(value: string): string {
|
||||
return parseKafkaBootstrapServers(value).bootstrapServers;
|
||||
}
|
||||
|
||||
export function resolveKafkaSecurityProtocol(configured: string, inferred?: KafkaSecurityProtocol): string {
|
||||
return configured.trim() || inferred || "";
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
import { parseKafkaBootstrapServers, resolveKafkaSecurityProtocol } from "@/lib/connection/kafkaBootstrapServers";
|
||||
import { firstZooKeeperEndpoint, normalizeZooKeeperConnectString } from "@/lib/zookeeper/zookeeperConnection";
|
||||
|
||||
export type MqKafkaConnectionSource = "bootstrap" | "zookeeper";
|
||||
|
||||
export interface MqKafkaConnectionInput {
|
||||
connectionSource: MqKafkaConnectionSource;
|
||||
bootstrapServers: string;
|
||||
zookeeperServers: string;
|
||||
securityProtocol?: string;
|
||||
}
|
||||
|
||||
export interface MqKafkaConnectionTarget {
|
||||
host: string;
|
||||
port: number;
|
||||
ssl: boolean;
|
||||
}
|
||||
|
||||
export function resolveMqKafkaConnectionSource(extra: Record<string, unknown>): MqKafkaConnectionSource {
|
||||
if (extra.connectionSource === "zookeeper") return "zookeeper";
|
||||
if (typeof extra.zookeeperServers === "string" && extra.zookeeperServers.trim() && !(typeof extra.bootstrapServers === "string" && extra.bootstrapServers.trim())) {
|
||||
return "zookeeper";
|
||||
}
|
||||
return "bootstrap";
|
||||
}
|
||||
|
||||
export function normalizeMqKafkaZooKeeperServers(value: string): string {
|
||||
const normalized = normalizeZooKeeperConnectString(value.trim());
|
||||
if (!normalized) throw new Error("Kafka ZooKeeper servers are required");
|
||||
|
||||
const chrootIndex = normalized.indexOf("/");
|
||||
const ensemble = chrootIndex >= 0 ? normalized.slice(0, chrootIndex) : normalized;
|
||||
const chroot = chrootIndex >= 0 ? normalized.slice(chrootIndex) : "";
|
||||
if (chroot && (!chroot.startsWith("/") || chroot.includes("//"))) {
|
||||
throw new Error("Kafka ZooKeeper servers are invalid");
|
||||
}
|
||||
|
||||
for (const endpoint of ensemble.split(",")) {
|
||||
const match = endpoint.match(/^(?:\[[^\]]+\]|[^:\s/?#]+):(\d+)$/u);
|
||||
const port = match ? Number(match[1]) : Number.NaN;
|
||||
if (!match || !Number.isInteger(port) || port < 1 || port > 65535) {
|
||||
throw new Error("Kafka ZooKeeper servers are invalid");
|
||||
}
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
export function buildMqKafkaConnectionExtra(input: MqKafkaConnectionInput): Record<string, string> {
|
||||
let extra: Record<string, string>;
|
||||
let securityProtocol = input.securityProtocol?.trim() || "";
|
||||
|
||||
if (input.connectionSource === "zookeeper") {
|
||||
extra = {
|
||||
connectionSource: "zookeeper",
|
||||
zookeeperServers: normalizeMqKafkaZooKeeperServers(input.zookeeperServers),
|
||||
};
|
||||
} else {
|
||||
const parsed = parseKafkaBootstrapServers(input.bootstrapServers);
|
||||
securityProtocol = resolveKafkaSecurityProtocol(securityProtocol, parsed.inferredSecurityProtocol);
|
||||
extra = { bootstrapServers: parsed.bootstrapServers };
|
||||
}
|
||||
|
||||
if (securityProtocol) extra.securityProtocol = securityProtocol;
|
||||
return extra;
|
||||
}
|
||||
|
||||
export function mqKafkaConnectionTarget(input: MqKafkaConnectionInput): MqKafkaConnectionTarget {
|
||||
if (input.connectionSource === "zookeeper") {
|
||||
const endpoint = firstZooKeeperEndpoint(normalizeMqKafkaZooKeeperServers(input.zookeeperServers));
|
||||
if (!endpoint) throw new Error("Kafka ZooKeeper servers are required");
|
||||
return { ...endpoint, ssl: false };
|
||||
}
|
||||
|
||||
const parsed = parseKafkaBootstrapServers(input.bootstrapServers);
|
||||
const first = parsed.bootstrapServers.split(",")[0];
|
||||
const endpoint = new URL(`kafka://${first}`);
|
||||
const securityProtocol = resolveKafkaSecurityProtocol(input.securityProtocol || "", parsed.inferredSecurityProtocol);
|
||||
return {
|
||||
host: endpoint.hostname,
|
||||
port: Number(endpoint.port),
|
||||
ssl: securityProtocol === "SSL" || securityProtocol === "SASL_SSL",
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user