feat(rocketmq): add agent and management console
This commit is contained in:
86 files changed
+12373
-505
No files matched your search
@@ -1716,6 +1716,11 @@
|
||||
"profile": "kafka",
|
||||
"label": "Apache Kafka",
|
||||
"agentKey": "kafka"
|
||||
},
|
||||
{
|
||||
"profile": "rocketmq",
|
||||
"label": "Apache RocketMQ",
|
||||
"agentKey": "rocketmq"
|
||||
}
|
||||
],
|
||||
"supportLevel": "connect",
|
||||
|
||||
@@ -39,10 +39,16 @@ const MONGODB_PROFILES: &[AgentDriverProfile] = &[AgentDriverProfile {
|
||||
const H2_PROFILES: &[AgentDriverProfile] =
|
||||
&[AgentDriverProfile { profile: "h2-legacy", key: "h2-legacy", label: "H2 2.1 Legacy", store_visible: true }];
|
||||
|
||||
const EXTRA_AGENT_LABELS: &[(&str, &str)] =
|
||||
&[("kafka", "Apache Kafka"), ("sqlserver-legacy", "SQL Server legacy compatibility component")];
|
||||
const EXTRA_DRIVER_STORE_ENTRIES: &[(&str, &str)] =
|
||||
&[("kafka", "Apache Kafka"), ("sqlserver-legacy", "SQL Server legacy compatibility component")];
|
||||
const EXTRA_AGENT_LABELS: &[(&str, &str)] = &[
|
||||
("kafka", "Apache Kafka"),
|
||||
("rocketmq", "Apache RocketMQ"),
|
||||
("sqlserver-legacy", "SQL Server legacy compatibility component"),
|
||||
];
|
||||
const EXTRA_DRIVER_STORE_ENTRIES: &[(&str, &str)] = &[
|
||||
("kafka", "Apache Kafka"),
|
||||
("rocketmq", "Apache RocketMQ"),
|
||||
("sqlserver-legacy", "SQL Server legacy compatibility component"),
|
||||
];
|
||||
|
||||
const AGENT_CATALOG: &[AgentCatalogEntry] = &[
|
||||
AgentCatalogEntry {
|
||||
@@ -301,7 +307,11 @@ pub fn entries() -> &'static [AgentCatalogEntry] {
|
||||
|
||||
pub fn agent_key(db_type: &DatabaseType, driver_profile: Option<&str>) -> Option<&'static str> {
|
||||
if *db_type == DatabaseType::MessageQueue {
|
||||
return (driver_profile == Some("kafka")).then_some("kafka");
|
||||
return match driver_profile {
|
||||
Some("kafka") => Some("kafka"),
|
||||
Some("rocketmq") => Some("rocketmq"),
|
||||
_ => None,
|
||||
};
|
||||
}
|
||||
if *db_type == DatabaseType::SqlServer {
|
||||
return driver_profile
|
||||
|
||||
@@ -1635,8 +1635,8 @@ impl AppState {
|
||||
// connectivity via the mq_registry and insert a marker so this
|
||||
// connection_id is recognized as valid.
|
||||
let mqc = self.mq_admin_config_for_connection(connection_id, &config).await?;
|
||||
let kafka_launch = crate::mq::service::resolve_kafka_launch_spec(&mqc, self);
|
||||
let adapter = match self.mq_registry.get_or_build_config(connection_id, mqc, kafka_launch).await {
|
||||
let agent_launch = crate::mq::service::resolve_mq_agent_launch_spec(&mqc, self);
|
||||
let adapter = match self.mq_registry.get_or_build_config(connection_id, mqc, agent_launch).await {
|
||||
Ok(adapter) => adapter,
|
||||
Err(err) => {
|
||||
self.mq_registry.drop_connection(connection_id).await;
|
||||
|
||||
@@ -21,7 +21,7 @@ use crate::mq::config::MqAdminConfig;
|
||||
use crate::mq::port::MessageQueueAdmin;
|
||||
use crate::mq::types::*;
|
||||
|
||||
/// Kafka capabilities — no tenants/namespaces, supports topics, consumer groups,
|
||||
/// Kafka capabilities - no tenants/namespaces, supports topics, consumer groups,
|
||||
/// ACLs, retention, and message production.
|
||||
const KAFKA_CAPABILITIES: MqCapabilities = MqCapabilities {
|
||||
supports_tenants: false,
|
||||
@@ -42,6 +42,9 @@ const KAFKA_CAPABILITIES: MqCapabilities = MqCapabilities {
|
||||
supports_token_management: false,
|
||||
supports_raw_admin_api: false,
|
||||
supports_send_message: true,
|
||||
supports_message_query: false,
|
||||
supports_dlq: false,
|
||||
supports_message_trace: false,
|
||||
};
|
||||
|
||||
pub struct KafkaAdmin {
|
||||
@@ -180,6 +183,8 @@ impl MessageQueueAdmin for KafkaAdmin {
|
||||
partitioned: partitions.map(|p| p > 1).unwrap_or(false),
|
||||
partitions,
|
||||
persistent: true,
|
||||
internal: t.get("internal").and_then(|v| v.as_bool()).unwrap_or(false),
|
||||
message_type: None,
|
||||
}
|
||||
})
|
||||
.collect())
|
||||
@@ -584,6 +589,7 @@ impl MessageQueueAdmin for KafkaAdmin {
|
||||
host: node.get("host")?.as_str()?.to_string(),
|
||||
port: node.get("port")?.as_i64()? as i32,
|
||||
rack: node.get("rack").and_then(|v| v.as_str()).map(String::from),
|
||||
..Default::default()
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
@@ -733,6 +739,7 @@ fn kafka_subscription_for_topic(
|
||||
msg_rate_out: 0.0,
|
||||
msg_throughput_out: 0.0,
|
||||
consumers: Vec::new(),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
|
||||
@@ -833,6 +840,8 @@ mod tests {
|
||||
topic: "events".to_string(),
|
||||
persistent: true,
|
||||
partitioned: None,
|
||||
message_type: None,
|
||||
..TopicRef::default()
|
||||
};
|
||||
|
||||
let params = reset_cursor_params(&topic, "group-a", ResetPosition::Timestamp { timestamp_ms: 1710000000000 })
|
||||
@@ -852,6 +861,8 @@ mod tests {
|
||||
topic: "events".to_string(),
|
||||
persistent: true,
|
||||
partitioned: None,
|
||||
message_type: None,
|
||||
..TopicRef::default()
|
||||
};
|
||||
|
||||
let err = reset_cursor_params(&topic, "group-a", ResetPosition::MessageId { ledger_id: 1, entry_id: 2 })
|
||||
|
||||
@@ -3,3 +3,4 @@
|
||||
pub mod kafka;
|
||||
pub mod pulsar;
|
||||
pub mod pulsar_version;
|
||||
pub mod rocketmq;
|
||||
@@ -291,6 +291,8 @@ impl MessageQueueAdmin for PulsarAdmin {
|
||||
partitioned: true,
|
||||
partitions,
|
||||
persistent,
|
||||
internal: false,
|
||||
message_type: None,
|
||||
})
|
||||
})
|
||||
.buffered(PARTITION_METADATA_CONCURRENCY)
|
||||
@@ -314,6 +316,8 @@ impl MessageQueueAdmin for PulsarAdmin {
|
||||
partitioned: false,
|
||||
partitions: None,
|
||||
persistent,
|
||||
internal: false,
|
||||
message_type: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -507,7 +511,7 @@ impl MessageQueueAdmin for PulsarAdmin {
|
||||
namespace: namespace.clone(),
|
||||
topic: topic.clone(),
|
||||
persistent: *persistent,
|
||||
partitioned: None,
|
||||
..Default::default()
|
||||
};
|
||||
self.profile.topic_publish_rate_path(topic_ref.domain(), &topic_ref.path())
|
||||
}
|
||||
@@ -527,7 +531,7 @@ impl MessageQueueAdmin for PulsarAdmin {
|
||||
namespace: namespace.clone(),
|
||||
topic: topic.clone(),
|
||||
persistent: *persistent,
|
||||
partitioned: None,
|
||||
..Default::default()
|
||||
};
|
||||
self.profile.topic_dispatch_rate_path(topic_ref.domain(), &topic_ref.path())
|
||||
}
|
||||
@@ -560,7 +564,7 @@ impl MessageQueueAdmin for PulsarAdmin {
|
||||
namespace: namespace.clone(),
|
||||
topic: topic.clone(),
|
||||
persistent: *persistent,
|
||||
partitioned: None,
|
||||
..Default::default()
|
||||
};
|
||||
self.profile.topic_backlog_quota_path(topic_ref.domain(), &topic_ref.path())
|
||||
}
|
||||
@@ -585,7 +589,7 @@ impl MessageQueueAdmin for PulsarAdmin {
|
||||
namespace: namespace.clone(),
|
||||
topic: topic.clone(),
|
||||
persistent: *persistent,
|
||||
partitioned: None,
|
||||
..Default::default()
|
||||
};
|
||||
self.profile.topic_retention_path(topic_ref.domain(), &topic_ref.path())
|
||||
}
|
||||
@@ -608,7 +612,7 @@ impl MessageQueueAdmin for PulsarAdmin {
|
||||
namespace: namespace.clone(),
|
||||
topic: topic.clone(),
|
||||
persistent: *persistent,
|
||||
partitioned: None,
|
||||
..Default::default()
|
||||
};
|
||||
let namespace_policies = self
|
||||
.get_value(&self.profile.namespace_policies_path(&topic_ref.tenant, &topic_ref.namespace))
|
||||
@@ -654,7 +658,7 @@ impl MessageQueueAdmin for PulsarAdmin {
|
||||
namespace: namespace.clone(),
|
||||
topic: topic.clone(),
|
||||
persistent: *persistent,
|
||||
partitioned: None,
|
||||
..Default::default()
|
||||
};
|
||||
self.profile.topic_permission_role_path(topic_ref.domain(), &topic_ref.path(), role)
|
||||
}
|
||||
@@ -674,7 +678,7 @@ impl MessageQueueAdmin for PulsarAdmin {
|
||||
namespace: namespace.clone(),
|
||||
topic: topic.clone(),
|
||||
persistent: *persistent,
|
||||
partitioned: None,
|
||||
..Default::default()
|
||||
};
|
||||
self.profile.topic_permission_role_path(topic_ref.domain(), &topic_ref.path(), role)
|
||||
}
|
||||
@@ -691,7 +695,7 @@ impl MessageQueueAdmin for PulsarAdmin {
|
||||
namespace: namespace.clone(),
|
||||
topic: topic.clone(),
|
||||
persistent: *persistent,
|
||||
partitioned: None,
|
||||
..Default::default()
|
||||
};
|
||||
self.profile.topic_permissions_path(topic_ref.domain(), &topic_ref.path())
|
||||
}
|
||||
@@ -1422,6 +1426,7 @@ mod tests {
|
||||
topic: "orders".to_string(),
|
||||
persistent: true,
|
||||
partitioned: Some(true),
|
||||
..Default::default()
|
||||
};
|
||||
admin.create_topic(&topic, Some(3)).await.expect("create_topic should succeed");
|
||||
admin
|
||||
|
||||
@@ -140,6 +140,9 @@ impl PulsarApiProfile {
|
||||
supports_token_management: false,
|
||||
supports_raw_admin_api: true,
|
||||
supports_send_message: false,
|
||||
supports_message_query: false,
|
||||
supports_dlq: false,
|
||||
supports_message_trace: false,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -416,6 +419,7 @@ fn subscription_from_stats(name: &str, body: &serde_json::Value) -> Subscription
|
||||
msg_rate_out: body.get("msgRateOut").and_then(serde_json::Value::as_f64).unwrap_or(0.0),
|
||||
msg_throughput_out: body.get("msgThroughputOut").and_then(serde_json::Value::as_f64).unwrap_or(0.0),
|
||||
consumers,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -63,9 +63,11 @@ impl MqAdminConfig {
|
||||
let mut parsed: MqAdminConfig = serde_json::from_value(raw.clone())
|
||||
.map_err(|e| format!("Failed to parse message queue admin config: {e}"))?;
|
||||
parsed.admin_url = parsed.admin_url.trim().to_string();
|
||||
// Kafka uses bootstrap servers from `extra` instead of an admin URL,
|
||||
// so allow an empty admin_url for Kafka connections.
|
||||
if parsed.admin_url.is_empty() && parsed.system_kind != MqSystemKind::Kafka {
|
||||
// Kafka and RocketMQ use namesrv/bootstrap from `extra` instead of an admin URL.
|
||||
if parsed.admin_url.is_empty()
|
||||
&& parsed.system_kind != MqSystemKind::Kafka
|
||||
&& parsed.system_kind != MqSystemKind::RocketMq
|
||||
{
|
||||
return Err("Message queue admin URL is empty".to_string());
|
||||
}
|
||||
Ok(parsed)
|
||||
@@ -198,6 +200,22 @@ mod tests {
|
||||
assert_eq!(mqc.extra.get("bootstrapServers").and_then(|v| v.as_str()), Some("broker1:9092,broker2:9092"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_rocketmq_config_with_empty_admin_url() {
|
||||
let cfg = connection_with_external(serde_json::json!({
|
||||
"systemKind": "rocketmq",
|
||||
"adminUrl": "",
|
||||
"auth": { "kind": "none" },
|
||||
"extra": {
|
||||
"namesrvAddr": "127.0.0.1:9876"
|
||||
}
|
||||
}));
|
||||
let mqc = MqAdminConfig::from_connection(&cfg).expect("should parse valid RocketMQ config");
|
||||
assert_eq!(mqc.system_kind, MqSystemKind::RocketMq);
|
||||
assert_eq!(mqc.admin_url, "");
|
||||
assert_eq!(mqc.extra.get("namesrvAddr").and_then(|v| v.as_str()), Some("127.0.0.1:9876"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn admin_url_with_endpoint_preserves_scheme_path_and_query() {
|
||||
let rewritten =
|
||||
|
||||
@@ -29,6 +29,7 @@ use crate::db::agent_driver::AgentLaunchSpec;
|
||||
use crate::models::connection::ConnectionConfig;
|
||||
use crate::mq::adapters::kafka::KafkaAdmin;
|
||||
use crate::mq::adapters::pulsar::PulsarAdmin;
|
||||
use crate::mq::adapters::rocketmq::RocketMqAdmin;
|
||||
use crate::mq::config::MqAdminConfig;
|
||||
use crate::mq::port::MessageQueueAdmin;
|
||||
use crate::mq::types::MqSystemKind as MqSystemKindInternal;
|
||||
@@ -70,9 +71,9 @@ impl MqAdminRegistry {
|
||||
&self,
|
||||
connection_id: &str,
|
||||
mqc: MqAdminConfig,
|
||||
kafka_launch: Option<AgentLaunchSpec>,
|
||||
agent_launch: Option<AgentLaunchSpec>,
|
||||
) -> Result<Arc<dyn MessageQueueAdmin>, String> {
|
||||
let fingerprint = adapter_fingerprint(&mqc, kafka_launch.as_ref());
|
||||
let fingerprint = adapter_fingerprint(&mqc, agent_launch.as_ref());
|
||||
|
||||
// Fast path: return the cached adapter.
|
||||
if let Some(entry) = self.instances.read().await.get(connection_id) {
|
||||
@@ -96,7 +97,7 @@ impl MqAdminRegistry {
|
||||
}
|
||||
}
|
||||
|
||||
let adapter = build_adapter(mqc, kafka_launch).await?;
|
||||
let adapter = build_adapter(mqc, agent_launch).await?;
|
||||
self.instances
|
||||
.write()
|
||||
.await
|
||||
@@ -120,22 +121,22 @@ impl MqAdminRegistry {
|
||||
pub async fn build_transient_config(
|
||||
&self,
|
||||
mqc: MqAdminConfig,
|
||||
kafka_launch: Option<AgentLaunchSpec>,
|
||||
agent_launch: Option<AgentLaunchSpec>,
|
||||
) -> Result<Arc<dyn MessageQueueAdmin>, String> {
|
||||
build_adapter(mqc, kafka_launch).await
|
||||
build_adapter(mqc, agent_launch).await
|
||||
}
|
||||
}
|
||||
|
||||
fn adapter_fingerprint(mqc: &MqAdminConfig, kafka_launch: Option<&AgentLaunchSpec>) -> u64 {
|
||||
fn adapter_fingerprint(mqc: &MqAdminConfig, agent_launch: Option<&AgentLaunchSpec>) -> u64 {
|
||||
let mut hasher = std::collections::hash_map::DefaultHasher::new();
|
||||
format!("{mqc:?}").hash(&mut hasher);
|
||||
format!("{kafka_launch:?}").hash(&mut hasher);
|
||||
format!("{agent_launch:?}").hash(&mut hasher);
|
||||
hasher.finish()
|
||||
}
|
||||
|
||||
async fn build_adapter(
|
||||
mqc: MqAdminConfig,
|
||||
kafka_launch: Option<AgentLaunchSpec>,
|
||||
agent_launch: Option<AgentLaunchSpec>,
|
||||
) -> Result<Arc<dyn MessageQueueAdmin>, String> {
|
||||
match mqc.system_kind {
|
||||
MqSystemKindInternal::Pulsar => {
|
||||
@@ -143,11 +144,17 @@ async fn build_adapter(
|
||||
Ok(Arc::new(adapter))
|
||||
}
|
||||
MqSystemKindInternal::Kafka => {
|
||||
let launch = kafka_launch
|
||||
let launch = agent_launch
|
||||
.ok_or("Kafka adapter requires an agent launch spec. The Kafka agent driver is not installed or not configured.")?;
|
||||
let adapter = KafkaAdmin::new(mqc, launch).await?;
|
||||
Ok(Arc::new(adapter))
|
||||
}
|
||||
MqSystemKindInternal::RocketMq => Err("RocketMQ admin is not yet implemented".to_string()),
|
||||
MqSystemKindInternal::RocketMq => {
|
||||
let launch = agent_launch.ok_or(
|
||||
"RocketMQ adapter requires an agent launch spec. The RocketMQ agent driver is not installed or not configured.",
|
||||
)?;
|
||||
let adapter = RocketMqAdmin::new(mqc, launch).await?;
|
||||
Ok(Arc::new(adapter))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -39,6 +39,51 @@ pub trait MessageQueueAdmin: Send + Sync {
|
||||
async fn get_topic_stats(&self, topic: &TopicRef) -> Result<TopicStats, String>;
|
||||
async fn get_topic_internal_stats(&self, topic: &TopicRef) -> Result<serde_json::Value, String>;
|
||||
|
||||
async fn get_topic_route(&self, _topic: &TopicRef) -> Result<serde_json::Value, String> {
|
||||
Err("Topic route is not supported by this MQ system".to_string())
|
||||
}
|
||||
|
||||
async fn alter_topic_config(&self, _topic: &TopicRef, _configs: serde_json::Value) -> Result<(), String> {
|
||||
Err("Alter topic config is not supported by this MQ system".to_string())
|
||||
}
|
||||
|
||||
async fn skip_topic_accumulation(&self, _topic: &TopicRef) -> Result<serde_json::Value, String> {
|
||||
Err("Skip topic accumulation is not supported by this MQ system".to_string())
|
||||
}
|
||||
|
||||
async fn view_message(&self, _topic: &TopicRef, _msg_id: &str) -> Result<serde_json::Value, String> {
|
||||
Err("View message is not supported by this MQ system".to_string())
|
||||
}
|
||||
|
||||
async fn query_messages_by_key(
|
||||
&self,
|
||||
_topic: &TopicRef,
|
||||
_key: &str,
|
||||
_begin: i64,
|
||||
_end: i64,
|
||||
_max_num: u32,
|
||||
) -> Result<serde_json::Value, String> {
|
||||
Err("Message query is not supported by this MQ system".to_string())
|
||||
}
|
||||
|
||||
async fn query_messages_by_topic(
|
||||
&self,
|
||||
_topic: &TopicRef,
|
||||
_begin: i64,
|
||||
_end: i64,
|
||||
_max_num: u32,
|
||||
) -> Result<serde_json::Value, String> {
|
||||
Err("Message query is not supported by this MQ system".to_string())
|
||||
}
|
||||
|
||||
async fn query_message_trace(
|
||||
&self,
|
||||
_msg_id: &str,
|
||||
_trace_topic: Option<&str>,
|
||||
) -> Result<serde_json::Value, String> {
|
||||
Err("Message trace is not supported by this MQ system".to_string())
|
||||
}
|
||||
|
||||
// ---- Subscriptions ----
|
||||
async fn list_subscriptions(&self, topic: &TopicRef) -> Result<Vec<SubscriptionInfo>, String>;
|
||||
async fn create_subscription(&self, topic: &TopicRef, sub: &str, pos: ResetPosition) -> Result<(), String>;
|
||||
@@ -55,6 +100,16 @@ pub trait MessageQueueAdmin: Send + Sync {
|
||||
) -> Result<Vec<PeekedMessage>, String>;
|
||||
async fn expire_messages(&self, topic: &TopicRef, sub: &str, expire_seconds: i64) -> Result<(), String>;
|
||||
|
||||
/// RocketMQ: read subscription group config from broker metadata.
|
||||
async fn get_consumer_group_config(&self, _group_id: &str) -> Result<serde_json::Value, String> {
|
||||
Err("Consumer group config is not supported by this MQ system".to_string())
|
||||
}
|
||||
|
||||
/// RocketMQ: update subscription group config on brokers.
|
||||
async fn alter_consumer_group_config(&self, _group_id: &str, _config: serde_json::Value) -> Result<(), String> {
|
||||
Err("Consumer group config is not supported by this MQ system".to_string())
|
||||
}
|
||||
|
||||
// ---- Producers / consumers (runtime, read from stats) ----
|
||||
async fn list_producers(&self, topic: &TopicRef) -> Result<Vec<ProducerInfo>, String>;
|
||||
async fn list_consumers(&self, topic: &TopicRef, sub: &str) -> Result<Vec<ConsumerInfo>, String>;
|
||||
|
||||
@@ -21,8 +21,8 @@ const MAX_PEEK_MESSAGES: u32 = 100;
|
||||
pub async fn mq_test_connection_core(state: &AppState, conn_id: &str) -> Result<MqClusterInfo, String> {
|
||||
let cfg = state.configs.read().await.get(conn_id).cloned().ok_or("Connection not found")?;
|
||||
let mqc = state.mq_admin_config_for_connection(conn_id, &cfg).await?;
|
||||
let kafka_launch = resolve_kafka_launch_spec(&mqc, state);
|
||||
let adapter = match state.mq_registry.get_or_build_config(conn_id, mqc, kafka_launch).await {
|
||||
let agent_launch = resolve_mq_agent_launch_spec(&mqc, state);
|
||||
let adapter = match state.mq_registry.get_or_build_config(conn_id, mqc, agent_launch).await {
|
||||
Ok(adapter) => adapter,
|
||||
Err(err) => {
|
||||
state.mq_registry.drop_connection(conn_id).await;
|
||||
@@ -244,6 +244,26 @@ pub async fn mq_clear_backlog_core(
|
||||
adapter.clear_backlog(&topic, &sub).await
|
||||
}
|
||||
|
||||
pub async fn mq_get_consumer_group_config_core(
|
||||
state: &AppState,
|
||||
conn_id: &str,
|
||||
group_id: String,
|
||||
) -> Result<serde_json::Value, String> {
|
||||
let adapter = get_adapter(state, conn_id).await?;
|
||||
adapter.get_consumer_group_config(&group_id).await
|
||||
}
|
||||
|
||||
pub async fn mq_alter_consumer_group_config_core(
|
||||
state: &AppState,
|
||||
conn_id: &str,
|
||||
group_id: String,
|
||||
config: serde_json::Value,
|
||||
) -> Result<(), String> {
|
||||
ensure_connection_writable(state, conn_id, "Alter consumer group config").await?;
|
||||
let adapter = get_adapter(state, conn_id).await?;
|
||||
adapter.alter_consumer_group_config(&group_id, config).await
|
||||
}
|
||||
|
||||
pub async fn mq_peek_messages_core(
|
||||
state: &AppState,
|
||||
conn_id: &str,
|
||||
@@ -467,6 +487,81 @@ pub async fn mq_get_cluster_info_core(state: &AppState, conn_id: &str) -> Result
|
||||
adapter.get_cluster_info().await
|
||||
}
|
||||
|
||||
pub async fn mq_get_topic_route_core(
|
||||
state: &AppState,
|
||||
conn_id: &str,
|
||||
topic: TopicRef,
|
||||
) -> Result<serde_json::Value, String> {
|
||||
let adapter = get_adapter(state, conn_id).await?;
|
||||
adapter.get_topic_route(&topic).await
|
||||
}
|
||||
|
||||
pub async fn mq_alter_topic_config_core(
|
||||
state: &AppState,
|
||||
conn_id: &str,
|
||||
topic: TopicRef,
|
||||
configs: serde_json::Value,
|
||||
) -> Result<(), String> {
|
||||
ensure_connection_writable(state, conn_id, "Alter topic config").await?;
|
||||
let adapter = get_adapter(state, conn_id).await?;
|
||||
adapter.alter_topic_config(&topic, configs).await
|
||||
}
|
||||
|
||||
pub async fn mq_skip_topic_accumulation_core(
|
||||
state: &AppState,
|
||||
conn_id: &str,
|
||||
topic: TopicRef,
|
||||
) -> Result<serde_json::Value, String> {
|
||||
ensure_connection_writable(state, conn_id, "Skip topic accumulation").await?;
|
||||
let adapter = get_adapter(state, conn_id).await?;
|
||||
adapter.skip_topic_accumulation(&topic).await
|
||||
}
|
||||
|
||||
pub async fn mq_view_message_core(
|
||||
state: &AppState,
|
||||
conn_id: &str,
|
||||
topic: TopicRef,
|
||||
msg_id: String,
|
||||
) -> Result<serde_json::Value, String> {
|
||||
let adapter = get_adapter(state, conn_id).await?;
|
||||
adapter.view_message(&topic, &msg_id).await
|
||||
}
|
||||
|
||||
pub async fn mq_query_messages_by_key_core(
|
||||
state: &AppState,
|
||||
conn_id: &str,
|
||||
topic: TopicRef,
|
||||
key: String,
|
||||
begin: i64,
|
||||
end: i64,
|
||||
max_num: u32,
|
||||
) -> Result<serde_json::Value, String> {
|
||||
let adapter = get_adapter(state, conn_id).await?;
|
||||
adapter.query_messages_by_key(&topic, &key, begin, end, max_num).await
|
||||
}
|
||||
|
||||
pub async fn mq_query_messages_by_topic_core(
|
||||
state: &AppState,
|
||||
conn_id: &str,
|
||||
topic: TopicRef,
|
||||
begin: i64,
|
||||
end: i64,
|
||||
max_num: u32,
|
||||
) -> Result<serde_json::Value, String> {
|
||||
let adapter = get_adapter(state, conn_id).await?;
|
||||
adapter.query_messages_by_topic(&topic, begin, end, max_num).await
|
||||
}
|
||||
|
||||
pub async fn mq_query_message_trace_core(
|
||||
state: &AppState,
|
||||
conn_id: &str,
|
||||
msg_id: String,
|
||||
trace_topic: Option<String>,
|
||||
) -> Result<serde_json::Value, String> {
|
||||
let adapter = get_adapter(state, conn_id).await?;
|
||||
adapter.query_message_trace(&msg_id, trace_topic.as_deref()).await
|
||||
}
|
||||
|
||||
// ---- Raw request (escape hatch) ----
|
||||
|
||||
pub async fn mq_raw_request_core(state: &AppState, conn_id: &str, req: MqRawRequest) -> Result<MqRawResponse, String> {
|
||||
@@ -500,31 +595,38 @@ async fn get_adapter(
|
||||
) -> Result<std::sync::Arc<dyn crate::mq::port::MessageQueueAdmin>, String> {
|
||||
let cfg = state.configs.read().await.get(conn_id).cloned().ok_or("Connection not found")?;
|
||||
let mqc = state.mq_admin_config_for_connection(conn_id, &cfg).await?;
|
||||
let kafka_launch = resolve_kafka_launch_spec(&mqc, state);
|
||||
state.mq_registry.get_or_build_config(conn_id, mqc, kafka_launch).await
|
||||
let agent_launch = resolve_mq_agent_launch_spec(&mqc, state);
|
||||
state.mq_registry.get_or_build_config(conn_id, mqc, agent_launch).await
|
||||
}
|
||||
|
||||
/// Resolve the Kafka agent launch spec if the config targets Kafka.
|
||||
/// Returns `None` for non-Kafka systems so the registry skips agent resolution.
|
||||
pub fn resolve_kafka_launch_spec(mqc: &MqAdminConfig, state: &AppState) -> Option<AgentLaunchSpec> {
|
||||
if mqc.system_kind != MqSystemKind::Kafka {
|
||||
return None;
|
||||
}
|
||||
/// Resolve the MQ agent launch spec for agent-backed systems (Kafka, RocketMQ).
|
||||
/// Returns `None` for native REST systems so the registry skips agent resolution.
|
||||
pub fn resolve_mq_agent_launch_spec(mqc: &MqAdminConfig, state: &AppState) -> Option<AgentLaunchSpec> {
|
||||
let agent_key = match mqc.system_kind {
|
||||
MqSystemKind::Kafka => "kafka",
|
||||
MqSystemKind::RocketMq => "rocketmq",
|
||||
_ => return None,
|
||||
};
|
||||
let agent_state = state.agent_manager.load_state();
|
||||
let jre_key = agent_state
|
||||
.installed_drivers
|
||||
.get("kafka")
|
||||
.get(agent_key)
|
||||
.map(|driver| driver.jre.as_str())
|
||||
.unwrap_or(crate::agent_manager::DEFAULT_JRE_KEY);
|
||||
match state.agent_manager.resolve_agent_launch_spec(&agent_state, "kafka", jre_key) {
|
||||
match state.agent_manager.resolve_agent_launch_spec(&agent_state, agent_key, jre_key) {
|
||||
Ok(launch) => Some(launch),
|
||||
Err(err) => {
|
||||
log::warn!("Failed to resolve Kafka agent launch spec: {err}");
|
||||
log::warn!("Failed to resolve {agent_key} agent launch spec: {err}");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[deprecated(note = "use resolve_mq_agent_launch_spec")]
|
||||
pub fn resolve_kafka_launch_spec(mqc: &MqAdminConfig, state: &AppState) -> Option<AgentLaunchSpec> {
|
||||
resolve_mq_agent_launch_spec(mqc, state)
|
||||
}
|
||||
|
||||
async fn ensure_connection_writable(state: &AppState, conn_id: &str, operation: &str) -> Result<(), String> {
|
||||
let configs = state.configs.read().await;
|
||||
if let Some(config) = configs.get(conn_id) {
|
||||
@@ -640,6 +742,8 @@ mod tests {
|
||||
topic: "orders".to_string(),
|
||||
persistent: true,
|
||||
partitioned: None,
|
||||
message_type: None,
|
||||
..TopicRef::default()
|
||||
},
|
||||
None,
|
||||
)
|
||||
@@ -682,6 +786,8 @@ mod tests {
|
||||
topic: "orders".to_string(),
|
||||
persistent: true,
|
||||
partitioned: None,
|
||||
message_type: None,
|
||||
..TopicRef::default()
|
||||
},
|
||||
"sub-a".to_string(),
|
||||
101,
|
||||
|
||||
@@ -29,9 +29,8 @@ impl MqSystemKind {
|
||||
/// Capability flags. The frontend reads these to show/hide functionality, and
|
||||
/// the adapter computes them from the detected server version so unsupported
|
||||
/// features are hidden rather than failing at call time.
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
#[derive(Default)]
|
||||
pub struct MqCapabilities {
|
||||
pub supports_tenants: bool,
|
||||
pub supports_namespaces: bool,
|
||||
@@ -53,6 +52,15 @@ pub struct MqCapabilities {
|
||||
/// Whether the adapter supports producing messages to topics.
|
||||
#[serde(default)]
|
||||
pub supports_send_message: bool,
|
||||
/// RocketMQ: view/query messages by msgId or key.
|
||||
#[serde(default)]
|
||||
pub supports_message_query: bool,
|
||||
/// RocketMQ: dedicated dead-letter browsing.
|
||||
#[serde(default)]
|
||||
pub supports_dlq: bool,
|
||||
/// RocketMQ: message trace lookup (requires broker trace topic).
|
||||
#[serde(default)]
|
||||
pub supports_message_trace: bool,
|
||||
}
|
||||
|
||||
/// Result of a connectivity test, including the detected server version and how
|
||||
@@ -228,12 +236,41 @@ pub struct TopicRef {
|
||||
/// from the partitioned topic list.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub partitioned: Option<bool>,
|
||||
/// RocketMQ-only create hint: NORMAL / DELAY / FIFO / TRANSACTION.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub message_type: Option<String>,
|
||||
/// RocketMQ-only: target broker name for create/delete/update.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub broker_name: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub read_queue_nums: Option<u32>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub write_queue_nums: Option<u32>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub perm: Option<u32>,
|
||||
}
|
||||
|
||||
fn default_true() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
impl Default for TopicRef {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
tenant: String::new(),
|
||||
namespace: String::new(),
|
||||
topic: String::new(),
|
||||
persistent: true,
|
||||
partitioned: None,
|
||||
message_type: None,
|
||||
broker_name: None,
|
||||
read_queue_nums: None,
|
||||
write_queue_nums: None,
|
||||
perm: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TopicRef {
|
||||
/// The URL domain segment: `persistent` or `non-persistent`.
|
||||
pub fn domain(&self) -> &'static str {
|
||||
@@ -244,7 +281,7 @@ impl TopicRef {
|
||||
}
|
||||
}
|
||||
|
||||
/// `{tenant}/{namespace}/{topic}` — the path used by most Pulsar endpoints.
|
||||
/// `{tenant}/{namespace}/{topic}` - the path used by most Pulsar endpoints.
|
||||
pub fn path(&self) -> String {
|
||||
format!("{}/{}/{}", self.tenant, self.namespace, self.topic)
|
||||
}
|
||||
@@ -266,6 +303,12 @@ pub struct TopicInfo {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub partitions: Option<u32>,
|
||||
pub persistent: bool,
|
||||
/// Kafka/RocketMQ internal or system topic; hidden by default in the MQ console UI.
|
||||
#[serde(default)]
|
||||
pub internal: bool,
|
||||
/// RocketMQ message type from broker topic config (NORMAL, DELAY, FIFO, etc.).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub message_type: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize)]
|
||||
@@ -321,6 +364,10 @@ pub struct BrokerNode {
|
||||
pub host: String,
|
||||
pub port: i32,
|
||||
pub rack: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub broker_name: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub role: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
@@ -334,6 +381,18 @@ pub struct SubscriptionInfo {
|
||||
pub msg_throughput_out: f64,
|
||||
#[serde(default)]
|
||||
pub consumers: Vec<ConsumerInfo>,
|
||||
/// RocketMQ: subscribed topic names for this consumer group (cluster-wide listing).
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub topics: Vec<String>,
|
||||
/// RocketMQ: online consumer client count (cluster-wide listing).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub online_members: Option<u32>,
|
||||
/// RocketMQ: NORMAL / FIFO / SYSTEM (aligned with Dashboard).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub consumer_group_type: Option<String>,
|
||||
/// RocketMQ: CLUSTERING / BROADCASTING.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub message_model: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
@@ -364,7 +423,7 @@ pub struct ProducerInfo {
|
||||
|
||||
/// Where to position a cursor when creating a subscription or resetting it.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "camelCase")]
|
||||
#[serde(tag = "kind", rename_all = "camelCase", rename_all_fields = "camelCase")]
|
||||
pub enum ResetPosition {
|
||||
/// Earliest available message.
|
||||
Earliest,
|
||||
@@ -378,7 +437,7 @@ pub enum ResetPosition {
|
||||
|
||||
/// How many messages to skip on a subscription.
|
||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", rename_all = "camelCase")]
|
||||
#[serde(tag = "kind", rename_all = "camelCase", rename_all_fields = "camelCase")]
|
||||
pub enum SkipCount {
|
||||
All,
|
||||
Count { count: u32 },
|
||||
@@ -459,7 +518,7 @@ impl PolicyScope {
|
||||
namespace: namespace.clone(),
|
||||
topic: topic.clone(),
|
||||
persistent: *persistent,
|
||||
partitioned: None,
|
||||
..Default::default()
|
||||
}),
|
||||
_ => None,
|
||||
}
|
||||
@@ -547,7 +606,7 @@ pub type PermissionMap = HashMap<String, Vec<AuthAction>>;
|
||||
|
||||
/// A raw admin REST request, proxied through the adapter to cover any endpoint
|
||||
/// the typed methods do not. The path is appended to the connection's
|
||||
/// `admin_url` base — arbitrary hosts are not allowed (SSRF guard).
|
||||
/// `admin_url` base ? arbitrary hosts are not allowed (SSRF guard).
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct MqRawRequest {
|
||||
@@ -623,3 +682,19 @@ pub struct SendMessageResponse {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub timestamp: Option<String>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::ResetPosition;
|
||||
|
||||
#[test]
|
||||
fn reset_position_deserializes_camel_case_variant_fields() {
|
||||
let pos: ResetPosition = serde_json::from_str(r#"{"kind":"timestamp","timestampMs":1710000000000}"#)
|
||||
.expect("timestamp reset position");
|
||||
assert!(matches!(pos, ResetPosition::Timestamp { timestamp_ms: 1710000000000 }));
|
||||
|
||||
let pos: ResetPosition = serde_json::from_str(r#"{"kind":"messageId","ledgerId":5,"entryId":9}"#)
|
||||
.expect("message-id reset position");
|
||||
assert!(matches!(pos, ResetPosition::MessageId { ledger_id: 5, entry_id: 9 }));
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user