feat(rocketmq): improve connections, topics, and message queries

This commit is contained in:
AiLi authored and GitHub committed 2026-08-04 01:38:29 +08:00
1 parent 0ece86c478
commit ce7b8c7b74
22 files changed
+1166 -407

No files matched your search

+27 -16
View File
@@ -2170,23 +2170,37 @@ impl AppState {
// connection_id is recognized as valid.
let mqc = self.mq_admin_config_for_connection(connection_id, &config).await?;
let agent_launch = crate::mq::service::resolve_mq_agent_launch_spec(&mqc, self);
let adapter = match self.mq_registry.get_or_build_config(connection_id, mqc, agent_launch).await {
Ok(adapter) => adapter,
Err(err) => {
// Temporary "__test_*" probes must not retain agents in the registry.
// reconnect fast-path caching only applies to durable connection ids;
// drain_connection_pools no longer drops MQ adapters (reconnect reuse).
if connection_id.starts_with("__test_") {
let adapter = self.mq_registry.build_transient_config(mqc, agent_launch).await?;
adapter.test_connection().await?;
if let Err(err) = self.ensure_current_connection_attempt(connection_id, connection_attempt).await {
self.reset_connection_transport_for_config(connection_id, &db_config).await;
return Err(err);
}
// adapter drops here and kills agent-backed processes (RocketMQ/Kafka/RabbitMQ).
PoolKind::MessageQueue
} else {
let build = match self.mq_registry.get_or_build_config(connection_id, mqc, agent_launch).await {
Ok(build) => build,
Err(err) => {
self.mq_registry.drop_connection(connection_id).await;
return Err(err);
}
};
if let Err(err) = crate::mq::validate_mq_adapter_after_build(&build).await {
self.mq_registry.drop_connection(connection_id).await;
return Err(err);
}
};
if let Err(err) = adapter.test_connection().await {
self.mq_registry.drop_connection(connection_id).await;
return Err(err);
if let Err(err) = self.ensure_current_connection_attempt(connection_id, connection_attempt).await {
self.mq_registry.drop_connection(connection_id).await;
self.reset_connection_transport_for_config(connection_id, &db_config).await;
return Err(err);
}
PoolKind::MessageQueue
}
if let Err(err) = self.ensure_current_connection_attempt(connection_id, connection_attempt).await {
self.mq_registry.drop_connection(connection_id).await;
self.reset_connection_transport_for_config(connection_id, &db_config).await;
return Err(err);
}
PoolKind::MessageQueue
}
#[cfg(not(feature = "mq-admin"))]
DatabaseType::MessageQueue => {
@@ -3943,9 +3957,6 @@ impl AppState {
}
}
drop(conns);
// Also drop the MQ admin adapter if this is an MQ connection.
#[cfg(feature = "mq-admin")]
self.mq_registry.drop_connection(connection_id).await;
removed
}
+98 -90
View File
@@ -52,13 +52,46 @@ const ROCKETMQ_CAPABILITIES: MqCapabilities = MqCapabilities {
supports_cluster_monitoring: false,
};
const TOPIC_LIST_PAGE_SIZE: u32 = 200;
/// Prefer one RPC for the full topic catalog.
///
/// Use a large *positive* limit (not `0`): older agents coerce `limit <= 0` back to 200,
/// which truncates catalogs (e.g. 338 topics → 200 rows → ~167 after type filters).
/// Current agents treat `limit <= 0` as "all"; a large positive limit returns the same
/// full page via normal pagination math without that version skew.
const TOPIC_LIST_FETCH_LIMIT: i32 = i32::MAX;
/// Fallback page size when an agent still returns a truncated first page (`topics.len() < total`).
const TOPIC_LIST_FALLBACK_PAGE_SIZE: i32 = 200;
pub struct RocketMqAdmin {
client: Arc<Mutex<AgentDriverClient>>,
config: MqAdminConfig,
}
fn cluster_info_from_agent_result(result: &serde_json::Value) -> MqClusterInfo {
let cluster_id = result.get("clusterId").and_then(|v| v.as_str()).map(String::from);
let brokers = result.get("brokers").cloned().unwrap_or(serde_json::json!([]));
// When the broker has no authorizer configured, disable permissions in the UI
// so the frontend hides the tab instead of showing raw errors.
let acl_enabled = result.get("aclEnabled").and_then(|v| v.as_bool()).unwrap_or(true);
let mut caps = ROCKETMQ_CAPABILITIES;
if !acl_enabled {
caps.supports_permissions = false;
}
MqClusterInfo {
system_kind: MqSystemKind::RocketMq,
server_version: None,
resolved_profile: "rocketmq-agent".to_string(),
version_detection: "agent".to_string(),
capabilities: caps,
extra: serde_json::json!({
"clusterId": cluster_id,
"brokers": brokers,
}),
}
}
impl RocketMqAdmin {
/// Spawn the RocketMQ Java agent, perform handshake, and connect.
pub async fn new(cfg: MqAdminConfig, launch: AgentLaunchSpec) -> Result<Self, String> {
@@ -104,33 +137,16 @@ impl MessageQueueAdmin for RocketMqAdmin {
MqSystemKind::RocketMq
}
fn build_includes_connect_test(&self) -> bool {
true
}
async fn test_connection(&self) -> Result<MqClusterInfo, String> {
let conn_params = build_connection_params(&self.config);
let result: serde_json::Value =
self.call("test_connection", serde_json::json!({ "connection": conn_params })).await?;
let cluster_id = result.get("clusterId").and_then(|v| v.as_str()).map(String::from);
let brokers = result.get("brokers").cloned().unwrap_or(serde_json::json!([]));
// When the broker has no authorizer configured, disable permissions in the UI
// so the frontend hides the tab instead of showing raw errors.
let acl_enabled = result.get("aclEnabled").and_then(|v| v.as_bool()).unwrap_or(true);
let mut caps = ROCKETMQ_CAPABILITIES;
if !acl_enabled {
caps.supports_permissions = false;
}
Ok(MqClusterInfo {
system_kind: MqSystemKind::RocketMq,
server_version: None,
resolved_profile: "rocketmq-agent".to_string(),
version_detection: "agent".to_string(),
capabilities: caps,
extra: serde_json::json!({
"clusterId": cluster_id,
"brokers": brokers,
}),
})
Ok(cluster_info_from_agent_result(&result))
}
// ---- Tenants (not supported by RocketMQ) ----
@@ -176,35 +192,40 @@ impl MessageQueueAdmin for RocketMqAdmin {
// ---- Topics ----
async fn list_topics(&self, _ns: &NamespaceRef, _opts: ListTopicsOpts) -> Result<Vec<TopicInfo>, String> {
let mut all = Vec::new();
let mut offset: u32 = 0;
// Prefer a single RPC so the agent builds the catalog once. If the agent still
// truncates (old coerce-to-200 behavior), page the remainder using `total`.
let result: serde_json::Value = self
.call(
"mq_list_topics",
serde_json::json!({
"keyword": "",
"limit": TOPIC_LIST_FETCH_LIMIT,
"offset": 0,
}),
)
.await?;
loop {
let result: serde_json::Value = self
let mut all = topic_infos_from_agent_list_response(&result);
let total = agent_list_total(&result, all.len());
let mut offset = all.len();
while (offset as u64) < total {
let page: serde_json::Value = self
.call(
"mq_list_topics",
serde_json::json!({
"keyword": "",
"limit": TOPIC_LIST_PAGE_SIZE,
"limit": TOPIC_LIST_FALLBACK_PAGE_SIZE,
"offset": offset,
}),
)
.await?;
let topics = result.get("topics").and_then(|v| v.as_array()).cloned().unwrap_or_default();
let page_len = topics.len();
for topic in topics {
all.push(topic_info_from_agent_value(&topic));
}
let total = result.get("total").and_then(|v| v.as_u64()).unwrap_or(offset as u64 + page_len as u64);
let fetch_next = topic_list_should_fetch_next(offset, page_len, total);
offset = offset.saturating_add(page_len as u32);
if !fetch_next {
let batch = topic_infos_from_agent_list_response(&page);
if batch.is_empty() {
break;
}
offset = offset.saturating_add(batch.len());
all.extend(batch);
}
Ok(all)
}
@@ -735,28 +756,13 @@ fn topic_info_from_agent_value(t: &serde_json::Value) -> TopicInfo {
}
}
/// Whether another Agent topic list page should be requested after consuming `page_len` rows.
fn topic_list_should_fetch_next(offset: u32, page_len: usize, total: u64) -> bool {
page_len > 0 && u64::from(offset) + (page_len as u64) < total
fn topic_infos_from_agent_list_response(response: &serde_json::Value) -> Vec<TopicInfo> {
response.get("topics").and_then(|v| v.as_array()).into_iter().flatten().map(topic_info_from_agent_value).collect()
}
#[cfg(test)]
fn topic_infos_from_agent_pages(pages: &[serde_json::Value]) -> Vec<TopicInfo> {
let mut all = Vec::new();
let mut offset: u32 = 0;
for page in pages {
let topics = page.get("topics").and_then(|v| v.as_array()).cloned().unwrap_or_default();
let page_len = topics.len();
for topic in topics {
all.push(topic_info_from_agent_value(&topic));
}
let total = page.get("total").and_then(|v| v.as_u64()).unwrap_or(offset as u64 + page_len as u64);
offset = offset.saturating_add(page_len as u32);
if page_len == 0 || offset as u64 >= total {
break;
}
}
all
/// Prefer agent-reported `total`; fall back to the page length when absent.
fn agent_list_total(response: &serde_json::Value, page_len: usize) -> u64 {
response.get("total").and_then(|v| v.as_u64()).unwrap_or(page_len as u64)
}
/// Extract RocketMQ NameServer address from MqAdminConfig.extra.
@@ -1043,40 +1049,28 @@ mod tests {
}
#[test]
fn topic_list_should_fetch_next_when_more_rows_remain() {
assert!(!topic_list_should_fetch_next(200, 0, 201));
assert!(topic_list_should_fetch_next(0, 200, 201));
assert!(!topic_list_should_fetch_next(0, 200, 200));
assert!(topic_list_should_fetch_next(200, 1, 450));
assert!(!topic_list_should_fetch_next(400, 50, 450));
fn topic_infos_from_agent_list_response_parses_full_catalog() {
let topics: Vec<serde_json::Value> = (0..341)
.map(|i| serde_json::json!({ "name": format!("topic-{i}"), "partitions": 4, "messageType": "UNSPECIFIED" }))
.collect();
let response = serde_json::json!({ "topics": topics, "total": 341, "offset": 0, "limit": i32::MAX });
let parsed = topic_infos_from_agent_list_response(&response);
assert_eq!(parsed.len(), 341);
assert_eq!(agent_list_total(&response, parsed.len()), 341);
assert_eq!(parsed.first().map(|t| t.name.as_str()), Some("topic-0"));
assert_eq!(parsed.last().map(|t| t.name.as_str()), Some("topic-340"));
assert_eq!(parsed[0].message_type.as_deref(), Some("UNSPECIFIED"));
}
#[test]
fn topic_infos_from_agent_pages_merges_200_201_and_multi_page_totals() {
let page1: Vec<serde_json::Value> =
fn agent_list_total_detects_truncated_first_page() {
let page: Vec<serde_json::Value> =
(0..200).map(|i| serde_json::json!({ "name": format!("topic-{i}"), "partitions": 4 })).collect();
let page2_201 = serde_json::json!({ "name": "topic-200", "partitions": 4 });
let response_201_first = serde_json::json!({ "topics": page1, "total": 201, "offset": 0, "limit": 200 });
let response_201_second =
serde_json::json!({ "topics": [page2_201], "total": 201, "offset": 200, "limit": 200 });
let merged_201 = topic_infos_from_agent_pages(&[response_201_first, response_201_second]);
assert_eq!(merged_201.len(), 201);
assert_eq!(merged_201.last().map(|t| t.name.as_str()), Some("topic-200"));
let page_a: Vec<serde_json::Value> =
(0..200).map(|i| serde_json::json!({ "name": format!("p-{i}"), "partitions": 1 })).collect();
let page_b: Vec<serde_json::Value> =
(200..400).map(|i| serde_json::json!({ "name": format!("p-{i}"), "partitions": 1 })).collect();
let page_c: Vec<serde_json::Value> =
(400..450).map(|i| serde_json::json!({ "name": format!("p-{i}"), "partitions": 1 })).collect();
let merged_450 = topic_infos_from_agent_pages(&[
serde_json::json!({ "topics": page_a, "total": 450 }),
serde_json::json!({ "topics": page_b, "total": 450 }),
serde_json::json!({ "topics": page_c, "total": 450 }),
]);
assert_eq!(merged_450.len(), 450);
assert_eq!(merged_450.first().map(|t| t.name.as_str()), Some("p-0"));
assert_eq!(merged_450.last().map(|t| t.name.as_str()), Some("p-449"));
let response = serde_json::json!({ "topics": page, "total": 338, "offset": 0, "limit": 200 });
let parsed = topic_infos_from_agent_list_response(&response);
assert_eq!(parsed.len(), 200);
assert_eq!(agent_list_total(&response, parsed.len()), 338);
assert!((parsed.len() as u64) < agent_list_total(&response, parsed.len()));
}
#[test]
@@ -1102,4 +1096,18 @@ mod tests {
assert_eq!(peeked.properties.get("tag").map(String::as_str), Some("cs-pt-dlq-test"));
assert_eq!(peeked.publish_time.as_deref(), Some("1710000000000"));
}
#[test]
fn cluster_info_from_agent_result_parses_acl_and_brokers() {
let result = serde_json::json!({
"ok": true,
"clusterId": "DefaultCluster",
"brokers": [{"brokerName": "broker-a"}],
"aclEnabled": false
});
let info = super::cluster_info_from_agent_result(&result);
assert_eq!(info.system_kind, MqSystemKind::RocketMq);
assert!(!info.capabilities.supports_permissions);
assert_eq!(info.extra.get("clusterId").and_then(|v| v.as_str()), Some("DefaultCluster"));
}
}
+34 -5
View File
@@ -56,6 +56,13 @@ struct CachedMqAdmin {
adapter: Arc<dyn MessageQueueAdmin>,
}
/// Result of resolving an MQ admin adapter for a connection attempt.
pub struct MqBuildResult {
pub adapter: Arc<dyn MessageQueueAdmin>,
/// True when an existing cached adapter was reused (reconnect fast path).
pub was_cached: bool,
}
impl MqAdminRegistry {
pub fn new() -> Self {
Self { instances: RwLock::new(HashMap::new()), build_locks: RwLock::new(HashMap::new()) }
@@ -63,7 +70,7 @@ impl MqAdminRegistry {
/// Return the cached adapter for this connection, building it from the
/// connection's `external_config` if not already present.
pub async fn get_or_build(&self, cfg: &ConnectionConfig) -> Result<Arc<dyn MessageQueueAdmin>, String> {
pub async fn get_or_build(&self, cfg: &ConnectionConfig) -> Result<MqBuildResult, String> {
let mqc = MqAdminConfig::from_connection(cfg)?;
self.get_or_build_config(&cfg.id, mqc, None).await
}
@@ -73,13 +80,13 @@ impl MqAdminRegistry {
connection_id: &str,
mqc: MqAdminConfig,
agent_launch: Option<AgentLaunchSpec>,
) -> Result<Arc<dyn MessageQueueAdmin>, String> {
) -> Result<MqBuildResult, String> {
let fingerprint = adapter_fingerprint(&mqc, agent_launch.as_ref());
// Fast path: return the cached adapter.
if let Some(entry) = self.instances.read().await.get(connection_id) {
if entry.fingerprint == fingerprint {
return Ok(entry.adapter.clone());
return Ok(MqBuildResult { adapter: entry.adapter.clone(), was_cached: true });
}
}
@@ -94,16 +101,19 @@ impl MqAdminRegistry {
// Another task may have built it while we were waiting for the lock.
if let Some(entry) = self.instances.read().await.get(connection_id) {
if entry.fingerprint == fingerprint {
return Ok(entry.adapter.clone());
return Ok(MqBuildResult { adapter: entry.adapter.clone(), was_cached: true });
}
}
// Config changed — drop the stale adapter so its agent process is released.
self.instances.write().await.remove(connection_id);
let adapter = build_adapter(mqc, agent_launch).await?;
self.instances
.write()
.await
.insert(connection_id.to_string(), CachedMqAdmin { fingerprint, adapter: adapter.clone() });
Ok(adapter)
Ok(MqBuildResult { adapter, was_cached: false })
}
/// Drop the cached adapter for a connection (called on disconnect).
@@ -112,6 +122,16 @@ impl MqAdminRegistry {
self.build_locks.write().await.remove(connection_id);
}
/// Whether an adapter is currently cached for this connection id.
pub async fn has_cached_connection(&self, connection_id: &str) -> bool {
self.instances.read().await.contains_key(connection_id)
}
/// Connection ids currently holding a cached MQ adapter (for cleanup assertions).
pub async fn cached_connection_ids(&self) -> Vec<String> {
self.instances.read().await.keys().cloned().collect()
}
/// Build a fresh adapter without caching it — used for connection tests
/// where we don't want to retain state.
pub async fn build_transient(&self, cfg: &ConnectionConfig) -> Result<Arc<dyn MessageQueueAdmin>, String> {
@@ -128,6 +148,15 @@ impl MqAdminRegistry {
}
}
/// Validate connectivity after resolving an MQ adapter. Skips an immediate probe when
/// the adapter was just built and its connect path already verified the cluster.
pub async fn validate_mq_adapter_after_build(build: &MqBuildResult) -> Result<(), String> {
if build.was_cached || !build.adapter.build_includes_connect_test() {
build.adapter.test_connection().await?;
}
Ok(())
}
fn adapter_fingerprint(mqc: &MqAdminConfig, agent_launch: Option<&AgentLaunchSpec>) -> u64 {
let mut hasher = std::collections::hash_map::DefaultHasher::new();
format!("{mqc:?}").hash(&mut hasher);
+7
View File
@@ -18,6 +18,13 @@ pub trait MessageQueueAdmin: Send + Sync {
/// Connectivity test; returns cluster/version info.
async fn test_connection(&self) -> Result<MqClusterInfo, String>;
/// When true, the adapter's build path already validated connectivity (e.g.
/// RocketMQ agent `connect` RPC). Callers may skip an immediate follow-up
/// `test_connection` on first build, but should still test on cache reuse.
fn build_includes_connect_test(&self) -> bool {
false
}
// ---- Tenants ----
async fn list_tenants(&self) -> Result<Vec<TenantInfo>, String>;
async fn get_tenant(&self, name: &str) -> Result<TenantInfo, String>;
+4 -4
View File
@@ -22,14 +22,14 @@ pub async fn mq_test_connection_core(state: &AppState, conn_id: &str) -> Result<
let cfg = state.configs.read().await.get(conn_id).cloned().ok_or("Connection not found")?;
let mqc = state.mq_admin_config_for_connection(conn_id, &cfg).await?;
let agent_launch = resolve_mq_agent_launch_spec(&mqc, state);
let adapter = match state.mq_registry.get_or_build_config(conn_id, mqc, agent_launch).await {
Ok(adapter) => adapter,
let build = match state.mq_registry.get_or_build_config(conn_id, mqc, agent_launch).await {
Ok(build) => build,
Err(err) => {
state.mq_registry.drop_connection(conn_id).await;
return Err(err);
}
};
match adapter.test_connection().await {
match build.adapter.test_connection().await {
Ok(info) => Ok(info),
Err(err) => {
state.mq_registry.drop_connection(conn_id).await;
@@ -801,7 +801,7 @@ async fn get_adapter(
let cfg = state.configs.read().await.get(conn_id).cloned().ok_or("Connection not found")?;
let mqc = state.mq_admin_config_for_connection(conn_id, &cfg).await?;
let agent_launch = resolve_mq_agent_launch_spec(&mqc, state);
state.mq_registry.get_or_build_config(conn_id, mqc, agent_launch).await
state.mq_registry.get_or_build_config(conn_id, mqc, agent_launch).await.map(|build| build.adapter)
}
/// Resolve the MQ agent launch spec for agent-backed systems (Kafka, RocketMQ, RabbitMQ).