diff --git a/agents/drivers/rocketmq/src/main/java/com/dbx/agent/rocketmq/RocketMqAgent.java b/agents/drivers/rocketmq/src/main/java/com/dbx/agent/rocketmq/RocketMqAgent.java index b644299506..a428885d9f 100644 --- a/agents/drivers/rocketmq/src/main/java/com/dbx/agent/rocketmq/RocketMqAgent.java +++ b/agents/drivers/rocketmq/src/main/java/com/dbx/agent/rocketmq/RocketMqAgent.java @@ -63,15 +63,26 @@ public final class RocketMqAgent { private static final int DEFAULT_REQUEST_TIMEOUT_MS = 30_000; private static final int DEFAULT_LIST_LIMIT = 200; private static final int CONSUMER_GROUP_ENRICH_CONCURRENCY = 8; - private static final long CONSUMER_GROUP_ENRICH_BUDGET_MS = 8_000; - /** Wall-clock budget for parallel getAllSubscriptionGroup across masters. */ - private static final long CONSUMER_GROUP_COLLECT_BUDGET_MS = 12_000; private static final int CONSUMER_GROUP_COLLECT_CONCURRENCY = 8; - private static final long CONSUMER_LAG_BUDGET_MS = 10_000; private static final int CONSUMER_LAG_CONCURRENCY = 8; + /** + * Phase baselines at {@link #DEFAULT_REQUEST_TIMEOUT_MS}. Scaled via + * {@link #scaledOperationBudgetMs} so Advanced query timeout expands headroom without letting + * collect+enrich+lag each consume a full RPC window. + */ + private static final long CONSUMER_GROUP_COLLECT_BUDGET_BASELINE_MS = 12_000; + private static final long CONSUMER_GROUP_ENRICH_BUDGET_BASELINE_MS = 8_000; + private static final long CONSUMER_LAG_BUDGET_BASELINE_MS = 10_000; + private static final long TOPIC_LIST_ROUTE_BUDGET_BASELINE_MS = 8_000; + /** Host-side Docker collision-fallbacks are usually blackholed; keep remoting short. */ + private static final long COLLISION_FALLBACK_PROBE_TIMEOUT_MS = 2_000; + /** + * LitePullConsumer.poll wait for local pull cache — not an RPC budget. + * Must stay short: empty queues block for the full timeout, and peek walks queues serially. + */ + private static final long MESSAGE_POLL_TIMEOUT_MS = 3_000; /** Nameserver route enrichment for listTopics fallback when bulk broker config is unavailable. */ private static final int TOPIC_LIST_ROUTE_CONCURRENCY = 8; - private static final long TOPIC_LIST_ROUTE_BUDGET_MS = 8_000; private static final String AUTO_CREATE_TOPIC_KEY = "TBW102"; /** RocketMQ 5.x topic attribute key for message type (NORMAL/DELAY/FIFO/TRANSACTION). */ private static final String TOPIC_MESSAGE_TYPE_ATTRIBUTE = "message.type"; @@ -494,7 +505,7 @@ public final class RocketMqAgent { keyword, admin::examineTopicRouteInfo, TOPIC_LIST_ROUTE_CONCURRENCY, - TOPIC_LIST_ROUTE_BUDGET_MS + scaledOperationBudgetMs(conn, TOPIC_LIST_ROUTE_BUDGET_BASELINE_MS) ); topics.sort(Comparator.comparing(m -> String.valueOf(m.get("name")))); @@ -873,7 +884,7 @@ public final class RocketMqAgent { consumer.assign(Collections.singletonList(queue)); consumer.seek(queue, offset); consumer.setPullBatchSize(1); - List polled = consumer.poll(3000); + List polled = consumer.poll(MESSAGE_POLL_TIMEOUT_MS); if (polled == null || polled.isEmpty()) { continue; } @@ -1125,45 +1136,43 @@ public final class RocketMqAgent { private static Map collectConsumerGroupConfigs( DefaultMQAdminExt admin, JsonObject conn) throws Exception { - long requestTimeoutMs = intOrDefault(conn, "request_timeout_ms", DEFAULT_REQUEST_TIMEOUT_MS); - // Prefer connection RPC timeout over the legacy 5s hard-code so slow-but-alive brokers - // still contribute within the shared wall-clock budget. - long perBrokerMs = Math.min(Math.max(1_000L, requestTimeoutMs), CONSUMER_GROUP_COLLECT_BUDGET_MS); + long budgetMs = scaledOperationBudgetMs(conn, CONSUMER_GROUP_COLLECT_BUDGET_BASELINE_MS); + MasterBrokerAddrPlan plan = resolveMasterBrokerAddrPlan(admin, conn); + // Fallbacks are usually Docker-internal and blackholed from host agents; keep their + // remoting short so raising Advanced query timeout cannot stretch collect wall time. + long fallbackPerBrokerMs = Math.min(budgetMs, COLLISION_FALLBACK_PROBE_TIMEOUT_MS); return collectConsumerGroupConfigs( - admin, - conn, + plan, CONSUMER_GROUP_COLLECT_CONCURRENCY, - CONSUMER_GROUP_COLLECT_BUDGET_MS, - perBrokerMs + budgetMs, + budgetMs, + fallbackPerBrokerMs, + (brokerAddr, timeoutMs) -> { + SubscriptionGroupWrapper wrapper = admin.getAllSubscriptionGroup(brokerAddr, timeoutMs); + if (wrapper == null || wrapper.getSubscriptionGroupTable() == null) { + return Map.of(); + } + return wrapper.getSubscriptionGroupTable(); + } ); } /** - * Topic-filtered lists skip the parallel full-cluster collect for speed, but must still - * walk every reachable master. An intermediate broker can return a duplicate dump (map size - * unchanged) while a later master still holds FIFO-only groups. + * Topic-filtered lists still need every reachable master's subscription dump for FIFO typing, + * but must not serially walk unreachable collision-fallbacks. Reuse the two-phase parallel collect. */ private static Map collectConsumerGroupConfigsForTopicFilter( DefaultMQAdminExt admin, JsonObject conn) { - long timeoutMs = Math.max(1_000L, intOrDefault(conn, "request_timeout_ms", DEFAULT_REQUEST_TIMEOUT_MS)); - Map configs = new TreeMap<>(); try { - for (String brokerAddr : resolveMasterBrokerAddrs(admin, conn)) { - try { - SubscriptionGroupWrapper wrapper = admin.getAllSubscriptionGroup(brokerAddr, timeoutMs); - if (wrapper == null || wrapper.getSubscriptionGroupTable() == null - || wrapper.getSubscriptionGroupTable().isEmpty()) { - continue; - } - mergeSubscriptionGroupConfigs(configs, wrapper.getSubscriptionGroupTable()); - } catch (Exception ignored) { - // Try next master. - } - } + return collectConsumerGroupConfigs(admin, conn); } catch (Exception ignored) { - // No reachable masters. + return new TreeMap<>(); } - return configs; + } + + @FunctionalInterface + interface SubscriptionGroupBrokerProbe { + Map load(String brokerAddr, long timeoutMs) throws Exception; } /** @@ -1194,19 +1203,47 @@ public final class RocketMqAgent { } /** - * Parallel subscription-group dump with a total budget. Slow/unreachable brokers are - * skipped so a large cluster does not burn the entire Agent RPC window. + * Two-phase parallel subscription-group dump. Remapped masters first; collision-fallbacks + * only when the remapped merge is empty — same host-side Docker tradeoff as consume-stats. + * Slow/unreachable brokers are skipped so a large cluster does not burn the Agent RPC window. */ static Map collectConsumerGroupConfigs( - DefaultMQAdminExt admin, - JsonObject conn, + MasterBrokerAddrPlan plan, int concurrency, long budgetMs, - long perBrokerTimeoutMs + long remappedPerBrokerTimeoutMs, + long fallbackPerBrokerTimeoutMs, + SubscriptionGroupBrokerProbe probe + ) throws Exception { + Map configs = collectConsumerGroupConfigsFromBrokers( + new ArrayList<>(plan.remappedAddrs()), + concurrency, + budgetMs, + remappedPerBrokerTimeoutMs, + probe + ); + if (configs.isEmpty() && plan.fallbackCount() > 0) { + Map fallbackConfigs = collectConsumerGroupConfigsFromBrokers( + new ArrayList<>(plan.collisionFallbackAddrs()), + concurrency, + budgetMs, + fallbackPerBrokerTimeoutMs, + probe + ); + mergeSubscriptionGroupConfigs(configs, fallbackConfigs); + } + return configs; + } + + static Map collectConsumerGroupConfigsFromBrokers( + List brokers, + int concurrency, + long budgetMs, + long perBrokerTimeoutMs, + SubscriptionGroupBrokerProbe probe ) throws Exception { - List brokers = resolveMasterBrokerAddrs(admin, conn); Map configs = new TreeMap<>(); - if (brokers.isEmpty()) { + if (brokers == null || brokers.isEmpty()) { return configs; } int workers = Math.max(1, Math.min(concurrency, brokers.size())); @@ -1221,9 +1258,9 @@ public final class RocketMqAgent { tasks.add(() -> { Map partial = new HashMap<>(); try { - SubscriptionGroupWrapper wrapper = admin.getAllSubscriptionGroup(brokerAddr, brokerTimeout); - if (wrapper != null && wrapper.getSubscriptionGroupTable() != null) { - partial.putAll(wrapper.getSubscriptionGroupTable()); + Map loaded = probe.load(brokerAddr, brokerTimeout); + if (loaded != null && !loaded.isEmpty()) { + partial.putAll(loaded); } } catch (Exception ignored) { // Skip unreachable brokers; return whatever others collected. @@ -1370,6 +1407,11 @@ public final class RocketMqAgent { return examineConsumeStatsOnMasters(admin, groupId, topic, resolveMasterBrokerAddrPlan(admin, conn)); } + @FunctionalInterface + interface ConsumeStatsBrokerProbe { + ConsumeStats examine(String brokerAddr, String groupId, String topic, long timeout) throws Exception; + } + /** * Probe consume stats on a pre-resolved master plan. Callers that fan out per group * (lag attach) should resolve the plan once so each task does not re-hit clusterInfo. @@ -1379,19 +1421,39 @@ public final class RocketMqAgent { // RocketMQ 5.3.1: 3-arg examineConsumeStats(clusterName, group, topic) is NOT broker-scoped. // Must use 4-arg (brokerAddr, group, topic, timeout) or remapped Docker addrs never hit brokers. long timeout = adminTimeoutMillis(admin); + long fallbackTimeout = Math.min(timeout, COLLISION_FALLBACK_PROBE_TIMEOUT_MS); + return examineConsumeStatsOnMasters( + groupId, + topic, + plan, + timeout, + fallbackTimeout, + (brokerAddr, g, t, to) -> admin.examineConsumeStats(brokerAddr, g, t, to) + ); + } + + /** + * Two-phase probe: remapped masters first; collision-fallbacks only when the remapped merge + * is empty. Fail-closed ({@link #shouldFailClosedOnCollisionEmpty}) only guards empty merge, + * so non-empty remapped success must not spend adminTimeout on unreachable Docker IPs. + * Fallback remoting stays short — same host-side Docker tradeoff as subscription collect. + */ + static ConsumeStats examineConsumeStatsOnMasters( + String groupId, + String topic, + MasterBrokerAddrPlan plan, + long remappedTimeout, + long fallbackTimeout, + ConsumeStatsBrokerProbe probe + ) throws Exception { ConsumeStats merged = new ConsumeStats(); int remappedSuccess = 0; int fallbackSuccess = 0; Exception lastError = null; - for (String brokerAddr : plan.allAddrs()) { - boolean collisionFallback = plan.isCollisionFallback(brokerAddr); + for (String brokerAddr : plan.remappedAddrs()) { try { - ConsumeStats stats = admin.examineConsumeStats(brokerAddr, groupId, topic, timeout); - if (collisionFallback) { - fallbackSuccess++; - } else { - remappedSuccess++; - } + ConsumeStats stats = probe.examine(brokerAddr, groupId, topic, remappedTimeout); + remappedSuccess++; if (stats != null && stats.getOffsetTable() != null) { merged.getOffsetTable().putAll(stats.getOffsetTable()); merged.setConsumeTps(merged.getConsumeTps() + stats.getConsumeTps()); @@ -1402,10 +1464,24 @@ public final class RocketMqAgent { } boolean mergedEmpty = merged.getOffsetTable() == null || merged.getOffsetTable().isEmpty(); // Remap collisions append Docker-internal originals as best-effort fallbacks. Host-side - // agents usually cannot reach them; those failures must not trip empty-merge fail-closed - // when a remapped master already returned offsets. When remapped probes all fail, gate on - // fallbacks. Empty merge + unreachable collision siblings is still fail-closed: a single - // published host:port can answer for the wrong master and look like healthy zero lag. + // agents usually cannot reach them; skip those probes once remapped masters already + // returned offsets. Empty merge + unreachable collision siblings is still fail-closed: + // a single published host:port can answer for the wrong master and look like zero lag. + if (mergedEmpty) { + for (String brokerAddr : plan.collisionFallbackAddrs()) { + try { + ConsumeStats stats = probe.examine(brokerAddr, groupId, topic, fallbackTimeout); + fallbackSuccess++; + if (stats != null && stats.getOffsetTable() != null) { + merged.getOffsetTable().putAll(stats.getOffsetTable()); + merged.setConsumeTps(merged.getConsumeTps() + stats.getConsumeTps()); + } + } catch (Exception e) { + lastError = e; + } + } + mergedEmpty = merged.getOffsetTable() == null || merged.getOffsetTable().isEmpty(); + } if (shouldFailClosedOnCollisionEmpty(mergedEmpty, plan.fallbackCount(), fallbackSuccess)) { throw new MQClientException( "Failed to examine consume stats for group " + groupId @@ -1676,7 +1752,10 @@ public final class RocketMqAgent { } try { List> results = - executor.invokeAll(tasks, Math.max(1, CONSUMER_LAG_BUDGET_MS), TimeUnit.MILLISECONDS); + executor.invokeAll( + tasks, + Math.max(1, scaledOperationBudgetMs(conn, CONSUMER_LAG_BUDGET_BASELINE_MS)), + TimeUnit.MILLISECONDS); for (int i = 0; i < page.size(); i++) { Future future = i < results.size() ? results.get(i) : null; if (future == null || future.isCancelled()) { @@ -1717,7 +1796,7 @@ public final class RocketMqAgent { page, groupId -> examineConsumerConnectionInfoRemapped(admin, conn, groupId), CONSUMER_GROUP_ENRICH_CONCURRENCY, - CONSUMER_GROUP_ENRICH_BUDGET_MS + scaledOperationBudgetMs(conn, CONSUMER_GROUP_ENRICH_BUDGET_BASELINE_MS) ); } @@ -2373,7 +2452,7 @@ public final class RocketMqAgent { consumer.assign(Collections.singletonList(queue)); consumer.seek(queue, seekOffset); consumer.setPullBatchSize(count - messages.size()); - List polled = consumer.poll(3000); + List polled = consumer.poll(MESSAGE_POLL_TIMEOUT_MS); for (MessageExt message : polled) { messages.add(peekedMessageFromRecord(topic, message)); if (messages.size() >= count) { @@ -2557,6 +2636,8 @@ public final class RocketMqAgent { : new DefaultMQProducer("_DBX_ROCKETMQ_PRODUCER"); nextProducer.setNamesrvAddr(namesrvAddr(conn)); nextProducer.setInstanceName("DBX_" + UUID.randomUUID()); + // Follow Advanced query timeout — SDK default sendMsgTimeout is only 3s. + nextProducer.setSendMsgTimeout(operationBudgetMsAsInt(conn)); nextProducer.start(); return nextProducer; } @@ -2738,6 +2819,14 @@ public final class RocketMqAgent { return all; } + LinkedHashSet remappedAddrs() { + return new LinkedHashSet<>(remapped); + } + + LinkedHashSet collisionFallbackAddrs() { + return new LinkedHashSet<>(collisionFallbacks); + } + int remappedCount() { return remapped.size(); } @@ -3315,9 +3404,10 @@ public final class RocketMqAgent { private static BrokerTopicConfigSnapshot collectBrokerTopicConfigSnapshot( DefaultMQAdminExt admin, JsonObject conn, ClusterInfo clusterInfo) throws Exception { List brokerAddrs = resolveMasterBrokerAddrs(admin, conn, null, clusterInfo); + long timeoutMs = operationBudgetMs(conn); return collectBrokerTopicConfigs( brokerAddrs, - brokerAddr -> admin.getAllTopicConfig(brokerAddr, DEFAULT_REQUEST_TIMEOUT_MS) + brokerAddr -> admin.getAllTopicConfig(brokerAddr, timeoutMs) ); } @@ -3583,6 +3673,31 @@ public final class RocketMqAgent { return value == null ? fallback : value; } + /** + * Full Advanced query timeout via {@code request_timeout_ms}. Use for single-shot ops + * (send/admin construct/topic-config per call). Multi-phase list paths must use + * {@link #scaledOperationBudgetMs} so phases still fit under one Rust RPC window. + * Message {@code LitePullConsumer.poll} stays on {@link #MESSAGE_POLL_TIMEOUT_MS}. + */ + static long operationBudgetMs(JsonObject conn) { + return Math.max(1_000L, intOrDefault(conn, "request_timeout_ms", DEFAULT_REQUEST_TIMEOUT_MS)); + } + + /** + * Scale a legacy phase baseline with Advanced query timeout. At the 30s default, baselines are + * unchanged (12/8/10/8); at 120s they expand 4× but never exceed the RPC window. + */ + static long scaledOperationBudgetMs(JsonObject conn, long baselineMs) { + long requestMs = operationBudgetMs(conn); + long scaled = baselineMs * requestMs / DEFAULT_REQUEST_TIMEOUT_MS; + return Math.max(1_000L, Math.min(requestMs, scaled)); + } + + /** {@link #operationBudgetMs} clamped to {@code int} for APIs like {@code setSendMsgTimeout}. */ + static int operationBudgetMsAsInt(JsonObject conn) { + return (int) Math.min(Integer.MAX_VALUE, operationBudgetMs(conn)); + } + static long longOrDefault(JsonObject object, String key, long fallback) { Long value = longOrNull(object, key); return value == null ? fallback : value; diff --git a/agents/drivers/rocketmq/src/test/java/com/dbx/agent/rocketmq/RocketMqAgentTest.java b/agents/drivers/rocketmq/src/test/java/com/dbx/agent/rocketmq/RocketMqAgentTest.java index a10d9e17f0..83af091c84 100644 --- a/agents/drivers/rocketmq/src/test/java/com/dbx/agent/rocketmq/RocketMqAgentTest.java +++ b/agents/drivers/rocketmq/src/test/java/com/dbx/agent/rocketmq/RocketMqAgentTest.java @@ -895,6 +895,207 @@ class RocketMqAgentTest { } } + @Test + void operationBudgetMsFollowsRequestTimeoutMs() { + JsonObject conn120 = JsonParser.parseString(""" + {"request_timeout_ms":120000} + """).getAsJsonObject(); + assertEquals(120_000L, RocketMqAgent.operationBudgetMs(conn120)); + assertEquals(120_000, RocketMqAgent.operationBudgetMsAsInt(conn120)); + + JsonObject connDefault = JsonParser.parseString("{}").getAsJsonObject(); + assertEquals(30_000L, RocketMqAgent.operationBudgetMs(connDefault)); + assertEquals(30_000, RocketMqAgent.operationBudgetMsAsInt(connDefault)); + + JsonObject connTiny = JsonParser.parseString(""" + {"request_timeout_ms":100} + """).getAsJsonObject(); + // Floor at 1s so budgets cannot collapse to zero. + assertEquals(1_000L, RocketMqAgent.operationBudgetMs(connTiny)); + assertEquals(1_000, RocketMqAgent.operationBudgetMsAsInt(connTiny)); + } + + @Test + void scaledOperationBudgetMsKeepsDefaultPhasesUnderRpcWindow() { + JsonObject connDefault = JsonParser.parseString("{}").getAsJsonObject(); + assertEquals(12_000L, RocketMqAgent.scaledOperationBudgetMs(connDefault, 12_000L)); + assertEquals(8_000L, RocketMqAgent.scaledOperationBudgetMs(connDefault, 8_000L)); + assertEquals(10_000L, RocketMqAgent.scaledOperationBudgetMs(connDefault, 10_000L)); + + JsonObject conn120 = JsonParser.parseString(""" + {"request_timeout_ms":120000} + """).getAsJsonObject(); + assertEquals(48_000L, RocketMqAgent.scaledOperationBudgetMs(conn120, 12_000L)); + assertEquals(32_000L, RocketMqAgent.scaledOperationBudgetMs(conn120, 8_000L)); + assertEquals(40_000L, RocketMqAgent.scaledOperationBudgetMs(conn120, 10_000L)); + // Never exceed the RPC window even if baseline is larger. + assertEquals(120_000L, RocketMqAgent.scaledOperationBudgetMs(conn120, 200_000L)); + } + + @Test + void collectConsumerGroupConfigsSkipsFallbacksWhenRemappedNonEmpty() throws Exception { + RocketMqAgent.MasterBrokerAddrPlan plan = collisionPlan( + "127.0.0.1:10911", "172.18.0.3:10911"); + List probed = new ArrayList<>(); + SubscriptionGroupConfig group = new SubscriptionGroupConfig(); + group.setGroupName("GID_A"); + + Map configs = RocketMqAgent.collectConsumerGroupConfigs( + plan, + 2, + 30_000L, + 30_000L, + 2_000L, + (addr, timeout) -> { + probed.add(addr); + if (plan.isCollisionFallback(addr)) { + throw new AssertionError("collision-fallback must not be probed after non-empty remapped collect"); + } + return Map.of("GID_A", group); + } + ); + + assertEquals(List.of("127.0.0.1:10911"), probed); + assertEquals(1, configs.size()); + assertTrue(configs.containsKey("GID_A")); + } + + @Test + void collectConsumerGroupConfigsProbesFallbacksWhenRemappedEmpty() throws Exception { + RocketMqAgent.MasterBrokerAddrPlan plan = collisionPlan( + "127.0.0.1:10911", "172.18.0.3:10911"); + List probed = new ArrayList<>(); + SubscriptionGroupConfig group = new SubscriptionGroupConfig(); + group.setGroupName("GID_B"); + + Map configs = RocketMqAgent.collectConsumerGroupConfigs( + plan, + 2, + 30_000L, + 30_000L, + 2_000L, + (addr, timeout) -> { + probed.add(addr); + if (plan.isCollisionFallback(addr)) { + assertEquals(2_000L, timeout); + return Map.of("GID_B", group); + } + return Map.of(); + } + ); + + assertEquals(2, probed.size()); + assertTrue(probed.contains("127.0.0.1:10911")); + assertTrue(probed.contains("172.18.0.3:10911")); + assertEquals(1, configs.size()); + assertTrue(configs.containsKey("GID_B")); + } + + @Test + void examineConsumeStatsOnMastersSkipsFallbacksWhenRemappedHasOffsets() throws Exception { + RocketMqAgent.MasterBrokerAddrPlan plan = collisionPlan( + "127.0.0.1:10911", "172.18.0.3:10911"); + List probed = new ArrayList<>(); + ConsumeStats remappedStats = nonEmptyConsumeStats("TOPIC_A", "broker-a", 0); + + ConsumeStats merged = RocketMqAgent.examineConsumeStatsOnMasters( + "GID_A", + "TOPIC_A", + plan, + 30_000L, + 2_000L, + (addr, groupId, topic, timeout) -> { + probed.add(addr); + if (plan.isCollisionFallback(addr)) { + throw new AssertionError("collision-fallback must not be probed after non-empty remapped"); + } + assertEquals(30_000L, timeout); + return remappedStats; + } + ); + + assertEquals(List.of("127.0.0.1:10911"), probed); + assertEquals(1, merged.getOffsetTable().size()); + } + + @Test + void examineConsumeStatsOnMastersProbesFallbacksAndFailClosesOnEmptyUnreachable() { + RocketMqAgent.MasterBrokerAddrPlan plan = collisionPlan( + "127.0.0.1:10911", "172.18.0.3:10911"); + List probed = new ArrayList<>(); + + MQClientException err = assertThrows( + MQClientException.class, + () -> RocketMqAgent.examineConsumeStatsOnMasters( + "GID_A", + "TOPIC_A", + plan, + 30_000L, + 2_000L, + (addr, groupId, topic, timeout) -> { + probed.add(addr); + if (plan.isCollisionFallback(addr)) { + assertEquals(2_000L, timeout); + throw new RuntimeException("docker internal unreachable"); + } + assertEquals(30_000L, timeout); + return new ConsumeStats(); + } + ) + ); + + assertTrue(err.getMessage().contains("Docker remap address collision")); + assertEquals(2, probed.size()); + assertTrue(probed.contains("127.0.0.1:10911")); + assertTrue(probed.contains("172.18.0.3:10911")); + } + + @Test + void examineConsumeStatsOnMastersUsesFallbackWhenRemappedFails() throws Exception { + RocketMqAgent.MasterBrokerAddrPlan plan = collisionPlan( + "127.0.0.1:10911", "172.18.0.3:10911"); + List probed = new ArrayList<>(); + ConsumeStats fallbackStats = nonEmptyConsumeStats("TOPIC_A", "broker-b", 1); + + ConsumeStats merged = RocketMqAgent.examineConsumeStatsOnMasters( + "GID_A", + "TOPIC_A", + plan, + 30_000L, + 2_000L, + (addr, groupId, topic, timeout) -> { + probed.add(addr); + if (!plan.isCollisionFallback(addr)) { + assertEquals(30_000L, timeout); + throw new RuntimeException("published remap unreachable"); + } + assertEquals(2_000L, timeout); + return fallbackStats; + } + ); + + assertEquals(2, probed.size()); + assertEquals(1, merged.getOffsetTable().size()); + } + + private static RocketMqAgent.MasterBrokerAddrPlan collisionPlan( + String remapped, String fallback) { + RocketMqAgent.MasterBrokerAddrPlan plan = new RocketMqAgent.MasterBrokerAddrPlan(); + plan.addRemapped(remapped); + plan.addCollisionFallback(fallback); + return plan; + } + + private static ConsumeStats nonEmptyConsumeStats(String topic, String broker, int queueId) { + MessageQueue mq = new MessageQueue(topic, broker, queueId); + OffsetWrapper offset = new OffsetWrapper(); + offset.setBrokerOffset(10); + offset.setConsumerOffset(0); + ConsumeStats stats = new ConsumeStats(); + stats.getOffsetTable().put(mq, offset); + return stats; + } + private static org.apache.rocketmq.remoting.protocol.route.BrokerData brokerData( String name, String masterAddr) { org.apache.rocketmq.remoting.protocol.route.BrokerData data = diff --git a/crates/dbx-core/src/mq/adapters/rocketmq.rs b/crates/dbx-core/src/mq/adapters/rocketmq.rs index 33975e340f..d035ddd327 100644 --- a/crates/dbx-core/src/mq/adapters/rocketmq.rs +++ b/crates/dbx-core/src/mq/adapters/rocketmq.rs @@ -104,14 +104,15 @@ impl RocketMqAdmin { pub async fn new(cfg: MqAdminConfig, launch: AgentLaunchSpec) -> Result { let mut client = AgentDriverClient::spawn(launch).await?; - // Handshake + // Handshake / connect use Advanced connect timeout; ops RPC keep query timeout. let _: serde_json::Value = - client.call_with_timeout("handshake", serde_json::json!({}), cfg.rpc_timeout()).await?; + client.call_with_timeout("handshake", serde_json::json!({}), Some(cfg.connect_timeout())).await?; // Build the connection params from MqAdminConfig let conn_params = build_connection_params(&cfg); let connect_params = serde_json::json!({ "connection": conn_params }); - let _: serde_json::Value = client.call_with_timeout("connect", connect_params, cfg.rpc_timeout()).await?; + let _: serde_json::Value = + client.call_with_timeout("connect", connect_params, Some(cfg.connect_timeout())).await?; log::info!("RocketMQ admin connected via agent (namesrv: {})", namesrv_addr(&cfg)); @@ -840,6 +841,7 @@ fn build_connection_params(cfg: &MqAdminConfig) -> serde_json::Value { "secret_key": secret_key, "tls_skip_verify": cfg.tls_skip_verify, "request_timeout_ms": cfg.request_timeout_ms(), + "connect_timeout_ms": cfg.connect_timeout_ms(), }) } @@ -964,12 +966,10 @@ async fn probe_namesrv_tcp(namesrv: &str, budget: Duration) -> Result<(), String } let deadline = tokio::time::Instant::now() + budget; - // Multi-NS HA: cap each attempt so a blackholed first node leaves budget for later hosts. - let per_attempt = if hosts.len() <= 1 { - budget - } else { - (budget / hosts.len() as u32).clamp(Duration::from_millis(500), Duration::from_secs(3)) - }; + // Multi-NS HA: share budget across hosts so a blackholed first node leaves time for later ones. + // Floor 500ms; no hard 3s cap so Advanced connect timeout can raise the per-host attempt. + let per_attempt = + if hosts.len() <= 1 { budget } else { (budget / hosts.len() as u32).max(Duration::from_millis(500)) }; let mut last_error = String::new(); for host in &hosts { let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); @@ -1101,6 +1101,21 @@ mod tests { assert_eq!(params.get("access_key").and_then(|v| v.as_str()), Some("rocket")); assert_eq!(params.get("secret_key").and_then(|v| v.as_str()), Some("secret")); assert_eq!(params.get("request_timeout_ms").and_then(|v| v.as_u64()), Some(30_000)); + assert_eq!( + params.get("connect_timeout_ms").and_then(|v| v.as_u64()), + Some(crate::mq::config::DEFAULT_MQ_CONNECT_TIMEOUT_SECS.saturating_mul(1000).max(1_000)) + ); + } + + #[test] + fn connection_params_follow_advanced_timeouts() { + let mut cfg = rocketmq_config(serde_json::json!({ "namesrvAddr": "127.0.0.1:9876" }), MqAuth::None); + cfg.query_timeout_secs = 120; + cfg.connect_timeout_secs = 15; + + let params = build_connection_params(&cfg); + assert_eq!(params.get("request_timeout_ms").and_then(|v| v.as_u64()), Some(120_000)); + assert_eq!(params.get("connect_timeout_ms").and_then(|v| v.as_u64()), Some(15_000)); } #[test] diff --git a/crates/dbx-core/src/mq/config.rs b/crates/dbx-core/src/mq/config.rs index beb85d3df0..49e14cccbe 100644 --- a/crates/dbx-core/src/mq/config.rs +++ b/crates/dbx-core/src/mq/config.rs @@ -118,6 +118,11 @@ impl MqAdminConfig { Duration::from_secs(self.connect_timeout_secs.max(1)) } + /// Milliseconds for agent `connect_timeout_ms` (Advanced connect timeout). + pub fn connect_timeout_ms(&self) -> u64 { + self.connect_timeout_secs.saturating_mul(1000).max(1_000) + } + pub fn token_signing_configured(&self) -> bool { self.token_signing.as_ref().is_some_and(MqTokenSigningConfig::is_configured) } @@ -285,6 +290,7 @@ mod tests { assert_eq!(mqc.query_timeout_secs, 120); assert_eq!(mqc.connect_timeout_secs, 15); assert_eq!(mqc.request_timeout_ms(), 120_000); + assert_eq!(mqc.connect_timeout_ms(), 15_000); assert_eq!(mqc.rpc_timeout(), Some(std::time::Duration::from_secs(120))); } diff --git a/crates/dbx-core/src/mq/mod.rs b/crates/dbx-core/src/mq/mod.rs index 648a6b8f7a..e7217541ef 100644 --- a/crates/dbx-core/src/mq/mod.rs +++ b/crates/dbx-core/src/mq/mod.rs @@ -168,9 +168,8 @@ async fn build_adapter_with_connect_timeout( // RocketMQ: TCP-probe NameServer outside the connect wall so cold JVM spawn // retains the full connect_timeout (probe used to steal up to half of it). if mqc.system_kind == MqSystemKindInternal::RocketMq { - // Probe runs outside the connect wall; scale with connect_timeout so HA NameServer - // lists are not starved by a hard 2s cap, but keep an upper bound for snappy UX. - let probe_budget = (budget / 2).clamp(std::time::Duration::from_millis(500), std::time::Duration::from_secs(5)); + // Probe runs outside the connect wall; scale with Advanced connect_timeout (no hard 5s cap). + let probe_budget = (budget / 2).max(std::time::Duration::from_millis(500)); crate::mq::adapters::rocketmq::probe_namesrv_before_connect(&mqc, probe_budget).await?; } match tokio::time::timeout(budget, build_adapter(mqc, agent_launch)).await {