From 9fb70bcfc96e4368e02d8a77bd627acabda9cfa1 Mon Sep 17 00:00:00 2001 From: t8y2 <1156263951@qq.com> Date: Wed, 12 Aug 2026 23:15:20 +0800 Subject: [PATCH] feat(rocketmq): replace Java agent with native Go implementation --- .github/scripts/bump-agent-versions.mjs | 3 +- .github/scripts/bump-agent-versions.test.mjs | 12 + .../scripts/reuse-agent-release-assets.mjs | 2 +- .../reuse-agent-release-assets.test.mjs | 18 + .github/workflows/agents-release.yml | 46 +- .github/workflows/ci.yml | 17 + agents/README.md | 10 +- agents/README.zh-CN.md | 10 +- agents/build.gradle | 2 +- agents/drivers/rocketmq/acl.go | 240 ++ agents/drivers/rocketmq/build.gradle | 13 - agents/drivers/rocketmq/cluster.go | 67 + agents/drivers/rocketmq/connection.go | 295 ++ agents/drivers/rocketmq/connection_test.go | 118 + agents/drivers/rocketmq/consumers.go | 525 +++ agents/drivers/rocketmq/consumers_test.go | 70 + agents/drivers/rocketmq/go.mod | 28 + agents/drivers/rocketmq/go.sum | 183 + agents/drivers/rocketmq/helpers.go | 228 + agents/drivers/rocketmq/integration_test.go | 342 ++ agents/drivers/rocketmq/main.go | 73 + agents/drivers/rocketmq/main_test.go | 115 + agents/drivers/rocketmq/messages.go | 586 +++ agents/drivers/rocketmq/messages_test.go | 116 + agents/drivers/rocketmq/remoting.go | 42 + agents/drivers/rocketmq/routing.go | 526 +++ agents/drivers/rocketmq/routing_test.go | 136 + .../rocketmq/scripts/run-integration.sh | 130 + agents/drivers/rocketmq/server.go | 168 + agents/drivers/rocketmq/server_test.go | 34 + .../com/dbx/agent/rocketmq/RocketMqAgent.java | 3794 ----------------- .../dbx/agent/rocketmq/RocketMqAgentTest.java | 1272 ------ .../RocketMqLiveMessagePropertyTest.java | 179 - .../rocketmq/RocketMqLiveTopicTypeTest.java | 125 - agents/drivers/rocketmq/topics.go | 620 +++ agents/drivers/rocketmq/topics_test.go | 80 + .../scripts/driver_release_packages_test.py | 26 +- agents/scripts/validate_agents.py | 1 + agents/scripts/validate_agents_test.py | 3 +- agents/scripts/version_agent_artifacts.py | 2 +- agents/settings.gradle | 2 +- crates/dbx-core/src/mq/README.md | 2 + crates/dbx-core/src/mq/adapters/rocketmq.rs | 17 +- docs/mq-quick-start.md | 8 +- 44 files changed, 4872 insertions(+), 5414 deletions(-) create mode 100644 agents/drivers/rocketmq/acl.go delete mode 100644 agents/drivers/rocketmq/build.gradle create mode 100644 agents/drivers/rocketmq/cluster.go create mode 100644 agents/drivers/rocketmq/connection.go create mode 100644 agents/drivers/rocketmq/connection_test.go create mode 100644 agents/drivers/rocketmq/consumers.go create mode 100644 agents/drivers/rocketmq/consumers_test.go create mode 100644 agents/drivers/rocketmq/go.mod create mode 100644 agents/drivers/rocketmq/go.sum create mode 100644 agents/drivers/rocketmq/helpers.go create mode 100644 agents/drivers/rocketmq/integration_test.go create mode 100644 agents/drivers/rocketmq/main.go create mode 100644 agents/drivers/rocketmq/main_test.go create mode 100644 agents/drivers/rocketmq/messages.go create mode 100644 agents/drivers/rocketmq/messages_test.go create mode 100644 agents/drivers/rocketmq/remoting.go create mode 100644 agents/drivers/rocketmq/routing.go create mode 100644 agents/drivers/rocketmq/routing_test.go create mode 100755 agents/drivers/rocketmq/scripts/run-integration.sh create mode 100644 agents/drivers/rocketmq/server.go create mode 100644 agents/drivers/rocketmq/server_test.go delete mode 100644 agents/drivers/rocketmq/src/main/java/com/dbx/agent/rocketmq/RocketMqAgent.java delete mode 100644 agents/drivers/rocketmq/src/test/java/com/dbx/agent/rocketmq/RocketMqAgentTest.java delete mode 100644 agents/drivers/rocketmq/src/test/java/com/dbx/agent/rocketmq/RocketMqLiveMessagePropertyTest.java delete mode 100644 agents/drivers/rocketmq/src/test/java/com/dbx/agent/rocketmq/RocketMqLiveTopicTypeTest.java create mode 100644 agents/drivers/rocketmq/topics.go create mode 100644 agents/drivers/rocketmq/topics_test.go diff --git a/.github/scripts/bump-agent-versions.mjs b/.github/scripts/bump-agent-versions.mjs index be7b434d1e..44358a6967 100644 --- a/.github/scripts/bump-agent-versions.mjs +++ b/.github/scripts/bump-agent-versions.mjs @@ -58,9 +58,10 @@ const nativeDriverDirectories = { neo4j: "neo4j-go", vastbase: "vastbase-go", rabbitmq: "rabbitmq", + rocketmq: "rocketmq", tdengine: "tdengine", }; -const nativeDriverModules = new Set(["cassandra", "duckdb", "hive", "oracle", "xugu", "kingbase", "iotdb", "neo4j", "vastbase", "rabbitmq", "tdengine"]); +const nativeDriverModules = new Set(["cassandra", "duckdb", "hive", "oracle", "xugu", "kingbase", "iotdb", "neo4j", "vastbase", "rabbitmq", "rocketmq", "tdengine"]); const nativeDriverSharedPaths = { hive: [ "agents/go-common/go-gssapi", diff --git a/.github/scripts/bump-agent-versions.test.mjs b/.github/scripts/bump-agent-versions.test.mjs index f5be89613a..99ea4c20f4 100644 --- a/.github/scripts/bump-agent-versions.test.mjs +++ b/.github/scripts/bump-agent-versions.test.mjs @@ -59,6 +59,18 @@ test("bumps the native RabbitMQ agent from its Go directory", () => { assert.equal(result.versions.rabbitmq, "0.1.1"); }); +test("bumps the native RocketMQ agent from its Go directory", () => { + const result = evaluateAgentVersionBump({ + versions: { rocketmq: "0.1.0" }, + changedFiles: ["agents/drivers/rocketmq/main.go"], + moduleExists: (path) => path === "agents/drivers/rocketmq", + readModuleFile: () => "", + }); + + assert.equal(result.versions.rocketmq, "0.1.1"); + assert.deepEqual(result.nativeModules, ["rocketmq"]); +}); + test("bumps the native Vastbase agent from its independent Go directory", () => { const result = evaluateAgentVersionBump({ versions: { vastbase: "0.1.37" }, diff --git a/.github/scripts/reuse-agent-release-assets.mjs b/.github/scripts/reuse-agent-release-assets.mjs index 5c863ff2ac..21f72bde29 100644 --- a/.github/scripts/reuse-agent-release-assets.mjs +++ b/.github/scripts/reuse-agent-release-assets.mjs @@ -15,7 +15,7 @@ import { basename, join } from "node:path"; import { tmpdir } from "node:os"; const REGISTRY_ASSET = "agent-registry.json"; -const NATIVE_MODULES = new Set(["duckdb", "oracle", "xugu", "kingbase", "iotdb", "neo4j", "vastbase", "rabbitmq", "tdengine"]); +const NATIVE_MODULES = new Set(["duckdb", "oracle", "xugu", "kingbase", "iotdb", "neo4j", "vastbase", "rabbitmq", "rocketmq", "tdengine"]); const PLATFORMS = [ "macos-aarch64", "macos-x64", diff --git a/.github/scripts/reuse-agent-release-assets.test.mjs b/.github/scripts/reuse-agent-release-assets.test.mjs index 16c6453074..174a2865bb 100644 --- a/.github/scripts/reuse-agent-release-assets.test.mjs +++ b/.github/scripts/reuse-agent-release-assets.test.mjs @@ -114,6 +114,24 @@ test("requires all IoTDB native platforms when reusing a release", () => { ); }); +test("requires all RocketMQ native platforms when reusing a release", () => { + const native = Object.fromEntries( + platforms.slice(0, -1).map((platform, index) => [platform, artifact(`dbx-agent-rocketmq-0.1.40-${platform}.tar.zst`, String(index + 1))]), + ); + const registry = { drivers: { rocketmq: { version: "0.1.40", native } }, jres: {} }; + + assert.throws( + () => collectReusableAssetPlan({ + registry, + release: releaseFor(Object.values(native)), + versions: { rocketmq: "0.1.40" }, + modules: ["rocketmq"], + reuseJre: false, + }), + /missing=windows-x64/, + ); +}); + test("ignores zero-size legacy JAR placeholders for native-only modules", () => { const native = Object.fromEntries( platforms.map((platform, index) => [platform, artifact(`dbx-agent-duckdb-0.1.2-${platform}.tar.zst`, String(index + 1))]), diff --git a/.github/workflows/agents-release.yml b/.github/workflows/agents-release.yml index 78269b7a1c..86d94837b4 100644 --- a/.github/workflows/agents-release.yml +++ b/.github/workflows/agents-release.yml @@ -289,6 +289,46 @@ jobs: name: rabbitmq-native path: "release-native/dbx-agent-rabbitmq-*" + build-rocketmq-native: + needs: [bump-versions] + if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'rocketmq') }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: "1.25.x" + - name: Test RocketMQ native agent + working-directory: agents/drivers/rocketmq + run: go test -race ./... + - name: Cross-compile RocketMQ native agent + shell: bash + run: | + mkdir -p release-native + cd agents/drivers/rocketmq + declare -A TARGETS=( + ["macos-aarch64"]="darwin/arm64" + ["macos-x64"]="darwin/amd64" + ["linux-aarch64"]="linux/arm64" + ["linux-x64"]="linux/amd64" + ["windows-aarch64"]="windows/arm64" + ["windows-x64"]="windows/amd64" + ) + for platform in "${!TARGETS[@]}"; do + IFS=/ read -r goos goarch <<< "${TARGETS[$platform]}" + output="../../../release-native/dbx-agent-rocketmq-${platform}" + if [[ "$goos" == "windows" ]]; then + output="${output}.exe" + fi + echo "Building $platform ($goos/$goarch)" + CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -trimpath -ldflags="-s -w" -o "$output" . + done + ls -lh ../../../release-native + - uses: actions/upload-artifact@v4 + with: + name: rocketmq-native + path: "release-native/dbx-agent-rocketmq-*" + build-cassandra-native: needs: [bump-versions] if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'cassandra') }} @@ -868,7 +908,7 @@ jobs: retention-days: 1 release: - needs: [bump-versions, commit-versions, build-agents, build-oracle-native, build-xugu-native, build-rabbitmq-native, build-cassandra-native, build-hive-native, build-kingbase-native, build-vastbase-native, build-neo4j-native, build-iotdb-native, build-duckdb-native, build-tdengine-native, build-jre, reuse-previous-assets] + needs: [bump-versions, commit-versions, build-agents, build-oracle-native, build-xugu-native, build-rabbitmq-native, build-rocketmq-native, build-cassandra-native, build-hive-native, build-kingbase-native, build-vastbase-native, build-neo4j-native, build-iotdb-native, build-duckdb-native, build-tdengine-native, build-jre, reuse-previous-assets] if: ${{ always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }} runs-on: ubuntu-latest steps: @@ -990,6 +1030,7 @@ jobs: duckdb) echo "DuckDB" ;; xugu) echo "虚谷 XuguDB" ;; rabbitmq) echo "RabbitMQ" ;; + rocketmq) echo "Apache RocketMQ" ;; cassandra) echo "Apache Cassandra" ;; hive) echo "Apache Hive" ;; neo4j) echo "Neo4j" ;; @@ -1058,7 +1099,7 @@ jobs: [ -n "$DRIVERS" ] && DRIVERS="${DRIVERS},"$'\n' DRIVERS="${DRIVERS}$(generate_jar_entry "$name" "$label" "$f" "$jre_key" "$version" "$external_driver" "$native_json")" done - for name in oracle xugu kingbase vastbase neo4j iotdb duckdb rabbitmq cassandra hive tdengine; do + for name in oracle xugu kingbase vastbase neo4j iotdb duckdb rabbitmq rocketmq cassandra hive tdengine; do version=$(get_module_version "$name") [ -f "release/dbx-agent-${name}-${version}.jar" ] && continue native_json=$(generate_native_platforms "$name" "$version") @@ -1128,6 +1169,7 @@ jobs: oracle) echo "Oracle" ;; xugu) echo "虚谷 XuguDB" ;; rabbitmq) echo "RabbitMQ" ;; + rocketmq) echo "Apache RocketMQ" ;; cassandra) echo "Apache Cassandra" ;; hive) echo "Apache Hive" ;; neo4j) echo "Neo4j" ;; diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9eff4e5fdb..0b4bb879b6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -740,6 +740,10 @@ jobs: run: go test ./... working-directory: agents/drivers/rabbitmq + - name: RocketMQ native agent tests + run: go test -race ./... + working-directory: agents/drivers/rocketmq + - name: Cassandra native agent tests run: go test ./... working-directory: agents/drivers/cassandra-go @@ -772,6 +776,19 @@ jobs: run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-rabbitmq-linux-x64 . working-directory: agents/drivers/rabbitmq + - name: RocketMQ native agent build + run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-rocketmq-linux-x64 . + working-directory: agents/drivers/rocketmq + + - name: RocketMQ native agent integration tests + shell: bash + working-directory: agents/drivers/rocketmq + run: | + set -euo pipefail + for version in 4.9.8 5.3.1; do + ./scripts/run-integration.sh "$version" + done + - name: Cassandra native agent build run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-cassandra-linux-x64 . working-directory: agents/drivers/cassandra-go diff --git a/agents/README.md b/agents/README.md index fe4f45c66c..fbe698a3ee 100644 --- a/agents/README.md +++ b/agents/README.md @@ -45,11 +45,12 @@ Each agent runs as a standalone process and communicates with DBX via stdin/stdo | etcd | etcd | jetcd | | zookeeper | Apache ZooKeeper | Apache Curator | | rabbitmq | RabbitMQ | amqp091-go native agent | +| rocketmq | Apache RocketMQ | rocketmq-admin-go native agent | ## Multi-JRE Support -Most Java agents target JRE 21. Native agents, such as `cassandra`, `duckdb`, `hive`, `iotdb`, `oracle`, `kingbase`, `tdengine`, `xugu`, and `rabbitmq`, do not require a JRE. DBX downloads and manages the JRE 21 installation automatically for Java agents. +Most Java agents target JRE 21. Native agents, such as `cassandra`, `duckdb`, `hive`, `iotdb`, `oracle`, `kingbase`, `tdengine`, `xugu`, `rabbitmq`, and `rocketmq`, do not require a JRE. DBX downloads and manages the JRE 21 installation automatically for Java agents. ## JDBC Connection Pooling @@ -75,7 +76,7 @@ Set `DBX_AGENT_JDBC_POOL_ENABLED=false` for a runtime-level compatibility fallba For new agents, prefer a **native (Go or Rust) driver** over a Java/JDBC agent whenever a mature, license-compatible native driver is available. Native agents ship as a single self-contained executable with no JRE, which significantly reduces memory footprint and startup time — the JVM baseline that every Java agent pays even when idle is avoided entirely. -- **Native (C++/Go/Rust)** — preferred when a usable native driver exists. See `drivers/cassandra-go` (Apache cassandra-gocql-driver), `drivers/duckdb`, `drivers/hive-go` (native HS2), `drivers/iotdb` (Apache IoTDB Go Client), `drivers/oracle-go` (go-ora), `drivers/kingbase-go` (gokb), `drivers/vastbase-go` (openGauss connector), `drivers/tdengine` (taos-connector-rust), `drivers/xugu`, and `drivers/rabbitmq` (amqp091-go) as reference implementations. No JRE download or management is needed. +- **Native (C++/Go/Rust)** — preferred when a usable native driver exists. See `drivers/cassandra-go` (Apache cassandra-gocql-driver), `drivers/duckdb`, `drivers/hive-go` (native HS2), `drivers/iotdb` (Apache IoTDB Go Client), `drivers/oracle-go` (go-ora), `drivers/kingbase-go` (gokb), `drivers/vastbase-go` (openGauss connector), `drivers/tdengine` (taos-connector-rust), `drivers/xugu`, `drivers/rabbitmq` (amqp091-go), and `drivers/rocketmq` (rocketmq-admin-go) as reference implementations. No JRE download or management is needed. - **Java/JDBC** — the default fallback when only a JDBC driver exists for the database, or when the native driver is immature or unmaintained. Most agents still fall in this category. Native agents implement the same JSON-RPC contract and `versions.json` registration as Java agents; they ship an `agent` executable instead of `agent.jar`. If both native and Java source implementations exist for the same database, publish only the native artifact unless the Java variant has a separately registered compatibility profile, such as `oracle-legacy` / `oracle-10g`. @@ -95,9 +96,10 @@ Requires JDK 21 (Gradle toolchain auto-downloads if needed). (cargo build --manifest-path drivers/tdengine/Cargo.toml --release --locked) (cd drivers/xugu && go build -o agent .) (cd drivers/rabbitmq && go build -o agent .) +(cd drivers/rocketmq && go build -o agent .) ``` -Output JARs are in `drivers/{module}/build/libs/`. Native agents build from `drivers/cassandra-go`, `drivers/duckdb`, `drivers/hive-go`, `drivers/iotdb`, `drivers/oracle-go`, `drivers/kingbase-go`, `drivers/vastbase-go`, `drivers/tdengine`, `drivers/xugu`, and `drivers/rabbitmq`. +Output JARs are in `drivers/{module}/build/libs/`. Native agents build from `drivers/cassandra-go`, `drivers/duckdb`, `drivers/hive-go`, `drivers/iotdb`, `drivers/oracle-go`, `drivers/kingbase-go`, `drivers/vastbase-go`, `drivers/tdengine`, `drivers/xugu`, `drivers/rabbitmq`, and `drivers/rocketmq`. ### Local DBX Runtime Test @@ -111,7 +113,7 @@ cp agents/drivers//build/libs/*-all.jar ~/.dbx/agents/drivers/ Restart DBX or disconnect and reconnect the database so the new agent process loads the replacement JAR. -Native agents such as `cassandra`, `hive`, `iotdb`, `oracle`, `kingbase`, `tdengine`, `xugu`, and `rabbitmq` use an `agent` executable instead of `agent.jar`. TDengine builds `target/release/dbx-tdengine-driver` from `drivers/tdengine/Cargo.toml`. +Native agents such as `cassandra`, `hive`, `iotdb`, `oracle`, `kingbase`, `tdengine`, `xugu`, `rabbitmq`, and `rocketmq` use an `agent` executable instead of `agent.jar`. TDengine builds `target/release/dbx-tdengine-driver` from `drivers/tdengine/Cargo.toml`. ## Versioning diff --git a/agents/README.zh-CN.md b/agents/README.zh-CN.md index 076c7b5c83..1321edf436 100644 --- a/agents/README.zh-CN.md +++ b/agents/README.zh-CN.md @@ -45,11 +45,12 @@ DBX 的 Agent 驱动 —— 通过 JDBC 和原生数据库驱动支持各种数 | etcd | etcd | jetcd | | zookeeper | Apache ZooKeeper | Apache Curator | | rabbitmq | RabbitMQ | amqp091-go 原生 agent | +| rocketmq | Apache RocketMQ | rocketmq-admin-go 原生 agent | ## 多 JRE 支持 -多数 Java agent 以 JRE 21 为目标。原生 agent（如 `cassandra`、`duckdb`、`hive`、`iotdb`、`oracle`、`kingbase`、`tdengine`、`xugu` 和 `rabbitmq`）不需要 JRE。对 Java agent，DBX 会自动下载并管理 JRE 21 安装。 +多数 Java agent 以 JRE 21 为目标。原生 agent（如 `cassandra`、`duckdb`、`hive`、`iotdb`、`oracle`、`kingbase`、`tdengine`、`xugu`、`rabbitmq` 和 `rocketmq`）不需要 JRE。对 Java agent，DBX 会自动下载并管理 JRE 21 安装。 ## JDBC 连接池 @@ -75,7 +76,7 @@ HikariCP 会直接打进启用连接池的 Agent JAR。已经使用 DBX 托管 J 对于新 agent，只要存在成熟、许可证兼容的原生驱动，优先选择**原生（Go 或 Rust）驱动**而非 Java/JDBC agent。原生 agent 以单一自包含可执行文件发布，无需 JRE，可显著降低内存占用和启动时间 —— 完全避开 Java agent 即便空闲也要付出的 JVM 基线开销。 -- **原生（Go/Rust）** —— 存在可用原生驱动时首选。参考 `drivers/cassandra-go`（Apache cassandra-gocql-driver）、`drivers/duckdb`、`drivers/hive-go`（原生 HS2）、`drivers/iotdb`（Apache IoTDB Go Client）、`drivers/oracle-go`（go-ora）、`drivers/kingbase-go`（gokb）、`drivers/vastbase-go`（openGauss connector）、`drivers/tdengine`（taos-connector-rust）、`drivers/xugu` 和 `drivers/rabbitmq`（amqp091-go）。无需 JRE 下载与管理。 +- **原生（Go/Rust）** —— 存在可用原生驱动时首选。参考 `drivers/cassandra-go`（Apache cassandra-gocql-driver）、`drivers/duckdb`、`drivers/hive-go`（原生 HS2）、`drivers/iotdb`（Apache IoTDB Go Client）、`drivers/oracle-go`（go-ora）、`drivers/kingbase-go`（gokb）、`drivers/vastbase-go`（openGauss connector）、`drivers/tdengine`（taos-connector-rust）、`drivers/xugu`、`drivers/rabbitmq`（amqp091-go）和 `drivers/rocketmq`（rocketmq-admin-go）。无需 JRE 下载与管理。 - **Java/JDBC** —— 当某数据库只有 JDBC 驱动，或原生驱动不成熟、缺乏维护时的默认兜底方案。多数 agent 仍属此类。 原生 agent 实现与 Java agent 相同的 JSON-RPC 契约和 `versions.json` 登记；它发布的是 `agent` 可执行文件而非 `agent.jar`。若同一数据库同时保留原生和 Java 源码实现，默认只发布原生产物；只有 Java 变体以独立兼容配置登记时才同时发布，例如 `oracle-legacy` / `oracle-10g`。 @@ -95,9 +96,10 @@ HikariCP 会直接打进启用连接池的 Agent JAR。已经使用 DBX 托管 J (cargo build --manifest-path drivers/tdengine/Cargo.toml --release --locked) (cd drivers/xugu && go build -o agent .) (cd drivers/rabbitmq && go build -o agent .) +(cd drivers/rocketmq && go build -o agent .) ``` -产物 JAR 在 `drivers/{module}/build/libs/`。原生 agent 从 `drivers/cassandra-go`、`drivers/duckdb`、`drivers/hive-go`、`drivers/iotdb`、`drivers/oracle-go`、`drivers/kingbase-go`、`drivers/vastbase-go`、`drivers/tdengine`、`drivers/xugu` 和 `drivers/rabbitmq` 构建。 +产物 JAR 在 `drivers/{module}/build/libs/`。原生 agent 从 `drivers/cassandra-go`、`drivers/duckdb`、`drivers/hive-go`、`drivers/iotdb`、`drivers/oracle-go`、`drivers/kingbase-go`、`drivers/vastbase-go`、`drivers/tdengine`、`drivers/xugu`、`drivers/rabbitmq` 和 `drivers/rocketmq` 构建。 ### 本地 DBX 运行时测试 @@ -111,7 +113,7 @@ cp agents/drivers//build/libs/*-all.jar ~/.dbx/agents/drivers/ 重启 DBX 或断开重连数据库，使新 agent 进程加载替换后的 JAR。 -`cassandra`、`hive`、`iotdb`、`oracle`、`kingbase`、`tdengine`、`xugu` 和 `rabbitmq` 等原生 agent 使用可执行文件而非 `agent.jar`。TDengine 从 `drivers/tdengine/Cargo.toml` 构建 `target/release/dbx-tdengine-driver`。 +`cassandra`、`hive`、`iotdb`、`oracle`、`kingbase`、`tdengine`、`xugu`、`rabbitmq` 和 `rocketmq` 等原生 agent 使用可执行文件而非 `agent.jar`。TDengine 从 `drivers/tdengine/Cargo.toml` 构建 `target/release/dbx-tdengine-driver`。 ## 版本管理 diff --git a/agents/build.gradle b/agents/build.gradle index f218cc04df..6fa647d60a 100644 --- a/agents/build.gradle +++ b/agents/build.gradle @@ -3,7 +3,7 @@ plugins { } def infrastructureProjects = ['common', 'test-support'] as Set -def legacyStandaloneProjects = ['mongodb', 'kafka', 'rocketmq'] as Set +def legacyStandaloneProjects = ['mongodb', 'kafka'] as Set def pooledJdbcProjects = [ 'access', 'bigquery', 'dameng', 'databend', 'databricks', 'db2', 'exasol', 'firebird', 'gbase8a', 'gbase8s', 'goldendb', 'h2', 'h2-legacy', 'highgo', diff --git a/agents/drivers/rocketmq/acl.go b/agents/drivers/rocketmq/acl.go new file mode 100644 index 0000000000..dbd221e1c6 --- /dev/null +++ b/agents/drivers/rocketmq/acl.go @@ -0,0 +1,240 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + admin "github.com/amigoer/rocketmq-admin-go" + "github.com/amigoer/rocketmq-admin-go/protocol/remoting" +) + +type aclWire struct { + Subject string `json:"subject"` + Policies []aclPolicyWire `json:"policies"` +} + +type aclPolicyWire struct { + PolicyType string `json:"policyType,omitempty"` + Entries []aclEntryWire `json:"entries"` +} + +type aclEntryWire struct { + Resource string `json:"resource"` + Actions []string `json:"actions"` + SourceIPs []string `json:"sourceIps"` + Decision string `json:"decision"` +} + +func (a *rocketMQAgent) listACLs(params map[string]any) (any, error) { + _, config, err := a.requireClient() + if err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout) + defer cancel() + address, err := a.brokerAddressForName(stringValue(params, "brokerName")) + if err != nil { + return nil, err + } + response, err := invokeRemotingWithClient(ctx, address, remoting.NewRequest(remoting.ListAcl, map[string]string{ + "subjectFilter": strings.TrimPrefix(stringValue(params, "principal", "subject"), "User:"), + "resourceFilter": stringValue(params, "resourceName"), + })) + if err != nil { + return nil, err + } + var wrapper struct { + Acls []aclWire `json:"acls"` + } + body := repairRocketMQJSON(response.Body) + if len(body) > 0 && body[0] == '[' { + if err := json.Unmarshal(body, &wrapper.Acls); err != nil { + return nil, fmt.Errorf("decode ACL list: %w", err) + } + } else if err := json.Unmarshal(body, &wrapper); err != nil { + return nil, fmt.Errorf("decode ACL list: %w", err) + } + principalFilter := strings.TrimPrefix(stringValue(params, "principal", "subject"), "User:") + resourceFilter := stringValue(params, "resourceName") + rows := make([]map[string]any, 0) + for _, acl := range wrapper.Acls { + if principalFilter != "" && acl.Subject != principalFilter && acl.Subject != "User:"+principalFilter { + continue + } + for _, policy := range acl.Policies { + for _, entry := range policy.Entries { + if resourceFilter != "" && entry.Resource != resourceFilter { + continue + } + actions := entry.Actions + if len(actions) == 0 { + actions = []string{"ALL"} + } + for _, action := range actions { + host := "*" + if len(entry.SourceIPs) > 0 { + host = strings.Join(entry.SourceIPs, ",") + } + rows = append(rows, map[string]any{ + "resourceType": "TOPIC", "resourceName": entry.Resource, + "patternType": "LITERAL", "principal": acl.Subject, + "host": host, "operation": normalizeACLOperation(action), + "permissionType": entry.Decision, + }) + } + } + } + } + return map[string]any{"acls": rows}, nil +} + +func (a *rocketMQAgent) createACLs(params map[string]any) (any, error) { + client, config, err := a.requireClient() + if err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout) + defer cancel() + address, err := a.brokerAddressForName(stringValue(params, "brokerName")) + if err != nil { + return nil, err + } + entries, _ := params["acls"].([]any) + for _, raw := range entries { + entry, _ := raw.(map[string]any) + if _, hasAccessKey := entry["accessKey"]; hasAccessKey || entry["secretKey"] != nil { + if err := client.UpdatePlainAccessConfig(ctx, address, plainAccessConfig(entry)); err != nil { + return nil, err + } + continue + } + subject := strings.TrimPrefix(stringValue(entry, "principal", "subject"), "User:") + if subject == "" { + return nil, fmt.Errorf("ACL principal is required") + } + resource := stringValue(entry, "resourceName") + if resource == "" { + resource = "*" + } + host := stringValue(entry, "host") + if host == "" { + host = "*" + } + decision := stringValue(entry, "permissionType") + if decision == "" { + decision = "ALLOW" + } + acl := aclWire{Subject: subject, Policies: []aclPolicyWire{{Entries: []aclEntryWire{{ + Resource: resource, Actions: []string{mapACLOperation(stringValue(entry, "operation"))}, + SourceIPs: []string{host}, Decision: decision, + }}}}} + body, marshalErr := json.Marshal(acl) + if marshalErr != nil { + return nil, marshalErr + } + command := remoting.NewRequest(remoting.CreateAcl, map[string]string{"subject": subject}) + command.Body = body + if _, err := invokeRemotingWithClient(ctx, address, command); err != nil { + return nil, err + } + } + return okResult(), nil +} + +func (a *rocketMQAgent) deleteACLs(params map[string]any) (any, error) { + client, config, err := a.requireClient() + if err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout) + defer cancel() + address, err := a.brokerAddressForName(stringValue(params, "brokerName")) + if err != nil { + return nil, err + } + filters, _ := params["filters"].([]any) + if len(filters) == 0 { + listed, listErr := a.listACLs(params) + if listErr != nil { + return nil, listErr + } + for _, row := range listed.(map[string]any)["acls"].([]map[string]any) { + filters = append(filters, row) + } + } + deleted := 0 + seen := make(map[string]struct{}) + for _, raw := range filters { + filter, _ := raw.(map[string]any) + if accessKey := stringValue(filter, "accessKey"); accessKey != "" { + if err := client.DeletePlainAccessConfig(ctx, address, accessKey); err != nil { + return nil, err + } + deleted++ + continue + } + subject := strings.TrimPrefix(stringValue(filter, "principal", "subject"), "User:") + resource := stringValue(filter, "resourceName") + key := subject + "|" + resource + if subject == "" { + continue + } + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + fields := map[string]string{"subject": subject} + if resource != "" { + fields["resource"] = resource + } + if _, err := invokeRemotingWithClient(ctx, address, remoting.NewRequest(remoting.DeleteAcl, fields)); err != nil { + return nil, err + } + deleted++ + } + return map[string]any{"ok": true, "deleted": deleted}, nil +} + +func plainAccessConfig(entry map[string]any) admin.PlainAccessConfig { + return admin.PlainAccessConfig{ + AccessKey: stringValue(entry, "accessKey"), SecretKey: stringValue(entry, "secretKey"), + WhiteRemoteAddress: stringValue(entry, "whiteRemoteAddress"), + Admin: boolValue(entry, false, "admin"), + DefaultTopicPerm: stringValue(entry, "defaultTopicPerm"), + DefaultGroupPerm: stringValue(entry, "defaultGroupPerm"), + TopicPerms: stringSlice(entry["topicPerms"]), GroupPerms: stringSlice(entry["groupPerms"]), + } +} + +func stringSlice(value any) []string { + items, _ := value.([]any) + result := make([]string, 0, len(items)) + for _, item := range items { + result = append(result, fmt.Sprint(item)) + } + return result +} + +func mapACLOperation(operation string) string { + switch strings.ToUpper(operation) { + case "WRITE", "PRODUCE", "PUB": + return "PUB" + case "READ", "CONSUME", "SUB": + return "SUB" + default: + return "ALL" + } +} + +func normalizeACLOperation(operation string) string { + switch strings.ToUpper(operation) { + case "PUB": + return "WRITE" + case "SUB": + return "READ" + default: + return strings.ToUpper(operation) + } +} diff --git a/agents/drivers/rocketmq/build.gradle b/agents/drivers/rocketmq/build.gradle deleted file mode 100644 index 6846f1d825..0000000000 --- a/agents/drivers/rocketmq/build.gradle +++ /dev/null @@ -1,13 +0,0 @@ -dependencies { - implementation 'com.google.code.gson:gson:2.12.1' - implementation 'org.apache.rocketmq:rocketmq-client:5.3.1' - implementation 'org.apache.rocketmq:rocketmq-tools:5.3.1' - runtimeOnly 'org.slf4j:slf4j-simple:1.7.36' -} - -tasks.named('shadowJar') { - mergeServiceFiles() - manifest { - attributes('Agent-Label': 'Apache RocketMQ', 'Main-Class': 'com.dbx.agent.rocketmq.RocketMqAgent') - } -} diff --git a/agents/drivers/rocketmq/cluster.go b/agents/drivers/rocketmq/cluster.go new file mode 100644 index 0000000000..60bdd781ff --- /dev/null +++ b/agents/drivers/rocketmq/cluster.go @@ -0,0 +1,67 @@ +package main + +import ( + "context" + "strconv" + + admin "github.com/amigoer/rocketmq-admin-go" +) + +func (a *rocketMQAgent) describeCluster(_ map[string]any) (any, error) { + client, config, err := a.requireClient() + if err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout) + defer cancel() + info, err := client.ExamineBrokerClusterInfo(ctx) + if err != nil { + return nil, err + } + a.mu.RLock() + proxies := a.proxies + clusterName := a.clusterName + a.mu.RUnlock() + brokers := brokerNodes(info, proxies) + var controller any + if len(brokers) > 0 { + controller = brokers[0] + } + return map[string]any{ + "clusterId": clusterName, "controller": controller, + "brokers": brokers, "nodeCount": len(brokers), + }, nil +} + +func brokerNodes(info *admin.ClusterInfo, proxies *proxyManager) []map[string]any { + if info == nil { + return []map[string]any{} + } + nodes := make([]map[string]any, 0) + id := 0 + for _, brokerName := range sortedKeys(info.BrokerAddrTable) { + broker := info.BrokerAddrTable[brokerName] + if broker == nil { + continue + } + for _, brokerID := range sortedKeys(broker.BrokerAddrs) { + address := broker.BrokerAddrs[brokerID] + if original := proxies.OriginalForLocal(address); original != "" { + address = original + } + host, portText := parseSocketAddress(address) + port, _ := strconv.Atoi(portText) + brokerIDValue, _ := strconv.ParseInt(brokerID, 10, 64) + role := "SLAVE" + if brokerIDValue == 0 { + role = "MASTER" + } + nodes = append(nodes, map[string]any{ + "id": id, "host": host, "port": port, "rack": nil, + "brokerName": broker.BrokerName, "brokerId": brokerIDValue, "role": role, + }) + id++ + } + } + return nodes +} diff --git a/agents/drivers/rocketmq/connection.go b/agents/drivers/rocketmq/connection.go new file mode 100644 index 0000000000..b334332507 --- /dev/null +++ b/agents/drivers/rocketmq/connection.go @@ -0,0 +1,295 @@ +package main + +import ( + "context" + "fmt" + "time" + + admin "github.com/amigoer/rocketmq-admin-go" +) + +const ( + defaultRequestTimeout = 30 * time.Second + defaultConnectTimeout = 10 * time.Second + brokerRegistrationPollInterval = 100 * time.Millisecond +) + +type socksProxyConfig struct { + Host string + Port int + Username string + Password string +} + +type connectionConfig struct { + NameServers []string + ClusterName string + BrokerAddr string + AccessKey string + SecretKey string + RequestTimeout time.Duration + ConnectTimeout time.Duration + TLSSkipVerify bool + SocksProxy *socksProxyConfig +} + +func parseConnection(params map[string]any) (connectionConfig, error) { + connection := nestedMap(params, "connection") + if connection == nil { + connection = params + } + nameServers := splitAddresses(stringValue(connection, "namesrv_addr", "namesrvAddr")) + if len(nameServers) == 0 { + return connectionConfig{}, fmt.Errorf("namesrv_addr is required") + } + config := connectionConfig{ + NameServers: nameServers, + ClusterName: stringValue(connection, "cluster_name", "clusterName"), + BrokerAddr: stringValue(connection, "broker_addr", "brokerAddr"), + AccessKey: stringValue(connection, "access_key", "accessKey"), + SecretKey: stringValue(connection, "secret_key", "secretKey"), + RequestTimeout: time.Duration(intValue(connection, int(defaultRequestTimeout/time.Millisecond), "request_timeout_ms")) * time.Millisecond, + ConnectTimeout: time.Duration(intValue(connection, int(defaultConnectTimeout/time.Millisecond), "connect_timeout_ms")) * time.Millisecond, + TLSSkipVerify: boolValue(connection, false, "tls_skip_verify"), + } + if config.RequestTimeout <= 0 { + config.RequestTimeout = defaultRequestTimeout + } + if config.ConnectTimeout <= 0 { + config.ConnectTimeout = defaultConnectTimeout + } + if proxy := nestedMap(connection, "socks_proxy"); proxy != nil { + host := stringValue(proxy, "host") + port := intValue(proxy, 0, "port") + if host == "" || port <= 0 || port > 65535 { + return connectionConfig{}, fmt.Errorf("socks_proxy host and port are required") + } + config.SocksProxy = &socksProxyConfig{ + Host: host, Port: port, + Username: stringValue(proxy, "username"), + Password: stringValue(proxy, "password"), + } + } + return config, nil +} + +func (a *rocketMQAgent) connect(params map[string]any) (any, error) { + config, err := parseConnection(params) + if err != nil { + return nil, err + } + client, proxies, clusterInfo, err := buildClient(config) + if err != nil { + return nil, err + } + + clusterName := resolveClusterName(clusterInfo, config.ClusterName) + brokerAddr, err := resolveBrokerAddr(clusterInfo, config, proxies) + if err != nil { + client.Close() + proxies.Close() + return nil, err + } + + a.mu.Lock() + oldClient, oldProxies := a.client, a.proxies + a.client = client + a.proxies = proxies + a.connection = config + a.clusterName = clusterName + a.brokerAddr = brokerAddr + a.mu.Unlock() + if oldClient != nil { + _ = oldClient.Close() + } + if oldProxies != nil { + oldProxies.Close() + } + return clusterTestResult(client, clusterInfo, config, proxies), nil +} + +func (a *rocketMQAgent) testConnection(params map[string]any) (any, error) { + config, err := parseConnection(params) + if err != nil { + return nil, err + } + a.mu.RLock() + if a.client != nil && a.connection.equal(config) { + client, proxies := a.client, a.proxies + a.mu.RUnlock() + ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout) + defer cancel() + clusterInfo, err := client.ExamineBrokerClusterInfo(ctx) + if err != nil { + return nil, err + } + return clusterTestResult(client, clusterInfo, config, proxies), nil + } + a.mu.RUnlock() + + client, proxies, clusterInfo, err := buildClient(config) + if err != nil { + return nil, err + } + defer client.Close() + defer proxies.Close() + return clusterTestResult(client, clusterInfo, config, proxies), nil +} + +func buildClient(config connectionConfig) (*admin.Client, *proxyManager, *admin.ClusterInfo, error) { + proxies := newProxyManager(config) + localNameServers := make([]string, 0, len(config.NameServers)) + for _, address := range config.NameServers { + local, err := proxies.ProxyFor(address, proxyTargetNameServer) + if err != nil { + proxies.Close() + return nil, nil, nil, err + } + localNameServers = append(localNameServers, local) + } + client, err := admin.NewClient( + admin.WithNameServers(localNameServers), + admin.WithTimeout(config.RequestTimeout), + admin.WithRetryTimes(1), + ) + if err != nil { + proxies.Close() + return nil, nil, nil, err + } + if err := client.Start(); err != nil { + proxies.Close() + return nil, nil, nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), config.ConnectTimeout) + defer cancel() + clusterInfo, err := waitForBrokerRegistration(ctx, brokerRegistrationPollInterval, client.ExamineBrokerClusterInfo) + if err != nil { + client.Close() + proxies.Close() + return nil, nil, nil, err + } + return client, proxies, clusterInfo, nil +} + +func waitForBrokerRegistration( + ctx context.Context, + pollInterval time.Duration, + examine func(context.Context) (*admin.ClusterInfo, error), +) (*admin.ClusterInfo, error) { + if pollInterval <= 0 { + pollInterval = brokerRegistrationPollInterval + } + var lastErr error + for { + clusterInfo, err := examine(ctx) + if err == nil && hasMasterBroker(clusterInfo) { + return clusterInfo, nil + } + if err != nil { + lastErr = err + } + + timer := time.NewTimer(pollInterval) + select { + case <-ctx.Done(): + timer.Stop() + if lastErr != nil { + return nil, fmt.Errorf("wait for RocketMQ broker registration after %v: %w", lastErr, ctx.Err()) + } + return nil, fmt.Errorf("wait for RocketMQ broker registration: %w", ctx.Err()) + case <-timer.C: + } + } +} + +func hasMasterBroker(info *admin.ClusterInfo) bool { + if info == nil { + return false + } + for _, broker := range info.BrokerAddrTable { + if broker != nil && broker.BrokerAddrs["0"] != "" { + return true + } + } + return false +} + +func clusterTestResult(client *admin.Client, info *admin.ClusterInfo, config connectionConfig, proxies *proxyManager) map[string]any { + brokers := brokerNodes(info, proxies) + aclEnabled := false + if len(brokers) > 0 { + ctx, cancel := context.WithTimeout(context.Background(), minDuration(config.ConnectTimeout/2, 5*time.Second)) + defer cancel() + brokerAddr, err := resolveBrokerAddr(info, config, proxies) + if err == nil { + _, aclErr := client.GetBrokerClusterAclInfo(ctx, brokerAddr) + aclEnabled = aclErr == nil + } + } + var controller any + if len(brokers) > 0 { + controller = brokers[0] + } + return map[string]any{ + "ok": true, "clusterId": resolveClusterName(info, config.ClusterName), + "brokers": brokers, "nodeCount": len(brokers), "controller": controller, + "aclEnabled": aclEnabled, + } +} + +func (config connectionConfig) equal(other connectionConfig) bool { + if config.ClusterName != other.ClusterName || config.BrokerAddr != other.BrokerAddr || + config.AccessKey != other.AccessKey || config.SecretKey != other.SecretKey || + config.RequestTimeout != other.RequestTimeout || config.ConnectTimeout != other.ConnectTimeout || + config.TLSSkipVerify != other.TLSSkipVerify || + len(config.NameServers) != len(other.NameServers) { + return false + } + for index := range config.NameServers { + if config.NameServers[index] != other.NameServers[index] { + return false + } + } + if (config.SocksProxy == nil) != (other.SocksProxy == nil) { + return false + } + return config.SocksProxy == nil || *config.SocksProxy == *other.SocksProxy +} + +func resolveClusterName(info *admin.ClusterInfo, configured string) string { + if configured != "" { + return configured + } + for _, name := range sortedKeys(info.ClusterAddrTable) { + return name + } + return "DefaultCluster" +} + +func resolveBrokerAddr(info *admin.ClusterInfo, config connectionConfig, proxies *proxyManager) (string, error) { + if config.BrokerAddr != "" { + if local := proxies.LocalForOriginal(config.BrokerAddr); local != "" { + return local, nil + } + return proxies.ProxyFor(config.BrokerAddr, proxyTargetBroker) + } + for _, brokerName := range sortedKeys(info.BrokerAddrTable) { + broker := info.BrokerAddrTable[brokerName] + if address := broker.BrokerAddrs["0"]; address != "" { + return address, nil + } + for _, brokerID := range sortedKeys(broker.BrokerAddrs) { + if broker.BrokerAddrs[brokerID] != "" { + return broker.BrokerAddrs[brokerID], nil + } + } + } + return "", fmt.Errorf("no RocketMQ broker address found") +} + +func minDuration(left, right time.Duration) time.Duration { + if left <= 0 || left > right { + return right + } + return left +} diff --git a/agents/drivers/rocketmq/connection_test.go b/agents/drivers/rocketmq/connection_test.go new file mode 100644 index 0000000000..03357afebe --- /dev/null +++ b/agents/drivers/rocketmq/connection_test.go @@ -0,0 +1,118 @@ +package main + +import ( + "context" + "errors" + "reflect" + "testing" + "time" + + admin "github.com/amigoer/rocketmq-admin-go" +) + +func TestParseConnectionSupportsAliasesAndMultipleNameServers(t *testing.T) { + config, err := parseConnection(map[string]any{"connection": map[string]any{ + "namesrvAddr": "127.0.0.1:9876; [2001:db8::1]:9876,127.0.0.1:9876", + "clusterName": "DefaultCluster", + "brokerAddr": "broker.example.com:10911", + "accessKey": "access", + "secretKey": "secret", + "request_timeout_ms": float64(12_000), + "connect_timeout_ms": "4500", + "tls_skip_verify": true, + }}) + if err != nil { + t.Fatal(err) + } + wantServers := []string{"127.0.0.1:9876", "[2001:db8::1]:9876"} + if !reflect.DeepEqual(config.NameServers, wantServers) { + t.Fatalf("NameServers = %#v, want %#v", config.NameServers, wantServers) + } + if config.ClusterName != "DefaultCluster" || config.BrokerAddr != "broker.example.com:10911" { + t.Fatalf("unexpected route config: %#v", config) + } + if config.AccessKey != "access" || config.SecretKey != "secret" { + t.Fatalf("unexpected credentials: %#v", config) + } + if config.RequestTimeout != 12*time.Second || config.ConnectTimeout != 4500*time.Millisecond { + t.Fatalf("unexpected timeouts: %#v", config) + } + if !config.TLSSkipVerify { + t.Fatal("tls_skip_verify was not parsed") + } +} + +func TestParseConnectionValidatesNameServerAndSocksProxy(t *testing.T) { + if _, err := parseConnection(map[string]any{}); err == nil { + t.Fatal("expected missing namesrv_addr to fail") + } + if _, err := parseConnection(map[string]any{ + "namesrv_addr": "127.0.0.1:9876", + "socks_proxy": map[string]any{"host": "127.0.0.1"}, + }); err == nil { + t.Fatal("expected incomplete socks_proxy to fail") + } +} + +func TestConnectionConfigEqualIncludesTLSSkipVerify(t *testing.T) { + base := connectionConfig{NameServers: []string{"127.0.0.1:9876"}} + other := base + other.TLSSkipVerify = true + if base.equal(other) { + t.Fatal("configs with different tls_skip_verify values must not be equal") + } +} + +func TestWaitForBrokerRegistrationRetriesUntilMasterAppears(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + calls := 0 + info, err := waitForBrokerRegistration(ctx, time.Millisecond, func(context.Context) (*admin.ClusterInfo, error) { + calls++ + if calls < 3 { + return &admin.ClusterInfo{BrokerAddrTable: map[string]*admin.BrokerData{}}, nil + } + return &admin.ClusterInfo{BrokerAddrTable: map[string]*admin.BrokerData{ + "broker-a": {BrokerAddrs: map[string]string{"0": "127.0.0.1:10911"}}, + }}, nil + }) + if err != nil { + t.Fatal(err) + } + if calls != 3 || !hasMasterBroker(info) { + t.Fatalf("calls = %d, info = %#v", calls, info) + } +} + +func TestWaitForBrokerRegistrationTimesOut(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + _, err := waitForBrokerRegistration(ctx, time.Millisecond, func(context.Context) (*admin.ClusterInfo, error) { + return &admin.ClusterInfo{BrokerAddrTable: map[string]*admin.BrokerData{}}, nil + }) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("error = %v, want context deadline exceeded", err) + } +} + +func TestParseAndFormatSocketAddress(t *testing.T) { + tests := []struct { + address string + host string + port string + formatted string + }{ + {"127.0.0.1:9876", "127.0.0.1", "9876", "127.0.0.1:9876"}, + {"[2001:db8::1]:9876", "2001:db8::1", "9876", "[2001:db8::1]:9876"}, + {"broker.example.com", "broker.example.com", "", "broker.example.com"}, + } + for _, test := range tests { + host, port := parseSocketAddress(test.address) + if host != test.host || port != test.port { + t.Fatalf("parseSocketAddress(%q) = %q, %q", test.address, host, port) + } + if formatted := formatSocketAddress(host, port); formatted != test.formatted { + t.Fatalf("formatSocketAddress(%q, %q) = %q", host, port, formatted) + } + } +} diff --git a/agents/drivers/rocketmq/consumers.go b/agents/drivers/rocketmq/consumers.go new file mode 100644 index 0000000000..6e453abfdb --- /dev/null +++ b/agents/drivers/rocketmq/consumers.go @@ -0,0 +1,525 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "sync" + "time" + + admin "github.com/amigoer/rocketmq-admin-go" + "github.com/amigoer/rocketmq-admin-go/protocol/remoting" +) + +var systemConsumerGroups = map[string]struct{}{ + "TOOLS_CONSUMER": {}, "FILTERSRV_CONSUMER": {}, "SELF_TEST_C_GROUP": {}, + "CID_ONS-HTTP-PROXY": {}, "CID_ONSAPI_PULL": {}, "CID_ONSAPI_PERMISSION": {}, + "CID_ONSAPI_OWNER": {}, "CID_SYS_RMQ_TRANS": {}, "CID_DefaultHeartBeatSyncerTopic": {}, +} + +type subscriptionGroupConfig struct { + GroupName string `json:"groupName"` + ConsumeEnable bool `json:"consumeEnable"` + ConsumeFromMinEnable bool `json:"consumeFromMinEnable"` + ConsumeBroadcastEnable bool `json:"consumeBroadcastEnable"` + ConsumeMessageOrderly bool `json:"consumeMessageOrderly"` + RetryQueueNums int `json:"retryQueueNums"` + RetryMaxTimes int `json:"retryMaxTimes"` + BrokerID int64 `json:"brokerId"` + WhichBrokerWhenConsumeSlowly int64 `json:"whichBrokerWhenConsumeSlowly"` + NotifyConsumerIDsChangedEnable bool `json:"notifyConsumerIdsChangedEnable"` +} + +func (a *rocketMQAgent) listConsumerGroups(params map[string]any) (any, error) { + client, config, err := a.requireClient() + if err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout) + defer cancel() + + configs, err := a.collectSubscriptionGroupConfigs(ctx) + if err != nil { + return nil, err + } + topicFilter := stringValue(params, "topic") + groupNames := make([]string, 0) + if topicFilter != "" { + groups, queryErr := client.QueryTopicConsumeByWho(ctx, topicFilter) + if queryErr != nil { + return nil, queryErr + } + groupNames = append(groupNames, groups...) + } else { + for name := range configs { + groupNames = append(groupNames, name) + } + } + sort.Strings(groupNames) + groupNames = uniqueStrings(groupNames) + keyword := strings.ToLower(stringValue(params, "keyword")) + rows := make([]map[string]any, 0, len(groupNames)) + for _, groupID := range groupNames { + if keyword != "" && !strings.Contains(strings.ToLower(groupID), keyword) { + continue + } + row := map[string]any{ + "groupId": groupID, "state": "UNKNOWN", "simpleGroup": false, + "groupType": classifyConsumerGroup(groupID, configs[groupID]), "messageModel": "CLUSTERING", + } + if topicFilter != "" { + row["topics"] = []string{topicFilter} + } + rows = append(rows, row) + } + + offset := max(0, intValue(params, 0, "offset")) + limit := intValue(params, 200, "limit") + if limit <= 0 { + limit = 200 + } + page := paginate(rows, offset, limit) + if boolValue(params, false, "enrich") { + a.enrichConsumerGroups(ctx, client, page) + } + if topicFilter != "" && boolValue(params, false, "includeLag") { + a.attachConsumerGroupLag(page, topicFilter) + } + return map[string]any{ + "groups": page, "total": len(rows), "offset": offset, "limit": limit, + }, nil +} + +func (a *rocketMQAgent) describeConsumerGroup(params map[string]any) (any, error) { + groupID, err := requireString(params, "groupId") + if err != nil { + return nil, err + } + client, config, _ := a.requireClient() + ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout) + defer cancel() + connection, err := client.ExamineConsumerConnectionInfo(ctx, groupID) + if err != nil { + return nil, err + } + assignments := make([]map[string]any, 0, len(connection.SubscriptionTable)) + for _, topic := range sortedKeys(connection.SubscriptionTable) { + subscription := connection.SubscriptionTable[topic] + assignments = append(assignments, map[string]any{ + "topic": subscription.Topic, "subExpression": subscription.SubString, + }) + } + members := make([]map[string]any, 0, len(connection.ConnectionSet)) + for _, member := range connection.ConnectionSet { + members = append(members, map[string]any{ + "memberId": member.ClientId, "clientId": member.ClientId, + "host": member.ClientAddr, "assignments": assignments, + }) + } + return map[string]any{ + "groupId": groupID, "state": valueOrDefault(connection.ConsumeType, "UNKNOWN"), + "partitionAssignor": emptyToNil(connection.MessageModel), + "messageModel": emptyToNil(connection.MessageModel), "members": members, + }, nil +} + +func (a *rocketMQAgent) deleteConsumerGroup(params map[string]any) (any, error) { + groupID, err := requireString(params, "groupId") + if err != nil { + return nil, err + } + client, config, _ := a.requireClient() + ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout) + defer cancel() + addresses, err := a.masterBrokerAddresses("") + if err != nil { + return nil, err + } + successCount := 0 + var lastErr error + for _, address := range addresses { + if mutationErr := client.DeleteSubscriptionGroup(ctx, address, groupID); mutationErr != nil { + lastErr = mutationErr + continue + } + successCount++ + } + if err := ensureMutationCoverage("delete", "consumer group "+groupID, len(addresses), successCount, lastErr); err != nil { + return nil, err + } + return okResult(), nil +} + +func (a *rocketMQAgent) getSubscriptionGroupConfig(params map[string]any) (any, error) { + groupID, err := requireString(params, "groupId") + if err != nil { + return nil, err + } + _, config, _ := a.requireClient() + ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout) + defer cancel() + configs, err := a.collectSubscriptionGroupConfigs(ctx) + if err != nil { + return nil, err + } + groupConfig := configs[groupID] + if groupConfig == nil { + return nil, fmt.Errorf("consumer group not found: %s", groupID) + } + return subscriptionGroupConfigMap(groupConfig), nil +} + +func (a *rocketMQAgent) alterSubscriptionGroupConfig(params map[string]any) (any, error) { + groupID, err := requireString(params, "groupId") + if err != nil { + return nil, err + } + _, config, _ := a.requireClient() + ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout) + defer cancel() + configs, collectErr := a.collectSubscriptionGroupConfigs(ctx) + if collectErr != nil { + return nil, collectErr + } + groupConfig := configs[groupID] + if groupConfig == nil { + groupConfig = &subscriptionGroupConfig{ + GroupName: groupID, ConsumeEnable: true, ConsumeFromMinEnable: true, + RetryQueueNums: 1, RetryMaxTimes: 16, NotifyConsumerIDsChangedEnable: true, + } + } + applySubscriptionGroupUpdates(groupConfig, params) + addresses, err := a.masterBrokerAddresses("") + if err != nil { + return nil, err + } + successCount := 0 + var lastErr error + for _, address := range addresses { + if mutationErr := writeSubscriptionGroupConfig(ctx, address, groupConfig); mutationErr != nil { + lastErr = mutationErr + continue + } + successCount++ + } + if err := ensureMutationCoverage("update", "consumer group "+groupID, len(addresses), successCount, lastErr); err != nil { + return nil, err + } + return okResult(), nil +} + +func (a *rocketMQAgent) resetConsumerGroupOffsets(params map[string]any) (any, error) { + groupID, err := requireString(params, "groupId") + if err != nil { + return nil, err + } + topic, err := requireString(params, "topic") + if err != nil { + return nil, err + } + client, config, _ := a.requireClient() + ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout) + defer cancel() + if offsets, ok := params["offsets"].([]any); ok && len(offsets) > 0 { + address, addressErr := a.brokerAddressForName(stringValue(params, "brokerName")) + if addressErr != nil { + return nil, addressErr + } + for _, raw := range offsets { + offset, _ := raw.(map[string]any) + if err := client.UpdateConsumeOffset(ctx, address, groupID, topic, + intValue(offset, 0, "partition"), int64Value(offset, 0, "offset")); err != nil { + return nil, err + } + } + return okResult(), nil + } + position := strings.ToLower(stringValue(params, "position")) + if position == "" { + position = "latest" + } + var timestamp int64 + switch position { + case "earliest": + timestamp = 0 + case "latest": + timestamp = time.Now().UnixMilli() + case "timestamp": + timestamp = int64Value(params, time.Now().UnixMilli(), "timestampMs") + default: + return nil, fmt.Errorf("unsupported reset position: %s", position) + } + if _, err := client.ResetOffsetByTimestamp(ctx, topic, groupID, timestamp, true); err != nil { + return nil, err + } + return okResult(), nil +} + +func (a *rocketMQAgent) getConsumerLag(params map[string]any) (any, error) { + groupID, err := requireString(params, "groupId") + if err != nil { + return nil, err + } + topic, err := requireString(params, "topic") + if err != nil { + return nil, err + } + client, config, _ := a.requireClient() + ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout) + defer cancel() + stats, err := a.examineConsumeStatsByTopic(ctx, client, groupID, topic) + if err != nil { + return nil, err + } + partitions := make([]map[string]any, 0, len(stats.OffsetTable)) + var totalLag int64 + for key, offset := range stats.OffsetTable { + queue := parseMessageQueueKey(key) + lag := max(int64(0), offset.BrokerOffset-offset.ConsumerOffset) + totalLag += lag + partitions = append(partitions, map[string]any{ + "partition": queue.QueueID, "currentOffset": offset.ConsumerOffset, + "endOffset": offset.BrokerOffset, "lag": lag, "brokerName": queue.BrokerName, + "lastTimestamp": offset.LastTimestamp, "consumerClient": "", + }) + } + sort.Slice(partitions, func(i, j int) bool { + left, right := partitions[i], partitions[j] + if left["brokerName"] != right["brokerName"] { + return fmt.Sprint(left["brokerName"]) < fmt.Sprint(right["brokerName"]) + } + return left["partition"].(int) < right["partition"].(int) + }) + return map[string]any{"partitions": partitions, "totalLag": totalLag}, nil +} + +func (a *rocketMQAgent) examineConsumeStatsByTopic( + ctx context.Context, + client *admin.Client, + groupID string, + topic string, +) (*admin.ConsumeStats, error) { + route, err := client.ExamineTopicRouteInfo(ctx, topic) + if err != nil { + return nil, err + } + addresses := masterAddressesFromRoute(route) + if len(addresses) == 0 { + return nil, fmt.Errorf("no RocketMQ master broker found for topic %s", topic) + } + merged := &admin.ConsumeStats{OffsetTable: make(map[string]*admin.OffsetWrapper)} + successCount := 0 + var lastErr error + for _, address := range addresses { + response, requestErr := invokeRemotingWithClient(ctx, address, remoting.NewRequest( + remoting.GetConsumeStats, + map[string]string{"consumerGroup": groupID, "topic": topic}, + )) + if requestErr != nil { + lastErr = requestErr + continue + } + partial, decodeErr := decodeConsumeStats(response.Body) + if decodeErr != nil { + lastErr = decodeErr + continue + } + successCount++ + for key, offset := range partial.OffsetTable { + merged.OffsetTable[key] = offset + } + merged.ConsumeTps += partial.ConsumeTps + } + if successCount == 0 { + return nil, fmt.Errorf("query consumer lag for group %s on all masters: %w", groupID, lastErr) + } + if len(merged.OffsetTable) == 0 && successCount != len(addresses) { + return nil, fmt.Errorf("consumer lag for group %s is incomplete: %w", groupID, lastErr) + } + return merged, nil +} + +func decodeConsumeStats(body []byte) (*admin.ConsumeStats, error) { + var stats admin.ConsumeStats + if err := json.Unmarshal(repairRocketMQJSON(body), &stats); err != nil { + return nil, fmt.Errorf("decode consumer stats: %w", err) + } + if stats.OffsetTable == nil { + stats.OffsetTable = make(map[string]*admin.OffsetWrapper) + } + return &stats, nil +} + +func (a *rocketMQAgent) collectSubscriptionGroupConfigs(ctx context.Context) (map[string]*subscriptionGroupConfig, error) { + addresses, err := a.masterBrokerAddresses("") + if err != nil { + return nil, err + } + merged := make(map[string]*subscriptionGroupConfig) + var lastErr error + for _, address := range addresses { + configs, fetchErr := fetchSubscriptionGroupConfigs(ctx, address) + if fetchErr != nil { + lastErr = fetchErr + continue + } + for name, next := range configs { + current := merged[name] + if current == nil || next.ConsumeMessageOrderly { + merged[name] = next + } + } + } + if len(merged) == 0 && lastErr != nil { + return nil, lastErr + } + return merged, nil +} + +func fetchSubscriptionGroupConfigs(ctx context.Context, address string) (map[string]*subscriptionGroupConfig, error) { + response, err := invokeRemotingWithClient(ctx, address, + remoting.NewRequest(remoting.GetAllSubscriptionGroupConfig, nil)) + if err != nil { + return nil, err + } + var wrapper struct { + SubscriptionGroupTable map[string]*subscriptionGroupConfig `json:"subscriptionGroupTable"` + } + if err := json.Unmarshal(repairRocketMQJSON(response.Body), &wrapper); err != nil { + return nil, fmt.Errorf("decode subscription groups: %w", err) + } + return wrapper.SubscriptionGroupTable, nil +} + +func writeSubscriptionGroupConfig(ctx context.Context, address string, config *subscriptionGroupConfig) error { + body, err := json.Marshal(config) + if err != nil { + return fmt.Errorf("encode subscription group config: %w", err) + } + command := remoting.NewRequest(remoting.UpdateAndCreateSubscriptionGroup, nil) + command.Body = body + _, err = invokeRemotingWithClient(ctx, address, command) + return err +} + +func ensureMutationCoverage(action, resource string, attempted, succeeded int, lastErr error) error { + if attempted <= 0 { + return fmt.Errorf("no RocketMQ master brokers available to %s %s", action, resource) + } + if succeeded == attempted { + return nil + } + message := fmt.Sprintf("failed to %s %s on all masters: %d of %d succeeded", action, resource, succeeded, attempted) + if lastErr != nil { + return fmt.Errorf("%s: %w", message, lastErr) + } + return fmt.Errorf("%s", message) +} + +func (a *rocketMQAgent) enrichConsumerGroups(ctx context.Context, client *admin.Client, rows []map[string]any) { + for _, row := range rows { + groupID := fmt.Sprint(row["groupId"]) + connection, err := client.ExamineConsumerConnectionInfo(ctx, groupID) + if err != nil { + if _, ok := row["topics"]; !ok { + row["topics"] = []string{} + } + continue + } + row["consumeType"] = valueOrDefault(connection.ConsumeType, "UNKNOWN") + row["messageModel"] = valueOrDefault(connection.MessageModel, "CLUSTERING") + row["memberCount"] = len(connection.ConnectionSet) + topics := make([]string, 0, len(connection.SubscriptionTable)) + for topic := range connection.SubscriptionTable { + topics = append(topics, topic) + } + sort.Strings(topics) + row["topics"] = topics + } +} + +func (a *rocketMQAgent) attachConsumerGroupLag(rows []map[string]any, topic string) { + semaphore := make(chan struct{}, 8) + var waitGroup sync.WaitGroup + for _, row := range rows { + row := row + waitGroup.Add(1) + go func() { + defer waitGroup.Done() + semaphore <- struct{}{} + defer func() { <-semaphore }() + result, err := a.getConsumerLag(map[string]any{"groupId": row["groupId"], "topic": topic}) + if err != nil { + row["totalLagFailed"] = true + return + } + lag := result.(map[string]any) + row["totalLag"] = lag["totalLag"] + row["partitions"] = lag["partitions"] + }() + } + waitGroup.Wait() +} + +func classifyConsumerGroup(groupID string, config *subscriptionGroupConfig) string { + if _, ok := systemConsumerGroups[groupID]; ok { + return "SYSTEM" + } + if config == nil { + return "UNKNOWN" + } + if config.ConsumeMessageOrderly { + return "FIFO" + } + return "NORMAL" +} + +func subscriptionGroupConfigMap(config *subscriptionGroupConfig) map[string]any { + return map[string]any{ + "groupName": config.GroupName, "consumeEnable": config.ConsumeEnable, + "consumeFromMinEnable": config.ConsumeFromMinEnable, + "consumeBroadcastEnable": config.ConsumeBroadcastEnable, + "consumeMessageOrderly": config.ConsumeMessageOrderly, + "retryQueueNums": config.RetryQueueNums, "retryMaxTimes": config.RetryMaxTimes, + "brokerId": config.BrokerID, "whichBrokerWhenConsumeSlowly": config.WhichBrokerWhenConsumeSlowly, + } +} + +func applySubscriptionGroupUpdates(config *subscriptionGroupConfig, params map[string]any) { + config.ConsumeEnable = boolValue(params, config.ConsumeEnable, "consumeEnable") + config.ConsumeFromMinEnable = boolValue(params, config.ConsumeFromMinEnable, "consumeFromMinEnable") + config.ConsumeBroadcastEnable = boolValue(params, config.ConsumeBroadcastEnable, "consumeBroadcastEnable") + config.ConsumeMessageOrderly = boolValue(params, config.ConsumeMessageOrderly, "consumeMessageOrderly") + config.RetryQueueNums = intValue(params, config.RetryQueueNums, "retryQueueNums") + config.RetryMaxTimes = intValue(params, config.RetryMaxTimes, "retryMaxTimes") + config.BrokerID = int64Value(params, config.BrokerID, "brokerId") + config.WhichBrokerWhenConsumeSlowly = int64Value(params, config.WhichBrokerWhenConsumeSlowly, "whichBrokerWhenConsumeSlowly") +} + +func uniqueStrings(values []string) []string { + result := values[:0] + var previous string + for index, value := range values { + if index == 0 || value != previous { + result = append(result, value) + previous = value + } + } + return result +} + +func valueOrDefault(value, fallback string) string { + if value == "" { + return fallback + } + return value +} + +func emptyToNil(value string) any { + if value == "" { + return nil + } + return value +} diff --git a/agents/drivers/rocketmq/consumers_test.go b/agents/drivers/rocketmq/consumers_test.go new file mode 100644 index 0000000000..e198364e30 --- /dev/null +++ b/agents/drivers/rocketmq/consumers_test.go @@ -0,0 +1,70 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestClassifyConsumerGroup(t *testing.T) { + if got := classifyConsumerGroup("TOOLS_CONSUMER", nil); got != "SYSTEM" { + t.Fatalf("system group classified as %q", got) + } + if got := classifyConsumerGroup("GID_Orders", nil); got != "UNKNOWN" { + t.Fatalf("missing config classified as %q", got) + } + if got := classifyConsumerGroup("GID_Orders", &subscriptionGroupConfig{}); got != "NORMAL" { + t.Fatalf("normal group classified as %q", got) + } + if got := classifyConsumerGroup("GID_FIFO", &subscriptionGroupConfig{ConsumeMessageOrderly: true}); got != "FIFO" { + t.Fatalf("FIFO group classified as %q", got) + } +} + +func TestDecodeConsumeStatsRepairsObjectKeys(t *testing.T) { + body := []byte(`{"consumeTps":1.5,"offsetTable":{{"brokerName":"broker-a","queueId":2,"topic":"Orders"}:{"brokerOffset":20,"consumerOffset":12,"lastTimestamp":9,"pullOffset":13}}}`) + stats, err := decodeConsumeStats(body) + if err != nil { + t.Fatal(err) + } + if stats.ConsumeTps != 1.5 || len(stats.OffsetTable) != 1 { + t.Fatalf("unexpected consume stats: %#v", stats) + } + for key, offset := range stats.OffsetTable { + queue := parseMessageQueueKey(key) + if queue.Topic != "Orders" || queue.BrokerName != "broker-a" || queue.QueueID != 2 { + t.Fatalf("unexpected queue key %q: %#v", key, queue) + } + if offset.BrokerOffset != 20 || offset.ConsumerOffset != 12 { + t.Fatalf("unexpected offset: %#v", offset) + } + } +} + +func TestMutationCoverageFailureMessage(t *testing.T) { + err := ensureMutationCoverage("update", "consumer group GID_Orders", 2, 1, nil) + if err == nil || !strings.Contains(err.Error(), "1 of 2") { + t.Fatalf("unexpected partial mutation error: %v", err) + } + if err := ensureMutationCoverage("update", "consumer group GID_Orders", 2, 2, nil); err != nil { + t.Fatal(err) + } +} + +func TestSubscriptionGroupConfigWireShape(t *testing.T) { + config := subscriptionGroupConfig{ + GroupName: "GID_Orders", ConsumeEnable: true, ConsumeMessageOrderly: true, + RetryQueueNums: 2, RetryMaxTimes: 20, NotifyConsumerIDsChangedEnable: true, + } + body, err := json.Marshal(config) + if err != nil { + t.Fatal(err) + } + var decoded map[string]any + if err := json.Unmarshal(body, &decoded); err != nil { + t.Fatal(err) + } + if decoded["groupName"] != "GID_Orders" || decoded["consumeMessageOrderly"] != true { + t.Fatalf("unexpected group wire body: %#v", decoded) + } +} diff --git a/agents/drivers/rocketmq/go.mod b/agents/drivers/rocketmq/go.mod new file mode 100644 index 0000000000..358661354c --- /dev/null +++ b/agents/drivers/rocketmq/go.mod @@ -0,0 +1,28 @@ +module github.com/t8y2/dbx/agents/drivers/rocketmq + +go 1.25.0 + +require ( + github.com/amigoer/rocketmq-admin-go v1.1.1 + golang.org/x/net v0.44.0 +) + +require ( + github.com/apache/rocketmq-client-go/v2 v2.1.2 // indirect + github.com/emirpasic/gods v1.18.1 // indirect + github.com/golang/mock v1.6.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/patrickmn/go-cache v2.1.0+incompatible // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/sirupsen/logrus v1.9.4 // indirect + github.com/tidwall/gjson v1.18.0 // indirect + github.com/tidwall/match v1.2.0 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + go.uber.org/atomic v1.11.0 // indirect + golang.org/x/sys v0.42.0 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect + stathat.com/c/consistent v1.0.0 // indirect +) diff --git a/agents/drivers/rocketmq/go.sum b/agents/drivers/rocketmq/go.sum new file mode 100644 index 0000000000..3e53cfa55c --- /dev/null +++ b/agents/drivers/rocketmq/go.sum @@ -0,0 +1,183 @@ +github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/amigoer/rocketmq-admin-go v1.1.1 h1:DhV7c1lukiB5Qy115/b4a7yE3XXLMRYJ/Jz77miLZoc= +github.com/amigoer/rocketmq-admin-go v1.1.1/go.mod h1:I/N1RG6IvNEP2APjKh66Tg7YbZyW9aetiEEl8ziuYPs= +github.com/apache/rocketmq-client-go/v2 v2.1.2 h1:yt73olKe5N6894Dbm+ojRf/JPiP0cxfDNNffKwhpJVg= +github.com/apache/rocketmq-client-go/v2 v2.1.2/go.mod h1:6I6vgxHR3hzrvn+6n/4mrhS+UTulzK/X9LB2Vk1U5gE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= +github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= +github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tidwall/gjson v1.13.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/match v1.2.0 h1:0pt8FlkOwjN2fPt4bIl4BoNxb98gGHN2ObFEDkrfZnM= +github.com/tidwall/match v1.2.0/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= +golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c= +stathat.com/c/consistent v1.0.0/go.mod h1:QkzMWzcbB+yQBL2AttO6sgsQS/JSTapcDISJalmCDS0= diff --git a/agents/drivers/rocketmq/helpers.go b/agents/drivers/rocketmq/helpers.go new file mode 100644 index 0000000000..f22474f116 --- /dev/null +++ b/agents/drivers/rocketmq/helpers.go @@ -0,0 +1,228 @@ +package main + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "net" + "sort" + "strconv" + "strings" +) + +func nestedMap(params map[string]any, key string) map[string]any { + value, ok := params[key] + if !ok || value == nil { + return nil + } + if result, ok := value.(map[string]any); ok { + return result + } + return nil +} + +func stringValue(params map[string]any, keys ...string) string { + for _, key := range keys { + if value, ok := params[key]; ok && value != nil { + switch typed := value.(type) { + case string: + return strings.TrimSpace(typed) + case json.Number: + return typed.String() + case float64: + return strconv.FormatFloat(typed, 'f', -1, 64) + } + } + } + return "" +} + +func requireString(params map[string]any, keys ...string) (string, error) { + value := stringValue(params, keys...) + if value == "" { + return "", fmt.Errorf("%s is required", keys[0]) + } + return value, nil +} + +func intValue(params map[string]any, defaultValue int, keys ...string) int { + for _, key := range keys { + value, ok := params[key] + if !ok || value == nil { + continue + } + switch typed := value.(type) { + case float64: + return int(typed) + case json.Number: + parsed, err := typed.Int64() + if err == nil { + return int(parsed) + } + case string: + parsed, err := strconv.Atoi(strings.TrimSpace(typed)) + if err == nil { + return parsed + } + } + } + return defaultValue +} + +func int64Value(params map[string]any, defaultValue int64, keys ...string) int64 { + for _, key := range keys { + value, ok := params[key] + if !ok || value == nil { + continue + } + switch typed := value.(type) { + case float64: + return int64(typed) + case json.Number: + parsed, err := typed.Int64() + if err == nil { + return parsed + } + case string: + parsed, err := strconv.ParseInt(strings.TrimSpace(typed), 10, 64) + if err == nil { + return parsed + } + } + } + return defaultValue +} + +func boolValue(params map[string]any, defaultValue bool, keys ...string) bool { + for _, key := range keys { + value, ok := params[key] + if !ok || value == nil { + continue + } + switch typed := value.(type) { + case bool: + return typed + case string: + parsed, err := strconv.ParseBool(strings.TrimSpace(typed)) + if err == nil { + return parsed + } + } + } + return defaultValue +} + +func decodePayload(params map[string]any) ([]byte, error) { + encoded := stringValue(params, "payloadBase64") + if encoded != "" { + payload, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + return nil, fmt.Errorf("invalid payloadBase64: %w", err) + } + return payload, nil + } + return []byte(stringValue(params, "payload", "payloadText")), nil +} + +func splitAddresses(value string) []string { + fields := strings.FieldsFunc(value, func(char rune) bool { + return char == ';' || char == ',' || char == '\n' || char == '\r' || char == '\t' || char == ' ' + }) + seen := map[string]struct{}{} + result := make([]string, 0, len(fields)) + for _, field := range fields { + address := strings.TrimSpace(field) + if address == "" { + continue + } + if _, ok := seen[address]; ok { + continue + } + seen[address] = struct{}{} + result = append(result, address) + } + return result +} + +func parseSocketAddress(address string) (string, string) { + trimmed := strings.TrimSpace(address) + if trimmed == "" { + return "", "" + } + if host, port, err := net.SplitHostPort(trimmed); err == nil { + return strings.Trim(host, "[]"), port + } + colon := strings.LastIndex(trimmed, ":") + if colon <= 0 || colon == len(trimmed)-1 { + return trimmed, "" + } + if _, err := strconv.Atoi(trimmed[colon+1:]); err != nil { + return trimmed, "" + } + return strings.Trim(trimmed[:colon], "[]"), trimmed[colon+1:] +} + +func formatSocketAddress(host, port string) string { + if host == "" { + return port + } + if port == "" { + return host + } + return net.JoinHostPort(strings.Trim(host, "[]"), port) +} + +func isLikelyUnreachableBrokerHost(host string) bool { + host = strings.TrimSpace(strings.ToLower(host)) + if host == "" || host == "127.0.0.1" || host == "localhost" || host == "::1" { + return false + } + if strings.HasSuffix(host, ".docker") || strings.Contains(host, ".docker.") { + return true + } + ip := net.ParseIP(host) + return ip != nil && ip.IsPrivate() +} + +func sortedKeys[V any](values map[string]V) []string { + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + +func paginate[T any](items []T, offset, limit int) []T { + if offset < 0 { + offset = 0 + } + if offset >= len(items) { + return []T{} + } + end := len(items) + if limit > 0 && offset+limit < end { + end = offset + limit + } + return items[offset:end] +} + +func normalizeTopicPerm(perm int) int { + switch perm { + case 2, 4, 6: + return perm + default: + return 6 + } +} + +func normalizeMessageType(value string) string { + value = strings.ToUpper(strings.TrimSpace(value)) + if value == "" { + return "NORMAL" + } + if value == "ORDER" { + return "FIFO" + } + return value +} diff --git a/agents/drivers/rocketmq/integration_test.go b/agents/drivers/rocketmq/integration_test.go new file mode 100644 index 0000000000..42573228d5 --- /dev/null +++ b/agents/drivers/rocketmq/integration_test.go @@ -0,0 +1,342 @@ +package main + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "os" + "strings" + "testing" + "time" +) + +type integrationTopicList struct { + Topics []struct { + Name string `json:"name"` + Partitions int `json:"partitions"` + ReadQueueNums int `json:"readQueueNums"` + WriteQueueNums int `json:"writeQueueNums"` + MessageType string `json:"messageType"` + } `json:"topics"` +} + +type integrationTopicConfig struct { + Configs map[string]struct { + Value string `json:"value"` + } `json:"configs"` +} + +type integrationMessages struct { + Messages []struct { + MessageID string `json:"messageId"` + Partition int `json:"partition"` + Offset int64 `json:"offset"` + PayloadText string `json:"payloadText"` + Headers map[string]string `json:"headers"` + } `json:"messages"` +} + +type integrationLag struct { + TotalLag int64 `json:"totalLag"` +} + +type integrationConsumerGroups struct { + Groups []struct { + GroupID string `json:"groupId"` + GroupType string `json:"groupType"` + } `json:"groups"` +} + +func TestRocketMQIntegration(t *testing.T) { + if os.Getenv("ROCKETMQ_INTEGRATION") != "1" { + t.Skip("set ROCKETMQ_INTEGRATION=1 to run against a real RocketMQ broker") + } + nameServer := envString("ROCKETMQ_NAMESRV_ADDR", "127.0.0.1:9876") + version := envString("ROCKETMQ_VERSION", "unknown") + agent := newRocketMQAgent() + t.Cleanup(agent.close) + connection := map[string]any{ + "namesrv_addr": nameServer, "request_timeout_ms": 20_000, "connect_timeout_ms": 20_000, + } + + var connectionResult struct { + OK bool `json:"ok"` + NodeCount int `json:"nodeCount"` + } + integrationCall(t, agent, "test_connection", map[string]any{"connection": connection}, &connectionResult) + if !connectionResult.OK || connectionResult.NodeCount < 1 { + t.Fatalf("unexpected test connection result: %#v", connectionResult) + } + integrationCall(t, agent, "connect", map[string]any{"connection": connection}, &connectionResult) + if !connectionResult.OK || connectionResult.NodeCount < 1 { + t.Fatalf("unexpected connection result: %#v", connectionResult) + } + + var cluster struct { + ClusterID string `json:"clusterId"` + NodeCount int `json:"nodeCount"` + Brokers []any `json:"brokers"` + } + integrationCall(t, agent, "mq_describe_cluster", map[string]any{}, &cluster) + if cluster.ClusterID == "" || cluster.NodeCount < 1 || len(cluster.Brokers) < 1 { + t.Fatalf("unexpected cluster result: %#v", cluster) + } + + suffix := fmt.Sprintf("%s_%d", strings.ReplaceAll(version, ".", "_"), time.Now().UnixNano()) + topic := "DBX_GO_INTEGRATION_" + suffix + groupID := "GID_DBX_GO_" + suffix + t.Cleanup(func() { + integrationCleanupCall(agent, "mq_delete_consumer_group", map[string]any{"groupId": groupID}) + integrationCleanupCall(agent, "mq_delete_topic", map[string]any{"name": topic}) + }) + + integrationCall[map[string]any](t, agent, "mq_create_topic", map[string]any{ + "name": topic, "readQueueNums": 2, "writeQueueNums": 2, "perm": 6, "messageType": "NORMAL", + }, nil) + + var route struct { + Topic string `json:"topic"` + BrokerDatas []any `json:"brokerDatas"` + QueueDatas []struct { + ReadQueueNums int `json:"readQueueNums"` + WriteQueueNums int `json:"writeQueueNums"` + } `json:"queueDatas"` + } + integrationCall(t, agent, "mq_get_topic_route", map[string]any{"name": topic}, &route) + if route.Topic != topic || len(route.BrokerDatas) == 0 || len(route.QueueDatas) == 0 { + t.Fatalf("unexpected route: %#v", route) + } + if route.QueueDatas[0].ReadQueueNums != 2 || route.QueueDatas[0].WriteQueueNums != 2 { + t.Fatalf("unexpected initial route queues: %#v", route.QueueDatas) + } + var topicsAfterCreate integrationTopicList + integrationCall(t, agent, "mq_list_topics", map[string]any{"keyword": topic, "limit": 10}, &topicsAfterCreate) + if len(topicsAfterCreate.Topics) != 1 || topicsAfterCreate.Topics[0].Name != topic { + t.Fatalf("unexpected topic list after creation: %#v", topicsAfterCreate.Topics) + } + createdTopic := topicsAfterCreate.Topics[0] + if createdTopic.Partitions != 2 || createdTopic.ReadQueueNums != 2 || createdTopic.WriteQueueNums != 2 { + t.Fatalf("unexpected created topic queues: %#v", createdTopic) + } + if strings.HasPrefix(version, "5.") && createdTopic.MessageType != "NORMAL" { + t.Fatalf("RocketMQ %s lost the 5.x message.type attribute: %#v", version, createdTopic) + } + + var topicConfig integrationTopicConfig + integrationCall(t, agent, "mq_get_topic_config", map[string]any{"name": topic}, &topicConfig) + if topicConfig.Configs["readQueueNums"].Value != "2" || topicConfig.Configs["writeQueueNums"].Value != "2" { + t.Fatalf("unexpected topic config: %#v", topicConfig) + } + integrationCall[map[string]any](t, agent, "mq_update_partitions", map[string]any{ + "name": topic, "readQueueNums": 3, "writeQueueNums": 2, + }, nil) + integrationCall(t, agent, "mq_get_topic_config", map[string]any{"name": topic}, &topicConfig) + if topicConfig.Configs["readQueueNums"].Value != "3" || topicConfig.Configs["writeQueueNums"].Value != "2" { + t.Fatalf("unexpected topic config after partition update: %#v", topicConfig) + } + integrationCall[map[string]any](t, agent, "mq_alter_topic_config", map[string]any{ + "name": topic, + "configs": []map[string]any{ + {"key": "readQueueNums", "value": "4"}, + {"key": "writeQueueNums", "value": "3"}, + {"key": "perm", "value": "6"}, + }, + }, nil) + integrationCall(t, agent, "mq_get_topic_config", map[string]any{"name": topic}, &topicConfig) + if topicConfig.Configs["readQueueNums"].Value != "4" || topicConfig.Configs["writeQueueNums"].Value != "3" || topicConfig.Configs["perm"].Value != "6" { + t.Fatalf("unexpected topic config after alteration: %#v", topicConfig) + } + if strings.HasPrefix(version, "5.") { + var topicsAfterAlter integrationTopicList + integrationCall(t, agent, "mq_list_topics", map[string]any{"keyword": topic, "limit": 10}, &topicsAfterAlter) + if len(topicsAfterAlter.Topics) != 1 || topicsAfterAlter.Topics[0].MessageType != "NORMAL" { + t.Fatalf("RocketMQ %s lost message.type after config updates: %#v", version, topicsAfterAlter.Topics) + } + } + + payload := "RocketMQ Go Agent 世界 " + version + var sendResult struct { + OK bool `json:"ok"` + Partition int `json:"partition"` + Offset int64 `json:"offset"` + } + integrationCall(t, agent, "mq_send_message", map[string]any{ + "topic": topic, "partition": 0, "key": "dbx-integration-key", "tag": "dbx-integration", + "payloadBase64": base64.StdEncoding.EncodeToString([]byte(payload)), + "headers": map[string]any{"source": "integration", "empty": "", "nil": nil}, + }, &sendResult) + if !sendResult.OK || sendResult.Partition != 0 || sendResult.Offset < 0 { + t.Fatalf("unexpected send result: %#v", sendResult) + } + + peeked := waitForIntegrationMessages(t, agent, "mq_peek_messages", map[string]any{ + "topic": topic, "partition": 0, "count": 10, + }) + if peeked.Messages[0].PayloadText != payload || peeked.Messages[0].Headers["source"] != "integration" { + t.Fatalf("unexpected peeked message: %#v", peeked.Messages[0]) + } + if _, ok := peeked.Messages[0].Headers["empty"]; ok { + t.Fatalf("empty user property should be omitted: %#v", peeked.Messages[0].Headers) + } + if peeked.Messages[0].MessageID == "" { + t.Fatalf("peeked message has no message ID: %#v", peeked.Messages[0]) + } + + var viewed struct { + Message struct { + MessageID string `json:"messageId"` + PayloadText string `json:"payloadText"` + } `json:"message"` + } + integrationCall(t, agent, "mq_view_message", map[string]any{ + "topic": topic, "msgId": peeked.Messages[0].MessageID, + "partition": peeked.Messages[0].Partition, "offset": peeked.Messages[0].Offset, + }, &viewed) + if viewed.Message.MessageID == "" || viewed.Message.PayloadText != payload { + t.Fatalf("unexpected viewed message: %#v", viewed.Message) + } + + queried := waitForIntegrationMessages(t, agent, "mq_query_message_by_key", map[string]any{ + "topic": topic, "key": "dbx-integration-key", "maxNum": 10, + "begin": 0, "end": time.Now().Add(time.Minute).UnixMilli(), + }) + if queried.Messages[0].PayloadText != payload { + t.Fatalf("unexpected queried message: %#v", queried.Messages[0]) + } + + queriedByTopic := waitForIntegrationMessages(t, agent, "mq_query_message_by_topic", map[string]any{ + "topic": topic, "maxNum": 10, "begin": 0, "end": time.Now().Add(time.Minute).UnixMilli(), + }) + if queriedByTopic.Messages[0].PayloadText != payload { + t.Fatalf("unexpected topic query message: %#v", queriedByTopic.Messages[0]) + } + + var stats struct { + TotalMessages int64 `json:"totalMessages"` + } + integrationCall(t, agent, "mq_get_topic_stats", map[string]any{"name": topic}, &stats) + if stats.TotalMessages < 1 { + t.Fatalf("unexpected topic stats: %#v", stats) + } + + integrationCall[map[string]any](t, agent, "mq_alter_subscription_group_config", map[string]any{ + "groupId": groupID, "consumeEnable": true, "consumeMessageOrderly": true, + "retryQueueNums": 2, "retryMaxTimes": 20, + }, nil) + + var groupConfig struct { + GroupName string `json:"groupName"` + ConsumeMessageOrderly bool `json:"consumeMessageOrderly"` + } + integrationCall(t, agent, "mq_get_subscription_group_config", map[string]any{"groupId": groupID}, &groupConfig) + if groupConfig.GroupName != groupID { + t.Fatalf("unexpected group config: %#v", groupConfig) + } + if strings.HasPrefix(version, "5.") && !groupConfig.ConsumeMessageOrderly { + t.Fatalf("RocketMQ %s did not preserve FIFO group config: %#v", version, groupConfig) + } + + var groups integrationConsumerGroups + integrationCall(t, agent, "mq_list_consumer_groups", map[string]any{ + "keyword": groupID, "limit": 10, + }, &groups) + if len(groups.Groups) != 1 || groups.Groups[0].GroupID != groupID { + t.Fatalf("unexpected consumer groups: %#v", groups) + } + if strings.HasPrefix(version, "5.") && groups.Groups[0].GroupType != "FIFO" { + t.Fatalf("RocketMQ %s did not classify FIFO group: %#v", version, groups.Groups[0]) + } + + var producers struct { + Producers []any `json:"producers"` + } + integrationCall(t, agent, "mq_list_producers", map[string]any{}, &producers) + if producers.Producers == nil { + t.Fatal("producer list must be an array") + } + + var lagBefore integrationLag + integrationCall(t, agent, "mq_get_consumer_lag", map[string]any{"groupId": groupID, "topic": topic}, &lagBefore) + if lagBefore.TotalLag < 1 { + t.Fatalf("unexpected lag before reset: %#v", lagBefore) + } + integrationCall[map[string]any](t, agent, "mq_reset_consumer_group_offsets", map[string]any{ + "groupId": groupID, "topic": topic, + "offsets": []map[string]any{{"partition": 0, "offset": sendResult.Offset + 1}, {"partition": 1, "offset": 0}}, + }, nil) + var lagAfter integrationLag + integrationCall(t, agent, "mq_get_consumer_lag", map[string]any{"groupId": groupID, "topic": topic}, &lagAfter) + if lagAfter.TotalLag != 0 { + t.Fatalf("unexpected lag after reset: %#v", lagAfter) + } + + var skipped struct { + OK bool `json:"ok"` + ResetGroups int `json:"resetGroups"` + } + integrationCall(t, agent, "mq_skip_topic_accumulation", map[string]any{"topic": topic}, &skipped) + if !skipped.OK || skipped.ResetGroups < 0 { + t.Fatalf("unexpected skip accumulation result: %#v", skipped) + } + + integrationCall[map[string]any](t, agent, "mq_delete_consumer_group", map[string]any{"groupId": groupID}, nil) + integrationCall[map[string]any](t, agent, "mq_delete_topic", map[string]any{"name": topic}, nil) + var topics integrationTopicList + integrationCall(t, agent, "mq_list_topics", map[string]any{"keyword": topic, "limit": 10}, &topics) + if len(topics.Topics) != 0 { + t.Fatalf("topic still exists after deletion: %#v", topics.Topics) + } + integrationCall[map[string]any](t, agent, "disconnect", map[string]any{}, nil) +} + +func integrationCall[T any](t *testing.T, agent *rocketMQAgent, method string, params map[string]any, target *T) { + t.Helper() + raw, err := json.Marshal(params) + if err != nil { + t.Fatal(err) + } + result, err := agent.dispatch(method, raw) + if err != nil { + t.Fatalf("%s: %v", method, err) + } + if target == nil { + return + } + encoded, err := json.Marshal(result) + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(encoded, target); err != nil { + t.Fatalf("%s decode: %v; result=%s", method, err, encoded) + } +} + +func integrationCleanupCall(agent *rocketMQAgent, method string, params map[string]any) { + raw, err := json.Marshal(params) + if err == nil { + _, _ = agent.dispatch(method, raw) + } +} + +func waitForIntegrationMessages(t *testing.T, agent *rocketMQAgent, method string, params map[string]any) integrationMessages { + t.Helper() + deadline := time.Now().Add(15 * time.Second) + for { + var messages integrationMessages + integrationCall(t, agent, method, params, &messages) + if len(messages.Messages) > 0 { + return messages + } + if time.Now().After(deadline) { + t.Fatalf("%s returned no messages before timeout", method) + } + time.Sleep(250 * time.Millisecond) + } +} + +func envString(key, fallback string) string { + if value := strings.TrimSpace(os.Getenv(key)); value != "" { + return value + } + return fallback +} diff --git a/agents/drivers/rocketmq/main.go b/agents/drivers/rocketmq/main.go new file mode 100644 index 0000000000..316303fb69 --- /dev/null +++ b/agents/drivers/rocketmq/main.go @@ -0,0 +1,73 @@ +package main + +import ( + "bufio" + "encoding/json" + "fmt" + "os" +) + +const maxRPCMessageBytes = 32 * 1024 * 1024 + +type rpcRequest struct { + JSONRPC string `json:"jsonrpc"` + ID json.RawMessage `json:"id"` + Method string `json:"method"` + Params json.RawMessage `json:"params"` +} + +type rpcError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +type rpcResponse struct { + JSONRPC string `json:"jsonrpc"` + ID json.RawMessage `json:"id"` + Result any `json:"result,omitempty"` + Error *rpcError `json:"error,omitempty"` +} + +func main() { + agent := newRocketMQAgent() + defer agent.close() + + encoder := json.NewEncoder(os.Stdout) + encoder.SetEscapeHTML(false) + _ = encoder.Encode(map[string]bool{"ready": true}) + + scanner := bufio.NewScanner(os.Stdin) + scanner.Buffer(make([]byte, 64*1024), maxRPCMessageBytes) + for scanner.Scan() { + var request rpcRequest + if err := json.Unmarshal(scanner.Bytes(), &request); err != nil { + _ = encoder.Encode(rpcResponse{ + JSONRPC: "2.0", + ID: json.RawMessage("null"), + Error: &rpcError{Code: -32700, Message: normalizeError(err)}, + }) + continue + } + + result, err := agent.dispatch(request.Method, request.Params) + response := rpcResponse{JSONRPC: "2.0", ID: request.ID} + if len(response.ID) == 0 { + response.ID = json.RawMessage("null") + } + if err != nil { + response.Error = &rpcError{Code: -1, Message: normalizeError(err)} + } else { + response.Result = result + } + if err := encoder.Encode(response); err != nil { + fmt.Fprintln(os.Stderr, err) + return + } + if agent.shutdownRequested { + return + } + } + if err := scanner.Err(); err != nil { + fmt.Fprintln(os.Stderr, err) + } +} diff --git a/agents/drivers/rocketmq/main_test.go b/agents/drivers/rocketmq/main_test.go new file mode 100644 index 0000000000..543d69be2e --- /dev/null +++ b/agents/drivers/rocketmq/main_test.go @@ -0,0 +1,115 @@ +package main + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "strings" + "sync" + "testing" + "time" +) + +const rocketMQHelperProcessEnv = "DBX_ROCKETMQ_HELPER_PROCESS" + +type synchronizedBuffer struct { + mu sync.Mutex + buffer bytes.Buffer +} + +func (b *synchronizedBuffer) Write(data []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buffer.Write(data) +} + +func (b *synchronizedBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buffer.String() +} + +func TestRocketMQAgentProcessProtocol(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + command := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestRocketMQAgentHelperProcess$") + command.Env = append(os.Environ(), rocketMQHelperProcessEnv+"=1") + stdin, err := command.StdinPipe() + if err != nil { + t.Fatal(err) + } + stdout, err := command.StdoutPipe() + if err != nil { + t.Fatal(err) + } + var stderr synchronizedBuffer + command.Stderr = &stderr + if err := command.Start(); err != nil { + t.Fatal(err) + } + + scanner := bufio.NewScanner(stdout) + if !scanner.Scan() || scanner.Text() != `{"ready":true}` { + t.Fatalf("unexpected ready line %q: %s", scanner.Text(), stderr.String()) + } + + writeRPCRequest(t, stdin, `{"jsonrpc":"2.0","id":1,"method":"handshake","params":{}}`) + handshake := readRPCResponse(t, scanner, stderr.String()) + if string(handshake.ID) != "1" || handshake.Error != nil { + t.Fatalf("unexpected handshake response: %#v", handshake) + } + handshakeResult, ok := handshake.Result.(map[string]any) + if !ok || int(handshakeResult["protocolVersion"].(float64)) != protocolVersion { + t.Fatalf("unexpected handshake result: %#v", handshake.Result) + } + + writeRPCRequest(t, stdin, `{`) + parseFailure := readRPCResponse(t, scanner, stderr.String()) + if string(parseFailure.ID) != "null" || parseFailure.Error == nil || parseFailure.Error.Code != -32700 { + t.Fatalf("unexpected parse failure response: %#v", parseFailure) + } + + writeRPCRequest(t, stdin, `{"jsonrpc":"2.0","id":2,"method":"shutdown","params":{}}`) + shutdown := readRPCResponse(t, scanner, stderr.String()) + if string(shutdown.ID) != "2" || shutdown.Error != nil { + t.Fatalf("unexpected shutdown response: %#v", shutdown) + } + if err := command.Wait(); err != nil { + t.Fatalf("agent process failed: %v: %s", err, stderr.String()) + } + if ctx.Err() != nil { + t.Fatalf("agent process timed out: %v", ctx.Err()) + } +} + +func TestRocketMQAgentHelperProcess(t *testing.T) { + if os.Getenv(rocketMQHelperProcessEnv) != "1" { + return + } + main() + os.Exit(0) +} + +func writeRPCRequest(t *testing.T, stdin interface{ Write([]byte) (int, error) }, request string) { + t.Helper() + if _, err := fmt.Fprintln(stdin, request); err != nil { + t.Fatal(err) + } +} + +func readRPCResponse(t *testing.T, scanner *bufio.Scanner, stderr string) rpcResponse { + t.Helper() + if !scanner.Scan() { + t.Fatalf("agent closed stdout: %s", strings.TrimSpace(stderr)) + } + var response rpcResponse + if err := json.Unmarshal(scanner.Bytes(), &response); err != nil { + t.Fatalf("decode response %q: %v", scanner.Text(), err) + } + return response +} diff --git a/agents/drivers/rocketmq/messages.go b/agents/drivers/rocketmq/messages.go new file mode 100644 index 0000000000..3872237c5e --- /dev/null +++ b/agents/drivers/rocketmq/messages.go @@ -0,0 +1,586 @@ +package main + +import ( + "context" + "encoding/base64" + "fmt" + "sort" + "strconv" + "strings" + "time" + "unicode/utf8" + + admin "github.com/amigoer/rocketmq-admin-go" + "github.com/amigoer/rocketmq-admin-go/protocol/remoting" + "github.com/apache/rocketmq-client-go/v2/primitive" +) + +const sendMessageRequestCode = 10 +const queryMessageNotFoundCode = 208 + +type messageQueueTarget struct { + BrokerName string + Address string + QueueID int +} + +func (a *rocketMQAgent) listProducers(params map[string]any) (any, error) { + client, config, err := a.requireClient() + if err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout) + defer cancel() + topic := stringValue(params, "topic") + producerGroup := stringValue(params, "producerGroup", "group") + rows := make([]map[string]any, 0) + if producerGroup != "" { + if topic == "" { + return nil, fmt.Errorf("topic is required when producerGroup is specified") + } + connection, queryErr := client.ExamineProducerConnectionInfo(ctx, producerGroup, topic) + if queryErr != nil { + return nil, queryErr + } + for index, connectionInfo := range connection.ConnectionSet { + rows = append(rows, producerRow(int64(index+1), producerGroup, connectionInfo)) + } + return map[string]any{"producers": rows}, nil + } + + addresses := make([]string, 0) + if topic != "" { + stats, statsErr := a.examineTopicStats(ctx, client, topic) + if statsErr != nil { + return nil, statsErr + } + hasMessages := false + for _, offset := range stats { + if offset.MaxOffset > offset.MinOffset { + hasMessages = true + break + } + } + if !hasMessages { + return map[string]any{"producers": rows}, nil + } + route, routeErr := client.ExamineTopicRouteInfo(ctx, topic) + if routeErr != nil { + return nil, routeErr + } + for _, broker := range route.BrokerDatas { + if address := broker.BrokerAddrs["0"]; address != "" { + addresses = append(addresses, address) + } + } + } else { + a.mu.RLock() + addresses = append(addresses, a.brokerAddr) + a.mu.RUnlock() + } + seen := make(map[string]struct{}) + var producerID int64 = 1 + for _, address := range uniqueStrings(addresses) { + producerTable, tableErr := client.GetAllProducerInfo(ctx, address) + if tableErr != nil { + continue + } + for _, group := range sortedKeys(producerTable) { + for _, connectionInfo := range producerTable[group] { + key := group + "|" + connectionInfo.ClientAddr + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + rows = append(rows, producerRow(producerID, group, connectionInfo)) + producerID++ + } + } + } + return map[string]any{"producers": rows}, nil +} + +func (a *rocketMQAgent) peekMessages(params map[string]any) (any, error) { + topic, err := requireString(params, "topic") + if err != nil { + return nil, err + } + client, config, _ := a.requireClient() + ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout) + defer cancel() + count := min(max(1, intValue(params, 10, "count")), 100) + partition, hasPartition := optionalInt(params, "partition") + offset, hasOffset := optionalInt64(params, "offset") + targets, err := a.messageQueueTargets(ctx, client, topic, false) + if err != nil { + return nil, err + } + messages := make([]map[string]any, 0, count) + for _, target := range targets { + if len(messages) >= count { + break + } + if hasPartition && target.QueueID != partition { + continue + } + startOffset := offset + if !hasOffset { + startOffset, err = queueOffset(ctx, target.Address, topic, target.QueueID, remoting.GetMinOffset) + if err != nil { + continue + } + } + pullResult, pullErr := client.PullMessage(ctx, target.Address, topic, target.QueueID, startOffset, count-len(messages)) + if pullErr != nil { + continue + } + for _, message := range pullResult.Messages { + message.BrokerName = target.BrokerName + messages = append(messages, messageMap(topic, message)) + if len(messages) >= count { + break + } + } + } + sortMessageRows(messages) + return map[string]any{"messages": messages}, nil +} + +func (a *rocketMQAgent) viewMessage(params map[string]any) (any, error) { + topic, err := requireString(params, "topic") + if err != nil { + return nil, err + } + messageID, err := requireString(params, "msgId") + if err != nil { + return nil, err + } + client, config, _ := a.requireClient() + ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout) + defer cancel() + messages, queryErr := queryMessagesByKey(ctx, client, config.ConnectTimeout, + topic, messageID, 32, 0, time.Now().UnixMilli()) + if queryErr == nil { + for _, message := range messages { + if message.MsgId == messageID || message.OffsetMsgId == messageID || message.Properties[primitive.PropertyUniqueClientMessageIdKeyIndex] == messageID { + return map[string]any{"message": messageMap(topic, message)}, nil + } + } + if len(messages) > 0 { + return map[string]any{"message": messageMap(topic, messages[0])}, nil + } + } + partition, hasPartition := optionalInt(params, "partition") + offset, hasOffset := optionalInt64(params, "offset") + if hasPartition && hasOffset { + targets, targetErr := a.messageQueueTargets(ctx, client, topic, false) + if targetErr == nil { + for _, target := range targets { + if target.QueueID != partition { + continue + } + pullResult, pullErr := client.PullMessage(ctx, target.Address, topic, partition, offset, 1) + if pullErr == nil && len(pullResult.Messages) > 0 { + return map[string]any{"message": messageMap(topic, pullResult.Messages[0])}, nil + } + } + } + } + if queryErr != nil { + return nil, queryErr + } + return nil, fmt.Errorf("message not found: %s", messageID) +} + +func (a *rocketMQAgent) queryMessageByKey(params map[string]any) (any, error) { + topic, err := requireString(params, "topic") + if err != nil { + return nil, err + } + key, err := requireString(params, "key") + if err != nil { + return nil, err + } + client, config, _ := a.requireClient() + ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout) + defer cancel() + maxNum := min(max(1, intValue(params, 32, "maxNum")), 200) + messages, err := queryMessagesByKey(ctx, client, config.ConnectTimeout, topic, key, maxNum, + int64Value(params, 0, "begin"), int64Value(params, time.Now().UnixMilli(), "end")) + if err != nil { + if isEmptyMessageQueryError(err) { + return emptyMessageQuery(), nil + } + return nil, err + } + return messageQueryResult(topic, messages), nil +} + +func queryMessagesByKey( + ctx context.Context, + client *admin.Client, + connectTimeout time.Duration, + topic string, + key string, + maxNum int, + beginTimestamp int64, + endTimestamp int64, +) ([]*admin.MessageExt, error) { + route, err := client.ExamineTopicRouteInfo(ctx, topic) + if err != nil { + return nil, err + } + targets := make([]messageQueueTarget, 0, len(route.BrokerDatas)) + for _, broker := range route.BrokerDatas { + if address := broker.BrokerAddrs["0"]; address != "" { + targets = append(targets, messageQueueTarget{BrokerName: broker.BrokerName, Address: address}) + } + } + if len(targets) == 0 { + return nil, fmt.Errorf("no RocketMQ master broker found for topic %s", topic) + } + messages := make([]*admin.MessageExt, 0) + successCount := 0 + var lastErr error + for _, target := range targets { + response, requestErr := invokeRemotingAllowCodes(ctx, target.Address, connectTimeout, + buildQueryMessageCommand(topic, key, maxNum, beginTimestamp, endTimestamp), + remoting.Success, queryMessageNotFoundCode) + if requestErr != nil { + lastErr = requestErr + continue + } + successCount++ + if response.Code == queryMessageNotFoundCode || len(response.Body) == 0 { + continue + } + for _, message := range primitive.DecodeMessage(response.Body) { + queueID := 0 + if message.Queue != nil { + queueID = message.Queue.QueueId + } + messageTopic := message.Topic + if messageTopic == "" { + messageTopic = topic + } + messages = append(messages, &admin.MessageExt{ + Topic: messageTopic, QueueId: queueID, QueueOffset: message.QueueOffset, + MsgId: message.MsgId, OffsetMsgId: message.OffsetMsgId, Body: message.Body, + Flag: int(message.Flag), BornTimestamp: message.BornTimestamp, + StoreTimestamp: message.StoreTimestamp, BornHost: message.BornHost, + StoreHost: message.StoreHost, SysFlag: int(message.SysFlag), + BrokerName: target.BrokerName, Properties: message.GetProperties(), + }) + } + } + if successCount == 0 { + return nil, fmt.Errorf("query messages for topic %s on all masters: %w", topic, lastErr) + } + return messages, nil +} + +func buildQueryMessageCommand(topic, key string, maxNum int, beginTimestamp, endTimestamp int64) *remoting.RemotingCommand { + return remoting.NewRequest(remoting.QueryMessage, map[string]string{ + "topic": topic, "key": key, "maxNum": strconv.Itoa(maxNum), + "beginTimestamp": strconv.FormatInt(beginTimestamp, 10), + "endTimestamp": strconv.FormatInt(endTimestamp, 10), + }) +} + +func (a *rocketMQAgent) queryMessageByTopic(params map[string]any) (any, error) { + topic, err := requireString(params, "topic") + if err != nil { + return nil, err + } + client, config, _ := a.requireClient() + ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout) + defer cancel() + maxNum := min(max(1, intValue(params, 32, "maxNum")), 200) + messages, err := client.QueryMessageByTime(ctx, topic, + int64Value(params, 0, "begin"), int64Value(params, time.Now().UnixMilli(), "end"), maxNum) + if err != nil { + return nil, err + } + return messageQueryResult(topic, messages), nil +} + +func (a *rocketMQAgent) queryMessageTrace(params map[string]any) (any, error) { + messageID, err := requireString(params, "msgId") + if err != nil { + return nil, err + } + traceTopic := stringValue(params, "traceTopic") + if traceTopic == "" { + traceTopic = "RMQ_SYS_TRACE_TOPIC" + } + result, err := a.queryMessageByKey(map[string]any{ + "topic": traceTopic, "key": messageID, + "maxNum": intValue(params, 64, "maxNum"), + "begin": int64Value(params, 0, "begin"), + "end": int64Value(params, time.Now().UnixMilli(), "end"), + }) + if err != nil { + return nil, err + } + response := result.(map[string]any) + response["msgId"] = messageID + response["traceTopic"] = traceTopic + return response, nil +} + +func (a *rocketMQAgent) sendMessage(params map[string]any) (any, error) { + topic, err := requireString(params, "topic") + if err != nil { + return nil, err + } + payload, err := decodePayload(params) + if err != nil { + return nil, err + } + client, config, _ := a.requireClient() + ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout) + defer cancel() + targets, err := a.messageQueueTargets(ctx, client, topic, true) + if err != nil { + return nil, err + } + if len(targets) == 0 { + return nil, fmt.Errorf("no writable queue for topic %s", topic) + } + partition, hasPartition := optionalInt(params, "partition") + selected := targets[0] + if hasPartition { + found := false + for _, target := range targets { + if target.QueueID == partition { + selected = target + found = true + break + } + } + if !found { + return nil, fmt.Errorf("partition %d not found for topic %s", partition, topic) + } + } + command := buildSendMessageCommand(topic, payload, selected.QueueID, params, time.Now().UnixMilli()) + response, err := invokeRemotingAllowCodes(ctx, selected.Address, config.ConnectTimeout, command, 0, 10, 11, 12) + if err != nil { + return nil, err + } + queueID, _ := strconv.Atoi(response.ExtFields["queueId"]) + queueOffset, _ := strconv.ParseInt(response.ExtFields["queueOffset"], 10, 64) + return map[string]any{ + "ok": true, "topic": topic, "partition": queueID, + "offset": queueOffset, "timestamp": time.Now().UnixMilli(), + }, nil +} + +func buildSendMessageCommand(topic string, payload []byte, queueID int, params map[string]any, bornTimestamp int64) *remoting.RemotingCommand { + message := primitive.NewMessage(topic, payload) + message.WithProperty(primitive.PropertyUniqueClientMessageIdKeyIndex, primitive.CreateUniqID()) + headers := mapValue(params, "headers") + key := stringValue(params, "key") + if key == "" { + key = stringValue(headers, primitive.PropertyKeys) + } + tag := stringValue(params, "tag") + if tag == "" { + tag = stringValue(headers, primitive.PropertyTags) + } + if key != "" { + message.WithKeys([]string{key}) + } + if tag != "" { + message.WithTag(tag) + } + for header, raw := range headers { + if raw == nil { + continue + } + value := strings.TrimSpace(fmt.Sprint(raw)) + if value == "" || header == primitive.PropertyKeys || header == primitive.PropertyTags || isReservedMessageProperty(header) { + continue + } + message.WithProperty(header, value) + } + command := remoting.NewRequest(sendMessageRequestCode, map[string]string{ + "producerGroup": "_DBX_ROCKETMQ_PRODUCER", "topic": topic, + "defaultTopic": "TBW102", "defaultTopicQueueNums": "4", + "queueId": strconv.Itoa(queueID), "sysFlag": "0", + "bornTimestamp": strconv.FormatInt(bornTimestamp, 10), "flag": "0", + "properties": message.MarshallProperties(), "reconsumeTimes": "0", + "unitMode": "false", "maxReconsumeTimes": "0", "batch": "false", + }) + command.Body = payload + return command +} + +func (a *rocketMQAgent) messageQueueTargets(ctx context.Context, client *admin.Client, topic string, writable bool) ([]messageQueueTarget, error) { + route, err := client.ExamineTopicRouteInfo(ctx, topic) + if err != nil { + return nil, err + } + addresses := make(map[string]string, len(route.BrokerDatas)) + for _, broker := range route.BrokerDatas { + if address := broker.BrokerAddrs["0"]; address != "" { + addresses[broker.BrokerName] = address + } + } + targets := make([]messageQueueTarget, 0) + for _, queueData := range route.QueueDatas { + address := addresses[queueData.BrokerName] + if address == "" { + continue + } + queueCount := queueData.ReadQueueNums + if writable { + queueCount = queueData.WriteQueueNums + } + for queueID := 0; queueID < queueCount; queueID++ { + targets = append(targets, messageQueueTarget{ + BrokerName: queueData.BrokerName, Address: address, QueueID: queueID, + }) + } + } + sort.Slice(targets, func(i, j int) bool { + if targets[i].BrokerName != targets[j].BrokerName { + return targets[i].BrokerName < targets[j].BrokerName + } + return targets[i].QueueID < targets[j].QueueID + }) + return targets, nil +} + +func queueOffset(ctx context.Context, address, topic string, queueID, requestCode int) (int64, error) { + response, err := invokeRemotingWithClient(ctx, address, remoting.NewRequest(requestCode, map[string]string{ + "topic": topic, "queueId": strconv.Itoa(queueID), + })) + if err != nil { + return 0, err + } + offset, err := strconv.ParseInt(response.ExtFields["offset"], 10, 64) + if err != nil { + return 0, fmt.Errorf("invalid queue offset: %w", err) + } + return offset, nil +} + +func messageMap(topic string, message *admin.MessageExt) map[string]any { + headers := make(map[string]string, len(message.Properties)) + for key, value := range message.Properties { + headers[key] = value + } + row := map[string]any{ + "topic": topic, "messageId": message.MsgId, "partition": message.QueueId, + "offset": message.QueueOffset, "timestamp": message.StoreTimestamp, + "key": message.Properties[primitive.PropertyKeys], "tag": message.Properties[primitive.PropertyTags], + "headers": headers, "payloadBase64": base64.StdEncoding.EncodeToString(message.Body), + } + if utf8.Valid(message.Body) { + row["payloadText"] = string(message.Body) + } + return row +} + +func messageQueryResult(topic string, messages []*admin.MessageExt) map[string]any { + rows := make([]map[string]any, 0, len(messages)) + for _, message := range messages { + rows = append(rows, messageMap(topic, message)) + } + sortMessageRows(rows) + return map[string]any{"messages": rows, "indexLastUpdateTimestamp": int64(0)} +} + +func emptyMessageQuery() map[string]any { + return map[string]any{"messages": []map[string]any{}, "indexLastUpdateTimestamp": int64(0)} +} + +func sortMessageRows(messages []map[string]any) { + sort.Slice(messages, func(i, j int) bool { + leftTime := anyInt64(messages[i]["timestamp"]) + rightTime := anyInt64(messages[j]["timestamp"]) + if leftTime != rightTime { + return leftTime < rightTime + } + leftPartition := int(anyInt64(messages[i]["partition"])) + rightPartition := int(anyInt64(messages[j]["partition"])) + if leftPartition != rightPartition { + return leftPartition < rightPartition + } + return anyInt64(messages[i]["offset"]) < anyInt64(messages[j]["offset"]) + }) +} + +func producerRow(producerID int64, group string, connection admin.Connection) map[string]any { + return map[string]any{ + "producerId": producerID, "producerName": group, "msgRateIn": 0.0, + "msgThroughputIn": 0.0, "clientVersion": strconv.Itoa(connection.Version), + "address": connection.ClientAddr, "lastTimestamp": int64(0), + } +} + +func optionalInt(params map[string]any, key string) (int, bool) { + if _, ok := params[key]; !ok || params[key] == nil { + return 0, false + } + return intValue(params, 0, key), true +} + +func optionalInt64(params map[string]any, key string) (int64, bool) { + if _, ok := params[key]; !ok || params[key] == nil { + return 0, false + } + return int64Value(params, 0, key), true +} + +func mapValue(params map[string]any, key string) map[string]any { + if value, ok := params[key].(map[string]any); ok { + return value + } + return map[string]any{} +} + +func anyInt64(value any) int64 { + switch typed := value.(type) { + case int: + return int64(typed) + case int64: + return typed + case float64: + return int64(typed) + case string: + parsed, _ := strconv.ParseInt(typed, 10, 64) + return parsed + default: + return 0 + } +} + +func isEmptyMessageQueryError(err error) bool { + message := strings.ToLower(err.Error()) + return strings.Contains(message, "208") || strings.Contains(message, "no message") || strings.Contains(message, "未找到消息") +} + +func isReservedMessageProperty(key string) bool { + switch key { + case primitive.PropertyUniqueClientMessageIdKeyIndex, primitive.PropertyWaitStoreMsgOk, + primitive.PropertyDelayTimeLevel, primitive.PropertyRetryTopic, + primitive.PropertyRealTopic, primitive.PropertyRealQueueId, + primitive.PropertyTransactionPrepared, primitive.PropertyProducerGroup, + primitive.PropertyMinOffset, primitive.PropertyMaxOffset, + primitive.PropertyBuyerId, primitive.PropertyOriginMessageId, + primitive.PropertyTransferFlag, primitive.PropertyCorrectionFlag, + primitive.PropertyMQ2Flag, primitive.PropertyReconsumeTime, + primitive.PropertyMsgRegion, primitive.PropertyTraceSwitch, + primitive.PropertyMaxReconsumeTimes, primitive.PropertyConsumeStartTime, + primitive.PropertyTranscationPreparedQueueOffset, primitive.PropertyTranscationCheckTimes, + primitive.PropertyCheckImmunityTimeInSeconds, primitive.PropertyShardingKey, + primitive.PropertyTransactionID, primitive.PropertyCorrelationID, + primitive.PropertyMessageReplyToClient, primitive.PropertyMessageTTL, + primitive.PropertyReplyMessageArriveTime, primitive.PropertyMsgType, + primitive.PropertyCluster: + return true + default: + return false + } +} diff --git a/agents/drivers/rocketmq/messages_test.go b/agents/drivers/rocketmq/messages_test.go new file mode 100644 index 0000000000..a354f7efbb --- /dev/null +++ b/agents/drivers/rocketmq/messages_test.go @@ -0,0 +1,116 @@ +package main + +import ( + "context" + "encoding/base64" + "net" + "strings" + "testing" + "time" + + admin "github.com/amigoer/rocketmq-admin-go" + "github.com/amigoer/rocketmq-admin-go/protocol/remoting" + "github.com/apache/rocketmq-client-go/v2/primitive" +) + +func TestBuildSendMessageCommandCopiesOnlyUserProperties(t *testing.T) { + command := buildSendMessageCommand("Orders", []byte("payload"), 3, map[string]any{ + "headers": map[string]any{ + primitive.PropertyTags: "tag-a", + primitive.PropertyKeys: "key-a", + primitive.PropertyDelayTimeLevel: "18", + "Region": "Hangzhou", + "color": "blue", + "empty": " ", + "nil": nil, + }, + }, 1234) + if command.Code != sendMessageRequestCode || command.ExtFields["queueId"] != "3" { + t.Fatalf("unexpected send command: %#v", command) + } + properties := command.ExtFields["properties"] + for _, expected := range []string{"TAGS\x01tag-a\x02", "KEYS\x01key-a\x02", "Region\x01Hangzhou\x02", "color\x01blue\x02"} { + if !strings.Contains(properties, expected) { + t.Fatalf("missing property %q in %q", expected, properties) + } + } + for _, forbidden := range []string{"DELAY\x01", "empty\x01", "nil\x01"} { + if strings.Contains(properties, forbidden) { + t.Fatalf("reserved/empty property %q leaked into %q", forbidden, properties) + } + } +} + +func TestBuildQueryMessageCommandUsesBrokerHeaderNames(t *testing.T) { + command := buildQueryMessageCommand("Orders", "order-1", 32, 100, 200) + if command.ExtFields["beginTimestamp"] != "100" || command.ExtFields["endTimestamp"] != "200" { + t.Fatalf("unexpected query fields: %#v", command.ExtFields) + } + if _, exists := command.ExtFields["begin"]; exists { + t.Fatalf("legacy begin field leaked into query: %#v", command.ExtFields) + } + if _, exists := command.ExtFields["end"]; exists { + t.Fatalf("legacy end field leaked into query: %#v", command.ExtFields) + } +} + +func TestInvokeRemotingAllowCodes(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + done := make(chan error, 1) + go func() { + connection, acceptErr := listener.Accept() + if acceptErr != nil { + done <- acceptErr + return + } + defer connection.Close() + frame, readErr := readRemotingFrame(connection) + if readErr != nil { + done <- readErr + return + } + request, decodeErr := remoting.Decode(frame[4:]) + if decodeErr != nil { + done <- decodeErr + return + } + response := &remoting.RemotingCommand{Code: 11, Opaque: request.Opaque, Flag: 1, ExtFields: map[string]string{}} + encoded, encodeErr := response.Encode() + if encodeErr == nil { + _, encodeErr = connection.Write(encoded) + } + done <- encodeErr + }() + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if _, err := invokeRemotingAllowCodes(ctx, listener.Addr().String(), time.Second, + remoting.NewRequest(sendMessageRequestCode, nil), 0, 10, 11, 12); err != nil { + t.Fatal(err) + } + if err := <-done; err != nil { + t.Fatal(err) + } +} + +func TestMessageMapPreservesBinaryAndTextPayloads(t *testing.T) { + text := &admin.MessageExt{ + QueueId: 2, QueueOffset: 7, MsgId: "msg-1", Body: []byte("hello"), + StoreTimestamp: 123, Properties: map[string]string{primitive.PropertyKeys: "key-a", primitive.PropertyTags: "tag-a"}, + } + row := messageMap("Orders", text) + if row["payloadText"] != "hello" || row["payloadBase64"] != base64.StdEncoding.EncodeToString([]byte("hello")) { + t.Fatalf("unexpected text message row: %#v", row) + } + if row["key"] != "key-a" || row["tag"] != "tag-a" || row["partition"] != 2 { + t.Fatalf("unexpected message metadata: %#v", row) + } + binary := &admin.MessageExt{Body: []byte{0xff, 0xfe}, Properties: map[string]string{}} + binaryRow := messageMap("Binary", binary) + if _, ok := binaryRow["payloadText"]; ok { + t.Fatalf("binary payload exposed as text: %#v", binaryRow) + } +} diff --git a/agents/drivers/rocketmq/remoting.go b/agents/drivers/rocketmq/remoting.go new file mode 100644 index 0000000000..759192915b --- /dev/null +++ b/agents/drivers/rocketmq/remoting.go @@ -0,0 +1,42 @@ +package main + +import ( + "context" + "fmt" + "time" + + "github.com/amigoer/rocketmq-admin-go/protocol/remoting" +) + +func invokeRemotingWithClient(ctx context.Context, address string, command *remoting.RemotingCommand) (*remoting.RemotingCommand, error) { + return invokeRemotingAllowCodes(ctx, address, defaultRequestTimeout, command, remoting.Success) +} + +func invokeRemotingAllowCodes( + ctx context.Context, + address string, + connectTimeout time.Duration, + command *remoting.RemotingCommand, + allowedCodes ...int, +) (*remoting.RemotingCommand, error) { + client := remoting.NewClient(address, connectTimeout) + if err := client.Connect(); err != nil { + return nil, err + } + defer client.Close() + response, err := client.InvokeSync(ctx, command) + if err != nil { + return nil, err + } + allowed := false + for _, code := range allowedCodes { + if response.Code == code { + allowed = true + break + } + } + if !allowed { + return nil, fmt.Errorf("RocketMQ error %d: %s", response.Code, response.Remark) + } + return response, nil +} diff --git a/agents/drivers/rocketmq/routing.go b/agents/drivers/rocketmq/routing.go new file mode 100644 index 0000000000..e1e47c7cc5 --- /dev/null +++ b/agents/drivers/rocketmq/routing.go @@ -0,0 +1,526 @@ +package main + +import ( + "context" + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "encoding/binary" + "encoding/json" + "fmt" + "io" + "net" + "regexp" + "sort" + "strings" + "sync" + + "github.com/amigoer/rocketmq-admin-go/protocol/remoting" + "golang.org/x/net/proxy" +) + +type proxyTargetKind int + +const ( + proxyTargetNameServer proxyTargetKind = iota + proxyTargetBroker +) + +type proxyEndpoint struct { + original string + targets []string + fallback bool + listener net.Listener + manager *proxyManager + closeOnce sync.Once +} + +type proxyManager struct { + mu sync.RWMutex + config connectionConfig + byOriginal map[string]*proxyEndpoint + originalByLocal map[string]string + routedOwners map[string]string + closed bool +} + +func newProxyManager(config connectionConfig) *proxyManager { + return &proxyManager{ + config: config, byOriginal: map[string]*proxyEndpoint{}, + originalByLocal: map[string]string{}, routedOwners: map[string]string{}, + } +} + +func (manager *proxyManager) ProxyFor(original string, kind proxyTargetKind) (string, error) { + original = strings.TrimSpace(original) + if original == "" { + return "", fmt.Errorf("empty RocketMQ endpoint") + } + manager.mu.RLock() + if endpoint := manager.byOriginal[original]; endpoint != nil { + address := endpoint.listener.Addr().String() + manager.mu.RUnlock() + return address, nil + } + manager.mu.RUnlock() + + manager.mu.Lock() + defer manager.mu.Unlock() + if manager.closed { + return "", fmt.Errorf("RocketMQ proxy manager is closed") + } + if endpoint := manager.byOriginal[original]; endpoint != nil { + return endpoint.listener.Addr().String(), nil + } + targets, fallback := manager.routeTargetsLocked(original, kind) + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return "", fmt.Errorf("start RocketMQ endpoint proxy: %w", err) + } + endpoint := &proxyEndpoint{original: original, targets: targets, fallback: fallback, listener: listener, manager: manager} + local := listener.Addr().String() + manager.byOriginal[original] = endpoint + manager.originalByLocal[local] = original + go endpoint.serve() + return local, nil +} + +func (manager *proxyManager) routeTargetsLocked(original string, kind proxyTargetKind) ([]string, bool) { + if kind == proxyTargetNameServer || manager.config.SocksProxy != nil { + return []string{original}, false + } + if manager.config.BrokerAddr != "" { + return []string{manager.config.BrokerAddr}, false + } + host, port := parseSocketAddress(original) + if port == "" || !isLikelyUnreachableBrokerHost(host) { + return []string{original}, false + } + nameServerHost, _ := parseSocketAddress(manager.config.NameServers[0]) + routed := formatSocketAddress(nameServerHost, port) + if owner := manager.routedOwners[routed]; owner == "" || owner == original { + manager.routedOwners[routed] = original + return []string{routed, original}, false + } + // Multiple private brokers often advertise the same port. Routing a collision to the + // published host would silently contact the first broker, so keep the original only. + return []string{original}, true +} + +func (manager *proxyManager) LocalForOriginal(original string) string { + manager.mu.RLock() + defer manager.mu.RUnlock() + if endpoint := manager.byOriginal[original]; endpoint != nil { + return endpoint.listener.Addr().String() + } + return "" +} + +func (manager *proxyManager) OriginalForLocal(local string) string { + manager.mu.RLock() + defer manager.mu.RUnlock() + if original := manager.originalByLocal[local]; original != "" { + return original + } + host, port := parseSocketAddress(local) + if host == "localhost" { + return manager.originalByLocal[formatSocketAddress("127.0.0.1", port)] + } + return "" +} + +func (manager *proxyManager) IsCollisionFallback(local string) bool { + manager.mu.RLock() + defer manager.mu.RUnlock() + original := manager.originalByLocal[local] + return original != "" && manager.byOriginal[original] != nil && manager.byOriginal[original].fallback +} + +func (manager *proxyManager) Close() { + manager.mu.Lock() + if manager.closed { + manager.mu.Unlock() + return + } + manager.closed = true + endpoints := make([]*proxyEndpoint, 0, len(manager.byOriginal)) + for _, endpoint := range manager.byOriginal { + endpoints = append(endpoints, endpoint) + } + manager.mu.Unlock() + for _, endpoint := range endpoints { + endpoint.closeOnce.Do(func() { _ = endpoint.listener.Close() }) + } +} + +func (endpoint *proxyEndpoint) serve() { + for { + clientConn, err := endpoint.listener.Accept() + if err != nil { + return + } + go endpoint.handle(clientConn) + } +} + +func (endpoint *proxyEndpoint) handle(clientConn net.Conn) { + defer clientConn.Close() + remoteConn, err := endpoint.dialRemote() + if err != nil { + return + } + defer remoteConn.Close() + + done := make(chan struct{}, 2) + go func() { + _ = endpoint.forward(clientConn, remoteConn, false) + done <- struct{}{} + }() + go func() { + _ = endpoint.forward(remoteConn, clientConn, true) + done <- struct{}{} + }() + <-done +} + +func (endpoint *proxyEndpoint) dialRemote() (net.Conn, error) { + var lastErr error + for _, target := range endpoint.targets { + var conn net.Conn + var err error + if socks := endpoint.manager.config.SocksProxy; socks != nil { + var auth *proxy.Auth + if socks.Username != "" || socks.Password != "" { + auth = &proxy.Auth{User: socks.Username, Password: socks.Password} + } + dialer, dialErr := proxy.SOCKS5("tcp", formatSocketAddress(socks.Host, fmt.Sprintf("%d", socks.Port)), auth, &net.Dialer{Timeout: endpoint.manager.config.ConnectTimeout}) + if dialErr != nil { + lastErr = dialErr + continue + } + conn, err = dialer.Dial("tcp", target) + } else { + ctx, cancel := context.WithTimeout(context.Background(), endpoint.manager.config.ConnectTimeout) + conn, err = (&net.Dialer{}).DialContext(ctx, "tcp", target) + cancel() + } + if err == nil { + return conn, nil + } + lastErr = err + } + return nil, fmt.Errorf("connect RocketMQ endpoint %s: %w", endpoint.original, lastErr) +} + +func (endpoint *proxyEndpoint) forward(source, destination net.Conn, response bool) error { + for { + frame, err := readRemotingFrame(source) + if err != nil { + return err + } + command, err := remoting.Decode(frame[4:]) + if err != nil { + return err + } + if response { + if err := endpoint.manager.rewriteResponse(command); err != nil { + return err + } + } else { + signCommand(command, endpoint.manager.config.AccessKey, endpoint.manager.config.SecretKey) + } + encoded, err := command.Encode() + if err != nil { + return err + } + if _, err := destination.Write(encoded); err != nil { + return err + } + } +} + +func readRemotingFrame(reader io.Reader) ([]byte, error) { + header := make([]byte, 4) + if _, err := io.ReadFull(reader, header); err != nil { + return nil, err + } + length := int(binary.BigEndian.Uint32(header)) + if length <= 0 || length > 64*1024*1024 { + return nil, fmt.Errorf("invalid RocketMQ frame length: %d", length) + } + frame := make([]byte, 4+length) + copy(frame, header) + if _, err := io.ReadFull(reader, frame[4:]); err != nil { + return nil, err + } + return frame, nil +} + +func signCommand(command *remoting.RemotingCommand, accessKey, secretKey string) { + if accessKey == "" && secretKey == "" { + return + } + if command.ExtFields == nil { + command.ExtFields = map[string]string{} + } + delete(command.ExtFields, "Signature") + command.ExtFields["AccessKey"] = accessKey + keys := make([]string, 0, len(command.ExtFields)) + for key := range command.ExtFields { + if key != "Signature" { + keys = append(keys, key) + } + } + sort.Strings(keys) + var content strings.Builder + for _, key := range keys { + content.WriteString(command.ExtFields[key]) + } + data := append([]byte(content.String()), command.Body...) + mac := hmac.New(sha1.New, []byte(secretKey)) + _, _ = mac.Write(data) + command.ExtFields["Signature"] = base64.StdEncoding.EncodeToString(mac.Sum(nil)) +} + +func (manager *proxyManager) rewriteResponse(command *remoting.RemotingCommand) error { + if len(command.Body) == 0 { + return nil + } + body := repairRocketMQJSON(command.Body) + if !json.Valid(body) { + return nil + } + var value any + if err := json.Unmarshal(body, &value); err != nil { + return nil + } + changed, err := manager.rewriteJSON(value, "") + if err != nil || !changed { + return err + } + body, err = json.Marshal(value) + if err != nil { + return err + } + command.Body = body + return nil +} + +var ( + unquotedNumberKey = regexp.MustCompile(`([{,])(\d+):`) + unquotedStringKey = regexp.MustCompile(`([{,])([a-zA-Z_][a-zA-Z0-9_]*):`) +) + +func repairRocketMQJSON(body []byte) []byte { + repaired := unquotedNumberKey.ReplaceAll(body, []byte(`$1"$2":`)) + repaired = unquotedStringKey.ReplaceAll(repaired, []byte(`$1"$2":`)) + return repairObjectKeyedMaps(repaired) +} + +func repairObjectKeyedMaps(body []byte) []byte { + result := make([]byte, 0, len(body)+64) + for index := 0; index < len(body); { + if index+2 < len(body) && body[index] == ':' && body[index+1] == '{' && body[index+2] == '{' { + converted, next := convertObjectKeyedMap(body, index+1) + result = append(result, ':') + result = append(result, converted...) + index = next + continue + } + result = append(result, body[index]) + index++ + } + return result +} + +func convertObjectKeyedMap(body []byte, start int) ([]byte, int) { + if start >= len(body) || body[start] != '{' { + return []byte("{}"), start + } + result := []byte{'{'} + index := start + 1 + first := true + for index < len(body) { + for index < len(body) && isJSONSpace(body[index]) { + index++ + } + if index >= len(body) { + break + } + if body[index] == '}' { + return append(result, '}'), index + 1 + } + if body[index] == ',' { + index++ + continue + } + if body[index] != '{' { + return originalJSONMap(body, start) + } + keyEnd := matchingBrace(body, index) + if keyEnd < 0 { + return body[start:], len(body) + } + key := body[index : keyEnd+1] + index = keyEnd + 1 + for index < len(body) && isJSONSpace(body[index]) { + index++ + } + if index >= len(body) || body[index] != ':' { + return originalJSONMap(body, start) + } + index++ + for index < len(body) && isJSONSpace(body[index]) { + index++ + } + if index >= len(body) || body[index] != '{' { + return originalJSONMap(body, start) + } + valueEnd := matchingBrace(body, index) + if valueEnd < 0 { + return body[start:], len(body) + } + if !first { + result = append(result, ',') + } + first = false + result = append(result, '"') + result = append(result, escapeJSONString(key)...) + result = append(result, '"', ':') + result = append(result, body[index:valueEnd+1]...) + index = valueEnd + 1 + } + return append(result, '}'), len(body) +} + +func originalJSONMap(body []byte, start int) ([]byte, int) { + end := matchingBrace(body, start) + if end < 0 { + return body[start:], len(body) + } + return body[start : end+1], end + 1 +} + +func matchingBrace(body []byte, start int) int { + depth := 0 + inString := false + escaped := false + for index := start; index < len(body); index++ { + character := body[index] + if inString { + if escaped { + escaped = false + continue + } + if character == '\\' { + escaped = true + continue + } + if character == '"' { + inString = false + } + continue + } + switch character { + case '"': + inString = true + case '{': + depth++ + case '}': + depth-- + if depth == 0 { + return index + } + } + } + return -1 +} + +func escapeJSONString(value []byte) []byte { + result := make([]byte, 0, len(value)+8) + for _, character := range value { + switch character { + case '\\', '"': + result = append(result, '\\', character) + case '\n': + result = append(result, '\\', 'n') + case '\r': + result = append(result, '\\', 'r') + case '\t': + result = append(result, '\\', 't') + default: + result = append(result, character) + } + } + return result +} + +func isJSONSpace(character byte) bool { + return character == ' ' || character == '\n' || character == '\r' || character == '\t' +} + +func (manager *proxyManager) rewriteJSON(value any, parentKey string) (bool, error) { + changed := false + switch typed := value.(type) { + case map[string]any: + if parentKey == "brokerAddrs" { + for key, raw := range typed { + address, ok := raw.(string) + if !ok || address == "" { + continue + } + local, err := manager.ProxyFor(address, proxyTargetBroker) + if err != nil { + return changed, err + } + typed[key] = local + changed = true + } + return changed, nil + } + if parentKey == "filterServerTable" { + rewritten := make(map[string]any, len(typed)) + for address, raw := range typed { + local, err := manager.ProxyFor(address, proxyTargetBroker) + if err != nil { + return changed, err + } + rewritten[local] = raw + changed = true + } + for key := range typed { + delete(typed, key) + } + for key, raw := range rewritten { + typed[key] = raw + } + } + for key, child := range typed { + if key == "brokerAddr" { + if address, ok := child.(string); ok && address != "" { + local, err := manager.ProxyFor(address, proxyTargetBroker) + if err != nil { + return changed, err + } + typed[key] = local + changed = true + continue + } + } + childChanged, err := manager.rewriteJSON(child, key) + if err != nil { + return changed, err + } + changed = changed || childChanged + } + case []any: + for _, child := range typed { + childChanged, err := manager.rewriteJSON(child, parentKey) + if err != nil { + return changed, err + } + changed = changed || childChanged + } + } + return changed, nil +} diff --git a/agents/drivers/rocketmq/routing_test.go b/agents/drivers/rocketmq/routing_test.go new file mode 100644 index 0000000000..fb0fe80f5c --- /dev/null +++ b/agents/drivers/rocketmq/routing_test.go @@ -0,0 +1,136 @@ +package main + +import ( + "bytes" + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "encoding/json" + "reflect" + "sort" + "strings" + "testing" + + "github.com/amigoer/rocketmq-admin-go/protocol/remoting" +) + +func TestProxyRoutesPrivateBrokersAndFailsClosedOnCollision(t *testing.T) { + manager := newProxyManager(connectionConfig{NameServers: []string{"127.0.0.1:9876"}}) + defer manager.Close() + + firstLocal, err := manager.ProxyFor("172.18.0.2:10911", proxyTargetBroker) + if err != nil { + t.Fatal(err) + } + secondLocal, err := manager.ProxyFor("172.18.0.3:10911", proxyTargetBroker) + if err != nil { + t.Fatal(err) + } + first := manager.byOriginal["172.18.0.2:10911"] + second := manager.byOriginal["172.18.0.3:10911"] + if !reflect.DeepEqual(first.targets, []string{"127.0.0.1:10911", "172.18.0.2:10911"}) { + t.Fatalf("unexpected first targets: %#v", first.targets) + } + if !reflect.DeepEqual(second.targets, []string{"172.18.0.3:10911"}) || !second.fallback { + t.Fatalf("collision must keep only original target: %#v", second) + } + if manager.OriginalForLocal(firstLocal) != "172.18.0.2:10911" || manager.OriginalForLocal(secondLocal) != "172.18.0.3:10911" { + t.Fatal("local endpoint did not round-trip to original address") + } + if !manager.IsCollisionFallback(secondLocal) { + t.Fatal("second private broker should be marked as collision fallback") + } +} + +func TestProxyRoutingHonorsSocksAndExplicitBroker(t *testing.T) { + socksManager := newProxyManager(connectionConfig{ + NameServers: []string{"127.0.0.1:9876"}, + SocksProxy: &socksProxyConfig{Host: "127.0.0.1", Port: 1080}, + }) + defer socksManager.Close() + local, err := socksManager.ProxyFor("172.18.0.2:10911", proxyTargetBroker) + if err != nil { + t.Fatal(err) + } + if targets := socksManager.byOriginal[socksManager.OriginalForLocal(local)].targets; !reflect.DeepEqual(targets, []string{"172.18.0.2:10911"}) { + t.Fatalf("SOCKS route changed logical broker: %#v", targets) + } + + explicitManager := newProxyManager(connectionConfig{ + NameServers: []string{"127.0.0.1:9876"}, BrokerAddr: "broker.example.com:10911", + }) + defer explicitManager.Close() + local, err = explicitManager.ProxyFor("172.18.0.2:10911", proxyTargetBroker) + if err != nil { + t.Fatal(err) + } + if targets := explicitManager.byOriginal[explicitManager.OriginalForLocal(local)].targets; !reflect.DeepEqual(targets, []string{"broker.example.com:10911"}) { + t.Fatalf("explicit broker was not honored: %#v", targets) + } +} + +func TestSignCommandMatchesRocketMQACLAlgorithm(t *testing.T) { + command := remoting.NewRequest(17, map[string]string{"topic": "Orders", "perm": "6"}) + command.Body = []byte("payload") + signCommand(command, "access", "secret") + + keys := make([]string, 0, len(command.ExtFields)) + for key := range command.ExtFields { + if key != "Signature" { + keys = append(keys, key) + } + } + sort.Strings(keys) + var content strings.Builder + for _, key := range keys { + content.WriteString(command.ExtFields[key]) + } + mac := hmac.New(sha1.New, []byte("secret")) + _, _ = mac.Write(append([]byte(content.String()), command.Body...)) + want := base64.StdEncoding.EncodeToString(mac.Sum(nil)) + if command.ExtFields["AccessKey"] != "access" || command.ExtFields["Signature"] != want { + t.Fatalf("unexpected signed fields: %#v", command.ExtFields) + } +} + +func TestRepairRocketMQJSONQuotesNonStandardKeys(t *testing.T) { + body := repairRocketMQJSON([]byte(`{"offsetTable":{{"brokerName":"broker-a","queueId":0,"topic":"Order-Events"}:{"brokerOffset":10}},brokerAddrs:{0:"127.0.0.1:10911"}}`)) + if !json.Valid(body) { + t.Fatalf("repaired body is invalid JSON: %s", body) + } + if !bytes.Contains(body, []byte(`"0":`)) || !bytes.Contains(body, []byte(`"{\"brokerName\":\"broker-a\"`)) { + t.Fatalf("missing repaired keys: %s", body) + } +} + +func TestRewriteResponseProxiesBrokerAddresses(t *testing.T) { + manager := newProxyManager(connectionConfig{NameServers: []string{"127.0.0.1:9876"}}) + defer manager.Close() + command := &remoting.RemotingCommand{Body: []byte(`{ + "brokerDatas":[{"brokerAddrs":{"0":"172.18.0.2:10911"}}], + "brokerAddr":"172.18.0.3:10912", + "filterServerTable":{"172.18.0.4:12000":[]} + }`)} + if err := manager.rewriteResponse(command); err != nil { + t.Fatal(err) + } + var value map[string]any + if err := json.Unmarshal(command.Body, &value); err != nil { + t.Fatal(err) + } + brokerDatas := value["brokerDatas"].([]any) + brokerLocal := brokerDatas[0].(map[string]any)["brokerAddrs"].(map[string]any)["0"].(string) + if manager.OriginalForLocal(brokerLocal) != "172.18.0.2:10911" { + t.Fatalf("broker address not proxied: %s", brokerLocal) + } + directLocal := value["brokerAddr"].(string) + if manager.OriginalForLocal(directLocal) != "172.18.0.3:10912" { + t.Fatalf("direct broker address not proxied: %s", directLocal) + } + filterTable := value["filterServerTable"].(map[string]any) + for local := range filterTable { + if manager.OriginalForLocal(local) != "172.18.0.4:12000" { + t.Fatalf("filter server address not proxied: %s", local) + } + } +} diff --git a/agents/drivers/rocketmq/scripts/run-integration.sh b/agents/drivers/rocketmq/scripts/run-integration.sh new file mode 100755 index 0000000000..b73706c509 --- /dev/null +++ b/agents/drivers/rocketmq/scripts/run-integration.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +set -euo pipefail + +# The DBX RocketMQ agent under test is the Go binary in this directory. Maven +# only supplies disposable official RocketMQ NameServer/Broker server JARs. +version="${1:?RocketMQ version is required}" +module_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +work_dir=$(mktemp -d "${TMPDIR:-/tmp}/dbx-rocketmq-${version}.XXXXXX") +namesrv_port="${ROCKETMQ_NAMESRV_PORT:-19876}" +broker_port="${ROCKETMQ_BROKER_PORT:-20911}" +store_root="${ROCKETMQ_STORE_ROOT:-${work_dir}/store}" +namesrv_pid="" +broker_pid="" + +cleanup() { + if [[ -n "$broker_pid" ]] && kill -0 "$broker_pid" 2>/dev/null; then + kill "$broker_pid" 2>/dev/null || true + fi + if [[ -n "$namesrv_pid" ]] && kill -0 "$namesrv_pid" 2>/dev/null; then + kill "$namesrv_pid" 2>/dev/null || true + fi + wait "$broker_pid" 2>/dev/null || true + wait "$namesrv_pid" 2>/dev/null || true + rm -rf "$work_dir" +} +trap cleanup EXIT + +cat >"$work_dir/pom.xml" < + 4.0.0 + com.dbx.test + rocketmq-integration + 1.0.0 + + + org.apache.rocketmq + rocketmq-namesrv + ${version} + + + org.apache.rocketmq + rocketmq-broker + ${version} + + + +EOF + +mvn -q -f "$work_dir/pom.xml" dependency:copy-dependencies -DoutputDirectory="$work_dir/lib" + +mkdir -p "$work_dir/conf" +for component in namesrv broker; do + cat >"$work_dir/conf/logback_${component}.xml" <<'EOF' + + + + %date %-5level %logger - %msg%n + + + + + + +EOF +done + +cat >"$work_dir/namesrv.conf" <"$work_dir/broker.conf" <"$work_dir/namesrv.stdout.log" 2>&1 & +namesrv_pid=$! + +java -Xms256m -Xmx256m -Xmn128m \ + --add-exports=java.base/jdk.internal.ref=ALL-UNNAMED \ + --add-opens=java.base/java.nio=ALL-UNNAMED \ + --add-opens=java.base/sun.nio.ch=ALL-UNNAMED \ + -Drocketmq.home.dir="$work_dir" -Duser.home="$work_dir" \ + -cp "$work_dir/lib/*" org.apache.rocketmq.broker.BrokerStartup -c "$work_dir/broker.conf" \ + >"$work_dir/broker.stdout.log" 2>&1 & +broker_pid=$! + +for _ in $(seq 1 120); do + if ! kill -0 "$namesrv_pid" 2>/dev/null; then + cat "$work_dir/namesrv.stdout.log" + exit 1 + fi + if ! kill -0 "$broker_pid" 2>/dev/null; then + cat "$work_dir/broker.stdout.log" + exit 1 + fi + if grep -Rqs "boot success" "$work_dir/logs" "$work_dir/broker.stdout.log" 2>/dev/null; then + break + fi + sleep 1 +done + +if ! grep -Rqs "boot success" "$work_dir/logs" "$work_dir/broker.stdout.log" 2>/dev/null; then + cat "$work_dir/namesrv.stdout.log" + cat "$work_dir/broker.stdout.log" + find "$work_dir/logs" -type f -maxdepth 3 -print -exec tail -80 {} \; 2>/dev/null || true + exit 1 +fi + +ROCKETMQ_INTEGRATION=1 \ +ROCKETMQ_VERSION="$version" \ +ROCKETMQ_NAMESRV_ADDR="127.0.0.1:${namesrv_port}" \ +go test -C "$module_dir" -run '^TestRocketMQIntegration$' -count=1 -v . diff --git a/agents/drivers/rocketmq/server.go b/agents/drivers/rocketmq/server.go new file mode 100644 index 0000000000..3b5520ba27 --- /dev/null +++ b/agents/drivers/rocketmq/server.go @@ -0,0 +1,168 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + + admin "github.com/amigoer/rocketmq-admin-go" +) + +const ( + protocolVersion = 1 + agentProtocolVersion = 1 +) + +var capabilities = []string{ + "mq_connect", "mq_test_connection", "mq_topics", "mq_consumer_groups", + "mq_messages", "mq_acl", "mq_config", "mq_monitoring", +} + +type rocketMQAgent struct { + mu sync.RWMutex + client *admin.Client + connection connectionConfig + proxies *proxyManager + clusterName string + brokerAddr string + shutdownRequested bool +} + +func newRocketMQAgent() *rocketMQAgent { + return &rocketMQAgent{} +} + +func (a *rocketMQAgent) dispatch(method string, raw json.RawMessage) (any, error) { + params, err := decodeParams(raw) + if err != nil { + return nil, err + } + switch method { + case "handshake": + return map[string]any{ + "protocolVersion": protocolVersion, + "agentProtocolVersion": agentProtocolVersion, + "capabilities": capabilities, + }, nil + case "connect": + return a.connect(params) + case "test_connection": + return a.testConnection(params) + case "disconnect": + a.close() + return okResult(), nil + case "shutdown": + a.close() + a.shutdownRequested = true + return okResult(), nil + case "mq_list_topics": + return a.listTopics(params) + case "mq_create_topic": + return a.createTopic(params) + case "mq_delete_topic": + return a.deleteTopic(params) + case "mq_update_partitions": + return a.updatePartitions(params) + case "mq_get_topic_stats": + return a.getTopicStats(params) + case "mq_get_topic_route": + return a.getTopicRoute(params) + case "mq_get_topic_config": + return a.getTopicConfig(params) + case "mq_alter_topic_config": + return a.alterTopicConfig(params) + case "mq_skip_topic_accumulation": + return a.skipTopicAccumulation(params) + case "mq_list_consumer_groups": + return a.listConsumerGroups(params) + case "mq_describe_consumer_group": + return a.describeConsumerGroup(params) + case "mq_delete_consumer_group": + return a.deleteConsumerGroup(params) + case "mq_get_subscription_group_config": + return a.getSubscriptionGroupConfig(params) + case "mq_alter_subscription_group_config": + return a.alterSubscriptionGroupConfig(params) + case "mq_reset_consumer_group_offsets": + return a.resetConsumerGroupOffsets(params) + case "mq_get_consumer_lag": + return a.getConsumerLag(params) + case "mq_list_producers": + return a.listProducers(params) + case "mq_peek_messages": + return a.peekMessages(params) + case "mq_view_message": + return a.viewMessage(params) + case "mq_query_message_by_key": + return a.queryMessageByKey(params) + case "mq_query_message_by_topic": + return a.queryMessageByTopic(params) + case "mq_query_message_trace": + return a.queryMessageTrace(params) + case "mq_send_message": + return a.sendMessage(params) + case "mq_list_acls": + return a.listACLs(params) + case "mq_create_acls": + return a.createACLs(params) + case "mq_delete_acls": + return a.deleteACLs(params) + case "mq_describe_cluster": + return a.describeCluster(params) + default: + return nil, fmt.Errorf("unknown method: %s", method) + } +} + +func (a *rocketMQAgent) requireClient() (*admin.Client, connectionConfig, error) { + a.mu.RLock() + defer a.mu.RUnlock() + if a.client == nil { + return nil, connectionConfig{}, errors.New("RocketMQ agent is not connected") + } + return a.client, a.connection, nil +} + +func (a *rocketMQAgent) close() { + a.mu.Lock() + defer a.mu.Unlock() + if a.client != nil { + _ = a.client.Close() + } + if a.proxies != nil { + a.proxies.Close() + } + a.client = nil + a.proxies = nil + a.connection = connectionConfig{} + a.clusterName = "" + a.brokerAddr = "" +} + +func decodeParams(raw json.RawMessage) (map[string]any, error) { + if len(raw) == 0 || string(raw) == "null" { + return map[string]any{}, nil + } + var params map[string]any + if err := json.Unmarshal(raw, ¶ms); err != nil { + return nil, fmt.Errorf("invalid params: %w", err) + } + return params, nil +} + +func normalizeError(err error) string { + if err == nil { + return "" + } + message := strings.TrimSpace(err.Error()) + if message == "" { + return "RocketMQ operation failed" + } + return message +} + +func okResult() map[string]any { + return map[string]any{"ok": true} +} diff --git a/agents/drivers/rocketmq/server_test.go b/agents/drivers/rocketmq/server_test.go new file mode 100644 index 0000000000..fe6d416be3 --- /dev/null +++ b/agents/drivers/rocketmq/server_test.go @@ -0,0 +1,34 @@ +package main + +import ( + "encoding/json" + "testing" +) + +func TestHandshakeAndShutdown(t *testing.T) { + agent := newRocketMQAgent() + result, err := agent.dispatch("handshake", json.RawMessage(`{}`)) + if err != nil { + t.Fatal(err) + } + handshake := result.(map[string]any) + if handshake["protocolVersion"] != protocolVersion || handshake["agentProtocolVersion"] != agentProtocolVersion { + t.Fatalf("unexpected handshake: %#v", handshake) + } + if _, err := agent.dispatch("shutdown", nil); err != nil { + t.Fatal(err) + } + if !agent.shutdownRequested { + t.Fatal("shutdown flag was not set") + } +} + +func TestDispatchRejectsInvalidParamsAndUnknownMethods(t *testing.T) { + agent := newRocketMQAgent() + if _, err := agent.dispatch("handshake", json.RawMessage(`[]`)); err == nil { + t.Fatal("expected array params to fail") + } + if _, err := agent.dispatch("does_not_exist", json.RawMessage(`{}`)); err == nil { + t.Fatal("expected unknown method to fail") + } +} diff --git a/agents/drivers/rocketmq/src/main/java/com/dbx/agent/rocketmq/RocketMqAgent.java b/agents/drivers/rocketmq/src/main/java/com/dbx/agent/rocketmq/RocketMqAgent.java deleted file mode 100644 index 35e9e5a0ef..0000000000 --- a/agents/drivers/rocketmq/src/main/java/com/dbx/agent/rocketmq/RocketMqAgent.java +++ /dev/null @@ -1,3794 +0,0 @@ -package com.dbx.agent.rocketmq; - -import com.google.gson.*; -import org.apache.rocketmq.acl.common.AclClientRPCHook; -import org.apache.rocketmq.acl.common.SessionCredentials; -import org.apache.rocketmq.client.ClientConfig; -import org.apache.rocketmq.client.QueryResult; -import org.apache.rocketmq.client.exception.MQBrokerException; -import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; -import org.apache.rocketmq.client.consumer.DefaultMQPullConsumer; -import org.apache.rocketmq.client.consumer.PullResult; -import org.apache.rocketmq.client.consumer.PullStatus; -import org.apache.rocketmq.client.producer.DefaultMQProducer; -import org.apache.rocketmq.client.producer.SendResult; -import org.apache.rocketmq.common.MixAll; -import org.apache.rocketmq.common.PlainAccessConfig; -import org.apache.rocketmq.common.TopicConfig; -import org.apache.rocketmq.common.message.Message; -import org.apache.rocketmq.common.message.MessageConst; -import org.apache.rocketmq.common.message.MessageExt; -import org.apache.rocketmq.common.message.MessageQueue; -import org.apache.rocketmq.remoting.RPCHook; -import org.apache.rocketmq.remoting.protocol.admin.ConsumeStats; -import org.apache.rocketmq.remoting.protocol.admin.OffsetWrapper; -import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable; -import org.apache.rocketmq.remoting.protocol.body.AclInfo; -import org.apache.rocketmq.remoting.protocol.body.ClusterInfo; -import org.apache.rocketmq.remoting.protocol.body.Connection; -import org.apache.rocketmq.remoting.protocol.body.ConsumerConnection; -import org.apache.rocketmq.remoting.protocol.body.ConsumerRunningInfo; -import org.apache.rocketmq.remoting.protocol.body.GroupList; -import org.apache.rocketmq.remoting.protocol.body.ProducerConnection; -import org.apache.rocketmq.remoting.protocol.body.ProducerInfo; -import org.apache.rocketmq.remoting.protocol.body.ProducerTableInfo; -import org.apache.rocketmq.remoting.protocol.body.SubscriptionGroupWrapper; -import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper; -import org.apache.rocketmq.remoting.protocol.body.TopicList; -import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData; -import org.apache.rocketmq.remoting.protocol.route.BrokerData; -import org.apache.rocketmq.remoting.protocol.route.QueueData; -import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; -import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; -import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; - -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.util.*; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; - -/** - * RocketMQ admin agent for DBX. Communicates with the Rust bridge via JSON-RPC - * over stdin/stdout. Uses DefaultMQAdminExt for admin operations and - * DefaultMQProducer for message production. - */ -public final class RocketMqAgent { - - private static final Gson GSON = new GsonBuilder().serializeNulls().create(); - private static final int DEFAULT_REQUEST_TIMEOUT_MS = 30_000; - private static final int DEFAULT_CONNECT_TIMEOUT_MS = 10_000; - private static final long MIN_ACL_PROBE_TIMEOUT_MS = 500; - private static final long MAX_ACL_PROBE_TIMEOUT_MS = 5_000; - private static final int DEFAULT_LIST_LIMIT = 200; - private static final int CONSUMER_GROUP_ENRICH_CONCURRENCY = 8; - private static final int CONSUMER_GROUP_COLLECT_CONCURRENCY = 8; - private static final int CONSUMER_LAG_CONCURRENCY = 8; - /** - * Phase baselines at {@link #DEFAULT_REQUEST_TIMEOUT_MS}. Scaled via - * {@link #scaledOperationBudgetMs} so Advanced query timeout expands headroom without letting - * collect+enrich+lag each consume a full RPC window. - */ - private static final long CONSUMER_GROUP_COLLECT_BUDGET_BASELINE_MS = 12_000; - private static final long CONSUMER_GROUP_ENRICH_BUDGET_BASELINE_MS = 8_000; - private static final long CONSUMER_LAG_BUDGET_BASELINE_MS = 10_000; - private static final long TOPIC_LIST_ROUTE_BUDGET_BASELINE_MS = 8_000; - /** Host-side Docker collision-fallbacks are usually blackholed; keep remoting short. */ - private static final long COLLISION_FALLBACK_PROBE_TIMEOUT_MS = 2_000; - /** - * LitePullConsumer.poll wait for local pull cache — not an RPC budget. - * Must stay short: empty queues block for the full timeout, and peek walks queues serially. - */ - private static final long MESSAGE_POLL_TIMEOUT_MS = 3_000; - /** Nameserver route enrichment for listTopics fallback when bulk broker config is unavailable. */ - private static final int TOPIC_LIST_ROUTE_CONCURRENCY = 8; - private static final String AUTO_CREATE_TOPIC_KEY = "TBW102"; - /** RocketMQ 5.x topic attribute key for message type (NORMAL/DELAY/FIFO/TRANSACTION). */ - private static final String TOPIC_MESSAGE_TYPE_ATTRIBUTE = "message.type"; - private static final Set RESERVED_TOPIC_NAMES = Set.of( - AUTO_CREATE_TOPIC_KEY, - "BenchmarkTest", - "SELF_TEST_TOPIC", - "OFFSET_MOVED_EVENT", - "DefaultHeartBeatSyncerTopic" - ); - /** Same set as rocketmq-dashboard ConsumerServiceImpl.SYSTEM_GROUP_SET. */ - private static final Set SYSTEM_CONSUMER_GROUPS = Set.of( - MixAll.TOOLS_CONSUMER_GROUP, - MixAll.FILTERSRV_CONSUMER_GROUP, - MixAll.SELF_TEST_CONSUMER_GROUP, - MixAll.ONS_HTTP_PROXY_GROUP, - MixAll.CID_ONSAPI_PULL_GROUP, - MixAll.CID_ONSAPI_PERMISSION_GROUP, - MixAll.CID_ONSAPI_OWNER_GROUP, - MixAll.CID_SYS_RMQ_TRANS, - "CID_DefaultHeartBeatSyncerTopic" - ); - - static boolean isSystemConsumerGroup(String groupId) { - return groupId != null && SYSTEM_CONSUMER_GROUPS.contains(groupId); - } - - /** - * Align with rocketmq-dashboard ConsumerServiceImpl group type: SYSTEM / FIFO / NORMAL. - * When the subscription dump is missing for a discovered group, return UNKNOWN instead of - * silently labeling FIFO groups as NORMAL. - */ - static String classifyConsumerGroupType(String groupId, SubscriptionGroupConfig config) { - if (isSystemConsumerGroup(groupId)) { - return "SYSTEM"; - } - if (config == null) { - return "UNKNOWN"; - } - if (config.isConsumeMessageOrderly()) { - return "FIFO"; - } - return "NORMAL"; - } - - static boolean isUserTopic(String topic) { - if (topic == null || topic.isBlank()) { - return false; - } - if (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) - || topic.startsWith(MixAll.DLQ_GROUP_TOPIC_PREFIX) - || topic.startsWith("%")) { - return false; - } - // RocketMQ system topics use RMQ_SYS / rmq_sys prefixes (see apache/rocketmq#1203). - if (topic.startsWith("RMQ_SYS") || topic.startsWith("rmq_sys")) { - return false; - } - if (topic.startsWith("SCHEDULE_TOPIC_") || topic.startsWith("rocketmq-broker-")) { - return false; - } - if (topic.endsWith("_REPLY_TOPIC")) { - return false; - } - if (RESERVED_TOPIC_NAMES.contains(topic)) { - return false; - } - return true; - } - - /** - * RocketMQ Dashboard Publishing Management queries producers via - * {@code mqadmin producerConnection -g -t }. Without a producer group the - * broker cannot filter producers by topic (apache/rocketmq#6371); list broker tables on - * topic-route brokers instead. - */ - static boolean shouldQueryClusterProducerTable(String topic, String producerGroup) { - return (topic == null || topic.isBlank()) && (producerGroup == null || producerGroup.isBlank()); - } - - static boolean isRocketMqSystemTopic( - String topic, - Set brokerSystemTopics, - Set brokerNames, - String cluster - ) { - return !isUserTopic(topic) - || brokerSystemTopics.contains(topic) - || brokerNames.contains(topic) - || (!cluster.isBlank() && cluster.equals(topic)); - } - - /** - * Classify RocketMQ topic message type using the same rules as rocketmq-dashboard - * {@code TopicServiceImpl#classifyTopicType}, plus broker/cluster reserved names. - */ - static String classifyTopicMessageType( - String topicName, - Map attributes, - Set brokerSystemTopics, - Set brokerNames, - String cluster - ) { - if (topicName == null || topicName.isBlank()) { - return "UNSPECIFIED"; - } - if (topicName.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) || topicName.startsWith("%R")) { - return "RETRY"; - } - if (topicName.startsWith(MixAll.DLQ_GROUP_TOPIC_PREFIX) || topicName.startsWith("%D")) { - return "DLQ"; - } - if (brokerSystemTopics.contains(topicName) - || topicName.startsWith("RMQ_SYS") - || topicName.startsWith("rmq_sys") - || topicName.equals("DefaultHeartBeatSyncerTopic") - || brokerNames.contains(topicName) - || (!cluster.isBlank() && cluster.equals(topicName)) - || !isUserTopic(topicName)) { - return "SYSTEM"; - } - String messageType = readTopicMessageTypeAttribute(attributes); - if (messageType == null || messageType.isBlank()) { - return "UNSPECIFIED"; - } - return normalizeMessageType(messageType); - } - - private static String readTopicMessageTypeAttribute(Map attributes) { - if (attributes == null || attributes.isEmpty()) { - return null; - } - String attrName = TOPIC_MESSAGE_TYPE_ATTRIBUTE; - String messageType = attributes.get(attrName); - if (messageType == null || messageType.isBlank()) { - messageType = attributes.get("+" + attrName); - } - if (messageType == null || messageType.isBlank()) { - for (Map.Entry entry : attributes.entrySet()) { - String key = entry.getKey(); - if (key != null && key.endsWith(attrName) && entry.getValue() != null && !entry.getValue().isBlank()) { - messageType = entry.getValue(); - break; - } - } - } - return messageType; - } - - private static String readTopicMessageType(TopicConfig config) { - if (config == null) { - return null; - } - // Prefer raw broker attributes: getTopicMessageType() may return UNSPECIFIED - // even when attributes already contain message.type=NORMAL (RocketMQ 5.x stores without '+'). - String fromAttributes = readTopicMessageTypeAttribute(config.getAttributes()); - if (fromAttributes != null && !fromAttributes.isBlank()) { - return fromAttributes; - } - return null; - } - - /** Test hook: run listTopics against an existing admin client. */ - static Object listTopicsWithAdminForTest(DefaultMQAdminExt admin, JsonObject params) throws Exception { - DefaultMQAdminExt previous = adminClient; - adminClient = admin; - try { - return listTopics(params); - } finally { - adminClient = previous; - } - } - - /** Test hook: send via an existing producer instance. */ - static Object sendMessageForTest(DefaultMQProducer activeProducer, JsonObject params) throws Exception { - DefaultMQProducer previous = producer; - producer = activeProducer; - try { - return sendMessage(params); - } finally { - producer = previous; - } - } - - /** Test hook: peek messages with an existing admin client. */ - static Object peekMessagesForTest(DefaultMQAdminExt admin, JsonObject params) throws Exception { - DefaultMQAdminExt previous = adminClient; - adminClient = admin; - try { - return peekMessages(params); - } finally { - adminClient = previous; - } - } - - static String normalizeMessageType(String raw) { - if (raw == null || raw.isBlank()) { - return "NORMAL"; - } - String upper = raw.trim().toUpperCase(Locale.ROOT); - if ("ORDER".equals(upper)) { - return "FIFO"; - } - return upper; - } - - private static final List CAPABILITIES = Collections.unmodifiableList(Arrays.asList( - "mq_connect", "mq_test_connection", "mq_topics", "mq_consumer_groups", - "mq_messages", "mq_acl", "mq_config", "mq_monitoring" - )); - - private static DefaultMQAdminExt adminClient; - private static DefaultMQProducer producer; - private static JsonObject cachedConnection; - private static String cachedBrokerAddr; - private static String cachedClusterName; - private static volatile boolean shutdownRequested; - - private RocketMqAgent() {} - - public static void main(String[] args) throws Exception { - System.setProperty("org.slf4j.simpleLogger.logFile", "System.err"); - System.out.println("{\"ready\":true}"); - System.out.flush(); - - BufferedReader reader = new BufferedReader(new InputStreamReader(System.in)); - while (true) { - String line = reader.readLine(); - if (line == null) { - break; - } - String response = handleRequest(line); - System.out.println(response); - System.out.flush(); - if (shutdownRequested) { - System.exit(0); - } - } - } - - static String handleRequest(String line) { - JsonObject req = JsonParser.parseString(line).getAsJsonObject(); - JsonElement id = req.get("id"); - String method = req.get("method").getAsString(); - JsonObject params = req.has("params") && req.get("params").isJsonObject() - ? req.getAsJsonObject("params") : new JsonObject(); - - JsonObject response = new JsonObject(); - response.addProperty("jsonrpc", "2.0"); - response.add("id", id); - - try { - Object result = dispatch(method, params); - response.add("result", GSON.toJsonTree(result)); - } catch (Exception e) { - JsonObject error = new JsonObject(); - error.addProperty("code", -1); - error.addProperty("message", normalizeErrorMessage(e)); - response.add("error", error); - } - return GSON.toJson(response); - } - - private static Object dispatch(String method, JsonObject params) throws Exception { - return switch (method) { - case "handshake" -> handshakeResult(); - case "connect" -> connect(params); - case "test_connection" -> testConnection(params); - case "disconnect" -> { closeClients(); yield Collections.singletonMap("ok", true); } - case "shutdown" -> { closeClients(); shutdownRequested = true; yield Collections.singletonMap("ok", true); } - case "mq_list_topics" -> listTopics(params); - case "mq_create_topic" -> createTopic(params); - case "mq_delete_topic" -> deleteTopic(params); - case "mq_update_partitions" -> updatePartitions(params); - case "mq_get_topic_stats" -> getTopicStats(params); - case "mq_get_topic_route" -> getTopicRoute(params); - case "mq_get_topic_config" -> getTopicConfig(params); - case "mq_alter_topic_config" -> alterTopicConfig(params); - case "mq_skip_topic_accumulation" -> skipTopicAccumulation(params); - case "mq_list_consumer_groups" -> listConsumerGroups(params); - case "mq_describe_consumer_group" -> describeConsumerGroup(params); - case "mq_delete_consumer_group" -> deleteConsumerGroup(params); - case "mq_get_subscription_group_config" -> getSubscriptionGroupConfig(params); - case "mq_alter_subscription_group_config" -> alterSubscriptionGroupConfig(params); - case "mq_reset_consumer_group_offsets" -> resetConsumerGroupOffsets(params); - case "mq_get_consumer_lag" -> getConsumerLag(params); - case "mq_list_producers" -> listProducers(params); - case "mq_peek_messages" -> peekMessages(params); - case "mq_view_message" -> viewMessage(params); - case "mq_query_message_by_key" -> queryMessageByKey(params); - case "mq_query_message_by_topic" -> queryMessageByTopic(params); - case "mq_query_message_trace" -> queryMessageTrace(params); - case "mq_send_message" -> sendMessage(params); - case "mq_list_acls" -> listAcls(params); - case "mq_create_acls" -> createAcls(params); - case "mq_delete_acls" -> deleteAcls(params); - case "mq_describe_cluster" -> describeCluster(params); - default -> throw new IllegalArgumentException("Unknown method: " + method); - }; - } - - private static Object handshakeResult() { - return new HandshakeResult(1, 1, CAPABILITIES); - } - - private static Object connect(JsonObject params) throws Exception { - JsonObject conn = connectionObject(params); - DefaultMQAdminExt nextAdmin = null; - try { - nextAdmin = buildAdminClient(conn); - // One ClusterInfo per connect — reuse for name/broker resolution and test payload. - ClusterInfo clusterInfo = nextAdmin.examineBrokerClusterInfo(); - closeClients(); - adminClient = nextAdmin; - cachedConnection = conn.deepCopy(); - cachedClusterName = resolveClusterName(clusterInfo, conn); - cachedBrokerAddr = resolveBrokerAddr(nextAdmin, conn, clusterInfo); - return buildClusterTestResult(clusterInfo, nextAdmin, conn); - } catch (Exception e) { - if (nextAdmin != null) { - nextAdmin.shutdown(); - } - throw e; - } - } - - private static Object testConnection(JsonObject params) throws Exception { - JsonObject conn = connectionObject(params); - if (adminClient != null && cachedConnection != null && connectionMatches(cachedConnection, conn)) { - ClusterInfo clusterInfo = adminClient.examineBrokerClusterInfo(); - return buildClusterTestResult(clusterInfo, adminClient, conn); - } - DefaultMQAdminExt probe = null; - try { - probe = buildAdminClient(conn); - ClusterInfo clusterInfo = probe.examineBrokerClusterInfo(); - return buildClusterTestResult(clusterInfo, probe, conn); - } finally { - if (probe != null) { - probe.shutdown(); - } - } - } - - private static Map buildClusterTestResult( - ClusterInfo clusterInfo, - DefaultMQAdminExt admin, - JsonObject conn - ) throws Exception { - String clusterName = resolveClusterName(clusterInfo, conn); - List> brokers = brokerNodes(clusterInfo); - boolean aclEnabled = probeAclSupport(admin, clusterInfo, conn); - - Map result = new LinkedHashMap<>(); - result.put("ok", true); - result.put("clusterId", clusterName); - result.put("brokers", brokers); - result.put("nodeCount", brokers.size()); - result.put("controller", brokers.isEmpty() ? null : brokers.get(0)); - result.put("aclEnabled", aclEnabled); - return result; - } - - static boolean connectionMatches(JsonObject cached, JsonObject requested) { - return namesrvAddr(cached).equals(namesrvAddr(requested)) - && clusterName(cached).equals(clusterName(requested)) - && brokerAddress(cached).equals(brokerAddress(requested)) - && credential(cached, "access_key", "accessKey").equals(credential(requested, "access_key", "accessKey")) - && credential(cached, "secret_key", "secretKey").equals(credential(requested, "secret_key", "secretKey")); - } - - private static void closeClients() { - if (adminClient != null) { - adminClient.shutdown(); - adminClient = null; - } - if (producer != null) { - producer.shutdown(); - producer = null; - } - cachedConnection = null; - cachedBrokerAddr = null; - cachedClusterName = null; - } - - private static Object listTopics(JsonObject params) throws Exception { - DefaultMQAdminExt admin = requireAdmin(); - String keyword = stringOrEmpty(params, "keyword").toLowerCase(Locale.ROOT); - int offset = Math.max(0, intOrDefault(params, "offset", 0)); - // limit <= 0: return all rows from offset (legacy single-shot signal). - // Large positive limits (e.g. Integer.MAX_VALUE) also return the full remaining - // catalog and stay compatible with older agents that coerced limit<=0 to 200. - int limit = intOrDefault(params, "limit", DEFAULT_LIST_LIMIT); - JsonObject conn = connectionObject(params); - - // One ClusterInfo per list call — avoid repeated examineBrokerClusterInfo in helpers. - ClusterInfo clusterInfo = admin.examineBrokerClusterInfo(); - Set brokerSystemTopics = collectBrokerSystemTopics(admin); - Set brokerNames = collectBrokerNames(clusterInfo); - String cluster = resolveClusterName(clusterInfo, conn); - BrokerTopicConfigSnapshot brokerSnapshot = collectBrokerTopicConfigSnapshot(admin, conn, clusterInfo); - Map brokerTopics = brokerSnapshot.topics(); - Map> topicAttributes = topicAttributesFromConfigs(brokerTopics); - - Set nameserverTopics = Set.of(); - if (!brokerSnapshot.complete() || brokerTopics.isEmpty()) { - // Reuse resolved cluster — do not call examineBrokerClusterInfo again via clusterName(conn, admin). - TopicList topicList = fetchTopicList(admin, cluster); - nameserverTopics = topicList.getTopicList(); - } - Set topicNames = topicCatalogNames(brokerSnapshot, nameserverTopics); - - List> topics = buildTopicCatalogRows( - topicNames, - brokerTopics, - topicAttributes, - brokerSystemTopics, - brokerNames, - cluster, - keyword, - admin::examineTopicRouteInfo, - TOPIC_LIST_ROUTE_CONCURRENCY, - scaledOperationBudgetMs(conn, TOPIC_LIST_ROUTE_BUDGET_BASELINE_MS) - ); - topics.sort(Comparator.comparing(m -> String.valueOf(m.get("name")))); - - int total = topics.size(); - List> page = paginate(topics, offset, limit); - Map result = new LinkedHashMap<>(); - result.put("topics", page); - result.put("total", total); - result.put("offset", offset); - result.put("limit", limit); - return result; - } - - @FunctionalInterface - interface TopicRouteLookup { - TopicRouteData load(String topic) throws Exception; - } - - @FunctionalInterface - interface BrokerTopicConfigLookup { - TopicConfigSerializeWrapper load(String brokerAddr) throws Exception; - } - - static final class BrokerTopicConfigSnapshot { - private final Map topics; - private final boolean complete; - - private BrokerTopicConfigSnapshot(Map topics, boolean complete) { - this.topics = topics; - this.complete = complete; - } - - Map topics() { - return topics; - } - - boolean complete() { - return complete; - } - } - - static Set topicCatalogNames( - BrokerTopicConfigSnapshot brokerSnapshot, Set nameserverTopics) { - if (brokerSnapshot.complete() && !brokerSnapshot.topics().isEmpty()) { - return new TreeSet<>(brokerSnapshot.topics().keySet()); - } - Set topics = new TreeSet<>(brokerSnapshot.topics().keySet()); - topics.addAll(nameserverTopics); - return topics; - } - - /** - * Build topic catalog rows from names + optional bulk broker configs. - * When bulk configs are empty, partition counts are filled via a single budgeted route RPC - * per topic; per-topic examineTopicConfig / resolveTopicMessageType is never used (that path - * hits the same broker that already failed getAllTopicConfig and can stall for minutes). - */ - static List> buildTopicCatalogRows( - Set topicNames, - Map brokerTopics, - Map> topicAttributes, - Set brokerSystemTopics, - Set brokerNames, - String cluster, - String keyword, - TopicRouteLookup routeLookup, - int routeConcurrency, - long routeBudgetMs - ) { - String keywordLower = keyword == null ? "" : keyword.toLowerCase(Locale.ROOT); - boolean hasBrokerCatalog = brokerTopics != null && !brokerTopics.isEmpty(); - List> topics = new ArrayList<>(); - List> needsRoute = new ArrayList<>(); - for (String topic : topicNames) { - if (!keywordLower.isBlank() && !topic.toLowerCase(Locale.ROOT).contains(keywordLower)) { - continue; - } - TopicConfig brokerConfig = hasBrokerCatalog ? brokerTopics.get(topic) : null; - // Nameserver-only stale topics are skipped when broker catalog is the source of truth. - if (hasBrokerCatalog && brokerConfig == null) { - continue; - } - int partitions = brokerConfig == null ? 1 : Math.max(brokerConfig.getReadQueueNums(), 1); - Map attributes = topicAttributes == null ? null : topicAttributes.get(topic); - // Type from bulk config attributes only — never examineTopicConfig per topic on fallback. - if (brokerConfig != null - && isUserTopic(topic) - && readTopicMessageTypeAttribute(attributes) == null) { - String resolved = readTopicMessageType(brokerConfig); - if (resolved != null && !resolved.isBlank()) { - attributes = attributes == null ? new HashMap<>() : new HashMap<>(attributes); - attributes.put("+" + TOPIC_MESSAGE_TYPE_ATTRIBUTE, resolved); - } - } - String messageType = classifyTopicMessageType( - topic, attributes, brokerSystemTopics, brokerNames, cluster); - boolean internal = "SYSTEM".equals(messageType) - || "RETRY".equals(messageType) - || "DLQ".equals(messageType); - Map row = new LinkedHashMap<>(); - row.put("name", topic); - row.put("partitions", partitions); - row.put("replicationFactor", 1); - row.put("internal", internal); - row.put("messageType", messageType); - topics.add(row); - if (brokerConfig == null) { - needsRoute.add(row); - } - } - if (!needsRoute.isEmpty() && routeLookup != null) { - enrichTopicPartitions(needsRoute, routeLookup, routeConcurrency, routeBudgetMs); - } - return topics; - } - - static int partitionsFromRoute(TopicRouteData route) { - if (route == null || route.getQueueDatas() == null || route.getQueueDatas().isEmpty()) { - return 1; - } - return Math.max(route.getQueueDatas().get(0).getReadQueueNums(), 1); - } - - /** - * Fill {@code partitions} via nameserver route lookups with bounded concurrency and a global - * time budget. Unfinished/failed topics keep the default partitions=1. - */ - static void enrichTopicPartitions( - List> rows, - TopicRouteLookup lookup, - int concurrency, - long budgetMs) { - if (rows.isEmpty() || lookup == null) { - return; - } - int workers = Math.max(1, Math.min(concurrency, rows.size())); - ExecutorService executor = Executors.newFixedThreadPool(workers, runnable -> { - Thread thread = new Thread(runnable, "dbx-rocketmq-topic-route"); - thread.setDaemon(true); - return thread; - }); - List> tasks = new ArrayList<>(rows.size()); - for (Map row : rows) { - String topic = String.valueOf(row.get("name")); - tasks.add(() -> { - try { - return partitionsFromRoute(lookup.load(topic)); - } catch (Exception ignored) { - return 1; - } - }); - } - List> results = Collections.emptyList(); - try { - results = executor.invokeAll(tasks, Math.max(1, budgetMs), TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } finally { - executor.shutdownNow(); - } - for (int index = 0; index < rows.size(); index++) { - Map row = rows.get(index); - if (index < results.size() && !results.get(index).isCancelled()) { - try { - row.put("partitions", results.get(index).get()); - } catch (Exception ignored) { - // Keep default partitions=1 when enrichment fails. - } - } - row.putIfAbsent("partitions", 1); - } - } - - private static Object createTopic(JsonObject params) throws Exception { - DefaultMQAdminExt admin = requireAdmin(); - JsonObject conn = connectionObject(params); - String name = requireString(params, "name"); - int readQueues = intOrDefault(params, "readQueueNums", intOrDefault(params, "partitions", 8)); - int writeQueues = intOrDefault(params, "writeQueueNums", readQueues); - int perm = normalizeTopicPerm(intOrDefault(params, "perm", 6)); - String messageType = normalizeMessageType(stringOrDefault(params, "messageType", "NORMAL")); - String brokerName = stringOrEmpty(params, "brokerName"); - TopicConfig config = buildTopicConfigForCreate(name, readQueues, writeQueues, messageType, perm); - for (String brokerAddr : resolveMasterBrokerAddrs( - admin, conn, brokerName.isBlank() ? null : brokerName)) { - admin.createAndUpdateTopicConfig(brokerAddr, config); - } - return Collections.singletonMap("ok", true); - } - - private static Object deleteTopic(JsonObject params) throws Exception { - DefaultMQAdminExt admin = requireAdmin(); - JsonObject conn = connectionObject(params); - String name = requireString(params, "name"); - String brokerName = stringOrEmpty(params, "brokerName"); - // Match RocketMQ Dashboard: delete broker configs and nameserver routes (remapped addrs for docker). - Set masterAddrs = new LinkedHashSet<>(resolveMasterBrokerAddrs( - admin, conn, brokerName.isBlank() ? null : brokerName)); - if (!masterAddrs.isEmpty()) { - admin.deleteTopicInBroker(masterAddrs, name); - } - Set nameServerSet = resolveNameServerAddrSet(conn); - if (!nameServerSet.isEmpty()) { - admin.deleteTopicInNameServer(nameServerSet, name); - } - return Collections.singletonMap("ok", true); - } - - private static Object updatePartitions(JsonObject params) throws Exception { - DefaultMQAdminExt admin = requireAdmin(); - JsonObject conn = connectionObject(params); - String name = requireString(params, "name"); - int totalPartitions = intOrDefault(params, "totalPartitions", 1); - int readQueues = intOrDefault(params, "readQueueNums", totalPartitions); - int writeQueues = intOrDefault(params, "writeQueueNums", totalPartitions); - String brokerAddr = brokerAddr(params, admin, conn); - TopicConfig config = loadTopicConfig(admin, brokerAddr, name); - config.setReadQueueNums(readQueues); - config.setWriteQueueNums(writeQueues); - admin.createAndUpdateTopicConfig(brokerAddr, config); - return Collections.singletonMap("ok", true); - } - - private static Object getTopicRoute(JsonObject params) throws Exception { - DefaultMQAdminExt admin = requireAdmin(); - String name = requireString(params, "name"); - TopicRouteData route = admin.examineTopicRouteInfo(name); - List> queueDatas = new ArrayList<>(); - if (route.getQueueDatas() != null) { - for (QueueData queueData : route.getQueueDatas()) { - Map row = new LinkedHashMap<>(); - row.put("brokerName", queueData.getBrokerName()); - row.put("readQueueNums", queueData.getReadQueueNums()); - row.put("writeQueueNums", queueData.getWriteQueueNums()); - row.put("perm", queueData.getPerm()); - queueDatas.add(row); - } - } - List> brokerDatas = new ArrayList<>(); - if (route.getBrokerDatas() != null) { - for (BrokerData brokerData : route.getBrokerDatas()) { - Map row = new LinkedHashMap<>(); - row.put("brokerName", brokerData.getBrokerName()); - row.put("cluster", brokerData.getCluster()); - row.put("brokerAddrs", brokerData.getBrokerAddrs()); - brokerDatas.add(row); - } - } - Map result = new LinkedHashMap<>(); - result.put("topic", name); - result.put("queueDatas", queueDatas); - result.put("brokerDatas", brokerDatas); - return result; - } - - private static Object skipTopicAccumulation(JsonObject params) throws Exception { - DefaultMQAdminExt admin = requireAdmin(); - JsonObject conn = connectionObject(params); - String topic = requireString(params, "topic"); - Set groupIds = queryTopicConsumeByWhoRemapped(admin, conn, topic); - if (groupIds.isEmpty()) { - return Map.of("ok", true, "resetGroups", 0); - } - long timestamp = System.currentTimeMillis(); - int resetGroups = 0; - for (String groupId : groupIds) { - admin.resetOffsetByTimestamp(topic, groupId, timestamp, true); - resetGroups++; - } - Map result = new LinkedHashMap<>(); - result.put("ok", true); - result.put("resetGroups", resetGroups); - return result; - } - - private static Object viewMessage(JsonObject params) throws Exception { - DefaultMQAdminExt admin = requireAdmin(); - JsonObject conn = connectionObject(params); - String topic = requireString(params, "topic"); - String msgId = requireString(params, "msgId"); - Integer partition = integerOrNull(params, "partition"); - Long offset = longOrNull(params, "offset"); - MessageExt message = resolveViewMessage(admin, conn, topic, msgId, partition, offset); - Map result = new LinkedHashMap<>(); - result.put("message", peekedMessageFromRecord(topic, message)); - return result; - } - - /** - * Match rocketmq-dashboard 5.x: query by msgId across topic clusters, then fall back to - * queue offset when peek/list responses already include partition + offset. - */ - static MessageExt resolveViewMessage( - DefaultMQAdminExt admin, - JsonObject conn, - String topic, - String msgId, - Integer partition, - Long offset - ) throws Exception { - MessageExt message = queryMessageByIdAcrossClusters(admin, topic, msgId); - if (message != null) { - return message; - } - if (partition != null && offset != null) { - message = readMessageAtQueueOffset(conn, admin, topic, partition, offset); - if (message != null) { - return message; - } - } - throw new MQClientException( - 208, - "query message by key finished, but no message. For more information, please visit the url, " - + "https://rocketmq.apache.org/docs/bestPractice/06FAQ" - ); - } - - static MessageExt queryMessageByIdAcrossClusters(DefaultMQAdminExt admin, String topic, String msgId) { - try { - return admin.viewMessage(topic, msgId); - } catch (Exception ignored) { - } - try { - Set clusters = admin.getTopicClusterList(topic); - if (clusters != null && !clusters.isEmpty()) { - for (String cluster : clusters) { - MessageExt message = queryMessageByIdInCluster(admin, cluster, topic, msgId); - if (message != null) { - return message; - } - } - return null; - } - return queryMessageByIdInCluster(admin, "", topic, msgId); - } catch (Exception ignored) { - return null; - } - } - - private static MessageExt queryMessageByIdInCluster( - DefaultMQAdminExt admin, - String cluster, - String topic, - String msgId - ) { - try { - return admin.queryMessage(cluster, topic, msgId); - } catch (MQClientException e) { - if (isEmptyQueryMessageResult(e)) { - return null; - } - return null; - } catch (Exception ignored) { - return null; - } - } - - private static MessageExt readMessageAtQueueOffset( - JsonObject conn, - DefaultMQAdminExt admin, - String topic, - int partition, - long offset - ) throws Exception { - List queues = resolvePeekQueues(admin, topic, partition); - if (queues.isEmpty()) { - return null; - } - DefaultLitePullConsumer consumer = buildLitePullConsumer(conn); - consumer.start(); - try { - for (MessageQueue queue : queues) { - if (queue.getQueueId() != partition) { - continue; - } - consumer.assign(Collections.singletonList(queue)); - consumer.seek(queue, offset); - consumer.setPullBatchSize(1); - List polled = consumer.poll(MESSAGE_POLL_TIMEOUT_MS); - if (polled == null || polled.isEmpty()) { - continue; - } - for (MessageExt message : polled) { - if (message.getQueueOffset() == offset) { - return message; - } - } - return polled.get(0); - } - return null; - } finally { - consumer.shutdown(); - } - } - - private static Object queryMessageByKey(JsonObject params) throws Exception { - DefaultMQAdminExt admin = requireAdmin(); - String topic = requireString(params, "topic"); - String key = requireString(params, "key"); - int maxNum = Math.max(1, Math.min(intOrDefault(params, "maxNum", 32), 200)); - long begin = longOrDefault(params, "begin", 0L); - long end = longOrDefault(params, "end", System.currentTimeMillis()); - try { - QueryResult queryResult = admin.queryMessage(topic, key, maxNum, begin, end); - return queryResultToMap(topic, queryResult); - } catch (MQClientException e) { - if (isEmptyQueryMessageResult(e)) { - return emptyQueryResultMap(); - } - throw e; - } - } - - private static Object queryMessageByTopic(JsonObject params) throws Exception { - JsonObject conn = connectionObject(params); - String topic = requireString(params, "topic"); - int maxNum = Math.max(1, Math.min(intOrDefault(params, "maxNum", 32), 200)); - long begin = longOrDefault(params, "begin", 0L); - long end = longOrDefault(params, "end", System.currentTimeMillis()); - // Match rocketmq-dashboard: scan queues by store timestamp instead of queryMessage(topic, topic, ...). - List> messages = queryMessagesByTopicTimeRange(conn, topic, begin, end, maxNum); - Map result = new LinkedHashMap<>(); - result.put("messages", messages); - result.put("indexLastUpdateTimestamp", 0L); - return result; - } - - private static Object queryMessageTrace(JsonObject params) throws Exception { - DefaultMQAdminExt admin = requireAdmin(); - String msgId = requireString(params, "msgId"); - String traceTopic = stringOrDefault(params, "traceTopic", "RMQ_SYS_TRACE_TOPIC"); - int maxNum = Math.max(1, Math.min(intOrDefault(params, "maxNum", 64), 200)); - long begin = longOrDefault(params, "begin", 0L); - long end = longOrDefault(params, "end", System.currentTimeMillis()); - try { - QueryResult queryResult = admin.queryMessage(traceTopic, msgId, maxNum, begin, end); - Map result = queryResultToMap(traceTopic, queryResult); - result.put("msgId", msgId); - result.put("traceTopic", traceTopic); - return result; - } catch (MQClientException e) { - if (isEmptyQueryMessageResult(e)) { - Map result = emptyQueryResultMap(); - result.put("msgId", msgId); - result.put("traceTopic", traceTopic); - return result; - } - throw e; - } - } - - /** - * RocketMQ client code 208 means the key/index lookup completed with zero matches. - * Dashboard treats this as an empty result, not a hard failure. - */ - static boolean isEmptyQueryMessageResult(MQClientException e) { - return e != null && e.getResponseCode() == 208; - } - - private static Map emptyQueryResultMap() { - Map result = new LinkedHashMap<>(); - result.put("messages", Collections.emptyList()); - result.put("indexLastUpdateTimestamp", 0L); - return result; - } - - private static List> queryMessagesByTopicTimeRange( - JsonObject conn, - String topic, - long begin, - long end, - int maxNum - ) throws Exception { - DefaultMQPullConsumer consumer = buildPullConsumer(conn); - consumer.start(); - try { - List> messages = new ArrayList<>(); - Set queues = consumer.fetchSubscribeMessageQueues(topic); - for (MessageQueue queue : queues) { - if (messages.size() >= maxNum) { - break; - } - long minOffset; - long maxOffset; - try { - minOffset = consumer.searchOffset(queue, begin); - maxOffset = consumer.searchOffset(queue, end); - } catch (Exception ignored) { - continue; - } - if (maxOffset < minOffset) { - continue; - } - READQ: - for (long offset = minOffset; offset <= maxOffset; ) { - if (messages.size() >= maxNum) { - break; - } - PullResult pullResult = consumer.pull(queue, "*", offset, 32); - offset = pullResult.getNextBeginOffset(); - switch (pullResult.getPullStatus()) { - case FOUND -> { - for (MessageExt message : pullResult.getMsgFoundList()) { - long storeTimestamp = message.getStoreTimestamp(); - if (storeTimestamp >= begin && storeTimestamp <= end) { - messages.add(peekedMessageFromRecord(topic, message)); - if (messages.size() >= maxNum) { - break READQ; - } - } - } - } - case NO_MATCHED_MSG, NO_NEW_MSG, OFFSET_ILLEGAL -> { - break READQ; - } - default -> { - } - } - } - } - sortPeekedMessages(messages); - if (messages.size() > maxNum) { - messages = new ArrayList<>(messages.subList(0, maxNum)); - } - return messages; - } finally { - consumer.shutdown(); - } - } - - private static Map queryResultToMap(String topic, QueryResult queryResult) { - List> messages = new ArrayList<>(); - if (queryResult.getMessageList() != null) { - for (MessageExt message : queryResult.getMessageList()) { - messages.add(peekedMessageFromRecord(topic, message)); - } - } - sortPeekedMessages(messages); - Map result = new LinkedHashMap<>(); - result.put("messages", messages); - result.put("indexLastUpdateTimestamp", queryResult.getIndexLastUpdateTimestamp()); - return result; - } - - private static Object getTopicStats(JsonObject params) throws Exception { - DefaultMQAdminExt admin = requireAdmin(); - JsonObject conn = connectionObject(params); - String name = requireString(params, "name"); - TopicStatsTable stats = examineTopicStatsRemapped(admin, conn, name); - - long totalMessages = 0; - List> partitionStats = new ArrayList<>(); - if (stats.getOffsetTable() != null) { - for (var entry : stats.getOffsetTable().entrySet()) { - long begin = entry.getValue().getMinOffset(); - long end = entry.getValue().getMaxOffset(); - long count = Math.max(0, end - begin); - totalMessages += count; - - Map ps = new LinkedHashMap<>(); - ps.put("partition", entry.getKey().getQueueId()); - ps.put("brokerName", entry.getKey().getBrokerName()); - ps.put("beginOffset", begin); - ps.put("endOffset", end); - ps.put("messageCount", count); - partitionStats.add(ps); - } - } - partitionStats.sort(Comparator.comparingInt(a -> (int) a.get("partition"))); - - Map result = new LinkedHashMap<>(); - result.put("name", name); - result.put("partitions", partitionStats.size()); - result.put("replicationFactor", 1); - result.put("totalMessages", totalMessages); - result.put("partitionStats", partitionStats); - return result; - } - - /** - * DefaultMQAdminExt.examineTopicStats(topic) uses broker addresses from NameServer route - * directly. Docker brokers often register unreachable internal IPs; remap to the namesrv host. - */ - private static TopicStatsTable examineTopicStatsRemapped( - DefaultMQAdminExt admin, JsonObject conn, String topic) throws Exception { - TopicRouteData route = admin.examineTopicRouteInfo(topic); - TopicStatsTable merged = new TopicStatsTable(); - if (route.getBrokerDatas() != null) { - for (BrokerData brokerData : route.getBrokerDatas()) { - String rawAddr = brokerData.selectBrokerAddr(); - if (rawAddr == null || rawAddr.isBlank()) { - continue; - } - String brokerAddr = remapBrokerAddrForClient(rawAddr, conn); - TopicStatsTable partial = admin.examineTopicStats(brokerAddr, topic); - if (partial != null && partial.getOffsetTable() != null) { - merged.getOffsetTable().putAll(partial.getOffsetTable()); - } - } - } - if (merged.getOffsetTable().isEmpty()) { - throw new MQClientException("Not found the topic stats info", null); - } - return merged; - } - - private static SubscriptionGroupConfig findSubscriptionGroupConfig( - DefaultMQAdminExt admin, JsonObject conn, String groupId) throws Exception { - // Walk every master and FIFO-merge so an earlier NORMAL stub cannot hide orderly=true. - long timeoutMs = Math.max(1_000L, intOrDefault(conn, "request_timeout_ms", DEFAULT_REQUEST_TIMEOUT_MS)); - Map merged = new HashMap<>(); - for (String brokerAddr : resolveMasterBrokerAddrs(admin, conn)) { - try { - SubscriptionGroupWrapper wrapper = admin.getAllSubscriptionGroup(brokerAddr, timeoutMs); - if (wrapper == null || wrapper.getSubscriptionGroupTable() == null) { - continue; - } - SubscriptionGroupConfig config = wrapper.getSubscriptionGroupTable().get(groupId); - if (config != null) { - mergeSubscriptionGroupConfigs(merged, Map.of(groupId, config)); - } - } catch (Exception ignored) { - // Try next broker. - } - } - return merged.get(groupId); - } - - private static Map collectConsumerGroupConfigs( - DefaultMQAdminExt admin, JsonObject conn) throws Exception { - long budgetMs = scaledOperationBudgetMs(conn, CONSUMER_GROUP_COLLECT_BUDGET_BASELINE_MS); - MasterBrokerAddrPlan plan = resolveMasterBrokerAddrPlan(admin, conn); - // Fallbacks are usually Docker-internal and blackholed from host agents; keep their - // remoting short so raising Advanced query timeout cannot stretch collect wall time. - long fallbackPerBrokerMs = Math.min(budgetMs, COLLISION_FALLBACK_PROBE_TIMEOUT_MS); - return collectConsumerGroupConfigs( - plan, - CONSUMER_GROUP_COLLECT_CONCURRENCY, - budgetMs, - budgetMs, - fallbackPerBrokerMs, - (brokerAddr, timeoutMs) -> { - SubscriptionGroupWrapper wrapper = admin.getAllSubscriptionGroup(brokerAddr, timeoutMs); - if (wrapper == null || wrapper.getSubscriptionGroupTable() == null) { - return Map.of(); - } - return wrapper.getSubscriptionGroupTable(); - } - ); - } - - /** - * Topic-filtered lists still need every reachable master's subscription dump for FIFO typing, - * but must not serially walk unreachable collision-fallbacks. Reuse the two-phase parallel collect. - */ - private static Map collectConsumerGroupConfigsForTopicFilter( - DefaultMQAdminExt admin, JsonObject conn) { - try { - return collectConsumerGroupConfigs(admin, conn); - } catch (Exception ignored) { - return new TreeMap<>(); - } - } - - @FunctionalInterface - interface SubscriptionGroupBrokerProbe { - Map load(String brokerAddr, long timeoutMs) throws Exception; - } - - /** - * Merge broker subscription dumps. Prefer {@code consumeMessageOrderly=true} so a NORMAL - * stub on an earlier master cannot hide a FIFO group that exists on another broker. - */ - static void mergeSubscriptionGroupConfigs( - Map target, - Map incoming - ) { - if (incoming == null || incoming.isEmpty()) { - return; - } - for (Map.Entry entry : incoming.entrySet()) { - String groupId = entry.getKey(); - SubscriptionGroupConfig next = entry.getValue(); - if (groupId == null || next == null) { - continue; - } - target.merge(groupId, next, (existing, candidate) -> { - if (candidate != null && candidate.isConsumeMessageOrderly() - && (existing == null || !existing.isConsumeMessageOrderly())) { - return candidate; - } - return existing != null ? existing : candidate; - }); - } - } - - /** - * Two-phase parallel subscription-group dump. Remapped masters first; collision-fallbacks - * only when the remapped merge is empty — same host-side Docker tradeoff as consume-stats. - * Slow/unreachable brokers are skipped so a large cluster does not burn the Agent RPC window. - */ - static Map collectConsumerGroupConfigs( - MasterBrokerAddrPlan plan, - int concurrency, - long budgetMs, - long remappedPerBrokerTimeoutMs, - long fallbackPerBrokerTimeoutMs, - SubscriptionGroupBrokerProbe probe - ) throws Exception { - Map configs = collectConsumerGroupConfigsFromBrokers( - new ArrayList<>(plan.remappedAddrs()), - concurrency, - budgetMs, - remappedPerBrokerTimeoutMs, - probe - ); - if (configs.isEmpty() && plan.fallbackCount() > 0) { - Map fallbackConfigs = collectConsumerGroupConfigsFromBrokers( - new ArrayList<>(plan.collisionFallbackAddrs()), - concurrency, - budgetMs, - fallbackPerBrokerTimeoutMs, - probe - ); - mergeSubscriptionGroupConfigs(configs, fallbackConfigs); - } - return configs; - } - - static Map collectConsumerGroupConfigsFromBrokers( - List brokers, - int concurrency, - long budgetMs, - long perBrokerTimeoutMs, - SubscriptionGroupBrokerProbe probe - ) throws Exception { - Map configs = new TreeMap<>(); - if (brokers == null || brokers.isEmpty()) { - return configs; - } - int workers = Math.max(1, Math.min(concurrency, brokers.size())); - ExecutorService executor = Executors.newFixedThreadPool(workers, runnable -> { - Thread thread = new Thread(runnable, "dbx-rocketmq-consumer-collect"); - thread.setDaemon(true); - return thread; - }); - List>> tasks = new ArrayList<>(brokers.size()); - long brokerTimeout = Math.max(1_000, perBrokerTimeoutMs); - for (String brokerAddr : brokers) { - tasks.add(() -> { - Map partial = new HashMap<>(); - try { - Map loaded = probe.load(brokerAddr, brokerTimeout); - if (loaded != null && !loaded.isEmpty()) { - partial.putAll(loaded); - } - } catch (Exception ignored) { - // Skip unreachable brokers; return whatever others collected. - } - return partial; - }); - } - try { - List>> futures = - executor.invokeAll(tasks, Math.max(1, budgetMs), TimeUnit.MILLISECONDS); - for (Future> future : futures) { - if (future.isCancelled()) { - continue; - } - try { - Map partial = future.get(); - mergeSubscriptionGroupConfigs(configs, partial); - } catch (Exception ignored) { - // Partial page is better than failing the whole list. - } - } - } finally { - executor.shutdownNow(); - } - return configs; - } - - /** - * DefaultMQAdminExt.examineConsumerConnectionInfo(group) picks a broker from route using - * NameServer-registered addresses. Remap to the client-reachable host before querying. - * Live members short-circuit. Empty / 206 offline claims require full remapped coverage and - * complete collision fallbacks (same class of gate as lag/mutate) so a single remapped 206 - * cannot paint memberCount 0 while consumers may be online on an unreached sibling. - */ - private static ConsumerConnection examineConsumerConnectionInfoRemapped( - DefaultMQAdminExt admin, JsonObject conn, String groupId) throws Exception { - MasterBrokerAddrPlan plan = resolveMasterBrokerAddrPlan(admin, conn); - if (plan.isEmpty()) { - throw new MQClientException( - "No reachable RocketMQ master brokers for consumer connection of group " + groupId, null); - } - Exception lastError = null; - int remappedSuccess = 0; - int fallbackSuccess = 0; - ConsumerConnection emptySuccess = null; - for (String brokerAddr : plan.allAddrs()) { - boolean collisionFallback = plan.isCollisionFallback(brokerAddr); - try { - ConsumerConnection connection = admin.examineConsumerConnectionInfo(groupId, brokerAddr); - if (connection.getConnectionSet() != null && !connection.getConnectionSet().isEmpty()) { - return connection; - } - emptySuccess = connection != null ? connection : new ConsumerConnection(); - if (collisionFallback) { - fallbackSuccess++; - } else { - remappedSuccess++; - } - } catch (Exception e) { - // 206 = group has no live clients on that broker — count as probed offline. - if (isConsumerGroupNotOnline(e)) { - if (emptySuccess == null) { - emptySuccess = new ConsumerConnection(); - } - if (collisionFallback) { - fallbackSuccess++; - } else { - remappedSuccess++; - } - continue; - } - lastError = e; - } - } - return requireConsumerConnectionProbeResult( - plan.remappedCount(), - remappedSuccess, - plan.fallbackCount(), - fallbackSuccess, - emptySuccess, - lastError, - groupId - ); - } - - /** - * RocketMQ broker/client code 206: consumer group has no online clients (definitive offline). - */ - static boolean isConsumerGroupNotOnline(Throwable error) { - Throwable cursor = error; - while (cursor != null) { - if (cursor instanceof MQBrokerException brokerEx && brokerEx.getResponseCode() == 206) { - return true; - } - if (cursor instanceof MQClientException clientEx && clientEx.getResponseCode() == 206) { - return true; - } - cursor = cursor.getCause(); - } - return false; - } - - /** - * Accept empty/206 offline only when every remapped master (and every collision fallback) - * answered. Partial coverage must throw so enrich omits memberCount instead of fake 0. - */ - static ConsumerConnection requireConsumerConnectionProbeResult( - int remappedCount, - int remappedSuccess, - int fallbackCount, - int fallbackSuccess, - ConsumerConnection emptySuccess, - Exception lastError, - String groupId - ) throws MQClientException { - if (remappedCount <= 0 && fallbackCount <= 0) { - throw new MQClientException( - "No reachable RocketMQ master brokers for consumer connection of group " + groupId, null); - } - if (remappedSuccess <= 0 && fallbackSuccess <= 0) { - throw new MQClientException( - "Failed to examine consumer connection for group " + groupId + " on all masters", - lastError); - } - if (remappedCount > 0 && remappedSuccess < remappedCount) { - throw new MQClientException( - "Failed to examine consumer connection for group " + groupId - + " on some masters (partial remapped probe)", - lastError); - } - // Docker remap collisions: remapped host:port 206/empty is not enough — sibling masters - // may still have online consumers on unreachable internal addresses. - if (shouldFailClosedOnCollisionPartialMutation(fallbackCount, fallbackSuccess)) { - throw new MQClientException( - "Failed to examine consumer connection for group " + groupId - + " after Docker remap address collision (unreachable sibling masters)", - lastError); - } - return emptySuccess != null ? emptySuccess : new ConsumerConnection(); - } - - private static ConsumeStats examineConsumeStatsRemapped( - DefaultMQAdminExt admin, JsonObject conn, String groupId, String topic) throws Exception { - return examineConsumeStatsOnMasters(admin, groupId, topic, resolveMasterBrokerAddrPlan(admin, conn)); - } - - @FunctionalInterface - interface ConsumeStatsBrokerProbe { - ConsumeStats examine(String brokerAddr, String groupId, String topic, long timeout) throws Exception; - } - - /** - * Probe consume stats on a pre-resolved master plan. Callers that fan out per group - * (lag attach) should resolve the plan once so each task does not re-hit clusterInfo. - */ - private static ConsumeStats examineConsumeStatsOnMasters( - DefaultMQAdminExt admin, String groupId, String topic, MasterBrokerAddrPlan plan) throws Exception { - // RocketMQ 5.3.1: 3-arg examineConsumeStats(clusterName, group, topic) is NOT broker-scoped. - // Must use 4-arg (brokerAddr, group, topic, timeout) or remapped Docker addrs never hit brokers. - long timeout = adminTimeoutMillis(admin); - long fallbackTimeout = Math.min(timeout, COLLISION_FALLBACK_PROBE_TIMEOUT_MS); - return examineConsumeStatsOnMasters( - groupId, - topic, - plan, - timeout, - fallbackTimeout, - (brokerAddr, g, t, to) -> admin.examineConsumeStats(brokerAddr, g, t, to) - ); - } - - /** - * Two-phase probe: remapped masters first; collision-fallbacks only when the remapped merge - * is empty. Fail-closed ({@link #shouldFailClosedOnCollisionEmpty}) only guards empty merge, - * so non-empty remapped success must not spend adminTimeout on unreachable Docker IPs. - * Fallback remoting stays short — same host-side Docker tradeoff as subscription collect. - */ - static ConsumeStats examineConsumeStatsOnMasters( - String groupId, - String topic, - MasterBrokerAddrPlan plan, - long remappedTimeout, - long fallbackTimeout, - ConsumeStatsBrokerProbe probe - ) throws Exception { - ConsumeStats merged = new ConsumeStats(); - int remappedSuccess = 0; - int fallbackSuccess = 0; - Exception lastError = null; - for (String brokerAddr : plan.remappedAddrs()) { - try { - ConsumeStats stats = probe.examine(brokerAddr, groupId, topic, remappedTimeout); - remappedSuccess++; - if (stats != null && stats.getOffsetTable() != null) { - merged.getOffsetTable().putAll(stats.getOffsetTable()); - merged.setConsumeTps(merged.getConsumeTps() + stats.getConsumeTps()); - } - } catch (Exception e) { - lastError = e; - } - } - boolean mergedEmpty = merged.getOffsetTable() == null || merged.getOffsetTable().isEmpty(); - // Remap collisions append Docker-internal originals as best-effort fallbacks. Host-side - // agents usually cannot reach them; skip those probes once remapped masters already - // returned offsets. Empty merge + unreachable collision siblings is still fail-closed: - // a single published host:port can answer for the wrong master and look like zero lag. - if (mergedEmpty) { - for (String brokerAddr : plan.collisionFallbackAddrs()) { - try { - ConsumeStats stats = probe.examine(brokerAddr, groupId, topic, fallbackTimeout); - fallbackSuccess++; - if (stats != null && stats.getOffsetTable() != null) { - merged.getOffsetTable().putAll(stats.getOffsetTable()); - merged.setConsumeTps(merged.getConsumeTps() + stats.getConsumeTps()); - } - } catch (Exception e) { - lastError = e; - } - } - mergedEmpty = merged.getOffsetTable() == null || merged.getOffsetTable().isEmpty(); - } - if (shouldFailClosedOnCollisionEmpty(mergedEmpty, plan.fallbackCount(), fallbackSuccess)) { - throw new MQClientException( - "Failed to examine consume stats for group " + groupId - + " after Docker remap address collision (unreachable sibling masters)", - lastError); - } - int attemptedForGate; - int successForGate; - if (remappedSuccess > 0) { - attemptedForGate = plan.remappedCount(); - successForGate = remappedSuccess; - } else if (fallbackSuccess > 0) { - attemptedForGate = plan.fallbackCount(); - successForGate = fallbackSuccess; - } else { - attemptedForGate = plan.size(); - successForGate = 0; - } - ensureConsumeStatsProbeSucceeded(attemptedForGate, successForGate, mergedEmpty, lastError, groupId); - return merged; - } - - /** - * Host-side Docker remap collisions collapse N masters onto one published address. An empty - * merge is only trustworthy when every collision-fallback original was also reachable (or - * there were no collisions). - */ - static boolean shouldFailClosedOnCollisionEmpty( - boolean mergedEmpty, int fallbackCount, int fallbackSuccess) { - return mergedEmpty && fallbackCount > 0 && fallbackSuccess < fallbackCount; - } - - /** - * Fail closed when no master answered, or when some masters failed and the merge is empty - * (partial outage otherwise looks like healthy zero lag). - */ - static void ensureConsumeStatsProbeSucceeded( - int attemptedBrokers, - int successCount, - boolean mergedEmpty, - Exception lastError, - String groupId - ) throws MQClientException { - if (attemptedBrokers <= 0) { - throw new MQClientException( - "No reachable RocketMQ master brokers for consume stats of group " + groupId, null); - } - if (successCount <= 0) { - throw new MQClientException( - "Failed to examine consume stats for group " + groupId + " on all masters", - lastError); - } - if (mergedEmpty && successCount < attemptedBrokers) { - throw new MQClientException( - "Failed to examine consume stats for group " + groupId - + " on some masters (empty merge after partial failure)", - lastError); - } - } - - /** - * RocketMQ admin {@code queryTopicConsumeByWho(topic)} contacts the first broker using - * NameServer-registered addresses (often Docker-internal IPs). Remap each broker before - * querying, matching Dashboard {@code queryTopicConsumerInfo}. - */ - private static Set queryTopicConsumeByWhoRemapped( - DefaultMQAdminExt admin, JsonObject conn, String topic) { - Set groups = new TreeSet<>(); - try { - TopicRouteData route = admin.examineTopicRouteInfo(topic); - if (route != null && route.getBrokerDatas() != null) { - for (BrokerData brokerData : route.getBrokerDatas()) { - String rawAddr = brokerData.selectBrokerAddr(); - if (rawAddr == null || rawAddr.isBlank()) { - continue; - } - String brokerAddr = remapBrokerAddrForClient(rawAddr, conn); - try { - GroupList groupList = queryTopicConsumeByWhoOnBroker(admin, brokerAddr, topic); - if (groupList != null && groupList.getGroupList() != null) { - groups.addAll(groupList.getGroupList()); - } - } catch (Exception ignored) { - // Try next broker. - } - } - } - } catch (Exception ignored) { - // Fall through to aggregate admin call below. - } - if (groups.isEmpty()) { - try { - GroupList groupList = admin.queryTopicConsumeByWho(topic); - if (groupList != null && groupList.getGroupList() != null) { - groups.addAll(groupList.getGroupList()); - } - } catch (Exception ignored) { - // Topic may have no registered consumer groups yet. - } - } - return groups; - } - - /** Broker-scoped query; DefaultMQAdminExt only exposes topic-level aggregation. */ - private static GroupList queryTopicConsumeByWhoOnBroker( - DefaultMQAdminExt admin, String brokerAddr, String topic) throws Exception { - try { - Object api = mqClientApiImpl(admin); - long timeout = adminTimeoutMillis(admin); - var queryMethod = api.getClass().getMethod( - "queryTopicConsumeByWho", String.class, String.class, long.class); - return (GroupList) queryMethod.invoke(api, brokerAddr, topic, timeout); - } catch (ReflectiveOperationException e) { - throw new MQClientException("Failed to query topic consumers on broker " + brokerAddr, e); - } - } - - /** Topic config dialog exposes the same editable fields as rocketmq-dashboard. */ - private static final Set EDITABLE_TOPIC_CONFIG_KEYS = Set.of( - "readQueueNums", "writeQueueNums", "perm" - ); - - private static Object getTopicConfig(JsonObject params) throws Exception { - DefaultMQAdminExt admin = requireAdmin(); - String name = requireString(params, "name"); - String brokerAddr = brokerAddr(params, admin); - TopicConfig config = loadTopicConfig(admin, brokerAddr, name); - - Map configs = new LinkedHashMap<>(); - putConfigEntry(configs, "readQueueNums", String.valueOf(config.getReadQueueNums())); - putConfigEntry(configs, "writeQueueNums", String.valueOf(config.getWriteQueueNums())); - putConfigEntry(configs, "perm", String.valueOf(config.getPerm())); - return Collections.singletonMap("configs", configs); - } - - private static Object alterTopicConfig(JsonObject params) throws Exception { - DefaultMQAdminExt admin = requireAdmin(); - String name = requireString(params, "name"); - String brokerAddr = brokerAddr(params, admin); - TopicConfig config = loadTopicConfig(admin, brokerAddr, name); - - JsonArray entries = params.has("configs") && params.get("configs").isJsonArray() - ? params.getAsJsonArray("configs") : new JsonArray(); - for (JsonElement element : entries) { - JsonObject entry = element.getAsJsonObject(); - String key = entry.get("key").getAsString(); - String op = stringOrDefault(entry, "op", "set"); - if ("delete".equalsIgnoreCase(op)) { - if (config.getAttributes() != null) { - config.getAttributes().remove(key); - } - continue; - } - String value = entry.has("value") && !entry.get("value").isJsonNull() - ? entry.get("value").getAsString() : null; - if (!EDITABLE_TOPIC_CONFIG_KEYS.contains(key)) { - continue; - } - applyTopicConfigValue(config, key, value); - } - - admin.createAndUpdateTopicConfig(brokerAddr, config); - return Collections.singletonMap("ok", true); - } - - private static Object listConsumerGroups(JsonObject params) throws Exception { - DefaultMQAdminExt admin = requireAdmin(); - JsonObject conn = connectionObject(params); - String topicFilter = stringOrEmpty(params, "topic"); - String keyword = stringOrEmpty(params, "keyword").toLowerCase(Locale.ROOT); - int offset = Math.max(0, intOrDefault(params, "offset", 0)); - int limit = intOrDefault(params, "limit", DEFAULT_LIST_LIMIT); - if (limit <= 0) { - limit = DEFAULT_LIST_LIMIT; - } - boolean includeLag = boolOrDefault(params, "includeLag", false); - - Map configs; - Set groups = new TreeSet<>(); - if (!topicFilter.isBlank()) { - // Dashboard-style: discover groups via topic; skip full-cluster subscription dump. - groups.addAll(queryTopicConsumeByWhoRemapped(admin, conn, topicFilter)); - // Merge every reachable master dump so FIFO typing is not stuck on the first broker. - configs = collectConsumerGroupConfigsForTopicFilter(admin, conn); - } else { - configs = collectConsumerGroupConfigs(admin, conn); - groups.addAll(configs.keySet()); - } - - List> rows = new ArrayList<>(); - for (String groupId : groups) { - if (!keyword.isBlank() && !groupId.toLowerCase(Locale.ROOT).contains(keyword)) { - continue; - } - Map row = new LinkedHashMap<>(); - row.put("groupId", groupId); - row.put("state", "UNKNOWN"); - row.put("simpleGroup", false); - // Placeholder until config classify runs; missing dump → UNKNOWN, not NORMAL. - row.put("groupType", "UNKNOWN"); - row.put("messageModel", "CLUSTERING"); - rows.add(row); - } - - int total = rows.size(); - List> page = paginate(rows, offset, limit); - for (Map row : page) { - String groupId = String.valueOf(row.get("groupId")); - SubscriptionGroupConfig config = configs.get(groupId); - row.put("groupType", classifyConsumerGroupType(groupId, config)); - } - if (boolOrDefault(params, "enrich", false)) { - enrichConsumerGroupRows(admin, conn, page); - } - if (includeLag && !topicFilter.isBlank()) { - attachConsumerLags(admin, conn, page, topicFilter); - } - - Map result = new LinkedHashMap<>(); - result.put("groups", page); - result.put("total", total); - result.put("offset", offset); - result.put("limit", limit); - return result; - } - - /** Concurrent lag lookup for the current page; budget-capped like enrich. */ - private static void attachConsumerLags( - DefaultMQAdminExt admin, - JsonObject conn, - List> page, - String topic - ) { - if (page.isEmpty() || topic == null || topic.isBlank()) { - return; - } - final MasterBrokerAddrPlan masterPlan; - try { - // Resolve/remap masters once — per-group examineBrokerClusterInfo burns the lag budget. - masterPlan = resolveMasterBrokerAddrPlan(admin, conn); - } catch (Exception e) { - for (Map row : page) { - row.put("totalLagFailed", true); - } - return; - } - int workers = Math.max(1, Math.min(CONSUMER_LAG_CONCURRENCY, page.size())); - ExecutorService executor = Executors.newFixedThreadPool(workers, runnable -> { - Thread thread = new Thread(runnable, "dbx-rocketmq-consumer-lag"); - thread.setDaemon(true); - return thread; - }); - // Nullable Long: omit totalLag on failure; mark totalLagFailed so list UI does not show 0. - List> tasks = new ArrayList<>(page.size()); - for (Map row : page) { - String groupId = String.valueOf(row.get("groupId")); - tasks.add(() -> { - // examineConsumeStatsOnMasters throws when no master succeeds. - ConsumeStats stats = examineConsumeStatsOnMasters(admin, groupId, topic, masterPlan); - long totalLag = 0; - if (stats.getOffsetTable() != null) { - for (var entry : stats.getOffsetTable().entrySet()) { - totalLag += Math.max(0, entry.getValue().getBrokerOffset() - entry.getValue().getConsumerOffset()); - } - } - return totalLag; - }); - } - try { - List> results = - executor.invokeAll( - tasks, - Math.max(1, scaledOperationBudgetMs(conn, CONSUMER_LAG_BUDGET_BASELINE_MS)), - TimeUnit.MILLISECONDS); - for (int i = 0; i < page.size(); i++) { - Future future = i < results.size() ? results.get(i) : null; - if (future == null || future.isCancelled()) { - page.get(i).put("totalLagFailed", true); - continue; - } - try { - Long lag = future.get(); - if (lag != null) { - page.get(i).put("totalLag", lag); - } else { - page.get(i).put("totalLagFailed", true); - } - } catch (Exception ignored) { - page.get(i).put("totalLagFailed", true); - } - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - for (Map row : page) { - if (!row.containsKey("totalLag")) { - row.put("totalLagFailed", true); - } - } - } finally { - executor.shutdownNow(); - } - } - - @FunctionalInterface - interface ConsumerConnectionLookup { - ConsumerConnection load(String groupId) throws Exception; - } - - private static void enrichConsumerGroupRows( - DefaultMQAdminExt admin, JsonObject conn, List> page) { - enrichConsumerGroupRows( - page, - groupId -> examineConsumerConnectionInfoRemapped(admin, conn, groupId), - CONSUMER_GROUP_ENRICH_CONCURRENCY, - scaledOperationBudgetMs(conn, CONSUMER_GROUP_ENRICH_BUDGET_BASELINE_MS) - ); - } - - static void enrichConsumerGroupRows( - List> page, - ConsumerConnectionLookup lookup, - int concurrency, - long budgetMs) { - if (page.isEmpty()) { - return; - } - int workers = Math.max(1, Math.min(concurrency, page.size())); - ExecutorService executor = Executors.newFixedThreadPool(workers, runnable -> { - Thread thread = new Thread(runnable, "dbx-rocketmq-consumer-enrich"); - thread.setDaemon(true); - return thread; - }); - List>> tasks = new ArrayList<>(page.size()); - for (Map row : page) { - tasks.add(() -> consumerGroupEnrichment(String.valueOf(row.get("groupId")), lookup)); - } - List>> results = Collections.emptyList(); - try { - results = executor.invokeAll(tasks, Math.max(1, budgetMs), TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } finally { - executor.shutdownNow(); - } - for (int index = 0; index < page.size(); index++) { - Map row = page.get(index); - if (index < results.size() && !results.get(index).isCancelled()) { - try { - row.putAll(results.get(index).get()); - } catch (Exception ignored) { - // Keep the base group row when enrichment fails. - } - } - // Omit memberCount on cancel/timeout/failure so UI shows '-' instead of fake offline 0. - row.putIfAbsent("topics", Collections.emptyList()); - } - } - - private static Map consumerGroupEnrichment( - String groupId, ConsumerConnectionLookup lookup) { - Map enrichment = new LinkedHashMap<>(); - try { - ConsumerConnection connection = lookup.load(groupId); - enrichment.put("consumeType", connection.getConsumeType() != null ? connection.getConsumeType().name() : "UNKNOWN"); - if (connection.getMessageModel() != null) { - enrichment.put("messageModel", connection.getMessageModel().name()); - } - int memberCount = connection.getConnectionSet() == null ? 0 : connection.getConnectionSet().size(); - enrichment.put("memberCount", memberCount); - List topics = new ArrayList<>(); - if (connection.getSubscriptionTable() != null) { - for (SubscriptionData sub : connection.getSubscriptionTable().values()) { - if (sub.getTopic() != null && !sub.getTopic().isBlank()) { - topics.add(sub.getTopic()); - } - } - } - enrichment.put("topics", topics); - } catch (Exception ignored) { - // Leave memberCount unset — probe did not finish; list UI must not show 0. - enrichment.put("topics", Collections.emptyList()); - } - return enrichment; - } - - private static Object describeConsumerGroup(JsonObject params) throws Exception { - DefaultMQAdminExt admin = requireAdmin(); - JsonObject conn = connectionObject(params); - String groupId = requireString(params, "groupId"); - ConsumerConnection connection = examineConsumerConnectionInfoRemapped(admin, conn, groupId); - - List> sharedAssignments = new ArrayList<>(); - if (connection.getSubscriptionTable() != null) { - for (SubscriptionData sub : connection.getSubscriptionTable().values()) { - Map assignment = new LinkedHashMap<>(); - assignment.put("topic", sub.getTopic()); - assignment.put("subExpression", sub.getSubString()); - sharedAssignments.add(assignment); - } - } - - List> members = new ArrayList<>(); - if (connection.getConnectionSet() != null) { - for (Connection clientConn : connection.getConnectionSet()) { - Map member = new LinkedHashMap<>(); - member.put("memberId", clientConn.getClientId()); - member.put("clientId", clientConn.getClientId()); - member.put("host", clientConn.getClientAddr()); - member.put("assignments", sharedAssignments); - members.add(member); - } - } - - Map result = new LinkedHashMap<>(); - result.put("groupId", groupId); - result.put("state", connection.getConsumeType() != null ? connection.getConsumeType().name() : "UNKNOWN"); - result.put("partitionAssignor", connection.getMessageModel() != null - ? connection.getMessageModel().name() : null); - result.put("messageModel", connection.getMessageModel() != null - ? connection.getMessageModel().name() : null); - result.put("members", members); - return result; - } - - private static Object deleteConsumerGroup(JsonObject params) throws Exception { - DefaultMQAdminExt admin = requireAdmin(); - JsonObject conn = connectionObject(params); - String groupId = requireString(params, "groupId"); - // Dashboard deletes per broker; wipe every reachable master so the group does not reappear. - mutateSubscriptionGroupOnMasters( - admin, - conn, - groupId, - "delete", - brokerAddr -> admin.deleteSubscriptionGroup(brokerAddr, groupId) - ); - return Collections.singletonMap("ok", true); - } - - private static Object getSubscriptionGroupConfig(JsonObject params) throws Exception { - DefaultMQAdminExt admin = requireAdmin(); - JsonObject conn = connectionObject(params); - String groupId = requireString(params, "groupId"); - SubscriptionGroupConfig config = findSubscriptionGroupConfig(admin, conn, groupId); - if (config == null) { - throw new IllegalArgumentException("Consumer group not found: " + groupId); - } - return subscriptionGroupConfigToMap(config); - } - - private static Object alterSubscriptionGroupConfig(JsonObject params) throws Exception { - DefaultMQAdminExt admin = requireAdmin(); - JsonObject conn = connectionObject(params); - String groupId = requireString(params, "groupId"); - SubscriptionGroupConfig config = findSubscriptionGroupConfig(admin, conn, groupId); - if (config == null) { - config = new SubscriptionGroupConfig(); - config.setGroupName(groupId); - } - applySubscriptionGroupConfigUpdates(config, params); - SubscriptionGroupConfig toWrite = config; - mutateSubscriptionGroupOnMasters( - admin, - conn, - groupId, - "update", - brokerAddr -> admin.createAndUpdateSubscriptionGroupConfig(brokerAddr, toWrite) - ); - return Collections.singletonMap("ok", true); - } - - @FunctionalInterface - interface BrokerSubscriptionMutation { - void apply(String brokerAddr) throws Exception; - } - - /** - * Apply a subscription-group mutation on every remapped master (and collision fallbacks). - * Require full coverage of remapped addrs; Docker remap collisions must also succeed on - * fallback originals — a single published host:port hit must not report ok while sibling - * masters still hold the old group/config. - */ - static void mutateSubscriptionGroupOnMasters( - DefaultMQAdminExt admin, - JsonObject conn, - String groupId, - String action, - BrokerSubscriptionMutation mutation - ) throws Exception { - MasterBrokerAddrPlan plan = resolveMasterBrokerAddrPlan(admin, conn); - int remappedSuccess = 0; - int fallbackSuccess = 0; - Exception lastError = null; - for (String brokerAddr : plan.allAddrs()) { - try { - mutation.apply(brokerAddr); - if (plan.isCollisionFallback(brokerAddr)) { - fallbackSuccess++; - } else { - remappedSuccess++; - } - } catch (Exception e) { - lastError = e; - } - } - ensureSubscriptionGroupMutationSucceeded( - plan.remappedCount(), - remappedSuccess, - plan.fallbackCount(), - fallbackSuccess, - lastError, - action, - groupId - ); - } - - /** - * Gate subscription-group mutate/delete. Partial remapped updates and Docker collision - * fallbacks that never answered must fail closed (group would otherwise reappear / diverge). - */ - static void ensureSubscriptionGroupMutationSucceeded( - int remappedCount, - int remappedSuccess, - int fallbackCount, - int fallbackSuccess, - Exception lastError, - String action, - String groupId - ) throws MQClientException { - if (remappedCount <= 0 && fallbackCount <= 0) { - throw new MQClientException( - "No reachable RocketMQ master brokers for " + action + " of group " + groupId, null); - } - if (remappedSuccess <= 0 && fallbackSuccess <= 0) { - throw new MQClientException( - "Failed to " + action + " consumer group " + groupId + " on all masters", - lastError); - } - if (remappedCount > 0 && remappedSuccess < remappedCount) { - throw new MQClientException( - "Failed to " + action + " consumer group " + groupId - + " on some masters (partial remapped update)", - lastError); - } - // Remap collisions collapse N masters onto one published addr. Updating that addr once - // leaves sibling masters untouched unless collision-fallback originals also succeed. - if (shouldFailClosedOnCollisionPartialMutation(fallbackCount, fallbackSuccess)) { - throw new MQClientException( - "Failed to " + action + " consumer group " + groupId - + " after Docker remap address collision (unreachable sibling masters)", - lastError); - } - } - - /** True when collision fallbacks exist but were not all successfully mutated. */ - static boolean shouldFailClosedOnCollisionPartialMutation(int fallbackCount, int fallbackSuccess) { - return fallbackCount > 0 && fallbackSuccess < fallbackCount; - } - - private static Map subscriptionGroupConfigToMap(SubscriptionGroupConfig config) { - Map map = new LinkedHashMap<>(); - map.put("groupName", config.getGroupName()); - map.put("consumeEnable", config.isConsumeEnable()); - map.put("consumeFromMinEnable", config.isConsumeFromMinEnable()); - map.put("consumeBroadcastEnable", config.isConsumeBroadcastEnable()); - map.put("consumeMessageOrderly", config.isConsumeMessageOrderly()); - map.put("retryQueueNums", config.getRetryQueueNums()); - map.put("retryMaxTimes", config.getRetryMaxTimes()); - map.put("brokerId", config.getBrokerId()); - map.put("whichBrokerWhenConsumeSlowly", config.getWhichBrokerWhenConsumeSlowly()); - return map; - } - - private static void applySubscriptionGroupConfigUpdates(SubscriptionGroupConfig config, JsonObject params) { - if (params.has("consumeEnable") && !params.get("consumeEnable").isJsonNull()) { - config.setConsumeEnable(params.get("consumeEnable").getAsBoolean()); - } - if (params.has("consumeFromMinEnable") && !params.get("consumeFromMinEnable").isJsonNull()) { - config.setConsumeFromMinEnable(params.get("consumeFromMinEnable").getAsBoolean()); - } - if (params.has("consumeBroadcastEnable") && !params.get("consumeBroadcastEnable").isJsonNull()) { - config.setConsumeBroadcastEnable(params.get("consumeBroadcastEnable").getAsBoolean()); - } - if (params.has("consumeMessageOrderly") && !params.get("consumeMessageOrderly").isJsonNull()) { - config.setConsumeMessageOrderly(params.get("consumeMessageOrderly").getAsBoolean()); - } - if (params.has("retryQueueNums") && !params.get("retryQueueNums").isJsonNull()) { - config.setRetryQueueNums(params.get("retryQueueNums").getAsInt()); - } - if (params.has("retryMaxTimes") && !params.get("retryMaxTimes").isJsonNull()) { - config.setRetryMaxTimes(params.get("retryMaxTimes").getAsInt()); - } - if (params.has("brokerId") && !params.get("brokerId").isJsonNull()) { - config.setBrokerId(params.get("brokerId").getAsLong()); - } - if (params.has("whichBrokerWhenConsumeSlowly") && !params.get("whichBrokerWhenConsumeSlowly").isJsonNull()) { - config.setWhichBrokerWhenConsumeSlowly(params.get("whichBrokerWhenConsumeSlowly").getAsLong()); - } - } - - private static Object resetConsumerGroupOffsets(JsonObject params) throws Exception { - DefaultMQAdminExt admin = requireAdmin(); - String groupId = requireString(params, "groupId"); - String topic = requireString(params, "topic"); - String brokerAddr = brokerAddr(params, admin); - - JsonArray offsetArray = params.has("offsets") && params.get("offsets").isJsonArray() - ? params.getAsJsonArray("offsets") : new JsonArray(); - if (!offsetArray.isEmpty()) { - for (JsonElement element : offsetArray) { - JsonObject offsetObj = element.getAsJsonObject(); - int partition = offsetObj.get("partition").getAsInt(); - long offset = offsetObj.get("offset").getAsLong(); - admin.resetOffsetByQueueId(brokerAddr, groupId, topic, partition, offset); - } - return Collections.singletonMap("ok", true); - } - - String position = stringOrDefault(params, "position", "latest"); - long timestamp = switch (position.toLowerCase(Locale.ROOT)) { - case "earliest" -> 0L; - case "latest" -> System.currentTimeMillis(); - case "timestamp" -> longOrDefault(params, "timestampMs", System.currentTimeMillis()); - default -> throw new IllegalArgumentException("Unsupported reset position: " + position); - }; - admin.resetOffsetByTimestamp(topic, groupId, timestamp, true); - return Collections.singletonMap("ok", true); - } - - private static Object getConsumerLag(JsonObject params) throws Exception { - DefaultMQAdminExt admin = requireAdmin(); - JsonObject conn = connectionObject(params); - String groupId = requireString(params, "groupId"); - String topic = requireString(params, "topic"); - ConsumeStats stats = examineConsumeStatsRemapped(admin, conn, groupId, topic); - // Best-effort queue→client mapping (Dashboard getClientConnection); failures must not hide offsets. - Map queueClientMap = resolveQueueClientMap(admin, conn, groupId); - return buildConsumerLagResult(stats, queueClientMap); - } - - /** - * Build lag payload from ConsumeStats. Keeps Kafka-compatible field names - * ({@code currentOffset}/{@code endOffset}/{@code lag}) and adds Dashboard fields - * ({@code brokerName}/{@code lastTimestamp}/{@code consumerClient}). - */ - static Map buildConsumerLagResult( - ConsumeStats stats, Map queueClientMap) { - long totalLag = 0; - List> partitions = new ArrayList<>(); - if (stats != null && stats.getOffsetTable() != null) { - for (var entry : stats.getOffsetTable().entrySet()) { - MessageQueue mq = entry.getKey(); - OffsetWrapper offset = entry.getValue(); - long consumerOffset = offset.getConsumerOffset(); - long brokerOffset = offset.getBrokerOffset(); - long lag = Math.max(0, brokerOffset - consumerOffset); - totalLag += lag; - - Map partition = new LinkedHashMap<>(); - partition.put("partition", mq.getQueueId()); - partition.put("currentOffset", consumerOffset); - partition.put("endOffset", brokerOffset); - partition.put("lag", lag); - // Dashboard-compatible consume-detail columns. - partition.put("brokerName", mq.getBrokerName() != null ? mq.getBrokerName() : ""); - partition.put("lastTimestamp", offset.getLastTimestamp()); - String clientId = queueClientMap != null ? queueClientMap.get(mq) : null; - partition.put("consumerClient", clientId != null ? clientId : ""); - partitions.add(partition); - } - } - // Sort by broker then queue so multi-broker topics stay readable. - partitions.sort(Comparator - .comparing((Map a) -> String.valueOf(a.get("brokerName"))) - .thenComparingInt(a -> (int) a.get("partition"))); - - Map result = new LinkedHashMap<>(); - result.put("partitions", partitions); - result.put("totalLag", totalLag); - return result; - } - - /** - * Map MessageQueue → online consumer clientId via ConsumerRunningInfo.mqTable. - * Mirrors RocketMQ Dashboard ConsumerServiceImpl#getClientConnection; swallows errors. - */ - static Map resolveQueueClientMap( - DefaultMQAdminExt admin, JsonObject conn, String groupId) { - Map results = new HashMap<>(); - try { - ConsumerConnection connection = examineConsumerConnectionInfoRemapped(admin, conn, groupId); - if (connection.getConnectionSet() == null || connection.getConnectionSet().isEmpty()) { - return results; - } - for (Connection clientConn : connection.getConnectionSet()) { - String clientId = clientConn.getClientId(); - if (clientId == null || clientId.isBlank()) { - continue; - } - try { - // DefaultMQAdminExt picks the first NameServer-registered broker (often Docker-internal). - ConsumerRunningInfo runningInfo = - getConsumerRunningInfoRemapped(admin, conn, groupId, clientId, false); - if (runningInfo == null || runningInfo.getMqTable() == null) { - continue; - } - for (MessageQueue mq : runningInfo.getMqTable().keySet()) { - results.put(mq, clientId); - } - } catch (Exception ignored) { - // Offline/unreachable client: keep offsets without client assignment. - } - } - } catch (Exception ignored) { - // Connection lookup failed; return empty map. - } - return results; - } - - /** - * Like {@link DefaultMQAdminExt#getConsumerRunningInfo}, but remaps broker addresses for - * Docker/host-published ports before invoking {@code MQClientAPIImpl}. - */ - static ConsumerRunningInfo getConsumerRunningInfoRemapped( - DefaultMQAdminExt admin, - JsonObject conn, - String groupId, - String clientId, - boolean jstack - ) { - // Upstream walks %RETRY%{group} route and contacts the first broker addr as registered. - try { - String retryTopic = MixAll.RETRY_GROUP_TOPIC_PREFIX + groupId; - TopicRouteData route = admin.examineTopicRouteInfo(retryTopic); - if (route != null && route.getBrokerDatas() != null) { - for (BrokerData brokerData : route.getBrokerDatas()) { - String rawAddr = brokerData.selectBrokerAddr(); - if (rawAddr == null || rawAddr.isBlank()) { - continue; - } - String brokerAddr = remapBrokerAddrForClient(rawAddr, conn); - try { - ConsumerRunningInfo info = - getConsumerRunningInfoOnBroker(admin, brokerAddr, groupId, clientId, jstack); - if (info != null) { - return info; - } - } catch (Exception ignored) { - // Try next remapped broker. - } - } - } - } catch (Exception ignored) { - // Fall through to master-broker scan. - } - try { - for (String brokerAddr : resolveMasterBrokerAddrs(admin, conn)) { - try { - ConsumerRunningInfo info = - getConsumerRunningInfoOnBroker(admin, brokerAddr, groupId, clientId, jstack); - if (info != null) { - return info; - } - } catch (Exception ignored) { - // Try next master. - } - } - } catch (Exception ignored) { - // No reachable masters. - } - return null; - } - - /** Broker-scoped ConsumerRunningInfo; DefaultMQAdminExt has no remapped overload. */ - private static ConsumerRunningInfo getConsumerRunningInfoOnBroker( - DefaultMQAdminExt admin, - String brokerAddr, - String groupId, - String clientId, - boolean jstack - ) throws Exception { - try { - Object api = mqClientApiImpl(admin); - long timeout = adminTimeoutMillis(admin); - var method = api.getClass().getMethod( - "getConsumerRunningInfo", - String.class, - String.class, - String.class, - boolean.class, - long.class - ); - return (ConsumerRunningInfo) method.invoke(api, brokerAddr, groupId, clientId, jstack, timeout); - } catch (ReflectiveOperationException e) { - throw new MQClientException( - "Failed to get consumer running info on broker " + brokerAddr, e); - } - } - - private static Object mqClientApiImpl(DefaultMQAdminExt admin) throws ReflectiveOperationException { - var implField = DefaultMQAdminExt.class.getDeclaredField("defaultMQAdminExtImpl"); - implField.setAccessible(true); - Object impl = implField.get(admin); - var mqClientMethod = impl.getClass().getMethod("getMqClientInstance"); - Object mqClient = mqClientMethod.invoke(impl); - var apiMethod = mqClient.getClass().getMethod("getMQClientAPIImpl"); - return apiMethod.invoke(mqClient); - } - - private static long adminTimeoutMillis(DefaultMQAdminExt admin) throws ReflectiveOperationException { - var implField = DefaultMQAdminExt.class.getDeclaredField("defaultMQAdminExtImpl"); - implField.setAccessible(true); - Object impl = implField.get(admin); - var timeoutField = impl.getClass().getDeclaredField("timeoutMillis"); - timeoutField.setAccessible(true); - return timeoutField.getLong(impl); - } - - private static Object listProducers(JsonObject params) throws Exception { - DefaultMQAdminExt admin = requireAdmin(); - JsonObject conn = connectionObject(params); - String topic = stringOrEmpty(params, "topic"); - String producerGroup = stringOrEmpty(params, "producerGroup"); - if (producerGroup.isBlank()) { - producerGroup = stringOrEmpty(params, "group"); - } - - if (!producerGroup.isBlank()) { - if (topic.isBlank()) { - throw new IllegalArgumentException("topic is required when producerGroup is specified"); - } - ProducerConnection connection = admin.examineProducerConnectionInfo(producerGroup, topic); - List> producers = new ArrayList<>(); - long producerId = 1; - if (connection.getConnectionSet() != null) { - for (Connection client : connection.getConnectionSet()) { - producers.add(producerRow(producerId++, producerGroup, client)); - } - } - return Collections.singletonMap("producers", producers); - } - - List brokerAddrs; - if (!topic.isBlank()) { - // Broker APIs cannot map producer connections to a topic (apache/rocketmq#6371). - // Gate on topic produce stats: topics with no queued messages have no producers to show. - TopicStatsTable stats = examineTopicStatsRemapped(admin, conn, topic); - if (!topicHasProduceActivity(stats)) { - return Collections.singletonMap("producers", Collections.emptyList()); - } - brokerAddrs = resolveTopicRouteBrokerAddrs(admin, conn, topic); - } else if (shouldQueryClusterProducerTable(topic, producerGroup)) { - brokerAddrs = List.of(brokerAddr(params, admin, conn)); - } else { - return Collections.singletonMap("producers", Collections.emptyList()); - } - - return Collections.singletonMap("producers", listProducersFromBrokers(admin, brokerAddrs)); - } - - /** - * Returns true when the topic has ever received messages on at least one queue. - * RocketMQ tracks this via {@code examineTopicStats}; empty topics should not inherit - * broker-wide producer tables from unrelated groups. - */ - static boolean topicHasProduceActivity(TopicStatsTable stats) { - if (stats == null || stats.getOffsetTable() == null || stats.getOffsetTable().isEmpty()) { - return false; - } - for (var entry : stats.getOffsetTable().entrySet()) { - if (entry.getValue().getMaxOffset() > entry.getValue().getMinOffset()) { - return true; - } - } - return false; - } - - static List resolveTopicRouteBrokerAddrs( - DefaultMQAdminExt admin, JsonObject conn, String topic) throws Exception { - TopicRouteData route = admin.examineTopicRouteInfo(topic); - LinkedHashSet addrs = new LinkedHashSet<>(); - if (route.getBrokerDatas() != null) { - for (BrokerData brokerData : route.getBrokerDatas()) { - String rawAddr = brokerData.selectBrokerAddr(); - if (rawAddr == null || rawAddr.isBlank()) { - continue; - } - addrs.add(remapBrokerAddrForClient(rawAddr, conn)); - } - } - return new ArrayList<>(addrs); - } - - static List> listProducersFromBrokers( - DefaultMQAdminExt admin, List brokerAddrs) { - List> producers = new ArrayList<>(); - Set seen = new LinkedHashSet<>(); - long producerId = 1; - for (String brokerAddr : brokerAddrs) { - try { - ProducerTableInfo tableInfo = admin.getAllProducerInfo(brokerAddr); - producerId = appendProducerTableRows(producers, seen, producerId, tableInfo); - } catch (Exception ignored) { - // Try next broker when Docker/internal broker addresses are unreachable. - } - } - return producers; - } - - static long appendProducerTableRows( - List> producers, - Set seen, - long producerId, - ProducerTableInfo tableInfo) { - if (tableInfo == null || tableInfo.getData() == null) { - return producerId; - } - for (Map.Entry> entry : tableInfo.getData().entrySet()) { - String group = entry.getKey(); - if (entry.getValue() == null) { - continue; - } - for (ProducerInfo info : entry.getValue()) { - String dedupeKey = group + "|" + info.getRemoteIP(); - if (!seen.add(dedupeKey)) { - continue; - } - producers.add(producerRow(producerId++, group, info)); - } - } - return producerId; - } - - private static Object peekMessages(JsonObject params) throws Exception { - JsonObject conn = connectionObject(params); - DefaultMQAdminExt admin = requireAdmin(); - String topic = requireString(params, "topic"); - int count = Math.max(1, Math.min(intOrDefault(params, "count", 10), 100)); - Integer partition = integerOrNull(params, "partition"); - Long offset = longOrNull(params, "offset"); - - List queues = resolvePeekQueues(admin, topic, partition); - if (queues.isEmpty()) { - return Collections.singletonMap("messages", Collections.emptyList()); - } - - DefaultLitePullConsumer consumer = buildLitePullConsumer(conn); - consumer.start(); - try { - List> messages = new ArrayList<>(); - for (MessageQueue queue : queues) { - if (messages.size() >= count) { - break; - } - long minOffset = admin.minOffset(queue); - long maxOffset = admin.maxOffset(queue); - if (maxOffset <= minOffset) { - continue; - } - long seekOffset = offset != null ? offset : minOffset; - if (seekOffset < minOffset) { - seekOffset = minOffset; - } - if (seekOffset >= maxOffset) { - continue; - } - - consumer.assign(Collections.singletonList(queue)); - consumer.seek(queue, seekOffset); - consumer.setPullBatchSize(count - messages.size()); - List polled = consumer.poll(MESSAGE_POLL_TIMEOUT_MS); - for (MessageExt message : polled) { - messages.add(peekedMessageFromRecord(topic, message)); - if (messages.size() >= count) { - break; - } - } - } - sortPeekedMessages(messages); - if (messages.size() > count) { - messages = new ArrayList<>(messages.subList(0, count)); - } - return Collections.singletonMap("messages", messages); - } finally { - consumer.shutdown(); - } - } - - private static Object sendMessage(JsonObject params) throws Exception { - DefaultMQProducer activeProducer = requireProducer(); - String topic = requireString(params, "topic"); - String key = params.has("key") && !params.get("key").isJsonNull() - ? params.get("key").getAsString() : null; - String tag = stringOrEmpty(params, "tag"); - if (tag.isBlank() && params.has("headers") && params.get("headers").isJsonObject()) { - tag = stringOrEmpty(params.getAsJsonObject("headers"), "TAGS"); - } - byte[] payload = decodePayload(params); - Integer partition = integerOrNull(params, "partition"); - - Message message = tag.isBlank() ? new Message(topic, payload) : new Message(topic, tag, payload); - if (key != null && !key.isBlank()) { - message.setKeys(key); - } else if (params.has("headers") && params.get("headers").isJsonObject()) { - String headerKey = stringOrEmpty(params.getAsJsonObject("headers"), "KEYS"); - if (!headerKey.isBlank()) { - message.setKeys(headerKey); - } - } - applySendHeaders(message, params); - if (partition != null) { - message.setWaitStoreMsgOK(true); - } - - SendResult result = activeProducer.send(message); - Map response = new LinkedHashMap<>(); - response.put("ok", true); - response.put("topic", topic); - response.put("partition", result.getMessageQueue().getQueueId()); - response.put("offset", result.getQueueOffset()); - response.put("timestamp", System.currentTimeMillis()); - return response; - } - - private static Object listAcls(JsonObject params) throws Exception { - DefaultMQAdminExt admin = requireAdmin(); - String brokerAddr = brokerAddr(params, admin); - String subjectFilter = stringOrNull(params, "principal"); - if (subjectFilter != null && subjectFilter.startsWith("User:")) { - subjectFilter = subjectFilter.substring("User:".length()); - } - if (subjectFilter == null) { - subjectFilter = stringOrEmpty(params, "subject"); - } - String resourceFilter = stringOrNull(params, "resourceName"); - - List> acls = new ArrayList<>(); - try { - List aclInfos = admin.listAcl( - brokerAddr, - subjectFilter == null ? "" : subjectFilter, - resourceFilter == null ? "" : resourceFilter - ); - if (aclInfos != null) { - for (AclInfo aclInfo : aclInfos) { - acls.addAll(flattenAclInfo(aclInfo)); - } - } - } catch (Exception acl2Error) { - // Fall back to empty list when ACL 2.0 APIs are unavailable. - if (acls.isEmpty()) { - throw acl2Error; - } - } - return Collections.singletonMap("acls", acls); - } - - private static Object createAcls(JsonObject params) throws Exception { - DefaultMQAdminExt admin = requireAdmin(); - String brokerAddr = brokerAddr(params, admin); - JsonArray aclsArray = params.has("acls") && params.get("acls").isJsonArray() - ? params.getAsJsonArray("acls") : new JsonArray(); - - for (JsonElement element : aclsArray) { - JsonObject acl = element.getAsJsonObject(); - if (acl.has("accessKey") || acl.has("secretKey")) { - PlainAccessConfig config = plainAccessConfigFromJson(acl); - admin.createAndUpdatePlainAccessConfig(brokerAddr, config); - continue; - } - - String subject = stringOrEmpty(acl, "principal"); - if (subject.startsWith("User:")) { - subject = subject.substring("User:".length()); - } - if (subject.isBlank()) { - subject = stringOrEmpty(acl, "subject"); - } - String resourceName = stringOrDefault(acl, "resourceName", "*"); - String operation = stringOrDefault(acl, "operation", "ALL"); - String permissionType = stringOrDefault(acl, "permissionType", "ALLOW"); - AclInfo aclInfo = AclInfo.of( - subject, - List.of(resourceName), - List.of(mapOperation(operation)), - List.of(stringOrDefault(acl, "host", "*")), - permissionType - ); - admin.createAcl(brokerAddr, aclInfo); - } - return Collections.singletonMap("ok", true); - } - - private static Object deleteAcls(JsonObject params) throws Exception { - DefaultMQAdminExt admin = requireAdmin(); - String brokerAddr = brokerAddr(params, admin); - JsonArray filtersArray = params.has("filters") && params.get("filters").isJsonArray() - ? params.getAsJsonArray("filters") : new JsonArray(); - - int deleted = 0; - if (filtersArray.isEmpty()) { - @SuppressWarnings("unchecked") - Map listed = (Map) listAcls(params); - List> existing = (List>) listed.get("acls"); - for (Map acl : existing) { - deleted += deleteAclEntry(admin, brokerAddr, acl); - } - } else { - for (JsonElement element : filtersArray) { - deleted += deleteAclEntry(admin, brokerAddr, jsonToMap(element.getAsJsonObject())); - } - } - - Map result = new LinkedHashMap<>(); - result.put("ok", true); - result.put("deleted", deleted); - return result; - } - - private static Object describeCluster(JsonObject params) throws Exception { - DefaultMQAdminExt admin = requireAdmin(); - ClusterInfo clusterInfo = admin.examineBrokerClusterInfo(); - String clusterName = clusterName(params, admin); - List> brokers = brokerNodes(clusterInfo); - - Map result = new LinkedHashMap<>(); - result.put("clusterId", clusterName); - result.put("controller", brokers.isEmpty() ? null : brokers.get(0)); - result.put("brokers", brokers); - result.put("nodeCount", brokers.size()); - return result; - } - - static DefaultMQAdminExt buildAdminClient(JsonObject conn) throws Exception { - long timeoutMs = intOrDefault(conn, "request_timeout_ms", DEFAULT_REQUEST_TIMEOUT_MS); - RPCHook rpcHook = buildRpcHook(conn); - // Always pass timeout — DefaultMQAdminExt() ignores request_timeout_ms. - DefaultMQAdminExt admin = rpcHook != null - ? new DefaultMQAdminExt(rpcHook, timeoutMs) - : new DefaultMQAdminExt(timeoutMs); - admin.setNamesrvAddr(namesrvAddr(conn)); - applySocksProxy(admin, conn); - admin.setAdminExtGroup("_DBX_ROCKETMQ_ADMIN_" + UUID.randomUUID()); - admin.setInstanceName("DBX_" + UUID.randomUUID()); - admin.start(); - return admin; - } - - static DefaultMQProducer buildProducer(JsonObject conn) throws Exception { - RPCHook rpcHook = buildRpcHook(conn); - DefaultMQProducer nextProducer = rpcHook != null - ? new DefaultMQProducer("_DBX_ROCKETMQ_PRODUCER", rpcHook) - : new DefaultMQProducer("_DBX_ROCKETMQ_PRODUCER"); - nextProducer.setNamesrvAddr(namesrvAddr(conn)); - applySocksProxy(nextProducer, conn); - nextProducer.setInstanceName("DBX_" + UUID.randomUUID()); - // Follow Advanced query timeout — SDK default sendMsgTimeout is only 3s. - nextProducer.setSendMsgTimeout(operationBudgetMsAsInt(conn)); - nextProducer.start(); - return nextProducer; - } - - static DefaultLitePullConsumer buildLitePullConsumer(JsonObject conn) throws Exception { - RPCHook rpcHook = buildRpcHook(conn); - DefaultLitePullConsumer consumer = rpcHook != null - ? new DefaultLitePullConsumer(rpcHook) - : new DefaultLitePullConsumer(); - consumer.setNamesrvAddr(namesrvAddr(conn)); - applySocksProxy(consumer, conn); - consumer.setConsumerGroup("_DBX_PEEK_" + UUID.randomUUID()); - consumer.setInstanceName("DBX_" + UUID.randomUUID()); - consumer.setAutoCommit(false); - return consumer; - } - - static DefaultMQPullConsumer buildPullConsumer(JsonObject conn) throws Exception { - RPCHook rpcHook = buildRpcHook(conn); - DefaultMQPullConsumer consumer = rpcHook != null - ? new DefaultMQPullConsumer(MixAll.TOOLS_CONSUMER_GROUP, rpcHook) - : new DefaultMQPullConsumer(MixAll.TOOLS_CONSUMER_GROUP); - consumer.setNamesrvAddr(namesrvAddr(conn)); - applySocksProxy(consumer, conn); - consumer.setInstanceName("DBX_" + UUID.randomUUID()); - return consumer; - } - - static RPCHook buildRpcHook(JsonObject conn) { - String accessKey = credential(conn, "access_key", "accessKey"); - String secretKey = credential(conn, "secret_key", "secretKey"); - if (accessKey.isBlank() && secretKey.isBlank()) { - return null; - } - return new AclClientRPCHook(new SessionCredentials(accessKey, secretKey)); - } - - static String namesrvAddr(JsonObject conn) { - String addr = stringOrEmpty(conn, "namesrv_addr"); - if (addr.isBlank()) { - addr = stringOrEmpty(conn, "namesrvAddr"); - } - if (addr.isBlank()) { - throw new IllegalArgumentException("namesrv_addr is required"); - } - return addr; - } - - static boolean applySocksProxy(ClientConfig client, JsonObject conn) { - if (!conn.has("socks_proxy") || !conn.get("socks_proxy").isJsonObject()) { - return false; - } - JsonObject proxy = conn.getAsJsonObject("socks_proxy"); - String host = stringOrEmpty(proxy, "host"); - int port = intOrDefault(proxy, "port", 0); - if (host.isBlank() || port <= 0 || port > 65_535) { - throw new IllegalArgumentException("socks_proxy host and port are required"); - } - - JsonObject route = new JsonObject(); - route.addProperty("addr", formatSocketAddress(host, Integer.toString(port))); - String username = stringOrEmpty(proxy, "username"); - String password = stringOrEmpty(proxy, "password"); - if (!username.isBlank()) { - route.addProperty("username", username); - } - if (!password.isBlank()) { - route.addProperty("password", password); - } - JsonObject routes = new JsonObject(); - routes.add("0.0.0.0/0", route); - client.setSocksProxyConfig(GSON.toJson(routes)); - return true; - } - - private static TopicList fetchTopicList(DefaultMQAdminExt admin, String cluster) throws Exception { - if (cluster != null && !cluster.isBlank()) { - return admin.fetchTopicsByCLuster(cluster); - } - return admin.fetchAllTopicList(); - } - - private static TopicConfig loadTopicConfig(DefaultMQAdminExt admin, String brokerAddr, String topic) - throws Exception { - TopicConfig config = admin.examineTopicConfig(brokerAddr, topic); - if (config == null) { - config = new TopicConfig(topic); - } - return config; - } - - static TopicConfig buildTopicConfigForCreate(String topicName, int partitions, String messageType) { - return buildTopicConfigForCreate(topicName, partitions, partitions, messageType, 6); - } - - static TopicConfig buildTopicConfigForCreate( - String topicName, int readQueueNums, int writeQueueNums, String messageType, int perm) { - TopicConfig config = new TopicConfig(topicName); - config.setReadQueueNums(Math.max(readQueueNums, 1)); - config.setWriteQueueNums(Math.max(writeQueueNums, 1)); - config.setPerm(normalizeTopicPerm(perm)); - Map attributes = new HashMap<>(); - attributes.put("+" + TOPIC_MESSAGE_TYPE_ATTRIBUTE, messageType); - config.setAttributes(attributes); - return config; - } - - static int normalizeTopicPerm(int perm) { - return switch (perm) { - case 2, 4, 6 -> perm; - default -> 6; - }; - } - - private static List resolveMasterBrokerAddrs(DefaultMQAdminExt admin, JsonObject conn) throws Exception { - return resolveMasterBrokerAddrs(admin, conn, null); - } - - private static MasterBrokerAddrPlan resolveMasterBrokerAddrPlan( - DefaultMQAdminExt admin, JsonObject conn) throws Exception { - return resolveMasterBrokerAddrPlan(admin, conn, null, admin.examineBrokerClusterInfo()); - } - - static List resolveMasterBrokerAddrs( - DefaultMQAdminExt admin, JsonObject conn, String brokerNameFilter) throws Exception { - return resolveMasterBrokerAddrs(admin, conn, brokerNameFilter, admin.examineBrokerClusterInfo()); - } - - static List resolveMasterBrokerAddrs( - DefaultMQAdminExt admin, JsonObject conn, String brokerNameFilter, ClusterInfo clusterInfo) - throws Exception { - return new ArrayList<>(resolveMasterBrokerAddrPlan(admin, conn, brokerNameFilter, clusterInfo).allAddrs()); - } - - static MasterBrokerAddrPlan resolveMasterBrokerAddrPlan( - DefaultMQAdminExt admin, JsonObject conn, String brokerNameFilter, ClusterInfo clusterInfo) - throws Exception { - MasterBrokerAddrPlan plan = masterBrokerAddrsFromClusterInfo(clusterInfo, conn, brokerNameFilter); - if (plan.isEmpty()) { - plan.addRemapped(resolveBrokerAddr(admin, conn, clusterInfo)); - } - return plan; - } - - /** Extract remapped master broker addresses from a ClusterInfo snapshot. */ - static MasterBrokerAddrPlan masterBrokerAddrsFromClusterInfo( - ClusterInfo clusterInfo, JsonObject conn, String brokerNameFilter) { - MasterBrokerAddrPlan plan = new MasterBrokerAddrPlan(); - if (clusterInfo == null || clusterInfo.getBrokerAddrTable() == null) { - return plan; - } - LinkedHashSet remappedSeen = new LinkedHashSet<>(); - for (BrokerData broker : clusterInfo.getBrokerAddrTable().values()) { - if (brokerNameFilter != null && !brokerNameFilter.isBlank() - && !brokerNameFilter.equals(broker.getBrokerName())) { - continue; - } - String masterAddr = null; - if (broker.getBrokerAddrs() != null && broker.getBrokerAddrs().containsKey(0L)) { - masterAddr = broker.getBrokerAddrs().get(0L); - } - if (masterAddr == null || masterAddr.isBlank()) { - masterAddr = broker.selectBrokerAddr(); - } - if (masterAddr == null || masterAddr.isBlank()) { - continue; - } - String remapped = remapBrokerAddrForClient(masterAddr, conn); - // Multi-broker Docker often remaps every master to the same host:port. Keep the - // original address for collisions so in-network agents can still reach each broker. - if (remappedSeen.add(remapped)) { - plan.addRemapped(remapped); - } else if (!remapped.equals(masterAddr)) { - plan.addCollisionFallback(masterAddr); - } - } - return plan; - } - - /** - * Remapped masters plus optional collision-fallback originals. Fail-closed consume-stats - * gating treats fallbacks as best-effort so host-side Docker agents do not mark lag - * unavailable when only unreachable internal IPs fail. - */ - static final class MasterBrokerAddrPlan { - private final LinkedHashSet remapped = new LinkedHashSet<>(); - private final LinkedHashSet collisionFallbacks = new LinkedHashSet<>(); - - void addRemapped(String addr) { - if (addr != null && !addr.isBlank()) { - remapped.add(addr); - } - } - - void addCollisionFallback(String addr) { - if (addr != null && !addr.isBlank() && !remapped.contains(addr)) { - collisionFallbacks.add(addr); - } - } - - boolean isCollisionFallback(String addr) { - return collisionFallbacks.contains(addr); - } - - LinkedHashSet allAddrs() { - LinkedHashSet all = new LinkedHashSet<>(remapped); - all.addAll(collisionFallbacks); - return all; - } - - LinkedHashSet remappedAddrs() { - return new LinkedHashSet<>(remapped); - } - - LinkedHashSet collisionFallbackAddrs() { - return new LinkedHashSet<>(collisionFallbacks); - } - - int remappedCount() { - return remapped.size(); - } - - int fallbackCount() { - return collisionFallbacks.size(); - } - - int size() { - return remapped.size() + collisionFallbacks.size(); - } - - boolean isEmpty() { - return remapped.isEmpty() && collisionFallbacks.isEmpty(); - } - } - - private static void applyTopicConfigValue(TopicConfig config, String key, String value) { - if (value == null) { - return; - } - switch (key) { - case "readQueueNums" -> config.setReadQueueNums(Integer.parseInt(value)); - case "writeQueueNums" -> config.setWriteQueueNums(Integer.parseInt(value)); - case "perm" -> config.setPerm(Integer.parseInt(value)); - default -> { - } - } - } - - private static List resolvePeekQueues(DefaultMQAdminExt admin, String topic, Integer partition) - throws Exception { - TopicRouteData route = admin.examineTopicRouteInfo(topic); - if (route.getQueueDatas() == null || route.getQueueDatas().isEmpty()) { - return Collections.emptyList(); - } - String brokerName = route.getQueueDatas().get(0).getBrokerName(); - if (partition != null) { - return List.of(new MessageQueue(topic, brokerName, partition)); - } - int queueCount = Math.max(route.getQueueDatas().get(0).getReadQueueNums(), 1); - List queues = new ArrayList<>(); - for (int queueId = 0; queueId < queueCount; queueId++) { - queues.add(new MessageQueue(topic, brokerName, queueId)); - } - return queues; - } - - private static Map peekedMessageFromRecord(String topic, MessageExt message) { - Map msg = new LinkedHashMap<>(); - msg.put("topic", topic); - msg.put("messageId", message.getMsgId()); - msg.put("partition", message.getQueueId()); - msg.put("offset", message.getQueueOffset()); - msg.put("timestamp", message.getStoreTimestamp()); - msg.put("key", message.getKeys()); - msg.put("tag", message.getTags()); - Map headers = new LinkedHashMap<>(); - if (message.getProperties() != null) { - headers.putAll(message.getProperties()); - } - msg.put("headers", headers); - if (message.getBody() != null) { - msg.put("payloadBase64", Base64.getEncoder().encodeToString(message.getBody())); - String text = tryDecodeUtf8(message.getBody()); - if (text != null) { - msg.put("payloadText", text); - } - } else { - msg.put("payloadBase64", ""); - } - return msg; - } - - static void sortPeekedMessages(List> messages) { - messages.sort((left, right) -> { - long leftTs = ((Number) left.getOrDefault("timestamp", 0L)).longValue(); - long rightTs = ((Number) right.getOrDefault("timestamp", 0L)).longValue(); - int byTs = Long.compare(leftTs, rightTs); - if (byTs != 0) { - return byTs; - } - int leftPartition = ((Number) left.getOrDefault("partition", 0)).intValue(); - int rightPartition = ((Number) right.getOrDefault("partition", 0)).intValue(); - int byPartition = Integer.compare(leftPartition, rightPartition); - if (byPartition != 0) { - return byPartition; - } - long leftOffset = ((Number) left.getOrDefault("offset", 0L)).longValue(); - long rightOffset = ((Number) right.getOrDefault("offset", 0L)).longValue(); - return Long.compare(leftOffset, rightOffset); - }); - } - - private static Map producerRow(long producerId, String producerGroup, Connection conn) { - Map producer = new LinkedHashMap<>(); - producer.put("producerId", producerId); - producer.put("producerName", producerGroup); - producer.put("msgRateIn", 0.0); - producer.put("msgThroughputIn", 0.0); - producer.put("clientVersion", String.valueOf(conn.getVersion())); - producer.put("address", conn.getClientAddr()); - producer.put("lastTimestamp", 0L); - return producer; - } - - private static Map producerRow(long producerId, String producerGroup, ProducerInfo info) { - Map producer = new LinkedHashMap<>(); - producer.put("producerId", producerId); - producer.put("producerName", producerGroup); - producer.put("msgRateIn", 0.0); - producer.put("msgThroughputIn", 0.0); - producer.put("clientVersion", String.valueOf(info.getVersion())); - producer.put("address", info.getRemoteIP()); - producer.put("lastTimestamp", info.getLastUpdateTimestamp()); - return producer; - } - - private static List> flattenAclInfo(AclInfo aclInfo) { - List> rows = new ArrayList<>(); - if (aclInfo.getPolicies() == null) { - return rows; - } - for (AclInfo.PolicyInfo policy : aclInfo.getPolicies()) { - if (policy.getEntries() == null) { - continue; - } - for (AclInfo.PolicyEntryInfo entry : policy.getEntries()) { - List actions = entry.getActions() == null ? List.of("ALL") : entry.getActions(); - for (String action : actions) { - Map row = new LinkedHashMap<>(); - row.put("resourceType", "TOPIC"); - row.put("resourceName", entry.getResource()); - row.put("patternType", "LITERAL"); - row.put("principal", aclInfo.getSubject()); - row.put("host", entry.getSourceIps() == null || entry.getSourceIps().isEmpty() - ? "*" : String.join(",", entry.getSourceIps())); - row.put("operation", action); - row.put("permissionType", entry.getDecision()); - rows.add(row); - } - } - } - return rows; - } - - private static PlainAccessConfig plainAccessConfigFromJson(JsonObject acl) { - PlainAccessConfig config = new PlainAccessConfig(); - config.setAccessKey(stringOrEmpty(acl, "accessKey")); - config.setSecretKey(stringOrEmpty(acl, "secretKey")); - config.setAdmin(boolOrDefault(acl, "admin", false)); - config.setWhiteRemoteAddress(stringOrDefault(acl, "whiteRemoteAddress", "*")); - config.setDefaultTopicPerm(stringOrDefault(acl, "defaultTopicPerm", "DENY")); - config.setDefaultGroupPerm(stringOrDefault(acl, "defaultGroupPerm", "DENY")); - if (acl.has("topicPerms") && acl.get("topicPerms").isJsonArray()) { - List topicPerms = new ArrayList<>(); - for (JsonElement element : acl.getAsJsonArray("topicPerms")) { - topicPerms.add(element.getAsString()); - } - config.setTopicPerms(topicPerms); - } - if (acl.has("groupPerms") && acl.get("groupPerms").isJsonArray()) { - List groupPerms = new ArrayList<>(); - for (JsonElement element : acl.getAsJsonArray("groupPerms")) { - groupPerms.add(element.getAsString()); - } - config.setGroupPerms(groupPerms); - } - return config; - } - - private static int deleteAclEntry(DefaultMQAdminExt admin, String brokerAddr, Map acl) throws Exception { - String accessKey = stringValue(acl.get("accessKey")); - if (accessKey == null) { - accessKey = stringValue(acl.get("principal")); - } - if (accessKey != null && !accessKey.isBlank()) { - admin.deletePlainAccessConfig(brokerAddr, accessKey); - return 1; - } - String subject = stringValue(acl.get("principal")); - String resource = stringValue(acl.get("resourceName")); - if (subject != null && resource != null) { - admin.deleteAcl(brokerAddr, subject, resource); - return 1; - } - return 0; - } - - private static String mapOperation(String operation) { - return switch (operation.toUpperCase(Locale.ROOT)) { - case "WRITE", "PRODUCE" -> "PUB"; - case "READ", "CONSUME" -> "SUB"; - default -> "ALL"; - }; - } - - private static boolean probeAclSupport(DefaultMQAdminExt admin, ClusterInfo clusterInfo, JsonObject conn) { - return probeAclSupport(clusterInfo, conn, (brokerAddr, timeoutMs) -> { - admin.getDefaultMQAdminExtImpl() - .getMqClientInstance() - .getMQClientAPIImpl() - .getBrokerClusterAclInfo(brokerAddr, timeoutMs); - }); - } - - @FunctionalInterface - interface BrokerAclProbe { - void probe(String brokerAddr, long timeoutMs) throws Exception; - } - - static long aclProbeTimeoutMs(JsonObject conn) { - long connectTimeoutMs = Math.max( - 1_000L, - (long) intOrDefault(conn, "connect_timeout_ms", DEFAULT_CONNECT_TIMEOUT_MS) - ); - return Math.max( - MIN_ACL_PROBE_TIMEOUT_MS, - Math.min(MAX_ACL_PROBE_TIMEOUT_MS, connectTimeoutMs / 2) - ); - } - - static boolean probeAclSupport(ClusterInfo clusterInfo, JsonObject conn, BrokerAclProbe probe) { - try { - if (clusterInfo == null || clusterInfo.getBrokerAddrTable() == null) { - return false; - } - long timeoutMs = aclProbeTimeoutMs(conn); - for (BrokerData broker : clusterInfo.getBrokerAddrTable().values()) { - String brokerAddr = broker.selectBrokerAddr(); - if (brokerAddr == null || brokerAddr.isBlank()) { - continue; - } - probe.probe(remapBrokerAddrForClient(brokerAddr, conn), timeoutMs); - return true; - } - } catch (Exception ignored) { - // ACL not enabled or broker does not expose ACL metadata. - } - return false; - } - - private static String resolveClusterName(DefaultMQAdminExt admin, JsonObject conn) throws Exception { - String configured = clusterName(conn); - if (!configured.isBlank()) { - return configured; - } - return resolveClusterName(admin.examineBrokerClusterInfo(), conn); - } - - private static String resolveClusterName(ClusterInfo clusterInfo, JsonObject conn) { - String configured = clusterName(conn); - if (!configured.isBlank()) { - return configured; - } - if (clusterInfo != null - && clusterInfo.getClusterAddrTable() != null - && !clusterInfo.getClusterAddrTable().isEmpty()) { - return clusterInfo.getClusterAddrTable().keySet().iterator().next(); - } - return "DefaultCluster"; - } - - private static String resolveBrokerAddr(DefaultMQAdminExt admin, JsonObject conn) throws Exception { - return resolveBrokerAddr(admin, conn, admin.examineBrokerClusterInfo()); - } - - private static String resolveBrokerAddr(DefaultMQAdminExt admin, JsonObject conn, ClusterInfo clusterInfo) - throws Exception { - String explicit = brokerAddress(conn); - if (!explicit.isBlank()) { - return explicit; - } - if (clusterInfo != null && clusterInfo.getBrokerAddrTable() != null) { - for (BrokerData broker : clusterInfo.getBrokerAddrTable().values()) { - String addr = broker.selectBrokerAddr(); - if (addr != null && !addr.isBlank()) { - return remapBrokerAddrForClient(addr, conn); - } - } - } - throw new IllegalStateException("No RocketMQ broker address found"); - } - - private static String clusterName(JsonObject params, DefaultMQAdminExt admin) throws Exception { - String configured = clusterName(params); - if (!configured.isBlank()) { - return configured; - } - if (cachedClusterName != null && !cachedClusterName.isBlank()) { - return cachedClusterName; - } - return resolveClusterName(admin, params); - } - - private static String clusterName(JsonObject conn) { - String cluster = stringOrEmpty(conn, "cluster_name"); - if (cluster.isBlank()) { - cluster = stringOrEmpty(conn, "clusterName"); - } - return cluster; - } - - private static String brokerAddr(JsonObject params, DefaultMQAdminExt admin) throws Exception { - return brokerAddr(params, admin, connectionObject(params)); - } - - private static String brokerAddr(JsonObject params, DefaultMQAdminExt admin, JsonObject conn) throws Exception { - String brokerName = stringOrEmpty(params, "brokerName"); - if (!brokerName.isBlank()) { - List addrs = resolveMasterBrokerAddrs(admin, conn, brokerName); - if (!addrs.isEmpty()) { - return addrs.get(0); - } - } - String explicit = brokerAddress(conn); - if (!explicit.isBlank()) { - return explicit; - } - if (cachedBrokerAddr != null && !cachedBrokerAddr.isBlank()) { - return cachedBrokerAddr; - } - return resolveBrokerAddr(admin, conn); - } - - private static String brokerAddress(JsonObject conn) { - String broker = stringOrEmpty(conn, "broker_addr"); - if (broker.isBlank()) { - broker = stringOrEmpty(conn, "brokerAddr"); - } - return broker; - } - - /** - * RocketMQ brokers often register docker/internal IPs with NameServer. Remap to the - * namesrv host with the same broker port so host-side agents can reach published ports. - */ - static String remapBrokerAddrForClient(String brokerAddr, JsonObject conn) { - if (brokerAddr == null || brokerAddr.isBlank()) { - return brokerAddr; - } - String explicit = brokerAddress(conn); - if (!explicit.isBlank()) { - return explicit; - } - if (conn.has("socks_proxy") && conn.get("socks_proxy").isJsonObject()) { - return brokerAddr; - } - String brokerHost = parseHostFromSocketAddress(brokerAddr); - String brokerPort = parsePortFromSocketAddress(brokerAddr); - if (brokerPort.isBlank()) { - return brokerAddr; - } - if (!isLikelyUnreachableBrokerHost(brokerHost)) { - return brokerAddr; - } - String namesrvHost = primaryNamesrvHost(conn); - return formatSocketAddress(namesrvHost, brokerPort); - } - - /** - * Copy non-system UI headers onto the message as user properties so SQL92 filters work. - * TAGS/KEYS are applied via tag/keys APIs and must not be duplicated here. - */ - static void applySendHeaders(Message message, JsonObject params) { - if (!params.has("headers") || !params.get("headers").isJsonObject()) { - return; - } - JsonObject headers = params.getAsJsonObject("headers"); - for (Map.Entry entry : headers.entrySet()) { - String key = entry.getKey(); - if (key == null || key.isBlank() || entry.getValue() == null || entry.getValue().isJsonNull()) { - continue; - } - if (MessageConst.PROPERTY_TAGS.equals(key) || MessageConst.PROPERTY_KEYS.equals(key)) { - continue; - } - if (isRocketMqSystemMessageProperty(key)) { - continue; - } - String value = entry.getValue().getAsString(); - if (value.isBlank()) { - continue; - } - message.putUserProperty(key, value); - } - } - - private static boolean isRocketMqSystemMessageProperty(String key) { - return MessageConst.STRING_HASH_SET.contains(key); - } - - /** Host part of a single {@code host:port} or {@code [ipv6]:port} socket address. */ - static String parseHostFromSocketAddress(String hostPort) { - if (hostPort == null) { - return ""; - } - String trimmed = hostPort.trim(); - if (trimmed.isEmpty()) { - return ""; - } - if (trimmed.startsWith("[")) { - int close = trimmed.indexOf(']'); - if (close > 0) { - return trimmed.substring(1, close); - } - } - int colon = trimmed.lastIndexOf(':'); - if (colon <= 0) { - return trimmed; - } - String portPart = trimmed.substring(colon + 1); - if (isNumericPort(portPart)) { - return trimmed.substring(0, colon); - } - return trimmed; - } - - static String parsePortFromSocketAddress(String hostPort) { - if (hostPort == null) { - return ""; - } - String trimmed = hostPort.trim(); - if (trimmed.isEmpty()) { - return ""; - } - if (trimmed.startsWith("[")) { - int close = trimmed.indexOf(']'); - if (close > 0 && close + 1 < trimmed.length() && trimmed.charAt(close + 1) == ':') { - return trimmed.substring(close + 2); - } - return ""; - } - int colon = trimmed.lastIndexOf(':'); - if (colon <= 0 || colon >= trimmed.length() - 1) { - return ""; - } - return trimmed.substring(colon + 1); - } - - static String formatSocketAddress(String host, String port) { - if (host == null || host.isBlank()) { - return port == null ? "" : port; - } - if (port == null || port.isBlank()) { - return host; - } - if (host.indexOf(':') >= 0) { - return "[" + host + "]:" + port; - } - return host + ":" + port; - } - - /** - * First reachable NameServer host for Docker broker remap. When auto-remap is wrong, - * set {@code broker_addr} / {@code brokerAddr} on the connection explicitly. - */ - static String primaryNamesrvHost(JsonObject conn) { - for (String entry : resolveNameServerAddrSet(conn)) { - String host = parseHostFromSocketAddress(entry); - if (!host.isBlank()) { - return host; - } - } - return "127.0.0.1"; - } - - private static boolean isNumericPort(String portPart) { - if (portPart == null || portPart.isEmpty()) { - return false; - } - for (int i = 0; i < portPart.length(); i++) { - if (!Character.isDigit(portPart.charAt(i))) { - return false; - } - } - return true; - } - - /** Package-visible for tests; RFC1918 + common Docker DNS names. */ - static boolean isLikelyUnreachableBrokerHost(String host) { - if (host == null || host.isBlank()) { - return false; - } - if ("127.0.0.1".equals(host) || "localhost".equalsIgnoreCase(host) || "::1".equals(host)) { - return false; - } - // RFC1918: 10/8, 172.16/12, 192.168/16 — not every 172.* address. - return isRfc1918PrivateIpv4(host) - || host.endsWith(".docker") - || host.contains(".docker."); - } - - static boolean isRfc1918PrivateIpv4(String host) { - if (host == null || host.isBlank()) { - return false; - } - String[] parts = host.split("\\."); - if (parts.length != 4) { - return false; - } - int[] octets = new int[4]; - for (int i = 0; i < 4; i++) { - try { - octets[i] = Integer.parseInt(parts[i]); - } catch (NumberFormatException e) { - return false; - } - if (octets[i] < 0 || octets[i] > 255) { - return false; - } - } - if (octets[0] == 10) { - return true; - } - if (octets[0] == 192 && octets[1] == 168) { - return true; - } - // 172.16.0.0 – 172.31.255.255 - return octets[0] == 172 && octets[1] >= 16 && octets[1] <= 31; - } - - private static List> brokerNodes(ClusterInfo clusterInfo) { - List> brokers = new ArrayList<>(); - int id = 0; - for (BrokerData broker : clusterInfo.getBrokerAddrTable().values()) { - String brokerName = broker.getBrokerName(); - if (broker.getBrokerAddrs() != null && !broker.getBrokerAddrs().isEmpty()) { - for (Map.Entry entry : broker.getBrokerAddrs().entrySet()) { - brokers.add(brokerNode(id++, brokerName, entry.getKey(), entry.getValue())); - } - continue; - } - String addr = broker.selectBrokerAddr(); - if (addr == null || addr.isBlank()) { - continue; - } - brokers.add(brokerNode(id++, brokerName, 0L, addr)); - } - return brokers; - } - - private static Map brokerNode(int id, String brokerName, long brokerId, String addr) { - String host = parseHostFromSocketAddress(addr); - String portText = parsePortFromSocketAddress(addr); - int port = portText.isBlank() ? 0 : Integer.parseInt(portText); - Map node = new LinkedHashMap<>(); - node.put("id", id); - node.put("host", host); - node.put("port", port); - node.put("rack", null); - node.put("brokerName", brokerName); - node.put("brokerId", brokerId); - node.put("role", brokerId == 0L ? "MASTER" : "SLAVE"); - return node; - } - - private static Set collectBrokerNames(DefaultMQAdminExt admin) { - try { - return collectBrokerNames(admin.examineBrokerClusterInfo()); - } catch (Exception ignored) { - // Fall back to static reserved-topic filtering only. - return Set.of(); - } - } - - static Set collectBrokerNames(ClusterInfo clusterInfo) { - Set names = new HashSet<>(); - if (clusterInfo == null || clusterInfo.getBrokerAddrTable() == null) { - return names; - } - for (BrokerData brokerData : clusterInfo.getBrokerAddrTable().values()) { - if (brokerData.getBrokerName() != null && !brokerData.getBrokerName().isBlank()) { - names.add(brokerData.getBrokerName()); - } - } - return names; - } - - private static Set collectBrokerSystemTopics(DefaultMQAdminExt admin) { - // DefaultMQAdminExt 5.3.x does not expose getSystemTopicListFromBroker; reserved-name rules cover most cases. - return Set.of(); - } - - static Set resolveNameServerAddrSet(JsonObject conn) { - Set addrs = new LinkedHashSet<>(); - for (String part : namesrvAddr(conn).split(";")) { - String trimmed = part.trim(); - if (!trimmed.isEmpty()) { - addrs.add(trimmed); - } - } - return addrs; - } - - private static Map collectBrokerTopicConfigs(DefaultMQAdminExt admin, JsonObject conn) { - try { - return collectBrokerTopicConfigSnapshot(admin, conn, admin.examineBrokerClusterInfo()).topics(); - } catch (Exception ignored) { - return new LinkedHashMap<>(); - } - } - - private static BrokerTopicConfigSnapshot collectBrokerTopicConfigSnapshot( - DefaultMQAdminExt admin, JsonObject conn, ClusterInfo clusterInfo) throws Exception { - List brokerAddrs = resolveMasterBrokerAddrs(admin, conn, null, clusterInfo); - long timeoutMs = operationBudgetMs(conn); - return collectBrokerTopicConfigs( - brokerAddrs, - brokerAddr -> admin.getAllTopicConfig(brokerAddr, timeoutMs) - ); - } - - static BrokerTopicConfigSnapshot collectBrokerTopicConfigs( - List brokerAddrs, BrokerTopicConfigLookup lookup) { - Map merged = new LinkedHashMap<>(); - boolean complete = !brokerAddrs.isEmpty(); - for (String brokerAddr : brokerAddrs) { - try { - TopicConfigSerializeWrapper wrapper = lookup.load(brokerAddr); - if (wrapper == null || wrapper.getTopicConfigTable() == null) { - complete = false; - continue; - } - for (TopicConfig config : wrapper.getTopicConfigTable().values()) { - if (config.getTopicName() == null || config.getTopicName().isBlank()) { - continue; - } - merged.merge(config.getTopicName(), config, RocketMqAgent::preferTopicConfig); - } - } catch (Exception ignored) { - complete = false; - } - } - return new BrokerTopicConfigSnapshot(merged, complete); - } - - private static TopicConfig preferTopicConfig(TopicConfig left, TopicConfig right) { - String leftType = readTopicMessageType(left); - String rightType = readTopicMessageType(right); - if ((leftType == null || leftType.isBlank()) && rightType != null && !rightType.isBlank()) { - return right; - } - if (right.getReadQueueNums() > left.getReadQueueNums()) { - return right; - } - return left; - } - - private static Map> topicAttributesFromConfigs(Map brokerTopics) { - Map> topicAttributes = new HashMap<>(); - Map topicMessageTypes = new HashMap<>(); - for (TopicConfig config : brokerTopics.values()) { - mergeTopicAttributes(topicAttributes, topicMessageTypes, config); - } - for (Map.Entry entry : topicMessageTypes.entrySet()) { - Map attrs = topicAttributes.computeIfAbsent(entry.getKey(), ignored -> new HashMap<>()); - attrs.putIfAbsent("+" + TOPIC_MESSAGE_TYPE_ATTRIBUTE, entry.getValue()); - attrs.putIfAbsent(TOPIC_MESSAGE_TYPE_ATTRIBUTE, entry.getValue()); - } - return topicAttributes; - } - - private static Map> collectTopicAttributes(DefaultMQAdminExt admin, JsonObject conn) { - return topicAttributesFromConfigs(collectBrokerTopicConfigs(admin, conn)); - } - - private static void mergeTopicAttributes( - Map> topicAttributes, - Map topicMessageTypes, - TopicConfig config - ) { - String topicName = config.getTopicName(); - if (topicName == null || topicName.isBlank()) { - return; - } - String messageType = readTopicMessageType(config); - if (messageType != null && !messageType.isBlank()) { - topicMessageTypes.putIfAbsent(topicName, messageType); - } - Map attrs = config.getAttributes(); - Map existing = topicAttributes.get(topicName); - if (existing == null) { - topicAttributes.put(topicName, attrs == null ? new HashMap<>() : new HashMap<>(attrs)); - return; - } - if (readTopicMessageTypeAttribute(existing) == null && attrs != null && !attrs.isEmpty()) { - existing.putAll(attrs); - } - } - - private static String resolveTopicMessageType(DefaultMQAdminExt admin, JsonObject conn, String topic) { - try { - return resolveTopicMessageType(admin, conn, topic, admin.examineBrokerClusterInfo()); - } catch (Exception ignored) { - return null; - } - } - - private static String resolveTopicMessageType( - DefaultMQAdminExt admin, JsonObject conn, String topic, ClusterInfo clusterInfo) { - try { - TopicRouteData route = admin.examineTopicRouteInfo(topic); - if (route.getQueueDatas() == null || route.getQueueDatas().isEmpty()) { - return null; - } - String brokerName = route.getQueueDatas().get(0).getBrokerName(); - if (clusterInfo == null || clusterInfo.getBrokerAddrTable() == null) { - return null; - } - BrokerData brokerData = clusterInfo.getBrokerAddrTable().get(brokerName); - if (brokerData == null) { - return null; - } - String brokerAddr = remapBrokerAddrForClient(brokerData.selectBrokerAddr(), conn); - if (brokerAddr == null || brokerAddr.isBlank()) { - return null; - } - TopicConfig config = admin.examineTopicConfig(brokerAddr, topic); - return readTopicMessageType(config); - } catch (Exception ignored) { - return null; - } - } - - /** - * Slice {@code items} from {@code offset}. When {@code limit <= 0}, return all remaining items - * (used by the Rust adapter's single-shot full list request). - */ - static List paginate(List items, int offset, int limit) { - if (offset >= items.size()) { - return Collections.emptyList(); - } - if (limit <= 0) { - return new ArrayList<>(items.subList(offset, items.size())); - } - int end = Math.min(items.size(), offset + limit); - return new ArrayList<>(items.subList(offset, end)); - } - - private static DefaultMQAdminExt requireAdmin() { - if (adminClient == null) { - throw new IllegalStateException("Not connected. Call connect first."); - } - return adminClient; - } - - private static DefaultMQProducer requireProducer() throws Exception { - if (producer == null) { - if (cachedConnection == null) { - throw new IllegalStateException("Not connected. Call connect first."); - } - producer = buildProducer(cachedConnection); - } - return producer; - } - - private static JsonObject connectionObject(JsonObject params) { - if (params.has("connection") && params.get("connection").isJsonObject()) { - return params.getAsJsonObject("connection"); - } - if (cachedConnection != null) { - return cachedConnection; - } - return params; - } - - private static String credential(JsonObject conn, String snakeCase, String camelCase) { - String value = stringOrEmpty(conn, snakeCase); - if (value.isBlank()) { - value = stringOrEmpty(conn, camelCase); - } - return value; - } - - private static byte[] decodePayload(JsonObject params) { - String payloadBase64 = stringOrEmpty(params, "payloadBase64"); - if (!payloadBase64.isBlank()) { - return Base64.getDecoder().decode(payloadBase64); - } - String text = stringOrEmpty(params, "payloadText"); - if (!text.isBlank()) { - return text.getBytes(StandardCharsets.UTF_8); - } - return new byte[0]; - } - - private static void putConfigEntry(Map configs, String key, String value) { - Map entry = new LinkedHashMap<>(); - entry.put("value", value); - entry.put("source", "USER"); - entry.put("isSensitive", false); - entry.put("isReadOnly", false); - entry.put("isDefault", false); - configs.put(key, entry); - } - - private static Map jsonToMap(JsonObject object) { - Map map = new LinkedHashMap<>(); - for (Map.Entry entry : object.entrySet()) { - if (entry.getValue().isJsonNull()) { - map.put(entry.getKey(), null); - } else if (entry.getValue().isJsonPrimitive()) { - map.put(entry.getKey(), entry.getValue().getAsString()); - } - } - return map; - } - - private static String stringValue(Object value) { - return value == null ? null : String.valueOf(value); - } - - private static String tryDecodeUtf8(byte[] bytes) { - try { - String text = new String(bytes, StandardCharsets.UTF_8); - byte[] reEncoded = text.getBytes(StandardCharsets.UTF_8); - if (Arrays.equals(bytes, reEncoded)) { - return text; - } - } catch (Exception ignored) { - // Not valid UTF-8 text. - } - return null; - } - - private static String normalizeErrorMessage(Exception e) { - String message = e.getMessage() == null || e.getMessage().isBlank() - ? e.getClass().getName() - : e.getMessage(); - Throwable root = rootCause(e); - if (root != e && root.getMessage() != null && !root.getMessage().isBlank() - && !message.contains(root.getMessage())) { - message = message + ": " + root.getMessage(); - } - return message; - } - - private static Throwable rootCause(Throwable error) { - Throwable current = error; - while (current.getCause() != null && current.getCause() != current) { - current = current.getCause(); - } - return current; - } - - static String stringOrEmpty(JsonObject object, String key) { - return stringOrDefault(object, key, ""); - } - - private static String stringOrNull(JsonObject object, String key) { - JsonElement element = object.get(key); - return element == null || element.isJsonNull() ? null : element.getAsString(); - } - - private static String stringOrDefault(JsonObject object, String key, String fallback) { - String value = stringOrNull(object, key); - return value == null ? fallback : value; - } - - private static Integer integerOrNull(JsonObject object, String key) { - JsonElement element = object.get(key); - return element == null || element.isJsonNull() ? null : element.getAsInt(); - } - - private static Long longOrNull(JsonObject object, String key) { - JsonElement element = object.get(key); - return element == null || element.isJsonNull() ? null : element.getAsLong(); - } - - static int intOrDefault(JsonObject object, String key, int fallback) { - Integer value = integerOrNull(object, key); - return value == null ? fallback : value; - } - - /** - * Full Advanced query timeout via {@code request_timeout_ms}. Use for single-shot ops - * (send/admin construct/topic-config per call). Multi-phase list paths must use - * {@link #scaledOperationBudgetMs} so phases still fit under one Rust RPC window. - * Message {@code LitePullConsumer.poll} stays on {@link #MESSAGE_POLL_TIMEOUT_MS}. - */ - static long operationBudgetMs(JsonObject conn) { - return Math.max(1_000L, intOrDefault(conn, "request_timeout_ms", DEFAULT_REQUEST_TIMEOUT_MS)); - } - - /** - * Scale a legacy phase baseline with Advanced query timeout. At the 30s default, baselines are - * unchanged (12/8/10/8); at 120s they expand 4× but never exceed the RPC window. - */ - static long scaledOperationBudgetMs(JsonObject conn, long baselineMs) { - long requestMs = operationBudgetMs(conn); - long scaled = baselineMs * requestMs / DEFAULT_REQUEST_TIMEOUT_MS; - return Math.max(1_000L, Math.min(requestMs, scaled)); - } - - /** {@link #operationBudgetMs} clamped to {@code int} for APIs like {@code setSendMsgTimeout}. */ - static int operationBudgetMsAsInt(JsonObject conn) { - return (int) Math.min(Integer.MAX_VALUE, operationBudgetMs(conn)); - } - - static long longOrDefault(JsonObject object, String key, long fallback) { - Long value = longOrNull(object, key); - return value == null ? fallback : value; - } - - private static boolean boolOrDefault(JsonObject object, String key, boolean fallback) { - JsonElement element = object.get(key); - return element == null || element.isJsonNull() ? fallback : element.getAsBoolean(); - } - - private static String requireString(JsonObject object, String key) { - String value = stringOrEmpty(object, key); - if (value.isBlank()) { - throw new IllegalArgumentException(key + " is required"); - } - return value; - } - - private static final class HandshakeResult { - private final int protocolVersion; - private final int agentProtocolVersion; - private final List capabilities; - - private HandshakeResult(int protocolVersion, int agentProtocolVersion, List capabilities) { - this.protocolVersion = protocolVersion; - this.agentProtocolVersion = agentProtocolVersion; - this.capabilities = capabilities; - } - } -} diff --git a/agents/drivers/rocketmq/src/test/java/com/dbx/agent/rocketmq/RocketMqAgentTest.java b/agents/drivers/rocketmq/src/test/java/com/dbx/agent/rocketmq/RocketMqAgentTest.java deleted file mode 100644 index 3ff6b81c24..0000000000 --- a/agents/drivers/rocketmq/src/test/java/com/dbx/agent/rocketmq/RocketMqAgentTest.java +++ /dev/null @@ -1,1272 +0,0 @@ -package com.dbx.agent.rocketmq; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertSame; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import org.apache.rocketmq.client.exception.MQBrokerException; -import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.common.TopicConfig; -import org.apache.rocketmq.common.MixAll; -import org.apache.rocketmq.common.message.Message; -import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; -import org.apache.rocketmq.remoting.protocol.admin.ConsumeStats; -import org.apache.rocketmq.remoting.protocol.admin.OffsetWrapper; -import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable; -import org.apache.rocketmq.remoting.protocol.admin.TopicOffset; -import org.apache.rocketmq.remoting.protocol.body.ProducerInfo; -import org.apache.rocketmq.remoting.protocol.body.ProducerTableInfo; -import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper; -import org.apache.rocketmq.remoting.protocol.body.ConsumerConnection; -import org.apache.rocketmq.remoting.protocol.route.QueueData; -import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; -import org.apache.rocketmq.common.message.MessageQueue; -import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.IntStream; -import org.junit.jupiter.api.Test; - -class RocketMqAgentTest { - @Test - void namesrvAddrAcceptsSnakeCaseField() { - assertEquals( - "127.0.0.1:9876", - RocketMqAgent.namesrvAddr(JsonParser.parseString(""" - {"namesrv_addr":"127.0.0.1:9876"} - """).getAsJsonObject()) - ); - } - - @Test - void namesrvAddrAcceptsCamelCaseField() { - assertEquals( - "127.0.0.1:9876", - RocketMqAgent.namesrvAddr(JsonParser.parseString(""" - {"namesrvAddr":"127.0.0.1:9876"} - """).getAsJsonObject()) - ); - } - - @Test - void namesrvAddrRequiresValue() { - assertThrows( - IllegalArgumentException.class, - () -> RocketMqAgent.namesrvAddr(JsonParser.parseString("{}").getAsJsonObject()) - ); - } - - @Test - void applySocksProxyConfiguresAllRocketMqDestinations() { - JsonObject conn = JsonParser.parseString(""" - { - "socks_proxy": { - "host": "127.0.0.1", - "port": 41080, - "username": "proxy-user", - "password": "proxy-secret" - } - } - """).getAsJsonObject(); - DefaultMQAdminExt admin = new DefaultMQAdminExt(); - - assertTrue(RocketMqAgent.applySocksProxy(admin, conn)); - - JsonObject routes = JsonParser.parseString(admin.getSocksProxyConfig()).getAsJsonObject(); - assertEquals("127.0.0.1:41080", routes.getAsJsonObject("0.0.0.0/0").get("addr").getAsString()); - assertEquals("proxy-user", routes.getAsJsonObject("0.0.0.0/0").get("username").getAsString()); - assertEquals("proxy-secret", routes.getAsJsonObject("0.0.0.0/0").get("password").getAsString()); - } - - @Test - void isUserTopicFiltersRocketMqSystemTopics() { - assertEquals(false, RocketMqAgent.isUserTopic("RMQ_SYS_TRANS_HALF_TOPIC")); - assertEquals(false, RocketMqAgent.isUserTopic("BenchmarkTest")); - assertEquals(false, RocketMqAgent.isUserTopic("SCHEDULE_TOPIC_XXXX")); - assertEquals(false, RocketMqAgent.isUserTopic("DefaultCluster_REPLY_TOPIC")); - assertEquals(false, RocketMqAgent.isUserTopic("%RETRY%MyGroup")); - assertEquals(true, RocketMqAgent.isUserTopic("CS123")); - assertEquals(true, RocketMqAgent.isUserTopic("OrderCreated")); - } - - @Test - void classifyTopicMessageTypeMatchesDashboardRules() { - assertEquals("RETRY", RocketMqAgent.classifyTopicMessageType( - "%RETRY%MyGroup", null, Set.of(), Set.of(), "")); - assertEquals("DLQ", RocketMqAgent.classifyTopicMessageType( - "%DLQ%MyGroup", null, Set.of(), Set.of(), "")); - assertEquals("SYSTEM", RocketMqAgent.classifyTopicMessageType( - "RMQ_SYS_TRANS_HALF_TOPIC", null, Set.of(), Set.of(), "")); - assertEquals("UNSPECIFIED", RocketMqAgent.classifyTopicMessageType( - "CS123", null, Set.of(), Set.of(), "")); - assertEquals("UNSPECIFIED", RocketMqAgent.classifyTopicMessageType( - "CS123", Map.of(), Set.of(), Set.of(), "")); - assertEquals("NORMAL", RocketMqAgent.classifyTopicMessageType( - "CS123", - Map.of("+message.type", "NORMAL"), - Set.of(), - Set.of(), - "")); - assertEquals("NORMAL", RocketMqAgent.classifyTopicMessageType( - "OrderCreated", - Map.of("message.type", "NORMAL"), - Set.of(), - Set.of(), - "")); - assertEquals("DELAY", RocketMqAgent.classifyTopicMessageType( - "DelayTopic", - Map.of("message.type", "DELAY"), - Set.of(), - Set.of(), - "")); - assertEquals("UNSPECIFIED", RocketMqAgent.classifyTopicMessageType( - "LegacyTopic", - Map.of("message.type", "UNSPECIFIED"), - Set.of(), - Set.of(), - "")); - assertEquals("FIFO", RocketMqAgent.classifyTopicMessageType( - "OrderedTopic", - Map.of("message.type", "ORDER"), - Set.of(), - Set.of(), - "")); - } - - @Test - void remapBrokerAddrUsesNamesrvHostForDockerBrokerIp() { - JsonObject conn = JsonParser.parseString(""" - {"namesrv_addr":"127.0.0.1:9876"} - """).getAsJsonObject(); - assertEquals("127.0.0.1:10911", RocketMqAgent.remapBrokerAddrForClient("172.18.0.3:10911", conn)); - assertEquals("127.0.0.1:10911", RocketMqAgent.remapBrokerAddrForClient("10.0.0.5:10911", conn)); - assertEquals("broker.example.com:10911", RocketMqAgent.remapBrokerAddrForClient("broker.example.com:10911", conn)); - // 172.15/172.32 are not RFC1918 — leave reachable public/non-private hosts alone. - assertEquals("172.15.0.1:10911", RocketMqAgent.remapBrokerAddrForClient("172.15.0.1:10911", conn)); - assertEquals("172.32.0.1:10911", RocketMqAgent.remapBrokerAddrForClient("172.32.0.1:10911", conn)); - } - - @Test - void remapBrokerAddrPreservesAdvertisedAddressWhenSocksProxyIsActive() { - JsonObject conn = JsonParser.parseString(""" - { - "namesrv_addr": "172.19.191.166:9876", - "socks_proxy": {"host":"127.0.0.1","port":41080} - } - """).getAsJsonObject(); - - assertEquals("172.18.0.3:10911", RocketMqAgent.remapBrokerAddrForClient("172.18.0.3:10911", conn)); - } - - @Test - void isRfc1918PrivateIpv4MatchesOnlyPrivateRanges() { - assertTrue(RocketMqAgent.isRfc1918PrivateIpv4("10.0.0.1")); - assertTrue(RocketMqAgent.isRfc1918PrivateIpv4("172.16.0.1")); - assertTrue(RocketMqAgent.isRfc1918PrivateIpv4("172.31.255.255")); - assertTrue(RocketMqAgent.isRfc1918PrivateIpv4("192.168.1.1")); - assertFalse(RocketMqAgent.isRfc1918PrivateIpv4("172.15.0.1")); - assertFalse(RocketMqAgent.isRfc1918PrivateIpv4("172.32.0.1")); - assertFalse(RocketMqAgent.isRfc1918PrivateIpv4("8.8.8.8")); - assertFalse(RocketMqAgent.isRfc1918PrivateIpv4("broker.example.com")); - } - - @Test - void remapBrokerAddrUsesFirstNamesrvHostWhenMultipleConfigured() { - JsonObject conn = JsonParser.parseString(""" - {"namesrv_addr":"ns1:9876;ns2:9876"} - """).getAsJsonObject(); - assertEquals("ns1:10911", RocketMqAgent.remapBrokerAddrForClient("172.18.0.3:10911", conn)); - } - - @Test - void remapBrokerAddrSupportsIpv6NamesrvHost() { - JsonObject conn = JsonParser.parseString(""" - {"namesrv_addr":"[2001:db8::1]:9876"} - """).getAsJsonObject(); - assertEquals("[2001:db8::1]:10911", RocketMqAgent.remapBrokerAddrForClient("10.0.0.5:10911", conn)); - assertEquals("[::1]:10911", RocketMqAgent.remapBrokerAddrForClient("172.18.0.2:10911", JsonParser.parseString(""" - {"namesrv_addr":"[::1]:9876"} - """).getAsJsonObject())); - } - - @Test - void remapBrokerAddrHonorsExplicitBrokerAddress() { - JsonObject conn = JsonParser.parseString(""" - {"namesrv_addr":"127.0.0.1:9876","broker_addr":"published.example.com:10911"} - """).getAsJsonObject(); - assertEquals("published.example.com:10911", RocketMqAgent.remapBrokerAddrForClient("172.18.0.3:10911", conn)); - } - - @Test - void aclProbeTimeoutUsesConnectBudgetWithoutReducingOperationBudget() { - JsonObject conn = JsonParser.parseString(""" - {"connect_timeout_ms":10000,"request_timeout_ms":120000} - """).getAsJsonObject(); - assertEquals(5_000L, RocketMqAgent.aclProbeTimeoutMs(conn)); - assertEquals(120_000L, RocketMqAgent.operationBudgetMs(conn)); - - assertEquals(500L, RocketMqAgent.aclProbeTimeoutMs(JsonParser.parseString(""" - {"connect_timeout_ms":1000} - """).getAsJsonObject())); - assertEquals(5_000L, RocketMqAgent.aclProbeTimeoutMs(JsonParser.parseString(""" - {"connect_timeout_ms":120000} - """).getAsJsonObject())); - assertEquals(500L, RocketMqAgent.aclProbeTimeoutMs(JsonParser.parseString(""" - {"connect_timeout_ms":-2147483648} - """).getAsJsonObject())); - assertEquals(5_000L, RocketMqAgent.aclProbeTimeoutMs(JsonParser.parseString(""" - {"connect_timeout_ms":2147483647} - """).getAsJsonObject())); - } - - @Test - void probeAclSupportRemapsBrokerAndPassesIndependentTimeout() { - var clusterInfo = clusterInfoWithBrokers("172.18.0.3:10911"); - JsonObject conn = JsonParser.parseString(""" - { - "namesrv_addr":"127.0.0.1:9876", - "connect_timeout_ms":10000, - "request_timeout_ms":120000 - } - """).getAsJsonObject(); - List addresses = new ArrayList<>(); - List timeouts = new ArrayList<>(); - - assertTrue(RocketMqAgent.probeAclSupport(clusterInfo, conn, (address, timeoutMs) -> { - addresses.add(address); - timeouts.add(timeoutMs); - })); - assertEquals(List.of("127.0.0.1:10911"), addresses); - assertEquals(List.of(5_000L), timeouts); - } - - @Test - void probeAclSupportPreservesPublicExplicitAndSocksAddresses() { - List addresses = new ArrayList<>(); - RocketMqAgent.BrokerAclProbe capture = (address, timeoutMs) -> addresses.add(address); - - assertTrue(RocketMqAgent.probeAclSupport( - clusterInfoWithBrokers("broker.example.com:10911"), - JsonParser.parseString(""" - {"namesrv_addr":"ns.example.com:9876","connect_timeout_ms":10000} - """).getAsJsonObject(), - capture)); - assertTrue(RocketMqAgent.probeAclSupport( - clusterInfoWithBrokers("172.18.0.3:10911"), - JsonParser.parseString(""" - { - "namesrv_addr":"127.0.0.1:9876", - "broker_addr":"published.example.com:10911", - "connect_timeout_ms":10000 - } - """).getAsJsonObject(), - capture)); - assertTrue(RocketMqAgent.probeAclSupport( - clusterInfoWithBrokers("172.18.0.3:10911"), - JsonParser.parseString(""" - { - "namesrv_addr":"172.19.191.166:9876", - "socks_proxy":{"host":"127.0.0.1","port":41080}, - "connect_timeout_ms":10000 - } - """).getAsJsonObject(), - capture)); - - assertEquals(List.of( - "broker.example.com:10911", - "published.example.com:10911", - "172.18.0.3:10911" - ), addresses); - } - - @Test - void probeAclSupportReturnsFalseWithoutRetryOnTimeoutOrError() { - var clusterInfo = clusterInfoWithBrokers("broker-a:10911", "broker-b:10911"); - JsonObject conn = JsonParser.parseString(""" - {"namesrv_addr":"ns.example.com:9876","connect_timeout_ms":10000} - """).getAsJsonObject(); - AtomicInteger calls = new AtomicInteger(); - - assertFalse(RocketMqAgent.probeAclSupport(clusterInfo, conn, (address, timeoutMs) -> { - calls.incrementAndGet(); - throw new java.util.concurrent.TimeoutException("blackholed broker"); - })); - assertEquals(1, calls.get()); - - assertFalse(RocketMqAgent.probeAclSupport(clusterInfo, conn, (address, timeoutMs) -> { - throw new IllegalStateException("ACL unavailable"); - })); - } - - @Test - void probeAclSupportReturnsFalseForMissingBrokersWithoutNetworkCall() { - AtomicInteger calls = new AtomicInteger(); - RocketMqAgent.BrokerAclProbe probe = (address, timeoutMs) -> calls.incrementAndGet(); - JsonObject conn = JsonParser.parseString(""" - {"namesrv_addr":"ns.example.com:9876","connect_timeout_ms":10000} - """).getAsJsonObject(); - - assertFalse(RocketMqAgent.probeAclSupport(null, conn, probe)); - assertFalse(RocketMqAgent.probeAclSupport( - new org.apache.rocketmq.remoting.protocol.body.ClusterInfo(), conn, probe)); - assertEquals(0, calls.get()); - } - - @Test - void parseHostFromSocketAddressHandlesIpv6AndPorts() { - assertEquals("127.0.0.1", RocketMqAgent.parseHostFromSocketAddress("127.0.0.1:9876")); - assertEquals("2001:db8::1", RocketMqAgent.parseHostFromSocketAddress("[2001:db8::1]:9876")); - assertEquals("::1", RocketMqAgent.parseHostFromSocketAddress("[::1]:9876")); - } - - @Test - void applySendHeadersSetsUserPropertiesAndSkipsSystemKeys() { - Message message = new Message("TopicA", "tag-a", "body".getBytes()); - JsonObject params = JsonParser.parseString(""" - {"headers":{"TAGS":"tag-a","KEYS":"k1","Region":"Hangzhou","color":"blue"}} - """).getAsJsonObject(); - RocketMqAgent.applySendHeaders(message, params); - assertEquals("Hangzhou", message.getProperty("Region")); - assertEquals("blue", message.getProperty("color")); - assertEquals("tag-a", message.getTags()); - assertEquals(null, message.getProperty("KEYS")); - } - - @Test - void paginateReturnsSecondPageAfter200Topics() { - List items = IntStream.range(0, 201).boxed().toList(); - assertEquals(200, RocketMqAgent.paginate(items, 0, 200).size()); - assertEquals(1, RocketMqAgent.paginate(items, 200, 200).size()); - assertEquals(200, RocketMqAgent.paginate(items, 200, 200).get(0)); - } - - @Test - void paginateLimitZeroReturnsAllFromOffset() { - List items = IntStream.range(0, 341).boxed().toList(); - assertEquals(341, RocketMqAgent.paginate(items, 0, 0).size()); - assertEquals(141, RocketMqAgent.paginate(items, 200, 0).size()); - assertEquals(200, RocketMqAgent.paginate(items, 200, 0).get(0)); - } - - @Test - void collectBrokerNamesFromClusterInfoSnapshot() { - org.apache.rocketmq.remoting.protocol.body.ClusterInfo clusterInfo = - new org.apache.rocketmq.remoting.protocol.body.ClusterInfo(); - java.util.HashMap table = - new java.util.HashMap<>(); - org.apache.rocketmq.remoting.protocol.route.BrokerData broker = - new org.apache.rocketmq.remoting.protocol.route.BrokerData(); - broker.setBrokerName("broker-a"); - table.put("broker-a", broker); - clusterInfo.setBrokerAddrTable(table); - assertEquals(Set.of("broker-a"), RocketMqAgent.collectBrokerNames(clusterInfo)); - assertEquals(Set.of(), RocketMqAgent.collectBrokerNames(null)); - } - - @Test - void resolveNameServerAddrSetSplitsMultiAddr() { - JsonObject conn = JsonParser.parseString(""" - {"namesrv_addr":"127.0.0.1:9876;192.168.1.2:9876"} - """).getAsJsonObject(); - assertEquals( - Set.of("127.0.0.1:9876", "192.168.1.2:9876"), - RocketMqAgent.resolveNameServerAddrSet(conn) - ); - } - - @Test - void buildTopicConfigForCreateSetsPartitionsAndMessageType() { - TopicConfig config = RocketMqAgent.buildTopicConfigForCreate("OrderCreated", 8, "DELAY"); - assertEquals("OrderCreated", config.getTopicName()); - assertEquals(8, config.getReadQueueNums()); - assertEquals(8, config.getWriteQueueNums()); - assertEquals(6, config.getPerm()); - assertEquals("DELAY", config.getAttributes().get("+message.type")); - } - - @Test - void buildTopicConfigForCreateSupportsSeparateQueuesAndPerm() { - TopicConfig config = RocketMqAgent.buildTopicConfigForCreate("T1", 8, 4, "NORMAL", 4); - assertEquals(8, config.getReadQueueNums()); - assertEquals(4, config.getWriteQueueNums()); - assertEquals(4, config.getPerm()); - } - - @Test - void normalizeTopicPermAllowsReadWriteValues() { - assertEquals(6, RocketMqAgent.normalizeTopicPerm(6)); - assertEquals(4, RocketMqAgent.normalizeTopicPerm(4)); - assertEquals(2, RocketMqAgent.normalizeTopicPerm(2)); - assertEquals(6, RocketMqAgent.normalizeTopicPerm(7)); - } - - @Test - void classifyConsumerGroupTypeMatchesDashboard() { - assertEquals("SYSTEM", RocketMqAgent.classifyConsumerGroupType(MixAll.CID_SYS_RMQ_TRANS, null)); - SubscriptionGroupConfig fifo = new SubscriptionGroupConfig(); - fifo.setConsumeMessageOrderly(true); - assertEquals("FIFO", RocketMqAgent.classifyConsumerGroupType("OrderGroup", fifo)); - SubscriptionGroupConfig normal = new SubscriptionGroupConfig(); - normal.setConsumeMessageOrderly(false); - assertEquals("NORMAL", RocketMqAgent.classifyConsumerGroupType("MyGroup", normal)); - // Missing dump must not look like NORMAL (FIFO groups would be mislabeled). - assertEquals("UNKNOWN", RocketMqAgent.classifyConsumerGroupType("MissingConfigGroup", null)); - } - - @Test - void enrichConsumerGroupRowsRunsIndependentLookupsConcurrently() throws Exception { - List> rows = IntStream.range(0, 4) - .mapToObj(index -> { - Map row = new LinkedHashMap<>(); - row.put("groupId", "group-" + index); - return row; - }) - .toList(); - CountDownLatch started = new CountDownLatch(rows.size()); - CountDownLatch release = new CountDownLatch(1); - AtomicInteger active = new AtomicInteger(); - AtomicInteger maxActive = new AtomicInteger(); - - CompletableFuture enrichment = CompletableFuture.runAsync(() -> - RocketMqAgent.enrichConsumerGroupRows(rows, groupId -> { - int current = active.incrementAndGet(); - maxActive.accumulateAndGet(current, Math::max); - started.countDown(); - try { - release.await(1, TimeUnit.SECONDS); - return new ConsumerConnection(); - } finally { - active.decrementAndGet(); - } - }, 4, 1_000) - ); - - assertTrue(started.await(1, TimeUnit.SECONDS)); - release.countDown(); - enrichment.get(2, TimeUnit.SECONDS); - - assertTrue(maxActive.get() > 1); - for (Map row : rows) { - // Successful empty connection probe → genuine offline 0. - assertEquals(0, row.get("memberCount")); - assertEquals(List.of(), row.get("topics")); - } - } - - @Test - void requireConsumerConnectionProbeResultFailsClosedWhenNoMasterAnswers() throws Exception { - assertThrows( - MQClientException.class, - () -> RocketMqAgent.requireConsumerConnectionProbeResult( - 1, 0, 0, 0, null, new IllegalStateException("timeout"), "g1") - ); - ConsumerConnection offline = new ConsumerConnection(); - assertSame( - offline, - RocketMqAgent.requireConsumerConnectionProbeResult(1, 1, 0, 0, offline, null, "g1") - ); - } - - @Test - void requireConsumerConnectionProbeResultFailsClosedOnRemapped206WithUnreachableFallback() { - Exception cause = new IllegalStateException("fallback unreachable"); - MQClientException err = assertThrows( - MQClientException.class, - () -> RocketMqAgent.requireConsumerConnectionProbeResult( - 1, 1, 1, 0, new ConsumerConnection(), cause, "g1") - ); - assertTrue(err.getMessage().contains("Docker remap address collision")); - assertSame(cause, err.getCause()); - } - - @Test - void requireConsumerConnectionProbeResultFailsClosedOnPartialRemappedOffline() { - Exception cause = new IllegalStateException("broker-b timeout"); - MQClientException err = assertThrows( - MQClientException.class, - () -> RocketMqAgent.requireConsumerConnectionProbeResult( - 2, 1, 0, 0, new ConsumerConnection(), cause, "g1") - ); - assertTrue(err.getMessage().contains("partial remapped probe")); - assertSame(cause, err.getCause()); - } - - @Test - void requireConsumerConnectionProbeResultAllowsFullOfflineCoverage() throws Exception { - ConsumerConnection offline = new ConsumerConnection(); - assertSame( - offline, - RocketMqAgent.requireConsumerConnectionProbeResult(2, 2, 0, 0, offline, null, "g1") - ); - assertSame( - offline, - RocketMqAgent.requireConsumerConnectionProbeResult(1, 1, 1, 1, offline, null, "g1") - ); - } - - @Test - void isConsumerGroupNotOnlineDetectsBrokerCode206() { - assertTrue(RocketMqAgent.isConsumerGroupNotOnline( - new MQBrokerException(206, "the consumer group[g1] not online"))); - assertTrue(RocketMqAgent.isConsumerGroupNotOnline( - new MQClientException(206, "the consumer group[g1] not online"))); - assertTrue(RocketMqAgent.isConsumerGroupNotOnline( - new RuntimeException(new MQBrokerException(206, "not online")))); - assertFalse(RocketMqAgent.isConsumerGroupNotOnline( - new MQBrokerException(1, "system error"))); - assertFalse(RocketMqAgent.isConsumerGroupNotOnline(new IllegalStateException("timeout"))); - } - - @Test - void enrichConsumerGroupRowsOmitsMemberCountWhenConnectionProbeFails() { - List> rows = new ArrayList<>(); - Map row = new LinkedHashMap<>(); - row.put("groupId", "unreachable-group"); - rows.add(row); - - RocketMqAgent.enrichConsumerGroupRows(rows, groupId -> { - throw new MQClientException("Failed to examine consumer connection on all masters", null); - }, 1, 1_000); - - // All-master connection probe failure must not look like offline 0. - assertFalse(row.containsKey("memberCount")); - assertEquals(List.of(), row.get("topics")); - } - - @Test - void enrichConsumerGroupRowsOmitsMemberCountWhenBudgetExpires() { - List> rows = IntStream.range(0, 4) - .mapToObj(index -> { - Map row = new LinkedHashMap<>(); - row.put("groupId", "slow-group-" + index); - return row; - }) - .toList(); - CountDownLatch blocked = new CountDownLatch(1); - long startedAt = System.nanoTime(); - - RocketMqAgent.enrichConsumerGroupRows(rows, groupId -> { - blocked.await(); - return new ConsumerConnection(); - }, 2, 50); - - long elapsedMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAt); - assertTrue(elapsedMs < 1_000, "enrichment exceeded its response budget: " + elapsedMs + "ms"); - for (Map row : rows) { - // Cancelled probes must not look offline (0); UI shows '-' when memberCount is absent. - assertFalse(row.containsKey("memberCount")); - assertEquals(List.of(), row.get("topics")); - } - } - - @Test - void enrichTopicPartitionsRunsIndependentLookupsConcurrently() throws Exception { - List> rows = IntStream.range(0, 4) - .mapToObj(index -> { - Map row = new LinkedHashMap<>(); - row.put("name", "topic-" + index); - row.put("partitions", 1); - return row; - }) - .toList(); - CountDownLatch started = new CountDownLatch(rows.size()); - CountDownLatch release = new CountDownLatch(1); - AtomicInteger active = new AtomicInteger(); - AtomicInteger maxActive = new AtomicInteger(); - - CompletableFuture enrichment = CompletableFuture.runAsync(() -> - RocketMqAgent.enrichTopicPartitions(rows, topic -> { - int current = active.incrementAndGet(); - maxActive.accumulateAndGet(current, Math::max); - started.countDown(); - try { - release.await(1, TimeUnit.SECONDS); - return routeWithReadQueues(8); - } finally { - active.decrementAndGet(); - } - }, 4, 1_000) - ); - - assertTrue(started.await(1, TimeUnit.SECONDS)); - release.countDown(); - enrichment.get(2, TimeUnit.SECONDS); - - assertTrue(maxActive.get() > 1); - for (Map row : rows) { - assertEquals(8, row.get("partitions")); - } - } - - @Test - void enrichTopicPartitionsReturnsDefaultsWhenBudgetExpires() { - List> rows = IntStream.range(0, 4) - .mapToObj(index -> { - Map row = new LinkedHashMap<>(); - row.put("name", "slow-topic-" + index); - row.put("partitions", 1); - return row; - }) - .toList(); - CountDownLatch blocked = new CountDownLatch(1); - long startedAt = System.nanoTime(); - - RocketMqAgent.enrichTopicPartitions(rows, topic -> { - blocked.await(); - return routeWithReadQueues(16); - }, 2, 50); - - long elapsedMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startedAt); - assertTrue(elapsedMs < 1_000, "topic route enrichment exceeded its response budget: " + elapsedMs + "ms"); - for (Map row : rows) { - assertEquals(1, row.get("partitions")); - } - } - - @Test - void buildTopicCatalogRowsFallbackIssuesAtMostOneRouteRpcPerTopic() { - Set topicNames = new TreeSet<>(List.of("Alpha", "Beta", "Gamma")); - Map routeCalls = new ConcurrentHashMap<>(); - AtomicInteger totalRouteCalls = new AtomicInteger(); - - List> rows = RocketMqAgent.buildTopicCatalogRows( - topicNames, - Collections.emptyMap(), - Collections.emptyMap(), - Set.of(), - Set.of(), - "DefaultCluster", - "", - topic -> { - totalRouteCalls.incrementAndGet(); - routeCalls.computeIfAbsent(topic, ignored -> new AtomicInteger()).incrementAndGet(); - return routeWithReadQueues(4); - }, - 8, - 2_000 - ); - - assertEquals(3, rows.size()); - assertEquals(3, totalRouteCalls.get(), "fallback must not repeat examineTopicRouteInfo per topic"); - for (String topic : topicNames) { - assertEquals(1, routeCalls.get(topic).get()); - } - for (Map row : rows) { - assertEquals(4, row.get("partitions")); - // Bulk config unavailable — classify without per-topic examineTopicConfig. - assertEquals("UNSPECIFIED", row.get("messageType")); - } - } - - @Test - void buildTopicCatalogRowsUsesBrokerConfigWithoutRouteLookup() { - TopicConfig config = new TopicConfig("Orders"); - config.setReadQueueNums(12); - config.setAttributes(Map.of("message.type", "NORMAL")); - Map brokerTopics = Map.of("Orders", config); - AtomicInteger routeCalls = new AtomicInteger(); - - List> rows = RocketMqAgent.buildTopicCatalogRows( - Set.of("Orders"), - brokerTopics, - Map.of("Orders", Map.of("message.type", "NORMAL")), - Set.of(), - Set.of(), - "DefaultCluster", - "", - topic -> { - routeCalls.incrementAndGet(); - return routeWithReadQueues(99); - }, - 8, - 2_000 - ); - - assertEquals(1, rows.size()); - assertEquals(0, routeCalls.get(), "broker catalog path must not call examineTopicRouteInfo"); - assertEquals(12, rows.get(0).get("partitions")); - assertEquals("NORMAL", rows.get(0).get("messageType")); - } - - @Test - void partialBrokerTopicConfigsMergeNameServerCatalog() { - TopicConfig orders = new TopicConfig("Orders"); - TopicConfigSerializeWrapper wrapper = new TopicConfigSerializeWrapper(); - wrapper.setTopicConfigTable(new ConcurrentHashMap<>(Map.of("Orders", orders))); - - RocketMqAgent.BrokerTopicConfigSnapshot snapshot = RocketMqAgent.collectBrokerTopicConfigs( - List.of("broker-a", "broker-b"), - brokerAddr -> { - if (brokerAddr.equals("broker-b")) { - throw new IllegalStateException("broker unavailable"); - } - return wrapper; - } - ); - - assertFalse(snapshot.complete()); - assertEquals(Set.of("Orders"), snapshot.topics().keySet()); - assertEquals( - Set.of("Invoices", "Orders"), - RocketMqAgent.topicCatalogNames(snapshot, Set.of("Invoices", "Orders")) - ); - } - - @Test - void completeBrokerTopicConfigsRemainAuthoritative() { - TopicConfig orders = new TopicConfig("Orders"); - TopicConfigSerializeWrapper wrapper = new TopicConfigSerializeWrapper(); - wrapper.setTopicConfigTable(new ConcurrentHashMap<>(Map.of("Orders", orders))); - - RocketMqAgent.BrokerTopicConfigSnapshot snapshot = RocketMqAgent.collectBrokerTopicConfigs( - List.of("broker-a"), - brokerAddr -> wrapper - ); - - assertTrue(snapshot.complete()); - assertEquals( - Set.of("Orders"), - RocketMqAgent.topicCatalogNames(snapshot, Set.of("DeletedButStillRouted", "Orders")) - ); - } - - private static TopicRouteData routeWithReadQueues(int readQueueNums) { - QueueData queueData = new QueueData(); - queueData.setBrokerName("broker-a"); - queueData.setReadQueueNums(readQueueNums); - queueData.setWriteQueueNums(readQueueNums); - TopicRouteData route = new TopicRouteData(); - route.setQueueDatas(List.of(queueData)); - return route; - } - - @Test - void isEmptyQueryMessageResultDetectsRocketMqCode208() { - MQClientException empty = new MQClientException(208, "query message by key finished, but no message"); - assertTrue(RocketMqAgent.isEmptyQueryMessageResult(empty)); - MQClientException other = new MQClientException(1, "other error"); - assertFalse(RocketMqAgent.isEmptyQueryMessageResult(other)); - } - - @Test - void shouldQueryClusterProducerTableOnlyWithoutTopicOrGroup() { - assertTrue(RocketMqAgent.shouldQueryClusterProducerTable("", "")); - assertTrue(RocketMqAgent.shouldQueryClusterProducerTable(null, null)); - assertFalse(RocketMqAgent.shouldQueryClusterProducerTable("CS-SW", "")); - assertFalse(RocketMqAgent.shouldQueryClusterProducerTable("CS-SW", null)); - assertFalse(RocketMqAgent.shouldQueryClusterProducerTable("", "p-test")); - assertFalse(RocketMqAgent.shouldQueryClusterProducerTable("CS-SW", "p-test")); - } - - @Test - void topicHasProduceActivityDetectsQueuedMessages() { - TopicStatsTable empty = new TopicStatsTable(); - MessageQueue queue = new MessageQueue("CS-SX", "broker-a", 0); - TopicOffset offset = new TopicOffset(); - offset.setMinOffset(0); - offset.setMaxOffset(0); - empty.getOffsetTable().put(queue, offset); - assertFalse(RocketMqAgent.topicHasProduceActivity(empty)); - assertFalse(RocketMqAgent.topicHasProduceActivity(null)); - - TopicStatsTable active = new TopicStatsTable(); - TopicOffset activeOffset = new TopicOffset(); - activeOffset.setMinOffset(0); - activeOffset.setMaxOffset(2); - active.getOffsetTable().put(new MessageQueue("CS-PT", "broker-a", 0), activeOffset); - assertTrue(RocketMqAgent.topicHasProduceActivity(active)); - } - - @Test - void appendProducerTableRowsDedupesDuplicateConnections() { - ProducerInfo first = new ProducerInfo("client-1", "127.0.0.1:39688", null, 5050, 1L); - ProducerInfo duplicate = new ProducerInfo("client-2", "127.0.0.1:39688", null, 5050, 2L); - - Map> data = new LinkedHashMap<>(); - data.put("CLIENT_INNER_PRODUCER", List.of(first, duplicate)); - ProducerTableInfo tableInfo = new ProducerTableInfo(data); - - List> producers = new ArrayList<>(); - Set seen = new LinkedHashSet<>(); - RocketMqAgent.appendProducerTableRows(producers, seen, 1L, tableInfo); - - assertEquals(1, producers.size()); - assertEquals("CLIENT_INNER_PRODUCER", producers.get(0).get("producerName")); - assertEquals("127.0.0.1:39688", producers.get(0).get("address")); - } - - @Test - void connectionMatchesComparesRocketMqConnectFields() { - JsonObject base = JsonParser.parseString(""" - { - "namesrv_addr": "127.0.0.1:9876", - "cluster_name": "DefaultCluster", - "broker_addr": "", - "access_key": "ak", - "secret_key": "sk" - } - """).getAsJsonObject(); - JsonObject same = base.deepCopy(); - JsonObject differentNamesrv = base.deepCopy(); - differentNamesrv.addProperty("namesrv_addr", "127.0.0.1:9877"); - - assertTrue(RocketMqAgent.connectionMatches(base, same)); - assertFalse(RocketMqAgent.connectionMatches(base, differentNamesrv)); - } - - @Test - void buildConsumerLagResultIncludesBrokerClientAndTimestamp() { - MessageQueue mq0 = new MessageQueue("TX_TOPIC", "broker-a", 0); - MessageQueue mq1 = new MessageQueue("TX_TOPIC", "broker-a", 1); - OffsetWrapper offset0 = new OffsetWrapper(); - offset0.setBrokerOffset(100); - offset0.setConsumerOffset(90); - offset0.setLastTimestamp(1_725_000_000_000L); - OffsetWrapper offset1 = new OffsetWrapper(); - offset1.setBrokerOffset(50); - offset1.setConsumerOffset(50); - offset1.setLastTimestamp(0L); - - ConsumeStats stats = new ConsumeStats(); - stats.getOffsetTable().put(mq0, offset0); - stats.getOffsetTable().put(mq1, offset1); - - Map clients = new HashMap<>(); - clients.put(mq0, "172.18.2.212@7#1"); - - Map result = RocketMqAgent.buildConsumerLagResult(stats, clients); - assertEquals(10L, result.get("totalLag")); - - @SuppressWarnings("unchecked") - List> partitions = (List>) result.get("partitions"); - assertEquals(2, partitions.size()); - assertEquals("broker-a", partitions.get(0).get("brokerName")); - assertEquals(0, partitions.get(0).get("partition")); - assertEquals(90L, partitions.get(0).get("currentOffset")); - assertEquals(100L, partitions.get(0).get("endOffset")); - assertEquals(10L, partitions.get(0).get("lag")); - assertEquals(1_725_000_000_000L, partitions.get(0).get("lastTimestamp")); - assertEquals("172.18.2.212@7#1", partitions.get(0).get("consumerClient")); - // Missing client mapping still returns offsets with empty consumerClient. - assertEquals("", partitions.get(1).get("consumerClient")); - assertEquals(0L, partitions.get(1).get("lastTimestamp")); - } - - @Test - void buildConsumerLagResultHandlesNullStatsAndEmptyClientMap() { - Map empty = RocketMqAgent.buildConsumerLagResult(null, null); - assertEquals(0L, empty.get("totalLag")); - @SuppressWarnings("unchecked") - List> partitions = (List>) empty.get("partitions"); - assertTrue(partitions.isEmpty()); - } - - @Test - void mergeSubscriptionGroupConfigsPrefersFifoAndKeepsLaterOnlyGroups() { - Map merged = new LinkedHashMap<>(); - - SubscriptionGroupConfig normalA = new SubscriptionGroupConfig(); - normalA.setGroupName("group-a"); - normalA.setConsumeMessageOrderly(false); - - SubscriptionGroupConfig fifoA = new SubscriptionGroupConfig(); - fifoA.setGroupName("group-a"); - fifoA.setConsumeMessageOrderly(true); - - SubscriptionGroupConfig onlyOnLater = new SubscriptionGroupConfig(); - onlyOnLater.setGroupName("group-b"); - onlyOnLater.setConsumeMessageOrderly(true); - - // Intermediate broker repeats group-a as NORMAL (size would not grow), then later adds FIFO-only group-b. - RocketMqAgent.mergeSubscriptionGroupConfigs(merged, Map.of("group-a", normalA)); - RocketMqAgent.mergeSubscriptionGroupConfigs(merged, Map.of("group-a", normalA)); - RocketMqAgent.mergeSubscriptionGroupConfigs( - merged, - Map.of("group-a", fifoA, "group-b", onlyOnLater) - ); - - assertEquals(2, merged.size()); - assertTrue(merged.get("group-a").isConsumeMessageOrderly()); - assertTrue(merged.get("group-b").isConsumeMessageOrderly()); - assertEquals("FIFO", RocketMqAgent.classifyConsumerGroupType("group-a", merged.get("group-a"))); - assertEquals("FIFO", RocketMqAgent.classifyConsumerGroupType("group-b", merged.get("group-b"))); - } - - @Test - void mergeSubscriptionGroupConfigsDoesNotDowngradeFifoToNormal() { - Map merged = new LinkedHashMap<>(); - SubscriptionGroupConfig fifo = new SubscriptionGroupConfig(); - fifo.setGroupName("ordered"); - fifo.setConsumeMessageOrderly(true); - SubscriptionGroupConfig normal = new SubscriptionGroupConfig(); - normal.setGroupName("ordered"); - normal.setConsumeMessageOrderly(false); - - RocketMqAgent.mergeSubscriptionGroupConfigs(merged, Map.of("ordered", fifo)); - RocketMqAgent.mergeSubscriptionGroupConfigs(merged, Map.of("ordered", normal)); - - assertTrue(merged.get("ordered").isConsumeMessageOrderly()); - } - - @Test - void ensureConsumeStatsProbeSucceededRejectsAllBrokerFailures() { - MQClientException noMasters = assertThrows( - MQClientException.class, - () -> RocketMqAgent.ensureConsumeStatsProbeSucceeded(0, 0, true, null, "GID_A")); - assertTrue(noMasters.getMessage().contains("No reachable RocketMQ master")); - - Exception cause = new RuntimeException("broker down"); - MQClientException allFailed = assertThrows( - MQClientException.class, - () -> RocketMqAgent.ensureConsumeStatsProbeSucceeded(2, 0, true, cause, "GID_A")); - assertTrue(allFailed.getMessage().contains("Failed to examine consume stats")); - assertSame(cause, allFailed.getCause()); - } - - @Test - void ensureConsumeStatsProbeSucceededRejectsPartialFailureWithEmptyMerge() { - Exception cause = new RuntimeException("one broker down"); - MQClientException partialEmpty = assertThrows( - MQClientException.class, - () -> RocketMqAgent.ensureConsumeStatsProbeSucceeded(3, 1, true, cause, "GID_A")); - assertTrue(partialEmpty.getMessage().contains("partial failure")); - assertSame(cause, partialEmpty.getCause()); - } - - @Test - void ensureConsumeStatsProbeSucceededAllowsEmptyOffsetsWhenAllBrokersSucceed() throws Exception { - // Genuine offline / unused group: every broker answered, offset table empty. - RocketMqAgent.ensureConsumeStatsProbeSucceeded(1, 1, true, null, "GID_A"); - RocketMqAgent.ensureConsumeStatsProbeSucceeded(3, 3, true, null, "GID_A"); - } - - @Test - void ensureConsumeStatsProbeSucceededAllowsPartialFailureWhenMergeHasOffsets() throws Exception { - // One broker failed but another returned queue offsets — keep partial lag. - RocketMqAgent.ensureConsumeStatsProbeSucceeded( - 3, 1, false, new RuntimeException("partial"), "GID_A"); - } - - @Test - void shouldFailClosedOnCollisionEmptyWhenSiblingFallbacksUnreachable() { - assertTrue(RocketMqAgent.shouldFailClosedOnCollisionEmpty(true, 1, 0)); - assertTrue(RocketMqAgent.shouldFailClosedOnCollisionEmpty(true, 2, 1)); - assertFalse(RocketMqAgent.shouldFailClosedOnCollisionEmpty(false, 1, 0)); - assertFalse(RocketMqAgent.shouldFailClosedOnCollisionEmpty(true, 0, 0)); - assertFalse(RocketMqAgent.shouldFailClosedOnCollisionEmpty(true, 2, 2)); - } - - @Test - void shouldFailClosedOnCollisionPartialMutationWhenSiblingFallbacksUnreachable() { - assertTrue(RocketMqAgent.shouldFailClosedOnCollisionPartialMutation(1, 0)); - assertTrue(RocketMqAgent.shouldFailClosedOnCollisionPartialMutation(2, 1)); - assertFalse(RocketMqAgent.shouldFailClosedOnCollisionPartialMutation(0, 0)); - assertFalse(RocketMqAgent.shouldFailClosedOnCollisionPartialMutation(2, 2)); - } - - @Test - void ensureSubscriptionGroupMutationSucceededRejectsCollisionPartialUpdate() { - Exception cause = new RuntimeException("fallback unreachable"); - MQClientException err = assertThrows( - MQClientException.class, - () -> RocketMqAgent.ensureSubscriptionGroupMutationSucceeded( - 1, 1, 1, 0, cause, "delete", "GID_A")); - assertTrue(err.getMessage().contains("Docker remap address collision")); - assertSame(cause, err.getCause()); - } - - @Test - void ensureSubscriptionGroupMutationSucceededRejectsPartialRemappedUpdate() { - Exception cause = new RuntimeException("broker-b down"); - MQClientException err = assertThrows( - MQClientException.class, - () -> RocketMqAgent.ensureSubscriptionGroupMutationSucceeded( - 2, 1, 0, 0, cause, "update", "GID_A")); - assertTrue(err.getMessage().contains("partial remapped update")); - assertSame(cause, err.getCause()); - } - - @Test - void ensureSubscriptionGroupMutationSucceededAllowsFullCoverage() throws Exception { - RocketMqAgent.ensureSubscriptionGroupMutationSucceeded(2, 2, 0, 0, null, "delete", "GID_A"); - RocketMqAgent.ensureSubscriptionGroupMutationSucceeded(1, 1, 1, 1, null, "update", "GID_A"); - } - - @Test - void masterBrokerAddrsFromClusterInfoKeepsCollisionFallbacksSeparate() { - org.apache.rocketmq.remoting.protocol.body.ClusterInfo clusterInfo = - new org.apache.rocketmq.remoting.protocol.body.ClusterInfo(); - HashMap table = new HashMap<>(); - table.put("broker-a", brokerData("broker-a", "172.18.0.2:10911")); - table.put("broker-b", brokerData("broker-b", "172.18.0.3:10911")); - clusterInfo.setBrokerAddrTable(table); - - JsonObject conn = JsonParser.parseString(""" - {"namesrvAddr":"127.0.0.1:9876"} - """).getAsJsonObject(); - RocketMqAgent.MasterBrokerAddrPlan plan = - RocketMqAgent.masterBrokerAddrsFromClusterInfo(clusterInfo, conn, null); - - // Both masters remap to the same published host:port; one remapped + one fallback. - assertEquals(1, plan.remappedCount()); - assertEquals(1, plan.fallbackCount()); - assertTrue(plan.allAddrs().contains("127.0.0.1:10911")); - assertTrue(plan.isCollisionFallback("172.18.0.3:10911") || plan.isCollisionFallback("172.18.0.2:10911")); - // Remapped answered with offsets: collision fallbacks may stay unreachable on host agents. - try { - RocketMqAgent.ensureConsumeStatsProbeSucceeded( - plan.remappedCount(), 1, false, new RuntimeException("fallback unreachable"), "GID_A"); - } catch (MQClientException e) { - throw new AssertionError("non-empty remapped merge should be allowed", e); - } - // Empty merge is still allowed by the shared gate when remappedCount==success; the - // examineConsumeStatsOnMasters collision check rejects empty+unreachable siblings. - try { - RocketMqAgent.ensureConsumeStatsProbeSucceeded( - plan.remappedCount(), 1, true, null, "GID_A"); - } catch (MQClientException e) { - throw new AssertionError("empty remapped-only success remains allowed at gate layer", e); - } - } - - @Test - void operationBudgetMsFollowsRequestTimeoutMs() { - JsonObject conn120 = JsonParser.parseString(""" - {"request_timeout_ms":120000} - """).getAsJsonObject(); - assertEquals(120_000L, RocketMqAgent.operationBudgetMs(conn120)); - assertEquals(120_000, RocketMqAgent.operationBudgetMsAsInt(conn120)); - - JsonObject connDefault = JsonParser.parseString("{}").getAsJsonObject(); - assertEquals(30_000L, RocketMqAgent.operationBudgetMs(connDefault)); - assertEquals(30_000, RocketMqAgent.operationBudgetMsAsInt(connDefault)); - - JsonObject connTiny = JsonParser.parseString(""" - {"request_timeout_ms":100} - """).getAsJsonObject(); - // Floor at 1s so budgets cannot collapse to zero. - assertEquals(1_000L, RocketMqAgent.operationBudgetMs(connTiny)); - assertEquals(1_000, RocketMqAgent.operationBudgetMsAsInt(connTiny)); - } - - @Test - void scaledOperationBudgetMsKeepsDefaultPhasesUnderRpcWindow() { - JsonObject connDefault = JsonParser.parseString("{}").getAsJsonObject(); - assertEquals(12_000L, RocketMqAgent.scaledOperationBudgetMs(connDefault, 12_000L)); - assertEquals(8_000L, RocketMqAgent.scaledOperationBudgetMs(connDefault, 8_000L)); - assertEquals(10_000L, RocketMqAgent.scaledOperationBudgetMs(connDefault, 10_000L)); - - JsonObject conn120 = JsonParser.parseString(""" - {"request_timeout_ms":120000} - """).getAsJsonObject(); - assertEquals(48_000L, RocketMqAgent.scaledOperationBudgetMs(conn120, 12_000L)); - assertEquals(32_000L, RocketMqAgent.scaledOperationBudgetMs(conn120, 8_000L)); - assertEquals(40_000L, RocketMqAgent.scaledOperationBudgetMs(conn120, 10_000L)); - // Never exceed the RPC window even if baseline is larger. - assertEquals(120_000L, RocketMqAgent.scaledOperationBudgetMs(conn120, 200_000L)); - } - - @Test - void collectConsumerGroupConfigsSkipsFallbacksWhenRemappedNonEmpty() throws Exception { - RocketMqAgent.MasterBrokerAddrPlan plan = collisionPlan( - "127.0.0.1:10911", "172.18.0.3:10911"); - List probed = new ArrayList<>(); - SubscriptionGroupConfig group = new SubscriptionGroupConfig(); - group.setGroupName("GID_A"); - - Map configs = RocketMqAgent.collectConsumerGroupConfigs( - plan, - 2, - 30_000L, - 30_000L, - 2_000L, - (addr, timeout) -> { - probed.add(addr); - if (plan.isCollisionFallback(addr)) { - throw new AssertionError("collision-fallback must not be probed after non-empty remapped collect"); - } - return Map.of("GID_A", group); - } - ); - - assertEquals(List.of("127.0.0.1:10911"), probed); - assertEquals(1, configs.size()); - assertTrue(configs.containsKey("GID_A")); - } - - @Test - void collectConsumerGroupConfigsProbesFallbacksWhenRemappedEmpty() throws Exception { - RocketMqAgent.MasterBrokerAddrPlan plan = collisionPlan( - "127.0.0.1:10911", "172.18.0.3:10911"); - List probed = new ArrayList<>(); - SubscriptionGroupConfig group = new SubscriptionGroupConfig(); - group.setGroupName("GID_B"); - - Map configs = RocketMqAgent.collectConsumerGroupConfigs( - plan, - 2, - 30_000L, - 30_000L, - 2_000L, - (addr, timeout) -> { - probed.add(addr); - if (plan.isCollisionFallback(addr)) { - assertEquals(2_000L, timeout); - return Map.of("GID_B", group); - } - return Map.of(); - } - ); - - assertEquals(2, probed.size()); - assertTrue(probed.contains("127.0.0.1:10911")); - assertTrue(probed.contains("172.18.0.3:10911")); - assertEquals(1, configs.size()); - assertTrue(configs.containsKey("GID_B")); - } - - @Test - void examineConsumeStatsOnMastersSkipsFallbacksWhenRemappedHasOffsets() throws Exception { - RocketMqAgent.MasterBrokerAddrPlan plan = collisionPlan( - "127.0.0.1:10911", "172.18.0.3:10911"); - List probed = new ArrayList<>(); - ConsumeStats remappedStats = nonEmptyConsumeStats("TOPIC_A", "broker-a", 0); - - ConsumeStats merged = RocketMqAgent.examineConsumeStatsOnMasters( - "GID_A", - "TOPIC_A", - plan, - 30_000L, - 2_000L, - (addr, groupId, topic, timeout) -> { - probed.add(addr); - if (plan.isCollisionFallback(addr)) { - throw new AssertionError("collision-fallback must not be probed after non-empty remapped"); - } - assertEquals(30_000L, timeout); - return remappedStats; - } - ); - - assertEquals(List.of("127.0.0.1:10911"), probed); - assertEquals(1, merged.getOffsetTable().size()); - } - - @Test - void examineConsumeStatsOnMastersProbesFallbacksAndFailClosesOnEmptyUnreachable() { - RocketMqAgent.MasterBrokerAddrPlan plan = collisionPlan( - "127.0.0.1:10911", "172.18.0.3:10911"); - List probed = new ArrayList<>(); - - MQClientException err = assertThrows( - MQClientException.class, - () -> RocketMqAgent.examineConsumeStatsOnMasters( - "GID_A", - "TOPIC_A", - plan, - 30_000L, - 2_000L, - (addr, groupId, topic, timeout) -> { - probed.add(addr); - if (plan.isCollisionFallback(addr)) { - assertEquals(2_000L, timeout); - throw new RuntimeException("docker internal unreachable"); - } - assertEquals(30_000L, timeout); - return new ConsumeStats(); - } - ) - ); - - assertTrue(err.getMessage().contains("Docker remap address collision")); - assertEquals(2, probed.size()); - assertTrue(probed.contains("127.0.0.1:10911")); - assertTrue(probed.contains("172.18.0.3:10911")); - } - - @Test - void examineConsumeStatsOnMastersUsesFallbackWhenRemappedFails() throws Exception { - RocketMqAgent.MasterBrokerAddrPlan plan = collisionPlan( - "127.0.0.1:10911", "172.18.0.3:10911"); - List probed = new ArrayList<>(); - ConsumeStats fallbackStats = nonEmptyConsumeStats("TOPIC_A", "broker-b", 1); - - ConsumeStats merged = RocketMqAgent.examineConsumeStatsOnMasters( - "GID_A", - "TOPIC_A", - plan, - 30_000L, - 2_000L, - (addr, groupId, topic, timeout) -> { - probed.add(addr); - if (!plan.isCollisionFallback(addr)) { - assertEquals(30_000L, timeout); - throw new RuntimeException("published remap unreachable"); - } - assertEquals(2_000L, timeout); - return fallbackStats; - } - ); - - assertEquals(2, probed.size()); - assertEquals(1, merged.getOffsetTable().size()); - } - - private static RocketMqAgent.MasterBrokerAddrPlan collisionPlan( - String remapped, String fallback) { - RocketMqAgent.MasterBrokerAddrPlan plan = new RocketMqAgent.MasterBrokerAddrPlan(); - plan.addRemapped(remapped); - plan.addCollisionFallback(fallback); - return plan; - } - - private static ConsumeStats nonEmptyConsumeStats(String topic, String broker, int queueId) { - MessageQueue mq = new MessageQueue(topic, broker, queueId); - OffsetWrapper offset = new OffsetWrapper(); - offset.setBrokerOffset(10); - offset.setConsumerOffset(0); - ConsumeStats stats = new ConsumeStats(); - stats.getOffsetTable().put(mq, offset); - return stats; - } - - private static org.apache.rocketmq.remoting.protocol.route.BrokerData brokerData( - String name, String masterAddr) { - org.apache.rocketmq.remoting.protocol.route.BrokerData data = - new org.apache.rocketmq.remoting.protocol.route.BrokerData(); - data.setBrokerName(name); - HashMap addrs = new HashMap<>(); - addrs.put(0L, masterAddr); - data.setBrokerAddrs(addrs); - return data; - } - - private static org.apache.rocketmq.remoting.protocol.body.ClusterInfo clusterInfoWithBrokers( - String... masterAddrs) { - org.apache.rocketmq.remoting.protocol.body.ClusterInfo clusterInfo = - new org.apache.rocketmq.remoting.protocol.body.ClusterInfo(); - HashMap table = new HashMap<>(); - for (int index = 0; index < masterAddrs.length; index++) { - String name = "broker-" + index; - table.put(name, brokerData(name, masterAddrs[index])); - } - clusterInfo.setBrokerAddrTable(table); - return clusterInfo; - } -} diff --git a/agents/drivers/rocketmq/src/test/java/com/dbx/agent/rocketmq/RocketMqLiveMessagePropertyTest.java b/agents/drivers/rocketmq/src/test/java/com/dbx/agent/rocketmq/RocketMqLiveMessagePropertyTest.java deleted file mode 100644 index 99f39cf38f..0000000000 --- a/agents/drivers/rocketmq/src/test/java/com/dbx/agent/rocketmq/RocketMqLiveMessagePropertyTest.java +++ /dev/null @@ -1,179 +0,0 @@ -package com.dbx.agent.rocketmq; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import java.nio.charset.StandardCharsets; -import java.util.Base64; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; -import org.apache.rocketmq.client.consumer.MessageSelector; -import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; -import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; -import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; -import org.apache.rocketmq.client.exception.MQClientException; -import org.apache.rocketmq.client.producer.DefaultMQProducer; -import org.apache.rocketmq.common.consumer.ConsumeFromWhere; -import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; - -/** - * Live RocketMQ tests on 127.0.0.1:9876. SQL92 filtering requires broker - * {@code enablePropertyFilter=true} (skipped when unsupported). - */ -class RocketMqLiveMessagePropertyTest { - private static final String NAMESRV = "127.0.0.1:9876"; - private static final String TOPIC = "CS-PT"; - private static DefaultMQAdminExt admin; - private static JsonObject conn; - private static boolean available; - - @BeforeAll - static void connect() { - conn = JsonParser.parseString(""" - {"namesrv_addr":"127.0.0.1:9876"} - """).getAsJsonObject(); - admin = new DefaultMQAdminExt(); - admin.setNamesrvAddr(NAMESRV); - try { - admin.start(); - admin.examineBrokerClusterInfo(); - available = true; - } catch (Exception e) { - available = false; - } - } - - @AfterAll - static void shutdown() { - if (admin != null) { - admin.shutdown(); - } - } - - @Test - void sendMessageUserPropertyRoundTripViaPeek() throws Exception { - assumeTrue(available, "RocketMQ namesrv not available on " + NAMESRV); - DefaultMQProducer producer = RocketMqAgent.buildProducer(conn); - try { - String marker = "dbx-prop-" + UUID.randomUUID(); - JsonObject sendParams = JsonParser.parseString(""" - { - "topic":"%s", - "tag":"prop-test", - "payloadBase64":"%s", - "headers":{"TAGS":"prop-test","Region":"Hangzhou","marker":"%s"} - } - """.formatted( - TOPIC, - Base64.getEncoder().encodeToString(marker.getBytes(StandardCharsets.UTF_8)), - marker - )).getAsJsonObject(); - - @SuppressWarnings("unchecked") - Map sendResult = (Map) RocketMqAgent.sendMessageForTest(producer, sendParams); - Number partition = (Number) sendResult.get("partition"); - Number offset = (Number) sendResult.get("offset"); - assertNotNull(partition); - assertNotNull(offset); - - JsonObject peekParams = JsonParser.parseString(""" - { - "connection":{"namesrv_addr":"127.0.0.1:9876"}, - "topic":"%s", - "count":1, - "partition":%d, - "offset":%d - } - """.formatted(TOPIC, partition.intValue(), offset.longValue())).getAsJsonObject(); - - @SuppressWarnings("unchecked") - Map peekResult = (Map) RocketMqAgent.peekMessagesForTest(admin, peekParams); - @SuppressWarnings("unchecked") - List> messages = (List>) peekResult.get("messages"); - assumeTrue(messages != null && !messages.isEmpty(), "peek returned no messages at sent offset"); - @SuppressWarnings("unchecked") - Map headers = (Map) messages.get(0).get("headers"); - assertEquals("Hangzhou", headers.get("Region")); - assertEquals(marker, headers.get("marker")); - } finally { - producer.shutdown(); - } - } - - @Test - void sql92ConsumerFiltersOnUserProperty() throws Exception { - assumeTrue(available, "RocketMQ namesrv not available on " + NAMESRV); - String group = "dbx-sql92-" + UUID.randomUUID(); - String markerMatch = "match-" + UUID.randomUUID(); - String markerSkip = "skip-" + UUID.randomUUID(); - CountDownLatch matched = new CountDownLatch(1); - AtomicInteger received = new AtomicInteger(); - - DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(group); - consumer.setNamesrvAddr(NAMESRV); - consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET); - try { - consumer.subscribe( - TOPIC, - MessageSelector.bySql("Region IS NOT NULL AND Region='Hangzhou' AND marker IS NOT NULL") - ); - } catch (MQClientException e) { - assumeTrue(false, "Broker does not support SQL92 property filter: " + e.getMessage()); - } - consumer.registerMessageListener((MessageListenerConcurrently) (messages, context) -> { - for (var message : messages) { - String marker = message.getProperty("marker"); - if (markerMatch.equals(marker)) { - received.incrementAndGet(); - matched.countDown(); - } - } - return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; - }); - - try { - consumer.start(); - } catch (MQClientException e) { - assumeTrue(false, "Consumer start failed for SQL92: " + e.getMessage()); - } - - DefaultMQProducer producer = RocketMqAgent.buildProducer(conn); - try { - sendWithMarker(producer, markerSkip, "Shanghai"); - sendWithMarker(producer, markerMatch, "Hangzhou"); - assumeTrue(matched.await(15, TimeUnit.SECONDS), "SQL92 consumer did not receive matching message"); - assertEquals(1, received.get(), "only the Hangzhou message should match the filter"); - } finally { - producer.shutdown(); - consumer.shutdown(); - } - } - - private static void sendWithMarker(DefaultMQProducer producer, String marker, String region) throws Exception { - JsonObject sendParams = JsonParser.parseString(""" - { - "topic":"%s", - "tag":"sql92-test", - "payloadBase64":"%s", - "headers":{"TAGS":"sql92-test","Region":"%s","marker":"%s"} - } - """.formatted( - TOPIC, - Base64.getEncoder().encodeToString(marker.getBytes(StandardCharsets.UTF_8)), - region, - marker - )).getAsJsonObject(); - RocketMqAgent.sendMessageForTest(producer, sendParams); - } -} diff --git a/agents/drivers/rocketmq/src/test/java/com/dbx/agent/rocketmq/RocketMqLiveTopicTypeTest.java b/agents/drivers/rocketmq/src/test/java/com/dbx/agent/rocketmq/RocketMqLiveTopicTypeTest.java deleted file mode 100644 index 96ac1e78ff..0000000000 --- a/agents/drivers/rocketmq/src/test/java/com/dbx/agent/rocketmq/RocketMqLiveTopicTypeTest.java +++ /dev/null @@ -1,125 +0,0 @@ -package com.dbx.agent.rocketmq; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import java.util.List; -import java.util.Map; -import org.apache.rocketmq.common.TopicConfig; -import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper; -import org.apache.rocketmq.tools.admin.DefaultMQAdminExt; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; - -/** - * Live verification against local RocketMQ (127.0.0.1:9876). Skipped when namesrv is unavailable. - */ -class RocketMqLiveTopicTypeTest { - private static DefaultMQAdminExt admin; - private static boolean available; - - @BeforeAll - static void connect() { - admin = new DefaultMQAdminExt(); - admin.setNamesrvAddr("127.0.0.1:9876"); - try { - admin.start(); - admin.examineBrokerClusterInfo(); - available = true; - } catch (Exception e) { - available = false; - } - } - - @AfterAll - static void shutdown() { - if (admin != null) { - admin.shutdown(); - } - } - - @Test - void cs123BrokerConfigHasNormalMessageType() throws Exception { - assumeTrue(available, "RocketMQ namesrv not available on 127.0.0.1:9876"); - JsonObject conn = JsonParser.parseString(""" - {"namesrv_addr":"127.0.0.1:9876"} - """).getAsJsonObject(); - String brokerAddr = RocketMqAgent.remapBrokerAddrForClient( - admin.examineBrokerClusterInfo().getBrokerAddrTable().values().iterator().next().selectBrokerAddr(), - conn); - TopicConfig config = admin.examineTopicConfig(brokerAddr, "CS123"); - assumeTrue(config != null, "CS123 topic config missing"); - String fromAttributes = config.getAttributes() == null ? null : config.getAttributes().get("message.type"); - assertEquals("NORMAL", fromAttributes, "attributes.message.type"); - } - - @Test - void getAllTopicConfigIncludesCs123MessageType() throws Exception { - assumeTrue(available, "RocketMQ namesrv not available on 127.0.0.1:9876"); - JsonObject conn = JsonParser.parseString(""" - {"namesrv_addr":"127.0.0.1:9876"} - """).getAsJsonObject(); - String brokerAddr = RocketMqAgent.remapBrokerAddrForClient( - admin.examineBrokerClusterInfo().getBrokerAddrTable().values().iterator().next().selectBrokerAddr(), - conn); - TopicConfigSerializeWrapper wrapper = admin.getAllTopicConfig(brokerAddr, 10_000L); - TopicConfig config = wrapper.getTopicConfigTable().get("CS123"); - assumeTrue(config != null, "CS123 missing from getAllTopicConfig"); - String fromAttributes = config.getAttributes() == null ? null : config.getAttributes().get("message.type"); - assertEquals("NORMAL", fromAttributes, "bulk attributes.message.type"); - } - - @Test - void agentListTopicsClassifiesAllTopicsAgainstBroker() throws Exception { - assumeTrue(available, "RocketMQ namesrv not available on 127.0.0.1:9876"); - JsonObject params = JsonParser.parseString(""" - {"connection":{"namesrv_addr":"127.0.0.1:9876"},"limit":500,"offset":0} - """).getAsJsonObject(); - Object result = RocketMqAgent.listTopicsWithAdminForTest(admin, params); - assumeTrue(result instanceof Map, "unexpected listTopics result"); - @SuppressWarnings("unchecked") - List> topics = (List>) ((Map) result).get("topics"); - assumeTrue(topics != null && !topics.isEmpty(), "no topics returned"); - - Map cs123 = topics.stream() - .filter(row -> "CS123".equals(String.valueOf(row.get("name")))) - .findFirst() - .orElse(null); - assumeTrue(cs123 != null, "CS123 not in agent topic list"); - assertEquals("NORMAL", String.valueOf(cs123.get("messageType")), "agent messageType for CS123"); - - Map expected = Map.ofEntries( - Map.entry("CS123", "NORMAL"), - Map.entry("BenchmarkTest", "SYSTEM"), - Map.entry("SCHEDULE_TOPIC_XXXX", "SYSTEM"), - Map.entry("RMQ_SYS_TRANS_HALF_TOPIC", "SYSTEM"), - Map.entry("RMQ_SYS_TRANS_OP_HALF_TOPIC", "SYSTEM"), - Map.entry("RMQ_SYS_ROCKSDB_TRANS_HALF_TOPIC", "SYSTEM"), - Map.entry("RMQ_SYS_ROCKSDB_TRANS_OP_HALF_TOPIC", "SYSTEM"), - Map.entry("rmq_sys_wheel_timer", "SYSTEM"), - Map.entry("rmq_sys_REVIVE_LOG_DefaultCluster", "SYSTEM"), - Map.entry("rmq_sys_SYNC_BROKER_MEMBER_37e06a7b803c", "SYSTEM"), - Map.entry("rmq_sys_SYNC_BROKER_MEMBER_broker-a", "SYSTEM"), - Map.entry("DefaultHeartBeatSyncerTopic", "SYSTEM"), - Map.entry("DefaultCluster_REPLY_TOPIC", "SYSTEM"), - Map.entry("SELF_TEST_TOPIC", "SYSTEM"), - Map.entry("OFFSET_MOVED_EVENT", "SYSTEM"), - Map.entry("TBW102", "SYSTEM"), - Map.entry("DefaultCluster", "SYSTEM"), - Map.entry("broker-a", "SYSTEM"), - Map.entry("37e06a7b803c", "UNSPECIFIED"), - Map.entry("%RETRY%CID_DefaultHeartBeatSyncerTopic", "RETRY") - ); - - for (Map row : topics) { - String name = String.valueOf(row.get("name")); - String messageType = String.valueOf(row.get("messageType")); - if (expected.containsKey(name)) { - assertEquals(expected.get(name), messageType, "messageType for " + name); - } - } - } -} diff --git a/agents/drivers/rocketmq/topics.go b/agents/drivers/rocketmq/topics.go new file mode 100644 index 0000000000..8894b9df7e --- /dev/null +++ b/agents/drivers/rocketmq/topics.go @@ -0,0 +1,620 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strconv" + "strings" + "time" + + admin "github.com/amigoer/rocketmq-admin-go" + "github.com/amigoer/rocketmq-admin-go/protocol/remoting" +) + +var reservedTopicNames = map[string]struct{}{ + "TBW102": {}, "BenchmarkTest": {}, "SELF_TEST_TOPIC": {}, + "OFFSET_MOVED_EVENT": {}, "DefaultHeartBeatSyncerTopic": {}, +} + +type topicConfigWire struct { + TopicName string `json:"topicName"` + ReadQueueNums int `json:"readQueueNums"` + WriteQueueNums int `json:"writeQueueNums"` + Perm int `json:"perm"` + TopicFilterType string `json:"topicFilterType"` + TopicSysFlag int `json:"topicSysFlag"` + Order bool `json:"order"` + Attributes map[string]string `json:"attributes"` +} + +func (a *rocketMQAgent) listTopics(params map[string]any) (any, error) { + client, config, err := a.requireClient() + if err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout) + defer cancel() + + a.mu.RLock() + clusterName, brokerAddr := a.clusterName, a.brokerAddr + a.mu.RUnlock() + topicList, err := client.FetchTopicsByCluster(ctx, clusterName) + if err != nil { + topicList, err = client.FetchAllTopicList(ctx) + } + if err != nil { + return nil, err + } + _ = brokerAddr + configs, complete := a.collectTopicConfigs(ctx) + keyword := strings.ToLower(stringValue(params, "keyword")) + rows := make([]map[string]any, 0, len(topicList.TopicList)) + names := make(map[string]struct{}, len(topicList.TopicList)+len(configs)) + for _, name := range topicList.TopicList { + names[name] = struct{}{} + } + for name := range configs { + names[name] = struct{}{} + } + if complete && len(configs) > 0 { + names = make(map[string]struct{}, len(configs)) + for name := range configs { + names[name] = struct{}{} + } + } + for _, name := range sortedKeys(names) { + if keyword != "" && !strings.Contains(strings.ToLower(name), keyword) { + continue + } + configEntry := configs[name] + partitions := 0 + perm := 6 + messageType := classifyTopicMessageType(name, clusterName, nil) + if configEntry != nil { + partitions = max(configEntry.ReadQueueNums, 1) + perm = configEntry.Perm + messageType = classifyTopicMessageType(name, clusterName, configEntry.Attributes) + } + if partitions <= 0 { + if route, routeErr := client.ExamineTopicRouteInfo(ctx, name); routeErr == nil { + if len(route.QueueDatas) > 0 { + partitions = max(route.QueueDatas[0].ReadQueueNums, 1) + } + } + } + if partitions <= 0 { + partitions = 1 + } + rows = append(rows, map[string]any{ + "name": name, "partitions": partitions, "replicationFactor": 1, + "internal": messageType == "SYSTEM" || messageType == "RETRY" || messageType == "DLQ", + "messageType": messageType, "readQueueNums": partitions, + "writeQueueNums": writeQueueNums(configEntry, partitions), "perm": perm, + }) + } + sort.Slice(rows, func(i, j int) bool { return rows[i]["name"].(string) < rows[j]["name"].(string) }) + offset := max(0, intValue(params, 0, "offset")) + limit := intValue(params, 200, "limit") + return map[string]any{ + "topics": paginate(rows, offset, limit), "total": len(rows), "offset": offset, "limit": limit, + }, nil +} + +func (a *rocketMQAgent) createTopic(params map[string]any) (any, error) { + name, err := requireString(params, "name") + if err != nil { + return nil, err + } + readQueues := max(1, intValue(params, 8, "readQueueNums", "partitions")) + writeQueues := max(1, intValue(params, readQueues, "writeQueueNums")) + perm := normalizeTopicPerm(intValue(params, 6, "perm")) + messageType := normalizeMessageType(stringValue(params, "messageType")) + addresses, err := a.masterBrokerAddresses(stringValue(params, "brokerName")) + if err != nil { + return nil, err + } + _, config, _ := a.requireClient() + ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout) + defer cancel() + for _, address := range addresses { + if err := writeTopicConfig(ctx, address, &topicConfigWire{ + TopicName: name, ReadQueueNums: readQueues, WriteQueueNums: writeQueues, + Perm: perm, TopicFilterType: "SINGLE_TAG", Attributes: map[string]string{"+message.type": messageType}, + }); err != nil { + return nil, err + } + } + return okResult(), nil +} + +func writeTopicConfig(ctx context.Context, address string, config *topicConfigWire) error { + fields := map[string]string{ + "topic": config.TopicName, "defaultTopic": "TBW102", "readQueueNums": strconv.Itoa(config.ReadQueueNums), + "writeQueueNums": strconv.Itoa(config.WriteQueueNums), "perm": strconv.Itoa(config.Perm), + "topicFilterType": valueOrDefault(config.TopicFilterType, "SINGLE_TAG"), + "topicSysFlag": strconv.Itoa(config.TopicSysFlag), "order": strconv.FormatBool(config.Order), + "attributes": topicAttributesString(config.Attributes), + } + command := remoting.NewRequest(remoting.UpdateAndCreateTopic, fields) + _, err := invokeRemotingWithClient(ctx, address, command) + return err +} + +func (a *rocketMQAgent) deleteTopic(params map[string]any) (any, error) { + name, err := requireString(params, "name") + if err != nil { + return nil, err + } + client, config, _ := a.requireClient() + ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout) + defer cancel() + addresses, err := a.masterBrokerAddresses(stringValue(params, "brokerName")) + if err != nil { + return nil, err + } + for _, address := range addresses { + if err := client.DeleteTopicInBroker(ctx, address, name); err != nil { + return nil, err + } + } + if err := client.DeleteTopicInNameServer(ctx, name); err != nil { + return nil, err + } + return okResult(), nil +} + +func (a *rocketMQAgent) updatePartitions(params map[string]any) (any, error) { + name, err := requireString(params, "name") + if err != nil { + return nil, err + } + _, config, _ := a.requireClient() + ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout) + defer cancel() + addresses, err := a.masterBrokerAddresses(stringValue(params, "brokerName")) + if err != nil { + return nil, err + } + readQueues := max(1, intValue(params, intValue(params, 1, "totalPartitions"), "readQueueNums")) + writeQueues := max(1, intValue(params, readQueues, "writeQueueNums")) + for _, address := range addresses { + current, configErr := readTopicConfig(ctx, address, name) + if configErr != nil { + current = &topicConfigWire{TopicName: name, Perm: 6, TopicFilterType: "SINGLE_TAG"} + } + current.ReadQueueNums = readQueues + current.WriteQueueNums = writeQueues + if err := writeTopicConfig(ctx, address, current); err != nil { + return nil, err + } + } + return okResult(), nil +} + +func (a *rocketMQAgent) getTopicRoute(params map[string]any) (any, error) { + name, err := requireString(params, "name", "topic") + if err != nil { + return nil, err + } + client, config, _ := a.requireClient() + ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout) + defer cancel() + route, err := client.ExamineTopicRouteInfo(ctx, name) + if err != nil { + return nil, err + } + a.mu.RLock() + proxies := a.proxies + a.mu.RUnlock() + brokerDatas := make([]map[string]any, 0, len(route.BrokerDatas)) + for _, broker := range route.BrokerDatas { + addresses := make(map[string]string, len(broker.BrokerAddrs)) + for brokerID, address := range broker.BrokerAddrs { + if original := proxies.OriginalForLocal(address); original != "" { + address = original + } + addresses[brokerID] = address + } + brokerDatas = append(brokerDatas, map[string]any{ + "brokerName": broker.BrokerName, "cluster": broker.Cluster, "brokerAddrs": addresses, + }) + } + queueDatas := make([]map[string]any, 0, len(route.QueueDatas)) + for _, queue := range route.QueueDatas { + queueDatas = append(queueDatas, map[string]any{ + "brokerName": queue.BrokerName, "readQueueNums": queue.ReadQueueNums, + "writeQueueNums": queue.WriteQueueNums, "perm": queue.Perm, + }) + } + return map[string]any{"topic": name, "queueDatas": queueDatas, "brokerDatas": brokerDatas}, nil +} + +func (a *rocketMQAgent) getTopicStats(params map[string]any) (any, error) { + name, err := requireString(params, "name", "topic") + if err != nil { + return nil, err + } + client, config, _ := a.requireClient() + ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout) + defer cancel() + stats, err := a.examineTopicStats(ctx, client, name) + if err != nil { + return nil, err + } + partitionStats := make([]map[string]any, 0, len(stats)) + var total int64 + for key, offset := range stats { + queue := parseMessageQueueKey(key) + count := max(int64(0), offset.MaxOffset-offset.MinOffset) + total += count + partitionStats = append(partitionStats, map[string]any{ + "partition": queue.QueueID, "brokerName": queue.BrokerName, + "beginOffset": offset.MinOffset, "endOffset": offset.MaxOffset, + "messageCount": count, "lastTimestamp": offset.LastUpdateTimestamp, + }) + } + sort.Slice(partitionStats, func(i, j int) bool { + return partitionStats[i]["partition"].(int) < partitionStats[j]["partition"].(int) + }) + return map[string]any{ + "name": name, "partitions": len(partitionStats), "replicationFactor": 1, + "totalMessages": total, "partitionStats": partitionStats, + }, nil +} + +func (a *rocketMQAgent) examineTopicStats( + ctx context.Context, + client *admin.Client, + topic string, +) (map[string]*admin.TopicOffset, error) { + route, err := client.ExamineTopicRouteInfo(ctx, topic) + if err != nil { + return nil, err + } + addresses := masterAddressesFromRoute(route) + if len(addresses) == 0 { + return nil, fmt.Errorf("no RocketMQ master broker found for topic %s", topic) + } + merged := make(map[string]*admin.TopicOffset) + var lastErr error + successCount := 0 + for _, address := range addresses { + response, requestErr := invokeRemotingWithClient(ctx, address, + remoting.NewRequest(remoting.GetTopicStatsInfo, map[string]string{"topic": topic})) + if requestErr != nil { + lastErr = requestErr + continue + } + partial, decodeErr := decodeTopicStats(response.Body) + if decodeErr != nil { + lastErr = decodeErr + continue + } + successCount++ + for key, offset := range partial { + merged[key] = offset + } + } + if successCount != len(addresses) { + return nil, fmt.Errorf("query topic stats for %s on all masters: %w", topic, lastErr) + } + if len(merged) == 0 { + return nil, fmt.Errorf("topic stats not found: %s", topic) + } + return merged, nil +} + +func decodeTopicStats(body []byte) (map[string]*admin.TopicOffset, error) { + var stats admin.TopicStatsTable + if err := json.Unmarshal(repairRocketMQJSON(body), &stats); err != nil { + return nil, fmt.Errorf("decode topic stats: %w", err) + } + if stats.OffsetTable == nil { + stats.OffsetTable = make(map[string]*admin.TopicOffset) + } + return stats.OffsetTable, nil +} + +func masterAddressesFromRoute(route *admin.TopicRouteData) []string { + addresses := make(map[string]struct{}) + for _, broker := range route.BrokerDatas { + if address := broker.BrokerAddrs["0"]; address != "" { + addresses[address] = struct{}{} + } + } + return sortedKeys(addresses) +} + +func (a *rocketMQAgent) getTopicConfig(params map[string]any) (any, error) { + name, err := requireString(params, "name", "topic") + if err != nil { + return nil, err + } + _, config, _ := a.requireClient() + ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout) + defer cancel() + address, err := a.brokerAddressForName(stringValue(params, "brokerName")) + if err != nil { + return nil, err + } + topicConfig, err := readTopicConfig(ctx, address, name) + if err != nil { + return nil, err + } + configs := map[string]any{ + "readQueueNums": configEntry(strconv.Itoa(topicConfig.ReadQueueNums)), + "writeQueueNums": configEntry(strconv.Itoa(topicConfig.WriteQueueNums)), + "perm": configEntry(strconv.Itoa(topicConfig.Perm)), + } + return map[string]any{"configs": configs}, nil +} + +func (a *rocketMQAgent) alterTopicConfig(params map[string]any) (any, error) { + name, err := requireString(params, "name", "topic") + if err != nil { + return nil, err + } + _, config, _ := a.requireClient() + ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout) + defer cancel() + address, err := a.brokerAddressForName(stringValue(params, "brokerName")) + if err != nil { + return nil, err + } + topicConfig, err := readTopicConfig(ctx, address, name) + if err != nil { + return nil, err + } + if entries, ok := params["configs"].([]any); ok { + for _, raw := range entries { + entry, _ := raw.(map[string]any) + value := stringValue(entry, "value") + switch stringValue(entry, "key") { + case "readQueueNums": + topicConfig.ReadQueueNums = intValue(entry, topicConfig.ReadQueueNums, "value") + case "writeQueueNums": + topicConfig.WriteQueueNums = intValue(entry, topicConfig.WriteQueueNums, "value") + case "perm": + topicConfig.Perm = normalizeTopicPerm(intValue(entry, topicConfig.Perm, "value")) + case "retention.ms", "retention.bytes": + if value != "" { + return nil, fmt.Errorf("RocketMQ topic retention is broker-level and cannot be changed per topic") + } + } + } + } + if err := writeTopicConfig(ctx, address, topicConfig); err != nil { + return nil, err + } + return okResult(), nil +} + +func (a *rocketMQAgent) skipTopicAccumulation(params map[string]any) (any, error) { + topic, err := requireString(params, "topic", "name") + if err != nil { + return nil, err + } + client, config, _ := a.requireClient() + ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout) + defer cancel() + groups, err := client.QueryTopicConsumeByWho(ctx, topic) + if err != nil { + return nil, err + } + reset := make([]string, 0, len(groups)) + for _, group := range groups { + if _, resetErr := client.ResetOffsetByTimestamp(ctx, topic, group, time.Now().UnixMilli(), true); resetErr == nil { + reset = append(reset, group) + } + } + return map[string]any{"ok": true, "resetGroups": len(reset)}, nil +} + +func (a *rocketMQAgent) masterBrokerAddresses(brokerName string) ([]string, error) { + client, config, err := a.requireClient() + if err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout) + defer cancel() + info, err := client.ExamineBrokerClusterInfo(ctx) + if err != nil { + return nil, err + } + addresses := make([]string, 0) + for _, name := range sortedKeys(info.BrokerAddrTable) { + if brokerName != "" && brokerName != name { + continue + } + if address := info.BrokerAddrTable[name].BrokerAddrs["0"]; address != "" { + addresses = append(addresses, address) + } + } + if len(addresses) == 0 { + a.mu.RLock() + fallback := a.brokerAddr + a.mu.RUnlock() + if fallback != "" { + addresses = append(addresses, fallback) + } + } + if len(addresses) == 0 { + return nil, fmt.Errorf("no RocketMQ master broker found") + } + return addresses, nil +} + +func (a *rocketMQAgent) brokerAddressForName(brokerName string) (string, error) { + addresses, err := a.masterBrokerAddresses(brokerName) + if err != nil { + return "", err + } + return addresses[0], nil +} + +type parsedMessageQueue struct { + Topic string + BrokerName string + QueueID int +} + +func parseMessageQueueKey(value string) parsedMessageQueue { + value = strings.TrimSpace(value) + var queue struct { + Topic string `json:"topic"` + BrokerName string `json:"brokerName"` + QueueID int `json:"queueId"` + } + if strings.HasPrefix(value, "{") && json.Unmarshal([]byte(value), &queue) == nil { + return parsedMessageQueue(queue) + } + if strings.HasPrefix(value, "MessageQueue [") && strings.HasSuffix(value, "]") { + fields := strings.Split(strings.TrimSuffix(strings.TrimPrefix(value, "MessageQueue ["), "]"), ",") + for _, field := range fields { + keyValue := strings.SplitN(strings.TrimSpace(field), "=", 2) + if len(keyValue) != 2 { + continue + } + switch keyValue[0] { + case "topic": + queue.Topic = keyValue[1] + case "brokerName": + queue.BrokerName = keyValue[1] + case "queueId": + queue.QueueID, _ = strconv.Atoi(keyValue[1]) + } + } + return parsedMessageQueue(queue) + } + parts := strings.Split(value, "-") + if len(parts) >= 3 { + queueID, _ := strconv.Atoi(parts[len(parts)-1]) + return parsedMessageQueue{Topic: strings.Join(parts[:len(parts)-2], "-"), BrokerName: parts[len(parts)-2], QueueID: queueID} + } + return parsedMessageQueue{} +} + +func configEntry(value string) map[string]any { + return map[string]any{ + "value": value, "isDefault": false, + "isReadOnly": false, "isSensitive": false, "source": "USER", + } +} + +func isSystemTopic(topic, cluster string) bool { + if topic == "" { + return true + } + if _, ok := reservedTopicNames[topic]; ok { + return true + } + return strings.HasPrefix(topic, "%") || strings.HasPrefix(topic, "RMQ_SYS") || + strings.HasPrefix(topic, "rmq_sys") || strings.HasPrefix(topic, "SCHEDULE_TOPIC_") || + strings.HasPrefix(topic, "rocketmq-broker-") || strings.HasSuffix(topic, "_REPLY_TOPIC") || topic == cluster +} + +func classifyTopicMessageType(topic, cluster string, attributes map[string]string) string { + if strings.HasPrefix(topic, "%RETRY%") || strings.HasPrefix(topic, "%R") { + return "RETRY" + } + if strings.HasPrefix(topic, "%DLQ%") || strings.HasPrefix(topic, "%D") { + return "DLQ" + } + if isSystemTopic(topic, cluster) { + return "SYSTEM" + } + for _, key := range []string{"message.type", "+message.type"} { + if value := normalizeMessageType(attributes[key]); value != "NORMAL" || attributes[key] != "" { + return value + } + } + return "UNSPECIFIED" +} + +func (a *rocketMQAgent) collectTopicConfigs(ctx context.Context) (map[string]*topicConfigWire, bool) { + addresses, err := a.masterBrokerAddresses("") + if err != nil { + return map[string]*topicConfigWire{}, false + } + merged := make(map[string]*topicConfigWire) + complete := true + for _, address := range addresses { + if a.proxies.IsCollisionFallback(address) && len(merged) > 0 { + complete = false + continue + } + configs, fetchErr := fetchAllTopicConfigs(ctx, address) + if fetchErr != nil { + complete = false + continue + } + for name, config := range configs { + if merged[name] == nil { + merged[name] = config + } + } + } + return merged, complete +} + +func fetchAllTopicConfigs(ctx context.Context, address string) (map[string]*topicConfigWire, error) { + response, err := invokeRemotingWithClient(ctx, address, remoting.NewRequest(remoting.GetAllTopicConfig, nil)) + if err != nil { + return nil, err + } + var wrapper struct { + TopicConfigTable map[string]*topicConfigWire `json:"topicConfigTable"` + } + if err := json.Unmarshal(repairRocketMQJSON(response.Body), &wrapper); err != nil { + return nil, err + } + return wrapper.TopicConfigTable, nil +} + +func readTopicConfig(ctx context.Context, address, topic string) (*topicConfigWire, error) { + response, err := invokeRemotingWithClient(ctx, address, remoting.NewRequest(remoting.GetTopicConfig, map[string]string{"topic": topic})) + if err == nil { + var config topicConfigWire + if decodeErr := json.Unmarshal(repairRocketMQJSON(response.Body), &config); decodeErr != nil { + return nil, decodeErr + } + return &config, nil + } + configs, fallbackErr := fetchAllTopicConfigs(ctx, address) + if fallbackErr != nil { + return nil, fmt.Errorf("read topic config %s: %v; fallback snapshot: %w", topic, err, fallbackErr) + } + config := configs[topic] + if config == nil { + return nil, fmt.Errorf("topic config not found: %s", topic) + } + if config.TopicName == "" { + config.TopicName = topic + } + return config, nil +} + +func topicAttributesString(attributes map[string]string) string { + keys := sortedKeys(attributes) + parts := make([]string, 0, len(keys)) + for _, key := range keys { + if !strings.HasPrefix(key, "+") && !strings.HasPrefix(key, "-") { + continue + } + if attributes[key] == "" { + parts = append(parts, key) + } else { + parts = append(parts, key+"="+attributes[key]) + } + } + return strings.Join(parts, ",") +} + +func writeQueueNums(config *topicConfigWire, fallback int) int { + if config == nil || config.WriteQueueNums <= 0 { + return fallback + } + return config.WriteQueueNums +} diff --git a/agents/drivers/rocketmq/topics_test.go b/agents/drivers/rocketmq/topics_test.go new file mode 100644 index 0000000000..f32d9ed211 --- /dev/null +++ b/agents/drivers/rocketmq/topics_test.go @@ -0,0 +1,80 @@ +package main + +import ( + "reflect" + "testing" +) + +func TestTopicAttributesStringEmitsOnlyRocketMQ5Modifications(t *testing.T) { + attributes := map[string]string{ + "+message.type": "FIFO", + "cleanup.policy": "DELETE", + "-reserve.time": "", + } + if got := topicAttributesString(attributes); got != "+message.type=FIFO,-reserve.time" { + t.Fatalf("topicAttributesString() = %q", got) + } +} + +func TestClassifyTopicMessageType(t *testing.T) { + tests := map[string]string{ + "%RETRY%GID_Orders": "RETRY", + "%DLQ%GID_Orders": "DLQ", + "RMQ_SYS_TRANS_HALF_TOPIC": "SYSTEM", + "Orders": "UNSPECIFIED", + } + for topic, want := range tests { + if got := classifyTopicMessageType(topic, "DefaultCluster", nil); got != want { + t.Fatalf("classifyTopicMessageType(%q) = %q, want %q", topic, got, want) + } + } + attributes := map[string]string{"message.type": "ORDER"} + if got := classifyTopicMessageType("Ordered", "DefaultCluster", attributes); got != "FIFO" { + t.Fatalf("ORDER attribute classified as %q", got) + } +} + +func TestPaginationAndMessageQueueParsing(t *testing.T) { + items := []int{0, 1, 2, 3} + if got := paginate(items, 2, 0); !reflect.DeepEqual(got, []int{2, 3}) { + t.Fatalf("paginate all from offset = %#v", got) + } + queue := parseMessageQueueKey(`{"topic":"Orders","brokerName":"broker-a","queueId":3}`) + if queue.Topic != "Orders" || queue.BrokerName != "broker-a" || queue.QueueID != 3 { + t.Fatalf("unexpected JSON queue: %#v", queue) + } + queue = parseMessageQueueKey("MessageQueue [topic=Order-Events, brokerName=broker-a, queueId=7]") + if queue.Topic != "Order-Events" || queue.BrokerName != "broker-a" || queue.QueueID != 7 { + t.Fatalf("unexpected text queue: %#v", queue) + } +} + +func TestDecodeTopicStatsRepairsObjectKeys(t *testing.T) { + body := []byte(`{"offsetTable":{{"brokerName":"broker-a","queueId":1,"topic":"Orders"}:{"minOffset":2,"maxOffset":9,"lastUpdateTimestamp":10}}}`) + stats, err := decodeTopicStats(body) + if err != nil { + t.Fatal(err) + } + if len(stats) != 1 { + t.Fatalf("unexpected stats: %#v", stats) + } + for key, offset := range stats { + queue := parseMessageQueueKey(key) + if queue.Topic != "Orders" || queue.BrokerName != "broker-a" || queue.QueueID != 1 { + t.Fatalf("unexpected queue key %q: %#v", key, queue) + } + if offset.MinOffset != 2 || offset.MaxOffset != 9 { + t.Fatalf("unexpected offset: %#v", offset) + } + } +} + +func TestConfigEntryMatchesAgentContract(t *testing.T) { + entry := configEntry("8") + if entry["value"] != "8" || entry["source"] != "USER" { + t.Fatalf("unexpected config entry: %#v", entry) + } + if _, exists := entry["key"]; exists { + t.Fatalf("config entry must be keyed by its parent object: %#v", entry) + } +} diff --git a/agents/scripts/driver_release_packages_test.py b/agents/scripts/driver_release_packages_test.py index be89f9ae9d..7c471ddda4 100644 --- a/agents/scripts/driver_release_packages_test.py +++ b/agents/scripts/driver_release_packages_test.py @@ -24,6 +24,8 @@ class DriverReleasePackagesTest(unittest.TestCase): duckdb_source.write_bytes(b"\xcf\xfa\xed\xfetest-duckdb-agent") rabbitmq_source = release_dir / "dbx-agent-rabbitmq-linux-x64" rabbitmq_source.write_bytes(b"\x7fELFtest-rabbitmq-agent") + rocketmq_source = release_dir / "dbx-agent-rocketmq-windows-x64.exe" + rocketmq_source.write_bytes(b"MZtest-rocketmq-agent") cassandra_source = release_dir / "dbx-agent-cassandra-linux-x64" cassandra_source.write_bytes(b"\x7fELFtest-cassandra-agent") tdengine_source = release_dir / "dbx-agent-tdengine-windows-aarch64.exe" @@ -40,6 +42,7 @@ class DriverReleasePackagesTest(unittest.TestCase): "vastbase": "0.1.37", "duckdb": "0.1.0", "rabbitmq": "0.1.0", + "rocketmq": "0.1.0", "cassandra": "0.1.37", "hive": "0.1.43", "tdengine": "0.1.0", @@ -51,6 +54,7 @@ class DriverReleasePackagesTest(unittest.TestCase): versioned_vastbase = release_dir / "dbx-agent-vastbase-0.1.37-linux-x64" versioned_duckdb = release_dir / "dbx-agent-duckdb-0.1.0-macos-aarch64" versioned_rabbitmq = release_dir / "dbx-agent-rabbitmq-0.1.0-linux-x64" + versioned_rocketmq = release_dir / "dbx-agent-rocketmq-0.1.0-windows-x64.exe" versioned_cassandra = release_dir / "dbx-agent-cassandra-0.1.37-linux-x64" versioned_tdengine = release_dir / "dbx-agent-tdengine-0.1.0-windows-aarch64.exe" self.assertEqual( @@ -62,6 +66,7 @@ class DriverReleasePackagesTest(unittest.TestCase): versioned_vastbase, versioned_duckdb, versioned_rabbitmq, + versioned_rocketmq, versioned_tdengine, ], ) @@ -141,6 +146,19 @@ class DriverReleasePackagesTest(unittest.TestCase): } }, }, + "rocketmq": { + "version": "0.1.0", + "label": "Apache RocketMQ", + "min_app_version": "0.6.0", + "jre": "21", + "jar": {"url": "https://example.com/legacy-placeholder.jar", "size": 0}, + "native": { + "windows-x64": { + "url": f"https://example.com/{versioned_rocketmq.name}", + "size": versioned_rocketmq.stat().st_size, + } + }, + }, "tdengine": { "version": "0.1.0", "label": "TDengine", @@ -169,6 +187,7 @@ class DriverReleasePackagesTest(unittest.TestCase): release_dir / "dbx-agent-vastbase-0.1.37-linux-x64.tar.zst", release_dir / "dbx-agent-duckdb-0.1.0-macos-aarch64.tar.zst", release_dir / "dbx-agent-rabbitmq-0.1.0-linux-x64.tar.zst", + release_dir / "dbx-agent-rocketmq-0.1.0-windows-x64.tar.zst", release_dir / "dbx-agent-tdengine-0.1.0-windows-aarch64.tar.zst", ], ) @@ -179,7 +198,8 @@ class DriverReleasePackagesTest(unittest.TestCase): (outputs[3], "vastbase", versioned_vastbase, "native", "linux-x64"), (outputs[4], "duckdb", versioned_duckdb, "native", "macos-aarch64"), (outputs[5], "rabbitmq", versioned_rabbitmq, "native", "linux-x64"), - (outputs[6], "tdengine", versioned_tdengine, "native", "windows-aarch64"), + (outputs[6], "rocketmq", versioned_rocketmq, "native", "windows-x64"), + (outputs[7], "tdengine", versioned_tdengine, "native", "windows-aarch64"), ] for output, driver_name, source, artifact_type, platform in package_cases: tar_bytes = subprocess.run( @@ -209,7 +229,8 @@ class DriverReleasePackagesTest(unittest.TestCase): (final_registry["drivers"]["vastbase"]["native"]["linux-x64"], outputs[3]), (final_registry["drivers"]["duckdb"]["native"]["macos-aarch64"], outputs[4]), (final_registry["drivers"]["rabbitmq"]["native"]["linux-x64"], outputs[5]), - (final_registry["drivers"]["tdengine"]["native"]["windows-aarch64"], outputs[6]), + (final_registry["drivers"]["rocketmq"]["native"]["windows-x64"], outputs[6]), + (final_registry["drivers"]["tdengine"]["native"]["windows-aarch64"], outputs[7]), ] for artifact, output in release_artifacts: self.assertEqual(artifact["url"], f"https://example.com/{output.name}") @@ -226,6 +247,7 @@ class DriverReleasePackagesTest(unittest.TestCase): versioned_java, versioned_native, versioned_rabbitmq, + versioned_rocketmq, versioned_tdengine, versioned_vastbase, ], diff --git a/agents/scripts/validate_agents.py b/agents/scripts/validate_agents.py index e4c51fa0b1..ef8698d832 100644 --- a/agents/scripts/validate_agents.py +++ b/agents/scripts/validate_agents.py @@ -24,6 +24,7 @@ NATIVE_ONLY_AGENT_MODULES = { "tdengine": "drivers/tdengine", "xugu": "drivers/xugu", "rabbitmq": "drivers/rabbitmq", + "rocketmq": "drivers/rocketmq", } AUTO_VERSIONED_NATIVE_MODULES = {"duckdb"} JDBC_ARCHITECTURE_ALLOWLIST = { diff --git a/agents/scripts/validate_agents_test.py b/agents/scripts/validate_agents_test.py index 402ecbf02d..66ebac8bbd 100644 --- a/agents/scripts/validate_agents_test.py +++ b/agents/scripts/validate_agents_test.py @@ -147,11 +147,12 @@ class ValidateAgentsTest(unittest.TestCase): "xugu", "duckdb", "rabbitmq", + "rocketmq", "tdengine", ): (root / "drivers" / driver).mkdir(parents=True) (root / "versions.json").write_text( - json.dumps({"h2": "0.1.0", "cassandra": "0.1.0", "hive": "0.1.0", "oracle": "0.1.0", "kingbase": "0.1.0", "iotdb": "0.1.0", "neo4j": "0.1.0", "vastbase": "0.1.0", "xugu": "0.1.0", "rabbitmq": "0.1.0", "tdengine": "0.1.0"}), + json.dumps({"h2": "0.1.0", "cassandra": "0.1.0", "hive": "0.1.0", "oracle": "0.1.0", "kingbase": "0.1.0", "iotdb": "0.1.0", "neo4j": "0.1.0", "vastbase": "0.1.0", "xugu": "0.1.0", "rabbitmq": "0.1.0", "rocketmq": "0.1.0", "tdengine": "0.1.0"}), encoding="utf-8", ) diff --git a/agents/scripts/version_agent_artifacts.py b/agents/scripts/version_agent_artifacts.py index c4ee4f4eae..91de053acc 100644 --- a/agents/scripts/version_agent_artifacts.py +++ b/agents/scripts/version_agent_artifacts.py @@ -4,7 +4,7 @@ import json from pathlib import Path -NATIVE_DRIVERS = ("cassandra", "hive", "oracle", "xugu", "kingbase", "iotdb", "neo4j", "vastbase", "duckdb", "rabbitmq", "tdengine") +NATIVE_DRIVERS = ("cassandra", "hive", "oracle", "xugu", "kingbase", "iotdb", "neo4j", "vastbase", "duckdb", "rabbitmq", "rocketmq", "tdengine") PLATFORMS = ( "macos-aarch64", "macos-x64", diff --git a/agents/settings.gradle b/agents/settings.gradle index cf73b32c8d..7b6ca2846e 100644 --- a/agents/settings.gradle +++ b/agents/settings.gradle @@ -6,7 +6,7 @@ def driverModules = [ 'teradata', 'vertica', 'firebird', 'exasol', 'oceanbase-oracle', 'gbase8a', 'gbase8s', 'bigquery', 'kylin', 'sundb', 'h2', 'h2-legacy', 'snowflake', 'trino', 'spark', 'db2', 'informix', 'mongodb', 'highgo', 'uxdb', 'yashandb', 'oscar', - 'iris', 'etcd', 'zookeeper', 'kafka', 'rocketmq', 'sqlserver-legacy' + 'iris', 'etcd', 'zookeeper', 'kafka', 'sqlserver-legacy' ] include(*(infrastructureModules + driverModules)) diff --git a/crates/dbx-core/src/mq/README.md b/crates/dbx-core/src/mq/README.md index 09ff7333a7..a2027184ce 100644 --- a/crates/dbx-core/src/mq/README.md +++ b/crates/dbx-core/src/mq/README.md @@ -14,7 +14,9 @@ mq/ ├── service.rs - 服务层函数 └── adapters/ ├── pulsar.rs - Pulsar 实现 + ├── kafka.rs - Kafka 实现 (Java agent) ├── rabbitmq.rs - RabbitMQ 实现 (Go native agent) + ├── rocketmq.rs - RocketMQ 实现 (Go native agent) └── pulsar_version.rs - 版本探测 ``` diff --git a/crates/dbx-core/src/mq/adapters/rocketmq.rs b/crates/dbx-core/src/mq/adapters/rocketmq.rs index 48fdde5dff..ae32e5d08c 100644 --- a/crates/dbx-core/src/mq/adapters/rocketmq.rs +++ b/crates/dbx-core/src/mq/adapters/rocketmq.rs @@ -1,10 +1,9 @@ -//! Apache RocketMQ admin adapter. Communicates with a Java agent process -//! (`RocketMqAgent.java`) via JSON-RPC over stdin/stdout. The Java agent uses -//! `DefaultMQAdminExt` for admin operations and `DefaultMQProducer` for -//! message production. +//! Apache RocketMQ admin adapter. Communicates with the Go native agent via +//! JSON-RPC over stdin/stdout. The agent uses `rocketmq-admin-go` plus direct +//! remoting calls for compatibility gaps in the upstream admin API. //! -//! This adapter follows the same pattern as the ZooKeeper/Etcd agents: -//! 1. Spawn a Java agent process via `AgentDriverClient` +//! This adapter follows the same process-agent pattern as the other drivers: +//! 1. Spawn the native agent process via `AgentDriverClient` //! 2. Perform JSON-RPC handshake + connect //! 3. Delegate all `MessageQueueAdmin` trait methods to JSON-RPC calls @@ -97,10 +96,10 @@ fn cluster_info_from_agent_result(result: &serde_json::Value) -> MqClusterInfo { } impl RocketMqAdmin { - /// Spawn the RocketMQ Java agent, perform handshake, and connect. + /// Spawn the RocketMQ native agent, perform handshake, and connect. /// /// Callers must probe NameServer reachability before invoking this so the - /// connect-timeout wall covers only JVM spawn + handshake + connect. + /// connect-timeout wall covers only process spawn + handshake + connect. pub async fn new(cfg: MqAdminConfig, launch: AgentLaunchSpec) -> Result { let mut client = AgentDriverClient::spawn(launch).await?; @@ -814,7 +813,7 @@ fn rocketmq_topic_admin_params(topic: &TopicRef, partitions: Option) -> ser params } -/// Build the connection params JSON from MqAdminConfig for the Java agent. +/// Build the connection params JSON from MqAdminConfig for the native agent. fn build_connection_params(cfg: &MqAdminConfig) -> serde_json::Value { let extra = &cfg.extra; let access_key = extra_str(extra, "accessKey") diff --git a/docs/mq-quick-start.md b/docs/mq-quick-start.md index e1e15ed8e5..adf77f48af 100644 --- a/docs/mq-quick-start.md +++ b/docs/mq-quick-start.md @@ -378,7 +378,7 @@ cargo check --package dbx-web --features mq-admin ### 添加新的 MQ 系统 -Kafka 与 RocketMQ 已实现 Java Agent + Rust 适配器模式。新增 MQ 系统时： +Kafka 使用 Java Agent，RocketMQ 使用 Go 原生 Agent；两者均通过 Rust 适配器接入。新增 MQ 系统时： 1. 在 `agents/drivers//` 实现 JSON-RPC Agent（参考 `kafka` / `rocketmq`） 2. 在 `crates/dbx-core/src/mq/adapters/` 实现 `MessageQueueAdmin` @@ -407,9 +407,9 @@ Kafka 与 RocketMQ 已实现 Java Agent + Rust 适配器模式。新增 MQ 系 Agent 构建与安装： ```bash -cd agents -./gradlew :rocketmq:shadowJar -# 将 shadow JAR 安装到 DBX 数据目录 agents/drivers/rocketmq/agent.jar +cd agents/drivers/rocketmq +CGO_ENABLED=0 go build -trimpath -ldflags='-s -w' -o agent . +# 将可执行文件安装到 DBX 数据目录 agents/drivers/rocketmq/agent ``` Docker 快速启动（NameServer 9876 + Broker 10911，仅用于本地验证）：