github/dbx
4
0
Fork 0
mirror of https://github.com/t8y2/dbx.git synced 2026-10-21 17:09:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(rocketmq): replace Java agent with native Go implementation

Browse Source
This commit is contained in:
t8y2
2026-08-12 23:15:20 +08:00
parent baec487a21
commit 9fb70bcfc9
44 changed files with 4872 additions and 5414 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/scripts/bump-agent-versions.mjs
+2 -1
View File
@@ -58,9 +58,10 @@ const nativeDriverDirectories = {
neo4j: "neo4j-go",
vastbase: "vastbase-go",
rabbitmq: "rabbitmq",
rocketmq: "rocketmq",
tdengine: "tdengine",
};
const nativeDriverModules = new Set(["cassandra", "duckdb", "hive", "oracle", "xugu", "kingbase", "iotdb", "neo4j", "vastbase", "rabbitmq", "tdengine"]);
const nativeDriverModules = new Set(["cassandra", "duckdb", "hive", "oracle", "xugu", "kingbase", "iotdb", "neo4j", "vastbase", "rabbitmq", "rocketmq", "tdengine"]);
const nativeDriverSharedPaths = {
hive: [
"agents/go-common/go-gssapi",
.github/scripts/bump-agent-versions.test.mjs
+12
View File
@@ -59,6 +59,18 @@ test("bumps the native RabbitMQ agent from its Go directory", () => {
assert.equal(result.versions.rabbitmq, "0.1.1");
});
test("bumps the native RocketMQ agent from its Go directory", () => {
const result = evaluateAgentVersionBump({
versions: { rocketmq: "0.1.0" },
changedFiles: ["agents/drivers/rocketmq/main.go"],
moduleExists: (path) => path === "agents/drivers/rocketmq",
readModuleFile: () => "",
});
assert.equal(result.versions.rocketmq, "0.1.1");
assert.deepEqual(result.nativeModules, ["rocketmq"]);
});
test("bumps the native Vastbase agent from its independent Go directory", () => {
const result = evaluateAgentVersionBump({
versions: { vastbase: "0.1.37" },
.github/scripts/reuse-agent-release-assets.mjs
+1 -1
View File
@@ -15,7 +15,7 @@ import { basename, join } from "node:path";
import { tmpdir } from "node:os";
const REGISTRY_ASSET = "agent-registry.json";
const NATIVE_MODULES = new Set(["duckdb", "oracle", "xugu", "kingbase", "iotdb", "neo4j", "vastbase", "rabbitmq", "tdengine"]);
const NATIVE_MODULES = new Set(["duckdb", "oracle", "xugu", "kingbase", "iotdb", "neo4j", "vastbase", "rabbitmq", "rocketmq", "tdengine"]);
const PLATFORMS = [
"macos-aarch64",
"macos-x64",
.github/scripts/reuse-agent-release-assets.test.mjs
+18
View File
@@ -114,6 +114,24 @@ test("requires all IoTDB native platforms when reusing a release", () => {
);
});
test("requires all RocketMQ native platforms when reusing a release", () => {
const native = Object.fromEntries(
platforms.slice(0, -1).map((platform, index) => [platform, artifact(`dbx-agent-rocketmq-0.1.40-${platform}.tar.zst`, String(index + 1))]),
);
const registry = { drivers: { rocketmq: { version: "0.1.40", native } }, jres: {} };
assert.throws(
() => collectReusableAssetPlan({
registry,
release: releaseFor(Object.values(native)),
versions: { rocketmq: "0.1.40" },
modules: ["rocketmq"],
reuseJre: false,
}),
/missing=windows-x64/,
);
});
test("ignores zero-size legacy JAR placeholders for native-only modules", () => {
const native = Object.fromEntries(
platforms.map((platform, index) => [platform, artifact(`dbx-agent-duckdb-0.1.2-${platform}.tar.zst`, String(index + 1))]),
.github/workflows/agents-release.yml
+44 -2
View File
@@ -289,6 +289,46 @@ jobs:
name: rabbitmq-native
path: "release-native/dbx-agent-rabbitmq-*"
build-rocketmq-native:
needs: [bump-versions]
if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'rocketmq') }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: "1.25.x"
- name: Test RocketMQ native agent
working-directory: agents/drivers/rocketmq
run: go test -race ./...
- name: Cross-compile RocketMQ native agent
shell: bash
run: |
mkdir -p release-native
cd agents/drivers/rocketmq
declare -A TARGETS=(
["macos-aarch64"]="darwin/arm64"
["macos-x64"]="darwin/amd64"
["linux-aarch64"]="linux/arm64"
["linux-x64"]="linux/amd64"
["windows-aarch64"]="windows/arm64"
["windows-x64"]="windows/amd64"
)
for platform in "${!TARGETS[@]}"; do
IFS=/ read -r goos goarch <<< "${TARGETS[$platform]}"
output="../../../release-native/dbx-agent-rocketmq-${platform}"
if [[ "$goos" == "windows" ]]; then
output="${output}.exe"
fi
echo "Building $platform ($goos/$goarch)"
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -trimpath -ldflags="-s -w" -o "$output" .
done
ls -lh ../../../release-native
- uses: actions/upload-artifact@v4
with:
name: rocketmq-native
path: "release-native/dbx-agent-rocketmq-*"
build-cassandra-native:
needs: [bump-versions]
if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'cassandra') }}
@@ -868,7 +908,7 @@ jobs:
retention-days: 1
release:
needs: [bump-versions, commit-versions, build-agents, build-oracle-native, build-xugu-native, build-rabbitmq-native, build-cassandra-native, build-hive-native, build-kingbase-native, build-vastbase-native, build-neo4j-native, build-iotdb-native, build-duckdb-native, build-tdengine-native, build-jre, reuse-previous-assets]
needs: [bump-versions, commit-versions, build-agents, build-oracle-native, build-xugu-native, build-rabbitmq-native, build-rocketmq-native, build-cassandra-native, build-hive-native, build-kingbase-native, build-vastbase-native, build-neo4j-native, build-iotdb-native, build-duckdb-native, build-tdengine-native, build-jre, reuse-previous-assets]
if: ${{ always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}
runs-on: ubuntu-latest
steps:
@@ -990,6 +1030,7 @@ jobs:
duckdb) echo "DuckDB" ;;
xugu) echo "虚谷 XuguDB" ;;
rabbitmq) echo "RabbitMQ" ;;
rocketmq) echo "Apache RocketMQ" ;;
cassandra) echo "Apache Cassandra" ;;
hive) echo "Apache Hive" ;;
neo4j) echo "Neo4j" ;;
@@ -1058,7 +1099,7 @@ jobs:
[ -n "$DRIVERS" ] && DRIVERS="${DRIVERS},"$'\n'
DRIVERS="${DRIVERS}$(generate_jar_entry "$name" "$label" "$f" "$jre_key" "$version" "$external_driver" "$native_json")"
done
for name in oracle xugu kingbase vastbase neo4j iotdb duckdb rabbitmq cassandra hive tdengine; do
for name in oracle xugu kingbase vastbase neo4j iotdb duckdb rabbitmq rocketmq cassandra hive tdengine; do
version=$(get_module_version "$name")
[ -f "release/dbx-agent-${name}-${version}.jar" ] && continue
native_json=$(generate_native_platforms "$name" "$version")
@@ -1128,6 +1169,7 @@ jobs:
oracle) echo "Oracle" ;;
xugu) echo "虚谷 XuguDB" ;;
rabbitmq) echo "RabbitMQ" ;;
rocketmq) echo "Apache RocketMQ" ;;
cassandra) echo "Apache Cassandra" ;;
hive) echo "Apache Hive" ;;
neo4j) echo "Neo4j" ;;
.github/workflows/ci.yml
+17
View File
@@ -740,6 +740,10 @@ jobs:
run: go test ./...
working-directory: agents/drivers/rabbitmq
- name: RocketMQ native agent tests
run: go test -race ./...
working-directory: agents/drivers/rocketmq
- name: Cassandra native agent tests
run: go test ./...
working-directory: agents/drivers/cassandra-go
@@ -772,6 +776,19 @@ jobs:
run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-rabbitmq-linux-x64 .
working-directory: agents/drivers/rabbitmq
- name: RocketMQ native agent build
run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-rocketmq-linux-x64 .
working-directory: agents/drivers/rocketmq
- name: RocketMQ native agent integration tests
shell: bash
working-directory: agents/drivers/rocketmq
run: |
set -euo pipefail
for version in 4.9.8 5.3.1; do
./scripts/run-integration.sh "$version"
done
- name: Cassandra native agent build
run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o /tmp/dbx-agent-cassandra-linux-x64 .
working-directory: agents/drivers/cassandra-go
agents/README.md
+6 -4
View File
@@ -45,11 +45,12 @@ Each agent runs as a standalone process and communicates with DBX via stdin/stdo
| etcd | etcd | jetcd |
| zookeeper | Apache ZooKeeper | Apache Curator |
| rabbitmq | RabbitMQ | amqp091-go native agent |
| rocketmq | Apache RocketMQ | rocketmq-admin-go native agent |
## Multi-JRE Support
Most Java agents target JRE 21. Native agents, such as `cassandra`, `duckdb`, `hive`, `iotdb`, `oracle`, `kingbase`, `tdengine`, `xugu`, and `rabbitmq`, do not require a JRE. DBX downloads and manages the JRE 21 installation automatically for Java agents.
Most Java agents target JRE 21. Native agents, such as `cassandra`, `duckdb`, `hive`, `iotdb`, `oracle`, `kingbase`, `tdengine`, `xugu`, `rabbitmq`, and `rocketmq`, do not require a JRE. DBX downloads and manages the JRE 21 installation automatically for Java agents.
## JDBC Connection Pooling
@@ -75,7 +76,7 @@ Set `DBX_AGENT_JDBC_POOL_ENABLED=false` for a runtime-level compatibility fallba
For new agents, prefer a **native (Go or Rust) driver** over a Java/JDBC agent whenever a mature, license-compatible native driver is available. Native agents ship as a single self-contained executable with no JRE, which significantly reduces memory footprint and startup time — the JVM baseline that every Java agent pays even when idle is avoided entirely.
- **Native (C++/Go/Rust)** — preferred when a usable native driver exists. See `drivers/cassandra-go` (Apache cassandra-gocql-driver), `drivers/duckdb`, `drivers/hive-go` (native HS2), `drivers/iotdb` (Apache IoTDB Go Client), `drivers/oracle-go` (go-ora), `drivers/kingbase-go` (gokb), `drivers/vastbase-go` (openGauss connector), `drivers/tdengine` (taos-connector-rust), `drivers/xugu`, and `drivers/rabbitmq` (amqp091-go) as reference implementations. No JRE download or management is needed.
- **Native (C++/Go/Rust)** — preferred when a usable native driver exists. See `drivers/cassandra-go` (Apache cassandra-gocql-driver), `drivers/duckdb`, `drivers/hive-go` (native HS2), `drivers/iotdb` (Apache IoTDB Go Client), `drivers/oracle-go` (go-ora), `drivers/kingbase-go` (gokb), `drivers/vastbase-go` (openGauss connector), `drivers/tdengine` (taos-connector-rust), `drivers/xugu`, `drivers/rabbitmq` (amqp091-go), and `drivers/rocketmq` (rocketmq-admin-go) as reference implementations. No JRE download or management is needed.
- **Java/JDBC** — the default fallback when only a JDBC driver exists for the database, or when the native driver is immature or unmaintained. Most agents still fall in this category.
Native agents implement the same JSON-RPC contract and `versions.json` registration as Java agents; they ship an `agent` executable instead of `agent.jar`. If both native and Java source implementations exist for the same database, publish only the native artifact unless the Java variant has a separately registered compatibility profile, such as `oracle-legacy` / `oracle-10g`.
@@ -95,9 +96,10 @@ Requires JDK 21 (Gradle toolchain auto-downloads if needed).
(cargo build --manifest-path drivers/tdengine/Cargo.toml --release --locked)
(cd drivers/xugu && go build -o agent .)
(cd drivers/rabbitmq && go build -o agent .)
(cd drivers/rocketmq && go build -o agent .)
```
Output JARs are in `drivers/{module}/build/libs/`. Native agents build from `drivers/cassandra-go`, `drivers/duckdb`, `drivers/hive-go`, `drivers/iotdb`, `drivers/oracle-go`, `drivers/kingbase-go`, `drivers/vastbase-go`, `drivers/tdengine`, `drivers/xugu`, and `drivers/rabbitmq`.
Output JARs are in `drivers/{module}/build/libs/`. Native agents build from `drivers/cassandra-go`, `drivers/duckdb`, `drivers/hive-go`, `drivers/iotdb`, `drivers/oracle-go`, `drivers/kingbase-go`, `drivers/vastbase-go`, `drivers/tdengine`, `drivers/xugu`, `drivers/rabbitmq`, and `drivers/rocketmq`.
### Local DBX Runtime Test
@@ -111,7 +113,7 @@ cp agents/drivers/<db_type>/build/libs/*-all.jar ~/.dbx/agents/drivers/<db_type>
Restart DBX or disconnect and reconnect the database so the new agent process loads the replacement JAR.
Native agents such as `cassandra`, `hive`, `iotdb`, `oracle`, `kingbase`, `tdengine`, `xugu`, and `rabbitmq` use an `agent` executable instead of `agent.jar`. TDengine builds `target/release/dbx-tdengine-driver` from `drivers/tdengine/Cargo.toml`.
Native agents such as `cassandra`, `hive`, `iotdb`, `oracle`, `kingbase`, `tdengine`, `xugu`, `rabbitmq`, and `rocketmq` use an `agent` executable instead of `agent.jar`. TDengine builds `target/release/dbx-tdengine-driver` from `drivers/tdengine/Cargo.toml`.
## Versioning
agents/README.zh-CN.md
+6 -4
View File
@@ -45,11 +45,12 @@ DBX 的 Agent 驱动 —— 通过 JDBC 和原生数据库驱动支持各种数
| etcd | etcd | jetcd |
| zookeeper | Apache ZooKeeper | Apache Curator |
| rabbitmq | RabbitMQ | amqp091-go 原生 agent |
| rocketmq | Apache RocketMQ | rocketmq-admin-go 原生 agent |
## 多 JRE 支持
多数 Java agent 以 JRE 21 为目标。原生 agent（如 `cassandra`、`duckdb`、`hive`、`iotdb`、`oracle`、`kingbase`、`tdengine`、`xugu` 和 `rabbitmq`）不需要 JRE。对 Java agent，DBX 会自动下载并管理 JRE 21 安装。
多数 Java agent 以 JRE 21 为目标。原生 agent（如 `cassandra`、`duckdb`、`hive`、`iotdb`、`oracle`、`kingbase`、`tdengine`、`xugu`、`rabbitmq` 和 `rocketmq`）不需要 JRE。对 Java agent，DBX 会自动下载并管理 JRE 21 安装。
## JDBC 连接池
@@ -75,7 +76,7 @@ HikariCP 会直接打进启用连接池的 Agent JAR。已经使用 DBX 托管 J
对于新 agent，只要存在成熟、许可证兼容的原生驱动，优先选择**原生（Go 或 Rust）驱动**而非 Java/JDBC agent。原生 agent 以单一自包含可执行文件发布，无需 JRE，可显著降低内存占用和启动时间 —— 完全避开 Java agent 即便空闲也要付出的 JVM 基线开销。
- **原生（Go/Rust）** —— 存在可用原生驱动时首选。参考 `drivers/cassandra-go`（Apache cassandra-gocql-driver）、`drivers/duckdb`、`drivers/hive-go`（原生 HS2）、`drivers/iotdb`（Apache IoTDB Go Client）、`drivers/oracle-go`（go-ora）、`drivers/kingbase-go`（gokb）、`drivers/vastbase-go`（openGauss connector）、`drivers/tdengine`（taos-connector-rust）、`drivers/xugu` 和 `drivers/rabbitmq`（amqp091-go）。无需 JRE 下载与管理。
- **原生（Go/Rust）** —— 存在可用原生驱动时首选。参考 `drivers/cassandra-go`（Apache cassandra-gocql-driver）、`drivers/duckdb`、`drivers/hive-go`（原生 HS2）、`drivers/iotdb`（Apache IoTDB Go Client）、`drivers/oracle-go`（go-ora）、`drivers/kingbase-go`（gokb）、`drivers/vastbase-go`（openGauss connector）、`drivers/tdengine`（taos-connector-rust）、`drivers/xugu`、`drivers/rabbitmq`（amqp091-go）和 `drivers/rocketmq`（rocketmq-admin-go）。无需 JRE 下载与管理。
- **Java/JDBC** —— 当某数据库只有 JDBC 驱动，或原生驱动不成熟、缺乏维护时的默认兜底方案。多数 agent 仍属此类。
原生 agent 实现与 Java agent 相同的 JSON-RPC 契约和 `versions.json` 登记；它发布的是 `agent` 可执行文件而非 `agent.jar`。若同一数据库同时保留原生和 Java 源码实现，默认只发布原生产物；只有 Java 变体以独立兼容配置登记时才同时发布，例如 `oracle-legacy` / `oracle-10g`。
@@ -95,9 +96,10 @@ HikariCP 会直接打进启用连接池的 Agent JAR。已经使用 DBX 托管 J
(cargo build --manifest-path drivers/tdengine/Cargo.toml --release --locked)
(cd drivers/xugu && go build -o agent .)
(cd drivers/rabbitmq && go build -o agent .)
(cd drivers/rocketmq && go build -o agent .)
```
产物 JAR 在 `drivers/{module}/build/libs/`。原生 agent 从 `drivers/cassandra-go`、`drivers/duckdb`、`drivers/hive-go`、`drivers/iotdb`、`drivers/oracle-go`、`drivers/kingbase-go`、`drivers/vastbase-go`、`drivers/tdengine`、`drivers/xugu` 和 `drivers/rabbitmq` 构建。
产物 JAR 在 `drivers/{module}/build/libs/`。原生 agent 从 `drivers/cassandra-go`、`drivers/duckdb`、`drivers/hive-go`、`drivers/iotdb`、`drivers/oracle-go`、`drivers/kingbase-go`、`drivers/vastbase-go`、`drivers/tdengine`、`drivers/xugu`、`drivers/rabbitmq` 和 `drivers/rocketmq` 构建。
### 本地 DBX 运行时测试
@@ -111,7 +113,7 @@ cp agents/drivers/<db_type>/build/libs/*-all.jar ~/.dbx/agents/drivers/<db_type>
重启 DBX 或断开重连数据库，使新 agent 进程加载替换后的 JAR。
`cassandra`、`hive`、`iotdb`、`oracle`、`kingbase`、`tdengine`、`xugu` 和 `rabbitmq` 等原生 agent 使用可执行文件而非 `agent.jar`。TDengine 从 `drivers/tdengine/Cargo.toml` 构建 `target/release/dbx-tdengine-driver`。
`cassandra`、`hive`、`iotdb`、`oracle`、`kingbase`、`tdengine`、`xugu`、`rabbitmq` 和 `rocketmq` 等原生 agent 使用可执行文件而非 `agent.jar`。TDengine 从 `drivers/tdengine/Cargo.toml` 构建 `target/release/dbx-tdengine-driver`。
## 版本管理
agents/build.gradle
+1 -1
View File
@@ -3,7 +3,7 @@ plugins {
}
def infrastructureProjects = ['common', 'test-support'] as Set
def legacyStandaloneProjects = ['mongodb', 'kafka', 'rocketmq'] as Set
def legacyStandaloneProjects = ['mongodb', 'kafka'] as Set
def pooledJdbcProjects = [
'access', 'bigquery', 'dameng', 'databend', 'databricks', 'db2', 'exasol',
'firebird', 'gbase8a', 'gbase8s', 'goldendb', 'h2', 'h2-legacy', 'highgo',
agents/drivers/rocketmq/acl.go
+240
View File
@@ -0,0 +1,240 @@
package main
import (
"context"
"encoding/json"
"fmt"
"strings"
admin "github.com/amigoer/rocketmq-admin-go"
"github.com/amigoer/rocketmq-admin-go/protocol/remoting"
)
type aclWire struct {
Subject string `json:"subject"`
Policies []aclPolicyWire `json:"policies"`
}
type aclPolicyWire struct {
PolicyType string `json:"policyType,omitempty"`
Entries []aclEntryWire `json:"entries"`
}
type aclEntryWire struct {
Resource string `json:"resource"`
Actions []string `json:"actions"`
SourceIPs []string `json:"sourceIps"`
Decision string `json:"decision"`
}
func (a *rocketMQAgent) listACLs(params map[string]any) (any, error) {
_, config, err := a.requireClient()
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout)
defer cancel()
address, err := a.brokerAddressForName(stringValue(params, "brokerName"))
if err != nil {
return nil, err
}
response, err := invokeRemotingWithClient(ctx, address, remoting.NewRequest(remoting.ListAcl, map[string]string{
"subjectFilter": strings.TrimPrefix(stringValue(params, "principal", "subject"), "User:"),
"resourceFilter": stringValue(params, "resourceName"),
}))
if err != nil {
return nil, err
}
var wrapper struct {
Acls []aclWire `json:"acls"`
}
body := repairRocketMQJSON(response.Body)
if len(body) > 0 && body[0] == '[' {
if err := json.Unmarshal(body, &wrapper.Acls); err != nil {
return nil, fmt.Errorf("decode ACL list: %w", err)
}
} else if err := json.Unmarshal(body, &wrapper); err != nil {
return nil, fmt.Errorf("decode ACL list: %w", err)
}
principalFilter := strings.TrimPrefix(stringValue(params, "principal", "subject"), "User:")
resourceFilter := stringValue(params, "resourceName")
rows := make([]map[string]any, 0)
for _, acl := range wrapper.Acls {
if principalFilter != "" && acl.Subject != principalFilter && acl.Subject != "User:"+principalFilter {
continue
}
for _, policy := range acl.Policies {
for _, entry := range policy.Entries {
if resourceFilter != "" && entry.Resource != resourceFilter {
continue
}
actions := entry.Actions
if len(actions) == 0 {
actions = []string{"ALL"}
}
for _, action := range actions {
host := "*"
if len(entry.SourceIPs) > 0 {
host = strings.Join(entry.SourceIPs, ",")
}
rows = append(rows, map[string]any{
"resourceType": "TOPIC", "resourceName": entry.Resource,
"patternType": "LITERAL", "principal": acl.Subject,
"host": host, "operation": normalizeACLOperation(action),
"permissionType": entry.Decision,
})
}
}
}
}
return map[string]any{"acls": rows}, nil
}
func (a *rocketMQAgent) createACLs(params map[string]any) (any, error) {
client, config, err := a.requireClient()
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout)
defer cancel()
address, err := a.brokerAddressForName(stringValue(params, "brokerName"))
if err != nil {
return nil, err
}
entries, _ := params["acls"].([]any)
for _, raw := range entries {
entry, _ := raw.(map[string]any)
if _, hasAccessKey := entry["accessKey"]; hasAccessKey || entry["secretKey"] != nil {
if err := client.UpdatePlainAccessConfig(ctx, address, plainAccessConfig(entry)); err != nil {
return nil, err
}
continue
}
subject := strings.TrimPrefix(stringValue(entry, "principal", "subject"), "User:")
if subject == "" {
return nil, fmt.Errorf("ACL principal is required")
}
resource := stringValue(entry, "resourceName")
if resource == "" {
resource = "*"
}
host := stringValue(entry, "host")
if host == "" {
host = "*"
}
decision := stringValue(entry, "permissionType")
if decision == "" {
decision = "ALLOW"
}
acl := aclWire{Subject: subject, Policies: []aclPolicyWire{{Entries: []aclEntryWire{{
Resource: resource, Actions: []string{mapACLOperation(stringValue(entry, "operation"))},
SourceIPs: []string{host}, Decision: decision,
}}}}}
body, marshalErr := json.Marshal(acl)
if marshalErr != nil {
return nil, marshalErr
}
command := remoting.NewRequest(remoting.CreateAcl, map[string]string{"subject": subject})
command.Body = body
if _, err := invokeRemotingWithClient(ctx, address, command); err != nil {
return nil, err
}
}
return okResult(), nil
}
func (a *rocketMQAgent) deleteACLs(params map[string]any) (any, error) {
client, config, err := a.requireClient()
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout)
defer cancel()
address, err := a.brokerAddressForName(stringValue(params, "brokerName"))
if err != nil {
return nil, err
}
filters, _ := params["filters"].([]any)
if len(filters) == 0 {
listed, listErr := a.listACLs(params)
if listErr != nil {
return nil, listErr
}
for _, row := range listed.(map[string]any)["acls"].([]map[string]any) {
filters = append(filters, row)
}
}
deleted := 0
seen := make(map[string]struct{})
for _, raw := range filters {
filter, _ := raw.(map[string]any)
if accessKey := stringValue(filter, "accessKey"); accessKey != "" {
if err := client.DeletePlainAccessConfig(ctx, address, accessKey); err != nil {
return nil, err
}
deleted++
continue
}
subject := strings.TrimPrefix(stringValue(filter, "principal", "subject"), "User:")
resource := stringValue(filter, "resourceName")
key := subject + "|" + resource
if subject == "" {
continue
}
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
fields := map[string]string{"subject": subject}
if resource != "" {
fields["resource"] = resource
}
if _, err := invokeRemotingWithClient(ctx, address, remoting.NewRequest(remoting.DeleteAcl, fields)); err != nil {
return nil, err
}
deleted++
}
return map[string]any{"ok": true, "deleted": deleted}, nil
}
func plainAccessConfig(entry map[string]any) admin.PlainAccessConfig {
return admin.PlainAccessConfig{
AccessKey: stringValue(entry, "accessKey"), SecretKey: stringValue(entry, "secretKey"),
WhiteRemoteAddress: stringValue(entry, "whiteRemoteAddress"),
Admin: boolValue(entry, false, "admin"),
DefaultTopicPerm: stringValue(entry, "defaultTopicPerm"),
DefaultGroupPerm: stringValue(entry, "defaultGroupPerm"),
TopicPerms: stringSlice(entry["topicPerms"]), GroupPerms: stringSlice(entry["groupPerms"]),
}
}
func stringSlice(value any) []string {
items, _ := value.([]any)
result := make([]string, 0, len(items))
for _, item := range items {
result = append(result, fmt.Sprint(item))
}
return result
}
func mapACLOperation(operation string) string {
switch strings.ToUpper(operation) {
case "WRITE", "PRODUCE", "PUB":
return "PUB"
case "READ", "CONSUME", "SUB":
return "SUB"
default:
return "ALL"
}
}
func normalizeACLOperation(operation string) string {
switch strings.ToUpper(operation) {
case "PUB":
return "WRITE"
case "SUB":
return "READ"
default:
return strings.ToUpper(operation)
}
}
agents/drivers/rocketmq/build.gradle
-13
View File
@@ -1,13 +0,0 @@
dependencies {
implementation 'com.google.code.gson:gson:2.12.1'
implementation 'org.apache.rocketmq:rocketmq-client:5.3.1'
implementation 'org.apache.rocketmq:rocketmq-tools:5.3.1'
runtimeOnly 'org.slf4j:slf4j-simple:1.7.36'
}
tasks.named('shadowJar') {
mergeServiceFiles()
manifest {
attributes('Agent-Label': 'Apache RocketMQ', 'Main-Class': 'com.dbx.agent.rocketmq.RocketMqAgent')
}
}
agents/drivers/rocketmq/cluster.go
+67
View File
@@ -0,0 +1,67 @@
package main
import (
"context"
"strconv"
admin "github.com/amigoer/rocketmq-admin-go"
)
func (a *rocketMQAgent) describeCluster(_ map[string]any) (any, error) {
client, config, err := a.requireClient()
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout)
defer cancel()
info, err := client.ExamineBrokerClusterInfo(ctx)
if err != nil {
return nil, err
}
a.mu.RLock()
proxies := a.proxies
clusterName := a.clusterName
a.mu.RUnlock()
brokers := brokerNodes(info, proxies)
var controller any
if len(brokers) > 0 {
controller = brokers[0]
}
return map[string]any{
"clusterId": clusterName, "controller": controller,
"brokers": brokers, "nodeCount": len(brokers),
}, nil
}
func brokerNodes(info *admin.ClusterInfo, proxies *proxyManager) []map[string]any {
if info == nil {
return []map[string]any{}
}
nodes := make([]map[string]any, 0)
id := 0
for _, brokerName := range sortedKeys(info.BrokerAddrTable) {
broker := info.BrokerAddrTable[brokerName]
if broker == nil {
continue
}
for _, brokerID := range sortedKeys(broker.BrokerAddrs) {
address := broker.BrokerAddrs[brokerID]
if original := proxies.OriginalForLocal(address); original != "" {
address = original
}
host, portText := parseSocketAddress(address)
port, _ := strconv.Atoi(portText)
brokerIDValue, _ := strconv.ParseInt(brokerID, 10, 64)
role := "SLAVE"
if brokerIDValue == 0 {
role = "MASTER"
}
nodes = append(nodes, map[string]any{
"id": id, "host": host, "port": port, "rack": nil,
"brokerName": broker.BrokerName, "brokerId": brokerIDValue, "role": role,
})
id++
}
}
return nodes
}
agents/drivers/rocketmq/connection.go
+295
View File
@@ -0,0 +1,295 @@
package main
import (
"context"
"fmt"
"time"
admin "github.com/amigoer/rocketmq-admin-go"
)
const (
defaultRequestTimeout = 30 * time.Second
defaultConnectTimeout = 10 * time.Second
brokerRegistrationPollInterval = 100 * time.Millisecond
)
type socksProxyConfig struct {
Host string
Port int
Username string
Password string
}
type connectionConfig struct {
NameServers []string
ClusterName string
BrokerAddr string
AccessKey string
SecretKey string
RequestTimeout time.Duration
ConnectTimeout time.Duration
TLSSkipVerify bool
SocksProxy *socksProxyConfig
}
func parseConnection(params map[string]any) (connectionConfig, error) {
connection := nestedMap(params, "connection")
if connection == nil {
connection = params
}
nameServers := splitAddresses(stringValue(connection, "namesrv_addr", "namesrvAddr"))
if len(nameServers) == 0 {
return connectionConfig{}, fmt.Errorf("namesrv_addr is required")
}
config := connectionConfig{
NameServers: nameServers,
ClusterName: stringValue(connection, "cluster_name", "clusterName"),
BrokerAddr: stringValue(connection, "broker_addr", "brokerAddr"),
AccessKey: stringValue(connection, "access_key", "accessKey"),
SecretKey: stringValue(connection, "secret_key", "secretKey"),
RequestTimeout: time.Duration(intValue(connection, int(defaultRequestTimeout/time.Millisecond), "request_timeout_ms")) * time.Millisecond,
ConnectTimeout: time.Duration(intValue(connection, int(defaultConnectTimeout/time.Millisecond), "connect_timeout_ms")) * time.Millisecond,
TLSSkipVerify: boolValue(connection, false, "tls_skip_verify"),
}
if config.RequestTimeout <= 0 {
config.RequestTimeout = defaultRequestTimeout
}
if config.ConnectTimeout <= 0 {
config.ConnectTimeout = defaultConnectTimeout
}
if proxy := nestedMap(connection, "socks_proxy"); proxy != nil {
host := stringValue(proxy, "host")
port := intValue(proxy, 0, "port")
if host == "" || port <= 0 || port > 65535 {
return connectionConfig{}, fmt.Errorf("socks_proxy host and port are required")
}
config.SocksProxy = &socksProxyConfig{
Host: host, Port: port,
Username: stringValue(proxy, "username"),
Password: stringValue(proxy, "password"),
}
}
return config, nil
}
func (a *rocketMQAgent) connect(params map[string]any) (any, error) {
config, err := parseConnection(params)
if err != nil {
return nil, err
}
client, proxies, clusterInfo, err := buildClient(config)
if err != nil {
return nil, err
}
clusterName := resolveClusterName(clusterInfo, config.ClusterName)
brokerAddr, err := resolveBrokerAddr(clusterInfo, config, proxies)
if err != nil {
client.Close()
proxies.Close()
return nil, err
}
a.mu.Lock()
oldClient, oldProxies := a.client, a.proxies
a.client = client
a.proxies = proxies
a.connection = config
a.clusterName = clusterName
a.brokerAddr = brokerAddr
a.mu.Unlock()
if oldClient != nil {
_ = oldClient.Close()
}
if oldProxies != nil {
oldProxies.Close()
}
return clusterTestResult(client, clusterInfo, config, proxies), nil
}
func (a *rocketMQAgent) testConnection(params map[string]any) (any, error) {
config, err := parseConnection(params)
if err != nil {
return nil, err
}
a.mu.RLock()
if a.client != nil && a.connection.equal(config) {
client, proxies := a.client, a.proxies
a.mu.RUnlock()
ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout)
defer cancel()
clusterInfo, err := client.ExamineBrokerClusterInfo(ctx)
if err != nil {
return nil, err
}
return clusterTestResult(client, clusterInfo, config, proxies), nil
}
a.mu.RUnlock()
client, proxies, clusterInfo, err := buildClient(config)
if err != nil {
return nil, err
}
defer client.Close()
defer proxies.Close()
return clusterTestResult(client, clusterInfo, config, proxies), nil
}
func buildClient(config connectionConfig) (*admin.Client, *proxyManager, *admin.ClusterInfo, error) {
proxies := newProxyManager(config)
localNameServers := make([]string, 0, len(config.NameServers))
for _, address := range config.NameServers {
local, err := proxies.ProxyFor(address, proxyTargetNameServer)
if err != nil {
proxies.Close()
return nil, nil, nil, err
}
localNameServers = append(localNameServers, local)
}
client, err := admin.NewClient(
admin.WithNameServers(localNameServers),
admin.WithTimeout(config.RequestTimeout),
admin.WithRetryTimes(1),
)
if err != nil {
proxies.Close()
return nil, nil, nil, err
}
if err := client.Start(); err != nil {
proxies.Close()
return nil, nil, nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), config.ConnectTimeout)
defer cancel()
clusterInfo, err := waitForBrokerRegistration(ctx, brokerRegistrationPollInterval, client.ExamineBrokerClusterInfo)
if err != nil {
client.Close()
proxies.Close()
return nil, nil, nil, err
}
return client, proxies, clusterInfo, nil
}
func waitForBrokerRegistration(
ctx context.Context,
pollInterval time.Duration,
examine func(context.Context) (*admin.ClusterInfo, error),
) (*admin.ClusterInfo, error) {
if pollInterval <= 0 {
pollInterval = brokerRegistrationPollInterval
}
var lastErr error
for {
clusterInfo, err := examine(ctx)
if err == nil && hasMasterBroker(clusterInfo) {
return clusterInfo, nil
}
if err != nil {
lastErr = err
}
timer := time.NewTimer(pollInterval)
select {
case <-ctx.Done():
timer.Stop()
if lastErr != nil {
return nil, fmt.Errorf("wait for RocketMQ broker registration after %v: %w", lastErr, ctx.Err())
}
return nil, fmt.Errorf("wait for RocketMQ broker registration: %w", ctx.Err())
case <-timer.C:
}
}
}
func hasMasterBroker(info *admin.ClusterInfo) bool {
if info == nil {
return false
}
for _, broker := range info.BrokerAddrTable {
if broker != nil && broker.BrokerAddrs["0"] != "" {
return true
}
}
return false
}
func clusterTestResult(client *admin.Client, info *admin.ClusterInfo, config connectionConfig, proxies *proxyManager) map[string]any {
brokers := brokerNodes(info, proxies)
aclEnabled := false
if len(brokers) > 0 {
ctx, cancel := context.WithTimeout(context.Background(), minDuration(config.ConnectTimeout/2, 5*time.Second))
defer cancel()
brokerAddr, err := resolveBrokerAddr(info, config, proxies)
if err == nil {
_, aclErr := client.GetBrokerClusterAclInfo(ctx, brokerAddr)
aclEnabled = aclErr == nil
}
}
var controller any
if len(brokers) > 0 {
controller = brokers[0]
}
return map[string]any{
"ok": true, "clusterId": resolveClusterName(info, config.ClusterName),
"brokers": brokers, "nodeCount": len(brokers), "controller": controller,
"aclEnabled": aclEnabled,
}
}
func (config connectionConfig) equal(other connectionConfig) bool {
if config.ClusterName != other.ClusterName || config.BrokerAddr != other.BrokerAddr ||
config.AccessKey != other.AccessKey || config.SecretKey != other.SecretKey ||
config.RequestTimeout != other.RequestTimeout || config.ConnectTimeout != other.ConnectTimeout ||
config.TLSSkipVerify != other.TLSSkipVerify ||
len(config.NameServers) != len(other.NameServers) {
return false
}
for index := range config.NameServers {
if config.NameServers[index] != other.NameServers[index] {
return false
}
}
if (config.SocksProxy == nil) != (other.SocksProxy == nil) {
return false
}
return config.SocksProxy == nil || *config.SocksProxy == *other.SocksProxy
}
func resolveClusterName(info *admin.ClusterInfo, configured string) string {
if configured != "" {
return configured
}
for _, name := range sortedKeys(info.ClusterAddrTable) {
return name
}
return "DefaultCluster"
}
func resolveBrokerAddr(info *admin.ClusterInfo, config connectionConfig, proxies *proxyManager) (string, error) {
if config.BrokerAddr != "" {
if local := proxies.LocalForOriginal(config.BrokerAddr); local != "" {
return local, nil
}
return proxies.ProxyFor(config.BrokerAddr, proxyTargetBroker)
}
for _, brokerName := range sortedKeys(info.BrokerAddrTable) {
broker := info.BrokerAddrTable[brokerName]
if address := broker.BrokerAddrs["0"]; address != "" {
return address, nil
}
for _, brokerID := range sortedKeys(broker.BrokerAddrs) {
if broker.BrokerAddrs[brokerID] != "" {
return broker.BrokerAddrs[brokerID], nil
}
}
}
return "", fmt.Errorf("no RocketMQ broker address found")
}
func minDuration(left, right time.Duration) time.Duration {
if left <= 0 || left > right {
return right
}
return left
}
agents/drivers/rocketmq/connection_test.go
+118
View File
@@ -0,0 +1,118 @@
package main
import (
"context"
"errors"
"reflect"
"testing"
"time"
admin "github.com/amigoer/rocketmq-admin-go"
)
func TestParseConnectionSupportsAliasesAndMultipleNameServers(t *testing.T) {
config, err := parseConnection(map[string]any{"connection": map[string]any{
"namesrvAddr": "127.0.0.1:9876; [2001:db8::1]:9876,127.0.0.1:9876",
"clusterName": "DefaultCluster",
"brokerAddr": "broker.example.com:10911",
"accessKey": "access",
"secretKey": "secret",
"request_timeout_ms": float64(12_000),
"connect_timeout_ms": "4500",
"tls_skip_verify": true,
}})
if err != nil {
t.Fatal(err)
}
wantServers := []string{"127.0.0.1:9876", "[2001:db8::1]:9876"}
if !reflect.DeepEqual(config.NameServers, wantServers) {
t.Fatalf("NameServers = %#v, want %#v", config.NameServers, wantServers)
}
if config.ClusterName != "DefaultCluster" || config.BrokerAddr != "broker.example.com:10911" {
t.Fatalf("unexpected route config: %#v", config)
}
if config.AccessKey != "access" || config.SecretKey != "secret" {
t.Fatalf("unexpected credentials: %#v", config)
}
if config.RequestTimeout != 12*time.Second || config.ConnectTimeout != 4500*time.Millisecond {
t.Fatalf("unexpected timeouts: %#v", config)
}
if !config.TLSSkipVerify {
t.Fatal("tls_skip_verify was not parsed")
}
}
func TestParseConnectionValidatesNameServerAndSocksProxy(t *testing.T) {
if _, err := parseConnection(map[string]any{}); err == nil {
t.Fatal("expected missing namesrv_addr to fail")
}
if _, err := parseConnection(map[string]any{
"namesrv_addr": "127.0.0.1:9876",
"socks_proxy": map[string]any{"host": "127.0.0.1"},
}); err == nil {
t.Fatal("expected incomplete socks_proxy to fail")
}
}
func TestConnectionConfigEqualIncludesTLSSkipVerify(t *testing.T) {
base := connectionConfig{NameServers: []string{"127.0.0.1:9876"}}
other := base
other.TLSSkipVerify = true
if base.equal(other) {
t.Fatal("configs with different tls_skip_verify values must not be equal")
}
}
func TestWaitForBrokerRegistrationRetriesUntilMasterAppears(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
calls := 0
info, err := waitForBrokerRegistration(ctx, time.Millisecond, func(context.Context) (*admin.ClusterInfo, error) {
calls++
if calls < 3 {
return &admin.ClusterInfo{BrokerAddrTable: map[string]*admin.BrokerData{}}, nil
}
return &admin.ClusterInfo{BrokerAddrTable: map[string]*admin.BrokerData{
"broker-a": {BrokerAddrs: map[string]string{"0": "127.0.0.1:10911"}},
}}, nil
})
if err != nil {
t.Fatal(err)
}
if calls != 3 || !hasMasterBroker(info) {
t.Fatalf("calls = %d, info = %#v", calls, info)
}
}
func TestWaitForBrokerRegistrationTimesOut(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
defer cancel()
_, err := waitForBrokerRegistration(ctx, time.Millisecond, func(context.Context) (*admin.ClusterInfo, error) {
return &admin.ClusterInfo{BrokerAddrTable: map[string]*admin.BrokerData{}}, nil
})
if !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("error = %v, want context deadline exceeded", err)
}
}
func TestParseAndFormatSocketAddress(t *testing.T) {
tests := []struct {
address string
host string
port string
formatted string
}{
{"127.0.0.1:9876", "127.0.0.1", "9876", "127.0.0.1:9876"},
{"[2001:db8::1]:9876", "2001:db8::1", "9876", "[2001:db8::1]:9876"},
{"broker.example.com", "broker.example.com", "", "broker.example.com"},
}
for _, test := range tests {
host, port := parseSocketAddress(test.address)
if host != test.host || port != test.port {
t.Fatalf("parseSocketAddress(%q) = %q, %q", test.address, host, port)
}
if formatted := formatSocketAddress(host, port); formatted != test.formatted {
t.Fatalf("formatSocketAddress(%q, %q) = %q", host, port, formatted)
}
}
}
agents/drivers/rocketmq/consumers.go
+525
View File
@@ -0,0 +1,525 @@
package main
import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"
"sync"
"time"
admin "github.com/amigoer/rocketmq-admin-go"
"github.com/amigoer/rocketmq-admin-go/protocol/remoting"
)
var systemConsumerGroups = map[string]struct{}{
"TOOLS_CONSUMER": {}, "FILTERSRV_CONSUMER": {}, "SELF_TEST_C_GROUP": {},
"CID_ONS-HTTP-PROXY": {}, "CID_ONSAPI_PULL": {}, "CID_ONSAPI_PERMISSION": {},
"CID_ONSAPI_OWNER": {}, "CID_SYS_RMQ_TRANS": {}, "CID_DefaultHeartBeatSyncerTopic": {},
}
type subscriptionGroupConfig struct {
GroupName string `json:"groupName"`
ConsumeEnable bool `json:"consumeEnable"`
ConsumeFromMinEnable bool `json:"consumeFromMinEnable"`
ConsumeBroadcastEnable bool `json:"consumeBroadcastEnable"`
ConsumeMessageOrderly bool `json:"consumeMessageOrderly"`
RetryQueueNums int `json:"retryQueueNums"`
RetryMaxTimes int `json:"retryMaxTimes"`
BrokerID int64 `json:"brokerId"`
WhichBrokerWhenConsumeSlowly int64 `json:"whichBrokerWhenConsumeSlowly"`
NotifyConsumerIDsChangedEnable bool `json:"notifyConsumerIdsChangedEnable"`
}
func (a *rocketMQAgent) listConsumerGroups(params map[string]any) (any, error) {
client, config, err := a.requireClient()
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout)
defer cancel()
configs, err := a.collectSubscriptionGroupConfigs(ctx)
if err != nil {
return nil, err
}
topicFilter := stringValue(params, "topic")
groupNames := make([]string, 0)
if topicFilter != "" {
groups, queryErr := client.QueryTopicConsumeByWho(ctx, topicFilter)
if queryErr != nil {
return nil, queryErr
}
groupNames = append(groupNames, groups...)
} else {
for name := range configs {
groupNames = append(groupNames, name)
}
}
sort.Strings(groupNames)
groupNames = uniqueStrings(groupNames)
keyword := strings.ToLower(stringValue(params, "keyword"))
rows := make([]map[string]any, 0, len(groupNames))
for _, groupID := range groupNames {
if keyword != "" && !strings.Contains(strings.ToLower(groupID), keyword) {
continue
}
row := map[string]any{
"groupId": groupID, "state": "UNKNOWN", "simpleGroup": false,
"groupType": classifyConsumerGroup(groupID, configs[groupID]), "messageModel": "CLUSTERING",
}
if topicFilter != "" {
row["topics"] = []string{topicFilter}
}
rows = append(rows, row)
}
offset := max(0, intValue(params, 0, "offset"))
limit := intValue(params, 200, "limit")
if limit <= 0 {
limit = 200
}
page := paginate(rows, offset, limit)
if boolValue(params, false, "enrich") {
a.enrichConsumerGroups(ctx, client, page)
}
if topicFilter != "" && boolValue(params, false, "includeLag") {
a.attachConsumerGroupLag(page, topicFilter)
}
return map[string]any{
"groups": page, "total": len(rows), "offset": offset, "limit": limit,
}, nil
}
func (a *rocketMQAgent) describeConsumerGroup(params map[string]any) (any, error) {
groupID, err := requireString(params, "groupId")
if err != nil {
return nil, err
}
client, config, _ := a.requireClient()
ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout)
defer cancel()
connection, err := client.ExamineConsumerConnectionInfo(ctx, groupID)
if err != nil {
return nil, err
}
assignments := make([]map[string]any, 0, len(connection.SubscriptionTable))
for _, topic := range sortedKeys(connection.SubscriptionTable) {
subscription := connection.SubscriptionTable[topic]
assignments = append(assignments, map[string]any{
"topic": subscription.Topic, "subExpression": subscription.SubString,
})
}
members := make([]map[string]any, 0, len(connection.ConnectionSet))
for _, member := range connection.ConnectionSet {
members = append(members, map[string]any{
"memberId": member.ClientId, "clientId": member.ClientId,
"host": member.ClientAddr, "assignments": assignments,
})
}
return map[string]any{
"groupId": groupID, "state": valueOrDefault(connection.ConsumeType, "UNKNOWN"),
"partitionAssignor": emptyToNil(connection.MessageModel),
"messageModel": emptyToNil(connection.MessageModel), "members": members,
}, nil
}
func (a *rocketMQAgent) deleteConsumerGroup(params map[string]any) (any, error) {
groupID, err := requireString(params, "groupId")
if err != nil {
return nil, err
}
client, config, _ := a.requireClient()
ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout)
defer cancel()
addresses, err := a.masterBrokerAddresses("")
if err != nil {
return nil, err
}
successCount := 0
var lastErr error
for _, address := range addresses {
if mutationErr := client.DeleteSubscriptionGroup(ctx, address, groupID); mutationErr != nil {
lastErr = mutationErr
continue
}
successCount++
}
if err := ensureMutationCoverage("delete", "consumer group "+groupID, len(addresses), successCount, lastErr); err != nil {
return nil, err
}
return okResult(), nil
}
func (a *rocketMQAgent) getSubscriptionGroupConfig(params map[string]any) (any, error) {
groupID, err := requireString(params, "groupId")
if err != nil {
return nil, err
}
_, config, _ := a.requireClient()
ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout)
defer cancel()
configs, err := a.collectSubscriptionGroupConfigs(ctx)
if err != nil {
return nil, err
}
groupConfig := configs[groupID]
if groupConfig == nil {
return nil, fmt.Errorf("consumer group not found: %s", groupID)
}
return subscriptionGroupConfigMap(groupConfig), nil
}
func (a *rocketMQAgent) alterSubscriptionGroupConfig(params map[string]any) (any, error) {
groupID, err := requireString(params, "groupId")
if err != nil {
return nil, err
}
_, config, _ := a.requireClient()
ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout)
defer cancel()
configs, collectErr := a.collectSubscriptionGroupConfigs(ctx)
if collectErr != nil {
return nil, collectErr
}
groupConfig := configs[groupID]
if groupConfig == nil {
groupConfig = &subscriptionGroupConfig{
GroupName: groupID, ConsumeEnable: true, ConsumeFromMinEnable: true,
RetryQueueNums: 1, RetryMaxTimes: 16, NotifyConsumerIDsChangedEnable: true,
}
}
applySubscriptionGroupUpdates(groupConfig, params)
addresses, err := a.masterBrokerAddresses("")
if err != nil {
return nil, err
}
successCount := 0
var lastErr error
for _, address := range addresses {
if mutationErr := writeSubscriptionGroupConfig(ctx, address, groupConfig); mutationErr != nil {
lastErr = mutationErr
continue
}
successCount++
}
if err := ensureMutationCoverage("update", "consumer group "+groupID, len(addresses), successCount, lastErr); err != nil {
return nil, err
}
return okResult(), nil
}
func (a *rocketMQAgent) resetConsumerGroupOffsets(params map[string]any) (any, error) {
groupID, err := requireString(params, "groupId")
if err != nil {
return nil, err
}
topic, err := requireString(params, "topic")
if err != nil {
return nil, err
}
client, config, _ := a.requireClient()
ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout)
defer cancel()
if offsets, ok := params["offsets"].([]any); ok && len(offsets) > 0 {
address, addressErr := a.brokerAddressForName(stringValue(params, "brokerName"))
if addressErr != nil {
return nil, addressErr
}
for _, raw := range offsets {
offset, _ := raw.(map[string]any)
if err := client.UpdateConsumeOffset(ctx, address, groupID, topic,
intValue(offset, 0, "partition"), int64Value(offset, 0, "offset")); err != nil {
return nil, err
}
}
return okResult(), nil
}
position := strings.ToLower(stringValue(params, "position"))
if position == "" {
position = "latest"
}
var timestamp int64
switch position {
case "earliest":
timestamp = 0
case "latest":
timestamp = time.Now().UnixMilli()
case "timestamp":
timestamp = int64Value(params, time.Now().UnixMilli(), "timestampMs")
default:
return nil, fmt.Errorf("unsupported reset position: %s", position)
}
if _, err := client.ResetOffsetByTimestamp(ctx, topic, groupID, timestamp, true); err != nil {
return nil, err
}
return okResult(), nil
}
func (a *rocketMQAgent) getConsumerLag(params map[string]any) (any, error) {
groupID, err := requireString(params, "groupId")
if err != nil {
return nil, err
}
topic, err := requireString(params, "topic")
if err != nil {
return nil, err
}
client, config, _ := a.requireClient()
ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout)
defer cancel()
stats, err := a.examineConsumeStatsByTopic(ctx, client, groupID, topic)
if err != nil {
return nil, err
}
partitions := make([]map[string]any, 0, len(stats.OffsetTable))
var totalLag int64
for key, offset := range stats.OffsetTable {
queue := parseMessageQueueKey(key)
lag := max(int64(0), offset.BrokerOffset-offset.ConsumerOffset)
totalLag += lag
partitions = append(partitions, map[string]any{
"partition": queue.QueueID, "currentOffset": offset.ConsumerOffset,
"endOffset": offset.BrokerOffset, "lag": lag, "brokerName": queue.BrokerName,
"lastTimestamp": offset.LastTimestamp, "consumerClient": "",
})
}
sort.Slice(partitions, func(i, j int) bool {
left, right := partitions[i], partitions[j]
if left["brokerName"] != right["brokerName"] {
return fmt.Sprint(left["brokerName"]) < fmt.Sprint(right["brokerName"])
}
return left["partition"].(int) < right["partition"].(int)
})
return map[string]any{"partitions": partitions, "totalLag": totalLag}, nil
}
func (a *rocketMQAgent) examineConsumeStatsByTopic(
ctx context.Context,
client *admin.Client,
groupID string,
topic string,
) (*admin.ConsumeStats, error) {
route, err := client.ExamineTopicRouteInfo(ctx, topic)
if err != nil {
return nil, err
}
addresses := masterAddressesFromRoute(route)
if len(addresses) == 0 {
return nil, fmt.Errorf("no RocketMQ master broker found for topic %s", topic)
}
merged := &admin.ConsumeStats{OffsetTable: make(map[string]*admin.OffsetWrapper)}
successCount := 0
var lastErr error
for _, address := range addresses {
response, requestErr := invokeRemotingWithClient(ctx, address, remoting.NewRequest(
remoting.GetConsumeStats,
map[string]string{"consumerGroup": groupID, "topic": topic},
))
if requestErr != nil {
lastErr = requestErr
continue
}
partial, decodeErr := decodeConsumeStats(response.Body)
if decodeErr != nil {
lastErr = decodeErr
continue
}
successCount++
for key, offset := range partial.OffsetTable {
merged.OffsetTable[key] = offset
}
merged.ConsumeTps += partial.ConsumeTps
}
if successCount == 0 {
return nil, fmt.Errorf("query consumer lag for group %s on all masters: %w", groupID, lastErr)
}
if len(merged.OffsetTable) == 0 && successCount != len(addresses) {
return nil, fmt.Errorf("consumer lag for group %s is incomplete: %w", groupID, lastErr)
}
return merged, nil
}
func decodeConsumeStats(body []byte) (*admin.ConsumeStats, error) {
var stats admin.ConsumeStats
if err := json.Unmarshal(repairRocketMQJSON(body), &stats); err != nil {
return nil, fmt.Errorf("decode consumer stats: %w", err)
}
if stats.OffsetTable == nil {
stats.OffsetTable = make(map[string]*admin.OffsetWrapper)
}
return &stats, nil
}
func (a *rocketMQAgent) collectSubscriptionGroupConfigs(ctx context.Context) (map[string]*subscriptionGroupConfig, error) {
addresses, err := a.masterBrokerAddresses("")
if err != nil {
return nil, err
}
merged := make(map[string]*subscriptionGroupConfig)
var lastErr error
for _, address := range addresses {
configs, fetchErr := fetchSubscriptionGroupConfigs(ctx, address)
if fetchErr != nil {
lastErr = fetchErr
continue
}
for name, next := range configs {
current := merged[name]
if current == nil || next.ConsumeMessageOrderly {
merged[name] = next
}
}
}
if len(merged) == 0 && lastErr != nil {
return nil, lastErr
}
return merged, nil
}
func fetchSubscriptionGroupConfigs(ctx context.Context, address string) (map[string]*subscriptionGroupConfig, error) {
response, err := invokeRemotingWithClient(ctx, address,
remoting.NewRequest(remoting.GetAllSubscriptionGroupConfig, nil))
if err != nil {
return nil, err
}
var wrapper struct {
SubscriptionGroupTable map[string]*subscriptionGroupConfig `json:"subscriptionGroupTable"`
}
if err := json.Unmarshal(repairRocketMQJSON(response.Body), &wrapper); err != nil {
return nil, fmt.Errorf("decode subscription groups: %w", err)
}
return wrapper.SubscriptionGroupTable, nil
}
func writeSubscriptionGroupConfig(ctx context.Context, address string, config *subscriptionGroupConfig) error {
body, err := json.Marshal(config)
if err != nil {
return fmt.Errorf("encode subscription group config: %w", err)
}
command := remoting.NewRequest(remoting.UpdateAndCreateSubscriptionGroup, nil)
command.Body = body
_, err = invokeRemotingWithClient(ctx, address, command)
return err
}
func ensureMutationCoverage(action, resource string, attempted, succeeded int, lastErr error) error {
if attempted <= 0 {
return fmt.Errorf("no RocketMQ master brokers available to %s %s", action, resource)
}
if succeeded == attempted {
return nil
}
message := fmt.Sprintf("failed to %s %s on all masters: %d of %d succeeded", action, resource, succeeded, attempted)
if lastErr != nil {
return fmt.Errorf("%s: %w", message, lastErr)
}
return fmt.Errorf("%s", message)
}
func (a *rocketMQAgent) enrichConsumerGroups(ctx context.Context, client *admin.Client, rows []map[string]any) {
for _, row := range rows {
groupID := fmt.Sprint(row["groupId"])
connection, err := client.ExamineConsumerConnectionInfo(ctx, groupID)
if err != nil {
if _, ok := row["topics"]; !ok {
row["topics"] = []string{}
}
continue
}
row["consumeType"] = valueOrDefault(connection.ConsumeType, "UNKNOWN")
row["messageModel"] = valueOrDefault(connection.MessageModel, "CLUSTERING")
row["memberCount"] = len(connection.ConnectionSet)
topics := make([]string, 0, len(connection.SubscriptionTable))
for topic := range connection.SubscriptionTable {
topics = append(topics, topic)
}
sort.Strings(topics)
row["topics"] = topics
}
}
func (a *rocketMQAgent) attachConsumerGroupLag(rows []map[string]any, topic string) {
semaphore := make(chan struct{}, 8)
var waitGroup sync.WaitGroup
for _, row := range rows {
row := row
waitGroup.Add(1)
go func() {
defer waitGroup.Done()
semaphore <- struct{}{}
defer func() { <-semaphore }()
result, err := a.getConsumerLag(map[string]any{"groupId": row["groupId"], "topic": topic})
if err != nil {
row["totalLagFailed"] = true
return
}
lag := result.(map[string]any)
row["totalLag"] = lag["totalLag"]
row["partitions"] = lag["partitions"]
}()
}
waitGroup.Wait()
}
func classifyConsumerGroup(groupID string, config *subscriptionGroupConfig) string {
if _, ok := systemConsumerGroups[groupID]; ok {
return "SYSTEM"
}
if config == nil {
return "UNKNOWN"
}
if config.ConsumeMessageOrderly {
return "FIFO"
}
return "NORMAL"
}
func subscriptionGroupConfigMap(config *subscriptionGroupConfig) map[string]any {
return map[string]any{
"groupName": config.GroupName, "consumeEnable": config.ConsumeEnable,
"consumeFromMinEnable": config.ConsumeFromMinEnable,
"consumeBroadcastEnable": config.ConsumeBroadcastEnable,
"consumeMessageOrderly": config.ConsumeMessageOrderly,
"retryQueueNums": config.RetryQueueNums, "retryMaxTimes": config.RetryMaxTimes,
"brokerId": config.BrokerID, "whichBrokerWhenConsumeSlowly": config.WhichBrokerWhenConsumeSlowly,
}
}
func applySubscriptionGroupUpdates(config *subscriptionGroupConfig, params map[string]any) {
config.ConsumeEnable = boolValue(params, config.ConsumeEnable, "consumeEnable")
config.ConsumeFromMinEnable = boolValue(params, config.ConsumeFromMinEnable, "consumeFromMinEnable")
config.ConsumeBroadcastEnable = boolValue(params, config.ConsumeBroadcastEnable, "consumeBroadcastEnable")
config.ConsumeMessageOrderly = boolValue(params, config.ConsumeMessageOrderly, "consumeMessageOrderly")
config.RetryQueueNums = intValue(params, config.RetryQueueNums, "retryQueueNums")
config.RetryMaxTimes = intValue(params, config.RetryMaxTimes, "retryMaxTimes")
config.BrokerID = int64Value(params, config.BrokerID, "brokerId")
config.WhichBrokerWhenConsumeSlowly = int64Value(params, config.WhichBrokerWhenConsumeSlowly, "whichBrokerWhenConsumeSlowly")
}
func uniqueStrings(values []string) []string {
result := values[:0]
var previous string
for index, value := range values {
if index == 0 || value != previous {
result = append(result, value)
previous = value
}
}
return result
}
func valueOrDefault(value, fallback string) string {
if value == "" {
return fallback
}
return value
}
func emptyToNil(value string) any {
if value == "" {
return nil
}
return value
}
agents/drivers/rocketmq/consumers_test.go
+70
View File
@@ -0,0 +1,70 @@
package main
import (
"encoding/json"
"strings"
"testing"
)
func TestClassifyConsumerGroup(t *testing.T) {
if got := classifyConsumerGroup("TOOLS_CONSUMER", nil); got != "SYSTEM" {
t.Fatalf("system group classified as %q", got)
}
if got := classifyConsumerGroup("GID_Orders", nil); got != "UNKNOWN" {
t.Fatalf("missing config classified as %q", got)
}
if got := classifyConsumerGroup("GID_Orders", &subscriptionGroupConfig{}); got != "NORMAL" {
t.Fatalf("normal group classified as %q", got)
}
if got := classifyConsumerGroup("GID_FIFO", &subscriptionGroupConfig{ConsumeMessageOrderly: true}); got != "FIFO" {
t.Fatalf("FIFO group classified as %q", got)
}
}
func TestDecodeConsumeStatsRepairsObjectKeys(t *testing.T) {
body := []byte(`{"consumeTps":1.5,"offsetTable":{{"brokerName":"broker-a","queueId":2,"topic":"Orders"}:{"brokerOffset":20,"consumerOffset":12,"lastTimestamp":9,"pullOffset":13}}}`)
stats, err := decodeConsumeStats(body)
if err != nil {
t.Fatal(err)
}
if stats.ConsumeTps != 1.5 || len(stats.OffsetTable) != 1 {
t.Fatalf("unexpected consume stats: %#v", stats)
}
for key, offset := range stats.OffsetTable {
queue := parseMessageQueueKey(key)
if queue.Topic != "Orders" || queue.BrokerName != "broker-a" || queue.QueueID != 2 {
t.Fatalf("unexpected queue key %q: %#v", key, queue)
}
if offset.BrokerOffset != 20 || offset.ConsumerOffset != 12 {
t.Fatalf("unexpected offset: %#v", offset)
}
}
}
func TestMutationCoverageFailureMessage(t *testing.T) {
err := ensureMutationCoverage("update", "consumer group GID_Orders", 2, 1, nil)
if err == nil || !strings.Contains(err.Error(), "1 of 2") {
t.Fatalf("unexpected partial mutation error: %v", err)
}
if err := ensureMutationCoverage("update", "consumer group GID_Orders", 2, 2, nil); err != nil {
t.Fatal(err)
}
}
func TestSubscriptionGroupConfigWireShape(t *testing.T) {
config := subscriptionGroupConfig{
GroupName: "GID_Orders", ConsumeEnable: true, ConsumeMessageOrderly: true,
RetryQueueNums: 2, RetryMaxTimes: 20, NotifyConsumerIDsChangedEnable: true,
}
body, err := json.Marshal(config)
if err != nil {
t.Fatal(err)
}
var decoded map[string]any
if err := json.Unmarshal(body, &decoded); err != nil {
t.Fatal(err)
}
if decoded["groupName"] != "GID_Orders" || decoded["consumeMessageOrderly"] != true {
t.Fatalf("unexpected group wire body: %#v", decoded)
}
}
agents/drivers/rocketmq/go.mod
+28
View File
@@ -0,0 +1,28 @@
module github.com/t8y2/dbx/agents/drivers/rocketmq
go 1.25.0
require (
github.com/amigoer/rocketmq-admin-go v1.1.1
golang.org/x/net v0.44.0
)
require (
github.com/apache/rocketmq-client-go/v2 v2.1.2 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/golang/mock v1.6.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/sirupsen/logrus v1.9.4 // indirect
github.com/tidwall/gjson v1.18.0 // indirect
github.com/tidwall/match v1.2.0 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
go.uber.org/atomic v1.11.0 // indirect
golang.org/x/sys v0.42.0 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
stathat.com/c/consistent v1.0.0 // indirect
)
agents/drivers/rocketmq/go.sum
+183
View File
@@ -0,0 +1,183 @@
github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/amigoer/rocketmq-admin-go v1.1.1 h1:DhV7c1lukiB5Qy115/b4a7yE3XXLMRYJ/Jz77miLZoc=
github.com/amigoer/rocketmq-admin-go v1.1.1/go.mod h1:I/N1RG6IvNEP2APjKh66Tg7YbZyW9aetiEEl8ziuYPs=
github.com/apache/rocketmq-client-go/v2 v2.1.2 h1:yt73olKe5N6894Dbm+ojRf/JPiP0cxfDNNffKwhpJVg=
github.com/apache/rocketmq-client-go/v2 v2.1.2/go.mod h1:6I6vgxHR3hzrvn+6n/4mrhS+UTulzK/X9LB2Vk1U5gE=
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs=
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tidwall/gjson v1.13.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/match v1.2.0 h1:0pt8FlkOwjN2fPt4bIl4BoNxb98gGHN2ObFEDkrfZnM=
github.com/tidwall/match v1.2.0/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I=
golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
stathat.com/c/consistent v1.0.0 h1:ezyc51EGcRPJUxfHGSgJjWzJdj3NiMU9pNfLNGiXV0c=
stathat.com/c/consistent v1.0.0/go.mod h1:QkzMWzcbB+yQBL2AttO6sgsQS/JSTapcDISJalmCDS0=
agents/drivers/rocketmq/helpers.go
+228
View File
@@ -0,0 +1,228 @@
package main
import (
"encoding/base64"
"encoding/json"
"fmt"
"net"
"sort"
"strconv"
"strings"
)
func nestedMap(params map[string]any, key string) map[string]any {
value, ok := params[key]
if !ok || value == nil {
return nil
}
if result, ok := value.(map[string]any); ok {
return result
}
return nil
}
func stringValue(params map[string]any, keys ...string) string {
for _, key := range keys {
if value, ok := params[key]; ok && value != nil {
switch typed := value.(type) {
case string:
return strings.TrimSpace(typed)
case json.Number:
return typed.String()
case float64:
return strconv.FormatFloat(typed, 'f', -1, 64)
}
}
}
return ""
}
func requireString(params map[string]any, keys ...string) (string, error) {
value := stringValue(params, keys...)
if value == "" {
return "", fmt.Errorf("%s is required", keys[0])
}
return value, nil
}
func intValue(params map[string]any, defaultValue int, keys ...string) int {
for _, key := range keys {
value, ok := params[key]
if !ok || value == nil {
continue
}
switch typed := value.(type) {
case float64:
return int(typed)
case json.Number:
parsed, err := typed.Int64()
if err == nil {
return int(parsed)
}
case string:
parsed, err := strconv.Atoi(strings.TrimSpace(typed))
if err == nil {
return parsed
}
}
}
return defaultValue
}
func int64Value(params map[string]any, defaultValue int64, keys ...string) int64 {
for _, key := range keys {
value, ok := params[key]
if !ok || value == nil {
continue
}
switch typed := value.(type) {
case float64:
return int64(typed)
case json.Number:
parsed, err := typed.Int64()
if err == nil {
return parsed
}
case string:
parsed, err := strconv.ParseInt(strings.TrimSpace(typed), 10, 64)
if err == nil {
return parsed
}
}
}
return defaultValue
}
func boolValue(params map[string]any, defaultValue bool, keys ...string) bool {
for _, key := range keys {
value, ok := params[key]
if !ok || value == nil {
continue
}
switch typed := value.(type) {
case bool:
return typed
case string:
parsed, err := strconv.ParseBool(strings.TrimSpace(typed))
if err == nil {
return parsed
}
}
}
return defaultValue
}
func decodePayload(params map[string]any) ([]byte, error) {
encoded := stringValue(params, "payloadBase64")
if encoded != "" {
payload, err := base64.StdEncoding.DecodeString(encoded)
if err != nil {
return nil, fmt.Errorf("invalid payloadBase64: %w", err)
}
return payload, nil
}
return []byte(stringValue(params, "payload", "payloadText")), nil
}
func splitAddresses(value string) []string {
fields := strings.FieldsFunc(value, func(char rune) bool {
return char == ';' || char == ',' || char == '\n' || char == '\r' || char == '\t' || char == ' '
})
seen := map[string]struct{}{}
result := make([]string, 0, len(fields))
for _, field := range fields {
address := strings.TrimSpace(field)
if address == "" {
continue
}
if _, ok := seen[address]; ok {
continue
}
seen[address] = struct{}{}
result = append(result, address)
}
return result
}
func parseSocketAddress(address string) (string, string) {
trimmed := strings.TrimSpace(address)
if trimmed == "" {
return "", ""
}
if host, port, err := net.SplitHostPort(trimmed); err == nil {
return strings.Trim(host, "[]"), port
}
colon := strings.LastIndex(trimmed, ":")
if colon <= 0 || colon == len(trimmed)-1 {
return trimmed, ""
}
if _, err := strconv.Atoi(trimmed[colon+1:]); err != nil {
return trimmed, ""
}
return strings.Trim(trimmed[:colon], "[]"), trimmed[colon+1:]
}
func formatSocketAddress(host, port string) string {
if host == "" {
return port
}
if port == "" {
return host
}
return net.JoinHostPort(strings.Trim(host, "[]"), port)
}
func isLikelyUnreachableBrokerHost(host string) bool {
host = strings.TrimSpace(strings.ToLower(host))
if host == "" || host == "127.0.0.1" || host == "localhost" || host == "::1" {
return false
}
if strings.HasSuffix(host, ".docker") || strings.Contains(host, ".docker.") {
return true
}
ip := net.ParseIP(host)
return ip != nil && ip.IsPrivate()
}
func sortedKeys[V any](values map[string]V) []string {
keys := make([]string, 0, len(values))
for key := range values {
keys = append(keys, key)
}
sort.Strings(keys)
return keys
}
func paginate[T any](items []T, offset, limit int) []T {
if offset < 0 {
offset = 0
}
if offset >= len(items) {
return []T{}
}
end := len(items)
if limit > 0 && offset+limit < end {
end = offset + limit
}
return items[offset:end]
}
func normalizeTopicPerm(perm int) int {
switch perm {
case 2, 4, 6:
return perm
default:
return 6
}
}
func normalizeMessageType(value string) string {
value = strings.ToUpper(strings.TrimSpace(value))
if value == "" {
return "NORMAL"
}
if value == "ORDER" {
return "FIFO"
}
return value
}
agents/drivers/rocketmq/integration_test.go
+342
View File
@@ -0,0 +1,342 @@
package main
import (
"encoding/base64"
"encoding/json"
"fmt"
"os"
"strings"
"testing"
"time"
)
type integrationTopicList struct {
Topics []struct {
Name string `json:"name"`
Partitions int `json:"partitions"`
ReadQueueNums int `json:"readQueueNums"`
WriteQueueNums int `json:"writeQueueNums"`
MessageType string `json:"messageType"`
} `json:"topics"`
}
type integrationTopicConfig struct {
Configs map[string]struct {
Value string `json:"value"`
} `json:"configs"`
}
type integrationMessages struct {
Messages []struct {
MessageID string `json:"messageId"`
Partition int `json:"partition"`
Offset int64 `json:"offset"`
PayloadText string `json:"payloadText"`
Headers map[string]string `json:"headers"`
} `json:"messages"`
}
type integrationLag struct {
TotalLag int64 `json:"totalLag"`
}
type integrationConsumerGroups struct {
Groups []struct {
GroupID string `json:"groupId"`
GroupType string `json:"groupType"`
} `json:"groups"`
}
func TestRocketMQIntegration(t *testing.T) {
if os.Getenv("ROCKETMQ_INTEGRATION") != "1" {
t.Skip("set ROCKETMQ_INTEGRATION=1 to run against a real RocketMQ broker")
}
nameServer := envString("ROCKETMQ_NAMESRV_ADDR", "127.0.0.1:9876")
version := envString("ROCKETMQ_VERSION", "unknown")
agent := newRocketMQAgent()
t.Cleanup(agent.close)
connection := map[string]any{
"namesrv_addr": nameServer, "request_timeout_ms": 20_000, "connect_timeout_ms": 20_000,
}
var connectionResult struct {
OK bool `json:"ok"`
NodeCount int `json:"nodeCount"`
}
integrationCall(t, agent, "test_connection", map[string]any{"connection": connection}, &connectionResult)
if !connectionResult.OK || connectionResult.NodeCount < 1 {
t.Fatalf("unexpected test connection result: %#v", connectionResult)
}
integrationCall(t, agent, "connect", map[string]any{"connection": connection}, &connectionResult)
if !connectionResult.OK || connectionResult.NodeCount < 1 {
t.Fatalf("unexpected connection result: %#v", connectionResult)
}
var cluster struct {
ClusterID string `json:"clusterId"`
NodeCount int `json:"nodeCount"`
Brokers []any `json:"brokers"`
}
integrationCall(t, agent, "mq_describe_cluster", map[string]any{}, &cluster)
if cluster.ClusterID == "" || cluster.NodeCount < 1 || len(cluster.Brokers) < 1 {
t.Fatalf("unexpected cluster result: %#v", cluster)
}
suffix := fmt.Sprintf("%s_%d", strings.ReplaceAll(version, ".", "_"), time.Now().UnixNano())
topic := "DBX_GO_INTEGRATION_" + suffix
groupID := "GID_DBX_GO_" + suffix
t.Cleanup(func() {
integrationCleanupCall(agent, "mq_delete_consumer_group", map[string]any{"groupId": groupID})
integrationCleanupCall(agent, "mq_delete_topic", map[string]any{"name": topic})
})
integrationCall[map[string]any](t, agent, "mq_create_topic", map[string]any{
"name": topic, "readQueueNums": 2, "writeQueueNums": 2, "perm": 6, "messageType": "NORMAL",
}, nil)
var route struct {
Topic string `json:"topic"`
BrokerDatas []any `json:"brokerDatas"`
QueueDatas []struct {
ReadQueueNums int `json:"readQueueNums"`
WriteQueueNums int `json:"writeQueueNums"`
} `json:"queueDatas"`
}
integrationCall(t, agent, "mq_get_topic_route", map[string]any{"name": topic}, &route)
if route.Topic != topic || len(route.BrokerDatas) == 0 || len(route.QueueDatas) == 0 {
t.Fatalf("unexpected route: %#v", route)
}
if route.QueueDatas[0].ReadQueueNums != 2 || route.QueueDatas[0].WriteQueueNums != 2 {
t.Fatalf("unexpected initial route queues: %#v", route.QueueDatas)
}
var topicsAfterCreate integrationTopicList
integrationCall(t, agent, "mq_list_topics", map[string]any{"keyword": topic, "limit": 10}, &topicsAfterCreate)
if len(topicsAfterCreate.Topics) != 1 || topicsAfterCreate.Topics[0].Name != topic {
t.Fatalf("unexpected topic list after creation: %#v", topicsAfterCreate.Topics)
}
createdTopic := topicsAfterCreate.Topics[0]
if createdTopic.Partitions != 2 || createdTopic.ReadQueueNums != 2 || createdTopic.WriteQueueNums != 2 {
t.Fatalf("unexpected created topic queues: %#v", createdTopic)
}
if strings.HasPrefix(version, "5.") && createdTopic.MessageType != "NORMAL" {
t.Fatalf("RocketMQ %s lost the 5.x message.type attribute: %#v", version, createdTopic)
}
var topicConfig integrationTopicConfig
integrationCall(t, agent, "mq_get_topic_config", map[string]any{"name": topic}, &topicConfig)
if topicConfig.Configs["readQueueNums"].Value != "2" || topicConfig.Configs["writeQueueNums"].Value != "2" {
t.Fatalf("unexpected topic config: %#v", topicConfig)
}
integrationCall[map[string]any](t, agent, "mq_update_partitions", map[string]any{
"name": topic, "readQueueNums": 3, "writeQueueNums": 2,
}, nil)
integrationCall(t, agent, "mq_get_topic_config", map[string]any{"name": topic}, &topicConfig)
if topicConfig.Configs["readQueueNums"].Value != "3" || topicConfig.Configs["writeQueueNums"].Value != "2" {
t.Fatalf("unexpected topic config after partition update: %#v", topicConfig)
}
integrationCall[map[string]any](t, agent, "mq_alter_topic_config", map[string]any{
"name": topic,
"configs": []map[string]any{
{"key": "readQueueNums", "value": "4"},
{"key": "writeQueueNums", "value": "3"},
{"key": "perm", "value": "6"},
},
}, nil)
integrationCall(t, agent, "mq_get_topic_config", map[string]any{"name": topic}, &topicConfig)
if topicConfig.Configs["readQueueNums"].Value != "4" || topicConfig.Configs["writeQueueNums"].Value != "3" || topicConfig.Configs["perm"].Value != "6" {
t.Fatalf("unexpected topic config after alteration: %#v", topicConfig)
}
if strings.HasPrefix(version, "5.") {
var topicsAfterAlter integrationTopicList
integrationCall(t, agent, "mq_list_topics", map[string]any{"keyword": topic, "limit": 10}, &topicsAfterAlter)
if len(topicsAfterAlter.Topics) != 1 || topicsAfterAlter.Topics[0].MessageType != "NORMAL" {
t.Fatalf("RocketMQ %s lost message.type after config updates: %#v", version, topicsAfterAlter.Topics)
}
}
payload := "RocketMQ Go Agent 世界 " + version
var sendResult struct {
OK bool `json:"ok"`
Partition int `json:"partition"`
Offset int64 `json:"offset"`
}
integrationCall(t, agent, "mq_send_message", map[string]any{
"topic": topic, "partition": 0, "key": "dbx-integration-key", "tag": "dbx-integration",
"payloadBase64": base64.StdEncoding.EncodeToString([]byte(payload)),
"headers": map[string]any{"source": "integration", "empty": "", "nil": nil},
}, &sendResult)
if !sendResult.OK || sendResult.Partition != 0 || sendResult.Offset < 0 {
t.Fatalf("unexpected send result: %#v", sendResult)
}
peeked := waitForIntegrationMessages(t, agent, "mq_peek_messages", map[string]any{
"topic": topic, "partition": 0, "count": 10,
})
if peeked.Messages[0].PayloadText != payload || peeked.Messages[0].Headers["source"] != "integration" {
t.Fatalf("unexpected peeked message: %#v", peeked.Messages[0])
}
if _, ok := peeked.Messages[0].Headers["empty"]; ok {
t.Fatalf("empty user property should be omitted: %#v", peeked.Messages[0].Headers)
}
if peeked.Messages[0].MessageID == "" {
t.Fatalf("peeked message has no message ID: %#v", peeked.Messages[0])
}
var viewed struct {
Message struct {
MessageID string `json:"messageId"`
PayloadText string `json:"payloadText"`
} `json:"message"`
}
integrationCall(t, agent, "mq_view_message", map[string]any{
"topic": topic, "msgId": peeked.Messages[0].MessageID,
"partition": peeked.Messages[0].Partition, "offset": peeked.Messages[0].Offset,
}, &viewed)
if viewed.Message.MessageID == "" || viewed.Message.PayloadText != payload {
t.Fatalf("unexpected viewed message: %#v", viewed.Message)
}
queried := waitForIntegrationMessages(t, agent, "mq_query_message_by_key", map[string]any{
"topic": topic, "key": "dbx-integration-key", "maxNum": 10,
"begin": 0, "end": time.Now().Add(time.Minute).UnixMilli(),
})
if queried.Messages[0].PayloadText != payload {
t.Fatalf("unexpected queried message: %#v", queried.Messages[0])
}
queriedByTopic := waitForIntegrationMessages(t, agent, "mq_query_message_by_topic", map[string]any{
"topic": topic, "maxNum": 10, "begin": 0, "end": time.Now().Add(time.Minute).UnixMilli(),
})
if queriedByTopic.Messages[0].PayloadText != payload {
t.Fatalf("unexpected topic query message: %#v", queriedByTopic.Messages[0])
}
var stats struct {
TotalMessages int64 `json:"totalMessages"`
}
integrationCall(t, agent, "mq_get_topic_stats", map[string]any{"name": topic}, &stats)
if stats.TotalMessages < 1 {
t.Fatalf("unexpected topic stats: %#v", stats)
}
integrationCall[map[string]any](t, agent, "mq_alter_subscription_group_config", map[string]any{
"groupId": groupID, "consumeEnable": true, "consumeMessageOrderly": true,
"retryQueueNums": 2, "retryMaxTimes": 20,
}, nil)
var groupConfig struct {
GroupName string `json:"groupName"`
ConsumeMessageOrderly bool `json:"consumeMessageOrderly"`
}
integrationCall(t, agent, "mq_get_subscription_group_config", map[string]any{"groupId": groupID}, &groupConfig)
if groupConfig.GroupName != groupID {
t.Fatalf("unexpected group config: %#v", groupConfig)
}
if strings.HasPrefix(version, "5.") && !groupConfig.ConsumeMessageOrderly {
t.Fatalf("RocketMQ %s did not preserve FIFO group config: %#v", version, groupConfig)
}
var groups integrationConsumerGroups
integrationCall(t, agent, "mq_list_consumer_groups", map[string]any{
"keyword": groupID, "limit": 10,
}, &groups)
if len(groups.Groups) != 1 || groups.Groups[0].GroupID != groupID {
t.Fatalf("unexpected consumer groups: %#v", groups)
}
if strings.HasPrefix(version, "5.") && groups.Groups[0].GroupType != "FIFO" {
t.Fatalf("RocketMQ %s did not classify FIFO group: %#v", version, groups.Groups[0])
}
var producers struct {
Producers []any `json:"producers"`
}
integrationCall(t, agent, "mq_list_producers", map[string]any{}, &producers)
if producers.Producers == nil {
t.Fatal("producer list must be an array")
}
var lagBefore integrationLag
integrationCall(t, agent, "mq_get_consumer_lag", map[string]any{"groupId": groupID, "topic": topic}, &lagBefore)
if lagBefore.TotalLag < 1 {
t.Fatalf("unexpected lag before reset: %#v", lagBefore)
}
integrationCall[map[string]any](t, agent, "mq_reset_consumer_group_offsets", map[string]any{
"groupId": groupID, "topic": topic,
"offsets": []map[string]any{{"partition": 0, "offset": sendResult.Offset + 1}, {"partition": 1, "offset": 0}},
}, nil)
var lagAfter integrationLag
integrationCall(t, agent, "mq_get_consumer_lag", map[string]any{"groupId": groupID, "topic": topic}, &lagAfter)
if lagAfter.TotalLag != 0 {
t.Fatalf("unexpected lag after reset: %#v", lagAfter)
}
var skipped struct {
OK bool `json:"ok"`
ResetGroups int `json:"resetGroups"`
}
integrationCall(t, agent, "mq_skip_topic_accumulation", map[string]any{"topic": topic}, &skipped)
if !skipped.OK || skipped.ResetGroups < 0 {
t.Fatalf("unexpected skip accumulation result: %#v", skipped)
}
integrationCall[map[string]any](t, agent, "mq_delete_consumer_group", map[string]any{"groupId": groupID}, nil)
integrationCall[map[string]any](t, agent, "mq_delete_topic", map[string]any{"name": topic}, nil)
var topics integrationTopicList
integrationCall(t, agent, "mq_list_topics", map[string]any{"keyword": topic, "limit": 10}, &topics)
if len(topics.Topics) != 0 {
t.Fatalf("topic still exists after deletion: %#v", topics.Topics)
}
integrationCall[map[string]any](t, agent, "disconnect", map[string]any{}, nil)
}
func integrationCall[T any](t *testing.T, agent *rocketMQAgent, method string, params map[string]any, target *T) {
t.Helper()
raw, err := json.Marshal(params)
if err != nil {
t.Fatal(err)
}
result, err := agent.dispatch(method, raw)
if err != nil {
t.Fatalf("%s: %v", method, err)
}
if target == nil {
return
}
encoded, err := json.Marshal(result)
if err != nil {
t.Fatal(err)
}
if err := json.Unmarshal(encoded, target); err != nil {
t.Fatalf("%s decode: %v; result=%s", method, err, encoded)
}
}
func integrationCleanupCall(agent *rocketMQAgent, method string, params map[string]any) {
raw, err := json.Marshal(params)
if err == nil {
_, _ = agent.dispatch(method, raw)
}
}
func waitForIntegrationMessages(t *testing.T, agent *rocketMQAgent, method string, params map[string]any) integrationMessages {
t.Helper()
deadline := time.Now().Add(15 * time.Second)
for {
var messages integrationMessages
integrationCall(t, agent, method, params, &messages)
if len(messages.Messages) > 0 {
return messages
}
if time.Now().After(deadline) {
t.Fatalf("%s returned no messages before timeout", method)
}
time.Sleep(250 * time.Millisecond)
}
}
func envString(key, fallback string) string {
if value := strings.TrimSpace(os.Getenv(key)); value != "" {
return value
}
return fallback
}
agents/drivers/rocketmq/main.go
+73
View File
@@ -0,0 +1,73 @@
package main
import (
"bufio"
"encoding/json"
"fmt"
"os"
)
const maxRPCMessageBytes = 32 * 1024 * 1024
type rpcRequest struct {
JSONRPC string `json:"jsonrpc"`
ID json.RawMessage `json:"id"`
Method string `json:"method"`
Params json.RawMessage `json:"params"`
}
type rpcError struct {
Code int `json:"code"`
Message string `json:"message"`
}
type rpcResponse struct {
JSONRPC string `json:"jsonrpc"`
ID json.RawMessage `json:"id"`
Result any `json:"result,omitempty"`
Error *rpcError `json:"error,omitempty"`
}
func main() {
agent := newRocketMQAgent()
defer agent.close()
encoder := json.NewEncoder(os.Stdout)
encoder.SetEscapeHTML(false)
_ = encoder.Encode(map[string]bool{"ready": true})
scanner := bufio.NewScanner(os.Stdin)
scanner.Buffer(make([]byte, 64*1024), maxRPCMessageBytes)
for scanner.Scan() {
var request rpcRequest
if err := json.Unmarshal(scanner.Bytes(), &request); err != nil {
_ = encoder.Encode(rpcResponse{
JSONRPC: "2.0",
ID: json.RawMessage("null"),
Error: &rpcError{Code: -32700, Message: normalizeError(err)},
})
continue
}
result, err := agent.dispatch(request.Method, request.Params)
response := rpcResponse{JSONRPC: "2.0", ID: request.ID}
if len(response.ID) == 0 {
response.ID = json.RawMessage("null")
}
if err != nil {
response.Error = &rpcError{Code: -1, Message: normalizeError(err)}
} else {
response.Result = result
}
if err := encoder.Encode(response); err != nil {
fmt.Fprintln(os.Stderr, err)
return
}
if agent.shutdownRequested {
return
}
}
if err := scanner.Err(); err != nil {
fmt.Fprintln(os.Stderr, err)
}
}
agents/drivers/rocketmq/main_test.go
+115
View File
@@ -0,0 +1,115 @@
package main
import (
"bufio"
"bytes"
"context"
"encoding/json"
"fmt"
"os"
"os/exec"
"strings"
"sync"
"testing"
"time"
)
const rocketMQHelperProcessEnv = "DBX_ROCKETMQ_HELPER_PROCESS"
type synchronizedBuffer struct {
mu sync.Mutex
buffer bytes.Buffer
}
func (b *synchronizedBuffer) Write(data []byte) (int, error) {
b.mu.Lock()
defer b.mu.Unlock()
return b.buffer.Write(data)
}
func (b *synchronizedBuffer) String() string {
b.mu.Lock()
defer b.mu.Unlock()
return b.buffer.String()
}
func TestRocketMQAgentProcessProtocol(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
command := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestRocketMQAgentHelperProcess$")
command.Env = append(os.Environ(), rocketMQHelperProcessEnv+"=1")
stdin, err := command.StdinPipe()
if err != nil {
t.Fatal(err)
}
stdout, err := command.StdoutPipe()
if err != nil {
t.Fatal(err)
}
var stderr synchronizedBuffer
command.Stderr = &stderr
if err := command.Start(); err != nil {
t.Fatal(err)
}
scanner := bufio.NewScanner(stdout)
if !scanner.Scan() || scanner.Text() != `{"ready":true}` {
t.Fatalf("unexpected ready line %q: %s", scanner.Text(), stderr.String())
}
writeRPCRequest(t, stdin, `{"jsonrpc":"2.0","id":1,"method":"handshake","params":{}}`)
handshake := readRPCResponse(t, scanner, stderr.String())
if string(handshake.ID) != "1" || handshake.Error != nil {
t.Fatalf("unexpected handshake response: %#v", handshake)
}
handshakeResult, ok := handshake.Result.(map[string]any)
if !ok || int(handshakeResult["protocolVersion"].(float64)) != protocolVersion {
t.Fatalf("unexpected handshake result: %#v", handshake.Result)
}
writeRPCRequest(t, stdin, `{`)
parseFailure := readRPCResponse(t, scanner, stderr.String())
if string(parseFailure.ID) != "null" || parseFailure.Error == nil || parseFailure.Error.Code != -32700 {
t.Fatalf("unexpected parse failure response: %#v", parseFailure)
}
writeRPCRequest(t, stdin, `{"jsonrpc":"2.0","id":2,"method":"shutdown","params":{}}`)
shutdown := readRPCResponse(t, scanner, stderr.String())
if string(shutdown.ID) != "2" || shutdown.Error != nil {
t.Fatalf("unexpected shutdown response: %#v", shutdown)
}
if err := command.Wait(); err != nil {
t.Fatalf("agent process failed: %v: %s", err, stderr.String())
}
if ctx.Err() != nil {
t.Fatalf("agent process timed out: %v", ctx.Err())
}
}
func TestRocketMQAgentHelperProcess(t *testing.T) {
if os.Getenv(rocketMQHelperProcessEnv) != "1" {
return
}
main()
os.Exit(0)
}
func writeRPCRequest(t *testing.T, stdin interface{ Write([]byte) (int, error) }, request string) {
t.Helper()
if _, err := fmt.Fprintln(stdin, request); err != nil {
t.Fatal(err)
}
}
func readRPCResponse(t *testing.T, scanner *bufio.Scanner, stderr string) rpcResponse {
t.Helper()
if !scanner.Scan() {
t.Fatalf("agent closed stdout: %s", strings.TrimSpace(stderr))
}
var response rpcResponse
if err := json.Unmarshal(scanner.Bytes(), &response); err != nil {
t.Fatalf("decode response %q: %v", scanner.Text(), err)
}
return response
}
agents/drivers/rocketmq/messages.go
+586
View File
@@ -0,0 +1,586 @@
package main
import (
"context"
"encoding/base64"
"fmt"
"sort"
"strconv"
"strings"
"time"
"unicode/utf8"
admin "github.com/amigoer/rocketmq-admin-go"
"github.com/amigoer/rocketmq-admin-go/protocol/remoting"
"github.com/apache/rocketmq-client-go/v2/primitive"
)
const sendMessageRequestCode = 10
const queryMessageNotFoundCode = 208
type messageQueueTarget struct {
BrokerName string
Address string
QueueID int
}
func (a *rocketMQAgent) listProducers(params map[string]any) (any, error) {
client, config, err := a.requireClient()
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout)
defer cancel()
topic := stringValue(params, "topic")
producerGroup := stringValue(params, "producerGroup", "group")
rows := make([]map[string]any, 0)
if producerGroup != "" {
if topic == "" {
return nil, fmt.Errorf("topic is required when producerGroup is specified")
}
connection, queryErr := client.ExamineProducerConnectionInfo(ctx, producerGroup, topic)
if queryErr != nil {
return nil, queryErr
}
for index, connectionInfo := range connection.ConnectionSet {
rows = append(rows, producerRow(int64(index+1), producerGroup, connectionInfo))
}
return map[string]any{"producers": rows}, nil
}
addresses := make([]string, 0)
if topic != "" {
stats, statsErr := a.examineTopicStats(ctx, client, topic)
if statsErr != nil {
return nil, statsErr
}
hasMessages := false
for _, offset := range stats {
if offset.MaxOffset > offset.MinOffset {
hasMessages = true
break
}
}
if !hasMessages {
return map[string]any{"producers": rows}, nil
}
route, routeErr := client.ExamineTopicRouteInfo(ctx, topic)
if routeErr != nil {
return nil, routeErr
}
for _, broker := range route.BrokerDatas {
if address := broker.BrokerAddrs["0"]; address != "" {
addresses = append(addresses, address)
}
}
} else {
a.mu.RLock()
addresses = append(addresses, a.brokerAddr)
a.mu.RUnlock()
}
seen := make(map[string]struct{})
var producerID int64 = 1
for _, address := range uniqueStrings(addresses) {
producerTable, tableErr := client.GetAllProducerInfo(ctx, address)
if tableErr != nil {
continue
}
for _, group := range sortedKeys(producerTable) {
for _, connectionInfo := range producerTable[group] {
key := group + "|" + connectionInfo.ClientAddr
if _, ok := seen[key]; ok {
continue
}
seen[key] = struct{}{}
rows = append(rows, producerRow(producerID, group, connectionInfo))
producerID++
}
}
}
return map[string]any{"producers": rows}, nil
}
func (a *rocketMQAgent) peekMessages(params map[string]any) (any, error) {
topic, err := requireString(params, "topic")
if err != nil {
return nil, err
}
client, config, _ := a.requireClient()
ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout)
defer cancel()
count := min(max(1, intValue(params, 10, "count")), 100)
partition, hasPartition := optionalInt(params, "partition")
offset, hasOffset := optionalInt64(params, "offset")
targets, err := a.messageQueueTargets(ctx, client, topic, false)
if err != nil {
return nil, err
}
messages := make([]map[string]any, 0, count)
for _, target := range targets {
if len(messages) >= count {
break
}
if hasPartition && target.QueueID != partition {
continue
}
startOffset := offset
if !hasOffset {
startOffset, err = queueOffset(ctx, target.Address, topic, target.QueueID, remoting.GetMinOffset)
if err != nil {
continue
}
}
pullResult, pullErr := client.PullMessage(ctx, target.Address, topic, target.QueueID, startOffset, count-len(messages))
if pullErr != nil {
continue
}
for _, message := range pullResult.Messages {
message.BrokerName = target.BrokerName
messages = append(messages, messageMap(topic, message))
if len(messages) >= count {
break
}
}
}
sortMessageRows(messages)
return map[string]any{"messages": messages}, nil
}
func (a *rocketMQAgent) viewMessage(params map[string]any) (any, error) {
topic, err := requireString(params, "topic")
if err != nil {
return nil, err
}
messageID, err := requireString(params, "msgId")
if err != nil {
return nil, err
}
client, config, _ := a.requireClient()
ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout)
defer cancel()
messages, queryErr := queryMessagesByKey(ctx, client, config.ConnectTimeout,
topic, messageID, 32, 0, time.Now().UnixMilli())
if queryErr == nil {
for _, message := range messages {
if message.MsgId == messageID || message.OffsetMsgId == messageID || message.Properties[primitive.PropertyUniqueClientMessageIdKeyIndex] == messageID {
return map[string]any{"message": messageMap(topic, message)}, nil
}
}
if len(messages) > 0 {
return map[string]any{"message": messageMap(topic, messages[0])}, nil
}
}
partition, hasPartition := optionalInt(params, "partition")
offset, hasOffset := optionalInt64(params, "offset")
if hasPartition && hasOffset {
targets, targetErr := a.messageQueueTargets(ctx, client, topic, false)
if targetErr == nil {
for _, target := range targets {
if target.QueueID != partition {
continue
}
pullResult, pullErr := client.PullMessage(ctx, target.Address, topic, partition, offset, 1)
if pullErr == nil && len(pullResult.Messages) > 0 {
return map[string]any{"message": messageMap(topic, pullResult.Messages[0])}, nil
}
}
}
}
if queryErr != nil {
return nil, queryErr
}
return nil, fmt.Errorf("message not found: %s", messageID)
}
func (a *rocketMQAgent) queryMessageByKey(params map[string]any) (any, error) {
topic, err := requireString(params, "topic")
if err != nil {
return nil, err
}
key, err := requireString(params, "key")
if err != nil {
return nil, err
}
client, config, _ := a.requireClient()
ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout)
defer cancel()
maxNum := min(max(1, intValue(params, 32, "maxNum")), 200)
messages, err := queryMessagesByKey(ctx, client, config.ConnectTimeout, topic, key, maxNum,
int64Value(params, 0, "begin"), int64Value(params, time.Now().UnixMilli(), "end"))
if err != nil {
if isEmptyMessageQueryError(err) {
return emptyMessageQuery(), nil
}
return nil, err
}
return messageQueryResult(topic, messages), nil
}
func queryMessagesByKey(
ctx context.Context,
client *admin.Client,
connectTimeout time.Duration,
topic string,
key string,
maxNum int,
beginTimestamp int64,
endTimestamp int64,
) ([]*admin.MessageExt, error) {
route, err := client.ExamineTopicRouteInfo(ctx, topic)
if err != nil {
return nil, err
}
targets := make([]messageQueueTarget, 0, len(route.BrokerDatas))
for _, broker := range route.BrokerDatas {
if address := broker.BrokerAddrs["0"]; address != "" {
targets = append(targets, messageQueueTarget{BrokerName: broker.BrokerName, Address: address})
}
}
if len(targets) == 0 {
return nil, fmt.Errorf("no RocketMQ master broker found for topic %s", topic)
}
messages := make([]*admin.MessageExt, 0)
successCount := 0
var lastErr error
for _, target := range targets {
response, requestErr := invokeRemotingAllowCodes(ctx, target.Address, connectTimeout,
buildQueryMessageCommand(topic, key, maxNum, beginTimestamp, endTimestamp),
remoting.Success, queryMessageNotFoundCode)
if requestErr != nil {
lastErr = requestErr
continue
}
successCount++
if response.Code == queryMessageNotFoundCode || len(response.Body) == 0 {
continue
}
for _, message := range primitive.DecodeMessage(response.Body) {
queueID := 0
if message.Queue != nil {
queueID = message.Queue.QueueId
}
messageTopic := message.Topic
if messageTopic == "" {
messageTopic = topic
}
messages = append(messages, &admin.MessageExt{
Topic: messageTopic, QueueId: queueID, QueueOffset: message.QueueOffset,
MsgId: message.MsgId, OffsetMsgId: message.OffsetMsgId, Body: message.Body,
Flag: int(message.Flag), BornTimestamp: message.BornTimestamp,
StoreTimestamp: message.StoreTimestamp, BornHost: message.BornHost,
StoreHost: message.StoreHost, SysFlag: int(message.SysFlag),
BrokerName: target.BrokerName, Properties: message.GetProperties(),
})
}
}
if successCount == 0 {
return nil, fmt.Errorf("query messages for topic %s on all masters: %w", topic, lastErr)
}
return messages, nil
}
func buildQueryMessageCommand(topic, key string, maxNum int, beginTimestamp, endTimestamp int64) *remoting.RemotingCommand {
return remoting.NewRequest(remoting.QueryMessage, map[string]string{
"topic": topic, "key": key, "maxNum": strconv.Itoa(maxNum),
"beginTimestamp": strconv.FormatInt(beginTimestamp, 10),
"endTimestamp": strconv.FormatInt(endTimestamp, 10),
})
}
func (a *rocketMQAgent) queryMessageByTopic(params map[string]any) (any, error) {
topic, err := requireString(params, "topic")
if err != nil {
return nil, err
}
client, config, _ := a.requireClient()
ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout)
defer cancel()
maxNum := min(max(1, intValue(params, 32, "maxNum")), 200)
messages, err := client.QueryMessageByTime(ctx, topic,
int64Value(params, 0, "begin"), int64Value(params, time.Now().UnixMilli(), "end"), maxNum)
if err != nil {
return nil, err
}
return messageQueryResult(topic, messages), nil
}
func (a *rocketMQAgent) queryMessageTrace(params map[string]any) (any, error) {
messageID, err := requireString(params, "msgId")
if err != nil {
return nil, err
}
traceTopic := stringValue(params, "traceTopic")
if traceTopic == "" {
traceTopic = "RMQ_SYS_TRACE_TOPIC"
}
result, err := a.queryMessageByKey(map[string]any{
"topic": traceTopic, "key": messageID,
"maxNum": intValue(params, 64, "maxNum"),
"begin": int64Value(params, 0, "begin"),
"end": int64Value(params, time.Now().UnixMilli(), "end"),
})
if err != nil {
return nil, err
}
response := result.(map[string]any)
response["msgId"] = messageID
response["traceTopic"] = traceTopic
return response, nil
}
func (a *rocketMQAgent) sendMessage(params map[string]any) (any, error) {
topic, err := requireString(params, "topic")
if err != nil {
return nil, err
}
payload, err := decodePayload(params)
if err != nil {
return nil, err
}
client, config, _ := a.requireClient()
ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout)
defer cancel()
targets, err := a.messageQueueTargets(ctx, client, topic, true)
if err != nil {
return nil, err
}
if len(targets) == 0 {
return nil, fmt.Errorf("no writable queue for topic %s", topic)
}
partition, hasPartition := optionalInt(params, "partition")
selected := targets[0]
if hasPartition {
found := false
for _, target := range targets {
if target.QueueID == partition {
selected = target
found = true
break
}
}
if !found {
return nil, fmt.Errorf("partition %d not found for topic %s", partition, topic)
}
}
command := buildSendMessageCommand(topic, payload, selected.QueueID, params, time.Now().UnixMilli())
response, err := invokeRemotingAllowCodes(ctx, selected.Address, config.ConnectTimeout, command, 0, 10, 11, 12)
if err != nil {
return nil, err
}
queueID, _ := strconv.Atoi(response.ExtFields["queueId"])
queueOffset, _ := strconv.ParseInt(response.ExtFields["queueOffset"], 10, 64)
return map[string]any{
"ok": true, "topic": topic, "partition": queueID,
"offset": queueOffset, "timestamp": time.Now().UnixMilli(),
}, nil
}
func buildSendMessageCommand(topic string, payload []byte, queueID int, params map[string]any, bornTimestamp int64) *remoting.RemotingCommand {
message := primitive.NewMessage(topic, payload)
message.WithProperty(primitive.PropertyUniqueClientMessageIdKeyIndex, primitive.CreateUniqID())
headers := mapValue(params, "headers")
key := stringValue(params, "key")
if key == "" {
key = stringValue(headers, primitive.PropertyKeys)
}
tag := stringValue(params, "tag")
if tag == "" {
tag = stringValue(headers, primitive.PropertyTags)
}
if key != "" {
message.WithKeys([]string{key})
}
if tag != "" {
message.WithTag(tag)
}
for header, raw := range headers {
if raw == nil {
continue
}
value := strings.TrimSpace(fmt.Sprint(raw))
if value == "" || header == primitive.PropertyKeys || header == primitive.PropertyTags || isReservedMessageProperty(header) {
continue
}
message.WithProperty(header, value)
}
command := remoting.NewRequest(sendMessageRequestCode, map[string]string{
"producerGroup": "_DBX_ROCKETMQ_PRODUCER", "topic": topic,
"defaultTopic": "TBW102", "defaultTopicQueueNums": "4",
"queueId": strconv.Itoa(queueID), "sysFlag": "0",
"bornTimestamp": strconv.FormatInt(bornTimestamp, 10), "flag": "0",
"properties": message.MarshallProperties(), "reconsumeTimes": "0",
"unitMode": "false", "maxReconsumeTimes": "0", "batch": "false",
})
command.Body = payload
return command
}
func (a *rocketMQAgent) messageQueueTargets(ctx context.Context, client *admin.Client, topic string, writable bool) ([]messageQueueTarget, error) {
route, err := client.ExamineTopicRouteInfo(ctx, topic)
if err != nil {
return nil, err
}
addresses := make(map[string]string, len(route.BrokerDatas))
for _, broker := range route.BrokerDatas {
if address := broker.BrokerAddrs["0"]; address != "" {
addresses[broker.BrokerName] = address
}
}
targets := make([]messageQueueTarget, 0)
for _, queueData := range route.QueueDatas {
address := addresses[queueData.BrokerName]
if address == "" {
continue
}
queueCount := queueData.ReadQueueNums
if writable {
queueCount = queueData.WriteQueueNums
}
for queueID := 0; queueID < queueCount; queueID++ {
targets = append(targets, messageQueueTarget{
BrokerName: queueData.BrokerName, Address: address, QueueID: queueID,
})
}
}
sort.Slice(targets, func(i, j int) bool {
if targets[i].BrokerName != targets[j].BrokerName {
return targets[i].BrokerName < targets[j].BrokerName
}
return targets[i].QueueID < targets[j].QueueID
})
return targets, nil
}
func queueOffset(ctx context.Context, address, topic string, queueID, requestCode int) (int64, error) {
response, err := invokeRemotingWithClient(ctx, address, remoting.NewRequest(requestCode, map[string]string{
"topic": topic, "queueId": strconv.Itoa(queueID),
}))
if err != nil {
return 0, err
}
offset, err := strconv.ParseInt(response.ExtFields["offset"], 10, 64)
if err != nil {
return 0, fmt.Errorf("invalid queue offset: %w", err)
}
return offset, nil
}
func messageMap(topic string, message *admin.MessageExt) map[string]any {
headers := make(map[string]string, len(message.Properties))
for key, value := range message.Properties {
headers[key] = value
}
row := map[string]any{
"topic": topic, "messageId": message.MsgId, "partition": message.QueueId,
"offset": message.QueueOffset, "timestamp": message.StoreTimestamp,
"key": message.Properties[primitive.PropertyKeys], "tag": message.Properties[primitive.PropertyTags],
"headers": headers, "payloadBase64": base64.StdEncoding.EncodeToString(message.Body),
}
if utf8.Valid(message.Body) {
row["payloadText"] = string(message.Body)
}
return row
}
func messageQueryResult(topic string, messages []*admin.MessageExt) map[string]any {
rows := make([]map[string]any, 0, len(messages))
for _, message := range messages {
rows = append(rows, messageMap(topic, message))
}
sortMessageRows(rows)
return map[string]any{"messages": rows, "indexLastUpdateTimestamp": int64(0)}
}
func emptyMessageQuery() map[string]any {
return map[string]any{"messages": []map[string]any{}, "indexLastUpdateTimestamp": int64(0)}
}
func sortMessageRows(messages []map[string]any) {
sort.Slice(messages, func(i, j int) bool {
leftTime := anyInt64(messages[i]["timestamp"])
rightTime := anyInt64(messages[j]["timestamp"])
if leftTime != rightTime {
return leftTime < rightTime
}
leftPartition := int(anyInt64(messages[i]["partition"]))
rightPartition := int(anyInt64(messages[j]["partition"]))
if leftPartition != rightPartition {
return leftPartition < rightPartition
}
return anyInt64(messages[i]["offset"]) < anyInt64(messages[j]["offset"])
})
}
func producerRow(producerID int64, group string, connection admin.Connection) map[string]any {
return map[string]any{
"producerId": producerID, "producerName": group, "msgRateIn": 0.0,
"msgThroughputIn": 0.0, "clientVersion": strconv.Itoa(connection.Version),
"address": connection.ClientAddr, "lastTimestamp": int64(0),
}
}
func optionalInt(params map[string]any, key string) (int, bool) {
if _, ok := params[key]; !ok || params[key] == nil {
return 0, false
}
return intValue(params, 0, key), true
}
func optionalInt64(params map[string]any, key string) (int64, bool) {
if _, ok := params[key]; !ok || params[key] == nil {
return 0, false
}
return int64Value(params, 0, key), true
}
func mapValue(params map[string]any, key string) map[string]any {
if value, ok := params[key].(map[string]any); ok {
return value
}
return map[string]any{}
}
func anyInt64(value any) int64 {
switch typed := value.(type) {
case int:
return int64(typed)
case int64:
return typed
case float64:
return int64(typed)
case string:
parsed, _ := strconv.ParseInt(typed, 10, 64)
return parsed
default:
return 0
}
}
func isEmptyMessageQueryError(err error) bool {
message := strings.ToLower(err.Error())
return strings.Contains(message, "208") || strings.Contains(message, "no message") || strings.Contains(message, "未找到消息")
}
func isReservedMessageProperty(key string) bool {
switch key {
case primitive.PropertyUniqueClientMessageIdKeyIndex, primitive.PropertyWaitStoreMsgOk,
primitive.PropertyDelayTimeLevel, primitive.PropertyRetryTopic,
primitive.PropertyRealTopic, primitive.PropertyRealQueueId,
primitive.PropertyTransactionPrepared, primitive.PropertyProducerGroup,
primitive.PropertyMinOffset, primitive.PropertyMaxOffset,
primitive.PropertyBuyerId, primitive.PropertyOriginMessageId,
primitive.PropertyTransferFlag, primitive.PropertyCorrectionFlag,
primitive.PropertyMQ2Flag, primitive.PropertyReconsumeTime,
primitive.PropertyMsgRegion, primitive.PropertyTraceSwitch,
primitive.PropertyMaxReconsumeTimes, primitive.PropertyConsumeStartTime,
primitive.PropertyTranscationPreparedQueueOffset, primitive.PropertyTranscationCheckTimes,
primitive.PropertyCheckImmunityTimeInSeconds, primitive.PropertyShardingKey,
primitive.PropertyTransactionID, primitive.PropertyCorrelationID,
primitive.PropertyMessageReplyToClient, primitive.PropertyMessageTTL,
primitive.PropertyReplyMessageArriveTime, primitive.PropertyMsgType,
primitive.PropertyCluster:
return true
default:
return false
}
}
agents/drivers/rocketmq/messages_test.go
+116
View File
@@ -0,0 +1,116 @@
package main
import (
"context"
"encoding/base64"
"net"
"strings"
"testing"
"time"
admin "github.com/amigoer/rocketmq-admin-go"
"github.com/amigoer/rocketmq-admin-go/protocol/remoting"
"github.com/apache/rocketmq-client-go/v2/primitive"
)
func TestBuildSendMessageCommandCopiesOnlyUserProperties(t *testing.T) {
command := buildSendMessageCommand("Orders", []byte("payload"), 3, map[string]any{
"headers": map[string]any{
primitive.PropertyTags: "tag-a",
primitive.PropertyKeys: "key-a",
primitive.PropertyDelayTimeLevel: "18",
"Region": "Hangzhou",
"color": "blue",
"empty": " ",
"nil": nil,
},
}, 1234)
if command.Code != sendMessageRequestCode || command.ExtFields["queueId"] != "3" {
t.Fatalf("unexpected send command: %#v", command)
}
properties := command.ExtFields["properties"]
for _, expected := range []string{"TAGS\x01tag-a\x02", "KEYS\x01key-a\x02", "Region\x01Hangzhou\x02", "color\x01blue\x02"} {
if !strings.Contains(properties, expected) {
t.Fatalf("missing property %q in %q", expected, properties)
}
}
for _, forbidden := range []string{"DELAY\x01", "empty\x01", "nil\x01"} {
if strings.Contains(properties, forbidden) {
t.Fatalf("reserved/empty property %q leaked into %q", forbidden, properties)
}
}
}
func TestBuildQueryMessageCommandUsesBrokerHeaderNames(t *testing.T) {
command := buildQueryMessageCommand("Orders", "order-1", 32, 100, 200)
if command.ExtFields["beginTimestamp"] != "100" || command.ExtFields["endTimestamp"] != "200" {
t.Fatalf("unexpected query fields: %#v", command.ExtFields)
}
if _, exists := command.ExtFields["begin"]; exists {
t.Fatalf("legacy begin field leaked into query: %#v", command.ExtFields)
}
if _, exists := command.ExtFields["end"]; exists {
t.Fatalf("legacy end field leaked into query: %#v", command.ExtFields)
}
}
func TestInvokeRemotingAllowCodes(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer listener.Close()
done := make(chan error, 1)
go func() {
connection, acceptErr := listener.Accept()
if acceptErr != nil {
done <- acceptErr
return
}
defer connection.Close()
frame, readErr := readRemotingFrame(connection)
if readErr != nil {
done <- readErr
return
}
request, decodeErr := remoting.Decode(frame[4:])
if decodeErr != nil {
done <- decodeErr
return
}
response := &remoting.RemotingCommand{Code: 11, Opaque: request.Opaque, Flag: 1, ExtFields: map[string]string{}}
encoded, encodeErr := response.Encode()
if encodeErr == nil {
_, encodeErr = connection.Write(encoded)
}
done <- encodeErr
}()
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
if _, err := invokeRemotingAllowCodes(ctx, listener.Addr().String(), time.Second,
remoting.NewRequest(sendMessageRequestCode, nil), 0, 10, 11, 12); err != nil {
t.Fatal(err)
}
if err := <-done; err != nil {
t.Fatal(err)
}
}
func TestMessageMapPreservesBinaryAndTextPayloads(t *testing.T) {
text := &admin.MessageExt{
QueueId: 2, QueueOffset: 7, MsgId: "msg-1", Body: []byte("hello"),
StoreTimestamp: 123, Properties: map[string]string{primitive.PropertyKeys: "key-a", primitive.PropertyTags: "tag-a"},
}
row := messageMap("Orders", text)
if row["payloadText"] != "hello" || row["payloadBase64"] != base64.StdEncoding.EncodeToString([]byte("hello")) {
t.Fatalf("unexpected text message row: %#v", row)
}
if row["key"] != "key-a" || row["tag"] != "tag-a" || row["partition"] != 2 {
t.Fatalf("unexpected message metadata: %#v", row)
}
binary := &admin.MessageExt{Body: []byte{0xff, 0xfe}, Properties: map[string]string{}}
binaryRow := messageMap("Binary", binary)
if _, ok := binaryRow["payloadText"]; ok {
t.Fatalf("binary payload exposed as text: %#v", binaryRow)
}
}
agents/drivers/rocketmq/remoting.go
+42
View File
@@ -0,0 +1,42 @@
package main
import (
"context"
"fmt"
"time"
"github.com/amigoer/rocketmq-admin-go/protocol/remoting"
)
func invokeRemotingWithClient(ctx context.Context, address string, command *remoting.RemotingCommand) (*remoting.RemotingCommand, error) {
return invokeRemotingAllowCodes(ctx, address, defaultRequestTimeout, command, remoting.Success)
}
func invokeRemotingAllowCodes(
ctx context.Context,
address string,
connectTimeout time.Duration,
command *remoting.RemotingCommand,
allowedCodes ...int,
) (*remoting.RemotingCommand, error) {
client := remoting.NewClient(address, connectTimeout)
if err := client.Connect(); err != nil {
return nil, err
}
defer client.Close()
response, err := client.InvokeSync(ctx, command)
if err != nil {
return nil, err
}
allowed := false
for _, code := range allowedCodes {
if response.Code == code {
allowed = true
break
}
}
if !allowed {
return nil, fmt.Errorf("RocketMQ error %d: %s", response.Code, response.Remark)
}
return response, nil
}
agents/drivers/rocketmq/routing.go
+526
View File
@@ -0,0 +1,526 @@
package main
import (
"context"
"crypto/hmac"
"crypto/sha1"
"encoding/base64"
"encoding/binary"
"encoding/json"
"fmt"
"io"
"net"
"regexp"
"sort"
"strings"
"sync"
"github.com/amigoer/rocketmq-admin-go/protocol/remoting"
"golang.org/x/net/proxy"
)
type proxyTargetKind int
const (
proxyTargetNameServer proxyTargetKind = iota
proxyTargetBroker
)
type proxyEndpoint struct {
original string
targets []string
fallback bool
listener net.Listener
manager *proxyManager
closeOnce sync.Once
}
type proxyManager struct {
mu sync.RWMutex
config connectionConfig
byOriginal map[string]*proxyEndpoint
originalByLocal map[string]string
routedOwners map[string]string
closed bool
}
func newProxyManager(config connectionConfig) *proxyManager {
return &proxyManager{
config: config, byOriginal: map[string]*proxyEndpoint{},
originalByLocal: map[string]string{}, routedOwners: map[string]string{},
}
}
func (manager *proxyManager) ProxyFor(original string, kind proxyTargetKind) (string, error) {
original = strings.TrimSpace(original)
if original == "" {
return "", fmt.Errorf("empty RocketMQ endpoint")
}
manager.mu.RLock()
if endpoint := manager.byOriginal[original]; endpoint != nil {
address := endpoint.listener.Addr().String()
manager.mu.RUnlock()
return address, nil
}
manager.mu.RUnlock()
manager.mu.Lock()
defer manager.mu.Unlock()
if manager.closed {
return "", fmt.Errorf("RocketMQ proxy manager is closed")
}
if endpoint := manager.byOriginal[original]; endpoint != nil {
return endpoint.listener.Addr().String(), nil
}
targets, fallback := manager.routeTargetsLocked(original, kind)
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
return "", fmt.Errorf("start RocketMQ endpoint proxy: %w", err)
}
endpoint := &proxyEndpoint{original: original, targets: targets, fallback: fallback, listener: listener, manager: manager}
local := listener.Addr().String()
manager.byOriginal[original] = endpoint
manager.originalByLocal[local] = original
go endpoint.serve()
return local, nil
}
func (manager *proxyManager) routeTargetsLocked(original string, kind proxyTargetKind) ([]string, bool) {
if kind == proxyTargetNameServer || manager.config.SocksProxy != nil {
return []string{original}, false
}
if manager.config.BrokerAddr != "" {
return []string{manager.config.BrokerAddr}, false
}
host, port := parseSocketAddress(original)
if port == "" || !isLikelyUnreachableBrokerHost(host) {
return []string{original}, false
}
nameServerHost, _ := parseSocketAddress(manager.config.NameServers[0])
routed := formatSocketAddress(nameServerHost, port)
if owner := manager.routedOwners[routed]; owner == "" || owner == original {
manager.routedOwners[routed] = original
return []string{routed, original}, false
}
// Multiple private brokers often advertise the same port. Routing a collision to the
// published host would silently contact the first broker, so keep the original only.
return []string{original}, true
}
func (manager *proxyManager) LocalForOriginal(original string) string {
manager.mu.RLock()
defer manager.mu.RUnlock()
if endpoint := manager.byOriginal[original]; endpoint != nil {
return endpoint.listener.Addr().String()
}
return ""
}
func (manager *proxyManager) OriginalForLocal(local string) string {
manager.mu.RLock()
defer manager.mu.RUnlock()
if original := manager.originalByLocal[local]; original != "" {
return original
}
host, port := parseSocketAddress(local)
if host == "localhost" {
return manager.originalByLocal[formatSocketAddress("127.0.0.1", port)]
}
return ""
}
func (manager *proxyManager) IsCollisionFallback(local string) bool {
manager.mu.RLock()
defer manager.mu.RUnlock()
original := manager.originalByLocal[local]
return original != "" && manager.byOriginal[original] != nil && manager.byOriginal[original].fallback
}
func (manager *proxyManager) Close() {
manager.mu.Lock()
if manager.closed {
manager.mu.Unlock()
return
}
manager.closed = true
endpoints := make([]*proxyEndpoint, 0, len(manager.byOriginal))
for _, endpoint := range manager.byOriginal {
endpoints = append(endpoints, endpoint)
}
manager.mu.Unlock()
for _, endpoint := range endpoints {
endpoint.closeOnce.Do(func() { _ = endpoint.listener.Close() })
}
}
func (endpoint *proxyEndpoint) serve() {
for {
clientConn, err := endpoint.listener.Accept()
if err != nil {
return
}
go endpoint.handle(clientConn)
}
}
func (endpoint *proxyEndpoint) handle(clientConn net.Conn) {
defer clientConn.Close()
remoteConn, err := endpoint.dialRemote()
if err != nil {
return
}
defer remoteConn.Close()
done := make(chan struct{}, 2)
go func() {
_ = endpoint.forward(clientConn, remoteConn, false)
done <- struct{}{}
}()
go func() {
_ = endpoint.forward(remoteConn, clientConn, true)
done <- struct{}{}
}()
<-done
}
func (endpoint *proxyEndpoint) dialRemote() (net.Conn, error) {
var lastErr error
for _, target := range endpoint.targets {
var conn net.Conn
var err error
if socks := endpoint.manager.config.SocksProxy; socks != nil {
var auth *proxy.Auth
if socks.Username != "" || socks.Password != "" {
auth = &proxy.Auth{User: socks.Username, Password: socks.Password}
}
dialer, dialErr := proxy.SOCKS5("tcp", formatSocketAddress(socks.Host, fmt.Sprintf("%d", socks.Port)), auth, &net.Dialer{Timeout: endpoint.manager.config.ConnectTimeout})
if dialErr != nil {
lastErr = dialErr
continue
}
conn, err = dialer.Dial("tcp", target)
} else {
ctx, cancel := context.WithTimeout(context.Background(), endpoint.manager.config.ConnectTimeout)
conn, err = (&net.Dialer{}).DialContext(ctx, "tcp", target)
cancel()
}
if err == nil {
return conn, nil
}
lastErr = err
}
return nil, fmt.Errorf("connect RocketMQ endpoint %s: %w", endpoint.original, lastErr)
}
func (endpoint *proxyEndpoint) forward(source, destination net.Conn, response bool) error {
for {
frame, err := readRemotingFrame(source)
if err != nil {
return err
}
command, err := remoting.Decode(frame[4:])
if err != nil {
return err
}
if response {
if err := endpoint.manager.rewriteResponse(command); err != nil {
return err
}
} else {
signCommand(command, endpoint.manager.config.AccessKey, endpoint.manager.config.SecretKey)
}
encoded, err := command.Encode()
if err != nil {
return err
}
if _, err := destination.Write(encoded); err != nil {
return err
}
}
}
func readRemotingFrame(reader io.Reader) ([]byte, error) {
header := make([]byte, 4)
if _, err := io.ReadFull(reader, header); err != nil {
return nil, err
}
length := int(binary.BigEndian.Uint32(header))
if length <= 0 || length > 64*1024*1024 {
return nil, fmt.Errorf("invalid RocketMQ frame length: %d", length)
}
frame := make([]byte, 4+length)
copy(frame, header)
if _, err := io.ReadFull(reader, frame[4:]); err != nil {
return nil, err
}
return frame, nil
}
func signCommand(command *remoting.RemotingCommand, accessKey, secretKey string) {
if accessKey == "" && secretKey == "" {
return
}
if command.ExtFields == nil {
command.ExtFields = map[string]string{}
}
delete(command.ExtFields, "Signature")
command.ExtFields["AccessKey"] = accessKey
keys := make([]string, 0, len(command.ExtFields))
for key := range command.ExtFields {
if key != "Signature" {
keys = append(keys, key)
}
}
sort.Strings(keys)
var content strings.Builder
for _, key := range keys {
content.WriteString(command.ExtFields[key])
}
data := append([]byte(content.String()), command.Body...)
mac := hmac.New(sha1.New, []byte(secretKey))
_, _ = mac.Write(data)
command.ExtFields["Signature"] = base64.StdEncoding.EncodeToString(mac.Sum(nil))
}
func (manager *proxyManager) rewriteResponse(command *remoting.RemotingCommand) error {
if len(command.Body) == 0 {
return nil
}
body := repairRocketMQJSON(command.Body)
if !json.Valid(body) {
return nil
}
var value any
if err := json.Unmarshal(body, &value); err != nil {
return nil
}
changed, err := manager.rewriteJSON(value, "")
if err != nil || !changed {
return err
}
body, err = json.Marshal(value)
if err != nil {
return err
}
command.Body = body
return nil
}
var (
unquotedNumberKey = regexp.MustCompile(`([{,])(\d+):`)
unquotedStringKey = regexp.MustCompile(`([{,])([a-zA-Z_][a-zA-Z0-9_]*):`)
)
func repairRocketMQJSON(body []byte) []byte {
repaired := unquotedNumberKey.ReplaceAll(body, []byte(`$1"$2":`))
repaired = unquotedStringKey.ReplaceAll(repaired, []byte(`$1"$2":`))
return repairObjectKeyedMaps(repaired)
}
func repairObjectKeyedMaps(body []byte) []byte {
result := make([]byte, 0, len(body)+64)
for index := 0; index < len(body); {
if index+2 < len(body) && body[index] == ':' && body[index+1] == '{' && body[index+2] == '{' {
converted, next := convertObjectKeyedMap(body, index+1)
result = append(result, ':')
result = append(result, converted...)
index = next
continue
}
result = append(result, body[index])
index++
}
return result
}
func convertObjectKeyedMap(body []byte, start int) ([]byte, int) {
if start >= len(body) || body[start] != '{' {
return []byte("{}"), start
}
result := []byte{'{'}
index := start + 1
first := true
for index < len(body) {
for index < len(body) && isJSONSpace(body[index]) {
index++
}
if index >= len(body) {
break
}
if body[index] == '}' {
return append(result, '}'), index + 1
}
if body[index] == ',' {
index++
continue
}
if body[index] != '{' {
return originalJSONMap(body, start)
}
keyEnd := matchingBrace(body, index)
if keyEnd < 0 {
return body[start:], len(body)
}
key := body[index : keyEnd+1]
index = keyEnd + 1
for index < len(body) && isJSONSpace(body[index]) {
index++
}
if index >= len(body) || body[index] != ':' {
return originalJSONMap(body, start)
}
index++
for index < len(body) && isJSONSpace(body[index]) {
index++
}
if index >= len(body) || body[index] != '{' {
return originalJSONMap(body, start)
}
valueEnd := matchingBrace(body, index)
if valueEnd < 0 {
return body[start:], len(body)
}
if !first {
result = append(result, ',')
}
first = false
result = append(result, '"')
result = append(result, escapeJSONString(key)...)
result = append(result, '"', ':')
result = append(result, body[index:valueEnd+1]...)
index = valueEnd + 1
}
return append(result, '}'), len(body)
}
func originalJSONMap(body []byte, start int) ([]byte, int) {
end := matchingBrace(body, start)
if end < 0 {
return body[start:], len(body)
}
return body[start : end+1], end + 1
}
func matchingBrace(body []byte, start int) int {
depth := 0
inString := false
escaped := false
for index := start; index < len(body); index++ {
character := body[index]
if inString {
if escaped {
escaped = false
continue
}
if character == '\\' {
escaped = true
continue
}
if character == '"' {
inString = false
}
continue
}
switch character {
case '"':
inString = true
case '{':
depth++
case '}':
depth--
if depth == 0 {
return index
}
}
}
return -1
}
func escapeJSONString(value []byte) []byte {
result := make([]byte, 0, len(value)+8)
for _, character := range value {
switch character {
case '\\', '"':
result = append(result, '\\', character)
case '\n':
result = append(result, '\\', 'n')
case '\r':
result = append(result, '\\', 'r')
case '\t':
result = append(result, '\\', 't')
default:
result = append(result, character)
}
}
return result
}
func isJSONSpace(character byte) bool {
return character == ' ' || character == '\n' || character == '\r' || character == '\t'
}
func (manager *proxyManager) rewriteJSON(value any, parentKey string) (bool, error) {
changed := false
switch typed := value.(type) {
case map[string]any:
if parentKey == "brokerAddrs" {
for key, raw := range typed {
address, ok := raw.(string)
if !ok || address == "" {
continue
}
local, err := manager.ProxyFor(address, proxyTargetBroker)
if err != nil {
return changed, err
}
typed[key] = local
changed = true
}
return changed, nil
}
if parentKey == "filterServerTable" {
rewritten := make(map[string]any, len(typed))
for address, raw := range typed {
local, err := manager.ProxyFor(address, proxyTargetBroker)
if err != nil {
return changed, err
}
rewritten[local] = raw
changed = true
}
for key := range typed {
delete(typed, key)
}
for key, raw := range rewritten {
typed[key] = raw
}
}
for key, child := range typed {
if key == "brokerAddr" {
if address, ok := child.(string); ok && address != "" {
local, err := manager.ProxyFor(address, proxyTargetBroker)
if err != nil {
return changed, err
}
typed[key] = local
changed = true
continue
}
}
childChanged, err := manager.rewriteJSON(child, key)
if err != nil {
return changed, err
}
changed = changed || childChanged
}
case []any:
for _, child := range typed {
childChanged, err := manager.rewriteJSON(child, parentKey)
if err != nil {
return changed, err
}
changed = changed || childChanged
}
}
return changed, nil
}
agents/drivers/rocketmq/routing_test.go
+136
View File
@@ -0,0 +1,136 @@
package main
import (
"bytes"
"crypto/hmac"
"crypto/sha1"
"encoding/base64"
"encoding/json"
"reflect"
"sort"
"strings"
"testing"
"github.com/amigoer/rocketmq-admin-go/protocol/remoting"
)
func TestProxyRoutesPrivateBrokersAndFailsClosedOnCollision(t *testing.T) {
manager := newProxyManager(connectionConfig{NameServers: []string{"127.0.0.1:9876"}})
defer manager.Close()
firstLocal, err := manager.ProxyFor("172.18.0.2:10911", proxyTargetBroker)
if err != nil {
t.Fatal(err)
}
secondLocal, err := manager.ProxyFor("172.18.0.3:10911", proxyTargetBroker)
if err != nil {
t.Fatal(err)
}
first := manager.byOriginal["172.18.0.2:10911"]
second := manager.byOriginal["172.18.0.3:10911"]
if !reflect.DeepEqual(first.targets, []string{"127.0.0.1:10911", "172.18.0.2:10911"}) {
t.Fatalf("unexpected first targets: %#v", first.targets)
}
if !reflect.DeepEqual(second.targets, []string{"172.18.0.3:10911"}) || !second.fallback {
t.Fatalf("collision must keep only original target: %#v", second)
}
if manager.OriginalForLocal(firstLocal) != "172.18.0.2:10911" || manager.OriginalForLocal(secondLocal) != "172.18.0.3:10911" {
t.Fatal("local endpoint did not round-trip to original address")
}
if !manager.IsCollisionFallback(secondLocal) {
t.Fatal("second private broker should be marked as collision fallback")
}
}
func TestProxyRoutingHonorsSocksAndExplicitBroker(t *testing.T) {
socksManager := newProxyManager(connectionConfig{
NameServers: []string{"127.0.0.1:9876"},
SocksProxy: &socksProxyConfig{Host: "127.0.0.1", Port: 1080},
})
defer socksManager.Close()
local, err := socksManager.ProxyFor("172.18.0.2:10911", proxyTargetBroker)
if err != nil {
t.Fatal(err)
}
if targets := socksManager.byOriginal[socksManager.OriginalForLocal(local)].targets; !reflect.DeepEqual(targets, []string{"172.18.0.2:10911"}) {
t.Fatalf("SOCKS route changed logical broker: %#v", targets)
}
explicitManager := newProxyManager(connectionConfig{
NameServers: []string{"127.0.0.1:9876"}, BrokerAddr: "broker.example.com:10911",
})
defer explicitManager.Close()
local, err = explicitManager.ProxyFor("172.18.0.2:10911", proxyTargetBroker)
if err != nil {
t.Fatal(err)
}
if targets := explicitManager.byOriginal[explicitManager.OriginalForLocal(local)].targets; !reflect.DeepEqual(targets, []string{"broker.example.com:10911"}) {
t.Fatalf("explicit broker was not honored: %#v", targets)
}
}
func TestSignCommandMatchesRocketMQACLAlgorithm(t *testing.T) {
command := remoting.NewRequest(17, map[string]string{"topic": "Orders", "perm": "6"})
command.Body = []byte("payload")
signCommand(command, "access", "secret")
keys := make([]string, 0, len(command.ExtFields))
for key := range command.ExtFields {
if key != "Signature" {
keys = append(keys, key)
}
}
sort.Strings(keys)
var content strings.Builder
for _, key := range keys {
content.WriteString(command.ExtFields[key])
}
mac := hmac.New(sha1.New, []byte("secret"))
_, _ = mac.Write(append([]byte(content.String()), command.Body...))
want := base64.StdEncoding.EncodeToString(mac.Sum(nil))
if command.ExtFields["AccessKey"] != "access" || command.ExtFields["Signature"] != want {
t.Fatalf("unexpected signed fields: %#v", command.ExtFields)
}
}
func TestRepairRocketMQJSONQuotesNonStandardKeys(t *testing.T) {
body := repairRocketMQJSON([]byte(`{"offsetTable":{{"brokerName":"broker-a","queueId":0,"topic":"Order-Events"}:{"brokerOffset":10}},brokerAddrs:{0:"127.0.0.1:10911"}}`))
if !json.Valid(body) {
t.Fatalf("repaired body is invalid JSON: %s", body)
}
if !bytes.Contains(body, []byte(`"0":`)) || !bytes.Contains(body, []byte(`"{\"brokerName\":\"broker-a\"`)) {
t.Fatalf("missing repaired keys: %s", body)
}
}
func TestRewriteResponseProxiesBrokerAddresses(t *testing.T) {
manager := newProxyManager(connectionConfig{NameServers: []string{"127.0.0.1:9876"}})
defer manager.Close()
command := &remoting.RemotingCommand{Body: []byte(`{
"brokerDatas":[{"brokerAddrs":{"0":"172.18.0.2:10911"}}],
"brokerAddr":"172.18.0.3:10912",
"filterServerTable":{"172.18.0.4:12000":[]}
}`)}
if err := manager.rewriteResponse(command); err != nil {
t.Fatal(err)
}
var value map[string]any
if err := json.Unmarshal(command.Body, &value); err != nil {
t.Fatal(err)
}
brokerDatas := value["brokerDatas"].([]any)
brokerLocal := brokerDatas[0].(map[string]any)["brokerAddrs"].(map[string]any)["0"].(string)
if manager.OriginalForLocal(brokerLocal) != "172.18.0.2:10911" {
t.Fatalf("broker address not proxied: %s", brokerLocal)
}
directLocal := value["brokerAddr"].(string)
if manager.OriginalForLocal(directLocal) != "172.18.0.3:10912" {
t.Fatalf("direct broker address not proxied: %s", directLocal)
}
filterTable := value["filterServerTable"].(map[string]any)
for local := range filterTable {
if manager.OriginalForLocal(local) != "172.18.0.4:12000" {
t.Fatalf("filter server address not proxied: %s", local)
}
}
}
agents/drivers/rocketmq/scripts/run-integration.sh
Executable
+130
View File
@@ -0,0 +1,130 @@
#!/usr/bin/env bash
set -euo pipefail
# The DBX RocketMQ agent under test is the Go binary in this directory. Maven
# only supplies disposable official RocketMQ NameServer/Broker server JARs.
version="${1:?RocketMQ version is required}"
module_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
work_dir=$(mktemp -d "${TMPDIR:-/tmp}/dbx-rocketmq-${version}.XXXXXX")
namesrv_port="${ROCKETMQ_NAMESRV_PORT:-19876}"
broker_port="${ROCKETMQ_BROKER_PORT:-20911}"
store_root="${ROCKETMQ_STORE_ROOT:-${work_dir}/store}"
namesrv_pid=""
broker_pid=""
cleanup() {
if [[ -n "$broker_pid" ]] && kill -0 "$broker_pid" 2>/dev/null; then
kill "$broker_pid" 2>/dev/null || true
fi
if [[ -n "$namesrv_pid" ]] && kill -0 "$namesrv_pid" 2>/dev/null; then
kill "$namesrv_pid" 2>/dev/null || true
fi
wait "$broker_pid" 2>/dev/null || true
wait "$namesrv_pid" 2>/dev/null || true
rm -rf "$work_dir"
}
trap cleanup EXIT
cat >"$work_dir/pom.xml" <<EOF
<project xmlns="http://maven.apache.org/POM/4.0.0">
<modelVersion>4.0.0</modelVersion>
<groupId>com.dbx.test</groupId>
<artifactId>rocketmq-integration</artifactId>
<version>1.0.0</version>
<dependencies>
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-namesrv</artifactId>
<version>${version}</version>
</dependency>
<dependency>
<groupId>org.apache.rocketmq</groupId>
<artifactId>rocketmq-broker</artifactId>
<version>${version}</version>
</dependency>
</dependencies>
</project>
EOF
mvn -q -f "$work_dir/pom.xml" dependency:copy-dependencies -DoutputDirectory="$work_dir/lib"
mkdir -p "$work_dir/conf"
for component in namesrv broker; do
cat >"$work_dir/conf/logback_${component}.xml" <<'EOF'
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%date %-5level %logger - %msg%n</pattern>
</encoder>
</appender>
<root level="INFO">
<appender-ref ref="STDOUT" />
</root>
</configuration>
EOF
done
cat >"$work_dir/namesrv.conf" <<EOF
listenPort=${namesrv_port}
EOF
cat >"$work_dir/broker.conf" <<EOF
brokerClusterName=DefaultCluster
brokerName=broker-a
brokerId=0
brokerIP1=127.0.0.1
namesrvAddr=127.0.0.1:${namesrv_port}
listenPort=${broker_port}
haListenPort=$((broker_port + 1))
autoCreateTopicEnable=true
autoCreateSubscriptionGroup=true
deleteWhen=04
fileReservedTime=1
brokerRole=ASYNC_MASTER
flushDiskType=ASYNC_FLUSH
storePathRootDir=${store_root}
storePathCommitLog=${store_root}/commitlog
maxMessageSize=4194304
diskMaxUsedSpaceRatio=95
EOF
java -Xms128m -Xmx128m -Xmn64m \
-Drocketmq.home.dir="$work_dir" -Duser.home="$work_dir" \
-cp "$work_dir/lib/*" org.apache.rocketmq.namesrv.NamesrvStartup -c "$work_dir/namesrv.conf" \
>"$work_dir/namesrv.stdout.log" 2>&1 &
namesrv_pid=$!
java -Xms256m -Xmx256m -Xmn128m \
--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED \
--add-opens=java.base/java.nio=ALL-UNNAMED \
--add-opens=java.base/sun.nio.ch=ALL-UNNAMED \
-Drocketmq.home.dir="$work_dir" -Duser.home="$work_dir" \
-cp "$work_dir/lib/*" org.apache.rocketmq.broker.BrokerStartup -c "$work_dir/broker.conf" \
>"$work_dir/broker.stdout.log" 2>&1 &
broker_pid=$!
for _ in $(seq 1 120); do
if ! kill -0 "$namesrv_pid" 2>/dev/null; then
cat "$work_dir/namesrv.stdout.log"
exit 1
fi
if ! kill -0 "$broker_pid" 2>/dev/null; then
cat "$work_dir/broker.stdout.log"
exit 1
fi
if grep -Rqs "boot success" "$work_dir/logs" "$work_dir/broker.stdout.log" 2>/dev/null; then
break
fi
sleep 1
done
if ! grep -Rqs "boot success" "$work_dir/logs" "$work_dir/broker.stdout.log" 2>/dev/null; then
cat "$work_dir/namesrv.stdout.log"
cat "$work_dir/broker.stdout.log"
find "$work_dir/logs" -type f -maxdepth 3 -print -exec tail -80 {} \; 2>/dev/null || true
exit 1
fi
ROCKETMQ_INTEGRATION=1 \
ROCKETMQ_VERSION="$version" \
ROCKETMQ_NAMESRV_ADDR="127.0.0.1:${namesrv_port}" \
go test -C "$module_dir" -run '^TestRocketMQIntegration$' -count=1 -v .
agents/drivers/rocketmq/server.go
+168
View File
@@ -0,0 +1,168 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"strings"
"sync"
admin "github.com/amigoer/rocketmq-admin-go"
)
const (
protocolVersion = 1
agentProtocolVersion = 1
)
var capabilities = []string{
"mq_connect", "mq_test_connection", "mq_topics", "mq_consumer_groups",
"mq_messages", "mq_acl", "mq_config", "mq_monitoring",
}
type rocketMQAgent struct {
mu sync.RWMutex
client *admin.Client
connection connectionConfig
proxies *proxyManager
clusterName string
brokerAddr string
shutdownRequested bool
}
func newRocketMQAgent() *rocketMQAgent {
return &rocketMQAgent{}
}
func (a *rocketMQAgent) dispatch(method string, raw json.RawMessage) (any, error) {
params, err := decodeParams(raw)
if err != nil {
return nil, err
}
switch method {
case "handshake":
return map[string]any{
"protocolVersion": protocolVersion,
"agentProtocolVersion": agentProtocolVersion,
"capabilities": capabilities,
}, nil
case "connect":
return a.connect(params)
case "test_connection":
return a.testConnection(params)
case "disconnect":
a.close()
return okResult(), nil
case "shutdown":
a.close()
a.shutdownRequested = true
return okResult(), nil
case "mq_list_topics":
return a.listTopics(params)
case "mq_create_topic":
return a.createTopic(params)
case "mq_delete_topic":
return a.deleteTopic(params)
case "mq_update_partitions":
return a.updatePartitions(params)
case "mq_get_topic_stats":
return a.getTopicStats(params)
case "mq_get_topic_route":
return a.getTopicRoute(params)
case "mq_get_topic_config":
return a.getTopicConfig(params)
case "mq_alter_topic_config":
return a.alterTopicConfig(params)
case "mq_skip_topic_accumulation":
return a.skipTopicAccumulation(params)
case "mq_list_consumer_groups":
return a.listConsumerGroups(params)
case "mq_describe_consumer_group":
return a.describeConsumerGroup(params)
case "mq_delete_consumer_group":
return a.deleteConsumerGroup(params)
case "mq_get_subscription_group_config":
return a.getSubscriptionGroupConfig(params)
case "mq_alter_subscription_group_config":
return a.alterSubscriptionGroupConfig(params)
case "mq_reset_consumer_group_offsets":
return a.resetConsumerGroupOffsets(params)
case "mq_get_consumer_lag":
return a.getConsumerLag(params)
case "mq_list_producers":
return a.listProducers(params)
case "mq_peek_messages":
return a.peekMessages(params)
case "mq_view_message":
return a.viewMessage(params)
case "mq_query_message_by_key":
return a.queryMessageByKey(params)
case "mq_query_message_by_topic":
return a.queryMessageByTopic(params)
case "mq_query_message_trace":
return a.queryMessageTrace(params)
case "mq_send_message":
return a.sendMessage(params)
case "mq_list_acls":
return a.listACLs(params)
case "mq_create_acls":
return a.createACLs(params)
case "mq_delete_acls":
return a.deleteACLs(params)
case "mq_describe_cluster":
return a.describeCluster(params)
default:
return nil, fmt.Errorf("unknown method: %s", method)
}
}
func (a *rocketMQAgent) requireClient() (*admin.Client, connectionConfig, error) {
a.mu.RLock()
defer a.mu.RUnlock()
if a.client == nil {
return nil, connectionConfig{}, errors.New("RocketMQ agent is not connected")
}
return a.client, a.connection, nil
}
func (a *rocketMQAgent) close() {
a.mu.Lock()
defer a.mu.Unlock()
if a.client != nil {
_ = a.client.Close()
}
if a.proxies != nil {
a.proxies.Close()
}
a.client = nil
a.proxies = nil
a.connection = connectionConfig{}
a.clusterName = ""
a.brokerAddr = ""
}
func decodeParams(raw json.RawMessage) (map[string]any, error) {
if len(raw) == 0 || string(raw) == "null" {
return map[string]any{}, nil
}
var params map[string]any
if err := json.Unmarshal(raw, &params); err != nil {
return nil, fmt.Errorf("invalid params: %w", err)
}
return params, nil
}
func normalizeError(err error) string {
if err == nil {
return ""
}
message := strings.TrimSpace(err.Error())
if message == "" {
return "RocketMQ operation failed"
}
return message
}
func okResult() map[string]any {
return map[string]any{"ok": true}
}
agents/drivers/rocketmq/server_test.go
+34
View File
@@ -0,0 +1,34 @@
package main
import (
"encoding/json"
"testing"
)
func TestHandshakeAndShutdown(t *testing.T) {
agent := newRocketMQAgent()
result, err := agent.dispatch("handshake", json.RawMessage(`{}`))
if err != nil {
t.Fatal(err)
}
handshake := result.(map[string]any)
if handshake["protocolVersion"] != protocolVersion || handshake["agentProtocolVersion"] != agentProtocolVersion {
t.Fatalf("unexpected handshake: %#v", handshake)
}
if _, err := agent.dispatch("shutdown", nil); err != nil {
t.Fatal(err)
}
if !agent.shutdownRequested {
t.Fatal("shutdown flag was not set")
}
}
func TestDispatchRejectsInvalidParamsAndUnknownMethods(t *testing.T) {
agent := newRocketMQAgent()
if _, err := agent.dispatch("handshake", json.RawMessage(`[]`)); err == nil {
t.Fatal("expected array params to fail")
}
if _, err := agent.dispatch("does_not_exist", json.RawMessage(`{}`)); err == nil {
t.Fatal("expected unknown method to fail")
}
}
agents/drivers/rocketmq/src/main/java/com/dbx/agent/rocketmq/RocketMqAgent.java
-3794
View File
File diff suppressed because it is too large Load Diff
agents/drivers/rocketmq/src/test/java/com/dbx/agent/rocketmq/RocketMqAgentTest.java
-1272
View File
File diff suppressed because it is too large Load Diff
agents/drivers/rocketmq/src/test/java/com/dbx/agent/rocketmq/RocketMqLiveMessagePropertyTest.java
-179
View File
@@ -1,179 +0,0 @@
package com.dbx.agent.rocketmq;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assumptions.assumeTrue;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer;
import org.apache.rocketmq.client.consumer.MessageSelector;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext;
import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus;
import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently;
import org.apache.rocketmq.client.exception.MQClientException;
import org.apache.rocketmq.client.producer.DefaultMQProducer;
import org.apache.rocketmq.common.consumer.ConsumeFromWhere;
import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
/**
* Live RocketMQ tests on 127.0.0.1:9876. SQL92 filtering requires broker
* {@code enablePropertyFilter=true} (skipped when unsupported).
*/
class RocketMqLiveMessagePropertyTest {
private static final String NAMESRV = "127.0.0.1:9876";
private static final String TOPIC = "CS-PT";
private static DefaultMQAdminExt admin;
private static JsonObject conn;
private static boolean available;
@BeforeAll
static void connect() {
conn = JsonParser.parseString("""
{"namesrv_addr":"127.0.0.1:9876"}
""").getAsJsonObject();
admin = new DefaultMQAdminExt();
admin.setNamesrvAddr(NAMESRV);
try {
admin.start();
admin.examineBrokerClusterInfo();
available = true;
} catch (Exception e) {
available = false;
}
}
@AfterAll
static void shutdown() {
if (admin != null) {
admin.shutdown();
}
}
@Test
void sendMessageUserPropertyRoundTripViaPeek() throws Exception {
assumeTrue(available, "RocketMQ namesrv not available on " + NAMESRV);
DefaultMQProducer producer = RocketMqAgent.buildProducer(conn);
try {
String marker = "dbx-prop-" + UUID.randomUUID();
JsonObject sendParams = JsonParser.parseString("""
{
"topic":"%s",
"tag":"prop-test",
"payloadBase64":"%s",
"headers":{"TAGS":"prop-test","Region":"Hangzhou","marker":"%s"}
}
""".formatted(
TOPIC,
Base64.getEncoder().encodeToString(marker.getBytes(StandardCharsets.UTF_8)),
marker
)).getAsJsonObject();
@SuppressWarnings("unchecked")
Map<String, Object> sendResult = (Map<String, Object>) RocketMqAgent.sendMessageForTest(producer, sendParams);
Number partition = (Number) sendResult.get("partition");
Number offset = (Number) sendResult.get("offset");
assertNotNull(partition);
assertNotNull(offset);
JsonObject peekParams = JsonParser.parseString("""
{
"connection":{"namesrv_addr":"127.0.0.1:9876"},
"topic":"%s",
"count":1,
"partition":%d,
"offset":%d
}
""".formatted(TOPIC, partition.intValue(), offset.longValue())).getAsJsonObject();
@SuppressWarnings("unchecked")
Map<String, Object> peekResult = (Map<String, Object>) RocketMqAgent.peekMessagesForTest(admin, peekParams);
@SuppressWarnings("unchecked")
List<Map<String, Object>> messages = (List<Map<String, Object>>) peekResult.get("messages");
assumeTrue(messages != null && !messages.isEmpty(), "peek returned no messages at sent offset");
@SuppressWarnings("unchecked")
Map<String, String> headers = (Map<String, String>) messages.get(0).get("headers");
assertEquals("Hangzhou", headers.get("Region"));
assertEquals(marker, headers.get("marker"));
} finally {
producer.shutdown();
}
}
@Test
void sql92ConsumerFiltersOnUserProperty() throws Exception {
assumeTrue(available, "RocketMQ namesrv not available on " + NAMESRV);
String group = "dbx-sql92-" + UUID.randomUUID();
String markerMatch = "match-" + UUID.randomUUID();
String markerSkip = "skip-" + UUID.randomUUID();
CountDownLatch matched = new CountDownLatch(1);
AtomicInteger received = new AtomicInteger();
DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(group);
consumer.setNamesrvAddr(NAMESRV);
consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET);
try {
consumer.subscribe(
TOPIC,
MessageSelector.bySql("Region IS NOT NULL AND Region='Hangzhou' AND marker IS NOT NULL")
);
} catch (MQClientException e) {
assumeTrue(false, "Broker does not support SQL92 property filter: " + e.getMessage());
}
consumer.registerMessageListener((MessageListenerConcurrently) (messages, context) -> {
for (var message : messages) {
String marker = message.getProperty("marker");
if (markerMatch.equals(marker)) {
received.incrementAndGet();
matched.countDown();
}
}
return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
});
try {
consumer.start();
} catch (MQClientException e) {
assumeTrue(false, "Consumer start failed for SQL92: " + e.getMessage());
}
DefaultMQProducer producer = RocketMqAgent.buildProducer(conn);
try {
sendWithMarker(producer, markerSkip, "Shanghai");
sendWithMarker(producer, markerMatch, "Hangzhou");
assumeTrue(matched.await(15, TimeUnit.SECONDS), "SQL92 consumer did not receive matching message");
assertEquals(1, received.get(), "only the Hangzhou message should match the filter");
} finally {
producer.shutdown();
consumer.shutdown();
}
}
private static void sendWithMarker(DefaultMQProducer producer, String marker, String region) throws Exception {
JsonObject sendParams = JsonParser.parseString("""
{
"topic":"%s",
"tag":"sql92-test",
"payloadBase64":"%s",
"headers":{"TAGS":"sql92-test","Region":"%s","marker":"%s"}
}
""".formatted(
TOPIC,
Base64.getEncoder().encodeToString(marker.getBytes(StandardCharsets.UTF_8)),
region,
marker
)).getAsJsonObject();
RocketMqAgent.sendMessageForTest(producer, sendParams);
}
}
agents/drivers/rocketmq/src/test/java/com/dbx/agent/rocketmq/RocketMqLiveTopicTypeTest.java
-125
View File
@@ -1,125 +0,0 @@
package com.dbx.agent.rocketmq;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assumptions.assumeTrue;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import java.util.List;
import java.util.Map;
import org.apache.rocketmq.common.TopicConfig;
import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper;
import org.apache.rocketmq.tools.admin.DefaultMQAdminExt;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
/**
* Live verification against local RocketMQ (127.0.0.1:9876). Skipped when namesrv is unavailable.
*/
class RocketMqLiveTopicTypeTest {
private static DefaultMQAdminExt admin;
private static boolean available;
@BeforeAll
static void connect() {
admin = new DefaultMQAdminExt();
admin.setNamesrvAddr("127.0.0.1:9876");
try {
admin.start();
admin.examineBrokerClusterInfo();
available = true;
} catch (Exception e) {
available = false;
}
}
@AfterAll
static void shutdown() {
if (admin != null) {
admin.shutdown();
}
}
@Test
void cs123BrokerConfigHasNormalMessageType() throws Exception {
assumeTrue(available, "RocketMQ namesrv not available on 127.0.0.1:9876");
JsonObject conn = JsonParser.parseString("""
{"namesrv_addr":"127.0.0.1:9876"}
""").getAsJsonObject();
String brokerAddr = RocketMqAgent.remapBrokerAddrForClient(
admin.examineBrokerClusterInfo().getBrokerAddrTable().values().iterator().next().selectBrokerAddr(),
conn);
TopicConfig config = admin.examineTopicConfig(brokerAddr, "CS123");
assumeTrue(config != null, "CS123 topic config missing");
String fromAttributes = config.getAttributes() == null ? null : config.getAttributes().get("message.type");
assertEquals("NORMAL", fromAttributes, "attributes.message.type");
}
@Test
void getAllTopicConfigIncludesCs123MessageType() throws Exception {
assumeTrue(available, "RocketMQ namesrv not available on 127.0.0.1:9876");
JsonObject conn = JsonParser.parseString("""
{"namesrv_addr":"127.0.0.1:9876"}
""").getAsJsonObject();
String brokerAddr = RocketMqAgent.remapBrokerAddrForClient(
admin.examineBrokerClusterInfo().getBrokerAddrTable().values().iterator().next().selectBrokerAddr(),
conn);
TopicConfigSerializeWrapper wrapper = admin.getAllTopicConfig(brokerAddr, 10_000L);
TopicConfig config = wrapper.getTopicConfigTable().get("CS123");
assumeTrue(config != null, "CS123 missing from getAllTopicConfig");
String fromAttributes = config.getAttributes() == null ? null : config.getAttributes().get("message.type");
assertEquals("NORMAL", fromAttributes, "bulk attributes.message.type");
}
@Test
void agentListTopicsClassifiesAllTopicsAgainstBroker() throws Exception {
assumeTrue(available, "RocketMQ namesrv not available on 127.0.0.1:9876");
JsonObject params = JsonParser.parseString("""
{"connection":{"namesrv_addr":"127.0.0.1:9876"},"limit":500,"offset":0}
""").getAsJsonObject();
Object result = RocketMqAgent.listTopicsWithAdminForTest(admin, params);
assumeTrue(result instanceof Map, "unexpected listTopics result");
@SuppressWarnings("unchecked")
List<Map<String, Object>> topics = (List<Map<String, Object>>) ((Map<?, ?>) result).get("topics");
assumeTrue(topics != null && !topics.isEmpty(), "no topics returned");
Map<String, Object> cs123 = topics.stream()
.filter(row -> "CS123".equals(String.valueOf(row.get("name"))))
.findFirst()
.orElse(null);
assumeTrue(cs123 != null, "CS123 not in agent topic list");
assertEquals("NORMAL", String.valueOf(cs123.get("messageType")), "agent messageType for CS123");
Map<String, String> expected = Map.ofEntries(
Map.entry("CS123", "NORMAL"),
Map.entry("BenchmarkTest", "SYSTEM"),
Map.entry("SCHEDULE_TOPIC_XXXX", "SYSTEM"),
Map.entry("RMQ_SYS_TRANS_HALF_TOPIC", "SYSTEM"),
Map.entry("RMQ_SYS_TRANS_OP_HALF_TOPIC", "SYSTEM"),
Map.entry("RMQ_SYS_ROCKSDB_TRANS_HALF_TOPIC", "SYSTEM"),
Map.entry("RMQ_SYS_ROCKSDB_TRANS_OP_HALF_TOPIC", "SYSTEM"),
Map.entry("rmq_sys_wheel_timer", "SYSTEM"),
Map.entry("rmq_sys_REVIVE_LOG_DefaultCluster", "SYSTEM"),
Map.entry("rmq_sys_SYNC_BROKER_MEMBER_37e06a7b803c", "SYSTEM"),
Map.entry("rmq_sys_SYNC_BROKER_MEMBER_broker-a", "SYSTEM"),
Map.entry("DefaultHeartBeatSyncerTopic", "SYSTEM"),
Map.entry("DefaultCluster_REPLY_TOPIC", "SYSTEM"),
Map.entry("SELF_TEST_TOPIC", "SYSTEM"),
Map.entry("OFFSET_MOVED_EVENT", "SYSTEM"),
Map.entry("TBW102", "SYSTEM"),
Map.entry("DefaultCluster", "SYSTEM"),
Map.entry("broker-a", "SYSTEM"),
Map.entry("37e06a7b803c", "UNSPECIFIED"),
Map.entry("%RETRY%CID_DefaultHeartBeatSyncerTopic", "RETRY")
);
for (Map<String, Object> row : topics) {
String name = String.valueOf(row.get("name"));
String messageType = String.valueOf(row.get("messageType"));
if (expected.containsKey(name)) {
assertEquals(expected.get(name), messageType, "messageType for " + name);
}
}
}
}
agents/drivers/rocketmq/topics.go
+620
View File
@@ -0,0 +1,620 @@
package main
import (
"context"
"encoding/json"
"fmt"
"sort"
"strconv"
"strings"
"time"
admin "github.com/amigoer/rocketmq-admin-go"
"github.com/amigoer/rocketmq-admin-go/protocol/remoting"
)
var reservedTopicNames = map[string]struct{}{
"TBW102": {}, "BenchmarkTest": {}, "SELF_TEST_TOPIC": {},
"OFFSET_MOVED_EVENT": {}, "DefaultHeartBeatSyncerTopic": {},
}
type topicConfigWire struct {
TopicName string `json:"topicName"`
ReadQueueNums int `json:"readQueueNums"`
WriteQueueNums int `json:"writeQueueNums"`
Perm int `json:"perm"`
TopicFilterType string `json:"topicFilterType"`
TopicSysFlag int `json:"topicSysFlag"`
Order bool `json:"order"`
Attributes map[string]string `json:"attributes"`
}
func (a *rocketMQAgent) listTopics(params map[string]any) (any, error) {
client, config, err := a.requireClient()
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout)
defer cancel()
a.mu.RLock()
clusterName, brokerAddr := a.clusterName, a.brokerAddr
a.mu.RUnlock()
topicList, err := client.FetchTopicsByCluster(ctx, clusterName)
if err != nil {
topicList, err = client.FetchAllTopicList(ctx)
}
if err != nil {
return nil, err
}
_ = brokerAddr
configs, complete := a.collectTopicConfigs(ctx)
keyword := strings.ToLower(stringValue(params, "keyword"))
rows := make([]map[string]any, 0, len(topicList.TopicList))
names := make(map[string]struct{}, len(topicList.TopicList)+len(configs))
for _, name := range topicList.TopicList {
names[name] = struct{}{}
}
for name := range configs {
names[name] = struct{}{}
}
if complete && len(configs) > 0 {
names = make(map[string]struct{}, len(configs))
for name := range configs {
names[name] = struct{}{}
}
}
for _, name := range sortedKeys(names) {
if keyword != "" && !strings.Contains(strings.ToLower(name), keyword) {
continue
}
configEntry := configs[name]
partitions := 0
perm := 6
messageType := classifyTopicMessageType(name, clusterName, nil)
if configEntry != nil {
partitions = max(configEntry.ReadQueueNums, 1)
perm = configEntry.Perm
messageType = classifyTopicMessageType(name, clusterName, configEntry.Attributes)
}
if partitions <= 0 {
if route, routeErr := client.ExamineTopicRouteInfo(ctx, name); routeErr == nil {
if len(route.QueueDatas) > 0 {
partitions = max(route.QueueDatas[0].ReadQueueNums, 1)
}
}
}
if partitions <= 0 {
partitions = 1
}
rows = append(rows, map[string]any{
"name": name, "partitions": partitions, "replicationFactor": 1,
"internal": messageType == "SYSTEM" || messageType == "RETRY" || messageType == "DLQ",
"messageType": messageType, "readQueueNums": partitions,
"writeQueueNums": writeQueueNums(configEntry, partitions), "perm": perm,
})
}
sort.Slice(rows, func(i, j int) bool { return rows[i]["name"].(string) < rows[j]["name"].(string) })
offset := max(0, intValue(params, 0, "offset"))
limit := intValue(params, 200, "limit")
return map[string]any{
"topics": paginate(rows, offset, limit), "total": len(rows), "offset": offset, "limit": limit,
}, nil
}
func (a *rocketMQAgent) createTopic(params map[string]any) (any, error) {
name, err := requireString(params, "name")
if err != nil {
return nil, err
}
readQueues := max(1, intValue(params, 8, "readQueueNums", "partitions"))
writeQueues := max(1, intValue(params, readQueues, "writeQueueNums"))
perm := normalizeTopicPerm(intValue(params, 6, "perm"))
messageType := normalizeMessageType(stringValue(params, "messageType"))
addresses, err := a.masterBrokerAddresses(stringValue(params, "brokerName"))
if err != nil {
return nil, err
}
_, config, _ := a.requireClient()
ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout)
defer cancel()
for _, address := range addresses {
if err := writeTopicConfig(ctx, address, &topicConfigWire{
TopicName: name, ReadQueueNums: readQueues, WriteQueueNums: writeQueues,
Perm: perm, TopicFilterType: "SINGLE_TAG", Attributes: map[string]string{"+message.type": messageType},
}); err != nil {
return nil, err
}
}
return okResult(), nil
}
func writeTopicConfig(ctx context.Context, address string, config *topicConfigWire) error {
fields := map[string]string{
"topic": config.TopicName, "defaultTopic": "TBW102", "readQueueNums": strconv.Itoa(config.ReadQueueNums),
"writeQueueNums": strconv.Itoa(config.WriteQueueNums), "perm": strconv.Itoa(config.Perm),
"topicFilterType": valueOrDefault(config.TopicFilterType, "SINGLE_TAG"),
"topicSysFlag": strconv.Itoa(config.TopicSysFlag), "order": strconv.FormatBool(config.Order),
"attributes": topicAttributesString(config.Attributes),
}
command := remoting.NewRequest(remoting.UpdateAndCreateTopic, fields)
_, err := invokeRemotingWithClient(ctx, address, command)
return err
}
func (a *rocketMQAgent) deleteTopic(params map[string]any) (any, error) {
name, err := requireString(params, "name")
if err != nil {
return nil, err
}
client, config, _ := a.requireClient()
ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout)
defer cancel()
addresses, err := a.masterBrokerAddresses(stringValue(params, "brokerName"))
if err != nil {
return nil, err
}
for _, address := range addresses {
if err := client.DeleteTopicInBroker(ctx, address, name); err != nil {
return nil, err
}
}
if err := client.DeleteTopicInNameServer(ctx, name); err != nil {
return nil, err
}
return okResult(), nil
}
func (a *rocketMQAgent) updatePartitions(params map[string]any) (any, error) {
name, err := requireString(params, "name")
if err != nil {
return nil, err
}
_, config, _ := a.requireClient()
ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout)
defer cancel()
addresses, err := a.masterBrokerAddresses(stringValue(params, "brokerName"))
if err != nil {
return nil, err
}
readQueues := max(1, intValue(params, intValue(params, 1, "totalPartitions"), "readQueueNums"))
writeQueues := max(1, intValue(params, readQueues, "writeQueueNums"))
for _, address := range addresses {
current, configErr := readTopicConfig(ctx, address, name)
if configErr != nil {
current = &topicConfigWire{TopicName: name, Perm: 6, TopicFilterType: "SINGLE_TAG"}
}
current.ReadQueueNums = readQueues
current.WriteQueueNums = writeQueues
if err := writeTopicConfig(ctx, address, current); err != nil {
return nil, err
}
}
return okResult(), nil
}
func (a *rocketMQAgent) getTopicRoute(params map[string]any) (any, error) {
name, err := requireString(params, "name", "topic")
if err != nil {
return nil, err
}
client, config, _ := a.requireClient()
ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout)
defer cancel()
route, err := client.ExamineTopicRouteInfo(ctx, name)
if err != nil {
return nil, err
}
a.mu.RLock()
proxies := a.proxies
a.mu.RUnlock()
brokerDatas := make([]map[string]any, 0, len(route.BrokerDatas))
for _, broker := range route.BrokerDatas {
addresses := make(map[string]string, len(broker.BrokerAddrs))
for brokerID, address := range broker.BrokerAddrs {
if original := proxies.OriginalForLocal(address); original != "" {
address = original
}
addresses[brokerID] = address
}
brokerDatas = append(brokerDatas, map[string]any{
"brokerName": broker.BrokerName, "cluster": broker.Cluster, "brokerAddrs": addresses,
})
}
queueDatas := make([]map[string]any, 0, len(route.QueueDatas))
for _, queue := range route.QueueDatas {
queueDatas = append(queueDatas, map[string]any{
"brokerName": queue.BrokerName, "readQueueNums": queue.ReadQueueNums,
"writeQueueNums": queue.WriteQueueNums, "perm": queue.Perm,
})
}
return map[string]any{"topic": name, "queueDatas": queueDatas, "brokerDatas": brokerDatas}, nil
}
func (a *rocketMQAgent) getTopicStats(params map[string]any) (any, error) {
name, err := requireString(params, "name", "topic")
if err != nil {
return nil, err
}
client, config, _ := a.requireClient()
ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout)
defer cancel()
stats, err := a.examineTopicStats(ctx, client, name)
if err != nil {
return nil, err
}
partitionStats := make([]map[string]any, 0, len(stats))
var total int64
for key, offset := range stats {
queue := parseMessageQueueKey(key)
count := max(int64(0), offset.MaxOffset-offset.MinOffset)
total += count
partitionStats = append(partitionStats, map[string]any{
"partition": queue.QueueID, "brokerName": queue.BrokerName,
"beginOffset": offset.MinOffset, "endOffset": offset.MaxOffset,
"messageCount": count, "lastTimestamp": offset.LastUpdateTimestamp,
})
}
sort.Slice(partitionStats, func(i, j int) bool {
return partitionStats[i]["partition"].(int) < partitionStats[j]["partition"].(int)
})
return map[string]any{
"name": name, "partitions": len(partitionStats), "replicationFactor": 1,
"totalMessages": total, "partitionStats": partitionStats,
}, nil
}
func (a *rocketMQAgent) examineTopicStats(
ctx context.Context,
client *admin.Client,
topic string,
) (map[string]*admin.TopicOffset, error) {
route, err := client.ExamineTopicRouteInfo(ctx, topic)
if err != nil {
return nil, err
}
addresses := masterAddressesFromRoute(route)
if len(addresses) == 0 {
return nil, fmt.Errorf("no RocketMQ master broker found for topic %s", topic)
}
merged := make(map[string]*admin.TopicOffset)
var lastErr error
successCount := 0
for _, address := range addresses {
response, requestErr := invokeRemotingWithClient(ctx, address,
remoting.NewRequest(remoting.GetTopicStatsInfo, map[string]string{"topic": topic}))
if requestErr != nil {
lastErr = requestErr
continue
}
partial, decodeErr := decodeTopicStats(response.Body)
if decodeErr != nil {
lastErr = decodeErr
continue
}
successCount++
for key, offset := range partial {
merged[key] = offset
}
}
if successCount != len(addresses) {
return nil, fmt.Errorf("query topic stats for %s on all masters: %w", topic, lastErr)
}
if len(merged) == 0 {
return nil, fmt.Errorf("topic stats not found: %s", topic)
}
return merged, nil
}
func decodeTopicStats(body []byte) (map[string]*admin.TopicOffset, error) {
var stats admin.TopicStatsTable
if err := json.Unmarshal(repairRocketMQJSON(body), &stats); err != nil {
return nil, fmt.Errorf("decode topic stats: %w", err)
}
if stats.OffsetTable == nil {
stats.OffsetTable = make(map[string]*admin.TopicOffset)
}
return stats.OffsetTable, nil
}
func masterAddressesFromRoute(route *admin.TopicRouteData) []string {
addresses := make(map[string]struct{})
for _, broker := range route.BrokerDatas {
if address := broker.BrokerAddrs["0"]; address != "" {
addresses[address] = struct{}{}
}
}
return sortedKeys(addresses)
}
func (a *rocketMQAgent) getTopicConfig(params map[string]any) (any, error) {
name, err := requireString(params, "name", "topic")
if err != nil {
return nil, err
}
_, config, _ := a.requireClient()
ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout)
defer cancel()
address, err := a.brokerAddressForName(stringValue(params, "brokerName"))
if err != nil {
return nil, err
}
topicConfig, err := readTopicConfig(ctx, address, name)
if err != nil {
return nil, err
}
configs := map[string]any{
"readQueueNums": configEntry(strconv.Itoa(topicConfig.ReadQueueNums)),
"writeQueueNums": configEntry(strconv.Itoa(topicConfig.WriteQueueNums)),
"perm": configEntry(strconv.Itoa(topicConfig.Perm)),
}
return map[string]any{"configs": configs}, nil
}
func (a *rocketMQAgent) alterTopicConfig(params map[string]any) (any, error) {
name, err := requireString(params, "name", "topic")
if err != nil {
return nil, err
}
_, config, _ := a.requireClient()
ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout)
defer cancel()
address, err := a.brokerAddressForName(stringValue(params, "brokerName"))
if err != nil {
return nil, err
}
topicConfig, err := readTopicConfig(ctx, address, name)
if err != nil {
return nil, err
}
if entries, ok := params["configs"].([]any); ok {
for _, raw := range entries {
entry, _ := raw.(map[string]any)
value := stringValue(entry, "value")
switch stringValue(entry, "key") {
case "readQueueNums":
topicConfig.ReadQueueNums = intValue(entry, topicConfig.ReadQueueNums, "value")
case "writeQueueNums":
topicConfig.WriteQueueNums = intValue(entry, topicConfig.WriteQueueNums, "value")
case "perm":
topicConfig.Perm = normalizeTopicPerm(intValue(entry, topicConfig.Perm, "value"))
case "retention.ms", "retention.bytes":
if value != "" {
return nil, fmt.Errorf("RocketMQ topic retention is broker-level and cannot be changed per topic")
}
}
}
}
if err := writeTopicConfig(ctx, address, topicConfig); err != nil {
return nil, err
}
return okResult(), nil
}
func (a *rocketMQAgent) skipTopicAccumulation(params map[string]any) (any, error) {
topic, err := requireString(params, "topic", "name")
if err != nil {
return nil, err
}
client, config, _ := a.requireClient()
ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout)
defer cancel()
groups, err := client.QueryTopicConsumeByWho(ctx, topic)
if err != nil {
return nil, err
}
reset := make([]string, 0, len(groups))
for _, group := range groups {
if _, resetErr := client.ResetOffsetByTimestamp(ctx, topic, group, time.Now().UnixMilli(), true); resetErr == nil {
reset = append(reset, group)
}
}
return map[string]any{"ok": true, "resetGroups": len(reset)}, nil
}
func (a *rocketMQAgent) masterBrokerAddresses(brokerName string) ([]string, error) {
client, config, err := a.requireClient()
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), config.RequestTimeout)
defer cancel()
info, err := client.ExamineBrokerClusterInfo(ctx)
if err != nil {
return nil, err
}
addresses := make([]string, 0)
for _, name := range sortedKeys(info.BrokerAddrTable) {
if brokerName != "" && brokerName != name {
continue
}
if address := info.BrokerAddrTable[name].BrokerAddrs["0"]; address != "" {
addresses = append(addresses, address)
}
}
if len(addresses) == 0 {
a.mu.RLock()
fallback := a.brokerAddr
a.mu.RUnlock()
if fallback != "" {
addresses = append(addresses, fallback)
}
}
if len(addresses) == 0 {
return nil, fmt.Errorf("no RocketMQ master broker found")
}
return addresses, nil
}
func (a *rocketMQAgent) brokerAddressForName(brokerName string) (string, error) {
addresses, err := a.masterBrokerAddresses(brokerName)
if err != nil {
return "", err
}
return addresses[0], nil
}
type parsedMessageQueue struct {
Topic string
BrokerName string
QueueID int
}
func parseMessageQueueKey(value string) parsedMessageQueue {
value = strings.TrimSpace(value)
var queue struct {
Topic string `json:"topic"`
BrokerName string `json:"brokerName"`
QueueID int `json:"queueId"`
}
if strings.HasPrefix(value, "{") && json.Unmarshal([]byte(value), &queue) == nil {
return parsedMessageQueue(queue)
}
if strings.HasPrefix(value, "MessageQueue [") && strings.HasSuffix(value, "]") {
fields := strings.Split(strings.TrimSuffix(strings.TrimPrefix(value, "MessageQueue ["), "]"), ",")
for _, field := range fields {
keyValue := strings.SplitN(strings.TrimSpace(field), "=", 2)
if len(keyValue) != 2 {
continue
}
switch keyValue[0] {
case "topic":
queue.Topic = keyValue[1]
case "brokerName":
queue.BrokerName = keyValue[1]
case "queueId":
queue.QueueID, _ = strconv.Atoi(keyValue[1])
}
}
return parsedMessageQueue(queue)
}
parts := strings.Split(value, "-")
if len(parts) >= 3 {
queueID, _ := strconv.Atoi(parts[len(parts)-1])
return parsedMessageQueue{Topic: strings.Join(parts[:len(parts)-2], "-"), BrokerName: parts[len(parts)-2], QueueID: queueID}
}
return parsedMessageQueue{}
}
func configEntry(value string) map[string]any {
return map[string]any{
"value": value, "isDefault": false,
"isReadOnly": false, "isSensitive": false, "source": "USER",
}
}
func isSystemTopic(topic, cluster string) bool {
if topic == "" {
return true
}
if _, ok := reservedTopicNames[topic]; ok {
return true
}
return strings.HasPrefix(topic, "%") || strings.HasPrefix(topic, "RMQ_SYS") ||
strings.HasPrefix(topic, "rmq_sys") || strings.HasPrefix(topic, "SCHEDULE_TOPIC_") ||
strings.HasPrefix(topic, "rocketmq-broker-") || strings.HasSuffix(topic, "_REPLY_TOPIC") || topic == cluster
}
func classifyTopicMessageType(topic, cluster string, attributes map[string]string) string {
if strings.HasPrefix(topic, "%RETRY%") || strings.HasPrefix(topic, "%R") {
return "RETRY"
}
if strings.HasPrefix(topic, "%DLQ%") || strings.HasPrefix(topic, "%D") {
return "DLQ"
}
if isSystemTopic(topic, cluster) {
return "SYSTEM"
}
for _, key := range []string{"message.type", "+message.type"} {
if value := normalizeMessageType(attributes[key]); value != "NORMAL" || attributes[key] != "" {
return value
}
}
return "UNSPECIFIED"
}
func (a *rocketMQAgent) collectTopicConfigs(ctx context.Context) (map[string]*topicConfigWire, bool) {
addresses, err := a.masterBrokerAddresses("")
if err != nil {
return map[string]*topicConfigWire{}, false
}
merged := make(map[string]*topicConfigWire)
complete := true
for _, address := range addresses {
if a.proxies.IsCollisionFallback(address) && len(merged) > 0 {
complete = false
continue
}
configs, fetchErr := fetchAllTopicConfigs(ctx, address)
if fetchErr != nil {
complete = false
continue
}
for name, config := range configs {
if merged[name] == nil {
merged[name] = config
}
}
}
return merged, complete
}
func fetchAllTopicConfigs(ctx context.Context, address string) (map[string]*topicConfigWire, error) {
response, err := invokeRemotingWithClient(ctx, address, remoting.NewRequest(remoting.GetAllTopicConfig, nil))
if err != nil {
return nil, err
}
var wrapper struct {
TopicConfigTable map[string]*topicConfigWire `json:"topicConfigTable"`
}
if err := json.Unmarshal(repairRocketMQJSON(response.Body), &wrapper); err != nil {
return nil, err
}
return wrapper.TopicConfigTable, nil
}
func readTopicConfig(ctx context.Context, address, topic string) (*topicConfigWire, error) {
response, err := invokeRemotingWithClient(ctx, address, remoting.NewRequest(remoting.GetTopicConfig, map[string]string{"topic": topic}))
if err == nil {
var config topicConfigWire
if decodeErr := json.Unmarshal(repairRocketMQJSON(response.Body), &config); decodeErr != nil {
return nil, decodeErr
}
return &config, nil
}
configs, fallbackErr := fetchAllTopicConfigs(ctx, address)
if fallbackErr != nil {
return nil, fmt.Errorf("read topic config %s: %v; fallback snapshot: %w", topic, err, fallbackErr)
}
config := configs[topic]
if config == nil {
return nil, fmt.Errorf("topic config not found: %s", topic)
}
if config.TopicName == "" {
config.TopicName = topic
}
return config, nil
}
func topicAttributesString(attributes map[string]string) string {
keys := sortedKeys(attributes)
parts := make([]string, 0, len(keys))
for _, key := range keys {
if !strings.HasPrefix(key, "+") && !strings.HasPrefix(key, "-") {
continue
}
if attributes[key] == "" {
parts = append(parts, key)
} else {
parts = append(parts, key+"="+attributes[key])
}
}
return strings.Join(parts, ",")
}
func writeQueueNums(config *topicConfigWire, fallback int) int {
if config == nil || config.WriteQueueNums <= 0 {
return fallback
}
return config.WriteQueueNums
}
agents/drivers/rocketmq/topics_test.go
+80
View File
@@ -0,0 +1,80 @@
package main
import (
"reflect"
"testing"
)
func TestTopicAttributesStringEmitsOnlyRocketMQ5Modifications(t *testing.T) {
attributes := map[string]string{
"+message.type": "FIFO",
"cleanup.policy": "DELETE",
"-reserve.time": "",
}
if got := topicAttributesString(attributes); got != "+message.type=FIFO,-reserve.time" {
t.Fatalf("topicAttributesString() = %q", got)
}
}
func TestClassifyTopicMessageType(t *testing.T) {
tests := map[string]string{
"%RETRY%GID_Orders": "RETRY",
"%DLQ%GID_Orders": "DLQ",
"RMQ_SYS_TRANS_HALF_TOPIC": "SYSTEM",
"Orders": "UNSPECIFIED",
}
for topic, want := range tests {
if got := classifyTopicMessageType(topic, "DefaultCluster", nil); got != want {
t.Fatalf("classifyTopicMessageType(%q) = %q, want %q", topic, got, want)
}
}
attributes := map[string]string{"message.type": "ORDER"}
if got := classifyTopicMessageType("Ordered", "DefaultCluster", attributes); got != "FIFO" {
t.Fatalf("ORDER attribute classified as %q", got)
}
}
func TestPaginationAndMessageQueueParsing(t *testing.T) {
items := []int{0, 1, 2, 3}
if got := paginate(items, 2, 0); !reflect.DeepEqual(got, []int{2, 3}) {
t.Fatalf("paginate all from offset = %#v", got)
}
queue := parseMessageQueueKey(`{"topic":"Orders","brokerName":"broker-a","queueId":3}`)
if queue.Topic != "Orders" || queue.BrokerName != "broker-a" || queue.QueueID != 3 {
t.Fatalf("unexpected JSON queue: %#v", queue)
}
queue = parseMessageQueueKey("MessageQueue [topic=Order-Events, brokerName=broker-a, queueId=7]")
if queue.Topic != "Order-Events" || queue.BrokerName != "broker-a" || queue.QueueID != 7 {
t.Fatalf("unexpected text queue: %#v", queue)
}
}
func TestDecodeTopicStatsRepairsObjectKeys(t *testing.T) {
body := []byte(`{"offsetTable":{{"brokerName":"broker-a","queueId":1,"topic":"Orders"}:{"minOffset":2,"maxOffset":9,"lastUpdateTimestamp":10}}}`)
stats, err := decodeTopicStats(body)
if err != nil {
t.Fatal(err)
}
if len(stats) != 1 {
t.Fatalf("unexpected stats: %#v", stats)
}
for key, offset := range stats {
queue := parseMessageQueueKey(key)
if queue.Topic != "Orders" || queue.BrokerName != "broker-a" || queue.QueueID != 1 {
t.Fatalf("unexpected queue key %q: %#v", key, queue)
}
if offset.MinOffset != 2 || offset.MaxOffset != 9 {
t.Fatalf("unexpected offset: %#v", offset)
}
}
}
func TestConfigEntryMatchesAgentContract(t *testing.T) {
entry := configEntry("8")
if entry["value"] != "8" || entry["source"] != "USER" {
t.Fatalf("unexpected config entry: %#v", entry)
}
if _, exists := entry["key"]; exists {
t.Fatalf("config entry must be keyed by its parent object: %#v", entry)
}
}
agents/scripts/driver_release_packages_test.py
+24 -2
View File
@@ -24,6 +24,8 @@ class DriverReleasePackagesTest(unittest.TestCase):
duckdb_source.write_bytes(b"\xcf\xfa\xed\xfetest-duckdb-agent")
rabbitmq_source = release_dir / "dbx-agent-rabbitmq-linux-x64"
rabbitmq_source.write_bytes(b"\x7fELFtest-rabbitmq-agent")
rocketmq_source = release_dir / "dbx-agent-rocketmq-windows-x64.exe"
rocketmq_source.write_bytes(b"MZtest-rocketmq-agent")
cassandra_source = release_dir / "dbx-agent-cassandra-linux-x64"
cassandra_source.write_bytes(b"\x7fELFtest-cassandra-agent")
tdengine_source = release_dir / "dbx-agent-tdengine-windows-aarch64.exe"
@@ -40,6 +42,7 @@ class DriverReleasePackagesTest(unittest.TestCase):
"vastbase": "0.1.37",
"duckdb": "0.1.0",
"rabbitmq": "0.1.0",
"rocketmq": "0.1.0",
"cassandra": "0.1.37",
"hive": "0.1.43",
"tdengine": "0.1.0",
@@ -51,6 +54,7 @@ class DriverReleasePackagesTest(unittest.TestCase):
versioned_vastbase = release_dir / "dbx-agent-vastbase-0.1.37-linux-x64"
versioned_duckdb = release_dir / "dbx-agent-duckdb-0.1.0-macos-aarch64"
versioned_rabbitmq = release_dir / "dbx-agent-rabbitmq-0.1.0-linux-x64"
versioned_rocketmq = release_dir / "dbx-agent-rocketmq-0.1.0-windows-x64.exe"
versioned_cassandra = release_dir / "dbx-agent-cassandra-0.1.37-linux-x64"
versioned_tdengine = release_dir / "dbx-agent-tdengine-0.1.0-windows-aarch64.exe"
self.assertEqual(
@@ -62,6 +66,7 @@ class DriverReleasePackagesTest(unittest.TestCase):
versioned_vastbase,
versioned_duckdb,
versioned_rabbitmq,
versioned_rocketmq,
versioned_tdengine,
],
)
@@ -141,6 +146,19 @@ class DriverReleasePackagesTest(unittest.TestCase):
}
},
},
"rocketmq": {
"version": "0.1.0",
"label": "Apache RocketMQ",
"min_app_version": "0.6.0",
"jre": "21",
"jar": {"url": "https://example.com/legacy-placeholder.jar", "size": 0},
"native": {
"windows-x64": {
"url": f"https://example.com/{versioned_rocketmq.name}",
"size": versioned_rocketmq.stat().st_size,
}
},
},
"tdengine": {
"version": "0.1.0",
"label": "TDengine",
@@ -169,6 +187,7 @@ class DriverReleasePackagesTest(unittest.TestCase):
release_dir / "dbx-agent-vastbase-0.1.37-linux-x64.tar.zst",
release_dir / "dbx-agent-duckdb-0.1.0-macos-aarch64.tar.zst",
release_dir / "dbx-agent-rabbitmq-0.1.0-linux-x64.tar.zst",
release_dir / "dbx-agent-rocketmq-0.1.0-windows-x64.tar.zst",
release_dir / "dbx-agent-tdengine-0.1.0-windows-aarch64.tar.zst",
],
)
@@ -179,7 +198,8 @@ class DriverReleasePackagesTest(unittest.TestCase):
(outputs[3], "vastbase", versioned_vastbase, "native", "linux-x64"),
(outputs[4], "duckdb", versioned_duckdb, "native", "macos-aarch64"),
(outputs[5], "rabbitmq", versioned_rabbitmq, "native", "linux-x64"),
(outputs[6], "tdengine", versioned_tdengine, "native", "windows-aarch64"),
(outputs[6], "rocketmq", versioned_rocketmq, "native", "windows-x64"),
(outputs[7], "tdengine", versioned_tdengine, "native", "windows-aarch64"),
]
for output, driver_name, source, artifact_type, platform in package_cases:
tar_bytes = subprocess.run(
@@ -209,7 +229,8 @@ class DriverReleasePackagesTest(unittest.TestCase):
(final_registry["drivers"]["vastbase"]["native"]["linux-x64"], outputs[3]),
(final_registry["drivers"]["duckdb"]["native"]["macos-aarch64"], outputs[4]),
(final_registry["drivers"]["rabbitmq"]["native"]["linux-x64"], outputs[5]),
(final_registry["drivers"]["tdengine"]["native"]["windows-aarch64"], outputs[6]),
(final_registry["drivers"]["rocketmq"]["native"]["windows-x64"], outputs[6]),
(final_registry["drivers"]["tdengine"]["native"]["windows-aarch64"], outputs[7]),
]
for artifact, output in release_artifacts:
self.assertEqual(artifact["url"], f"https://example.com/{output.name}")
@@ -226,6 +247,7 @@ class DriverReleasePackagesTest(unittest.TestCase):
versioned_java,
versioned_native,
versioned_rabbitmq,
versioned_rocketmq,
versioned_tdengine,
versioned_vastbase,
],
agents/scripts/validate_agents.py
+1
View File
@@ -24,6 +24,7 @@ NATIVE_ONLY_AGENT_MODULES = {
"tdengine": "drivers/tdengine",
"xugu": "drivers/xugu",
"rabbitmq": "drivers/rabbitmq",
"rocketmq": "drivers/rocketmq",
}
AUTO_VERSIONED_NATIVE_MODULES = {"duckdb"}
JDBC_ARCHITECTURE_ALLOWLIST = {
agents/scripts/validate_agents_test.py
+2 -1
View File
@@ -147,11 +147,12 @@ class ValidateAgentsTest(unittest.TestCase):
"xugu",
"duckdb",
"rabbitmq",
"rocketmq",
"tdengine",
):
(root / "drivers" / driver).mkdir(parents=True)
(root / "versions.json").write_text(
json.dumps({"h2": "0.1.0", "cassandra": "0.1.0", "hive": "0.1.0", "oracle": "0.1.0", "kingbase": "0.1.0", "iotdb": "0.1.0", "neo4j": "0.1.0", "vastbase": "0.1.0", "xugu": "0.1.0", "rabbitmq": "0.1.0", "tdengine": "0.1.0"}),
json.dumps({"h2": "0.1.0", "cassandra": "0.1.0", "hive": "0.1.0", "oracle": "0.1.0", "kingbase": "0.1.0", "iotdb": "0.1.0", "neo4j": "0.1.0", "vastbase": "0.1.0", "xugu": "0.1.0", "rabbitmq": "0.1.0", "rocketmq": "0.1.0", "tdengine": "0.1.0"}),
encoding="utf-8",
)
agents/scripts/version_agent_artifacts.py
+1 -1
View File
@@ -4,7 +4,7 @@ import json
from pathlib import Path
NATIVE_DRIVERS = ("cassandra", "hive", "oracle", "xugu", "kingbase", "iotdb", "neo4j", "vastbase", "duckdb", "rabbitmq", "tdengine")
NATIVE_DRIVERS = ("cassandra", "hive", "oracle", "xugu", "kingbase", "iotdb", "neo4j", "vastbase", "duckdb", "rabbitmq", "rocketmq", "tdengine")
PLATFORMS = (
"macos-aarch64",
"macos-x64",
agents/settings.gradle
+1 -1
View File
@@ -6,7 +6,7 @@ def driverModules = [
'teradata', 'vertica', 'firebird', 'exasol', 'oceanbase-oracle', 'gbase8a', 'gbase8s',
'bigquery', 'kylin', 'sundb', 'h2', 'h2-legacy', 'snowflake', 'trino', 'spark',
'db2', 'informix', 'mongodb', 'highgo', 'uxdb', 'yashandb', 'oscar',
'iris', 'etcd', 'zookeeper', 'kafka', 'rocketmq', 'sqlserver-legacy'
'iris', 'etcd', 'zookeeper', 'kafka', 'sqlserver-legacy'
]
include(*(infrastructureModules + driverModules))
crates/dbx-core/src/mq/README.md
+2
View File
@@ -14,7 +14,9 @@ mq/
├── service.rs - 服务层函数
└── adapters/
├── pulsar.rs - Pulsar 实现
├── kafka.rs - Kafka 实现 (Java agent)
├── rabbitmq.rs - RabbitMQ 实现 (Go native agent)
├── rocketmq.rs - RocketMQ 实现 (Go native agent)
└── pulsar_version.rs - 版本探测
```
crates/dbx-core/src/mq/adapters/rocketmq.rs
+8 -9
View File
@@ -1,10 +1,9 @@
//! Apache RocketMQ admin adapter. Communicates with a Java agent process
//! (`RocketMqAgent.java`) via JSON-RPC over stdin/stdout. The Java agent uses
//! `DefaultMQAdminExt` for admin operations and `DefaultMQProducer` for
//! message production.
//! Apache RocketMQ admin adapter. Communicates with the Go native agent via
//! JSON-RPC over stdin/stdout. The agent uses `rocketmq-admin-go` plus direct
//! remoting calls for compatibility gaps in the upstream admin API.
//!
//! This adapter follows the same pattern as the ZooKeeper/Etcd agents:
//! 1. Spawn a Java agent process via `AgentDriverClient`
//! This adapter follows the same process-agent pattern as the other drivers:
//! 1. Spawn the native agent process via `AgentDriverClient`
//! 2. Perform JSON-RPC handshake + connect
//! 3. Delegate all `MessageQueueAdmin` trait methods to JSON-RPC calls
@@ -97,10 +96,10 @@ fn cluster_info_from_agent_result(result: &serde_json::Value) -> MqClusterInfo {
}
impl RocketMqAdmin {
/// Spawn the RocketMQ Java agent, perform handshake, and connect.
/// Spawn the RocketMQ native agent, perform handshake, and connect.
///
/// Callers must probe NameServer reachability before invoking this so the
/// connect-timeout wall covers only JVM spawn + handshake + connect.
/// connect-timeout wall covers only process spawn + handshake + connect.
pub async fn new(cfg: MqAdminConfig, launch: AgentLaunchSpec) -> Result<Self, String> {
let mut client = AgentDriverClient::spawn(launch).await?;
@@ -814,7 +813,7 @@ fn rocketmq_topic_admin_params(topic: &TopicRef, partitions: Option<u32>) -> ser
params
}
/// Build the connection params JSON from MqAdminConfig for the Java agent.
/// Build the connection params JSON from MqAdminConfig for the native agent.
fn build_connection_params(cfg: &MqAdminConfig) -> serde_json::Value {
let extra = &cfg.extra;
let access_key = extra_str(extra, "accessKey")
docs/mq-quick-start.md
+4 -4
View File
@@ -378,7 +378,7 @@ cargo check --package dbx-web --features mq-admin
### 添加新的 MQ 系统
Kafka 与 RocketMQ 已实现 Java Agent + Rust 适配器模式。新增 MQ 系统时：
Kafka 使用 Java Agent，RocketMQ 使用 Go 原生 Agent；两者均通过 Rust 适配器接入。新增 MQ 系统时：
1. 在 `agents/drivers/<name>/` 实现 JSON-RPC Agent（参考 `kafka` / `rocketmq`）
2. 在 `crates/dbx-core/src/mq/adapters/` 实现 `MessageQueueAdmin`
@@ -407,9 +407,9 @@ Kafka 与 RocketMQ 已实现 Java Agent + Rust 适配器模式。新增 MQ 系
Agent 构建与安装：
```bash
cd agents
./gradlew :rocketmq:shadowJar
# 将 shadow JAR 安装到 DBX 数据目录 agents/drivers/rocketmq/agent.jar
cd agents/drivers/rocketmq
CGO_ENABLED=0 go build -trimpath -ldflags='-s -w' -o agent .
# 将可执行文件安装到 DBX 数据目录 agents/drivers/rocketmq/agent
```
Docker 快速启动（NameServer 9876 + Broker 10911，仅用于本地验证）：