github/dbx
4
0
Fork 0
mirror of https://github.com/t8y2/dbx.git synced 2026-10-21 17:09:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(etcd): rewrite agent in Go and add v2 API support

Browse Source
This commit is contained in:
t8y2
2026-09-02 18:04:25 +08:00
parent 41fc755135
commit 72dcd8355a
63 changed files with 8085 additions and 2333 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/scripts/bump-agent-versions.mjs
+6 -1
View File
@@ -61,11 +61,13 @@ const nativeDriverDirectories = {
rocketmq: "rocketmq",
zookeeper: "zookeeper",
tdengine: "tdengine",
etcd: "etcd-go",
etcd2: "etcd2-go",
};
const crateNativeDriverDirectories = {
"sqlite-worker": "crates/dbx-sqlite-worker",
};
const nativeDriverModules = new Set(["cassandra", "duckdb", "hive", "oracle", "xugu", "kingbase", "iotdb", "neo4j", "vastbase", "rabbitmq", "rocketmq", "zookeeper", "tdengine", "sqlite-worker"]);
const nativeDriverModules = new Set(["cassandra", "duckdb", "hive", "oracle", "xugu", "kingbase", "iotdb", "neo4j", "vastbase", "rabbitmq", "rocketmq", "zookeeper", "tdengine", "etcd", "etcd2", "sqlite-worker"]);
const nativeDriverSharedPaths = {
hive: [
"agents/go-common/go-gssapi",
@@ -76,6 +78,9 @@ const nativeDriverSharedPaths = {
"agents/go-common/go-gssapi",
"agents/go-common/gosasl",
],
etcd: [
"agents/go-common/go-semver",
],
};
function resolveAgentModule(moduleName, { legacyStandaloneModules, moduleExists, readModuleFile }) {
.github/scripts/reuse-agent-release-assets.mjs
+1 -1
View File
@@ -15,7 +15,7 @@ import { basename, join } from "node:path";
import { tmpdir } from "node:os";
const REGISTRY_ASSET = "agent-registry.json";
const NATIVE_MODULES = new Set(["duckdb", "oracle", "xugu", "kingbase", "iotdb", "neo4j", "vastbase", "rabbitmq", "rocketmq", "zookeeper", "tdengine"]);
const NATIVE_MODULES = new Set(["duckdb", "oracle", "xugu", "kingbase", "iotdb", "neo4j", "vastbase", "rabbitmq", "rocketmq", "zookeeper", "tdengine", "etcd", "etcd2"]);
const PLATFORMS = [
"macos-aarch64",
"macos-x64",
.github/workflows/agents-release.yml
+86 -2
View File
@@ -347,6 +347,86 @@ jobs:
name: rocketmq-native
path: "release-native/dbx-agent-rocketmq-*"
build-etcd-native:
needs: [bump-versions]
if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'etcd') }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: "1.26.x"
- name: Test etcd native agent
working-directory: agents/drivers/etcd-go
run: go test ./...
- name: Cross-compile etcd native agent
shell: bash
run: |
mkdir -p release-native
cd agents/drivers/etcd-go
declare -A TARGETS=(
["macos-aarch64"]="darwin/arm64"
["macos-x64"]="darwin/amd64"
["linux-aarch64"]="linux/arm64"
["linux-x64"]="linux/amd64"
["windows-aarch64"]="windows/arm64"
["windows-x64"]="windows/amd64"
)
for platform in "${!TARGETS[@]}"; do
IFS=/ read -r goos goarch <<< "${TARGETS[$platform]}"
output="../../../release-native/dbx-agent-etcd-${platform}"
if [[ "$goos" == "windows" ]]; then
output="${output}.exe"
fi
echo "Building $platform ($goos/$goarch)"
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -trimpath -ldflags="-s -w" -o "$output" .
done
ls -lh ../../../release-native
- uses: actions/upload-artifact@v4
with:
name: etcd-native
path: "release-native/dbx-agent-etcd-*"
build-etcd2-native:
needs: [bump-versions]
if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'etcd2') }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: "1.24.x"
- name: Test etcd v2 native agent
working-directory: agents/drivers/etcd2-go
run: go test ./...
- name: Cross-compile etcd v2 native agent
shell: bash
run: |
mkdir -p release-native
cd agents/drivers/etcd2-go
declare -A TARGETS=(
["macos-aarch64"]="darwin/arm64"
["macos-x64"]="darwin/amd64"
["linux-aarch64"]="linux/arm64"
["linux-x64"]="linux/amd64"
["windows-aarch64"]="windows/arm64"
["windows-x64"]="windows/amd64"
)
for platform in "${!TARGETS[@]}"; do
IFS=/ read -r goos goarch <<< "${TARGETS[$platform]}"
output="../../../release-native/dbx-agent-etcd2-${platform}"
if [[ "$goos" == "windows" ]]; then
output="${output}.exe"
fi
echo "Building $platform ($goos/$goarch)"
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -trimpath -ldflags="-s -w" -o "$output" .
done
ls -lh ../../../release-native
- uses: actions/upload-artifact@v4
with:
name: etcd2-native
path: "release-native/dbx-agent-etcd2-*"
build-zookeeper-native:
needs: [bump-versions]
if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'zookeeper') }}
@@ -1137,7 +1217,7 @@ jobs:
retention-days: 1
release:
needs: [bump-versions, commit-versions, build-agents, build-oracle-native, build-xugu-native, build-rabbitmq-native, build-rocketmq-native, build-zookeeper-native, build-cassandra-native, build-hive-native, build-kingbase-native, build-vastbase-native, build-neo4j-native, build-iotdb-native, build-duckdb-native, build-sqlite-worker-native, build-tdengine-native, build-jre, reuse-previous-assets]
needs: [bump-versions, commit-versions, build-agents, build-oracle-native, build-xugu-native, build-rabbitmq-native, build-rocketmq-native, build-etcd-native, build-etcd2-native, build-zookeeper-native, build-cassandra-native, build-hive-native, build-kingbase-native, build-vastbase-native, build-neo4j-native, build-iotdb-native, build-duckdb-native, build-sqlite-worker-native, build-tdengine-native, build-jre, reuse-previous-assets]
if: ${{ always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}
runs-on: ubuntu-latest
steps:
@@ -1272,6 +1352,8 @@ jobs:
neo4j) echo "Neo4j" ;;
iotdb) echo "Apache IoTDB" ;;
tdengine) echo "TDengine" ;;
etcd) echo "etcd" ;;
etcd2) echo "etcd 2.x (v2 API)" ;;
*) echo "$name" ;;
esac
}
@@ -1335,7 +1417,7 @@ jobs:
[ -n "$DRIVERS" ] && DRIVERS="${DRIVERS},"$'\n'
DRIVERS="${DRIVERS}$(generate_jar_entry "$name" "$label" "$f" "$jre_key" "$version" "$external_driver" "$native_json")"
done
for name in oracle xugu kingbase vastbase neo4j iotdb duckdb sqlite-worker rabbitmq rocketmq zookeeper cassandra hive tdengine; do
for name in oracle xugu kingbase vastbase neo4j iotdb duckdb sqlite-worker rabbitmq rocketmq zookeeper cassandra hive tdengine etcd etcd2; do
version=$(get_module_version "$name")
[ -f "release/dbx-agent-${name}-${version}.jar" ] && continue
native_json=$(generate_native_platforms "$name" "$version")
@@ -1426,6 +1508,8 @@ jobs:
neo4j) echo "Neo4j" ;;
iotdb) echo "Apache IoTDB" ;;
tdengine) echo "TDengine" ;;
etcd) echo "etcd" ;;
etcd2) echo "etcd 2.x (v2 API)" ;;
*) echo "$name" ;;
esac
}
.gitignore
+3
View File
@@ -8,6 +8,7 @@ node_modules
.trellis/
.worktrees/
openspec/
.zcode/
.mcp.json
AGENTS.md
CLAUDE.md
@@ -23,6 +24,8 @@ plugins/jdbc/target/
plugins/jdbc/dependency-reduced-pom.xml
plugins/jdbc/lib/*.jar
agents/drivers/duckdb/target/
agents/drivers/etcd-go/etcd-go
agents/drivers/etcd2-go/etcd2-go
plugins/jdbc/.gradle/
plugins/jdbc/build/
plugins/jdbc/dist/
agents/README.md
+6 -5
View File
@@ -45,7 +45,8 @@ Each agent runs as a standalone process and communicates with DBX via stdin/stdo
| yashandb | 崖山 YashanDB | YashanDB JDBC |
| xugu | 虚谷 XuguDB | XuguDB Go native agent |
| iotdb | Apache IoTDB | Apache IoTDB Go Client native agent |
| etcd | etcd | jetcd |
| etcd | etcd 3.3 degraded, 3.4+ full (v3 API) | etcd client/v3 native Go agent |
| etcd2 | etcd 2.0–2.3, 3.0–3.5 with --enable-v2 (v2 API) | etcd v2 HTTP/JSON native Go agent |
| zookeeper | Apache ZooKeeper | go-zookeeper native agent |
| rabbitmq | RabbitMQ | amqp091-go native agent |
| rocketmq | Apache RocketMQ | rocketmq-admin-go native agent |
@@ -53,7 +54,7 @@ Each agent runs as a standalone process and communicates with DBX via stdin/stdo
## Multi-JRE Support
Most Java agents target JRE 21. Native agents, such as `cassandra`, `duckdb`, `hive`, `iotdb`, `oracle`, `kingbase`, `tdengine`, `xugu`, `rabbitmq`, `rocketmq`, and `zookeeper`, do not require a JRE. DBX downloads and manages the JRE 21 installation automatically for Java agents.
Most Java agents target JRE 21. Native agents, such as `cassandra`, `duckdb`, `hive`, `iotdb`, `oracle`, `kingbase`, `tdengine`, `xugu`, `rabbitmq`, `rocketmq`, `zookeeper`, `etcd`, and `etcd2`, do not require a JRE. DBX downloads and manages the JRE 21 installation automatically for Java agents.
## JDBC Connection Pooling
@@ -79,7 +80,7 @@ Set `DBX_AGENT_JDBC_POOL_ENABLED=false` for a runtime-level compatibility fallba
For new agents, prefer a **native (Go or Rust) driver** over a Java/JDBC agent whenever a mature, license-compatible native driver is available. Native agents ship as a single self-contained executable with no JRE, which significantly reduces memory footprint and startup time — the JVM baseline that every Java agent pays even when idle is avoided entirely.
- **Native (C++/Go/Rust)** — preferred when a usable native driver exists. See `drivers/cassandra-go` (Apache cassandra-gocql-driver), `drivers/duckdb`, `drivers/hive-go` (native HS2), `drivers/iotdb` (Apache IoTDB Go Client), `drivers/oracle-go` (go-ora), `drivers/kingbase-go` (gokb), `drivers/vastbase-go` (openGauss connector), `drivers/tdengine` (taos-connector-rust), `drivers/xugu`, `drivers/rabbitmq` (amqp091-go), `drivers/rocketmq` (rocketmq-admin-go), and `drivers/zookeeper` (go-zookeeper) as reference implementations. No JRE download or management is needed.
- **Native (C++/Go/Rust)** — preferred when a usable native driver exists. See `drivers/cassandra-go` (Apache cassandra-gocql-driver), `drivers/duckdb`, `drivers/hive-go` (native HS2), `drivers/iotdb` (Apache IoTDB Go Client), `drivers/oracle-go` (go-ora), `drivers/kingbase-go` (gokb), `drivers/vastbase-go` (openGauss connector), `drivers/tdengine` (taos-connector-rust), `drivers/xugu`, `drivers/rabbitmq` (amqp091-go), `drivers/rocketmq` (rocketmq-admin-go), and `drivers/zookeeper` (go-zookeeper), `drivers/etcd-go` (etcd client/v3), and `drivers/etcd2-go` (etcd v2 HTTP API) as reference implementations. No JRE download or management is needed.
- **Java/JDBC** — the default fallback when only a JDBC driver exists for the database, or when the native driver is immature or unmaintained. Most agents still fall in this category.
Native agents implement the same JSON-RPC contract and `versions.json` registration as Java agents; they ship an `agent` executable instead of `agent.jar`. If both native and Java source implementations exist for the same database, publish only the native artifact unless the Java variant has a separately registered compatibility profile, such as `oracle-legacy` / `oracle-10g`.
@@ -103,7 +104,7 @@ Requires JDK 21 (Gradle toolchain auto-downloads if needed).
(cd drivers/zookeeper && go build -o agent .)
```
Output JARs are in `drivers/{module}/build/libs/`. Native agents build from `drivers/cassandra-go`, `drivers/duckdb`, `drivers/hive-go`, `drivers/iotdb`, `drivers/oracle-go`, `drivers/kingbase-go`, `drivers/vastbase-go`, `drivers/tdengine`, `drivers/xugu`, `drivers/rabbitmq`, `drivers/rocketmq`, and `drivers/zookeeper`.
Output JARs are in `drivers/{module}/build/libs/`. Native agents build from `drivers/cassandra-go`, `drivers/duckdb`, `drivers/hive-go`, `drivers/iotdb`, `drivers/oracle-go`, `drivers/kingbase-go`, `drivers/vastbase-go`, `drivers/tdengine`, `drivers/xugu`, `drivers/rabbitmq`, `drivers/rocketmq`, `drivers/zookeeper`, `drivers/etcd-go`, and `drivers/etcd2-go`.
### Local DBX Runtime Test
@@ -117,7 +118,7 @@ cp agents/drivers/<db_type>/build/libs/*-all.jar ~/.dbx/agents/drivers/<db_type>
Restart DBX or disconnect and reconnect the database so the new agent process loads the replacement JAR.
Native agents such as `cassandra`, `hive`, `iotdb`, `oracle`, `kingbase`, `tdengine`, `xugu`, `rabbitmq`, `rocketmq`, and `zookeeper` use an `agent` executable instead of `agent.jar`. TDengine builds `target/release/dbx-tdengine-driver` from `drivers/tdengine/Cargo.toml`.
Native agents such as `cassandra`, `hive`, `iotdb`, `oracle`, `kingbase`, `tdengine`, `xugu`, `rabbitmq`, `rocketmq`, `zookeeper`, `etcd`, and `etcd2` use an `agent` executable instead of `agent.jar`. TDengine builds `target/release/dbx-tdengine-driver` from `drivers/tdengine/Cargo.toml`.
## Versioning
agents/docs/agent-protocol-v2.md
+1 -1
View File
@@ -37,7 +37,7 @@ DBX uses unique short-lived logical sessions for independent Agent metadata task
Runtime reuse keys include the Agent driver key, executable or JAR path, launch arguments, working directory, JRE selection, JVM options, classpath-affecting options, and native executable version boundary. Host, account, schema, and credentials belong to sessions and are not part of the runtime key.
Etcd and ZooKeeper retain the legacy path because they use the key-value Agent protocol rather than the SQL session contract. Older Agent binaries and JARs also remain on the legacy path.
ZooKeeper retains the legacy single-session path because its Agent does not advertise the `multi_session` capability; etcd Agents (v3 and v2) run on the shared multi-session path like the SQL Agents. Older Agent binaries and JARs also remain on the legacy path.
## Resource limits and recovery
agents/drivers/etcd-go/MIGRATION_PARITY.md
+53
View File
@@ -0,0 +1,53 @@
# etcd Java to Go migration parity
Status date: 2026-09-02.
The Java `EtcdAgent` (jetcd 0.8.x, 33 protocol methods) was replaced by this
native Go agent. The migration is complete only where a capability is
implemented, covered by automated tests, validated against a real etcd cluster,
and wired into the native build and release path.
## Baseline
- DBX Java baseline: jetcd 0.8.x `EtcdAgent`; all protocol shapes, capability
strings, and error signal texts were copied verbatim.
- Go client: `go.etcd.io/etcd/client/v3` v3.7.1 over gRPC.
- Capabilities (exact Java set, no `structured_error_v1`): `connect`,
`test_connection`, `kv`, `kv_ttl`, `kv_cas`, `kv_list_values`, `kv_status`,
`kv_history`, `etcd_compaction`, `etcd_defrag`, `etcd_watch`, `etcd_lease`,
`etcd_auth`, `multi_session`.
- Error signal strings (`ETCD_CAS_CONFLICT`, `ETCD_NOT_FOUND`,
`ETCD_COMPACTED`, `ETCD_WATCH_LIMIT`, …) are matched verbatim because the
host normalizes agent errors by message text
(`agent_kv.rs::normalize_agent_kv_error`).
- The shared `go-semver` module is vendored under `agents/go-common/go-semver`
because the upstream `v0.3.1` tag declares a broken module path.
## Current matrix
| Capability | Go code | Automated | Live server | Status |
| --- | --- | --- | --- | --- |
| connect / endpoints / TLS / basic auth | yes | yes | etcd 3.7.0, 3.5.21 | parity passed (auth enabled) |
| test_connection probe (PERMISSION_DENIED → limited) | yes | yes | yes | parity passed |
| kv list/get/put/delete/rename with CAS | yes | yes | yes | parity passed |
| TTL + preserveLease (3-attempt retry) | yes | yes | yes | parity passed |
| Lease list/get/grant/keepalive/revoke | yes | yes | yes | parity passed |
| LeaseLeases fallback on 3.3 (UNIMPLEMENTED/timeout → partial) | yes | yes | n/a (3.3 fixture not retained) | coded + unit tested |
| LeaseGrant with custom ID (raw etcdserverpb RPC) | yes | yes | yes | replaces Java reflection wire handling |
| kv_history (temporary watch, prevKV, progress notify) | yes | yes | yes | requires `clientv3.WithCreatedNotify()`; jetcd parity |
| kv_history compacted/future-revision errors | yes | yes | yes | parity passed |
| Watch start/poll/stop with full budget system | yes | yes | yes | 256 batches / 10k events / 8MiB per watch, 16MiB session aggregate |
| Status: members, alarms, latency fan-out, member dedup | yes | yes | yes | parity passed |
| Status raft fields as unsigned decimal strings | yes | yes | yes | bit-level parity with Java `longString(int64(uint64))` |
| Compaction (pre-validation, error propagation) | yes | yes | yes | Java check order preserved (params before connect) |
| Defrag (followers first, leader last, per-endpoint errors) | yes | yes | yes | parity passed |
| Auth users/roles (14 methods) | yes | yes | yes | parity passed |
| Multi-session protocol runtime (256 sessions, cancel_session) | yes | yes | yes | protocol flow live test passed |
| etcd 3.3 degraded mode | partial | partial | no | lease/history fallbacks coded; no retained 3.3 fixture |
| etcd 2.x | n/a | n/a | n/a | served by the dedicated `etcd2` agent, not this one |
## Known divergences
- None intentional. All observable protocol outputs are byte-for-byte
compatible with the Java agent; the raw protobuf RPCs used for lease
operations are an internal implementation change only.
agents/drivers/etcd-go/auth.go
+261
View File
@@ -0,0 +1,261 @@
package main
import (
"encoding/json"
"fmt"
"strings"
clientv3 "go.etcd.io/etcd/client/v3"
)
func (s *etcdSession) authUserList(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
response, err := client.Auth.UserList(ctx)
if err != nil {
return nil, err
}
return map[string]any{"users": response.Users}, nil
}
func (s *etcdSession) authUserGet(params map[string]json.RawMessage) (any, error) {
user, err := requiredString(params, "user")
if err != nil {
return nil, err
}
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
response, err := client.Auth.UserGet(ctx, user)
if err != nil {
return nil, err
}
return map[string]any{"user": user, "roles": response.Roles}, nil
}
func (s *etcdSession) authUserAdd(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
user, err := requiredString(params, "user")
if err != nil {
return nil, err
}
password, err := requiredString(params, "password")
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
if _, err := client.Auth.UserAdd(ctx, user, password); err != nil {
return nil, err
}
return map[string]bool{"created": true}, nil
}
func (s *etcdSession) authUserDelete(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
user, err := requiredString(params, "user")
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
if _, err := client.Auth.UserDelete(ctx, user); err != nil {
return nil, err
}
return map[string]bool{"deleted": true}, nil
}
func (s *etcdSession) authUserChangePassword(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
user, err := requiredString(params, "user")
if err != nil {
return nil, err
}
password, err := requiredString(params, "password")
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
if _, err := client.Auth.UserChangePassword(ctx, user, password); err != nil {
return nil, err
}
return map[string]bool{"changed": true}, nil
}
func (s *etcdSession) authUserGrantRevokeRole(params map[string]json.RawMessage, grant bool) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
user, err := requiredString(params, "user")
if err != nil {
return nil, err
}
role, err := requiredString(params, "role")
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
if grant {
_, err = client.Auth.UserGrantRole(ctx, user, role)
} else {
_, err = client.Auth.UserRevokeRole(ctx, user, role)
}
if err != nil {
return nil, err
}
return map[string]bool{"updated": true}, nil
}
func (s *etcdSession) authRoleList(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
response, err := client.Auth.RoleList(ctx)
if err != nil {
return nil, err
}
return map[string]any{"roles": response.Roles}, nil
}
func (s *etcdSession) authRoleGet(params map[string]json.RawMessage) (any, error) {
role, err := requiredString(params, "role")
if err != nil {
return nil, err
}
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
response, err := client.Auth.RoleGet(ctx, role)
if err != nil {
return nil, err
}
permissions := []map[string]any{}
for _, permission := range response.Perm {
key := []byte(permission.Key)
rangeEnd := []byte(permission.RangeEnd)
resource := "prefix"
if len(key) == 1 && key[0] == 0 && len(rangeEnd) == 1 && rangeEnd[0] == 0 {
resource = "all"
} else if len(rangeEnd) == 0 {
resource = "key"
}
permissions = append(permissions, map[string]any{
"access": strings.ToLower(permission.PermType.String()),
"key": bytesObject(key),
"rangeEnd": bytesObject(rangeEnd),
"resource": resource,
})
}
return map[string]any{"role": role, "permissions": permissions}, nil
}
func (s *etcdSession) authRoleAdd(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
role, err := requiredString(params, "role")
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
if _, err := client.Auth.RoleAdd(ctx, role); err != nil {
return nil, err
}
return map[string]bool{"created": true}, nil
}
func (s *etcdSession) authRoleDelete(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
role, err := requiredString(params, "role")
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
if _, err := client.Auth.RoleDelete(ctx, role); err != nil {
return nil, err
}
return map[string]bool{"deleted": true}, nil
}
func (s *etcdSession) authRolePermission(params map[string]json.RawMessage, grant bool) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
role, err := requiredString(params, "role")
if err != nil {
return nil, err
}
resource := stringOrDefault(params, "resource", "key")
all := resource == "all"
prefix := resource == "prefix"
var key string
var rangeEnd string
if all {
key = "\x00"
rangeEnd = "\x00"
} else {
key, err = keyBytesParam(params)
if err != nil {
return nil, err
}
if prefix {
rangeEnd = prefixEnd(key)
}
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
if grant {
access, err := permissionType(stringOrDefault(params, "access", ""))
if err != nil {
return nil, err
}
if _, err := client.Auth.RoleGrantPermission(ctx, role, key, rangeEnd, access); err != nil {
return nil, err
}
} else {
if _, err := client.Auth.RoleRevokePermission(ctx, role, key, rangeEnd); err != nil {
return nil, err
}
}
return map[string]bool{"updated": true}, nil
}
func permissionType(access string) (clientv3.PermissionType, error) {
permission, err := clientv3.StrToPermissionType(strings.ToUpper(access))
if err != nil {
return 0, fmt.Errorf("ETCD_INVALID_ACCESS: access must be READ, WRITE, or READWRITE, got %s", access)
}
return permission, nil
}
agents/drivers/etcd-go/client.go
+321
View File
@@ -0,0 +1,321 @@
package main
import (
"context"
"crypto/tls"
"crypto/x509"
"encoding/json"
"errors"
"fmt"
"net/url"
"os"
"strconv"
"strings"
"time"
clientv3 "go.etcd.io/etcd/client/v3"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
const (
defaultGrpcMaxInboundMessageSize = 32 * 1024 * 1024
minGrpcMaxInboundMessageSize = 1024 * 1024
maxGrpcMaxInboundMessageSize = 256 * 1024 * 1024
grpcMaxInboundMessageSizeKey = "grpc_max_inbound_message_size"
defaultPort = 2379
)
// connectionParams mirrors the host-side etcd connection configuration.
type connectionParams struct {
EtcdEndpoints string `json:"etcd_endpoints"`
Endpoints string `json:"endpoints"`
ConnectionString string `json:"connection_string"`
Host string `json:"host"`
Port int `json:"port"`
Username string `json:"username"`
Password string `json:"password"`
SSL bool `json:"ssl"`
CACertPath string `json:"ca_cert_path"`
ClientCertPath string `json:"client_cert_path"`
ClientKeyPath string `json:"client_key_path"`
CertPath string `json:"cert_path"`
KeyPath string `json:"key_path"`
ConnectTimeoutSecs int `json:"connect_timeout_secs"`
GrpcMaxInboundMessageSize int `json:"grpc_max_inbound_message_size"`
URLParams string `json:"url_params"`
}
func connectionObject(params map[string]json.RawMessage) (connectionParams, error) {
var connection connectionParams
if raw, ok := params["connection"]; ok && raw != nil {
if err := json.Unmarshal(raw, &connection); err != nil {
return connection, err
}
return connection, nil
}
if err := decodeParams(params, &connection); err != nil {
return connection, err
}
return connection, nil
}
func (s *etcdSession) connectMap(params map[string]json.RawMessage) (any, error) {
if _, err := s.connect(params); err != nil {
return nil, err
}
return map[string]bool{"ok": true}, nil
}
func (s *etcdSession) connect(params map[string]json.RawMessage) (any, error) {
connection, err := connectionObject(params)
if err != nil {
return nil, err
}
nextClient, err := buildClient(connection)
if err != nil {
return nil, err
}
endpointList := connectionEndpoints(connection)
if _, err := probeClient(nextClient, endpointList); err != nil {
_ = nextClient.Close()
return nil, err
}
s.close()
s.clientMu.Lock()
s.client = nextClient
s.connectedEndpoints = endpointList
s.clientMu.Unlock()
return map[string]bool{"ok": true}, nil
}
func (s *etcdSession) activeClient() (*clientv3.Client, error) {
s.clientMu.Lock()
client := s.client
s.clientMu.Unlock()
if client == nil {
return nil, errors.New("Not connected")
}
return client, nil
}
func (s *etcdSession) connectedEndpointList() []string {
s.clientMu.Lock()
defer s.clientMu.Unlock()
return append([]string(nil), s.connectedEndpoints...)
}
func (s *etcdSession) validateConnection() (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
return probeClient(client, s.connectedEndpointList())
}
func (s *etcdSession) close() error {
s.watchesMu.Lock()
watches := s.watches
s.watches = map[string]*watchState{}
s.knownLeases = map[uint64]struct{}{}
s.watchBufferedBytes = 0
s.watchesMu.Unlock()
for _, watch := range watches {
watch.close()
}
s.clientMu.Lock()
client := s.client
s.client = nil
s.connectedEndpoints = nil
s.clientMu.Unlock()
if client != nil {
_ = client.Close()
}
return nil
}
func buildClient(connection connectionParams) (*clientv3.Client, error) {
endpoints := connectionEndpoints(connection)
config := clientv3.Config{
Endpoints: endpoints,
DialTimeout: time.Duration(connectTimeoutSeconds(connection)) * time.Second,
}
if size := grpcMaxInboundMessageSize(connection); size > 0 {
config.MaxCallSendMsgSize = size
config.MaxCallRecvMsgSize = size
}
if strings.TrimSpace(connection.Username) != "" {
config.Username = connection.Username
config.Password = connection.Password
}
if connection.SSL {
tlsConfig, err := tlsConfigFor(connection)
if err != nil {
return nil, err
}
config.TLS = tlsConfig
}
return clientv3.New(config)
}
func connectTimeoutSeconds(connection connectionParams) int {
seconds := connection.ConnectTimeoutSecs
if seconds == 0 {
seconds = rpcTimeoutSeconds
}
if seconds < 1 {
seconds = 1
}
if seconds > 300 {
seconds = 300
}
return seconds
}
func grpcMaxInboundMessageSize(connection connectionParams) int {
configured := connection.GrpcMaxInboundMessageSize
if configured == 0 {
configured = intURLParamOrDefault(connection.URLParams, grpcMaxInboundMessageSizeKey, defaultGrpcMaxInboundMessageSize)
}
if configured < minGrpcMaxInboundMessageSize {
configured = minGrpcMaxInboundMessageSize
}
if configured > maxGrpcMaxInboundMessageSize {
configured = maxGrpcMaxInboundMessageSize
}
return configured
}
func intURLParamOrDefault(params, key string, fallback int) int {
if strings.TrimSpace(params) == "" {
return fallback
}
for _, entry := range strings.Split(strings.TrimPrefix(params, "?"), "&") {
separator := strings.Index(entry, "=")
entryKey := entry
if separator >= 0 {
entryKey = entry[:separator]
}
if entryKey != key {
continue
}
if separator < 0 {
return fallback
}
value, err := strconv.Atoi(entry[separator+1:])
if err != nil {
return fallback
}
return value
}
return fallback
}
func tlsConfigFor(connection connectionParams) (*tls.Config, error) {
tlsConfig := &tls.Config{}
if ca := strings.TrimSpace(connection.CACertPath); ca != "" {
authorityPEM, err := os.ReadFile(ca)
if err != nil {
return nil, err
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(authorityPEM) {
return nil, fmt.Errorf("failed to parse CA certificate at %s", ca)
}
tlsConfig.RootCAs = pool
}
certPath := firstNonBlank(connection.ClientCertPath, connection.CertPath)
keyPath := firstNonBlank(connection.ClientKeyPath, connection.KeyPath)
if (certPath == "") != (keyPath == "") {
return nil, errors.New("Client certificate and key must be provided together")
}
if certPath != "" {
pair, err := tls.LoadX509KeyPair(certPath, keyPath)
if err != nil {
return nil, err
}
tlsConfig.Certificates = []tls.Certificate{pair}
}
return tlsConfig, nil
}
func probeClient(client *clientv3.Client, endpoints []string) (map[string]any, error) {
var lastFailure error
for _, endpoint := range endpoints {
ctx, cancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
_, err := client.Maintenance.Status(ctx, endpoint)
cancel()
if err == nil {
return map[string]any{"ok": true, "endpoint": endpoint}, nil
}
// A restricted etcd user may not be allowed to call Maintenance.Status.
// PERMISSION_DENIED still proves that the channel reached an etcd server.
if status.Code(err) == codes.PermissionDenied {
return map[string]any{"ok": true, "endpoint": endpoint, "limited": true}, nil
}
lastFailure = err
}
if lastFailure != nil {
return nil, lastFailure
}
return nil, errors.New("No etcd endpoint configured")
}
func connectionEndpoints(connection connectionParams) []string {
configured := firstNonBlank(connection.EtcdEndpoints, connection.Endpoints, connection.ConnectionString)
var result []string
if configured != "" {
for _, endpoint := range strings.FieldsFunc(configured, func(r rune) bool { return r == ',' || r == '\n' }) {
normalized := normalizeEndpoint(strings.TrimSpace(endpoint), connection.SSL)
if normalized != "" {
result = append(result, normalized)
}
}
}
if len(result) == 0 {
host := connection.Host
if host == "" {
host = "127.0.0.1"
}
port := connection.Port
if port == 0 {
port = defaultPort
}
result = append(result, normalizeEndpoint(fmt.Sprintf("%s:%d", host, port), connection.SSL))
}
return result
}
func normalizeEndpoint(endpoint string, tlsEnabled bool) string {
if strings.TrimSpace(endpoint) == "" {
return ""
}
if strings.HasPrefix(endpoint, "http://") || strings.HasPrefix(endpoint, "https://") {
return endpoint
}
if strings.HasPrefix(endpoint, "unix://") || strings.HasPrefix(endpoint, "unixs://") {
return endpoint
}
scheme := "http"
if tlsEnabled {
scheme = "https"
}
return scheme + "://" + endpoint
}
func endpointHost(rawEndpoint string) string {
parsed, err := url.Parse(rawEndpoint)
if err != nil {
return ""
}
return parsed.Hostname()
}
func firstNonBlank(values ...string) string {
for _, value := range values {
if strings.TrimSpace(value) != "" {
return value
}
}
return ""
}
agents/drivers/etcd-go/go.mod
+34
View File
@@ -0,0 +1,34 @@
module github.com/t8y2/dbx/agents/drivers/etcd-go
go 1.26
// The v0.3.1 tag of coreos/go-semver declares the root module without the
// /semver suffix, so the nested module path etcd requires no longer resolves
// upstream. Vendored under go-common from the v0.3.1 tag, package unchanged.
replace github.com/coreos/go-semver/semver => ../../go-common/go-semver
// The pseudo-version recorded in etcd's go.mod predates second-precision
// timestamps and is rejected by modern Go; point at the corrected form.
replace github.com/modern-go/concurrent => github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd
require (
go.etcd.io/etcd/api/v3 v3.7.1
go.etcd.io/etcd/client/v3 v3.7.1
google.golang.org/grpc v1.82.1
)
require (
github.com/coreos/go-semver/semver v0.3.1 // indirect
github.com/coreos/go-systemd/v22 v22.7.0 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.7.1 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.1 // indirect
golang.org/x/net v0.55.0 // indirect
golang.org/x/sys v0.45.0 // indirect
golang.org/x/text v0.37.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect
google.golang.org/protobuf v1.36.11 // indirect
)
agents/drivers/etcd-go/go.sum
+64
View File
@@ -0,0 +1,64 @@
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA=
github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
go.etcd.io/etcd/api/v3 v3.7.1 h1:KJG0/DcWGfe3Y1otDf/fsBf0TSSgpxZ5RO/L8SFt73E=
go.etcd.io/etcd/api/v3 v3.7.1/go.mod h1:8bXIpCMeV7E3/XL0Ix123ATn3dB+0V7d9zklHbB0m78=
go.etcd.io/etcd/client/pkg/v3 v3.7.1 h1:rKYsj3pRkR0eK3yjT3XOgrhqfmIfj9pzNgxjh7mfFv4=
go.etcd.io/etcd/client/pkg/v3 v3.7.1/go.mod h1:cnzZGIUzSfjEwLC6UBVsSXlEK1eepS/JUD7wE6PLRT0=
go.etcd.io/etcd/client/v3 v3.7.1 h1:0PEMMC0KuZmVIN+RAbdqfkZ45pYTgKVtmBEbRCvZFUg=
go.etcd.io/etcd/client/v3 v3.7.1/go.mod h1:ffNqALa8tRCYhYo1F9oR489y23K39Gz+BSR3ApAGYq0=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM=
go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg=
go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw=
go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A=
go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A=
go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc=
go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 h1:yQugLulqltosq0B/f8l4w9VryjV+N/5gcW0jQ3N8Qec=
google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478/go.mod h1:C6ADNqOxbgdUUeRTU+LCHDPB9ttAMCTff6auwCVa4uc=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE=
google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
agents/drivers/etcd-go/history.go
+255
View File
@@ -0,0 +1,255 @@
package main
import (
"context"
"encoding/json"
"errors"
"sort"
"strconv"
"sync"
"time"
"go.etcd.io/etcd/api/v3/mvccpb"
"go.etcd.io/etcd/api/v3/v3rpc/rpctypes"
clientv3 "go.etcd.io/etcd/client/v3"
)
const (
historyDefaultRevisionWindow = int64(10000)
historyLimitMax = 500
)
type historyCollector struct {
mu sync.Mutex
rows []map[string]any
limit int
truncated bool
}
func (c *historyCollector) append(row map[string]any) {
c.mu.Lock()
defer c.mu.Unlock()
if len(c.rows) == c.limit {
c.rows = c.rows[1:]
c.truncated = true
}
c.rows = append(c.rows, row)
}
func (s *etcdSession) history(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
key, err := keyBytesParam(params)
if err != nil {
return nil, err
}
limit := intOrDefault(params, "limit", 100)
if limit < 1 {
limit = 1
}
if limit > historyLimitMax {
limit = historyLimitMax
}
requestedEnd := longOrNull(params, "endRevision")
ctx, cancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
latestOptions := []clientv3.OpOption{}
if requestedEnd != nil && *requestedEnd > 0 {
latestOptions = append(latestOptions, clientv3.WithRev(*requestedEnd))
}
latest, err := client.Get(ctx, key, latestOptions...)
cancel()
if err != nil {
if errors.Is(err, rpctypes.ErrCompacted) {
if revision, ok := compactedRevisionOf(client, key); ok {
return nil, errors.New("ETCD_COMPACTED: requested history was compacted at revision " + longString(revision))
}
return nil, errors.New("ETCD_COMPACTED: requested history was compacted")
}
return nil, err
}
var endRevision int64
if requestedEnd != nil {
endRevision = *requestedEnd
} else {
endRevision = latest.Header.Revision
}
var targetKeyRevision int64
if len(latest.Kvs) == 0 {
targetKeyRevision = endRevision
} else {
targetKeyRevision = latest.Kvs[0].ModRevision
}
requestedStart := longOrNull(params, "startRevision")
startRevision := historyStartRevision(requestedStart, targetKeyRevision)
if startRevision > targetKeyRevision {
return map[string]any{
"events": []any{},
"observedRevision": longString(endRevision),
"truncated": false,
}, nil
}
collector := &historyCollector{limit: limit, truncated: requestedStart == nil && startRevision > 1}
created := make(chan struct{})
var createdOnce sync.Once
completed := make(chan struct{})
var completedOnce sync.Once
var failure error
var failureMu sync.Mutex
watchCtx, watchCancel := context.WithCancel(context.Background())
channel := client.Watch(watchCtx, key,
clientv3.WithRev(startRevision),
clientv3.WithPrevKV(),
clientv3.WithProgressNotify(),
// clientv3 only posts the created response when this option is set;
// the created gate relies on it (jetcd withCreateNotify parity).
clientv3.WithCreatedNotify(),
)
go func() {
for response := range channel {
if response.Err() != nil {
failureMu.Lock()
failure = response.Err()
failureMu.Unlock()
completedOnce.Do(func() { close(completed) })
continue
}
if response.Canceled {
if response.CompactRevision > 0 {
err := errors.New("ETCD_COMPACTED: requested history was compacted at revision " + longString(response.CompactRevision))
failureMu.Lock()
failure = err
failureMu.Unlock()
}
completedOnce.Do(func() { close(completed) })
continue
}
createdOnce.Do(func() { close(created) })
for _, event := range response.Events {
item := event.Kv
revision := item.ModRevision
if revision > endRevision {
continue
}
var previous *mvccpb.KeyValue
if event.PrevKv != nil {
previous = event.PrevKv
}
row := map[string]any{
"eventType": eventType(event),
"revision": longString(revision),
}
if event.Type == mvccpb.DELETE {
row["value"] = nil
} else {
row["value"] = valueObject(item.Value)
}
if previous != nil && previous.Version > 0 {
row["previousValue"] = valueObject(previous.Value)
} else {
row["previousValue"] = nil
}
if event.Type == mvccpb.DELETE && previous != nil {
row["metadata"] = metadataMap(previous)
} else {
row["metadata"] = metadataMap(item)
}
collector.append(row)
if revision >= targetKeyRevision {
completedOnce.Do(func() { close(completed) })
}
}
if response.IsProgressNotify() && response.Header.Revision >= endRevision {
completedOnce.Do(func() { close(completed) })
}
}
completedOnce.Do(func() { close(completed) })
}()
defer watchCancel()
select {
case <-created:
case <-time.After(5 * time.Second):
return nil, errors.New("ETCD_HISTORY_TIMEOUT: watcher was not created")
}
// For an existing exact key, its latest mod revision is an explicit
// replay boundary. This avoids relying on progress notifications,
// which older etcd/jetcd combinations do not consistently emit.
progressCtx, progressCancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
_ = client.RequestProgress(progressCtx)
progressCancel()
select {
case <-completed:
case <-time.After(15 * time.Second):
return nil, errors.New("ETCD_HISTORY_TIMEOUT: history replay did not reach the requested revision")
}
failureMu.Lock()
historyFailure := failure
failureMu.Unlock()
if historyFailure != nil {
if errors.Is(historyFailure, rpctypes.ErrCompacted) {
if revision, ok := compactedRevisionOf(client, key); ok {
return nil, errors.New("ETCD_COMPACTED: requested history was compacted at revision " + longString(revision))
}
return nil, errors.New("ETCD_COMPACTED: requested history was compacted")
}
return nil, errors.New("ETCD_HISTORY_FAILED: " + historyFailure.Error())
}
collector.mu.Lock()
rows := make([]map[string]any, len(collector.rows))
copy(rows, collector.rows)
truncated := collector.truncated
collector.mu.Unlock()
sort.SliceStable(rows, func(i, j int) bool {
a, _ := strconv.ParseInt(rows[i]["revision"].(string), 10, 64)
b, _ := strconv.ParseInt(rows[j]["revision"].(string), 10, 64)
return a > b
})
serialized := make([]any, len(rows))
for i, row := range rows {
serialized[i] = row
}
return map[string]any{
"events": serialized,
"observedRevision": longString(endRevision),
"truncated": truncated,
}, nil
}
func historyStartRevision(requestedStart *int64, targetRevision int64) int64 {
if requestedStart != nil {
if *requestedStart < 1 {
return 1
}
return *requestedStart
}
start := targetRevision - historyDefaultRevisionWindow + 1
if start < 1 {
return 1
}
return start
}
// compactedRevisionOf recovers the compaction boundary that gRPC errors do not
// carry: a watch from revision 1 is cancelled immediately with the compacted
// revision in its response.
func compactedRevisionOf(client *clientv3.Client, key string) (int64, bool) {
watchCtx, watchCancel := context.WithTimeout(context.Background(), 5*time.Second)
defer watchCancel()
channel := client.Watch(watchCtx, key, clientv3.WithRev(1))
select {
case response := <-channel:
if response.CompactRevision > 0 {
return response.CompactRevision, true
}
return 0, false
case <-watchCtx.Done():
return 0, false
}
}
agents/drivers/etcd-go/integration_test.go
+515
View File
@@ -0,0 +1,515 @@
package main
import (
"encoding/json"
"fmt"
"os"
"strings"
"testing"
"time"
)
// TestLiveEtcdAgent exercises the full agent surface against a real etcd
// server. Enable with DBX_ETCD_LIVE=1; configure via DBX_ETCD_ENDPOINTS,
// DBX_ETCD_USER, and DBX_ETCD_PASSWORD (defaults match the deploy recipes:
// root/123456 on 127.0.0.1:10700 for 3.7).
func TestLiveEtcdAgent(t *testing.T) {
if os.Getenv("DBX_ETCD_LIVE") != "1" {
t.Skip("set DBX_ETCD_LIVE=1 to run the live etcd agent test")
}
endpoints := envOrDefault("DBX_ETCD_ENDPOINTS", "http://127.0.0.1:10700")
user := envOrDefault("DBX_ETCD_USER", "root")
password := envOrDefault("DBX_ETCD_PASSWORD", "123456")
state := newEtcdSession()
connectParams := map[string]json.RawMessage{
"etcd_endpoints": json.RawMessage(`"` + endpoints + `"`),
"username": json.RawMessage(`"` + user + `"`),
"password": json.RawMessage(`"` + password + `"`),
}
if _, err := state.connect(connectParams); err != nil {
t.Fatalf("connect failed: %v", err)
}
defer state.close()
if result, err := state.validateConnection(); err != nil {
t.Fatalf("validate_connection failed: %v", err)
} else {
probe := result.(map[string]any)
if probe["ok"] != true {
t.Fatalf("unexpected probe result: %#v", probe)
}
}
prefix := fmt.Sprintf("dbx/live/%d/", time.Now().UnixNano())
// --- KV basics -----------------------------------------------------
putResult, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "hello"),
"value": jsonValue("world"),
}))
if err != nil {
t.Fatalf("put failed: %v", err)
}
baseRevision := revisionOf(t, putResult)
binaryEncoded := base64Encode([]byte{0x00, 0xff, 0x81})
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "binary"),
"value": json.RawMessage(`{"encoding":"base64","data":"` + binaryEncoded + `"}`),
})); err != nil {
t.Fatalf("binary put failed: %v", err)
}
fetched, err := state.get(paramsWith(map[string]json.RawMessage{"key": jsonString(prefix + "hello")}))
if err != nil {
t.Fatalf("get failed: %v", err)
}
row := fetched.(map[string]any)
if row["found"] != true || row["key"] != prefix+"hello" {
t.Fatalf("unexpected get result: %#v", row)
}
value := row["value"].(map[string]any)
if value["encoding"] != "utf8" || value["data"] != "world" {
t.Fatalf("unexpected value: %#v", value)
}
metadata := row["metadata"].(map[string]any)
if metadata["createRevision"] != metadata["modRevision"] {
t.Fatalf("fresh key revisions should match: %#v", metadata)
}
binaryRow, err := state.get(paramsWith(map[string]json.RawMessage{"key": jsonString(prefix + "binary")}))
if err != nil {
t.Fatalf("binary get failed: %v", err)
}
binaryValue := binaryRow.(map[string]any)["value"].(map[string]any)
if binaryValue["encoding"] != "base64" || binaryValue["data"] != binaryEncoded {
t.Fatalf("binary round trip broken: %#v", binaryValue)
}
// CAS conflict on stale revision.
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "hello"),
"value": jsonValue("v2"),
"expectedModRevision": json.RawMessage(`1`),
})); err == nil || err.Error() != "ETCD_CAS_CONFLICT: key changed after it was loaded" {
t.Fatalf("expected CAS conflict, got %v", err)
}
// CAS success with the fresh revision.
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "hello"),
"value": jsonValue("v3"),
"expectedModRevision": json.RawMessage(metadata["modRevision"].(string)),
})); err != nil {
t.Fatalf("CAS put failed: %v", err)
}
// TTL put, lease metadata, and preserveLease.
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "leased"),
"value": jsonValue("expiring"),
"ttl": json.RawMessage(`300`),
})); err != nil {
t.Fatalf("ttl put failed: %v", err)
}
leased, err := state.get(paramsWith(map[string]json.RawMessage{"key": jsonString(prefix + "leased")}))
if err != nil {
t.Fatalf("leased get failed: %v", err)
}
leasedMetadata := leased.(map[string]any)["metadata"].(map[string]any)
if _, hasTTL := leasedMetadata["ttl"]; !hasTTL {
t.Fatalf("leased key must expose ttl metadata: %#v", leasedMetadata)
}
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "leased"),
"value": jsonValue("renewed"),
"preserveLease": json.RawMessage(`true`),
})); err != nil {
t.Fatalf("preserveLease put failed: %v", err)
}
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "hello"),
"value": jsonValue("nope"),
"preserveLease": json.RawMessage(`true`),
})); err == nil || err.Error() != "Cannot preserve lease: key does not exist or has no lease" {
t.Fatalf("expected preserveLease rejection, got %v", err)
}
// Rename, including conflict and missing-source paths.
if _, err := state.rename(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "old"),
"newKey": jsonString(prefix + "new"),
})); err == nil || err.Error() != "ETCD_NOT_FOUND: source key does not exist" {
t.Fatalf("expected rename not-found, got %v", err)
}
if _, err := state.rename(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "hello"),
"newKey": jsonString(prefix + "renamed"),
})); err != nil {
t.Fatalf("rename failed: %v", err)
}
if _, err := state.rename(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "binary"),
"newKey": jsonString(prefix + "renamed"),
})); err == nil || err.Error() != "ETCD_CAS_CONFLICT: source changed or target already exists" {
t.Fatalf("expected rename conflict, got %v", err)
}
// list_prefix with continuation.
listed, err := state.listPrefix(paramsWith(map[string]json.RawMessage{
"prefix": jsonString(prefix),
"limit": json.RawMessage(`2`),
"includeValues": json.RawMessage(`true`),
}))
if err != nil {
t.Fatalf("list_prefix failed: %v", err)
}
list := listed.(map[string]any)
if len(list["keys"].([]any)) != 2 {
t.Fatalf("expected limited list, got %#v", list)
}
continuation, ok := list["continuation"].(string)
if !ok || continuation == "" {
t.Fatalf("expected continuation, got %#v", list)
}
nextPage, err := state.listPrefix(paramsWith(map[string]json.RawMessage{
"prefix": jsonString(prefix),
"limit": json.RawMessage(`100`),
"includeValues": json.RawMessage(`true`),
"continuation": json.RawMessage(`"` + continuation + `"`),
}))
if err != nil {
t.Fatalf("continuation list failed: %v", err)
}
nextKeys := nextPage.(map[string]any)["keys"].([]any)
if len(nextKeys) == 0 {
t.Fatalf("continuation must resume after the first page")
}
firstContinued := nextKeys[0].(map[string]any)["key"].(string)
if firstContinued == list["keys"].([]any)[1].(map[string]any)["key"].(string) {
t.Fatalf("continuation overlapped the first page")
}
// --- history -------------------------------------------------------
// Explicit startRevision keeps the window above any revision compacted by
// earlier verification runs (the flow itself compacts at the end).
history, err := state.history(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "renamed"),
"startRevision": json.RawMessage(longString(baseRevision)),
}))
if err != nil {
t.Fatalf("history failed: %v", err)
}
historyRow := history.(map[string]any)
events := historyRow["events"].([]any)
// The renamed key only exists from the rename revision onward; the old
// key's earlier revisions belong to the old key.
if len(events) != 1 {
t.Fatalf("expected exactly the rename event, got %#v", historyRow)
}
if historyRow["truncated"] != false {
t.Fatalf("history should not be truncated for a short window: %#v", historyRow)
}
firstEvent := events[0].(map[string]any)
if _, err := strconvParse(firstEvent["revision"].(string)); err != nil {
t.Fatalf("revisions must be strings: %#v", firstEvent)
}
// --- watch ---------------------------------------------------------
watchStarted, err := state.watchStart(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "watched"),
"includePrevKv": json.RawMessage(`true`),
}))
if err != nil {
t.Fatalf("watch_start failed: %v", err)
}
watchID := watchStarted.(map[string]any)["watchId"].(string)
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "watched"),
"value": jsonValue("first"),
})); err != nil {
t.Fatalf("watched put failed: %v", err)
}
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "watched"),
"value": jsonValue("second"),
})); err != nil {
t.Fatalf("watched put 2 failed: %v", err)
}
deadline := time.Now().Add(10 * time.Second)
var watchBatches []any
for time.Now().Before(deadline) {
polled, err := state.watchPoll(paramsWith(map[string]json.RawMessage{"watchId": json.RawMessage(`"` + watchID + `"`)}))
if err != nil {
t.Fatalf("watch_poll failed: %v", err)
}
batches := polled.(map[string]any)["batches"].([]any)
watchBatches = append(watchBatches, batches...)
if len(watchBatches) >= 2 {
break
}
time.Sleep(100 * time.Millisecond)
}
if len(watchBatches) < 2 {
t.Fatalf("watch did not deliver both events: %#v", watchBatches)
}
secondBatch := watchBatches[len(watchBatches)-1].(map[string]any)
secondEvent := secondBatch["events"].([]map[string]any)[0]
if secondEvent["previousValue"].(map[string]any)["data"] != "first" {
t.Fatalf("prevKv missing on watch event: %#v", secondEvent)
}
stopped, err := state.watchStop(paramsWith(map[string]json.RawMessage{"watchId": json.RawMessage(`"` + watchID + `"`)}))
if err != nil || stopped.(map[string]bool)["stopped"] != true {
t.Fatalf("watch_stop failed: %#v %v", stopped, err)
}
if _, err := state.watchPoll(paramsWith(map[string]json.RawMessage{"watchId": json.RawMessage(`"` + watchID + `"`)})); err == nil || err.Error() != "ETCD_WATCH_NOT_FOUND: watch does not exist" {
t.Fatalf("expected watch-not-found after stop, got %v", err)
}
// --- lease ---------------------------------------------------------
granted, err := state.leaseGrant(paramsWith(map[string]json.RawMessage{"ttl": json.RawMessage(`120`)}))
if err != nil {
t.Fatalf("lease_grant failed: %v", err)
}
leaseID := granted.(map[string]any)["id"].(string)
if _, err := state.leaseKeepAlive(paramsWith(map[string]json.RawMessage{"id": json.RawMessage(leaseID)})); err != nil {
t.Fatalf("lease_keepalive_once failed: %v", err)
}
leaseListed, err := state.leaseList(paramsWith(map[string]json.RawMessage{}))
if err != nil {
t.Fatalf("lease_list failed: %v", err)
}
leaseListRow := leaseListed.(map[string]any)
if leaseListRow["partial"] != false {
t.Fatalf("lease list must be complete on etcd >= 3.4: %#v", leaseListRow)
}
foundLease := false
for _, lease := range leaseListRow["leases"].([]map[string]any) {
if lease["id"] == leaseID {
foundLease = true
}
}
if !foundLease {
t.Fatalf("granted lease %s missing from list: %#v", leaseID, leaseListRow)
}
leaseDetail, err := state.leaseGet(paramsWith(map[string]json.RawMessage{
"id": json.RawMessage(leaseID),
"includeKeys": json.RawMessage(`true`),
}))
if err != nil {
t.Fatalf("lease_get failed: %v", err)
}
detail := leaseDetail.(map[string]any)
if detail["grantedTtl"].(int64) != 120 && detail["grantedTtl"].(int64) < 100 {
t.Fatalf("unexpected granted ttl: %#v", detail)
}
if _, err := state.leaseRevoke(paramsWith(map[string]json.RawMessage{"id": json.RawMessage(leaseID)})); err != nil {
t.Fatalf("lease_revoke failed: %v", err)
}
// --- auth ----------------------------------------------------------
userName := fmt.Sprintf("dbx_live_%d", time.Now().UnixNano()%100000)
if _, err := state.authUserAdd(paramsWith(map[string]json.RawMessage{
"user": jsonString(userName),
"password": jsonString("secret123"),
})); err != nil {
t.Fatalf("user add failed: %v", err)
}
if _, err := state.authRoleAdd(paramsWith(map[string]json.RawMessage{"role": jsonString("dbx_live_role")})); err != nil {
t.Fatalf("role add failed: %v", err)
}
if _, err := state.authRolePermission(paramsWith(map[string]json.RawMessage{
"role": jsonString("dbx_live_role"),
"resource": jsonString("prefix"),
"key": jsonString(prefix),
"access": jsonString("READWRITE"),
}), true); err != nil {
t.Fatalf("role grant permission failed: %v", err)
}
roleDetail, err := state.authRoleGet(paramsWith(map[string]json.RawMessage{"role": jsonString("dbx_live_role")}))
if err != nil {
t.Fatalf("role get failed: %v", err)
}
permissions := roleDetail.(map[string]any)["permissions"].([]map[string]any)
if len(permissions) != 1 || permissions[0]["access"] != "readwrite" || permissions[0]["resource"] != "prefix" {
t.Fatalf("unexpected permission shape: %#v", permissions)
}
if _, err := state.authUserGrantRevokeRole(paramsWith(map[string]json.RawMessage{
"user": jsonString(userName),
"role": jsonString("dbx_live_role"),
}), true); err != nil {
t.Fatalf("grant role failed: %v", err)
}
userDetail, err := state.authUserGet(paramsWith(map[string]json.RawMessage{"user": jsonString(userName)}))
if err != nil {
t.Fatalf("user get failed: %v", err)
}
if len(userDetail.(map[string]any)["roles"].([]string)) == 0 {
t.Fatalf("expected granted role: %#v", userDetail)
}
users, err := state.authUserList(paramsWith(map[string]json.RawMessage{}))
if err != nil {
t.Fatalf("user list failed: %v", err)
}
if !containsAny(users.(map[string]any)["users"].([]string), userName) {
t.Fatalf("user list missing created user: %#v", users)
}
if _, err := state.authUserGrantRevokeRole(paramsWith(map[string]json.RawMessage{
"user": jsonString(userName),
"role": jsonString("dbx_live_role"),
}), false); err != nil {
t.Fatalf("revoke role failed: %v", err)
}
if _, err := state.authRolePermission(paramsWith(map[string]json.RawMessage{
"role": jsonString("dbx_live_role"),
"resource": jsonString("prefix"),
"key": jsonString(prefix),
}), false); err != nil {
t.Fatalf("role revoke permission failed: %v", err)
}
if _, err := state.authUserDelete(paramsWith(map[string]json.RawMessage{"user": jsonString(userName)})); err != nil {
t.Fatalf("user delete failed: %v", err)
}
if _, err := state.authRoleDelete(paramsWith(map[string]json.RawMessage{"role": jsonString("dbx_live_role")})); err != nil {
t.Fatalf("role delete failed: %v", err)
}
// --- status / maintenance ------------------------------------------
status, err := state.status(paramsWith(map[string]json.RawMessage{}))
if err != nil {
t.Fatalf("kv_status failed: %v", err)
}
statusRow := status.(map[string]any)
if statusRow["clusterId"] == nil || statusRow["leaderId"] == nil {
t.Fatalf("status missing cluster identity: %#v", statusRow)
}
members := statusRow["members"].([]map[string]any)
if len(members) == 0 || members[0]["reachable"] != true || members[0]["version"] == nil {
t.Fatalf("unexpected member rows: %#v", members)
}
if keyCount, err := strconvParse(statusRow["keyCount"].(string)); err != nil || keyCount <= 0 {
t.Fatalf("keyCount must be a positive numeric string: %#v", statusRow["keyCount"])
}
compactTarget := revisionOf(t, putResult)
if compactTarget < baseRevision {
compactTarget = baseRevision
}
if _, err := state.compact(paramsWith(map[string]json.RawMessage{"revision": json.RawMessage(statusRow["revision"].(string))})); err != nil {
t.Fatalf("compact failed: %v", err)
}
if _, err := state.compact(paramsWith(map[string]json.RawMessage{"revision": json.RawMessage(`999999999999`)})); err == nil || !strings.Contains(err.Error(), "future revision") {
t.Fatalf("expected raw future-revision error (Java parity), got %v", err)
}
defragged, err := state.defrag(paramsWith(map[string]json.RawMessage{
"endpoints": json.RawMessage(`["` + endpoints + `"]`),
}))
if err != nil {
t.Fatalf("defrag failed: %v", err)
}
defragRow := defragged.(map[string]any)["members"].([]map[string]any)[0]
if defragRow["status"] != "succeeded" {
t.Fatalf("defrag should succeed on a live member: %#v", defragRow)
}
// --- delete + cleanup ----------------------------------------------
for _, key := range []string{prefix + "binary", prefix + "leased", prefix + "renamed", prefix + "watched"} {
if _, err := state.delete(paramsWith(map[string]json.RawMessage{"key": jsonString(key)})); err != nil {
t.Fatalf("delete %s failed: %v", key, err)
}
}
if _, err := state.delete(paramsWith(map[string]json.RawMessage{"key": jsonString(prefix + "renamed")})); err != nil {
t.Fatalf("delete missing key failed: %v", err)
}
}
// TestLiveEtcdProtocolFlow drives the multi-session runtime exactly like the
// Rust host does: NDJSON lines with agentSessionId routing.
func TestLiveEtcdProtocolFlow(t *testing.T) {
if os.Getenv("DBX_ETCD_LIVE") != "1" {
t.Skip("set DBX_ETCD_LIVE=1 to run the live etcd agent test")
}
endpoints := envOrDefault("DBX_ETCD_ENDPOINTS", "http://127.0.0.1:10700")
user := envOrDefault("DBX_ETCD_USER", "root")
password := envOrDefault("DBX_ETCD_PASSWORD", "123456")
server := newRuntimeServer()
connection := fmt.Sprintf(`{"etcd_endpoints":%q,"username":%q,"password":%q}`, endpoints, user, password)
handshake, _ := server.handleLine(`{"id":1,"method":"handshake","params":{}}`)
if handshake.Error != nil {
t.Fatalf("handshake failed: %#v", handshake)
}
openLine := fmt.Sprintf(`{"id":2,"method":"open_session","params":{"agentSessionId":"s1","connection":%s}}`, connection)
opened, _ := server.handleLine(openLine)
if opened.Error != nil {
t.Fatalf("open_session failed: %#v", opened)
}
putLine := fmt.Sprintf(`{"id":3,"method":"kv_put","params":{"agentSessionId":"s1","key":"dbx:proto","value":{"encoding":"utf8","data":"flow"}}}`)
put, _ := server.handleLine(putLine)
if put.Error != nil {
t.Fatalf("kv_put failed: %#v", put)
}
unknown, _ := server.handleLine(`{"id":4,"method":"kv_describe","params":{"agentSessionId":"s1"}}`)
if unknown.Error == nil || !strings.Contains(unknown.Error.Message, "unknown method") {
t.Fatalf("expected unknown-method error, got %#v", unknown)
}
closed, _ := server.handleLine(`{"id":5,"method":"close_session","params":{"agentSessionId":"s1"}}`)
if closed.Error != nil {
t.Fatalf("close_session failed: %#v", closed)
}
}
func envOrDefault(key, fallback string) string {
if value := strings.TrimSpace(os.Getenv(key)); value != "" {
return value
}
return fallback
}
func paramsWith(params map[string]json.RawMessage) map[string]json.RawMessage {
return params
}
func jsonString(value string) json.RawMessage {
encoded, _ := json.Marshal(value)
return json.RawMessage(encoded)
}
func jsonValue(value string) json.RawMessage {
encoded, _ := json.Marshal(value)
return json.RawMessage(`{"encoding":"utf8","data":` + string(encoded) + `}`)
}
func revisionOf(t *testing.T, result any) int64 {
t.Helper()
row, ok := result.(map[string]any)
if !ok {
t.Fatalf("expected map result, got %#v", result)
}
revision, err := strconvParse(row["revision"].(string))
if err != nil {
t.Fatalf("revision not numeric: %#v", row["revision"])
}
return revision
}
func strconvParse(value string) (int64, error) {
var parsed int64
_, err := fmt.Sscan(value, &parsed)
return parsed, err
}
func containsAny(values []string, target string) bool {
for _, value := range values {
if value == target {
return true
}
}
return false
}
agents/drivers/etcd-go/kv.go
+524
View File
@@ -0,0 +1,524 @@
package main
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
"time"
"unicode/utf8"
"go.etcd.io/etcd/api/v3/mvccpb"
clientv3 "go.etcd.io/etcd/client/v3"
)
const defaultListLimit = 100
const preserveLeaseMaxAttempts = 3
func (s *etcdSession) listPrefix(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
prefix := stringOrDefault(params, "prefix", "")
limit := intOrDefault(params, "limit", defaultListLimit)
if limit < 1 {
limit = 1
}
revision := longOrNull(params, "revision")
includeValues := boolOrDefault(params, "includeValues", false)
continuation := stringOrNull(params, "continuation")
start := prefixStart(prefix)
if continuation != nil && *continuation != "" {
decoded, err := base64.StdEncoding.DecodeString(*continuation)
if err != nil {
return nil, err
}
start = string(decoded)
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
options := []clientv3.OpOption{
clientv3.WithRange(prefixEnd(prefix)),
clientv3.WithLimit(int64(limit)),
clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend),
}
if revision != nil && *revision > 0 {
options = append(options, clientv3.WithRev(*revision))
}
response, err := client.Get(ctx, start, options...)
if err != nil {
return nil, err
}
keys := make([]any, 0, len(response.Kvs))
for _, item := range response.Kvs {
row := metadataMap(item)
row["key"] = displayBytes(item.Key)
row["keyBytes"] = bytesObject(item.Key)
if includeValues {
row["value"] = valueObject(item.Value)
}
keys = append(keys, row)
}
result := map[string]any{"keys": keys}
if response.More && len(response.Kvs) > 0 {
result["continuation"] = nextContinuation(response.Kvs[len(response.Kvs)-1].Key)
} else {
result["continuation"] = nil
}
result["revision"] = longString(response.Header.Revision)
return result, nil
}
func (s *etcdSession) get(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
key, err := keyBytesParam(params)
if err != nil {
return nil, err
}
revision := longOrNull(params, "revision")
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
options := []clientv3.OpOption{}
if revision != nil && *revision > 0 {
options = append(options, clientv3.WithRev(*revision))
}
response, err := client.Get(ctx, key, options...)
if err != nil {
return nil, err
}
if len(response.Kvs) == 0 {
return map[string]any{"found": false, "key": nil, "value": nil, "metadata": nil}, nil
}
item := response.Kvs[0]
result := map[string]any{
"found": true,
"key": displayBytes(item.Key),
"keyBytes": bytesObject(item.Key),
}
if !boolOrDefault(params, "metadataOnly", false) {
result["value"] = valueObject(item.Value)
} else {
result["value"] = nil
}
metadata, err := s.metadataWithTtl(item)
if err != nil {
return nil, err
}
result["metadata"] = metadata
return result, nil
}
func (s *etcdSession) put(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
key, err := keyBytesParam(params)
if err != nil {
return nil, err
}
value, err := parseValueObject(rawObject(params, "value"))
if err != nil {
return nil, err
}
expectedModRevision := longOrNull(params, "expectedModRevision")
expectedCreateRevision := longOrNull(params, "expectedCreateRevision")
leaseValue := longOrNull(params, "lease")
ttlValue := longOrNull(params, "ttl")
preserveLease := boolOrDefault(params, "preserveLease", false)
hasLease := leaseValue != nil
hasTtl := ttlValue != nil
if (hasLease && hasTtl) || (preserveLease && (hasLease || hasTtl)) {
return nil, errors.New("lease, ttl, and preserveLease cannot be specified together")
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
if preserveLease {
revision, err := putPreservingLease(client, ctx, key, value)
if err != nil {
return nil, err
}
return map[string]any{"revision": longString(revision)}, nil
}
var leaseID clientv3.LeaseID
var grantedLeaseID clientv3.LeaseID
if hasTtl {
if *ttlValue <= 0 {
return nil, errors.New("ttl must be a positive integer")
}
grant, err := client.Lease.Grant(ctx, *ttlValue)
if err != nil {
return nil, err
}
grantedLeaseID = grant.ID
leaseID = grant.ID
} else if hasLease {
leaseID = clientv3.LeaseID(*leaseValue)
}
revision, err := func() (int64, error) {
if expectedModRevision != nil || expectedCreateRevision != nil {
var comparisons []clientv3.Cmp
if expectedModRevision != nil {
comparisons = append(comparisons, clientv3.Compare(clientv3.ModRevision(key), "=", *expectedModRevision))
}
if expectedCreateRevision != nil {
comparisons = append(comparisons, clientv3.Compare(clientv3.CreateRevision(key), "=", *expectedCreateRevision))
}
txn := client.Txn(ctx).If(comparisons...).Then(clientv3.OpPut(key, value, clientv3.WithLease(leaseID)))
response, err := txn.Commit()
if err != nil {
return 0, err
}
if !response.Succeeded {
return 0, errors.New("ETCD_CAS_CONFLICT: key changed after it was loaded")
}
return response.Header.Revision, nil
}
response, err := client.Put(ctx, key, value, clientv3.WithLease(leaseID))
if err != nil {
return 0, err
}
return response.Header.Revision, nil
}()
if err != nil && grantedLeaseID != 0 {
_, _ = client.Lease.Revoke(context.Background(), grantedLeaseID)
}
if err != nil {
return nil, err
}
return map[string]any{"revision": longString(revision)}, nil
}
func putPreservingLease(client *clientv3.Client, ctx context.Context, key, value string) (int64, error) {
for attempt := 0; attempt < preserveLeaseMaxAttempts; attempt++ {
existing, err := client.Get(ctx, key)
if err != nil {
return 0, err
}
if len(existing.Kvs) == 0 || existing.Kvs[0].Lease <= 0 {
return 0, errors.New("Cannot preserve lease: key does not exist or has no lease")
}
current := existing.Kvs[0]
txn := client.Txn(ctx).
If(clientv3.Compare(clientv3.ModRevision(key), "=", current.ModRevision)).
Then(clientv3.OpPut(key, value, clientv3.WithLease(clientv3.LeaseID(current.Lease))))
response, err := txn.Commit()
if err != nil {
return 0, err
}
if response.Succeeded {
return response.Header.Revision, nil
}
}
return 0, errors.New("Cannot preserve lease: key changed concurrently; retry the save")
}
func (s *etcdSession) delete(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
key, err := keyBytesParam(params)
if err != nil {
return nil, err
}
expectedModRevision := longOrNull(params, "expectedModRevision")
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
if expectedModRevision != nil {
txn := client.Txn(ctx).
If(clientv3.Compare(clientv3.ModRevision(key), "=", *expectedModRevision)).
Then(clientv3.OpDelete(key))
response, err := txn.Commit()
if err != nil {
return nil, err
}
if !response.Succeeded {
return nil, errors.New("ETCD_CAS_CONFLICT: key changed after it was loaded")
}
return map[string]any{"deleted": int64(1), "revision": longString(response.Header.Revision)}, nil
}
response, err := client.Delete(ctx, key)
if err != nil {
return nil, err
}
return map[string]any{"deleted": response.Deleted, "revision": longString(response.Header.Revision)}, nil
}
func (s *etcdSession) rename(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
sourceKey, err := keyBytesParam(params)
if err != nil {
return nil, err
}
newKey := stringOrNull(params, "newKey")
if newKey == nil || *newKey == "" {
return nil, errors.New("ETCD_NEWKEY_REQUIRED")
}
targetKey := *newKey
if sourceKey == targetKey {
return map[string]any{"renamed": true, "revision": nil}, nil
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
sourceResponse, err := client.Get(ctx, sourceKey)
if err != nil {
return nil, err
}
if len(sourceResponse.Kvs) == 0 {
return nil, errors.New("ETCD_NOT_FOUND: source key does not exist")
}
source := sourceResponse.Kvs[0]
expected := longOrNull(params, "expectedModRevision")
expectedRevision := source.ModRevision
if expected != nil {
expectedRevision = *expected
}
putOption := []clientv3.OpOption{}
if source.Lease != 0 {
putOption = append(putOption, clientv3.WithLease(clientv3.LeaseID(source.Lease)))
}
txn := client.Txn(ctx).
If(
clientv3.Compare(clientv3.ModRevision(sourceKey), "=", expectedRevision),
clientv3.Compare(clientv3.CreateRevision(targetKey), "=", 0),
).
Then(
clientv3.OpPut(targetKey, string(source.Value), putOption...),
clientv3.OpDelete(sourceKey),
)
response, err := txn.Commit()
if err != nil {
return nil, err
}
if !response.Succeeded {
return nil, errors.New("ETCD_CAS_CONFLICT: source changed or target already exists")
}
return map[string]any{"renamed": true, "revision": longString(response.Header.Revision)}, nil
}
func (s *etcdSession) metadataWithTtl(item *mvccpb.KeyValue) (map[string]any, error) {
metadata := metadataMap(item)
if item.Lease > 0 {
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
defer cancel()
lease, err := client.Lease.TimeToLive(ctx, clientv3.LeaseID(item.Lease))
if err != nil {
return nil, err
}
metadata["ttl"] = lease.TTL
}
return metadata, nil
}
func metadataMap(item *mvccpb.KeyValue) map[string]any {
return map[string]any{
"createRevision": longString(item.CreateRevision),
"modRevision": longString(item.ModRevision),
"version": longString(item.Version),
"lease": longString(item.Lease),
"valueSize": len(item.Value),
}
}
func prefixEnd(prefix string) string {
if prefix == "" {
return "\x00"
}
end := []byte(prefix)
for i := len(end) - 1; i >= 0; i-- {
if end[i] < 0xff {
end[i]++
return string(end[:i+1])
}
}
return "\x00"
}
func prefixStart(prefix string) string {
if prefix == "" {
return "\x00"
}
return prefix
}
func nextContinuation(key []byte) string {
next := make([]byte, len(key)+1)
copy(next, key)
return base64.StdEncoding.EncodeToString(next)
}
func keyBytesParam(params map[string]json.RawMessage) (string, error) {
if encoded := rawObject(params, "keyBytes"); encoded != nil {
value, err := parseValueObject(encoded)
if err != nil {
return "", err
}
return value, nil
}
if raw, ok := params["key"]; ok && raw != nil && string(raw) != "null" {
var key string
if err := json.Unmarshal(raw, &key); err != nil {
return "", err
}
return key, nil
}
return "", errors.New("Key is required")
}
func rawObject(params map[string]json.RawMessage, key string) map[string]json.RawMessage {
raw := params[key]
if len(raw) == 0 || string(raw) == "null" {
return nil
}
var object map[string]json.RawMessage
if err := json.Unmarshal(raw, &object); err != nil || object == nil {
return nil
}
return object
}
func bytesObject(bytes []byte) map[string]any {
return map[string]any{
"encoding": "base64",
"data": base64.StdEncoding.EncodeToString(bytes),
}
}
func valueObject(value []byte) map[string]any {
if utf8.Valid(value) {
return map[string]any{"encoding": "utf8", "data": string(value)}
}
return bytesObject(value)
}
func displayBytes(bytes []byte) string {
if utf8.Valid(bytes) {
return string(bytes)
}
return base64.StdEncoding.EncodeToString(bytes)
}
func parseValueObject(value map[string]json.RawMessage) (string, error) {
if value == nil {
value = map[string]json.RawMessage{}
}
encoding := stringOrDefault(value, "encoding", "utf8")
data := stringOrDefault(value, "data", "")
if encoding == "base64" {
decoded, err := base64.StdEncoding.DecodeString(data)
if err != nil {
return "", err
}
return string(decoded), nil
}
if encoding != "utf8" {
return "", fmt.Errorf("Unsupported value encoding: %s", encoding)
}
return data, nil
}
func longString(value int64) string {
return strconv.FormatInt(value, 10)
}
func unsignedLongString(value int64) string {
return strconv.FormatUint(uint64(value), 10)
}
func stringOrNull(params map[string]json.RawMessage, key string) *string {
raw := params[key]
if len(raw) == 0 || string(raw) == "null" {
return nil
}
var value string
if err := json.Unmarshal(raw, &value); err != nil {
var text *string
return text
}
return &value
}
func stringOrDefault(params map[string]json.RawMessage, key string, fallback string) string {
value := stringOrNull(params, key)
if value == nil {
return fallback
}
return *value
}
func intOrDefault(params map[string]json.RawMessage, key string, fallback int) int {
raw := params[key]
if len(raw) == 0 || string(raw) == "null" {
return fallback
}
var value int
if err := json.Unmarshal(raw, &value); err != nil {
return fallback
}
return value
}
func boolOrDefault(params map[string]json.RawMessage, key string, fallback bool) bool {
raw := params[key]
if len(raw) == 0 || string(raw) == "null" {
return fallback
}
var value bool
if err := json.Unmarshal(raw, &value); err != nil {
return fallback
}
return value
}
func longOrNull(params map[string]json.RawMessage, key string) *int64 {
raw := params[key]
if len(raw) == 0 || string(raw) == "null" {
return nil
}
var value int64
if err := json.Unmarshal(raw, &value); err != nil {
return nil
}
return &value
}
func requiredPositiveLong(params map[string]json.RawMessage, field string) (int64, error) {
value := longOrNull(params, field)
if value == nil || *value <= 0 {
return 0, fmt.Errorf("ETCD_INVALID_%s: a positive integer is required", strings.ToUpper(field))
}
return *value, nil
}
func requiredString(params map[string]json.RawMessage, field string) (string, error) {
value := stringOrNull(params, field)
if value == nil || *value == "" {
return "", fmt.Errorf("ETCD_%s_REQUIRED", strings.ToUpper(field))
}
return *value, nil
}
agents/drivers/etcd-go/lease.go
+339
View File
@@ -0,0 +1,339 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"strconv"
"time"
"go.etcd.io/etcd/api/v3/etcdserverpb"
clientv3 "go.etcd.io/etcd/client/v3"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
const (
maxLeaseAttachedKeys = 256
defaultLeaseListLimit = 100
maxLeaseListLimit = 200
leaseListConcurrency = 8
leaseListDeadlineSeconds = 5
)
type leaseListDeadline struct {
deadline time.Time
}
func (d leaseListDeadline) remaining() (time.Duration, error) {
remaining := time.Until(d.deadline)
if remaining <= 0 {
return 0, errors.New("ETCD_LEASE_LIST_TIMEOUT")
}
return remaining, nil
}
func (s *etcdSession) leaseList(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
deadline := leaseListDeadline{deadline: time.Now().Add(leaseListDeadlineSeconds * time.Second)}
limit := intOrDefault(params, "limit", defaultLeaseListLimit)
if limit < 1 {
limit = 1
}
if limit > maxLeaseListLimit {
limit = maxLeaseListLimit
}
continuation := stringOrNull(params, "continuation")
var afterLeaseID *uint64
if continuation != nil && *continuation != "" {
parsed, err := parseUnsignedLong(*continuation)
if err != nil {
return nil, err
}
afterLeaseID = &parsed
}
partial := false
leaseIDs, err := clusterLeaseIDs(client, deadline)
if err != nil {
if isLeaseListFallbackError(err) {
leaseIDs = s.knownLeaseIDs()
partial = true
} else {
return nil, err
}
} else {
for _, id := range leaseIDs {
s.rememberLease(id)
}
}
leaseIDs = leasePageIDs(leaseIDs, afterLeaseID, limit+1)
hasMore := len(leaseIDs) > limit
if len(leaseIDs) > limit {
leaseIDs = leaseIDs[:limit]
}
leases := []map[string]any{}
var lastProcessedID *uint64
deadlineReached := false
chunkLoop:
for offset := 0; offset < len(leaseIDs); offset += leaseListConcurrency {
end := offset + leaseListConcurrency
if end > len(leaseIDs) {
end = len(leaseIDs)
}
chunkIDs := leaseIDs[offset:end]
type ttlResult struct {
id uint64
ttl int64
granted int64
err error
}
results := make([]chan ttlResult, len(chunkIDs))
for index, id := range chunkIDs {
channel := make(chan ttlResult, 1)
results[index] = channel
go func(id uint64, channel chan ttlResult) {
remaining, err := deadline.remaining()
if err != nil {
channel <- ttlResult{id: id, err: err}
return
}
ctx, cancel := context.WithTimeout(context.Background(), remaining)
defer cancel()
response, err := client.Lease.TimeToLive(ctx, clientv3.LeaseID(int64(id)))
if err != nil {
channel <- ttlResult{id: id, err: err}
return
}
channel <- ttlResult{id: id, ttl: response.TTL, granted: response.GrantedTTL}
}(id, channel)
}
for _, channel := range results {
result := <-channel
id := result.id
if result.err != nil {
if errors.Is(result.err, errLeaseListDeadline) || isDeadlineError(result.err) {
partial = true
deadlineReached = true
break chunkLoop
}
if status.Code(result.err) == codes.NotFound {
s.forgetLease(id)
} else {
partial = true
}
lastProcessedID = &id
continue
}
leases = append(leases, map[string]any{
"id": unsignedLongString(int64(id)),
"ttl": result.ttl,
"grantedTtl": result.granted,
})
processed := id
lastProcessedID = &processed
}
}
var nextContinuation any
if deadlineReached && len(leaseIDs) > 0 {
if lastProcessedID != nil {
nextContinuation = unsignedLongString(int64(*lastProcessedID))
} else if afterLeaseID != nil {
nextContinuation = unsignedLongString(int64(*afterLeaseID))
} else {
nextContinuation = "0"
}
} else if hasMore && len(leaseIDs) > 0 {
nextContinuation = unsignedLongString(int64(leaseIDs[len(leaseIDs)-1]))
}
return map[string]any{
"leases": leases,
"partial": partial,
"nextContinuation": nextContinuation,
}, nil
}
var errLeaseListDeadline = errors.New("ETCD_LEASE_LIST_TIMEOUT")
func isDeadlineError(err error) bool {
return errors.Is(err, context.DeadlineExceeded) || status.Code(err) == codes.DeadlineExceeded
}
func isLeaseListFallbackError(err error) bool {
if errors.Is(err, errLeaseListDeadline) || errors.Is(err, context.DeadlineExceeded) {
return true
}
code := status.Code(err)
return code == codes.Unimplemented || code == codes.DeadlineExceeded
}
func parseUnsignedLong(value string) (uint64, error) {
parsed, err := strconv.ParseUint(value, 10, 64)
if err != nil {
return 0, fmt.Errorf("invalid lease continuation: %s", value)
}
return parsed, nil
}
func leasePageIDs(leaseIDs []uint64, afterLeaseID *uint64, fetchLimit int) []uint64 {
sorted := make([]uint64, len(leaseIDs))
copy(sorted, leaseIDs)
sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] })
result := []uint64{}
for _, id := range sorted {
if afterLeaseID != nil && id <= *afterLeaseID {
continue
}
if len(result) >= fetchLimit {
break
}
result = append(result, id)
}
return result
}
func clusterLeaseIDs(client *clientv3.Client, deadline leaseListDeadline) ([]uint64, error) {
remaining, err := deadline.remaining()
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), remaining)
defer cancel()
leaseClient := etcdserverpb.NewLeaseClient(client.ActiveConnection())
response, err := leaseClient.LeaseLeases(ctx, &etcdserverpb.LeaseLeasesRequest{})
if err != nil {
return nil, err
}
ids := make([]uint64, 0, len(response.Leases))
for _, lease := range response.Leases {
ids = append(ids, uint64(lease.ID))
}
return ids, nil
}
func (s *etcdSession) leaseGet(params map[string]json.RawMessage) (any, error) {
id, err := requiredPositiveLong(params, "id")
if err != nil {
return nil, err
}
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
options := []clientv3.LeaseOption{}
if boolOrDefault(params, "includeKeys", false) {
options = append(options, clientv3.WithAttachedKeys())
}
response, err := client.Lease.TimeToLive(ctx, clientv3.LeaseID(id), options...)
if err != nil {
return nil, err
}
s.rememberLease(uint64(id))
keys := []map[string]any{}
for index, key := range response.Keys {
if index >= maxLeaseAttachedKeys {
break
}
keys = append(keys, bytesObject(key))
}
return map[string]any{
"id": unsignedLongString(int64(response.ID)),
"ttl": response.TTL,
"grantedTtl": response.GrantedTTL,
"keys": keys,
"truncated": len(response.Keys) > maxLeaseAttachedKeys,
}, nil
}
func (s *etcdSession) leaseGrant(params map[string]json.RawMessage) (any, error) {
ttl, err := requiredPositiveLong(params, "ttl")
if err != nil {
return nil, err
}
requestedID := longOrNull(params, "id")
if requestedID != nil && *requestedID < 0 {
return nil, errors.New("id must be a positive integer or 0")
}
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
var grantedID clientv3.LeaseID
var grantedTTL int64
if requestedID == nil || *requestedID == 0 {
response, err := client.Lease.Grant(ctx, ttl)
if err != nil {
return nil, err
}
grantedID = response.ID
grantedTTL = response.TTL
} else {
leaseClient := etcdserverpb.NewLeaseClient(client.ActiveConnection())
response, err := leaseClient.LeaseGrant(ctx, &etcdserverpb.LeaseGrantRequest{TTL: ttl, ID: *requestedID})
if err != nil {
return nil, err
}
grantedID = clientv3.LeaseID(response.ID)
grantedTTL = response.TTL
}
s.rememberLease(uint64(grantedID))
return map[string]any{
"id": unsignedLongString(int64(grantedID)),
"ttl": grantedTTL,
}, nil
}
func (s *etcdSession) leaseKeepAlive(params map[string]json.RawMessage) (any, error) {
id, err := requiredPositiveLong(params, "id")
if err != nil {
return nil, err
}
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
response, err := client.Lease.KeepAliveOnce(ctx, clientv3.LeaseID(id))
if err != nil {
return nil, err
}
s.rememberLease(uint64(id))
return map[string]any{
"id": unsignedLongString(id),
"ttl": response.TTL,
}, nil
}
func (s *etcdSession) leaseRevoke(params map[string]json.RawMessage) (any, error) {
id, err := requiredPositiveLong(params, "id")
if err != nil {
return nil, err
}
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
if _, err := client.Lease.Revoke(ctx, clientv3.LeaseID(id)); err != nil {
return nil, err
}
s.forgetLease(uint64(id))
return map[string]any{
"id": unsignedLongString(id),
"revoked": true,
}, nil
}
agents/drivers/etcd-go/main.go
+468
View File
@@ -0,0 +1,468 @@
package main
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"os"
"runtime"
"strconv"
"strings"
"sync"
"time"
clientv3 "go.etcd.io/etcd/client/v3"
)
const (
protocolVersion = 2
legacyAgentSessionID = "__legacy__"
maxAgentSessions = 256
rpcTimeoutSeconds = 30
)
var capabilities = []string{
"connect", "test_connection", "kv", "kv_ttl", "kv_cas", "kv_list_values", "kv_status",
"kv_history", "etcd_compaction", "etcd_defrag", "etcd_watch", "etcd_lease", "etcd_auth",
"multi_session",
}
type request struct {
ID json.RawMessage `json:"id"`
Method string `json:"method"`
Params map[string]json.RawMessage `json:"params"`
}
type response struct {
JSONRPC string `json:"jsonrpc,omitempty"`
ID json.RawMessage `json:"id,omitempty"`
Result any `json:"result,omitempty"`
Error *rpcError `json:"error,omitempty"`
}
// etcdSession carries the per-connection etcd client and its derived state.
// Requests for one session are serialized by agentSession.mu; watch callbacks
// reach the maps through watchesMu instead so buffering never blocks dispatch.
type etcdSession struct {
clientMu sync.Mutex
client *clientv3.Client
connectedEndpoints []string
watchesMu sync.Mutex
watches map[string]*watchState
knownLeases map[uint64]struct{}
watchBufferedBytes int64
activeCancelMu sync.Mutex
activeCancel context.CancelFunc
}
func (s *etcdSession) reserveWatchBuffer(bytes int64) bool {
s.watchesMu.Lock()
defer s.watchesMu.Unlock()
if bytes < 0 || s.watchBufferedBytes+bytes > maxSessionWatchBufferBytes {
return false
}
s.watchBufferedBytes += bytes
return true
}
func (s *etcdSession) releaseWatchBuffer(bytes int64) {
s.watchesMu.Lock()
defer s.watchesMu.Unlock()
s.watchBufferedBytes -= bytes
if s.watchBufferedBytes < 0 {
s.watchBufferedBytes = 0
}
}
func (s *etcdSession) rememberLease(id uint64) {
s.watchesMu.Lock()
defer s.watchesMu.Unlock()
s.knownLeases[id] = struct{}{}
}
func (s *etcdSession) forgetLease(id uint64) {
s.watchesMu.Lock()
defer s.watchesMu.Unlock()
delete(s.knownLeases, id)
}
func (s *etcdSession) knownLeaseIDs() []uint64 {
s.watchesMu.Lock()
defer s.watchesMu.Unlock()
ids := make([]uint64, 0, len(s.knownLeases))
for id := range s.knownLeases {
ids = append(ids, id)
}
return ids
}
type agentSession struct {
state *etcdSession
mu sync.Mutex
}
func (a *agentSession) cancelActive() {
a.state.activeCancelMu.Lock()
cancel := a.state.activeCancel
a.state.activeCancelMu.Unlock()
if cancel != nil {
cancel()
}
}
type runtimeServer struct {
mu sync.RWMutex
sessions map[string]*agentSession
}
func main() {
configureRuntimeParallelism()
server := newRuntimeServer()
encoder := json.NewEncoder(os.Stdout)
var encoderMu sync.Mutex
var requests sync.WaitGroup
fmt.Fprintln(os.Stdout, `{"ready":true}`)
scanner := bufio.NewScanner(os.Stdin)
scanner.Buffer(make([]byte, 0, 64*1024), 512*1024*1024)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
}
var envelope request
if json.Unmarshal([]byte(line), &envelope) == nil && envelope.Method == "shutdown" {
requests.Wait()
result, _ := server.handleLine(line)
encoderMu.Lock()
_ = encoder.Encode(result)
encoderMu.Unlock()
server.closeAll()
return
}
requests.Add(1)
go func(line string) {
defer requests.Done()
result, _ := server.handleLine(line)
encoderMu.Lock()
defer encoderMu.Unlock()
if err := encoder.Encode(result); err != nil {
fmt.Fprintf(os.Stderr, "failed to write response: %v\n", err)
}
}(line)
}
requests.Wait()
server.closeAll()
}
func configureRuntimeParallelism() {
if raw := strings.TrimSpace(os.Getenv("DBX_AGENT_ETCD_GOMAXPROCS")); raw != "" {
if configured, err := strconv.Atoi(raw); err == nil && configured > 0 {
runtime.GOMAXPROCS(configured)
return
}
}
if strings.TrimSpace(os.Getenv("GOMAXPROCS")) != "" {
return
}
runtime.GOMAXPROCS(min(runtime.NumCPU(), 4))
}
func newRuntimeServer() *runtimeServer {
return &runtimeServer{sessions: map[string]*agentSession{}}
}
func (r *runtimeServer) handleLine(line string) (response, bool) {
var req request
if err := json.Unmarshal([]byte(line), &req); err != nil {
return errorResponse(nil, "", "", err), false
}
if len(req.ID) == 0 {
req.ID = json.RawMessage("1")
}
result, shutdown, err := r.dispatch(req.Method, req.Params)
if err != nil {
return errorResponse(req.ID, req.Method, stringParam(req.Params, "agentSessionId"), err), false
}
return response{JSONRPC: "2.0", ID: req.ID, Result: result}, shutdown
}
func (r *runtimeServer) dispatch(method string, params map[string]json.RawMessage) (any, bool, error) {
switch method {
case "handshake":
return handshakeResult(), false, nil
case "open_session":
id := requiredSessionID(params)
if id == "" {
return nil, false, errors.New("agentSessionId is required")
}
return r.openSession(id, params)
case "close_session":
return r.closeSession(stringParam(params, "agentSessionId")), false, nil
case "validate_session":
session, err := r.session(requiredSessionID(params))
if err != nil {
return nil, false, err
}
session.mu.Lock()
defer session.mu.Unlock()
result, err := session.state.validateConnection()
return result, false, err
case "cancel_session":
session, err := r.session(requiredSessionID(params))
if err != nil {
return nil, false, err
}
session.cancelActive()
return map[string]bool{"ok": true}, false, nil
case "test_connection":
return r.testConnection(params)
case "connect":
_ = r.closeSession(legacyAgentSessionID)
return r.openSession(legacyAgentSessionID, params)
case "disconnect":
return r.closeSession(legacyAgentSessionID), false, nil
case "shutdown":
return map[string]bool{"ok": true}, true, r.closeAll()
default:
id := stringParam(params, "agentSessionId")
if id == "" {
id = legacyAgentSessionID
}
session, err := r.session(id)
if err != nil {
return nil, false, err
}
session.mu.Lock()
defer session.mu.Unlock()
result, err := session.state.handle(method, params)
return result, false, err
}
}
func requiredSessionID(params map[string]json.RawMessage) string {
return strings.TrimSpace(stringParam(params, "agentSessionId"))
}
func (r *runtimeServer) openSession(id string, params map[string]json.RawMessage) (any, bool, error) {
r.mu.Lock()
if _, exists := r.sessions[id]; exists {
r.mu.Unlock()
return nil, false, fmt.Errorf("Agent session already exists: %s", id)
}
if len(r.sessions) >= maxAgentSessions {
r.mu.Unlock()
return nil, false, fmt.Errorf("Agent session limit reached: %d", maxAgentSessions)
}
session := &agentSession{state: newEtcdSession()}
r.sessions[id] = session
r.mu.Unlock()
if _, err := session.state.connect(params); err != nil {
r.mu.Lock()
delete(r.sessions, id)
r.mu.Unlock()
session.state.close()
return nil, false, err
}
return map[string]bool{"ok": true}, false, nil
}
func (r *runtimeServer) session(id string) (*agentSession, error) {
r.mu.RLock()
session := r.sessions[id]
r.mu.RUnlock()
if session == nil {
return nil, fmt.Errorf("Agent session not found: %s", id)
}
return session, nil
}
func (r *runtimeServer) closeSession(id string) any {
r.mu.Lock()
session := r.sessions[id]
delete(r.sessions, id)
r.mu.Unlock()
if session != nil {
session.cancelActive()
session.mu.Lock()
session.state.close()
session.mu.Unlock()
}
return map[string]bool{"ok": true}
}
func (r *runtimeServer) closeAll() error {
r.mu.Lock()
sessions := r.sessions
r.sessions = map[string]*agentSession{}
r.mu.Unlock()
var firstErr error
for _, session := range sessions {
session.cancelActive()
if err := session.state.close(); err != nil && firstErr == nil {
firstErr = err
}
}
return firstErr
}
func (r *runtimeServer) testConnection(params map[string]json.RawMessage) (any, bool, error) {
state := newEtcdSession()
defer state.close()
if _, err := state.connect(params); err != nil {
return nil, false, err
}
return map[string]bool{"ok": true}, false, nil
}
func newEtcdSession() *etcdSession {
return &etcdSession{
watches: map[string]*watchState{},
knownLeases: map[uint64]struct{}{},
}
}
func handshakeResult() map[string]any {
return map[string]any{
"protocolVersion": protocolVersion,
"agentProtocolVersion": protocolVersion,
"capabilities": capabilities,
}
}
func (s *etcdSession) beginOperation() (context.Context, context.CancelFunc) {
ctx, cancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
s.activeCancelMu.Lock()
s.activeCancel = cancel
s.activeCancelMu.Unlock()
return ctx, cancel
}
func (s *etcdSession) endOperation(cancel context.CancelFunc) {
cancel()
s.activeCancelMu.Lock()
s.activeCancel = nil
s.activeCancelMu.Unlock()
}
func (s *etcdSession) handle(method string, params map[string]json.RawMessage) (any, error) {
switch method {
case "handshake":
return handshakeResult(), nil
case "connect", "test_connection":
return s.connectMap(params)
case "validate_connection":
return s.validateConnection()
case "kv_list_prefix":
return s.listPrefix(params)
case "kv_get":
return s.get(params)
case "kv_put":
return s.put(params)
case "kv_delete":
return s.delete(params)
case "kv_rename":
return s.rename(params)
case "kv_history":
return s.history(params)
case "kv_status":
return s.status(params)
case "etcd_compact":
return s.compact(params)
case "etcd_defrag":
return s.defrag(params)
case "etcd_watch_start":
return s.watchStart(params)
case "etcd_watch_poll":
return s.watchPoll(params)
case "etcd_watch_stop":
return s.watchStop(params)
case "etcd_lease_list":
return s.leaseList(params)
case "etcd_lease_get":
return s.leaseGet(params)
case "etcd_lease_grant":
return s.leaseGrant(params)
case "etcd_lease_keepalive_once":
return s.leaseKeepAlive(params)
case "etcd_lease_revoke":
return s.leaseRevoke(params)
case "etcd_auth_user_list":
return s.authUserList(params)
case "etcd_auth_user_get":
return s.authUserGet(params)
case "etcd_auth_user_add":
return s.authUserAdd(params)
case "etcd_auth_user_delete":
return s.authUserDelete(params)
case "etcd_auth_user_change_password":
return s.authUserChangePassword(params)
case "etcd_auth_user_grant_role":
return s.authUserGrantRevokeRole(params, true)
case "etcd_auth_user_revoke_role":
return s.authUserGrantRevokeRole(params, false)
case "etcd_auth_role_list":
return s.authRoleList(params)
case "etcd_auth_role_get":
return s.authRoleGet(params)
case "etcd_auth_role_add":
return s.authRoleAdd(params)
case "etcd_auth_role_delete":
return s.authRoleDelete(params)
case "etcd_auth_role_grant_permission":
return s.authRolePermission(params, true)
case "etcd_auth_role_revoke_permission":
return s.authRolePermission(params, false)
case "disconnect":
s.close()
return map[string]bool{"ok": true}, nil
case "shutdown":
s.close()
return map[string]bool{"ok": true}, nil
default:
return nil, fmt.Errorf("unknown method: %s", method)
}
}
func decodeParams(params map[string]json.RawMessage, target any) error {
if params == nil {
params = map[string]json.RawMessage{}
}
data, err := json.Marshal(params)
if err != nil {
return err
}
return json.Unmarshal(data, target)
}
func stringParam(params map[string]json.RawMessage, key string) string {
if params == nil {
return ""
}
var result string
_ = json.Unmarshal(params[key], &result)
return result
}
func intParam(params map[string]json.RawMessage, key string) int {
if params == nil {
return 0
}
var result int
_ = json.Unmarshal(params[key], &result)
return result
}
func boolParam(params map[string]json.RawMessage, key string) bool {
if params == nil {
return false
}
var result bool
_ = json.Unmarshal(params[key], &result)
return result
}
agents/drivers/etcd-go/main_test.go
+502
View File
@@ -0,0 +1,502 @@
package main
import (
"encoding/base64"
"encoding/json"
"strings"
"testing"
"go.etcd.io/etcd/api/v3/mvccpb"
)
func TestHandshakeAdvertisesCapabilities(t *testing.T) {
server := newRuntimeServer()
result, _, err := server.dispatch("handshake", nil)
if err != nil {
t.Fatalf("handshake failed: %v", err)
}
handshake := result.(map[string]any)
if handshake["protocolVersion"] != 2 || handshake["agentProtocolVersion"] != 2 {
t.Fatalf("unexpected protocol versions: %#v", handshake)
}
want := []string{
"connect", "test_connection", "kv", "kv_ttl", "kv_cas", "kv_list_values", "kv_status",
"kv_history", "etcd_compaction", "etcd_defrag", "etcd_watch", "etcd_lease", "etcd_auth",
"multi_session",
}
got, ok := handshake["capabilities"].([]string)
if !ok {
t.Fatalf("capabilities missing: %#v", handshake)
}
if strings.Join(got, ",") != strings.Join(want, ",") {
t.Fatalf("capability mismatch:\n got %v\nwant %v", got, want)
}
for _, capability := range got {
if capability == "structured_error_v1" {
t.Fatalf("structured_error_v1 must not be advertised (Java parity)")
}
}
}
func TestHandleLineReportsSessionNotFound(t *testing.T) {
server := newRuntimeServer()
response, _ := server.handleLine(`{"id":7,"method":"kv_get","params":{"key":"a"}}`)
if response.Error == nil {
t.Fatalf("expected error response, got %#v", response)
}
if response.Error.Code != -1 {
t.Fatalf("expected code -1, got %d", response.Error.Code)
}
if response.Error.Message != "Agent session not found: __legacy__" {
t.Fatalf("unexpected message: %q", response.Error.Message)
}
var raw map[string]json.RawMessage
if err := json.Unmarshal(response.ID, &raw); err == nil {
_ = raw
}
if string(response.ID) != "7" {
t.Fatalf("expected id 7, got %s", response.ID)
}
}
func TestOpenSessionValidatesAgentSessionID(t *testing.T) {
server := newRuntimeServer()
_, _, err := server.dispatch("open_session", map[string]json.RawMessage{})
if err == nil || !strings.Contains(err.Error(), "agentSessionId is required") {
t.Fatalf("expected agentSessionId requirement, got %v", err)
}
}
func TestOpenSessionRejectsDuplicate(t *testing.T) {
server := newRuntimeServer()
server.sessions["dup"] = &agentSession{state: newEtcdSession()}
_, _, err := server.dispatch("open_session", map[string]json.RawMessage{
"agentSessionId": json.RawMessage(`"dup"`),
})
if err == nil || !strings.Contains(err.Error(), "Agent session already exists") {
t.Fatalf("expected duplicate rejection, got %v", err)
}
}
func TestUnconnectedSessionErrors(t *testing.T) {
state := newEtcdSession()
if _, err := state.get(map[string]json.RawMessage{"key": json.RawMessage(`"k"`)}); err == nil || err.Error() != "Not connected" {
t.Fatalf("kv_get before connect: %v", err)
}
if _, err := state.history(map[string]json.RawMessage{"key": json.RawMessage(`"k"`)}); err == nil || err.Error() != "Not connected" {
t.Fatalf("kv_history before connect: %v", err)
}
if _, err := state.validateConnection(); err == nil || err.Error() != "Not connected" {
t.Fatalf("validate_connection before connect: %v", err)
}
}
func TestConnectTimeoutBounds(t *testing.T) {
cases := []struct {
configured int
want int
}{
{0, 30},
{-5, 1},
{1, 1},
{30, 30},
{300, 300},
{999, 300},
}
for _, testCase := range cases {
got := connectTimeoutSeconds(connectionParams{ConnectTimeoutSecs: testCase.configured})
if got != testCase.want {
t.Fatalf("connectTimeoutSeconds(%d) = %d, want %d", testCase.configured, got, testCase.want)
}
}
}
func TestGrpcMaxInboundMessageSize(t *testing.T) {
cases := []struct {
connection connectionParams
want int
}{
{connectionParams{}, defaultGrpcMaxInboundMessageSize},
{connectionParams{GrpcMaxInboundMessageSize: 1024}, minGrpcMaxInboundMessageSize},
{connectionParams{GrpcMaxInboundMessageSize: 999 * 1024 * 1024}, maxGrpcMaxInboundMessageSize},
{connectionParams{URLParams: "grpc_max_inbound_message_size=104857600"}, 100 * 1024 * 1024},
{connectionParams{URLParams: "?grpc_max_inbound_message_size=64"}, minGrpcMaxInboundMessageSize},
{connectionParams{URLParams: "grpc_max_inbound_message_size=abc"}, defaultGrpcMaxInboundMessageSize},
{connectionParams{GrpcMaxInboundMessageSize: 52428800, URLParams: "grpc_max_inbound_message_size=1024"}, 50 * 1024 * 1024},
}
for _, testCase := range cases {
got := grpcMaxInboundMessageSize(testCase.connection)
if got != testCase.want {
t.Fatalf("grpcMaxInboundMessageSize(%+v) = %d, want %d", testCase.connection, got, testCase.want)
}
}
}
func TestConnectionEndpoints(t *testing.T) {
cases := []struct {
name string
connection connectionParams
want []string
}{
{"host port fallback", connectionParams{Port: 2379}, []string{"http://127.0.0.1:2379"}},
{"ssl scheme", connectionParams{Host: "db", Port: 1, SSL: true}, []string{"https://db:1"}},
{"comma list", connectionParams{EtcdEndpoints: "a:1,b:2"}, []string{"http://a:1", "http://b:2"}},
{"newline list", connectionParams{Endpoints: "a:1\nb:2"}, []string{"http://a:1", "http://b:2"}},
{"precedence", connectionParams{EtcdEndpoints: "a:1", Endpoints: "b:2", ConnectionString: "c:3"}, []string{"http://a:1"}},
{"existing scheme", connectionParams{EtcdEndpoints: "https://a:1,http://b:2"}, []string{"https://a:1", "http://b:2"}},
{"blank entries", connectionParams{EtcdEndpoints: " a:1 , ,b:2"}, []string{"http://a:1", "http://b:2"}},
}
for _, testCase := range cases {
got := connectionEndpoints(testCase.connection)
if strings.Join(got, "|") != strings.Join(testCase.want, "|") {
t.Fatalf("%s: got %v, want %v", testCase.name, got, testCase.want)
}
}
}
func TestTLSConfigRequiresCertAndKeyTogether(t *testing.T) {
_, err := tlsConfigFor(connectionParams{SSL: true, ClientCertPath: "/tmp/cert.pem"})
if err == nil || err.Error() != "Client certificate and key must be provided together" {
t.Fatalf("expected paired cert/key error, got %v", err)
}
_, err = tlsConfigFor(connectionParams{SSL: true, CertPath: "/tmp/cert.pem", KeyPath: "/tmp/key.pem"})
if err == nil {
t.Fatalf("expected key pair load failure for missing files")
}
}
func TestHistoryStartRevision(t *testing.T) {
cases := []struct {
requested *int64
target int64
want int64
}{
{nil, 5000, 1},
{nil, 10000, 1},
{nil, 10001, 2},
{nil, 15000, 5001},
{int64Ptr(200), 15000, 200},
{int64Ptr(-5), 15000, 1},
}
for _, testCase := range cases {
got := historyStartRevision(testCase.requested, testCase.target)
if got != testCase.want {
t.Fatalf("historyStartRevision(%v, %d) = %d, want %d", testCase.requested, testCase.target, got, testCase.want)
}
}
}
func int64Ptr(value int64) *int64 {
return &value
}
func TestHistoryCollectorBounds(t *testing.T) {
collector := &historyCollector{limit: 3}
for i := int64(1); i <= 5; i++ {
collector.append(map[string]any{"revision": longString(i)})
}
if len(collector.rows) != 3 {
t.Fatalf("expected 3 retained rows, got %d", len(collector.rows))
}
if !collector.truncated {
t.Fatalf("expected truncated flag")
}
if collector.rows[0]["revision"] != "3" || collector.rows[2]["revision"] != "5" {
t.Fatalf("expected newest rows retained, got %v", collector.rows)
}
}
func TestPrefixEnd(t *testing.T) {
cases := []struct{ prefix, want string }{
{"", "\x00"},
{"abc", "abd"},
{"a\xff", "b"},
{"\xff\xff", "\x00"},
{"a\xffb", "a\xffc"},
}
for _, testCase := range cases {
if got := prefixEnd(testCase.prefix); got != testCase.want {
t.Fatalf("prefixEnd(%q) = %q, want %q", testCase.prefix, got, testCase.want)
}
}
}
func TestValueEncodings(t *testing.T) {
utf8Value := valueObject([]byte("hello"))
if utf8Value["encoding"] != "utf8" || utf8Value["data"] != "hello" {
t.Fatalf("unexpected utf8 value: %#v", utf8Value)
}
binaryValue := valueObject([]byte{0xff, 0xfe})
if binaryValue["encoding"] != "base64" {
t.Fatalf("expected base64 for binary: %#v", binaryValue)
}
roundTrip, err := parseValueObject(map[string]json.RawMessage{
"encoding": json.RawMessage(`"base64"`),
"data": json.RawMessage(`"` + binaryValue["data"].(string) + `"`),
})
if err != nil || roundTrip != string([]byte{0xff, 0xfe}) {
t.Fatalf("base64 round trip failed: %q %v", roundTrip, err)
}
if _, err := parseValueObject(map[string]json.RawMessage{"encoding": json.RawMessage(`"hex"`)}); err == nil {
t.Fatalf("expected unsupported encoding error")
}
if displayBytes([]byte{0xff}) != valueObject([]byte{0xff})["data"] {
t.Fatalf("displayBytes should fall back to base64")
}
}
func TestNextContinuation(t *testing.T) {
encoded := nextContinuation([]byte("key"))
decoded, err := decodeBase64String(encoded)
if err != nil {
t.Fatalf("invalid continuation: %v", err)
}
if decoded != "key\x00" {
t.Fatalf("unexpected continuation start: %q", decoded)
}
}
func decodeBase64String(value string) (string, error) {
decoded, err := base64Decode(value)
if err != nil {
return "", err
}
return string(decoded), nil
}
func TestWatchEventBufferBytes(t *testing.T) {
item := testKeyValue(3, 10, 20)
previous := testKeyValue(2, 5, 7)
// 512 + 4*3 (key) + 4*10 (value) + 4*5 (previous value) = 584
if got := watchEventBufferBytes(item, previous); got != 584 {
t.Fatalf("watchEventBufferBytes = %d, want 584", got)
}
previous.Version = 0
if got := watchEventBufferBytes(item, previous); got != 564 {
t.Fatalf("zero-version previous ignored: %d, want 564", got)
}
}
func TestWatchOverflowPreservesBufferedBatch(t *testing.T) {
session := newEtcdSession()
state := &watchState{watchID: "w", session: session}
smallBatch := []map[string]any{{"eventType": "put"}}
state.append(10, smallBatch, 128)
state.append(11, []map[string]any{{"eventType": "put"}}, maxWatchBufferBytes+1)
// The buffered batch stays pollable; the terminal rides along once the
// buffer has fully drained within the same poll (Java parity).
polled := state.poll()
if len(polled["batches"].([]any)) != 1 {
t.Fatalf("overflow must keep the buffered batch pollable: %#v", polled)
}
terminal, ok := polled["terminal"].(map[string]any)
if !ok || terminal["reason"] != "overflow" {
t.Fatalf("expected overflow terminal after drain, got %#v", polled)
}
if terminal["message"] != "ETCD_WATCH_OVERFLOW: the event buffer reached its byte or event limit" {
t.Fatalf("unexpected overflow message: %v", terminal["message"])
}
if session.watchBufferedBytesSnapshot() != 0 {
t.Fatalf("session budget not released: %d", session.watchBufferedBytesSnapshot())
}
}
func TestWatchPollReleasesSessionBudget(t *testing.T) {
session := newEtcdSession()
state := &watchState{watchID: "w", session: session}
state.append(1, []map[string]any{{"eventType": "put"}}, 1000)
if session.watchBufferedBytesSnapshot() != 1000 {
t.Fatalf("expected reserved budget 1000, got %d", session.watchBufferedBytesSnapshot())
}
state.poll()
if session.watchBufferedBytesSnapshot() != 0 {
t.Fatalf("expected budget released after poll, got %d", session.watchBufferedBytesSnapshot())
}
}
func TestSessionAggregateBudgetTerminatesOnlyOffendingWatch(t *testing.T) {
session := newEtcdSession()
first := &watchState{watchID: "a", session: session}
second := &watchState{watchID: "b", session: session}
if !session.reserveWatchBuffer(maxSessionWatchBufferBytes / 2) {
t.Fatalf("first reservation should succeed")
}
second.append(1, []map[string]any{{"eventType": "put"}}, maxSessionWatchBufferBytes/2+1)
polled := second.poll()
terminal, ok := polled["terminal"].(map[string]any)
if !ok || terminal["reason"] != "overflow" {
t.Fatalf("expected aggregate overflow on second watch: %#v", polled)
}
if first.hasTerminal() {
t.Fatalf("first watch must stay healthy")
}
session.releaseWatchBuffer(maxSessionWatchBufferBytes / 2)
}
func TestWatchPollRemovesTerminalWatch(t *testing.T) {
session := newEtcdSession()
state := &watchState{watchID: "w", session: session}
session.registerWatch("w", state)
state.fail("closed", "watch closed", nil)
params := map[string]json.RawMessage{"watchId": json.RawMessage(`"w"`)}
result, err := session.watchPoll(params)
if err != nil {
t.Fatalf("watchPoll failed: %v", err)
}
if _, ok := result.(map[string]any)["terminal"]; !ok {
t.Fatalf("expected terminal in poll result: %#v", result)
}
if session.watchCount() != 0 {
t.Fatalf("terminal watch must be removed from the session")
}
}
func TestWatchPollUnknownID(t *testing.T) {
session := newEtcdSession()
_, err := session.watchPoll(map[string]json.RawMessage{"watchId": json.RawMessage(`"missing"`)})
if err == nil || err.Error() != "ETCD_WATCH_NOT_FOUND: watch does not exist" {
t.Fatalf("expected watch-not-found, got %v", err)
}
}
func TestWatchStartLimitReached(t *testing.T) {
session := newEtcdSession()
for i := 0; i < maxWatches; i++ {
session.registerWatch(string(rune('a'+i)), &watchState{watchID: string(rune('a' + i)), session: session})
}
_, err := session.watchStart(map[string]json.RawMessage{"key": json.RawMessage(`"k"`)})
if err == nil || !strings.HasPrefix(err.Error(), "ETCD_WATCH_LIMIT") {
t.Fatalf("expected watch limit error, got %v", err)
}
}
func TestWatchScopeValidation(t *testing.T) {
session := newEtcdSession()
_, err := session.watchStart(map[string]json.RawMessage{"key": json.RawMessage(`"k"`), "scope": json.RawMessage(`"glob"`)})
if err == nil || err.Error() != "ETCD_WATCH_SCOPE_INVALID: scope must be key or prefix" {
t.Fatalf("expected scope error, got %v", err)
}
}
func TestLeasePageIDs(t *testing.T) {
ids := []uint64{30, 10, 20}
page := leasePageIDs(ids, nil, 2)
if page[0] != 10 || page[1] != 20 {
t.Fatalf("expected unsigned ascending order, got %v", page)
}
after := uint64(20)
page = leasePageIDs(ids, &after, 10)
if len(page) != 1 || page[0] != 30 {
t.Fatalf("expected only 30 after cursor 20, got %v", page)
}
// 2^63 boundary must compare unsigned, not signed.
big := []uint64{1, uint64(1) << 63}
page = leasePageIDs(big, nil, 2)
if page[0] != 1 || page[1] != uint64(1)<<63 {
t.Fatalf("unsigned ordering broken: %v", page)
}
}
func TestParseUnsignedLong(t *testing.T) {
parsed, err := parseUnsignedLong("18446744073709551615")
if err != nil || parsed != 18446744073709551615 {
t.Fatalf("uint64 max parse failed: %d %v", parsed, err)
}
if _, err := parseUnsignedLong("nope"); err == nil {
t.Fatalf("expected parse failure")
}
}
func TestAppendUnexecutedDefragMembers(t *testing.T) {
members := []map[string]any{{"endpoint": "a", "status": "failed"}}
appendUnexecutedDefragMembers(&members, []string{"a", "b", "c"}, "a")
if len(members) != 3 {
t.Fatalf("expected 3 rows, got %d", len(members))
}
for _, row := range members[1:] {
if row["status"] != "not_executed" || row["durationMs"] != nil || row["error"] != nil {
t.Fatalf("unexpected not_executed row: %#v", row)
}
}
}
func TestDefragRequiresEndpoints(t *testing.T) {
state := newEtcdSession()
_, err := state.defrag(map[string]json.RawMessage{})
if err == nil || err.Error() != "ETCD_DEFRAG_TARGET_REQUIRED: at least one endpoint is required" {
t.Fatalf("expected defrag target error, got %v", err)
}
}
func TestRequiredPositiveLongAndString(t *testing.T) {
_, err := requiredPositiveLong(map[string]json.RawMessage{"id": json.RawMessage(`0`)}, "id")
if err == nil || err.Error() != "ETCD_INVALID_ID: a positive integer is required" {
t.Fatalf("unexpected error: %v", err)
}
_, err = requiredString(map[string]json.RawMessage{}, "user")
if err == nil || err.Error() != "ETCD_USER_REQUIRED" {
t.Fatalf("unexpected error: %v", err)
}
}
func TestKeyBytesParam(t *testing.T) {
if _, err := keyBytesParam(map[string]json.RawMessage{}); err == nil || err.Error() != "Key is required" {
t.Fatalf("expected Key is required, got %v", err)
}
encoded := base64Encode([]byte{0x01, 0x02})
key, err := keyBytesParam(map[string]json.RawMessage{
"keyBytes": json.RawMessage(`{"encoding":"base64","data":"` + encoded + `"}`),
})
if err != nil || key != string([]byte{0x01, 0x02}) {
t.Fatalf("keyBytes decode failed: %q %v", key, err)
}
}
func TestPutRejectsConflictingLeaseOptions(t *testing.T) {
state := newEtcdSession()
params := map[string]json.RawMessage{
"key": json.RawMessage(`"k"`),
"value": json.RawMessage(`{"encoding":"utf8","data":"v"}`),
"ttl": json.RawMessage(`5`),
"lease": json.RawMessage(`7`),
}
// Java checks the active client before option exclusivity; without a
// connection the error must stay "Not connected".
_, err := state.put(params)
if err == nil || err.Error() != "Not connected" {
t.Fatalf("expected Not connected before option validation, got %v", err)
}
}
func (s *etcdSession) watchBufferedBytesSnapshot() int64 {
s.watchesMu.Lock()
defer s.watchesMu.Unlock()
return s.watchBufferedBytes
}
func (w *watchState) hasTerminal() bool {
w.mu.Lock()
defer w.mu.Unlock()
return w.terminalReason != ""
}
func testKeyValue(keySize, valueSize, version int64) *mvccpb.KeyValue {
return &mvccpb.KeyValue{
Key: bytesOfLength(keySize),
Value: bytesOfLength(valueSize),
Version: version,
}
}
func bytesOfLength(size int64) []byte {
return make([]byte, size)
}
func base64Decode(value string) ([]byte, error) {
return base64.StdEncoding.DecodeString(value)
}
func base64Encode(value []byte) string {
return base64.StdEncoding.EncodeToString(value)
}
agents/drivers/etcd-go/maintenance.go
+166
View File
@@ -0,0 +1,166 @@
package main
import (
"context"
"encoding/json"
"errors"
"time"
"go.etcd.io/etcd/api/v3/v3rpc/rpctypes"
clientv3 "go.etcd.io/etcd/client/v3"
)
func jsonUnmarshal(raw json.RawMessage, target any) error {
return json.Unmarshal(raw, target)
}
func (s *etcdSession) compact(params map[string]json.RawMessage) (any, error) {
revision, err := requiredPositiveLong(params, "revision")
if err != nil {
return nil, err
}
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
_, err = client.Get(ctx, "\x00", clientv3.WithRange("\x00"), clientv3.WithCountOnly(), clientv3.WithRev(revision))
cancel()
if err != nil {
if errors.Is(err, rpctypes.ErrCompacted) {
if compacted, ok := compactedRevisionOf(client, "\x00"); ok {
return nil, errors.New("ETCD_INVALID_REVISION: revision was already compacted at " + longString(compacted))
}
return nil, errors.New("ETCD_INVALID_REVISION: revision was already compacted")
}
return nil, err
}
currentCtx, currentCancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
current, err := client.Get(currentCtx, "\x00", clientv3.WithRange("\x00"), clientv3.WithCountOnly())
currentCancel()
if err != nil {
return nil, err
}
if revision > current.Header.Revision {
return nil, errors.New("ETCD_INVALID_REVISION: revision is newer than the current revision")
}
compactCtx, compactCancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
defer compactCancel()
if _, err := client.Compact(compactCtx, revision); err != nil {
return nil, err
}
return map[string]any{"revision": longString(revision)}, nil
}
func (s *etcdSession) defrag(params map[string]json.RawMessage) (any, error) {
var targets []string
if raw := params["endpoints"]; len(raw) > 0 && string(raw) != "null" {
var list []string
if jsonUnmarshal(raw, &list) == nil {
for _, endpoint := range list {
if endpoint != "" && !containsString(targets, endpoint) {
targets = append(targets, endpoint)
}
}
}
}
if len(targets) == 0 {
return nil, errors.New("ETCD_DEFRAG_TARGET_REQUIRED: at least one endpoint is required")
}
client, err := s.activeClient()
if err != nil {
return nil, err
}
members := []map[string]any{}
remaining := targets
for len(remaining) > 0 {
endpoint, err := nextDefragEndpoint(client, remaining)
if err != nil {
return nil, err
}
started := time.Now()
ctx, cancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
_, defragErr := client.Maintenance.Defragment(ctx, endpoint)
cancel()
durationMs := time.Since(started).Milliseconds()
if defragErr != nil {
members = append(members, map[string]any{
"endpoint": endpoint,
"status": "failed",
"durationMs": durationMs,
"error": defragErr.Error(),
})
appendUnexecutedDefragMembers(&members, remaining, endpoint)
break
}
members = append(members, map[string]any{
"endpoint": endpoint,
"status": "succeeded",
"durationMs": durationMs,
})
remaining = removeString(remaining, endpoint)
}
return map[string]any{"members": members}, nil
}
func appendUnexecutedDefragMembers(members *[]map[string]any, remaining []string, failedEndpoint string) {
for _, endpoint := range remaining {
if endpoint == failedEndpoint {
continue
}
*members = append(*members, map[string]any{
"endpoint": endpoint,
"status": "not_executed",
"durationMs": nil,
"error": nil,
})
}
}
// nextDefragEndpoint re-evaluates leadership before each member so a leader
// change is not defragmented early.
func nextDefragEndpoint(client *clientv3.Client, remaining []string) (string, error) {
var leaderEndpoint string
for _, endpoint := range remaining {
ctx, cancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
statusResponse, err := client.Maintenance.Status(ctx, endpoint)
cancel()
if err != nil {
// The following defragment call reports this member as the failed step.
continue
}
if statusResponse.Header.MemberId == statusResponse.Leader {
leaderEndpoint = endpoint
break
}
}
for _, endpoint := range remaining {
if endpoint != leaderEndpoint {
return endpoint, nil
}
}
if leaderEndpoint == "" {
return remaining[0], nil
}
return leaderEndpoint, nil
}
func containsString(values []string, target string) bool {
for _, value := range values {
if value == target {
return true
}
}
return false
}
func removeString(values []string, target string) []string {
result := make([]string, 0, len(values))
for _, value := range values {
if value != target {
result = append(result, value)
}
}
return result
}
agents/drivers/etcd-go/protocol_error.go
+143
View File
@@ -0,0 +1,143 @@
package main
import (
"context"
"encoding/json"
"errors"
"io"
"net"
"strings"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
type rpcError struct {
Code int `json:"code"`
Message string `json:"message"`
Data *rpcErrorData `json:"data,omitempty"`
}
type rpcErrorData struct {
Category string `json:"category"`
Retryable bool `json:"retryable"`
SessionDisposition string `json:"sessionDisposition"`
Stage string `json:"stage"`
ContractVersion int `json:"contractVersion"`
OperationOutcome string `json:"operationOutcome"`
ExceptionClass string `json:"exceptionClass,omitempty"`
AgentSessionID string `json:"agentSessionId,omitempty"`
}
func errorResponse(id json.RawMessage, method, agentSessionID string, err error) response {
return response{JSONRPC: "2.0", ID: id, Error: classifyRPCError(method, agentSessionID, err)}
}
func classifyRPCError(method, agentSessionID string, err error) *rpcError {
stage := rpcErrorStage(method)
data := &rpcErrorData{
Category: "protocol",
Retryable: false,
SessionDisposition: "keep",
Stage: stage,
ContractVersion: 1,
OperationOutcome: rpcOperationOutcome(stage),
ExceptionClass: safeRPCDiagnostic(errTypeName(err), 160),
AgentSessionID: strings.TrimSpace(agentSessionID),
}
switch {
case errors.Is(err, context.Canceled):
data.Category = "canceled"
data.SessionDisposition = "quarantine"
case errors.Is(err, context.DeadlineExceeded) || isTimeoutError(err):
data.Category = "timeout"
data.SessionDisposition = "quarantine"
case isUnavailableError(err):
data.Category = "connection"
data.Retryable = stage == "connect" || stage == "validate"
if stage != "connect" {
data.SessionDisposition = "quarantine"
}
}
return &rpcError{Code: -1, Message: err.Error(), Data: data}
}
// isUnavailableError treats gRPC UNAVAILABLE (etcd server unreachable) as a
// connection failure, matching the Java agent's transport classification.
func isUnavailableError(err error) bool {
if status.Code(err) == codes.Unavailable {
return true
}
if errors.Is(err, io.EOF) || errors.Is(err, net.ErrClosed) {
return true
}
var networkError *net.OpError
if errors.As(err, &networkError) {
return true
}
lower := strings.ToLower(err.Error())
for _, marker := range []string{
"connection refused", "connection reset", "broken pipe", "connection closed",
"connection lost", "unexpected eof", "no route to host",
} {
if strings.Contains(lower, marker) {
return true
}
}
return false
}
func errTypeName(err error) string {
if err == nil {
return ""
}
return strings.ReplaceAll(strings.TrimSpace(err.Error()), "\n", " ")
}
func rpcErrorStage(method string) string {
switch method {
case "connect", "open_session", "test_connection":
return "connect"
case "validate_connection", "validate_session":
return "validate"
case "cancel_session":
return "cancel"
case "close_session", "disconnect", "shutdown":
return "close"
case "etcd_watch_poll", "etcd_lease_get":
return "fetch"
case "handshake", "":
return "request"
default:
return "execute"
}
}
func rpcOperationOutcome(stage string) string {
switch stage {
case "request", "connect", "validate":
return "not_started"
default:
return "unknown"
}
}
func isTimeoutError(err error) bool {
var timeout interface{ Timeout() bool }
return errors.As(err, &timeout) && timeout.Timeout()
}
func safeRPCDiagnostic(value string, maxLength int) string {
var result strings.Builder
for _, char := range value {
if result.Len() >= maxLength {
break
}
if char >= 0x21 && char <= 0x7e {
result.WriteRune(char)
}
}
return result.String()
}
agents/drivers/etcd-go/status.go
+148
View File
@@ -0,0 +1,148 @@
package main
import (
"context"
"encoding/json"
"time"
"go.etcd.io/etcd/api/v3/etcdserverpb"
clientv3 "go.etcd.io/etcd/client/v3"
)
func (s *etcdSession) status(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
maintenance := client.Maintenance
membersByID := map[uint64]*etcdserverpb.Member{}
endpoints := s.connectedEndpointList()
memberCtx, memberCancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
memberList, err := client.Cluster.MemberList(memberCtx)
memberCancel()
if err == nil {
for _, member := range memberList.Members {
membersByID[member.ID] = member
for _, clientURL := range member.ClientURLs {
host := endpointHost(clientURL)
if host == "" || host == "0.0.0.0" || host == "::" {
continue
}
known := false
for _, existing := range endpoints {
if existing == clientURL {
known = true
break
}
}
if !known {
endpoints = append(endpoints, clientURL)
}
}
}
}
alarms := []string{}
alarmCtx, alarmCancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
alarmResponse, err := maintenance.AlarmList(alarmCtx)
alarmCancel()
if err != nil {
alarms = append(alarms, "UNAVAILABLE: "+err.Error())
} else {
for _, alarm := range alarmResponse.Alarms {
alarms = append(alarms, alarm.Alarm.String()+"@"+unsignedLongString(int64(alarm.MemberID)))
}
}
countCtx, countCancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
countResponse, err := client.Get(countCtx, "\x00", clientv3.WithRange("\x00"), clientv3.WithCountOnly())
countCancel()
if err != nil {
return nil, err
}
type statusResult struct {
response *clientv3.StatusResponse
err error
started time.Time
}
statusRequests := make([]chan *statusResult, len(endpoints))
for index, endpoint := range endpoints {
channel := make(chan *statusResult, 1)
statusRequests[index] = channel
go func(endpoint string, channel chan *statusResult) {
started := time.Now()
ctx, cancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
response, err := client.Maintenance.Status(ctx, endpoint)
cancel()
channel <- &statusResult{response: response, err: err, started: started}
}(endpoint, channel)
}
statusMembers := []map[string]any{}
observedMemberIDs := map[string]struct{}{}
var clusterID *string
var leaderID *string
for index, channel := range statusRequests {
result := <-channel
row := map[string]any{"endpoint": endpoints[index]}
if result.err != nil {
row["memberId"] = nil
row["name"] = nil
row["version"] = nil
row["leaderId"] = nil
row["revision"] = nil
row["raftTerm"] = nil
row["raftIndex"] = nil
row["raftAppliedIndex"] = nil
row["dbSize"] = nil
row["dbSizeInUse"] = nil
row["learner"] = false
row["reachable"] = false
row["latencyMs"] = nil
row["errors"] = []string{result.err.Error()}
statusMembers = append(statusMembers, row)
continue
}
memberStatus := result.response
memberID := memberStatus.Header.MemberId
member := membersByID[memberID]
clusterIDValue := unsignedLongString(int64(memberStatus.Header.ClusterId))
leaderIDValue := unsignedLongString(int64(memberStatus.Leader))
clusterID = &clusterIDValue
leaderID = &leaderIDValue
row["memberId"] = unsignedLongString(int64(memberID))
if member == nil {
row["name"] = nil
} else {
row["name"] = member.Name
}
row["version"] = memberStatus.Version
row["leaderId"] = unsignedLongString(int64(memberStatus.Leader))
row["revision"] = longString(memberStatus.Header.Revision)
row["raftTerm"] = longString(int64(memberStatus.RaftTerm))
row["raftIndex"] = longString(int64(memberStatus.RaftIndex))
row["raftAppliedIndex"] = longString(int64(memberStatus.RaftAppliedIndex))
row["dbSize"] = longString(memberStatus.DbSize)
row["dbSizeInUse"] = longString(memberStatus.DbSizeInUse)
row["learner"] = memberStatus.IsLearner
row["reachable"] = true
row["latencyMs"] = time.Since(result.started).Milliseconds()
memberIDKey := unsignedLongString(int64(memberID))
if _, seen := observedMemberIDs[memberIDKey]; seen {
continue
}
observedMemberIDs[memberIDKey] = struct{}{}
statusMembers = append(statusMembers, row)
}
result := map[string]any{
"clusterId": clusterID,
"revision": longString(countResponse.Header.Revision),
"leaderId": leaderID,
"keyCount": longString(countResponse.Count),
"alarms": alarms,
"members": statusMembers,
}
return result, nil
}
agents/drivers/etcd-go/watch.go
+366
View File
@@ -0,0 +1,366 @@
package main
import (
"context"
"crypto/rand"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"sync"
"time"
"go.etcd.io/etcd/api/v3/mvccpb"
clientv3 "go.etcd.io/etcd/client/v3"
)
const (
maxWatches = 4
maxWatchBatches = 256
maxWatchEvents = 10000
maxWatchBufferBytes = int64(8 * 1024 * 1024)
maxSessionWatchBufferBytes = int64(16 * 1024 * 1024)
)
type bufferedWatchBatch struct {
payload map[string]any
bufferedBytes int64
}
// watchState buffers watch events for the polling host. Terminal states are
// only surfaced after the buffer drains, mirroring the Java implementation.
type watchState struct {
watchID string
session *etcdSession
mu sync.Mutex
batches []bufferedWatchBatch
eventCount int
bufferedBytes int64
terminalReason string
terminalMessage string
compactedRevision *int64
watchCancel context.CancelFunc
closed bool
}
func (w *watchState) append(revision int64, events []map[string]any, batchBytes int64) {
w.mu.Lock()
defer w.mu.Unlock()
if w.terminalReason != "" {
return
}
if int64(len(w.batches)) >= maxWatchBatches ||
int64(w.eventCount+len(events)) > maxWatchEvents ||
batchBytes > maxWatchBufferBytes ||
w.bufferedBytes+batchBytes > maxWatchBufferBytes ||
!w.session.reserveWatchBuffer(batchBytes) {
w.overflowLocked()
return
}
w.batches = append(w.batches, bufferedWatchBatch{
payload: map[string]any{"revision": longString(revision), "events": events},
bufferedBytes: batchBytes,
})
w.eventCount += len(events)
w.bufferedBytes += batchBytes
}
func (w *watchState) overflowLocked() {
if w.terminalReason != "" {
return
}
w.terminalReason = "overflow"
w.terminalMessage = "ETCD_WATCH_OVERFLOW: the event buffer reached its byte or event limit"
w.closeWatcherLocked()
}
func (w *watchState) fail(reason, message string, compacted *int64) {
w.mu.Lock()
defer w.mu.Unlock()
if w.terminalReason == "" {
w.terminalReason = reason
w.terminalMessage = message
w.compactedRevision = compacted
}
}
func (w *watchState) poll() map[string]any {
w.mu.Lock()
defer w.mu.Unlock()
page := make([]any, 0, 64)
for len(w.batches) > 0 && len(page) < 64 {
batch := w.batches[0]
w.batches = w.batches[1:]
events := batch.payload["events"].([]map[string]any)
w.eventCount -= len(events)
w.bufferedBytes -= batch.bufferedBytes
w.session.releaseWatchBuffer(batch.bufferedBytes)
page = append(page, batch.payload)
}
result := map[string]any{"watchId": w.watchID, "batches": page}
if w.terminalReason != "" && len(w.batches) == 0 {
terminal := map[string]any{
"reason": w.terminalReason,
"message": w.terminalMessage,
}
if w.compactedRevision != nil {
terminal["compactedRevision"] = longString(*w.compactedRevision)
} else {
terminal["compactedRevision"] = nil
}
result["terminal"] = terminal
}
return result
}
func (w *watchState) close() {
w.mu.Lock()
w.failLocked("stopped", "watch stopped", nil)
w.clearBufferedLocked()
w.closed = true
cancel := w.watchCancel
w.watchCancel = nil
w.mu.Unlock()
if cancel != nil {
cancel()
}
}
func (w *watchState) failLocked(reason, message string, compacted *int64) {
if w.terminalReason == "" {
w.terminalReason = reason
w.terminalMessage = message
w.compactedRevision = compacted
}
}
func (w *watchState) clearBufferedLocked() {
if w.bufferedBytes > 0 {
w.session.releaseWatchBuffer(w.bufferedBytes)
}
w.batches = nil
w.eventCount = 0
w.bufferedBytes = 0
}
func (w *watchState) attachWatcher(cancel context.CancelFunc) {
w.mu.Lock()
defer w.mu.Unlock()
if w.closed || w.terminalReason != "" {
cancel()
return
}
w.watchCancel = cancel
}
func (w *watchState) closeWatcherLocked() {
cancel := w.watchCancel
w.watchCancel = nil
if cancel != nil {
go cancel()
}
}
func watchEventBufferBytes(item, previous *mvccpb.KeyValue) int64 {
bytes := int64(512 + estimatedBufferedBytes(len(item.Key)))
bytes += estimatedBufferedBytes(len(item.Value))
if previous != nil && previous.Version > 0 {
bytes += estimatedBufferedBytes(len(previous.Value))
}
return bytes
}
func estimatedBufferedBytes(sourceBytes int) int64 {
if sourceBytes < 0 {
return 0
}
return int64(sourceBytes) * 4
}
func newWatchID() string {
var bytes [16]byte
if _, err := rand.Read(bytes[:]); err != nil {
return base64.StdEncoding.EncodeToString(bytes[:])
}
bytes[6] = (bytes[6] & 0x0f) | 0x40
bytes[8] = (bytes[8] & 0x3f) | 0x80
hexed := hex.EncodeToString(bytes[:])
return fmt.Sprintf("%s-%s-%s-%s-%s", hexed[0:8], hexed[8:12], hexed[12:16], hexed[16:20], hexed[20:32])
}
func (s *etcdSession) watchCount() int {
s.watchesMu.Lock()
defer s.watchesMu.Unlock()
return len(s.watches)
}
func (s *etcdSession) registerWatch(id string, state *watchState) bool {
s.watchesMu.Lock()
defer s.watchesMu.Unlock()
if _, exists := s.watches[id]; exists {
return false
}
s.watches[id] = state
return true
}
func (s *etcdSession) removeWatch(id string) *watchState {
s.watchesMu.Lock()
defer s.watchesMu.Unlock()
state := s.watches[id]
delete(s.watches, id)
return state
}
func (s *etcdSession) watchStart(params map[string]json.RawMessage) (any, error) {
if s.watchCount() >= maxWatches {
return nil, fmt.Errorf("ETCD_WATCH_LIMIT: at most %d watches are allowed per connection", maxWatches)
}
key, err := keyBytesParam(params)
if err != nil {
return nil, err
}
scope := stringOrDefault(params, "scope", "key")
if scope != "key" && scope != "prefix" {
return nil, errors.New("ETCD_WATCH_SCOPE_INVALID: scope must be key or prefix")
}
client, err := s.activeClient()
if err != nil {
return nil, err
}
requestedRevision := longOrNull(params, "startRevision")
var startedRevision int64
if requestedRevision != nil && *requestedRevision > 0 {
startedRevision = *requestedRevision
} else {
ctx, cancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
response, err := client.Get(ctx, "\x00", clientv3.WithRange("\x00"), clientv3.WithCountOnly())
cancel()
if err != nil {
return nil, err
}
startedRevision = response.Header.Revision + 1
}
watchID := newWatchID()
state := &watchState{watchID: watchID, session: s}
options := []clientv3.OpOption{clientv3.WithRev(startedRevision)}
if boolOrDefault(params, "includePrevKv", false) {
options = append(options, clientv3.WithPrevKV())
}
if scope == "prefix" {
options = append(options, clientv3.WithRange(prefixEnd(key)))
}
watchCtx, watchCancel := context.WithCancel(context.Background())
channel := client.Watch(watchCtx, key, options...)
go consumeWatchChannel(channel, state)
state.attachWatcher(watchCancel)
s.registerWatch(watchID, state)
return map[string]any{"watchId": watchID, "startedRevision": longString(startedRevision)}, nil
}
func consumeWatchChannel(channel clientv3.WatchChan, state *watchState) {
for response := range channel {
if response.Err() != nil {
state.fail("error", response.Err().Error(), nil)
continue
}
if response.Canceled {
if response.CompactRevision > 0 {
compacted := response.CompactRevision
state.fail("compacted", "ETCD_COMPACTED", &compacted)
} else {
state.fail("closed", "watch closed", nil)
}
continue
}
if len(response.Events) == 0 {
continue
}
events := make([]map[string]any, 0, len(response.Events))
bufferedBytes := int64(128)
overflowed := false
for _, event := range response.Events {
item := event.Kv
var previous *mvccpb.KeyValue
if event.PrevKv != nil {
previous = event.PrevKv
}
bufferedBytes += watchEventBufferBytes(item, previous)
if bufferedBytes > maxWatchBufferBytes {
state.failOverflow()
overflowed = true
break
}
events = append(events, watchEventRow(event, item, previous))
}
if overflowed {
continue
}
state.append(response.Header.Revision, events, bufferedBytes)
}
}
func (w *watchState) failOverflow() {
w.mu.Lock()
defer w.mu.Unlock()
w.overflowLocked()
}
func watchEventRow(event *clientv3.Event, item, previous *mvccpb.KeyValue) map[string]any {
row := map[string]any{
"eventType": eventType(event),
"revision": longString(item.ModRevision),
"key": displayBytes(item.Key),
"keyBytes": bytesObject(item.Key),
}
if event.Type == mvccpb.DELETE {
row["value"] = nil
} else {
row["value"] = valueObject(item.Value)
}
if previous != nil && previous.Version > 0 {
row["previousValue"] = valueObject(previous.Value)
} else {
row["previousValue"] = nil
}
if event.Type == mvccpb.DELETE && previous != nil {
row["metadata"] = metadataMap(previous)
} else {
row["metadata"] = metadataMap(item)
}
return row
}
func eventType(event *clientv3.Event) string {
if event.Type == mvccpb.DELETE {
return "delete"
}
return "put"
}
func (s *etcdSession) watchPoll(params map[string]json.RawMessage) (any, error) {
watchID := stringOrDefault(params, "watchId", "")
s.watchesMu.Lock()
state := s.watches[watchID]
s.watchesMu.Unlock()
if state == nil {
return nil, errors.New("ETCD_WATCH_NOT_FOUND: watch does not exist")
}
result := state.poll()
if _, terminal := result["terminal"]; terminal {
if removed := s.removeWatch(watchID); removed != nil {
removed.close()
}
}
return result, nil
}
func (s *etcdSession) watchStop(params map[string]json.RawMessage) (any, error) {
if state := s.removeWatch(stringOrDefault(params, "watchId", "")); state != nil {
state.close()
}
return map[string]bool{"stopped": true}, nil
}
agents/drivers/etcd/build.gradle
-11
View File
@@ -1,11 +0,0 @@
dependencies {
implementation 'com.google.code.gson:gson:2.12.1'
implementation 'io.etcd:jetcd-core:0.8.6'
}
tasks.named('shadowJar') {
mergeServiceFiles()
manifest {
attributes('Agent-Label': 'etcd', 'Main-Class': 'com.dbx.agent.etcd.EtcdAgent')
}
}
agents/drivers/etcd/src/main/java/com/dbx/agent/etcd/EtcdAgent.java
-1796
View File
File diff suppressed because it is too large Load Diff
agents/drivers/etcd/src/test/java/com/dbx/agent/etcd/EtcdAgentTest.java
-475
View File
@@ -1,475 +0,0 @@
package com.dbx.agent.etcd;
import com.google.protobuf.ByteString;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import io.etcd.jetcd.ByteSequence;
import io.etcd.jetcd.KV;
import io.etcd.jetcd.Lease;
import io.etcd.jetcd.Txn;
import io.etcd.jetcd.api.KeyValue;
import io.etcd.jetcd.api.RangeResponse;
import io.etcd.jetcd.api.ResponseHeader;
import io.etcd.jetcd.kv.GetResponse;
import io.etcd.jetcd.kv.TxnResponse;
import java.lang.reflect.Proxy;
import java.nio.charset.StandardCharsets;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Deque;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
final class EtcdAgentTest {
@Test
void handshakeAdvertisesKvCapability() {
String response = EtcdAgent.handleRequest(
"{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"handshake\",\"params\":{}}"
);
JsonObject result = JsonParser.parseString(response).getAsJsonObject().getAsJsonObject("result");
Assertions.assertEquals(2, result.get("protocolVersion").getAsInt());
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"kv\"")));
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"kv_ttl\"")));
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"kv_cas\"")));
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"kv_list_values\"")));
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"kv_status\"")));
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"kv_history\"")));
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"etcd_compaction\"")));
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"etcd_defrag\"")));
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"etcd_watch\"")));
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"etcd_lease\"")));
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"etcd_auth\"")));
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"multi_session\"")));
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"connect\"")));
}
@Test
void endpointsUseConfiguredListAndScheme() {
JsonObject connection = JsonParser.parseString(
"{\"endpoints\":\"etcd-1:2379,https://etcd-2:2379\",\"ssl\":true}"
).getAsJsonObject();
Assertions.assertEquals(
List.of("https://etcd-1:2379", "https://etcd-2:2379"),
EtcdAgent.endpoints(connection)
);
}
@Test
void endpointsFallbackToHostPort() {
JsonObject connection = JsonParser.parseString(
"{\"host\":\"127.0.0.1\",\"port\":2379}"
).getAsJsonObject();
Assertions.assertEquals(List.of("http://127.0.0.1:2379"), EtcdAgent.endpoints(connection));
}
@Test
void connectTimeoutUsesConfiguredValueAndSafeBounds() {
Assertions.assertEquals(
45,
EtcdAgent.connectTimeoutSeconds(JsonParser.parseString("{\"connect_timeout_secs\":45}").getAsJsonObject())
);
Assertions.assertEquals(
1,
EtcdAgent.connectTimeoutSeconds(JsonParser.parseString("{\"connect_timeout_secs\":0}").getAsJsonObject())
);
Assertions.assertEquals(
300,
EtcdAgent.connectTimeoutSeconds(JsonParser.parseString("{\"connect_timeout_secs\":999}").getAsJsonObject())
);
}
@Test
void grpcInboundLimitDefaultsTo32MiBAndUsesSafeBounds() {
Assertions.assertEquals(
32 * 1024 * 1024,
EtcdAgent.grpcMaxInboundMessageSize(new JsonObject())
);
Assertions.assertEquals(
64 * 1024 * 1024,
EtcdAgent.grpcMaxInboundMessageSize(
JsonParser.parseString("{\"grpc_max_inbound_message_size\":67108864}").getAsJsonObject()
)
);
Assertions.assertEquals(
1024 * 1024,
EtcdAgent.grpcMaxInboundMessageSize(
JsonParser.parseString("{\"grpc_max_inbound_message_size\":0}").getAsJsonObject()
)
);
Assertions.assertEquals(
256 * 1024 * 1024,
EtcdAgent.grpcMaxInboundMessageSize(
JsonParser.parseString("{\"grpc_max_inbound_message_size\":536870912}").getAsJsonObject()
)
);
}
@Test
void grpcInboundLimitCanBeConfiguredThroughConnectionUrlParams() {
JsonObject connection = JsonParser.parseString(
"{\"url_params\":\"foo=bar&grpc_max_inbound_message_size=50331648\"}"
).getAsJsonObject();
Assertions.assertEquals(48 * 1024 * 1024, EtcdAgent.grpcMaxInboundMessageSize(connection));
}
@Test
void validateConnectionRequiresAnActiveSession() {
String response = EtcdAgent.handleRequest(
"{\"jsonrpc\":\"2.0\",\"id\":2,\"method\":\"validate_connection\",\"params\":{}}"
);
JsonObject error = JsonParser.parseString(response).getAsJsonObject().getAsJsonObject("error");
Assertions.assertEquals(-1, error.get("code").getAsInt());
Assertions.assertEquals("Not connected", error.get("message").getAsString());
}
@Test
void kvMethodDispatchReturnsJsonRpcErrorWhenDisconnected() {
String response = EtcdAgent.handleRequest(
"{\"jsonrpc\":\"2.0\",\"id\":2,\"method\":\"kv_get\",\"params\":{\"key\":\"/app/name\"}}"
);
JsonObject error = JsonParser.parseString(response).getAsJsonObject().getAsJsonObject("error");
Assertions.assertEquals(-1, error.get("code").getAsInt());
Assertions.assertEquals("Not connected", error.get("message").getAsString());
}
@Test
void historyMethodIsRegistered() {
String response = EtcdAgent.handleRequest(
"{\"jsonrpc\":\"2.0\",\"id\":3,\"method\":\"kv_history\",\"params\":{\"key\":\"/app/name\",\"limit\":100}}"
);
JsonObject error = JsonParser.parseString(response).getAsJsonObject().getAsJsonObject("error");
Assertions.assertEquals(-1, error.get("code").getAsInt());
Assertions.assertEquals("Not connected", error.get("message").getAsString());
}
@Test
void historyDefaultsToABoundedRevisionWindow() {
Assertions.assertEquals(1L, EtcdAgent.historyStartRevision(null, 100L));
Assertions.assertEquals(10_001L, EtcdAgent.historyStartRevision(null, 20_000L));
Assertions.assertEquals(42L, EtcdAgent.historyStartRevision(42L, 20_000L));
}
@Test
void historyRetainsOnlyTheNewestRequestedEvents() {
Deque<Integer> events = new ArrayDeque<>();
AtomicBoolean truncated = new AtomicBoolean();
for (int value = 1; value <= 5; value++) {
EtcdAgent.appendBoundedHistory(events, value, 3, truncated);
}
Assertions.assertEquals(List.of(3, 4, 5), List.copyOf(events));
Assertions.assertTrue(truncated.get());
}
@Test
void preserveLeaseRetriesWithTheLatestLeaseAfterConcurrentChange() throws Exception {
AtomicInteger getCalls = new AtomicInteger();
AtomicInteger txnCalls = new AtomicInteger();
KV active = scriptedKv(
getCalls,
txnCalls,
List.of(getResponse("/app/name", 10, 101), getResponse("/app/name", 11, 202)),
List.of(txnResponse(false, 10), txnResponse(true, 12))
);
long revision = EtcdAgent.putPreservingLease(active, bytes("/app/name"), bytes("updated"));
Assertions.assertEquals(12, revision);
Assertions.assertEquals(2, getCalls.get());
Assertions.assertEquals(2, txnCalls.get());
}
@Test
void preserveLeaseRejectsMissingOrUnleasedKeys() {
KV missing = scriptedKv(
new AtomicInteger(),
new AtomicInteger(),
List.of(emptyGetResponse()),
List.of()
);
KV unleased = scriptedKv(
new AtomicInteger(),
new AtomicInteger(),
List.of(getResponse("/app/name", 10, 0)),
List.of()
);
IllegalStateException missingError = Assertions.assertThrows(
IllegalStateException.class,
() -> EtcdAgent.putPreservingLease(missing, bytes("/app/name"), bytes("updated"))
);
IllegalStateException unleasedError = Assertions.assertThrows(
IllegalStateException.class,
() -> EtcdAgent.putPreservingLease(unleased, bytes("/app/name"), bytes("updated"))
);
Assertions.assertEquals("Cannot preserve lease: key does not exist or has no lease", missingError.getMessage());
Assertions.assertEquals("Cannot preserve lease: key does not exist or has no lease", unleasedError.getMessage());
}
@Test
void preserveLeaseFailsClearlyAfterBoundedConcurrentChanges() {
AtomicInteger getCalls = new AtomicInteger();
AtomicInteger txnCalls = new AtomicInteger();
KV active = scriptedKv(
getCalls,
txnCalls,
List.of(
getResponse("/app/name", 10, 101),
getResponse("/app/name", 11, 202),
getResponse("/app/name", 12, 303)
),
List.of(txnResponse(false, 10), txnResponse(false, 11), txnResponse(false, 12))
);
IllegalStateException error = Assertions.assertThrows(
IllegalStateException.class,
() -> EtcdAgent.putPreservingLease(active, bytes("/app/name"), bytes("updated"))
);
Assertions.assertEquals("Cannot preserve lease: key changed concurrently; retry the save", error.getMessage());
Assertions.assertEquals(3, getCalls.get());
Assertions.assertEquals(3, txnCalls.get());
}
@Test
void grantedLeaseIsRevokedWhenWriteFailsAfterCreation() {
AtomicInteger revokeCalls = new AtomicInteger();
AtomicInteger revokedLeaseId = new AtomicInteger();
Lease lease = (Lease) Proxy.newProxyInstance(
Lease.class.getClassLoader(),
new Class<?>[] { Lease.class },
(proxy, method, args) -> {
if (method.getName().equals("revoke")) {
revokeCalls.incrementAndGet();
revokedLeaseId.set(Math.toIntExact((long) args[0]));
return CompletableFuture.completedFuture(null);
}
if (method.getName().equals("close")) return null;
throw new UnsupportedOperationException(method.getName());
}
);
TimeoutException error = Assertions.assertThrows(
TimeoutException.class,
() -> EtcdAgent.cleanUpGrantedLeaseOnFailure(123, lease, () -> {
throw new TimeoutException("write timed out");
})
);
Assertions.assertEquals("write timed out", error.getMessage());
Assertions.assertEquals(1, revokeCalls.get());
Assertions.assertEquals(123, revokedLeaseId.get());
}
@Test
void defragFailureMarksEveryRemainingEndpointAsNotExecuted() {
List<Map<String, Object>> members = new ArrayList<>();
EtcdAgent.appendUnexecutedDefragMembers(
members,
List.of("http://follower-2:2379", "http://follower-3:2379", "http://leader:2379"),
"http://follower-2:2379"
);
Assertions.assertEquals(
List.of("http://follower-3:2379", "http://leader:2379"),
members.stream().map(member -> member.get("endpoint")).toList()
);
Assertions.assertTrue(members.stream().allMatch(member -> "not_executed".equals(member.get("status"))));
}
@Test
void leaseListResponseParserReadsEveryLeaseId() throws Exception {
byte[] response = new byte[] {
0x12, 0x02, 0x08, 0x7b,
0x12, 0x03, 0x08, (byte) 0xc8, 0x03
};
Assertions.assertEquals(List.of(123L, 456L), EtcdAgent.leaseIdsFromResponse(response));
}
@Test
void leaseListPageUsesAStableCursorAndBoundsTheResult() {
List<Long> leaseIds = new ArrayList<>();
for (long id = 1000; id >= 1; id--) leaseIds.add(id);
List<Long> first = EtcdAgent.leasePageIds(leaseIds, null, 101);
List<Long> second = EtcdAgent.leasePageIds(leaseIds, first.get(99), 101);
Assertions.assertEquals(101, first.size());
Assertions.assertEquals(1L, first.get(0));
Assertions.assertEquals(101L, first.get(100));
Assertions.assertEquals(101L, second.get(0));
Assertions.assertEquals(201L, second.get(100));
}
@Test
void watchBufferRejectsLargeValueAndPreviousValuePayloads() {
long payloadBytes = EtcdAgent.estimatedBufferedBytes(2 * 1024 * 1024)
+ EtcdAgent.estimatedBufferedBytes(2 * 1024 * 1024);
Assertions.assertTrue(payloadBytes > EtcdAgent.MAX_WATCH_BUFFER_BYTES);
}
@Test
void watchOverflowPreservesBufferedPayloadAndReportsTerminalState() {
EtcdAgent.EtcdSessionState session = new EtcdAgent.EtcdSessionState();
EtcdAgent.EtcdWatchState watch = new EtcdAgent.EtcdWatchState("watch-1", session);
List<Map<String, Object>> events = List.of(Map.of("eventType", "put"));
watch.append(1, events, 4L * 1024 * 1024);
Assertions.assertEquals(4L * 1024 * 1024, session.watchBufferedBytes());
watch.append(2, events, 5L * 1024 * 1024);
Map<String, Object> result = watch.poll();
Assertions.assertEquals(0, session.watchBufferedBytes());
List<?> batches = (List<?>) result.get("batches");
Assertions.assertEquals(1, batches.size());
Assertions.assertEquals("1", ((Map<?, ?>) batches.get(0)).get("revision"));
Assertions.assertEquals("overflow", ((Map<?, ?>) result.get("terminal")).get("reason"));
}
@Test
void watchPollReleasesTheSessionByteBudget() {
EtcdAgent.EtcdSessionState session = new EtcdAgent.EtcdSessionState();
EtcdAgent.EtcdWatchState watch = new EtcdAgent.EtcdWatchState("watch-1", session);
watch.append(1, List.of(Map.of("eventType", "put")), 1024);
Assertions.assertEquals(1024, session.watchBufferedBytes());
watch.poll();
Assertions.assertEquals(0, session.watchBufferedBytes());
}
@Test
void aggregateSessionBudgetTerminatesOnlyTheWatchThatExceedsIt() {
EtcdAgent.EtcdSessionState session = new EtcdAgent.EtcdSessionState();
List<Map<String, Object>> events = List.of(Map.of("eventType", "put"));
EtcdAgent.EtcdWatchState first = new EtcdAgent.EtcdWatchState("watch-1", session);
EtcdAgent.EtcdWatchState second = new EtcdAgent.EtcdWatchState("watch-2", session);
EtcdAgent.EtcdWatchState third = new EtcdAgent.EtcdWatchState("watch-3", session);
first.append(1, events, EtcdAgent.MAX_WATCH_BUFFER_BYTES);
second.append(1, events, EtcdAgent.MAX_WATCH_BUFFER_BYTES);
third.append(1, events, 1);
Assertions.assertEquals(EtcdAgent.MAX_SESSION_WATCH_BUFFER_BYTES, session.watchBufferedBytes());
Assertions.assertEquals("overflow", ((Map<?, ?>) third.poll().get("terminal")).get("reason"));
Assertions.assertEquals(EtcdAgent.MAX_SESSION_WATCH_BUFFER_BYTES, session.watchBufferedBytes());
}
@Test
void terminalPollRemovesTheWatchSlot() {
EtcdAgent.EtcdSessionState session = new EtcdAgent.EtcdSessionState();
EtcdAgent.EtcdWatchState watch = new EtcdAgent.EtcdWatchState("watch-1", session);
session.addWatch("watch-1", watch);
watch.overflow();
Map<String, Object> result = EtcdAgent.pollWatchState(session, "watch-1");
Assertions.assertEquals("overflow", ((Map<?, ?>) result.get("terminal")).get("reason"));
Assertions.assertEquals(0, session.watchCount());
}
private static KV scriptedKv(
AtomicInteger getCalls,
AtomicInteger txnCalls,
List<GetResponse> getResponses,
List<TxnResponse> txnResponses
) {
Deque<GetResponse> gets = new ArrayDeque<>(getResponses);
Deque<TxnResponse> txns = new ArrayDeque<>(txnResponses);
return (KV) Proxy.newProxyInstance(
KV.class.getClassLoader(),
new Class<?>[] { KV.class },
(proxy, method, args) -> {
if (method.getName().equals("get")) {
getCalls.incrementAndGet();
return CompletableFuture.completedFuture(gets.removeFirst());
}
if (method.getName().equals("txn")) {
return scriptedTxn(txnCalls, txns);
}
if (method.getName().equals("close")) {
return null;
}
throw new UnsupportedOperationException(method.getName());
}
);
}
private static Txn scriptedTxn(AtomicInteger txnCalls, Deque<TxnResponse> responses) {
return (Txn) Proxy.newProxyInstance(
Txn.class.getClassLoader(),
new Class<?>[] { Txn.class },
(proxy, method, args) -> {
if (Arrays.asList("If", "Then", "Else").contains(method.getName())) {
return proxy;
}
if (method.getName().equals("commit")) {
txnCalls.incrementAndGet();
return CompletableFuture.completedFuture(responses.removeFirst());
}
throw new UnsupportedOperationException(method.getName());
}
);
}
private static ByteSequence bytes(String value) {
return ByteSequence.from(value, StandardCharsets.UTF_8);
}
private static GetResponse getResponse(String key, long modRevision, long lease) {
KeyValue keyValue = KeyValue.newBuilder()
.setKey(ByteString.copyFromUtf8(key))
.setModRevision(modRevision)
.setLease(lease)
.build();
RangeResponse response = RangeResponse.newBuilder()
.setHeader(header(modRevision))
.addKvs(keyValue)
.build();
return new GetResponse(response, ByteSequence.EMPTY);
}
private static GetResponse emptyGetResponse() {
return new GetResponse(
RangeResponse.newBuilder().setHeader(header(1)).build(),
ByteSequence.EMPTY
);
}
private static TxnResponse txnResponse(boolean succeeded, long revision) {
io.etcd.jetcd.api.TxnResponse response = io.etcd.jetcd.api.TxnResponse.newBuilder()
.setHeader(header(revision))
.setSucceeded(succeeded)
.build();
return new TxnResponse(response, ByteSequence.EMPTY);
}
private static ResponseHeader header(long revision) {
return ResponseHeader.newBuilder().setRevision(revision).build();
}
}
agents/drivers/etcd2-go/README.md
+70
View File
@@ -0,0 +1,70 @@
# etcd v2 native agent (`etcd2`)
Native Go agent speaking the etcd **v2 HTTP/JSON API** with the standard DBX
agent protocol (NDJSON JSON-RPC over stdio). Registered as agentKey `etcd2`,
selected through the `etcd-v2` driver profile of the `etcd` connection type.
## Supported servers
| Server | Support |
| --- | --- |
| etcd 2.0–2.3 (pure v2) | fully supported; validated against 2.3.8 |
| etcd 3.0–3.5 started with `--enable-v2` | same code path |
| etcd 3.4–3.5 default configuration | rejected at connect: v2 is disabled by default (`ETCD_V2_API_DISABLED`) |
| etcd 3.6+ | impossible: the v2 API was removed upstream |
## Capabilities
`connect`, `test_connection`, `kv`, `kv_ttl`, `kv_cas`, `kv_list_values`,
`kv_status`, `etcd_watch`, `etcd_auth`, `multi_session`. The v3-only
capabilities (`kv_history`, `etcd_lease`, `etcd_compaction`, `etcd_defrag`)
are deliberately absent; the host blocks those calls before dispatch and the
UI hides the corresponding workspaces for `etcd-v2` connections.
## v2 API semantics and deliberate limits
- **TTL**: v2 attaches TTLs directly to keys (`?ttl=`). Key editing offers
permanent/TTL expiry only; binding a lease is rejected with
`ETCD_V2_LEASE_UNSUPPORTED` because v2 has no lease concept.
- **CAS**: writes use `prevIndex` (modifiedIndex) and `prevExist`, matching
the v3 modRevision contract. `expectedCreateRevision` is only honored when
`0` (key-must-not-exist); a nonzero value is rejected with
`ETCD_V2_CAS_UNSUPPORTED` because v2 does not expose createdIndex for
compare-and-swap.
- **Rename**: check-then-set, **not atomic** — v2 has no transaction
primitive. The source is verified by CAS before the target is created, but
a concurrent writer between the two requests is not serialized.
- **Watch**: long-poll loop (`?wait=true&waitIndex=N`, recursive for prefix
scopes). Idle poll timeouts are retried transparently. There are no
progress notifications; the same batch/event/byte budget system as the v3
agent applies.
- **Hidden directories**: v2 hides `_`-prefixed directory entries from
non-recursive listings; such keys are only reachable when addressed
explicitly, matching `etcdctl` behavior.
- **Auth**: users hold roles; roles hold read/write permissions over exact
keys and `/prefix/*` globs. Grants and revokes use the server's incremental
`grant`/`revoke` role PUT (`{"role":"r","grant":{"kv":{"read":["/p/*"]}}}`),
matching the etcd 2.3 client wire format. Resource mapping: `key` → bare
path, `prefix`/`all` → `/path/*` / `/*` globs.
- **Status**: reduced dashboard. `clusterId` comes from the
`X-Etcd-Cluster-Id` header, `revision` from `X-Etcd-Index`; Prometheus
metrics, alarms, dbSize, and raft details do not exist in v2 and stay null.
Raw `/v2/stats/store` counters are passed through as the `store` object.
## Connect probing
`connect` fetches `/version`, then probes `GET /v2/members`:
- `200` → ok; `403` → ok with `limited: true` (credentials work, root access
missing); `404` → `ETCD_V2_API_DISABLED`; `401` → `ETCD_UNAUTHENTICATED`.
## Testing
```bash
go test ./... # unit tests
DBX_ETCD2_LIVE=1 go test -run Live ./... # live flow (default endpoint below)
```
Live test environment variables: `DBX_ETCD2_ENDPOINTS`, `DBX_ETCD2_USER`,
`DBX_ETCD2_PASSWORD`. The default endpoint points at the team test server's
etcd 2.3.8 instance; see `docs/testing-databases.md` in the workspace root.
agents/drivers/etcd2-go/auth.go
+390
View File
@@ -0,0 +1,390 @@
package main
import (
"context"
"encoding/json"
"fmt"
"net/http"
"strings"
)
// v2 auth wire format (etcd 2.3 etcdserver/etcdhttp/client_auth.go):
// a role PUT body without grant/revoke creates the role; a body carrying
// grant/revoke updates it incrementally and must not carry permissions.
// Permission entries are glob patterns: a trailing "*" grants a prefix,
// anything else matches a single key exactly.
type v2RWPermission struct {
Read []string `json:"read"`
Write []string `json:"write"`
}
type v2Permissions struct {
KV v2RWPermission `json:"kv"`
}
type v2Role struct {
Role string `json:"role"`
Permissions *v2Permissions `json:"permissions"`
Grant *v2Permissions `json:"grant,omitempty"`
Revoke *v2Permissions `json:"revoke,omitempty"`
}
// v2UserRecord is the GET shape of /v2/auth/users/<name>: roles come back
// as full role objects, not plain names.
type v2UserRecord struct {
User string `json:"user"`
Roles []struct {
Role string `json:"role"`
} `json:"roles"`
}
// v2UserDocument is the PUT shape: grant/revoke carry role names.
type v2UserDocument struct {
User string `json:"user"`
Password string `json:"password,omitempty"`
Roles []string `json:"roles,omitempty"`
Grant []string `json:"grant,omitempty"`
Revoke []string `json:"revoke,omitempty"`
}
func (s *etcd2Session) authUserList(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
body, _, err := client.do(ctx, http.MethodGet, "/v2/auth/users", "", nil)
if err != nil {
return nil, err
}
var parsed struct {
Users []v2UserRecord `json:"users"`
}
if err := json.Unmarshal(body, &parsed); err != nil {
return nil, err
}
users := make([]string, 0, len(parsed.Users))
for _, user := range parsed.Users {
users = append(users, user.User)
}
return map[string]any{"users": users}, nil
}
func (s *etcd2Session) authUserGet(params map[string]json.RawMessage) (any, error) {
user, err := requiredString(params, "user")
if err != nil {
return nil, err
}
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
body, _, err := client.do(ctx, http.MethodGet, "/v2/auth/users/"+escapePathSegment(user), "", nil)
if err != nil {
return nil, err
}
var parsed v2UserRecord
if err := json.Unmarshal(body, &parsed); err != nil {
return nil, err
}
roles := make([]string, 0, len(parsed.Roles))
for _, role := range parsed.Roles {
roles = append(roles, role.Role)
}
return map[string]any{"user": parsed.User, "roles": roles}, nil
}
// authUserAdd relies on the create-or-update PUT: HTTP 201 means created,
// HTTP 200 means the user existed and only the password would change.
func (s *etcd2Session) authUserAdd(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
user, err := requiredString(params, "user")
if err != nil {
return nil, err
}
password, err := requiredString(params, "password")
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
payload := v2UserDocument{User: user, Password: password}
_, response, err := client.doJSON(ctx, http.MethodPut, "/v2/auth/users/"+escapePathSegment(user), payload)
if err != nil {
return nil, err
}
if response != nil && response.StatusCode == http.StatusOK {
return nil, fmt.Errorf("auth: User %s already exists.", user)
}
return map[string]bool{"created": true}, nil
}
func (s *etcd2Session) authUserDelete(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
user, err := requiredString(params, "user")
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
if _, _, err := client.do(ctx, http.MethodDelete, "/v2/auth/users/"+escapePathSegment(user), "", nil); err != nil {
return nil, err
}
return map[string]bool{"deleted": true}, nil
}
// authUserChangePassword reuses the create-or-update PUT: the server
// replaces the stored password hash when the user exists.
func (s *etcd2Session) authUserChangePassword(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
user, err := requiredString(params, "user")
if err != nil {
return nil, err
}
password, err := requiredString(params, "password")
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
payload := v2UserDocument{User: user, Password: password}
if _, _, err := client.doJSON(ctx, http.MethodPut, "/v2/auth/users/"+escapePathSegment(user), payload); err != nil {
return nil, err
}
return map[string]bool{"changed": true}, nil
}
func (s *etcd2Session) authUserGrantRevokeRole(params map[string]json.RawMessage, grant bool) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
user, err := requiredString(params, "user")
if err != nil {
return nil, err
}
role, err := requiredString(params, "role")
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
payload := v2UserDocument{User: user, Grant: []string{role}}
if !grant {
payload = v2UserDocument{User: user, Revoke: []string{role}}
}
if _, _, err := client.doJSON(ctx, http.MethodPut, "/v2/auth/users/"+escapePathSegment(user), payload); err != nil {
return nil, err
}
return map[string]bool{"updated": true}, nil
}
func (s *etcd2Session) authRoleList(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
body, _, err := client.do(ctx, http.MethodGet, "/v2/auth/roles", "", nil)
if err != nil {
return nil, err
}
var parsed struct {
Roles []v2Role `json:"roles"`
}
if err := json.Unmarshal(body, &parsed); err != nil {
return nil, err
}
roles := make([]string, 0, len(parsed.Roles))
for _, role := range parsed.Roles {
roles = append(roles, role.Role)
}
return map[string]any{"roles": roles}, nil
}
func (s *etcd2Session) authRoleGet(params map[string]json.RawMessage) (any, error) {
role, err := requiredString(params, "role")
if err != nil {
return nil, err
}
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
body, _, err := client.do(ctx, http.MethodGet, "/v2/auth/roles/"+escapePathSegment(role), "", nil)
if err != nil {
return nil, err
}
var parsed v2Role
if err := json.Unmarshal(body, &parsed); err != nil {
return nil, err
}
permissions := []map[string]any{}
if parsed.Permissions != nil {
permissions = append(permissions, v2PermissionRows("read", parsed.Permissions.KV.Read)...)
permissions = append(permissions, v2PermissionRows("write", parsed.Permissions.KV.Write)...)
}
return map[string]any{"role": parsed.Role, "permissions": permissions}, nil
}
// v2PermissionRows converts stored v2 patterns into the
// {access, key, rangeEnd, resource} rows the v3 agent emits: a trailing
// "*" displays as a prefix grant, anything else as an exact key.
func v2PermissionRows(access string, entries []string) []map[string]any {
rows := []map[string]any{}
for _, entry := range entries {
if strings.HasSuffix(entry, "*") {
base := strings.TrimSuffix(entry, "*")
rows = append(rows, map[string]any{
"access": access,
"key": bytesObject([]byte(base)),
"rangeEnd": bytesObject([]byte(prefixEnd(base))),
"resource": "prefix",
})
} else {
rows = append(rows, map[string]any{
"access": access,
"key": bytesObject([]byte(entry)),
"rangeEnd": bytesObject(nil),
"resource": "key",
})
}
}
return rows
}
func (s *etcd2Session) authRoleAdd(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
role, err := requiredString(params, "role")
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
payload := v2Role{Role: role}
if _, _, err := client.doJSON(ctx, http.MethodPut, "/v2/auth/roles/"+escapePathSegment(role), payload); err != nil {
return nil, err
}
return map[string]bool{"created": true}, nil
}
func (s *etcd2Session) authRoleDelete(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
role, err := requiredString(params, "role")
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
if _, _, err := client.do(ctx, http.MethodDelete, "/v2/auth/roles/"+escapePathSegment(role), "", nil); err != nil {
return nil, err
}
return map[string]bool{"deleted": true}, nil
}
// v2PermissionPattern maps the protocol's resource model onto the v2 glob
// convention: exact keys stay bare, prefixes and "all" become "/path/*".
func v2PermissionPattern(resource, key string) string {
switch resource {
case "all":
return "/*"
case "prefix":
return strings.TrimSuffix(key, "/") + "/*"
default:
return key
}
}
// authRolePermission maps grants/revokes onto the server's incremental
// grant/revoke role update (one PUT per access direction).
func (s *etcd2Session) authRolePermission(params map[string]json.RawMessage, grant bool) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
role, err := requiredString(params, "role")
if err != nil {
return nil, err
}
resource := stringOrDefault(params, "resource", "key")
key, err := keyBytesParam(params)
if err != nil {
return nil, err
}
if !strings.HasPrefix(key, "/") {
key = "/" + key
}
pattern := v2PermissionPattern(resource, key)
var accesses []string
if grant {
access := strings.ToUpper(stringOrDefault(params, "access", ""))
switch access {
case "READ":
accesses = []string{"read"}
case "WRITE":
accesses = []string{"write"}
case "READWRITE":
accesses = []string{"read", "write"}
default:
return nil, fmt.Errorf("ETCD_INVALID_ACCESS: access must be READ, WRITE, or READWRITE, got %s", access)
}
} else {
accesses = []string{"read", "write"}
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
for _, access := range accesses {
document := v2Role{Role: role}
direction := v2Permissions{KV: v2RWPermission{}}
if access == "read" {
direction.KV.Read = []string{pattern}
} else {
direction.KV.Write = []string{pattern}
}
if grant {
document.Grant = &direction
} else {
document.Revoke = &direction
}
if _, _, err := client.doJSON(ctx, http.MethodPut, "/v2/auth/roles/"+escapePathSegment(role), document); err != nil {
return nil, err
}
}
return map[string]bool{"updated": true}, nil
}
func (c *authenticatedClient) doJSON(ctx context.Context, method, path string, payload any) ([]byte, *http.Response, error) {
encoded, err := json.Marshal(payload)
if err != nil {
return nil, nil, err
}
return c.do(ctx, method, path, string(encoded), map[string]string{"Content-Type": "application/json"})
}
func escapePathSegment(segment string) string {
return strings.ReplaceAll(segment, "/", "%2F")
}
agents/drivers/etcd2-go/client.go
+457
View File
@@ -0,0 +1,457 @@
package main
import (
"context"
"crypto/tls"
"crypto/x509"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"strings"
"time"
)
const (
defaultPort = 2379
operationTimeout = 30 * time.Second
)
// connectionParams mirrors the host-side etcd connection configuration.
// gRPC-specific options are accepted and ignored: the v2 API is plain HTTP.
type connectionParams struct {
EtcdEndpoints string `json:"etcd_endpoints"`
Endpoints string `json:"endpoints"`
ConnectionString string `json:"connection_string"`
Host string `json:"host"`
Port int `json:"port"`
Username string `json:"username"`
Password string `json:"password"`
SSL bool `json:"ssl"`
CACertPath string `json:"ca_cert_path"`
ClientCertPath string `json:"client_cert_path"`
ClientKeyPath string `json:"client_key_path"`
CertPath string `json:"cert_path"`
KeyPath string `json:"key_path"`
ConnectTimeoutSecs int `json:"connect_timeout_secs"`
GrpcMaxInboundMessageSize int `json:"grpc_max_inbound_message_size"`
URLParams string `json:"url_params"`
}
// authenticatedClient issues v2 API requests against one etcd endpoint with
// optional basic auth and TLS.
type authenticatedClient struct {
endpoint string
http *http.Client
username string
password string
serverVersion string
}
func connectionObject(params map[string]json.RawMessage) (connectionParams, error) {
var connection connectionParams
if raw, ok := params["connection"]; ok && raw != nil {
if err := json.Unmarshal(raw, &connection); err != nil {
return connection, err
}
return connection, nil
}
if err := decodeParams(params, &connection); err != nil {
return connection, err
}
return connection, nil
}
func (s *etcd2Session) connect(params map[string]json.RawMessage) (any, error) {
connection, err := connectionObject(params)
if err != nil {
return nil, err
}
endpointList := connectionEndpoints(connection)
var lastErr error
for _, endpoint := range endpointList {
client, _, err := probeClient(endpoint, connection)
if err != nil {
lastErr = err
continue
}
s.close()
s.clientMu.Lock()
s.httpClient = client
s.connectedEndpoints = endpointList
s.serverVersion = client.serverVersion
s.clientMu.Unlock()
return map[string]bool{"ok": true}, nil
}
if lastErr == nil {
lastErr = errors.New("No etcd endpoint configured")
}
return nil, lastErr
}
func (s *etcd2Session) activeClient() (*authenticatedClient, error) {
s.clientMu.Lock()
client := s.httpClient
s.clientMu.Unlock()
if client == nil {
return nil, errors.New("Not connected")
}
return client, nil
}
func (s *etcd2Session) connectedEndpointList() []string {
s.clientMu.Lock()
defer s.clientMu.Unlock()
return append([]string(nil), s.connectedEndpoints...)
}
func (s *etcd2Session) validateConnection() (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), operationTimeout)
defer cancel()
probe, err := client.probeV2(ctx)
if err != nil {
return nil, err
}
return probe, nil
}
func (s *etcd2Session) close() error {
s.watchesMu.Lock()
watches := s.watches
s.watches = map[string]*watchState{}
s.watchBufferedBytes = 0
s.watchesMu.Unlock()
for _, watch := range watches {
watch.close()
}
s.clientMu.Lock()
s.httpClient = nil
s.connectedEndpoints = nil
s.serverVersion = ""
s.clientMu.Unlock()
return nil
}
func buildHTTPClient(connection connectionParams) (*http.Client, error) {
transport := &http.Transport{
MaxIdleConns: 4,
MaxIdleConnsPerHost: 4,
}
if connection.SSL {
tlsConfig, err := tlsConfigFor(connection)
if err != nil {
return nil, err
}
transport.TLSClientConfig = tlsConfig
}
return &http.Client{
Transport: transport,
// The request context governs per-call timeouts; the client-level
// timeout stays high so long-poll watches are not cut short.
Timeout: 10 * time.Minute,
}, nil
}
func connectTimeoutSeconds(connection connectionParams) time.Duration {
seconds := connection.ConnectTimeoutSecs
if seconds == 0 {
seconds = 30
}
if seconds < 1 {
seconds = 1
}
if seconds > 300 {
seconds = 300
}
return time.Duration(seconds) * time.Second
}
func tlsConfigFor(connection connectionParams) (*tls.Config, error) {
tlsConfig := &tls.Config{}
if ca := strings.TrimSpace(connection.CACertPath); ca != "" {
authorityPEM, err := os.ReadFile(ca)
if err != nil {
return nil, err
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(authorityPEM) {
return nil, fmt.Errorf("failed to parse CA certificate at %s", ca)
}
tlsConfig.RootCAs = pool
}
certPath := firstNonBlank(connection.ClientCertPath, connection.CertPath)
keyPath := firstNonBlank(connection.ClientKeyPath, connection.KeyPath)
if (certPath == "") != (keyPath == "") {
return nil, errors.New("Client certificate and key must be provided together")
}
if certPath != "" {
pair, err := tls.LoadX509KeyPair(certPath, keyPath)
if err != nil {
return nil, err
}
tlsConfig.Certificates = []tls.Certificate{pair}
}
return tlsConfig, nil
}
// probeClient verifies the endpoint speaks the v2 API. It returns the
// connected client and a probe result shaped like the v3 agent's:
// {ok, endpoint, limited?}.
func probeClient(endpoint string, connection connectionParams) (*authenticatedClient, map[string]any, error) {
httpClient, err := buildHTTPClient(connection)
if err != nil {
return nil, nil, err
}
client := &authenticatedClient{
endpoint: strings.TrimSuffix(endpoint, "/"),
http: httpClient,
username: connection.Username,
password: connection.Password,
}
dialCtx, dialCancel := context.WithTimeout(context.Background(), connectTimeoutSeconds(connection))
defer dialCancel()
version, err := client.fetchVersion(dialCtx)
if err != nil {
return nil, nil, err
}
client.serverVersion = version.etcdserver
probe, err := client.probeV2(dialCtx)
if err != nil {
return nil, nil, err
}
return client, probe, nil
}
type etcdVersion struct {
etcdserver string
etcdcluster string
}
func (c *authenticatedClient) fetchVersion(ctx context.Context) (etcdVersion, error) {
body, _, err := c.do(ctx, http.MethodGet, "/version", "", nil)
if err != nil {
return etcdVersion{}, err
}
var parsed struct {
Etcdserver string `json:"etcdserver"`
Etcdcluster string `json:"etcdcluster"`
}
if err := json.Unmarshal(body, &parsed); err != nil {
return etcdVersion{}, fmt.Errorf("unrecognized etcd /version response from %s: %w", c.endpoint, err)
}
return etcdVersion{etcdserver: parsed.Etcdserver, etcdcluster: parsed.Etcdcluster}, nil
}
// probeV2 checks that the v2 keys API is actually served. A 403 proves the
// channel and credentials reached etcd, mirroring the v3 agent's
// PERMISSION_DENIED handling for restricted users.
func (c *authenticatedClient) probeV2(ctx context.Context) (map[string]any, error) {
response, err := c.request(ctx, http.MethodGet, "/v2/members", "", nil)
if err != nil {
return nil, err
}
defer drainClose(response.Body)
switch response.StatusCode {
case http.StatusOK:
return map[string]any{"ok": true, "endpoint": c.endpoint}, nil
case http.StatusForbidden:
return map[string]any{"ok": true, "endpoint": c.endpoint, "limited": true}, nil
case http.StatusNotFound:
return nil, fmt.Errorf("ETCD_V2_API_DISABLED: %s does not expose the etcd v2 API (removed in etcd 3.6+)", c.endpoint)
case http.StatusUnauthorized:
return nil, fmt.Errorf("ETCD_UNAUTHENTICATED: authentication failed against %s", c.endpoint)
default:
body, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
return nil, fmt.Errorf("etcd v2 probe against %s failed: HTTP %d %s", c.endpoint, response.StatusCode, strings.TrimSpace(string(body)))
}
}
// do performs a v2 API request and returns the body. Non-2xx responses are
// converted into etcdError values carrying the server's errorCode/message.
func (c *authenticatedClient) do(ctx context.Context, method, path, body string, header map[string]string) ([]byte, *http.Response, error) {
response, err := c.request(ctx, method, path, body, header)
if err != nil {
return nil, response, err
}
payload, readErr := io.ReadAll(response.Body)
_ = response.Body.Close()
if response.StatusCode < 200 || response.StatusCode >= 300 {
return nil, response, errorFromResponse(response.StatusCode, payload)
}
if readErr != nil {
return nil, response, readErr
}
return payload, response, nil
}
func (c *authenticatedClient) request(ctx context.Context, method, path, body string, header map[string]string) (*http.Response, error) {
return c.requestAt(ctx, method, c.endpoint, path, body, header)
}
// requestAt targets a specific endpoint, used for per-member status fan-out.
func (c *authenticatedClient) requestAt(ctx context.Context, method, endpoint, path, body string, header map[string]string) (*http.Response, error) {
var reader io.Reader
if body != "" {
reader = strings.NewReader(body)
}
req, err := http.NewRequestWithContext(ctx, method, strings.TrimSuffix(endpoint, "/")+path, reader)
if err != nil {
return nil, err
}
if c.username != "" {
req.SetBasicAuth(c.username, c.password)
}
for key, value := range header {
req.Header.Set(key, value)
}
if body != "" && req.Header.Get("Content-Type") == "" {
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
}
response, err := c.http.Do(req)
if err != nil {
// gRPC-style sentinel so the host-side transient-error handling keeps working.
return nil, fmt.Errorf("etcd connection error: %w", err)
}
return response, nil
}
// doAt performs a v2 API request against a specific endpoint.
func (c *authenticatedClient) doAt(ctx context.Context, method, endpoint, path, body string) ([]byte, *http.Response, error) {
response, err := c.requestAt(ctx, method, endpoint, path, body, nil)
if err != nil {
return nil, nil, err
}
payload, readErr := io.ReadAll(response.Body)
_ = response.Body.Close()
if response.StatusCode < 200 || response.StatusCode >= 300 {
return nil, response, errorFromResponse(response.StatusCode, payload)
}
if readErr != nil {
return nil, response, readErr
}
return payload, response, nil
}
// etcdError carries the v2 API error payload (errorCode + message + index).
type etcdError struct {
statusCode int
errorCode int64
message string
cause string
index int64
}
func (e *etcdError) Error() string {
if e.message == "" {
return fmt.Sprintf("etcd v2 error (HTTP %d)", e.statusCode)
}
if e.cause != "" {
return fmt.Sprintf("%s (%s)", e.message, e.cause)
}
return e.message
}
func errorFromResponse(statusCode int, body []byte) *etcdError {
parsed := &etcdError{statusCode: statusCode}
if len(body) > 0 {
var payload struct {
ErrorCode int64 `json:"errorCode"`
Message string `json:"message"`
Cause string `json:"cause"`
Index int64 `json:"index"`
}
if json.Unmarshal(body, &payload) == nil && payload.Message != "" {
parsed.errorCode = payload.ErrorCode
parsed.message = payload.Message
parsed.cause = payload.Cause
parsed.index = payload.Index
return parsed
}
}
parsed.message = strings.TrimSpace(string(body))
return parsed
}
func isEtcdErrorCode(err error, code int64) bool {
var etcdErr *etcdError
if errors.As(err, &etcdErr) {
return etcdErr.errorCode == code
}
return false
}
func isNotFound(err error) bool {
return isEtcdErrorCode(err, 100)
}
func isCompareFailed(err error) bool {
return isEtcdErrorCode(err, 101)
}
func drainClose(body io.ReadCloser) {
_, _ = io.Copy(io.Discard, io.LimitReader(body, 4096))
_ = body.Close()
}
func connectionEndpoints(connection connectionParams) []string {
configured := firstNonBlank(connection.EtcdEndpoints, connection.Endpoints, connection.ConnectionString)
var result []string
if configured != "" {
for _, endpoint := range strings.FieldsFunc(configured, func(r rune) bool { return r == ',' || r == '\n' }) {
normalized := normalizeEndpoint(strings.TrimSpace(endpoint), connection.SSL)
if normalized != "" {
result = append(result, normalized)
}
}
}
if len(result) == 0 {
host := connection.Host
if host == "" {
host = "127.0.0.1"
}
port := connection.Port
if port == 0 {
port = defaultPort
}
result = append(result, normalizeEndpoint(fmt.Sprintf("%s:%d", host, port), connection.SSL))
}
return result
}
func normalizeEndpoint(endpoint string, tlsEnabled bool) string {
if strings.TrimSpace(endpoint) == "" {
return ""
}
if strings.HasPrefix(endpoint, "http://") || strings.HasPrefix(endpoint, "https://") {
return strings.TrimSuffix(endpoint, "/")
}
scheme := "http"
if tlsEnabled {
scheme = "https"
}
return scheme + "://" + endpoint
}
func endpointHost(rawEndpoint string) string {
parsed, err := url.Parse(rawEndpoint)
if err != nil {
return ""
}
return parsed.Hostname()
}
func firstNonBlank(values ...string) string {
for _, value := range values {
if strings.TrimSpace(value) != "" {
return value
}
}
return ""
}
agents/drivers/etcd2-go/go.mod
+3
View File
@@ -0,0 +1,3 @@
module github.com/t8y2/dbx/agents/drivers/etcd2-go
go 1.24
agents/drivers/etcd2-go/integration_test.go
+412
View File
@@ -0,0 +1,412 @@
package main
import (
"encoding/json"
"fmt"
"os"
"strings"
"testing"
"time"
)
// TestLiveEtcd2Agent exercises the v2 agent surface against a real etcd 2.x
// server. Enable with DBX_ETCD2_LIVE=1; configure via DBX_ETCD2_ENDPOINTS,
// DBX_ETCD2_USER, and DBX_ETCD2_PASSWORD (defaults match the server
// deployment: root/123456).
func TestLiveEtcd2Agent(t *testing.T) {
if os.Getenv("DBX_ETCD2_LIVE") != "1" {
t.Skip("set DBX_ETCD2_LIVE=1 to run the live etcd v2 agent test")
}
endpoints := envOrDefault("DBX_ETCD2_ENDPOINTS", "http://172.26.129.83:20041")
user := envOrDefault("DBX_ETCD2_USER", "root")
password := envOrDefault("DBX_ETCD2_PASSWORD", "123456")
state := newEtcd2Session()
connectParams := map[string]json.RawMessage{
"etcd_endpoints": json.RawMessage(`"` + endpoints + `"`),
"username": json.RawMessage(`"` + user + `"`),
"password": json.RawMessage(`"` + password + `"`),
}
if _, err := state.connect(connectParams); err != nil {
t.Fatalf("connect failed: %v", err)
}
defer state.close()
if result, err := state.validateConnection(); err != nil {
t.Fatalf("validate_connection failed: %v", err)
} else if probe := result.(map[string]any); probe["ok"] != true {
t.Fatalf("unexpected probe result: %#v", probe)
}
prefix := fmt.Sprintf("/dbx/live/%d/", time.Now().UnixNano())
// --- KV basics -----------------------------------------------------
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "hello"),
"value": jsonValue("world"),
})); err != nil {
t.Fatalf("put failed: %v", err)
}
fetched, err := state.get(paramsWith(map[string]json.RawMessage{"key": jsonString(prefix + "hello")}))
if err != nil {
t.Fatalf("get failed: %v", err)
}
row := fetched.(map[string]any)
if row["found"] != true || row["key"] != prefix+"hello" {
t.Fatalf("unexpected get result: %#v", row)
}
value := row["value"].(map[string]any)
if value["encoding"] != "utf8" || value["data"] != "world" {
t.Fatalf("unexpected value: %#v", value)
}
metadata := row["metadata"].(map[string]any)
if metadata["createRevision"] != metadata["modRevision"] {
t.Fatalf("fresh key indexes should match: %#v", metadata)
}
missing, err := state.get(paramsWith(map[string]json.RawMessage{"key": jsonString(prefix + "missing")}))
if err != nil {
t.Fatalf("missing get failed: %v", err)
}
if missing.(map[string]any)["found"] != false {
t.Fatalf("expected found=false: %#v", missing)
}
// CAS conflict on stale index.
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "hello"),
"value": jsonValue("v2"),
"expectedModRevision": json.RawMessage(`1`),
})); err == nil || err.Error() != "ETCD_CAS_CONFLICT: key changed after it was loaded" {
t.Fatalf("expected CAS conflict, got %v", err)
}
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "hello"),
"value": jsonValue("v3"),
"expectedModRevision": json.RawMessage(metadata["modRevision"].(string)),
})); err != nil {
t.Fatalf("CAS put failed: %v", err)
}
// TTL put and metadata.
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "leased"),
"value": jsonValue("expiring"),
"ttl": json.RawMessage(`300`),
})); err != nil {
t.Fatalf("ttl put failed: %v", err)
}
leased, err := state.get(paramsWith(map[string]json.RawMessage{"key": jsonString(prefix + "leased")}))
if err != nil {
t.Fatalf("leased get failed: %v", err)
}
if _, hasTTL := leased.(map[string]any)["metadata"].(map[string]any)["ttl"]; !hasTTL {
t.Fatalf("leased key must expose ttl metadata: %#v", leased)
}
// Lease objects are a v3 concept and must be rejected clearly.
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "nolease"),
"value": jsonValue("v"),
"lease": json.RawMessage(`5`),
})); err == nil || !strings.HasPrefix(err.Error(), "ETCD_V2_LEASE_UNSUPPORTED") {
t.Fatalf("expected lease rejection, got %v", err)
}
// Rename including conflict and missing-source paths.
if _, err := state.rename(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "old"),
"newKey": jsonString(prefix + "new"),
})); err == nil || err.Error() != "ETCD_NOT_FOUND: source key does not exist" {
t.Fatalf("expected rename not-found, got %v", err)
}
if _, err := state.rename(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "hello"),
"newKey": jsonString(prefix + "renamed"),
})); err != nil {
t.Fatalf("rename failed: %v", err)
}
renamed, err := state.get(paramsWith(map[string]json.RawMessage{"key": jsonString(prefix + "renamed")}))
if err != nil || renamed.(map[string]any)["found"] != true {
t.Fatalf("rename target missing: %#v %v", renamed, err)
}
if renamed.(map[string]any)["value"].(map[string]any)["data"] != "v3" {
t.Fatalf("rename must carry the value: %#v", renamed)
}
source, err := state.get(paramsWith(map[string]json.RawMessage{"key": jsonString(prefix + "hello")}))
if err != nil || source.(map[string]any)["found"] != false {
t.Fatalf("rename must remove the source: %#v %v", source, err)
}
if _, err := state.rename(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "leased"),
"newKey": jsonString(prefix + "renamed"),
})); err == nil || err.Error() != "ETCD_CAS_CONFLICT: source changed or target already exists" {
t.Fatalf("expected rename conflict, got %v", err)
}
// list_prefix with continuation.
for i := 0; i < 3; i++ {
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(fmt.Sprintf("%sbulk%d", prefix, i)),
"value": jsonValue(fmt.Sprintf("v%d", i)),
})); err != nil {
t.Fatalf("bulk put %d failed: %v", i, err)
}
}
listed, err := state.listPrefix(paramsWith(map[string]json.RawMessage{
"prefix": jsonString(prefix),
"limit": json.RawMessage(`2`),
"includeValues": json.RawMessage(`true`),
}))
if err != nil {
t.Fatalf("list_prefix failed: %v", err)
}
list := listed.(map[string]any)
if len(list["keys"].([]any)) != 2 {
t.Fatalf("expected limited list, got %#v", list)
}
continuation, ok := list["continuation"].(string)
if !ok || continuation == "" {
t.Fatalf("expected continuation, got %#v", list)
}
nextPage, err := state.listPrefix(paramsWith(map[string]json.RawMessage{
"prefix": jsonString(prefix),
"limit": json.RawMessage(`100`),
"includeValues": json.RawMessage(`true`),
"continuation": json.RawMessage(`"` + continuation + `"`),
}))
if err != nil {
t.Fatalf("continuation list failed: %v", err)
}
nextKeys := nextPage.(map[string]any)["keys"].([]any)
if len(nextKeys) == 0 {
t.Fatalf("continuation must resume after the first page")
}
if nextKeys[0].(map[string]any)["key"].(string) == list["keys"].([]any)[1].(map[string]any)["key"].(string) {
t.Fatalf("continuation overlapped the first page")
}
// --- watch (long poll) --------------------------------------------
watchStarted, err := state.watchStart(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "watched"),
"includePrevKv": json.RawMessage(`true`),
}))
if err != nil {
t.Fatalf("watch_start failed: %v", err)
}
watchID := watchStarted.(map[string]any)["watchId"].(string)
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "watched"),
"value": jsonValue("first"),
})); err != nil {
t.Fatalf("watched put failed: %v", err)
}
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "watched"),
"value": jsonValue("second"),
})); err != nil {
t.Fatalf("watched put 2 failed: %v", err)
}
deadline := time.Now().Add(20 * time.Second)
var watchBatches []any
for time.Now().Before(deadline) {
polled, err := state.watchPoll(paramsWith(map[string]json.RawMessage{"watchId": json.RawMessage(`"` + watchID + `"`)}))
if err != nil {
t.Fatalf("watch_poll failed: %v", err)
}
batches := polled.(map[string]any)["batches"].([]any)
watchBatches = append(watchBatches, batches...)
if len(watchBatches) >= 2 {
break
}
time.Sleep(150 * time.Millisecond)
}
if len(watchBatches) < 2 {
t.Fatalf("watch did not deliver both events: %#v", watchBatches)
}
lastBatch := watchBatches[len(watchBatches)-1].(map[string]any)
lastEvent := lastBatch["events"].([]map[string]any)[0]
if lastEvent["value"].(map[string]any)["data"] != "second" {
t.Fatalf("unexpected last watch event: %#v", lastEvent)
}
if lastEvent["previousValue"].(map[string]any)["data"] != "first" {
t.Fatalf("prevKv missing on watch event: %#v", lastEvent)
}
stopped, err := state.watchStop(paramsWith(map[string]json.RawMessage{"watchId": json.RawMessage(`"` + watchID + `"`)}))
if err != nil || stopped.(map[string]bool)["stopped"] != true {
t.Fatalf("watch_stop failed: %#v %v", stopped, err)
}
// --- auth ----------------------------------------------------------
userName := fmt.Sprintf("dbxlive%d", time.Now().UnixNano()%100000)
// Idempotent pre-cleanup: an interrupted earlier run may have left the
// fixed-named role behind.
_, _ = state.authRoleDelete(paramsWith(map[string]json.RawMessage{"role": jsonString("dbx_live_role")}))
if _, err := state.authUserAdd(paramsWith(map[string]json.RawMessage{
"user": jsonString(userName),
"password": jsonString("secret123"),
})); err != nil {
t.Fatalf("user add failed: %v", err)
}
if _, err := state.authRoleAdd(paramsWith(map[string]json.RawMessage{"role": jsonString("dbx_live_role")})); err != nil {
t.Fatalf("role add failed: %v", err)
}
if _, err := state.authRolePermission(paramsWith(map[string]json.RawMessage{
"role": jsonString("dbx_live_role"),
"resource": jsonString("prefix"),
"key": jsonString(prefix),
"access": jsonString("READWRITE"),
}), true); err != nil {
t.Fatalf("role grant permission failed: %v", err)
}
roleDetail, err := state.authRoleGet(paramsWith(map[string]json.RawMessage{"role": jsonString("dbx_live_role")}))
if err != nil {
t.Fatalf("role get failed: %v", err)
}
permissions := roleDetail.(map[string]any)["permissions"].([]map[string]any)
if len(permissions) == 0 {
t.Fatalf("expected permission rows after grant: %#v", roleDetail)
}
if _, err := state.authUserGrantRevokeRole(paramsWith(map[string]json.RawMessage{
"user": jsonString(userName),
"role": jsonString("dbx_live_role"),
}), true); err != nil {
t.Fatalf("grant role failed: %v", err)
}
userDetail, err := state.authUserGet(paramsWith(map[string]json.RawMessage{"user": jsonString(userName)}))
if err != nil {
t.Fatalf("user get failed: %v", err)
}
if len(userDetail.(map[string]any)["roles"].([]string)) == 0 {
t.Fatalf("expected granted role: %#v", userDetail)
}
users, err := state.authUserList(paramsWith(map[string]json.RawMessage{}))
if err != nil {
t.Fatalf("user list failed: %v", err)
}
if !containsAny(users.(map[string]any)["users"].([]string), userName) {
t.Fatalf("user list missing created user: %#v", users)
}
roles, err := state.authRoleList(paramsWith(map[string]json.RawMessage{}))
if err != nil || !containsAny(roles.(map[string]any)["roles"].([]string), "dbx_live_role") {
t.Fatalf("role list missing role: %#v %v", roles, err)
}
if _, err := state.authRolePermission(paramsWith(map[string]json.RawMessage{
"role": jsonString("dbx_live_role"),
"resource": jsonString("prefix"),
"key": jsonString(prefix),
}), false); err != nil {
t.Fatalf("role revoke permission failed: %v", err)
}
afterRevoke, err := state.authRoleGet(paramsWith(map[string]json.RawMessage{"role": jsonString("dbx_live_role")}))
if err != nil {
t.Fatalf("role get after revoke failed: %v", err)
}
if len(afterRevoke.(map[string]any)["permissions"].([]map[string]any)) != 0 {
t.Fatalf("permissions must be empty after revoke: %#v", afterRevoke)
}
if _, err := state.authUserGrantRevokeRole(paramsWith(map[string]json.RawMessage{
"user": jsonString(userName),
"role": jsonString("dbx_live_role"),
}), false); err != nil {
t.Fatalf("revoke role failed: %v", err)
}
if _, err := state.authUserChangePassword(paramsWith(map[string]json.RawMessage{
"user": jsonString(userName),
"password": jsonString("rotated456"),
})); err != nil {
t.Fatalf("change password failed: %v", err)
}
if _, err := state.authUserDelete(paramsWith(map[string]json.RawMessage{"user": jsonString(userName)})); err != nil {
t.Fatalf("user delete failed: %v", err)
}
if _, err := state.authRoleDelete(paramsWith(map[string]json.RawMessage{"role": jsonString("dbx_live_role")})); err != nil {
t.Fatalf("role delete failed: %v", err)
}
// --- status --------------------------------------------------------
status, err := state.status(paramsWith(map[string]json.RawMessage{}))
if err != nil {
t.Fatalf("kv_status failed: %v", err)
}
statusRow := status.(map[string]any)
if statusRow["clusterId"] == nil || statusRow["leaderId"] == nil {
t.Fatalf("status missing cluster identity: %#v", statusRow)
}
members := statusRow["members"].([]map[string]any)
if len(members) == 0 || members[0]["reachable"] != true {
t.Fatalf("unexpected member rows: %#v", members)
}
// --- delete + cleanup ----------------------------------------------
for _, key := range []string{prefix + "leased", prefix + "renamed", prefix + "watched", prefix + "bulk0", prefix + "bulk1", prefix + "bulk2"} {
if _, err := state.delete(paramsWith(map[string]json.RawMessage{"key": jsonString(key)})); err != nil {
t.Fatalf("delete %s failed: %v", key, err)
}
}
}
func TestLiveEtcd2ProtocolFlow(t *testing.T) {
if os.Getenv("DBX_ETCD2_LIVE") != "1" {
t.Skip("set DBX_ETCD2_LIVE=1 to run the live etcd v2 agent test")
}
endpoints := envOrDefault("DBX_ETCD2_ENDPOINTS", "http://172.26.129.83:20041")
user := envOrDefault("DBX_ETCD2_USER", "root")
password := envOrDefault("DBX_ETCD2_PASSWORD", "123456")
server := newRuntimeServer()
connection := fmt.Sprintf(`{"etcd_endpoints":%q,"username":%q,"password":%q}`, endpoints, user, password)
handshake, _ := server.handleLine(`{"id":1,"method":"handshake","params":{}}`)
if handshake.Error != nil {
t.Fatalf("handshake failed: %#v", handshake)
}
openLine := fmt.Sprintf(`{"id":2,"method":"open_session","params":{"agentSessionId":"s1","connection":%s}}`, connection)
opened, _ := server.handleLine(openLine)
if opened.Error != nil {
t.Fatalf("open_session failed: %#v", opened)
}
putLine := fmt.Sprintf(`{"id":3,"method":"kv_put","params":{"agentSessionId":"s1","key":"/dbx:proto","value":{"encoding":"utf8","data":"flow"}}}`)
put, _ := server.handleLine(putLine)
if put.Error != nil {
t.Fatalf("kv_put failed: %#v", put)
}
v3Only, _ := server.handleLine(`{"id":4,"method":"kv_history","params":{"agentSessionId":"s1","key":"/dbx:proto"}}`)
if v3Only.Error == nil || !strings.HasPrefix(v3Only.Error.Message, "ETCD_V2_UNSUPPORTED") {
t.Fatalf("expected ETCD_V2_UNSUPPORTED, got %#v", v3Only)
}
closed, _ := server.handleLine(`{"id":5,"method":"close_session","params":{"agentSessionId":"s1"}}`)
if closed.Error != nil {
t.Fatalf("close_session failed: %#v", closed)
}
}
func envOrDefault(key, fallback string) string {
if value := strings.TrimSpace(os.Getenv(key)); value != "" {
return value
}
return fallback
}
func paramsWith(params map[string]json.RawMessage) map[string]json.RawMessage {
return params
}
func jsonString(value string) json.RawMessage {
encoded, _ := json.Marshal(value)
return json.RawMessage(encoded)
}
func jsonValue(value string) json.RawMessage {
encoded, _ := json.Marshal(value)
return json.RawMessage(`{"encoding":"utf8","data":` + string(encoded) + `}`)
}
func containsAny(values []string, target string) bool {
for _, value := range values {
if value == target {
return true
}
}
return false
}
agents/drivers/etcd2-go/kv.go
+524
View File
@@ -0,0 +1,524 @@
package main
import (
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"sort"
"strconv"
"strings"
"unicode/utf8"
)
const defaultListLimit = 100
// v2Node is one node of the etcd v2 keys API tree.
type v2Node struct {
Key string `json:"key"`
Value string `json:"value"`
Dir bool `json:"dir"`
Expiration *string `json:"expiration"`
TTL *int64 `json:"ttl"`
CreatedIndex int64 `json:"createdIndex"`
ModifiedIndex int64 `json:"modifiedIndex"`
Nodes []v2Node `json:"nodes"`
}
type v2KeysResponse struct {
Action string `json:"action"`
Node *v2Node `json:"node"`
PrevNode *v2Node `json:"prevNode"`
}
// v2KeyPath normalizes a protocol key into a v2 keys API path. v2 keys are
// slash paths; an empty prefix maps to the root directory.
func v2KeyPath(key string) string {
if key == "" {
return "/v2/keys/"
}
if !strings.HasPrefix(key, "/") {
key = "/" + key
}
return "/v2/keys" + key
}
func (s *etcd2Session) listPrefix(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
prefix := stringOrDefault(params, "prefix", "")
limit := intOrDefault(params, "limit", defaultListLimit)
if limit < 1 {
limit = 1
}
includeValues := boolOrDefault(params, "includeValues", false)
continuation := stringOrNull(params, "continuation")
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
// v2 has no revisioned reads; the revision parameter is accepted and ignored.
query := url.Values{}
query.Set("recursive", "true")
query.Set("sorted", "true")
body, _, err := client.do(ctx, http.MethodGet, v2KeyPath(prefix)+"?"+query.Encode(), "", nil)
if err != nil {
if isNotFound(err) {
return map[string]any{"keys": []any{}, "continuation": nil, "revision": nil}, nil
}
return nil, err
}
var parsed v2KeysResponse
if err := json.Unmarshal(body, &parsed); err != nil {
return nil, err
}
startAfter := ""
if continuation != nil && *continuation != "" {
startAfter = *continuation
}
rows := []map[string]any{}
flattenNodes(parsed.Node, includeValues, func(row map[string]any) {
key := row["key"].(string)
if startAfter != "" && key <= startAfter {
return
}
if len(rows) < limit {
rows = append(rows, row)
}
})
result := map[string]any{"keys": toAnySlice(rows)}
if len(rows) == limit {
result["continuation"] = rows[len(rows)-1]["key"]
} else {
result["continuation"] = nil
}
// etcd v2 exposes the current index on the listing's node headers; the
// modifiedIndex of the returned root is the closest analog of a revision.
var revision any
if parsed.Node != nil {
revision = longString(parsed.Node.ModifiedIndex)
}
result["revision"] = revision
return result, nil
}
func flattenNodes(node *v2Node, includeValues bool, emit func(map[string]any)) {
if node == nil {
return
}
if !node.Dir {
row := nodeMetadata(node)
row["key"] = node.Key
if includeValues {
row["value"] = valueObject([]byte(node.Value))
}
emit(row)
return
}
children := append([]v2Node(nil), node.Nodes...)
sort.SliceStable(children, func(i, j int) bool { return children[i].Key < children[j].Key })
for index := range children {
flattenNodes(&children[index], includeValues, emit)
}
}
func toAnySlice(rows []map[string]any) []any {
result := make([]any, len(rows))
for index, row := range rows {
result[index] = row
}
return result
}
func (s *etcd2Session) get(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
key, err := keyBytesParam(params)
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
body, _, err := client.do(ctx, http.MethodGet, v2KeyPath(key), "", nil)
if err != nil {
if isNotFound(err) {
return map[string]any{"found": false, "key": nil, "value": nil, "metadata": nil}, nil
}
return nil, err
}
var parsed v2KeysResponse
if err := json.Unmarshal(body, &parsed); err != nil {
return nil, err
}
if parsed.Node == nil || parsed.Node.Dir {
return map[string]any{"found": false, "key": nil, "value": nil, "metadata": nil}, nil
}
node := parsed.Node
result := map[string]any{
"found": true,
"key": node.Key,
"keyBytes": bytesObject([]byte(node.Key)),
}
if !boolOrDefault(params, "metadataOnly", false) {
result["value"] = valueObject([]byte(node.Value))
} else {
result["value"] = nil
}
result["metadata"] = nodeMetadata(node)
return result, nil
}
func (s *etcd2Session) put(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
key, err := keyBytesParam(params)
if err != nil {
return nil, err
}
value, err := parseValueObject(rawObject(params, "value"))
if err != nil {
return nil, err
}
expectedModRevision := longOrNull(params, "expectedModRevision")
expectedCreateRevision := longOrNull(params, "expectedCreateRevision")
ttlValue := longOrNull(params, "ttl")
preserveLease := boolOrDefault(params, "preserveLease", false)
leaseValue := longOrNull(params, "lease")
hasLease := leaseValue != nil
hasTtl := ttlValue != nil
if (hasLease && hasTtl) || (preserveLease && (hasLease || hasTtl)) {
return nil, errors.New("lease, ttl, and preserveLease cannot be specified together")
}
if hasLease {
return nil, errors.New("ETCD_V2_LEASE_UNSUPPORTED: the etcd v2 API has no lease objects; use ttl instead")
}
if preserveLease {
return nil, errors.New("ETCD_V2_LEASE_UNSUPPORTED: the etcd v2 API has no lease objects; use ttl instead")
}
if expectedCreateRevision != nil && *expectedCreateRevision != 0 {
return nil, errors.New("ETCD_V2_CAS_UNSUPPORTED: the etcd v2 API cannot compare createdIndex; use expectedModRevision")
}
if hasTtl && *ttlValue <= 0 {
return nil, errors.New("ttl must be a positive integer")
}
form := url.Values{}
form.Set("value", value)
if hasTtl {
form.Set("ttl", strconv.FormatInt(*ttlValue, 10))
}
if expectedModRevision != nil {
form.Set("prevIndex", strconv.FormatInt(*expectedModRevision, 10))
}
if expectedCreateRevision != nil && *expectedCreateRevision == 0 {
form.Set("prevExist", "false")
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
body, _, err := client.do(ctx, http.MethodPut, v2KeyPath(key), form.Encode(), nil)
if err != nil {
if isCompareFailed(err) {
return nil, errors.New("ETCD_CAS_CONFLICT: key changed after it was loaded")
}
return nil, err
}
var parsed v2KeysResponse
if err := json.Unmarshal(body, &parsed); err != nil {
return nil, err
}
var revision int64
if parsed.Node != nil {
revision = parsed.Node.ModifiedIndex
}
return map[string]any{"revision": longString(revision)}, nil
}
func (s *etcd2Session) delete(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
key, err := keyBytesParam(params)
if err != nil {
return nil, err
}
expectedModRevision := longOrNull(params, "expectedModRevision")
path := v2KeyPath(key)
if expectedModRevision != nil {
query := url.Values{}
query.Set("prevIndex", strconv.FormatInt(*expectedModRevision, 10))
path += "?" + query.Encode()
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
body, _, err := client.do(ctx, http.MethodDelete, path, "", nil)
if err != nil {
if isCompareFailed(err) {
return nil, errors.New("ETCD_CAS_CONFLICT: key changed after it was loaded")
}
return nil, err
}
var parsed v2KeysResponse
if err := json.Unmarshal(body, &parsed); err != nil {
return nil, err
}
var revision int64
if parsed.Node != nil {
revision = parsed.Node.ModifiedIndex
}
return map[string]any{"deleted": int64(1), "revision": longString(revision)}, nil
}
// rename is a non-atomic check-then-set: the v2 API has no transactions, so a
// concurrent writer between the target check and the delete can be lost. The
// protocol exposes this honestly instead of pretending atomicity.
func (s *etcd2Session) rename(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
sourceKey, err := keyBytesParam(params)
if err != nil {
return nil, err
}
newKey := stringOrNull(params, "newKey")
if newKey == nil || *newKey == "" {
return nil, errors.New("ETCD_NEWKEY_REQUIRED")
}
targetKey := *newKey
if sourceKey == targetKey {
return map[string]any{"renamed": true, "revision": nil}, nil
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
body, _, err := client.do(ctx, http.MethodGet, v2KeyPath(sourceKey), "", nil)
if err != nil {
if isNotFound(err) {
return nil, errors.New("ETCD_NOT_FOUND: source key does not exist")
}
return nil, err
}
var source v2KeysResponse
if err := json.Unmarshal(body, &source); err != nil {
return nil, err
}
if source.Node == nil || source.Node.Dir {
return nil, errors.New("ETCD_NOT_FOUND: source key does not exist")
}
if _, _, err := client.do(ctx, http.MethodGet, v2KeyPath(targetKey), "", nil); err == nil {
return nil, errors.New("ETCD_CAS_CONFLICT: source changed or target already exists")
} else if !isNotFound(err) {
return nil, err
}
expected := longOrNull(params, "expectedModRevision")
expectedRevision := source.Node.ModifiedIndex
if expected != nil {
expectedRevision = *expected
}
form := url.Values{}
form.Set("value", source.Node.Value)
if source.Node.TTL != nil {
form.Set("ttl", strconv.FormatInt(*source.Node.TTL, 10))
}
form.Set("prevExist", "false")
if _, _, err := client.do(ctx, http.MethodPut, v2KeyPath(targetKey), form.Encode(), nil); err != nil {
if isCompareFailed(err) {
return nil, errors.New("ETCD_CAS_CONFLICT: source changed or target already exists")
}
return nil, err
}
deleteQuery := url.Values{}
deleteQuery.Set("prevIndex", strconv.FormatInt(expectedRevision, 10))
if _, _, err := client.do(ctx, http.MethodDelete, v2KeyPath(sourceKey)+"?"+deleteQuery.Encode(), "", nil); err != nil {
if isCompareFailed(err) {
return nil, errors.New("ETCD_CAS_CONFLICT: source changed or target already exists")
}
return nil, err
}
return map[string]any{"renamed": true, "revision": nil}, nil
}
func nodeMetadata(node *v2Node) map[string]any {
metadata := map[string]any{
"createRevision": longString(node.CreatedIndex),
"modRevision": longString(node.ModifiedIndex),
"version": longString(int64(1)),
"lease": longString(0),
"valueSize": len(node.Value),
}
if node.TTL != nil {
metadata["ttl"] = *node.TTL
}
return metadata
}
func longString(value int64) string {
return strconv.FormatInt(value, 10)
}
func unsignedLongString(value int64) string {
return strconv.FormatUint(uint64(value), 10)
}
// prefixEnd mirrors the v3 agent's byte-increment range end; for v2 rows it
// only feeds UI metadata, where the explicit resource field carries the truth.
func prefixEnd(prefix string) string {
if prefix == "" {
return "\x00"
}
end := []byte(prefix)
for i := len(end) - 1; i >= 0; i-- {
if end[i] < 0xff {
end[i]++
return string(end[:i+1])
}
}
return "\x00"
}
func keyBytesParam(params map[string]json.RawMessage) (string, error) {
if encoded := rawObject(params, "keyBytes"); encoded != nil {
value, err := parseValueObject(encoded)
if err != nil {
return "", err
}
return value, nil
}
if raw, ok := params["key"]; ok && raw != nil && string(raw) != "null" {
var key string
if err := json.Unmarshal(raw, &key); err != nil {
return "", err
}
return key, nil
}
return "", errors.New("Key is required")
}
func rawObject(params map[string]json.RawMessage, key string) map[string]json.RawMessage {
raw := params[key]
if len(raw) == 0 || string(raw) == "null" {
return nil
}
var object map[string]json.RawMessage
if err := json.Unmarshal(raw, &object); err != nil || object == nil {
return nil
}
return object
}
func bytesObject(bytes []byte) map[string]any {
return map[string]any{
"encoding": "base64",
"data": base64.StdEncoding.EncodeToString(bytes),
}
}
func valueObject(value []byte) map[string]any {
if utf8.Valid(value) {
return map[string]any{"encoding": "utf8", "data": string(value)}
}
return bytesObject(value)
}
func displayBytes(bytes []byte) string {
if utf8.Valid(bytes) {
return string(bytes)
}
return base64.StdEncoding.EncodeToString(bytes)
}
func parseValueObject(value map[string]json.RawMessage) (string, error) {
if value == nil {
value = map[string]json.RawMessage{}
}
encoding := stringOrDefault(value, "encoding", "utf8")
data := stringOrDefault(value, "data", "")
if encoding == "base64" {
decoded, err := base64.StdEncoding.DecodeString(data)
if err != nil {
return "", err
}
return string(decoded), nil
}
if encoding != "utf8" {
return "", fmt.Errorf("Unsupported value encoding: %s", encoding)
}
return data, nil
}
func stringOrNull(params map[string]json.RawMessage, key string) *string {
raw := params[key]
if len(raw) == 0 || string(raw) == "null" {
return nil
}
var value string
if err := json.Unmarshal(raw, &value); err != nil {
return nil
}
return &value
}
func stringOrDefault(params map[string]json.RawMessage, key string, fallback string) string {
value := stringOrNull(params, key)
if value == nil {
return fallback
}
return *value
}
func intOrDefault(params map[string]json.RawMessage, key string, fallback int) int {
raw := params[key]
if len(raw) == 0 || string(raw) == "null" {
return fallback
}
var value int
if err := json.Unmarshal(raw, &value); err != nil {
return fallback
}
return value
}
func boolOrDefault(params map[string]json.RawMessage, key string, fallback bool) bool {
raw := params[key]
if len(raw) == 0 || string(raw) == "null" {
return fallback
}
var value bool
if err := json.Unmarshal(raw, &value); err != nil {
return fallback
}
return value
}
func longOrNull(params map[string]json.RawMessage, key string) *int64 {
raw := params[key]
if len(raw) == 0 || string(raw) == "null" {
return nil
}
var value int64
if err := json.Unmarshal(raw, &value); err != nil {
return nil
}
return &value
}
func requiredString(params map[string]json.RawMessage, field string) (string, error) {
value := stringOrNull(params, field)
if value == nil || *value == "" {
return "", fmt.Errorf("ETCD_%s_REQUIRED", strings.ToUpper(field))
}
return *value, nil
}
agents/drivers/etcd2-go/main.go
+422
View File
@@ -0,0 +1,422 @@
package main
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"os"
"runtime"
"strconv"
"strings"
"sync"
)
const (
protocolVersion = 2
legacyAgentSessionID = "__legacy__"
maxAgentSessions = 256
)
// Capabilities advertise the v2 API surface: no history, no lease objects,
// no compaction, and no defrag because etcd v2 semantics do not have them.
var capabilities = []string{
"connect", "test_connection", "kv", "kv_ttl", "kv_cas", "kv_list_values",
"kv_status", "etcd_watch", "etcd_auth", "multi_session",
}
type request struct {
ID json.RawMessage `json:"id"`
Method string `json:"method"`
Params map[string]json.RawMessage `json:"params"`
}
type response struct {
JSONRPC string `json:"jsonrpc,omitempty"`
ID json.RawMessage `json:"id,omitempty"`
Result any `json:"result,omitempty"`
Error *rpcError `json:"error,omitempty"`
}
// etcd2Session carries the per-connection HTTP client and derived state.
// Requests for one session are serialized by agentSession.mu; watch pump
// goroutines reach the maps through watchesMu instead.
type etcd2Session struct {
clientMu sync.Mutex
httpClient *authenticatedClient
connectedEndpoints []string
serverVersion string
watchesMu sync.Mutex
watches map[string]*watchState
watchBufferedBytes int64
activeCancelMu sync.Mutex
activeCancel context.CancelFunc
}
func (s *etcd2Session) reserveWatchBuffer(bytes int64) bool {
s.watchesMu.Lock()
defer s.watchesMu.Unlock()
if bytes < 0 || s.watchBufferedBytes+bytes > maxSessionWatchBufferBytes {
return false
}
s.watchBufferedBytes += bytes
return true
}
func (s *etcd2Session) releaseWatchBuffer(bytes int64) {
s.watchesMu.Lock()
defer s.watchesMu.Unlock()
s.watchBufferedBytes -= bytes
if s.watchBufferedBytes < 0 {
s.watchBufferedBytes = 0
}
}
type agentSession struct {
state *etcd2Session
mu sync.Mutex
}
func (a *agentSession) cancelActive() {
a.state.activeCancelMu.Lock()
cancel := a.state.activeCancel
a.state.activeCancelMu.Unlock()
if cancel != nil {
cancel()
}
}
type runtimeServer struct {
mu sync.RWMutex
sessions map[string]*agentSession
}
func main() {
configureRuntimeParallelism()
server := newRuntimeServer()
encoder := json.NewEncoder(os.Stdout)
var encoderMu sync.Mutex
var requests sync.WaitGroup
fmt.Fprintln(os.Stdout, `{"ready":true}`)
scanner := bufio.NewScanner(os.Stdin)
scanner.Buffer(make([]byte, 0, 64*1024), 512*1024*1024)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
}
var envelope request
if json.Unmarshal([]byte(line), &envelope) == nil && envelope.Method == "shutdown" {
requests.Wait()
result, _ := server.handleLine(line)
encoderMu.Lock()
_ = encoder.Encode(result)
encoderMu.Unlock()
server.closeAll()
return
}
requests.Add(1)
go func(line string) {
defer requests.Done()
result, _ := server.handleLine(line)
encoderMu.Lock()
defer encoderMu.Unlock()
if err := encoder.Encode(result); err != nil {
fmt.Fprintf(os.Stderr, "failed to write response: %v\n", err)
}
}(line)
}
requests.Wait()
server.closeAll()
}
func configureRuntimeParallelism() {
if raw := strings.TrimSpace(os.Getenv("DBX_AGENT_ETCD2_GOMAXPROCS")); raw != "" {
if configured, err := strconv.Atoi(raw); err == nil && configured > 0 {
runtime.GOMAXPROCS(configured)
return
}
}
if strings.TrimSpace(os.Getenv("GOMAXPROCS")) != "" {
return
}
runtime.GOMAXPROCS(min(runtime.NumCPU(), 4))
}
func newRuntimeServer() *runtimeServer {
return &runtimeServer{sessions: map[string]*agentSession{}}
}
func (r *runtimeServer) handleLine(line string) (response, bool) {
var req request
if err := json.Unmarshal([]byte(line), &req); err != nil {
return errorResponse(nil, "", "", err), false
}
if len(req.ID) == 0 {
req.ID = json.RawMessage("1")
}
result, shutdown, err := r.dispatch(req.Method, req.Params)
if err != nil {
return errorResponse(req.ID, req.Method, stringParam(req.Params, "agentSessionId"), err), false
}
return response{JSONRPC: "2.0", ID: req.ID, Result: result}, shutdown
}
func (r *runtimeServer) dispatch(method string, params map[string]json.RawMessage) (any, bool, error) {
switch method {
case "handshake":
return handshakeResult(), false, nil
case "open_session":
id := requiredSessionID(params)
if id == "" {
return nil, false, errors.New("agentSessionId is required")
}
return r.openSession(id, params)
case "close_session":
return r.closeSession(stringParam(params, "agentSessionId")), false, nil
case "validate_session":
session, err := r.session(requiredSessionID(params))
if err != nil {
return nil, false, err
}
session.mu.Lock()
defer session.mu.Unlock()
result, err := session.state.validateConnection()
return result, false, err
case "cancel_session":
session, err := r.session(requiredSessionID(params))
if err != nil {
return nil, false, err
}
session.cancelActive()
return map[string]bool{"ok": true}, false, nil
case "test_connection":
return r.testConnection(params)
case "connect":
_ = r.closeSession(legacyAgentSessionID)
return r.openSession(legacyAgentSessionID, params)
case "disconnect":
return r.closeSession(legacyAgentSessionID), false, nil
case "shutdown":
return map[string]bool{"ok": true}, true, r.closeAll()
default:
id := stringParam(params, "agentSessionId")
if id == "" {
id = legacyAgentSessionID
}
session, err := r.session(id)
if err != nil {
return nil, false, err
}
session.mu.Lock()
defer session.mu.Unlock()
result, err := session.state.handle(method, params)
return result, false, err
}
}
func requiredSessionID(params map[string]json.RawMessage) string {
return strings.TrimSpace(stringParam(params, "agentSessionId"))
}
func (r *runtimeServer) openSession(id string, params map[string]json.RawMessage) (any, bool, error) {
r.mu.Lock()
if _, exists := r.sessions[id]; exists {
r.mu.Unlock()
return nil, false, fmt.Errorf("Agent session already exists: %s", id)
}
if len(r.sessions) >= maxAgentSessions {
r.mu.Unlock()
return nil, false, fmt.Errorf("Agent session limit reached: %d", maxAgentSessions)
}
session := &agentSession{state: newEtcd2Session()}
r.sessions[id] = session
r.mu.Unlock()
if _, err := session.state.connect(params); err != nil {
r.mu.Lock()
delete(r.sessions, id)
r.mu.Unlock()
session.state.close()
return nil, false, err
}
return map[string]bool{"ok": true}, false, nil
}
func (r *runtimeServer) session(id string) (*agentSession, error) {
r.mu.RLock()
session := r.sessions[id]
r.mu.RUnlock()
if session == nil {
return nil, fmt.Errorf("Agent session not found: %s", id)
}
return session, nil
}
func (r *runtimeServer) closeSession(id string) any {
r.mu.Lock()
session := r.sessions[id]
delete(r.sessions, id)
r.mu.Unlock()
if session != nil {
session.cancelActive()
session.mu.Lock()
session.state.close()
session.mu.Unlock()
}
return map[string]bool{"ok": true}
}
func (r *runtimeServer) closeAll() error {
r.mu.Lock()
sessions := r.sessions
r.sessions = map[string]*agentSession{}
r.mu.Unlock()
var firstErr error
for _, session := range sessions {
session.cancelActive()
if err := session.state.close(); err != nil && firstErr == nil {
firstErr = err
}
}
return firstErr
}
func (r *runtimeServer) testConnection(params map[string]json.RawMessage) (any, bool, error) {
state := newEtcd2Session()
defer state.close()
if _, err := state.connect(params); err != nil {
return nil, false, err
}
return map[string]bool{"ok": true}, false, nil
}
func newEtcd2Session() *etcd2Session {
return &etcd2Session{watches: map[string]*watchState{}}
}
func handshakeResult() map[string]any {
return map[string]any{
"protocolVersion": protocolVersion,
"agentProtocolVersion": protocolVersion,
"capabilities": capabilities,
}
}
func (s *etcd2Session) beginOperation() (context.Context, context.CancelFunc) {
ctx, cancel := context.WithTimeout(context.Background(), operationTimeout)
s.activeCancelMu.Lock()
s.activeCancel = cancel
s.activeCancelMu.Unlock()
return ctx, cancel
}
func (s *etcd2Session) endOperation(cancel context.CancelFunc) {
cancel()
s.activeCancelMu.Lock()
s.activeCancel = nil
s.activeCancelMu.Unlock()
}
func (s *etcd2Session) handle(method string, params map[string]json.RawMessage) (any, error) {
switch method {
case "handshake":
return handshakeResult(), nil
case "connect", "test_connection":
if _, err := s.connect(params); err != nil {
return nil, err
}
return map[string]bool{"ok": true}, nil
case "validate_connection":
return s.validateConnection()
case "kv_list_prefix":
return s.listPrefix(params)
case "kv_get":
return s.get(params)
case "kv_put":
return s.put(params)
case "kv_delete":
return s.delete(params)
case "kv_rename":
return s.rename(params)
case "kv_status":
return s.status(params)
case "etcd_watch_start":
return s.watchStart(params)
case "etcd_watch_poll":
return s.watchPoll(params)
case "etcd_watch_stop":
return s.watchStop(params)
case "etcd_auth_user_list":
return s.authUserList(params)
case "etcd_auth_user_get":
return s.authUserGet(params)
case "etcd_auth_user_add":
return s.authUserAdd(params)
case "etcd_auth_user_delete":
return s.authUserDelete(params)
case "etcd_auth_user_change_password":
return s.authUserChangePassword(params)
case "etcd_auth_user_grant_role":
return s.authUserGrantRevokeRole(params, true)
case "etcd_auth_user_revoke_role":
return s.authUserGrantRevokeRole(params, false)
case "etcd_auth_role_list":
return s.authRoleList(params)
case "etcd_auth_role_get":
return s.authRoleGet(params)
case "etcd_auth_role_add":
return s.authRoleAdd(params)
case "etcd_auth_role_delete":
return s.authRoleDelete(params)
case "etcd_auth_role_grant_permission":
return s.authRolePermission(params, true)
case "etcd_auth_role_revoke_permission":
return s.authRolePermission(params, false)
case "disconnect":
s.close()
return map[string]bool{"ok": true}, nil
case "shutdown":
s.close()
return map[string]bool{"ok": true}, nil
default:
if isEtcd3OnlyMethod(method) {
return nil, fmt.Errorf("ETCD_V2_UNSUPPORTED: %s is not available on the etcd v2 API", method)
}
return nil, fmt.Errorf("unknown method: %s", method)
}
}
func isEtcd3OnlyMethod(method string) bool {
switch method {
case "kv_history", "etcd_compact", "etcd_defrag",
"etcd_lease_list", "etcd_lease_get", "etcd_lease_grant",
"etcd_lease_keepalive_once", "etcd_lease_revoke":
return true
}
return false
}
func decodeParams(params map[string]json.RawMessage, target any) error {
if params == nil {
params = map[string]json.RawMessage{}
}
data, err := json.Marshal(params)
if err != nil {
return err
}
return json.Unmarshal(data, target)
}
func stringParam(params map[string]json.RawMessage, key string) string {
if params == nil {
return ""
}
var result string
_ = json.Unmarshal(params[key], &result)
return result
}
agents/drivers/etcd2-go/main_test.go
+267
View File
@@ -0,0 +1,267 @@
package main
import (
"encoding/json"
"strings"
"testing"
)
func TestHandshakeAdvertisesV2Capabilities(t *testing.T) {
server := newRuntimeServer()
result, _, err := server.dispatch("handshake", nil)
if err != nil {
t.Fatalf("handshake failed: %v", err)
}
handshake := result.(map[string]any)
want := []string{
"connect", "test_connection", "kv", "kv_ttl", "kv_cas", "kv_list_values",
"kv_status", "etcd_watch", "etcd_auth", "multi_session",
}
got := handshake["capabilities"].([]string)
if strings.Join(got, ",") != strings.Join(want, ",") {
t.Fatalf("capability mismatch:\n got %v\nwant %v", got, want)
}
for _, forbidden := range []string{"kv_history", "etcd_lease", "etcd_compaction", "etcd_defrag", "structured_error_v1"} {
for _, capability := range got {
if capability == forbidden {
t.Fatalf("%s must not be advertised on the v2 agent", forbidden)
}
}
}
}
func TestHandleLineReportsSessionNotFound(t *testing.T) {
server := newRuntimeServer()
response, _ := server.handleLine(`{"id":7,"method":"kv_get","params":{"key":"a"}}`)
if response.Error == nil || response.Error.Code != -1 {
t.Fatalf("expected code -1 error, got %#v", response)
}
if response.Error.Message != "Agent session not found: __legacy__" {
t.Fatalf("unexpected message: %q", response.Error.Message)
}
}
func TestUnconnectedSessionErrors(t *testing.T) {
state := newEtcd2Session()
if _, err := state.get(map[string]json.RawMessage{"key": json.RawMessage(`"k"`)}); err == nil || err.Error() != "Not connected" {
t.Fatalf("kv_get before connect: %v", err)
}
if _, err := state.validateConnection(); err == nil || err.Error() != "Not connected" {
t.Fatalf("validate_connection before connect: %v", err)
}
}
func TestEtcd3OnlyMethodsRejected(t *testing.T) {
state := newEtcd2Session()
for _, method := range []string{"kv_history", "etcd_lease_list", "etcd_lease_grant", "etcd_compact", "etcd_defrag"} {
_, err := state.handle(method, map[string]json.RawMessage{})
if err == nil || !strings.HasPrefix(err.Error(), "ETCD_V2_UNSUPPORTED") {
t.Fatalf("%s should report ETCD_V2_UNSUPPORTED, got %v", method, err)
}
}
_, err := state.handle("etcd_nonsense", map[string]json.RawMessage{})
if err == nil || !strings.Contains(err.Error(), "unknown method") {
t.Fatalf("expected unknown method error, got %v", err)
}
}
func TestConnectionEndpoints(t *testing.T) {
cases := []struct {
name string
connection connectionParams
want []string
}{
{"host port fallback", connectionParams{Port: 2379}, []string{"http://127.0.0.1:2379"}},
{"ssl scheme", connectionParams{Host: "db", Port: 1, SSL: true}, []string{"https://db:1"}},
{"comma list", connectionParams{EtcdEndpoints: "a:1,b:2"}, []string{"http://a:1", "http://b:2"}},
{"existing scheme trimmed", connectionParams{EtcdEndpoints: "https://a:1/"}, []string{"https://a:1"}},
}
for _, testCase := range cases {
got := connectionEndpoints(testCase.connection)
if strings.Join(got, "|") != strings.Join(testCase.want, "|") {
t.Fatalf("%s: got %v, want %v", testCase.name, got, testCase.want)
}
}
}
func TestTLSConfigRequiresCertAndKeyTogether(t *testing.T) {
_, err := tlsConfigFor(connectionParams{SSL: true, ClientCertPath: "/tmp/cert.pem"})
if err == nil || err.Error() != "Client certificate and key must be provided together" {
t.Fatalf("expected paired cert/key error, got %v", err)
}
}
func TestV2KeyPath(t *testing.T) {
cases := []struct{ key, want string }{
{"", "/v2/keys/"},
{"foo", "/v2/keys/foo"},
{"/foo/bar", "/v2/keys/foo/bar"},
}
for _, testCase := range cases {
if got := v2KeyPath(testCase.key); got != testCase.want {
t.Fatalf("v2KeyPath(%q) = %q, want %q", testCase.key, got, testCase.want)
}
}
}
func TestErrorFromResponse(t *testing.T) {
err := errorFromResponse(404, []byte(`{"errorCode":100,"message":"Key not found","cause":"/foo","index":37}`))
if !isNotFound(err) {
t.Fatalf("expected key-not-found classification: %#v", err)
}
if isCompareFailed(err) {
t.Fatalf("errorCode 100 is not compare failure")
}
if err.Error() != "Key not found (/foo)" {
t.Fatalf("unexpected message: %q", err.Error())
}
compare := errorFromResponse(412, []byte(`{"errorCode":101,"message":"Compare failed","cause":"[1 != 2]"}`))
if !isCompareFailed(compare) {
t.Fatalf("expected compare failure classification: %#v", compare)
}
plain := errorFromResponse(500, []byte("boom"))
if plain.Error() != "boom" {
t.Fatalf("plain body passthrough broken: %q", plain.Error())
}
}
func TestV2PermissionRows(t *testing.T) {
rows := v2PermissionRows("read", []string{"/a", "/p/*"})
if len(rows) != 2 {
t.Fatalf("expected 2 permission rows, got %d", len(rows))
}
first := rows[0]
if first["access"] != "read" || first["resource"] != "key" {
t.Fatalf("unexpected exact-key row: %#v", first)
}
second := rows[1]
if second["access"] != "read" || second["resource"] != "prefix" {
t.Fatalf("unexpected prefix row: %#v", second)
}
end := second["rangeEnd"].(map[string]any)
if end["data"] != "L3Aw" { // prefixEnd("/p/") == "/p0"
t.Fatalf("prefix rangeEnd wrong: %#v", end)
}
}
func TestV2PermissionPattern(t *testing.T) {
if got := v2PermissionPattern("key", "/foo"); got != "/foo" {
t.Fatalf("key pattern must stay bare: %q", got)
}
if got := v2PermissionPattern("prefix", "/foo/"); got != "/foo/*" {
t.Fatalf("prefix pattern must become glob: %q", got)
}
if got := v2PermissionPattern("all", "/ignored"); got != "/*" {
t.Fatalf("all pattern must be root glob: %q", got)
}
}
func TestV2RoleGrantDocument(t *testing.T) {
encoded, err := json.Marshal(v2Role{
Role: "r1",
Grant: &v2Permissions{KV: v2RWPermission{Read: []string{"/p/*"}}},
})
if err != nil {
t.Fatal(err)
}
var round v2Role
if err := json.Unmarshal(encoded, &round); err != nil {
t.Fatal(err)
}
if round.Role != "r1" || len(round.Grant.KV.Read) != 1 || round.Grant.KV.Read[0] != "/p/*" {
t.Fatalf("grant document round-trip broken: %s", encoded)
}
if round.Permissions != nil {
t.Fatalf("grant documents must not carry permissions: %s", encoded)
}
}
func TestMemberHexToUnsigned(t *testing.T) {
if got := memberHexToUnsigned("272e204152"); got != "168277590354" {
t.Fatalf("unexpected conversion: %v", got)
}
if got := memberHexToUnsigned(""); got != nil {
t.Fatalf("empty id must map to nil, got %v", got)
}
if got := memberHexToUnsigned("zzz"); got != nil {
t.Fatalf("invalid hex must map to nil, got %v", got)
}
}
func TestWatchBudgetState(t *testing.T) {
session := newEtcd2Session()
state := &watchState{watchID: "w", session: session}
state.append(10, []map[string]any{{"eventType": "put"}}, 128)
state.append(11, []map[string]any{{"eventType": "put"}}, maxWatchBufferBytes+1)
polled := state.poll()
if len(polled["batches"].([]any)) != 1 {
t.Fatalf("overflow must keep buffered batch: %#v", polled)
}
terminal, ok := polled["terminal"].(map[string]any)
if !ok || terminal["reason"] != "overflow" {
t.Fatalf("expected overflow terminal: %#v", polled)
}
if session.watchBufferedBytesSnapshot() != 0 {
t.Fatalf("budget not released: %d", session.watchBufferedBytesSnapshot())
}
}
func TestWatchLimits(t *testing.T) {
session := newEtcd2Session()
for i := 0; i < maxWatches; i++ {
session.registerWatch(string(rune('a'+i)), &watchState{watchID: string(rune('a' + i)), session: session})
}
_, err := session.watchStart(map[string]json.RawMessage{"key": json.RawMessage(`"k"`)})
if err == nil || !strings.HasPrefix(err.Error(), "ETCD_WATCH_LIMIT") {
t.Fatalf("expected watch limit error, got %v", err)
}
fresh := newEtcd2Session()
_, err = fresh.watchStart(map[string]json.RawMessage{"key": json.RawMessage(`"k"`), "scope": json.RawMessage(`"glob"`)})
if err == nil || err.Error() != "ETCD_WATCH_SCOPE_INVALID: scope must be key or prefix" {
t.Fatalf("expected scope error, got %v", err)
}
_, err = fresh.watchPoll(map[string]json.RawMessage{"watchId": json.RawMessage(`"none"`)})
if err == nil || err.Error() != "ETCD_WATCH_NOT_FOUND: watch does not exist" {
t.Fatalf("expected watch-not-found, got %v", err)
}
}
func TestPutRejectsLeaseOptions(t *testing.T) {
state := newEtcd2Session()
// Client check comes first (Java parity); without a connection the error
// stays "Not connected".
params := map[string]json.RawMessage{
"key": json.RawMessage(`"k"`),
"value": json.RawMessage(`{"encoding":"utf8","data":"v"}`),
"ttl": json.RawMessage(`5`),
"lease": json.RawMessage(`7`),
}
if _, err := state.put(params); err == nil || err.Error() != "Not connected" {
t.Fatalf("expected Not connected before option validation, got %v", err)
}
}
func TestPrefixEnd(t *testing.T) {
if prefixEnd("") != "\x00" || prefixEnd("ab") != "ac" || prefixEnd("a\xff") != "b" {
t.Fatalf("prefixEnd algorithm broken")
}
}
func TestValueEncodings(t *testing.T) {
utf8Value := valueObject([]byte("hello"))
if utf8Value["encoding"] != "utf8" || utf8Value["data"] != "hello" {
t.Fatalf("unexpected utf8 value: %#v", utf8Value)
}
if valueObject([]byte{0xff})["encoding"] != "base64" {
t.Fatalf("binary must be base64")
}
if displayBytes([]byte{0xff}) == "\xff" {
t.Fatalf("displayBytes must fall back to base64")
}
}
func (s *etcd2Session) watchBufferedBytesSnapshot() int64 {
s.watchesMu.Lock()
defer s.watchesMu.Unlock()
return s.watchBufferedBytes
}
agents/drivers/etcd2-go/protocol_error.go
+135
View File
@@ -0,0 +1,135 @@
package main
import (
"context"
"encoding/json"
"errors"
"io"
"net"
"strings"
)
type rpcError struct {
Code int `json:"code"`
Message string `json:"message"`
Data *rpcErrorData `json:"data,omitempty"`
}
type rpcErrorData struct {
Category string `json:"category"`
Retryable bool `json:"retryable"`
SessionDisposition string `json:"sessionDisposition"`
Stage string `json:"stage"`
ContractVersion int `json:"contractVersion"`
OperationOutcome string `json:"operationOutcome"`
ExceptionClass string `json:"exceptionClass,omitempty"`
AgentSessionID string `json:"agentSessionId,omitempty"`
}
func errorResponse(id json.RawMessage, method, agentSessionID string, err error) response {
return response{JSONRPC: "2.0", ID: id, Error: classifyRPCError(method, agentSessionID, err)}
}
func classifyRPCError(method, agentSessionID string, err error) *rpcError {
stage := rpcErrorStage(method)
data := &rpcErrorData{
Category: "protocol",
Retryable: false,
SessionDisposition: "keep",
Stage: stage,
ContractVersion: 1,
OperationOutcome: rpcOperationOutcome(stage),
ExceptionClass: safeRPCDiagnostic(errTypeName(err), 160),
AgentSessionID: strings.TrimSpace(agentSessionID),
}
switch {
case errors.Is(err, context.Canceled):
data.Category = "canceled"
data.SessionDisposition = "quarantine"
case errors.Is(err, context.DeadlineExceeded) || isTimeoutError(err):
data.Category = "timeout"
data.SessionDisposition = "quarantine"
case isConnectionError(err):
data.Category = "connection"
data.Retryable = stage == "connect" || stage == "validate"
if stage != "connect" {
data.SessionDisposition = "quarantine"
}
}
return &rpcError{Code: -1, Message: err.Error(), Data: data}
}
func isConnectionError(err error) bool {
if errors.Is(err, io.EOF) || errors.Is(err, net.ErrClosed) {
return true
}
var networkError *net.OpError
if errors.As(err, &networkError) {
return true
}
lower := strings.ToLower(err.Error())
for _, marker := range []string{
"connection refused", "connection reset", "broken pipe", "connection closed",
"connection lost", "unexpected eof", "no route to host", "etcd connection error",
} {
if strings.Contains(lower, marker) {
return true
}
}
return false
}
func errTypeName(err error) string {
if err == nil {
return ""
}
return strings.ReplaceAll(strings.TrimSpace(err.Error()), "\n", " ")
}
func rpcErrorStage(method string) string {
switch method {
case "connect", "open_session", "test_connection":
return "connect"
case "validate_connection", "validate_session":
return "validate"
case "cancel_session":
return "cancel"
case "close_session", "disconnect", "shutdown":
return "close"
case "etcd_watch_poll":
return "fetch"
case "handshake", "":
return "request"
default:
return "execute"
}
}
func rpcOperationOutcome(stage string) string {
switch stage {
case "request", "connect", "validate":
return "not_started"
default:
return "unknown"
}
}
func isTimeoutError(err error) bool {
var timeout interface{ Timeout() bool }
return errors.As(err, &timeout) && timeout.Timeout()
}
func safeRPCDiagnostic(value string, maxLength int) string {
var result strings.Builder
for _, char := range value {
if result.Len() >= maxLength {
break
}
if char >= 0x21 && char <= 0x7e {
result.WriteRune(char)
}
}
return result.String()
}
agents/drivers/etcd2-go/status.go
+179
View File
@@ -0,0 +1,179 @@
package main
import (
"encoding/json"
"net/http"
"strconv"
"strings"
"time"
)
// status assembles a reduced kv_status document from the v2 endpoints:
// /v2/members, /v2/stats/self, /v2/stats/leader, and /v2/stats/store. Fields
// without a v2 analog (alarms, dbSize, raft detail) stay null.
func (s *etcd2Session) status(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
type member struct {
ID string `json:"id"`
Name string `json:"name"`
PeerURLs []string `json:"peerURLs"`
ClientURLs []string `json:"clientURLs"`
}
membersByID := map[string]*member{}
var memberList []member
var clusterID *string
if body, response, err := client.do(ctx, http.MethodGet, "/v2/members", "", nil); err == nil {
var parsed struct {
Members []member `json:"members"`
}
if json.Unmarshal(body, &parsed) == nil {
memberList = parsed.Members
for index := range memberList {
membersByID[memberList[index].ID] = &memberList[index]
}
}
if header := response.Header.Get("X-Etcd-Cluster-Id"); header != "" {
clusterID = &header
}
}
// The v2 API exposes the current index only as a response header on the
// keys endpoints, not in any stats document.
var revision *string
if _, response, err := client.do(ctx, http.MethodGet, "/v2/keys/", "", nil); err == nil {
if header := response.Header.Get("X-Etcd-Index"); header != "" {
revision = &header
}
}
type selfStats struct {
Name string `json:"name"`
ID string `json:"id"`
State string `json:"state"`
LeaderInfo struct {
Leader string `json:"leader"`
} `json:"leaderInfo"`
RecvAppendRequestCnt int64 `json:"recvAppendRequestCnt"`
SendAppendRequestCnt int64 `json:"sendAppendRequestCnt"`
}
var leaderID *string
selfByEndpoint := map[string]*selfStats{}
endpoints := s.connectedEndpointList()
for _, endpoint := range endpoints {
self := &selfStats{}
body, _, err := client.doAt(ctx, http.MethodGet, endpoint, "/v2/stats/self", "")
if err != nil {
continue
}
if json.Unmarshal(body, self) != nil {
continue
}
selfByEndpoint[endpoint] = self
if self.LeaderInfo.Leader != "" {
value := self.LeaderInfo.Leader
leaderID = &value
}
}
storeStats := map[string]any{}
if body, _, err := client.do(ctx, http.MethodGet, "/v2/stats/store", "", nil); err == nil {
var parsed map[string]any
if json.Unmarshal(body, &parsed) == nil {
storeStats = parsed
}
}
statusMembers := []map[string]any{}
observedIDs := map[string]struct{}{}
for _, endpoint := range endpoints {
row := map[string]any{"endpoint": endpoint}
started := time.Now()
self := selfByEndpoint[endpoint]
if self == nil {
row["memberId"] = nil
row["name"] = nil
row["version"] = nil
row["leaderId"] = nil
row["revision"] = nil
row["raftTerm"] = nil
row["raftIndex"] = nil
row["raftAppliedIndex"] = nil
row["dbSize"] = nil
row["dbSizeInUse"] = nil
row["learner"] = false
row["reachable"] = false
row["latencyMs"] = nil
row["errors"] = []string{"stats request failed"}
statusMembers = append(statusMembers, row)
continue
}
member := membersByID[self.ID]
row["memberId"] = memberHexToUnsigned(self.ID)
if member == nil {
row["name"] = self.Name
} else {
row["name"] = member.Name
}
row["version"] = client.serverVersion
row["leaderId"] = memberHexToUnsigned(self.LeaderInfo.Leader)
row["revision"] = nil
row["raftTerm"] = nil
row["raftIndex"] = nil
row["raftAppliedIndex"] = nil
row["dbSize"] = nil
row["dbSizeInUse"] = nil
row["learner"] = false
row["reachable"] = true
row["latencyMs"] = time.Since(started).Milliseconds()
row["errors"] = []string{}
row["state"] = strings.TrimPrefix(self.State, "State")
if _, seen := observedIDs[self.ID]; seen {
continue
}
observedIDs[self.ID] = struct{}{}
statusMembers = append(statusMembers, row)
}
var clusterIDValue any
if clusterID != nil {
clusterIDValue = memberHexToUnsigned(*clusterID)
}
var leaderIDValue any
if leaderID != nil {
leaderIDValue = memberHexToUnsigned(*leaderID)
}
var revisionValue any
if revision != nil {
revisionValue = *revision
}
// The v2 API has no key-count equivalent of the v3 count-only range;
// the raw store counters are exposed separately below.
var keyCountValue any
return map[string]any{
"clusterId": clusterIDValue,
"revision": revisionValue,
"leaderId": leaderIDValue,
"keyCount": keyCountValue,
"alarms": []string{},
"members": statusMembers,
"store": storeStats,
}, nil
}
// memberHexToUnsigned converts v2 hex member/cluster IDs (e.g. "272e204152")
// to the unsigned decimal strings the v3 agent emits.
func memberHexToUnsigned(hex string) any {
if hex == "" {
return nil
}
parsed, err := strconv.ParseUint(strings.TrimPrefix(hex, "0x"), 16, 64)
if err != nil {
return nil
}
return unsignedLongString(int64(parsed))
}
agents/drivers/etcd2-go/watch.go
+395
View File
@@ -0,0 +1,395 @@
package main
import (
"context"
"crypto/rand"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"sync"
"time"
)
const (
maxWatches = 4
maxWatchBatches = 256
maxWatchEvents = 10000
maxWatchBufferBytes = int64(8 * 1024 * 1024)
maxSessionWatchBufferBytes = int64(16 * 1024 * 1024)
// v2 long polls hold until an event arrives; re-issue the request when a
// poll is cut off so the pump survives idle periods.
watchPollTimeout = 5 * time.Minute
)
type bufferedWatchBatch struct {
payload map[string]any
bufferedBytes int64
}
// watchState buffers watch events for the polling host, mirroring the v3
// agent's budget system so host-side expectations stay identical.
type watchState struct {
watchID string
session *etcd2Session
mu sync.Mutex
batches []bufferedWatchBatch
eventCount int
bufferedBytes int64
terminalReason string
terminalMessage string
pumpCancel context.CancelFunc
closed bool
}
func (w *watchState) append(revision int64, events []map[string]any, batchBytes int64) {
w.mu.Lock()
defer w.mu.Unlock()
if w.terminalReason != "" {
return
}
if int64(len(w.batches)) >= maxWatchBatches ||
int64(w.eventCount+len(events)) > maxWatchEvents ||
batchBytes > maxWatchBufferBytes ||
w.bufferedBytes+batchBytes > maxWatchBufferBytes ||
!w.session.reserveWatchBuffer(batchBytes) {
w.overflowLocked()
return
}
w.batches = append(w.batches, bufferedWatchBatch{
payload: map[string]any{"revision": longString(revision), "events": events},
bufferedBytes: batchBytes,
})
w.eventCount += len(events)
w.bufferedBytes += batchBytes
}
func (w *watchState) overflowLocked() {
if w.terminalReason != "" {
return
}
w.terminalReason = "overflow"
w.terminalMessage = "ETCD_WATCH_OVERFLOW: the event buffer reached its byte or event limit"
w.stopPumpLocked()
}
func (w *watchState) fail(reason, message string) {
w.mu.Lock()
defer w.mu.Unlock()
if w.terminalReason == "" {
w.terminalReason = reason
w.terminalMessage = message
}
}
func (w *watchState) poll() map[string]any {
w.mu.Lock()
defer w.mu.Unlock()
page := make([]any, 0, 64)
for len(w.batches) > 0 && len(page) < 64 {
batch := w.batches[0]
w.batches = w.batches[1:]
events := batch.payload["events"].([]map[string]any)
w.eventCount -= len(events)
w.bufferedBytes -= batch.bufferedBytes
w.session.releaseWatchBuffer(batch.bufferedBytes)
page = append(page, batch.payload)
}
result := map[string]any{"watchId": w.watchID, "batches": page}
if w.terminalReason != "" && len(w.batches) == 0 {
result["terminal"] = map[string]any{
"reason": w.terminalReason,
"message": w.terminalMessage,
}
}
return result
}
func (w *watchState) close() {
w.mu.Lock()
w.failLocked("stopped", "watch stopped")
w.clearBufferedLocked()
w.closed = true
cancel := w.pumpCancel
w.pumpCancel = nil
w.mu.Unlock()
if cancel != nil {
cancel()
}
}
func (w *watchState) failLocked(reason, message string) {
if w.terminalReason == "" {
w.terminalReason = reason
w.terminalMessage = message
}
}
func (w *watchState) clearBufferedLocked() {
if w.bufferedBytes > 0 {
w.session.releaseWatchBuffer(w.bufferedBytes)
}
w.batches = nil
w.eventCount = 0
w.bufferedBytes = 0
}
func (w *watchState) attachPump(cancel context.CancelFunc) {
w.mu.Lock()
defer w.mu.Unlock()
if w.closed || w.terminalReason != "" {
cancel()
return
}
w.pumpCancel = cancel
}
func (w *watchState) stopPumpLocked() {
cancel := w.pumpCancel
w.pumpCancel = nil
if cancel != nil {
go cancel()
}
}
func watchEventBufferBytes(key, value, previousValue string) int64 {
bytes := int64(512 + 4*len(key))
bytes += estimatedBufferedBytes(len(value))
if previousValue != "" {
bytes += estimatedBufferedBytes(len(previousValue))
}
return bytes
}
func estimatedBufferedBytes(sourceBytes int) int64 {
if sourceBytes < 0 {
return 0
}
return int64(sourceBytes) * 4
}
func newWatchID() string {
var bytes [16]byte
if _, err := rand.Read(bytes[:]); err != nil {
return base64.StdEncoding.EncodeToString(bytes[:])
}
bytes[6] = (bytes[6] & 0x0f) | 0x40
bytes[8] = (bytes[8] & 0x3f) | 0x80
hexed := hex.EncodeToString(bytes[:])
return fmt.Sprintf("%s-%s-%s-%s-%s", hexed[0:8], hexed[8:12], hexed[12:16], hexed[16:20], hexed[20:32])
}
func (s *etcd2Session) watchCount() int {
s.watchesMu.Lock()
defer s.watchesMu.Unlock()
return len(s.watches)
}
func (s *etcd2Session) registerWatch(id string, state *watchState) bool {
s.watchesMu.Lock()
defer s.watchesMu.Unlock()
if _, exists := s.watches[id]; exists {
return false
}
s.watches[id] = state
return true
}
func (s *etcd2Session) removeWatch(id string) *watchState {
s.watchesMu.Lock()
defer s.watchesMu.Unlock()
state := s.watches[id]
delete(s.watches, id)
return state
}
func (s *etcd2Session) watchStart(params map[string]json.RawMessage) (any, error) {
if s.watchCount() >= maxWatches {
return nil, fmt.Errorf("ETCD_WATCH_LIMIT: at most %d watches are allowed per connection", maxWatches)
}
key, err := keyBytesParam(params)
if err != nil {
return nil, err
}
scope := stringOrDefault(params, "scope", "key")
if scope != "key" && scope != "prefix" {
return nil, errors.New("ETCD_WATCH_SCOPE_INVALID: scope must be key or prefix")
}
client, err := s.activeClient()
if err != nil {
return nil, err
}
// startRevision maps to the v2 waitIndex: default to the current index+1.
requestedRevision := longOrNull(params, "startRevision")
var waitIndex int64
if requestedRevision != nil && *requestedRevision > 0 {
waitIndex = *requestedRevision
} else {
current, err := client.currentEtcdIndex()
if err != nil {
return nil, err
}
waitIndex = current + 1
}
watchID := newWatchID()
state := &watchState{watchID: watchID, session: s}
pumpCtx, pumpCancel := context.WithCancel(context.Background())
go pumpLongPollWatch(pumpCtx, client, key, scope, waitIndex, boolOrDefault(params, "includePrevKv", false), state)
state.attachPump(pumpCancel)
s.registerWatch(watchID, state)
return map[string]any{"watchId": watchID, "startedRevision": longString(waitIndex)}, nil
}
// currentEtcdIndex reads the X-Etcd-Index header from a lightweight GET.
func (c *authenticatedClient) currentEtcdIndex() (int64, error) {
ctx, cancel := context.WithTimeout(context.Background(), operationTimeout)
defer cancel()
response, err := c.request(ctx, http.MethodGet, "/v2/keys/", "", nil)
if err != nil {
return 0, err
}
defer drainClose(response.Body)
if response.StatusCode < 200 || response.StatusCode >= 300 {
body, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
return 0, errorFromResponse(response.StatusCode, body)
}
index, err := strconv.ParseInt(response.Header.Get("X-Etcd-Index"), 10, 64)
if err != nil {
return 0, fmt.Errorf("etcd v2 server did not report X-Etcd-Index")
}
return index, nil
}
// pumpLongPollWatch drives the v2 wait loop: each response becomes one batch;
// timeouts and connection blips re-issue the same waitIndex.
func pumpLongPollWatch(ctx context.Context, client *authenticatedClient, key, scope string, waitIndex int64, includePrevKv bool, state *watchState) {
for {
if ctx.Err() != nil || state.hasTerminal() {
return
}
query := url.Values{}
query.Set("wait", "true")
query.Set("waitIndex", strconv.FormatInt(waitIndex, 10))
if scope == "prefix" {
query.Set("recursive", "true")
}
pollCtx, pollCancel := context.WithTimeout(ctx, watchPollTimeout)
body, _, err := client.do(pollCtx, http.MethodGet, v2KeyPath(key)+"?"+query.Encode(), "", nil)
pollCancel()
if err != nil {
if errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil {
// Idle long poll cut off: keep waiting from the same index.
continue
}
if ctx.Err() != nil {
return
}
var etcdErr *etcdError
if errors.As(err, &etcdErr) && etcdErr.errorCode == 401 {
state.fail("error", "ETCD_WATCH_EXPIRED: the watched index was cleared from the etcd v2 event history")
return
}
state.fail("error", "watch failed: "+err.Error())
return
}
var parsed v2KeysResponse
if jsonErr := json.Unmarshal(body, &parsed); jsonErr != nil {
state.fail("error", "watch response was not valid etcd v2 JSON: "+jsonErr.Error())
return
}
if parsed.Node == nil {
waitIndex++
continue
}
events := []map[string]any{}
bufferedBytes := int64(128)
node := parsed.Node
previous := parsed.PrevNode
deleted := parsed.Action == "delete" || parsed.Action == "expire"
previousValue := ""
if previous != nil {
previousValue = previous.Value
}
bufferedBytes += watchEventBufferBytes(node.Key, node.Value, previousValue)
if bufferedBytes > maxWatchBufferBytes {
state.mu.Lock()
state.overflowLocked()
state.mu.Unlock()
return
}
row := map[string]any{
"eventType": eventTypeForAction(parsed.Action),
"revision": longString(node.ModifiedIndex),
"key": node.Key,
"keyBytes": bytesObject([]byte(node.Key)),
}
if deleted {
row["value"] = nil
} else {
row["value"] = valueObject([]byte(node.Value))
}
if includePrevKv && previous != nil && previous.Value != "" {
row["previousValue"] = valueObject([]byte(previous.Value))
} else {
row["previousValue"] = nil
}
if deleted && previous != nil {
row["metadata"] = nodeMetadata(previous)
} else {
row["metadata"] = nodeMetadata(node)
}
events = append(events, row)
state.append(node.ModifiedIndex, events, bufferedBytes)
nextIndex := node.ModifiedIndex + 1
if nextIndex > waitIndex {
waitIndex = nextIndex
} else {
waitIndex++
}
}
}
func eventTypeForAction(action string) string {
if action == "delete" || action == "expire" || action == "compareAndDelete" {
return "delete"
}
return "put"
}
func (w *watchState) hasTerminal() bool {
w.mu.Lock()
defer w.mu.Unlock()
return w.terminalReason != ""
}
func (s *etcd2Session) watchPoll(params map[string]json.RawMessage) (any, error) {
watchID := stringOrDefault(params, "watchId", "")
s.watchesMu.Lock()
state := s.watches[watchID]
s.watchesMu.Unlock()
if state == nil {
return nil, errors.New("ETCD_WATCH_NOT_FOUND: watch does not exist")
}
result := state.poll()
if _, terminal := result["terminal"]; terminal {
if removed := s.removeWatch(watchID); removed != nil {
removed.close()
}
}
return result, nil
}
func (s *etcd2Session) watchStop(params map[string]json.RawMessage) (any, error) {
if state := s.removeWatch(stringOrDefault(params, "watchId", "")); state != nil {
state.close()
}
return map[string]bool{"stopped": true}, nil
}
agents/go-common/go-semver/go.mod
+3
View File
@@ -0,0 +1,3 @@
module github.com/coreos/go-semver/semver
go 1.24
agents/go-common/go-semver/semver.go
+296
View File
@@ -0,0 +1,296 @@
// Copyright 2013-2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Semantic Versions http://semver.org
package semver
import (
"bytes"
"errors"
"fmt"
"regexp"
"strconv"
"strings"
)
type Version struct {
Major int64
Minor int64
Patch int64
PreRelease PreRelease
Metadata string
}
type PreRelease string
func splitOff(input *string, delim string) (val string) {
parts := strings.SplitN(*input, delim, 2)
if len(parts) == 2 {
*input = parts[0]
val = parts[1]
}
return val
}
func New(version string) *Version {
return Must(NewVersion(version))
}
func NewVersion(version string) (*Version, error) {
v := Version{}
if err := v.Set(version); err != nil {
return nil, err
}
return &v, nil
}
// Must is a helper for wrapping NewVersion and will panic if err is not nil.
func Must(v *Version, err error) *Version {
if err != nil {
panic(err)
}
return v
}
// Set parses and updates v from the given version string. Implements flag.Value
func (v *Version) Set(version string) error {
metadata := splitOff(&version, "+")
preRelease := PreRelease(splitOff(&version, "-"))
dotParts := strings.SplitN(version, ".", 3)
if len(dotParts) != 3 {
return fmt.Errorf("%s is not in dotted-tri format", version)
}
if err := validateIdentifier(string(preRelease)); err != nil {
return fmt.Errorf("failed to validate pre-release: %v", err)
}
if err := validateIdentifier(metadata); err != nil {
return fmt.Errorf("failed to validate metadata: %v", err)
}
parsed := make([]int64, 3)
for i, v := range dotParts[:3] {
val, err := strconv.ParseInt(v, 10, 64)
parsed[i] = val
if err != nil {
return err
}
}
v.Metadata = metadata
v.PreRelease = preRelease
v.Major = parsed[0]
v.Minor = parsed[1]
v.Patch = parsed[2]
return nil
}
func (v Version) String() string {
var buffer bytes.Buffer
fmt.Fprintf(&buffer, "%d.%d.%d", v.Major, v.Minor, v.Patch)
if v.PreRelease != "" {
fmt.Fprintf(&buffer, "-%s", v.PreRelease)
}
if v.Metadata != "" {
fmt.Fprintf(&buffer, "+%s", v.Metadata)
}
return buffer.String()
}
func (v *Version) UnmarshalYAML(unmarshal func(interface{}) error) error {
var data string
if err := unmarshal(&data); err != nil {
return err
}
return v.Set(data)
}
func (v Version) MarshalJSON() ([]byte, error) {
return []byte(`"` + v.String() + `"`), nil
}
func (v *Version) UnmarshalJSON(data []byte) error {
l := len(data)
if l == 0 || string(data) == `""` {
return nil
}
if l < 2 || data[0] != '"' || data[l-1] != '"' {
return errors.New("invalid semver string")
}
return v.Set(string(data[1 : l-1]))
}
// Compare tests if v is less than, equal to, or greater than versionB,
// returning -1, 0, or +1 respectively.
func (v Version) Compare(versionB Version) int {
if cmp := recursiveCompare(v.Slice(), versionB.Slice()); cmp != 0 {
return cmp
}
return preReleaseCompare(v, versionB)
}
// Equal tests if v is equal to versionB.
func (v Version) Equal(versionB Version) bool {
return v.Compare(versionB) == 0
}
// LessThan tests if v is less than versionB.
func (v Version) LessThan(versionB Version) bool {
return v.Compare(versionB) < 0
}
// Slice converts the comparable parts of the semver into a slice of integers.
func (v Version) Slice() []int64 {
return []int64{v.Major, v.Minor, v.Patch}
}
func (p PreRelease) Slice() []string {
preRelease := string(p)
return strings.Split(preRelease, ".")
}
func preReleaseCompare(versionA Version, versionB Version) int {
a := versionA.PreRelease
b := versionB.PreRelease
/* Handle the case where if two versions are otherwise equal it is the
* one without a PreRelease that is greater */
if len(a) == 0 && (len(b) > 0) {
return 1
} else if len(b) == 0 && (len(a) > 0) {
return -1
}
// If there is a prerelease, check and compare each part.
return recursivePreReleaseCompare(a.Slice(), b.Slice())
}
func recursiveCompare(versionA []int64, versionB []int64) int {
if len(versionA) == 0 {
return 0
}
a := versionA[0]
b := versionB[0]
if a > b {
return 1
} else if a < b {
return -1
}
return recursiveCompare(versionA[1:], versionB[1:])
}
func recursivePreReleaseCompare(versionA []string, versionB []string) int {
// A larger set of pre-release fields has a higher precedence than a smaller set,
// if all of the preceding identifiers are equal.
if len(versionA) == 0 {
if len(versionB) > 0 {
return -1
}
return 0
} else if len(versionB) == 0 {
// We're longer than versionB so return 1.
return 1
}
a := versionA[0]
b := versionB[0]
aInt := false
bInt := false
aI, err := strconv.Atoi(versionA[0])
if err == nil {
aInt = true
}
bI, err := strconv.Atoi(versionB[0])
if err == nil {
bInt = true
}
// Numeric identifiers always have lower precedence than non-numeric identifiers.
if aInt && !bInt {
return -1
} else if !aInt && bInt {
return 1
}
// Handle Integer Comparison
if aInt && bInt {
if aI > bI {
return 1
} else if aI < bI {
return -1
}
}
// Handle String Comparison
if a > b {
return 1
} else if a < b {
return -1
}
return recursivePreReleaseCompare(versionA[1:], versionB[1:])
}
// BumpMajor increments the Major field by 1 and resets all other fields to their default values
func (v *Version) BumpMajor() {
v.Major += 1
v.Minor = 0
v.Patch = 0
v.PreRelease = PreRelease("")
v.Metadata = ""
}
// BumpMinor increments the Minor field by 1 and resets all other fields to their default values
func (v *Version) BumpMinor() {
v.Minor += 1
v.Patch = 0
v.PreRelease = PreRelease("")
v.Metadata = ""
}
// BumpPatch increments the Patch field by 1 and resets all other fields to their default values
func (v *Version) BumpPatch() {
v.Patch += 1
v.PreRelease = PreRelease("")
v.Metadata = ""
}
// validateIdentifier makes sure the provided identifier satisfies semver spec
func validateIdentifier(id string) error {
if id != "" && !reIdentifier.MatchString(id) {
return fmt.Errorf("%s is not a valid semver identifier", id)
}
return nil
}
// reIdentifier is a regular expression used to check that pre-release and metadata
// identifiers satisfy the spec requirements
var reIdentifier = regexp.MustCompile(`^[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*$`)
agents/go-common/go-semver/sort.go
+38
View File
@@ -0,0 +1,38 @@
// Copyright 2013-2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package semver
import (
"sort"
)
type Versions []*Version
func (s Versions) Len() int {
return len(s)
}
func (s Versions) Swap(i, j int) {
s[i], s[j] = s[j], s[i]
}
func (s Versions) Less(i, j int) bool {
return s[i].LessThan(*s[j])
}
// Sort sorts the given slice of Version
func Sort(versions []*Version) {
sort.Sort(Versions(versions))
}
agents/scripts/driver_release_packages_test.py
+12
View File
@@ -30,6 +30,10 @@ class DriverReleasePackagesTest(unittest.TestCase):
cassandra_source.write_bytes(b"\x7fELFtest-cassandra-agent")
tdengine_source = release_dir / "dbx-agent-tdengine-windows-aarch64.exe"
tdengine_source.write_bytes(b"MZtest-tdengine-agent")
etcd_source = release_dir / "dbx-agent-etcd-linux-x64"
etcd_source.write_bytes(b"\x7fELFtest-etcd-agent")
etcd2_source = release_dir / "dbx-agent-etcd2-macos-aarch64"
etcd2_source.write_bytes(b"test-etcd2-agent")
java_source = release_dir / "dbx-agent-h2.jar"
java_source.write_bytes(b"test-jar")
versions = {
@@ -47,6 +51,8 @@ class DriverReleasePackagesTest(unittest.TestCase):
"cassandra": "0.1.37",
"hive": "0.1.43",
"tdengine": "0.1.0",
"etcd": "0.1.40",
"etcd2": "0.1.0",
"sqlite-worker": "0.1.0",
}
@@ -59,6 +65,8 @@ class DriverReleasePackagesTest(unittest.TestCase):
versioned_rocketmq = release_dir / "dbx-agent-rocketmq-0.1.0-windows-x64.exe"
versioned_cassandra = release_dir / "dbx-agent-cassandra-0.1.37-linux-x64"
versioned_tdengine = release_dir / "dbx-agent-tdengine-0.1.0-windows-aarch64.exe"
versioned_etcd = release_dir / "dbx-agent-etcd-0.1.40-linux-x64"
versioned_etcd2 = release_dir / "dbx-agent-etcd2-0.1.0-macos-aarch64"
self.assertEqual(
renamed,
[
@@ -70,6 +78,8 @@ class DriverReleasePackagesTest(unittest.TestCase):
versioned_rabbitmq,
versioned_rocketmq,
versioned_tdengine,
versioned_etcd,
versioned_etcd2,
],
)
@@ -246,6 +256,8 @@ class DriverReleasePackagesTest(unittest.TestCase):
[
versioned_cassandra,
versioned_duckdb,
versioned_etcd,
versioned_etcd2,
versioned_java,
versioned_native,
versioned_rabbitmq,
agents/scripts/validate_agents.py
+3 -1
View File
@@ -11,7 +11,7 @@ SOURCE_GLOBS = ("*/src/main/**/*.java", "drivers/*/src/main/**/*.java")
KOTLIN_FILE_SUFFIXES = (".kt", ".kts")
KOTLIN_SCAN_EXCLUDED_PARTS = {".git", ".gradle", "build"}
DEFAULT_AGENT_JRE_KEY = "21"
NON_JDBC_AGENT_MODULES = {"mongodb", "etcd", "zookeeper", "kafka", "rocketmq", "rabbitmq"}
NON_JDBC_AGENT_MODULES = {"mongodb", "zookeeper", "kafka", "rocketmq", "rabbitmq"}
NATIVE_ONLY_AGENT_MODULES = {
"cassandra": "drivers/cassandra-go",
"duckdb": "drivers/duckdb",
@@ -26,6 +26,8 @@ NATIVE_ONLY_AGENT_MODULES = {
"rabbitmq": "drivers/rabbitmq",
"rocketmq": "drivers/rocketmq",
"zookeeper": "drivers/zookeeper",
"etcd": "drivers/etcd-go",
"etcd2": "drivers/etcd2-go",
}
CRATE_NATIVE_AGENT_MODULES = {
"sqlite-worker": Path("..") / "crates" / "dbx-sqlite-worker",
agents/scripts/version_agent_artifacts.py
+1 -1
View File
@@ -4,7 +4,7 @@ import json
from pathlib import Path
NATIVE_DRIVERS = ("cassandra", "hive", "oracle", "xugu", "kingbase", "iotdb", "neo4j", "vastbase", "duckdb", "rabbitmq", "rocketmq", "zookeeper", "tdengine", "sqlite-worker")
NATIVE_DRIVERS = ("cassandra", "hive", "oracle", "xugu", "kingbase", "iotdb", "neo4j", "vastbase", "duckdb", "rabbitmq", "rocketmq", "zookeeper", "tdengine", "etcd", "etcd2", "sqlite-worker")
PLATFORMS = (
"macos-aarch64",
"macos-x64",
agents/settings.gradle
+1 -1
View File
@@ -6,7 +6,7 @@ def driverModules = [
'teradata', 'vertica', 'firebird', 'exasol', 'oceanbase-oracle', 'gbase8a', 'gbase8s',
'bigquery', 'kylin', 'ignite', 'ignite3', 'spanner', 'sundb', 'h2', 'h2-legacy', 'snowflake', 'trino', 'spark',
'db2', 'informix', 'mongodb', 'highgo', 'uxdb', 'yashandb', 'oscar',
'iris', 'etcd', 'kafka', 'sqlserver-legacy'
'iris', 'kafka', 'sqlserver-legacy'
]
include(*(infrastructureModules + driverModules))
agents/versions.json
+1
View File
@@ -43,6 +43,7 @@
"iris": "0.1.43",
"iotdb": "0.1.38",
"etcd": "0.1.39",
"etcd2": "0.1.0",
"zookeeper": "0.1.27",
"kafka": "0.1.14",
"rocketmq": "0.1.10",
apps/desktop/src/components/connection/ConnectionDialog.vue
+17
View File
@@ -2786,6 +2786,11 @@ function switchH2ConnectionMode(mode: H2ConnectionMode) {
resetTestState();
}
function switchEtcdApiVersion(profile: "etcd" | "etcd-v2") {
form.value.driver_profile = profile;
resetTestState();
}
function switchH2DriverProfile(profile: "h2" | "h2-v1" | "h2-v2" | "h2-v3" | "h2-custom") {
form.value.driver_profile = profile;
if (profile === "h2-custom") {
@@ -6872,6 +6877,18 @@ function openExternalUrl(url: string) {
<!-- etcd: endpoints, user, password, TLS -->
<template v-else-if="form.db_type === 'etcd'">
<div class="grid grid-cols-4 items-center gap-4">
<Label :class="connectionLabelClass">API</Label>
<div class="col-span-3 space-y-1.5">
<div class="flex flex-wrap gap-2">
<Button size="sm" :variant="!form.driver_profile || form.driver_profile === 'etcd' ? 'default' : 'outline'" @click="switchEtcdApiVersion('etcd')">v3 (etcd 3.x)</Button>
<Button size="sm" :variant="form.driver_profile === 'etcd-v2' ? 'default' : 'outline'" @click="switchEtcdApiVersion('etcd-v2')">v2 (etcd 2.x)</Button>
</div>
<p v-if="form.driver_profile === 'etcd-v2'" class="text-xs text-muted-foreground">
{{ t("connection.etcdV2ApiHint") }}
</p>
</div>
</div>
<div class="grid grid-cols-4 items-center gap-4">
<Label :class="connectionLabelClass">{{ t("connection.host") }}</Label>
<Input v-model="form.host" class="col-span-2" />
apps/desktop/src/components/etcd/EtcdKeyBrowser.vue
+10 -6
View File
@@ -155,6 +155,8 @@ const transferPreviewLoaded = ref(false);
const syncConfigurationExpanded = ref(true);
const readOnly = computed(() => connectionIsEffectivelyReadOnly(connectionStore.getConfig(props.connectionId)));
// etcd 2.x connections speak the v2 API: no key history, no leases, no maintenance.
const isV2Api = computed(() => connectionStore.getConfig(props.connectionId)?.driver_profile === "etcd-v2");
const etcdConnections = computed(() => connectionStore.connections.filter((connection) => connection.db_type === "etcd"));
const targetReadOnly = computed(() => connectionIsEffectivelyReadOnly(connectionStore.getConfig(targetConnectionId.value)));
const selectedTransferRows = computed(() => transferRows.value.filter((row) => row.selected && isTransferRowSelectable(row)));
@@ -259,7 +261,7 @@ function keyOptions(key: string): api.KvGetOptions {
return { keyBytes: candidates?.size === 1 ? [...candidates.values()][0] : undefined };
}
const etcdApi = {
const etcdApi = computed(() => ({
async listPrefix(connectionId: string, prefix: string, limit: number, continuation?: string | null, options?: api.KvListPrefixOptions | null) {
const response = await api.etcdListPrefix(connectionId, prefix, limit, continuation, options);
return { ...response, keys: response.keys.map(rememberSummary) };
@@ -279,9 +281,11 @@ const etcdApi = {
}),
deleteKey: (connectionId: string, key: string, options?: api.KvDeleteOptions | null) => api.etcdDelete(connectionId, key, { ...options, keyBytes: options?.keyBytes ?? keyOptions(key).keyBytes }),
rename: api.etcdRename,
history: api.etcdHistory,
// The v2 API has no event history; hiding the entry keeps KvKeyBrowser from
// offering restore/compare actions it cannot fulfill.
history: isV2Api.value ? undefined : api.etcdHistory,
exportScope: exportEtcdNodeScope,
};
}));
const labels = computed(() => ({
prefixPlaceholder: t("etcd.prefixPlaceholder"),
@@ -1030,7 +1034,7 @@ defineExpose({ focusSearch, refresh });
<Button size="sm" :variant="mode === 'keys' ? 'secondary' : 'ghost'" class="h-8 gap-1.5 px-3 text-sm" @click="mode = 'keys'"><KeyRound class="h-4 w-4" /> {{ t("etcd.key") }}</Button>
<Button size="sm" :variant="mode === 'search' ? 'secondary' : 'ghost'" class="h-8 gap-1.5 px-3 text-sm" @click="mode = 'search'"><Search class="h-4 w-4" /> {{ t("etcd.globalSearch") }}</Button>
</div>
<Button size="sm" :variant="mode === 'maintenance' ? 'secondary' : 'ghost'" class="h-8 gap-1.5 px-2.5 text-sm" @click="openOperations('maintenance')"><Wrench class="h-4 w-4" />{{ t("etcd.admin.maintenance") }}</Button>
<Button v-if="!isV2Api" size="sm" :variant="mode === 'maintenance' ? 'secondary' : 'ghost'" class="h-8 gap-1.5 px-2.5 text-sm" @click="openOperations('maintenance')"><Wrench class="h-4 w-4" />{{ t("etcd.admin.maintenance") }}</Button>
<Button
size="sm"
:variant="mode === 'watch' ? 'secondary' : 'ghost'"
@@ -1041,7 +1045,7 @@ defineExpose({ focusSearch, refresh });
"
><Activity class="h-4 w-4" />{{ t("etcd.admin.watch") }}</Button
>
<Button size="sm" :variant="mode === 'lease' ? 'secondary' : 'ghost'" class="h-8 gap-1.5 px-2.5 text-sm" @click="openOperations('lease')"><KeyRound class="h-4 w-4" />{{ t("etcd.admin.lease") }}</Button>
<Button v-if="!isV2Api" size="sm" :variant="mode === 'lease' ? 'secondary' : 'ghost'" class="h-8 gap-1.5 px-2.5 text-sm" @click="openOperations('lease')"><KeyRound class="h-4 w-4" />{{ t("etcd.admin.lease") }}</Button>
<div class="flex-1" />
<Badge v-if="readOnly" variant="outline">{{ t("connection.readOnly") }}</Badge>
<DropdownMenu>
@@ -1074,7 +1078,7 @@ defineExpose({ focusSearch, refresh });
:api="etcdApi"
:labels="labels"
:supports-ttl="supportsTtl"
:supports-lease-binding="true"
:supports-lease-binding="!isV2Api"
:ttl-capability-known="ttlCapabilityKnown"
:enable-node-actions="true"
:safe-write="true"
apps/desktop/src/components/icons/DatabaseIcon.vue
+1
View File
@@ -100,6 +100,7 @@ const assetIcons: Record<string, string> = {
xugu: "xugu.png",
iotdb: "iotdb",
etcd: "etcd",
etcd2: "etcd",
qdrant: "qdrant",
milvus: "milvus.png",
weaviate: "weaviate",
apps/desktop/src/i18n/locales/en.ts
+1
View File
@@ -645,6 +645,7 @@ export default {
redisKeyTemplatesHint: "One template per line. When set, these replace the global Redis key templates for this connection.",
etcdEndpoints: "Endpoints",
etcdEndpointsHint: "One endpoint per line. Leave blank to use the host and port above.",
etcdV2ApiHint: "The v2 API is only available on etcd 2.x, or etcd 3.0–3.5 started with --enable-v2. Key history, leases, compaction, and defragmentation are not available.",
etcdGrpcMaxInbound: "gRPC receive limit (MiB)",
etcdGrpcMaxInboundHint: "Maximum size of one etcd response. Defaults to 32 MiB and is capped at 256 MiB.",
etcdCaCertPlaceholder: "/path/to/ca.crt",
apps/desktop/src/i18n/locales/es.ts
+1
View File
@@ -615,6 +615,7 @@ export default withEnglishFallback({
redisKeyTemplatesHint: "Una plantilla por línea. Si se define, reemplaza las plantillas globales para esta conexión.",
etcdEndpoints: "Endpoints",
etcdEndpointsHint: "Una entrada por línea. Déjalo en blanco para usar el host y puerto de arriba.",
etcdV2ApiHint: "La API v2 solo está disponible en etcd 2.x, o etcd 3.0–3.5 iniciado con --enable-v2. El historial de claves, los leases, la compactación y la desfragmentación no están disponibles.",
etcdGrpcMaxInbound: "Límite de recepción gRPC (MiB)",
etcdGrpcMaxInboundHint: "Tamaño máximo de una respuesta de etcd. El valor predeterminado es 32 MiB y el máximo es 256 MiB.",
etcdCaCertPlaceholder: "/ruta/a/ca.crt",
apps/desktop/src/i18n/locales/it.ts
+1
View File
@@ -614,6 +614,7 @@ export default withEnglishFallback({
redisKeyTemplatesHint: "Un modello per riga. Se impostato, sostituisce i modelli globali per questa connessione.",
etcdEndpoints: "Endpoint",
etcdEndpointsHint: "Un endpoint per riga. Lascia vuoto per usare l'host e la porta sopra.",
etcdV2ApiHint: "L'API v2 è disponibile solo su etcd 2.x, o etcd 3.0–3.5 avviato con --enable-v2. Cronologia chiavi, lease, compattazione e deframmentazione non sono disponibili.",
etcdGrpcMaxInbound: "Limite ricezione gRPC (MiB)",
etcdGrpcMaxInboundHint: "Dimensione massima di una risposta etcd. Il valore predefinito è 32 MiB e il limite è 256 MiB.",
etcdCaCertPlaceholder: "/percorso/per/ca.crt",
apps/desktop/src/i18n/locales/ja.ts
+1
View File
@@ -614,6 +614,7 @@ export default withEnglishFallback({
redisKeyTemplatesHint: "1行に1つのテンプレート。設定するとこの接続のグローバルテンプレートを上書きします。",
etcdEndpoints: "エンドポイント",
etcdEndpointsHint: "1行に1つのエンドポイント。空白の場合、上のホストとポートを使用します。",
etcdV2ApiHint: "v2 API は etcd 2.x、または --enable-v2 を指定して起動した etcd 3.0–3.5 でのみ使用できます。キー履歴、リース、コンパクション、デフラグは利用できません。",
etcdGrpcMaxInbound: "gRPC 受信上限（MiB）",
etcdGrpcMaxInboundHint: "etcd 応答1件の最大サイズです。既定値は32 MiB、上限は256 MiBです。",
etcdCaCertPlaceholder: "/path/to/ca.crt",
apps/desktop/src/i18n/locales/ko.ts
+1
View File
@@ -639,6 +639,7 @@ export default withEnglishFallback({
redisKeyTemplatesHint: "한 줄에 하나의 템플릿. 설정하면 이 연결의 전역 템플릿을 덮어씁니다.",
etcdEndpoints: "엔드포인트",
etcdEndpointsHint: "한 줄에 하나의 엔드포인트. 비워두면 위의 호스트와 포트를 사용합니다.",
etcdV2ApiHint: "v2 API는 etcd 2.x 또는 --enable-v2로 시작한 etcd 3.0–3.5에서만 사용할 수 있습니다. 키 히스토리, 리스, 컨팩션, 조각모음은 지원되지 않습니다.",
etcdGrpcMaxInbound: "gRPC 수신 제한 (MiB)",
etcdGrpcMaxInboundHint: "단일 etcd 응답의 최대 크기입니다. 기본값은 32 MiB이며 최대 256 MiB입니다.",
etcdCaCertPlaceholder: "/path/to/ca.crt",
apps/desktop/src/i18n/locales/pt-BR.ts
+1
View File
@@ -615,6 +615,7 @@ export default withEnglishFallback({
redisKeyTemplatesHint: "Um modelo por linha. Quando definido, substitui os modelos globais nesta conexão.",
etcdEndpoints: "Endpoints",
etcdEndpointsHint: "Um endpoint por linha. Deixe em branco para usar o host e porta acima.",
etcdV2ApiHint: "A API v2 está disponível apenas no etcd 2.x, ou no etcd 3.0–3.5 iniciado com --enable-v2. Histórico de chaves, leases, compactação e desfragmentação não estão disponíveis.",
etcdGrpcMaxInbound: "Limite de recebimento gRPC (MiB)",
etcdGrpcMaxInboundHint: "Tamanho máximo de uma resposta do etcd. O padrão é 32 MiB e o limite é 256 MiB.",
etcdCaCertPlaceholder: "/caminho/para/ca.crt",
apps/desktop/src/i18n/locales/zh-CN.ts
+1
View File
@@ -570,6 +570,7 @@ export default withEnglishFallback({
redisKeyTemplatesHint: "每行一个模板。填写后将覆盖全局 Redis Key 模板，仅对本连接生效。",
etcdEndpoints: "Endpoints",
etcdEndpointsHint: "每行一个 endpoint。留空时使用上面的 host 和端口。",
etcdV2ApiHint: "v2 API 仅适用于 etcd 2.x，或以 --enable-v2 启动的 etcd 3.0–3.5。不支持键历史、Lease、压缩和碎片整理。",
etcdGrpcMaxInbound: "gRPC 接收上限（MiB）",
etcdGrpcMaxInboundHint: "单条 etcd 响应的最大大小。默认 32 MiB，最高 256 MiB。",
etcdCaCertPlaceholder: "/path/to/ca.crt",
apps/desktop/src/i18n/locales/zh-TW.ts
+1
View File
@@ -615,6 +615,7 @@ export default withEnglishFallback({
redisKeyTemplatesHint: "每行一個模板。設定後會覆寫全域 Redis Key 模板，僅對此連線生效。",
etcdEndpoints: "端點",
etcdEndpointsHint: "每行一個端點。留空則使用上方的主機和連接埠。",
etcdV2ApiHint: "v2 API 僅適用於 etcd 2.x，或以 --enable-v2 啟動的 etcd 3.0–3.5。不支援鍵歷史、Lease、壓縮和碎片整理。",
etcdGrpcMaxInbound: "gRPC 接收上限（MiB）",
etcdGrpcMaxInboundHint: "單筆 etcd 回應的最大大小。預設 32 MiB，最高 256 MiB。",
etcdCaCertPlaceholder: "/path/to/ca.crt",
apps/desktop/src/lib/__tests__/connection/etcdConnectionDialog.spec.ts
Vendored
+35
View File
@@ -0,0 +1,35 @@
import { readFileSync } from "node:fs";
import { describe, expect, it } from "vitest";
import { CONNECTION_PICKER_OPTIONS, CONNECTION_PROFILES } from "@/types/generated/connectionProfiles";
const dialogSource = readFileSync(new URL("../../../components/connection/ConnectionDialog.vue", import.meta.url), "utf8");
const browserSource = readFileSync(new URL("../../../components/etcd/EtcdKeyBrowser.vue", import.meta.url), "utf8");
describe("etcd connection dialog API versions", () => {
it("offers v3 and v2 API profile switches", () => {
expect(dialogSource).toContain("@click=\"switchEtcdApiVersion('etcd')\"");
expect(dialogSource).toContain("@click=\"switchEtcdApiVersion('etcd-v2')\"");
expect(dialogSource).toContain("v3 (etcd 3.x)");
expect(dialogSource).toContain("v2 (etcd 2.x)");
});
it("hydrates the v2 profile without exposing a second catalog entry", () => {
expect(CONNECTION_PROFILES["etcd-v2"]).toMatchObject({ type: "etcd", port: 2379 });
expect(CONNECTION_PICKER_OPTIONS.some((option) => option.value === "etcd-v2")).toBe(false);
});
});
describe("etcd v2 feature trimming", () => {
it("detects the v2 profile from the connection config", () => {
expect(browserSource).toContain('connectionStore.getConfig(props.connectionId)?.driver_profile === "etcd-v2"');
});
it("hides key history for v2 connections", () => {
expect(browserSource).toContain("history: isV2Api.value ? undefined : api.etcdHistory");
});
it("hides lease binding and the maintenance/lease workspaces for v2 connections", () => {
expect(browserSource).toContain(':supports-lease-binding="!isV2Api"');
expect(browserSource).toContain('<Button v-if="!isV2Api"');
});
});
apps/desktop/src/types/generated/connectionProfiles.ts
+2
View File
@@ -107,6 +107,7 @@ export const CONNECTION_PROFILES = {
xugu: { type: "xugu", port: 5138, user: "", label: "虚谷 XuguDB", icon: "xugu" },
iotdb: { type: "iotdb", port: 6667, user: "root", label: "Apache IoTDB", icon: "iotdb" },
etcd: { type: "etcd", port: 2379, user: "", label: "etcd", icon: "etcd" },
"etcd-v2": { type: "etcd", port: 2379, user: "", label: "etcd 2.x (v2 API)", icon: "etcd" },
zookeeper: { type: "zookeeper", port: 2181, user: "", label: "Apache ZooKeeper", icon: "zookeeper" },
mq: { type: "mq", port: 8080, user: "", label: "Apache Pulsar", icon: "pulsar", host: "127.0.0.1" },
kafka: { type: "mq", port: 9092, user: "", label: "Apache Kafka", icon: "kafka", host: "127.0.0.1" },
@@ -212,6 +213,7 @@ export const CONNECTION_PROFILE_ICONS = {
xugu: "xugu",
iotdb: "iotdb",
etcd: "etcd",
"etcd-v2": "etcd",
zookeeper: "zookeeper",
mq: "mq",
kafka: "kafka",
crates/dbx-core/assets/database-drivers.manifest.json
+9
View File
@@ -2397,6 +2397,15 @@
"runtimeMode": "agent",
"mcpMode": "bridge",
"agentKey": "etcd",
"driverProfiles": [
{
"profile": "etcd-v2",
"agentKey": "etcd2",
"label": "etcd 2.x (v2 API)",
"storeVisible": true,
"storeOrder": 49
}
],
"driverStoreVisible": true,
"driverStoreOrder": 39,
"singleConnectionPool": false,
crates/dbx-core/src/agent_catalog.rs
+12
View File
@@ -86,6 +86,18 @@ mod tests {
assert!(!driver_store_entries().any(|(key, _)| key == "h2-legacy"));
}
#[test]
fn etcd_v2_profile_uses_dedicated_agent() {
assert_eq!(agent_key(&DatabaseType::Etcd, None), Some("etcd"));
assert_eq!(agent_key(&DatabaseType::Etcd, Some("etcd")), Some("etcd"));
assert_eq!(agent_key(&DatabaseType::Etcd, Some("etcd-v2")), Some("etcd2"));
assert_eq!(agent_key(&DatabaseType::Etcd, Some("etcd-custom")), Some("etcd"));
assert_eq!(label_for_key("etcd2"), Some("etcd 2.x (v2 API)"));
// etcd2 ships its own binary, version, and registry entry, so it must
// stay visible in the driver store for install/upgrade/uninstall.
assert!(driver_store_entries().any(|(key, label)| key == "etcd2" && label == "etcd 2.x (v2 API)"));
}
#[test]
fn duckdb_is_available_in_driver_store_without_using_agent_runtime() {
assert!(driver_store_entries().any(|(key, label)| key == "duckdb" && label == "DuckDB"));
crates/dbx-core/src/agent_kv.rs
+53 -32
View File
@@ -1007,13 +1007,20 @@ async fn ensure_etcd_dangerous_action_capability(
_ => AgentCapability::EtcdAuth,
};
ensure_agent_kv_pool(state, connection_id).await?;
let agent_key = {
let configs = state.configs.read().await;
configs
.get(connection_id)
.and_then(|config| crate::agent_catalog::agent_key(&config.db_type, config.driver_profile.as_deref()))
};
let connections = state.connections.read().await;
let PoolKind::Agent(client) = connections.get(connection_id).ok_or("Connection not found")? else {
return Err("Not an agent key-value connection".to_string());
};
if !client.lock().await.supports_capability(capability) {
let note = etcd_capability_unavailable_note(agent_key);
return Err(format!(
"ETCD_CAPABILITY_UNSUPPORTED: Installed etcd Agent does not support {}. Update the etcd driver and reconnect.",
"ETCD_CAPABILITY_UNSUPPORTED: Installed etcd Agent does not support {}.{note}",
capability.as_str()
));
}
@@ -1724,6 +1731,12 @@ async fn call_agent_kv<T: serde::de::DeserializeOwned + Send + 'static>(
.is_some_and(|value| value.as_str() == Some("0") || value.as_i64() == Some(0));
ensure_agent_kv_pool(state, connection_id).await?;
let agent_key = {
let configs = state.configs.read().await;
configs
.get(connection_id)
.and_then(|config| crate::agent_catalog::agent_key(&config.db_type, config.driver_profile.as_deref()))
};
let client = {
let connections = state.connections.read().await;
match connections.get(connection_id) {
@@ -1740,38 +1753,35 @@ async fn call_agent_kv<T: serde::de::DeserializeOwned + Send + 'static>(
if let Some(capability) =
required_capabilities.into_iter().find(|capability| !agent.supports_capability(*capability))
{
let note = etcd_capability_unavailable_note(agent_key);
return Err(match capability {
AgentCapability::KvTtl => {
"Installed etcd Agent does not support TTL. Update the etcd driver and reconnect.".to_string()
}
AgentCapability::KvCas => {
"ETCD_CAS_UNSUPPORTED: Installed etcd Agent cannot safely compare and update Keys. Update the etcd driver and reconnect."
.to_string()
}
AgentCapability::KvListValues => {
"ETCD_LIST_VALUES_UNSUPPORTED: Installed etcd Agent cannot export or search Key values safely. Update the etcd driver and reconnect."
.to_string()
}
AgentCapability::KvStatus => {
"ETCD_STATUS_UNSUPPORTED: Installed etcd Agent does not support Dashboard status. Update the etcd driver and reconnect."
.to_string()
}
AgentCapability::KvHistory => {
"ETCD_HISTORY_UNSUPPORTED: Installed etcd Agent does not support key history. Update the etcd driver and reconnect."
.to_string()
}
AgentCapability::EtcdCompaction
| AgentCapability::EtcdDefrag
| AgentCapability::EtcdWatch
| AgentCapability::EtcdLease
| AgentCapability::EtcdAuth => {
format!(
"ETCD_CAPABILITY_UNSUPPORTED: Installed etcd Agent does not support {}. Update the etcd driver and reconnect.",
capability.as_str()
)
}
_ => format!("Agent does not support required capability: {}", capability.as_str()),
});
AgentCapability::KvTtl => {
format!("Installed etcd Agent does not support TTL.{note}")
}
AgentCapability::KvCas => {
format!("ETCD_CAS_UNSUPPORTED: Installed etcd Agent cannot safely compare and update Keys.{note}")
}
AgentCapability::KvListValues => {
format!("ETCD_LIST_VALUES_UNSUPPORTED: Installed etcd Agent cannot export or search Key values safely.{note}")
}
AgentCapability::KvStatus => {
format!("ETCD_STATUS_UNSUPPORTED: Installed etcd Agent does not support Dashboard status.{note}")
}
AgentCapability::KvHistory => {
format!("ETCD_HISTORY_UNSUPPORTED: Installed etcd Agent does not support key history.{note}")
}
AgentCapability::EtcdCompaction
| AgentCapability::EtcdDefrag
| AgentCapability::EtcdWatch
| AgentCapability::EtcdLease
| AgentCapability::EtcdAuth => {
format!(
"ETCD_CAPABILITY_UNSUPPORTED: Installed etcd Agent does not support {}.{note}",
capability.as_str()
)
}
_ => format!("Agent does not support required capability: {}", capability.as_str()),
});
}
agent.call_kv_method(method, params).await
};
@@ -1788,6 +1798,17 @@ async fn call_agent_kv<T: serde::de::DeserializeOwned + Send + 'static>(
}
}
/// Capability-blocked etcd calls explain themselves differently depending on
/// the backend: a missing capability on the v3 agent suggests upgrading the
/// driver, while the dedicated v2 agent simply cannot offer it.
fn etcd_capability_unavailable_note(agent_key: Option<&str>) -> &'static str {
if agent_key == Some("etcd2") {
" The etcd v2 API does not provide this capability."
} else {
" Update the etcd driver and reconnect."
}
}
fn normalize_agent_kv_error(error: &str, method: AgentKvMethod, create_only_write: bool) -> String {
let without_stderr = strip_recent_agent_stderr(error);
let message = strip_agent_rpc_error_prefix(without_stderr);
deploy/database/etcd/3.5/compose.yaml
+32
View File
@@ -0,0 +1,32 @@
services:
database:
image: quay.io/coreos/etcd:v3.5.21
container_name: dbx-etcd-3.5
restart: always
command:
- /usr/local/bin/etcd
- --name=s1
- --data-dir=/etcd-data
- --listen-client-urls=http://0.0.0.0:2379
- --advertise-client-urls=http://0.0.0.0:2379
- --listen-peer-urls=http://0.0.0.0:2380
- --initial-advertise-peer-urls=http://0.0.0.0:2380
- --initial-cluster=s1=http://0.0.0.0:2380
- --initial-cluster-token=tkn
- --initial-cluster-state=new
- --log-level=info
- --logger=zap
- --log-outputs=stderr
ports:
- "${DB_BIND_ADDRESS:-127.0.0.1}:${DB_PORT:-10710}:2379"
- "${DB_BIND_ADDRESS:-127.0.0.1}:${ETCD_PEER_PORT:-10711}:2380"
volumes:
- data:/etcd-data
healthcheck:
test: ["CMD", "/usr/local/bin/etcdctl", "--endpoints=http://127.0.0.1:2379", "endpoint", "health"]
interval: 5s
timeout: 5s
retries: 30
start_period: 10s
volumes:
data:
deploy/database/etcd/3.5/recipe.json
+23
View File
@@ -0,0 +1,23 @@
{
"database": "etcd",
"name": "etcd",
"version": "3.5.21",
"displayVersion": "3.5",
"image": "quay.io/coreos/etcd:v3.5.21",
"platforms": ["linux/amd64", "linux/arm64"],
"service": "database",
"defaultPort": 2379,
"connection": { "host": "127.0.0.1", "port": 10710, "username": "root", "password": "123456", "database": "dbx", "peerPort": 10711 },
"hostPorts": { "DB_PORT": 10710, "ETCD_PEER_PORT": 10711 },
"bootstrap": {
"check": { "command": ["/usr/local/bin/etcdctl", "--endpoints=http://127.0.0.1:2379", "--user", "root:${DB_PASSWORD}", "auth", "status"], "expect": "Authentication Status: true" },
"steps": [
{ "name": "create root user", "command": ["/usr/local/bin/etcdctl", "--endpoints=http://127.0.0.1:2379", "user", "add", "root:${DB_PASSWORD}"], "expect": "User root created" },
{ "name": "create root role", "command": ["/usr/local/bin/etcdctl", "--endpoints=http://127.0.0.1:2379", "role", "add", "root"], "expect": "Role root created" },
{ "name": "grant root role", "command": ["/usr/local/bin/etcdctl", "--endpoints=http://127.0.0.1:2379", "user", "grant-role", "root", "root"], "expect": "Role root is granted to user root" },
{ "name": "enable authentication", "command": ["/usr/local/bin/etcdctl", "--endpoints=http://127.0.0.1:2379", "auth", "enable"], "expect": "Authentication Enabled" }
]
},
"shell": ["/usr/local/bin/etcdctl", "--endpoints=http://127.0.0.1:2379", "--user", "root:${DB_PASSWORD}"],
"smoke": { "steps": [{ "name": "write authenticated smoke key", "command": ["/usr/local/bin/etcdctl", "--endpoints=http://127.0.0.1:2379", "--user", "root:${DB_PASSWORD}", "put", "dbx:smoke", "DBX smoke"], "expect": "OK" }, { "name": "read authenticated smoke key", "command": ["/usr/local/bin/etcdctl", "--endpoints=http://127.0.0.1:2379", "--user", "root:${DB_PASSWORD}", "get", "dbx:smoke"], "expect": "DBX smoke" }] }
}
plugins/connection-types/etcd.yaml
+6
View File
@@ -6,6 +6,12 @@ label: etcd
runtimeMode: agent
mcpMode: bridge
agentKey: etcd
driverProfiles:
- profile: etcd-v2
agentKey: etcd2
label: etcd 2.x (v2 API)
storeVisible: true
storeOrder: 49
driverStoreVisible: true
driverStoreOrder: 39
singleConnectionPool: false
plugins/connection-types/profiles/catalog.yaml
+6
View File
@@ -621,6 +621,12 @@ profiles:
port: 2379
user: ""
category: registry_config
- id: etcd-v2
dbType: etcd
label: etcd 2.x (v2 API)
icon: etcd
port: 2379
user: ""
- id: zookeeper
dbType: zookeeper
label: Apache ZooKeeper