feat(iotdb): replace JDBC agent with native Go driver
This commit is contained in:
40 files changed
+4750
-111
No files matched your search
@@ -0,0 +1,7 @@
|
||||
module github.com/t8y2/dbx/agents/drivers/iotdb/bench
|
||||
|
||||
go 1.23
|
||||
|
||||
require github.com/apache/iotdb-client-go/v2 v2.0.8
|
||||
|
||||
require github.com/apache/thrift v0.15.0 // indirect
|
||||
@@ -0,0 +1,29 @@
|
||||
github.com/apache/iotdb-client-go/v2 v2.0.8 h1:4Z2bmnoeH9gPNz0cDSfqbvwfyY0Ocbo18vtPfjKo34k=
|
||||
github.com/apache/iotdb-client-go/v2 v2.0.8/go.mod h1:keIG+1Xfw8n2NukUbrEcYnnyN9VxP8OJLYvnHsg1k2s=
|
||||
github.com/apache/thrift v0.15.0 h1:aGvdaR0v1t9XLgjtBYwxcBvBOTMqClzwE26CHOgjW1Y=
|
||||
github.com/apache/thrift v0.15.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -0,0 +1,284 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/apache/iotdb-client-go/v2/client"
|
||||
)
|
||||
|
||||
const (
|
||||
clientVersion = "2.0.8"
|
||||
database = "root.dbx_bench"
|
||||
device = database + ".d1"
|
||||
)
|
||||
|
||||
type config struct {
|
||||
Mode string
|
||||
Host string
|
||||
Port string
|
||||
Username string
|
||||
Password string
|
||||
Rows int
|
||||
FetchSize int32
|
||||
ConnectTimeoutMS int
|
||||
QueryTimeoutMS int64
|
||||
Warmups int
|
||||
MetadataIterations int
|
||||
PointIterations int
|
||||
RangeIterations int
|
||||
ScanIterations int
|
||||
}
|
||||
|
||||
type workload struct {
|
||||
Name string
|
||||
SQL string
|
||||
Iterations int
|
||||
}
|
||||
|
||||
type workloadResult struct {
|
||||
Name string `json:"name"`
|
||||
Iterations int `json:"iterations"`
|
||||
Rows int64 `json:"rows"`
|
||||
DecodedCells int64 `json:"decoded_cells"`
|
||||
MeanMS float64 `json:"mean_ms"`
|
||||
P50MS float64 `json:"p50_ms"`
|
||||
P95MS float64 `json:"p95_ms"`
|
||||
MinMS float64 `json:"min_ms"`
|
||||
MaxMS float64 `json:"max_ms"`
|
||||
}
|
||||
|
||||
type benchmarkResult struct {
|
||||
Driver string `json:"driver"`
|
||||
ClientVersion string `json:"client_version"`
|
||||
ConnectMS float64 `json:"connect_ms"`
|
||||
FetchSize int32 `json:"fetch_size"`
|
||||
Workloads []workloadResult `json:"workloads"`
|
||||
}
|
||||
|
||||
type queryObservation struct {
|
||||
Rows int64
|
||||
DecodedCells int64
|
||||
}
|
||||
|
||||
func main() {
|
||||
conf := loadConfig()
|
||||
session := client.NewSession(&client.Config{
|
||||
Host: conf.Host,
|
||||
Port: conf.Port,
|
||||
UserName: conf.Username,
|
||||
Password: conf.Password,
|
||||
FetchSize: conf.FetchSize,
|
||||
TimeZone: client.DefaultTimeZone,
|
||||
})
|
||||
|
||||
connectStarted := time.Now()
|
||||
if err := session.Open(false, conf.ConnectTimeoutMS); err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
connectMS := elapsedMS(connectStarted)
|
||||
defer session.Close()
|
||||
|
||||
if conf.Mode == "probe" {
|
||||
observation, err := executeQuery(&session, conf, "SHOW DATABASES")
|
||||
if err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
writeJSON(map[string]any{
|
||||
"driver": "go",
|
||||
"client_version": clientVersion,
|
||||
"connect_ms": roundMillis(connectMS),
|
||||
"rows": observation.Rows,
|
||||
})
|
||||
return
|
||||
}
|
||||
if conf.Mode == "hold" {
|
||||
observation, err := executeQuery(&session, conf, "SHOW DATABASES")
|
||||
if err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
writeJSON(map[string]any{
|
||||
"driver": "go",
|
||||
"client_version": clientVersion,
|
||||
"ready": true,
|
||||
"rows": observation.Rows,
|
||||
})
|
||||
time.Sleep(time.Duration(envInt("BENCH_HOLD_MS", 30_000)) * time.Millisecond)
|
||||
return
|
||||
}
|
||||
if conf.Mode != "benchmark" {
|
||||
fatal(fmt.Errorf("unsupported IOTDB_BENCH_MODE: %s", conf.Mode))
|
||||
}
|
||||
|
||||
workloads := []workload{
|
||||
{Name: "show_databases", SQL: "SHOW DATABASES", Iterations: conf.MetadataIterations},
|
||||
{
|
||||
Name: "point_query",
|
||||
SQL: fmt.Sprintf("SELECT s1,s2,s3 FROM %s WHERE time = %d", device, max(1, conf.Rows/2)),
|
||||
Iterations: conf.PointIterations,
|
||||
},
|
||||
{Name: "range_100", SQL: "SELECT s1,s2,s3 FROM " + device + " LIMIT 100", Iterations: conf.RangeIterations},
|
||||
{Name: "scan_all", SQL: "SELECT s1,s2,s3 FROM " + device, Iterations: conf.ScanIterations},
|
||||
}
|
||||
|
||||
results := make([]workloadResult, 0, len(workloads))
|
||||
for _, item := range workloads {
|
||||
warmups := conf.Warmups
|
||||
if item.Name == "scan_all" && warmups > 1 {
|
||||
warmups = 1
|
||||
}
|
||||
for index := 0; index < warmups; index++ {
|
||||
if _, err := executeQuery(&session, conf, item.SQL); err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
}
|
||||
result, err := runWorkload(&session, conf, item)
|
||||
if err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
results = append(results, result)
|
||||
}
|
||||
|
||||
writeJSON(benchmarkResult{
|
||||
Driver: "go",
|
||||
ClientVersion: clientVersion,
|
||||
ConnectMS: roundMillis(connectMS),
|
||||
FetchSize: conf.FetchSize,
|
||||
Workloads: results,
|
||||
})
|
||||
}
|
||||
|
||||
func runWorkload(session *client.Session, conf config, item workload) (workloadResult, error) {
|
||||
samples := make([]float64, 0, item.Iterations)
|
||||
var expected queryObservation
|
||||
for index := 0; index < item.Iterations; index++ {
|
||||
started := time.Now()
|
||||
observation, err := executeQuery(session, conf, item.SQL)
|
||||
if err != nil {
|
||||
return workloadResult{}, err
|
||||
}
|
||||
samples = append(samples, elapsedMS(started))
|
||||
if index == 0 {
|
||||
expected = observation
|
||||
} else if expected != observation {
|
||||
return workloadResult{}, fmt.Errorf("unstable result shape for %s", item.Name)
|
||||
}
|
||||
}
|
||||
|
||||
ordered := append([]float64(nil), samples...)
|
||||
sort.Float64s(ordered)
|
||||
var total float64
|
||||
for _, sample := range samples {
|
||||
total += sample
|
||||
}
|
||||
return workloadResult{
|
||||
Name: item.Name,
|
||||
Iterations: item.Iterations,
|
||||
Rows: expected.Rows,
|
||||
DecodedCells: expected.DecodedCells,
|
||||
MeanMS: roundMillis(total / float64(len(samples))),
|
||||
P50MS: roundMillis(percentile(ordered, 0.50)),
|
||||
P95MS: roundMillis(percentile(ordered, 0.95)),
|
||||
MinMS: roundMillis(ordered[0]),
|
||||
MaxMS: roundMillis(ordered[len(ordered)-1]),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func executeQuery(session *client.Session, conf config, sql string) (queryObservation, error) {
|
||||
timeout := conf.QueryTimeoutMS
|
||||
dataset, err := session.ExecuteQueryStatement(sql, &timeout)
|
||||
if err != nil {
|
||||
return queryObservation{}, err
|
||||
}
|
||||
defer dataset.Close()
|
||||
|
||||
columns := len(dataset.GetColumnNames())
|
||||
var observation queryObservation
|
||||
for {
|
||||
hasNext, nextErr := dataset.Next()
|
||||
if nextErr != nil {
|
||||
return queryObservation{}, nextErr
|
||||
}
|
||||
if !hasNext {
|
||||
break
|
||||
}
|
||||
observation.Rows++
|
||||
for column := 1; column <= columns; column++ {
|
||||
if _, valueErr := dataset.GetObjectByIndex(int32(column)); valueErr != nil {
|
||||
return queryObservation{}, valueErr
|
||||
}
|
||||
observation.DecodedCells++
|
||||
}
|
||||
}
|
||||
return observation, nil
|
||||
}
|
||||
|
||||
func loadConfig() config {
|
||||
return config{
|
||||
Mode: env("IOTDB_BENCH_MODE", "benchmark"),
|
||||
Host: env("IOTDB_HOST", "127.0.0.1"),
|
||||
Port: env("IOTDB_PORT", "6667"),
|
||||
Username: env("IOTDB_USERNAME", "root"),
|
||||
Password: env("IOTDB_PASSWORD", "root"),
|
||||
Rows: envInt("BENCH_ROWS", 10_000),
|
||||
FetchSize: int32(envInt("BENCH_FETCH_SIZE", 1_024)),
|
||||
ConnectTimeoutMS: envInt("BENCH_CONNECT_TIMEOUT_MS", 5_000),
|
||||
QueryTimeoutMS: int64(envInt("BENCH_QUERY_TIMEOUT_MS", 30_000)),
|
||||
Warmups: envInt("BENCH_WARMUPS", 3),
|
||||
MetadataIterations: envInt("BENCH_METADATA_ITERATIONS", 20),
|
||||
PointIterations: envInt("BENCH_POINT_ITERATIONS", 100),
|
||||
RangeIterations: envInt("BENCH_RANGE_ITERATIONS", 30),
|
||||
ScanIterations: envInt("BENCH_SCAN_ITERATIONS", 5),
|
||||
}
|
||||
}
|
||||
|
||||
func percentile(values []float64, fraction float64) float64 {
|
||||
index := int(float64(len(values))*fraction+0.999999) - 1
|
||||
if index < 0 {
|
||||
index = 0
|
||||
}
|
||||
if index >= len(values) {
|
||||
index = len(values) - 1
|
||||
}
|
||||
return values[index]
|
||||
}
|
||||
|
||||
func elapsedMS(started time.Time) float64 {
|
||||
return float64(time.Since(started).Microseconds()) / 1_000
|
||||
}
|
||||
|
||||
func roundMillis(value float64) float64 {
|
||||
return float64(int64(value*1_000+0.5)) / 1_000
|
||||
}
|
||||
|
||||
func env(name, fallback string) string {
|
||||
if value := os.Getenv(name); value != "" {
|
||||
return value
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func envInt(name string, fallback int) int {
|
||||
value, err := strconv.Atoi(env(name, strconv.Itoa(fallback)))
|
||||
if err != nil || value <= 0 {
|
||||
fatal(fmt.Errorf("%s must be a positive integer", name))
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func writeJSON(value any) {
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
encoder.SetEscapeHTML(false)
|
||||
if err := encoder.Encode(value); err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func fatal(err error) {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
Reference in new issue
Block a user