diff --git a/.github/scripts/ci-workflow.test.mjs b/.github/scripts/ci-workflow.test.mjs index 986dce858e..a2b53f030e 100644 --- a/.github/scripts/ci-workflow.test.mjs +++ b/.github/scripts/ci-workflow.test.mjs @@ -145,6 +145,22 @@ test("Windows compatibility jobs cache Rust compilation without wrapping C or C+ assert.ok(win7.includes("--timings")); assert.ok(win7.includes("name: DBX-win7-cargo-timings")); assert.ok(win7.includes("path: target/cargo-timings/")); + assert.ok(win7.includes("name: Measure dbx library command registry expansion")); + assert.ok(win7.includes("if: vars.WIN7_REGISTRY_AB == 'true'")); + assert.ok(win7.includes("--package dbx")); + assert.ok(win7.includes("--lib")); + assert.ok(win7.includes('$env:RUSTC_WRAPPER = ""')); + assert.ok(win7.includes('Measure-Registry "full" $fullCommandCount')); + assert.ok(win7.includes('Measure-Registry "half" $keptCommandCount')); + assert.ok(win7.includes('"-Zdump-mono-stats=$outputDir"')); + assert.ok(win7.includes('"-Zdump-mono-stats-format=json"')); + assert.ok(win7.includes("git restore --source=HEAD -- $sourcePath")); + assert.ok(win7.includes("name: DBX-win7-dbx-lib-command-registry-ab")); + assert.ok(win7.includes("if: always() && vars.WIN7_REGISTRY_AB == 'true'")); + assert.ok(win7.includes("path: ${{ runner.temp }}/dbx-lib-command-registry-ab/")); + assert.ok(win7.includes("retention-days: 3")); + assert.ok(win7.indexOf("name: Build DBX for Windows 7") < win7.indexOf("name: Measure dbx library command registry expansion")); + assert.ok(win7.indexOf("name: Measure dbx library command registry expansion") < win7.indexOf("name: Upload dbx library command registry A/B")); assert.doesNotMatch(win7, /^\s+(?:CC|CXX):/m); }); diff --git a/.github/scripts/label-pull-request.mjs b/.github/scripts/label-pull-request.mjs index e5bf852279..a81e06b25d 100644 --- a/.github/scripts/label-pull-request.mjs +++ b/.github/scripts/label-pull-request.mjs @@ -151,6 +151,7 @@ export function inferAreaLabels(changedFiles) { if (has((file) => file.startsWith("crates/dbx-cli/") || file.startsWith("packages/cli-") || file.startsWith("packages/cli/") + || file.startsWith("skills/dbx/") || file.startsWith("examples/cli/"))) { labels.add("area/cli"); } diff --git a/.github/scripts/label-pull-request.test.mjs b/.github/scripts/label-pull-request.test.mjs index aee05725e4..1779fadea5 100644 --- a/.github/scripts/label-pull-request.test.mjs +++ b/.github/scripts/label-pull-request.test.mjs @@ -53,6 +53,16 @@ test("labels documentation-only changes without a conventional title", () => { assert.deepEqual(result.labels, ["area/docs", "documentation"]); }); +test("labels the bundled DBX skill as CLI work", () => { + const result = evaluatePullRequestLabels({ + title: "feat(cli): improve the bundled agent skill", + changedFiles: ["skills/dbx/SKILL.md", "skills/dbx/references/commands.md"], + knownDatabaseTypes, + }); + + assert.deepEqual(result.labels, ["area/cli", "enhancement"]); +}); + test("maps agent and dialect paths to existing database types", () => { assert.deepEqual( inferDatabaseTypes([ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67cad05e6b..7c313c8da5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -577,6 +577,122 @@ jobs: if-no-files-found: error retention-days: 7 + - name: Measure dbx library command registry expansion + if: vars.WIN7_REGISTRY_AB == 'true' + shell: pwsh + run: | + $resultRoot = Join-Path $env:RUNNER_TEMP "dbx-lib-command-registry-ab" + $sourcePath = "src-tauri/src/lib.rs" + New-Item -ItemType Directory -Force -Path $resultRoot | Out-Null + $env:TAURI_CONFIG = Get-Content src-tauri/tauri.webview2-win7-fixed.conf.json -Raw + + $results = @{} + function Measure-Registry([string] $name, [int] $commandCount) { + $outputDir = Join-Path $resultRoot $name + New-Item -ItemType Directory -Force -Path $outputDir | Out-Null + $timer = [System.Diagnostics.Stopwatch]::StartNew() + cargo rustc ` + --locked ` + --package dbx ` + --release ` + --lib ` + --features custom-protocol ` + --target x86_64-win7-windows-msvc ` + -Z build-std=std,panic_abort ` + -- ` + "-Zdump-mono-stats=$outputDir" ` + "-Zdump-mono-stats-format=json" + $exitCode = $LASTEXITCODE + $timer.Stop() + if ($exitCode -ne 0) { + throw "$name command registry compile failed with exit code $exitCode." + } + + $monoFile = Get-ChildItem $outputDir -File -Filter "*.mono_items.json" | + Select-Object -First 1 + if (!$monoFile) { + throw "$name command registry compile did not create mono statistics." + } + $monoItems = Get-Content $monoFile.FullName -Raw | ConvertFrom-Json + $results[$name] = @{ + Commands = $commandCount + Definitions = $monoItems.Count + Seconds = [math]::Round($timer.Elapsed.TotalSeconds, 2) + TotalEstimate = ($monoItems | Measure-Object -Property total_estimate -Sum).Sum + } + } + + $savedWrapper = $env:RUSTC_WRAPPER + try { + $env:RUSTC_WRAPPER = "" + $sourceLines = Get-Content $sourcePath + $insideHandler = $false + $fullCommandCount = 0 + foreach ($line in $sourceLines) { + if ($line -match 'tauri::generate_handler!\[') { + $insideHandler = $true + continue + } + if ($insideHandler -and $line -match '^\s*\]\)\)') { + break + } + if ($insideHandler -and $line -match '^\s+commands::') { + $fullCommandCount++ + } + } + Measure-Registry "full" $fullCommandCount + + $insideHandler = $false + $handlerCommandCount = 0 + $keptCommandCount = 0 + $reducedLines = foreach ($line in $sourceLines) { + if ($line -match 'tauri::generate_handler!\[') { + $insideHandler = $true + $line + continue + } + if ($insideHandler -and $line -match '^\s*\]\)\)') { + $insideHandler = $false + $line + continue + } + if ($insideHandler -and $line -match '^\s+commands::') { + $handlerCommandCount++ + if ($handlerCommandCount % 2 -eq 0) { + continue + } + $keptCommandCount++ + } + $line + } + if ($handlerCommandCount -ne $fullCommandCount -or $keptCommandCount -eq $fullCommandCount) { + throw "Could not create the reduced command registry." + } + $reducedLines | Set-Content $sourcePath -Encoding utf8NoBOM + Measure-Registry "half" $keptCommandCount + } finally { + $env:RUSTC_WRAPPER = $savedWrapper + git restore --source=HEAD -- $sourcePath + } + + @( + "### dbx library command registry A/B" + "" + "| Registry | Commands | Definitions | Estimated cost | Seconds |" + "| --- | ---: | ---: | ---: | ---: |" + "| Full | $($results['full'].Commands) | $($results['full'].Definitions) | $($results['full'].TotalEstimate) | $($results['full'].Seconds) |" + "| Half | $($results['half'].Commands) | $($results['half'].Definitions) | $($results['half'].TotalEstimate) | $($results['half'].Seconds) |" + ) | Add-Content $env:GITHUB_STEP_SUMMARY + + - name: Upload dbx library command registry A/B + if: always() && vars.WIN7_REGISTRY_AB == 'true' + uses: actions/upload-artifact@v6 + with: + name: DBX-win7-dbx-lib-command-registry-ab + path: ${{ runner.temp }}/dbx-lib-command-registry-ab/ + if-no-files-found: error + retention-days: 3 + duckdb-windows-driver: needs: changes if: needs.changes.outputs.duckdb_windows == 'true' @@ -973,6 +1089,7 @@ jobs: - 'packages/mcp-server/**' - 'crates/dbx-cli/**' - 'crates/dbx-mcp/**' + - 'skills/dbx/**' - 'scripts/verify-package-install.mjs' - 'package.json' - 'pnpm-lock.yaml' diff --git a/Cargo.lock b/Cargo.lock index 48949e29c8..3aa488362d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2047,7 +2047,7 @@ dependencies = [ [[package]] name = "dbx" -version = "0.6.25" +version = "0.6.26" dependencies = [ "anyhow", "axum", @@ -2057,6 +2057,7 @@ dependencies = [ "csv", "dbx-core", "dbx-mcp", + "dbx-tauri-consul", "deadpool-postgres", "font-kit", "futures", @@ -2128,14 +2129,16 @@ dependencies = [ [[package]] name = "dbx-cli" -version = "0.4.99" +version = "0.4.100" dependencies = [ "async-trait", "dbx-core", "dbx-mcp", + "dirs", "rustls 0.23.43", "serde", "serde_json", + "sha2 0.10.9", "tempfile", "tokio", ] @@ -2460,7 +2463,7 @@ dependencies = [ [[package]] name = "dbx-mcp" -version = "0.4.99" +version = "0.4.100" dependencies = [ "async-trait", "axum", @@ -2607,6 +2610,16 @@ dependencies = [ "sqlparser", ] +[[package]] +name = "dbx-tauri-consul" +version = "0.1.0" +dependencies = [ + "dbx-core", + "serde", + "serde_json", + "tauri", +] + [[package]] name = "dbx-types" version = "0.1.0" @@ -2621,7 +2634,7 @@ dependencies = [ [[package]] name = "dbx-web" -version = "0.6.25" +version = "0.6.26" dependencies = [ "aes-gcm 0.10.3", "argon2", diff --git a/Cargo.toml b/Cargo.toml index fa0a54ef80..4b25f4fd72 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "2" -members = ["src-tauri", "crates/dbx-core", "crates/dbx-driver-agent", "crates/dbx-driver-support", "crates/dbx-driver-elasticsearch", "crates/dbx-driver-mongodb", "crates/dbx-driver-mysql", "crates/dbx-driver-postgres", "crates/dbx-driver-redis", "crates/dbx-driver-sqlserver", "crates/dbx-drivers", "crates/dbx-sql", "crates/dbx-sql-core", "crates/dbx-sql-data", "crates/dbx-sql-dialect", "crates/dbx-sql-schema", "crates/dbx-plugin-runtime", "crates/dbx-formats", "crates/dbx-types", "crates/dbx-platform", "crates/dbx-ai-provider", "crates/dbx-web", "crates/dbx-mcp", "crates/dbx-cli", "crates/dbx-sqlite-worker"] +members = ["src-tauri", "crates/dbx-core", "crates/dbx-driver-agent", "crates/dbx-driver-support", "crates/dbx-driver-elasticsearch", "crates/dbx-driver-mongodb", "crates/dbx-driver-mysql", "crates/dbx-driver-postgres", "crates/dbx-driver-redis", "crates/dbx-driver-sqlserver", "crates/dbx-drivers", "crates/dbx-sql", "crates/dbx-sql-core", "crates/dbx-sql-data", "crates/dbx-sql-dialect", "crates/dbx-sql-schema", "crates/dbx-plugin-runtime", "crates/dbx-formats", "crates/dbx-types", "crates/dbx-platform", "crates/dbx-ai-provider", "crates/dbx-web", "crates/dbx-mcp", "crates/dbx-cli", "crates/dbx-sqlite-worker", "crates/dbx-tauri-consul"] exclude = ["vendor/tauri-plugin-updater", "vendor/ctor", "vendor/dirs-sys", "vendor/pageant", "vendor/rumqttc", "vendor/tiberius", "vendor/webview2-com-sys", "vendor/wry"] [patch.crates-io] diff --git a/README.md b/README.md index eff8a98ab6..a31b05f76b 100644 --- a/README.md +++ b/README.md @@ -335,10 +335,13 @@ DBX also provides a dedicated CLI package for terminal, script, and Codex workfl npm install -g @dbx-app/cli # or via Homebrew brew tap t8y2/tap && brew install dbx-cli +dbx agent setup dbx connections list --json dbx query local "select 1" --json ``` +The CLI includes the official DBX Agent Skill. `dbx agent setup` installs or updates it offline under `~/.agents/skills/dbx`, so shell-capable AI agents know how to use DBX CLI safely. + See the [MCP server README](packages/mcp-server/README.md) and [CLI README](packages/cli/README.md) for details. ## Install diff --git a/README.zh-CN.md b/README.zh-CN.md index 2fd52577d3..05dfc88a70 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -333,10 +333,13 @@ DBX 也提供独立 CLI 包，适合终端、脚本和 Codex 工作流： npm install -g @dbx-app/cli # 或通过 Homebrew brew tap t8y2/tap && brew install dbx-cli +dbx agent setup dbx connections list --json dbx query local "select 1" --json ``` +CLI 已内置官方 DBX Agent Skill；`dbx agent setup` 会离线安装或更新到 `~/.agents/skills/dbx`，让支持 Shell 的 AI Agent 知道如何安全调用 DBX CLI。 + 详见 [MCP Server 说明](packages/mcp-server/README.md) 和 [CLI 说明](packages/cli/README.md)。 ## 安装 diff --git a/agents/drivers/cassandra-go/integration_test.go b/agents/drivers/cassandra-go/integration_test.go index 017c13097e..0172808822 100644 --- a/agents/drivers/cassandra-go/integration_test.go +++ b/agents/drivers/cassandra-go/integration_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "os" + "sort" "strconv" "strings" "testing" @@ -42,7 +43,7 @@ func TestCassandraIntegration(t *testing.T) { if err != nil { t.Fatal(err) } - defer runtime.close() + t.Cleanup(runtime.close) server := newServer(runtime, connection) if err := server.validateConnection(); err != nil { t.Fatal(err) @@ -54,7 +55,9 @@ func TestCassandraIntegration(t *testing.T) { pagedTable := "paged_rows" mustCQL(t, server, "CREATE KEYSPACE "+quoteCQLIdentifier(keyspace)+" WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}", "") t.Cleanup(func() { - _, _ = server.executeQuery(queryOptions{SQL: "DROP KEYSPACE IF EXISTS " + quoteCQLIdentifier(keyspace)}) + if _, err := server.executeQuery(queryOptions{SQL: "DROP KEYSPACE IF EXISTS " + quoteCQLIdentifier(keyspace)}); err != nil { + t.Errorf("drop integration keyspace %s: %v", keyspace, err) + } }) mustCQL(t, server, "CREATE TABLE "+qualifiedCQLName(keyspace, table)+" ("+ "id int PRIMARY KEY, txt text, flag boolean, amount decimal, payload blob, created timestamp, address inet, "+ @@ -139,6 +142,187 @@ func TestCassandraIntegration(t *testing.T) { } } +func TestCassandraMaterializedViewDDLIntegration(t *testing.T) { + host := strings.TrimSpace(os.Getenv("CASSANDRA_TEST_HOST")) + if host == "" { + t.Skip("Cassandra integration environment is not configured") + } + port := 9042 + if rawPort := strings.TrimSpace(os.Getenv("CASSANDRA_TEST_PORT")); rawPort != "" { + parsedPort, err := strconv.Atoi(rawPort) + if err != nil { + t.Fatal(err) + } + port = parsedPort + } + ssl, err := strconv.ParseBool(envDefault("CASSANDRA_TEST_SSL", "false")) + if err != nil { + t.Fatal(err) + } + connection := connectParams{ + Host: host, + Port: port, + Username: os.Getenv("CASSANDRA_TEST_USERNAME"), + Password: os.Getenv("CASSANDRA_TEST_PASSWORD"), + URLParams: os.Getenv("CASSANDRA_TEST_URL_PARAMS"), + SSL: ssl, + CACertPath: os.Getenv("CASSANDRA_TEST_CA_CERT_PATH"), + ClientCertPath: os.Getenv("CASSANDRA_TEST_CLIENT_CERT_PATH"), + ClientKeyPath: os.Getenv("CASSANDRA_TEST_CLIENT_KEY_PATH"), + } + runtime, err := newConnectionRuntime(connection) + if err != nil { + t.Fatal(err) + } + t.Cleanup(runtime.close) + server := newServer(runtime, connection) + if err := server.validateConnection(); err != nil { + t.Fatal(err) + } + connectionInfo, err := server.connectionInfo() + if err != nil { + t.Fatal(err) + } + t.Logf("Cassandra runtime version: %v", connectionInfo["version"]) + + suffix := strconv.FormatInt(time.Now().UnixNano(), 36) + keyspace := "dbx_issue_7475_regression_" + suffix + baseTable := "BaseEvents" + view := "EventsByCategory" + allColumnsView := "AllEventsByCategory" + mustCQL(t, server, "CREATE KEYSPACE "+quoteCQLIdentifier(keyspace)+" WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}", "") + t.Cleanup(func() { + if _, err := server.executeQuery(queryOptions{SQL: "DROP KEYSPACE IF EXISTS " + quoteCQLIdentifier(keyspace)}); err != nil { + t.Errorf("drop integration keyspace %s: %v", keyspace, err) + } + }) + mustCQL(t, server, "CREATE TABLE "+qualifiedCQLName(keyspace, baseTable)+" ("+ + `"Tenant Id" text, "Bucket" int, "Event Time" timestamp, "Category" text, "Payload" text, `+ + `PRIMARY KEY ("Tenant Id", "Event Time", "Bucket")) `+ + `WITH CLUSTERING ORDER BY ("Event Time" DESC, "Bucket" ASC)`, keyspace) + mustCQL(t, server, "CREATE MATERIALIZED VIEW "+qualifiedCQLName(keyspace, view)+" AS "+ + `SELECT "Tenant Id", "Bucket", "Event Time", "Category" FROM `+qualifiedCQLName(keyspace, baseTable)+" "+ + `WHERE "Tenant Id" IS NOT NULL AND "Event Time" IS NOT NULL AND "Bucket" IS NOT NULL AND "Category" IS NOT NULL `+ + `PRIMARY KEY (("Tenant Id", "Category"), "Event Time", "Bucket") `+ + `WITH CLUSTERING ORDER BY ("Event Time" DESC, "Bucket" ASC) AND comment = 'issue 7475 round trip' AND gc_grace_seconds = 86401`, keyspace) + mustCQL(t, server, "CREATE MATERIALIZED VIEW "+qualifiedCQLName(keyspace, allColumnsView)+" AS "+ + `SELECT * FROM `+qualifiedCQLName(keyspace, baseTable)+" "+ + `WHERE "Tenant Id" IS NOT NULL AND "Event Time" IS NOT NULL AND "Bucket" IS NOT NULL AND "Category" IS NOT NULL `+ + `PRIMARY KEY (("Tenant Id", "Category"), "Event Time", "Bucket") `+ + `WITH CLUSTERING ORDER BY ("Event Time" DESC, "Bucket" ASC)`, keyspace) + + ddl, err := server.getTableDDL(keyspace, view) + if err != nil { + t.Fatalf("materialized view DDL failed: %v", err) + } + for _, fragment := range []string{ + "CREATE MATERIALIZED VIEW", + "FROM " + keyspace + "." + quoteCQLIdentifier(baseTable), + `PRIMARY KEY (("Tenant Id", "Category"), "Event Time", "Bucket")`, + `CLUSTERING ORDER BY ("Event Time" DESC, "Bucket" ASC)`, + "comment = 'issue 7475 round trip'", + "gc_grace_seconds = 86401", + } { + if !strings.Contains(ddl, fragment) { + t.Fatalf("materialized view DDL omitted %q:\n%s", fragment, ddl) + } + } + projection := materializedViewProjection(t, ddl) + wantProjection := []string{`"Bucket"`, `"Category"`, `"Event Time"`, `"Tenant Id"`} + sort.Strings(projection) + sort.Strings(wantProjection) + if strings.Join(projection, ",") != strings.Join(wantProjection, ",") || strings.Contains(ddl, `"Payload"`) || strings.Contains(ddl, "SELECT *") { + t.Fatalf("materialized view DDL lost its selected-column subset:\n%s", ddl) + } + metadataSession, err := runtime.sessionFor("") + if err != nil { + t.Fatal(err) + } + legacyCatalog, err := legacyMaterializedViewCatalog(metadataSession, keyspace, view) + if err != nil { + t.Fatalf("legacy materialized view catalog failed: %v", err) + } + legacyDDL, err := materializedViewDDLFromCatalog(keyspace, view, legacyCatalog) + if err != nil { + t.Fatalf("legacy materialized view DDL failed: %v", err) + } + legacyProjection := materializedViewProjection(t, legacyDDL) + sort.Strings(legacyProjection) + if strings.Join(legacyProjection, ",") != strings.Join(wantProjection, ",") || strings.Contains(legacyDDL, `"Payload"`) || strings.Contains(legacyDDL, "SELECT *") { + t.Fatalf("legacy materialized view DDL lost its selected-column subset:\n%s", legacyDDL) + } + allColumnsDDL, err := server.getTableDDL(keyspace, allColumnsView) + if err != nil || !strings.Contains(allColumnsDDL, "SELECT *") { + t.Fatalf("materialized view SELECT * was not preserved: ddl=%q err=%v", allColumnsDDL, err) + } + + result, _, err := server.dispatch("get_object_source", map[string]json.RawMessage{ + "schema": json.RawMessage(strconv.Quote(keyspace)), + "name": json.RawMessage(strconv.Quote(view)), + "object_type": json.RawMessage(`"MATERIALIZED_VIEW"`), + }) + if err != nil { + t.Fatalf("materialized view object-source RPC failed: %v", err) + } + source, ok := result.(objectSource) + if !ok || source.Source != ddl || source.Name != view || source.ObjectType != "MATERIALIZED_VIEW" { + t.Fatalf("unexpected materialized view object-source response: %#v", result) + } + + tableDDL, err := server.getTableDDL(keyspace, baseTable) + if err != nil || !strings.HasPrefix(tableDDL, "CREATE TABLE ") || strings.Contains(tableDDL, "CREATE MATERIALIZED VIEW") { + t.Fatalf("ordinary table DDL changed: ddl=%q err=%v", tableDDL, err) + } + if _, err := server.getTableDDL(keyspace, "MissingObject"); err == nil { + t.Fatal("missing schema object unexpectedly returned DDL") + } + if _, err := server.getMaterializedViewDDL(keyspace, "MissingObject"); err == nil { + t.Fatal("missing materialized view unexpectedly returned DDL") + } + + mustCQL(t, server, "DROP MATERIALIZED VIEW "+qualifiedCQLName(keyspace, view), keyspace) + mustCQL(t, server, ddl, keyspace) + recreatedDDL, err := server.getTableDDL(keyspace, view) + if err != nil { + t.Fatalf("materialized view DDL after recreation failed: %v", err) + } + if recreatedDDL != ddl { + t.Fatalf("materialized view changed after DDL round trip:\nbefore:\n%s\nafter:\n%s", ddl, recreatedDDL) + } + mustCQL(t, server, "DROP MATERIALIZED VIEW "+qualifiedCQLName(keyspace, view), keyspace) + mustCQL(t, server, legacyDDL, keyspace) + legacyRecreatedDDL, err := server.getTableDDL(keyspace, view) + if err != nil { + t.Fatalf("materialized view DDL after legacy recreation failed: %v", err) + } + if legacyRecreatedDDL != ddl { + t.Fatalf("legacy materialized view DDL changed the definition:\nbefore:\n%s\nafter:\n%s", ddl, legacyRecreatedDDL) + } +} + +func materializedViewProjection(t *testing.T, ddl string) []string { + t.Helper() + upper := strings.ToUpper(ddl) + selectIndex := strings.Index(upper, "SELECT ") + if selectIndex < 0 { + t.Fatalf("DDL has no SELECT clause:\n%s", ddl) + } + projectionStart := selectIndex + len("SELECT ") + fromOffset := strings.Index(upper[projectionStart:], "FROM ") + if fromOffset < 0 { + t.Fatalf("DDL has no FROM clause:\n%s", ddl) + } + projection := strings.TrimSpace(ddl[projectionStart : projectionStart+fromOffset]) + if projection == "" { + t.Fatalf("DDL has an empty projection:\n%s", ddl) + } + columns := strings.Split(projection, ",") + for index := range columns { + columns[index] = strings.TrimSpace(columns[index]) + } + return columns +} + func envDefault(name, fallback string) string { if value := strings.TrimSpace(os.Getenv(name)); value != "" { return value diff --git a/agents/drivers/cassandra-go/main.go b/agents/drivers/cassandra-go/main.go index fc264cb690..4d596f3360 100644 --- a/agents/drivers/cassandra-go/main.go +++ b/agents/drivers/cassandra-go/main.go @@ -365,7 +365,12 @@ func (s *server) dispatch(method string, params map[string]json.RawMessage) (any result, err := s.listTriggers(stringParam(params, "schema"), stringParam(params, "table")) return result, false, err case "get_object_source": - return nil, false, errors.New("object source is not supported by Cassandra") + result, err := s.getObjectSource( + firstNonEmpty(stringParam(params, "schema"), stringParam(params, "database"), s.defaultKeyspace()), + firstNonEmpty(stringParam(params, "name"), stringParam(params, "table")), + stringParam(params, "object_type"), + ) + return result, false, err case "get_table_ddl": result, err := s.getTableDDL(stringParam(params, "schema"), stringParam(params, "table")) return result, false, err diff --git a/agents/drivers/cassandra-go/materialized_view.go b/agents/drivers/cassandra-go/materialized_view.go new file mode 100644 index 0000000000..9f7e77e8c2 --- /dev/null +++ b/agents/drivers/cassandra-go/materialized_view.go @@ -0,0 +1,394 @@ +package main + +import ( + "errors" + "fmt" + "sort" + "strings" + + gocql "github.com/apache/cassandra-gocql-driver/v2" +) + +const maxLegacyMaterializedViewColumns = 4096 + +const cassandraMaterializedViewObjectType = "MATERIALIZED_VIEW" + +var legacyMaterializedViewOptionNames = []string{ + "additional_write_policy", + "bloom_filter_fp_chance", + "caching", + "cdc", + "comment", + "compaction", + "compression", + "crc_check_chance", + "dclocal_read_repair_chance", + "gc_grace_seconds", + "max_index_interval", + "memtable", + "memtable_flush_period_in_ms", + "min_index_interval", + "read_repair", + "read_repair_chance", + "speculative_retry", +} + +type objectSource struct { + Name string `json:"name"` + ObjectType string `json:"object_type"` + Schema *string `json:"schema"` + Source string `json:"source"` +} + +type materializedViewColumn struct { + Name string + Kind string + Position int + ClusteringOrder string +} + +type materializedViewCatalog struct { + BaseTable string + IncludeAllColumns bool + WhereClause string + Columns []materializedViewColumn + Options map[string]any +} + +func (s *server) getObjectSource(schema, name, objectType string) (objectSource, error) { + return materializedViewObjectSource(schema, name, objectType, func() (string, error) { + return s.getMaterializedViewDDL(schema, name) + }) +} + +func materializedViewObjectSource(schema, name, objectType string, loadDDL func() (string, error)) (objectSource, error) { + if normalizeCassandraObjectType(objectType) != cassandraMaterializedViewObjectType { + return objectSource{}, errors.New("object source is not supported by Cassandra") + } + source, err := loadDDL() + if err != nil { + return objectSource{}, err + } + return objectSource{ + Name: name, + ObjectType: cassandraMaterializedViewObjectType, + Schema: optionalString(schema), + Source: source, + }, nil +} + +func normalizeCassandraObjectType(value string) string { + value = strings.ToUpper(strings.TrimSpace(value)) + value = strings.ReplaceAll(value, "-", "_") + return strings.ReplaceAll(value, " ", "_") +} + +func (s *server) getMaterializedViewDDL(schema, view string) (string, error) { + session, err := s.runtime.sessionFor("") + if err != nil { + return "", err + } + return materializedViewDDLWithFallback( + func() (string, error) { + return describeMaterializedViewDDL(session, schema, view) + }, + func() (string, error) { + catalog, err := legacyMaterializedViewCatalog(session, schema, view) + if err != nil { + return "", err + } + return materializedViewDDLFromCatalog(schema, view, catalog) + }, + ) +} + +func materializedViewDDLWithFallback(describe, legacy func() (string, error)) (string, error) { + ddl, err := describe() + if err == nil { + if strings.TrimSpace(ddl) == "" { + return "", errors.New("Cassandra returned an empty materialized view definition") + } + return ddl, nil + } + if !isUnsupportedDescribeSyntax(err) { + return "", err + } + return legacy() +} + +func isUnsupportedDescribeSyntax(err error) bool { + var requestError gocql.RequestError + return errors.As(err, &requestError) && requestError.Code() == gocql.ErrCodeSyntax +} + +func describeMaterializedViewDDL(session *gocql.Session, schema, view string) (string, error) { + statement := "DESCRIBE MATERIALIZED VIEW " + qualifiedCQLIdentifier(schema, view) + var describedSchema, objectType, describedName, ddl string + if err := session.Query(statement).Scan(&describedSchema, &objectType, &describedName, &ddl); err != nil { + return "", fmt.Errorf("describe Cassandra materialized view %s: %w", qualifiedCQLIdentifier(schema, view), err) + } + if normalizeCassandraObjectType(objectType) != cassandraMaterializedViewObjectType { + return "", fmt.Errorf("Cassandra DESCRIBE returned object type %q for materialized view %s", objectType, qualifiedCQLIdentifier(schema, view)) + } + if strings.TrimSpace(ddl) == "" { + return "", fmt.Errorf("Cassandra returned an empty materialized view definition for %s", qualifiedCQLIdentifier(schema, view)) + } + return ddl, nil +} + +func legacyMaterializedViewCatalog(session *gocql.Session, schema, view string) (materializedViewCatalog, error) { + row := map[string]any{} + err := session.Query( + "SELECT * FROM system_schema.views WHERE keyspace_name = ? AND view_name = ?", + schema, + view, + ).MapScan(row) + if errors.Is(err, gocql.ErrNotFound) { + return materializedViewCatalog{}, fmt.Errorf("Cassandra materialized view not found: %s.%s", schema, view) + } + if err != nil { + return materializedViewCatalog{}, fmt.Errorf("read Cassandra materialized view catalog for %s: %w", qualifiedCQLIdentifier(schema, view), err) + } + + baseTable, ok := row["base_table_name"].(string) + if !ok || strings.TrimSpace(baseTable) == "" { + return materializedViewCatalog{}, fmt.Errorf("Cassandra materialized view catalog has no base table for %s", qualifiedCQLIdentifier(schema, view)) + } + includeAllColumns, ok := row["include_all_columns"].(bool) + if !ok { + return materializedViewCatalog{}, fmt.Errorf("Cassandra materialized view catalog has no projection mode for %s", qualifiedCQLIdentifier(schema, view)) + } + whereClause, ok := row["where_clause"].(string) + if !ok || strings.TrimSpace(whereClause) == "" { + return materializedViewCatalog{}, fmt.Errorf("Cassandra materialized view catalog has no WHERE clause for %s", qualifiedCQLIdentifier(schema, view)) + } + + options, err := legacyMaterializedViewOptions(row) + if err != nil { + return materializedViewCatalog{}, fmt.Errorf("read Cassandra materialized view options for %s: %w", qualifiedCQLIdentifier(schema, view), err) + } + columns, err := legacyMaterializedViewColumns(session, schema, view) + if err != nil { + return materializedViewCatalog{}, err + } + return materializedViewCatalog{ + BaseTable: baseTable, + IncludeAllColumns: includeAllColumns, + WhereClause: whereClause, + Columns: columns, + Options: options, + }, nil +} + +func legacyMaterializedViewOptions(row map[string]any) (map[string]any, error) { + metadataColumns := map[string]bool{ + "base_table_id": true, + "base_table_name": true, + "id": true, + "include_all_columns": true, + "keyspace_name": true, + "view_name": true, + "where_clause": true, + } + knownOptions := make(map[string]bool, len(legacyMaterializedViewOptionNames)+1) + for _, name := range legacyMaterializedViewOptionNames { + knownOptions[name] = true + } + knownOptions["default_time_to_live"] = true + knownOptions["extensions"] = true + + for name, value := range row { + if value != nil && !metadataColumns[name] && !knownOptions[name] { + return nil, fmt.Errorf("unsupported system_schema.views column %q", name) + } + } + if value, exists := row["default_time_to_live"]; exists && value != nil && fmt.Sprint(value) != "0" { + return nil, fmt.Errorf("unsupported non-zero default_time_to_live %v", value) + } + if value, exists := row["extensions"]; exists && value != nil && !isEmptyCQLCollection(value) { + return nil, errors.New("non-empty materialized view extensions cannot be reproduced safely") + } + + options := make(map[string]any, len(legacyMaterializedViewOptionNames)) + _, hasModernReadRepair := row["read_repair"] + for _, name := range legacyMaterializedViewOptionNames { + if value, exists := row[name]; exists && value != nil { + if hasModernReadRepair && (name == "dclocal_read_repair_chance" || name == "read_repair_chance") { + continue + } + if name == "memtable" && strings.TrimSpace(fmt.Sprint(value)) == "" { + continue + } + options[name] = value + } + } + return options, nil +} + +func legacyMaterializedViewColumns(session *gocql.Session, schema, view string) ([]materializedViewColumn, error) { + iter := session.Query( + "SELECT column_name, kind, position, clustering_order FROM system_schema.columns WHERE keyspace_name = ? AND table_name = ?", + schema, + view, + ).PageSize(512).Iter() + columns := make([]materializedViewColumn, 0) + var column materializedViewColumn + for iter.Scan(&column.Name, &column.Kind, &column.Position, &column.ClusteringOrder) { + if len(columns) >= maxLegacyMaterializedViewColumns { + _ = iter.Close() + return nil, fmt.Errorf("Cassandra materialized view %s has more than %d columns", qualifiedCQLIdentifier(schema, view), maxLegacyMaterializedViewColumns) + } + columns = append(columns, column) + column = materializedViewColumn{} + } + if err := iter.Close(); err != nil { + return nil, fmt.Errorf("read Cassandra materialized view columns for %s: %w", qualifiedCQLIdentifier(schema, view), err) + } + if len(columns) == 0 { + return nil, fmt.Errorf("Cassandra materialized view catalog has no columns for %s", qualifiedCQLIdentifier(schema, view)) + } + return columns, nil +} + +func materializedViewDDLFromCatalog(schema, view string, catalog materializedViewCatalog) (string, error) { + if strings.TrimSpace(catalog.BaseTable) == "" { + return "", fmt.Errorf("Cassandra materialized view catalog has no base table for %s", qualifiedCQLIdentifier(schema, view)) + } + if strings.TrimSpace(catalog.WhereClause) == "" { + return "", fmt.Errorf("Cassandra materialized view catalog has no WHERE clause for %s", qualifiedCQLIdentifier(schema, view)) + } + + partitionKeys, clusteringColumns, selectedColumns, err := classifyMaterializedViewColumns(catalog.Columns) + if err != nil { + return "", fmt.Errorf("invalid Cassandra materialized view columns for %s: %w", qualifiedCQLIdentifier(schema, view), err) + } + if len(partitionKeys) == 0 { + return "", fmt.Errorf("Cassandra materialized view has no partition key: %s", qualifiedCQLIdentifier(schema, view)) + } + + projection := "*" + if !catalog.IncludeAllColumns { + quoted := make([]string, len(selectedColumns)) + for index, column := range selectedColumns { + quoted[index] = quoteCQLIdentifier(column.Name) + } + projection = strings.Join(quoted, ", ") + } + + primaryParts := make([]string, 0, len(clusteringColumns)+1) + if len(partitionKeys) == 1 { + primaryParts = append(primaryParts, quoteCQLIdentifier(partitionKeys[0].Name)) + } else { + quoted := make([]string, len(partitionKeys)) + for index, column := range partitionKeys { + quoted[index] = quoteCQLIdentifier(column.Name) + } + primaryParts = append(primaryParts, "("+strings.Join(quoted, ", ")+")") + } + for _, column := range clusteringColumns { + primaryParts = append(primaryParts, quoteCQLIdentifier(column.Name)) + } + + properties := make([]string, 0, len(catalog.Options)+1) + if len(clusteringColumns) > 0 { + orders := make([]string, len(clusteringColumns)) + for index, column := range clusteringColumns { + order := strings.ToUpper(strings.TrimSpace(column.ClusteringOrder)) + if order != "ASC" && order != "DESC" { + return "", fmt.Errorf("invalid clustering order %q for column %q", column.ClusteringOrder, column.Name) + } + orders[index] = quoteCQLIdentifier(column.Name) + " " + order + } + properties = append(properties, "CLUSTERING ORDER BY ("+strings.Join(orders, ", ")+")") + } + for _, name := range legacyMaterializedViewOptionNames { + value, exists := catalog.Options[name] + if !exists || value == nil { + continue + } + properties = append(properties, name+" = "+cqlLiteral(value)) + } + if len(properties) == 0 { + return "", fmt.Errorf("Cassandra materialized view has no reproducible table options: %s", qualifiedCQLIdentifier(schema, view)) + } + + return "CREATE MATERIALIZED VIEW " + qualifiedCQLIdentifier(schema, view) + " AS\n" + + " SELECT " + projection + "\n" + + " FROM " + qualifiedCQLIdentifier(schema, catalog.BaseTable) + "\n" + + " WHERE " + strings.TrimSpace(catalog.WhereClause) + "\n" + + " PRIMARY KEY (" + strings.Join(primaryParts, ", ") + ")\n" + + " WITH " + strings.Join(properties, "\n AND ") + ";", nil +} + +func qualifiedCQLIdentifier(schema, name string) string { + return quoteCQLIdentifier(schema) + "." + quoteCQLIdentifier(name) +} + +func classifyMaterializedViewColumns(columns []materializedViewColumn) ([]materializedViewColumn, []materializedViewColumn, []materializedViewColumn, error) { + partitionKeys := make([]materializedViewColumn, 0) + clusteringColumns := make([]materializedViewColumn, 0) + regularColumns := make([]materializedViewColumn, 0) + seenNames := make(map[string]bool, len(columns)) + for _, column := range columns { + if strings.TrimSpace(column.Name) == "" { + return nil, nil, nil, errors.New("column name is empty") + } + if seenNames[column.Name] { + return nil, nil, nil, fmt.Errorf("duplicate column %q", column.Name) + } + seenNames[column.Name] = true + switch strings.ToLower(strings.TrimSpace(column.Kind)) { + case "partition_key": + partitionKeys = append(partitionKeys, column) + case "clustering", "clustering_key": + clusteringColumns = append(clusteringColumns, column) + case "regular", "static": + regularColumns = append(regularColumns, column) + default: + return nil, nil, nil, fmt.Errorf("unsupported kind %q for column %q", column.Kind, column.Name) + } + } + if err := sortMaterializedViewKeyColumns(partitionKeys, "partition"); err != nil { + return nil, nil, nil, err + } + if err := sortMaterializedViewKeyColumns(clusteringColumns, "clustering"); err != nil { + return nil, nil, nil, err + } + sort.Slice(regularColumns, func(left, right int) bool { return regularColumns[left].Name < regularColumns[right].Name }) + selectedColumns := make([]materializedViewColumn, 0, len(columns)) + selectedColumns = append(selectedColumns, partitionKeys...) + selectedColumns = append(selectedColumns, clusteringColumns...) + selectedColumns = append(selectedColumns, regularColumns...) + return partitionKeys, clusteringColumns, selectedColumns, nil +} + +func sortMaterializedViewKeyColumns(columns []materializedViewColumn, kind string) error { + sort.Slice(columns, func(left, right int) bool { + if columns[left].Position == columns[right].Position { + return columns[left].Name < columns[right].Name + } + return columns[left].Position < columns[right].Position + }) + for index, column := range columns { + if column.Position < 0 { + return fmt.Errorf("invalid %s-key position %d for column %q", kind, column.Position, column.Name) + } + if index > 0 && columns[index-1].Position == column.Position { + return fmt.Errorf("duplicate %s-key position %d", kind, column.Position) + } + } + return nil +} + +func isEmptyCQLCollection(value any) bool { + switch typed := value.(type) { + case map[string]string: + return len(typed) == 0 + case map[string][]byte: + return len(typed) == 0 + case map[string]any: + return len(typed) == 0 + default: + return false + } +} diff --git a/agents/drivers/cassandra-go/metadata.go b/agents/drivers/cassandra-go/metadata.go index d8ec9eb82b..23deacc922 100644 --- a/agents/drivers/cassandra-go/metadata.go +++ b/agents/drivers/cassandra-go/metadata.go @@ -470,11 +470,23 @@ func indexesFromMetadata(metadata *gocql.TableMetadata) []indexInfo { } func (s *server) getTableDDL(schema, table string) (string, error) { - metadata, err := s.tableMetadata(schema, table) + keyspace, err := s.keyspaceMetadata(schema) if err != nil { return "", err } - return tableDDLFromMetadata(schema, table, metadata) + return schemaObjectDDLFromMetadata(schema, table, keyspace, func() (string, error) { + return s.getMaterializedViewDDL(schema, table) + }) +} + +func schemaObjectDDLFromMetadata(schema, table string, keyspace *gocql.KeyspaceMetadata, materializedViewDDL func() (string, error)) (string, error) { + if keyspace.MaterializedViews[table] != nil { + return materializedViewDDL() + } + if metadata := keyspace.Tables[table]; metadata != nil { + return tableDDLFromMetadata(schema, table, metadata) + } + return "", fmt.Errorf("Cassandra table not found: %s.%s", schema, table) } func tableDDLFromMetadata(schema, table string, metadata *gocql.TableMetadata) (string, error) { diff --git a/agents/drivers/cassandra-go/metadata_test.go b/agents/drivers/cassandra-go/metadata_test.go index a699d3f081..465e090882 100644 --- a/agents/drivers/cassandra-go/metadata_test.go +++ b/agents/drivers/cassandra-go/metadata_test.go @@ -3,11 +3,21 @@ package main import ( "errors" "reflect" + "strings" "testing" gocql "github.com/apache/cassandra-gocql-driver/v2" ) +type testCassandraRequestError struct { + code int + message string +} + +func (e *testCassandraRequestError) Error() string { return e.message } +func (e *testCassandraRequestError) Code() int { return e.code } +func (e *testCassandraRequestError) Message() string { return e.message } + type triggerMetadataIterator struct { rows []struct { name string @@ -80,6 +90,250 @@ func TestColumnsIndexesAndDDLFromMetadata(t *testing.T) { } } +func TestSchemaObjectDDLChoosesMaterializedViewBeforeMirroredTableMetadata(t *testing.T) { + id := &gocql.ColumnMetadata{Name: "id", Kind: gocql.ColumnPartitionKey, Type: gocql.NewNativeType(4, gocql.TypeInt, "")} + tableMetadata := &gocql.TableMetadata{ + OrderedColumns: []string{"id"}, + PartitionKey: []*gocql.ColumnMetadata{id}, + Columns: map[string]*gocql.ColumnMetadata{"id": id}, + } + keyspace := &gocql.KeyspaceMetadata{ + Tables: map[string]*gocql.TableMetadata{ + "users": tableMetadata, + "users_by_email": tableMetadata, + }, + MaterializedViews: map[string]*gocql.MaterializedViewMetadata{ + "users_by_email": {Name: "users_by_email", BaseTable: tableMetadata}, + }, + } + + viewCalls := 0 + ddl, err := schemaObjectDDLFromMetadata("app", "users_by_email", keyspace, func() (string, error) { + viewCalls++ + return "CREATE MATERIALIZED VIEW app.users_by_email", nil + }) + if err != nil || ddl != "CREATE MATERIALIZED VIEW app.users_by_email" || viewCalls != 1 { + t.Fatalf("materialized view dispatch = %q, calls = %d, err = %v", ddl, viewCalls, err) + } + + tableDDL, err := schemaObjectDDLFromMetadata("app", "users", keyspace, func() (string, error) { + t.Fatal("ordinary table DDL must not use the materialized-view loader") + return "", nil + }) + if err != nil || !strings.HasPrefix(tableDDL, `CREATE TABLE "app"."users"`) { + t.Fatalf("ordinary table DDL = %q, err = %v", tableDDL, err) + } + if _, err := schemaObjectDDLFromMetadata("app", "missing", keyspace, func() (string, error) { + return "", errors.New("unexpected") + }); err == nil || !strings.Contains(err.Error(), "table not found") { + t.Fatalf("missing object error = %v", err) + } +} + +func TestMaterializedViewObjectSourcePreservesProtocolShape(t *testing.T) { + result, err := materializedViewObjectSource("app", "users_by_email", "MATERIALIZED_VIEW", func() (string, error) { + return "CREATE MATERIALIZED VIEW app.users_by_email", nil + }) + if err != nil { + t.Fatal(err) + } + if result.Name != "users_by_email" || result.ObjectType != "MATERIALIZED_VIEW" || result.Schema == nil || *result.Schema != "app" || result.Source != "CREATE MATERIALIZED VIEW app.users_by_email" { + t.Fatalf("unexpected object source: %#v", result) + } + + loaderCalled := false + _, err = materializedViewObjectSource("app", "users", "TABLE", func() (string, error) { + loaderCalled = true + return "unexpected", nil + }) + if err == nil || loaderCalled || err.Error() != "object source is not supported by Cassandra" { + t.Fatalf("unsupported source err = %v, loader called = %v", err, loaderCalled) + } + + wantErr := errors.New("definition unavailable") + _, err = materializedViewObjectSource("app", "users_by_email", "materialized view", func() (string, error) { + return "", wantErr + }) + if !errors.Is(err, wantErr) { + t.Fatalf("object-source loader error = %v", err) + } +} + +func TestMaterializedViewDDLFromLegacyCatalogPreservesDefinition(t *testing.T) { + catalog := materializedViewCatalog{ + BaseTable: `Base"Events`, + IncludeAllColumns: false, + WhereClause: `"Tenant Id" IS NOT NULL AND "Category" IS NOT NULL`, + Columns: []materializedViewColumn{ + {Name: "Bucket", Kind: "clustering", Position: 1, ClusteringOrder: "asc"}, + {Name: "Payload", Kind: "regular", Position: -1}, + {Name: "Category", Kind: "partition_key", Position: 1}, + {Name: "Event Time", Kind: "clustering", Position: 0, ClusteringOrder: "desc"}, + {Name: "Tenant Id", Kind: "partition_key", Position: 0}, + }, + Options: map[string]any{ + "caching": map[string]string{"rows_per_partition": "NONE", "keys": "ALL"}, + "comment": "owner's view", + "default_time_to_live": 0, + "extensions": map[string][]byte{}, + "gc_grace_seconds": 86401, + "speculative_retry": "99PERCENTILE", + "read_repair_chance": 0.0, + "dclocal_read_repair_chance": 0.1, + }, + } + + ddl, err := materializedViewDDLFromCatalog(`App"Ks`, `Events"ByCategory`, catalog) + if err != nil { + t.Fatal(err) + } + for _, fragment := range []string{ + `CREATE MATERIALIZED VIEW "App""Ks"."Events""ByCategory" AS`, + `SELECT "Tenant Id", "Category", "Event Time", "Bucket", "Payload"`, + `FROM "App""Ks"."Base""Events"`, + `WHERE "Tenant Id" IS NOT NULL AND "Category" IS NOT NULL`, + `PRIMARY KEY (("Tenant Id", "Category"), "Event Time", "Bucket")`, + `CLUSTERING ORDER BY ("Event Time" DESC, "Bucket" ASC)`, + `caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}`, + `comment = 'owner''s view'`, + `dclocal_read_repair_chance = 0.1`, + `gc_grace_seconds = 86401`, + `read_repair_chance = 0`, + `speculative_retry = '99PERCENTILE'`, + } { + if !strings.Contains(ddl, fragment) { + t.Fatalf("materialized view DDL omitted %q:\n%s", fragment, ddl) + } + } + if strings.Contains(ddl, "default_time_to_live") || strings.Contains(ddl, "extensions") { + t.Fatalf("materialized view DDL emitted non-applicable empty options:\n%s", ddl) + } +} + +func TestLegacyMaterializedViewOptionsUseVersionAppropriateReadRepair(t *testing.T) { + modern, err := legacyMaterializedViewOptions(map[string]any{ + "read_repair": "BLOCKING", + "read_repair_chance": 0.0, + "dclocal_read_repair_chance": 0.0, + "memtable": "", + "default_time_to_live": 0, + "extensions": map[string][]byte{}, + }) + if err != nil { + t.Fatal(err) + } + if modern["read_repair"] != "BLOCKING" || modern["read_repair_chance"] != nil || modern["dclocal_read_repair_chance"] != nil || modern["memtable"] != nil { + t.Fatalf("unexpected modern options: %#v", modern) + } + + legacy, err := legacyMaterializedViewOptions(map[string]any{ + "read_repair_chance": 0.0, + "dclocal_read_repair_chance": 0.1, + "default_time_to_live": 0, + "extensions": map[string][]byte{}, + }) + if err != nil { + t.Fatal(err) + } + if legacy["read_repair_chance"] != 0.0 || legacy["dclocal_read_repair_chance"] != 0.1 { + t.Fatalf("unexpected Cassandra 3 options: %#v", legacy) + } +} + +func TestMaterializedViewDDLFromLegacyCatalogPreservesSelectStar(t *testing.T) { + ddl, err := materializedViewDDLFromCatalog("app", "users_by_email", materializedViewCatalog{ + BaseTable: "users", + IncludeAllColumns: true, + WhereClause: "email IS NOT NULL AND id IS NOT NULL", + Columns: []materializedViewColumn{ + {Name: "email", Kind: "partition_key", Position: 0}, + {Name: "id", Kind: "clustering", Position: 0, ClusteringOrder: "asc"}, + }, + }) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(ddl, "SELECT *") { + t.Fatalf("materialized view DDL did not preserve SELECT *:\n%s", ddl) + } +} + +func TestMaterializedViewDDLOnlyFallsBackForUnsupportedDescribeSyntax(t *testing.T) { + legacyCalls := 0 + ddl, err := materializedViewDDLWithFallback( + func() (string, error) { return "CREATE MATERIALIZED VIEW native", nil }, + func() (string, error) { + legacyCalls++ + return "unexpected", nil + }, + ) + if err != nil || ddl != "CREATE MATERIALIZED VIEW native" || legacyCalls != 0 { + t.Fatalf("native DDL = %q, legacy calls = %d, err = %v", ddl, legacyCalls, err) + } + + legacyCalls = 0 + ddl, err = materializedViewDDLWithFallback( + func() (string, error) { + return "", &testCassandraRequestError{code: gocql.ErrCodeSyntax, message: "no viable alternative at input 'DESCRIBE'"} + }, + func() (string, error) { + legacyCalls++ + return "CREATE MATERIALIZED VIEW legacy", nil + }, + ) + if err != nil || ddl != "CREATE MATERIALIZED VIEW legacy" || legacyCalls != 1 { + t.Fatalf("syntax fallback = %q, calls = %d, err = %v", ddl, legacyCalls, err) + } + + permissionErr := &testCassandraRequestError{code: gocql.ErrCodeUnauthorized, message: "not authorized"} + legacyCalls = 0 + _, err = materializedViewDDLWithFallback( + func() (string, error) { return "", permissionErr }, + func() (string, error) { + legacyCalls++ + return "unexpected", nil + }, + ) + if !errors.Is(err, permissionErr) || legacyCalls != 0 { + t.Fatalf("permission error = %v, legacy calls = %d", err, legacyCalls) + } + + networkErr := errors.New("connection reset") + _, err = materializedViewDDLWithFallback( + func() (string, error) { return "", networkErr }, + func() (string, error) { + legacyCalls++ + return "unexpected", nil + }, + ) + if !errors.Is(err, networkErr) || legacyCalls != 0 { + t.Fatalf("network error = %v, legacy calls = %d", err, legacyCalls) + } +} + +func TestLegacyMaterializedViewOptionsRejectLossyDefinitions(t *testing.T) { + for name, row := range map[string]map[string]any{ + "unknown option": { + "base_table_name": "users", + "future_option": "cannot safely reproduce", + }, + "non-zero default TTL": { + "base_table_name": "users", + "default_time_to_live": 60, + }, + "non-empty extensions": { + "base_table_name": "users", + "extensions": map[string][]byte{"custom": {0x01}}, + }, + } { + t.Run(name, func(t *testing.T) { + if _, err := legacyMaterializedViewOptions(row); err == nil { + t.Fatal("expected an explicit error instead of a lossy legacy DDL") + } + }) + } +} + func TestMetadataWindowAndFilter(t *testing.T) { values := []string{"a", "b", "c", "d"} if got := applyMetadataWindow(values, 1, 2); !reflect.DeepEqual(got, []string{"b", "c"}) { diff --git a/agents/drivers/dameng/src/main/java/com/dbx/agent/dameng/DamengAgent.java b/agents/drivers/dameng/src/main/java/com/dbx/agent/dameng/DamengAgent.java index cddacb8c70..4e4d2538a8 100644 --- a/agents/drivers/dameng/src/main/java/com/dbx/agent/dameng/DamengAgent.java +++ b/agents/drivers/dameng/src/main/java/com/dbx/agent/dameng/DamengAgent.java @@ -2244,52 +2244,124 @@ public final class DamengAgent extends AbstractJdbcAgent { table ); } - return unchecked(() -> { - List result = new ArrayList<>(); - // The LEFT JOIN also matches 'U': a UNIQUE constraint owns its backing index the same - // way a primary key does, and neither can be altered with index-level DDL (#7959). - // The constraint type therefore drives two flags — IS_PK (only 'P') and - // CONSTRAINT_BACKED ('P' or 'U'), the same split `independentIndexes` already uses to - // keep constraint-backed indexes out of the generated table DDL. A unique index - // created with CREATE UNIQUE INDEX has no ALL_CONSTRAINTS row, so it stays false and - // keeps the index-level DDL path. - String sql = """ - SELECT /*+ PARALLEL(1) */ i.INDEX_NAME, - LISTAGG(ic.COLUMN_NAME, ',') WITHIN GROUP (ORDER BY ic.COLUMN_POSITION) AS COLUMNS, - i.UNIQUENESS, - CASE WHEN c.CONSTRAINT_TYPE = 'P' THEN 1 ELSE 0 END AS IS_PK, - i.INDEX_TYPE, - CASE WHEN c.CONSTRAINT_TYPE IN ('P', 'U') THEN 1 ELSE 0 END AS CONSTRAINT_BACKED - FROM ALL_INDEXES i - JOIN ALL_IND_COLUMNS ic ON i.INDEX_NAME = ic.INDEX_NAME AND i.OWNER = ic.INDEX_OWNER AND i.TABLE_OWNER = ic.TABLE_OWNER - LEFT JOIN ALL_CONSTRAINTS c ON i.INDEX_NAME = c.INDEX_NAME AND i.TABLE_OWNER = c.OWNER - AND c.CONSTRAINT_TYPE IN ('P', 'U') - WHERE i.TABLE_OWNER = ? AND i.TABLE_NAME = ? - GROUP BY i.INDEX_NAME, i.UNIQUENESS, c.CONSTRAINT_TYPE, i.INDEX_TYPE - ORDER BY i.INDEX_NAME - """.stripIndent().trim(); - try (PreparedStatement stmt = requireConnected().prepareStatement(sql)) { - stmt.setString(1, schema); - stmt.setString(2, table); - try (ResultSet rs = stmt.executeQuery()) { - while (rs.next()) { - IndexInfo index = new IndexInfo( - rs.getString(1), - splitNonEmpty(coalesce(rs.getString(2)), ","), - "UNIQUE".equals(rs.getString(3)), - "1".equals(rs.getString(4)), - null, - rs.getString(5), - null, - null - ); - index.setConstraint_backed("1".equals(rs.getString(6))); - result.add(index); - } + return unchecked(() -> tableIndexes(schema, table)); + } + + private List tableIndexes(String schema, String table) throws Exception { + try { + return indexesFromSystemCatalog(schema, table); + } catch (Exception systemCatalogError) { + if (isDamengConnectionError(systemCatalogError)) { + throw systemCatalogError; + } + try { + return indexesFromDictionaryViews(schema, table); + } catch (Exception dictionaryViewError) { + dictionaryViewError.addSuppressed(systemCatalogError); + throw dictionaryViewError; + } + } + } + + private List indexesFromSystemCatalog(String schema, String table) throws Exception { + List result = new ArrayList<>(); + // ALL_INDEXES applies visibility checks across the complete catalog. Resolve the one table + // first, then inspect only its index children; the official catalog exposes key order via + // SF_GET_INDEX_KEY_SEQ and constraint ownership through SYSCONS. + String sql = """ + SELECT /*+ PARALLEL(1) */ index_object.NAME, + LISTAGG(column_object.NAME, ',') WITHIN GROUP ( + ORDER BY SF_GET_INDEX_KEY_SEQ(index_metadata.KEYNUM, index_metadata.KEYINFO, column_object.COLID) + ) AS COLUMNS, + index_metadata.ISUNIQUE, + MAX(CASE WHEN constraint_metadata.TYPE$ = 'P' THEN 1 ELSE 0 END) AS IS_PK, + index_metadata.TYPE$, + MAX(CASE WHEN constraint_metadata.TYPE$ IN ('P', 'U') THEN 1 ELSE 0 END) AS CONSTRAINT_BACKED, + index_metadata.FLAG + FROM SYS.SYSOBJECTS schema_object + JOIN SYS.SYSOBJECTS table_object ON table_object.SCHID = schema_object.ID + AND table_object.TYPE$ = 'SCHOBJ' AND table_object.SUBTYPE$ = 'UTAB' + JOIN SYS.SYSOBJECTS index_object ON index_object.PID = table_object.ID + AND index_object.SUBTYPE$ = 'INDEX' + JOIN SYS.SYSINDEXES index_metadata ON index_metadata.ID = index_object.ID + JOIN SYS.SYSCOLUMNS column_object ON column_object.ID = table_object.ID + AND SF_COL_IS_IDX_KEY( + index_metadata.KEYNUM, + index_metadata.KEYINFO, + column_object.COLID + ) = 1 + LEFT JOIN SYS.SYSCONS constraint_metadata ON constraint_metadata.TABLEID = table_object.ID + AND constraint_metadata.INDEXID = index_metadata.ID + AND constraint_metadata.TYPE$ IN ('P', 'U') + WHERE schema_object.TYPE$ = 'SCH' AND schema_object.NAME = ? AND table_object.NAME = ? + GROUP BY index_object.NAME, index_metadata.ISUNIQUE, index_metadata.TYPE$, index_metadata.FLAG + ORDER BY index_object.NAME + """.stripIndent().trim(); + try (PreparedStatement stmt = requireConnected().prepareStatement(sql)) { + stmt.setString(1, schema); + stmt.setString(2, table); + try (ResultSet rs = stmt.executeQuery()) { + while (rs.next()) { + boolean isPrimary = "1".equals(rs.getString(4)); + boolean constraintBacked = "1".equals(rs.getString(6)); + IndexInfo index = new IndexInfo( + rs.getString(1), + splitNonEmpty(coalesce(rs.getString(2)), ","), + "Y".equalsIgnoreCase(rs.getString(3)), + isPrimary, + null, + damengSystemIndexType(rs.getString(5), rs.getInt(7), constraintBacked), + null, + null + ); + index.setConstraint_backed(constraintBacked); + result.add(index); } } - return result; - }); + } + return result; + } + + private List indexesFromDictionaryViews(String schema, String table) throws Exception { + List result = new ArrayList<>(); + // The LEFT JOIN also matches 'U': a UNIQUE constraint owns its backing index the same + // way a primary key does, and neither can be altered with index-level DDL (#7959). + String sql = """ + SELECT /*+ PARALLEL(1) */ i.INDEX_NAME, + LISTAGG(ic.COLUMN_NAME, ',') WITHIN GROUP (ORDER BY ic.COLUMN_POSITION) AS COLUMNS, + i.UNIQUENESS, + CASE WHEN c.CONSTRAINT_TYPE = 'P' THEN 1 ELSE 0 END AS IS_PK, + i.INDEX_TYPE, + CASE WHEN c.CONSTRAINT_TYPE IN ('P', 'U') THEN 1 ELSE 0 END AS CONSTRAINT_BACKED + FROM ALL_INDEXES i + JOIN ALL_IND_COLUMNS ic ON i.INDEX_NAME = ic.INDEX_NAME AND i.OWNER = ic.INDEX_OWNER AND i.TABLE_OWNER = ic.TABLE_OWNER + LEFT JOIN ALL_CONSTRAINTS c ON i.INDEX_NAME = c.INDEX_NAME AND i.TABLE_OWNER = c.OWNER + AND c.CONSTRAINT_TYPE IN ('P', 'U') + WHERE i.TABLE_OWNER = ? AND i.TABLE_NAME = ? + GROUP BY i.INDEX_NAME, i.UNIQUENESS, c.CONSTRAINT_TYPE, i.INDEX_TYPE + ORDER BY i.INDEX_NAME + """.stripIndent().trim(); + try (PreparedStatement stmt = requireConnected().prepareStatement(sql)) { + stmt.setString(1, schema); + stmt.setString(2, table); + try (ResultSet rs = stmt.executeQuery()) { + while (rs.next()) { + IndexInfo index = new IndexInfo( + rs.getString(1), + splitNonEmpty(coalesce(rs.getString(2)), ","), + "UNIQUE".equals(rs.getString(3)), + "1".equals(rs.getString(4)), + null, + rs.getString(5), + null, + null + ); + index.setConstraint_backed("1".equals(rs.getString(6))); + result.add(index); + } + } + } + return result; } @Override @@ -2298,33 +2370,108 @@ public final class DamengAgent extends AbstractJdbcAgent { if (legacyJdbcMetadata) { return StandardJdbcMetadata.INSTANCE.listForeignKeys(requireConnected(), schema, table); } - return unchecked(() -> { - List result = new ArrayList<>(); - String sql = """ - SELECT c.CONSTRAINT_NAME, cc.COLUMN_NAME, rc.TABLE_NAME, rcc.COLUMN_NAME - FROM ALL_CONSTRAINTS c - JOIN ALL_CONS_COLUMNS cc ON c.CONSTRAINT_NAME = cc.CONSTRAINT_NAME AND c.OWNER = cc.OWNER - JOIN ALL_CONSTRAINTS rc ON c.R_CONSTRAINT_NAME = rc.CONSTRAINT_NAME AND c.R_OWNER = rc.OWNER - JOIN ALL_CONS_COLUMNS rcc ON rc.CONSTRAINT_NAME = rcc.CONSTRAINT_NAME AND rc.OWNER = rcc.OWNER - WHERE c.CONSTRAINT_TYPE = 'R' AND c.OWNER = ? AND c.TABLE_NAME = ? - ORDER BY c.CONSTRAINT_NAME - """.stripIndent().trim(); - try (PreparedStatement stmt = requireConnected().prepareStatement(sql)) { - stmt.setString(1, schema); - stmt.setString(2, table); - try (ResultSet rs = stmt.executeQuery()) { - while (rs.next()) { - result.add(new ForeignKeyInfo( - rs.getString(1), - rs.getString(2), - rs.getString(3), - rs.getString(4) - )); - } + return unchecked(() -> foreignKeys(schema, table)); + } + + private List foreignKeys(String schema, String table) throws Exception { + try { + return foreignKeysFromSystemCatalog(schema, table); + } catch (Exception systemCatalogError) { + if (isDamengConnectionError(systemCatalogError)) { + throw systemCatalogError; + } + try { + return foreignKeysFromDictionaryViews(schema, table); + } catch (Exception dictionaryViewError) { + dictionaryViewError.addSuppressed(systemCatalogError); + throw dictionaryViewError; + } + } + } + + private List foreignKeysFromSystemCatalog(String schema, String table) throws Exception { + List result = new ArrayList<>(); + String sql = """ + SELECT /*+ MAX_OPT_N_TABLES(5) PARALLEL(1) */ constraint_object.NAME, + local_column.NAME, + referenced_table.NAME, + referenced_column.NAME + FROM SYS.SYSOBJECTS schema_object + JOIN SYS.SYSOBJECTS table_object ON table_object.SCHID = schema_object.ID + AND table_object.TYPE$ = 'SCHOBJ' AND table_object.SUBTYPE$ = 'UTAB' + JOIN SYS.SYSCONS foreign_key ON foreign_key.TABLEID = table_object.ID + AND foreign_key.TYPE$ = 'F' + JOIN SYS.SYSOBJECTS constraint_object ON constraint_object.ID = foreign_key.ID + AND constraint_object.PID = table_object.ID + AND constraint_object.SUBTYPE$ = 'CONS' + JOIN SYS.SYSINDEXES local_index ON local_index.ID = foreign_key.INDEXID + JOIN SYS.SYSCOLUMNS local_column ON local_column.ID = table_object.ID + AND SF_COL_IS_IDX_KEY(local_index.KEYNUM, local_index.KEYINFO, local_column.COLID) = 1 + JOIN SYS.SYSOBJECTS referenced_index_object ON referenced_index_object.ID = foreign_key.FINDEXID + AND referenced_index_object.SUBTYPE$ = 'INDEX' + JOIN SYS.SYSINDEXES referenced_index ON referenced_index.ID = referenced_index_object.ID + JOIN SYS.SYSOBJECTS referenced_table ON referenced_table.ID = referenced_index_object.PID + AND referenced_table.TYPE$ = 'SCHOBJ' AND referenced_table.SUBTYPE$ = 'UTAB' + JOIN SYS.SYSCOLUMNS referenced_column ON referenced_column.ID = referenced_table.ID + AND SF_COL_IS_IDX_KEY( + referenced_index.KEYNUM, + referenced_index.KEYINFO, + referenced_column.COLID + ) = 1 + AND SF_GET_INDEX_KEY_SEQ(local_index.KEYNUM, local_index.KEYINFO, local_column.COLID) + = SF_GET_INDEX_KEY_SEQ( + referenced_index.KEYNUM, + referenced_index.KEYINFO, + referenced_column.COLID + ) + WHERE schema_object.TYPE$ = 'SCH' AND schema_object.NAME = ? AND table_object.NAME = ? + ORDER BY constraint_object.NAME, + SF_GET_INDEX_KEY_SEQ(local_index.KEYNUM, local_index.KEYINFO, local_column.COLID) + """.stripIndent().trim(); + try (PreparedStatement stmt = requireConnected().prepareStatement(sql)) { + stmt.setString(1, schema); + stmt.setString(2, table); + try (ResultSet rs = stmt.executeQuery()) { + while (rs.next()) { + result.add(new ForeignKeyInfo( + rs.getString(1), + rs.getString(2), + rs.getString(3), + rs.getString(4) + )); } } - return result; - }); + } + return result; + } + + private List foreignKeysFromDictionaryViews(String schema, String table) throws Exception { + List result = new ArrayList<>(); + String sql = """ + SELECT /*+ PARALLEL(1) */ c.CONSTRAINT_NAME, cc.COLUMN_NAME, rc.TABLE_NAME, rcc.COLUMN_NAME + FROM ALL_CONSTRAINTS c + JOIN ALL_CONS_COLUMNS cc ON c.CONSTRAINT_NAME = cc.CONSTRAINT_NAME AND c.OWNER = cc.OWNER + JOIN ALL_CONSTRAINTS rc ON c.R_CONSTRAINT_NAME = rc.CONSTRAINT_NAME AND c.R_OWNER = rc.OWNER + JOIN ALL_CONS_COLUMNS rcc ON rc.CONSTRAINT_NAME = rcc.CONSTRAINT_NAME AND rc.OWNER = rcc.OWNER + AND cc.POSITION = rcc.POSITION + WHERE c.CONSTRAINT_TYPE = 'R' AND c.OWNER = ? AND c.TABLE_NAME = ? + ORDER BY c.CONSTRAINT_NAME, cc.POSITION + """.stripIndent().trim(); + try (PreparedStatement stmt = requireConnected().prepareStatement(sql)) { + stmt.setString(1, schema); + stmt.setString(2, table); + try (ResultSet rs = stmt.executeQuery()) { + while (rs.next()) { + result.add(new ForeignKeyInfo( + rs.getString(1), + rs.getString(2), + rs.getString(3), + rs.getString(4) + )); + } + } + } + return result; } @Override @@ -2895,134 +3042,22 @@ public final class DamengAgent extends AbstractJdbcAgent { } private List independentIndexes(String schema, String table) throws Exception { - try { - return independentIndexesFromSystemCatalog(schema, table); - } catch (Exception systemCatalogError) { - if (isDamengConnectionError(systemCatalogError)) { - throw systemCatalogError; - } - try { - return independentIndexesFromDictionaryViews(schema, table); - } catch (Exception dictionaryViewError) { - dictionaryViewError.addSuppressed(systemCatalogError); - throw dictionaryViewError; - } - } + return tableIndexes(schema, table).stream() + .filter(index -> !index.getConstraint_backed()) + .toList(); } - private List independentIndexesFromSystemCatalog(String schema, String table) throws Exception { - List result = new ArrayList<>(); - // DM's ALL_INDEXES view performs privilege filtering across the complete catalog and can - // take tens of seconds in installations with very large index dictionaries (#10075). - // The DM JDBC driver and DBeaver instead resolve the requested table id first and read its - // SYS index rows directly. Keep the ALL_* implementation below as a compatibility and - // permission fallback for deployments that do not expose these catalog tables. - String sql = """ - SELECT /*+ PARALLEL(1) */ index_object.NAME, - LISTAGG(column_object.NAME, ',') WITHIN GROUP ( - ORDER BY SF_GET_INDEX_KEY_SEQ(index_metadata.KEYNUM, index_metadata.KEYINFO, column_object.COLID) - ) AS COLUMNS, - index_metadata.ISUNIQUE, - index_metadata.TYPE$, - index_metadata.FLAG - FROM SYS.SYSOBJECTS schema_object - JOIN SYS.SYSOBJECTS table_object ON table_object.SCHID = schema_object.ID - AND table_object.TYPE$ = 'SCHOBJ' AND table_object.SUBTYPE$ = 'UTAB' - JOIN SYS.SYSOBJECTS index_object ON index_object.PID = table_object.ID - AND index_object.SUBTYPE$ = 'INDEX' - JOIN SYS.SYSINDEXES index_metadata ON index_metadata.ID = index_object.ID - JOIN SYS.SYSCOLUMNS column_object ON column_object.ID = table_object.ID - AND SF_COL_IS_IDX_KEY( - index_metadata.KEYNUM, - index_metadata.KEYINFO, - column_object.COLID - ) = 1 - WHERE schema_object.TYPE$ = 'SCH' AND schema_object.NAME = ? AND table_object.NAME = ? - AND NOT EXISTS ( - SELECT 1 - FROM SYS.SYSCONS constraint_metadata - WHERE constraint_metadata.TABLEID = table_object.ID - AND constraint_metadata.INDEXID = index_metadata.ID - AND constraint_metadata.TYPE$ IN ('P', 'U') - ) - GROUP BY index_object.NAME, index_metadata.ISUNIQUE, index_metadata.TYPE$, index_metadata.FLAG - ORDER BY index_object.NAME - """.stripIndent().trim(); - try (PreparedStatement stmt = requireConnected().prepareStatement(sql)) { - stmt.setString(1, schema); - stmt.setString(2, table); - try (ResultSet rs = stmt.executeQuery()) { - while (rs.next()) { - String indexName = rs.getString(1); - if (notBlank(indexName)) { - result.add(new IndexInfo( - indexName, - splitNonEmpty(coalesce(rs.getString(2)), ","), - "Y".equalsIgnoreCase(rs.getString(3)), - false, - null, - damengSystemIndexType(rs.getString(4), rs.getInt(5)), - null, - null - )); - } - } - } - } - return result; - } - - private List independentIndexesFromDictionaryViews(String schema, String table) throws Exception { - List result = new ArrayList<>(); - // Primary-key and unique-constraint backing indexes are already represented in table DDL. - String sql = """ - SELECT /*+ PARALLEL(1) */ i.INDEX_NAME, - LISTAGG(ic.COLUMN_NAME, ',') WITHIN GROUP (ORDER BY ic.COLUMN_POSITION) AS COLUMNS, - i.UNIQUENESS, - i.INDEX_TYPE - FROM ALL_INDEXES i - JOIN ALL_IND_COLUMNS ic ON i.INDEX_NAME = ic.INDEX_NAME AND i.OWNER = ic.INDEX_OWNER AND i.TABLE_OWNER = ic.TABLE_OWNER - WHERE i.TABLE_OWNER = ? AND i.TABLE_NAME = ? - AND NOT EXISTS ( - SELECT 1 - FROM ALL_CONSTRAINTS c - WHERE c.OWNER = i.TABLE_OWNER - AND c.TABLE_NAME = i.TABLE_NAME - AND c.INDEX_NAME = i.INDEX_NAME - AND c.CONSTRAINT_TYPE IN ('P', 'U') - ) - GROUP BY i.INDEX_NAME, i.UNIQUENESS, i.INDEX_TYPE - ORDER BY i.INDEX_NAME - """.stripIndent().trim(); - try (PreparedStatement stmt = requireConnected().prepareStatement(sql)) { - stmt.setString(1, schema); - stmt.setString(2, table); - try (ResultSet rs = stmt.executeQuery()) { - while (rs.next()) { - String indexName = rs.getString(1); - if (notBlank(indexName)) { - result.add(new IndexInfo( - indexName, - splitNonEmpty(coalesce(rs.getString(2)), ","), - "UNIQUE".equals(rs.getString(3)), - false, - null, - rs.getString(4), - null, - null - )); - } - } - } - } - return result; - } - - private static String damengSystemIndexType(String catalogType, int flags) { - if ((flags & 3) != 0) { + private static String damengSystemIndexType(String catalogType, int flags, boolean constraintBacked) { + if ((flags & 1) != 0 || ((flags & 2) != 0 && !constraintBacked)) { return "INTERNAL"; } - return "ST".equalsIgnoreCase(coalesce(catalogType).trim()) ? "SPATIAL" : catalogType; + String normalized = coalesce(catalogType).trim().toUpperCase(Locale.ROOT); + return switch (normalized) { + case "BT" -> "NORMAL"; + case "BM" -> "BITMAP"; + case "ST" -> "SPATIAL"; + default -> catalogType; + }; } static String indexDdl(String schema, String table, IndexInfo index) { diff --git a/agents/drivers/dameng/src/test/java/com/dbx/agent/dameng/DamengAgentMetadataTest.java b/agents/drivers/dameng/src/test/java/com/dbx/agent/dameng/DamengAgentMetadataTest.java index 0c2c90b4e4..f46aaad394 100644 --- a/agents/drivers/dameng/src/test/java/com/dbx/agent/dameng/DamengAgentMetadataTest.java +++ b/agents/drivers/dameng/src/test/java/com/dbx/agent/dameng/DamengAgentMetadataTest.java @@ -2,6 +2,7 @@ package com.dbx.agent.dameng; import com.dbx.agent.ColumnInfo; import com.dbx.agent.DatabaseInfo; +import com.dbx.agent.ForeignKeyInfo; import com.dbx.agent.IndexInfo; import com.dbx.agent.MetadataListConstraints; import com.dbx.agent.ObjectInfo; @@ -188,17 +189,25 @@ class DamengAgentMetadataTest { } @Test - void disablesParallelExecutionForIndexMetadataQuery() { + void usesBoundedSystemCatalogForIndexMetadata() { DamengAgent agent = new DamengAgent(); TestSupport.setPrivateConnection(agent, JdbcMetadataSqlFake.connection()); agent.listIndexes("APP", "USERS"); String indexesSql = JdbcMetadataSqlFake.statements.stream() - .filter(sql -> sql.contains("ALL_INDEXES")) + .filter(sql -> sql.contains("SYS.SYSINDEXES")) .findFirst() .orElseThrow(); Assertions.assertTrue(indexesSql.startsWith("SELECT /*+ PARALLEL(1) */"), indexesSql); + Assertions.assertTrue(indexesSql.contains("schema_object.NAME = ?"), indexesSql); + Assertions.assertTrue(indexesSql.contains("table_object.NAME = ?"), indexesSql); + Assertions.assertTrue(indexesSql.contains("SF_COL_IS_IDX_KEY"), indexesSql); + Assertions.assertTrue(indexesSql.contains("SF_GET_INDEX_KEY_SEQ"), indexesSql); + Assertions.assertTrue( + JdbcMetadataSqlFake.statements.stream().noneMatch(sql -> sql.contains("ALL_INDEXES")), + String.join("\n", JdbcMetadataSqlFake.statements) + ); } @Test @@ -209,12 +218,15 @@ class DamengAgentMetadataTest { agent.listIndexes("APP", "USERS"); String indexesSql = JdbcMetadataSqlFake.statements.stream() - .filter(sql -> sql.contains("ALL_INDEXES") && sql.contains("ALL_CONSTRAINTS")) + .filter(sql -> sql.contains("SYS.SYSINDEXES") && sql.contains("SYS.SYSCONS")) .findFirst() .orElseThrow(); - Assertions.assertTrue(indexesSql.contains("AND c.CONSTRAINT_TYPE IN ('P', 'U')"), indexesSql); + Assertions.assertTrue(indexesSql.contains("constraint_metadata.TYPE$ IN ('P', 'U')"), indexesSql); Assertions.assertTrue(indexesSql.contains("AS CONSTRAINT_BACKED"), indexesSql); - Assertions.assertTrue(indexesSql.contains("CASE WHEN c.CONSTRAINT_TYPE = 'P' THEN 1 ELSE 0 END AS IS_PK"), indexesSql); + Assertions.assertTrue( + indexesSql.contains("MAX(CASE WHEN constraint_metadata.TYPE$ = 'P' THEN 1 ELSE 0 END) AS IS_PK"), + indexesSql + ); } @Test @@ -222,10 +234,10 @@ class DamengAgentMetadataTest { DamengAgent agent = new DamengAgent(); TestSupport.setPrivateConnection(agent, indexMetadataConnection(List.of( // UNIQUE constraint (虚索引) / standalone CREATE UNIQUE INDEX (实索引) / plain index. - Arrays.asList("UX_USERS_EMAIL", "EMAIL", "UNIQUE", "0", "NORMAL", "1"), - Arrays.asList("UX_USERS_CODE", "CODE", "UNIQUE", "0", "NORMAL", "0"), - Arrays.asList("IDX_USERS_NAME", "NAME", "NONUNIQUE", "0", "NORMAL", "0"), - Arrays.asList("PK_USERS", "ID", "UNIQUE", "1", "NORMAL", "1") + Arrays.asList("UX_USERS_EMAIL", "EMAIL", "Y", "0", "BT", "1", 0), + Arrays.asList("UX_USERS_CODE", "CODE", "Y", "0", "BT", "0", 0), + Arrays.asList("IDX_USERS_NAME", "NAME", "N", "0", "BT", "0", 0), + Arrays.asList("PK_USERS", "ID", "Y", "1", "BT", "1", 4) ))); List indexes = agent.listIndexes("APP", "USERS"); @@ -246,6 +258,142 @@ class DamengAgentMetadataTest { List.of(true, true, false, true), indexes.stream().map(IndexInfo::getIs_unique).toList() ); + Assertions.assertEquals( + List.of("NORMAL", "NORMAL", "NORMAL", "NORMAL"), + indexes.stream().map(IndexInfo::getIndex_type).toList() + ); + } + + @Test + void fallsBackToDictionaryViewsForIndexMetadataWithoutSystemCatalogAccess() { + DamengAgent agent = new DamengAgent(); + List sqls = new ArrayList<>(); + TestSupport.setPrivateConnection(agent, catalogFallbackConnection( + sqls, + "SYS.SYSINDEXES index_metadata", + List.of(), + new SQLException("no SYS index catalog privilege"), + "FROM ALL_INDEXES i", + List.of(Arrays.asList("IDX_USERS_NAME", "NAME", "NONUNIQUE", "0", "NORMAL", "0")) + )); + + List indexes = agent.listIndexes("APP", "USERS"); + + Assertions.assertEquals(List.of("IDX_USERS_NAME"), indexes.stream().map(IndexInfo::getName).toList()); + Assertions.assertTrue(indexOfSql(sqls, "SYS.SYSINDEXES") >= 0, String.join("\n", sqls)); + Assertions.assertTrue( + indexOfSql(sqls, "ALL_INDEXES") > indexOfSql(sqls, "SYS.SYSINDEXES"), + String.join("\n", sqls) + ); + } + + @Test + void acceptsEmptySystemIndexMetadataWithoutRunningExpensiveFallback() { + DamengAgent agent = new DamengAgent(); + List sqls = new ArrayList<>(); + TestSupport.setPrivateConnection(agent, catalogFallbackConnection( + sqls, + "SYS.SYSINDEXES index_metadata", + List.of(), + null, + "FROM ALL_INDEXES i", + List.of(Arrays.asList("SHOULD_NOT_LOAD", "ID", "NONUNIQUE", "0", "NORMAL", "0")) + )); + + Assertions.assertTrue(agent.listIndexes("APP", "USERS").isEmpty()); + Assertions.assertTrue(sqls.stream().noneMatch(sql -> sql.contains("ALL_INDEXES")), String.join("\n", sqls)); + } + + @Test + void usesBoundedSystemCatalogForForeignKeyMetadata() { + DamengAgent agent = new DamengAgent(); + List sqls = new ArrayList<>(); + TestSupport.setPrivateConnection(agent, catalogFallbackConnection( + sqls, + "SYS.SYSCONS foreign_key", + List.of( + List.of("FK_ORDER_CUSTOMER", "TENANT_ID", "CUSTOMERS", "TENANT_ID"), + List.of("FK_ORDER_CUSTOMER", "CUSTOMER_ID", "CUSTOMERS", "ID") + ), + null, + "FROM ALL_CONSTRAINTS c", + List.of() + )); + + List foreignKeys = agent.listForeignKeys("APP", "ORDERS"); + + Assertions.assertEquals(List.of("TENANT_ID", "CUSTOMER_ID"), foreignKeys.stream().map(ForeignKeyInfo::getColumn).toList()); + Assertions.assertEquals(List.of("TENANT_ID", "ID"), foreignKeys.stream().map(ForeignKeyInfo::getRef_column).toList()); + String sql = sqls.get(0); + Assertions.assertTrue(sql.contains("foreign_key.TABLEID = table_object.ID"), sql); + Assertions.assertTrue(sql.contains("foreign_key.FINDEXID"), sql); + Assertions.assertTrue(sql.contains("foreign_key.TYPE$ = 'F'"), sql); + Assertions.assertTrue(sql.contains("SF_GET_INDEX_KEY_SEQ"), sql); + Assertions.assertTrue(sql.contains("schema_object.NAME = ?"), sql); + Assertions.assertTrue(sql.contains("table_object.NAME = ?"), sql); + Assertions.assertTrue(sqls.stream().noneMatch(query -> query.contains("ALL_CONSTRAINTS")), String.join("\n", sqls)); + } + + @Test + void fallsBackToPositionMatchedDictionaryForeignKeysWithoutSystemCatalogAccess() { + DamengAgent agent = new DamengAgent(); + List sqls = new ArrayList<>(); + TestSupport.setPrivateConnection(agent, catalogFallbackConnection( + sqls, + "SYS.SYSCONS foreign_key", + List.of(), + new SQLException("no SYS constraint catalog privilege"), + "FROM ALL_CONSTRAINTS c", + List.of(List.of("FK_ORDER_CUSTOMER", "CUSTOMER_ID", "CUSTOMERS", "ID")) + )); + + List foreignKeys = agent.listForeignKeys("APP", "ORDERS"); + + Assertions.assertEquals(1, foreignKeys.size()); + Assertions.assertEquals("FK_ORDER_CUSTOMER", foreignKeys.get(0).getName()); + int systemQuery = indexOfSql(sqls, "SYS.SYSCONS foreign_key"); + int fallbackQuery = indexOfSql(sqls, "ALL_CONSTRAINTS"); + Assertions.assertTrue(fallbackQuery > systemQuery, String.join("\n", sqls)); + Assertions.assertTrue(sqls.get(fallbackQuery).contains("cc.POSITION = rcc.POSITION"), sqls.get(fallbackQuery)); + } + + @Test + void acceptsEmptySystemForeignKeysWithoutRunningExpensiveFallback() { + DamengAgent agent = new DamengAgent(); + List sqls = new ArrayList<>(); + TestSupport.setPrivateConnection(agent, catalogFallbackConnection( + sqls, + "SYS.SYSCONS foreign_key", + List.of(), + null, + "FROM ALL_CONSTRAINTS c", + List.of(List.of("SHOULD_NOT_LOAD", "ID", "PARENT", "ID")) + )); + + Assertions.assertTrue(agent.listForeignKeys("APP", "ORDERS").isEmpty()); + Assertions.assertTrue(sqls.stream().noneMatch(sql -> sql.contains("ALL_CONSTRAINTS")), String.join("\n", sqls)); + } + + @Test + void doesNotRetryForeignKeyCatalogAfterConnectionFailure() { + DamengAgent agent = new DamengAgent(); + List sqls = new ArrayList<>(); + TestSupport.setPrivateConnection(agent, catalogFallbackConnection( + sqls, + "SYS.SYSCONS foreign_key", + List.of(), + new SQLNonTransientConnectionException("connection lost"), + "FROM ALL_CONSTRAINTS c", + List.of() + )); + + RuntimeException error = Assertions.assertThrows( + RuntimeException.class, + () -> agent.listForeignKeys("APP", "ORDERS") + ); + + Assertions.assertEquals("connection lost", error.getCause().getMessage()); + Assertions.assertTrue(sqls.stream().noneMatch(sql -> sql.contains("ALL_CONSTRAINTS")), String.join("\n", sqls)); } @Test @@ -2178,7 +2326,8 @@ class DamengAgentMetadataTest { List.of( indexRow("IDX_USERS_NAME", "NAME", "N", "BT", 0), indexRow("UX_USERS_EMAIL", "EMAIL", "Y", "BT", 0), - indexRow("IDX_USERS_GEO", "GEO", "N", "ST", 0) + indexRow("IDX_USERS_GEO", "GEO", "N", "ST", 0), + Arrays.asList("PK_USERS", "ID", "Y", "1", "BT", "1", 4) ), sqls ); @@ -2480,7 +2629,7 @@ class DamengAgentMetadataTest { } private static List indexRow(String name, String columns, String uniqueness, String indexType, int flags) { - return List.of(name, columns, uniqueness, indexType, flags); + return List.of(name, columns, uniqueness, "0", indexType, "0", flags); } private static List> dictionaryIndexRows(List> systemRows) { @@ -2488,10 +2637,45 @@ class DamengAgentMetadataTest { row.get(0), row.get(1), "Y".equals(row.get(2)) ? "UNIQUE" : "NONUNIQUE", - "ST".equals(row.get(3)) ? "SPATIAL" : "NORMAL" + row.get(3), + "ST".equals(row.get(4)) ? "SPATIAL" : "NORMAL", + row.get(5) )).toList(); } + private static Connection catalogFallbackConnection( + List sqls, + String systemQueryMarker, + List> systemRows, + SQLException systemError, + String fallbackQueryMarker, + List> fallbackRows + ) { + return proxy(Connection.class, (method, args) -> { + String name = method.getName(); + if ("prepareStatement".equals(name)) { + String sql = (String) args[0]; + sqls.add(sql); + if (sql.contains(systemQueryMarker)) { + return systemError == null + ? metadataStatement(systemRows) + : failingMetadataStatement(systemError); + } + if (sql.contains(fallbackQueryMarker)) { + return metadataStatement(fallbackRows); + } + throw new AssertionError("Unexpected metadata query: " + sql); + } + if ("close".equals(name)) { + return null; + } + if ("isClosed".equals(name)) { + return false; + } + return defaultValue(method.getReturnType()); + }); + } + private static int indexOfSql(List sqls, String fragment) { for (int i = 0; i < sqls.size(); i++) { if (sqls.get(i).contains(fragment)) { diff --git a/agents/drivers/iotdb/main_test.go b/agents/drivers/iotdb/main_test.go index 4586b52fa0..20c0a4e501 100644 --- a/agents/drivers/iotdb/main_test.go +++ b/agents/drivers/iotdb/main_test.go @@ -290,6 +290,31 @@ func TestDecodeQueryOptions(t *testing.T) { } } +func TestIsQueryStatementRecognizesListStatements(t *testing.T) { + tests := []struct { + sql string + want bool + }{ + {sql: "LIST USER", want: true}, + {sql: "list role", want: true}, + {sql: " LiSt PRIVILEGES OF USER root; ", want: true}, + {sql: "\tLIST PRIVILEGES OF ROLE dbx7359_role\n", want: true}, + {sql: "SHOW VERSION", want: true}, + {sql: "SELECT * FROM root.dbx_7359.d", want: true}, + {sql: ""}, + {sql: " ; "}, + {sql: "INSERT INTO root.dbx_7359.d(time,s) VALUES(1,1)"}, + } + + for _, test := range tests { + t.Run(test.sql, func(t *testing.T) { + if got := isQueryStatement(trimStatementSQL(test.sql)); got != test.want { + t.Fatalf("isQueryStatement(%q) = %t, want %t", test.sql, got, test.want) + } + }) + } +} + func TestCancelActiveOperationInvalidatesClient(t *testing.T) { blocking := &blockingIoTDBSession{closed: make(chan struct{})} connected := &sessionClient{session: blocking, dialect: client.TreeSqlDialect} diff --git a/agents/drivers/iotdb/query.go b/agents/drivers/iotdb/query.go index 65877d5d9a..74ab201d5c 100644 --- a/agents/drivers/iotdb/query.go +++ b/agents/drivers/iotdb/query.go @@ -556,7 +556,7 @@ func isQueryStatement(sql string) bool { return false } switch strings.ToUpper(fields[0]) { - case "SELECT", "SHOW", "DESC", "DESCRIBE", "EXPLAIN", "WITH": + case "SELECT", "SHOW", "DESC", "DESCRIBE", "EXPLAIN", "WITH", "LIST": return true default: return false diff --git a/agents/drivers/kafka/build.gradle b/agents/drivers/kafka/build.gradle index cbea97ab7d..4c854c47ce 100644 --- a/agents/drivers/kafka/build.gradle +++ b/agents/drivers/kafka/build.gradle @@ -1,6 +1,6 @@ dependencies { implementation 'com.google.code.gson:gson:2.12.1' - implementation 'org.apache.kafka:kafka-clients:3.9.0' + implementation 'org.apache.kafka:kafka-clients:3.9.1' implementation('org.apache.zookeeper:zookeeper:3.8.4') { exclude group: 'log4j', module: 'log4j' exclude group: 'org.slf4j', module: 'slf4j-log4j12' diff --git a/agents/drivers/kafka/src/test/java/com/dbx/agent/kafka/KafkaClientCompatibilityTest.java b/agents/drivers/kafka/src/test/java/com/dbx/agent/kafka/KafkaClientCompatibilityTest.java new file mode 100644 index 0000000000..7043e53bba --- /dev/null +++ b/agents/drivers/kafka/src/test/java/com/dbx/agent/kafka/KafkaClientCompatibilityTest.java @@ -0,0 +1,40 @@ +package com.dbx.agent.kafka; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.List; +import java.util.Map; +import javax.security.auth.Subject; +import javax.security.auth.callback.Callback; +import javax.security.auth.callback.NameCallback; +import javax.security.auth.callback.PasswordCallback; +import org.apache.kafka.common.internals.SecurityManagerCompatibility; +import org.apache.kafka.common.security.authenticator.SaslClientCallbackHandler; +import org.junit.jupiter.api.Test; + +class KafkaClientCompatibilityTest { + @Test + void passwordSaslCallbacksUseTheCurrentSubject() { + // KAFKA-17078 routes Subject access through this shim when newer JDKs + // disable the legacy Subject.getSubject API. + Subject subject = new Subject(); + subject.getPublicCredentials().add("dbx-user"); + subject.getPrivateCredentials().add("dbx-password"); + + for (String mechanism : List.of("PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512")) { + SaslClientCallbackHandler handler = new SaslClientCallbackHandler(); + handler.configure(Map.of(), mechanism, List.of()); + NameCallback name = new NameCallback("username"); + PasswordCallback password = new PasswordCallback("password", false); + + assertDoesNotThrow(() -> SecurityManagerCompatibility.get().callAs(subject, () -> { + handler.handle(new Callback[] { name, password }); + return null; + })); + assertEquals("dbx-user", name.getName()); + assertArrayEquals("dbx-password".toCharArray(), password.getPassword()); + } + } +} diff --git a/agents/drivers/rabbitmq/management.go b/agents/drivers/rabbitmq/management.go index 16eda1d389..69df16798c 100644 --- a/agents/drivers/rabbitmq/management.go +++ b/agents/drivers/rabbitmq/management.go @@ -11,6 +11,9 @@ import ( "io" "net" "net/http" + "net/url" + "regexp" + "sort" "strconv" "strings" "time" @@ -21,10 +24,28 @@ const ( defaultManagementTLSPort = 15671 defaultAMQPTLSPort = 5671 managementPageSize = 100 + managementMaxPageSize = 500 managementConnectTimeout = 10 * time.Second managementRequestTimeout = 20 * time.Second ) +type managementPageRequest struct { + Page int + PageSize int + Search string + UseRegex bool + Sort string + SortReverse bool +} + +type managementPageResult struct { + Items []any + Page int + PageSize int + TotalCount int + HasMore bool +} + type managementStatusError struct { status int method string @@ -172,6 +193,204 @@ func managementGetAll(connection jsonObject, path string) ([]any, error) { } } +// managementGetPage requests exactly one RabbitMQ management API page. Queue +// and exchange endpoints in RabbitMQ 3.8.x return a pagination envelope when +// page/page_size are present. The plain-array fallback keeps the agent usable +// with endpoints or older brokers that ignore those parameters, while still +// presenting the same explicit page contract to callers. +func managementGetPage(connection jsonObject, path string, request managementPageRequest) (managementPageResult, error) { + if err := validateManagementPageRequest(request); err != nil { + return managementPageResult{}, err + } + query := url.Values{} + query.Set("page", strconv.Itoa(request.Page)) + query.Set("page_size", strconv.Itoa(request.PageSize)) + if request.Search != "" { + query.Set("name", request.Search) + query.Set("use_regex", strconv.FormatBool(request.UseRegex)) + } + if request.Sort != "" { + query.Set("sort", request.Sort) + } + if request.SortReverse { + query.Set("sort_reverse", "true") + } + + response, err := managementGet(connection, appendManagementQuery(path, query)) + if err != nil { + return managementPageResult{}, err + } + switch typed := response.(type) { + case []any: + return paginateManagementArray(typed, request) + case map[string]any: + object := jsonObject(typed) + items, exists := typed["items"] + if !exists { + return managementPageResult{}, fmt.Errorf("Unexpected management API response for list endpoint %s", path) + } + array, ok := items.([]any) + if !ok { + return managementPageResult{}, fmt.Errorf("Unexpected management API items for list endpoint %s", path) + } + page := intOrDefault(object, "page", request.Page) + pageSize := intOrDefault(object, "page_size", request.PageSize) + totalCount := integerOrNull(object, "filtered_count") + if totalCount == nil { + totalCount = integerOrNull(object, "total_count") + } + if totalCount == nil { + inferred := (page-1)*pageSize + len(array) + totalCount = &inferred + } + pageCount := integerOrNull(object, "page_count") + hasMore := pageCount != nil && page < *pageCount + if pageCount == nil { + hasMore = page*pageSize < *totalCount + } + return managementPageResult{ + Items: array, + Page: page, + PageSize: pageSize, + TotalCount: *totalCount, + HasMore: hasMore, + }, nil + default: + return managementPageResult{}, fmt.Errorf("Unexpected management API response for list endpoint %s", path) + } +} + +func managementPageRequestFromParams(params jsonObject, allowedSorts ...string) (*managementPageRequest, error) { + page := integerOrNull(params, "page") + pageSize := integerOrNull(params, "page_size") + if page == nil && pageSize == nil { + return nil, nil + } + if page == nil || pageSize == nil { + return nil, errors.New("page and page_size must be provided together") + } + sortName := stringOrDefault(params, "sort", "name") + allowed := false + for _, candidate := range allowedSorts { + if sortName == candidate { + allowed = true + break + } + } + if !allowed { + return nil, fmt.Errorf("Unsupported list sort %q", sortName) + } + request := managementPageRequest{ + Page: *page, + PageSize: *pageSize, + Search: strings.TrimSpace(stringOrEmpty(params, "search")), + Sort: sortName, + SortReverse: boolOrDefault(params, "sort_reverse", false), + } + if err := validateManagementPageRequest(request); err != nil { + return nil, err + } + return &request, nil +} + +func validateManagementPageRequest(request managementPageRequest) error { + if request.Page < 1 { + return errors.New("page must be at least 1") + } + if request.PageSize < 1 || request.PageSize > managementMaxPageSize { + return fmt.Errorf("page_size must be between 1 and %d", managementMaxPageSize) + } + maxInt := int(^uint(0) >> 1) + if request.Page-1 > maxInt/request.PageSize { + return errors.New("page is too large") + } + return nil +} + +func appendManagementQuery(path string, values url.Values) string { + separator := "?" + if strings.Contains(path, "?") { + separator = "&" + } + return path + separator + values.Encode() +} + +func paginateManagementArray(items []any, request managementPageRequest) (managementPageResult, error) { + filtered := make([]any, 0, len(items)) + var expression *regexp.Regexp + if request.Search != "" && request.UseRegex { + compiled, err := regexp.Compile("(?i)" + request.Search) + if err != nil { + return managementPageResult{}, fmt.Errorf("Invalid management list search: %w", err) + } + expression = compiled + } + search := strings.ToLower(request.Search) + for _, item := range items { + if request.Search != "" { + object, ok := item.(map[string]any) + if !ok { + continue + } + name := stringOrEmpty(jsonObject(object), "name") + if expression != nil { + if !expression.MatchString(name) { + continue + } + } else if !strings.Contains(strings.ToLower(name), search) { + continue + } + } + filtered = append(filtered, item) + } + sort.SliceStable(filtered, func(left, right int) bool { + leftObject, _ := filtered[left].(map[string]any) + rightObject, _ := filtered[right].(map[string]any) + leftJSON := jsonObject(leftObject) + rightJSON := jsonObject(rightObject) + comparison := 0 + if request.Sort == "messages_ready" { + leftCount := longOrDefault(leftJSON, "messages_ready", 0) + rightCount := longOrDefault(rightJSON, "messages_ready", 0) + if leftCount < rightCount { + comparison = -1 + } else if leftCount > rightCount { + comparison = 1 + } + } + if comparison == 0 { + comparison = strings.Compare(stringOrEmpty(leftJSON, "name"), stringOrEmpty(rightJSON, "name")) + } + if request.SortReverse { + comparison = -comparison + } + return comparison < 0 + }) + + totalCount := len(filtered) + start := (request.Page - 1) * request.PageSize + if start >= totalCount { + return managementPageResult{ + Items: []any{}, + Page: request.Page, + PageSize: request.PageSize, + TotalCount: totalCount, + HasMore: false, + }, nil + } + end := start + request.PageSize + if end > totalCount { + end = totalCount + } + return managementPageResult{ + Items: filtered[start:end], + Page: request.Page, + PageSize: request.PageSize, + TotalCount: totalCount, + HasMore: end < totalCount, + }, nil +} + func managementBaseURLs(connection jsonObject) ([]string, error) { if explicit := stringOrNull(connection, "management_url"); explicit != nil && strings.TrimSpace(*explicit) != "" { return []string{normalizeManagementURL(*explicit)}, nil diff --git a/agents/drivers/rabbitmq/management_test.go b/agents/drivers/rabbitmq/management_test.go index c412bc1462..67c007a1f4 100644 --- a/agents/drivers/rabbitmq/management_test.go +++ b/agents/drivers/rabbitmq/management_test.go @@ -135,6 +135,134 @@ func TestManagementGetAllPlainArray(t *testing.T) { } } +func TestManagementGetPageRequestsOnlyOneRabbitMqPage(t *testing.T) { + requestCount := 0 + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + requestCount++ + query := request.URL.Query() + if query.Get("page") != "2" || query.Get("page_size") != "100" { + t.Fatalf("unexpected pagination query %q", request.URL.RawQuery) + } + if query.Get("name") != "orders & paid" || query.Get("use_regex") != "false" { + t.Fatalf("unexpected search query %q", request.URL.RawQuery) + } + if query.Get("sort") != "messages_ready" || query.Get("sort_reverse") != "true" { + t.Fatalf("unexpected sort query %q", request.URL.RawQuery) + } + writer.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(writer, `{"items":[{"name":"orders.paid"}],"page":2,"page_size":100,"page_count":3,"total_count":10000,"filtered_count":250}`) + })) + defer server.Close() + + page, err := managementGetPage(jsonObject{"management_url": server.URL}, "/api/queues?columns=name", managementPageRequest{ + Page: 2, PageSize: 100, Search: "orders & paid", Sort: "messages_ready", SortReverse: true, + }) + if err != nil { + t.Fatal(err) + } + if requestCount != 1 { + t.Fatalf("expected one management request, got %d", requestCount) + } + if len(page.Items) != 1 || page.Page != 2 || page.PageSize != 100 || page.TotalCount != 250 || !page.HasMore { + t.Fatalf("unexpected page %#v", page) + } +} + +func TestManagementGetPageFallsBackForPlainArrayResponses(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + writer.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(writer, `[{"name":"alpha","messages_ready":1},{"name":"beta","messages_ready":9},{"name":"gamma","messages_ready":4}]`) + })) + defer server.Close() + + page, err := managementGetPage(jsonObject{"management_url": server.URL}, "/api/queues", managementPageRequest{ + Page: 1, PageSize: 2, Search: "a", Sort: "messages_ready", SortReverse: true, + }) + if err != nil { + t.Fatal(err) + } + if page.TotalCount != 3 || !page.HasMore || len(page.Items) != 2 { + t.Fatalf("unexpected page %#v", page) + } + first, _ := page.Items[0].(map[string]any) + second, _ := page.Items[1].(map[string]any) + if stringOrEmpty(jsonObject(first), "name") != "beta" || stringOrEmpty(jsonObject(second), "name") != "gamma" { + t.Fatalf("unexpected sorted items %#v", page.Items) + } +} + +func TestManagementPageRequestValidation(t *testing.T) { + for _, params := range []jsonObject{ + {"page": 0, "page_size": 100, "sort": "name"}, + {"page": 1, "page_size": managementMaxPageSize + 1, "sort": "name"}, + {"page": 1}, + {"page": 1, "page_size": 100, "sort": "unsupported"}, + } { + if _, err := managementPageRequestFromParams(params, "name"); err == nil { + t.Fatalf("expected invalid page request %#v to fail", params) + } + } + if page, err := managementPageRequestFromParams(jsonObject{}, "name"); err != nil || page != nil { + t.Fatalf("legacy unpaged request changed: %#v, %v", page, err) + } +} + +func TestListTopicsPagePreservesAllVhostIdentityWithoutDrainingPages(t *testing.T) { + requestCount := 0 + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + requestCount++ + if request.URL.Path != "/api/queues" { + t.Fatalf("unexpected all-vhosts path %q", request.URL.Path) + } + writer.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(writer, `{"items":[{"name":"orders","vhost":"staging","durable":true}],"page":1,"page_size":100,"page_count":100,"total_count":10000,"filtered_count":10000}`) + })) + defer server.Close() + service := newServer() + service.cachedConnection = jsonObject{"management_url": server.URL} + + listed, err := service.listTopics(jsonObject{ + "all_vhosts": true, "page": 1, "page_size": 100, "sort": "name", + }) + if err != nil { + t.Fatal(err) + } + result := listed.(jsonObject) + topics := result["topics"].([]jsonObject) + if requestCount != 1 || len(topics) != 1 || stringOrEmpty(topics[0], "vhost") != "staging" { + t.Fatalf("unexpected paged topics %#v after %d requests", result, requestCount) + } + if intOrDefault(result, "totalCount", 0) != 10000 || !boolOrDefault(result, "hasMore", false) { + t.Fatalf("unexpected page metadata %#v", result) + } +} + +func TestListExchangesPageSearchIncludesDisplayedDefaultExchange(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + query := request.URL.Query() + if query.Get("use_regex") != "true" || !strings.Contains(query.Get("name"), "^$") { + t.Fatalf("default exchange search was not translated to a regex: %q", request.URL.RawQuery) + } + writer.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(writer, `{"items":[{"name":"","type":"direct","durable":true}],"page":1,"page_size":100,"page_count":1,"total_count":8,"filtered_count":1}`) + })) + defer server.Close() + service := newServer() + service.cachedConnection = jsonObject{"management_url": server.URL} + + listed, err := service.listExchanges(jsonObject{ + "virtual_host": "/", "page": 1, "page_size": 100, "search": "default", "sort": "name", + }) + if err != nil { + t.Fatal(err) + } + result := listed.(jsonObject) + exchanges := result["exchanges"].([]jsonObject) + if len(exchanges) != 1 || stringOrEmpty(exchanges[0], "name") != "" || intOrDefault(result, "totalCount", 0) != 1 { + t.Fatalf("unexpected exchange page %#v", result) + } +} + func TestManagementRequestFailsOverConnectionErrors(t *testing.T) { listener, err := net.Listen("tcp", "127.0.0.2:0") if err != nil { diff --git a/agents/drivers/rabbitmq/operations.go b/agents/drivers/rabbitmq/operations.go index fdcdf75482..87205d5cd5 100644 --- a/agents/drivers/rabbitmq/operations.go +++ b/agents/drivers/rabbitmq/operations.go @@ -8,6 +8,7 @@ import ( "fmt" "net/http" "os" + "regexp" "sort" "strings" "unicode/utf8" @@ -30,7 +31,22 @@ func (s *server) listTopics(params jsonObject) (any, error) { return nil, err } allVhosts := allVhostsRequested(params) - queues, err := managementGetAll(connection, managementListPath(params, connection, "queues")) + pageRequest, err := managementPageRequestFromParams(params, "name", "messages_ready") + if err != nil { + return nil, err + } + var queues []any + var pageResult *managementPageResult + if pageRequest == nil { + queues, err = managementGetAll(connection, managementListPath(params, connection, "queues")) + } else { + page, pageErr := managementGetPage(connection, managementListPath(params, connection, "queues"), *pageRequest) + if pageErr != nil { + return nil, pageErr + } + queues = page.Items + pageResult = &page + } if err != nil { return nil, err } @@ -46,10 +62,12 @@ func (s *server) listTopics(params jsonObject) (any, error) { } topics = append(topics, info) } - sort.SliceStable(topics, func(left, right int) bool { - return stringOrEmpty(topics[left], "name") < stringOrEmpty(topics[right], "name") - }) - return jsonObject{"topics": topics}, nil + if pageRequest == nil { + sort.SliceStable(topics, func(left, right int) bool { + return stringOrEmpty(topics[left], "name") < stringOrEmpty(topics[right], "name") + }) + } + return managementListResult("topics", topics, pageResult), nil } func topicInfoFromJSON(queue jsonObject) jsonObject { @@ -382,7 +400,29 @@ func (s *server) listExchanges(params jsonObject) (any, error) { return nil, err } allVhosts := allVhostsRequested(params) - exchanges, err := managementGetAll(connection, managementListPath(params, connection, "exchanges")) + pageRequest, err := managementPageRequestFromParams(params, "name") + if err != nil { + return nil, err + } + if pageRequest != nil && pageRequest.Search != "" && strings.Contains(strings.ToLower("(AMQP default)"), strings.ToLower(pageRequest.Search)) { + // The UI displays the empty-name default exchange as "(AMQP default)". + // Include it in the broker-side name search whenever that display label + // matches, while retaining ordinary substring matches. + pageRequest.Search = "(" + regexp.QuoteMeta(pageRequest.Search) + "|^$)" + pageRequest.UseRegex = true + } + var exchanges []any + var pageResult *managementPageResult + if pageRequest == nil { + exchanges, err = managementGetAll(connection, managementListPath(params, connection, "exchanges")) + } else { + page, pageErr := managementGetPage(connection, managementListPath(params, connection, "exchanges"), *pageRequest) + if pageErr != nil { + return nil, pageErr + } + exchanges = page.Items + pageResult = &page + } if err != nil { return nil, err } @@ -398,10 +438,24 @@ func (s *server) listExchanges(params jsonObject) (any, error) { } result = append(result, info) } - sort.SliceStable(result, func(left, right int) bool { - return stringOrEmpty(result[left], "name") < stringOrEmpty(result[right], "name") - }) - return jsonObject{"exchanges": result}, nil + if pageRequest == nil { + sort.SliceStable(result, func(left, right int) bool { + return stringOrEmpty(result[left], "name") < stringOrEmpty(result[right], "name") + }) + } + return managementListResult("exchanges", result, pageResult), nil +} + +func managementListResult(key string, items []jsonObject, page *managementPageResult) jsonObject { + result := jsonObject{key: items} + if page == nil { + return result + } + result["page"] = page.Page + result["pageSize"] = page.PageSize + result["totalCount"] = page.TotalCount + result["hasMore"] = page.HasMore + return result } func exchangeInfoFromJSON(exchange jsonObject) jsonObject { diff --git a/agents/versions.json b/agents/versions.json index 32a50eb101..6f7ec6bd56 100644 --- a/agents/versions.json +++ b/agents/versions.json @@ -3,7 +3,7 @@ "bigquery": "0.1.69", "cache": "0.1.13", "cassandra": "0.1.50", - "dameng": "0.1.78", + "dameng": "0.1.79", "databend": "0.1.56", "databricks": "0.1.63", "db2": "0.1.69", @@ -15,7 +15,7 @@ "goldendb": "0.1.68", "h2": "0.1.69", "h2-legacy": "0.1.38", - "hive": "0.1.55", + "hive": "0.1.56", "argo": "0.1.0", "ignite": "0.1.21", "ignite3": "0.1.21", @@ -43,12 +43,12 @@ "oscar": "0.1.43", "xugu": "0.1.47", "iris": "0.1.62", - "iotdb": "0.1.42", + "iotdb": "0.1.43", "etcd": "0.1.43", "etcd2": "0.1.1", "zookeeper": "0.1.31", - "kafka": "0.1.15", + "kafka": "0.1.16", "rocketmq": "0.1.13", - "rabbitmq": "0.1.11", - "sqlserver-legacy": "0.1.41" + "rabbitmq": "0.1.12", + "sqlserver-legacy": "0.1.42" } diff --git a/apps/desktop/src/App.vue b/apps/desktop/src/App.vue index 85cbcf7dc3..c44ead14e2 100644 --- a/apps/desktop/src/App.vue +++ b/apps/desktop/src/App.vue @@ -93,7 +93,7 @@ import { uuid } from "@/lib/common/utils"; import { isMacOS, isWindows } from "@/lib/backend/platform"; import { isTauriRuntime } from "@/lib/backend/tauriRuntime"; import { openQueryResultArchiveFile } from "@/lib/query/queryResultArchiveFile"; -import { rememberExternalSqlFileTarget, resolveExternalSqlFileTarget, unassociatedExternalSqlFileTarget } from "@/lib/sql/externalSqlFileTarget"; +import { activeTabExternalSqlFileTarget, rememberExternalSqlFileTarget, resolveExternalSqlFileTarget, resolveExternalSqlFileTargetForActiveTab, unassociatedExternalSqlFileTarget, type ExternalSqlFileTarget } from "@/lib/sql/externalSqlFileTarget"; import { externalSqlFileOpenErrorMessage, externalSqlEditorMaxBytes, isSqlFilePath, readBrowserSqlFile, sqlFileTitleFromPath } from "@/lib/sql/sqlFileOpen"; import type { ConnectionConfig, DatabaseType, ObjectBrowserFilter, ObjectSourceKind, QueryTab, TabOutputView, TreeNode } from "@/types/database"; import type { PluginCenterFocus } from "@/lib/plugins/pluginCenterNavigation"; @@ -2435,8 +2435,7 @@ async function saveActiveSqlAsLocalFile() { if (tab) await saveExternalSqlTabAs(tab); } -function applyExternalSqlFileTarget(tab: QueryTab, path: string) { - const target = resolveExternalSqlFileTarget(path, (savedConnectionId) => !!connectionStore.getConfig(savedConnectionId), unassociatedExternalSqlFileTarget()); +function applyExternalSqlTarget(tab: QueryTab, target: ExternalSqlFileTarget) { if (target.connectionId !== tab.connectionId) { queryStore.updateConnection(tab.id, target.connectionId, target.database); } @@ -2449,6 +2448,13 @@ function applyExternalSqlFileTarget(tab: QueryTab, path: string) { if (target.schema !== tab.schema) queryStore.updateSchema(tab.id, target.schema); } +function applyExternalSqlFileTarget(tab: QueryTab, path: string) { + applyExternalSqlTarget( + tab, + resolveExternalSqlFileTargetForActiveTab(path, queryStore.tabs, tab.id, (connectionId) => connectionStore.getConfig(connectionId)), + ); +} + async function openSqlFile() { const tab = activeTab.value; if (!tab) return; @@ -2477,6 +2483,10 @@ async function openSqlFile() { if (!file) return; try { queryStore.updateSql(tab.id, await readBrowserSqlFile(file, externalSqlEditorMaxBytes(settingsStore.editorSettings.externalSqlEditorMaxMb))); + applyExternalSqlTarget( + tab, + activeTabExternalSqlFileTarget(queryStore.tabs, tab.id, (connectionId) => connectionStore.getConfig(connectionId)), + ); } catch (e: any) { toast(t("toolbar.sqlOpenFailed", { message: externalSqlFileOpenErrorMessage(e, (key, params) => t(key, params)) }), 5000); } @@ -2520,6 +2530,8 @@ async function openSqlFilePath(path: string) { try { await desktopOpenTabsRestorationBarrier?.settled; const snapshot = await api.readExternalSqlFileSnapshot(path, externalSqlEditorMaxBytes(settingsStore.editorSettings.externalSqlEditorMaxMb)); + // Startup and OS-open events have no initiating editor tab. Keep new files + // unassociated even if persisted tab restoration has since selected one. const target = resolveExternalSqlFileTarget(path, (savedConnectionId) => !!connectionStore.getConfig(savedConnectionId), unassociatedExternalSqlFileTarget()); queryStore.openExternalSqlFile(target.connectionId, target.database, path, snapshot.content, snapshot.version, target.catalog, target.schema); } catch (e: any) { @@ -3290,7 +3302,7 @@ async function handleQuickOpenSelect(item: any) { if (item.type === "content_match" && item.filePath) { try { const snapshot = await api.readExternalSqlFileSnapshot(item.filePath, externalSqlEditorMaxBytes(settingsStore.editorSettings.externalSqlEditorMaxMb)); - const target = resolveExternalSqlFileTarget(item.filePath, (savedConnectionId) => !!connectionStore.getConfig(savedConnectionId), unassociatedExternalSqlFileTarget()); + const target = resolveExternalSqlFileTargetForActiveTab(item.filePath, queryStore.tabs, queryStore.activeTabId, (connectionId) => connectionStore.getConfig(connectionId)); queryStore.openExternalSqlFile(target.connectionId, target.database, item.filePath, snapshot.content, snapshot.version, target.catalog, target.schema, { line: item.line ?? 1, column: item.column }); } catch (e: any) { toast( @@ -3304,7 +3316,7 @@ async function handleQuickOpenSelect(item: any) { if (item.type === "sql_file" && item.filePath) { try { const snapshot = await api.readExternalSqlFileSnapshot(item.filePath, externalSqlEditorMaxBytes(settingsStore.editorSettings.externalSqlEditorMaxMb)); - const target = resolveExternalSqlFileTarget(item.filePath, (savedConnectionId) => !!connectionStore.getConfig(savedConnectionId), unassociatedExternalSqlFileTarget()); + const target = resolveExternalSqlFileTargetForActiveTab(item.filePath, queryStore.tabs, queryStore.activeTabId, (connectionId) => connectionStore.getConfig(connectionId)); queryStore.openExternalSqlFile(target.connectionId, target.database, item.filePath, snapshot.content, snapshot.version, target.catalog, target.schema); } catch (e: any) { toast( @@ -4198,6 +4210,8 @@ onUnmounted(() => { @open-sql-file="dialogs.showSqlFileDialog.value = true" @open-schema-diff="dialogs.showSchemaDiffDialog.value = true" @open-data-compare="dialogs.showDataCompareDialog.value = true" + @open-backups="openSettings('backups')" + @open-mcp-settings="openSettings('mcp')" />
diff --git a/apps/desktop/src/components/config/DriverStoreDialog.vue b/apps/desktop/src/components/config/DriverStoreDialog.vue index 1e0b429b67..89a5f63084 100644 --- a/apps/desktop/src/components/config/DriverStoreDialog.vue +++ b/apps/desktop/src/components/config/DriverStoreDialog.vue @@ -2231,8 +2231,10 @@ html.dark .driver-store-agent-row--installed::before { overflow-x: hidden; } -/* Storage tab: cards scroll together inside the tab */ -.driver-store-storage-tab { +/* Storage tab: cards scroll together inside the tab. The selector must out-specify + `.driver-store-tabs-root > [data-slot="tabs-content"]` above, otherwise the + `overflow: hidden` there wins and the cards get clipped with no way to scroll. */ +.driver-store-tabs-root > [data-slot="tabs-content"].driver-store-storage-tab { display: flex !important; flex-direction: column; flex: 1 1 0; @@ -2241,6 +2243,12 @@ html.dark .driver-store-agent-row--installed::before { overflow-x: hidden; } +/* Cards must keep their natural height, otherwise `flex` squeezes the runtime + list inside the last card instead of letting the tab scroll. */ +.driver-store-tabs-root > [data-slot="tabs-content"].driver-store-storage-tab > * { + flex-shrink: 0; +} + .driver-store-agent-name, .driver-store-jdbc-name { flex: 1 1 auto !important; diff --git a/apps/desktop/src/components/connection/ConnectionDialog.vue b/apps/desktop/src/components/connection/ConnectionDialog.vue index d3a5c68764..8c7ed5872d 100644 --- a/apps/desktop/src/components/connection/ConnectionDialog.vue +++ b/apps/desktop/src/components/connection/ConnectionDialog.vue @@ -148,7 +148,7 @@ import { } from "@lucide/vue"; import { buildDraftVisibleDatabasesConnectionId, connectionCanChooseVisibleDatabases, initialVisibleDatabaseSelection, visibleObjectFiltersNeedReset } from "@/lib/connection/connectionVisibleDatabases"; import { canSaveVisibleDatabaseSelection, connectionUsesVisibleSchemaFilter, filterDatabaseNamesForVisiblePicker, filterSchemaNamesForVisiblePicker, normalizeVisibleDatabaseSelection, buildDraftVisibleSchemasConnectionId, normalizeVisibleSchemaSelection } from "@/lib/database/visibleDatabases"; -import { isSchemaAware, isSingleDatabase } from "@/lib/database/databaseFeatureSupport"; +import { isSchemaAware, isSingleDatabase, supportsDataDictionary } from "@/lib/database/databaseFeatureSupport"; import { normalizeConnectionScope, normalizeConnectionTimeouts } from "@/lib/connection/connectionSubmitNormalization"; import { databaseConnectionFormKind } from "@/lib/database/databaseDriverManifest"; import VisibleSchemasDialog from "@/components/sidebar/VisibleSchemasDialog.vue"; @@ -231,6 +231,7 @@ const DREMIO_ARROW_FLIGHT_SQL_JDBC_DRIVER_CLASS = "org.apache.arrow.driver.jdbc. const DREMIO_LEGACY_JDBC_URL = "jdbc:dremio:direct=127.0.0.1:31010"; const DREMIO_LEGACY_JDBC_DRIVER_CLASS = "com.dremio.jdbc.Driver"; const DEFAULT_SSH_USER = "root"; +const DIRECT_SIDEBAR_OBJECT_TYPES = new Set(["redis", "etcd", "zookeeper", "consul", "mongodb", "dynamodb", "elasticsearch", "easysearch", "meilisearch", "solr", "milvus", "qdrant", "weaviate", "chromadb", "mq", "mqtt", "nacos", "plugin"]); const ETCD_GRPC_MAX_INBOUND_DEFAULT_MIB = 32; const ETCD_GRPC_MAX_INBOUND_MIN_MIB = 1; const ETCD_GRPC_MAX_INBOUND_MAX_MIB = 256; @@ -441,6 +442,7 @@ const defaultForm = (): ConnectionForm => ({ docs_notes_path: undefined, read_only: false, show_system_schemas: false, + sidebar_auto_load_all_tables: false, is_production: false, production_databases: [], visible_databases: undefined, @@ -632,6 +634,7 @@ function sshLayersForConfig(config: LegacyConnectionConfig): SshTunnelConfig[] { } const form = ref(defaultForm()); +const supportsAutomaticTableLoading = computed(() => supportsDataDictionary(form.value.db_type) && !DIRECT_SIDEBAR_OBJECT_TYPES.has(form.value.db_type)); const redisKeyTemplatesText = ref(""); const noteTextareaRef = ref(null); const showGaussdbConnectionMode = computed(() => form.value.db_type === "gaussdb"); @@ -3110,6 +3113,7 @@ watch( docs_notes_path: config.docs_notes_path, read_only: config.read_only || false, show_system_schemas: config.show_system_schemas || false, + sidebar_auto_load_all_tables: config.sidebar_auto_load_all_tables === true, is_production: config.is_production || false, production_databases: config.production_databases || [], visible_databases: config.visible_databases, @@ -5141,6 +5145,7 @@ function connectionConfigForSubmit(id: string, generatedName = "", validatePlugi config.visible_databases = Array.isArray(config.visible_databases) && config.visible_databases.length > 0 ? config.visible_databases : undefined; } if (!config.show_system_schemas) config.show_system_schemas = undefined; + if (!config.sidebar_auto_load_all_tables) config.sidebar_auto_load_all_tables = undefined; if (config.visible_schemas && Object.keys(config.visible_schemas).length === 0) config.visible_schemas = undefined; if (config.agent_java_options && config.agent_java_options.length === 0) config.agent_java_options = undefined; // Pasted credentials may carry invisible characters that trim() keeps (#9043). @@ -9878,6 +9883,16 @@ function openExternalUrl(url: string) { {{ t("connection.showSystemSchemasHint") }}
+
+ +
+ +

{{ t("connection.autoLoadAllTablesHint") }}

+
+
diff --git a/apps/desktop/src/components/connection/__tests__/ConnectionDialog.deepLinkEdit.spec.ts b/apps/desktop/src/components/connection/__tests__/ConnectionDialog.deepLinkEdit.spec.ts index 3b77a65f8b..28153a5f18 100644 --- a/apps/desktop/src/components/connection/__tests__/ConnectionDialog.deepLinkEdit.spec.ts +++ b/apps/desktop/src/components/connection/__tests__/ConnectionDialog.deepLinkEdit.spec.ts @@ -234,7 +234,7 @@ describe("ConnectionDialog deep-link edit confirmation", () => { }); it("updates the existing ID only after Save and preserves omitted fields", async () => { - await mountDialog(savedConnection(), "dbx://connection/new?id=saved-production&password=new-password"); + await mountDialog(savedConnection({ sidebar_auto_load_all_tables: true }), "dbx://connection/new?id=saved-production&password=new-password"); const save = [...document.querySelectorAll("button")].find((button) => button.textContent?.trim() === "Save"); expect(save).toBeTruthy(); save!.click(); @@ -255,6 +255,7 @@ describe("ConnectionDialog deep-link edit confirmation", () => { is_production: true, production_databases: ["orders"], visible_databases: ["orders"], + sidebar_auto_load_all_tables: true, url_params: "connectTimeout=25", }), ); diff --git a/apps/desktop/src/components/diff/SchemaDiffDialog.vue b/apps/desktop/src/components/diff/SchemaDiffDialog.vue index ffa3c21772..3c0b4db928 100644 --- a/apps/desktop/src/components/diff/SchemaDiffDialog.vue +++ b/apps/desktop/src/components/diff/SchemaDiffDialog.vue @@ -121,7 +121,7 @@ watch([sourceDbType, targetDbType], ([src, tgt]) => { }); const optionTree = computed(() => { const targetConfig = store.getConfig(targetConnectionId.value); - const dbType = targetConfig?.db_type || "postgres"; + const dbType = schemaDiffEngineDatabaseType(targetConfig) || targetConfig?.db_type || "postgres"; return getSchemaDiffOptionsForDbType(dbType); }); @@ -1647,12 +1647,14 @@ const targetConnectionInfo = computed(() => {
-
-
+
+

{{ t("schemaDiff.optionsTitle") }}

- +
+ +
diff --git a/apps/desktop/src/components/diff/SchemaDiffOptionsPanel.vue b/apps/desktop/src/components/diff/SchemaDiffOptionsPanel.vue index 1230d20f21..3510c12191 100644 --- a/apps/desktop/src/components/diff/SchemaDiffOptionsPanel.vue +++ b/apps/desktop/src/components/diff/SchemaDiffOptionsPanel.vue @@ -1,5 +1,6 @@