Compare commits

...
Author SHA1 Message Date
github-actions[bot] 8f54385b97 chore(packages): release 0.4.75 [skip node-packages-release] 2026-08-28 10:27:48 +00:00
github-actions[bot] cb9f87031f chore(jdbc): bump plugin version [skip ci] 2026-08-28 09:35:31 +00:00
t8y2 5c738f5bf2 chore(release): prepare v0.5.98 2026-08-28 17:34:50 +08:00
二丫讲梵 038ff4e773 fix(editor): align wrapped line numbers 2026-08-28 17:19:40 +08:00
t8y2 9c56de2e9f chore(query): gate test-only execution helper 2026-08-28 09:17:46 +00:00
zipg ef87e45cbc fix(windows): recover hidden startup window 2026-08-28 17:16:55 +08:00
t8y2 f83f707fbc fix(kingbase): honor driver identifier quote for schema context
Closes #7505
2026-08-28 09:13:09 +00:00
zipg 7a4edb6c19 fix(postgres): preserve configured search path 2026-08-28 16:55:04 +08:00
二丫讲梵 b4787a63af feat(editor): configure multi-statement result view 2026-08-28 16:52:02 +08:00
Guoyu Su fd17037a2a fix(linux): bundle AppImage input method modules 2026-08-28 16:51:33 +08:00
二丫讲梵 41db375ee6 style(editor): align line number gutter items 2026-08-28 16:51:03 +08:00
t8y2 1498b27314 fix(query): avoid IRIS derived count aliases
Closes #7486
2026-08-28 08:25:31 +00:00
jearhe 6d8e667bed feat(structure): support search in DDL view 2026-08-28 15:30:08 +08:00
StaravenX df88d7baa7 fix(jdbc): fall back to DatabaseMetaData when oracle dictionary views are missing 2026-08-28 15:29:06 +08:00
t8y2 659fb7f330 feat(mcp): expose message queue send tool
Closes #7236
2026-08-28 07:27:51 +00:00
二丫讲梵 c237ecbfca fix(nacos): allow manual scope when namespace directory is missing 2026-08-28 15:10:25 +08:00
t8y2 bcc64274e8 feat(ci): sign arm64 Windows builds and NSIS uninstallers 2026-08-28 15:05:45 +08:00
Mo-Moore 802019daa3 feat(editor): execute custom sql template by shortcut on selection 2026-08-28 14:27:55 +08:00
jearhe 60bfb41f13 feat(editor): add clear-all action for sql parameter values 2026-08-28 14:25:42 +08:00
t8y2 17e2573fbe fix(kafka): fall back to topic names when metadata times out 2026-08-28 06:11:26 +00:00
t8y2 2467d212d5 fix(dameng): normalize legacy DBX connection URLs
Closes #7482
2026-08-28 14:07:10 +08:00
t8y2 80224ab1d9 fix(oracle): support 11g data compare pagination
Closes #7470
2026-08-28 05:42:17 +00:00
t8y2 89086cc22b fix(schema): list HighGo PostgreSQL extensions
Closes #7448
2026-08-28 05:02:43 +00:00
Abeautifulsnow fdfad88590 feat(editor): add global default transaction commit mode 2026-08-28 12:37:11 +08:00
李子凡 d3a97164aa fix(sql): preserve empty raw placeholders 2026-08-28 12:36:37 +08:00
t8y2 d4245f0d58 fix(dameng): open java.net module for multi-NIC hosts 2026-08-28 12:36:00 +08:00
李子凡 2c5f926bd8 fix(grid): pad short timestamp fractions 2026-08-28 12:35:47 +08:00
amwps290 20a2994fa1 feat(structure): show table constraints for postgres-compatible dbs 2026-08-28 12:14:04 +08:00
t8y2 7ad05b178f fix(mysql): support DRDS index expression fallback
Closes #7363
2026-08-28 03:42:29 +00:00
t8y2 be307ba2d7 fix(query): collapse duplicated external-driver query branch 2026-08-28 11:41:24 +08:00
zipg 10bce698ae fix(mcp): install rustls crypto provider for TLS connections 2026-08-28 11:13:14 +08:00
Diego Fabricio c0abc26fe8 fix(grid): allow rerunning exact total row count from status 2026-08-28 11:12:41 +08:00
zipg 425ca055ea fix(mysql): keep pool when health probe times out during export 2026-08-28 11:12:06 +08:00
dadong 99a9961712 fix(export): harden config export passphrase handling 2026-08-28 11:11:35 +08:00
zipg d8ec9bb8d0 feat(connection): choose group when creating a connection 2026-08-28 11:11:00 +08:00
t8y2 4636788f49 fix(export): fall back when Doris rejects read-only snapshot
Closes #7446
2026-08-28 02:36:06 +00:00
t8y2 22305b7d0f fix(jdbc): retry invalid UTF-8 table previews without left wrappers
Closes #7268
2026-08-28 02:05:30 +00:00
zipg 71d07348ad perf(oracle): stop slow index metadata from blocking first query 2026-08-27 22:25:39 +08:00
Melody Ruch b198dd1327 fix(postgres): affected rows for WITH-wrapped DML and CTE completion 2026-08-27 22:25:09 +08:00
Melody Ruch 9290391985 fix(structure): emit NULL keyword for nullable MySQL timestamp columns 2026-08-27 22:24:40 +08:00
Melody Ruch 1016ccc352 fix(backup): match db.schema.table excludes when database equals schema 2026-08-27 22:24:09 +08:00
Abeautifulsnow ec0b6768be feat(grid): add row/column crosshair highlight setting 2026-08-27 22:23:13 +08:00
t8y2 b09c6f4f21 fix(export): stabilize Kingbase large result pagination
Closes #7377
2026-08-27 19:08:05 +08:00
t8y2 74ffab29ca fix(transfer): enable identity insert for SQL Server
Closes #7335
2026-08-27 18:52:15 +08:00
t8y2 e996f8aefb fix(transfer): use rownum pagination for OceanBase Oracle
Closes #7209
2026-08-27 18:47:57 +08:00
zipg 4fed29fba2 fix(structure): improve mysql auto-increment editing and field shortcuts 2026-08-27 18:21:52 +08:00
二丫讲梵 214bf86a6f feat(desktop): add zen mode toggle for data tabs 2026-08-27 17:59:10 +08:00
Farahani 955558c81f feat(safety): temporary write-unlock window for read-only connections 2026-08-27 17:37:54 +08:00
Mo-Moore 7933b7656f feat(jdbc,desktop): browse Hive/Inceptor routines from system views 2026-08-27 17:32:56 +08:00
t8y2 e5572ead7a fix(grid): parse pg timestamps deterministically for local date sort
Closes #7281
2026-08-27 17:32:09 +08:00
t8y2 bd6c780aba fix(core): uppercase dameng column types in structure ddl
Closes #7343
2026-08-27 17:23:40 +08:00
dadong dfd4d3b7ff feat(import): add SQL script import with dialect-aware parsing 2026-08-27 17:17:41 +08:00
Yang Fei ab4e747bf9 feat(editor): drag result grid columns into the SQL editor 2026-08-27 17:16:57 +08:00
Mo-Moore 9e4fe8ce56 fix(jdbc): connect Hive2 JDBC through registered driver 2026-08-27 17:16:32 +08:00
t8y2 6a6e5fbad9 fix(core): use row_number pagination for sqlserver table transfers
Closes #7356
2026-08-27 17:10:10 +08:00
t8y2 e8d6370b1e fix(grid): reset scroll to top on external result replacement
Results replaced without an internal grid action (SQL re-executed from
the editor or a grid-embedded run button) kept the previous viewport, so
DOM-mode grids stayed stranded at the old scroll offset after running an
ORDER BY query. The result watch now resets to the first row unless the
replacement is an in-place refresh (prepareFullReload markers) or an
infinite-scroll append completion. A remounted grid also no longer
adopts a scroll-only pending-changes snapshot; cached scroll is only
replayed alongside unsaved edit state, while the KeepAlive reactivate
path still restores pure scroll.

Closes #7341
2026-08-27 16:33:23 +08:00
t8y2 4eeeafa858 fix(sidebar): expand collapsed groups when locating tab
Closes #7387
2026-08-27 16:33:23 +08:00
t8y2 e4dc5bace8 fix(core): use drop user cascade for oceanbase oracle schema drop
Closes #7351
2026-08-27 16:33:23 +08:00
zipg 920ab62cf1 feat(grid): support deleting rows from JOIN query results 2026-08-27 16:28:14 +08:00
t8y2 36a7b7bd3a feat(ai): localize custom provider label 2026-08-27 16:23:51 +08:00
t8y2 565feb8745 feat(ci): cache native c deps through sccache 2026-08-27 16:14:42 +08:00
Abeautifulsnow 40549c9573 fix(dialog): clamp grid track and wrap footer so dialogs stop clipping 2026-08-27 16:06:25 +08:00
zipg eb0994e980 feat(grid): support composite unique reference keys 2026-08-27 16:00:00 +08:00
t8y2 cf03de122a fix(ci): keep sccache server alive for post-build stats 2026-08-27 15:56:48 +08:00
t8y2 d9beaaaa1c fix(agents): raise gradle daemon heap for parallel builds 2026-08-27 14:38:53 +08:00
github-actions[bot] 5cca5e0b86 chore(packages): release 0.4.74 [skip node-packages-release] 2026-08-27 06:25:47 +00:00
github-actions[bot] 6e644c0eea chore: bump module versions [skip ci] 2026-08-27 06:07:01 +00:00
t8y2 f3effbc76b fix(release): bundle sqlite in static browser builds 2026-08-27 13:25:41 +08:00
github-actions[bot] 81172688b8 chore(jdbc): bump plugin version [skip ci] 2026-08-27 04:44:08 +00:00
t8y2 b5bac9cafc chore(release): prepare v0.5.97 2026-08-27 12:43:26 +08:00
hanxuanyu 704f817b4d feat(grid): add snapshots for selected data 2026-08-27 12:21:28 +08:00
t8y2 e6d25b016e fix(jdbc): pair oracle ojdbc maven installs with orai18n
Closes #7261
2026-08-27 12:19:16 +08:00
二丫讲梵 d70a1231b0 feat(xlsx): add optional auto filter to xlsx export 2026-08-27 12:18:42 +08:00
t8y2 185db5e8c6 docs(sponsors): add TrustAsia sponsor to READMEs and site 2026-08-27 12:09:57 +08:00
Abeautifulsnow bd05c0350d feat(oracle): hide commit/rollback for clean manual-transaction state 2026-08-27 11:30:27 +08:00
t8y2 d2326acb81 fix(grid): keep DOM results at the first row 2026-08-27 11:08:42 +08:00
onenewcode 8bc31d3319 fix(sql-library): use delete confirm label in delete dialogs 2026-08-27 10:55:20 +08:00
知食分子 ba8df70fe6 fix(desktop): add legacy webview fallbacks for snapshots and dialogs 2026-08-27 10:55:05 +08:00
t8y2 afcfa59d0b fix(oracle): reload large values through rowid-wrapped select
Refs: #7082
2026-08-27 10:43:54 +08:00
Melody Ruch 4a3d63b09a fix(completion): mask strings and comments in clause detection 2026-08-27 09:21:54 +08:00
t8y2 caaa8932a1 fix(tests): add missing Keyboard icon to structure editor mocks 2026-08-27 09:11:37 +08:00
jearhe f41b5dbdf6 fix(transfer): preserve PostgreSQL indexes in structure-only 2026-08-27 08:40:27 +08:00
zipg e7191b1c4b fix(completion): add datepart args for sqlserver date functions 2026-08-27 08:40:12 +08:00
zipg 2138f59da4 fix(compare): keep oracle prefill selections during async loads 2026-08-27 08:39:45 +08:00
zipg 2711646d48 fix(datagen): stop column defaults overriding generation rules 2026-08-27 08:39:30 +08:00
二丫讲梵 3d77966f52 feat(structure-editor): add quick field add and duplicate shortcuts 2026-08-27 08:39:15 +08:00
二丫讲梵 62f2185f44 feat(sql-formatter): add setting to preserve blank lines 2026-08-27 08:38:50 +08:00
Melody Ruch 699043ff07 fix(import): add default length to bare varchar on mysql targets 2026-08-27 08:38:35 +08:00
jearhe 6875ca4000 fix(schema-diff): refresh table lists before comparing 2026-08-27 08:38:20 +08:00
t8y2 168e7bcb54 fix(docs): sync lightweight dev command with sqlite-bundled feature 2026-08-26 22:09:26 +08:00
Melody Ruch ff88516235 fix(elasticsearch): clamp page size to max_result_window 2026-08-26 22:07:34 +08:00
二丫讲梵 aac09d3cb7 fix(grid): keep canvas hover state across drawing surface flips 2026-08-26 22:04:42 +08:00
二丫讲梵 174f786483 build(sqlite): enable bundled sqlite in fast dev and check targets 2026-08-26 22:03:52 +08:00
Zzz 4b454b72b9 fix(http-tunnel): reduce round-trip latency under active traffic 2026-08-26 22:00:04 +08:00
t8y2 ac2af84e7b fix(desktop): reformat redis key expansion spec for CI 2026-08-26 21:56:55 +08:00
t8y2 8333c78320 fix(docs): keep pnpm lockfile patchedDependencies format CI expects 2026-08-26 21:44:10 +08:00
t8y2 3a0e86956c feat(docs): redesign changelog page and add route progress bar 2026-08-26 21:38:27 +08:00
GIWTO 96d2f0dd1c feat(transfer): swap source and target connections 2026-08-26 21:38:10 +08:00
StaravenX bf09181ba5 fix(jdbc): fall back to jdbc schemas without all_users 2026-08-26 21:37:29 +08:00
zipg e93b41a4ff feat(editor): preselect the first completion candidate 2026-08-26 21:31:54 +08:00
zipg 589fbd9de8 feat(ui): streamline sidebar and settings layout 2026-08-26 21:31:26 +08:00
zipg cb4cc43685 fix(build): resolve windows desktop sqlite link failure 2026-08-26 21:07:45 +08:00
t8y2 8382d85e19 fix(schema-diff): keep both-side tables out of delete group
Closes #7225
2026-08-26 20:52:38 +08:00
t8y2 10df1b07d1 fix(redis): keep key tree groups expanded across refresh
Closes #7173
2026-08-26 20:52:37 +08:00
t8y2 7231c41618 fix(settings): make live preview read-only and hint intentional typo
Closes #7233
2026-08-26 20:52:37 +08:00
t8y2 2e739d5902 fix: initialize visible_database_patterns in remaining initializers 2026-08-26 20:41:31 +08:00
t8y2 c00003e4d7 fix(mq): initialize visible_database_patterns in test configs 2026-08-26 20:10:10 +08:00
Melody Ruch a33f7d3c56 fix(diff): keep self-referencing foreign keys inside the target database 2026-08-26 19:26:39 +08:00
t8y2 0739048425 fix(readme): avoid duplicate sponsor logos 2026-08-26 18:52:46 +08:00
t8y2 d06f62f9f4 feat(sidebar): wildcard patterns for visible databases
Closes #7164
2026-08-26 18:43:31 +08:00
t8y2 71344c637c fix(grid): hint truncated values in grid search
Closes #7279
2026-08-26 18:43:21 +08:00
t8y2 138e48d347 fix(export): clamp diagram png scale to canvas limits
Closes #7248
2026-08-26 18:43:09 +08:00
t8y2 0588a67449 feat(readme): reorganize sponsor section 2026-08-26 18:33:35 +08:00
二丫讲梵 25ea7b21eb feat(connection): show server versions for more datasources 2026-08-26 18:29:13 +08:00
zipg fb907a5866 fix(oracle): apply default orcl database 2026-08-26 17:47:40 +08:00
jearhe affcb7fa7e perf(transfer): virtualize large object lists in the picker 2026-08-26 17:47:14 +08:00
jearhe d15605cee9 fix(editor): match replace shortcut modifiers strictly 2026-08-26 17:46:50 +08:00
Melody Ruch 95537957cc fix(diff): qualify deploy ddl with the target schema 2026-08-26 17:46:24 +08:00
t8y2 c153ca8f57 feat(grid): add refresh to data grid context menu
Closes #7273
2026-08-26 17:28:39 +08:00
zipg b5124baca9 feat(sqlite): restore rc4 legacy support via optional feature 2026-08-26 17:21:53 +08:00
t8y2 4b4b4ac2dd fix(redis): fill expanded group subtree via targeted scan
Closes #7162
2026-08-26 17:14:12 +08:00
t8y2 83a6ad005b feat(docs): add faq page for data storage and data directory 2026-08-26 17:00:08 +08:00
Melody Ruch b77328f03e fix(jdbc): render large oracle numbers without scientific notation 2026-08-26 16:40:52 +08:00
t8y2 d85816c3ed feat(grid): add begins-with and ends-with filter modes
Closes #7282
2026-08-26 16:40:17 +08:00
jearhe f12ac21453 fix(diff): sync horizontal scroll in schema comparison 2026-08-26 15:50:14 +08:00
Guoyu Su bc18bbfe5c fix(redis): scan cluster masters fairly for matching keys 2026-08-26 15:49:49 +08:00
Melody Ruch 8c735206e0 fix(redis): pair withscores members and scores in the console 2026-08-26 15:31:42 +08:00
t8y2 8475c84d48 fix(duckdb): checkpoint wal on worker shutdown 2026-08-26 15:23:29 +08:00
zipg 6686d1628e fix(completion): resolve ob-oracle schema for alias column completion 2026-08-26 15:07:37 +08:00
Melody Ruch 1e9babca45 fix(editor): skip statement frame for oversized oracle package bodies 2026-08-26 15:07:07 +08:00
onenewcode 0f9d32e1f5 feat(redis): edit hash fields 2026-08-26 14:44:19 +08:00
李达千 75c2dc5e17 fix(mongo): do not force directConnection when loadBalanced 2026-08-26 14:43:52 +08:00
Abeautifulsnow eeeb4e5011 feat(ai): run agent sessions as background tasks 2026-08-26 14:09:08 +08:00
Allen b37a1f8538 feat(grid): color quick condition bar keywords, fields and values 2026-08-26 13:52:41 +08:00
Melody Ruch 4c28492f2c fix(sidebar): refresh tree after dropping the last visible database 2026-08-26 13:52:16 +08:00
Melody Ruch ca0d4356c0 fix(editor): highlight pg and mysql builtin functions 2026-08-26 13:51:51 +08:00
jearhe ef22b84fbe perf(kafka): batch consumer group subscriptions into one agent round trip 2026-08-26 13:51:26 +08:00
KuriyamaMirai a4bf8442d7 fix(sqlserver): complete schema diff DDL support 2026-08-26 13:21:50 +08:00
t8y2 2b7d353313 feat(docs): refine landing page presentation 2026-08-26 13:14:38 +08:00
zipg 5b61f17c49 feat(dameng): support dm6 connections via custom jdbc drivers 2026-08-26 12:43:31 +08:00
zipg c8c9870dba fix(schema): converge mysql structure sync and refresh metadata 2026-08-26 12:30:10 +08:00
zipg 43289e2ce1 fix(oracle): 加速大存储过程源码读取 2026-08-26 11:05:36 +08:00
zipg 60e3d61934 feat(mysql): 支持查看和修改表存储引擎 2026-08-26 11:05:08 +08:00
zipg 2e8673a7ea fix(kafka): 修复跳过证书校验未生效 2026-08-26 11:04:40 +08:00
zipg a6c386a067 fix(editor): 避免惯性滚动误触发字号缩放 2026-08-26 11:04:12 +08:00
zipg 68eae4ad4d feat(gaussdb): 支持带 psql 控制命令的匿名块执行 2026-08-26 11:03:44 +08:00
t8y2 8db66db8ca fix(release): ignore desktop-only Cargo.lock changes 2026-08-26 10:54:31 +08:00
t8y2 f1578a6e9e chore(release): prepare v0.5.96 2026-08-26 10:36:27 +08:00
zipg dc10323b7c fix(grid): restore first double-click cell editing 2026-08-26 10:04:44 +08:00
github-actions[bot] 8bcbfd7ae7 fix(ci): disable sccache for Windows 7 release build 2026-08-26 03:54:19 +08:00
github-actions[bot] 43c896053b chore(packages): release 0.4.73 [skip node-packages-release] 2026-08-25 19:19:09 +00:00
github-actions[bot] 084f872a80 chore: bump module versions [skip ci] 2026-08-25 19:01:19 +00:00
github-actions[bot] 8e47296e02 fix(release): restore Windows signing build 2026-08-26 02:46:11 +08:00
github-actions[bot] 706334434a revert(sqlite): remove RC4 Legacy database support 2026-08-26 02:46:04 +08:00
github-actions[bot] d362c8efb8 chore(release): prepare v0.5.95 2026-08-26 00:42:11 +08:00
zipg 771a7f03a0 fix(editor): restore SQL selection case shortcuts 2026-08-26 00:27:27 +08:00
zipg 2a2357e2f1 feat(sidebar): refine active connection filter indicator 2026-08-26 00:25:10 +08:00
skyler 1c2e5bb07c feat(ci): sign x64 Windows releases with TrustAsia VSign 2026-08-26 00:07:30 +08:00
Guoyu Su a9c6c7b5e4 fix(core): support named-user tilde paths 2026-08-25 23:57:02 +08:00
zipg 1ed20e424e feat(sqlite): support RC4 Legacy databases 2026-08-25 23:37:52 +08:00
jearhe 0dc1f0c14c fix(editor): guard SQL shortcuts during IME composition 2026-08-25 22:51:26 +08:00
Guoyu Su 0658d18922 fix(mysql): prevent stale export pool cleanup 2026-08-25 22:50:40 +08:00
Melody Ruch aa91a0aaa9 fix(duckdb): preserve CURRENT_DATE arithmetic literals 2026-08-25 22:36:40 +08:00
Klinge cb581baf20 fix(grid): avoid canvas resize observer loops 2026-08-25 22:35:39 +08:00
zipg 19f8736b67 fix(editor): improve MySQL routine syntax diagnostics 2026-08-25 22:34:57 +08:00
whoops 5e307f56cc feat(redis): add configurable key search templates 2026-08-25 22:34:04 +08:00
高瑞强 bfd7c7def2 feat(mongo): open object browser from database tree nodes 2026-08-25 22:15:40 +08:00
二丫讲梵 46111850db feat(export): support XLSX comment headers 2026-08-25 22:00:26 +08:00
zipg 86f36e51ad fix(sidebar): improve active connection indicator 2026-08-25 21:54:53 +08:00
zipg c6e9e94c58 fix(editor): restore configurable completion acceptance 2026-08-25 21:52:01 +08:00
jearhe f9c4b19a60 fix(grid): use native context menu in filter inputs 2026-08-25 21:28:55 +08:00
Zzz a66590868e fix(kingbase): resolve schema for unqualified grid edits 2026-08-25 21:26:00 +08:00
github-actions[bot] 7494cf8011 feat(ci): bootstrap VSign operator credentials 2026-08-25 20:29:34 +08:00
github-actions[bot] 18ee40bb5d feat(ci): add VSign operator bootstrap probe 2026-08-25 20:29:34 +08:00
t8y2 ef9d0b8042 fix(export): preserve PostgreSQL bytea values
Closes #7184
2026-08-25 12:24:11 +00:00
t8y2 988fa1aabb fix(postgres): support PostgreSQL 9.2 batched DDL metadata
Closes #7142
2026-08-25 11:49:52 +00:00
github-actions[bot] 9b4f94cbb5 fix(completion): trigger quoted table suggestions 2026-08-25 19:22:29 +08:00
t8y2 151172f81a fix(postgres): qualify view source relation references
Closes #7137
2026-08-25 10:44:25 +00:00
github-actions[bot] 51f7b1d8b5 fix(docs): read generated connection options in content check 2026-08-25 18:17:39 +08:00
github-actions[bot] c27246ca01 fix(ci): restore JDBC files before branch switch 2026-08-25 17:43:33 +08:00
github-actions[bot] 8d2e627efe chore(packages): release 0.4.72 [skip node-packages-release] 2026-08-25 17:31:10 +08:00
github-actions[bot] 5c29af80ec fix(ci): authenticate bot pushes through checkout 2026-08-25 17:28:14 +08:00
github-actions[bot] 013fb011e5 chore: bump module versions [skip ci] 2026-08-25 17:28:13 +08:00
github-actions[bot] 79af004b6e fix(ci): push agent version bump with release token 2026-08-25 17:22:49 +08:00
github-actions[bot] eae1fa0c75 fix(ci): push JDBC bump with release token 2026-08-25 17:14:50 +08:00
github-actions[bot] 33c0cf2050 chore(jdbc): bump plugin version [skip ci] 2026-08-25 17:14:49 +08:00
t8y2 dfbdcfc04f chore(release): prepare v0.5.94 2026-08-25 16:15:51 +08:00
zipg 81c3cc15d2 fix(memory): release closed query and connection resources 2026-08-25 15:47:23 +08:00
t8y2 88e659833d fix(sql): format Oracle queries with PL/SQL dialect
Closes #7138
2026-08-25 07:17:11 +00:00
Elias e9fea304ba fix(xugu): preserve spatial SRIDs in exports 2026-08-25 15:15:41 +08:00
t8y2 343d006300 feat(editor): require explicit SQL completion selection
Closes #1191
2026-08-25 06:49:45 +00:00
zipg 9b8721e0e8 fix(sql): preserve quoted parameter null semantics 2026-08-25 14:46:46 +08:00
lizzzzz777 876f850e2d feat(editor): format SQL files on save 2026-08-25 14:46:25 +08:00
二丫讲梵 249896dddf fix(data-grid): preserve scoped context selections 2026-08-25 14:45:15 +08:00
t8y2 7dfd0fdd5a fix(tests): cover explicit app close prompt mode 2026-08-25 06:31:35 +00:00
Zzz 316a1f2a7c fix(mcp): support pnpm global shim with separate node runtime 2026-08-25 14:26:13 +08:00
zipg b488942d82 feat(connection): preserve failed one-time connections 2026-08-25 14:02:51 +08:00
t8y2 d1d8dc4aec feat(tabs): keep SQL drafts on app exit by default
Closes #7134
2026-08-25 05:45:31 +00:00
t8y2 cf652f212e feat(completion): support OceanBase Oracle alias columns
Closes #7126
2026-08-25 05:33:12 +00:00
t8y2 e977e886c3 fix(query): skip metadata for constant projections
Closes #7111
2026-08-25 05:05:55 +00:00
t8y2 eccbc5a315 fix(editor): support Spark datasource DDL for Kyuubi
Closes #7120
2026-08-25 04:21:42 +00:00
t8y2 4eabeb6a15 fix(settings): allow option card text to wrap
Closes #7121
2026-08-25 04:07:31 +00:00
t8y2 9c90edabe3 fix(completion): replace partially quoted identifiers 2026-08-25 11:57:52 +08:00
t8y2 e7ef9df85c fix(transfer): support default tree-schema databases
Closes #7104
2026-08-25 03:19:26 +00:00
t8y2 7c8e0a88cf fix(grid): refresh saved rows in place 2026-08-25 11:05:38 +08:00
t8y2 08365d73eb fix(ci): sync frontend tests with grid changes 2026-08-25 11:02:27 +08:00
t8y2 4be1928d15 chore(github): stop automatic review requests 2026-08-25 10:55:06 +08:00
t8y2 2cf79b3c9f fix(editor): persist transaction mode changes 2026-08-25 10:49:58 +08:00
t8y2 0aa18257e7 fix(sidebar): restore database-first sticky headers 2026-08-25 10:33:13 +08:00
t8y2 418951c1e2 fix(settings): sync live preview line numbers 2026-08-25 10:31:46 +08:00
t8y2 7686d7dc13 fix(tabs): replace tab icon during query execution 2026-08-25 10:20:47 +08:00
t8y2 a2f8824e6a feat(tabs): show query execution status across tab surfaces
Closes #4986
2026-08-25 01:34:00 +00:00
t8y2 277386ec81 feat(mysql): edit table auto-increment counter
Closes #7065
2026-08-25 01:24:09 +00:00
t8y2 44dc2a88f8 feat(grid): add edit structure shortcut
Closes #4303
2026-08-25 00:41:11 +00:00
t8y2 2f464e0d33 feat(settings): move editor live preview near appearance controls
Closes #4346
2026-08-24 23:59:42 +00:00
t8y2 a6edbd7fcb feat(transfer): make data transfer dialog resizable
Closes #1149
2026-08-24 23:53:13 +00:00
t8y2 efba56c8c0 feat(grid): compare original and edited JSON values
Closes #1638
2026-08-24 23:25:24 +00:00
t8y2 b3e8709272 feat(grid): show average for numeric selections
Closes #4929
2026-08-24 22:53:11 +00:00
t8y2 e2fa810f2e feat(grid): distinguish blank and null filters
Closes #5006
2026-08-24 22:33:21 +00:00
t8y2 f621e40655 feat(grid): add pagination shortcut mappings
Closes #4743
2026-08-24 20:59:29 +00:00
t8y2 cd18b7a3db feat(grid): add go-to-column shortcut mapping
Closes #4108
2026-08-24 20:41:29 +00:00
t8y2 5c75c2391c feat(editor): select SQL string content on double click
Closes #4261
2026-08-24 20:22:38 +00:00
t8y2 68b0ab0476 feat(grid): configure cell detail button visibility
Closes #5593
2026-08-24 19:54:35 +00:00
t8y2 3232195e1f fix(editor): preserve visible cursor after query execution
Closes #7037
2026-08-24 19:32:48 +00:00
t8y2 9af98c2db2 feat(shortcuts): toggle query results pane
Closes #6193
2026-08-24 19:24:54 +00:00
t8y2 adb7578998 feat(grid): paste rows from blank new cells
Closes #6144
2026-08-24 18:56:44 +00:00
t8y2 332daba302 fix(query): protect unsaved tabs on connection close
Closes #4641
2026-08-24 18:44:23 +00:00
t8y2 492c9727ae feat(editor): set all SQL parameters to raw
Closes #5823
2026-08-24 18:32:50 +00:00
t8y2 6b5b33540f feat(grid): show column defaults in new row placeholders
Closes #3096
2026-08-24 18:22:10 +00:00
t8y2 9e135f7609 fix(export): bound stalled PostgreSQL backup streams
Closes #7087
2026-08-24 16:55:00 +00:00
t8y2 2305f2de22 fix(tdsql): preserve directives in generated query SQL
Closes #7097
2026-08-24 16:38:12 +00:00
1d3d6c8bb1 fix(jdbc): expose Oracle indexes and objects (#7096)
* fix(jdbc): expose Oracle indexes and objects

* fix(jdbc): degrade unsupported index metadata to empty list

---------

Co-authored-by: zipg <4047349+zipg@users.noreply.github.com>
Co-authored-by: t8y2 <t8y2@users.noreply.github.com>
2026-08-25 00:10:22 +08:00
t8y2 d9674029f3 fix(mysql): enable cleartext authentication for Doris profiles
Closes #5355
2026-08-24 14:31:06 +00:00
zipg 7904231566 fix(kingbase): display char column lengths from bpchar metadata 2026-08-24 21:42:46 +08:00
zipg 992ac00aff fix(backup): hide unsupported database backup entries 2026-08-24 21:29:21 +08:00
t8y2 6baf17f609 fix(ssh): trim Pageant username buffer padding
Closes #7091
2026-08-24 13:19:25 +00:00
t8y2 ba25a871fd fix(query): guard unsupported stale manual transaction state
Closes #7045
2026-08-24 12:55:43 +00:00
t8y2 255faf0e4c fix(elasticsearch): support OpenSearch Dashboards proxy XSRF
Closes #7075
2026-08-24 12:37:43 +00:00
二丫讲梵 8d23216d60 feat(completion): show table metadata comments in SQL completions 2026-08-24 20:36:58 +08:00
zipg f45fcdb69d feat(grid): preview text stored in MySQL BLOB cells 2026-08-24 20:34:04 +08:00
onenewcode 566bc87014 fix(grid): chunk Oracle CLOB literals in data grid SQL 2026-08-24 20:32:19 +08:00
zipg d8e3d041cc feat(saphana): open stored procedure source 2026-08-24 20:32:05 +08:00
zipg 0b4159372e fix(formatter): keep ClickHouse date abbreviation alias case 2026-08-24 20:31:51 +08:00
Guoyu Su e0ffbe8992 fix(ssh): expand custom agent socket paths 2026-08-24 20:31:35 +08:00
jearhe ba60ffb954 fix(grid): ignore unchanged cell blur commits 2026-08-24 20:28:24 +08:00
Abeautifulsnow ff31f44b42 feat(ai): surface live AI generation status with smooth timing 2026-08-24 20:28:12 +08:00
Guoyu Su 697a8efe6e feat(kafka): filter loaded messages 2026-08-24 20:27:26 +08:00
jearhe ae3b991bd5 fix(ui): prevent connection select dialog overflow 2026-08-24 20:27:14 +08:00
t8y2 a62a51b922 fix(agent): tolerate post-startup driver stdout noise
Closes #7039
2026-08-24 12:03:12 +00:00
t8y2 d6bb83cad7 fix(oracle): reload deferred values by synthetic rowid
Closes #7082
2026-08-24 11:53:16 +00:00
t8y2 6cf8d8e035 fix(sqlserver): generate valid schema diff column DDL
Closes #7051
2026-08-24 11:46:55 +00:00
t8y2 d29c2d85cb chore(github): add repository code ownership 2026-08-24 19:19:05 +08:00
t8y2 e1a4242729 fix(grid): keep grouped column comments in lookup
Closes #7071
2026-08-24 11:07:55 +00:00
t8y2 7a098a276a fix(tidb): preserve new view create statements
Closes #7050
2026-08-24 11:01:55 +00:00
GIWTO 57b350eb49 fix(sidebar): Dameng tree blank band, admin node duplication and schema auto-expand 2026-08-24 18:58:15 +08:00
jearhe 707296c71d fix(data-grid): preserve resized column widths across query reruns 2026-08-24 18:47:31 +08:00
二丫讲梵 ff4667b93f feat(structure): search and paginate source table picker for copy columns 2026-08-24 18:47:13 +08:00
zipg 5bafb68919 fix(grid): continue Oracle manual transaction cursor for infinite scroll 2026-08-24 18:37:30 +08:00
zipg 4021216155 fix(sidebar): refresh local table search index on tree changes 2026-08-24 18:37:13 +08:00
jearhe 4a478acf07 feat(editor): allow hiding query line numbers 2026-08-24 18:35:34 +08:00
miracle eaad88cc80 fix(objects): refresh event editor on selection and fix docs-export manifest 2026-08-24 18:35:17 +08:00
Melody Ruch c1888a2364 fix(storage): use WAL and immediate transactions for shared SQLite data dir 2026-08-24 18:34:49 +08:00
t8y2 bdba8ace7d fix(milvus): accept REST success response code
Closes #7074
2026-08-24 10:31:56 +00:00
github-actions[bot] 3f960f51ab chore(packages): release 0.4.71 [skip node-packages-release] 2026-08-24 09:02:04 +00:00
github-actions[bot] d7c81195d9 chore: bump module versions [skip ci] 2026-08-24 08:44:31 +00:00
t8y2 0aa1fb9d7e fix(ci): copy connection-types in docker image and align auto-commit tests 2026-08-24 15:55:05 +08:00
t8y2 357fcbbcf7 chore: bump version to 0.5.93 2026-08-24 15:31:26 +08:00
KlingeandWenJiu 332ce7405d fix(postgres): generate table DDL on PostgreSQL 9.x (#7042)
Co-authored-by: WenJiu <jilin.zhou@winkeytech.com>
2026-08-24 15:10:10 +08:00
t8y2 781cf01c78 fix(query): default query tabs to auto-commit 2026-08-24 15:01:55 +08:00
zipgandzipg c021fe009b fix(jdbc): support manual transactions and auto-commit control (#6997)
* fix(jdbc): support manual transactions

* fix(jdbc): preserve paged query transaction state

---------

Co-authored-by: zipg <4047349+zipg@users.noreply.github.com>
2026-08-24 14:50:18 +08:00
zipg c8dbeeaf7d fix(ui): keep driver store/settings page switching and tab active state (#7035) 2026-08-24 14:34:59 +08:00
t8y2 0efce9e9a9 fix(tests): update sidebar paste CI assertions 2026-08-24 14:04:19 +08:00
t8y2 63c19a9230 fix(docker): copy scripts and plugins for vite connection types sync 2026-08-24 13:59:25 +08:00
二丫讲梵 05d8e1333e fix(nacos): adapt service workbench layout to narrow panes 2026-08-24 13:34:29 +08:00
jearhe a8369390df feat(sidebar): close connected connections in group 2026-08-24 13:33:37 +08:00
zipg 917bbc1238 fix(oracle): speed up Oracle 11g system object column metadata loading 2026-08-24 13:32:42 +08:00
Elias d7adff54b7 feat(xugu): expose spatial query values as WKT 2026-08-24 13:31:48 +08:00
zipg 8cc924270c fix(editor): preserve user-adjusted SQL editor position during execution 2026-08-24 13:30:50 +08:00
zhangyc 4d7d4ab74e feat(web): support WebDAV config sync on web 2026-08-24 13:29:23 +08:00
Melody Ruch 8187ef9baf fix(sql): avoid O(n^2) re-tokenization in MySQL routine editing 2026-08-24 13:28:41 +08:00
onenewcode 2f47a23c4c fix(postgres): display money query values 2026-08-24 13:27:57 +08:00
zipg bd9f10cb5a fix(agents): build Win7-compatible RabbitMQ agent with Go 1.20 2026-08-24 13:27:12 +08:00
Allen 29b527fd3a feat(transfer): show source/target pickers as group trees 2026-08-24 13:26:08 +08:00
lizzzzz777 80240d0ba6 feat(structure-editor): support ctrl/shift multi-select for columns 2026-08-24 13:08:36 +08:00
zipg 4c938328b7 fix(oracle): avoid Oracle 11g table tree timeout 2026-08-24 12:51:11 +08:00
zipg c0665d69f3 fix(ci): sync sidebar table paste assertions 2026-08-24 12:50:58 +08:00
zipg 2124cccef7 fix(oracle): support editing query results without primary key 2026-08-24 12:50:45 +08:00
二丫讲梵 c462272d8e feat(editor): add copy button for floating DDL hint 2026-08-24 12:50:31 +08:00
t8y2 7e1e9053f8 fix(query): box transaction command errors 2026-08-24 12:38:05 +08:00
t8y2 e03cc2d269 fix(query): recover expired manual transaction sessions 2026-08-24 12:19:27 +08:00
t8y2 0047d79974 fix(grid): preserve MySQL tinyint numeric edits
Closes #7015
2026-08-24 03:12:16 +00:00
t8y2 c1e21c7261 fix(docs): avoid duplicate column type modifiers
Closes #7010
2026-08-24 02:29:34 +00:00
t8y2 55c2b63c70 fix(export): make database export cancellation responsive
Closes #7003
2026-08-24 01:45:32 +00:00
t8y2 497f070254 fix(ssh): support Pageant for Windows local accounts
Closes #6998
2026-08-24 00:45:09 +00:00
t8y2 3e9c383732 fix(oracle): prefer physical row identifiers over ROWID
Closes #6575
2026-08-23 22:53:52 +00:00
t8y2 e0be55ddf3 fix(ci): preserve table paste success contract
Refs: #4832
2026-08-23 22:17:22 +00:00
t8y2 e11a0d467d feat(ai): add generic AI configuration deep links 2026-08-24 06:05:24 +08:00
github-actions[bot] 3ece54d07e chore(docs): refresh contributor activity 2026-08-23 18:05:28 +00:00
t8y2 72bb880ee7 fix(table-paste): surface first batch paste error
Closes #4832
2026-08-23 16:24:19 +00:00
t8y2 e1eb44f626 fix(ci): format export selection helper 2026-08-23 22:25:15 +08:00
t8y2 e1f48bdd60 fix(ci): restore Rust lint and docs bundle checks 2026-08-23 14:03:53 +00:00
t8y2 b4b04d30b8 fix(export): allow routine-only database exports
Closes #6888
2026-08-23 22:00:42 +08:00
t8y2 e15ade130d fix(linux): disable DMABuf for Strix Halo on Wayland
Closes #6709
2026-08-23 11:09:14 +00:00
Xinbeok 2927e01591 feat(ai): add resizable maximized chat panel 2026-08-23 18:29:35 +08:00
t8y2 add44cf39e fix(duckdb): preserve prefix aliases during SQL formatting
Closes #6977
2026-08-23 10:11:25 +00:00
jearhe 351dc81a30 fix(schema-diff): align table selection layout 2026-08-23 18:08:17 +08:00
t8y2 6a06a12455 feat(nacos): support DataId mapping in config sync
Closes #6439
2026-08-23 10:00:41 +00:00
t8y2 d616e6b1a8 fix(iotdb): expose Tree Time in data grid filters
Closes #6434
2026-08-23 09:56:15 +00:00
t8y2 493e18755d fix(rocketmq): decode consumer client assignments
Closes #6967
2026-08-23 09:50:46 +00:00
t8y2 6e189723bd fix(sql): recognize @set declarations after comments
Closes #6974
2026-08-23 09:44:33 +00:00
t8y2 4dd107484f fix(ci): merge supplemental database issue aliases 2026-08-23 16:41:46 +08:00
t8y2 0827478078 feat(connection): centralize connection type registration 2026-08-23 16:38:06 +08:00
Melody Ruch 38664e6aea fix(codeSnapshot): preserve line structure for line numbers 2026-08-23 12:25:45 +08:00
zhangyc 726f86e24c feat(schema-diff): select fields and indexes for synchronization 2026-08-23 12:24:47 +08:00
miracle d8bc5153d2 feat(mysql): manage and edit MySQL events 2026-08-23 12:23:46 +08:00
t8y2 701f188e45 fix(ui): restore query tab activation
Closes #6968
2026-08-23 03:54:17 +00:00
t8y2 4e09b007c0 fix(ci): correct Windows Perl setup syntax 2026-08-23 11:39:43 +08:00
t8y2 bc19d25ccd fix(transfer): strip Dameng storage clauses from reused DDL
Closes #6833
2026-08-23 11:34:22 +08:00
t8y2 3b66e88356 fix(ci): provide full Perl for Windows package builds 2026-08-23 11:12:38 +08:00
t8y2 1db4148a93 fix(ci): make package release preparation idempotent 2026-08-23 10:24:06 +08:00
二丫讲梵 e55ee6054d feat(meilisearch): add workspace, key, and task management 2026-08-23 02:53:52 +08:00
二丫讲梵 93dc455c72 feat(result-tabs): support pinned tabs and batch closing 2026-08-23 02:53:07 +08:00
github-actions[bot] 9815e5ac81 chore(docs): refresh contributor activity 2026-08-22 18:05:20 +00:00
Allen 096e0ac409 feat(ai): show connection picker as a group tree 2026-08-23 01:40:51 +08:00
二丫讲梵 c4630540dd feat(export): support name and comment XLSX headers 2026-08-23 01:40:22 +08:00
jearhe 0908a944e8 fix(sidebar): keep table search results on Enter 2026-08-23 01:39:53 +08:00
zipg acc2137cde feat(postgres): support editing table owners 2026-08-23 01:39:23 +08:00
zipg 4a1f3e7141 feat(data-grid): support manual reference display values 2026-08-23 01:38:50 +08:00
jearhe b74d49e982 feat(sidebar): extend filters to views and routines 2026-08-23 01:38:19 +08:00
t8y2 3dbcc2d0dc fix(ci): install OpenSSL Perl module on Windows 2026-08-22 17:10:06 +00:00
github-actions[bot] c9f184ac5d chore(packages): release 0.4.70 [skip node-packages-release] 2026-08-22 15:34:17 +00:00
github-actions[bot] a1a4415425 chore: bump module versions [skip ci] 2026-08-22 15:17:50 +00:00
github-actions[bot] 9f638f9c79 chore(jdbc): bump plugin version [skip ci] 2026-08-22 13:25:04 +00:00
t8y2 3bbcab3bab chore(release): prepare v0.5.92 2026-08-22 21:22:54 +08:00
t8y2 a1e94d8228 feat(oracle): support adding primary keys to existing tables
Closes #6922
2026-08-22 12:24:02 +00:00
t8y2 946677a579 feat(dameng): support toggling identity on existing columns
Closes #6906
2026-08-22 11:40:51 +00:00
Tara bf029cd342 feat(postgres): support safe database renaming 2026-08-22 19:30:38 +08:00
jearhe 3118e0d617 feat(config): support optional plaintext connection exports 2026-08-22 19:19:10 +08:00
二丫讲梵 904efff6b3 feat(sidebar): support hierarchical batch connection moves 2026-08-22 19:18:11 +08:00
zipg 36c9500a4e fix(tdengine): display timezone timestamps and tags correctly 2026-08-22 19:17:05 +08:00
t8y2 2fee435b33 feat(mcp): add ZCode configuration guidance
Closes #6763
2026-08-22 11:14:06 +00:00
t8y2 906ea8ba26 feat(tabs): locate tab objects in sidebar
Closes #6681
2026-08-22 10:31:17 +00:00
t8y2 53b7be3c6f fix(grid): handle horizontal wheel input
Closes #6793
2026-08-22 10:17:35 +00:00
t8y2 de99931c04 feat(sidebar): support quick connection rename
Closes #6792
2026-08-22 10:06:21 +00:00
t8y2 3f1544d45e fix(ai): align send button with prompt submission
Closes #6949
2026-08-22 09:31:12 +00:00
t8y2 756d4aedc9 fix(test): settle vector imports before teardown 2026-08-22 09:16:13 +00:00
t8y2 389faff8ee fix(sqlserver): propagate USE context through query batches
Closes #6947
2026-08-22 09:00:22 +00:00
zipg 0559f9f76e feat(query): resume failed SQL batches safely 2026-08-22 16:37:24 +08:00
jearhe 9774a194ec fix(connection): parse OceanBase JDBC URLs 2026-08-22 16:37:20 +08:00
t8y2 6304335fcc fix(test): await pending frontend module teardown 2026-08-22 08:33:19 +00:00
Guoyu Su 82e5756325 feat(agent): export managed drivers for offline use 2026-08-22 15:41:21 +08:00
Klinge 6eb8b43701 fix(macos): prevent Escape from exiting native fullscreen 2026-08-22 15:41:17 +08:00
miracle 3442acd52c feat(completion): optimize qualified SQL completion queries 2026-08-22 15:41:13 +08:00
t8y2 fe269a28db feat(zookeeper): add UTF-8 value preview
Closes #6897
2026-08-22 07:21:26 +00:00
t8y2 b2cf487734 feat(mcp): add CodeBuddy configuration guide
Closes #6729
2026-08-22 07:17:06 +00:00
t8y2 33dfaf957f test(vector): make data grid mock synchronous 2026-08-22 06:50:10 +00:00
t8y2 a0c9a1c878 feat(redis): insert command argument examples
Closes #6925
2026-08-22 06:04:07 +00:00
jearhe f274a599ed fix(sql): respect PostgreSQL ARRAY brackets 2026-08-22 14:01:46 +08:00
t8y2 11179676ed fix(sidebar): locate grouped connections after restart
Closes #6840
2026-08-22 05:46:37 +00:00
二丫讲梵 a6f4cebd18 feat(editor): include database name in generated SQL 2026-08-22 13:40:42 +08:00
t8y2 fcb6ea92db fix(editor): cache semantic highlight windows
Closes #6936
2026-08-22 05:07:09 +00:00
onenewcode 6aaac7133f fix(grid): unify long-cell double-click selection 2026-08-22 12:58:28 +08:00
zipg 3858cf1e59 fix(gbase): sync type label after version switch 2026-08-22 12:57:55 +08:00
zipg edffa6706d fix(editor): preserve active cell border on row highlight 2026-08-22 12:57:23 +08:00
jearhe e8c759e314 fix(phoenix): handle system catalog special types 2026-08-22 12:54:28 +08:00
Melody Ruch a2c9317c8b fix(sql): resolve MyBatis where tags before execution 2026-08-22 12:52:54 +08:00
t8y2 355f34f3c7 fix(sql): split declaration-only Oracle package bodies
Closes #6943
2026-08-22 04:37:01 +00:00
t8y2 f3d2b6cb01 fix(grid): clear stale header context menu targets
Closes #6869
2026-08-22 03:50:15 +00:00
t8y2 62657446f3 feat(sql): add group-by multi-column completion
Closes #6939
2026-08-22 03:02:42 +00:00
t8y2 d33161bb71 fix(databricks): use backticks for generated identifiers
Closes #6915
2026-08-22 02:18:10 +00:00
t8y2 5ca99c632b fix(iris): return column descriptions
Closes #6912
2026-08-22 02:00:21 +00:00
t8y2 621074beca fix(redis): allow db0 commands without SELECT support
Closes #6876
2026-08-22 01:50:50 +00:00
t8y2 509d28c69a fix(rocketmq): restore consumer client assignments
Closes #6898
2026-08-22 01:38:58 +00:00
t8y2 b015cfeef5 fix(tdengine): preserve FLOAT decimal precision
Closes #6937
2026-08-22 01:07:45 +00:00
t8y2 da49e03048 fix(mcp): resolve mise-managed server shims
Closes #6930
2026-08-22 01:03:17 +00:00
github-actions[bot] bb90c5febc chore(docs): refresh contributor activity 2026-08-21 18:08:57 +00:00
t8y2 3bec5091df fix(grid): use table-open page size for documents
Closes #6903
2026-08-21 17:32:32 +00:00
t8y2 f57cd3960b fix(oracle): expose sequences in schema tree
Closes #6893
2026-08-21 17:22:32 +00:00
t8y2 37ca4c31ad fix(oceanbase): expose Oracle-mode sequences in schema tree
Closes #6895
2026-08-21 16:38:16 +00:00
zipg 7d5cae6486 fix(mysql): preserve JDBC TLS parameter semantics 2026-08-21 19:57:22 +08:00
zipg 8671a2fb2a feat(grid): support deleting custom column formatters 2026-08-21 19:44:52 +08:00
zipg 47e02b8da7 fix(oracle): defer XMLTYPE in manual transactions 2026-08-21 19:44:13 +08:00
zipg e50bd26141 feat(grid): load large-value previews by viewport 2026-08-21 19:05:44 +08:00
Justin Gao 31a50fb460 feat(editor): preserve unsaved SQL drafts on exit 2026-08-21 18:38:53 +08:00
onenewcode 52c892676c feat(vector): support renaming Milvus collections 2026-08-21 18:19:47 +08:00
zipg 4b75334164 fix(editor): avoid quadratic SQL folding scans 2026-08-21 18:13:55 +08:00
tatuke 68acf9f132 fix(import): preserve explicit DBeaver database names 2026-08-21 18:12:39 +08:00
zipg 2bb85dcdc9 fix(agent): accept legacy H2 offline packages 2026-08-21 17:51:50 +08:00
t8y2 00b641aaec fix(table-editor): coalesce MySQL auto-increment key changes
Closes #6873
2026-08-21 09:09:39 +00:00
t8y2 102f24798c fix(snapshot): wire code snapshot option switches
Closes #6870
2026-08-21 08:27:23 +00:00
t8y2 49296c8930 fix(data-grid): serialize JSON documents in copied SQL assignments
Closes #6871
2026-08-21 07:59:26 +00:00
t8y2 0daa23ed8f fix(desktop): validate app-open links on secondary launch 2026-08-21 15:17:33 +08:00
t8y2 7d6a7c2321 feat(desktop): support dbx://open deep links 2026-08-21 14:55:46 +08:00
github-actions[bot] 81e174e332 chore(packages): release 0.4.69 [skip node-packages-release] 2026-08-21 06:34:29 +00:00
github-actions[bot] 655cd12d9d chore: bump module versions [skip ci] 2026-08-21 06:19:21 +00:00
t8y2 dbe653d3d2 chore(release): prepare v0.5.91 2026-08-21 13:24:41 +08:00
t8y2 2d321b3537 fix(sql): preserve consecutive Oracle PL/SQL blocks
Closes #6843
2026-08-21 05:04:02 +00:00
zipg 0ba2021181 fix(sqlserver): suppress diagnostics for routine batches 2026-08-21 13:03:48 +08:00
Allen 09b326558f feat(ai): drag scoped table references into chat 2026-08-21 13:03:14 +08:00
t8y2 cc78d4f528 fix(ai): include Node runtime in Codex CLI path
Closes #6849
2026-08-21 04:34:41 +00:00
zipg 4a22889b17 fix(editor): prevent stale completion metadata after refresh 2026-08-21 12:22:14 +08:00
Zhuxing 4bf7551b82 fix(redis): support legacy pickle and valid XML tags 2026-08-21 12:21:37 +08:00
二丫讲梵 b09fd35cd8 feat(redis): delete leaf keys from the key list 2026-08-21 12:20:57 +08:00
jearhe 25d5b885ba fix(kafka): support scoped IPv6 bootstrap addresses 2026-08-21 12:20:19 +08:00
t8y2 c210efbcc5 fix(mysql): complete built-in function suggestions
Closes #6846
2026-08-21 04:09:56 +00:00
t8y2 ec50662793 fix(oracle): avoid slow 11g synonym metadata lookup
Closes #6855
2026-08-21 03:47:35 +00:00
t8y2 02402de53f fix(spanner): handle array values and intrinsic numeric DDL
Closes #6841
2026-08-21 02:57:23 +00:00
zipg ab0ff28fe4 fix(postgres): 支持旧版 TLS 兼容模式 2026-08-21 10:52:27 +08:00
Abeautifulsnow 12da615daa fix(ai): require exact SQL confirmation for agent writes 2026-08-21 10:36:17 +08:00
jearhe f2e13922a2 fix(phoenix): preserve schema in all table references 2026-08-21 10:18:56 +08:00
t8y2 e139f63953 fix(postgres): preserve primary key constraint names
Closes #6821
2026-08-21 02:04:03 +00:00
jearhe 5792ce1244 fix(doris): preserve single external catalog visibility 2026-08-21 09:27:24 +08:00
github-actions[bot] 68179d16f5 chore(docs): refresh contributor activity 2026-08-20 18:34:34 +00:00
t8y2 4c13e21cca fix(transfer): avoid cloned csv regression fixture 2026-08-21 01:28:02 +08:00
二丫讲梵 7fa94152a8 feat(nacos): add access control and namespace administration 2026-08-21 01:15:46 +08:00
t8y2 30f75d8388 fix(transfer): reduce import and export allocations 2026-08-21 01:10:17 +08:00
t8y2 b5d3343f9a feat(oracle): support manual query transactions 2026-08-21 01:05:59 +08:00
二丫讲梵 a7edb8f8db feat(grid): add conditional bulk updates for MySQL 2026-08-21 01:04:16 +08:00
t8y2 10d1509d92 fix(sidebar): isolate virtual tree row identities
Closes #6773
2026-08-20 15:59:10 +00:00
t8y2 90a18a7b23 fix(editor): refresh context actions on first right-click
Closes #6811
2026-08-20 14:38:00 +00:00
t8y2 5767a0fd59 fix(snapshot): preserve previews when syntax highlighting fails
Closes #6777
2026-08-20 14:33:10 +00:00
t8y2 69e5e81071 fix(grid): preserve exact totals beyond the result row limit
Closes #6786
2026-08-20 14:26:52 +00:00
Melody Ruch b4015a4393 fix(sidebar): make destructive table operation cancellation reliable 2026-08-20 22:23:37 +08:00
Allen 7ce992b8f8 feat(tabs): add Ctrl+Tab MRU switcher 2026-08-20 22:12:52 +08:00
dadong 81872165da fix(snapshot): preserve code layout and export full images 2026-08-20 22:11:49 +08:00
t8y2 68d2919219 fix(sqlserver): skip rowversion columns when copying table data
Closes #6796
2026-08-20 14:05:47 +00:00
t8y2 b0448da0cc fix(tabs): distinguish structure editors by schema
Closes #6799
2026-08-20 13:51:45 +00:00
Melody Ruch c030835acf fix(jdbc): keep local plugin installation fully offline 2026-08-20 21:39:00 +08:00
zipg be9f8e72e4 fix(postgres): resolve result metadata through search_path 2026-08-20 21:38:20 +08:00
dadong 8a9b02fa07 fix(settings): keep color picker panels within the viewport 2026-08-20 21:37:34 +08:00
t8y2 751894d224 fix(editor): cover soft-wrapped statements with the current frame
Closes #6764
2026-08-20 13:25:41 +00:00
t8y2 6aee39e884 fix(sqlserver): tolerate malformed UTF-16 row values
Closes #6758
2026-08-20 13:12:18 +00:00
SCSTC e8e980c25c feat(connection): unify Ignite version selection 2026-08-20 21:01:23 +08:00
zipg 8031a9e651 fix(mysql): report batch DML progress without phantom failures 2026-08-20 21:00:41 +08:00
t8y2 1a3f20f6c3 fix(kingbase): return materialized view source definitions
Closes #6766
2026-08-20 12:52:01 +00:00
Melody Ruch e2a4b26379 fix(grid): show deleted row details in confirmation 2026-08-20 20:49:47 +08:00
jearhe b3acf1e558 fix(wry): recover from WebView2 process failures 2026-08-20 20:42:45 +08:00
大熊 d2e2270d84 fix(grid): edit multi-cell selections inline 2026-08-20 20:41:51 +08:00
Andrew 5741bc1256 feat(gaussdb): support M-mode index metadata and DDL 2026-08-20 18:50:22 +08:00
t8y2 ba0f864234 fix(kingbase): retry plaintext after V7 TLS failure
Closes #6803
2026-08-20 10:45:05 +00:00
onenewcode 439f2c2617 feat(milvus): add database and collection deletion 2026-08-20 18:25:12 +08:00
zipg db8f262c14 fix(grid): reset pagination when table filters change 2026-08-20 18:15:51 +08:00
Melody Ruch 3132b17323 feat(editor): add semantic diagnostics toolbar toggle 2026-08-20 17:59:35 +08:00
t8y2 345cd3f8b3 feat(h2): bundle multi-version JDBC drivers 2026-08-20 17:41:22 +08:00
github-actions[bot] a6a76640e0 chore(packages): release 0.4.68 [skip node-packages-release] 2026-08-20 09:16:36 +00:00
github-actions[bot] 321a8d03af chore: bump module versions [skip ci] 2026-08-20 09:02:12 +00:00
t8y2 501990b29b chore(release): prepare v0.5.90 2026-08-20 16:53:29 +08:00
Frost f494f90c30 feat(spanner): add Google Cloud Spanner support 2026-08-20 16:33:09 +08:00
zipg 2cc8d78e64 fix(grid): restore data grid rendering 2026-08-20 16:17:58 +08:00
amwps290 11407b537f feat(postgres): add table vacuum action 2026-08-20 16:04:22 +08:00
二丫讲梵 0e0edcd206 feat(ai): add connection tree context 2026-08-20 15:15:19 +08:00
jearhe e4577abeed feat(config): support selective connection import and export 2026-08-20 15:15:05 +08:00
Melody Ruch 89e0f1734b fix(sidebar): keep destructive table operations cancellable 2026-08-20 15:14:45 +08:00
Frost dd77817c7e docs(icons): narrow brand attribution claims 2026-08-20 15:14:32 +08:00
Elias 90af026193 fix(xugu): preserve type members during refresh 2026-08-20 15:14:19 +08:00
Elias 50607826d0 fix(xugu): reload package member groups from owner 2026-08-20 15:14:06 +08:00
Elias dcbd034afa feat(xugu): preserve index partition metadata 2026-08-20 15:13:07 +08:00
t8y2 d53c855b96 feat(docs): update database coverage to 90+ 2026-08-20 15:02:26 +08:00
t8y2 be1785d4fa fix(cassandra): list table triggers from system schema
Closes #6751
2026-08-20 06:34:13 +00:00
t8y2 5a921b3b76 fix(grid): show comments for keyless query results
Closes #6748
2026-08-20 06:28:04 +00:00
t8y2 12a2dd07f7 fix(editor): format PostgreSQL SQL with backticks
Closes #6741
2026-08-20 06:18:51 +00:00
github-actions[bot] dc74d2a77f chore(packages): release 0.4.67 [skip node-packages-release] 2026-08-20 06:08:23 +00:00
github-actions[bot] 8b0fd8d7d8 chore: bump module versions [skip ci] 2026-08-20 05:53:15 +00:00
t8y2 3f939e0458 fix(redshift): use compatible object metadata catalogs
Closes #6734
2026-08-20 05:45:44 +00:00
1296 changed files with 130185 additions and 25844 deletions
+44
View File
@@ -0,0 +1,44 @@
[CmdletBinding()]
param(
[Parameter(Mandatory = $true)]
[string[]]$Path,
[string]$ExpectedCertificateSha1 = $env:VSIGN_CERT_HASH
)
$ErrorActionPreference = "Stop"
Set-StrictMode -Version Latest
if ([string]::IsNullOrWhiteSpace($ExpectedCertificateSha1)) {
throw "The expected Authenticode certificate SHA1 is not configured"
}
$expectedSha1 = $ExpectedCertificateSha1.Replace(" ", "").ToUpperInvariant()
if ($expectedSha1 -notmatch '^[0-9A-F]{40}$') {
throw "The expected Authenticode certificate SHA1 is invalid"
}
foreach ($candidate in $Path) {
$resolvedPath = (Resolve-Path -LiteralPath $candidate -ErrorAction Stop).Path
if (!(Test-Path -LiteralPath $resolvedPath -PathType Leaf)) {
throw "Authenticode target is not a file: $resolvedPath"
}
$signature = Get-AuthenticodeSignature -LiteralPath $resolvedPath
if ($null -eq $signature.SignerCertificate) {
throw "No Authenticode signature was found on $resolvedPath"
}
$actualSha1 = $signature.SignerCertificate.Thumbprint.Replace(" ", "").ToUpperInvariant()
if ($actualSha1 -ne $expectedSha1) {
throw "Signing certificate mismatch on $resolvedPath. Expected $expectedSha1, got $actualSha1"
}
if ($signature.Status -ne "Valid") {
throw "Authenticode signature status for $resolvedPath is $($signature.Status): $($signature.StatusMessage)"
}
if ($null -eq $signature.TimeStamperCertificate) {
throw "The Authenticode signature on $resolvedPath does not contain a timestamp certificate"
}
Write-Host "Verified Authenticode signature: $resolvedPath"
}
+14 -1
View File
@@ -55,6 +55,7 @@ const extraAliases = {
db2: ["ibm db2"],
informix: ["ibm informix"],
bigquery: ["google bigquery"],
spanner: ["cloud spanner", "google spanner", "google cloud spanner"],
kylin: ["apache kylin"],
oscar: ["shentong", "oscar", "神通"],
xugu: ["xugudb", "xugu", "虚谷"],
@@ -90,7 +91,19 @@ const supplementalDrivers = [
const manifestUrl = new URL("../../crates/dbx-core/assets/database-drivers.manifest.json", import.meta.url);
const manifest = JSON.parse(fs.readFileSync(manifestUrl, "utf8"));
const catalogEntries = [...manifest.drivers, ...supplementalDrivers];
const manifestDbTypes = new Set(manifest.drivers.map((driver) => driver.dbType));
const supplementalByDbType = new Map(supplementalDrivers.map((driver) => [driver.dbType, driver]));
const catalogEntries = [
...manifest.drivers.map((driver) => {
const supplemental = supplementalByDbType.get(driver.dbType);
if (!supplemental) return driver;
return {
...driver,
aliases: [...new Set([...(driver.aliases || []), ...(supplemental.aliases || [])])],
};
}),
...supplementalDrivers.filter((driver) => !manifestDbTypes.has(driver.dbType)),
];
const duplicateDbTypes = catalogEntries
.map((driver) => driver.dbType)
.filter((dbType, index, values) => values.indexOf(dbType) !== index);
+6 -2
View File
@@ -6,12 +6,16 @@ import { dirname, join, resolve } from "node:path";
const LOCALES_DIR = "apps/desktop/src/i18n/locales";
const SOURCE_LOCALE = "zh-CN";
const TARGET_LOCALES = ["en", "es", "it", "ja", "pt-BR", "zh-TW"];
const TARGET_LOCALES = ["en", "es", "it", "ja", "ko", "pt-BR", "zh-TW"];
// `git show <ref>:<path>` needs forward slashes even on Windows, where
// `path.join()` yields backslashes. Allow an override (CI keeps the default).
const refPath = (path) => path.replaceAll("\\", "/");
const TARGET_LABELS = {
en: "English",
es: "Spanish",
it: "Italian",
ja: "Japanese",
ko: "Korean",
"pt-BR": "Brazilian Portuguese",
"zh-TW": "Traditional Chinese used in Taiwan",
};
@@ -106,7 +110,7 @@ function localePath(locale) {
function readBaseFile(path) {
try {
return execFileSync("git", ["show", `${baseRef}:${path}`], { encoding: "utf8", maxBuffer: 16 * 1024 * 1024 });
return execFileSync("git", ["show", `${baseRef}:${refPath(path)}`], { encoding: "utf8", maxBuffer: 16 * 1024 * 1024 });
} catch (error) {
throw new Error(`Unable to read ${path} from ${baseRef}: ${error.message}`);
}
@@ -50,6 +50,14 @@ test("Kingbase aliases retain historical and current company names", () => {
assert.equal(kingbase.aliases.includes("中电科金仓"), false);
});
test("supplemental aliases merge into manifest entries without duplicating labels", () => {
const mqttEntries = databaseIssueDrivers.filter((driver) => driver.dbType === "mqtt");
assert.equal(mqttEntries.length, 1);
assert.equal(mqttEntries[0].aliases.includes("emqx"), true);
assert.equal(mqttEntries[0].aliases.includes("hivemq"), true);
assert.equal(mqttEntries[0].aliases.includes("mosquitto"), true);
});
test("labels native and compatibility database products with their families", () => {
const cases = [
["Turso", ["db/turso"]],
+106
View File
@@ -0,0 +1,106 @@
[CmdletBinding()]
param()
$ErrorActionPreference = "Stop"
Set-StrictMode -Version Latest
$requiredVariables = @(
"GITHUB_ENV",
"RUNNER_TEMP",
"VSIGN_CERT_HASH",
"VSIGN_OPERATOR_PFX_B64",
"VSIGN_OPERATOR_PWD",
"VSIGN_OPERATOR_SHA1",
"VSIGN_SERVER",
"VSIGN_TOOL_SHA256"
)
foreach ($variableName in $requiredVariables) {
$value = [Environment]::GetEnvironmentVariable($variableName)
if ([string]::IsNullOrWhiteSpace($value)) {
throw "Required VSign configuration is missing: $variableName"
}
}
$serverUri = $null
if (![Uri]::TryCreate($env:VSIGN_SERVER, [UriKind]::Absolute, [ref]$serverUri) -or $serverUri.Scheme -ne "https") {
throw "VSIGN_SERVER must be an absolute HTTPS URL"
}
$expectedToolSha256 = $env:VSIGN_TOOL_SHA256.Replace(" ", "").ToUpperInvariant()
$expectedOperatorSha1 = $env:VSIGN_OPERATOR_SHA1.Replace(" ", "").ToUpperInvariant()
$expectedCertificateSha1 = $env:VSIGN_CERT_HASH.Replace(" ", "").ToUpperInvariant()
if ($expectedToolSha256 -notmatch '^[0-9A-F]{64}$') {
throw "VSIGN_TOOL_SHA256 is invalid"
}
if ($expectedOperatorSha1 -notmatch '^[0-9A-F]{40}$') {
throw "VSIGN_OPERATOR_SHA1 is invalid"
}
if ($expectedCertificateSha1 -notmatch '^[0-9A-F]{40}$') {
throw "VSIGN_CERT_HASH is invalid"
}
$runId = if ([string]::IsNullOrWhiteSpace($env:GITHUB_RUN_ID)) { "local" } else { $env:GITHUB_RUN_ID }
$runAttempt = if ([string]::IsNullOrWhiteSpace($env:GITHUB_RUN_ATTEMPT)) { "1" } else { $env:GITHUB_RUN_ATTEMPT }
$workDir = Join-Path $env:RUNNER_TEMP "vsign-$runId-$runAttempt"
$operatorDir = Join-Path $workDir "operator"
$operatorFile = Join-Path $operatorDir "dbx-github-ci@ssigncode.pfx"
$toolPath = Join-Path $workDir "ssigncode.exe"
$toolUrl = "https://github.com/g5wsg/vsign-github-test1/releases/download/vsign-cli-v1/ssigncode.exe"
New-Item -ItemType Directory -Force -Path $operatorDir | Out-Null
Invoke-WebRequest -Uri $toolUrl -OutFile $toolPath
$actualToolSha256 = (Get-FileHash -LiteralPath $toolPath -Algorithm SHA256).Hash
if ($actualToolSha256 -ne $expectedToolSha256) {
throw "VSign CLI SHA256 mismatch. Expected $expectedToolSha256, got $actualToolSha256"
}
& $toolPath version
if ($LASTEXITCODE -ne 0) {
throw "VSign CLI version check failed with exit code $LASTEXITCODE"
}
try {
$pfxBytes = [Convert]::FromBase64String($env:VSIGN_OPERATOR_PFX_B64.Trim())
[IO.File]::WriteAllBytes($operatorFile, $pfxBytes)
$flags = [System.Security.Cryptography.X509Certificates.X509KeyStorageFlags]::EphemeralKeySet
$operatorCertificate = [System.Security.Cryptography.X509Certificates.X509Certificate2]::new(
$operatorFile,
$env:VSIGN_OPERATOR_PWD,
$flags
)
}
catch {
throw "Cannot restore or decrypt the VSign operator PFX: $($_.Exception.Message)"
}
try {
$actualOperatorSha1 = $operatorCertificate.Thumbprint.Replace(" ", "").ToUpperInvariant()
if ($actualOperatorSha1 -ne $expectedOperatorSha1) {
throw "Operator certificate mismatch. Expected $expectedOperatorSha1, got $actualOperatorSha1"
}
if (!$operatorCertificate.HasPrivateKey) {
throw "The operator PFX does not contain a private key"
}
if ($operatorCertificate.NotAfter.ToUniversalTime() -le [DateTime]::UtcNow) {
throw "The operator certificate has expired"
}
}
finally {
$operatorCertificate.Dispose()
}
$githubEnvironment = [ordered]@{
VSIGN_WORK_DIR = $workDir
VSIGN_CLI = $toolPath
SSIGNCODE_OPERATOR_STORE = "file"
SSIGNCODE_OPERATOR_DIR = $operatorDir
SSIGNCODE_OPERATOR_FILE = $operatorFile
}
foreach ($entry in $githubEnvironment.GetEnumerator()) {
"$($entry.Key)=$($entry.Value)" | Out-File $env:GITHUB_ENV -Encoding utf8 -Append
}
Write-Host "VSign CLI and operator certificate are ready."
+34 -4
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env node
import { createHash } from "node:crypto";
import { writeFileSync } from "node:fs";
import { mkdirSync, rmSync, writeFileSync } from "node:fs";
import { resolve } from "node:path";
import { fileURLToPath } from "node:url";
@@ -10,6 +10,10 @@ const GITHUB_TOKEN = process.env.GITHUB_TOKEN || "";
const DEEPSEEK_API_KEY = process.env.DEEPSEEK_API_KEY || "";
const OUT_CN = "releases-cn.json";
const OUT_EN = "releases-en.json";
const OUT_INDEX_CN = "index-cn.json";
const OUT_INDEX_EN = "index-en.json";
const RELEASES_CN_DIR = "releases-cn";
const RELEASES_EN_DIR = "releases-en";
const LATEST_EN_OUT = "latest-en.json";
const LATEST_NOTES_OUT = "latest-notes.json";
const EN_CACHE_URL = process.env.CHANGELOG_EN_CACHE_URL || "https://dl.dbxio.com/changelog/releases-en.json";
@@ -68,6 +72,7 @@ function stripIssueRefs(text) {
// 可能是旧版脚本生成的(desc 仍含 closes #),在写文件前统一清理一次。
function stripIssueRefsInReleases(json) {
for (const release of json.releases || []) {
if (release.markdown) release.markdown = stripIssueRefs(release.markdown);
for (const section of release.sections || []) {
for (const item of section.items || []) {
if (item.desc) item.desc = stripIssueRefs(item.desc);
@@ -138,12 +143,30 @@ export function buildReleasesJson(releases, now = new Date()) {
name: r.name || r.tag_name,
date: r.published_at.slice(0, 10),
_sourceHash: buildReleaseSourceHash(r),
markdown: stripIssueRefs(stripDownloadSection(r.body || "")),
sections: parseBody(r.body || ""),
})),
};
}
export function buildChangelogIndex(json) {
return {
updatedAt: json.updatedAt,
releases: json.releases.map(({ tag, name, date }) => ({ tag, name, date })),
};
}
function writeReleaseFiles(json, directory) {
rmSync(directory, { recursive: true, force: true });
mkdirSync(directory, { recursive: true });
for (const release of json.releases) {
writeFileSync(resolve(directory, `${release.tag}.json`), JSON.stringify(release, null, 2));
}
}
function releaseToMarkdown(release) {
if (release.markdown !== undefined) return release.markdown;
return release.sections
.map((s) => {
const items = s.items.map((i) => (i.desc ? `- **${i.title}** — ${i.desc}` : `- ${i.title}`)).join("\n");
@@ -199,7 +222,7 @@ export async function translateToEnglish(cnJson, { cachedEnJson = null, deepseek
for (const release of cnJson.releases) {
const cachedRelease = cachedByTag.get(release.tag);
if (cachedRelease?._sourceHash === release._sourceHash) {
enReleases.push({ ...cachedRelease, name: release.name, date: release.date, _sourceHash: release._sourceHash });
enReleases.push({ ...cachedRelease, name: release.name, date: release.date, _sourceHash: release._sourceHash, markdown: cachedRelease.markdown || releaseToMarkdown(cachedRelease) });
reusedCount++;
} else {
enReleases.push(release);
@@ -218,6 +241,7 @@ export async function translateToEnglish(cnJson, { cachedEnJson = null, deepseek
name: release.name,
date: release.date,
_sourceHash: release._sourceHash,
markdown: cachedRelease.markdown || releaseToMarkdown(cachedRelease),
});
reusedCount++;
continue;
@@ -226,7 +250,7 @@ export async function translateToEnglish(cnJson, { cachedEnJson = null, deepseek
const sectionsText = releaseToMarkdown(release);
if (!sectionsText.trim()) {
enReleases.push({ ...release, sections: [] });
enReleases.push({ ...release, sections: [], markdown: "" });
continue;
}
@@ -256,7 +280,7 @@ export async function translateToEnglish(cnJson, { cachedEnJson = null, deepseek
const data = await res.json();
const translated = data.choices?.[0]?.message?.content || "";
const enSections = parseBody(translated);
enReleases.push({ ...release, sections: enSections.length > 0 ? enSections : release.sections });
enReleases.push({ ...release, sections: enSections.length > 0 ? enSections : release.sections, markdown: enSections.length > 0 ? translated.trim() : release.markdown });
translatedCount++;
await sleep(200);
@@ -276,6 +300,9 @@ async function main() {
writeFileSync(OUT_CN, JSON.stringify(cnJson, null, 2));
console.log(`Wrote ${OUT_CN}`);
writeFileSync(OUT_INDEX_CN, JSON.stringify(buildChangelogIndex(cnJson), null, 2));
writeReleaseFiles(cnJson, RELEASES_CN_DIR);
console.log(`Wrote ${OUT_INDEX_CN} and ${RELEASES_CN_DIR}/`);
const latestNotes = buildLatestReleaseNotes(releases);
if (latestNotes) {
@@ -293,6 +320,9 @@ async function main() {
stripIssueRefsInReleases(enJson);
writeFileSync(OUT_EN, JSON.stringify(enJson, null, 2));
console.log(`Wrote ${OUT_EN}`);
writeFileSync(OUT_INDEX_EN, JSON.stringify(buildChangelogIndex(enJson), null, 2));
writeReleaseFiles(enJson, RELEASES_EN_DIR);
console.log(`Wrote ${OUT_INDEX_EN} and ${RELEASES_EN_DIR}/`);
// 应用内更新提示用的英文 notes(单条最新版本)
const latestEn = buildLatestEnNotes(enJson);
+67
View File
@@ -0,0 +1,67 @@
#!/usr/bin/env bash
set -euo pipefail
die() {
echo "AppImage input-method verification failed: $*" >&2
exit 1
}
if [[ $# -ne 1 ]]; then
die "usage: $0 <AppImage file or bundle directory>"
fi
input_path=$1
if [[ -f "$input_path" ]]; then
appimage=$input_path
elif [[ -d "$input_path" ]]; then
appimages=()
while IFS= read -r -d '' candidate; do
appimages+=("$candidate")
done < <(find "$input_path" -maxdepth 1 -type f -name '*.AppImage' -print0)
[[ ${#appimages[@]} -eq 1 ]] \
|| die "expected exactly one AppImage in $input_path, found ${#appimages[@]}"
appimage=${appimages[0]}
else
die "path does not exist: $input_path"
fi
appimage=$(realpath "$appimage")
work_dir=$(mktemp -d "${TMPDIR:-/tmp}/dbx-appimage-input-methods.XXXXXX")
trap 'rm -rf "$work_dir"' EXIT
(
cd "$work_dir"
"$appimage" --appimage-extract >/dev/null
)
appdir="$work_dir/squashfs-root"
[[ -d "$appdir" ]] || die "AppImage extraction did not create squashfs-root"
appdir=$(realpath "$appdir")
require_artifact() {
local basename=$1
local match
local resolved
match=$(find "$appdir" \( -type f -o -type l \) -name "$basename" -print -quit)
[[ -n "$match" ]] || die "missing bundled artifact: $basename"
[[ -e "$match" ]] || die "bundled artifact is a broken symbolic link: ${match#"$appdir"/}"
resolved=$(realpath "$match")
[[ "$resolved" == "$appdir"/* ]] \
|| die "bundled artifact resolves outside the AppImage: ${match#"$appdir"/}"
echo "Found $basename at ${match#"$appdir"/}"
}
require_artifact im-fcitx5.so
require_artifact im-ibus.so
require_artifact libFcitx5GClient.so.2
require_artifact libibus-1.0.so.5
cache=''
while IFS= read -r -d '' candidate; do
if grep -Fq 'im-fcitx5.so' "$candidate" && grep -Fq 'im-ibus.so' "$candidate"; then
cache=$candidate
break
fi
done < <(find "$appdir" -type f -path '*/gtk-3.0/*/immodules.cache' -print0)
[[ -n "$cache" ]] || die "no private GTK 3 immodules.cache registers both Fcitx 5 and IBus"
echo "Verified GTK input-method cache at ${cache#"$appdir"/}"
+72
View File
@@ -0,0 +1,72 @@
[CmdletBinding()]
param(
[Parameter(Mandatory = $true, Position = 0)]
[string]$FilePath
)
$ErrorActionPreference = "Stop"
Set-StrictMode -Version Latest
$resolvedPath = (Resolve-Path -LiteralPath $FilePath -ErrorAction Stop).Path
if (!(Test-Path -LiteralPath $resolvedPath -PathType Leaf)) {
throw "VSign target is not a file: $resolvedPath"
}
$extension = [IO.Path]::GetExtension($resolvedPath).ToLowerInvariant()
if ($extension -eq ".dll") {
Write-Host "Skipping Authenticode signing for DLL: $resolvedPath"
return
}
if ($extension -notin @(".exe", ".msi")) {
throw "Unsupported VSign target type: $extension"
}
$requiredVariables = @(
"SSIGNCODE_OPERATOR_DIR",
"SSIGNCODE_OPERATOR_FILE",
"SSIGNCODE_OPERATOR_PWD",
"SSIGNCODE_OPERATOR_STORE",
"VSIGN_CERT_HASH",
"VSIGN_CLI",
"VSIGN_KEY_PIN",
"VSIGN_SERVER"
)
foreach ($variableName in $requiredVariables) {
$value = [Environment]::GetEnvironmentVariable($variableName)
if ([string]::IsNullOrWhiteSpace($value)) {
throw "Required VSign runtime configuration is missing: $variableName"
}
}
$directory = [IO.Path]::GetDirectoryName($resolvedPath)
$fileName = [IO.Path]::GetFileNameWithoutExtension($resolvedPath)
$signedPath = Join-Path $directory "$fileName.vsign-$([Guid]::NewGuid().ToString('N'))$extension"
try {
& $env:VSIGN_CLI vsign `
-s $env:VSIGN_SERVER `
--cert_from csign `
-u dbx-github-ci `
--cert_hash $env:VSIGN_CERT_HASH `
-k $env:VSIGN_KEY_PIN `
--hash sha256 `
-i $resolvedPath `
-o $signedPath
if ($LASTEXITCODE -ne 0) {
throw "VSign failed for $resolvedPath with exit code $LASTEXITCODE"
}
if (!(Test-Path -LiteralPath $signedPath -PathType Leaf)) {
throw "VSign did not create the signed output for $resolvedPath"
}
& "$PSScriptRoot/assert-authenticode.ps1" -Path $signedPath
Move-Item -LiteralPath $signedPath -Destination $resolvedPath -Force
Write-Host "Signed with TrustAsia VSign: $resolvedPath"
}
finally {
if (Test-Path -LiteralPath $signedPath) {
Remove-Item -LiteralPath $signedPath -Force
}
}
+20 -2
View File
@@ -83,6 +83,7 @@ jobs:
- uses: actions/checkout@v4
with:
fetch-depth: 0
token: ${{ secrets.MCP_RELEASE_TOKEN }}
- name: Download agent version patch
uses: actions/download-artifact@v4
@@ -146,7 +147,11 @@ jobs:
for module in "${MODULES[@]}"; do
TASKS+=(":${module}:shadowJar")
done
./gradlew "${TASKS[@]}" --parallel
# agents/common changes trigger a parallel mass rebuild of every
# packaged module; the Gradle daemon default heap is too small for
# ~25 shadowJar fat jars at once on a 16 GB runner.
./gradlew "${TASKS[@]}" --parallel \
-Dorg.gradle.jvmargs="-Xmx4g -XX:MaxMetaspaceSize=1g"
working-directory: agents
- run: python3 scripts/validate_agent_jars.py
working-directory: agents
@@ -272,7 +277,6 @@ jobs:
["linux-aarch64"]="linux/arm64"
["linux-x64"]="linux/amd64"
["windows-aarch64"]="windows/arm64"
["windows-x64"]="windows/amd64"
)
for platform in "${!TARGETS[@]}"; do
IFS=/ read -r goos goarch <<< "${TARGETS[$platform]}"
@@ -284,6 +288,20 @@ jobs:
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -trimpath -ldflags="-s -w" -o "$output" .
done
ls -lh ../../../release-native
# Go 1.20 is the last release that supports Windows 7; keep other targets on Go 1.22.
- name: Set up Go for Windows 7-compatible RabbitMQ agent
uses: actions/setup-go@v5
with:
go-version: "1.20.14"
cache-dependency-path: agents/drivers/rabbitmq/go.sum
- name: Build Windows 7-compatible RabbitMQ agent
shell: bash
working-directory: agents/drivers/rabbitmq
run: |
output="../../../release-native/dbx-agent-rabbitmq-windows-x64.exe"
CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o "$output" .
go version -m "$output" | tee /tmp/rabbitmq-windows-x64-build-info.txt
grep -q ': go1\.20\.14$' /tmp/rabbitmq-windows-x64-build-info.txt
- uses: actions/upload-artifact@v4
with:
name: rabbitmq-native
+21
View File
@@ -71,6 +71,13 @@ jobs:
echo "SCCACHE_S3_USE_SSL=true"
echo "AWS_ACCESS_KEY_ID=${CACHE_ACCESS_KEY_ID}"
echo "AWS_SECRET_ACCESS_KEY=${CACHE_SECRET_ACCESS_KEY}"
# Keep the server alive through post-compile test/lint phases so
# "Show sccache stats" reflects the real counters.
echo "SCCACHE_IDLE_TIMEOUT=0"
# Also cache the C/C++ compilation of -sys crates. These jobs are
# native Linux builds, so plain CC/CXX select the target compiler.
echo "CC=${SCCACHE_PATH} cc"
echo "CXX=${SCCACHE_PATH} c++"
} >> "$GITHUB_ENV"
- name: Rust cache
@@ -378,6 +385,13 @@ jobs:
echo "SCCACHE_S3_USE_SSL=true"
echo "AWS_ACCESS_KEY_ID=${CACHE_ACCESS_KEY_ID}"
echo "AWS_SECRET_ACCESS_KEY=${CACHE_SECRET_ACCESS_KEY}"
# Keep the server alive through post-compile test/lint phases so
# "Show sccache stats" reflects the real counters.
echo "SCCACHE_IDLE_TIMEOUT=0"
# Also cache the C/C++ compilation of -sys crates. These jobs are
# native Linux builds, so plain CC/CXX select the target compiler.
echo "CC=${SCCACHE_PATH} cc"
echo "CXX=${SCCACHE_PATH} c++"
} >> "$GITHUB_ENV"
- name: Rust cache
@@ -455,6 +469,13 @@ jobs:
echo "SCCACHE_S3_USE_SSL=true"
echo "AWS_ACCESS_KEY_ID=${CACHE_ACCESS_KEY_ID}"
echo "AWS_SECRET_ACCESS_KEY=${CACHE_SECRET_ACCESS_KEY}"
# Keep the server alive through post-compile test/lint phases so
# "Show sccache stats" reflects the real counters.
echo "SCCACHE_IDLE_TIMEOUT=0"
# Also cache the C/C++ compilation of -sys crates. These jobs are
# native Linux builds, so plain CC/CXX select the target compiler.
echo "CC=${SCCACHE_PATH} cc"
echo "CXX=${SCCACHE_PATH} c++"
} >> "$GITHUB_ENV"
- name: Rust cache
+91 -8
View File
@@ -117,11 +117,10 @@ jobs:
const updateLockDependency = (dependency, packageDirectory) => {
const escapedDependency = dependency.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
const dependencyPattern = new RegExp(`('${escapedDependency}':\\n\\s+specifier: )[^\\n]+(\\n\\s+version: )([^\\n]+)`);
const updated = lockfile.replace(dependencyPattern, `$1${version}$2link:../${packageDirectory}`);
if (updated === lockfile) {
if (!dependencyPattern.test(lockfile)) {
throw new Error(`Unable to update ${dependency} in pnpm-lock.yaml.`);
}
lockfile = updated;
lockfile = lockfile.replace(dependencyPattern, `$1${version}$2link:../${packageDirectory}`);
};
for (const dependency of Object.keys(mcpPackage.optionalDependencies ?? {})) {
updateLockDependency(dependency, dependency.replace("@dbx-app/", ""));
@@ -194,6 +193,7 @@ jobs:
- uses: actions/checkout@v5
with:
fetch-depth: 0
token: ${{ secrets.MCP_RELEASE_TOKEN }}
- name: Download package release patch
uses: actions/download-artifact@v4
@@ -201,15 +201,10 @@ jobs:
name: package-release-patch
- name: Apply package release patch and push
env:
RELEASE_TOKEN: ${{ secrets.MCP_RELEASE_TOKEN }}
run: |
set -euo pipefail
TAG="${{ needs.prepare.outputs.tag }}"
VERSION="${{ needs.prepare.outputs.version }}"
if [ -n "${RELEASE_TOKEN}" ]; then
git remote set-url origin "https://x-access-token:${RELEASE_TOKEN}@github.com/${GITHUB_REPOSITORY}.git"
fi
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
git fetch origin main --no-tags
@@ -348,6 +343,22 @@ jobs:
echo "SCCACHE_S3_USE_SSL=true"
echo "AWS_ACCESS_KEY_ID=${CACHE_ACCESS_KEY_ID}"
echo "AWS_SECRET_ACCESS_KEY=${CACHE_SECRET_ACCESS_KEY}"
# Keep the server alive through the post-compile packaging tail so
# "Show sccache stats" reflects the real counters.
echo "SCCACHE_IDLE_TIMEOUT=0"
# Also cache the C/C++ compilation of -sys crates. All matrix legs
# are native builds, so plain CC/CXX select the target compiler;
# MSVC stays unwrapped. SCCACHE_PATH avoids depending on PATH.
case "${RUNNER_OS}" in
Linux)
echo "CC=${SCCACHE_PATH} cc"
echo "CXX=${SCCACHE_PATH} c++"
;;
macOS)
echo "CC=${SCCACHE_PATH} clang"
echo "CXX=${SCCACHE_PATH} clang++"
;;
esac
} >> "$GITHUB_ENV"
- name: Rust cache
@@ -357,6 +368,34 @@ jobs:
cache-targets: false
cache-on-failure: true
- name: Setup full Perl for vendored OpenSSL on Windows
if: runner.os == 'Windows'
shell: pwsh
env:
PERL_ARCHIVE_URL: https://github.com/shogo82148/build-perl/releases/download/perl-5.42.3-20260803012115/perl-5.42.3-thr-win32-x64.zip
PERL_ARCHIVE_SHA256: 4f0ac6fe1a4221c1fa1804c2d81999bdecebd505a61d0d04720e9b3925cd6db4
run: |
$archive = Join-Path $env:RUNNER_TEMP "perl-5.42.3-thr-win32-x64.zip"
$destination = Join-Path $env:RUNNER_TEMP "openssl-perl"
curl.exe --fail --location --retry 5 --retry-all-errors $env:PERL_ARCHIVE_URL --output $archive
if ($LASTEXITCODE -ne 0) {
throw "Unable to download the full Perl runtime."
}
$actualHash = (Get-FileHash -Path $archive -Algorithm SHA256).Hash.ToLowerInvariant()
if ($actualHash -ne $env:PERL_ARCHIVE_SHA256) {
throw "Unexpected Perl archive SHA256: $actualHash"
}
Expand-Archive -Path $archive -DestinationPath $destination -Force
$perl = Get-ChildItem -Path $destination -Filter perl.exe -Recurse | Where-Object { $_.FullName -match '[\\/]bin[\\/]perl\.exe$' } | Select-Object -First 1
if (-not $perl) {
throw "Unable to locate perl.exe in the downloaded runtime."
}
& $perl.FullName -MParams::Check -e 1
if ($LASTEXITCODE -ne 0) {
throw "The downloaded Perl runtime cannot load Params::Check."
}
"OPENSSL_SRC_PERL=$($perl.FullName)" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
- name: Install Linux native dependencies
if: runner.os == 'Linux'
run: |
@@ -538,6 +577,22 @@ jobs:
echo "SCCACHE_S3_USE_SSL=true"
echo "AWS_ACCESS_KEY_ID=${CACHE_ACCESS_KEY_ID}"
echo "AWS_SECRET_ACCESS_KEY=${CACHE_SECRET_ACCESS_KEY}"
# Keep the server alive through the post-compile packaging tail so
# "Show sccache stats" reflects the real counters.
echo "SCCACHE_IDLE_TIMEOUT=0"
# Also cache the C/C++ compilation of -sys crates. All matrix legs
# are native builds, so plain CC/CXX select the target compiler;
# MSVC stays unwrapped. SCCACHE_PATH avoids depending on PATH.
case "${RUNNER_OS}" in
Linux)
echo "CC=${SCCACHE_PATH} cc"
echo "CXX=${SCCACHE_PATH} c++"
;;
macOS)
echo "CC=${SCCACHE_PATH} clang"
echo "CXX=${SCCACHE_PATH} clang++"
;;
esac
} >> "$GITHUB_ENV"
- name: Rust cache
@@ -547,6 +602,34 @@ jobs:
cache-targets: false
cache-on-failure: true
- name: Setup full Perl for vendored OpenSSL on Windows
if: runner.os == 'Windows'
shell: pwsh
env:
PERL_ARCHIVE_URL: https://github.com/shogo82148/build-perl/releases/download/perl-5.42.3-20260803012115/perl-5.42.3-thr-win32-x64.zip
PERL_ARCHIVE_SHA256: 4f0ac6fe1a4221c1fa1804c2d81999bdecebd505a61d0d04720e9b3925cd6db4
run: |
$archive = Join-Path $env:RUNNER_TEMP "perl-5.42.3-thr-win32-x64.zip"
$destination = Join-Path $env:RUNNER_TEMP "openssl-perl"
curl.exe --fail --location --retry 5 --retry-all-errors $env:PERL_ARCHIVE_URL --output $archive
if ($LASTEXITCODE -ne 0) {
throw "Unable to download the full Perl runtime."
}
$actualHash = (Get-FileHash -Path $archive -Algorithm SHA256).Hash.ToLowerInvariant()
if ($actualHash -ne $env:PERL_ARCHIVE_SHA256) {
throw "Unexpected Perl archive SHA256: $actualHash"
}
Expand-Archive -Path $archive -DestinationPath $destination -Force
$perl = Get-ChildItem -Path $destination -Filter perl.exe -Recurse | Where-Object { $_.FullName -match '[\\/]bin[\\/]perl\.exe$' } | Select-Object -First 1
if (-not $perl) {
throw "Unable to locate perl.exe in the downloaded runtime."
}
& $perl.FullName -MParams::Check -e 1
if ($LASTEXITCODE -ne 0) {
throw "The downloaded Perl runtime cannot load Params::Check."
}
"OPENSSL_SRC_PERL=$($perl.FullName)" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
- name: Install Linux native dependencies
if: runner.os == 'Linux'
run: |
+268 -78
View File
@@ -62,6 +62,7 @@ jobs:
- uses: actions/checkout@v5
with:
fetch-depth: 0
token: ${{ secrets.MCP_RELEASE_TOKEN }}
- name: Find previous release tag
id: prev-tag
@@ -89,6 +90,7 @@ jobs:
TMP_DIR="$(mktemp -d)"
cp plugins/jdbc/build.gradle "$TMP_DIR/build.gradle"
cp plugins/jdbc/manifest.json "$TMP_DIR/manifest.json"
git restore plugins/jdbc/build.gradle plugins/jdbc/manifest.json
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
@@ -128,12 +130,6 @@ jobs:
target: x86_64-unknown-linux-gnu
- platform: ubuntu-22.04-arm
target: aarch64-unknown-linux-gnu
- platform: windows-2022
target: x86_64-pc-windows-msvc
arch: x64
- platform: windows-2022
target: aarch64-pc-windows-msvc
arch: arm64
runs-on: ${{ matrix.platform }}
env:
@@ -184,6 +180,23 @@ jobs:
echo "SCCACHE_S3_USE_SSL=true"
echo "AWS_ACCESS_KEY_ID=${CACHE_ACCESS_KEY_ID}"
echo "AWS_SECRET_ACCESS_KEY=${CACHE_SECRET_ACCESS_KEY}"
# Linking, bundling, and signing run for 30+ minutes after the last
# rustc call; the server's 10-minute default idle timeout would
# exit first and zero the counters shown by "Show sccache stats".
echo "SCCACHE_IDLE_TIMEOUT=0"
# Also cache the C/C++ compilation of -sys crates. All matrix legs
# are native builds, so plain CC/CXX select the target compiler;
# MSVC stays unwrapped. SCCACHE_PATH avoids depending on PATH.
case "${RUNNER_OS}" in
Linux)
echo "CC=${SCCACHE_PATH} cc"
echo "CXX=${SCCACHE_PATH} c++"
;;
macOS)
echo "CC=${SCCACHE_PATH} clang"
echo "CXX=${SCCACHE_PATH} clang++"
;;
esac
} >> "$GITHUB_ENV"
- name: Compute Rust dependency hash
@@ -230,7 +243,9 @@ jobs:
if: startsWith(matrix.platform, 'ubuntu')
run: |
sudo apt-get update
sudo apt-get install -y libwebkit2gtk-4.1-dev libgtk-3-dev libappindicator3-dev librsvg2-dev patchelf libssl-dev xdg-utils
# Bundle GTK input-method modules from the same Jammy ABI baseline
# instead of loading potentially incompatible modules from the host.
sudo apt-get install -y fcitx5-frontend-gtk3 ibus-gtk3 libwebkit2gtk-4.1-dev libgtk-3-dev libappindicator3-dev librsvg2-dev patchelf libssl-dev xdg-utils
- name: Setup Tauri signing key
shell: bash
@@ -255,18 +270,212 @@ jobs:
releaseDraft: true
args: --target ${{ matrix.target }}
- name: Verify AppImage input methods
if: startsWith(matrix.platform, 'ubuntu')
shell: bash
run: ./.github/scripts/verify-appimage-input-methods.sh "target/${{ matrix.target }}/release/bundle/appimage"
- name: Show sccache stats
if: always()
continue-on-error: true
shell: bash
run: ${SCCACHE_PATH} --show-stats
build-windows:
needs: release-ready
environment: release-signing
strategy:
fail-fast: false
matrix:
include:
- target: x86_64-pc-windows-msvc
arch: x64
- target: aarch64-pc-windows-msvc
arch: arm64
runs-on: windows-2022
env:
CARGO_INCREMENTAL: "0"
RUSTC_WRAPPER: sccache
SCCACHE_GHA_ENABLED: ${{ secrets.SCCACHE_S3_BUCKET == '' && 'true' || 'false' }}
VSIGN_CERT_HASH: ${{ vars.VSIGN_CERT_HASH }}
VSIGN_OPERATOR_SHA1: ${{ vars.VSIGN_OPERATOR_SHA1 }}
VSIGN_SERVER: ${{ vars.VSIGN_SERVER }}
VSIGN_TOOL_SHA256: ${{ vars.VSIGN_TOOL_SHA256 }}
steps:
- uses: actions/checkout@v5
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version: 22
- name: Setup pnpm
uses: pnpm/action-setup@v6
- name: Install frontend dependencies
run: pnpm install
- name: Setup Rust
uses: dtolnay/rust-toolchain@1.97.1
with:
targets: ${{ matrix.target }}
- name: Setup sccache
uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10
with:
version: "v0.10.0"
- name: Configure S3 sccache
if: env.SCCACHE_GHA_ENABLED != 'true'
shell: bash
env:
CACHE_BUCKET: ${{ secrets.SCCACHE_S3_BUCKET }}
CACHE_ENDPOINT: ${{ secrets.SCCACHE_S3_ENDPOINT }}
CACHE_REGION: ${{ secrets.SCCACHE_S3_REGION }}
CACHE_KEY_PREFIX: ${{ secrets.SCCACHE_S3_KEY_PREFIX }}
CACHE_ACCESS_KEY_ID: ${{ secrets.SCCACHE_S3_ACCESS_KEY_ID }}
CACHE_SECRET_ACCESS_KEY: ${{ secrets.SCCACHE_S3_SECRET_ACCESS_KEY }}
run: |
{
echo "SCCACHE_BUCKET=${CACHE_BUCKET}"
echo "SCCACHE_ENDPOINT=${CACHE_ENDPOINT}"
echo "SCCACHE_REGION=${CACHE_REGION}"
echo "SCCACHE_S3_KEY_PREFIX=${CACHE_KEY_PREFIX}"
echo "SCCACHE_S3_USE_SSL=true"
echo "AWS_ACCESS_KEY_ID=${CACHE_ACCESS_KEY_ID}"
echo "AWS_SECRET_ACCESS_KEY=${CACHE_SECRET_ACCESS_KEY}"
# Linking, bundling, and signing run for 30+ minutes after the last
# rustc call; the server's 10-minute default idle timeout would
# exit first and zero the counters shown by "Show sccache stats".
echo "SCCACHE_IDLE_TIMEOUT=0"
# Also cache the C/C++ compilation of -sys crates. All matrix legs
# are native builds, so plain CC/CXX select the target compiler;
# MSVC stays unwrapped. SCCACHE_PATH avoids depending on PATH.
case "${RUNNER_OS}" in
Linux)
echo "CC=${SCCACHE_PATH} cc"
echo "CXX=${SCCACHE_PATH} c++"
;;
macOS)
echo "CC=${SCCACHE_PATH} clang"
echo "CXX=${SCCACHE_PATH} clang++"
;;
esac
} >> "$GITHUB_ENV"
- name: Compute Rust dependency hash
id: deps-hash
shell: bash
run: |
{
find . -name Cargo.toml -not -path './target/*' -print0 \
| sort -z \
| xargs -0 sed -E '/^version = /d'
grep -v '^version = ' Cargo.lock
} | sha256sum | cut -d' ' -f1 | {
read -r hash
echo "hash=${hash:0:20}" >> "$GITHUB_OUTPUT"
}
- name: Rust cache
uses: swatinem/rust-cache@v2
with:
workspaces: "./ -> target"
shared-key: release-${{ matrix.target }}-${{ steps.deps-hash.outputs.hash }}
add-rust-environment-hash-key: false
cache-targets: false
cache-on-failure: true
- name: Setup Tauri signing key
shell: bash
run: |
echo "${{ secrets.TAURI_SIGNING_PRIVATE_KEY_BASE64 }}" | base64 --decode > "$RUNNER_TEMP/updater.key"
KEY_B64=$(base64 < "$RUNNER_TEMP/updater.key" | tr -d '\r\n')
echo "TAURI_SIGNING_PRIVATE_KEY=$KEY_B64" >> "$GITHUB_ENV"
if [ -n "${{ secrets.TAURI_SIGNING_PRIVATE_KEY_PASSWORD }}" ]; then
echo "TAURI_SIGNING_PRIVATE_KEY_PASSWORD=${{ secrets.TAURI_SIGNING_PRIVATE_KEY_PASSWORD }}" >> "$GITHUB_ENV"
fi
- name: Setup TrustAsia VSign
shell: pwsh
env:
VSIGN_OPERATOR_PFX_B64: ${{ secrets.VSIGN_OPERATOR_PFX_B64 }}
VSIGN_OPERATOR_PWD: ${{ secrets.VSIGN_OPERATOR_PWD }}
run: ./.github/scripts/setup-vsign.ps1
- name: Build and sign Tauri app
uses: tauri-apps/tauri-action@v0
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SSIGNCODE_OPERATOR_PWD: ${{ secrets.VSIGN_OPERATOR_PWD }}
VSIGN_KEY_PIN: ${{ secrets.VSIGN_KEY_PIN }}
with:
tagName: ${{ github.ref_name }}
releaseDraft: true
args: --target ${{ matrix.target }} --config src-tauri/tauri.vsign.conf.json
- name: Verify Windows bundles
shell: pwsh
run: |
$bundleRoot = "target/${{ matrix.target }}/release/bundle"
$nsisInstallers = @(
Get-ChildItem (Join-Path $bundleRoot "nsis") -Filter "*.exe" -File -ErrorAction SilentlyContinue
)
$msiInstallers = @(
Get-ChildItem (Join-Path $bundleRoot "msi") -Filter "*.msi" -File -ErrorAction SilentlyContinue
)
if ($nsisInstallers.Count -eq 0) {
throw "No NSIS installer was produced for ${{ matrix.target }}"
}
if ($msiInstallers.Count -eq 0) {
throw "No MSI installer was produced for ${{ matrix.target }}"
}
./.github/scripts/assert-authenticode.ps1 -Path @(
$nsisInstallers.FullName
$msiInstallers.FullName
)
- name: Upload Windows WebView2 offline installer
shell: pwsh
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SSIGNCODE_OPERATOR_PWD: ${{ secrets.VSIGN_OPERATOR_PWD }}
VSIGN_KEY_PIN: ${{ secrets.VSIGN_KEY_PIN }}
run: |
$version = "${env:GITHUB_REF_NAME}".TrimStart("v")
$arch = "${{ matrix.arch }}"
$bundleDir = "target/${{ matrix.target }}/release/bundle/nsis"
$offlineName = "DBX_${version}_${arch}-webview2-offline-setup.exe"
$before = @{}
if (Test-Path $bundleDir) {
Get-ChildItem $bundleDir -Filter "*.exe" | ForEach-Object { $before[$_.FullName] = $_.LastWriteTimeUtc }
}
pnpm tauri bundle --bundles nsis --target ${{ matrix.target }} `
--config src-tauri/tauri.webview2-offline.conf.json `
--config src-tauri/tauri.vsign.conf.json
$installer = Get-ChildItem $bundleDir -Filter "*.exe" |
Where-Object { !$before.ContainsKey($_.FullName) -or $_.LastWriteTimeUtc -gt $before[$_.FullName] } |
Sort-Object LastWriteTimeUtc -Descending |
Select-Object -First 1
if (!$installer) {
throw "Missing WebView2 offline installer in ${bundleDir}"
}
./.github/scripts/assert-authenticode.ps1 -Path $installer.FullName
Copy-Item $installer.FullName $offlineName -Force
gh release upload "${env:GITHUB_REF_NAME}" $offlineName --repo "${env:GITHUB_REPOSITORY}" --clobber
- name: Upload Windows portable ZIP
if: matrix.arch == 'x64' || matrix.arch == 'arm64'
timeout-minutes: 20
shell: pwsh
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SSIGNCODE_OPERATOR_PWD: ${{ secrets.VSIGN_OPERATOR_PWD }}
VSIGN_KEY_PIN: ${{ secrets.VSIGN_KEY_PIN }}
run: |
$version = "${env:GITHUB_REF_NAME}".TrimStart("v")
$arch = "${{ matrix.arch }}"
@@ -278,14 +487,15 @@ jobs:
$appVersion = ($cargoMetadata.packages | Where-Object { $_.name -eq "dbx" } | Select-Object -First 1).version
if (!(Test-Path $exePath)) {
Write-Error "Missing Windows executable: $exePath"
exit 1
throw "Missing Windows executable: $exePath"
}
if (!$appVersion -or $appVersion -ne $version) {
Write-Error "Release tag version $version does not match the built DBX package version $appVersion"
exit 1
throw "Release tag version $version does not match the built DBX package version $appVersion"
}
./.github/scripts/vsign-sign.ps1 -FilePath $exePath
./.github/scripts/assert-authenticode.ps1 -Path $exePath
New-Item -ItemType Directory -Force -Path $portableDir | Out-Null
Copy-Item $exePath (Join-Path $portableDir "DBX.exe") -Force
Copy-Item "LICENSE" (Join-Path $portableDir "LICENSE") -Force
@@ -313,51 +523,41 @@ jobs:
$signatureName = "${zipName}.sig"
if (!(Test-Path $signatureName)) {
Write-Error "Missing portable update signature: $signatureName"
exit 1
throw "Missing portable update signature: $signatureName"
}
gh release upload "${env:GITHUB_REF_NAME}" $zipName $signatureName --repo "${env:GITHUB_REPOSITORY}" --clobber
- name: Upload Windows WebView2 offline installer
if: matrix.arch == 'x64' || matrix.arch == 'arm64'
- name: Show sccache stats
if: always()
continue-on-error: true
shell: bash
run: ${SCCACHE_PATH} --show-stats
- name: Remove temporary VSign files
if: always()
shell: pwsh
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
$version = "${env:GITHUB_REF_NAME}".TrimStart("v")
$arch = "${{ matrix.arch }}"
$bundleDir = "target/${{ matrix.target }}/release/bundle/nsis"
$offlineName = "DBX_${version}_${arch}-webview2-offline-setup.exe"
$before = @{}
if (Test-Path $bundleDir) {
Get-ChildItem $bundleDir -Filter "*.exe" | ForEach-Object { $before[$_.FullName] = $_.LastWriteTimeUtc }
$workDir = if ([string]::IsNullOrWhiteSpace($env:VSIGN_WORK_DIR)) {
Join-Path $env:RUNNER_TEMP "vsign-$env:GITHUB_RUN_ID-$env:GITHUB_RUN_ATTEMPT"
} else {
$env:VSIGN_WORK_DIR
}
pnpm tauri bundle --bundles nsis --target ${{ matrix.target }} --config src-tauri/tauri.webview2-offline.conf.json
$installer = Get-ChildItem $bundleDir -Filter "*.exe" |
Where-Object { !$before.ContainsKey($_.FullName) -or $_.LastWriteTimeUtc -gt $before[$_.FullName] } |
Sort-Object LastWriteTimeUtc -Descending |
Select-Object -First 1
if (!$installer) {
Write-Error "Missing WebView2 offline installer in ${bundleDir}"
exit 1
if (Test-Path -LiteralPath $workDir) {
Remove-Item -LiteralPath $workDir -Recurse -Force
}
Copy-Item $installer.FullName $offlineName -Force
gh release upload "${env:GITHUB_REF_NAME}" $offlineName --repo "${env:GITHUB_REPOSITORY}" --clobber
build-windows-7-offline:
needs: release-ready
runs-on: windows-2022
environment: release-signing
env:
CARGO_INCREMENTAL: "0"
RUSTC_WRAPPER: sccache
SCCACHE_GHA_ENABLED: ${{ secrets.SCCACHE_S3_BUCKET == '' && 'true' || 'false' }}
RUSTFLAGS: -C debuginfo=line-tables-only -C target-feature=+crt-static
VSIGN_CERT_HASH: ${{ vars.VSIGN_CERT_HASH }}
VSIGN_OPERATOR_SHA1: ${{ vars.VSIGN_OPERATOR_SHA1 }}
VSIGN_SERVER: ${{ vars.VSIGN_SERVER }}
VSIGN_TOOL_SHA256: ${{ vars.VSIGN_TOOL_SHA256 }}
steps:
- uses: actions/checkout@v5
@@ -378,35 +578,6 @@ jobs:
toolchain: nightly-2026-07-22
components: rust-src
# sccache must be set up before the WebView2 loader/runtime scripts: they
# run `cargo fetch` under the job-wide RUSTC_WRAPPER=sccache, so installing
# sccache only after them made cargo abort with "program not found".
- name: Setup sccache
uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10
with:
version: "v0.10.0"
- name: Configure S3 sccache
if: env.SCCACHE_GHA_ENABLED != 'true'
shell: bash
env:
CACHE_BUCKET: ${{ secrets.SCCACHE_S3_BUCKET }}
CACHE_ENDPOINT: ${{ secrets.SCCACHE_S3_ENDPOINT }}
CACHE_REGION: ${{ secrets.SCCACHE_S3_REGION }}
CACHE_KEY_PREFIX: ${{ secrets.SCCACHE_S3_KEY_PREFIX }}
CACHE_ACCESS_KEY_ID: ${{ secrets.SCCACHE_S3_ACCESS_KEY_ID }}
CACHE_SECRET_ACCESS_KEY: ${{ secrets.SCCACHE_S3_SECRET_ACCESS_KEY }}
run: |
{
echo "SCCACHE_BUCKET=${CACHE_BUCKET}"
echo "SCCACHE_ENDPOINT=${CACHE_ENDPOINT}"
echo "SCCACHE_REGION=${CACHE_REGION}"
echo "SCCACHE_S3_KEY_PREFIX=${CACHE_KEY_PREFIX}"
echo "SCCACHE_S3_USE_SSL=true"
echo "AWS_ACCESS_KEY_ID=${CACHE_ACCESS_KEY_ID}"
echo "AWS_SECRET_ACCESS_KEY=${CACHE_SECRET_ACCESS_KEY}"
} >> "$GITHUB_ENV"
- name: Prepare Win7-compatible WebView2 loader
shell: pwsh
run: ./.github/scripts/prepare-webview2-win7-loader.ps1
@@ -441,10 +612,19 @@ jobs:
shell: pwsh
run: ./.github/scripts/assert-win7-pe-compat.ps1 -BinaryPath target/x86_64-win7-windows-msvc/release/dbx.exe
- name: Setup TrustAsia VSign
shell: pwsh
env:
VSIGN_OPERATOR_PFX_B64: ${{ secrets.VSIGN_OPERATOR_PFX_B64 }}
VSIGN_OPERATOR_PWD: ${{ secrets.VSIGN_OPERATOR_PWD }}
run: ./.github/scripts/setup-vsign.ps1
- name: Bundle and upload Windows 7 fixed-runtime installer
shell: pwsh
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SSIGNCODE_OPERATOR_PWD: ${{ secrets.VSIGN_OPERATOR_PWD }}
VSIGN_KEY_PIN: ${{ secrets.VSIGN_KEY_PIN }}
run: |
$version = "${env:GITHUB_REF_NAME}".TrimStart("v")
$bundleDir = "target/x86_64-win7-windows-msvc/release/bundle/nsis"
@@ -457,7 +637,9 @@ jobs:
exit 1
}
pnpm tauri bundle --bundles nsis --target x86_64-win7-windows-msvc --config src-tauri/tauri.webview2-win7-fixed.conf.json
pnpm tauri bundle --bundles nsis --target x86_64-win7-windows-msvc `
--config src-tauri/tauri.webview2-win7-fixed.conf.json `
--config src-tauri/tauri.vsign.conf.json
$installer = Get-ChildItem $bundleDir -Filter "*.exe" |
Sort-Object LastWriteTimeUtc -Descending |
@@ -468,15 +650,23 @@ jobs:
}
./.github/scripts/assert-win7-installer-content.ps1 -InstallerPath $installer.FullName
./.github/scripts/assert-authenticode.ps1 -Path $installer.FullName
Copy-Item $installer.FullName $offlineName -Force
gh release upload "${env:GITHUB_REF_NAME}" $offlineName --repo "${env:GITHUB_REPOSITORY}" --clobber
- name: Show sccache stats
- name: Remove temporary VSign files
if: always()
continue-on-error: true
shell: bash
run: ${SCCACHE_PATH} --show-stats
shell: pwsh
run: |
$workDir = if ([string]::IsNullOrWhiteSpace($env:VSIGN_WORK_DIR)) {
Join-Path $env:RUNNER_TEMP "vsign-$env:GITHUB_RUN_ID-$env:GITHUB_RUN_ATTEMPT"
} else {
$env:VSIGN_WORK_DIR
}
if (Test-Path -LiteralPath $workDir) {
Remove-Item -LiteralPath $workDir -Recurse -Force
}
static-browser:
# Fully static musl builds of the browser (dbx-web) variant. No glibc
@@ -535,7 +725,7 @@ jobs:
RUSTFLAGS: ${{ matrix.rustflags }}
run: |
cargo zigbuild --release -p dbx-web --target ${{ matrix.target }} \
--no-default-features --features "duckdb-sidecar,dynamodb,mq-admin"
--no-default-features --features "duckdb-sidecar,dynamodb,mq-admin,dbx-core/sqlite-bundled"
- name: Package static browser runtime
env:
@@ -581,7 +771,7 @@ jobs:
--repo "${GITHUB_REPOSITORY}" --clobber
cleanup-release-signatures:
needs: [build, build-windows-7-offline]
needs: [build, build-windows, build-windows-7-offline]
runs-on: ubuntu-latest
steps:
- name: Remove standalone updater signature assets
+20 -2
View File
@@ -31,11 +31,29 @@ jobs:
pip install awscli --quiet
aws s3 cp releases-cn.json "s3://${R2_BUCKET_NAME}/changelog/releases-cn.json" \
--endpoint-url "https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com" \
--content-type application/json
--content-type application/json \
--cache-control "public, max-age=300, stale-while-revalidate=86400"
aws s3 cp index-cn.json "s3://${R2_BUCKET_NAME}/changelog/index-cn.json" \
--endpoint-url "https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com" \
--content-type application/json \
--cache-control "public, max-age=300, stale-while-revalidate=86400"
aws s3 sync releases-cn/ "s3://${R2_BUCKET_NAME}/changelog/releases-cn/" \
--endpoint-url "https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com" \
--delete --content-type application/json \
--cache-control "public, max-age=86400, stale-while-revalidate=604800"
if [ -f releases-en.json ]; then
aws s3 cp releases-en.json "s3://${R2_BUCKET_NAME}/changelog/releases-en.json" \
--endpoint-url "https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com" \
--content-type application/json
--content-type application/json \
--cache-control "public, max-age=300, stale-while-revalidate=86400"
aws s3 cp index-en.json "s3://${R2_BUCKET_NAME}/changelog/index-en.json" \
--endpoint-url "https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com" \
--content-type application/json \
--cache-control "public, max-age=300, stale-while-revalidate=86400"
aws s3 sync releases-en/ "s3://${R2_BUCKET_NAME}/changelog/releases-en/" \
--endpoint-url "https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com" \
--delete --content-type application/json \
--cache-control "public, max-age=86400, stale-while-revalidate=604800"
fi
if [ -f latest-en.json ]; then
aws s3 cp latest-en.json "s3://${R2_BUCKET_NAME}/changelog/latest-en.json" \
@@ -0,0 +1,191 @@
name: VSign Operator Bootstrap
on:
push:
tags:
- "trustasia-vsign-init-*"
permissions:
contents: read
jobs:
bootstrap-operator:
runs-on: windows-2022
environment: release-signing
timeout-minutes: 15
steps:
- name: Download and verify VSign CLI
shell: pwsh
env:
VSIGN_CLI_URL: https://github.com/g5wsg/vsign-github-test1/releases/download/vsign-cli-v1/ssigncode.exe
VSIGN_CLI_SHA256: 25A680C653CF0E89BFD53D0A2399F4D748E33DE6644E15B0DAB29974765BFA40
run: |
$toolPath = Join-Path $env:RUNNER_TEMP "ssigncode.exe"
Invoke-WebRequest -Uri $env:VSIGN_CLI_URL -OutFile $toolPath
$actualHash = (Get-FileHash -LiteralPath $toolPath -Algorithm SHA256).Hash
if ($actualHash -ne $env:VSIGN_CLI_SHA256) {
throw "VSign CLI SHA256 mismatch. Expected $env:VSIGN_CLI_SHA256, got $actualHash"
}
"VSIGN_CLI=$toolPath" | Out-File $env:GITHUB_ENV -Encoding utf8 -Append
Write-Host "VSign CLI SHA256: $actualHash"
- name: Inspect VSign CLI commands
if: startsWith(github.ref_name, 'trustasia-vsign-init-probe-')
shell: pwsh
run: |
& $env:VSIGN_CLI version
if ($LASTEXITCODE -ne 0) {
throw "VSign CLI version command failed with exit code $LASTEXITCODE"
}
& $env:VSIGN_CLI install --help
if ($LASTEXITCODE -ne 0) {
throw "VSign CLI install help failed with exit code $LASTEXITCODE"
}
& $env:VSIGN_CLI vsign --help
if ($LASTEXITCODE -ne 0) {
throw "VSign CLI vsign help failed with exit code $LASTEXITCODE"
}
- name: Install and export VSign operator
if: startsWith(github.ref_name, 'trustasia-vsign-init-run-')
shell: pwsh
env:
VSIGN_SERVER: ${{ vars.VSIGN_SERVER }}
VSIGN_OPERATOR_TOKEN: ${{ secrets.VSIGN_OPERATOR_TOKEN }}
VSIGN_KEY_PIN: ${{ secrets.VSIGN_KEY_PIN }}
VSIGN_OPERATOR_PWD: ${{ secrets.VSIGN_OPERATOR_PWD }}
VSIGN_ARTIFACT_PWD: ${{ secrets.VSIGN_ARTIFACT_PWD }}
run: |
$requiredValues = @{
VSIGN_SERVER = $env:VSIGN_SERVER
VSIGN_OPERATOR_TOKEN = $env:VSIGN_OPERATOR_TOKEN
VSIGN_KEY_PIN = $env:VSIGN_KEY_PIN
VSIGN_OPERATOR_PWD = $env:VSIGN_OPERATOR_PWD
VSIGN_ARTIFACT_PWD = $env:VSIGN_ARTIFACT_PWD
}
foreach ($entry in $requiredValues.GetEnumerator()) {
if ([string]::IsNullOrWhiteSpace($entry.Value)) {
throw "$($entry.Key) is not configured"
}
}
$operatorDir = Join-Path $env:RUNNER_TEMP "vsign-operator"
$operatorFile = Join-Path $operatorDir "dbx-github-ci@ssigncode.pfx"
$bundleDir = Join-Path $env:RUNNER_TEMP "vsign-bundle"
$bundlePath = Join-Path $env:RUNNER_TEMP "vsign-operator-bundle.7z"
New-Item -ItemType Directory -Force -Path $operatorDir, $bundleDir | Out-Null
$env:SSIGNCODE_OPERATOR_STORE = "file"
$env:SSIGNCODE_OPERATOR_DIR = $operatorDir
$env:SSIGNCODE_OPERATOR_FILE = $operatorFile
$env:SSIGNCODE_OPERATOR_PWD = $env:VSIGN_OPERATOR_PWD
& $env:VSIGN_CLI install `
-s $env:VSIGN_SERVER `
--auth $env:VSIGN_OPERATOR_TOKEN `
--vsign `
--key_pin $env:VSIGN_KEY_PIN
if ($LASTEXITCODE -ne 0) {
throw "VSign operator installation failed with exit code $LASTEXITCODE"
}
$pfxCandidates = @(
Get-ChildItem -LiteralPath $operatorDir -Filter "*.pfx" -File -Recurse -ErrorAction SilentlyContinue
Get-ChildItem -LiteralPath $env:USERPROFILE -Filter "*@ssigncode.pfx" -File -Recurse -ErrorAction SilentlyContinue
) | Sort-Object LastWriteTimeUtc -Descending -Unique
if ($pfxCandidates.Count -eq 0) {
throw "VSign operator installation completed but no PFX file was found"
}
$sourcePfx = $pfxCandidates[0]
$flags = [System.Security.Cryptography.X509Certificates.X509KeyStorageFlags]::Exportable `
-bor [System.Security.Cryptography.X509Certificates.X509KeyStorageFlags]::EphemeralKeySet
$operatorCert = $null
foreach ($candidatePassword in @($env:VSIGN_OPERATOR_PWD, $env:VSIGN_KEY_PIN, "")) {
try {
$operatorCert = [System.Security.Cryptography.X509Certificates.X509Certificate2]::new(
$sourcePfx.FullName,
$candidatePassword,
$flags
)
if ($operatorCert.HasPrivateKey) {
break
}
$operatorCert.Dispose()
$operatorCert = $null
}
catch {
$operatorCert = $null
}
}
if ($null -eq $operatorCert) {
throw "The generated operator PFX could not be opened with the configured passwords"
}
try {
$exportedPfx = Join-Path $bundleDir "dbx-github-ci@ssigncode.pfx"
$pfxBytes = $operatorCert.Export(
[System.Security.Cryptography.X509Certificates.X509ContentType]::Pfx,
$env:VSIGN_OPERATOR_PWD
)
[IO.File]::WriteAllBytes($exportedPfx, $pfxBytes)
$thumbprint = $operatorCert.Thumbprint.Replace(" ", "").ToUpperInvariant()
$metadata = [ordered]@{
operator = "dbx-github-ci"
sha1 = $thumbprint
expires = $operatorCert.NotAfter.ToUniversalTime().ToString("o")
pfx_sha256 = (Get-FileHash -LiteralPath $exportedPfx -Algorithm SHA256).Hash
run_id = $env:GITHUB_RUN_ID
}
$metadata | ConvertTo-Json | Set-Content -LiteralPath (Join-Path $bundleDir "metadata.json") -Encoding utf8
$sevenZip = "C:\Program Files\7-Zip\7z.exe"
if (!(Test-Path -LiteralPath $sevenZip -PathType Leaf)) {
throw "7-Zip was not found on the Windows runner"
}
& $sevenZip a -t7z "-p$env:VSIGN_ARTIFACT_PWD" -mhe=on $bundlePath (Join-Path $bundleDir "*")
if ($LASTEXITCODE -ne 0) {
throw "Encrypted operator bundle creation failed with exit code $LASTEXITCODE"
}
"VSIGN_OPERATOR_SHA1=$thumbprint" | Out-File $env:GITHUB_ENV -Encoding utf8 -Append
"VSIGN_BUNDLE_PATH=$bundlePath" | Out-File $env:GITHUB_ENV -Encoding utf8 -Append
Write-Host "Operator PFX created and encrypted for retrieval."
Write-Host "Operator certificate SHA1: $thumbprint"
}
finally {
$operatorCert.Dispose()
}
- name: Upload encrypted operator bundle
if: startsWith(github.ref_name, 'trustasia-vsign-init-run-')
uses: actions/upload-artifact@v4
with:
name: vsign-operator-${{ github.run_id }}
path: ${{ env.VSIGN_BUNDLE_PATH }}
if-no-files-found: error
retention-days: 1
- name: Remove temporary VSign files
if: always()
shell: pwsh
run: |
foreach ($path in @(
(Join-Path $env:RUNNER_TEMP "ssigncode.exe"),
(Join-Path $env:RUNNER_TEMP "vsign-operator"),
(Join-Path $env:RUNNER_TEMP "vsign-bundle"),
(Join-Path $env:RUNNER_TEMP "vsign-operator-bundle.7z")
)) {
if (Test-Path -LiteralPath $path) {
Remove-Item -LiteralPath $path -Recurse -Force
}
}
+10
View File
@@ -2,6 +2,7 @@
node_modules
.claude/
.codex/
.cortexkit/
.workbuddy/
.superpowers/
.trellis/
@@ -15,10 +16,13 @@ CLAUDE.md
dist/
coverage/
/target/
vendor/wry/target/
vendor/wry/Cargo.lock
src-tauri/webview2-fixed-runtime/
plugins/jdbc/target/
plugins/jdbc/dependency-reduced-pom.xml
plugins/jdbc/lib/*.jar
agents/drivers/duckdb/target/
plugins/jdbc/.gradle/
plugins/jdbc/build/
plugins/jdbc/dist/
@@ -46,6 +50,7 @@ Thumbs.db
# GitHub (keep workflows)
.github/*
!.github/CODEOWNERS
!.github/workflows/
!.github/scripts/
!.github/ISSUE_TEMPLATE/
@@ -58,6 +63,11 @@ Thumbs.db
# Temp
tmp/
# AI background-run dev artifacts (never commit)
.dlp-quarantine/
outputs/
target-ai-write-confirmation/
# Code review tool artifacts
review_diff.json
review_classified.json
Generated
+95 -8
View File
@@ -1568,10 +1568,29 @@ version = "0.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747"
dependencies = [
"percent-encoding",
"time",
"version_check",
]
[[package]]
name = "cookie_store"
version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15b2c103cf610ec6cae3da84a766285b42fd16aad564758459e6ecf128c75206"
dependencies = [
"cookie",
"document-features",
"idna",
"log",
"publicsuffix",
"serde",
"serde_derive",
"serde_json",
"time",
"url",
]
[[package]]
name = "core-foundation"
version = "0.9.4"
@@ -1983,7 +2002,7 @@ dependencies = [
[[package]]
name = "dbx"
version = "0.5.89"
version = "0.5.98"
dependencies = [
"anyhow",
"axum",
@@ -2033,12 +2052,13 @@ dependencies = [
"tokio-util",
"uuid",
"windows-sys 0.61.2",
"wry",
"zip",
]
[[package]]
name = "dbx-cli"
version = "0.4.66"
version = "0.4.75"
dependencies = [
"async-trait",
"dbx-core",
@@ -2076,15 +2096,19 @@ dependencies = [
"insta",
"json5",
"jsonwebtoken",
"libsqlite3-hotbundle",
"log",
"minijinja",
"mongodb",
"mysql_async",
"mysql_common",
"nix",
"notify",
"openssl",
"pageant",
"percent-encoding",
"portpicker",
"postgres-openssl",
"prometheus-parse",
"quick-xml 0.37.5",
"rayon",
@@ -2109,6 +2133,7 @@ dependencies = [
"tempfile",
"tiberius",
"tokio",
"tokio-openssl",
"tokio-postgres",
"tokio-postgres-rustls",
"tokio-util",
@@ -2127,7 +2152,7 @@ dependencies = [
[[package]]
name = "dbx-mcp"
version = "0.4.66"
version = "0.4.75"
dependencies = [
"async-trait",
"dbx-core",
@@ -2148,7 +2173,7 @@ dependencies = [
[[package]]
name = "dbx-web"
version = "0.5.89"
version = "0.5.98"
dependencies = [
"aes-gcm 0.10.3",
"argon2",
@@ -2439,6 +2464,15 @@ dependencies = [
"const-random",
]
[[package]]
name = "document-features"
version = "0.2.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61"
dependencies = [
"litrs",
]
[[package]]
name = "dom_query"
version = "0.27.0"
@@ -4578,6 +4612,14 @@ dependencies = [
"libc",
]
[[package]]
name = "libsqlite3-hotbundle"
version = "1.510300.0"
source = "git+https://github.com/zipg/libsqlite3-hotbundle.git?rev=7f12f203b4f4694a273410bd9b02d77f82b6114e#7f12f203b4f4694a273410bd9b02d77f82b6114e"
dependencies = [
"cc",
]
[[package]]
name = "libsqlite3-sys"
version = "0.30.1"
@@ -4602,6 +4644,12 @@ version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0"
[[package]]
name = "litrs"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092"
[[package]]
name = "lock_api"
version = "0.4.14"
@@ -5015,7 +5063,7 @@ dependencies = [
[[package]]
name = "mysql_async"
version = "0.37.0"
source = "git+https://github.com/t8y2/mysql_async.git?rev=b615bb0b10b5975182a4aa069cabc3d09021f6ab#b615bb0b10b5975182a4aa069cabc3d09021f6ab"
source = "git+https://github.com/zipg/mysql_async.git?rev=4a9e60a62f5a48e35ef9ccc9fc8277d5012b19dd#4a9e60a62f5a48e35ef9ccc9fc8277d5012b19dd"
dependencies = [
"bytes",
"crossbeam-queue",
@@ -6149,6 +6197,18 @@ dependencies = [
"rand 0.8.7",
]
[[package]]
name = "postgres-openssl"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06743eefaa1a5c0ef2ccb6d9abf6528790a229eabd62ddcabf9b2a3aeff09fa4"
dependencies = [
"openssl",
"tokio",
"tokio-openssl",
"tokio-postgres",
]
[[package]]
name = "postgres-protocol"
version = "0.6.12"
@@ -6343,6 +6403,12 @@ dependencies = [
"regex",
]
[[package]]
name = "psl-types"
version = "2.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac"
[[package]]
name = "psm"
version = "0.1.31"
@@ -6373,6 +6439,16 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "publicsuffix"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f42ea446cab60335f76979ec15e12619a2165b5ae2c12166bef27d283a9fadf"
dependencies = [
"idna",
"psl-types",
]
[[package]]
name = "pxfm"
version = "0.1.30"
@@ -6739,6 +6815,8 @@ checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147"
dependencies = [
"base64 0.22.1",
"bytes",
"cookie",
"cookie_store",
"encoding_rs",
"futures-core",
"futures-util",
@@ -8880,7 +8958,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
dependencies = [
"fastrand",
"getrandom 0.4.3",
"getrandom 0.3.4",
"once_cell",
"rustix",
"windows-sys 0.61.2",
@@ -8956,8 +9034,6 @@ dependencies = [
[[package]]
name = "tiberius"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1446cb4198848d1562301a3340424b4f425ef79f35ef9ee034769a9dd92c10d"
dependencies = [
"async-trait",
"asynchronous-codec",
@@ -9122,6 +9198,17 @@ dependencies = [
"tokio",
]
[[package]]
name = "tokio-openssl"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59df6849caa43bb7567f9a36f863c447d95a11d5903c9cc334ba32576a27eadd"
dependencies = [
"openssl",
"openssl-sys",
"tokio",
]
[[package]]
name = "tokio-postgres"
version = "0.7.18"
+13 -5
View File
@@ -1,7 +1,7 @@
[workspace]
resolver = "2"
members = ["src-tauri", "crates/dbx-core", "crates/dbx-web", "crates/dbx-mcp", "crates/dbx-cli"]
exclude = ["vendor/ctor", "vendor/dirs-sys", "vendor/pageant", "vendor/rumqttc", "vendor/wry"]
exclude = ["vendor/ctor", "vendor/dirs-sys", "vendor/pageant", "vendor/rumqttc", "vendor/tiberius", "vendor/wry"]
[patch.crates-io]
# Tauri 2.11 uses ctor 0.8, which excludes Rust's win7 vendor. This vendors
@@ -17,14 +17,22 @@ pageant = { path = "vendor/pageant" }
# bundled Fixed Runtime path explicitly so Windows 7 does not fall back to an
# unavailable system Runtime. The vendored copy also intercepts the WebView2
# F6 "Focus Next Pane" accelerator to prevent a black screen on frameless
# Overlay-titlebar windows (see vendor/wry/src/webview2/mod.rs).
# Overlay-titlebar windows, and registers a ProcessFailed handler that
# auto-recovers the webview when the renderer process fails while the host
# process stays alive (black screen after an overnight idle session, see
# vendor/wry/src/webview2/mod.rs). Recovery is layered by process kind
# (browser exit is fatal, GPU exits are self-healed by WebView2) and bounded
# by a cooldown plus a maximum reload budget.
wry = { path = "vendor/wry" }
tokio-postgres = { git = "https://github.com/t8y2/tokio-postgres-gaussdb.git", rev = "ac8ad476de59093d33ab9f294531c956e0c6a724" }
postgres-types = { git = "https://github.com/t8y2/tokio-postgres-gaussdb.git", rev = "ac8ad476de59093d33ab9f294531c956e0c6a724" }
postgres-protocol = { git = "https://github.com/t8y2/tokio-postgres-gaussdb.git", rev = "ac8ad476de59093d33ab9f294531c956e0c6a724" }
# Temporary compatibility patch for MySQL accounts using the deprecated
# sha256_password auth plugin. Remove after upstream support lands.
mysql_async = { git = "https://github.com/t8y2/mysql_async.git", rev = "b615bb0b10b5975182a4aa069cabc3d09021f6ab" }
# Temporary compatibility patches for deprecated sha256_password accounts and
# rustls dangerous-mode connections to legacy certificates. Remove after upstream support lands.
mysql_async = { git = "https://github.com/zipg/mysql_async.git", rev = "4a9e60a62f5a48e35ef9ccc9fc8277d5012b19dd" }
# Preserve readable SQL Server result sets when a row contains an unpaired
# UTF-16 surrogate. Remove after the behavior is available in upstream tiberius.
tiberius = { path = "vendor/tiberius" }
[profile.release]
panic = "abort"
+3 -3
View File
@@ -80,7 +80,7 @@ dev: node_modules/.modules.yaml check-tauri-dev-port
$(PNPM) dev:tauri
dev-fast: node_modules/.modules.yaml check-tauri-dev-port
$(PNPM) tauri dev -- --no-default-features --features duckdb-sidecar,dynamodb
$(PNPM) tauri dev -- --no-default-features --features duckdb-sidecar,dynamodb,sqlite-bundled
dev-web: node_modules/.modules.yaml
$(PNPM) dev:web
@@ -110,10 +110,10 @@ test: node_modules/.modules.yaml
$(PNPM) test
cargo-check-fast:
cargo check --no-default-features
cargo check --no-default-features --features sqlite-bundled
cargo-test-fast:
cargo test --no-default-features
cargo test --no-default-features --features sqlite-bundled
db-list:
@$(PNPM) db:env -- list
+77 -61
View File
@@ -1,8 +1,8 @@
<div align="center">
<p style="font-size: 18px; white-space: nowrap;"><strong>80+ databases in 20 MB. Desktop, Docker, CLI, built-in AI assistant, and MCP Server.</strong></p>
<p style="font-size: 18px; white-space: nowrap;"><strong>90+ databases in 20 MB. Desktop, Docker, CLI, built-in AI assistant, and MCP Server.</strong></p>
<p>
<img src="https://dl.dbxio.com/assets/readme-hero-20260816.png" alt="DBX screenshot" width="820" />
<img src="https://dl.dbxio.com/assets/readme-hero-20260820.png" alt="DBX screenshot" width="820" />
</p>
<p>
@@ -34,6 +34,7 @@
<img src="https://img.shields.io/badge/SelectDB-22C1C3?logoColor=white" />
<img src="https://img.shields.io/badge/StarRocks-5C2D91?logoColor=white" />
<img src="https://img.shields.io/badge/Redshift-8C4FFF?logo=amazonredshift&logoColor=white" />
<img src="https://img.shields.io/badge/Cloud%20Spanner-4285F4?logo=googlecloudspanner&logoColor=white" />
<img src="https://img.shields.io/badge/DM-3857FF?logoColor=white" />
<img src="https://img.shields.io/badge/OceanBase-006AFF?logoColor=white" />
<img src="https://img.shields.io/badge/openGauss-2B7BD9?logoColor=white" />
@@ -62,6 +63,78 @@
</p>
</div>
## Sponsors & Partners
<table>
<tr>
<td align="center" valign="middle" width="220">
<a href="https://www.rainyun.com/MTE5Mjc4Ng==_" target="_blank">
<picture>
<source media="(prefers-color-scheme: dark)" srcset="docs/public/sponsors/rainyun-dark.png" />
<img src="https://www.rainyun.com/img/logo.d193755d.png" alt="RainYun" height="48" />
</picture>
</a>
</td>
<td>
RainYun is a cloud service provider offering cloud servers, physical servers, game hosting, and developer-friendly infrastructure services.
<a href="https://www.rainyun.com/MTE5Mjc4Ng==_" target="_blank">Visit RainYun</a>
</td>
</tr>
<tr>
<td align="center" valign="middle" width="220">
<a href="https://www.qiniu.com/" target="_blank">
<img src="https://www-static.qbox.me/_next/static/media/logo.0fc18feaa621d2068a7180631f742256.jpg" alt="Qiniu Cloud" height="64" />
</a>
</td>
<td>
Qiniu Cloud provides DBX with object storage, CDN, and other cloud infrastructure resources.
<a href="https://www.qiniu.com/" target="_blank">Visit Qiniu Cloud</a>
</td>
</tr>
<tr>
<td align="center" valign="middle" width="220">
<a href="https://easysearch.cn" target="_blank">
<picture>
<source media="(prefers-color-scheme: dark)" srcset="docs/public/sponsors/easysearch-dark.png" />
<img src="docs/public/sponsors/easysearch.png" alt="Easysearch" width="150" />
</picture>
</a>
</td>
<td>
Easysearch is an enterprise-grade distributed search engine compatible with Elasticsearch APIs, combining full-text, vector, geospatial search, real-time analytics, and AI capabilities in one platform.
<a href="https://easysearch.cn" target="_blank">Visit Easysearch</a>
</td>
</tr>
<tr>
<td align="center" valign="middle" width="220">
<a href="https://www.atlascloud.ai/?ref=6YYXWA" target="_blank">
<picture>
<source media="(prefers-color-scheme: dark)" srcset="docs/public/sponsors/atlas-dark.svg" />
<img src="https://www.atlascloud.ai/logo.svg" alt="Atlas Cloud" width="150" />
</picture>
</a>
</td>
<td>
Atlas Cloud gives developers one unified API for 400+ AI models across chat, image, video, and audio.
<a href="https://www.atlascloud.ai/?ref=6YYXWA" target="_blank">Visit Atlas Cloud</a>
</td>
</tr>
<tr>
<td align="center" valign="middle" width="220">
<a href="https://www.trustasia.com/ssl/trustasia/code-signing" target="_blank">
<picture>
<source media="(prefers-color-scheme: dark)" srcset="docs/public/sponsors/trustasia-dark.svg" />
<img src="docs/public/sponsors/trustasia.png" alt="TrustAsia" height="44" />
</picture>
</a>
</td>
<td>
TrustAsia provides cloud-based code signing service for DBX, enabling trusted software through automated CI/CD builds.
<a href="https://www.trustasia.com/ssl/trustasia/code-signing" target="_blank">Visit TrustAsia</a>
</td>
</tr>
</table>
## Why DBX?
<table>
@@ -89,9 +162,9 @@
## Features
### 80+ Databases, One Tool
### 90+ Databases, One Tool
MySQL, PostgreSQL, SQLite, Cloudflare D1, Redis, MongoDB, DuckDB, ClickHouse, SQL Server, Oracle, Elasticsearch, Easysearch, Meilisearch, Qdrant, Milvus, Weaviate, MariaDB, TiDB, OceanBase, openGauss, GaussDB, KWDB, KingbaseES, Vastbase, GoldenDB, Doris, SelectDB, StarRocks, Manticore Search, Redshift, DM, TDengine, XuguDB, CockroachDB, Access, HighGo, UXDB, Dolt, and more. Agent-based profiles extend DBX to H2, Snowflake, Trino, PrestoSQL, Hive, DB2, Informix, Neo4j, Cassandra, BigQuery, Kylin, SunDB, JDBCX, and custom JDBC connections. New native and agent-driven drivers also cover Databricks, SAP HANA, Teradata, Vertica, Firebird, Exasol, YashanDB, GBase 8a/8s, Databend, RQLite, Turso, InfluxDB, QuestDB, IoTDB, etcd, ZooKeeper, Nacos, Consul KV, IRIS, and more. Message queue admin is also available for Pulsar, Kafka, and RocketMQ. All in a single ~20 MB app. No bundled Chromium.
MySQL, PostgreSQL, SQLite, Cloudflare D1, Redis, MongoDB, DuckDB, ClickHouse, SQL Server, Oracle, Elasticsearch, Easysearch, Meilisearch, Qdrant, Milvus, Weaviate, MariaDB, TiDB, OceanBase, openGauss, GaussDB, KWDB, KingbaseES, Vastbase, GoldenDB, Doris, SelectDB, StarRocks, Manticore Search, Redshift, DM, TDengine, XuguDB, CockroachDB, Access, HighGo, UXDB, Dolt, and more. Agent-based profiles extend DBX to H2, Snowflake, Trino, PrestoSQL, Hive, DB2, Informix, Neo4j, Cassandra, BigQuery, Cloud Spanner, Kylin, SunDB, JDBCX, and custom JDBC connections. New native and agent-driven drivers also cover Databricks, SAP HANA, Teradata, Vertica, Firebird, Exasol, YashanDB, GBase 8a/8s, Databend, RQLite, Turso, InfluxDB, QuestDB, IoTDB, etcd, ZooKeeper, Nacos, Consul KV, IRIS, and more. Message queue admin is also available for Pulsar, Kafka, and RocketMQ. All in a single ~20 MB app. No bundled Chromium.
### Query Editor
@@ -393,63 +466,6 @@ The installer will be in `src-tauri/target/release/bundle/`.
DBX is free and open source, but ongoing maintenance, database compatibility testing, infrastructure, and release work require sustained time and resources.
### Sponsors & Partners
<table>
<tr>
<td align="center" width="180">
<a href="https://www.rainyun.com/MTE5Mjc4Ng==_" target="_blank">
<img src="https://www.rainyun.com/img/logo.d193755d.png" alt="RainYun" height="48" />
</a>
</td>
<td>
RainYun is a cloud service provider offering cloud servers, physical servers, game hosting, and developer-friendly infrastructure services.
</td>
<td align="center" width="140">
<a href="https://www.rainyun.com/MTE5Mjc4Ng==_" target="_blank">Visit RainYun</a>
</td>
</tr>
<tr>
<td align="center" width="180">
<a href="https://www.qiniu.com/" target="_blank">
<img src="https://www-static.qbox.me/_next/static/media/logo.0fc18feaa621d2068a7180631f742256.jpg" alt="Qiniu Cloud" height="64" />
</a>
</td>
<td>
Qiniu Cloud provides DBX with object storage, CDN, and other cloud infrastructure resources.
</td>
<td align="center" width="140">
<a href="https://www.qiniu.com/" target="_blank">Visit Qiniu Cloud</a>
</td>
</tr>
<tr>
<td align="center" width="180">
<a href="https://easysearch.cn" target="_blank">
<img src="docs/public/sponsors/easysearch.png" alt="Easysearch" width="150" />
</a>
</td>
<td>
Easysearch is an enterprise-grade distributed search engine compatible with Elasticsearch APIs, combining full-text, vector, geospatial search, real-time analytics, and AI capabilities in one platform.
</td>
<td align="center" width="140">
<a href="https://easysearch.cn" target="_blank">Visit Easysearch</a>
</td>
</tr>
<tr>
<td align="center" width="180">
<a href="https://www.atlascloud.ai/?ref=6YYXWA" target="_blank">
<img src="https://www.atlascloud.ai/logo.svg" alt="Atlas Cloud" width="150" />
</a>
</td>
<td>
Atlas Cloud gives developers one unified API for 400+ AI models across chat, image, video, and audio.
</td>
<td align="center" width="140">
<a href="https://www.atlascloud.ai/?ref=6YYXWA" target="_blank">Visit Atlas Cloud</a>
</td>
</tr>
</table>
- [Support DBX](https://my.feishu.cn/wiki/WMTkwdATDiiu4rk14JMcoyhTnoh) — voluntary donations via WeChat or Alipay
- [Sponsors & Partners](https://my.feishu.cn/wiki/CgOWwwTzzify79k9Oq8cXpUNn6e) — sponsorship, infrastructure, tools, and community collaboration
+79 -63
View File
@@ -1,8 +1,8 @@
<div align="center">
<p style="font-size: 18px; white-space: nowrap;"><strong>20 MB 驾驭 80+ 种数据库。桌面端、Docker、CLI、内置 AI 助手与 MCP Server。</strong></p>
<p style="font-size: 18px; white-space: nowrap;"><strong>20 MB 驾驭 90+ 种数据库。桌面端、Docker、CLI、内置 AI 助手与 MCP Server。</strong></p>
<p>
<img src="https://dl.dbxio.com/assets/readme-hero-20260816.png" alt="DBX 截图" width="820" />
<img src="https://dl.dbxio.com/assets/readme-hero-20260820.png" alt="DBX 截图" width="820" />
</p>
<p>
@@ -34,6 +34,7 @@
<img src="https://img.shields.io/badge/SelectDB-22C1C3?logoColor=white" />
<img src="https://img.shields.io/badge/StarRocks-5C2D91?logoColor=white" />
<img src="https://img.shields.io/badge/Redshift-8C4FFF?logo=amazonredshift&logoColor=white" />
<img src="https://img.shields.io/badge/Cloud%20Spanner-4285F4?logo=googlecloudspanner&logoColor=white" />
<img src="https://img.shields.io/badge/DM-3857FF?logoColor=white" />
<img src="https://img.shields.io/badge/OceanBase-006AFF?logoColor=white" />
<img src="https://img.shields.io/badge/openGauss-2B7BD9?logoColor=white" />
@@ -62,6 +63,78 @@
</p>
</div>
## 赞助商与合作伙伴
<table>
<tr>
<td align="center" valign="middle" width="220">
<a href="https://www.rainyun.com/MTE5Mjc4Ng==_" target="_blank">
<picture>
<source media="(prefers-color-scheme: dark)" srcset="docs/public/sponsors/rainyun-dark.png" />
<img src="https://www.rainyun.com/img/logo.d193755d.png" alt="雨云" height="48" />
</picture>
</a>
</td>
<td>
雨云是面向开发者和站长的云服务提供商,提供云服务器、物理服务器、游戏云和配套基础设施服务。
<a href="https://www.rainyun.com/MTE5Mjc4Ng==_" target="_blank">访问雨云</a>
</td>
</tr>
<tr>
<td align="center" valign="middle" width="220">
<a href="https://www.qiniu.com/" target="_blank">
<img src="https://www-static.qbox.me/_next/static/media/logo.0fc18feaa621d2068a7180631f742256.jpg" alt="七牛云" height="64" />
</a>
</td>
<td>
七牛云为 DBX 提供对象存储、CDN 等云基础设施资源支持。
<a href="https://www.qiniu.com/" target="_blank">访问七牛云</a>
</td>
</tr>
<tr>
<td align="center" valign="middle" width="220">
<a href="https://easysearch.cn" target="_blank">
<picture>
<source media="(prefers-color-scheme: dark)" srcset="docs/public/sponsors/easysearch-dark.png" />
<img src="docs/public/sponsors/easysearch.png" alt="Easysearch" width="150" />
</picture>
</a>
</td>
<td>
Easysearch 是一款企业级分布式搜索引擎,兼容 ES API、融合全文检索、向量检索、地理空间位置检索、实时分析与 AI 能力,为企业提供统一的数据检索与智能分析基础设施。
<a href="https://easysearch.cn" target="_blank">访问 Easysearch</a>
</td>
</tr>
<tr>
<td align="center" valign="middle" width="220">
<a href="https://www.atlascloud.ai/?ref=6YYXWA" target="_blank">
<picture>
<source media="(prefers-color-scheme: dark)" srcset="docs/public/sponsors/atlas-dark.svg" />
<img src="https://www.atlascloud.ai/logo.svg" alt="Atlas Cloud" width="150" />
</picture>
</a>
</td>
<td>
Atlas Cloud 为开发者提供统一的多模态 AI API,可通过一个接口访问聊天、图像、视频和音频等 400+ 模型。
<a href="https://www.atlascloud.ai/?ref=6YYXWA" target="_blank">访问 Atlas Cloud</a>
</td>
</tr>
<tr>
<td align="center" valign="middle" width="220">
<a href="https://www.trustasia.com/ssl/trustasia/code-signing" target="_blank">
<picture>
<source media="(prefers-color-scheme: dark)" srcset="docs/public/sponsors/trustasia-dark.svg" />
<img src="docs/public/sponsors/trustasia.png" alt="TrustAsia" height="44" />
</picture>
</a>
</td>
<td>
由 TrustAsia 提供代码签名云签服务,实现 CICD 自动化构建可信软件。
<a href="https://www.trustasia.com/ssl/trustasia/code-signing" target="_blank">访问 TrustAsia</a>
</td>
</tr>
</table>
## 为什么选择 DBX?
<table>
@@ -89,9 +162,9 @@
## 功能特性
### 80+ 种数据库,一个工具搞定
### 90+ 种数据库,一个工具搞定
MySQL、PostgreSQL、SQLite、Cloudflare D1、Redis、MongoDB、DuckDB、ClickHouse、SQL Server、Oracle、Elasticsearch、Easysearch、Meilisearch、MariaDB、TiDB、OceanBase、openGauss、GaussDB、KWDB、KingbaseES、Vastbase、GoldenDB、Doris、SelectDB、StarRocks、Manticore Search、Redshift、DM、TDengine、虚谷 XuguDB、CockroachDB、Access、HighGo、UXDB、Dolt 等数据库都能直接连接。Agent 配置还可扩展到 H2、Snowflake、Trino、Hive、DB2、Informix、Neo4j、Cassandra、BigQuery、Kylin、SunDB、JDBCX 和自定义 JDBC。新增的原生与 Agent 驱动还覆盖了 Databricks、SAP HANA、Teradata、Vertica、Firebird、Exasol、崖山 YashanDB、GBase、Databend、RQLite、Turso、InfluxDB、QuestDB、IoTDB、etcd、ZooKeeper、Nacos、Consul KV、IRIS 等。全部装进约 20 MB 的应用里,不内嵌 Chromium。
MySQL、PostgreSQL、SQLite、Cloudflare D1、Redis、MongoDB、DuckDB、ClickHouse、SQL Server、Oracle、Elasticsearch、Easysearch、Meilisearch、MariaDB、TiDB、OceanBase、openGauss、GaussDB、KWDB、KingbaseES、Vastbase、GoldenDB、Doris、SelectDB、StarRocks、Manticore Search、Redshift、DM、TDengine、虚谷 XuguDB、CockroachDB、Access、HighGo、UXDB、Dolt 等数据库都能直接连接。Agent 配置还可扩展到 H2、Snowflake、Trino、Hive、DB2、Informix、Neo4j、Cassandra、BigQuery、Cloud Spanner、Kylin、SunDB、JDBCX 和自定义 JDBC。新增的原生与 Agent 驱动还覆盖了 Databricks、SAP HANA、Teradata、Vertica、Firebird、Exasol、崖山 YashanDB、GBase、Databend、RQLite、Turso、InfluxDB、QuestDB、IoTDB、etcd、ZooKeeper、Nacos、Consul KV、IRIS 等。全部装进约 20 MB 的应用里,不内嵌 Chromium。
### 查询编辑器
@@ -402,63 +475,6 @@ make package
DBX 是免费开源项目,但持续维护、数据库兼容性测试、基础设施建设和版本发布都需要长期投入时间与资源。
### 赞助商与合作伙伴
<table>
<tr>
<td align="center" width="180">
<a href="https://www.rainyun.com/MTE5Mjc4Ng==_" target="_blank">
<img src="https://www.rainyun.com/img/logo.d193755d.png" alt="雨云" height="48" />
</a>
</td>
<td>
雨云是面向开发者和站长的云服务提供商,提供云服务器、物理服务器、游戏云和配套基础设施服务。
</td>
<td align="center" width="140">
<a href="https://www.rainyun.com/MTE5Mjc4Ng==_" target="_blank">访问雨云</a>
</td>
</tr>
<tr>
<td align="center" width="180">
<a href="https://www.qiniu.com/" target="_blank">
<img src="https://www-static.qbox.me/_next/static/media/logo.0fc18feaa621d2068a7180631f742256.jpg" alt="七牛云" height="64" />
</a>
</td>
<td>
七牛云为 DBX 提供对象存储、CDN 等云基础设施资源支持。
</td>
<td align="center" width="140">
<a href="https://www.qiniu.com/" target="_blank">访问七牛云</a>
</td>
</tr>
<tr>
<td align="center" width="180">
<a href="https://easysearch.cn" target="_blank">
<img src="docs/public/sponsors/easysearch.png" alt="Easysearch" width="150" />
</a>
</td>
<td>
Easysearch 是一款企业级分布式搜索引擎,兼容 ES API、融合全文检索、向量检索、地理空间位置检索、实时分析与 AI 能力,为企业提供统一的数据检索与智能分析基础设施。
</td>
<td align="center" width="140">
<a href="https://easysearch.cn" target="_blank">访问 Easysearch</a>
</td>
</tr>
<tr>
<td align="center" width="180">
<a href="https://www.atlascloud.ai/?ref=6YYXWA" target="_blank">
<img src="https://www.atlascloud.ai/logo.svg" alt="Atlas Cloud" width="150" />
</a>
</td>
<td>
Atlas Cloud 为开发者提供统一的多模态 AI API,可通过一个接口访问聊天、图像、视频和音频等 400+ 模型。
</td>
<td align="center" width="140">
<a href="https://www.atlascloud.ai/?ref=6YYXWA" target="_blank">访问 Atlas Cloud</a>
</td>
</tr>
</table>
- [支持 DBX](https://my.feishu.cn/wiki/WMTkwdATDiiu4rk14JMcoyhTnoh) —— 通过微信或支付宝自愿捐助
- [赞助商与合作伙伴](https://my.feishu.cn/wiki/CgOWwwTzzify79k9Oq8cXpUNn6e) —— 支持基础设施、开发工具、服务或社区合作
@@ -482,12 +498,12 @@ DBX 是免费开源项目,但持续维护、数据库兼容性测试、基础
<details>
<summary><strong>DBX 和 DBeaver / TablePlus / Beekeeper Studio 有什么区别?</strong></summary>
DBX 仅 20 MB,无需运行时依赖(无需 Java、无需 Python)。AI 和 MCP 是原生内置功能,不是插件。单一代码库同时支持 80+ 数据库、桌面端、Docker 和 Web。
DBX 仅 20 MB,无需运行时依赖(无需 Java、无需 Python)。AI 和 MCP 是原生内置功能,不是插件。单一代码库同时支持 90+ 数据库、桌面端、Docker 和 Web。
</details>
<details>
<summary><strong>支持哪些数据库?</strong></summary>
MySQL、PostgreSQL、SQLite、Cloudflare D1、Redis、MongoDB、DuckDB、ClickHouse、SQL Server、Oracle、Elasticsearch、Easysearch、Meilisearch、Qdrant、Milvus、Weaviate、MariaDB、TiDB、OceanBase、openGauss、GaussDB、KWDB、KingbaseES、Vastbase、GoldenDB、Doris、SelectDB、StarRocks、Manticore Search、Redshift、DM、TDengine、虚谷 XuguDB、CockroachDB、Access、HighGo、UXDB 等。Agent 配置可扩展到 H2、Snowflake、Trino、PrestoSQL、Hive、DB2、Informix、Neo4j、Cassandra、BigQuery、Kylin、SunDB、JDBCX、Databricks、SAP HANA、Teradata、Vertica、Firebird、Exasol、崖山 YashanDB、GBase 8a/8s、Databend、RQLite、Turso、InfluxDB、QuestDB、IoTDB、etcd、ZooKeeper、Nacos、Consul KV、IRIS 及自定义 JDBC 连接,并支持消息队列管理(Pulsar、Kafka、RocketMQ)。
MySQL、PostgreSQL、SQLite、Cloudflare D1、Redis、MongoDB、DuckDB、ClickHouse、SQL Server、Oracle、Elasticsearch、Easysearch、Meilisearch、Qdrant、Milvus、Weaviate、MariaDB、TiDB、OceanBase、openGauss、GaussDB、KWDB、KingbaseES、Vastbase、GoldenDB、Doris、SelectDB、StarRocks、Manticore Search、Redshift、DM、TDengine、虚谷 XuguDB、CockroachDB、Access、HighGo、UXDB 等。Agent 配置可扩展到 H2、Snowflake、Trino、PrestoSQL、Hive、DB2、Informix、Neo4j、Cassandra、BigQuery、Cloud Spanner、Kylin、SunDB、JDBCX、Databricks、SAP HANA、Teradata、Vertica、Firebird、Exasol、崖山 YashanDB、GBase 8a/8s、Databend、RQLite、Turso、InfluxDB、QuestDB、IoTDB、etcd、ZooKeeper、Nacos、Consul KV、IRIS 及自定义 JDBC 连接,并支持消息队列管理(Pulsar、Kafka、RocketMQ)。
</details>
<details>
+1
View File
@@ -36,6 +36,7 @@ Each agent runs as a standalone process and communicates with DBX via stdin/stdo
| neo4j | Neo4j | Official Neo4j Go Driver native agent |
| cassandra | Apache Cassandra 2.1+ | Apache cassandra-gocql-driver native agent |
| bigquery | Google BigQuery | BigQuery JDBC |
| spanner | Google Cloud Spanner | Google Cloud Spanner JDBC |
| kylin | Apache Kylin | Kylin JDBC |
| ignite | Apache Ignite | Ignite JDBC thin (ignite-core) |
| ignite3 | Apache Ignite 3 | Ignite 3 JDBC thin (ignite-jdbc) |
+1
View File
@@ -36,6 +36,7 @@ DBX 的 Agent 驱动 —— 通过 JDBC 和原生数据库驱动支持各种数
| neo4j | Neo4j | 官方 Neo4j Go Driver 原生 Agent |
| cassandra | Apache Cassandra 2.1+ | Apache cassandra-gocql-driver 原生 Agent |
| bigquery | Google BigQuery | BigQuery JDBC |
| spanner | Google Cloud Spanner | Google Cloud Spanner JDBC |
| kylin | Apache Kylin | Kylin JDBC |
| ignite | Apache Ignite | Ignite JDBC thin(ignite-core) |
| ignite3 | Apache Ignite 3 | Ignite 3 JDBC thin(ignite-jdbc) |
+1 -1
View File
@@ -8,7 +8,7 @@ def pooledJdbcProjects = [
'access', 'bigquery', 'dameng', 'databend', 'databricks', 'db2', 'exasol',
'firebird', 'gbase8a', 'gbase8s', 'goldendb', 'h2', 'h2-legacy', 'highgo',
'ignite', 'ignite3', 'informix', 'iris', 'kylin', 'oceanbase-oracle', 'oscar', 'saphana',
'snowflake', 'spark', 'sqlserver-legacy', 'sundb', 'teradata', 'trino', 'uxdb',
'snowflake', 'spanner', 'spark', 'sqlserver-legacy', 'sundb', 'teradata', 'trino', 'uxdb',
'vertica', 'yashandb'
] as Set
def agentProjects = subprojects.findAll { !infrastructureProjects.contains(it.name) }
@@ -90,7 +90,7 @@ public abstract class AbstractJdbcAgent extends BaseDatabaseAgent {
return unchecked(() -> {
loadDriver(params);
try (Connection conn = openTestConnection(params)) {
boolean valid = conn != null && conn.isValid(5);
boolean valid = isConnectionValid(conn, 5);
Map<String, Object> result = new LinkedHashMap<>();
result.put("ok", valid);
if (valid) {
@@ -275,6 +275,31 @@ public abstract class AbstractJdbcAgent extends BaseDatabaseAgent {
return true;
}
final boolean isConnectionValid(Connection connection, int timeoutSecs) {
if (connection == null) {
return false;
}
try {
if (connection.isClosed()) {
return false;
}
String validationQuery = connectionValidationQuery();
if (validationQuery == null || validationQuery.isBlank()) {
return connection.isValid(timeoutSecs);
}
try (Statement statement = connection.createStatement()) {
try {
statement.setQueryTimeout(timeoutSecs);
} catch (Exception | AbstractMethodError ignored) {
}
statement.execute(validationQuery);
return true;
}
} catch (Exception | AbstractMethodError ignored) {
return false;
}
}
final synchronized boolean usesConnectionPool() {
return poolRegistry != null;
}
@@ -485,6 +510,10 @@ public abstract class AbstractJdbcAgent extends BaseDatabaseAgent {
protected void afterDisconnect() throws Exception {
}
protected String connectionValidationQuery() {
return null;
}
protected void beforeQueryExecution(Connection connection, int timeoutSecs) throws Exception {
}
@@ -586,6 +615,7 @@ public abstract class AbstractJdbcAgent extends BaseDatabaseAgent {
return registry.borrow(
identity,
JdbcSessionRole.from(params.getSessionRole()),
connectionValidationQuery(),
() -> openInitializedConnection(params)
);
}
@@ -650,6 +680,7 @@ public abstract class AbstractJdbcAgent extends BaseDatabaseAgent {
appendPoolIdentity(identity, "driverPath" + index, driverPaths.get(index));
}
}
appendPoolIdentity(identity, "driverProfile", params.getDriver_profile());
return identity.toString();
}
@@ -42,6 +42,9 @@ public final class AgentProtocol {
public static final String METHOD_GET_EXPLAIN_INFO = "get_explain_info";
public static final String METHOD_EXECUTE_BATCH = "execute_batch";
public static final String METHOD_EXECUTE_TRANSACTION = "execute_transaction";
public static final String METHOD_BEGIN_MANUAL_TRANSACTION = "begin_manual_transaction";
public static final String METHOD_COMMIT_MANUAL_TRANSACTION = "commit_manual_transaction";
public static final String METHOD_ROLLBACK_MANUAL_TRANSACTION = "rollback_manual_transaction";
public static final String METHOD_DISCONNECT = "disconnect";
public static final String METHOD_SHUTDOWN = "shutdown";
@@ -200,6 +203,9 @@ public final class AgentProtocol {
METHOD_GET_EXPLAIN_INFO,
METHOD_EXECUTE_BATCH,
METHOD_EXECUTE_TRANSACTION,
METHOD_BEGIN_MANUAL_TRANSACTION,
METHOD_COMMIT_MANUAL_TRANSACTION,
METHOD_ROLLBACK_MANUAL_TRANSACTION,
METHOD_DISCONNECT,
METHOD_SHUTDOWN
));
@@ -23,6 +23,7 @@ public final class ConnectParams {
private String gbase_server;
private String informix_server;
private String sessionRole;
private String driver_profile;
public ConnectParams() {
this("", 0, "", "", "", "", "", false, "", Collections.emptyList());
@@ -209,6 +210,14 @@ public final class ConnectParams {
this.sessionRole = sessionRole;
}
public String getDriver_profile() {
return driver_profile;
}
public void setDriver_profile(String driver_profile) {
this.driver_profile = driver_profile;
}
@Override
public boolean equals(Object other) {
if (this == other) return true;
@@ -231,13 +240,14 @@ public final class ConnectParams {
&& Objects.equals(client_key_path, that.client_key_path)
&& Objects.equals(gbase_server, that.gbase_server)
&& Objects.equals(informix_server, that.informix_server)
&& Objects.equals(sessionRole, that.sessionRole);
&& Objects.equals(sessionRole, that.sessionRole)
&& Objects.equals(driver_profile, that.driver_profile);
}
@Override
public int hashCode() {
return Objects.hash(host, port, database, username, password, url_params, connection_string,
port_explicit, mysql_compat_mode, jdbc_driver_class, jdbc_driver_paths, ssl, ca_cert_path, client_cert_path, client_key_path, gbase_server, informix_server, sessionRole);
port_explicit, mysql_compat_mode, jdbc_driver_class, jdbc_driver_paths, ssl, ca_cert_path, client_cert_path, client_key_path, gbase_server, informix_server, sessionRole, driver_profile);
}
@Override
@@ -260,6 +270,7 @@ public final class ConnectParams {
+ ", gbase_server=" + gbase_server
+ ", informix_server=" + informix_server
+ ", sessionRole=" + sessionRole
+ ", driver_profile=" + driver_profile
+ ")";
}
}
@@ -81,10 +81,14 @@ final class JdbcConnectionPoolRegistry implements AutoCloseable {
}
Lease borrow(String identity, ConnectionFactory connectionFactory) throws Exception {
return borrow(identity, JdbcSessionRole.WORKLOAD, connectionFactory);
return borrow(identity, JdbcSessionRole.WORKLOAD, null, connectionFactory);
}
Lease borrow(String identity, JdbcSessionRole role, ConnectionFactory connectionFactory) throws Exception {
return borrow(identity, role, null, connectionFactory);
}
Lease borrow(String identity, JdbcSessionRole role, String connectionTestQuery, ConnectionFactory connectionFactory) throws Exception {
String key = digest(identity);
while (true) {
if (closed.get()) {
@@ -96,7 +100,7 @@ final class JdbcConnectionPoolRegistry implements AutoCloseable {
}
PoolEntry entry;
try {
entry = pools.computeIfAbsent(key, ignored -> createPoolEntry(key, connectionFactory));
entry = pools.computeIfAbsent(key, ignored -> createPoolEntry(key, connectionTestQuery, connectionFactory));
} catch (PoolCreationException error) {
throw error.unwrap();
}
@@ -130,7 +134,7 @@ final class JdbcConnectionPoolRegistry implements AutoCloseable {
return entry != null && entry.hasActiveLeases();
}
private PoolEntry createPoolEntry(String key, ConnectionFactory connectionFactory) {
private PoolEntry createPoolEntry(String key, String connectionTestQuery, ConnectionFactory connectionFactory) {
try {
ConnectionFactoryDataSource factoryDataSource = new ConnectionFactoryDataSource(
connectionFactory,
@@ -146,6 +150,9 @@ final class JdbcConnectionPoolRegistry implements AutoCloseable {
config.setMinimumIdle(settings.minimumIdle);
config.setConnectionTimeout(settings.connectionTimeoutMillis);
config.setValidationTimeout(settings.validationTimeoutMillis);
if (connectionTestQuery != null && !connectionTestQuery.isBlank()) {
config.setConnectionTestQuery(connectionTestQuery);
}
config.setIdleTimeout(settings.idleTimeoutMillis);
config.setMaxLifetime(settings.maxLifetimeMillis);
config.setInitializationFailTimeout(-1L);
@@ -171,8 +171,10 @@ public final class JsonRpcServer {
boolean valid = false;
if (conn != null) {
try {
valid = !conn.isClosed() && conn.isValid(2);
} catch (Exception ignored) {
valid = agent instanceof AbstractJdbcAgent jdbcAgent
? jdbcAgent.isConnectionValid(conn, 2)
: !conn.isClosed() && conn.isValid(2);
} catch (Exception | AbstractMethodError ignored) {
}
}
if (!valid) {
@@ -370,8 +372,10 @@ public final class JsonRpcServer {
}
boolean valid = false;
try {
valid = !conn.isClosed() && conn.isValid(2);
} catch (Exception ignored) {
valid = agent instanceof AbstractJdbcAgent validationAgent
? validationAgent.isConnectionValid(conn, 2)
: !conn.isClosed() && conn.isValid(2);
} catch (Exception | AbstractMethodError ignored) {
}
if (valid) {
lastConnectionValidationTimeMillis = now;
@@ -544,7 +544,7 @@ public abstract class PostgresLikeAgent extends AbstractJdbcAgent {
// characters in the text (#6312 review).
Map<String, IndexBuilder> byName = new LinkedHashMap<>();
String sql = "SELECT i.relname AS index_name, am.amname AS index_type, " +
"ix.indisunique AS is_unique, ix.indisprimary AS is_primary, " +
"(ix.indisunique AND ix.indisvalid) AS is_unique, ix.indisprimary AS is_primary, " +
"COALESCE(a.attname, " + profile.catalogPrefixedFunction("get_indexdef") + "(ix.indexrelid, k.n, true)) AS column_text, " +
"(a.attname IS NULL) AS is_expression " +
"FROM " + profile.catalogRelation("index") + " ix " +
@@ -622,7 +622,7 @@ public abstract class PostgresLikeAgent extends AbstractJdbcAgent {
return unchecked(() -> {
List<CatalogIndex> catalogIndexes = new ArrayList<>();
String sql = "SELECT i.relname AS index_name, am.amname AS index_type, " +
"ix.indisunique AS is_unique, ix.indisprimary AS is_primary, " +
"(ix.indisunique AND ix.indisvalid) AS is_unique, ix.indisprimary AS is_primary, " +
"ix.indkey AS column_numbers " +
"FROM " + profile.catalogRelation("index") + " ix " +
"JOIN " + profile.catalogRelation("class") + " t ON t.oid = ix.indrelid " +
@@ -36,6 +36,9 @@
"get_explain_info",
"execute_batch",
"execute_transaction",
"begin_manual_transaction",
"commit_manual_transaction",
"rollback_manual_transaction",
"disconnect",
"shutdown"
],
@@ -40,6 +40,9 @@
"get_explain_info",
"execute_batch",
"execute_transaction",
"begin_manual_transaction",
"commit_manual_transaction",
"rollback_manual_transaction",
"disconnect",
"shutdown"
],
@@ -127,6 +127,19 @@ class AbstractJdbcAgentTest {
assertEquals(1, tracking.closeCount);
}
@Test
void validatesLegacyJdbcConnectionsWithConfiguredQueryInsteadOfIsValid() {
TrackingConnection tracking = new TrackingConnection();
tracking.isValidUnsupported = true;
TestAgent agent = new TestAgent(tracking);
agent.validationQuery = "SELECT 1";
assertTrue(agent.testConnection(new ConnectParams()));
assertEquals(0, tracking.isValidCount);
assertEquals(Arrays.asList("setQueryTimeout:5", "execute:SELECT 1"), tracking.calls);
}
@Test
void testConnectionCanSkipOpeningAPhysicalConnection() {
TrackingConnection tracking = new TrackingConnection();
@@ -339,6 +352,7 @@ class AbstractJdbcAgentTest {
private int afterConnectCount;
private int afterDisconnectCount;
private boolean skipTestConnectionOpen;
private String validationQuery;
private TestAgent(TrackingConnection tracking) {
this.tracking = tracking;
@@ -375,6 +389,11 @@ class AbstractJdbcAgentTest {
afterDisconnectCount += 1;
}
@Override
protected String connectionValidationQuery() {
return validationQuery;
}
@Override
public List<DatabaseInfo> listDatabases() {
return Collections.emptyList();
@@ -436,6 +455,7 @@ class AbstractJdbcAgentTest {
private String identifierQuote = "\"";
private boolean compatibilityQueryFails;
private int compatibilityQueryCount;
private boolean isValidUnsupported;
private Connection connection() {
return proxy(Connection.class, new MethodHandler() {
@@ -450,6 +470,9 @@ class AbstractJdbcAgentTest {
}
if ("isValid".equals(name)) {
isValidCount += 1;
if (isValidUnsupported) {
throw new AbstractMethodError("legacy JDBC driver");
}
return true;
}
if ("close".equals(name)) {
@@ -173,6 +173,7 @@ class PostgresLikeAgentTest {
String sql = String.join("\n", MetadataSqlFake.statements);
assertTrue(sql.contains("LEFT JOIN pg_catalog.pg_attribute a ON a.attrelid = t.oid AND a.attnum = k.attnum AND k.attnum > 0"), sql);
assertTrue(sql.contains("pg_catalog.pg_get_indexdef(ix.indexrelid, k.n, true)"), sql);
assertTrue(sql.contains("ix.indisunique AND ix.indisvalid"), sql);
}
@Test
+10
View File
@@ -11,6 +11,16 @@ Protocol v2 allows one Agent process to serve multiple isolated database session
- `close_session` closes the session resources, query cursors, and table-read cursors without affecting other sessions.
- `shutdown` closes all sessions and terminates the runtime.
## Manual (interactive) transactions
When a runtime advertises the `transaction` capability and supports sticky sessions, DBX may open a dedicated workload session and call:
- `begin_manual_transaction` `{ schema? }` — pin one physical connection / start an open transaction
- `execute_query` (and related query methods) — run on the open transaction until commit/rollback
- `commit_manual_transaction` / `rollback_manual_transaction` — end the interactive transaction
This is separate from one-shot `execute_transaction`, which begins, runs a statement list, and commits/rolls back inside a single RPC. Runtimes that reconnect a session (`validate_session`) must clear any open manual transaction on that session.
`agentSessionId` identifies a logical database connection. Existing `sessionId` fields remain pagination cursor identifiers and must not be used as logical connection identifiers.
`sessionRole` is `workload` by default. DBX sends `metadata` for object-tree, completion, and other read-only metadata sessions. New runtimes use this role to preserve metadata checkout capacity; older runtimes may ignore the field.
+2 -1
View File
@@ -358,7 +358,8 @@ func (s *server) dispatch(method string, params map[string]json.RawMessage) (any
case "list_foreign_keys":
return []foreignKeyInfo{}, false, nil
case "list_triggers":
return []triggerInfo{}, false, nil
result, err := s.listTriggers(stringParam(params, "schema"), stringParam(params, "table"))
return result, false, err
case "get_object_source":
return nil, false, errors.New("object source is not supported by Cassandra")
case "get_table_ddl":
+38
View File
@@ -82,6 +82,15 @@ type triggerInfo struct {
Timing string `json:"timing"`
}
const cassandraListTriggersCQL = "SELECT trigger_name, options FROM system_schema.triggers WHERE keyspace_name = ? AND table_name = ?"
type metadataIterator interface {
Scan(...any) bool
Close() error
}
type metadataQuery func(string, ...any) metadataIterator
type metadataListConstraints struct {
Filter string
Limit int
@@ -321,6 +330,35 @@ func (s *server) querySystemIndexes(schema, table string) ([]indexInfo, error) {
return result, nil
}
func (s *server) listTriggers(schema, table string) ([]triggerInfo, error) {
session, err := s.runtime.sessionFor("")
if err != nil {
return nil, err
}
return listTriggersWithQuery(func(query string, values ...any) metadataIterator {
return session.Query(query, values...).Iter()
}, schema, table)
}
func listTriggersWithQuery(query metadataQuery, schema, table string) ([]triggerInfo, error) {
iter := query(cassandraListTriggersCQL, schema, table)
result := []triggerInfo{}
var name string
var options map[string]string
for iter.Scan(&name, &options) {
// Cassandra trigger metadata stores only the trigger class. Cassandra
// invokes every trigger before any DML statement, so expose those fixed
// semantics through the common TriggerInfo contract.
result = append(result, triggerInfo{Name: name, Event: "DML", Timing: "BEFORE"})
options = nil
}
if err := iter.Close(); err != nil {
return nil, err
}
sort.Slice(result, func(left, right int) bool { return result[left].Name < result[right].Name })
return result, nil
}
func mergeIndexes(first, second []indexInfo) []indexInfo {
byName := make(map[string]indexInfo, len(first)+len(second))
for _, index := range first {
@@ -1,12 +1,37 @@
package main
import (
"errors"
"reflect"
"testing"
gocql "github.com/apache/cassandra-gocql-driver/v2"
)
type triggerMetadataIterator struct {
rows []struct {
name string
options map[string]string
}
index int
closeErr error
}
func (i *triggerMetadataIterator) Scan(dest ...any) bool {
if i.index >= len(i.rows) {
return false
}
row := i.rows[i.index]
i.index++
*dest[0].(*string) = row.name
*dest[1].(*map[string]string) = row.options
return true
}
func (i *triggerMetadataIterator) Close() error {
return i.closeErr
}
func TestColumnsIndexesAndDDLFromMetadata(t *testing.T) {
textType := gocql.NewNativeType(4, gocql.TypeVarchar, "")
intType := gocql.NewNativeType(4, gocql.TypeInt, "")
@@ -84,3 +109,53 @@ func TestQuoteCQLIdentifierEscapesQuotes(t *testing.T) {
t.Fatalf("unexpected quoted identifier: %s", got)
}
}
func TestListTriggersQueriesExactTableAndMapsMetadata(t *testing.T) {
var statement string
var values []any
triggers, err := listTriggersWithQuery(func(query string, args ...any) metadataIterator {
statement = query
values = args
return &triggerMetadataIterator{rows: []struct {
name string
options map[string]string
}{
{name: "capture_changes", options: map[string]string{"class": "example.CaptureDataTrigger"}},
}}
}, "dev", "example")
if err != nil {
t.Fatal(err)
}
if statement != cassandraListTriggersCQL {
t.Fatalf("unexpected trigger query: %s", statement)
}
if !reflect.DeepEqual(values, []any{"dev", "example"}) {
t.Fatalf("trigger filters = %#v", values)
}
want := []triggerInfo{{Name: "capture_changes", Event: "DML", Timing: "BEFORE"}}
if !reflect.DeepEqual(triggers, want) {
t.Fatalf("triggers = %#v, want %#v", triggers, want)
}
}
func TestListTriggersReturnsEmptySlice(t *testing.T) {
triggers, err := listTriggersWithQuery(func(string, ...any) metadataIterator {
return &triggerMetadataIterator{}
}, "dev", "empty_table")
if err != nil {
t.Fatal(err)
}
if triggers == nil || len(triggers) != 0 {
t.Fatalf("triggers = %#v, want a non-nil empty slice", triggers)
}
}
func TestListTriggersReturnsQueryError(t *testing.T) {
wantErr := errors.New("trigger metadata unavailable")
triggers, err := listTriggersWithQuery(func(string, ...any) metadataIterator {
return &triggerMetadataIterator{closeErr: wantErr}
}, "dev", "example")
if !errors.Is(err, wantErr) || triggers != nil {
t.Fatalf("triggers = %#v, err = %v", triggers, err)
}
}
@@ -7,6 +7,7 @@ import com.dbx.agent.DatabaseInfo;
import com.dbx.agent.ExecuteQueryOptions;
import com.dbx.agent.ForeignKeyInfo;
import com.dbx.agent.IndexInfo;
import com.dbx.agent.JdbcAgentProfile;
import com.dbx.agent.JdbcExecutor;
import com.dbx.agent.JdbcIdentifiers;
import com.dbx.agent.MultiSessionJsonRpcServer;
@@ -18,14 +19,18 @@ import com.dbx.agent.QueryPageResult;
import com.dbx.agent.QueryResult;
import com.dbx.agent.TableInfo;
import com.dbx.agent.TriggerInfo;
import dm.jdbc.driver.DmdbConnection;
import com.dbx.agent.StandardJdbcMetadata;
import java.io.PrintStream;
import java.io.Reader;
import java.net.URL;
import java.net.URLClassLoader;
import java.net.SocketTimeoutException;
import java.nio.file.Paths;
import java.sql.Blob;
import java.sql.Clob;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.Driver;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
@@ -49,16 +54,26 @@ import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.WeakHashMap;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public final class DamengAgent extends AbstractJdbcAgent {
private static final Logger LOGGER = Logger.getLogger("com.dbx.agent.dameng");
private static final String AGENT_VERSION = "9999.06.04.1-fix-default";
private static final int DBMS_OUTPUT_ENABLE_TIMEOUT_SECS = 5;
private static final int DBMS_OUTPUT_ENABLE_NETWORK_TIMEOUT_MILLIS = 5_000;
private static final Pattern DATABASE_VERSION_MAJOR_PATTERN = Pattern.compile("(\\d+)\\.");
private static final JdbcAgentProfile DM6_METADATA_PROFILE = new JdbcAgentProfile(
"dm6.jdbc.driver.DmDriver",
"jdbc:dm6://{host}:{port}/{database}",
5236,
true
);
private static final String DAMENG_CLASSIFIED_OBJECT_TYPE_SQL =
"CASE WHEN o.OBJECT_TYPE = 'MATERIALIZED VIEW' OR (o.OBJECT_TYPE = 'VIEW' AND mv.MVIEW_NAME IS NOT NULL) "
+ "THEN 'MATERIALIZED_VIEW' ELSE o.OBJECT_TYPE END";
@@ -90,6 +105,11 @@ public final class DamengAgent extends AbstractJdbcAgent {
) mv ON mv.OWNER = o.OWNER AND mv.MVIEW_NAME = o.OBJECT_NAME
""".stripIndent().trim();
private String connectedUsername;
private Driver externalDriver;
private URLClassLoader externalDriverLoader;
private List<URL> externalDriverUrls;
private String externalDriverClass;
private volatile boolean legacyJdbcMetadata;
private volatile boolean dbmsOutputInitializationSupported = true;
private final Map<Object, Boolean> dbmsOutputInitializedConnections =
Collections.synchronizedMap(new WeakHashMap<>());
@@ -106,19 +126,120 @@ public final class DamengAgent extends AbstractJdbcAgent {
@Override
protected void loadDriver(ConnectParams params) throws Exception {
withSuppressedStdout(() -> super.loadDriver(params));
withSuppressedStdout(() -> {
if (params.getJdbc_driver_paths() == null || params.getJdbc_driver_paths().isEmpty()) {
// Keep an existing external loader open: pooled connections may
// still hold classes loaded from it. Only stop using it here.
externalDriver = null;
super.loadDriver(params);
return;
}
List<URL> urls = new ArrayList<>();
for (String path : params.getJdbc_driver_paths()) {
urls.add(Paths.get(path).toUri().toURL());
}
String driverClass = params.getJdbc_driver_class();
if (driverClass == null || driverClass.trim().isEmpty()) {
driverClass = driverClass();
}
// Reuse the loader for unchanged paths and class: recreating it
// would strand classes that live pooled connections still need
// (lazy driver classes would fail with NoClassDefFoundError). A
// loader dropped for changed paths is never closed here for the
// same reason; disconnect() releases the active one.
if (externalDriverLoader != null
&& urls.equals(externalDriverUrls)
&& driverClass.equals(externalDriverClass)) {
if (externalDriver == null) {
externalDriver = (Driver) Class.forName(driverClass, true, externalDriverLoader)
.getDeclaredConstructor()
.newInstance();
}
return;
}
externalDriverLoader = new URLClassLoader(
urls.toArray(new URL[0]),
ClassLoader.getPlatformClassLoader()
);
externalDriverUrls = urls;
externalDriverClass = driverClass;
externalDriver = (Driver) Class.forName(driverClass, true, externalDriverLoader)
.getDeclaredConstructor()
.newInstance();
});
}
@Override
protected Connection openConnection(ConnectParams params) throws Exception {
return withSuppressedStdout(
() -> DriverManager.getConnection(buildUrl(params), params.getUsername(), params.getPassword())
);
return withSuppressedStdout(() -> {
if (externalDriver == null) {
return DriverManager.getConnection(buildUrl(params), params.getUsername(), params.getPassword());
}
Properties properties = new Properties();
if (params.getUsername() != null) {
properties.setProperty("user", params.getUsername());
}
if (params.getPassword() != null) {
properties.setProperty("password", params.getPassword());
}
Connection connection = externalDriver.connect(buildUrl(params), properties);
if (connection == null) {
throw new SQLException("Selected Dameng JDBC driver does not accept URL: " + buildUrl(params));
}
return connection;
});
}
@Override
protected String connectionValidationQuery() {
return "SELECT 1";
}
@Override
public synchronized void disconnect() {
super.disconnect();
try {
closeExternalDriverLoader();
} catch (Exception error) {
// Best-effort release during teardown.
}
}
private void closeExternalDriverLoader() throws Exception {
externalDriver = null;
externalDriverUrls = null;
externalDriverClass = null;
if (externalDriverLoader != null) {
externalDriverLoader.close();
externalDriverLoader = null;
}
}
@Override
protected void afterConnect(ConnectParams params, Connection connection) {
connectedUsername = params.getUsername();
legacyJdbcMetadata = usesLegacyJdbcMetadata(connection);
}
static boolean usesLegacyJdbcMetadata(Connection connection) {
try {
int major = connection.getMetaData().getDatabaseMajorVersion();
if (major > 0) {
return major < 7;
}
} catch (Exception | AbstractMethodError ignored) {
}
try {
String version = connection.getMetaData().getDatabaseProductVersion();
if (version == null) {
return false;
}
Matcher matcher = DATABASE_VERSION_MAJOR_PATTERN.matcher(version);
return matcher.find() && Integer.parseInt(matcher.group(1)) < 7;
} catch (Exception | AbstractMethodError ignored) {
return false;
}
}
@Override
@@ -127,12 +248,12 @@ public final class DamengAgent extends AbstractJdbcAgent {
}
private void initializeDbmsOutputIfNeeded(String sql) throws SQLException {
if (!dbmsOutputInitializationSupported || !isDbmsOutputStatement(sql)) {
if (legacyJdbcMetadata || !dbmsOutputInitializationSupported || !isDbmsOutputStatement(sql)) {
return;
}
Connection connection = requireConnected();
Object physicalConnection = connection.unwrap(DmdbConnection.class);
Object physicalConnection = physicalConnectionIdentity(connection);
synchronized (dbmsOutputInitializedConnections) {
if (dbmsOutputInitializedConnections.containsKey(physicalConnection)) {
return;
@@ -190,6 +311,22 @@ public final class DamengAgent extends AbstractJdbcAgent {
|| startsWithKeyword(sql, start, "EXECUTE");
}
private Object physicalConnectionIdentity(Connection connection) {
try {
String driverClassName = externalDriver == null ? driverClass() : externalDriver.getClass().getName();
int packageSeparator = driverClassName.lastIndexOf('.');
if (packageSeparator < 0) {
return connection;
}
String connectionClassName = driverClassName.substring(0, packageSeparator + 1) + "DmdbConnection";
ClassLoader loader = externalDriverLoader == null ? getClass().getClassLoader() : externalDriverLoader;
Class<?> connectionClass = Class.forName(connectionClassName, false, loader);
return connection.unwrap(connectionClass);
} catch (Exception | AbstractMethodError ignored) {
return connection;
}
}
private static boolean startsWithKeyword(String sql, int start, String keyword) {
int end = start + keyword.length();
return end <= sql.length()
@@ -321,11 +458,17 @@ public final class DamengAgent extends AbstractJdbcAgent {
@Override
public List<DatabaseInfo> listDatabases() {
if (legacyJdbcMetadata) {
return unchecked(() -> listJdbcSchemas().stream().map(DatabaseInfo::new).toList());
}
return unchecked(() -> listVisibleUsers().stream().map(DatabaseInfo::new).toList());
}
@Override
public List<String> listSchemas() {
if (legacyJdbcMetadata) {
return StandardJdbcMetadata.INSTANCE.listSchemas(requireConnected(), DM6_METADATA_PROFILE);
}
return unchecked(() -> {
try {
return listVisibleSchemas();
@@ -395,6 +538,9 @@ public final class DamengAgent extends AbstractJdbcAgent {
}
private List<TableInfo> queryConstrainedTables(String schema, MetadataListConstraints constraints) {
if (legacyJdbcMetadata) {
return executeJdbcMetadataTables(schema, constraints);
}
if (!constraints.includesTableLikeTypes()) {
return List.of();
}
@@ -913,6 +1059,9 @@ public final class DamengAgent extends AbstractJdbcAgent {
}
private List<ObjectInfo> queryConstrainedObjects(String schema, MetadataListConstraints constraints) {
if (legacyJdbcMetadata) {
return executeJdbcMetadataObjects(schema, constraints);
}
if (!includesSupportedObjectTypes(constraints)) {
return List.of();
}
@@ -1451,6 +1600,9 @@ public final class DamengAgent extends AbstractJdbcAgent {
@Override
public String getTableDdl(String schema, String table) {
if (legacyJdbcMetadata) {
return super.getTableDdl(schema, table);
}
try {
return unchecked(() -> {
String sql = "SELECT /*+ PARALLEL(1) */ DBMS_METADATA.GET_DDL(?, ?, ?) FROM DUAL";
@@ -1486,6 +1638,15 @@ public final class DamengAgent extends AbstractJdbcAgent {
@Override
public List<ColumnInfo> getColumns(String schema, String table) {
if (legacyJdbcMetadata) {
return StandardJdbcMetadata.INSTANCE.getColumns(
requireConnected(),
DM6_METADATA_PROFILE,
getConfiguredDatabase(),
schema,
table
);
}
return unchecked(() -> {
Set<String> pkColumns = new java.util.HashSet<>();
String pkSql = """
@@ -1590,6 +1751,15 @@ public final class DamengAgent extends AbstractJdbcAgent {
@Override
public List<IndexInfo> listIndexes(String schema, String table) {
if (legacyJdbcMetadata) {
return StandardJdbcMetadata.INSTANCE.listIndexes(
requireConnected(),
DM6_METADATA_PROFILE,
getConfiguredDatabase(),
schema,
table
);
}
return unchecked(() -> {
List<IndexInfo> result = new ArrayList<>();
String sql = """
@@ -1630,6 +1800,9 @@ public final class DamengAgent extends AbstractJdbcAgent {
@Override
public List<ForeignKeyInfo> listForeignKeys(String schema, String table) {
if (legacyJdbcMetadata) {
return StandardJdbcMetadata.INSTANCE.listForeignKeys(requireConnected(), schema, table);
}
return unchecked(() -> {
List<ForeignKeyInfo> result = new ArrayList<>();
String sql = """
@@ -1661,6 +1834,9 @@ public final class DamengAgent extends AbstractJdbcAgent {
@Override
public List<TriggerInfo> listTriggers(String schema, String table) {
if (legacyJdbcMetadata) {
return StandardJdbcMetadata.INSTANCE.listTriggers(schema, table);
}
return unchecked(() -> {
List<TriggerInfo> result = new ArrayList<>();
String sql = """
@@ -1722,7 +1898,7 @@ public final class DamengAgent extends AbstractJdbcAgent {
}
String message = (String) value;
return message.isEmpty() ? List.of() : message.lines().toList();
} catch (Exception ignored) {
} catch (Exception | AbstractMethodError ignored) {
return List.of();
}
}
@@ -1921,6 +2097,11 @@ public final class DamengAgent extends AbstractJdbcAgent {
}
private static String buildUrl(ConnectParams params) {
String connectionString = params.getConnection_string() == null ? "" : params.getConnection_string().trim();
if (!connectionString.isEmpty()
&& !connectionString.regionMatches(true, 0, "dm://", 0, "dm://".length())) {
return connectionString;
}
String database = params.getDatabase() == null ? "" : params.getDatabase().trim();
String suffix = database.isEmpty() ? "" : "/" + database;
String url = "jdbc:dm://" + params.getHost() + ":" + params.getPort() + suffix;
@@ -1,6 +1,7 @@
package com.dbx.agent.dameng;
import com.dbx.agent.ColumnInfo;
import com.dbx.agent.DatabaseInfo;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.ObjectInfo;
import com.dbx.agent.ObjectSource;
@@ -25,6 +26,31 @@ import java.util.Arrays;
import java.util.List;
class DamengAgentMetadataTest {
@Test
void detectsLegacyDamengMetadataFromDatabaseVersion() {
Assertions.assertTrue(DamengAgent.usesLegacyJdbcMetadata(versionConnection(6, "6.0.2.79")));
Assertions.assertFalse(DamengAgent.usesLegacyJdbcMetadata(versionConnection(8, "8.1.5.45")));
Assertions.assertTrue(DamengAgent.usesLegacyJdbcMetadata(versionConnection(0, "DM Database Server x64 V6.0.2.79")));
}
@Test
void legacyDamengUsesJdbcMetadataWithoutQueryingDm8CatalogViews() {
DamengAgent agent = new DamengAgent();
List<String> metadataCalls = new ArrayList<>();
TestSupport.setPrivateConnection(agent, legacyMetadataConnection(metadataCalls));
setLegacyJdbcMetadata(agent, true);
List<DatabaseInfo> databases = agent.listDatabases();
List<String> schemas = agent.listSchemas();
List<TableInfo> tables = agent.listTables("DBX_TEST");
Assertions.assertEquals(List.of("DBX_TEST", "SYSDBA"), databases.stream().map(DatabaseInfo::getName).toList());
Assertions.assertEquals(List.of("DBX_TEST", "SYSDBA"), schemas);
Assertions.assertEquals(List.of("CONNECTION_SMOKE"), tables.stream().map(TableInfo::getName).toList());
Assertions.assertTrue(metadataCalls.contains("getSchemas"), metadataCalls.toString());
Assertions.assertTrue(metadataCalls.contains("getTables:DBX\\_TEST"), metadataCalls.toString());
}
@Test
void usesColumnCommentsMetadataQuery() {
DamengAgent agent = new DamengAgent();
@@ -1335,6 +1361,56 @@ class DamengAgentMetadataTest {
}
}
private static void setLegacyJdbcMetadata(DamengAgent agent, boolean value) {
try {
Field field = DamengAgent.class.getDeclaredField("legacyJdbcMetadata");
field.setAccessible(true);
field.set(agent, value);
} catch (ReflectiveOperationException e) {
throw new IllegalStateException("Unable to set legacy JDBC metadata mode", e);
}
}
private static Connection versionConnection(int majorVersion, String productVersion) {
DatabaseMetaData metadata = proxy(DatabaseMetaData.class, (method, args) -> switch (method.getName()) {
case "getDatabaseMajorVersion" -> majorVersion;
case "getDatabaseProductVersion" -> productVersion;
default -> defaultValue(method.getReturnType());
});
return proxy(Connection.class, (method, args) ->
"getMetaData".equals(method.getName()) ? metadata : defaultValue(method.getReturnType())
);
}
private static Connection legacyMetadataConnection(List<String> calls) {
DatabaseMetaData metadata = proxy(DatabaseMetaData.class, (method, args) -> {
switch (method.getName()) {
case "getSchemas":
calls.add("getSchemas");
return metadataResultSet(List.of(List.of("SYSDBA"), List.of("DBX_TEST")));
case "getTableTypes":
return metadataResultSet(List.of(List.of("", "TABLE", "")));
case "getTables":
Assertions.assertNull(args[3], "DM6 JDBC rejects non-null getTables types");
calls.add("getTables:" + args[1]);
return metadataResultSet(List.of(List.of("CONNECTION_SMOKE", "TABLE", "smoke table")));
case "getSearchStringEscape":
return "\\";
default:
return defaultValue(method.getReturnType());
}
});
return proxy(Connection.class, (method, args) -> {
if ("getMetaData".equals(method.getName())) {
return metadata;
}
if ("prepareStatement".equals(method.getName())) {
throw new AssertionError("Legacy metadata mode must not query DM8 catalog views: " + args[0]);
}
return defaultValue(method.getReturnType());
});
}
private static Connection objectSourceConnection(List<String> params, String source) {
return proxy(Connection.class, (method, args) -> {
String name = method.getName();
@@ -12,6 +12,7 @@ import com.dbx.agent.test.JdbcAgentFake;
import com.dbx.agent.test.TestSupport;
import org.junit.jupiter.api.Test;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
@@ -100,6 +101,18 @@ class DamengAgentTest extends JdbcFakeExecutionBehaviorTest {
assertDoesNotEnableDbmsOutput("/* CALL hidden in a comment */ SELECT 1");
}
@Test
void legacyDamengSkipsUnavailableDbmsOutputInitialization() {
List<String> executedSql = new ArrayList<>();
DamengAgent agent = new DamengAgent();
TestSupport.setPrivateConnection(agent, printMessageConnection(null, executedSql));
setLegacyJdbcMetadata(agent, true);
agent.executeQuery("BEGIN NULL; END;", null, new ExecuteQueryOptions());
assertEquals(List.of("BEGIN NULL; END;"), executedSql);
}
@Test
void outputStatementsEnableDbmsOutputOncePerPhysicalConnection() {
List<String> firstConnectionSql = new ArrayList<>();
@@ -216,6 +229,22 @@ class DamengAgentTest extends JdbcFakeExecutionBehaviorTest {
), executedSql);
}
@Test
void legacyStatementWithoutJdbc4UnwrapReturnsNoPrintMessages() {
Statement statement = (Statement) Proxy.newProxyInstance(
DamengAgentTest.class.getClassLoader(),
new Class<?>[]{Statement.class},
(unused, method, args) -> {
if ("unwrap".equals(method.getName())) {
throw new AbstractMethodError("legacy JDBC driver");
}
return defaultValue(method.getReturnType());
}
);
assertEquals(List.of(), DamengAgent.statementPrintMessages(statement));
}
@Test
void pagedOutputStatementsEnableDbmsOutputLazily() {
List<String> executedSql = new ArrayList<>();
@@ -620,6 +649,16 @@ class DamengAgentTest extends JdbcFakeExecutionBehaviorTest {
assertEquals(List.of("BEGIN DBMS_OUTPUT.ENABLE(1000000); END;"), executedSql);
}
private static void setLegacyJdbcMetadata(DamengAgent agent, boolean value) {
try {
Field field = DamengAgent.class.getDeclaredField("legacyJdbcMetadata");
field.setAccessible(true);
field.set(agent, value);
} catch (ReflectiveOperationException error) {
throw new IllegalStateException("Unable to set legacy JDBC metadata mode", error);
}
}
private static Object defaultValue(Class<?> type) {
if (type == Boolean.TYPE) return false;
if (type == Byte.TYPE) return (byte) 0;
@@ -21,6 +21,42 @@ class DamengAgentUrlTest {
Assertions.assertEquals("jdbc:dm://127.0.0.1:5236/MAIN", url);
}
@Test
void rebuildsLegacyDbxUrlFromConnectionFields() throws Exception {
ConnectParams params = new ConnectParams(
"127.0.0.1",
5236,
"MAIN",
"SYSDBA",
"pwd",
"",
"dm://SYSDBA:SYSDBA@legacy-host:5236/OLD",
false
);
String url = invokeBuildUrl(params);
Assertions.assertEquals("jdbc:dm://127.0.0.1:5236/MAIN", url);
}
@Test
void usesCompleteCustomJdbcUrlWithoutRewritingIt() throws Exception {
ConnectParams params = new ConnectParams(
"ignored",
5236,
"IGNORED",
"",
"",
"ssl=true",
"jdbc:dm6://dm6.internal:5237/MAIN?compatibleMode=oracle",
false
);
String url = invokeBuildUrl(params);
Assertions.assertEquals("jdbc:dm6://dm6.internal:5237/MAIN?compatibleMode=oracle", url);
}
@Test
void appendsDmJdbcUrlParameters() throws Exception {
String url = invokeBuildUrl(new ConnectParams(
+157 -8
View File
@@ -875,6 +875,35 @@ dependencies = [
"unicode-segmentation",
]
[[package]]
name = "cookie"
version = "0.18.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a373e3602691c3cdea496d2f0ee5935151e6168fe87739483c463db1b2f2f87"
dependencies = [
"percent-encoding",
"time",
"version_check",
]
[[package]]
name = "cookie_store"
version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15b2c103cf610ec6cae3da84a766285b42fd16aad564758459e6ecf128c75206"
dependencies = [
"cookie",
"document-features",
"idna",
"log",
"publicsuffix",
"serde",
"serde_derive",
"serde_json",
"time",
"url",
]
[[package]]
name = "core-foundation"
version = "0.9.4"
@@ -1227,10 +1256,13 @@ dependencies = [
"minijinja",
"mongodb",
"mysql_async",
"nix",
"notify",
"openssl",
"pageant",
"percent-encoding",
"portpicker",
"postgres-openssl",
"prometheus-parse",
"quick-xml",
"rayon",
@@ -1468,6 +1500,15 @@ dependencies = [
"syn 3.0.3",
]
[[package]]
name = "document-features"
version = "0.2.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61"
dependencies = [
"litrs",
]
[[package]]
name = "duckdb"
version = "1.10505.0"
@@ -1621,7 +1662,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -1725,6 +1766,21 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
[[package]]
name = "foreign-types"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
dependencies = [
"foreign-types-shared",
]
[[package]]
name = "foreign-types-shared"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
[[package]]
name = "form_urlencoded"
version = "1.2.2"
@@ -2784,7 +2840,6 @@ version = "0.30.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149"
dependencies = [
"cc",
"pkg-config",
"vcpkg",
]
@@ -2807,6 +2862,12 @@ version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0"
[[package]]
name = "litrs"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092"
[[package]]
name = "lock_api"
version = "0.4.14"
@@ -3231,7 +3292,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -3357,6 +3418,31 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
[[package]]
name = "openssl"
version = "0.10.81"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45"
dependencies = [
"bitflags 2.13.1",
"cfg-if",
"foreign-types",
"libc",
"openssl-macros",
"openssl-sys",
]
[[package]]
name = "openssl-macros"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "openssl-probe"
version = "0.1.6"
@@ -3369,6 +3455,28 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
[[package]]
name = "openssl-src"
version = "300.6.1+3.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46eb8fb9fb3b61ce1c0f8a026c4c1a0714d3a9e138e7fbde78753ce2babc3846"
dependencies = [
"cc",
]
[[package]]
name = "openssl-sys"
version = "0.9.117"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695"
dependencies = [
"cc",
"libc",
"openssl-src",
"pkg-config",
"vcpkg",
]
[[package]]
name = "p256"
version = "0.14.0-rc.7"
@@ -3679,6 +3787,18 @@ dependencies = [
"rand 0.8.7",
]
[[package]]
name = "postgres-openssl"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06743eefaa1a5c0ef2ccb6d9abf6528790a229eabd62ddcabf9b2a3aeff09fa4"
dependencies = [
"openssl",
"tokio",
"tokio-openssl",
"tokio-postgres",
]
[[package]]
name = "postgres-protocol"
version = "0.6.12"
@@ -3817,6 +3937,12 @@ dependencies = [
"regex",
]
[[package]]
name = "psl-types"
version = "2.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac"
[[package]]
name = "psm"
version = "0.1.31"
@@ -3847,6 +3973,16 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "publicsuffix"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f42ea446cab60335f76979ec15e12619a2165b5ae2c12166bef27d283a9fadf"
dependencies = [
"idna",
"psl-types",
]
[[package]]
name = "quick-xml"
version = "0.37.5"
@@ -3910,7 +4046,7 @@ dependencies = [
"once_cell",
"socket2",
"tracing",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -4149,6 +4285,8 @@ checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147"
dependencies = [
"base64 0.22.1",
"bytes",
"cookie",
"cookie_store",
"futures-core",
"futures-util",
"http",
@@ -4467,7 +4605,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys 0.12.1",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -5269,10 +5407,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
dependencies = [
"fastrand",
"getrandom 0.4.3",
"getrandom 0.3.4",
"once_cell",
"rustix 1.1.4",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -5476,6 +5614,17 @@ dependencies = [
"syn 2.0.119",
]
[[package]]
name = "tokio-openssl"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59df6849caa43bb7567f9a36f863c447d95a11d5903c9cc334ba32576a27eadd"
dependencies = [
"openssl",
"openssl-sys",
"tokio",
]
[[package]]
name = "tokio-postgres"
version = "0.7.18"
@@ -6105,7 +6254,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
+1 -3
View File
@@ -23,7 +23,6 @@ impl DuckDbConnection {
}
// Preserve PoisonError ownership so close_connection can still release the contained DuckDB handle.
#[cfg(test)]
#[allow(clippy::result_large_err)]
fn into_inner(self) -> std::sync::LockResult<duckdb::Connection> {
self.connection.into_inner()
@@ -116,8 +115,7 @@ pub fn is_memory_database_path(path: &str) -> bool {
/// Unlike relying on Drop, this calls `duckdb_disconnect` synchronously
/// so the file handle is released before this function returns.
/// On Windows this prevents "file already in use" errors when reconnecting.
#[cfg(test)]
fn close_connection(con: Arc<DuckDbConnection>) {
pub(crate) fn close_connection(con: Arc<DuckDbConnection>) {
match Arc::try_unwrap(con) {
Ok(handle) => match handle.into_inner() {
Ok(conn) => {
+17
View File
@@ -144,6 +144,21 @@ mod tests {
);
}
#[test]
fn duckdb_execute_resolves_current_date_arithmetic() {
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
let result = duckdb_execute(&con, "SELECT CURRENT_DATE - 4 AS d").expect("execute CURRENT_DATE - literal");
let expected = (Utc::now().date_naive() - ChronoDuration::days(4)).to_string();
assert_eq!(result.rows[0][0], serde_json::Value::String(expected));
con.execute_batch("CREATE VIEW recent AS SELECT CURRENT_DATE - 4 AS start_date, CURRENT_DATE - 1 AS end_date")
.expect("create view using CURRENT_DATE arithmetic");
let view_result =
duckdb_execute(&con, "SELECT * FROM recent").expect("query view built on CURRENT_DATE arithmetic");
assert_eq!(view_result.rows.len(), 1);
}
#[test]
fn duckdb_execute_formats_temporal_values_by_column_type() {
let con = duckdb::Connection::open_in_memory().expect("connect in-memory DuckDB");
@@ -378,6 +393,8 @@ pub fn duckdb_execute_with_max_rows(
) -> Result<db::QueryResult, String> {
let start = std::time::Instant::now();
let row_limit = query_result_row_limit(max_rows);
let sql = crate::sql::rewrite_duckdb_current_date_literal_arithmetic(sql);
let sql = sql.as_ref();
if crate::sql::starts_with_duckdb_result_sql_keyword(sql) {
let mut stmt = con.prepare(sql).map_err(|e| e.to_string())?;
+57 -5
View File
@@ -148,6 +148,27 @@ impl DuckDbWorkerSession {
Ok(connection.interrupt_handle())
}
/// Takes and closes the connection so DuckDB's shutdown checkpoint runs and
/// the database WAL is removed before this worker exits. Returns false —
/// leaving the connection untouched — when the connection is poisoned
/// (dropping it aborts the process) so the caller falls back to an
/// immediate exit and the WAL replays on the next open.
fn close_connection_for_shutdown(&mut self) -> bool {
if self.connection.is_none() {
return true;
}
if !self.is_connection_healthy() {
return false;
}
match self.connection.take() {
Some(connection) => {
crate::connection::close_connection(connection);
true
}
None => true,
}
}
/// Probes whether the connection is still usable after an execute error.
///
/// duckdb-rs 1.10503.1 has a bug where `Connection::prepare()` failing at the
@@ -205,6 +226,21 @@ struct WorkerHandleResult {
}
impl DuckDbWorkerRuntime {
/// Closes the session connection when the worker is idle and the connection
/// is healthy, so the DuckDB shutdown checkpoint removes the WAL. Returns
/// false when a query is running (the session is locked or an interrupt is
/// registered) or the connection is poisoned; callers then keep the legacy
/// immediate-exit behavior and rely on WAL replay at the next open.
fn close_session_for_shutdown(&self) -> bool {
if self.active_interrupt.lock().unwrap_or_else(|e| e.into_inner()).is_some() {
return false;
}
match self.session.try_lock() {
Ok(mut session) => session.close_connection_for_shutdown(),
Err(_) => false,
}
}
async fn handle_request(
&self,
request: DuckDbWorkerRequest,
@@ -327,10 +363,18 @@ impl DuckDbWorkerRuntime {
shutdown: false,
}
}
DuckDbWorkerMethod::Shutdown => WorkerHandleResult {
response: Some(DuckDbWorkerResponse::ok(request.id, serde_json::json!({ "shutdown": true }))),
shutdown: true,
},
DuckDbWorkerMethod::Shutdown => {
// Close the connection before acking: the checkpoint that removes
// the WAL must finish before the parent observes this response
// (or the process exit it waits for). Busy or poisoned sessions
// skip the close; the parent's kill fallback and WAL replay on
// the next open cover them.
self.close_session_for_shutdown();
WorkerHandleResult {
response: Some(DuckDbWorkerResponse::ok(request.id, serde_json::json!({ "shutdown": true }))),
shutdown: true,
}
}
}
}
@@ -377,7 +421,15 @@ pub async fn run_stdio_worker() -> Result<(), String> {
loop {
let line = lines.next_line().await.map_err(|e| e.to_string())?;
let Some(line) = line else {
std::process::exit(0);
// Parent closed stdin (exit or crash). Idle workers checkpoint and
// remove their WAL on the normal return path; a worker still running
// a query must not wait for it (nor drop a poisoned connection),
// so it keeps the legacy immediate exit and the WAL replays on the
// next open.
if !runtime.close_session_for_shutdown() {
std::process::exit(0);
}
break;
};
if line.trim().is_empty() {
continue;
+188
View File
@@ -193,6 +193,127 @@ fn parse_identifier_after_as(input: &str) -> Option<String> {
(!name.is_empty()).then(|| name.to_string())
}
/// Works around a DuckDB binder bug: `CURRENT_DATE - 4` (a bare numeric literal directly
/// against the `CURRENT_DATE` keyword node) fails with "No function matches ... (DATE,
/// INTEGER_LITERAL)", while the functionally identical `current_date() - 4` binds fine.
/// Rewrites only that narrow pattern (`CURRENT_DATE` immediately followed by `+`/`-` and a raw
/// numeric literal) to the function-call form, so plain `CURRENT_DATE`, qualified references
/// (`t.current_date`), aliases (`AS current_date`), and already-cast literals are left untouched.
pub fn rewrite_duckdb_current_date_literal_arithmetic(sql: &str) -> std::borrow::Cow<'_, str> {
const KEYWORD: &str = "CURRENT_DATE";
let bytes = sql.as_bytes();
let mut rewritten: Option<String> = None;
let mut copied_to = 0usize;
let mut index = 0usize;
let mut in_single = false;
let mut in_double = false;
while index < bytes.len() {
match bytes[index] {
b'\'' if !in_double => {
if in_single && bytes.get(index + 1) == Some(&b'\'') {
index += 2;
continue;
}
in_single = !in_single;
index += 1;
continue;
}
b'"' if !in_single => {
if in_double && bytes.get(index + 1) == Some(&b'"') {
index += 2;
continue;
}
in_double = !in_double;
index += 1;
continue;
}
b'-' if !in_single && !in_double && bytes.get(index + 1) == Some(&b'-') => {
index += 2;
while index < bytes.len() && bytes[index] != b'\n' {
index += 1;
}
continue;
}
b'/' if !in_single && !in_double && bytes.get(index + 1) == Some(&b'*') => {
index += 2;
while index + 1 < bytes.len() && !(bytes[index] == b'*' && bytes[index + 1] == b'/') {
index += 1;
}
index = (index + 2).min(bytes.len());
continue;
}
b'$' if !in_single && !in_double => {
if let Some(end) = dollar_quote_end(bytes, index) {
index = end;
continue;
}
index += 1;
}
_ if !in_single && !in_double && matches_keyword_at(bytes, index, KEYWORD) => {
let word_end = index + KEYWORD.len();
let preceded_by_dot = index > 0 && bytes[index - 1] == b'.';
let preceded_by_ident = index > 0 && is_ident_byte(bytes[index - 1]);
let followed_by_ident = bytes.get(word_end).copied().is_some_and(is_ident_byte);
if !preceded_by_dot
&& !preceded_by_ident
&& !followed_by_ident
&& has_raw_numeric_arithmetic_after(bytes, word_end)
{
let buf = rewritten.get_or_insert_with(String::new);
buf.push_str(&sql[copied_to..index]);
buf.push_str("current_date()");
copied_to = word_end;
}
index = word_end;
continue;
}
_ => {
index += 1;
}
}
}
match rewritten {
Some(mut buf) => {
buf.push_str(&sql[copied_to..]);
std::borrow::Cow::Owned(buf)
}
None => std::borrow::Cow::Borrowed(sql),
}
}
fn matches_keyword_at(bytes: &[u8], index: usize, keyword: &str) -> bool {
let keyword_bytes = keyword.as_bytes();
bytes.len() >= index + keyword_bytes.len()
&& bytes[index..index + keyword_bytes.len()].eq_ignore_ascii_case(keyword_bytes)
}
fn is_ident_byte(byte: u8) -> bool {
byte.is_ascii_alphanumeric() || byte == b'_'
}
/// After the `CURRENT_DATE` token: skip whitespace, require a `+`/`-` that isn't the start of a
/// `--` comment, skip whitespace again, and require a raw numeric literal (not `CAST(`/`::`).
fn has_raw_numeric_arithmetic_after(bytes: &[u8], from: usize) -> bool {
let mut index = from;
while bytes.get(index).copied().is_some_and(|b| b.is_ascii_whitespace()) {
index += 1;
}
let Some(&op) = bytes.get(index) else { return false };
if op != b'+' && op != b'-' {
return false;
}
if op == b'-' && bytes.get(index + 1) == Some(&b'-') {
return false;
}
index += 1;
while bytes.get(index).copied().is_some_and(|b| b.is_ascii_whitespace()) {
index += 1;
}
bytes.get(index).copied().is_some_and(|b| b.is_ascii_digit())
}
pub fn starts_with_duckdb_result_sql_keyword(sql: &str) -> bool {
let Some(token) = first_executable_sql_token(sql) else {
return false;
@@ -329,4 +450,71 @@ mod tests {
assert!(!starts_with_duckdb_result_sql_keyword("INSERT INTO items(note) VALUES ('RETURNING')"));
assert!(!starts_with_duckdb_result_sql_keyword("INSERT INTO items VALUES (1)"));
}
#[test]
fn rewrites_current_date_literal_arithmetic() {
assert_eq!(
rewrite_duckdb_current_date_literal_arithmetic("SELECT CURRENT_DATE - 4"),
"SELECT current_date() - 4"
);
assert_eq!(rewrite_duckdb_current_date_literal_arithmetic("SELECT CURRENT_DATE-4"), "SELECT current_date()-4");
assert_eq!(
rewrite_duckdb_current_date_literal_arithmetic("SELECT current_date + 1.5 AS d"),
"SELECT current_date() + 1.5 AS d"
);
assert_eq!(
rewrite_duckdb_current_date_literal_arithmetic("SELECT CURRENT_DATE - 4, CURRENT_DATE - 1"),
"SELECT current_date() - 4, current_date() - 1"
);
// Already-cast literals still match the narrow "digit right after the operator" check;
// rewriting them too is harmless since current_date() binds fine against any numeric form.
assert_eq!(
rewrite_duckdb_current_date_literal_arithmetic("SELECT CURRENT_DATE - 4::INTEGER"),
"SELECT current_date() - 4::INTEGER"
);
}
#[test]
fn leaves_non_literal_current_date_usage_untouched() {
for sql in [
"SELECT CURRENT_DATE",
"SELECT CURRENT_DATE::VARCHAR",
"SELECT CURRENT_DATE - CAST(4 AS INTEGER)",
"SELECT t.current_date - 4 FROM t",
"SELECT 1 AS current_date",
"CREATE TABLE cd_test (current_date INTEGER)",
"SELECT 4 - CURRENT_DATE",
"SELECT current_date() - 4",
"-- CURRENT_DATE - 4\nSELECT 1",
"SELECT '-- CURRENT_DATE - 4'",
] {
assert_eq!(rewrite_duckdb_current_date_literal_arithmetic(sql), sql, "should not rewrite: {sql}");
}
}
#[test]
fn leaves_dollar_quoted_current_date_arithmetic_untouched() {
assert_eq!(
rewrite_duckdb_current_date_literal_arithmetic(
"SELECT $$CURRENT_DATE - 4$$ AS source, CURRENT_DATE - 1 AS actual"
),
"SELECT $$CURRENT_DATE - 4$$ AS source, current_date() - 1 AS actual"
);
assert_eq!(
rewrite_duckdb_current_date_literal_arithmetic(
"SELECT $body_1$CURRENT_DATE + 2$body_1$ AS source, CURRENT_DATE + 3 AS actual"
),
"SELECT $body_1$CURRENT_DATE + 2$body_1$ AS source, current_date() + 3 AS actual"
);
assert_eq!(
rewrite_duckdb_current_date_literal_arithmetic(
"SELECT $outer$CURRENT_DATE - 4 $inner$ CURRENT_DATE + 2 $inner$$outer$ AS source"
),
"SELECT $outer$CURRENT_DATE - 4 $inner$ CURRENT_DATE + 2 $inner$$outer$ AS source"
);
assert_eq!(
rewrite_duckdb_current_date_literal_arithmetic("SELECT $body$CURRENT_DATE - 4"),
"SELECT $body$CURRENT_DATE - 4"
);
}
}
@@ -499,6 +499,52 @@ async fn worker_process_retries_connect_after_transient_file_lock() {
assert_eq!(result.rows, vec![vec![serde_json::json!(42)]]);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn worker_shutdown_checkpoints_and_removes_wal() {
let _guard = duckdb_worker_process_test_guard().await;
let executable = PathBuf::from(env!("CARGO_BIN_EXE_dbx-duckdb-driver"));
let db_path = temp_duckdb_path();
let wal_path = wal_path_for(&db_path);
let _ = std::fs::remove_file(&db_path);
let _ = std::fs::remove_file(&wal_path);
let client =
DuckDbWorkerClient::open_with_executable(executable.clone(), db_path.to_string_lossy().to_string(), Vec::new(), None)
.await
.expect("worker process connects");
client
.execute(
None,
"CREATE TABLE events (id INTEGER); INSERT INTO events VALUES (7);".to_string(),
Some(10),
None,
Some(Duration::from_secs(5)),
)
.await
.expect("write events");
// Committed writes sit in the WAL while the connection is open.
assert!(wal_path.exists(), "committed writes should leave a WAL file next to the database");
client.shutdown().await;
assert!(!wal_path.exists(), "shutdown should checkpoint and remove the WAL file");
// The checkpointed data survives a fresh open of the database file.
let client =
DuckDbWorkerClient::open_with_executable(executable, db_path.to_string_lossy().to_string(), Vec::new(), None)
.await
.expect("reopen after shutdown");
let result = client
.execute(None, "SELECT id FROM events".to_string(), Some(10), None, Some(Duration::from_secs(5)))
.await
.expect("read events back");
assert_eq!(result.rows, vec![vec![serde_json::json!(7)]]);
client.shutdown().await;
assert!(!wal_path.exists(), "second shutdown should also remove the WAL file");
let _ = std::fs::remove_file(&db_path);
}
async fn write_worker_request(stdin: &mut tokio::process::ChildStdin, request: DuckDbWorkerRequest) {
let line = serde_json::to_string(&request).expect("request json");
stdin.write_all(line.as_bytes()).await.expect("write request");
@@ -517,6 +563,10 @@ async fn read_worker_response(
serde_json::from_str(&line).expect("parse worker response")
}
fn wal_path_for(db_path: &PathBuf) -> PathBuf {
PathBuf::from(format!("{}.wal", db_path.display()))
}
fn temp_duckdb_path() -> PathBuf {
let suffix = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_nanos();
let pid = std::process::id();
+24 -1
View File
@@ -2,8 +2,31 @@ sourceSets {
main.java.srcDirs = ['src/main/java', '../h2/src/main/java']
}
configurations {
h2V1
h2V2
h2V3
}
dependencies {
implementation 'com.h2database:h2:2.1.214'
h2V1 'com.h2database:h2:1.4.200'
h2V2 'com.h2database:h2:2.1.214'
h2V3 'com.h2database:h2:2.4.240'
}
tasks.named('processResources') {
from(configurations.h2V1) {
into 'drivers'
rename { 'h2-1.4.200.jar' }
}
from(configurations.h2V2) {
into 'drivers'
rename { 'h2-2.1.214.jar' }
}
from(configurations.h2V3) {
into 'drivers'
rename { 'h2-2.4.240.jar' }
}
}
tasks.named('shadowJar') {
+32 -2
View File
@@ -1,6 +1,29 @@
configurations {
h2V1
h2V2
h2V3
}
dependencies {
implementation fileTree(dir: 'libs', include: ['*.jar'])
implementation 'com.h2database:h2:2.3.232'
h2V1 'com.h2database:h2:1.4.200'
h2V2 'com.h2database:h2:2.1.214'
h2V3 'com.h2database:h2:2.4.240'
testImplementation 'com.google.code.gson:gson:2.12.1'
}
tasks.named('processResources') {
from(configurations.h2V1) {
into 'drivers'
rename { 'h2-1.4.200.jar' }
}
from(configurations.h2V2) {
into 'drivers'
rename { 'h2-2.1.214.jar' }
}
from(configurations.h2V3) {
into 'drivers'
rename { 'h2-2.4.240.jar' }
}
}
tasks.named('shadowJar') {
@@ -8,3 +31,10 @@ tasks.named('shadowJar') {
attributes('Agent-Label': 'H2', 'Main-Class': 'com.dbx.agent.h2.H2Agent')
}
}
tasks.named('test') {
dependsOn tasks.named('shadowJar')
doFirst {
systemProperty 'dbx.h2.agent.jar', tasks.named('shadowJar').get().archiveFile.get().asFile.absolutePath
}
}
@@ -19,6 +19,7 @@ import com.dbx.agent.TriggerInfo;
import java.nio.charset.StandardCharsets;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -30,6 +31,9 @@ import java.util.Set;
public class H2Agent extends AbstractJdbcAgent {
private String databaseName = "";
private H2DriverLoader.LoadedDriver loadedDriver;
private H2DriverVersion driverVersion = H2DriverVersion.V3;
private int databaseMajorVersion = 2;
@Override
protected String driverClass() {
@@ -41,9 +45,62 @@ public class H2Agent extends AbstractJdbcAgent {
return buildUrl(params);
}
@Override
protected void loadDriver(ConnectParams params) throws Exception {
H2DriverVersion selected = H2DriverVersion.select(params);
if (selected == H2DriverVersion.CUSTOM) {
H2DriverLoader.LoadedDriver external = H2DriverLoader.loadExternal(
params.getJdbc_driver_paths(),
params.getJdbc_driver_class()
);
if (loadedDriver == null || !loadedDriver.identity().equals(external.identity())) {
replaceLoadedDriver(external);
} else {
external.classLoader().close();
}
} else if (loadedDriver == null || loadedDriver.version() != selected) {
replaceLoadedDriver(H2DriverLoader.load(selected));
}
driverVersion = selected;
}
private void replaceLoadedDriver(H2DriverLoader.LoadedDriver replacement) throws Exception {
H2DriverLoader.LoadedDriver previous = loadedDriver;
if (previous != null) {
try {
previous.classLoader().close();
} catch (Exception error) {
replacement.classLoader().close();
throw error;
}
}
loadedDriver = replacement;
}
@Override
protected Connection openConnection(ConnectParams params) throws Exception {
if (loadedDriver == null) {
throw new IllegalStateException("H2 JDBC driver was not loaded");
}
Connection opened = loadedDriver.driver().connect(buildJdbcUrl(params), buildConnectionProperties(params));
if (opened == null) {
throw new SQLException("H2 JDBC driver rejected URL: " + buildJdbcUrl(params));
}
return opened;
}
@Override
protected void afterConnect(ConnectParams params, Connection connection) {
databaseName = params.getDatabase();
databaseMajorVersion = unchecked(() -> connection.getMetaData().getDatabaseMajorVersion());
}
H2DriverVersion driverVersion() {
return driverVersion;
}
boolean isVersion2OrLater() {
return databaseMajorVersion >= 2;
}
@Override
@@ -118,7 +175,7 @@ public class H2Agent extends AbstractJdbcAgent {
List<Object> args = new ArrayList<>();
StringBuilder sql = new StringBuilder("SELECT TABLE_NAME, TABLE_TYPE FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = ?");
args.add(effectiveSchema);
appendH2TableTypePredicate(sql, args, constraints);
appendH2TableTypePredicate(sql, args, constraints, isVersion2OrLater());
MetadataSqlSupport.appendNameFilter(sql, args, "TABLE_NAME", constraints);
sql.append(" ORDER BY TABLE_NAME");
MetadataSqlSupport.appendLiteralLimitOffset(sql, constraints);
@@ -143,9 +200,10 @@ public class H2Agent extends AbstractJdbcAgent {
result.add(new ObjectInfo(table.getName(), table.getTable_type(), schema, table.getComment()));
}
try (var stmt = requireConnected().prepareStatement(
"SELECT ROUTINE_NAME, ROUTINE_TYPE FROM INFORMATION_SCHEMA.ROUTINES WHERE ROUTINE_SCHEMA = ? ORDER BY ROUTINE_NAME"
)) {
String sql = isVersion2OrLater()
? "SELECT ROUTINE_NAME, ROUTINE_TYPE FROM INFORMATION_SCHEMA.ROUTINES WHERE ROUTINE_SCHEMA = ? ORDER BY ROUTINE_NAME"
: "SELECT ALIAS_NAME, CASE WHEN RETURNS_RESULT = 2 THEN 'FUNCTION' ELSE 'PROCEDURE' END FROM INFORMATION_SCHEMA.FUNCTION_ALIASES WHERE ALIAS_SCHEMA = ? ORDER BY ALIAS_NAME";
try (var stmt = requireConnected().prepareStatement(sql)) {
stmt.setString(1, effectiveSchema);
try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) {
@@ -174,6 +232,9 @@ public class H2Agent extends AbstractJdbcAgent {
}
private List<ObjectInfo> queryConstrainedObjects(String schema, MetadataListConstraints constraints) {
if (!isVersion2OrLater()) {
return constraints.filterObjects(listObjects(schema));
}
return unchecked(() -> {
String effectiveSchema = resolveSchema(schema);
List<ObjectInfo> result = new ArrayList<>();
@@ -182,7 +243,7 @@ public class H2Agent extends AbstractJdbcAgent {
if (constraints.includesTableLikeTypes()) {
StringBuilder tableSql = new StringBuilder("SELECT TABLE_NAME AS OBJECT_NAME, TABLE_TYPE AS OBJECT_TYPE FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = ?");
args.add(effectiveSchema);
appendH2TableTypePredicate(tableSql, args, constraints);
appendH2TableTypePredicate(tableSql, args, constraints, true);
MetadataSqlSupport.appendNameFilter(tableSql, args, "TABLE_NAME", constraints);
branches.add(tableSql.toString());
}
@@ -218,7 +279,9 @@ public class H2Agent extends AbstractJdbcAgent {
String effectiveSchema = resolveSchema(schema);
String sql = switch (objectType.toUpperCase(Locale.ROOT)) {
case "VIEW" -> "SELECT VIEW_DEFINITION FROM INFORMATION_SCHEMA.VIEWS WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?";
case "FUNCTION", "PROCEDURE" -> "SELECT ROUTINE_DEFINITION FROM INFORMATION_SCHEMA.ROUTINES WHERE ROUTINE_SCHEMA = ? AND ROUTINE_NAME = ?";
case "FUNCTION", "PROCEDURE" -> isVersion2OrLater()
? "SELECT ROUTINE_DEFINITION FROM INFORMATION_SCHEMA.ROUTINES WHERE ROUTINE_SCHEMA = ? AND ROUTINE_NAME = ?"
: "SELECT SOURCE FROM INFORMATION_SCHEMA.FUNCTION_ALIASES WHERE ALIAS_SCHEMA = ? AND ALIAS_NAME = ?";
default -> throw new IllegalArgumentException("Unsupported object type: " + objectType);
};
@@ -242,16 +305,17 @@ public class H2Agent extends AbstractJdbcAgent {
return unchecked(() -> {
String effectiveSchema = resolveSchema(schema);
Set<String> primaryKeys = new HashSet<>();
try (var stmt = requireConnected().prepareStatement(
"""
SELECT ic.COLUMN_NAME
FROM INFORMATION_SCHEMA.INDEX_COLUMNS ic
JOIN INFORMATION_SCHEMA.INDEXES i
ON ic.INDEX_SCHEMA = i.INDEX_SCHEMA AND ic.INDEX_NAME = i.INDEX_NAME
WHERE ic.TABLE_SCHEMA = ? AND ic.TABLE_NAME = ?
AND i.INDEX_TYPE_NAME = 'PRIMARY KEY'
"""
)) {
String primaryKeySql = isVersion2OrLater()
? """
SELECT ic.COLUMN_NAME
FROM INFORMATION_SCHEMA.INDEX_COLUMNS ic
JOIN INFORMATION_SCHEMA.INDEXES i
ON ic.INDEX_SCHEMA = i.INDEX_SCHEMA AND ic.INDEX_NAME = i.INDEX_NAME
WHERE ic.TABLE_SCHEMA = ? AND ic.TABLE_NAME = ?
AND i.INDEX_TYPE_NAME = 'PRIMARY KEY'
"""
: "SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.INDEXES WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? AND PRIMARY_KEY = TRUE";
try (var stmt = requireConnected().prepareStatement(primaryKeySql)) {
stmt.setString(1, effectiveSchema);
stmt.setString(2, table);
try (ResultSet rs = stmt.executeQuery()) {
@@ -262,15 +326,11 @@ public class H2Agent extends AbstractJdbcAgent {
}
List<ColumnInfo> result = new ArrayList<>();
try (var stmt = requireConnected().prepareStatement(
"""
SELECT COLUMN_NAME, DATA_TYPE, IS_NULLABLE, COLUMN_DEFAULT,
NUMERIC_PRECISION, NUMERIC_SCALE, CHARACTER_MAXIMUM_LENGTH
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?
ORDER BY ORDINAL_POSITION
"""
)) {
String typeColumn = isVersion2OrLater() ? "DATA_TYPE" : "TYPE_NAME";
String columnSql = "SELECT COLUMN_NAME, " + typeColumn + " AS DBX_DATA_TYPE, IS_NULLABLE, COLUMN_DEFAULT, "
+ "NUMERIC_PRECISION, NUMERIC_SCALE, CHARACTER_MAXIMUM_LENGTH, REMARKS "
+ "FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION";
try (var stmt = requireConnected().prepareStatement(columnSql)) {
stmt.setString(1, effectiveSchema);
stmt.setString(2, table);
try (ResultSet rs = stmt.executeQuery()) {
@@ -278,11 +338,11 @@ public class H2Agent extends AbstractJdbcAgent {
String columnName = rs.getString("COLUMN_NAME");
result.add(new ColumnInfo(
columnName,
rs.getString("DATA_TYPE"),
rs.getString("DBX_DATA_TYPE"),
"YES".equals(rs.getString("IS_NULLABLE")),
rs.getString("COLUMN_DEFAULT"),
primaryKeys.contains(columnName),
null,
rs.getString("REMARKS"),
null,
intOrNull(rs, "NUMERIC_PRECISION"),
intOrNull(rs, "NUMERIC_SCALE"),
@@ -304,16 +364,24 @@ public class H2Agent extends AbstractJdbcAgent {
Map<String, Boolean> primaryMap = new HashMap<>();
Map<String, String> typeMap = new HashMap<>();
try (var stmt = requireConnected().prepareStatement(
"""
SELECT i.INDEX_NAME, ic.COLUMN_NAME, ic.IS_UNIQUE, i.INDEX_TYPE_NAME
FROM INFORMATION_SCHEMA.INDEX_COLUMNS ic
JOIN INFORMATION_SCHEMA.INDEXES i
ON ic.INDEX_SCHEMA = i.INDEX_SCHEMA AND ic.INDEX_NAME = i.INDEX_NAME
WHERE ic.TABLE_SCHEMA = ? AND ic.TABLE_NAME = ?
ORDER BY i.INDEX_NAME, ic.ORDINAL_POSITION
"""
)) {
String indexSql = isVersion2OrLater()
? """
SELECT i.INDEX_NAME, ic.COLUMN_NAME, ic.IS_UNIQUE, i.INDEX_TYPE_NAME,
CASE WHEN i.INDEX_TYPE_NAME = 'PRIMARY KEY' THEN TRUE ELSE FALSE END AS IS_PRIMARY
FROM INFORMATION_SCHEMA.INDEX_COLUMNS ic
JOIN INFORMATION_SCHEMA.INDEXES i
ON ic.INDEX_SCHEMA = i.INDEX_SCHEMA AND ic.INDEX_NAME = i.INDEX_NAME
WHERE ic.TABLE_SCHEMA = ? AND ic.TABLE_NAME = ?
ORDER BY i.INDEX_NAME, ic.ORDINAL_POSITION
"""
: """
SELECT INDEX_NAME, COLUMN_NAME, CASE WHEN NON_UNIQUE THEN FALSE ELSE TRUE END AS IS_UNIQUE,
INDEX_TYPE_NAME, PRIMARY_KEY AS IS_PRIMARY
FROM INFORMATION_SCHEMA.INDEXES
WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?
ORDER BY INDEX_NAME, ORDINAL_POSITION
""";
try (var stmt = requireConnected().prepareStatement(indexSql)) {
stmt.setString(1, effectiveSchema);
stmt.setString(2, table);
try (ResultSet rs = stmt.executeQuery()) {
@@ -324,7 +392,7 @@ public class H2Agent extends AbstractJdbcAgent {
indexMap.computeIfAbsent(indexName, ignored -> new ArrayList<>()).add(columnName);
uniqueMap.put(indexName, rs.getBoolean("IS_UNIQUE"));
primaryMap.put(indexName, "PRIMARY KEY".equals(indexType));
primaryMap.put(indexName, rs.getBoolean("IS_PRIMARY"));
typeMap.put(indexName, indexType == null ? "" : indexType);
}
}
@@ -353,27 +421,17 @@ public class H2Agent extends AbstractJdbcAgent {
return unchecked(() -> {
String effectiveSchema = resolveSchema(schema);
List<ForeignKeyInfo> result = new ArrayList<>();
try (var stmt = requireConnected().prepareStatement(
"""
SELECT FK_NAME, FKCOLUMN_NAME, PKTABLE_NAME, PKCOLUMN_NAME
FROM INFORMATION_SCHEMA.CROSS_REFERENCES
WHERE FKTABLE_SCHEMA = ? AND FKTABLE_NAME = ?
ORDER BY FK_NAME
"""
)) {
stmt.setString(1, effectiveSchema);
stmt.setString(2, table);
try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) {
result.add(new ForeignKeyInfo(
rs.getString("FK_NAME"),
rs.getString("FKCOLUMN_NAME"),
rs.getString("PKTABLE_NAME"),
rs.getString("PKCOLUMN_NAME")
));
}
try (ResultSet rs = requireConnected().getMetaData().getImportedKeys(null, effectiveSchema, table)) {
while (rs.next()) {
result.add(new ForeignKeyInfo(
rs.getString("FK_NAME"),
rs.getString("FKCOLUMN_NAME"),
rs.getString("PKTABLE_NAME"),
rs.getString("PKCOLUMN_NAME")
));
}
}
result.sort(java.util.Comparator.comparing(ForeignKeyInfo::getName, java.util.Comparator.nullsLast(String::compareTo)));
return result;
});
}
@@ -383,14 +441,21 @@ public class H2Agent extends AbstractJdbcAgent {
return unchecked(() -> {
String effectiveSchema = resolveSchema(schema);
List<TriggerInfo> result = new ArrayList<>();
try (var stmt = requireConnected().prepareStatement(
"""
SELECT TRIGGER_NAME, EVENT_MANIPULATION, ACTION_TIMING
FROM INFORMATION_SCHEMA.TRIGGERS
WHERE TRIGGER_SCHEMA = ? AND EVENT_OBJECT_TABLE = ?
ORDER BY TRIGGER_NAME
"""
)) {
String triggerSql = isVersion2OrLater()
? """
SELECT TRIGGER_NAME, EVENT_MANIPULATION, ACTION_TIMING
FROM INFORMATION_SCHEMA.TRIGGERS
WHERE TRIGGER_SCHEMA = ? AND EVENT_OBJECT_TABLE = ?
ORDER BY TRIGGER_NAME
"""
: """
SELECT TRIGGER_NAME, TRIGGER_TYPE AS EVENT_MANIPULATION,
CASE WHEN BEFORE THEN 'BEFORE' ELSE 'AFTER' END AS ACTION_TIMING
FROM INFORMATION_SCHEMA.TRIGGERS
WHERE TRIGGER_SCHEMA = ? AND TABLE_NAME = ?
ORDER BY TRIGGER_NAME
""";
try (var stmt = requireConnected().prepareStatement(triggerSql)) {
stmt.setString(1, effectiveSchema);
stmt.setString(2, table);
try (ResultSet rs = stmt.executeQuery()) {
@@ -452,13 +517,18 @@ public class H2Agent extends AbstractJdbcAgent {
|| constraints.objectTypeAllowed("FUNCTION");
}
private static void appendH2TableTypePredicate(StringBuilder sql, List<Object> args, MetadataListConstraints constraints) {
private static void appendH2TableTypePredicate(
StringBuilder sql,
List<Object> args,
MetadataListConstraints constraints,
boolean version2OrLater
) {
if (!constraints.hasObjectTypes()) {
return;
}
List<String> types = new ArrayList<>();
if (constraints.tableTypeAllowed("TABLE")) {
types.add("BASE TABLE");
types.add(version2OrLater ? "BASE TABLE" : "TABLE");
}
if (constraints.tableTypeAllowed("VIEW")) {
types.add("VIEW");
@@ -0,0 +1,127 @@
package com.dbx.agent.h2;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.security.MessageDigest;
import java.sql.Driver;
import java.util.ArrayList;
import java.util.HexFormat;
import java.util.List;
import java.util.UUID;
final class H2DriverLoader {
private static final Object EXTRACTION_LOCK = new Object();
private static final Path CACHE_ROOT = Path.of(System.getProperty("java.io.tmpdir"), "dbx-h2-drivers");
private H2DriverLoader() {
}
static LoadedDriver load(H2DriverVersion version) throws Exception {
Path jar = extract(version);
URLClassLoader classLoader = new URLClassLoader(
new URL[]{jar.toUri().toURL()},
H2DriverLoader.class.getClassLoader()
);
try {
Driver driver = (Driver) Class.forName("org.h2.Driver", true, classLoader).getDeclaredConstructor().newInstance();
return new LoadedDriver(version, version.version(), driver, classLoader);
} catch (Exception error) {
closeAfterFailure(classLoader, error);
throw error;
} catch (LinkageError error) {
closeAfterFailure(classLoader, error);
throw error;
}
}
static LoadedDriver loadExternal(List<String> driverPaths, String driverClass) throws Exception {
if (driverPaths == null || driverPaths.isEmpty()) {
throw new IllegalArgumentException("Custom H2 driver profile requires at least one JDBC JAR path");
}
List<URL> urls = new ArrayList<>();
List<String> identities = new ArrayList<>();
for (String driverPath : driverPaths) {
Path path = Path.of(driverPath).toAbsolutePath().normalize();
if (!Files.isRegularFile(path)) {
throw new IOException("Custom H2 JDBC JAR does not exist: " + path);
}
urls.add(path.toUri().toURL());
identities.add(path + ":" + sha256(path));
}
String effectiveDriverClass = driverClass == null || driverClass.isBlank() ? "org.h2.Driver" : driverClass.trim();
URLClassLoader classLoader = new URLClassLoader(
urls.toArray(new URL[0]),
H2DriverLoader.class.getClassLoader()
);
try {
Driver driver = (Driver) Class.forName(effectiveDriverClass, true, classLoader).getDeclaredConstructor().newInstance();
return new LoadedDriver(H2DriverVersion.CUSTOM, effectiveDriverClass + "|" + String.join("|", identities), driver, classLoader);
} catch (Exception error) {
closeAfterFailure(classLoader, error);
throw error;
} catch (LinkageError error) {
closeAfterFailure(classLoader, error);
throw error;
}
}
private static Path extract(H2DriverVersion version) throws Exception {
byte[] bytes;
try (InputStream input = H2DriverLoader.class.getResourceAsStream(version.resourcePath())) {
if (input == null) {
throw new IOException("Bundled H2 driver is missing: " + version.resourcePath());
}
bytes = input.readAllBytes();
}
String digest = sha256(bytes);
Path directory = CACHE_ROOT.resolve(version.version() + "-" + digest.substring(0, 16));
Path target = directory.resolve("h2.jar");
synchronized (EXTRACTION_LOCK) {
Files.createDirectories(directory);
if (!Files.isRegularFile(target) || Files.size(target) != bytes.length || !digest.equals(sha256(target))) {
Path temporary = directory.resolve("h2.jar.tmp-" + UUID.randomUUID());
Files.write(temporary, bytes);
try {
Files.move(temporary, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
} catch (java.nio.file.AtomicMoveNotSupportedException ignored) {
Files.move(temporary, target, StandardCopyOption.REPLACE_EXISTING);
}
}
}
return target;
}
private static String sha256(Path path) throws Exception {
MessageDigest digest = MessageDigest.getInstance("SHA-256");
try (InputStream input = Files.newInputStream(path)) {
byte[] buffer = new byte[8192];
int read;
while ((read = input.read(buffer)) >= 0) {
if (read > 0) {
digest.update(buffer, 0, read);
}
}
}
return HexFormat.of().formatHex(digest.digest());
}
private static String sha256(byte[] bytes) throws Exception {
return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(bytes));
}
private static void closeAfterFailure(URLClassLoader classLoader, Throwable error) {
try {
classLoader.close();
} catch (IOException closeError) {
error.addSuppressed(closeError);
}
}
record LoadedDriver(H2DriverVersion version, String identity, Driver driver, URLClassLoader classLoader) {
}
}
@@ -0,0 +1,65 @@
package com.dbx.agent.h2;
import com.dbx.agent.ConnectParams;
import java.util.Locale;
enum H2DriverVersion {
V1("h2-v1", "1.4.200", "/drivers/h2-1.4.200.jar", 1),
V2("h2-v2", "2.1.214", "/drivers/h2-2.1.214.jar", 2),
V3("h2-v3", "2.4.240", "/drivers/h2-2.4.240.jar", 3),
CUSTOM("h2-custom", "custom", "", 0);
private final String profile;
private final String version;
private final String resourcePath;
private final int storageFormat;
H2DriverVersion(String profile, String version, String resourcePath, int storageFormat) {
this.profile = profile;
this.version = version;
this.resourcePath = resourcePath;
this.storageFormat = storageFormat;
}
String profile() {
return profile;
}
String version() {
return version;
}
String resourcePath() {
return resourcePath;
}
int storageFormat() {
return storageFormat;
}
static H2DriverVersion select(ConnectParams params) throws Exception {
String profile = params.getDriver_profile();
String normalized = profile == null ? "" : profile.trim().toLowerCase(Locale.ROOT);
return switch (normalized) {
case "h2-v1" -> V1;
case "h2-v2", "h2-legacy" -> V2;
case "h2-v3" -> V3;
case "h2-custom" -> CUSTOM;
case "", "h2", "h2-auto", "h2_embedded", "h2_server" -> {
java.util.OptionalInt detected = H2FileFormatDetector.detect(H2Agent.buildUrl(params));
yield detected.isPresent() ? fromStorageFormat(detected.getAsInt()) : V3;
}
default -> throw new IllegalArgumentException("Unsupported H2 driver profile: " + profile);
};
}
private static H2DriverVersion fromStorageFormat(int format) {
return switch (format) {
case 1 -> V1;
case 2 -> V2;
case 3 -> V3;
default -> throw new IllegalArgumentException("Unsupported H2 database file format: " + format);
};
}
}
@@ -0,0 +1,187 @@
package com.dbx.agent.h2;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.OptionalInt;
final class H2FileFormatDetector {
private static final int BLOCK_SIZE = 4096;
private H2FileFormatDetector() {
}
static OptionalInt detect(String jdbcUrl) throws IOException {
Path base = localDatabaseBasePath(jdbcUrl);
if (base == null) {
return OptionalInt.empty();
}
Path pageStore = withSuffix(base, ".h2.db");
Path mvStore = withSuffix(base, ".mv.db");
boolean pageStoreExists = Files.isRegularFile(pageStore);
boolean mvStoreExists = Files.isRegularFile(mvStore);
if (pageStoreExists && mvStoreExists) {
throw new IOException("Both H2 PageStore and MVStore files exist for " + base + "; choose an explicit H2 driver profile");
}
if (pageStoreExists) {
return OptionalInt.of(1);
}
if (!mvStoreExists) {
return OptionalInt.empty();
}
return OptionalInt.of(readMvStoreFormat(mvStore));
}
private static int readMvStoreFormat(Path file) throws IOException {
byte[] bytes;
try (InputStream input = Files.newInputStream(file)) {
bytes = input.readNBytes(BLOCK_SIZE * 2);
}
if (bytes.length < BLOCK_SIZE * 2) {
throw new IOException("H2 MVStore file is too small to contain valid headers: " + file);
}
List<StoreHeader> headers = new ArrayList<>();
for (int block = 0; block < 2; block++) {
StoreHeader header = parseHeader(bytes, block * BLOCK_SIZE);
if (header != null) {
headers.add(header);
}
}
return headers.stream()
.max(Comparator.comparingLong(StoreHeader::version))
.orElseThrow(() -> new IOException("Cannot determine the H2 MVStore format for " + file + "; choose an explicit H2 driver profile"))
.format();
}
private static StoreHeader parseHeader(byte[] fileBytes, int offset) {
byte[] block = java.util.Arrays.copyOfRange(fileBytes, offset, offset + BLOCK_SIZE);
int start = 0;
int end = block.length;
while (start < end && block[start] <= ' ') {
start++;
}
while (start < end && block[end - 1] <= ' ') {
end--;
}
if (start >= end) {
return null;
}
String text = new String(block, start, end - start, StandardCharsets.ISO_8859_1);
int fletcherIndex = text.indexOf("fletcher:");
if (fletcherIndex <= 0 || text.charAt(fletcherIndex - 1) != ',') {
return null;
}
int checksumEnd = text.indexOf(',', fletcherIndex);
if (checksumEnd < 0) {
checksumEnd = text.length();
}
try {
long expected = Long.parseUnsignedLong(text.substring(fletcherIndex + "fletcher:".length(), checksumEnd), 16);
int actual = fletcher32(block, start, fletcherIndex - 1);
if (Integer.toUnsignedLong(actual) != expected) {
return null;
}
long version = readHexValue(text, "version", 0);
int format = Math.toIntExact(readHexValue(text, "format", 1));
return new StoreHeader(version, format);
} catch (RuntimeException ignored) {
return null;
}
}
private static long readHexValue(String text, String key, long defaultValue) {
String marker = key + ':';
int index = text.indexOf(marker);
if (index < 0 || (index > 0 && text.charAt(index - 1) != ',')) {
return defaultValue;
}
int start = index + marker.length();
int end = text.indexOf(',', start);
if (end < 0) {
end = text.length();
}
return Long.parseUnsignedLong(text.substring(start, end), 16);
}
private static int fletcher32(byte[] bytes, int offset, int length) {
int s1 = 0xffff;
int s2 = 0xffff;
int index = offset;
int evenEnd = offset + (length & ~1);
while (index < evenEnd) {
int batchEnd = Math.min(index + 720, evenEnd);
while (index < batchEnd) {
int value = ((bytes[index++] & 0xff) << 8) | (bytes[index++] & 0xff);
s2 += s1 += value;
}
s1 = (s1 & 0xffff) + (s1 >>> 16);
s2 = (s2 & 0xffff) + (s2 >>> 16);
}
if ((length & 1) != 0) {
int value = (bytes[index] & 0xff) << 8;
s2 += s1 += value;
}
s1 = (s1 & 0xffff) + (s1 >>> 16);
s2 = (s2 & 0xffff) + (s2 >>> 16);
return (s2 << 16) | s1;
}
private static Path localDatabaseBasePath(String jdbcUrl) {
String prefix = "jdbc:h2:";
if (jdbcUrl == null || !jdbcUrl.regionMatches(true, 0, prefix, 0, prefix.length())) {
return null;
}
String target = jdbcUrl.substring(prefix.length());
String lower = target.toLowerCase(java.util.Locale.ROOT);
if (lower.startsWith("tcp:") || lower.startsWith("ssl:") || lower.startsWith("mem:")
|| lower.startsWith("zip:") || lower.startsWith("split:")) {
return null;
}
if (lower.startsWith("file:")) {
target = target.substring("file:".length());
}
target = beforeOptions(target);
if (target.startsWith("~" + java.io.File.separator) || target.startsWith("~/") || target.startsWith("~\\")) {
target = System.getProperty("user.home") + target.substring(1);
}
if (target.isBlank()) {
return null;
}
Path path = Path.of(target).toAbsolutePath().normalize();
String value = path.toString().toLowerCase(java.util.Locale.ROOT);
if (value.endsWith(".mv.db")) {
return Path.of(path.toString().substring(0, path.toString().length() - ".mv.db".length()));
}
if (value.endsWith(".h2.db")) {
return Path.of(path.toString().substring(0, path.toString().length() - ".h2.db".length()));
}
return path;
}
private static String beforeOptions(String value) {
boolean escaped = false;
for (int index = 0; index < value.length(); index++) {
char current = value.charAt(index);
if (current == ';' && !escaped) {
return value.substring(0, index);
}
escaped = current == '\\' && !escaped;
if (current != '\\') {
escaped = false;
}
}
return value;
}
private static Path withSuffix(Path base, String suffix) {
return Path.of(base.toString() + suffix);
}
private record StoreHeader(long version, int format) {
}
}
@@ -0,0 +1,262 @@
package com.dbx.agent.h2;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.Duration;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
class H2AgentProcessTest {
@Test
@Timeout(45)
void shadedAgentSupportsIsolatedMultiVersionSessions() throws Exception {
Path agentJar = Path.of(System.getProperty("dbx.h2.agent.jar"));
Assertions.assertTrue(Files.isRegularFile(agentJar), () -> "Missing shaded H2 Agent: " + agentJar);
try (RpcProcess rpc = new RpcProcess(agentJar)) {
JsonObject handshake = rpc.result(rpc.request("handshake", new JsonObject()));
Assertions.assertEquals(2, handshake.get("protocolVersion").getAsInt());
Map<String, String> versions = new LinkedHashMap<>();
versions.put("h2-v1", "1.4.200");
versions.put("h2-v2", "2.1.214");
versions.put("h2-v3", "2.4.240");
int index = 0;
for (Map.Entry<String, String> entry : versions.entrySet()) {
String sessionId = "session-" + entry.getKey();
JsonObject connected = rpc.result(rpc.request("open_session", connectParams(sessionId, entry.getKey())));
Assertions.assertTrue(connected.get("ok").getAsBoolean(), entry.getKey());
JsonObject info = rpc.result(rpc.request("connection_info", sessionParams(sessionId)));
String driverVersion = info.getAsJsonObject("databaseInfo").get("driverVersion").getAsString();
Assertions.assertTrue(driverVersion.startsWith(entry.getValue()), entry.getKey() + ": " + driverVersion);
String table = "RPC_TABLE_" + (++index);
rpc.result(rpc.request("execute_query", queryParams(sessionId, "CREATE TABLE " + table + " (ID INT PRIMARY KEY, TEXT_VALUE VARCHAR(32))")));
rpc.result(rpc.request("execute_query", queryParams(sessionId, "INSERT INTO " + table + " VALUES (1, '" + entry.getKey() + "')")));
JsonArray tables = rpc.resultArray(rpc.request("list_tables", metadataParams(sessionId, "PUBLIC")));
Assertions.assertTrue(tables.asList().stream().anyMatch(value -> table.equals(value.getAsJsonObject().get("name").getAsString())), entry.getKey());
Assertions.assertEquals(entry.getKey(), firstCell(rpc.result(rpc.request("execute_query", queryParams(sessionId, "SELECT TEXT_VALUE FROM " + table)))));
}
JsonObject transaction = sessionParams("session-h2-v2");
JsonArray statements = new JsonArray();
statements.add("CREATE TABLE TX_PROBE (ID INT PRIMARY KEY)");
statements.add("INSERT INTO TX_PROBE VALUES (1)");
statements.add("INSERT INTO TX_PROBE VALUES (2)");
transaction.add("statements", statements);
Assertions.assertEquals(2, rpc.result(rpc.request("execute_transaction", transaction)).get("affected_rows").getAsLong());
Assertions.assertEquals("2", firstCell(rpc.result(rpc.request("execute_query", queryParams("session-h2-v2", "SELECT COUNT(*) FROM TX_PROBE")))));
JsonObject firstPageParams = queryParams("session-h2-v3", "SELECT X FROM SYSTEM_RANGE(1, 5)");
firstPageParams.addProperty("pageSize", 2);
JsonObject firstPage = rpc.result(rpc.request("execute_query_page", firstPageParams));
Assertions.assertEquals(2, firstPage.getAsJsonArray("rows").size());
Assertions.assertTrue(firstPage.get("has_more").getAsBoolean());
String querySessionId = firstPage.get("session_id").getAsString();
JsonObject fetchParams = sessionParams("session-h2-v3");
fetchParams.addProperty("sessionId", querySessionId);
fetchParams.addProperty("pageSize", 2);
JsonObject secondPage = rpc.result(rpc.request("fetch_query_page", fetchParams));
Assertions.assertEquals(2, secondPage.getAsJsonArray("rows").size());
Assertions.assertTrue(secondPage.get("has_more").getAsBoolean());
JsonObject finalPage = rpc.result(rpc.request("fetch_query_page", fetchParams));
Assertions.assertEquals(1, finalPage.getAsJsonArray("rows").size());
Assertions.assertFalse(finalPage.get("has_more").getAsBoolean());
int slowQueryId = rpc.send("execute_query", queryParams("session-h2-v3", "SELECT SUM(SIN(X)) FROM SYSTEM_RANGE(1, 1000000000)"));
Thread.sleep(100);
JsonObject cancelled = rpc.result(rpc.request("cancel_session", sessionParams("session-h2-v3")));
Assertions.assertTrue(cancelled.get("ok").getAsBoolean());
JsonObject slowResponse = rpc.await(slowQueryId, Duration.ofSeconds(15));
Assertions.assertTrue(slowResponse.has("error"), () -> "Expected cancelled query error, got " + slowResponse);
Assertions.assertTrue(rpc.result(rpc.request("validate_session", sessionParams("session-h2-v3"))).get("ok").getAsBoolean());
for (String profile : versions.keySet()) {
String sessionId = "session-" + profile;
Assertions.assertTrue(rpc.result(rpc.request("close_session", sessionParams(sessionId))).get("ok").getAsBoolean());
Assertions.assertTrue(rpc.request("validate_session", sessionParams(sessionId)).has("error"));
}
}
}
private static JsonObject connectParams(String sessionId, String profile) {
JsonObject params = sessionParams(sessionId);
params.addProperty("host", "");
params.addProperty("port", 0);
params.addProperty("database", "mem:rpc-" + profile + ";DB_CLOSE_DELAY=-1");
params.addProperty("username", "sa");
params.addProperty("password", "");
params.addProperty("connection_string", "");
params.addProperty("driver_profile", profile);
params.addProperty("ssl", false);
return params;
}
private static JsonObject sessionParams(String sessionId) {
JsonObject params = new JsonObject();
params.addProperty("agentSessionId", sessionId);
return params;
}
private static JsonObject queryParams(String sessionId, String sql) {
JsonObject params = sessionParams(sessionId);
params.addProperty("sql", sql);
return params;
}
private static JsonObject metadataParams(String sessionId, String schema) {
JsonObject params = sessionParams(sessionId);
params.addProperty("schema", schema);
return params;
}
private static String firstCell(JsonObject result) {
return result.getAsJsonArray("rows").get(0).getAsJsonArray().get(0).getAsString();
}
private static final class RpcProcess implements AutoCloseable {
private final Process process;
private final BufferedWriter input;
private final BlockingQueue<JsonObject> responses = new LinkedBlockingQueue<>();
private final Map<Integer, JsonObject> pending = new ConcurrentHashMap<>();
private final CompletableFuture<Void> ready = new CompletableFuture<>();
private final AtomicInteger requestIds = new AtomicInteger();
private final StringBuilder stderr = new StringBuilder();
private RpcProcess(Path agentJar) throws Exception {
Path java = Path.of(System.getProperty("java.home"), "bin", "java");
process = new ProcessBuilder(java.toString(), "-jar", agentJar.toString()).start();
input = new BufferedWriter(new OutputStreamWriter(process.getOutputStream(), StandardCharsets.UTF_8));
startStdoutReader();
startStderrReader();
ready.get(10, TimeUnit.SECONDS);
}
private void startStdoutReader() {
Thread thread = new Thread(() -> {
try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8))) {
String line;
while ((line = reader.readLine()) != null) {
JsonObject message = JsonParser.parseString(line).getAsJsonObject();
if (message.has("ready")) {
ready.complete(null);
} else {
responses.add(message);
}
}
} catch (Throwable error) {
ready.completeExceptionally(error);
}
}, "h2-agent-rpc-stdout");
thread.setDaemon(true);
thread.start();
}
private void startStderrReader() {
Thread thread = new Thread(() -> {
try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getErrorStream(), StandardCharsets.UTF_8))) {
String line;
while ((line = reader.readLine()) != null) {
synchronized (stderr) {
stderr.append(line).append('\n');
}
}
} catch (Exception ignored) {
}
}, "h2-agent-rpc-stderr");
thread.setDaemon(true);
thread.start();
}
private JsonObject request(String method, JsonObject params) throws Exception {
return await(send(method, params), Duration.ofSeconds(15));
}
private int send(String method, JsonObject params) throws Exception {
int id = requestIds.incrementAndGet();
JsonObject request = new JsonObject();
request.addProperty("jsonrpc", "2.0");
request.addProperty("id", id);
request.addProperty("method", method);
request.add("params", params);
synchronized (input) {
input.write(request.toString());
input.newLine();
input.flush();
}
return id;
}
private JsonObject await(int id, Duration timeout) throws Exception {
JsonObject buffered = pending.remove(id);
if (buffered != null) {
return buffered;
}
long deadline = System.nanoTime() + timeout.toNanos();
while (System.nanoTime() < deadline) {
long remaining = deadline - System.nanoTime();
JsonObject response = responses.poll(Math.max(1, TimeUnit.NANOSECONDS.toMillis(remaining)), TimeUnit.MILLISECONDS);
if (response == null) {
break;
}
int responseId = response.get("id").getAsInt();
if (responseId == id) {
return response;
}
pending.put(responseId, response);
}
throw new AssertionError("Timed out waiting for H2 Agent response " + id + "; stderr=" + stderrText());
}
private JsonObject result(JsonObject response) {
Assertions.assertFalse(response.has("error"), () -> response + "\nstderr=" + stderrText());
return response.getAsJsonObject("result");
}
private JsonArray resultArray(JsonObject response) {
Assertions.assertFalse(response.has("error"), () -> response + "\nstderr=" + stderrText());
return response.getAsJsonArray("result");
}
private String stderrText() {
synchronized (stderr) {
return stderr.toString();
}
}
@Override
public void close() throws Exception {
if (process.isAlive()) {
try {
result(request("shutdown", new JsonObject()));
} catch (Throwable ignored) {
}
}
input.close();
if (!process.waitFor(5, TimeUnit.SECONDS)) {
process.destroyForcibly();
process.waitFor(5, TimeUnit.SECONDS);
}
}
}
}
@@ -1,20 +1,34 @@
package com.dbx.agent.h2;
import com.dbx.agent.BaseDatabaseAgent;
import com.dbx.agent.ColumnInfo;
import com.dbx.agent.ConnectParams;
import com.dbx.agent.DatabaseAgent;
import com.dbx.agent.ExecuteQueryOptions;
import com.dbx.agent.ForeignKeyInfo;
import com.dbx.agent.IndexInfo;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.ObjectInfo;
import com.dbx.agent.QueryResult;
import com.dbx.agent.TableInfo;
import com.dbx.agent.test.JdbcExecutionBehaviorTest;
import com.dbx.agent.test.JdbcMetadataBehaviorTest;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.nio.file.attribute.FileTime;
import java.util.List;
import java.util.Map;
import java.util.jar.JarOutputStream;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
class H2AgentMigrationTest {
@TempDir
Path tempDirectory;
@Test
void agentExtendsBaseDatabaseAgent() {
Assertions.assertTrue(BaseDatabaseAgent.class.isAssignableFrom(H2Agent.class));
@@ -62,6 +76,273 @@ class H2AgentMigrationTest {
}
}
@Test
void loadsEveryBundledH2Driver() {
assertBundledDriver("h2-v1", H2DriverVersion.V1, "1.4.200");
assertBundledDriver("h2-v2", H2DriverVersion.V2, "2.1.214");
assertBundledDriver("h2-v3", H2DriverVersion.V3, "2.4.240");
assertBundledDriver("h2-legacy", H2DriverVersion.V2, "2.1.214");
}
@Test
void autoSelectsBundledDriverFromMvStoreFormat() {
assertAutoDetectedFileDriver(H2DriverVersion.V1);
assertAutoDetectedFileDriver(H2DriverVersion.V2);
assertAutoDetectedFileDriver(H2DriverVersion.V3);
}
@Test
void autoSelectsH2V1ForPageStoreDatabase() {
Path base = tempDirectory.resolve("pagestore");
H2Agent creator = new H2Agent();
creator.connect(profileParams("h2-v1", "file:" + base + ";MV_STORE=FALSE"));
try {
creator.executeQuery("CREATE TABLE PAGESTORE_PROBE (ID INT PRIMARY KEY)", null, new ExecuteQueryOptions());
} finally {
creator.disconnect();
}
Assertions.assertTrue(Files.isRegularFile(Path.of(base + ".h2.db")));
H2Agent detected = new H2Agent();
detected.connect(profileParams("h2", "file:" + base + ";MV_STORE=FALSE;IFEXISTS=TRUE"));
try {
Assertions.assertEquals(H2DriverVersion.V1, detected.driverVersion());
Assertions.assertEquals(List.of("PAGESTORE_PROBE"), detected.listTables("PUBLIC").stream().map(TableInfo::getName).toList());
} finally {
detected.disconnect();
}
}
@Test
void autoUsesLatestDriverForNewLocalDatabase() throws Exception {
Path base = tempDirectory.resolve("new-auto");
H2Agent agent = new H2Agent();
agent.connect(profileParams("h2", "file:" + base));
try {
Assertions.assertEquals(H2DriverVersion.V3, agent.driverVersion());
agent.executeQuery("CREATE TABLE LATEST_PROBE (ID INT PRIMARY KEY)", null, new ExecuteQueryOptions());
} finally {
agent.disconnect();
}
Assertions.assertEquals(3, H2FileFormatDetector.detect("jdbc:h2:file:" + base).orElseThrow());
}
@Test
void autoRejectsCorruptMvStoreWithoutModifyingIt() throws Exception {
Path base = tempDirectory.resolve("corrupt");
Path file = Path.of(base + ".mv.db");
byte[] corrupt = new byte[8192];
java.util.Arrays.fill(corrupt, (byte) 0x5a);
Files.write(file, corrupt);
FileTime modified = Files.getLastModifiedTime(file);
RuntimeException error = Assertions.assertThrows(
RuntimeException.class,
() -> new H2Agent().connect(profileParams("h2", "file:" + base + ";IFEXISTS=TRUE"))
);
Assertions.assertTrue(hasCauseMessage(error, "Cannot determine the H2 MVStore format"));
Assertions.assertArrayEquals(corrupt, Files.readAllBytes(file));
Assertions.assertEquals(modified, Files.getLastModifiedTime(file));
}
@Test
void autoRejectsAmbiguousPageStoreAndMvStoreFiles() throws Exception {
Path base = tempDirectory.resolve("ambiguous");
Files.write(Path.of(base + ".h2.db"), new byte[]{1});
Files.write(Path.of(base + ".mv.db"), new byte[8192]);
RuntimeException error = Assertions.assertThrows(
RuntimeException.class,
() -> new H2Agent().connect(profileParams("h2", "file:" + base + ";IFEXISTS=TRUE"))
);
Assertions.assertTrue(hasCauseMessage(error, "Both H2 PageStore and MVStore files exist"));
}
@Test
void autoHonorsIfExistsWithoutCreatingMissingDatabase() {
Path base = tempDirectory.resolve("missing-ifexists");
Assertions.assertThrows(
RuntimeException.class,
() -> new H2Agent().connect(profileParams("h2", "file:" + base + ";IFEXISTS=TRUE"))
);
Assertions.assertFalse(Files.exists(Path.of(base + ".mv.db")));
Assertions.assertFalse(Files.exists(Path.of(base + ".h2.db")));
}
@Test
void autoOpensExistingMvStoreReadOnlyWithoutChangingFile() throws Exception {
Path base = tempDirectory.resolve("readonly");
H2Agent creator = new H2Agent();
creator.connect(profileParams("h2-v2", "file:" + base));
try {
creator.executeQuery("CREATE TABLE READONLY_PROBE (ID INT PRIMARY KEY)", null, new ExecuteQueryOptions());
} finally {
creator.disconnect();
}
Path file = Path.of(base + ".mv.db");
byte[] before = Files.readAllBytes(file);
H2Agent reader = new H2Agent();
reader.connect(profileParams("h2", "file:" + base + ";ACCESS_MODE_DATA=r;IFEXISTS=TRUE"));
try {
Assertions.assertEquals(H2DriverVersion.V2, reader.driverVersion());
Assertions.assertEquals(0, reader.executeQuery("SELECT * FROM READONLY_PROBE", null, new ExecuteQueryOptions()).getRows().size());
Assertions.assertThrows(
RuntimeException.class,
() -> reader.executeQuery("INSERT INTO READONLY_PROBE VALUES (1)", null, new ExecuteQueryOptions())
);
} finally {
reader.disconnect();
}
Assertions.assertArrayEquals(before, Files.readAllBytes(file));
}
@Test
void encryptedMvStoreRequiresExplicitDriverProfile() {
Path base = tempDirectory.resolve("encrypted");
ConnectParams createParams = profileParams("h2-v3", "file:" + base + ";CIPHER=AES");
createParams.setPassword("file-secret user-secret");
H2Agent creator = new H2Agent();
creator.connect(createParams);
try {
creator.executeQuery("CREATE TABLE ENCRYPTED_PROBE (ID INT PRIMARY KEY)", null, new ExecuteQueryOptions());
} finally {
creator.disconnect();
}
ConnectParams autoParams = profileParams("h2", "file:" + base + ";CIPHER=AES;IFEXISTS=TRUE");
autoParams.setPassword("file-secret user-secret");
RuntimeException error = Assertions.assertThrows(RuntimeException.class, () -> new H2Agent().connect(autoParams));
Assertions.assertTrue(hasCauseMessage(error, "choose an explicit H2 driver profile"));
ConnectParams explicitParams = profileParams("h2-v3", "file:" + base + ";CIPHER=AES;IFEXISTS=TRUE");
explicitParams.setPassword("file-secret user-secret");
H2Agent detected = new H2Agent();
detected.connect(explicitParams);
try {
Assertions.assertEquals(H2DriverVersion.V3, detected.driverVersion());
Assertions.assertEquals(List.of("ENCRYPTED_PROBE"), detected.listTables("PUBLIC").stream().map(TableInfo::getName).toList());
} finally {
detected.disconnect();
}
}
@Test
void loadsCustomH2DriverFromExternalClasspath() throws Exception {
Path helperJar = tempDirectory.resolve("helper.jar");
try (JarOutputStream ignored = new JarOutputStream(Files.newOutputStream(helperJar))) {
}
Path driverJar = copyBundledDriver("h2-2.1.214.jar");
ConnectParams params = profileParams("h2-custom", "mem:dbx-h2-custom;DB_CLOSE_DELAY=-1");
params.setJdbc_driver_paths(List.of(helperJar.toString(), driverJar.toString()));
params.setJdbc_driver_class("org.h2.Driver");
H2Agent agent = new H2Agent();
agent.connect(params);
try {
Assertions.assertEquals(H2DriverVersion.CUSTOM, agent.driverVersion());
Assertions.assertTrue(agent.getDatabaseInfo().get("driverVersion").startsWith("2.1.214"));
agent.executeQuery("CREATE TABLE CUSTOM_DRIVER_PROBE (ID INT PRIMARY KEY)", null, new ExecuteQueryOptions());
Assertions.assertEquals(List.of("CUSTOM_DRIVER_PROBE"), agent.listTables("PUBLIC").stream().map(TableInfo::getName).toList());
} finally {
agent.disconnect();
}
}
@Test
void rejectsCustomH2DriverWithoutClasspath() {
ConnectParams params = profileParams("h2-custom", "mem:dbx-h2-custom-missing;DB_CLOSE_DELAY=-1");
RuntimeException error = Assertions.assertThrows(RuntimeException.class, () -> new H2Agent().connect(params));
Assertions.assertTrue(hasCauseMessage(error, "requires at least one JDBC JAR path"));
}
@Test
void rejectsMissingCustomH2DriverJar() {
ConnectParams params = profileParams("h2-custom", "mem:dbx-h2-custom-missing-file;DB_CLOSE_DELAY=-1");
params.setJdbc_driver_paths(List.of(tempDirectory.resolve("missing-h2.jar").toString()));
RuntimeException error = Assertions.assertThrows(RuntimeException.class, () -> new H2Agent().connect(params));
Assertions.assertTrue(hasCauseMessage(error, "Custom H2 JDBC JAR does not exist"));
}
@Test
void rejectsInvalidCustomH2DriverClass() throws Exception {
ConnectParams params = profileParams("h2-custom", "mem:dbx-h2-custom-invalid-class;DB_CLOSE_DELAY=-1");
params.setJdbc_driver_paths(List.of(copyBundledDriver("h2-2.4.240.jar").toString()));
params.setJdbc_driver_class("example.MissingH2Driver");
RuntimeException error = Assertions.assertThrows(RuntimeException.class, () -> new H2Agent().connect(params));
Assertions.assertTrue(hasCauseMessage(error, "example.MissingH2Driver"));
}
private void assertBundledDriver(String profile, H2DriverVersion expected, String expectedVersion) {
ConnectParams params = profileParams(profile, "mem:dbx-" + profile + ";DB_CLOSE_DELAY=-1");
H2Agent agent = new H2Agent();
agent.connect(params);
try {
Assertions.assertEquals(expected, agent.driverVersion());
Assertions.assertTrue(agent.getDatabaseInfo().get("driverVersion").startsWith(expectedVersion));
} finally {
agent.disconnect();
}
}
private void assertAutoDetectedFileDriver(H2DriverVersion expected) {
Path base = tempDirectory.resolve("format-" + expected.storageFormat());
ConnectParams createParams = profileParams(expected.profile(), "file:" + base);
H2Agent creator = new H2Agent();
creator.connect(createParams);
try {
creator.executeQuery("CREATE TABLE VERSION_PROBE (ID INT PRIMARY KEY)", null, new ExecuteQueryOptions());
} finally {
creator.disconnect();
}
ConnectParams autoParams = profileParams("h2", "file:" + base + ";IFEXISTS=TRUE");
H2Agent detected = new H2Agent();
detected.connect(autoParams);
try {
Assertions.assertEquals(expected, detected.driverVersion());
Assertions.assertEquals(List.of("VERSION_PROBE"), detected.listTables("PUBLIC").stream().map(TableInfo::getName).toList());
} finally {
detected.disconnect();
}
}
private static ConnectParams profileParams(String profile, String database) {
ConnectParams params = new ConnectParams("", 0, database, "sa", "", "", "", false);
params.setDriver_profile(profile);
return params;
}
private Path copyBundledDriver(String name) throws Exception {
Path target = tempDirectory.resolve(name);
try (InputStream input = H2AgentMigrationTest.class.getResourceAsStream("/drivers/" + name)) {
Assertions.assertNotNull(input, "Missing bundled test driver " + name);
Files.copy(input, target, StandardCopyOption.REPLACE_EXISTING);
}
return target;
}
private static boolean hasCauseMessage(Throwable error, String expected) {
Throwable current = error;
while (current != null) {
if (current.getMessage() != null && current.getMessage().contains(expected)) {
return true;
}
current = current.getCause();
}
return false;
}
private static void assertH2DatabaseInfo(Map<String, String> info) {
Assertions.assertEquals("H2", info.get("productName"));
Assertions.assertFalse(info.get("productVersion").isEmpty());
@@ -73,6 +354,60 @@ class H2AgentMigrationTest {
}
}
class H2VersionMatrixTest {
@Test
void metadataAndExecutionWorkAcrossBundledVersions() {
for (String profile : List.of("h2-v1", "h2-v2", "h2-v3")) {
H2Agent agent = connect(profile);
try {
agent.executeQuery("CREATE TABLE PARENT_TABLE (ID INT PRIMARY KEY, NAME VARCHAR(64))", null, new ExecuteQueryOptions());
agent.executeQuery(
"CREATE TABLE CHILD_TABLE (ID INT PRIMARY KEY, PARENT_ID INT, AMOUNT DECIMAL(12, 2), "
+ "CONSTRAINT FK_CHILD_PARENT FOREIGN KEY (PARENT_ID) REFERENCES PARENT_TABLE(ID))",
null,
new ExecuteQueryOptions()
);
agent.executeQuery("CREATE INDEX IDX_CHILD_AMOUNT ON CHILD_TABLE(AMOUNT)", null, new ExecuteQueryOptions());
agent.executeQuery("CREATE VIEW CHILD_VIEW AS SELECT ID, AMOUNT FROM CHILD_TABLE", null, new ExecuteQueryOptions());
agent.executeQuery("CREATE ALIAS DBX_REVERSE FOR \"java.lang.Integer.reverse\"", null, new ExecuteQueryOptions());
agent.executeQuery("INSERT INTO PARENT_TABLE VALUES (1, 'parent')", null, new ExecuteQueryOptions());
agent.executeQuery("INSERT INTO CHILD_TABLE VALUES (1, 1, 12.34)", null, new ExecuteQueryOptions());
Assertions.assertTrue(agent.listSchemas().contains("PUBLIC"), profile);
Assertions.assertTrue(agent.listTables("PUBLIC").stream().map(TableInfo::getName).toList().containsAll(List.of("PARENT_TABLE", "CHILD_TABLE", "CHILD_VIEW")), profile);
List<ObjectInfo> objects = agent.listObjects("PUBLIC");
Assertions.assertTrue(objects.stream().anyMatch(object -> object.getName().equals("DBX_REVERSE")), profile);
Assertions.assertTrue(agent.getObjectSource("PUBLIC", "CHILD_VIEW", "VIEW").getSource().contains("CHILD_TABLE"), profile);
List<ColumnInfo> columns = agent.getColumns("PUBLIC", "CHILD_TABLE");
Assertions.assertEquals(List.of("ID", "PARENT_ID", "AMOUNT"), columns.stream().map(ColumnInfo::getName).toList(), profile);
Assertions.assertTrue(columns.stream().filter(column -> column.getName().equals("ID")).findFirst().orElseThrow().getIs_primary_key(), profile);
List<IndexInfo> indexes = agent.listIndexes("PUBLIC", "CHILD_TABLE");
Assertions.assertTrue(indexes.stream().anyMatch(index -> index.getName().equals("IDX_CHILD_AMOUNT")), profile);
Assertions.assertTrue(indexes.stream().anyMatch(IndexInfo::getIs_primary), profile);
List<ForeignKeyInfo> foreignKeys = agent.listForeignKeys("PUBLIC", "CHILD_TABLE");
Assertions.assertTrue(foreignKeys.stream().anyMatch(key -> key.getName().equals("FK_CHILD_PARENT")), profile);
QueryResult result = agent.executeQuery("SELECT DBX_REVERSE(1), AMOUNT FROM CHILD_TABLE", null, new ExecuteQueryOptions());
Assertions.assertEquals(1, result.getRows().size(), profile);
} finally {
agent.disconnect();
}
}
}
private static H2Agent connect(String profile) {
ConnectParams params = new ConnectParams("", 0, "mem:matrix-" + profile + ";DB_CLOSE_DELAY=-1", "sa", "", "", "", false);
params.setDriver_profile(profile);
H2Agent agent = new H2Agent();
agent.connect(params);
return agent;
}
}
class H2ExecutionBehaviorTest extends JdbcExecutionBehaviorTest {
@Override
protected DatabaseAgent createConnectedAgent(String databaseName) {
@@ -62,7 +62,7 @@ public final class IrisAgent extends ConfiguredJdbcAgent {
stringOrNull(rs, "COLUMN_DEFAULT"),
"YES".equalsIgnoreCase(stringOrNull(rs, "PRIMARY_KEY")),
null,
null,
stringOrNull(rs, "DESCRIPTION"),
intOrNull(rs, "NUMERIC_PRECISION"),
intOrNull(rs, "NUMERIC_SCALE"),
intOrNull(rs, "CHARACTER_MAXIMUM_LENGTH")
@@ -126,7 +126,7 @@ public final class IrisAgent extends ConfiguredJdbcAgent {
private static String columnSql(String schema) {
StringBuilder sql = new StringBuilder(
"SELECT COLUMN_NAME, DATA_TYPE, IS_NULLABLE, COLUMN_DEFAULT, "
+ "PRIMARY_KEY, NUMERIC_PRECISION, NUMERIC_SCALE, CHARACTER_MAXIMUM_LENGTH "
+ "PRIMARY_KEY, DESCRIPTION, NUMERIC_PRECISION, NUMERIC_SCALE, CHARACTER_MAXIMUM_LENGTH "
+ "FROM INFORMATION_SCHEMA.COLUMNS WHERE "
);
if (!isBlank(schema)) {
@@ -19,6 +19,7 @@ import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
class IrisAgentTest {
@@ -63,12 +64,45 @@ class IrisAgentTest {
assertEquals("INTEGER", columns.get(0).getData_type());
assertFalse(columns.get(0).getIs_nullable());
assertTrue(columns.get(0).getIs_primary_key());
assertEquals("42", columns.get(0).getColumn_default());
assertEquals("identifier", columns.get(0).getComment());
assertEquals(Integer.valueOf(10), columns.get(0).getNumeric_precision());
assertEquals(Integer.valueOf(0), columns.get(0).getNumeric_scale());
assertNull(columns.get(0).getCharacter_maximum_length());
assertEquals("NAME", columns.get(1).getName());
assertEquals("VARCHAR", columns.get(1).getData_type());
assertTrue(columns.get(1).getIs_nullable());
assertNull(columns.get(1).getColumn_default());
assertNull(columns.get(1).getComment());
assertNull(columns.get(1).getNumeric_precision());
assertNull(columns.get(1).getNumeric_scale());
assertEquals(Integer.valueOf(64), columns.get(1).getCharacter_maximum_length());
assertEquals(1, calls.size());
assertFalse(calls.get(0).contains("LIMIT"));
assertTrue(calls.get(0).contains("PRIMARY_KEY"));
assertTrue(calls.get(0).contains("DESCRIPTION"));
assertTrue(calls.get(0).contains("INFORMATION_SCHEMA.COLUMNS"));
assertTrue(calls.get(0).contains("UPPER(TABLE_SCHEMA) = UPPER(?)"));
}
@Test
void readsDescriptionsWithoutSchemaFilter() {
List<String> calls = new ArrayList<>();
Connection conn = fakeConnection(
calls,
true,
Collections.emptyList(),
null,
Collections.emptyList()
);
List<ColumnInfo> columns = IrisAgent.irisColumns(conn, null, "People");
assertEquals("identifier", columns.get(0).getComment());
assertNull(columns.get(1).getComment());
assertEquals(1, calls.size());
assertTrue(calls.get(0).contains("DESCRIPTION"));
assertFalse(calls.get(0).contains("TABLE_SCHEMA"));
}
@Test
@@ -185,8 +219,11 @@ class IrisAgentTest {
"COLUMN_NAME", "ID",
"DATA_TYPE", "INTEGER",
"IS_NULLABLE", "NO",
"COLUMN_DEFAULT", "42",
"PRIMARY_KEY", inlinePrimaryKey ? "YES" : "NO",
"NUMERIC_PRECISION", 10
"DESCRIPTION", "identifier",
"NUMERIC_PRECISION", 10,
"NUMERIC_SCALE", 0
),
Map.of(
"COLUMN_NAME", "NAME",
@@ -0,0 +1,72 @@
package com.dbx.agent.kafka;
import java.net.Socket;
import java.security.KeyStore;
import java.security.Provider;
import java.security.Security;
import java.security.cert.X509Certificate;
import javax.net.ssl.ManagerFactoryParameters;
import javax.net.ssl.SSLEngine;
import javax.net.ssl.TrustManager;
import javax.net.ssl.TrustManagerFactorySpi;
import javax.net.ssl.X509ExtendedTrustManager;
/** Trust manager used only when a Kafka connection explicitly skips TLS verification. */
public final class DbxInsecureTrustManagerFactory {
static final String ALGORITHM = "DBX_TRUST_ALL";
private static final String PROVIDER_NAME = "DBX_KAFKA_INSECURE_TLS";
private DbxInsecureTrustManagerFactory() {}
static void ensureRegistered() {
if (Security.getProvider(PROVIDER_NAME) != null) return;
synchronized (DbxInsecureTrustManagerFactory.class) {
if (Security.getProvider(PROVIDER_NAME) != null) return;
Provider provider = new Provider(PROVIDER_NAME, "1.0", "DBX Kafka trust-all provider") {};
provider.put("TrustManagerFactory." + ALGORITHM, Spi.class.getName());
Security.addProvider(provider);
}
}
public static final class Spi extends TrustManagerFactorySpi {
private static final TrustManager[] TRUST_MANAGERS = { new TrustAllManager() };
@Override
protected void engineInit(KeyStore keyStore) {}
@Override
protected void engineInit(ManagerFactoryParameters managerFactoryParameters) {}
@Override
protected TrustManager[] engineGetTrustManagers() {
return TRUST_MANAGERS.clone();
}
}
private static final class TrustAllManager extends X509ExtendedTrustManager {
private static final X509Certificate[] NO_ISSUERS = new X509Certificate[0];
@Override
public void checkClientTrusted(X509Certificate[] chain, String authType) {}
@Override
public void checkServerTrusted(X509Certificate[] chain, String authType) {}
@Override
public void checkClientTrusted(X509Certificate[] chain, String authType, Socket socket) {}
@Override
public void checkServerTrusted(X509Certificate[] chain, String authType, Socket socket) {}
@Override
public void checkClientTrusted(X509Certificate[] chain, String authType, SSLEngine engine) {}
@Override
public void checkServerTrusted(X509Certificate[] chain, String authType, SSLEngine engine) {}
@Override
public X509Certificate[] getAcceptedIssuers() {
return NO_ISSUERS.clone();
}
}
}
@@ -507,16 +507,9 @@ public final class KafkaAgent {
}
}
// TLS properties
JsonObject tls = conn.has("tls") && conn.get("tls").isJsonObject()
? conn.getAsJsonObject("tls") : null;
boolean skipVerify = boolOrDefault(conn, "tls_skip_verify", false)
|| boolOrDefault(conn, "tlsSkipVerify", false)
|| (tls != null && boolOrDefault(tls, "skip_verify", false));
if (skipVerify) {
props.put("ssl.endpoint.identification.algorithm", "");
}
// TLS properties
if (tls != null) {
String truststorePath = stringOrEmpty(tls, "truststore_path");
if (!truststorePath.isBlank()) {
@@ -540,6 +533,20 @@ public final class KafkaAgent {
static void applyConnectionProperties(JsonObject conn, Properties props) {
applySecurityProperties(conn, props);
applyExtraProperties(conn, props);
applyTlsSkipVerification(conn, props);
}
private static void applyTlsSkipVerification(JsonObject conn, Properties props) {
JsonObject tls = conn.has("tls") && conn.get("tls").isJsonObject()
? conn.getAsJsonObject("tls") : null;
boolean skipVerify = boolOrDefault(conn, "tls_skip_verify", false)
|| boolOrDefault(conn, "tlsSkipVerify", false)
|| (tls != null && boolOrDefault(tls, "skip_verify", false));
if (!skipVerify) return;
DbxInsecureTrustManagerFactory.ensureRegistered();
props.put("ssl.endpoint.identification.algorithm", "");
props.put("ssl.trustmanager.algorithm", DbxInsecureTrustManagerFactory.ALGORITHM);
}
static String jaasValue(String value) {
@@ -643,9 +650,15 @@ public final class KafkaAgent {
topics.add(topic);
}
} catch (Exception error) {
if (!isUnsupportedVersionError(error)) {
if (!isUnsupportedVersionError(error) && !isTimeoutError(error)) {
throw error;
}
if (isTimeoutError(error)) {
logger().warn(
"Kafka topic descriptions timed out; returning topic names without partition metadata",
error
);
}
for (TopicListing listing : listings) {
Map<String, Object> topic = new LinkedHashMap<>();
topic.put("name", listing.name());
@@ -956,22 +969,184 @@ public final class KafkaAgent {
private static Object listConsumerGroups(JsonObject params) throws Exception {
AdminClient admin = requireAdmin();
int timeout = requestTimeout(params);
String filterTopic = stringOrEmpty(params, "topic");
Collection<ConsumerGroupListing> groups = admin.listConsumerGroups(
new ListConsumerGroupsOptions().timeoutMs(timeout))
.all().get(timeout, TimeUnit.MILLISECONDS);
List<String> groupIds = groups.stream()
.map(ConsumerGroupListing::groupId)
.sorted()
.toList();
if (groupIds.isEmpty()) {
return Collections.singletonMap("groups", Collections.emptyList());
}
// Batch-describe every group in one Kafka Admin request. Per-group
// failures degrade to an empty member list instead of failing the batch.
Map<String, ConsumerGroupDescription> descriptions = new HashMap<>();
DescribeConsumerGroupsResult described = admin.describeConsumerGroups(
groupIds,
new DescribeConsumerGroupsOptions().timeoutMs(timeout)
);
for (String groupId : groupIds) {
try {
descriptions.put(
groupId,
described.describedGroups().get(groupId).get(timeout, TimeUnit.MILLISECONDS)
);
} catch (Exception error) {
logger().debug("Consumer group details unavailable for {}: {}", groupId, normalizeErrorMessage(error));
}
}
// Batch-resolve committed offsets for every group in one Admin request.
Map<String, ListConsumerGroupOffsetsSpec> offsetSpecs = new LinkedHashMap<>();
for (String groupId : groupIds) {
offsetSpecs.put(groupId, new ListConsumerGroupOffsetsSpec());
}
Map<String, Map<TopicPartition, OffsetAndMetadata>> committedOffsets = new HashMap<>();
ListConsumerGroupOffsetsResult offsetsResult = admin.listConsumerGroupOffsets(
offsetSpecs,
new ListConsumerGroupOffsetsOptions().timeoutMs(timeout)
);
for (String groupId : groupIds) {
try {
Map<TopicPartition, OffsetAndMetadata> rows = offsetsResult
.partitionsToOffsetAndMetadata(groupId)
.get(timeout, TimeUnit.MILLISECONDS);
committedOffsets.put(groupId, rows == null ? Collections.emptyMap() : rows);
} catch (Exception error) {
committedOffsets.put(groupId, Collections.emptyMap());
logger().debug("Committed offsets unavailable for {}: {}", groupId, normalizeErrorMessage(error));
}
}
// End offsets: one batched Admin request limited to the requested
// topic's partitions (deduplicated across groups). Without a topic
// filter no end offsets are resolved, keeping the listing cheap.
Map<TopicPartition, Long> endOffsets = new HashMap<>();
if (!filterTopic.isEmpty()) {
LinkedHashSet<TopicPartition> topicPartitions = new LinkedHashSet<>();
for (Map<TopicPartition, OffsetAndMetadata> rows : committedOffsets.values()) {
for (TopicPartition topicPartition : rows.keySet()) {
if (topicPartition.topic().equals(filterTopic)) {
topicPartitions.add(topicPartition);
}
}
}
if (!topicPartitions.isEmpty()) {
Map<TopicPartition, OffsetSpec> endOffsetSpecs = new LinkedHashMap<>();
for (TopicPartition topicPartition : topicPartitions) {
endOffsetSpecs.put(topicPartition, OffsetSpec.latest());
}
try {
ListOffsetsResult latestOffsets = admin.listOffsets(
endOffsetSpecs,
new ListOffsetsOptions().timeoutMs(timeout)
);
for (TopicPartition topicPartition : topicPartitions) {
try {
ListOffsetsResult.ListOffsetsResultInfo info = latestOffsets
.partitionResult(topicPartition)
.get(timeout, TimeUnit.MILLISECONDS);
endOffsets.put(topicPartition, info.offset());
} catch (Exception error) {
// Leave the partition absent so callers can tell an
// unknown end offset (no lag contribution) apart from zero lag.
logger().debug("End offset unavailable for {}: {}", topicPartition, normalizeErrorMessage(error));
}
}
} catch (Exception error) {
logger().debug("End offset batch unavailable: {}", normalizeErrorMessage(error));
}
}
}
Map<String, ConsumerGroupListing> listingByGroup = groups.stream()
.collect(Collectors.toMap(ConsumerGroupListing::groupId, listing -> listing));
List<Map<String, Object>> result = new ArrayList<>();
for (ConsumerGroupListing group : groups) {
for (String groupId : groupIds) {
ConsumerGroupListing listing = listingByGroup.get(groupId);
ConsumerGroupDescription description = descriptions.get(groupId);
Map<TopicPartition, OffsetAndMetadata> committed = committedOffsets.getOrDefault(groupId, Collections.emptyMap());
Map<String, Object> g = new LinkedHashMap<>();
g.put("groupId", group.groupId());
g.put("state", group.state().map(Enum::name).orElse("UNKNOWN"));
g.put("simpleGroup", group.isSimpleConsumerGroup());
g.put("groupId", groupId);
g.put("state", listing == null ? "UNKNOWN" : listing.state().map(Enum::name).orElse("UNKNOWN"));
g.put("simpleGroup", listing != null && listing.isSimpleConsumerGroup());
g.put("members", memberMaps(description == null ? Collections.emptyList() : description.members()));
g.put("committedOffsets", offsetRows(committed, filterTopic));
g.put("endOffsets", endOffsetRows(committed, endOffsets, filterTopic));
result.add(g);
}
result.sort(Comparator.comparing(m -> (String) m.get("groupId")));
return Collections.singletonMap("groups", result);
}
static List<Map<String, Object>> memberMaps(Collection<MemberDescription> members) {
List<Map<String, Object>> result = new ArrayList<>();
for (MemberDescription member : members) {
Map<String, Object> m = new LinkedHashMap<>();
m.put("memberId", member.consumerId());
m.put("clientId", member.clientId());
m.put("host", member.host());
List<Map<String, Object>> assignments = new ArrayList<>();
for (TopicPartition topicPartition : member.assignment().topicPartitions()) {
Map<String, Object> a = new LinkedHashMap<>();
a.put("topic", topicPartition.topic());
a.put("partition", topicPartition.partition());
assignments.add(a);
}
m.put("assignments", assignments);
result.add(m);
}
return result;
}
/** Committed offsets as rows, optionally restricted to one topic. */
static List<Map<String, Object>> offsetRows(Map<TopicPartition, OffsetAndMetadata> offsets, String topic) {
List<Map<String, Object>> rows = new ArrayList<>();
for (Map.Entry<TopicPartition, OffsetAndMetadata> entry : offsets.entrySet()) {
if (!topic.isEmpty() && !entry.getKey().topic().equals(topic)) {
continue;
}
Map<String, Object> row = new LinkedHashMap<>();
row.put("topic", entry.getKey().topic());
row.put("partition", entry.getKey().partition());
row.put("offset", entry.getValue().offset());
rows.add(row);
}
rows.sort(Comparator.comparingInt(row -> (int) row.get("partition")));
return rows;
}
/** End offsets for a group's committed partitions, restricted to one topic. */
static List<Map<String, Object>> endOffsetRows(
Map<TopicPartition, OffsetAndMetadata> committed,
Map<TopicPartition, Long> endOffsets,
String topic
) {
List<Map<String, Object>> rows = new ArrayList<>();
for (TopicPartition topicPartition : committed.keySet()) {
if (!topic.isEmpty() && !topicPartition.topic().equals(topic)) {
continue;
}
Long endOffset = endOffsets.get(topicPartition);
if (endOffset == null) {
continue;
}
Map<String, Object> row = new LinkedHashMap<>();
row.put("topic", topicPartition.topic());
row.put("partition", topicPartition.partition());
row.put("offset", endOffset);
rows.add(row);
}
rows.sort(Comparator.comparingInt(row -> (int) row.get("partition")));
return rows;
}
private static Object getConsumerGroupSnapshot(JsonObject params) throws Exception {
AdminClient admin = requireAdmin();
int timeout = requestTimeout(params);
@@ -2413,6 +2588,21 @@ public final class KafkaAgent {
return false;
}
/**
* A topic listing is still useful when the broker cannot describe every topic before the
* request deadline. Keep timeout detection narrow so authorization, transport, and unknown
* topic failures remain visible to the caller instead of being mistaken for a partial result.
*/
private static boolean isTimeoutError(Throwable error) {
for (Throwable current : causeChain(error)) {
if (current instanceof java.util.concurrent.TimeoutException
|| current instanceof org.apache.kafka.common.errors.TimeoutException) {
return true;
}
}
return false;
}
static boolean hasUnsupportedVersionException(Throwable error) {
return causeChain(error).stream().anyMatch(UnsupportedVersionException.class::isInstance);
}
@@ -13,6 +13,8 @@ import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.security.KeyStore;
import java.security.cert.X509Certificate;
import java.time.Duration;
import java.util.ArrayDeque;
import java.util.ArrayList;
@@ -27,6 +29,8 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import javax.net.ssl.TrustManagerFactory;
import javax.net.ssl.X509TrustManager;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
@@ -138,6 +142,65 @@ class KafkaAgentTest {
assertEquals("End offsets unavailable for 1 partition(s)", group.get("error"));
}
@Test
void consumerGroupListRowsFilterToTheRequestedTopicAndSortByPartition() {
Map<TopicPartition, OffsetAndMetadata> offsets = new HashMap<>();
offsets.put(new TopicPartition("orders", 2), new OffsetAndMetadata(20));
offsets.put(new TopicPartition("payments", 0), new OffsetAndMetadata(5));
offsets.put(new TopicPartition("orders", 0), new OffsetAndMetadata(8));
List<Map<String, Object>> rows = KafkaAgent.offsetRows(offsets, "orders");
assertEquals(List.of("orders:0", "orders:2"), rows.stream()
.map(row -> row.get("topic") + ":" + row.get("partition"))
.toList());
assertEquals(8L, rows.get(0).get("offset"));
assertEquals(20L, rows.get(1).get("offset"));
}
@Test
void consumerGroupListRowsKeepEveryPartitionWithoutATopicFilter() {
Map<TopicPartition, OffsetAndMetadata> offsets = new HashMap<>();
offsets.put(new TopicPartition("orders", 0), new OffsetAndMetadata(8));
offsets.put(new TopicPartition("payments", 1), new OffsetAndMetadata(3));
List<Map<String, Object>> rows = KafkaAgent.offsetRows(offsets, "");
assertEquals(2, rows.size());
}
@Test
void consumerGroupListEndOffsetRowsSkipPartitionsWithoutResolvedEndOffsets() {
Map<TopicPartition, OffsetAndMetadata> committed = new HashMap<>();
committed.put(new TopicPartition("orders", 0), new OffsetAndMetadata(8));
committed.put(new TopicPartition("orders", 1), new OffsetAndMetadata(20));
Map<TopicPartition, Long> endOffsets = new HashMap<>();
endOffsets.put(new TopicPartition("orders", 1), 24L);
List<Map<String, Object>> rows = KafkaAgent.endOffsetRows(committed, endOffsets, "orders");
assertEquals(List.of("orders:1"), rows.stream()
.map(row -> row.get("topic") + ":" + row.get("partition"))
.toList());
assertEquals(24L, rows.get(0).get("offset"));
}
@Test
void consumerGroupListEndOffsetRowsFilterToTheRequestedTopic() {
Map<TopicPartition, OffsetAndMetadata> committed = new HashMap<>();
committed.put(new TopicPartition("orders", 0), new OffsetAndMetadata(8));
committed.put(new TopicPartition("payments", 0), new OffsetAndMetadata(4));
Map<TopicPartition, Long> endOffsets = new HashMap<>();
endOffsets.put(new TopicPartition("orders", 0), 10L);
endOffsets.put(new TopicPartition("payments", 0), 9L);
List<Map<String, Object>> rows = KafkaAgent.endOffsetRows(committed, endOffsets, "orders");
assertEquals(List.of("orders:0"), rows.stream()
.map(row -> row.get("topic") + ":" + row.get("partition"))
.toList());
}
@Test
void explicitConsumerGroupOffsetsAcceptOneBoundedPartitionOffset() {
JsonObject params = JsonParser.parseString("""
@@ -192,6 +255,25 @@ class KafkaAgentTest {
assertFalse(topics.get(1).containsKey("partitions"));
}
@Test
void topicListingFallsBackWhenDescriptionsTimeOut() throws Exception {
Object result = KafkaAgent.topicListResult(
Arrays.asList(
new TopicListing("orders", Uuid.randomUuid(), false),
new TopicListing("payments", Uuid.randomUuid(), false)
),
names -> {
throw new java.util.concurrent.TimeoutException("describeTopics timed out");
}
);
@SuppressWarnings("unchecked")
List<Map<String, Object>> topics = (List<Map<String, Object>>) ((Map<String, Object>) result).get("topics");
assertEquals(Arrays.asList("orders", "payments"), topics.stream().map(topic -> topic.get("name")).toList());
assertFalse(topics.get(0).containsKey("partitions"));
assertFalse(topics.get(1).containsKey("replicationFactor"));
}
@Test
void topicListingPreservesNonVersionDescriptionErrors() {
IllegalStateException failure = new IllegalStateException("metadata authorization failed");
@@ -1292,6 +1374,46 @@ class KafkaAgentTest {
);
}
@Test
void skipTlsVerificationDisablesHostnameAndCertificateChainValidation() throws Exception {
Properties props = new Properties();
KafkaAgent.applyConnectionProperties(JsonParser.parseString("""
{
"security_protocol": "SASL_SSL",
"tls_skip_verify": true,
"properties": {
"ssl.endpoint.identification.algorithm": "https",
"ssl.trustmanager.algorithm": "PKIX"
}
}
""").getAsJsonObject(), props);
assertEquals("", props.getProperty("ssl.endpoint.identification.algorithm"));
assertEquals(
DbxInsecureTrustManagerFactory.ALGORITHM,
props.getProperty("ssl.trustmanager.algorithm")
);
TrustManagerFactory factory = TrustManagerFactory.getInstance(DbxInsecureTrustManagerFactory.ALGORITHM);
factory.init((KeyStore) null);
X509TrustManager trustManager = (X509TrustManager) factory.getTrustManagers()[0];
assertDoesNotThrow(() -> trustManager.checkServerTrusted(new X509Certificate[0], "RSA"));
}
@Test
void verifiedTlsKeepsKafkaDefaultTrustAndHostnameValidation() {
Properties props = new Properties();
KafkaAgent.applyConnectionProperties(JsonParser.parseString("""
{
"security_protocol": "SASL_SSL",
"tls_skip_verify": false
}
""").getAsJsonObject(), props);
assertNull(props.getProperty("ssl.endpoint.identification.algorithm"));
assertNull(props.getProperty("ssl.trustmanager.algorithm"));
}
@Test
void appliesAllowedKerberosSystemPropertiesFromConnectionProperties() {
Map<String, String> previous = KafkaAgent.applyKerberosSystemProperties(JsonParser.parseString("""
+148 -5
View File
@@ -58,7 +58,7 @@ func TestKingbaseIntegration(t *testing.T) {
}
})
mustExecute(t, server, "CREATE TABLE "+qualifiedSchema+"."+quoteIdentifier(parent)+" (id integer PRIMARY KEY, name varchar(64) NOT NULL)")
mustExecute(t, server, "CREATE TABLE "+qualifiedSchema+"."+quoteIdentifier(parent)+" (id integer PRIMARY KEY, name varchar(64) NOT NULL, code character(32))")
mustExecute(t, server, "COMMENT ON TABLE "+qualifiedSchema+"."+quoteIdentifier(parent)+" IS '订单父表'")
mustExecute(t, server, "COMMENT ON COLUMN "+qualifiedSchema+"."+quoteIdentifier(parent)+".id IS '主键编号'")
mustExecute(t, server, "COMMENT ON COLUMN "+qualifiedSchema+"."+quoteIdentifier(parent)+".name IS '客户''名称'")
@@ -76,18 +76,33 @@ func TestKingbaseIntegration(t *testing.T) {
t.Fatalf("get columns failed: columns=%v err=%v", columns, err)
}
parentColumns, err := server.getColumns(schema, parent)
if err != nil || len(parentColumns) != 2 || parentColumns[0].Comment == nil || *parentColumns[0].Comment != "主键编号" || parentColumns[1].Comment == nil || *parentColumns[1].Comment != "客户'名称" {
if err != nil || len(parentColumns) != 3 || parentColumns[0].Comment == nil || *parentColumns[0].Comment != "主键编号" || parentColumns[1].Comment == nil || *parentColumns[1].Comment != "客户'名称" {
t.Fatalf("get commented columns failed: columns=%v err=%v", parentColumns, err)
}
parentVarcharType := strings.ToLower(strings.TrimSpace(parentColumns[1].DataType))
if !strings.Contains(parentVarcharType, "varchar") && !strings.Contains(parentVarcharType, "character varying") {
t.Fatalf("varchar type metadata mismatch: column=%+v", parentColumns[1])
}
if parentColumns[1].CharacterMaximumLength == nil || *parentColumns[1].CharacterMaximumLength != 64 {
actualLength := -1
if parentColumns[1].CharacterMaximumLength != nil {
actualLength = *parentColumns[1].CharacterMaximumLength
}
t.Fatalf("varchar length metadata mismatch: length=%d column=%+v", actualLength, parentColumns[1])
}
parentCharacterType := strings.ToLower(strings.TrimSpace(parentColumns[2].DataType))
if ((!strings.HasPrefix(parentCharacterType, "char") && !strings.HasPrefix(parentCharacterType, "character") && parentCharacterType != "bpchar") || strings.Contains(parentCharacterType, "varying")) || parentColumns[2].CharacterMaximumLength == nil || *parentColumns[2].CharacterMaximumLength != 32 {
t.Fatalf("character length metadata mismatch: column=%+v", parentColumns[2])
}
ddl, err := server.getTableDDL(schema, parent)
if err != nil {
t.Fatalf("get table DDL failed: %v", err)
}
qualifiedParent := qualifiedSchema + "." + quoteIdentifier(parent)
qualifiedParent := server.quoteDDLIdentifier(schema) + "." + server.quoteDDLIdentifier(parent)
for _, expected := range []string{
"COMMENT ON TABLE " + qualifiedParent + " IS '订单父表';",
"COMMENT ON COLUMN " + qualifiedParent + "." + quoteIdentifier(parentColumns[0].Name) + " IS '主键编号';",
"COMMENT ON COLUMN " + qualifiedParent + "." + quoteIdentifier(parentColumns[1].Name) + " IS '客户''名称';",
"COMMENT ON COLUMN " + qualifiedParent + "." + server.quoteDDLIdentifier(parentColumns[0].Name) + " IS '主键编号';",
"COMMENT ON COLUMN " + qualifiedParent + "." + server.quoteDDLIdentifier(parentColumns[1].Name) + " IS '客户''名称';",
} {
if !strings.Contains(ddl, expected) {
t.Fatalf("table DDL missing %q:\n%s", expected, ddl)
@@ -149,6 +164,134 @@ func TestKingbaseIntegration(t *testing.T) {
}
}
func TestKingbaseConstraintsIntegration(t *testing.T) {
host := os.Getenv("KINGBASE_TEST_HOST")
portText := os.Getenv("KINGBASE_TEST_PORT")
username := os.Getenv("KINGBASE_TEST_USERNAME")
password := os.Getenv("KINGBASE_TEST_PASSWORD")
if host == "" || portText == "" || username == "" || password == "" {
t.Skip("Kingbase integration environment is not configured")
}
port, err := strconv.Atoi(portText)
if err != nil {
t.Fatal(err)
}
database := os.Getenv("KINGBASE_TEST_DATABASE")
if database == "" {
database = "test"
}
suffix := strconv.FormatInt(time.Now().UnixNano(), 36)
schema := "dbx_constraints_" + suffix
schemaIdent := quoteIdentifier(schema)
parent := "parent"
child := "child"
parentTable := schemaIdent + "." + quoteIdentifier(parent)
childTable := schemaIdent + "." + quoteIdentifier(child)
server := newServer()
cp := connectParams{
Host: host, Port: port, Database: database, Username: username, Password: password,
ConnectionString: fmt.Sprintf("jdbc:kingbase8://%s:%d/%s", host, port, database),
}
if err := server.connect(cp); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = server.disconnect() })
t.Cleanup(func() {
_, _ = server.executeQuery(queryOptions{SQL: "DROP SCHEMA IF EXISTS " + schemaIdent + " CASCADE"})
})
mustExecute(t, server, "CREATE SCHEMA "+schemaIdent)
mustExecute(t, server, "CREATE TABLE "+parentTable+" (a bigint NOT NULL, b bigint NOT NULL, CONSTRAINT parent_pk PRIMARY KEY (a, b))")
mustExecute(t, server, "CREATE TABLE "+childTable+" (id bigint PRIMARY KEY, parent_a bigint, parent_b bigint, code text NOT NULL, amount int, CONSTRAINT child_parent_fk FOREIGN KEY (parent_a, parent_b) REFERENCES "+parentTable+" (a, b) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, CONSTRAINT child_code_unique UNIQUE (code), CONSTRAINT child_code_check CHECK (length(code) > 0))")
constraints, err := server.listConstraints(schema, child)
if err != nil {
t.Fatalf("list constraints failed in %s mode: %v", server.mode.compatibilityMode, err)
}
byName := map[string]constraintInfo{}
for _, constraint := range constraints {
byName[constraint.Name] = constraint
}
pk := byName["child_pkey"]
if pk.ConstraintType != "PRIMARY KEY" || len(pk.Columns) != 1 || pk.Columns[0] != "id" {
t.Fatalf("primary key metadata mismatch in %s mode: %+v", server.mode.compatibilityMode, pk)
}
fk := byName["child_parent_fk"]
if fk.ConstraintType != "FOREIGN KEY" || !equalStringSlices(fk.Columns, []string{"parent_a", "parent_b"}) || !equalStringSlices(fk.RefColumns, []string{"a", "b"}) || fk.RefSchema == nil || *fk.RefSchema != schema || fk.RefTable == nil || *fk.RefTable != parent || fk.OnDelete == nil || *fk.OnDelete != "CASCADE" || !fk.Deferrable || !fk.InitiallyDeferred {
t.Fatalf("foreign key metadata mismatch in %s mode: %+v", server.mode.compatibilityMode, fk)
}
unique := byName["child_code_unique"]
if unique.ConstraintType != "UNIQUE" || !equalStringSlices(unique.Columns, []string{"code"}) {
t.Fatalf("unique metadata mismatch in %s mode: %+v", server.mode.compatibilityMode, unique)
}
check := byName["child_code_check"]
if check.ConstraintType != "CHECK" || !equalStringSlices(check.Columns, []string{"code"}) || !strings.Contains(check.Definition, "length(code) > 0") || !check.Enabled {
t.Fatalf("check metadata mismatch in %s mode: %+v", server.mode.compatibilityMode, check)
}
foreignKeys, err := server.listForeignKeys(schema, child)
if err != nil {
t.Fatalf("list foreign keys failed in %s mode: %v", server.mode.compatibilityMode, err)
}
foreignKeyNames := map[string]struct{}{}
for _, foreignKey := range foreignKeys {
foreignKeyNames[foreignKey.Name] = struct{}{}
}
constraintForeignKeyNames := map[string]struct{}{}
for _, constraint := range constraints {
if constraint.ConstraintType == "FOREIGN KEY" {
constraintForeignKeyNames[constraint.Name] = struct{}{}
}
}
if len(foreignKeyNames) != len(constraintForeignKeyNames) {
t.Fatalf("foreign-key metadata sources disagree in %s mode: constraints=%v foreign_keys=%v", server.mode.compatibilityMode, constraintForeignKeyNames, foreignKeyNames)
}
for name := range foreignKeyNames {
if _, ok := constraintForeignKeyNames[name]; !ok {
t.Fatalf("foreign-key %q missing from list_constraints in %s mode", name, server.mode.compatibilityMode)
}
}
if server.mode.compatibilityMode == "oracle" {
mustExecute(t, server, "ALTER TABLE "+childTable+" DISABLE CONSTRAINT child_code_check")
disabled, err := server.listConstraints(schema, child)
if err != nil {
t.Fatalf("list disabled constraints failed in %s mode: %v", server.mode.compatibilityMode, err)
}
var disabledCheck constraintInfo
found := false
for _, constraint := range disabled {
if constraint.Name == "child_code_check" {
disabledCheck = constraint
found = true
break
}
}
if !found || disabledCheck.Enabled {
t.Fatalf("disabled constraint state mismatch in %s mode: %+v", server.mode.compatibilityMode, disabledCheck)
}
mustExecute(t, server, "ALTER TABLE "+childTable+" ENABLE CONSTRAINT child_code_check")
}
parentConstraints, err := server.listConstraints(schema, parent)
if err != nil {
t.Fatalf("list parent constraints failed in %s mode: %v", server.mode.compatibilityMode, err)
}
if len(parentConstraints) != 1 || parentConstraints[0].Name != "parent_pk" || !equalStringSlices(parentConstraints[0].Columns, []string{"a", "b"}) {
t.Fatalf("composite primary key order mismatch in %s mode: %+v", server.mode.compatibilityMode, parentConstraints)
}
}
func equalStringSlices(left, right []string) bool {
if len(left) != len(right) {
return false
}
for index := range left {
if left[index] != right[index] {
return false
}
}
return true
}
func TestKingbaseCustomTypesIntegration(t *testing.T) {
host := os.Getenv("KINGBASE_TEST_HOST")
portText := os.Getenv("KINGBASE_TEST_PORT")
+357 -31
View File
@@ -88,6 +88,7 @@ type metadataListConstraints struct {
type columnInfo struct {
Name string `json:"name"`
DataType string `json:"data_type"`
ResolvedSchema *string `json:"resolved_schema,omitempty"`
FullDataType string `json:"-"`
IsNullable bool `json:"is_nullable"`
ColumnDefault *string `json:"column_default"`
@@ -129,6 +130,23 @@ type foreignKeyInfo struct {
RefColumn string `json:"ref_column"`
}
type constraintInfo struct {
Name string `json:"name"`
ConstraintType string `json:"constraint_type"`
Definition string `json:"definition"`
Columns []string `json:"columns"`
RefSchema *string `json:"ref_schema,omitempty"`
RefTable *string `json:"ref_table,omitempty"`
RefColumns []string `json:"ref_columns"`
MatchType *string `json:"match_type,omitempty"`
OnUpdate *string `json:"on_update,omitempty"`
OnDelete *string `json:"on_delete,omitempty"`
Deferrable bool `json:"deferrable"`
InitiallyDeferred bool `json:"initially_deferred"`
Enabled bool `json:"enabled"`
Valid bool `json:"valid"`
}
type triggerInfo struct {
Name string `json:"name"`
Event string `json:"event"`
@@ -204,10 +222,7 @@ func (s *server) identifierQuote() string {
// schema/table names containing hyphens or other special characters render as
// valid SQL instead of being parsed as operators or bare tokens.
func (s *server) quoteDDLIdentifier(value string) string {
if s.mode.mysqlCompat {
return "`" + strings.ReplaceAll(value, "`", "``") + "`"
}
return quoteIdentifier(value)
return s.quoteIdentifier(value)
}
func (s *server) connectionInfo() (map[string]any, error) {
@@ -1279,43 +1294,66 @@ func completionNameMatches(name string, request completionAssistantRequest) bool
}
func (s *server) getColumns(schema, table string) ([]columnInfo, error) {
effective, err := s.effectiveSchema(schema)
if err != nil {
return nil, err
}
primary, _ := s.primaryKeys(effective, table)
if s.mode.mysqlCompat {
effective, err := s.effectiveSchema(schema)
if err != nil {
return nil, err
}
primary, _ := s.primaryKeys(effective, table)
return s.informationSchemaColumns(effective, table, primary)
}
catalog, prefix := "sys_catalog", "sys"
if s.mode.postgresCatalog {
catalog, prefix = "pg_catalog", "pg"
return s.queryCatalogColumns(effective, table, primary, catalog, prefix, "pg_get_expr")
result, err := s.queryCatalogColumns(schema, table, catalog, prefix, "pg_get_expr")
return s.finishCatalogColumns(schema, table, result, err)
}
expression := "sys_get_expr"
if s.usePgDefaultExpression {
expression = "pg_get_expr"
}
result, err := s.queryCatalogColumns(effective, table, primary, catalog, prefix, expression)
result, err := s.queryCatalogColumns(schema, table, catalog, prefix, expression)
if err != nil && expression == "sys_get_expr" && isUndefinedFunction(err, expression) {
// Some V8R6 PostgreSQL-mode databases keep sys_catalog while adbin is
// pg_node_tree. Cache the compatible function after the exact failure.
s.usePgDefaultExpression = true
return s.queryCatalogColumns(effective, table, primary, catalog, prefix, "pg_get_expr")
result, err = s.queryCatalogColumns(schema, table, catalog, prefix, "pg_get_expr")
}
return result, err
return s.finishCatalogColumns(schema, table, result, err)
}
func (s *server) finishCatalogColumns(schema, table string, result []columnInfo, err error) ([]columnInfo, error) {
if err != nil || len(result) == 0 {
return result, err
}
resolvedSchema := strings.TrimSpace(schema)
if result[0].ResolvedSchema != nil {
resolvedSchema = *result[0].ResolvedSchema
}
primary, _ := s.primaryKeys(resolvedSchema, table)
for index := range result {
result[index].IsPrimaryKey = primary[strings.ToLower(result[index].Name)]
}
if s.mode.sqlServerIdentity {
s.applyIdentityMetadata(resolvedSchema, table, result)
}
return result, nil
}
func (s *server) queryCatalogColumns(
schema, table string,
primary map[string]bool,
catalog, prefix, expression string,
) ([]columnInfo, error) {
identityExpression := "a.attidentity"
if s.catalogIdentityUnsupported {
identityExpression = "CAST(NULL AS varchar(1)) AS attidentity"
}
query := fmt.Sprintf(`SELECT a.attname, format_type(a.atttypid, a.atttypmod), NOT a.attnotnull,
relationPredicate := fmt.Sprintf("n.nspname = %s AND c.relname = %s", quoteLiteral(schema), quoteLiteral(table))
if strings.TrimSpace(schema) == "" {
visibilityFunction := kingbaseCatalogFunction(catalog, "sys_table_is_visible", "pg_table_is_visible")
relationPredicate = fmt.Sprintf("c.relname = %s AND %s(c.oid)", quoteLiteral(table), visibilityFunction)
}
query := fmt.Sprintf(`SELECT n.nspname, a.attname, format_type(a.atttypid, a.atttypmod), NOT a.attnotnull,
%s(ad.adbin, ad.adrelid), col_description(a.attrelid, a.attnum),
CASE WHEN t.typname = 'numeric' AND a.atttypmod > 0 THEN ((a.atttypmod - 4) >> 16) & 65535 END,
CASE WHEN t.typname = 'numeric' AND a.atttypmod > 0 THEN (a.atttypmod - 4) & 65535 END,
@@ -1324,11 +1362,11 @@ func (s *server) queryCatalogColumns(
FROM %s.%s_attribute a JOIN %s.%s_type t ON t.oid = a.atttypid
JOIN %s.%s_class c ON c.oid = a.attrelid JOIN %s.%s_namespace n ON n.oid = c.relnamespace
LEFT JOIN %s.%s_attrdef ad ON ad.adrelid = a.attrelid AND ad.adnum = a.attnum
WHERE n.nspname = %s AND c.relname = %s AND a.attnum > 0 AND NOT a.attisdropped ORDER BY a.attnum`, expression, identityExpression, catalog, prefix, catalog, prefix, catalog, prefix, catalog, prefix, catalog, prefix, quoteLiteral(schema), quoteLiteral(table))
WHERE %s AND a.attnum > 0 AND NOT a.attisdropped ORDER BY a.attnum`, expression, identityExpression, catalog, prefix, catalog, prefix, catalog, prefix, catalog, prefix, catalog, prefix, relationPredicate)
rows, err := s.metadataQuery(query)
if err != nil && !s.catalogIdentityUnsupported && isUndefinedColumn(err, "attidentity") {
s.catalogIdentityUnsupported = true
return s.queryCatalogColumns(schema, table, primary, catalog, prefix, expression)
return s.queryCatalogColumns(schema, table, catalog, prefix, expression)
}
if err != nil {
return nil, err
@@ -1336,21 +1374,18 @@ WHERE n.nspname = %s AND c.relname = %s AND a.attnum > 0 AND NOT a.attisdropped
defer rows.Close()
result := []columnInfo{}
for rows.Next() {
var name, dataType string
var resolvedSchema, name, dataType string
var nullable bool
var defaultValue, comment, identity sql.NullString
var precision, scale, length sql.NullInt64
if err := rows.Scan(&name, &dataType, &nullable, &defaultValue, &comment, &precision, &scale, &length, &identity); err != nil {
if err := rows.Scan(&resolvedSchema, &name, &dataType, &nullable, &defaultValue, &comment, &precision, &scale, &length, &identity); err != nil {
return nil, err
}
result = append(result, columnInfo{Name: name, DataType: dataType, IsNullable: nullable, ColumnDefault: nullStringPtr(defaultValue), IsPrimaryKey: primary[strings.ToLower(name)], Extra: kingbaseIdentityClause(identity.String), Comment: nullStringPtr(comment), NumericPrecision: nullIntPtr(precision), NumericScale: nullIntPtr(scale), CharacterMaximumLength: nullIntPtr(length)})
result = append(result, columnInfo{Name: name, DataType: dataType, ResolvedSchema: stringPtr(resolvedSchema), IsNullable: nullable, ColumnDefault: nullStringPtr(defaultValue), Extra: kingbaseIdentityClause(identity.String), Comment: nullStringPtr(comment), NumericPrecision: nullIntPtr(precision), NumericScale: nullIntPtr(scale), CharacterMaximumLength: nullIntPtr(length)})
}
if err := rows.Err(); err != nil {
return nil, err
}
if s.mode.sqlServerIdentity {
s.applyIdentityMetadata(schema, table, result)
}
return result, nil
}
@@ -1432,10 +1467,14 @@ func (s *server) queryInformationSchemaColumns(schema, table string, primary map
if err := rows.Scan(&name, &dataType, &fullDataType, &nullable, &defaultValue, &comment, &precision, &scale, &length); err != nil {
return nil, err
}
if parsed := boundedVarcharLength(dataType); parsed != nil && !length.Valid {
length = sql.NullInt64{Int64: int64(*parsed), Valid: true}
if !length.Valid || length.Int64 < 0 {
if parsed := boundedVarcharLength(dataType); parsed != nil {
length = sql.NullInt64{Int64: int64(*parsed), Valid: true}
} else if parsed := boundedVarcharLength(fullDataType.String); parsed != nil {
length = sql.NullInt64{Int64: int64(*parsed), Valid: true}
}
}
result = append(result, columnInfo{Name: name, DataType: resolvedInformationSchemaDataType(dataType, fullDataType.String), FullDataType: fullDataType.String, IsNullable: strings.EqualFold(nullable, "YES"), ColumnDefault: nullStringPtr(defaultValue), IsPrimaryKey: primary[strings.ToLower(name)], Comment: nullStringPtr(comment), NumericPrecision: nullIntPtr(precision), NumericScale: nullIntPtr(scale), CharacterMaximumLength: nullIntPtr(length)})
result = append(result, columnInfo{Name: name, DataType: resolvedInformationSchemaDataType(dataType, fullDataType.String), ResolvedSchema: stringPtr(schema), FullDataType: fullDataType.String, IsNullable: strings.EqualFold(nullable, "YES"), ColumnDefault: nullStringPtr(defaultValue), IsPrimaryKey: primary[strings.ToLower(name)], Comment: nullStringPtr(comment), NumericPrecision: nullIntPtr(precision), NumericScale: nullIntPtr(scale), CharacterMaximumLength: nullIntPtr(length)})
}
return result, rows.Err()
}
@@ -1579,7 +1618,7 @@ func parseCatalogAttributeNumbers(raw any) ([]int, error) {
default:
value = fmt.Sprint(typed)
}
value = strings.TrimSpace(strings.Trim(value, "{}"))
value = strings.TrimSpace(strings.Trim(value, "{}[]"))
if value == "" {
return []int{}, nil
}
@@ -1740,6 +1779,248 @@ WHERE c.contype = 'f' AND n.nspname = %s AND t.relname = %s ORDER BY c.conname`,
return result, nil
}
func kingbaseConstraintFunctionName(catalog string) string {
if catalog == "pg_catalog" {
return "pg_get_constraintdef"
}
return "sys_get_constraintdef"
}
func kingbaseConstraintsQuery(catalog, prefix, schema, table string, definitionUnsupported, validatedUnsupported, statusUnsupported bool) string {
definitionExpression := fmt.Sprintf("COALESCE(%s(c.oid, true), '')", kingbaseCatalogFunction(catalog, "sys_get_constraintdef", "pg_get_constraintdef"))
validExpression := "COALESCE(CAST(c.convalidated AS text), 'T')"
statusExpression := "COALESCE(CAST(c.constatus AS text), 'E')"
if definitionUnsupported {
definitionExpression = "''"
}
if validatedUnsupported {
validExpression = "true"
}
if statusUnsupported {
statusExpression = "'E'"
}
return fmt.Sprintf(`SELECT COALESCE(c.conname, ''), c.contype::text, %s, c.conkey,
rn.nspname, rt.relname, c.confkey, c.confmatchtype::text, c.confupdtype::text, c.confdeltype::text,
c.condeferrable, c.condeferred, %s, %s
FROM %s.%s_constraint c
JOIN %s.%s_class t ON t.oid = c.conrelid
JOIN %s.%s_namespace n ON n.oid = t.relnamespace
LEFT JOIN %s.%s_class rt ON rt.oid = c.confrelid
LEFT JOIN %s.%s_namespace rn ON rn.oid = rt.relnamespace
WHERE n.nspname = %s AND t.relname = %s AND t.relkind IN ('r', 'p', 'f')
ORDER BY COALESCE(c.conname, '')`, definitionExpression, validExpression, statusExpression, catalog, prefix, catalog, prefix, catalog, prefix, catalog, prefix, catalog, prefix, quoteLiteral(schema), quoteLiteral(table))
}
func (s *server) listConstraints(schema, table string) ([]constraintInfo, error) {
effective, err := s.effectiveSchema(schema)
if err != nil {
return nil, err
}
catalog, prefix := "sys_catalog", "sys"
if s.mode.postgresCatalog {
catalog, prefix = "pg_catalog", "pg"
}
definitionUnsupported := s.constraintDefinitionUnsupported
validatedUnsupported := s.mode.legacyV7 || s.constraintValidatedUnsupported
statusUnsupported := s.mode.legacyV7 || s.constraintStatusUnsupported
var rows *sql.Rows
for {
query := kingbaseConstraintsQuery(catalog, prefix, effective, table, definitionUnsupported, validatedUnsupported, statusUnsupported)
rows, err = s.metadataQuery(query)
if err == nil {
break
}
changed := false
if !definitionUnsupported && isUndefinedFunction(err, kingbaseConstraintFunctionName(catalog)) {
definitionUnsupported = true
s.constraintDefinitionUnsupported = true
changed = true
}
if !validatedUnsupported && isUndefinedColumn(err, "convalidated") {
validatedUnsupported = true
s.constraintValidatedUnsupported = true
changed = true
}
if !statusUnsupported && isUndefinedColumn(err, "constatus") {
statusUnsupported = true
s.constraintStatusUnsupported = true
changed = true
}
if !changed {
return nil, err
}
}
defer rows.Close()
type rawConstraint struct {
name, kind, definition string
columns, refColumns []int
refSchema, refTable sql.NullString
matchType, onUpdate, onDelete sql.NullString
deferrable, initiallyDeferred bool
valid, enabled bool
}
raw := []rawConstraint{}
for rows.Next() {
var item rawConstraint
var columnsRaw, refColumnsRaw, validRaw, statusRaw any
if err := rows.Scan(&item.name, &item.kind, &item.definition, &columnsRaw, &item.refSchema, &item.refTable, &refColumnsRaw, &item.matchType, &item.onUpdate, &item.onDelete, &item.deferrable, &item.initiallyDeferred, &validRaw, &statusRaw); err != nil {
return nil, err
}
item.valid = parseConstraintEnabled(validRaw)
item.enabled = parseConstraintEnabled(statusRaw)
item.columns, err = parseCatalogAttributeNumbers(columnsRaw)
if err != nil {
return nil, fmt.Errorf("failed to parse constraint %s columns: %w", item.name, err)
}
item.refColumns, err = parseCatalogAttributeNumbers(refColumnsRaw)
if err != nil {
return nil, fmt.Errorf("failed to parse constraint %s referenced columns: %w", item.name, err)
}
raw = append(raw, item)
}
if err := rows.Err(); err != nil {
return nil, err
}
attributes, err := s.relationAttributesByNumber(catalog, prefix, effective, table)
if err != nil {
return nil, err
}
refAttributes := map[string]map[int]string{}
result := make([]constraintInfo, 0, len(raw))
for _, item := range raw {
constraint := constraintInfo{
Name: item.name, ConstraintType: kingbaseConstraintTypeName(item.kind), Definition: item.definition,
Columns: []string{}, RefColumns: []string{}, Deferrable: item.deferrable,
// FK details are retained for API completeness and future unified
// constraint/FK presentations. The current UI renders FK rows through
// list_foreign_keys, so these fields are not currently displayed in the
// Constraints tab.
InitiallyDeferred: item.initiallyDeferred, Enabled: item.enabled, Valid: item.valid,
}
for _, number := range item.columns {
if name := attributes[number]; name != "" {
constraint.Columns = append(constraint.Columns, name)
}
}
if item.refSchema.Valid {
constraint.RefSchema = stringPtr(item.refSchema.String)
}
if item.refTable.Valid {
constraint.RefTable = stringPtr(item.refTable.String)
}
if strings.EqualFold(strings.TrimSpace(item.kind), "f") && item.refSchema.Valid && item.refTable.Valid {
key := item.refSchema.String + "\x00" + item.refTable.String
ref := refAttributes[key]
if ref == nil {
ref, err = s.relationAttributesByNumber(catalog, prefix, item.refSchema.String, item.refTable.String)
if err != nil {
return nil, err
}
refAttributes[key] = ref
}
for _, number := range item.refColumns {
if name := ref[number]; name != "" {
constraint.RefColumns = append(constraint.RefColumns, name)
}
}
constraint.MatchType = kingbaseConstraintMatchType(item.matchType)
constraint.OnUpdate = kingbaseConstraintAction(item.onUpdate)
constraint.OnDelete = kingbaseConstraintAction(item.onDelete)
}
result = append(result, constraint)
}
return result, nil
}
func parseConstraintEnabled(raw any) bool {
if raw == nil {
return true
}
if enabled, ok := raw.(bool); ok {
return enabled
}
if bytes, ok := raw.([]byte); ok {
raw = string(bytes)
}
value := strings.ToLower(strings.TrimSpace(fmt.Sprint(raw)))
if value == "" {
return true
}
switch value {
case "1", "t", "true", "y", "yes", "e", "enabled", "enable", "on":
return true
case "0", "f", "false", "d", "disabled", "disable", "off":
return false
default:
// Unknown catalog states should not make the whole metadata request fail.
return true
}
}
func kingbaseConstraintTypeName(value string) string {
switch strings.ToLower(strings.TrimSpace(value)) {
case "p":
return "PRIMARY KEY"
case "f":
return "FOREIGN KEY"
case "u":
return "UNIQUE"
case "c":
return "CHECK"
case "t":
return "CONSTRAINT TRIGGER"
case "x":
return "EXCLUDE"
case "n":
return "NOT NULL"
default:
return strings.TrimSpace(value)
}
}
func kingbaseConstraintMatchType(value sql.NullString) *string {
if !value.Valid {
return nil
}
var result string
switch strings.ToLower(strings.TrimSpace(value.String)) {
case "f":
result = "FULL"
case "p":
result = "PARTIAL"
case "s":
result = "SIMPLE"
default:
return nil
}
return &result
}
func kingbaseConstraintAction(value sql.NullString) *string {
if !value.Valid {
return nil
}
var result string
switch strings.ToLower(strings.TrimSpace(value.String)) {
case "a":
result = "NO ACTION"
case "r":
result = "RESTRICT"
case "c":
result = "CASCADE"
case "n":
result = "SET NULL"
case "d":
result = "SET DEFAULT"
default:
return nil
}
return &result
}
func (s *server) listTriggers(schema, table string) ([]triggerInfo, error) {
effective, err := s.effectiveSchema(schema)
if err != nil {
@@ -1791,7 +2072,7 @@ func (s *server) getObjectSourceForRelation(schema, name, objectType, relationNa
}
source := ""
kind := strings.ToUpper(objectType)
if kind == "VIEW" || kind == "MATERIALIZED_VIEW" {
if kind == "VIEW" {
if s.mode.mysqlCompat {
err = s.requireDBQueryRow("SELECT view_definition FROM information_schema.views WHERE table_schema = "+quoteLiteral(effective)+" AND table_name = "+quoteLiteral(name), &source)
} else {
@@ -1811,6 +2092,8 @@ func (s *server) getObjectSourceForRelation(schema, name, objectType, relationNa
err = querySource("pg_get_viewdef")
}
}
} else if kind == "MATERIALIZED_VIEW" {
source, err = s.getMaterializedViewSource(effective, name)
} else if kind == "FUNCTION" || kind == "PROCEDURE" {
catalog, prefix, function := "sys_catalog", "sys", "sys_get_functiondef"
if s.mode.postgresCatalog {
@@ -1866,6 +2149,46 @@ func (s *server) getObjectSourceForRelation(schema, name, objectType, relationNa
return result, nil
}
func (s *server) getMaterializedViewSource(schema, name string) (string, error) {
catalog, prefix, function := "sys_catalog", "sys", "sys_get_viewdef"
if s.mode.postgresCatalog {
catalog, prefix, function = "pg_catalog", "pg", "pg_get_viewdef"
} else if s.usePgViewDefinition {
function = "pg_get_viewdef"
}
querySource := func(definitionFunction string) (string, error) {
query := fmt.Sprintf("SELECT %s(c.oid) FROM %s.%s_class c JOIN %s.%s_namespace n ON n.oid=c.relnamespace WHERE n.nspname=%s AND c.relname=%s AND c.relkind = 'm' LIMIT 1", definitionFunction, catalog, prefix, catalog, prefix, quoteLiteral(schema), quoteLiteral(name))
var source sql.NullString
if err := s.requireDBQueryRow(query, &source); err != nil {
return "", err
}
if !source.Valid || strings.TrimSpace(source.String) == "" {
return "", sql.ErrNoRows
}
return source.String, nil
}
source, err := querySource(function)
if err == nil {
return source, nil
}
if function == "sys_get_viewdef" {
if isUndefinedFunction(err, function) {
s.usePgViewDefinition = true
} else if !errors.Is(err, sql.ErrNoRows) {
return "", err
}
source, err = querySource("pg_get_viewdef")
if err == nil {
return source, nil
}
}
if errors.Is(err, sql.ErrNoRows) {
return "", fmt.Errorf("materialized view %q.%q returned an empty source definition", schema, name)
}
return "", err
}
func (s *server) getTableDDL(schema, table string) (string, error) {
effective, err := s.effectiveSchema(schema)
if err != nil {
@@ -2252,12 +2575,15 @@ func decodeTriggerTiming(triggerType int) string {
func boundedVarcharLength(dataType string) *int {
lower := strings.ToLower(strings.TrimSpace(dataType))
for _, prefix := range []string{"varchar", "character varying"} {
for _, prefix := range []string{"character varying", "varchar", "bpchar", "character", "char"} {
if strings.HasPrefix(lower, prefix) {
value := strings.TrimSpace(strings.TrimSuffix(strings.TrimPrefix(lower, prefix), ")"))
value = strings.TrimPrefix(value, "(")
if number, err := strconv.Atoi(strings.TrimSpace(value)); err == nil && number >= 0 {
return &number
fields := strings.Fields(value)
if len(fields) > 0 {
if number, err := strconv.Atoi(fields[0]); err == nil && number >= 0 {
return &number
}
}
}
}
+55 -23
View File
@@ -129,27 +129,30 @@ type querySession struct {
}
type server struct {
db *sql.DB
openDatabase kingbaseDBOpener
params connectParams
mode kingbaseMode
usePgDefaultExpression bool
usePgViewDefinition bool
usePgFunctionDefinition bool
useLegacyRoutineDefinition bool
catalogIdentityUnsupported bool
catalogOIDUnsupported bool
infoColumnTypeUnsupported bool
infoUdtNameUnsupported bool
indexOrdinalityUnsupported bool
triggerPrettyUnsupported bool
triggerInternalUnsupported bool
currentSchema string
schemaSet bool
sessions map[string]*querySession
nextSessionID uint64
activeCancelMu sync.Mutex
activeCancel context.CancelFunc
db *sql.DB
openDatabase kingbaseDBOpener
params connectParams
mode kingbaseMode
usePgDefaultExpression bool
usePgViewDefinition bool
usePgFunctionDefinition bool
useLegacyRoutineDefinition bool
catalogIdentityUnsupported bool
catalogOIDUnsupported bool
infoColumnTypeUnsupported bool
infoUdtNameUnsupported bool
indexOrdinalityUnsupported bool
triggerPrettyUnsupported bool
triggerInternalUnsupported bool
constraintDefinitionUnsupported bool
constraintValidatedUnsupported bool
constraintStatusUnsupported bool
currentSchema string
schemaSet bool
sessions map[string]*querySession
nextSessionID uint64
activeCancelMu sync.Mutex
activeCancel context.CancelFunc
}
type agentSession struct {
@@ -407,6 +410,9 @@ func (s *server) dispatch(method string, params map[string]json.RawMessage) (any
case "list_foreign_keys":
result, err := s.listForeignKeys(stringParam(params, "schema"), stringParam(params, "table"))
return result, false, err
case "list_constraints":
result, err := s.listConstraints(stringParam(params, "schema"), stringParam(params, "table"))
return result, false, err
case "list_triggers":
result, err := s.listTriggers(stringParam(params, "schema"), stringParam(params, "table"))
return result, false, err
@@ -477,6 +483,9 @@ func (s *server) connect(cp connectParams) error {
s.indexOrdinalityUnsupported = false
s.triggerPrettyUnsupported = false
s.triggerInternalUnsupported = false
s.constraintDefinitionUnsupported = false
s.constraintValidatedUnsupported = false
s.constraintStatusUnsupported = false
return nil
}
@@ -511,7 +520,7 @@ func openAndPingDB(cp connectParams, timeout time.Duration, opener kingbaseDBOpe
if db != nil {
_ = db.Close()
}
if index == 0 && len(attempts) == 2 && errors.Is(err, gokb.ErrSSLNotSupported) {
if index == 0 && len(attempts) == 2 && shouldRetryKingbaseWithoutSSL(err) {
continue
}
return nil, err
@@ -519,6 +528,19 @@ func openAndPingDB(cp connectParams, timeout time.Duration, opener kingbaseDBOpe
return nil, errors.New("kingbase connection failed")
}
func shouldRetryKingbaseWithoutSSL(err error) bool {
if errors.Is(err, gokb.ErrSSLNotSupported) {
return true
}
if err == nil {
return false
}
// KingBase V7 can accept the SSLRequest and then reject the TLS handshake,
// so gokb returns the TLS alert instead of ErrSSLNotSupported.
return strings.Contains(strings.ToLower(err.Error()), "remote error: tls: handshake failure")
}
func openDBWithSSLMode(cp connectParams, sslMode string) (*sql.DB, error) {
dsn := buildDSNWithSSLMode(cp, sslMode)
db, err := sql.Open("kingbase", dsn)
@@ -548,6 +570,9 @@ func (s *server) disconnect() error {
s.indexOrdinalityUnsupported = false
s.triggerPrettyUnsupported = false
s.triggerInternalUnsupported = false
s.constraintDefinitionUnsupported = false
s.constraintValidatedUnsupported = false
s.constraintStatusUnsupported = false
s.currentSchema = ""
s.schemaSet = false
if s.db == nil {
@@ -909,7 +934,7 @@ func (s *server) setSchema(ctx context.Context, conn *sql.Conn, schema string) e
if schema != "" {
// Kingbase implicitly prioritizes its system catalog when it is not
// listed explicitly, matching the JDBC agent and DBeaver behavior.
statement = "SET search_path TO " + quoteIdentifier(schema)
statement = "SET search_path TO " + s.quoteIdentifier(schema)
}
if _, err := conn.ExecContext(ctx, statement); err != nil {
return err
@@ -1868,6 +1893,13 @@ func quoteIdentifier(value string) string {
return `"` + strings.ReplaceAll(value, `"`, `""`) + `"`
}
func (s *server) quoteIdentifier(value string) string {
if s.mode.mysqlCompat {
return "`" + strings.ReplaceAll(value, "`", "``") + "`"
}
return quoteIdentifier(value)
}
func quoteLiteral(value string) string {
return "'" + strings.ReplaceAll(value, "'", "''") + "'"
}
+692 -2
View File
@@ -228,8 +228,8 @@ func (connection *fallbackConn) QueryContext(_ context.Context, query string, _
identity = nil
}
return &valueRows{
columns: []string{"column_name", "data_type", "is_nullable", "column_default", "column_comment", "numeric_precision", "numeric_scale", "character_maximum_length", "attidentity"},
rows: [][]driver.Value{{"id", "integer", false, nil, nil, int64(32), int64(0), nil, identity}},
columns: []string{"resolved_schema", "column_name", "data_type", "is_nullable", "column_default", "column_comment", "numeric_precision", "numeric_scale", "character_maximum_length", "attidentity"},
rows: [][]driver.Value{{"public", "id", "integer", false, nil, nil, int64(32), int64(0), nil, identity}},
}, nil
}
return nil, errors.New("unexpected query: " + query)
@@ -851,6 +851,21 @@ func TestOpenAndPingDBPreferFallbackUsesOneTimeoutBudget(t *testing.T) {
}
}
func TestOpenAndPingDBPreferFallbackHandlesKingbaseV7TLSFailure(t *testing.T) {
state := &connectionAttemptState{pingErrors: map[string]error{
"require": errors.New("remote error: tls: handshake failure"),
}}
db, err := openAndPingDB(connectParams{}, time.Second, state.open)
if err != nil {
t.Fatal(err)
}
defer db.Close()
attempts, _ := state.snapshot()
if strings.Join(attempts, ",") != "require,disable" {
t.Fatalf("unexpected KingBase V7 attempts: %v", attempts)
}
}
func TestOpenAndPingDBDoesNotDowngradeUnrelatedErrors(t *testing.T) {
authErr := errors.New("authentication failed")
state := &connectionAttemptState{pingErrors: map[string]error{"require": authErr}}
@@ -867,6 +882,33 @@ func TestOpenAndPingDBDoesNotDowngradeUnrelatedErrors(t *testing.T) {
}
}
func TestOpenAndPingDBDoesNotDowngradeNetworkErrors(t *testing.T) {
tests := []struct {
name string
err error
}{
{name: "timeout", err: context.DeadlineExceeded},
{name: "connection refused", err: errors.New("dial tcp 127.0.0.1:54321: connect: connection refused")},
{name: "generic handshake", err: errors.New("handshake failure")},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
state := &connectionAttemptState{pingErrors: map[string]error{"require": test.err}}
db, err := openAndPingDB(connectParams{}, time.Second, state.open)
if db != nil {
db.Close()
}
if !errors.Is(err, test.err) {
t.Fatalf("unexpected error: %v", err)
}
attempts, _ := state.snapshot()
if strings.Join(attempts, ",") != "require" {
t.Fatalf("network errors must not downgrade: %v", attempts)
}
})
}
}
func TestOpenAndPingDBExplicitModesNeverDowngrade(t *testing.T) {
for _, sslMode := range []string{"disable", "require", "verify-ca", "verify-full"} {
t.Run(sslMode, func(t *testing.T) {
@@ -886,6 +928,26 @@ func TestOpenAndPingDBExplicitModesNeverDowngrade(t *testing.T) {
}
}
func TestOpenAndPingDBExplicitModesNeverDowngradeKingbaseV7TLSFailure(t *testing.T) {
tlsErr := errors.New("remote error: tls: handshake failure")
for _, sslMode := range []string{"disable", "require", "verify-ca", "verify-full"} {
t.Run(sslMode, func(t *testing.T) {
state := &connectionAttemptState{pingErrors: map[string]error{sslMode: tlsErr}}
db, err := openAndPingDB(connectParams{URLParams: "sslmode=" + sslMode}, time.Second, state.open)
if db != nil {
db.Close()
}
if !errors.Is(err, tlsErr) {
t.Fatalf("unexpected error: %v", err)
}
attempts, _ := state.snapshot()
if len(attempts) != 1 || attempts[0] != sslMode {
t.Fatalf("explicit mode must use one attempt: %v", attempts)
}
})
}
}
func TestOpenAndPingDBSSLDefaultsToVerifyFull(t *testing.T) {
state := &connectionAttemptState{pingErrors: map[string]error{}}
db, err := openAndPingDB(connectParams{SSL: true}, time.Second, state.open)
@@ -944,6 +1006,8 @@ func TestParseCatalogAttributeNumbers(t *testing.T) {
{name: "int2vector string", raw: "1 2 4", expected: "1,2,4"},
{name: "array string", raw: "{3,5}", expected: "3,5"},
{name: "bytes", raw: []byte("6 7"), expected: "6,7"},
{name: "bracketed array string", raw: "[8 9]", expected: "8,9"},
{name: "int16 slice", raw: []int16{10, 11}, expected: "10,11"},
{name: "empty", raw: nil, expected: ""},
{name: "invalid", raw: "1 bad", wantErr: true},
}
@@ -970,6 +1034,32 @@ func TestParseCatalogAttributeNumbers(t *testing.T) {
}
}
func TestParseConstraintEnabled(t *testing.T) {
for _, test := range []struct {
name string
raw any
want bool
}{
{name: "boolean true", raw: true, want: true},
{name: "boolean false", raw: false, want: false},
{name: "enabled string", raw: "E", want: true},
{name: "disabled string", raw: "D", want: false},
{name: "enabled bytes", raw: []byte("enabled"), want: true},
{name: "disabled bytes", raw: []byte("disabled"), want: false},
{name: "one", raw: int64(1), want: true},
{name: "zero", raw: int64(0), want: false},
{name: "null", raw: nil, want: true},
{name: "unknown", raw: "future-state", want: true},
{name: "not validated label defaults enabled", raw: "N", want: true},
} {
t.Run(test.name, func(t *testing.T) {
if actual := parseConstraintEnabled(test.raw); actual != test.want {
t.Fatalf("parseConstraintEnabled(%#v) = %v, want %v", test.raw, actual, test.want)
}
})
}
}
func TestListIndexesFallsBackWhenWithOrdinalityIsUnsupported(t *testing.T) {
state := &metadataDriverState{query: func(query string) (driver.Rows, error) {
switch {
@@ -1016,6 +1106,74 @@ func TestListIndexesFallsBackWhenWithOrdinalityIsUnsupported(t *testing.T) {
}
}
func TestGetColumnsUsesResolvedSchemaAcrossCatalogMetadata(t *testing.T) {
tests := []struct {
name string
postgresCatalog bool
requestedSchema string
resolvedSchema string
visibilityFunction string
sqlServerIdentity bool
}{
{name: "sys catalog search path", resolvedSchema: "tenant_visible", visibilityFunction: "sys_catalog.sys_table_is_visible(c.oid)", sqlServerIdentity: true},
{name: "postgres catalog search path", postgresCatalog: true, resolvedSchema: "tenant_pg", visibilityFunction: "pg_catalog.pg_table_is_visible(c.oid)"},
{name: "explicit schema", requestedSchema: "tenant_explicit", resolvedSchema: "tenant_explicit"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
catalog := "sys_catalog"
prefix := "sys"
if test.postgresCatalog {
catalog = "pg_catalog"
prefix = "pg"
}
state := &metadataDriverState{query: func(query string) (driver.Rows, error) {
switch {
case strings.Contains(query, "FROM "+catalog+"."+prefix+"_attribute a"):
if test.visibilityFunction != "" {
if !strings.Contains(query, test.visibilityFunction) {
return nil, fmt.Errorf("unqualified columns query did not use the catalog visibility function: %s", query)
}
} else if strings.Contains(query, "table_is_visible") || !strings.Contains(query, "n.nspname = '"+test.requestedSchema+"'") {
return nil, fmt.Errorf("explicit-schema columns query changed resolution behavior: %s", query)
}
return &valueRows{
columns: []string{"nspname", "attname", "format_type", "nullable", "default", "comment", "precision", "scale", "length", "identity"},
rows: [][]driver.Value{{test.resolvedSchema, "feearea", "character varying", false, nil, nil, nil, nil, nil, nil}},
}, nil
case strings.Contains(query, "FROM information_schema.table_constraints"):
if !strings.Contains(query, "tc.table_schema='"+test.resolvedSchema+"'") {
return nil, fmt.Errorf("primary-key lookup did not use resolved schema: %s", query)
}
return &valueRows{columns: []string{"column_name"}, rows: [][]driver.Value{{"feearea"}}}, nil
case strings.Contains(query, "FROM sys.identity_columns"):
if !strings.Contains(query, "n.nspname='"+test.resolvedSchema+"'") {
return nil, fmt.Errorf("identity lookup did not use resolved schema: %s", query)
}
return &valueRows{columns: []string{"attname", "seed_value", "increment_value"}, rows: [][]driver.Value{{"feearea", "1", "1"}}}, nil
default:
return nil, fmt.Errorf("unexpected query: %s", query)
}
}}
server := newServer()
server.db = openMetadataDB(t, state)
server.mode.postgresCatalog = test.postgresCatalog
server.mode.sqlServerIdentity = test.sqlServerIdentity
columns, err := server.getColumns(test.requestedSchema, "m_workflow")
if err != nil {
t.Fatal(err)
}
if len(columns) != 1 || columns[0].ResolvedSchema == nil || *columns[0].ResolvedSchema != test.resolvedSchema || !columns[0].IsPrimaryKey {
t.Fatalf("resolved relation metadata was lost: %#v", columns)
}
if test.sqlServerIdentity && (columns[0].Extra == nil || *columns[0].Extra != "IDENTITY(1,1)") {
t.Fatalf("identity metadata did not use the resolved relation: %#v", columns)
}
})
}
}
func TestListIndexesDoesNotFallbackForUnrelatedErrors(t *testing.T) {
expectedErr := errors.New("metadata connection reset")
state := &metadataDriverState{query: func(string) (driver.Rows, error) { return nil, expectedErr }}
@@ -1080,6 +1238,262 @@ func TestListForeignKeysKeepsEmptyInformationSchemaResultOnV8(t *testing.T) {
}
}
func TestListConstraintsResolvesColumnsAndForeignKeyDetails(t *testing.T) {
state := &metadataDriverState{query: func(query string) (driver.Rows, error) {
switch {
case strings.Contains(query, "FROM sys_catalog.sys_constraint c"):
return &valueRows{
columns: []string{"conname", "contype", "definition", "conkey", "ref_schema", "ref_table", "confkey", "match_type", "on_update", "on_delete", "condeferrable", "condeferred", "convalidated", "constatus"},
rows: [][]driver.Value{
{"orders_amount_check", "c", "CHECK (amount > 0)", "{4}", nil, nil, nil, " ", " ", " ", false, false, []byte("f"), []byte("D")},
{"orders_customer_fkey", "f", "FOREIGN KEY (customer_id, customer_region) REFERENCES customers(id, region) ON DELETE CASCADE", "{2,3}", "PUBLIC", "customers", "{1,2}", "s", "a", "c", true, true, "t", "E"},
},
}, nil
case strings.Contains(query, "SELECT a.attnum, a.attname") && strings.Contains(query, "c.relname = 'orders'"):
return &valueRows{columns: []string{"attnum", "attname"}, rows: [][]driver.Value{{int64(2), "customer_id"}, {int64(3), "customer_region"}, {int64(4), "amount"}}}, nil
case strings.Contains(query, "SELECT a.attnum, a.attname") && strings.Contains(query, "c.relname = 'customers'"):
return &valueRows{columns: []string{"attnum", "attname"}, rows: [][]driver.Value{{int64(1), "id"}, {int64(2), "region"}}}, nil
default:
return nil, errors.New("unexpected query: " + query)
}
}}
server := newServer()
server.db = openMetadataDB(t, state)
constraints, err := server.listConstraints("PUBLIC", "orders")
if err != nil {
t.Fatal(err)
}
if len(constraints) != 2 {
t.Fatalf("unexpected constraints: %#v", constraints)
}
check, foreignKey := constraints[0], constraints[1]
if check.ConstraintType != "CHECK" || !equalStringSlices(check.Columns, []string{"amount"}) || check.Valid || check.Enabled {
t.Fatalf("unexpected check constraint: %#v", check)
}
if foreignKey.ConstraintType != "FOREIGN KEY" || !foreignKey.Valid || !foreignKey.Enabled || !equalStringSlices(foreignKey.Columns, []string{"customer_id", "customer_region"}) || !equalStringSlices(foreignKey.RefColumns, []string{"id", "region"}) {
t.Fatalf("unexpected foreign key columns: %#v", foreignKey)
}
if foreignKey.RefSchema == nil || *foreignKey.RefSchema != "PUBLIC" || foreignKey.RefTable == nil || *foreignKey.RefTable != "customers" || foreignKey.MatchType == nil || *foreignKey.MatchType != "SIMPLE" || foreignKey.OnUpdate == nil || *foreignKey.OnUpdate != "NO ACTION" || foreignKey.OnDelete == nil || *foreignKey.OnDelete != "CASCADE" || !foreignKey.Deferrable || !foreignKey.InitiallyDeferred {
t.Fatalf("unexpected foreign key details: %#v", foreignKey)
}
queries := strings.Join(state.snapshotQueries(), "\n")
if !strings.Contains(queries, "sys_catalog.sys_get_constraintdef") {
t.Fatalf("constraints must use the active catalog deparser: %s", queries)
}
}
func TestKingbaseConstraintQueryHasLegacyV7Fallback(t *testing.T) {
modern := kingbaseConstraintsQuery("sys_catalog", "sys", "public", "orders", false, false, false)
if !strings.Contains(modern, "sys_catalog.sys_get_constraintdef") || !strings.Contains(modern, "c.convalidated") || !strings.Contains(modern, "COALESCE(c.conname, '')") {
t.Fatalf("modern constraint query missing metadata fields: %s", modern)
}
if !strings.Contains(modern, "COALESCE(CAST(c.convalidated AS text), 'T')") {
t.Fatalf("modern constraint query must normalize convalidated to text: %s", modern)
}
if !strings.Contains(modern, "COALESCE(CAST(c.constatus AS text), 'E')") {
t.Fatalf("modern constraint query must normalize constatus to text: %s", modern)
}
legacy := kingbaseConstraintsQuery("sys_catalog", "sys", "public", "orders", true, true, true)
if strings.Contains(legacy, "sys_get_constraintdef") || strings.Contains(legacy, "c.convalidated") || !strings.Contains(legacy, "''") || !strings.Contains(legacy, "COALESCE(c.conname, '')") {
t.Fatalf("legacy V7 constraint query is not safe: %s", legacy)
}
}
func TestListConstraintsFallsBackWhenValidatedColumnIsUnsupported(t *testing.T) {
state := &metadataDriverState{query: func(query string) (driver.Rows, error) {
if strings.Contains(query, "c.convalidated") {
return nil, &gokb.Error{Code: gokb.ErrorCode("42703"), Message: "column c.convalidated does not exist"}
}
if strings.Contains(query, "FROM sys_catalog.sys_constraint c") {
return &valueRows{
columns: []string{"conname", "contype", "definition", "conkey", "ref_schema", "ref_table", "confkey", "match_type", "on_update", "on_delete", "condeferrable", "condeferred", "valid", "enabled"},
rows: [][]driver.Value{{"orders_pkey", "p", "PRIMARY KEY (id)", "{1}", nil, nil, nil, nil, nil, nil, false, false, true, true}},
}, nil
}
if strings.Contains(query, "SELECT a.attnum, a.attname") {
return &valueRows{columns: []string{"attnum", "attname"}, rows: [][]driver.Value{{int64(1), "id"}}}, nil
}
return nil, errors.New("unexpected query: " + query)
}}
server := newServer()
server.db = openMetadataDB(t, state)
constraints, err := server.listConstraints("PUBLIC", "orders")
if err != nil {
t.Fatal(err)
}
if len(constraints) != 1 || constraints[0].Definition != "PRIMARY KEY (id)" || !constraints[0].Valid {
t.Fatalf("unexpected fallback constraints: %#v", constraints)
}
if !server.constraintValidatedUnsupported {
t.Fatal("validated-column fallback was not cached")
}
if len(state.snapshotQueries()) != 3 {
t.Fatalf("expected failed modern query, fallback query and attribute query: %v", state.snapshotQueries())
}
if _, err := server.listConstraints("PUBLIC", "orders"); err != nil {
t.Fatal(err)
}
queries := state.snapshotQueries()
for _, query := range queries[3:] {
if strings.Contains(query, "c.convalidated") {
t.Fatalf("cached fallback queried unsupported convalidated column: %s", query)
}
}
}
func TestListConstraintsLegacyV7KeepsStructuralMetadata(t *testing.T) {
state := &metadataDriverState{query: func(query string) (driver.Rows, error) {
switch {
case strings.Contains(query, "FROM sys_catalog.sys_constraint c"):
if strings.Contains(query, "c.convalidated") {
return nil, errors.New("V7 query used unsupported constraint metadata: " + query)
}
if strings.Contains(query, "sys_get_constraintdef") {
return nil, &gokb.Error{Code: gokb.ErrorCode("42883"), Message: "function sys_get_constraintdef does not exist"}
}
return &valueRows{
columns: []string{"conname", "contype", "definition", "conkey", "ref_schema", "ref_table", "confkey", "match_type", "on_update", "on_delete", "condeferrable", "condeferred", "valid", "enabled"},
rows: [][]driver.Value{{"orders_pkey", "p", "", "[1]", nil, nil, nil, nil, nil, nil, false, false, true, true}},
}, nil
case strings.Contains(query, "SELECT a.attnum, a.attname"):
return &valueRows{columns: []string{"attnum", "attname"}, rows: [][]driver.Value{{int64(1), "id"}}}, nil
default:
return nil, errors.New("unexpected query: " + query)
}
}}
server := newServer()
server.db = openMetadataDB(t, state)
server.mode.legacyV7 = true
constraints, err := server.listConstraints("PUBLIC", "orders")
if err != nil {
t.Fatal(err)
}
if len(constraints) != 1 || constraints[0].Name != "orders_pkey" || constraints[0].Definition != "" || !constraints[0].Valid || !equalStringSlices(constraints[0].Columns, []string{"id"}) || !server.constraintDefinitionUnsupported {
t.Fatalf("unexpected V7 constraints: %#v", constraints)
}
}
func TestListConstraintsLegacyV7UsesSupportedDeparser(t *testing.T) {
state := &metadataDriverState{query: func(query string) (driver.Rows, error) {
switch {
case strings.Contains(query, "FROM sys_catalog.sys_constraint c"):
if !strings.Contains(query, "sys_catalog.sys_get_constraintdef") || strings.Contains(query, "COALESCE(CAST(c.convalidated AS text)") || strings.Contains(query, "c.constatus") {
return nil, errors.New("unexpected V7 constraint query: " + query)
}
return &valueRows{
columns: []string{"conname", "contype", "definition", "conkey", "ref_schema", "ref_table", "confkey", "match_type", "on_update", "on_delete", "condeferrable", "condeferred", "valid", "enabled"},
rows: [][]driver.Value{{"orders_pkey", "p", "PRIMARY KEY (id)", "[1]", nil, nil, nil, nil, nil, nil, false, false, true, "E"}},
}, nil
case strings.Contains(query, "SELECT a.attnum, a.attname"):
return &valueRows{columns: []string{"attnum", "attname"}, rows: [][]driver.Value{{int64(1), "id"}}}, nil
default:
return nil, errors.New("unexpected query: " + query)
}
}}
server := newServer()
server.db = openMetadataDB(t, state)
server.mode.legacyV7 = true
constraints, err := server.listConstraints("PUBLIC", "orders")
if err != nil {
t.Fatal(err)
}
if len(constraints) != 1 || constraints[0].Definition != "PRIMARY KEY (id)" || !constraints[0].Enabled || server.constraintDefinitionUnsupported {
t.Fatalf("unexpected V7 constraints: %#v", constraints)
}
}
func TestKingbaseConstraintQueryUsesPostgresCatalog(t *testing.T) {
query := kingbaseConstraintsQuery("pg_catalog", "pg", "public", "orders", false, false, false)
if !strings.Contains(query, "pg_catalog.pg_get_constraintdef") || strings.Contains(query, "sys_get_constraintdef") {
t.Fatalf("PostgreSQL catalog query used the wrong deparser: %s", query)
}
if !strings.Contains(query, "FROM pg_catalog.pg_constraint") || !strings.Contains(query, "pg_catalog.pg_namespace") {
t.Fatalf("PostgreSQL catalog query used the wrong catalog tables: %s", query)
}
}
func TestListConstraintsHandlesNullAttributeVectors(t *testing.T) {
state := &metadataDriverState{query: func(query string) (driver.Rows, error) {
switch {
case strings.Contains(query, "FROM sys_catalog.sys_constraint c"):
return &valueRows{
columns: []string{"conname", "contype", "definition", "conkey", "ref_schema", "ref_table", "confkey", "match_type", "on_update", "on_delete", "condeferrable", "condeferred", "valid", "enabled"},
rows: [][]driver.Value{{"table_check", "c", "CHECK (true)", nil, nil, nil, nil, nil, nil, nil, false, false, true, true}},
}, nil
case strings.Contains(query, "SELECT a.attnum, a.attname"):
return &valueRows{columns: []string{"attnum", "attname"}}, nil
default:
return nil, errors.New("unexpected query: " + query)
}
}}
server := newServer()
server.db = openMetadataDB(t, state)
constraints, err := server.listConstraints("PUBLIC", "orders")
if err != nil {
t.Fatal(err)
}
if len(constraints) != 1 || len(constraints[0].Columns) != 0 || len(constraints[0].RefColumns) != 0 {
t.Fatalf("NULL attribute vectors should decode as empty lists: %#v", constraints)
}
}
func TestListConstraintsCachesReferencedAttributes(t *testing.T) {
state := &metadataDriverState{query: func(query string) (driver.Rows, error) {
switch {
case strings.Contains(query, "FROM sys_catalog.sys_constraint c"):
return &valueRows{
columns: []string{"conname", "contype", "definition", "conkey", "ref_schema", "ref_table", "confkey", "match_type", "on_update", "on_delete", "condeferrable", "condeferred", "valid", "enabled"},
rows: [][]driver.Value{
{"orders_customer_fkey", "f", "FOREIGN KEY (customer_id) REFERENCES customers(id)", "{2}", "PUBLIC", "customers", "{1}", "s", "a", "a", false, false, true, true},
{"orders_region_fkey", "f", "FOREIGN KEY (customer_region) REFERENCES customers(region)", "{3}", "PUBLIC", "customers", "{2}", "s", "a", "a", false, false, true, true},
},
}, nil
case strings.Contains(query, "SELECT a.attnum, a.attname") && strings.Contains(query, "c.relname = 'orders'"):
return &valueRows{columns: []string{"attnum", "attname"}, rows: [][]driver.Value{{int64(2), "customer_id"}, {int64(3), "customer_region"}}}, nil
case strings.Contains(query, "SELECT a.attnum, a.attname") && strings.Contains(query, "c.relname = 'customers'"):
return &valueRows{columns: []string{"attnum", "attname"}, rows: [][]driver.Value{{int64(1), "id"}, {int64(2), "region"}}}, nil
default:
return nil, errors.New("unexpected query: " + query)
}
}}
server := newServer()
server.db = openMetadataDB(t, state)
constraints, err := server.listConstraints("PUBLIC", "orders")
if err != nil || len(constraints) != 2 {
t.Fatalf("unexpected constraints: %v %#v", err, constraints)
}
var referencedAttributeQueries int
for _, query := range state.snapshotQueries() {
if strings.Contains(query, "SELECT a.attnum, a.attname") && strings.Contains(query, "c.relname = 'customers'") {
referencedAttributeQueries++
}
}
if referencedAttributeQueries != 1 {
t.Fatalf("referenced relation attributes were not cached: %v", state.snapshotQueries())
}
}
func TestKingbaseConstraintLabels(t *testing.T) {
for input, expected := range map[string]string{"p": "PRIMARY KEY", "f": "FOREIGN KEY", "u": "UNIQUE", "c": "CHECK", "t": "CONSTRAINT TRIGGER", "x": "EXCLUDE", "n": "NOT NULL", "custom": "custom"} {
if actual := kingbaseConstraintTypeName(input); actual != expected {
t.Fatalf("type %q: expected %q, got %q", input, expected, actual)
}
}
for input, expected := range map[string]string{"a": "NO ACTION", "r": "RESTRICT", "c": "CASCADE", "n": "SET NULL", "d": "SET DEFAULT"} {
actual := kingbaseConstraintAction(sql.NullString{String: input, Valid: true})
if actual == nil || *actual != expected {
t.Fatalf("action %q: expected %q, got %v", input, expected, actual)
}
}
}
func TestKingbaseV7VersionPattern(t *testing.T) {
for _, test := range []struct {
version string
@@ -2287,6 +2701,220 @@ func TestRoutineSourceUsesKingbaseCatalogFunction(t *testing.T) {
}
}
func TestMaterializedViewSourceFallsBackFromEmptySysDefinition(t *testing.T) {
tests := []struct {
name string
primaryRows [][]driver.Value
}{
{name: "blank definition", primaryRows: [][]driver.Value{{" \n"}}},
{name: "missing definition", primaryRows: [][]driver.Value{}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
const expected = "SELECT product_id, sum(amount) FROM public.sales GROUP BY product_id"
state := &metadataDriverState{query: func(query string) (driver.Rows, error) {
switch {
case strings.Contains(query, "sys_get_viewdef("):
return &valueRows{columns: []string{"source"}, rows: test.primaryRows}, nil
case strings.Contains(query, "pg_get_viewdef("):
return &valueRows{columns: []string{"source"}, rows: [][]driver.Value{{expected}}}, nil
default:
return nil, errors.New("unexpected query: " + query)
}
}}
server := newServer()
server.db = openMetadataDB(t, state)
source, err := server.getObjectSource("public", "daily_sales", "MATERIALIZED_VIEW")
if err != nil {
t.Fatal(err)
}
if source["source"] != expected {
t.Fatalf("unexpected materialized view source: %#v", source)
}
queries := state.snapshotQueries()
if len(queries) != 2 || !strings.Contains(queries[0], "sys_get_viewdef(") || !strings.Contains(queries[1], "pg_get_viewdef(") {
t.Fatalf("unexpected materialized view source fallback: %v", queries)
}
for _, query := range queries {
if !strings.Contains(query, "c.relkind = 'm'") {
t.Fatalf("materialized view query was not constrained by relation kind: %s", query)
}
}
if server.usePgViewDefinition {
t.Fatal("an empty definition for one materialized view must not change the ordinary-view function cache")
}
})
}
}
func TestMaterializedViewSourceRejectsEmptyDefinitionAfterFallback(t *testing.T) {
tests := []struct {
name string
rows [][]driver.Value
}{
{name: "blank definitions", rows: [][]driver.Value{{"\t"}}},
{name: "missing definitions", rows: [][]driver.Value{}},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
state := &metadataDriverState{query: func(query string) (driver.Rows, error) {
if strings.Contains(query, "sys_get_viewdef(") || strings.Contains(query, "pg_get_viewdef(") {
return &valueRows{columns: []string{"source"}, rows: test.rows}, nil
}
return nil, errors.New("unexpected query: " + query)
}}
server := newServer()
server.db = openMetadataDB(t, state)
_, err := server.getObjectSource("public", "daily_sales", "MATERIALIZED_VIEW")
if err == nil || !strings.Contains(err.Error(), "materialized view") || !strings.Contains(err.Error(), "empty source") {
t.Fatalf("empty materialized view definitions must fail explicitly: %v", err)
}
if queries := state.snapshotQueries(); len(queries) != 2 {
t.Fatalf("expected the bounded sys/pg definition probes, got %v", queries)
}
if server.usePgViewDefinition {
t.Fatal("empty definitions must not change the ordinary-view function cache")
}
})
}
}
func TestMySQLCompatViewAndMaterializedViewSourcesUseSeparateCatalogs(t *testing.T) {
state := &metadataDriverState{query: func(query string) (driver.Rows, error) {
switch {
case strings.Contains(query, "information_schema.views") && strings.Contains(query, "table_name = 'active_orders'"):
return &valueRows{columns: []string{"view_definition"}, rows: [][]driver.Value{{"SELECT * FROM orders WHERE active"}}}, nil
case strings.Contains(query, "information_schema.views"):
return nil, errors.New("materialized views must not be read from information_schema.views: " + query)
case strings.Contains(query, "sys_get_viewdef(") && strings.Contains(query, "c.relkind = 'm'"):
return &valueRows{columns: []string{"source"}, rows: [][]driver.Value{{"SELECT product_id, sum(amount) FROM sales GROUP BY product_id"}}}, nil
default:
return nil, errors.New("unexpected query: " + query)
}
}}
server := newServer()
server.db = openMetadataDB(t, state)
server.mode.mysqlCompat = true
view, err := server.getObjectSource("app", "active_orders", "VIEW")
if err != nil {
t.Fatal(err)
}
if view["source"] != "SELECT * FROM orders WHERE active" {
t.Fatalf("unexpected ordinary view source: %#v", view)
}
materialized, err := server.getObjectSource("app", "daily_sales", "MATERIALIZED_VIEW")
if err != nil {
t.Fatal(err)
}
if !strings.HasPrefix(fmt.Sprint(materialized["source"]), "SELECT product_id") {
t.Fatalf("unexpected materialized view source: %#v", materialized)
}
queries := state.snapshotQueries()
if len(queries) != 2 || !strings.Contains(queries[0], "information_schema.views") || !strings.Contains(queries[1], "sys_get_viewdef(") {
t.Fatalf("view kinds used the wrong metadata paths: %v", queries)
}
}
func TestMaterializedViewSourceUsesPostgresCatalogDirectly(t *testing.T) {
state := &metadataDriverState{query: func(query string) (driver.Rows, error) {
if !strings.Contains(query, "SELECT pg_get_viewdef(c.oid)") ||
!strings.Contains(query, "FROM pg_catalog.pg_class") ||
!strings.Contains(query, "c.relkind = 'm'") {
return nil, errors.New("unexpected query: " + query)
}
return &valueRows{columns: []string{"source"}, rows: [][]driver.Value{{"SELECT * FROM public.sales"}}}, nil
}}
server := newServer()
server.db = openMetadataDB(t, state)
server.mode.postgresCatalog = true
source, err := server.getObjectSource("public", "daily_sales", "MATERIALIZED_VIEW")
if err != nil {
t.Fatal(err)
}
if source["source"] != "SELECT * FROM public.sales" {
t.Fatalf("unexpected materialized view source: %#v", source)
}
if queries := state.snapshotQueries(); len(queries) != 1 {
t.Fatalf("PostgreSQL catalog mode must use one direct pg_get_viewdef query: %v", queries)
}
}
func TestMaterializedViewUndefinedSysFunctionKeepsPgDefinitionCache(t *testing.T) {
state := &metadataDriverState{query: func(query string) (driver.Rows, error) {
switch {
case strings.Contains(query, "sys_get_viewdef("):
return nil, &gokb.Error{Code: gokb.ErrorCode("42883"), Message: "function sys_get_viewdef(oid) does not exist"}
case strings.Contains(query, "pg_get_viewdef("):
return &valueRows{columns: []string{"source"}, rows: [][]driver.Value{{"SELECT * FROM public.sales"}}}, nil
default:
return nil, errors.New("unexpected query: " + query)
}
}}
server := newServer()
server.db = openMetadataDB(t, state)
if _, err := server.getObjectSource("public", "daily_sales", "MATERIALIZED_VIEW"); err != nil {
t.Fatal(err)
}
if _, err := server.getObjectSource("public", "active_sales", "VIEW"); err != nil {
t.Fatal(err)
}
queries := state.snapshotQueries()
if len(queries) != 3 || !strings.Contains(queries[0], "sys_get_viewdef(") ||
!strings.Contains(queries[1], "pg_get_viewdef(") || !strings.Contains(queries[2], "pg_get_viewdef(") ||
!server.usePgViewDefinition {
t.Fatalf("undefined sys_get_viewdef fallback cache changed: %v", queries)
}
}
func TestMaterializedViewSourceDoesNotFallbackOnUnrelatedErrors(t *testing.T) {
permissionErr := errors.New("permission denied for sys_class")
state := &metadataDriverState{query: func(query string) (driver.Rows, error) {
if strings.Contains(query, "sys_get_viewdef(") {
return nil, permissionErr
}
return nil, errors.New("unexpected fallback query: " + query)
}}
server := newServer()
server.db = openMetadataDB(t, state)
_, err := server.getObjectSource("public", "daily_sales", "MATERIALIZED_VIEW")
if !errors.Is(err, permissionErr) {
t.Fatalf("unexpected error: %v", err)
}
if queries := state.snapshotQueries(); len(queries) != 1 || !strings.Contains(queries[0], "sys_get_viewdef(") {
t.Fatalf("unrelated error triggered a fallback: %v", queries)
}
}
func TestViewSourceUsesSysDefinitionOnce(t *testing.T) {
state := &metadataDriverState{query: func(query string) (driver.Rows, error) {
if !strings.Contains(query, "SELECT sys_get_viewdef(c.oid)") || !strings.Contains(query, "FROM sys_catalog.sys_class") {
return nil, errors.New("unexpected query: " + query)
}
return &valueRows{columns: []string{"source"}, rows: [][]driver.Value{{"SELECT * FROM public.orders WHERE active"}}}, nil
}}
server := newServer()
server.db = openMetadataDB(t, state)
source, err := server.getObjectSource("public", "active_orders", "VIEW")
if err != nil {
t.Fatal(err)
}
if source["source"] != "SELECT * FROM public.orders WHERE active" {
t.Fatalf("unexpected view source: %#v", source)
}
if queries := state.snapshotQueries(); len(queries) != 1 {
t.Fatalf("ordinary view source should use one query: %v", queries)
}
}
func TestTriggerObjectSourceUsesOwningRelation(t *testing.T) {
state := &metadataDriverState{query: func(query string) (driver.Rows, error) {
for _, fragment := range []string{
@@ -2978,6 +3606,32 @@ func TestConnectionLifecycleResetsCatalogOIDCapability(t *testing.T) {
}
}
func TestConnectionLifecycleResetsConstraintCapabilityCache(t *testing.T) {
state := &connectionAttemptState{pingErrors: map[string]error{}}
server := newServer()
server.openDatabase = state.open
server.constraintDefinitionUnsupported = true
server.constraintValidatedUnsupported = true
server.constraintStatusUnsupported = true
if err := server.connect(connectParams{MySQLCompatMode: true, URLParams: "sslmode=disable"}); err != nil {
t.Fatal(err)
}
if server.constraintDefinitionUnsupported || server.constraintValidatedUnsupported || server.constraintStatusUnsupported {
t.Fatal("connect must reset cached constraint capabilities")
}
server.constraintDefinitionUnsupported = true
server.constraintValidatedUnsupported = true
server.constraintStatusUnsupported = true
if err := server.disconnect(); err != nil {
t.Fatal(err)
}
if server.constraintDefinitionUnsupported || server.constraintValidatedUnsupported || server.constraintStatusUnsupported {
t.Fatal("disconnect must reset cached constraint capabilities")
}
}
func TestAppendDDLStatementEnsuresSingleTerminator(t *testing.T) {
got := appendDDLStatement("CREATE TABLE \"public\".\"orders\" (\n \"id\" integer\n)\n", "CREATE INDEX orders_id_idx ON public.orders (id)")
want := "CREATE TABLE \"public\".\"orders\" (\n \"id\" integer\n);\n\nCREATE INDEX orders_id_idx ON public.orders (id);"
@@ -3010,6 +3664,9 @@ func TestMySQLCompatColumnsUsePostgresColumnComments(t *testing.T) {
if columns[0].Extra != nil {
t.Fatalf("MySQL-compatible metadata must not infer PostgreSQL identity: %#v", columns[0].Extra)
}
if columns[0].ResolvedSchema == nil || *columns[0].ResolvedSchema != "public" {
t.Fatalf("MySQL-compatible metadata must keep its effective schema: %#v", columns[0])
}
state.mu.Lock()
defer state.mu.Unlock()
@@ -3297,6 +3954,39 @@ func TestSchemaConnResetsOnceAfterExplicitSchema(t *testing.T) {
}
}
func TestSchemaConnUsesBackticksInMySQLCompatMode(t *testing.T) {
db, state := openFakeDB(t, 0)
server := newServer()
server.db = db
server.mode.mysqlCompat = true
conn, err := server.schemaConn(context.Background(), "audit-schema")
if err != nil {
t.Fatal(err)
}
if err := conn.Close(); err != nil {
t.Fatal(err)
}
state.mu.Lock()
defer state.mu.Unlock()
if len(state.execStatements) != 1 || state.execStatements[0] != "SET search_path TO `audit-schema`" {
t.Fatalf("unexpected MySQL compatibility schema setup: %v", state.execStatements)
}
}
func TestKingbaseIdentifierQuoteEscapesModeSpecificDelimiter(t *testing.T) {
server := newServer()
server.mode.mysqlCompat = true
if got := server.quoteIdentifier("audit`schema"); got != "`audit``schema`" {
t.Fatalf("unexpected MySQL compatibility identifier: %s", got)
}
server.mode.mysqlCompat = false
if got := server.quoteIdentifier(`audit"schema`); got != `"audit""schema"` {
t.Fatalf("unexpected PostgreSQL-compatible identifier: %s", got)
}
}
func TestSchemaConnPropagatesSchemaErrors(t *testing.T) {
tests := []struct {
name string
@@ -398,6 +398,28 @@ class OceanBaseOracleAgentTest {
Assertions.assertTrue(sql.get(0).contains("c.DATA_DEFAULT"), sql.get(0));
}
@Test
void getColumnsResolvesTheCurrentSchemaWhenSchemaIsEmpty() {
List<String> sql = new ArrayList<>();
List<String> params = new ArrayList<>();
OceanBaseOracleAgent agent = new OceanBaseOracleAgent();
TestSupport.setPrivateConnection(agent, currentSchemaColumnConnection(
sql,
params,
resultSet(new String[]{"CURRENT_SCHEMA"}, new Object[][]{{"APP"}}),
columnResultSet(new Object[][]{
{"PARAM_VALUE", "VARCHAR2", "Y", null, null, 100, 100, null, null, 0}
})
));
List<ColumnInfo> columns = agent.getColumns("", "TBPARAM");
Assertions.assertEquals(List.of("PARAM_VALUE"), columns.stream().map(ColumnInfo::getName).toList());
Assertions.assertEquals(List.of("APP", "TBPARAM", "APP", "TBPARAM"), params);
Assertions.assertTrue(sql.get(0).contains("SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA')"), sql.get(0));
Assertions.assertTrue(sql.get(1).contains("FROM ALL_TAB_COLUMNS"), sql.get(1));
}
@Test
void tableDdlIncludesDefaultsAndOnlyNonBlankColumnComments() {
OceanBaseOracleAgent agent = new OceanBaseOracleAgent();
@@ -496,6 +518,45 @@ class OceanBaseOracleAgentTest {
});
}
private static Connection currentSchemaColumnConnection(List<String> sql, List<String> params, ResultSet currentSchema, ResultSet columns) {
Statement schemaStatement = proxy(Statement.class, (method, args) -> {
if ("executeQuery".equals(method.getName())) {
sql.add(String.valueOf(args[0]));
return currentSchema;
}
if ("close".equals(method.getName())) {
return null;
}
return defaultValue(method.getReturnType());
});
PreparedStatement columnStatement = proxy(PreparedStatement.class, (method, args) -> {
if ("executeQuery".equals(method.getName())) {
return columns;
}
if ("setString".equals(method.getName())) {
params.add(String.valueOf(args[1]));
return null;
}
if ("close".equals(method.getName())) {
return null;
}
return defaultValue(method.getReturnType());
});
return proxy(Connection.class, (method, args) -> {
if ("createStatement".equals(method.getName())) {
return schemaStatement;
}
if ("prepareStatement".equals(method.getName())) {
sql.add(String.valueOf(args[0]));
return columnStatement;
}
if ("isClosed".equals(method.getName())) {
return false;
}
return defaultValue(method.getReturnType());
});
}
private static Connection objectSourceConnection(List<String> sql, List<String> params, ResultSet resultSet) {
PreparedStatement statement = objectSourceStatement(params, resultSet, false);
return proxy(Connection.class, (method, args) -> {
+193 -36
View File
@@ -94,6 +94,8 @@ ORDER BY CASE
WHEN username = SYS_CONTEXT('USERENV', 'SESSION_USER') THEN 1
ELSE 2
END, username`
// Sorting SYS views through ALL_OBJECTS can take over a minute on Oracle 11g; use the specialized dictionaries.
const oracleListTablesBaseSQL = `
SELECT OBJECT_NAME, TABLE_TYPE, COMMENTS
FROM (
@@ -105,18 +107,22 @@ WHERE t.OWNER = :1
AND t.NESTED = 'NO'
AND NOT EXISTS (
SELECT 1
FROM ALL_OBJECTS mv
FROM ALL_MVIEWS mv
WHERE mv.OWNER = t.OWNER
AND mv.OBJECT_NAME = t.TABLE_NAME
AND mv.OBJECT_TYPE = 'MATERIALIZED VIEW'
AND mv.MVIEW_NAME = t.TABLE_NAME
)
UNION ALL
SELECT o.OBJECT_NAME,
CASE o.OBJECT_TYPE WHEN 'MATERIALIZED VIEW' THEN 'MATERIALIZED_VIEW' ELSE o.OBJECT_TYPE END AS TABLE_TYPE,
SELECT v.VIEW_NAME AS OBJECT_NAME,
'VIEW' AS TABLE_TYPE,
CAST(NULL AS VARCHAR2(4000)) AS COMMENTS
FROM ALL_OBJECTS o
WHERE o.OWNER = :2
AND o.OBJECT_TYPE IN ('VIEW', 'MATERIALIZED VIEW')
FROM ALL_VIEWS v
WHERE v.OWNER = :2
UNION ALL
SELECT mv.MVIEW_NAME AS OBJECT_NAME,
'MATERIALIZED_VIEW' AS TABLE_TYPE,
CAST(NULL AS VARCHAR2(4000)) AS COMMENTS
FROM ALL_MVIEWS mv
WHERE mv.OWNER = :3
)`
const oracleListTablesSessionUserBaseSQL = `
SELECT OBJECT_NAME, TABLE_TYPE, COMMENTS
@@ -128,16 +134,19 @@ FROM USER_TABLES t
WHERE t.NESTED = 'NO'
AND NOT EXISTS (
SELECT 1
FROM USER_OBJECTS mv
WHERE mv.OBJECT_NAME = t.TABLE_NAME
AND mv.OBJECT_TYPE = 'MATERIALIZED VIEW'
FROM USER_MVIEWS mv
WHERE mv.MVIEW_NAME = t.TABLE_NAME
)
UNION ALL
SELECT o.OBJECT_NAME,
CASE o.OBJECT_TYPE WHEN 'MATERIALIZED VIEW' THEN 'MATERIALIZED_VIEW' ELSE o.OBJECT_TYPE END AS TABLE_TYPE,
SELECT v.VIEW_NAME AS OBJECT_NAME,
'VIEW' AS TABLE_TYPE,
CAST(NULL AS VARCHAR2(4000)) AS COMMENTS
FROM USER_OBJECTS o
WHERE o.OBJECT_TYPE IN ('VIEW', 'MATERIALIZED VIEW')
FROM USER_VIEWS v
UNION ALL
SELECT mv.MVIEW_NAME AS OBJECT_NAME,
'MATERIALIZED_VIEW' AS TABLE_TYPE,
CAST(NULL AS VARCHAR2(4000)) AS COMMENTS
FROM USER_MVIEWS mv
)`
const oracleListTablesOrderSQL = `ORDER BY OBJECT_NAME`
const oracleListTablesSQL = oracleListTablesBaseSQL + "\n" + oracleListTablesOrderSQL
@@ -167,7 +176,7 @@ SELECT o.OBJECT_NAME,
CAST(NULL AS VARCHAR2(4000)) AS COMMENTS
FROM ALL_OBJECTS o
WHERE o.OWNER = :2
AND o.OBJECT_TYPE IN ('VIEW', 'MATERIALIZED VIEW', 'PROCEDURE', 'FUNCTION', 'PACKAGE', 'PACKAGE BODY')
AND o.OBJECT_TYPE IN ('VIEW', 'MATERIALIZED VIEW', 'PROCEDURE', 'FUNCTION', 'SEQUENCE', 'PACKAGE', 'PACKAGE BODY')
UNION ALL
SELECT s.SYNONYM_NAME AS OBJECT_NAME,
'SYNONYM' AS OBJECT_TYPE,
@@ -198,7 +207,7 @@ SELECT o.OBJECT_NAME,
END AS OBJECT_TYPE,
CAST(NULL AS VARCHAR2(4000)) AS COMMENTS
FROM USER_OBJECTS o
WHERE o.OBJECT_TYPE IN ('VIEW', 'MATERIALIZED VIEW', 'PROCEDURE', 'FUNCTION', 'PACKAGE', 'PACKAGE BODY')
WHERE o.OBJECT_TYPE IN ('VIEW', 'MATERIALIZED VIEW', 'PROCEDURE', 'FUNCTION', 'SEQUENCE', 'PACKAGE', 'PACKAGE BODY')
UNION ALL
SELECT s.SYNONYM_NAME AS OBJECT_NAME,
'SYNONYM' AS OBJECT_TYPE,
@@ -211,9 +220,10 @@ const oracleListObjectsOrderSQL = `ORDER BY CASE OBJECT_TYPE
WHEN 'MATERIALIZED_VIEW' THEN 2
WHEN 'PROCEDURE' THEN 3
WHEN 'FUNCTION' THEN 4
WHEN 'SYNONYM' THEN 5
WHEN 'PACKAGE' THEN 6
ELSE 7
WHEN 'SEQUENCE' THEN 5
WHEN 'SYNONYM' THEN 6
WHEN 'PACKAGE' THEN 7
ELSE 8
END, OBJECT_NAME`
const oracleListObjectsSQL = oracleListObjectsBaseSQL + "\n" + oracleListObjectsOrderSQL
const oracleListTriggersSQL = `
@@ -528,6 +538,10 @@ type server struct {
db *sql.DB
params connectParams
legacyLOBFetchDeferred bool
// manualConn + manualTx pin one physical Oracle session for interactive
// commit/rollback control across multiple execute_query RPCs.
manualConn *sql.Conn
manualTx *sql.Tx
sessions map[string]*querySession
tableReadSessions map[string]*querySession
nextSessionID int64
@@ -631,7 +645,7 @@ func (r *runtimeServer) dispatch(method string, params map[string]json.RawMessag
return map[string]any{
"protocolVersion": multiSessionProtocolVersion,
"agentProtocolVersion": multiSessionProtocolVersion,
"capabilities": []string{"connect", "test_connection", "metadata", "query", "ddl", "multi_session"},
"capabilities": []string{"connect", "test_connection", "metadata", "query", "transaction", "ddl", "multi_session"},
}, false, nil
case "open_session":
agentSessionID := stringParam(params, "agentSessionId")
@@ -817,7 +831,7 @@ func (s *server) dispatch(method string, params map[string]json.RawMessage) (any
return map[string]any{
"protocolVersion": protocolVersion,
"agentProtocolVersion": protocolVersion,
"capabilities": []string{"connect", "test_connection", "metadata", "query", "ddl"},
"capabilities": []string{"connect", "test_connection", "metadata", "query", "transaction", "ddl"},
}, false, nil
case "connect":
var cp connectParams
@@ -947,6 +961,12 @@ func (s *server) dispatch(method string, params map[string]json.RawMessage) (any
case "execute_transaction":
result, err := s.executeTransaction(params)
return result, false, err
case "begin_manual_transaction":
return map[string]bool{"ok": true}, false, s.beginManualTransaction(stringParam(params, "schema"))
case "commit_manual_transaction":
return map[string]bool{"ok": true}, false, s.commitManualTransaction()
case "rollback_manual_transaction":
return map[string]bool{"ok": true}, false, s.rollbackManualTransaction()
case "disconnect":
return map[string]bool{"ok": true}, false, s.disconnect()
case "shutdown":
@@ -1127,6 +1147,7 @@ func withOracleLOBFetchPost(params connectParams) connectParams {
func (s *server) disconnect() error {
s.closeAllQuerySessions()
_ = s.rollbackManualTransactionQuiet()
s.legacyLOBFetchDeferred = false
if s.db == nil {
return nil
@@ -1136,6 +1157,75 @@ func (s *server) disconnect() error {
return err
}
func (s *server) beginManualTransaction(schema string) error {
if s.manualTx != nil {
return errors.New("manual transaction already open")
}
db, err := s.requireDB()
if err != nil {
return err
}
// Hold one exclusive physical connection so DML/SELECT/schema stay on the
// same Oracle session for the life of the interactive transaction.
conn, err := db.Conn(context.Background())
if err != nil {
return fmt.Errorf("reserve connection for manual transaction: %w", err)
}
if strings.TrimSpace(schema) != "" {
if _, err := conn.ExecContext(context.Background(), "ALTER SESSION SET CURRENT_SCHEMA = "+quoteIdentifier(schema)); err != nil {
_ = conn.Close()
return err
}
}
tx, err := conn.BeginTx(context.Background(), nil)
if err != nil {
_ = conn.Close()
return fmt.Errorf("begin manual transaction: %w", err)
}
s.manualConn = conn
s.manualTx = tx
return nil
}
func (s *server) commitManualTransaction() error {
if s.manualTx == nil {
return errors.New("no manual transaction open")
}
err := s.manualTx.Commit()
s.clearManualTransaction()
return err
}
func (s *server) rollbackManualTransaction() error {
if s.manualTx == nil {
return errors.New("no manual transaction open")
}
err := s.manualTx.Rollback()
s.clearManualTransaction()
return err
}
func (s *server) rollbackManualTransactionQuiet() error {
if s.manualTx == nil {
return nil
}
err := s.manualTx.Rollback()
s.clearManualTransaction()
return err
}
func (s *server) clearManualTransaction() {
s.manualTx = nil
if s.manualConn != nil {
_ = s.manualConn.Close()
s.manualConn = nil
}
}
func (s *server) hasManualTransaction() bool {
return s.manualTx != nil
}
func openDB(params connectParams) (*sql.DB, error) {
return openDBWithStringConverter(params, nil)
}
@@ -1607,7 +1697,7 @@ func oracleListTablesQuery(schema string, constraints metadataListConstraints) o
"OBJECT_NAME, TABLE_TYPE, COMMENTS",
"TABLE_TYPE",
oracleListTablesOrderSQL,
[]any{schema, schema},
[]any{schema, schema, schema},
constraints,
)
}
@@ -2834,7 +2924,7 @@ func (s *server) getObjectSource(schema, name, objectType string) (map[string]an
}
return map[string]any{"name": name, "object_type": objectType, "schema": schema, "source": source}, nil
}
if upperType == "SYNONYM" {
if upperType == "SEQUENCE" || upperType == "SYNONYM" {
return s.getMetadataObjectSource(schema, name, upperType)
}
@@ -2869,6 +2959,39 @@ func (s *server) getMetadataObjectSource(schema, name, objectType string) (map[s
}
func (s *server) loadObjectSourceText(schema, name, objectType string) (string, bool, error) {
source, found, err := s.loadAggregatedObjectSourceText(schema, name, objectType)
if err == nil {
return source, found, nil
}
return s.loadObjectSourceTextRows(schema, name, objectType)
}
func (s *server) loadAggregatedObjectSourceText(schema, name, objectType string) (string, bool, error) {
rows, err := s.queryRows(`
SELECT DBMS_XMLGEN.CONVERT(
XMLAGG(XMLELEMENT(E, TEXT) ORDER BY LINE).EXTRACT('//text()').GETCLOBVAL(),
1
)
FROM ALL_SOURCE
WHERE OWNER = :1 AND NAME = :2 AND TYPE = :3`, []any{schema, name, objectType})
if err != nil {
return "", false, err
}
defer s.closeRows(rows)
if !rows.Next() {
return "", false, rows.Err()
}
var source sql.NullString
if err := rows.Scan(&source); err != nil {
return "", false, err
}
if err := rows.Err(); err != nil {
return "", false, err
}
return source.String, source.Valid, nil
}
func (s *server) loadObjectSourceTextRows(schema, name, objectType string) (string, bool, error) {
rows, err := s.queryRows(`
SELECT TEXT
FROM ALL_SOURCE
@@ -3552,6 +3675,9 @@ func restoreOracleCurrentSchema(conn *sql.Conn, schema string) {
}
func (s *server) executeTransaction(params map[string]json.RawMessage) (queryResult, error) {
if s.hasManualTransaction() {
return queryResult{}, errors.New("cannot start a one-shot transaction while a manual transaction is open")
}
var payload struct {
Statements []string `json:"statements"`
Schema string `json:"schema"`
@@ -3601,7 +3727,7 @@ func (s *server) executeTransaction(params map[string]json.RawMessage) (queryRes
func (s *server) executeQueryPage(opts queryOptions, pageSize int) (queryPageResult, error) {
start := time.Now()
if strings.TrimSpace(opts.Schema) != "" {
if strings.TrimSpace(opts.Schema) != "" && !s.hasManualTransaction() {
if err := s.setSchema(opts.Schema); err != nil {
return queryPageResult{}, err
}
@@ -3677,7 +3803,7 @@ func (s *server) storeQuerySession(session *querySession) string {
func (s *server) startTableRead(opts queryOptions, pageSize int) (queryPageResult, error) {
start := time.Now()
if strings.TrimSpace(opts.Schema) != "" {
if strings.TrimSpace(opts.Schema) != "" && !s.hasManualTransaction() {
if err := s.setSchema(opts.Schema); err != nil {
return queryPageResult{}, err
}
@@ -3810,7 +3936,7 @@ func readQuerySessionPage(session *querySession, pageSize int) (queryPageResult,
func (s *server) executeQuery(opts queryOptions) (queryResult, error) {
start := time.Now()
if strings.TrimSpace(opts.Schema) != "" {
if strings.TrimSpace(opts.Schema) != "" && !s.hasManualTransaction() {
if err := s.setSchema(opts.Schema); err != nil {
return queryResult{}, err
}
@@ -3825,8 +3951,7 @@ func (s *server) executeQuery(opts queryOptions) (queryResult, error) {
result.ExecutionTimeMS = time.Since(start).Milliseconds()
return result, err
}
db, err := s.requireDB()
if err != nil {
if _, err := s.requireDB(); err != nil {
return queryResult{}, err
}
ctx, cancel := context.WithCancel(context.Background())
@@ -3856,7 +3981,7 @@ func (s *server) executeQuery(opts queryOptions) (queryResult, error) {
}
s.activeCancelMu.Unlock()
}()
execResult, err := db.ExecContext(ctx, sqlText)
execResult, err := s.execContext(ctx, sqlText)
if err != nil {
return queryResult{}, err
}
@@ -3864,6 +3989,17 @@ func (s *server) executeQuery(opts queryOptions) (queryResult, error) {
return queryResult{Columns: []string{}, ColumnTypes: []string{}, Rows: [][]any{}, AffectedRows: affected, ExecutionTimeMS: time.Since(start).Milliseconds()}, nil
}
func (s *server) execContext(ctx context.Context, sqlText string) (sql.Result, error) {
if s.manualTx != nil {
return s.manualTx.ExecContext(ctx, sqlText)
}
db, err := s.requireDB()
if err != nil {
return nil, err
}
return db.ExecContext(ctx, sqlText)
}
func (s *server) executeSelect(sqlText string, maxRows int, timeoutSecs int, deferLOBs bool) (queryResult, error) {
return executeOracleSelectWithXMLTypeRetry(
sqlText,
@@ -4286,7 +4422,7 @@ func rewriteOracleSelectItems(items []string, columns []oracleColumnMeta, tableR
for _, column := range columns {
columnRef := oracleColumnRef(tableRef.AliasText, column.Name)
outputAlias := quoteIdentifier(column.Name)
if isOracleXMLType(column.DataType) {
if isOracleXMLType(column.DataType) && !deferLOBs {
rewritten = append(rewritten, oracleXMLSerializeExpression(columnRef, outputAlias))
} else if deferLOBs {
if expressions, ok := oracleDeferredLOBExpressions(columnRef, outputAlias, sourceIndex, column.DataType); ok {
@@ -4310,7 +4446,7 @@ func rewriteOracleSelectItems(items []string, columns []oracleColumnMeta, tableR
outputAlias = quoteIdentifier(meta.Name)
}
columnRef := oracleColumnRef(qualifier, meta.Name)
if isOracleXMLType(meta.DataType) {
if isOracleXMLType(meta.DataType) && !deferLOBs {
rewritten = append(rewritten, oracleXMLSerializeExpression(columnRef, outputAlias))
changed = true
sourceIndex++
@@ -4353,6 +4489,8 @@ func oracleDeferredLOBKind(dataType string) (kind, placeholder string, ok bool)
return "L", "<BLOB>", true
case "BFILE":
return "F", "<BFILE>", true
case "XMLTYPE", "SYS.XMLTYPE":
return "C", "<XMLTYPE>", true
default:
return "", "", false
}
@@ -4725,11 +4863,20 @@ func skipBlockCommentSQL(value string, pos int) int {
}
func (s *server) setSchema(schema string) error {
sqlText := "ALTER SESSION SET CURRENT_SCHEMA = " + quoteIdentifier(schema)
if s.manualTx != nil {
_, err := s.manualTx.Exec(sqlText)
return err
}
if s.manualConn != nil {
_, err := s.manualConn.ExecContext(context.Background(), sqlText)
return err
}
db, err := s.requireDB()
if err != nil {
return err
}
_, err = db.Exec("ALTER SESSION SET CURRENT_SCHEMA = " + quoteIdentifier(schema))
_, err = db.Exec(sqlText)
return err
}
@@ -4738,8 +4885,7 @@ func (s *server) queryRows(sqlText string, args []any) (*sql.Rows, error) {
}
func (s *server) queryRowsWithTimeout(sqlText string, args []any, timeoutSecs int) (*sql.Rows, error) {
db, err := s.requireDB()
if err != nil {
if _, err := s.requireDB(); err != nil {
return nil, err
}
ctx, cancel := context.WithCancel(context.Background())
@@ -4761,7 +4907,18 @@ func (s *server) queryRowsWithTimeout(sqlText string, args []any, timeoutSecs in
s.activeTimer = timer
s.activeTimedOut = false
s.activeCancelMu.Unlock()
rows, queryErr := db.QueryContext(ctx, sqlText, args...)
var rows *sql.Rows
var queryErr error
if s.manualTx != nil {
rows, queryErr = s.manualTx.QueryContext(ctx, sqlText, args...)
} else {
db, err := s.requireDB()
if err != nil {
cancel()
return nil, err
}
rows, queryErr = db.QueryContext(ctx, sqlText, args...)
}
s.activeCancelMu.Lock()
s.activeCancel = nil
if s.activeTimer != nil {
+467 -12
View File
@@ -12,6 +12,7 @@ import (
"os"
"reflect"
"strings"
"sync"
"testing"
"time"
@@ -1733,9 +1734,12 @@ func TestResolveOracleSchemaFallsBackToSessionUser(t *testing.T) {
func TestListTablesSQLUsesSplitDictionaryQuery(t *testing.T) {
sqlText := strings.ToUpper(oracleListTablesSQL)
if !strings.Contains(sqlText, "ALL_TABLES") || !strings.Contains(sqlText, "ALL_OBJECTS") {
if !strings.Contains(sqlText, "ALL_TABLES") || !strings.Contains(sqlText, "ALL_VIEWS") || !strings.Contains(sqlText, "ALL_MVIEWS") {
t.Fatalf("table listing should split tables and views, got: %s", oracleListTablesSQL)
}
if strings.Contains(sqlText, "ALL_OBJECTS") {
t.Fatalf("table listing should avoid sorting the expensive ALL_OBJECTS view, got: %s", oracleListTablesSQL)
}
if !strings.Contains(sqlText, "UNION ALL") {
t.Fatalf("table listing should union table and view metadata, got: %s", oracleListTablesSQL)
}
@@ -1750,26 +1754,32 @@ func TestOracleMetadataQueriesClassifyMaterializedViews(t *testing.T) {
sqlText string
tableView string
objectView string
objectName string
objectType bool
ownerMatch string
}{
{
name: "all list tables",
sqlText: oracleListTablesBaseSQL,
tableView: "ALL_TABLES",
objectView: "ALL_OBJECTS",
objectView: "ALL_MVIEWS",
objectName: "MV.MVIEW_NAME",
ownerMatch: "MV.OWNER = T.OWNER",
},
{
name: "user list tables",
sqlText: oracleListTablesSessionUserBaseSQL,
tableView: "USER_TABLES",
objectView: "USER_OBJECTS",
objectView: "USER_MVIEWS",
objectName: "MV.MVIEW_NAME",
},
{
name: "all list objects",
sqlText: oracleListObjectsBaseSQL,
tableView: "ALL_TABLES",
objectView: "ALL_OBJECTS",
objectName: "MV.OBJECT_NAME",
objectType: true,
ownerMatch: "MV.OWNER = T.OWNER",
},
{
@@ -1777,6 +1787,8 @@ func TestOracleMetadataQueriesClassifyMaterializedViews(t *testing.T) {
sqlText: oracleListObjectsSessionUserBaseSQL,
tableView: "USER_TABLES",
objectView: "USER_OBJECTS",
objectName: "MV.OBJECT_NAME",
objectType: true,
},
}
@@ -1786,15 +1798,21 @@ func TestOracleMetadataQueriesClassifyMaterializedViews(t *testing.T) {
if !strings.Contains(sqlText, "FROM "+test.tableView+" T") || !strings.Contains(sqlText, "FROM "+test.objectView+" MV") {
t.Fatalf("metadata query should classify tables against materialized-view objects: %s", test.sqlText)
}
if !strings.Contains(sqlText, "NOT EXISTS") || !strings.Contains(sqlText, "MV.OBJECT_NAME = T.TABLE_NAME") || !strings.Contains(sqlText, "MV.OBJECT_TYPE = 'MATERIALIZED VIEW'") {
if !strings.Contains(sqlText, "NOT EXISTS") || !strings.Contains(sqlText, test.objectName+" = T.TABLE_NAME") {
t.Fatalf("metadata query should exclude materialized-view storage tables: %s", test.sqlText)
}
if test.objectType && !strings.Contains(sqlText, "MV.OBJECT_TYPE = 'MATERIALIZED VIEW'") {
t.Fatalf("object query should identify materialized views by object type: %s", test.sqlText)
}
if test.ownerMatch != "" && !strings.Contains(sqlText, test.ownerMatch) {
t.Fatalf("cross-schema metadata query should keep materialized-view classification owner-scoped: %s", test.sqlText)
}
if !strings.Contains(sqlText, "'MATERIALIZED_VIEW'") || !strings.Contains(sqlText, "'MATERIALIZED VIEW'") {
if !strings.Contains(sqlText, "'MATERIALIZED_VIEW'") {
t.Fatalf("metadata query should return the normalized materialized-view type: %s", test.sqlText)
}
if test.objectType && !strings.Contains(sqlText, "'MATERIALIZED VIEW'") {
t.Fatalf("object query should select materialized-view objects: %s", test.sqlText)
}
})
}
}
@@ -1808,19 +1826,19 @@ func TestListTablesQueryAppliesMetadataConstraints(t *testing.T) {
})
sqlText := strings.ToUpper(query.SQL)
if !strings.Contains(sqlText, "UPPER(OBJECT_NAME) LIKE :3 ESCAPE '\\'") {
if !strings.Contains(sqlText, "UPPER(OBJECT_NAME) LIKE :4 ESCAPE '\\'") {
t.Fatalf("table listing should push filter predicate, got: %s", query.SQL)
}
if !strings.Contains(sqlText, "TABLE_TYPE IN (:4,:5)") {
if !strings.Contains(sqlText, "TABLE_TYPE IN (:5,:6)") {
t.Fatalf("table listing should push table type predicate, got: %s", query.SQL)
}
if !strings.Contains(sqlText, "ROWNUM <= :6") || !strings.Contains(sqlText, "DBX_RN > :7") {
if !strings.Contains(sqlText, "ROWNUM <= :7") || !strings.Contains(sqlText, "DBX_RN > :8") {
t.Fatalf("table listing should use rownum pagination, got: %s", query.SQL)
}
if len(query.Args) != 7 {
if len(query.Args) != 8 {
t.Fatalf("unexpected args: %#v", query.Args)
}
if query.Args[0] != "APP" || query.Args[1] != "APP" || query.Args[2] != "%U%\\_%R%" || query.Args[3] != "TABLE" || query.Args[4] != "VIEW" || query.Args[5] != 511 || query.Args[6] != 10 {
if query.Args[0] != "APP" || query.Args[1] != "APP" || query.Args[2] != "APP" || query.Args[3] != "%U%\\_%R%" || query.Args[4] != "TABLE" || query.Args[5] != "VIEW" || query.Args[6] != 511 || query.Args[7] != 10 {
t.Fatalf("constraints args were not normalized: %#v", query.Args)
}
}
@@ -1834,10 +1852,10 @@ func TestListSessionUserTablesQueryUsesUserDictionary(t *testing.T) {
})
sqlText := strings.ToUpper(query.SQL)
if !strings.Contains(sqlText, "USER_TABLES") || !strings.Contains(sqlText, "USER_OBJECTS") {
if !strings.Contains(sqlText, "USER_TABLES") || !strings.Contains(sqlText, "USER_VIEWS") || !strings.Contains(sqlText, "USER_MVIEWS") {
t.Fatalf("session-user table listing should use USER_* dictionaries, got: %s", query.SQL)
}
if strings.Contains(sqlText, "ALL_TABLES") || strings.Contains(sqlText, "ALL_OBJECTS") {
if strings.Contains(sqlText, "ALL_TABLES") || strings.Contains(sqlText, "ALL_VIEWS") || strings.Contains(sqlText, "ALL_MVIEWS") || strings.Contains(sqlText, "USER_OBJECTS") {
t.Fatalf("session-user table listing should avoid ALL_* dictionaries, got: %s", query.SQL)
}
if strings.Contains(sqlText, "OWNER =") {
@@ -1875,11 +1893,21 @@ func TestListObjectsSQLUsesSplitDictionaryQuery(t *testing.T) {
if !strings.Contains(sqlText, "'PACKAGE BODY'") || !strings.Contains(sqlText, "PACKAGE_BODY") {
t.Fatalf("object listing should include package bodies with normalized type, got: %s", oracleListObjectsSQL)
}
if !strings.Contains(sqlText, "'SEQUENCE'") {
t.Fatalf("object listing should include sequences, got: %s", oracleListObjectsSQL)
}
if !strings.Contains(sqlText, "ALL_SYNONYMS") || !strings.Contains(sqlText, "'SYNONYM' AS OBJECT_TYPE") {
t.Fatalf("object listing should include schema synonyms, got: %s", oracleListObjectsSQL)
}
}
func TestListSessionUserObjectsSQLIncludesSequences(t *testing.T) {
sqlText := strings.ToUpper(oracleListObjectsSessionUserBaseSQL)
if !strings.Contains(sqlText, "FROM USER_OBJECTS") || !strings.Contains(sqlText, "'SEQUENCE'") {
t.Fatalf("session-user object listing should include sequences, got: %s", oracleListObjectsSessionUserBaseSQL)
}
}
func TestListObjectsQueryAppliesMetadataConstraints(t *testing.T) {
query := oracleListObjectsQuery("APP", metadataListConstraints{
Filter: "pkg%",
@@ -2287,6 +2315,24 @@ func TestRewriteOracleLOBSelectStarAsDeferredValues(t *testing.T) {
}
}
func TestRewriteOracleXMLTypeAsDeferredValue(t *testing.T) {
sqlText, err := rewriteOracleSelectSQL(
`SELECT t.ID, t.XML_CONTENT FROM TEST_LOBS t WHERE t.ID = 1`,
fakeOracleColumnLoader([]oracleColumnMeta{
{Name: "ID", DataType: "NUMBER"},
{Name: "XML_CONTENT", DataType: "SYS.XMLTYPE"},
}),
true,
)
if err != nil {
t.Fatal(err)
}
want := `SELECT t.ID, CASE WHEN t."XML_CONTENT" IS NULL THEN NULL ELSE '<XMLTYPE>' END AS "XML_CONTENT", CASE WHEN t."XML_CONTENT" IS NULL THEN NULL ELSE 'D:1' END AS "__DBX_LARGE_VALUE_BYTES_C_1" FROM TEST_LOBS t WHERE t.ID = 1`
if sqlText != want {
t.Fatalf("rewriteOracleSelectSQL() = %s, want %s", sqlText, want)
}
}
func TestRewriteOracleLOBExplicitColumnUsesVisibleResultIndex(t *testing.T) {
sqlText, err := rewriteOracleSelectSQL(
`SELECT t.ID, t.PAYLOAD AS body, LENGTH(t.PAYLOAD) AS payload_length FROM TEST_LOBS t`,
@@ -2464,6 +2510,107 @@ func TestGetObjectSourceUsesDBMSMetadataForSynonym(t *testing.T) {
}
}
func TestGetObjectSourceUsesDBMSMetadataForSequence(t *testing.T) {
const ddl = `CREATE SEQUENCE "HR"."ORDER_SEQ" MINVALUE 1 MAXVALUE 9999999999999999999999999999 INCREMENT BY 1 START WITH 42 CACHE 20 NOORDER NOCYCLE NOKEEP NOSCALE GLOBAL`
db, scripted := openOracleViewSourceTestDB(t, []oracleViewSourceQueryStep{
{
queryContains: "DBMS_METADATA.GET_DDL(:1, :2, :3)",
args: []driver.Value{"SEQUENCE", "ORDER_SEQ", "HR"},
rows: [][]driver.Value{{ddl}},
},
})
s := newServer()
s.db = db
result, err := s.getObjectSource("HR", "ORDER_SEQ", "SEQUENCE")
if err != nil {
t.Fatal(err)
}
if result["source"] != ddl || result["object_type"] != "SEQUENCE" {
t.Fatalf("unexpected sequence source: %#v", result)
}
if scripted.next != len(scripted.steps) {
t.Fatalf("expected %d queries, got %d", len(scripted.steps), scripted.next)
}
}
func TestGetObjectSourceAggregatesProcedureSourceInOracle(t *testing.T) {
const source = "PROCEDURE DBX_LARGE_SOURCE AS\nBEGIN\n -- preserve <xml> & special characters\n NULL;\nEND;"
db, scripted := openOracleViewSourceTestDB(t, []oracleViewSourceQueryStep{
{
queryContains: "DBMS_XMLGEN.CONVERT",
args: []driver.Value{"DBX_TEST", "DBX_LARGE_SOURCE", "PROCEDURE"},
rows: [][]driver.Value{{source}},
},
})
s := newServer()
s.db = db
result, err := s.getObjectSource("DBX_TEST", "DBX_LARGE_SOURCE", "PROCEDURE")
if err != nil {
t.Fatal(err)
}
if result["source"] != source || result["object_type"] != "PROCEDURE" {
t.Fatalf("unexpected procedure source: %#v", result)
}
if scripted.next != len(scripted.steps) {
t.Fatalf("expected %d queries, got %d", len(scripted.steps), scripted.next)
}
}
func TestGetObjectSourceReturnsEmptyWhenAggregatedSourceIsNull(t *testing.T) {
db, scripted := openOracleViewSourceTestDB(t, []oracleViewSourceQueryStep{
{
queryContains: "DBMS_XMLGEN.CONVERT",
args: []driver.Value{"DBX_TEST", "MISSING_PROC", "PROCEDURE"},
rows: [][]driver.Value{{nil}},
},
})
s := newServer()
s.db = db
result, err := s.getObjectSource("DBX_TEST", "MISSING_PROC", "PROCEDURE")
if err != nil {
t.Fatal(err)
}
if result["source"] != "" {
t.Fatalf("missing procedure source should be empty: %#v", result)
}
if scripted.next != len(scripted.steps) {
t.Fatalf("expected %d queries, got %d", len(scripted.steps), scripted.next)
}
}
func TestGetObjectSourceFallsBackWhenOracleAggregationFails(t *testing.T) {
const firstLine = "PROCEDURE DBX_SOURCE_FALLBACK AS\n"
const secondLine = "BEGIN NULL; END;"
db, scripted := openOracleViewSourceTestDB(t, []oracleViewSourceQueryStep{
{
queryContains: "DBMS_XMLGEN.CONVERT",
args: []driver.Value{"DBX_TEST", "DBX_SOURCE_FALLBACK", "PROCEDURE"},
err: errors.New("ORA-19011: Character string buffer too small"),
},
{
queryContains: "SELECT TEXT",
args: []driver.Value{"DBX_TEST", "DBX_SOURCE_FALLBACK", "PROCEDURE"},
rows: [][]driver.Value{{firstLine}, {secondLine}},
},
})
s := newServer()
s.db = db
result, err := s.getObjectSource("DBX_TEST", "DBX_SOURCE_FALLBACK", "PROCEDURE")
if err != nil {
t.Fatal(err)
}
if result["source"] != firstLine+secondLine {
t.Fatalf("unexpected fallback procedure source: %#v", result)
}
if scripted.next != len(scripted.steps) {
t.Fatalf("expected %d queries, got %d", len(scripted.steps), scripted.next)
}
}
func TestGetObjectSourcePreservesQuotedSynonymName(t *testing.T) {
const synonym = "MixedSynonym"
const ddl = `CREATE OR REPLACE EDITIONABLE SYNONYM "AP"."MixedSynonym" FOR "ADM"."S_SPECS"`
@@ -2846,3 +2993,311 @@ func TestOracleCursorSurvivesDeadlineWindow(t *testing.T) {
t.Fatalf("expected 3 rows, got %d", rowCount)
}
}
func TestManualTransactionBeginCommitRollback(t *testing.T) {
db, driver := openOracleManualTxTestDB(t)
s := newServer()
s.db = db
if err := s.beginManualTransaction(""); err != nil {
t.Fatalf("begin: %v", err)
}
if !s.hasManualTransaction() {
t.Fatal("expected open manual transaction")
}
if err := s.beginManualTransaction(""); err == nil {
t.Fatal("second begin should fail")
}
result, err := s.executeQuery(queryOptions{SQL: "UPDATE t SET a = 1", MaxRows: 10})
if err != nil {
t.Fatalf("update in txn: %v", err)
}
if result.AffectedRows != 1 {
t.Fatalf("expected 1 affected row, got %d", result.AffectedRows)
}
selectResult, err := s.executeQuery(queryOptions{SQL: "SELECT a FROM t", MaxRows: 10})
if err != nil {
t.Fatalf("select in txn: %v", err)
}
if len(selectResult.Rows) != 1 {
t.Fatalf("expected 1 row, got %d", len(selectResult.Rows))
}
if err := s.commitManualTransaction(); err != nil {
t.Fatalf("commit: %v", err)
}
if s.hasManualTransaction() {
t.Fatal("transaction should be closed after commit")
}
if !driver.committed {
t.Fatal("expected driver commit")
}
if driver.rolledBack {
t.Fatal("commit path should not rollback")
}
if err := s.beginManualTransaction(""); err != nil {
t.Fatalf("re-begin: %v", err)
}
if _, err := s.executeQuery(queryOptions{SQL: "DELETE FROM t", MaxRows: 10}); err != nil {
t.Fatalf("delete in txn: %v", err)
}
if err := s.rollbackManualTransaction(); err != nil {
t.Fatalf("rollback: %v", err)
}
if !driver.rolledBack {
t.Fatal("expected driver rollback")
}
if err := s.commitManualTransaction(); err == nil {
t.Fatal("commit without open txn should fail")
}
if err := s.rollbackManualTransaction(); err == nil {
t.Fatal("rollback without open txn should fail")
}
}
// TestManualTransactionSkipsPerStatementSetSchema guards against Oracle's
// implicit COMMIT before+after DDL (ALTER SESSION SET CURRENT_SCHEMA is DDL).
// Once a manual transaction is open and the schema is pinned at BEGIN time,
// executeQuery/executeQueryPage/startTableRead must NOT re-issue ALTER SESSION
// per statement, otherwise prior DML in the same transaction is silently
// committed and Rollback can no longer undo it.
func TestManualTransactionSkipsPerStatementSetSchema(t *testing.T) {
db, driver := openOracleManualTxTestDB(t)
s := newServer()
s.db = db
if err := s.beginManualTransaction("APP_TEST"); err != nil {
t.Fatalf("begin: %v", err)
}
// Run a couple of statements with a non-empty schema; before the fix each
// call would issue an extra ALTER SESSION SET CURRENT_SCHEMA inside the tx.
if _, err := s.executeQuery(queryOptions{SQL: "UPDATE t SET a = 1", Schema: "APP_TEST", MaxRows: 10}); err != nil {
t.Fatalf("update in txn: %v", err)
}
if _, err := s.executeQuery(queryOptions{SQL: "DELETE FROM t", Schema: "APP_TEST", MaxRows: 10}); err != nil {
t.Fatalf("delete in txn: %v", err)
}
alterSessionCount := 0
for _, q := range driver.execs {
if strings.Contains(strings.ToUpper(q), "ALTER SESSION SET CURRENT_SCHEMA") {
alterSessionCount++
}
}
// Exactly one ALTER SESSION is expected: the one issued by
// beginManualTransaction when pinning the schema. Any additional one would
// mean per-statement setSchema ran inside the open transaction and would
// trigger an implicit COMMIT of the preceding DML.
if alterSessionCount != 1 {
t.Fatalf("expected exactly 1 ALTER SESSION (from begin), got %d; per-statement setSchema must be skipped during manual tx", alterSessionCount)
}
if err := s.rollbackManualTransaction(); err != nil {
t.Fatalf("rollback: %v", err)
}
}
func TestManualTransactionDisconnectRollsBack(t *testing.T) {
db, driver := openOracleManualTxTestDB(t)
s := newServer()
s.db = db
if err := s.beginManualTransaction(""); err != nil {
t.Fatalf("begin: %v", err)
}
if err := s.disconnect(); err != nil {
t.Fatalf("disconnect: %v", err)
}
if s.hasManualTransaction() {
t.Fatal("disconnect should clear manual transaction")
}
if !driver.rolledBack {
t.Fatal("disconnect should rollback open manual transaction")
}
}
func TestManualTransactionBlocksOneShotTransaction(t *testing.T) {
db, _ := openOracleManualTxTestDB(t)
s := newServer()
s.db = db
if err := s.beginManualTransaction(""); err != nil {
t.Fatalf("begin: %v", err)
}
_, err := s.executeTransaction(map[string]json.RawMessage{
"statements": json.RawMessage(`["UPDATE t SET a = 1"]`),
})
if err == nil || !strings.Contains(err.Error(), "manual transaction") {
t.Fatalf("expected manual transaction conflict, got %v", err)
}
}
func TestRuntimeHandshakeAdvertisesTransactionCapability(t *testing.T) {
runtime := newRuntimeServer()
resp, shutdown := runtime.handleLine(`{"jsonrpc":"2.0","id":7,"method":"handshake","params":{"appVersion":"dev"}}`)
if shutdown || resp.Error != nil {
t.Fatalf("unexpected handshake response: shutdown=%v error=%v", shutdown, resp.Error)
}
data, err := json.Marshal(resp.Result)
if err != nil {
t.Fatal(err)
}
var result struct {
Capabilities []string `json:"capabilities"`
}
if err := json.Unmarshal(data, &result); err != nil {
t.Fatal(err)
}
if !contains(result.Capabilities, "transaction") {
t.Fatalf("expected transaction capability, got %v", result.Capabilities)
}
}
func TestManualTransactionRPCDispatch(t *testing.T) {
db, driver := openOracleManualTxTestDB(t)
s := newServer()
s.db = db
resp, _ := s.handleLine(`{"jsonrpc":"2.0","id":1,"method":"begin_manual_transaction","params":{}}`)
if resp.Error != nil {
t.Fatalf("begin rpc: %v", resp.Error)
}
resp, _ = s.handleLine(`{"jsonrpc":"2.0","id":2,"method":"execute_query","params":{"sql":"UPDATE t SET a = 2","maxRows":10}}`)
if resp.Error != nil {
t.Fatalf("execute rpc: %v", resp.Error)
}
resp, _ = s.handleLine(`{"jsonrpc":"2.0","id":3,"method":"commit_manual_transaction","params":{}}`)
if resp.Error != nil {
t.Fatalf("commit rpc: %v", resp.Error)
}
if !driver.committed {
t.Fatal("expected commit via RPC")
}
}
type oracleManualTxDriver struct {
mu sync.Mutex
committed bool
rolledBack bool
execs []string
queries []string
}
func openOracleManualTxTestDB(t *testing.T) (*sql.DB, *oracleManualTxDriver) {
t.Helper()
driverName := "oracle-test-manual-tx-" + strings.ReplaceAll(t.Name(), "/", "-") + "-" + time.Now().Format("150405.000000000")
drv := &oracleManualTxDriver{}
sql.Register(driverName, drv)
db, err := sql.Open(driverName, "")
if err != nil {
t.Fatal(err)
}
db.SetMaxOpenConns(1)
t.Cleanup(func() {
_ = db.Close()
})
return db, drv
}
func (d *oracleManualTxDriver) Open(string) (driver.Conn, error) {
return &oracleManualTxConn{driver: d}, nil
}
type oracleManualTxConn struct {
driver *oracleManualTxDriver
closed bool
}
func (c *oracleManualTxConn) Prepare(string) (driver.Stmt, error) {
return nil, errors.New("use Query/Exec context APIs")
}
func (c *oracleManualTxConn) Close() error {
c.closed = true
return nil
}
func (c *oracleManualTxConn) Begin() (driver.Tx, error) {
return c.BeginTx(context.Background(), driver.TxOptions{})
}
func (c *oracleManualTxConn) BeginTx(context.Context, driver.TxOptions) (driver.Tx, error) {
c.driver.mu.Lock()
defer c.driver.mu.Unlock()
c.driver.committed = false
c.driver.rolledBack = false
return &oracleManualTx{driver: c.driver}, nil
}
var (
_ driver.ConnBeginTx = (*oracleManualTxConn)(nil)
_ driver.ExecerContext = (*oracleManualTxConn)(nil)
_ driver.QueryerContext = (*oracleManualTxConn)(nil)
)
func (c *oracleManualTxConn) ExecContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Result, error) {
c.driver.mu.Lock()
defer c.driver.mu.Unlock()
c.driver.execs = append(c.driver.execs, query)
return driver.RowsAffected(1), nil
}
func (c *oracleManualTxConn) QueryContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Rows, error) {
c.driver.mu.Lock()
defer c.driver.mu.Unlock()
c.driver.queries = append(c.driver.queries, query)
return &oracleManualTxRows{
columns: []string{"A"},
values: [][]driver.Value{{int64(1)}},
}, nil
}
type oracleManualTx struct {
driver *oracleManualTxDriver
done bool
}
func (t *oracleManualTx) Commit() error {
t.driver.mu.Lock()
defer t.driver.mu.Unlock()
if t.done {
return errors.New("transaction already finished")
}
t.done = true
t.driver.committed = true
return nil
}
func (t *oracleManualTx) Rollback() error {
t.driver.mu.Lock()
defer t.driver.mu.Unlock()
if t.done {
return errors.New("transaction already finished")
}
t.done = true
t.driver.rolledBack = true
return nil
}
// database/sql routes Exec/Query on *sql.Tx through the underlying conn while the
// transaction is open when the driver implements ExecerContext/QueryerContext.
// Keep explicit Tx methods unused; Conn methods above serve sticky TX execution.
type oracleManualTxRows struct {
columns []string
values [][]driver.Value
next int
}
func (r *oracleManualTxRows) Columns() []string { return r.columns }
func (r *oracleManualTxRows) Close() error { return nil }
func (r *oracleManualTxRows) Next(dest []driver.Value) error {
if r.next >= len(r.values) {
return io.EOF
}
copy(dest, r.values[r.next])
r.next++
return nil
}
+1 -1
View File
@@ -1,5 +1,5 @@
module github.com/t8y2/dbx/agents/drivers/rabbitmq
go 1.22
go 1.20
require github.com/rabbitmq/amqp091-go v1.13.0
+229 -2
View File
@@ -273,6 +273,233 @@ func (a *rocketMQAgent) getConsumerLag(params map[string]any) (any, error) {
if err != nil {
return nil, err
}
consumerClients := resolveConsumerClients(ctx, remotingConsumeStatusReader{
client: client,
invoke: invokeRemotingWithClient,
}, groupID, topic)
return buildConsumerLagResult(stats, consumerClients), nil
}
type consumeStatusReader interface {
GetConsumeStatus(context.Context, string, string, string) (map[string]map[string]int64, error)
}
type consumerStatusInvoker func(
context.Context,
string,
*remoting.RemotingCommand,
) (*remoting.RemotingCommand, error)
type remotingConsumeStatusReader struct {
client *admin.Client
invoke consumerStatusInvoker
}
func (r remotingConsumeStatusReader) GetConsumeStatus(
ctx context.Context,
topic string,
groupID string,
clientAddr string,
) (map[string]map[string]int64, error) {
route, err := r.client.ExamineTopicRouteInfo(ctx, topic)
if err != nil {
return nil, err
}
return readConsumerStatusFromBrokers(
ctx, masterAddressesFromRoute(route), topic, groupID, clientAddr, r.invoke,
)
}
func readConsumerStatusFromBrokers(
ctx context.Context,
addresses []string,
topic string,
groupID string,
clientAddr string,
invoke consumerStatusInvoker,
) (map[string]map[string]int64, error) {
if len(addresses) == 0 {
return nil, fmt.Errorf("no RocketMQ master broker found for topic %s", topic)
}
orderedAddresses := append([]string(nil), addresses...)
sort.Strings(orderedAddresses)
merged := make(map[string]map[string]int64)
successCount := 0
var lastErr error
for _, address := range orderedAddresses {
fields := map[string]string{"topic": topic, "group": groupID}
if clientAddr != "" {
fields["clientAddr"] = clientAddr
}
response, requestErr := invoke(ctx, address,
remoting.NewRequest(remoting.InvokeBrokerToGetConsumerStatus, fields))
if requestErr != nil {
lastErr = requestErr
continue
}
if response == nil {
lastErr = fmt.Errorf("empty consumer status response from %s", address)
continue
}
partial, decodeErr := decodeConsumerStatus(response.Body)
if decodeErr != nil {
lastErr = decodeErr
continue
}
successCount++
for _, clientID := range sortedKeys(partial) {
if merged[clientID] == nil {
merged[clientID] = make(map[string]int64)
}
for _, queueKey := range sortedKeys(partial[clientID]) {
merged[clientID][queueKey] = partial[clientID][queueKey]
}
}
}
if successCount == 0 {
return nil, fmt.Errorf("query consumer status for group %s on all masters: %w", groupID, lastErr)
}
return merged, nil
}
func decodeConsumerStatus(body []byte) (map[string]map[string]int64, error) {
// RocketMQ wraps assignments in GetConsumerStatusBody; admin-go v1.1.1
// incorrectly decodes the complete response as the inner table.
var wrapper struct {
ConsumerTable map[string]map[string]int64 `json:"consumerTable"`
}
if err := json.Unmarshal(repairConsumerStatusJSON(body), &wrapper); err != nil {
return nil, fmt.Errorf("decode consumer status: %w", err)
}
if wrapper.ConsumerTable == nil {
wrapper.ConsumerTable = make(map[string]map[string]int64)
}
return wrapper.ConsumerTable, nil
}
func repairConsumerStatusJSON(body []byte) []byte {
repaired := repairRocketMQJSON(body)
result := make([]byte, 0, len(repaired)+64)
for index := 0; index < len(repaired); {
mapStart := index + 1
for mapStart < len(repaired) && isJSONSpace(repaired[mapStart]) {
mapStart++
}
if repaired[index] == ':' && mapStart+1 < len(repaired) &&
repaired[mapStart] == '{' && repaired[mapStart+1] == '{' {
if converted, next, ok := convertObjectKeyedInt64Map(repaired, mapStart); ok {
result = append(result, repaired[index:mapStart]...)
result = append(result, converted...)
index = next
continue
}
}
result = append(result, repaired[index])
index++
}
return result
}
func convertObjectKeyedInt64Map(body []byte, start int) ([]byte, int, bool) {
result := []byte{'{'}
index := start + 1
first := true
for index < len(body) {
for index < len(body) && isJSONSpace(body[index]) {
index++
}
if index >= len(body) {
return nil, start, false
}
if body[index] == '}' {
return append(result, '}'), index + 1, true
}
if body[index] == ',' {
index++
continue
}
if body[index] != '{' {
return nil, start, false
}
keyEnd := matchingBrace(body, index)
if keyEnd < 0 {
return nil, start, false
}
key := body[index : keyEnd+1]
index = keyEnd + 1
for index < len(body) && isJSONSpace(body[index]) {
index++
}
if index >= len(body) || body[index] != ':' {
return nil, start, false
}
index++
for index < len(body) && isJSONSpace(body[index]) {
index++
}
valueStart := index
if index < len(body) && body[index] == '-' {
index++
}
digitStart := index
for index < len(body) && body[index] >= '0' && body[index] <= '9' {
index++
}
if index == digitStart {
return nil, start, false
}
valueEnd := index
for index < len(body) && isJSONSpace(body[index]) {
index++
}
if index >= len(body) || (body[index] != ',' && body[index] != '}') {
return nil, start, false
}
if !first {
result = append(result, ',')
}
first = false
result = append(result, '"')
result = append(result, escapeJSONString(key)...)
result = append(result, '"', ':')
result = append(result, body[valueStart:valueEnd]...)
}
return nil, start, false
}
func resolveConsumerClients(
ctx context.Context,
reader consumeStatusReader,
groupID string,
topic string,
) map[parsedMessageQueue]string {
clients := make(map[parsedMessageQueue]string)
status, err := reader.GetConsumeStatus(ctx, topic, groupID, "")
if err != nil {
return clients
}
clientIDs := sortedKeys(status)
for _, clientID := range clientIDs {
if clientID == "" {
continue
}
for _, key := range sortedKeys(status[clientID]) {
queue := parseMessageQueueKey(key)
if queue.Topic == "" || queue.BrokerName == "" || queue.QueueID < 0 {
continue
}
if _, exists := clients[queue]; !exists {
clients[queue] = clientID
}
}
}
return clients
}
func buildConsumerLagResult(
stats *admin.ConsumeStats,
consumerClients map[parsedMessageQueue]string,
) map[string]any {
partitions := make([]map[string]any, 0, len(stats.OffsetTable))
var totalLag int64
for key, offset := range stats.OffsetTable {
@@ -282,7 +509,7 @@ func (a *rocketMQAgent) getConsumerLag(params map[string]any) (any, error) {
partitions = append(partitions, map[string]any{
"partition": queue.QueueID, "currentOffset": offset.ConsumerOffset,
"endOffset": offset.BrokerOffset, "lag": lag, "brokerName": queue.BrokerName,
"lastTimestamp": offset.LastTimestamp, "consumerClient": "",
"lastTimestamp": offset.LastTimestamp, "consumerClient": consumerClients[queue],
})
}
sort.Slice(partitions, func(i, j int) bool {
@@ -292,7 +519,7 @@ func (a *rocketMQAgent) getConsumerLag(params map[string]any) (any, error) {
}
return left["partition"].(int) < right["partition"].(int)
})
return map[string]any{"partitions": partitions, "totalLag": totalLag}, nil
return map[string]any{"partitions": partitions, "totalLag": totalLag}
}
func (a *rocketMQAgent) examineConsumeStatsByTopic(
+232
View File
@@ -1,11 +1,39 @@
package main
import (
"context"
"encoding/json"
"errors"
"reflect"
"strings"
"testing"
admin "github.com/amigoer/rocketmq-admin-go"
"github.com/amigoer/rocketmq-admin-go/protocol/remoting"
)
type fakeConsumeStatusReader struct {
calls int
topic string
groupID string
clientAddr string
status map[string]map[string]int64
err error
}
func (f *fakeConsumeStatusReader) GetConsumeStatus(
_ context.Context,
topic string,
groupID string,
clientAddr string,
) (map[string]map[string]int64, error) {
f.calls++
f.topic = topic
f.groupID = groupID
f.clientAddr = clientAddr
return f.status, f.err
}
func TestClassifyConsumerGroup(t *testing.T) {
if got := classifyConsumerGroup("TOOLS_CONSUMER", nil); got != "SYSTEM" {
t.Fatalf("system group classified as %q", got)
@@ -41,6 +69,210 @@ func TestDecodeConsumeStatsRepairsObjectKeys(t *testing.T) {
}
}
func TestDecodeConsumerStatusUnwrapsRocketMQ48Body(t *testing.T) {
body := []byte(`{
"messageQueueTable": {},
"consumerTable": {
"client-b": {{"brokerName":"broker-a","queueId":0,"topic":"Orders"}:90},
"client-a": {
"MessageQueue [topic=Orders, brokerName=broker-a, queueId=1]": 50
}
}
}`)
status, err := decodeConsumerStatus(body)
if err != nil {
t.Fatal(err)
}
want := map[string]map[string]int64{
"client-b": {
`{"brokerName":"broker-a","queueId":0,"topic":"Orders"}`: 90,
},
"client-a": {
`MessageQueue [topic=Orders, brokerName=broker-a, queueId=1]`: 50,
},
}
if !reflect.DeepEqual(status, want) {
t.Fatalf("consumer status = %#v, want %#v", status, want)
}
}
func TestDecodeConsumerStatusHandlesEmptyAndMalformedBodies(t *testing.T) {
status, err := decodeConsumerStatus([]byte(`{"messageQueueTable":{},"consumerTable":{}}`))
if err != nil {
t.Fatal(err)
}
if len(status) != 0 {
t.Fatalf("empty consumer table decoded as %#v", status)
}
if _, err := decodeConsumerStatus([]byte(`{"consumerTable":[]}`)); err == nil {
t.Fatal("expected malformed consumer table to fail")
}
}
func TestReadConsumerStatusFromBrokersUsesBoundedRequestsAndMerges(t *testing.T) {
responses := map[string][]byte{
"broker-a:10911": []byte(`{"consumerTable":{"client-a":{"{\"topic\":\"Orders\",\"brokerName\":\"broker-a\",\"queueId\":0}":90,"{\"topic\":\"Orders\",\"brokerName\":\"broker-a\",\"queueId\":1}":50}}}`),
"broker-b:10911": []byte(`{"consumerTable":{"client-a":{"MessageQueue [topic=Orders, brokerName=broker-b, queueId=0]":20}}}`),
}
calls := make([]string, 0, len(responses))
invoke := func(_ context.Context, address string, command *remoting.RemotingCommand) (*remoting.RemotingCommand, error) {
calls = append(calls, address)
if command.Code != remoting.InvokeBrokerToGetConsumerStatus {
t.Fatalf("request code = %d, want %d", command.Code, remoting.InvokeBrokerToGetConsumerStatus)
}
if command.ExtFields["topic"] != "Orders" || command.ExtFields["group"] != "GID_Orders" {
t.Fatalf("unexpected request fields: %#v", command.ExtFields)
}
if _, exists := command.ExtFields["clientAddr"]; exists {
t.Fatalf("empty client address sent on wire: %#v", command.ExtFields)
}
return &remoting.RemotingCommand{Code: remoting.Success, Body: responses[address]}, nil
}
status, err := readConsumerStatusFromBrokers(
context.Background(), []string{"broker-b:10911", "broker-a:10911"},
"Orders", "GID_Orders", "", invoke,
)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(calls, []string{"broker-a:10911", "broker-b:10911"}) {
t.Fatalf("broker calls = %#v", calls)
}
if len(status["client-a"]) != 3 {
t.Fatalf("merged status = %#v", status)
}
}
func TestReadConsumerStatusFromBrokersKeepsPartialResults(t *testing.T) {
invoke := func(_ context.Context, address string, _ *remoting.RemotingCommand) (*remoting.RemotingCommand, error) {
if address == "broker-b:10911" {
return nil, errors.New("consumer offline")
}
return &remoting.RemotingCommand{Code: remoting.Success, Body: []byte(
`{"consumerTable":{"client-a":{"{\"topic\":\"Orders\",\"brokerName\":\"broker-a\",\"queueId\":0}":90}}}`,
)}, nil
}
status, err := readConsumerStatusFromBrokers(
context.Background(), []string{"broker-a:10911", "broker-b:10911"},
"Orders", "GID_Orders", "", invoke,
)
if err != nil {
t.Fatal(err)
}
if len(status["client-a"]) != 1 {
t.Fatalf("partial status = %#v", status)
}
_, err = readConsumerStatusFromBrokers(
context.Background(), []string{"broker-b:10911"},
"Orders", "GID_Orders", "", invoke,
)
if err == nil {
t.Fatal("expected all-broker request failure")
}
}
func TestResolveConsumerClientsUsesOneBestEffortLookup(t *testing.T) {
reader := &fakeConsumeStatusReader{status: map[string]map[string]int64{
"client-z": {
`{"topic":"Orders","brokerName":"broker-a","queueId":0}`: 90,
},
"client-a": {
`MessageQueue [topic=Orders, brokerName=broker-a, queueId=0]`: 90,
`{"topic":"Orders","brokerName":"broker-a","queueId":1}`: 50,
"malformed": 1,
},
"client-b": {
`{"topic":"Orders","brokerName":"broker-b","queueId":0}`: 7,
},
"": {
`{"topic":"Orders","brokerName":"broker-b","queueId":1}`: 3,
},
}}
clients := resolveConsumerClients(context.Background(), reader, "GID_Orders", "Orders")
if reader.calls != 1 {
t.Fatalf("consume status calls = %d, want 1", reader.calls)
}
if reader.topic != "Orders" || reader.groupID != "GID_Orders" || reader.clientAddr != "" {
t.Fatalf("unexpected consume status request: %#v", reader)
}
want := map[parsedMessageQueue]string{
{Topic: "Orders", BrokerName: "broker-a", QueueID: 0}: "client-a",
{Topic: "Orders", BrokerName: "broker-a", QueueID: 1}: "client-a",
{Topic: "Orders", BrokerName: "broker-b", QueueID: 0}: "client-b",
}
if !reflect.DeepEqual(clients, want) {
t.Fatalf("consumer clients = %#v, want %#v", clients, want)
}
}
func TestResolveConsumerClientsKeepsLagAvailableOnStatusFailure(t *testing.T) {
reader := &fakeConsumeStatusReader{err: errors.New("consumer offline")}
clients := resolveConsumerClients(context.Background(), reader, "GID_Orders", "Orders")
if reader.calls != 1 {
t.Fatalf("consume status calls = %d, want 1", reader.calls)
}
if len(clients) != 0 {
t.Fatalf("consumer clients after failed lookup = %#v", clients)
}
}
func TestBuildConsumerLagResultIncludesConsumerAssignments(t *testing.T) {
stats := &admin.ConsumeStats{OffsetTable: map[string]*admin.OffsetWrapper{
`{"topic":"Orders","brokerName":"broker-b","queueId":1}`: {
BrokerOffset: 2, ConsumerOffset: 3, LastTimestamp: 40,
},
`{"topic":"Orders","brokerName":"broker-a","queueId":1}`: {
BrokerOffset: 50, ConsumerOffset: 50, LastTimestamp: 20,
},
`{"topic":"Orders","brokerName":"broker-b","queueId":0}`: {
BrokerOffset: 12, ConsumerOffset: 7, LastTimestamp: 30,
},
`{"topic":"Orders","brokerName":"broker-a","queueId":0}`: {
BrokerOffset: 100, ConsumerOffset: 90, LastTimestamp: 10,
},
}}
clients := map[parsedMessageQueue]string{
{Topic: "Orders", BrokerName: "broker-a", QueueID: 0}: "client-a",
{Topic: "Orders", BrokerName: "broker-a", QueueID: 1}: "client-a",
{Topic: "Orders", BrokerName: "broker-b", QueueID: 0}: "client-b",
}
result := buildConsumerLagResult(stats, clients)
if result["totalLag"] != int64(15) {
t.Fatalf("total lag = %#v, want 15", result["totalLag"])
}
want := []map[string]any{
{
"partition": 0, "currentOffset": int64(90), "endOffset": int64(100),
"lag": int64(10), "brokerName": "broker-a", "lastTimestamp": int64(10),
"consumerClient": "client-a",
},
{
"partition": 1, "currentOffset": int64(50), "endOffset": int64(50),
"lag": int64(0), "brokerName": "broker-a", "lastTimestamp": int64(20),
"consumerClient": "client-a",
},
{
"partition": 0, "currentOffset": int64(7), "endOffset": int64(12),
"lag": int64(5), "brokerName": "broker-b", "lastTimestamp": int64(30),
"consumerClient": "client-b",
},
{
"partition": 1, "currentOffset": int64(3), "endOffset": int64(2),
"lag": int64(0), "brokerName": "broker-b", "lastTimestamp": int64(40),
"consumerClient": "",
},
}
if got := result["partitions"]; !reflect.DeepEqual(got, want) {
t.Fatalf("partitions = %#v, want %#v", got, want)
}
}
func TestMutationCoverageFailureMessage(t *testing.T) {
err := ensureMutationCoverage("update", "consumer group GID_Orders", 2, 1, nil)
if err == nil || !strings.Contains(err.Error(), "1 of 2") {
@@ -3,8 +3,12 @@ package com.dbx.agent.saphana;
import com.dbx.agent.ConfiguredJdbcAgent;
import com.dbx.agent.JdbcAgentProfile;
import com.dbx.agent.MultiSessionJsonRpcServer;
import com.dbx.agent.ObjectSource;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.Arrays;
import java.util.Collections;
import java.util.Locale;
public final class SapHanaAgent extends ConfiguredJdbcAgent {
public static final JdbcAgentProfile SAPHANA_PROFILE = new JdbcAgentProfile(
@@ -20,6 +24,30 @@ public final class SapHanaAgent extends ConfiguredJdbcAgent {
super(SAPHANA_PROFILE);
}
@Override
public ObjectSource getObjectSource(String schema, String name, String objectType) {
String normalizedType = objectType == null ? "" : objectType.trim().toUpperCase(Locale.ROOT);
if (!"PROCEDURE".equals(normalizedType)) {
return super.getObjectSource(schema, name, objectType);
}
return unchecked(() -> {
String sql = "SELECT DEFINITION FROM SYS.PROCEDURES WHERE SCHEMA_NAME = ? AND PROCEDURE_NAME = ?";
String source = "";
try (PreparedStatement stmt = requireConnection().prepareStatement(sql)) {
stmt.setString(1, schema);
stmt.setString(2, name);
try (ResultSet rs = stmt.executeQuery()) {
if (rs.next()) {
String definition = rs.getString(1);
source = definition == null ? "" : definition;
}
}
}
return new ObjectSource(name, normalizedType, schema, source);
});
}
public static void main(String[] args) {
new MultiSessionJsonRpcServer(SapHanaAgent::new).run();
}
@@ -0,0 +1,120 @@
package com.dbx.agent.saphana;
import com.dbx.agent.ObjectSource;
import com.dbx.agent.test.TestSupport;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.ArrayList;
import java.util.List;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
class SapHanaAgentTest {
@Test
void getObjectSourceReadsProcedureDefinitionFromSystemCatalog() {
List<String> calls = new ArrayList<>();
String definition = "CREATE PROCEDURE \"APP\".\"REFRESH_ORDERS\" () AS BEGIN END";
SapHanaAgent agent = agentWithDefinition(calls, definition, true);
ObjectSource source = agent.getObjectSource("APP", "REFRESH_ORDERS", " procedure ");
Assertions.assertEquals("REFRESH_ORDERS", source.getName());
Assertions.assertEquals("PROCEDURE", source.getObject_type());
Assertions.assertEquals("APP", source.getSchema());
Assertions.assertEquals(definition, source.getSource());
Assertions.assertEquals(
List.of(
"sql:SELECT DEFINITION FROM SYS.PROCEDURES WHERE SCHEMA_NAME = ? AND PROCEDURE_NAME = ?",
"param:1=APP",
"param:2=REFRESH_ORDERS"
),
calls
);
}
@Test
void getObjectSourceReturnsEmptyTextWhenHanaHidesProcedureDefinition() {
SapHanaAgent agent = agentWithDefinition(new ArrayList<>(), null, true);
ObjectSource source = agent.getObjectSource("SYS", "BACKUP_DATA", "PROCEDURE");
Assertions.assertEquals("", source.getSource());
}
@Test
void getObjectSourceKeepsOtherObjectTypesUnsupported() {
SapHanaAgent agent = new SapHanaAgent();
UnsupportedOperationException error = Assertions.assertThrows(
UnsupportedOperationException.class,
() -> agent.getObjectSource("APP", "ACTIVE_ORDERS", "VIEW")
);
Assertions.assertEquals("Object source is not supported", error.getMessage());
}
private static SapHanaAgent agentWithDefinition(List<String> calls, String definition, boolean hasRow) {
ResultSet resultSet = proxy(ResultSet.class, (method, args) -> {
if ("next".equals(method.getName())) {
return hasRow;
}
if ("getString".equals(method.getName())) {
return definition;
}
return defaultValue(method.getReturnType());
});
PreparedStatement statement = proxy(PreparedStatement.class, (method, args) -> {
if ("setString".equals(method.getName())) {
calls.add("param:" + args[0] + "=" + args[1]);
return null;
}
if ("executeQuery".equals(method.getName())) {
return resultSet;
}
return defaultValue(method.getReturnType());
});
Connection connection = proxy(Connection.class, (method, args) -> {
if ("prepareStatement".equals(method.getName())) {
calls.add("sql:" + args[0]);
return statement;
}
if ("isClosed".equals(method.getName())) {
return false;
}
return defaultValue(method.getReturnType());
});
SapHanaAgent agent = new SapHanaAgent();
TestSupport.setPrivateConnection(agent, connection);
return agent;
}
private static <T> T proxy(Class<T> type, Handler handler) {
return type.cast(Proxy.newProxyInstance(
type.getClassLoader(),
new Class<?>[]{type},
(proxy, method, args) -> handler.handle(method, args)
));
}
private static Object defaultValue(Class<?> type) {
if (!type.isPrimitive()) return null;
if (type == boolean.class) return false;
if (type == byte.class) return (byte) 0;
if (type == short.class) return (short) 0;
if (type == int.class) return 0;
if (type == long.class) return 0L;
if (type == float.class) return 0f;
if (type == double.class) return 0d;
if (type == char.class) return '\0';
return null;
}
@FunctionalInterface
private interface Handler {
Object handle(Method method, Object[] args) throws Throwable;
}
}
+31
View File
@@ -0,0 +1,31 @@
dependencies {
// Google publishes an officially shaded fat jar for the Spanner JDBC driver. It already bundles
// gRPC/protobuf/netty with relocated packages, which keeps shadowJar free of the relocation and
// META-INF/services conflicts a plain dependency tree would introduce. The transitive tree is
// dropped because the bundle already contains it, and resolving it would both duplicate every
// shaded class and upgrade the Gson version the shared agent runtime is built against.
implementation('com.google.cloud:google-cloud-spanner-jdbc:2.42.0:single-jar-with-dependencies') {
transitive = false
}
}
tasks.named('shadowJar') {
dependencies {
exclude(dependency('com.google.code.gson:gson'))
exclude(dependency('com.google.errorprone:error_prone_annotations'))
}
// The published fat jar still carries stale BouncyCastle signature files (META-INF/BC2048KE.SF
// and its .DSA). Their digests no longer match the rewritten manifest, so keeping them makes the
// JVM fail with "SecurityException: Invalid signature file digest for Manifest main attributes".
exclude 'META-INF/*.SF'
exclude 'META-INF/*.DSA'
exclude 'META-INF/*.RSA'
exclude 'META-INF/*.EC'
mergeServiceFiles()
manifest {
attributes(
'Agent-Label': 'Google Cloud Spanner',
'Main-Class': 'com.dbx.agent.spanner.SpannerAgent'
)
}
}
@@ -0,0 +1,819 @@
package com.dbx.agent.spanner;
import com.dbx.agent.ColumnInfo;
import com.dbx.agent.CompletionAssistantCandidate;
import com.dbx.agent.CompletionAssistantCandidateKind;
import com.dbx.agent.CompletionAssistantMatchMode;
import com.dbx.agent.CompletionAssistantObjectKind;
import com.dbx.agent.CompletionAssistantRequest;
import com.dbx.agent.CompletionAssistantResponse;
import com.dbx.agent.ConfiguredJdbcAgent;
import com.dbx.agent.ConnectParams;
import com.dbx.agent.DatabaseInfo;
import com.dbx.agent.DdlBuilder;
import com.dbx.agent.ForeignKeyInfo;
import com.dbx.agent.IndexInfo;
import com.dbx.agent.JdbcAgentProfile;
import com.dbx.agent.MetadataListConstraints;
import com.dbx.agent.MultiSessionJsonRpcServer;
import com.dbx.agent.ObjectInfo;
import com.dbx.agent.StandardJdbcMetadata;
import com.dbx.agent.TableInfo;
import com.google.cloud.spanner.Database;
import com.google.cloud.spanner.DatabaseId;
import com.google.cloud.spanner.Dialect;
import com.google.cloud.spanner.jdbc.CloudSpannerJdbcConnection;
import java.sql.Array;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Types;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
/**
* Google Cloud Spanner agent.
*
* <p>The official driver already ships dialect-aware {@code DatabaseMetaData} implementations
* (separate SQL resources for GoogleSQL and the PostgreSQL dialect), so no INFORMATION_SCHEMA SQL is
* written here. What the shared {@code StandardJdbcMetadata} adapter cannot express is Spanner's
* schema model: in GoogleSQL the user schema name <em>is</em> the empty string, while the shared
* adapter maps blank schemas to {@code null} ("unspecified") and therefore leaks the 28+
* INFORMATION_SCHEMA views into every listing. The thin adapter below passes the schema through
* verbatim and fixes three field-level mismatches: the primary key index is named
* {@code PRIMARY_KEY}, {@code ORDINAL_POSITION} is NULL for STORING/INCLUDE index columns, and
* GoogleSQL reports INTERLEAVE parent links as unnamed foreign keys.
*/
public final class SpannerAgent extends ConfiguredJdbcAgent {
/** GoogleSQL exposes user objects under the empty schema name. */
private static final String GOOGLE_SQL_SCHEMA = "";
/** The PostgreSQL dialect always reports {@code public}; used only if the driver stops telling us. */
private static final String POSTGRES_FALLBACK_SCHEMA = "public";
private static final String PRIMARY_KEY_INDEX = "PRIMARY_KEY";
private static final String URL_PREFIX = "jdbc:cloudspanner:";
/** A loopback endpoint is always the local emulator: the real service is addressed host-less. */
private static final Set<String> LOOPBACK_HOSTS =
new HashSet<>(Arrays.asList("localhost", "127.0.0.1", "::1", "[::1]"));
private static final String PLAIN_TEXT_PARAM = "usePlainText=true";
/**
* Works verbatim in both dialects: the PostgreSQL dialect folds the unquoted upper-case
* identifiers to lower case, so one statement serves GoogleSQL and PostgreSQL alike.
*/
private static final String SCHEMATA_SQL = "SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA";
/**
* The driver reports GoogleSQL spellings from {@code getTypeInfo()} for both dialects (only
* JSON/JSONB differs), so the PostgreSQL-dialect names are substituted here. Every entry was
* read back from {@code information_schema.columns} on a PostgreSQL-dialect database rather than
* transcribed from documentation.
*/
private static final Map<String, String> POSTGRES_DIALECT_TYPE_NAMES = postgresDialectTypeNames();
private static Map<String, String> postgresDialectTypeNames() {
Map<String, String> names = new LinkedHashMap<>();
names.put("STRING", "character varying");
names.put("INT64", "bigint");
names.put("BYTES", "bytea");
names.put("FLOAT32", "real");
names.put("FLOAT64", "double precision");
names.put("BOOL", "boolean");
names.put("DATE", "date");
names.put("TIMESTAMP", "timestamp with time zone");
names.put("NUMERIC", "numeric");
names.put("UUID", "uuid");
// The driver already answers JSONB for the PostgreSQL dialect, but in upper case.
names.put("JSON", "jsonb");
names.put("JSONB", "jsonb");
return Collections.unmodifiableMap(names);
}
private static final int DEFAULT_COMPLETION_LIMIT = 100;
private static final int MAX_COMPLETION_LIMIT = 1000;
public static final JdbcAgentProfile SPANNER_PROFILE = new JdbcAgentProfile(
"com.google.cloud.spanner.jdbc.JdbcDriver",
URL_PREFIX + "//{host}:{port}/{database}",
443,
// Spanner has no SQL-level schema switch statement; SET SCHEMA and set search_path are both
// rejected as unknown statements, so setSchemaSQL must resolve to an empty string.
true,
new HashSet<>(Arrays.asList("INFORMATION_SCHEMA", "SPANNER_SYS", "PG_CATALOG")),
Arrays.asList("TABLE", "VIEW"),
"`",
"SET SCHEMA",
// The catalog is the empty string in GoogleSQL and the database name in the PostgreSQL
// dialect; retrying metadata with it only adds unrelated rows.
false,
false,
false,
false
);
private volatile boolean postgresDialect;
private volatile String defaultSchema = GOOGLE_SQL_SCHEMA;
public SpannerAgent() {
super(SPANNER_PROFILE);
}
@Override
protected String buildJdbcUrl(ConnectParams params) {
return spannerUrl(params);
}
@Override
protected void afterConnect(ConnectParams params, Connection connection) {
super.afterConnect(params, connection);
detectDialect(connection);
}
@Override
public String getIdentifierQuote() {
return postgresDialect ? "\"" : "`";
}
@Override
public List<DatabaseInfo> listDatabases() {
// JDBC alone cannot enumerate an instance (and in the PostgreSQL dialect getCatalogs() plus
// the configured resource path would duplicate the current entry), but the bundled driver
// ships the Admin API and the connection already carries the credentials, so ask it.
List<DatabaseInfo> databases = adminListDatabases();
return databases.isEmpty() ? Collections.singletonList(new DatabaseInfo(configuredDatabaseName())) : databases;
}
/**
* Lists the sibling databases through {@code DatabaseAdminClient}, which the JDBC connection
* exposes via {@code getSpanner()}. Returns an empty list — never throws — when the caller lacks
* {@code spanner.databases.list}, when the driver is loaded from a classloader that cannot see
* the admin classes, or when anything else about the unwrap fails; {@link #listDatabases()} then
* falls back to the connected database alone, which is what shipped before.
*/
private List<DatabaseInfo> adminListDatabases() {
try {
CloudSpannerJdbcConnection connection = requireConnection().unwrap(CloudSpannerJdbcConnection.class);
DatabaseId databaseId = connection.getDatabaseId();
String instance = databaseId.getInstanceId().getInstance();
List<DatabaseInfo> result = new ArrayList<>();
for (Database database : connection.getSpanner().getDatabaseAdminClient().listDatabases(instance).iterateAll()) {
// getName(), not getDatabase(): the reported name round-trips as
// ConnectParams.database, and only the full resource path builds a valid JDBC URL.
// The sidebar shortens it for display through `spannerDisplayDatabase`.
result.add(new DatabaseInfo(database.getId().getName()));
}
result.sort(Comparator.comparing(DatabaseInfo::getName));
return result;
} catch (Throwable ignored) {
return Collections.emptyList();
}
}
@Override
public List<String> listDataTypes() {
// The driver reports GoogleSQL spellings for both dialects except JSON/JSONB, so a
// PostgreSQL-dialect database would otherwise offer INT64/STRING/BOOL in the schema-diff
// field mapping. Substitute the PostgreSQL names for the types the driver enumerates.
List<String> types = super.listDataTypes();
if (!postgresDialect) {
return types;
}
List<String> result = new ArrayList<>(types.size());
for (String type : types) {
result.add(POSTGRES_DIALECT_TYPE_NAMES.getOrDefault(type.toUpperCase(Locale.ROOT), type));
}
return result;
}
@Override
public String getTableDdl(String schema, String table) {
String resolved = resolveSchema(schema);
List<IndexInfo> indexes;
try {
indexes = listIndexes(resolved, table);
} catch (RuntimeException e) {
indexes = Collections.emptyList();
}
List<ForeignKeyInfo> foreignKeys;
try {
foreignKeys = listForeignKeys(resolved, table);
} catch (RuntimeException e) {
foreignKeys = Collections.emptyList();
}
String tableComment = null;
try {
tableComment = getTableComment(resolved, table);
} catch (RuntimeException e) {
// Optional; DDL generation should still succeed without it.
}
// Same as the inherited implementation except for the quote: it hardcodes ANSI double quotes,
// which GoogleSQL reads as a string literal rather than an identifier.
return DdlBuilder.buildTableDdl(
resolved,
table,
getColumns(resolved, table),
indexes,
foreignKeys,
Collections.emptyList(),
!postgresDialect,
false,
tableComment
);
}
@Override
public List<String> listSchemas() {
// Not getSchemas(): the shared adapter drops blank names, and GoogleSQL's user schema *is*
// the empty string, so the default schema would disappear and take the object tree with it.
// Query the catalog directly instead, the way H2, DB2, Trino and Snowflake already do.
Connection connection = requireConnection();
Set<String> names = new LinkedHashSet<>();
names.add(defaultSchema);
List<String> named = new ArrayList<>();
try (java.sql.Statement statement = connection.createStatement();
ResultSet rs = statement.executeQuery(SCHEMATA_SQL)) {
while (rs.next()) {
String name = rs.getString(1);
if (name == null || name.equals(defaultSchema)) {
continue;
}
// Upper-cased before the comparison because the PostgreSQL dialect reports its
// system schemas in lower case.
if (SPANNER_PROFILE.getExcludedSchemas().contains(name.toUpperCase(Locale.ROOT))) {
continue;
}
named.add(name);
}
} catch (Exception ignored) {
// A database that cannot be queried still browses under its default schema.
return new ArrayList<>(names);
}
Collections.sort(named);
names.addAll(named);
return new ArrayList<>(names);
}
@Override
public List<TableInfo> listTables(String schema) {
return listTables(schema, MetadataListConstraints.NONE);
}
@Override
public List<TableInfo> listTables(String schema, List<String> objectTypes) {
return listTables(schema, new MetadataListConstraints(null, null, null, objectTypes));
}
@Override
public List<TableInfo> listTables(String schema, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
if (!normalized.includesTableLikeTypes()) {
return Collections.emptyList();
}
return normalized.filterTables(spannerTables(requireConnection(), resolveSchema(schema)));
}
@Override
public List<ObjectInfo> listObjects(String schema) {
return listObjects(schema, MetadataListConstraints.NONE);
}
@Override
public List<ObjectInfo> listObjects(String schema, MetadataListConstraints constraints) {
MetadataListConstraints normalized = MetadataListConstraints.orNone(constraints);
// Spanner has no stored procedures or functions, so table-like objects are the whole set.
String resolved = resolveSchema(schema);
List<TableInfo> tables = spannerTables(requireConnection(), resolved);
return normalized.filterObjects(StandardJdbcMetadata.INSTANCE.listObjects(tables, resolved));
}
@Override
public List<ColumnInfo> getColumns(String schema, String table) {
return spannerColumns(requireConnection(), resolveSchema(schema), table);
}
@Override
protected Object resultValue(ResultSet rs, int index, int sqlType) {
if (sqlType != Types.ARRAY) {
return super.resultValue(rs, index, sqlType);
}
return unchecked(() -> {
Array array = rs.getArray(index);
if (array == null) {
return null;
}
try {
Object raw = array.getArray();
if (raw == null) {
return null;
}
int length = java.lang.reflect.Array.getLength(raw);
List<Object> values = new ArrayList<>(length);
for (int element = 0; element < length; element++) {
values.add(java.lang.reflect.Array.get(raw, element));
}
return values;
} finally {
try {
array.free();
} catch (SQLException ignored) {
// The value has already been read; cleanup failure must not discard it.
}
}
});
}
@Override
public List<IndexInfo> listIndexes(String schema, String table) {
return spannerIndexes(requireConnection(), resolveSchema(schema), table);
}
@Override
public List<ForeignKeyInfo> listForeignKeys(String schema, String table) {
return spannerForeignKeys(requireConnection(), resolveSchema(schema), table);
}
@Override
public CompletionAssistantResponse completionAssistantSearch(CompletionAssistantRequest request) {
Connection conn = requireConnection();
String schema = resolveSchema(completionSchema(request));
int limit = completionLimit(request.getMax_results());
List<CompletionAssistantObjectKind> kinds = completionKinds(request);
List<CompletionAssistantCandidate> candidates = new ArrayList<>();
if (kinds.contains(CompletionAssistantObjectKind.SCHEMA)
&& !schema.isEmpty()
&& completionMatches(schema, request)) {
candidates.add(new CompletionAssistantCandidate(
schema,
CompletionAssistantCandidateKind.SCHEMA,
request.getDatabase(),
schema,
null,
null,
null,
null
));
}
if (kinds.contains(CompletionAssistantObjectKind.TABLE) || kinds.contains(CompletionAssistantObjectKind.VIEW)) {
for (TableInfo table : spannerTables(conn, schema)) {
if (candidates.size() >= limit) {
break;
}
boolean view = "VIEW".equalsIgnoreCase(table.getTable_type());
CompletionAssistantObjectKind requiredKind = view
? CompletionAssistantObjectKind.VIEW
: CompletionAssistantObjectKind.TABLE;
if (!kinds.contains(requiredKind) || !completionMatches(table.getName(), request)) {
continue;
}
candidates.add(new CompletionAssistantCandidate(
table.getName(),
view ? CompletionAssistantCandidateKind.VIEW : CompletionAssistantCandidateKind.TABLE,
request.getDatabase(),
schema,
null,
null,
table.getComment(),
null
));
}
}
String parentName = request.getParent_name();
if (kinds.contains(CompletionAssistantObjectKind.COLUMN)
&& parentName != null
&& !parentName.trim().isEmpty()) {
for (ColumnInfo column : spannerColumns(conn, schema, parentName)) {
if (candidates.size() >= limit) {
break;
}
if (!completionMatches(column.getName(), request)) {
continue;
}
candidates.add(new CompletionAssistantCandidate(
column.getName(),
CompletionAssistantCandidateKind.COLUMN,
request.getDatabase(),
schema,
schema,
parentName,
column.getComment(),
column.getData_type()
));
}
}
return new CompletionAssistantResponse(candidates, candidates.size() >= limit, false);
}
/**
* Builds the Spanner JDBC URL. The profile template cannot express the endpoint-less form used
* for the real service, where the URL is {@code jdbc:cloudspanner:/<resource path>} without any
* {@code //host:port} authority.
*/
static String spannerUrl(ConnectParams params) {
if (!params.getConnection_string().trim().isEmpty()) {
return params.getConnection_string();
}
String host = trimmed(params.getHost());
StringBuilder url = new StringBuilder(URL_PREFIX);
if (!host.isEmpty()) {
int port = params.getPort() > 0 ? params.getPort() : SPANNER_PROFILE.getDefaultPort();
url.append("//").append(host).append(':').append(port);
}
// params.database carries the full resource path projects/{p}/instances/{i}/databases/{d}.
url.append('/').append(stripLeadingSlashes(trimmed(params.getDatabase())));
return appendUrlParams(url.toString(), emulatorAwareUrlParams(host, params.getUrl_params()));
}
/**
* The emulator speaks plaintext gRPC, but the driver defaults to TLS plus Application Default
* Credentials because nothing in the URL distinguishes an emulator endpoint from a private one.
* Pointing a connection at localhost without a plaintext flag therefore burns the whole connect
* timeout while gRPC retries a TLS handshake against a plaintext port, and reports only
* {@code UNAVAILABLE: io exception} — so a loopback host opts into plaintext by default.
*
* <p>An explicit {@code usePlainText} or {@code autoConfigEmulator} always wins, including
* {@code usePlainText=false}. {@code autoConfigEmulator} implies plaintext but additionally
* creates a missing instance and database, which is a choice only the user should make.
*/
private static String emulatorAwareUrlParams(String host, String urlParams) {
String configured = trimmed(urlParams);
if (!LOOPBACK_HOSTS.contains(host.toLowerCase(Locale.ROOT))) {
return configured;
}
if (hasUrlParam(configured, "usePlainText") || hasUrlParam(configured, "autoConfigEmulator")) {
return configured;
}
return configured.isEmpty() ? PLAIN_TEXT_PARAM : configured + ";" + PLAIN_TEXT_PARAM;
}
/**
* Matches a property name at a separator boundary so that {@code usePlainText} is not reported
* for an unrelated property that merely ends with it. Spanner property names are
* case-insensitive.
*/
private static boolean hasUrlParam(String urlParams, String name) {
String haystack = urlParams.toLowerCase(Locale.ROOT);
String needle = name.toLowerCase(Locale.ROOT);
for (int from = 0; from <= haystack.length() - needle.length(); ) {
int index = haystack.indexOf(needle, from);
if (index < 0) {
return false;
}
boolean startsToken = index == 0 || "?;&".indexOf(haystack.charAt(index - 1)) >= 0;
int after = index + needle.length();
boolean endsToken = after >= haystack.length() || haystack.charAt(after) == '=';
if (startsToken && endsToken) {
return true;
}
from = index + 1;
}
return false;
}
static List<TableInfo> spannerTables(Connection conn, String schema) {
return unchecked(() -> {
String[] tableTypes = SPANNER_PROFILE.getTableTypes().toArray(new String[0]);
List<TableInfo> result = new ArrayList<>();
// The schema is passed through verbatim: an empty string is GoogleSQL's user schema, and
// mapping it to null would return every INFORMATION_SCHEMA/SPANNER_SYS view as well.
try (ResultSet rs = conn.getMetaData().getTables(null, schema, "%", tableTypes)) {
while (rs.next()) {
result.add(new TableInfo(
rs.getString("TABLE_NAME"),
normalizeTableType(rs.getString("TABLE_TYPE")),
rs.getString("REMARKS")
));
}
}
result.sort(Comparator.comparing(TableInfo::getName));
return result;
});
}
static List<ColumnInfo> spannerColumns(Connection conn, String schema, String table) {
return unchecked(() -> {
DatabaseMetaData meta = conn.getMetaData();
Set<String> primaryKeys = new LinkedHashSet<>();
try (ResultSet rs = meta.getPrimaryKeys(null, schema, table)) {
while (rs.next()) {
String name = rs.getString("COLUMN_NAME");
if (name != null) {
primaryKeys.add(name);
}
}
}
List<ColumnInfo> result = new ArrayList<>();
try (ResultSet rs = meta.getColumns(null, schema, table, "%")) {
while (rs.next()) {
String name = rs.getString("COLUMN_NAME");
// TYPE_NAME already carries the native Spanner type (STRING(100), ARRAY<INT64>).
String typeName = rs.getString("TYPE_NAME");
Integer columnSize = intOrNull(rs, "COLUMN_SIZE");
boolean intrinsicNumeric = isIntrinsicNumericType(typeName);
result.add(new ColumnInfo(
name,
typeName,
rs.getInt("NULLABLE") != DatabaseMetaData.columnNoNulls,
rs.getString("COLUMN_DEF"),
primaryKeys.contains(name),
null,
rs.getString("REMARKS"),
intrinsicNumeric ? null : columnSize,
intrinsicNumeric ? null : intOrNull(rs, "DECIMAL_DIGITS"),
characterLength(typeName, columnSize)
));
}
}
return result;
});
}
static List<IndexInfo> spannerIndexes(Connection conn, String schema, String table) {
return unchecked(() -> {
Map<String, Boolean> uniqueByIndex = new LinkedHashMap<>();
Map<String, List<IndexColumn>> keyColumns = new LinkedHashMap<>();
Map<String, List<String>> storingColumns = new LinkedHashMap<>();
try (ResultSet rs = conn.getMetaData().getIndexInfo(null, schema, table, false, false)) {
while (rs.next()) {
String name = rs.getString("INDEX_NAME");
String column = rs.getString("COLUMN_NAME");
if (name == null || column == null) {
continue;
}
uniqueByIndex.putIfAbsent(name, !rs.getBoolean("NON_UNIQUE"));
Integer ordinal = intOrNull(rs, "ORDINAL_POSITION");
if (ordinal == null) {
// Spanner reports STORING (GoogleSQL) / INCLUDE (PostgreSQL) columns with a
// NULL ORDINAL_POSITION. Reading them with getShort() would turn them into
// key column 0 and put a non-key column in front of the real key columns.
storingColumns.computeIfAbsent(name, ignored -> new ArrayList<>()).add(column);
continue;
}
keyColumns.computeIfAbsent(name, ignored -> new ArrayList<>())
.add(new IndexColumn(ordinal, column));
}
}
List<IndexInfo> result = new ArrayList<>();
for (Map.Entry<String, Boolean> entry : uniqueByIndex.entrySet()) {
String name = entry.getKey();
List<IndexColumn> ordered = keyColumns.getOrDefault(name, new ArrayList<>());
ordered.sort(Comparator.comparingInt(IndexColumn::getOrdinal));
List<String> columns = new ArrayList<>();
for (IndexColumn column : ordered) {
columns.add(column.getName());
}
result.add(new IndexInfo(
name,
columns,
Boolean.TRUE.equals(entry.getValue()),
// Spanner names the primary key index PRIMARY_KEY, not PRIMARY.
PRIMARY_KEY_INDEX.equalsIgnoreCase(name),
null,
null,
storingColumns.get(name),
null
));
}
return result;
});
}
static List<ForeignKeyInfo> spannerForeignKeys(Connection conn, String schema, String table) {
return unchecked(() -> {
List<ForeignKeyInfo> result = new ArrayList<>();
try (ResultSet rs = conn.getMetaData().getImportedKeys(null, schema, table)) {
while (rs.next()) {
String name = rs.getString("FK_NAME");
if (name == null || name.trim().isEmpty()) {
// GoogleSQL reports INTERLEAVE IN PARENT links as unnamed imported keys while
// the PostgreSQL dialect omits them. Skipping keeps both dialects consistent
// and avoids showing an interleaving relationship as a nameless foreign key.
continue;
}
result.add(new ForeignKeyInfo(
name,
rs.getString("FKCOLUMN_NAME"),
rs.getString("PKTABLE_NAME"),
rs.getString("PKCOLUMN_NAME")
));
}
}
return result;
});
}
/** Probes the database dialect once per connection; visible for testing. */
void detectDialect(Connection connection) {
boolean postgres = isPostgresDialect(connection);
postgresDialect = postgres;
defaultSchema = postgres ? postgresSchema(connection) : GOOGLE_SQL_SCHEMA;
}
static boolean isPostgresDialect(Connection connection) {
Boolean fromDriver = driverDialectIsPostgres(connection);
if (fromDriver != null) {
return fromDriver.booleanValue();
}
return productNameIsPostgres(connection);
}
private static Boolean driverDialectIsPostgres(Connection connection) {
try {
CloudSpannerJdbcConnection spannerConnection =
unwrapConnection(connection, CloudSpannerJdbcConnection.class);
if (spannerConnection == null) {
return null;
}
Dialect dialect = spannerConnection.getDialect();
if (dialect == null) {
return null;
}
return Boolean.valueOf(dialect == Dialect.POSTGRESQL);
} catch (Throwable ignored) {
// Linking the driver interface is not always possible, so this probe stays optional:
// the published fat jar carries stale BouncyCastle signatures that only the shadow jar
// strips (a plain classpath, for example the unit tests, throws SecurityException while
// loading any driver class), and a user-supplied driver jar is loaded through a child
// class loader whose interface is a different type. Both cases use the product name.
return null;
}
}
private static boolean productNameIsPostgres(Connection connection) {
try {
// GoogleSQL reports "Google Cloud Spanner", the PostgreSQL dialect appends " PostgreSQL".
String product = connection.getMetaData().getDatabaseProductName();
return product != null && product.contains("PostgreSQL");
} catch (Exception ignored) {
return false;
}
}
private static String postgresSchema(Connection connection) {
try {
String schema = connection.getSchema();
if (schema != null && !schema.trim().isEmpty()) {
return schema;
}
} catch (Exception ignored) {
// Fall through to the dialect default.
}
return POSTGRES_FALLBACK_SCHEMA;
}
private String configuredDatabaseName() {
String database = trimmed(getConfiguredDatabase());
if (!database.isEmpty()) {
return database;
}
try {
return trimmed(requireConnection().getCatalog());
} catch (Exception ignored) {
return "";
}
}
/**
* Never maps a blank schema to {@code null}: GoogleSQL's user schema name is the empty string,
* and the PostgreSQL dialect needs the literal {@code public}.
*
* <p>A value that cannot be a Spanner schema name is treated as "unspecified" as well. Hosts
* that have no schema level commonly send the database identifier as the metadata schema, and
* for Spanner that identifier is the resource path {@code projects/{p}/instances/{i}/databases/{d}}.
* Passing it through would make every metadata lookup match nothing, and a Spanner schema name
* can never contain a slash (identifiers are letters, digits and underscores).
*/
private String resolveSchema(String schema) {
if (schema == null || schema.trim().isEmpty() || schema.indexOf('/') >= 0) {
return defaultSchema;
}
return schema;
}
private static List<CompletionAssistantObjectKind> completionKinds(CompletionAssistantRequest request) {
List<CompletionAssistantObjectKind> kinds = request.getObject_kinds();
if (kinds.isEmpty()) {
return Arrays.asList(CompletionAssistantObjectKind.TABLE, CompletionAssistantObjectKind.VIEW);
}
return kinds;
}
private static String completionSchema(CompletionAssistantRequest request) {
String parentSchema = request.getParent_schema();
if (parentSchema != null && !parentSchema.trim().isEmpty()) {
return parentSchema;
}
return request.getSchema();
}
private static int completionLimit(Integer requested) {
if (requested == null) {
return DEFAULT_COMPLETION_LIMIT;
}
return Math.max(1, Math.min(MAX_COMPLETION_LIMIT, requested));
}
private static boolean completionMatches(String name, CompletionAssistantRequest request) {
if (name == null) {
return false;
}
String mask = request.getMask();
if (mask.trim().isEmpty()) {
return true;
}
String candidate = request.getCase_sensitive() ? name : name.toLowerCase(Locale.ROOT);
String expected = request.getCase_sensitive() ? mask : mask.toLowerCase(Locale.ROOT);
if (request.getMatch_mode() == CompletionAssistantMatchMode.CONTAINS) {
return candidate.contains(expected);
}
return candidate.startsWith(expected);
}
private static String normalizeTableType(String type) {
if (type == null || type.trim().isEmpty()) {
return "TABLE";
}
if ("BASE TABLE".equalsIgnoreCase(type.trim())) {
return "TABLE";
}
return type;
}
private static Integer characterLength(String typeName, Integer columnSize) {
if (typeName == null) {
return null;
}
String normalized = typeName.toLowerCase(Locale.ROOT);
if (!normalized.contains("char") && !normalized.contains("text")) {
return null;
}
return columnSize;
}
private static boolean isIntrinsicNumericType(String typeName) {
return "NUMERIC".equalsIgnoreCase(typeName) || "DECIMAL".equalsIgnoreCase(typeName);
}
private static Integer intOrNull(ResultSet rs, String column) throws Exception {
Object value = rs.getObject(column);
return value instanceof Number ? ((Number) value).intValue() : null;
}
private static String appendUrlParams(String url, String urlParams) {
String params = trimmed(urlParams);
while (params.startsWith("?") || params.startsWith(";") || params.startsWith("&")) {
params = params.substring(1);
}
if (params.isEmpty()) {
return url;
}
// Spanner takes properties as ?name=value;name=value.
return url + (url.indexOf('?') >= 0 ? ";" : "?") + params;
}
private static String stripLeadingSlashes(String value) {
String result = value;
while (result.startsWith("/")) {
result = result.substring(1);
}
return result;
}
private static String trimmed(String value) {
return value == null ? "" : value.trim();
}
private static final class IndexColumn {
private final int ordinal;
private final String name;
private IndexColumn(int ordinal, String name) {
this.ordinal = ordinal;
this.name = name;
}
private int getOrdinal() {
return ordinal;
}
private String getName() {
return name;
}
}
public static void main(String[] args) {
new MultiSessionJsonRpcServer(SpannerAgent::new).run();
}
}
@@ -0,0 +1,767 @@
package com.dbx.agent.spanner;
import com.dbx.agent.ColumnInfo;
import com.dbx.agent.ConnectParams;
import com.dbx.agent.DatabaseAgent;
import com.dbx.agent.DatabaseInfo;
import com.dbx.agent.ForeignKeyInfo;
import com.dbx.agent.IndexInfo;
import com.dbx.agent.TableInfo;
import com.dbx.agent.test.JdbcFakeExecutionBehaviorTest;
import com.dbx.agent.test.TestSupport;
import org.junit.jupiter.api.Test;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import java.sql.Array;
import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Types;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
class SpannerAgentTest extends JdbcFakeExecutionBehaviorTest {
private static final String GOOGLE_SQL_PRODUCT = "Google Cloud Spanner";
private static final String POSTGRES_PRODUCT = "Google Cloud Spanner PostgreSQL";
@Override
protected DatabaseAgent createAgent() {
return new SpannerAgent();
}
@Override
protected String resultSetSql() {
return "SHOW VARIABLE AUTOCOMMIT";
}
@Test
void usesBacktickIdentifierQuoteForGoogleSqlDialect() {
SpannerAgent agent = new SpannerAgent();
agent.detectDialect(fakeConnection(GOOGLE_SQL_PRODUCT, null, new LinkedHashMap<>()));
assertEquals("`", agent.getIdentifierQuote());
}
@Test
void usesDoubleQuoteIdentifierQuoteForPostgresDialect() {
SpannerAgent agent = new SpannerAgent();
agent.detectDialect(fakeConnection(POSTGRES_PRODUCT, "public", new LinkedHashMap<>()));
assertEquals("\"", agent.getIdentifierQuote());
}
@Test
void keepsBacktickQuoteBeforeTheDialectIsProbed() {
assertEquals("`", new SpannerAgent().getIdentifierQuote());
}
@Test
void reportsTheEmptyGoogleSqlSchemaInsteadOfDroppingIt() {
SpannerAgent agent = connectedAgent(GOOGLE_SQL_PRODUCT, null, new LinkedHashMap<>());
// The fake connection cannot create statements, so this exercises the fallback taken when
// INFORMATION_SCHEMA.SCHEMATA is unreadable: the default schema is still reported, and it is
// the empty string that GoogleSQL uses for its user schema. Named-schema discovery runs
// against the emulator instead — a proxy cannot answer a real catalog query.
assertEquals(Collections.singletonList(""), agent.listSchemas());
}
@Test
void reportsThePostgresDialectSchema() {
SpannerAgent agent = connectedAgent(POSTGRES_PRODUCT, "public", new LinkedHashMap<>());
// Same fallback path as above; the PostgreSQL dialect's default schema is `public`.
assertEquals(Collections.singletonList("public"), agent.listSchemas());
}
@Test
void passesTheEmptyGoogleSqlSchemaVerbatimToTheDriver() {
List<String> calls = new ArrayList<>();
Map<String, List<Map<String, Object>>> rows = new LinkedHashMap<>();
rows.put("getTables", Arrays.asList(
row("TABLE_NAME", "singers", "TABLE_TYPE", "TABLE"),
row("TABLE_NAME", "v_singer_albums", "TABLE_TYPE", "VIEW")
));
SpannerAgent agent = connectedAgent(GOOGLE_SQL_PRODUCT, null, rows, calls);
List<TableInfo> tables = agent.listTables("");
assertEquals(Arrays.asList("singers", "v_singer_albums"), names(tables));
assertEquals("TABLE", tables.get(0).getTable_type());
assertEquals("VIEW", tables.get(1).getTable_type());
// Empty string, never null: null would also return the INFORMATION_SCHEMA/SPANNER_SYS views.
assertEquals(Collections.singletonList("getTables(catalog=null, schema=[], pattern=%, types=[TABLE, VIEW])"), calls);
}
@Test
void fallsBackToTheDialectSchemaWhenTheCallerSendsNone() {
List<String> calls = new ArrayList<>();
Map<String, List<Map<String, Object>>> rows = new LinkedHashMap<>();
rows.put("getTables", Collections.emptyList());
SpannerAgent agent = connectedAgent(POSTGRES_PRODUCT, "public", rows, calls);
agent.listTables(null);
assertEquals(
Collections.singletonList("getTables(catalog=null, schema=[public], pattern=%, types=[TABLE, VIEW])"),
calls
);
}
@Test
void treatsTheResourcePathAsAnUnspecifiedSchema() {
List<String> calls = new ArrayList<>();
Map<String, List<Map<String, Object>>> rows = new LinkedHashMap<>();
rows.put("getTables", Collections.singletonList(row("TABLE_NAME", "singers", "TABLE_TYPE", "TABLE")));
SpannerAgent agent = connectedAgent(GOOGLE_SQL_PRODUCT, null, rows, calls);
// Hosts without a schema level send the database identifier as the metadata schema, and for
// Spanner that identifier is the resource path. Forwarding it verbatim matches nothing.
assertEquals(
Collections.singletonList("singers"),
names(agent.listTables("projects/test-project/instances/test-instance/databases/gsqldb"))
);
assertEquals(Collections.singletonList("getTables(catalog=null, schema=[], pattern=%, types=[TABLE, VIEW])"), calls);
}
@Test
void resolvesTheSchemaBeforeRenderingTableDdl() {
Map<String, List<Map<String, Object>>> rows = new LinkedHashMap<>();
rows.put("getColumns", Collections.singletonList(row(
"COLUMN_NAME", "singer_id",
"TYPE_NAME", "INT64",
"NULLABLE", DatabaseMetaData.columnNoNulls
)));
SpannerAgent agent = connectedAgent(GOOGLE_SQL_PRODUCT, null, rows);
String ddl = agent.getTableDdl("projects/test-project/instances/test-instance/databases/gsqldb", "singers");
// The resource path must not become a DDL qualifier, and the columns must still resolve.
assertFalse(ddl.contains("projects/"));
assertTrue(ddl.contains("singers"));
assertTrue(ddl.contains("singer_id"));
}
@Test
void quotesGeneratedDdlWithTheDialectQuote() {
Map<String, List<Map<String, Object>>> rows = new LinkedHashMap<>();
rows.put("getColumns", Collections.singletonList(row(
"COLUMN_NAME", "singer_id",
"TYPE_NAME", "INT64",
"NULLABLE", DatabaseMetaData.columnNoNulls
)));
// The shared builder hardcodes ANSI double quotes, which GoogleSQL reads as a string
// literal rather than an identifier.
String googleSql = connectedAgent(GOOGLE_SQL_PRODUCT, null, rows).getTableDdl("", "singers");
assertTrue(googleSql.contains("`singers`"), googleSql);
assertFalse(googleSql.contains("\"singers\""), googleSql);
String postgres = connectedAgent(POSTGRES_PRODUCT, "public", rows).getTableDdl("public", "singers");
assertTrue(postgres.contains("\"singers\""), postgres);
assertFalse(postgres.contains("`singers`"), postgres);
}
@Test
void skipsStoringIndexColumnsThatHaveNoOrdinalPosition() {
Map<String, List<Map<String, Object>>> rows = new LinkedHashMap<>();
rows.put("getIndexInfo", Arrays.asList(
indexRow("PRIMARY_KEY", "singer_id", 1, false),
indexRow("PRIMARY_KEY", "album_id", 2, false),
// STORING (GoogleSQL) / INCLUDE (PostgreSQL) column: ORDINAL_POSITION is NULL and the
// driver returns it before the real key column.
indexRow("idx_albums_title", "release_ts", null, true),
indexRow("idx_albums_title", "title", 1, true)
));
SpannerAgent agent = connectedAgent(GOOGLE_SQL_PRODUCT, null, rows);
List<IndexInfo> indexes = agent.listIndexes("", "albums");
assertEquals(2, indexes.size());
IndexInfo primaryKey = indexes.get(0);
assertEquals("PRIMARY_KEY", primaryKey.getName());
assertEquals(Arrays.asList("singer_id", "album_id"), primaryKey.getColumns());
assertTrue(primaryKey.getIs_unique());
// Spanner names the primary key index PRIMARY_KEY, so a "PRIMARY" comparison never matches.
assertTrue(primaryKey.getIs_primary());
assertNull(primaryKey.getIncluded_columns());
IndexInfo secondary = indexes.get(1);
assertEquals("idx_albums_title", secondary.getName());
assertEquals(Collections.singletonList("title"), secondary.getColumns());
assertEquals(Collections.singletonList("release_ts"), secondary.getIncluded_columns());
assertFalse(secondary.getIs_unique());
assertFalse(secondary.getIs_primary());
}
@Test
void skipsUnnamedInterleaveRelationshipsReportedAsForeignKeys() {
Map<String, List<Map<String, Object>>> rows = new LinkedHashMap<>();
rows.put("getImportedKeys", Arrays.asList(
row(
"FK_NAME", null,
"FKCOLUMN_NAME", "singer_id",
"PKTABLE_NAME", "singers",
"PKCOLUMN_NAME", "singer_id"
),
row(
"FK_NAME", "fk_concerts_singer",
"FKCOLUMN_NAME", "singer_id",
"PKTABLE_NAME", "singers",
"PKCOLUMN_NAME", "singer_id"
)
));
SpannerAgent agent = connectedAgent(GOOGLE_SQL_PRODUCT, null, rows);
List<ForeignKeyInfo> foreignKeys = agent.listForeignKeys("", "albums");
assertEquals(1, foreignKeys.size());
assertEquals("fk_concerts_singer", foreignKeys.get(0).getName());
}
@Test
void keepsNativeSpannerTypeNamesOnColumns() {
Map<String, List<Map<String, Object>>> rows = new LinkedHashMap<>();
rows.put("getPrimaryKeys", Collections.singletonList(row("COLUMN_NAME", "singer_id")));
rows.put("getColumns", Arrays.asList(
row(
"COLUMN_NAME", "singer_id",
"TYPE_NAME", "INT64",
"NULLABLE", DatabaseMetaData.columnNoNulls
),
row(
"COLUMN_NAME", "tags",
"TYPE_NAME", "ARRAY<STRING(MAX)>",
"NULLABLE", DatabaseMetaData.columnNullable
)
));
SpannerAgent agent = connectedAgent(GOOGLE_SQL_PRODUCT, null, rows);
List<ColumnInfo> columns = agent.getColumns("", "singers");
assertEquals(Arrays.asList("singer_id", "tags"), Arrays.asList(
columns.get(0).getName(),
columns.get(1).getName()
));
assertEquals("INT64", columns.get(0).getData_type());
assertTrue(columns.get(0).getIs_primary_key());
assertFalse(columns.get(0).getIs_nullable());
assertEquals("ARRAY<STRING(MAX)>", columns.get(1).getData_type());
assertFalse(columns.get(1).getIs_primary_key());
assertTrue(columns.get(1).getIs_nullable());
}
@Test
void readsArrayValuesInOrderWithoutUsingGetStringAndFreesTheArray() {
List<String> calls = new ArrayList<>();
Array array = fakeArray(new Object[]{7L, "two", null}, calls, false);
ResultSet resultSet = arrayResultSet(array, calls);
Object value = resultValue(new SpannerAgent(), resultSet, Types.ARRAY);
assertEquals(Arrays.asList(7L, "two", null), value);
assertEquals(Arrays.asList("getArray(1)", "array.getArray()", "array.free()"), calls);
}
@Test
void returnsNullForNullArraysWithoutUsingGetString() {
List<String> calls = new ArrayList<>();
ResultSet resultSet = arrayResultSet(null, calls);
assertNull(resultValue(new SpannerAgent(), resultSet, Types.ARRAY));
assertEquals(Collections.singletonList("getArray(1)"), calls);
}
@Test
void keepsTheReadArrayValueWhenFreeFails() {
List<String> calls = new ArrayList<>();
Array array = fakeArray(new String[]{"first", "second"}, calls, true);
Object value = resultValue(new SpannerAgent(), arrayResultSet(array, calls), Types.ARRAY);
assertEquals(Arrays.asList("first", "second"), value);
assertEquals(Arrays.asList("getArray(1)", "array.getArray()", "array.free()"), calls);
}
@Test
void delegatesScalarValuesToTheSharedReader() {
List<String> calls = new ArrayList<>();
ResultSet resultSet = proxy(ResultSet.class, (method, args) -> {
if ("getInt".equals(method.getName())) {
calls.add("getInt(" + args[0] + ")");
return 42;
}
if ("wasNull".equals(method.getName())) {
calls.add("wasNull()");
return false;
}
if ("getArray".equals(method.getName()) || "getString".equals(method.getName())) {
throw new AssertionError("Unexpected " + method.getName());
}
return defaultValue(method.getReturnType());
});
assertEquals(42, resultValue(new SpannerAgent(), resultSet, Types.INTEGER));
assertEquals(Arrays.asList("getInt(1)", "wasNull()"), calls);
}
@Test
void omitsIntrinsicNumericParametersWhileKeepingNativeArrayAndStringTypes() {
Map<String, List<Map<String, Object>>> rows = new LinkedHashMap<>();
rows.put("getColumns", Arrays.asList(
row(
"COLUMN_NAME", "amount",
"TYPE_NAME", "NUMERIC",
"NULLABLE", DatabaseMetaData.columnNullable,
"COLUMN_SIZE", 15,
"DECIMAL_DIGITS", 0
),
row(
"COLUMN_NAME", "ratio",
"TYPE_NAME", "DECIMAL",
"NULLABLE", DatabaseMetaData.columnNullable,
"COLUMN_SIZE", 15,
"DECIMAL_DIGITS", 0
),
row(
"COLUMN_NAME", "tags",
"TYPE_NAME", "ARRAY<STRING(MAX)>",
"NULLABLE", DatabaseMetaData.columnNullable
),
row(
"COLUMN_NAME", "nickname",
"TYPE_NAME", "STRING(100)",
"NULLABLE", DatabaseMetaData.columnNullable,
"COLUMN_SIZE", 100
)
));
SpannerAgent agent = connectedAgent(GOOGLE_SQL_PRODUCT, null, rows);
List<ColumnInfo> columns = agent.getColumns("", "singers");
String ddl = agent.getTableDdl("", "singers");
assertNull(columns.get(0).getNumeric_precision());
assertNull(columns.get(0).getNumeric_scale());
assertNull(columns.get(1).getNumeric_precision());
assertNull(columns.get(1).getNumeric_scale());
assertTrue(ddl.contains("`amount` NUMERIC"), ddl);
assertTrue(ddl.contains("`ratio` DECIMAL"), ddl);
assertFalse(ddl.contains("NUMERIC("), ddl);
assertFalse(ddl.contains("DECIMAL("), ddl);
assertTrue(ddl.contains("`tags` ARRAY<STRING(MAX)>"), ddl);
assertTrue(ddl.contains("`nickname` STRING(100)"), ddl);
}
@Test
void reportsTheConfiguredDatabaseOnlyOnce() {
SpannerAgent agent = connectedAgent(POSTGRES_PRODUCT, "public", new LinkedHashMap<>());
String resourcePath = "projects/test-project/instances/test-instance/databases/pgdb";
setConfiguredDatabase(agent, resourcePath);
// The shared adapter reports [pgdb, projects/.../databases/pgdb] for the PostgreSQL dialect.
// The fake connection cannot unwrap to CloudSpannerJdbcConnection, so this also pins the
// fallback taken whenever the Admin API is unavailable — a caller without
// spanner.databases.list still sees the database it connected to.
//
// The invariant both paths share: a reported name is the *full resource path*, because it
// round-trips as ConnectParams.database and only the path builds a resolvable JDBC URL. A
// bare database id would make every click on a sibling database hang until the connect
// timeout. The Admin API branch cannot be reached through this proxy, so that half is
// verified against the emulator by reconnecting with each name it reports.
assertEquals(Collections.singletonList(resourcePath), databaseNames(agent.listDatabases()));
}
@Test
void translatesDataTypeNamesForThePostgresDialect() {
Map<String, List<Map<String, Object>>> rows = new LinkedHashMap<>();
rows.put("getTypeInfo", Arrays.asList(
row("TYPE_NAME", "INT64"),
row("TYPE_NAME", "STRING"),
row("TYPE_NAME", "FLOAT64"),
row("TYPE_NAME", "JSON")
));
Map<String, List<Map<String, Object>>> postgresRows = new LinkedHashMap<>();
postgresRows.put("getTypeInfo", Arrays.asList(
row("TYPE_NAME", "INT64"),
row("TYPE_NAME", "STRING"),
row("TYPE_NAME", "FLOAT64"),
// The driver answers JSONB here rather than JSON, but still in upper case.
row("TYPE_NAME", "JSONB")
));
// The driver answers getTypeInfo() with GoogleSQL spellings for both dialects, so a
// PostgreSQL-dialect database would otherwise be offered INT64/STRING in field mapping.
assertEquals(
Arrays.asList("INT64", "STRING", "FLOAT64", "JSON"),
connectedAgent(GOOGLE_SQL_PRODUCT, null, rows).listDataTypes()
);
assertEquals(
Arrays.asList("bigint", "character varying", "double precision", "jsonb"),
connectedAgent(POSTGRES_PRODUCT, "public", postgresRows).listDataTypes()
);
}
@Test
void omitsTheEndpointWhenNoHostIsConfigured() {
ConnectParams params = new ConnectParams();
params.setDatabase("projects/p/instances/i/databases/d");
assertEquals("jdbc:cloudspanner:/projects/p/instances/i/databases/d", SpannerAgent.spannerUrl(params));
}
@Test
void buildsAnEndpointUrlForTheEmulator() {
ConnectParams params = new ConnectParams();
params.setHost("localhost");
params.setPort(9010);
params.setDatabase("projects/p/instances/i/databases/d");
params.setUrl_params("usePlainText=true;autoConfigEmulator=true");
assertEquals(
"jdbc:cloudspanner://localhost:9010/projects/p/instances/i/databases/d"
+ "?usePlainText=true;autoConfigEmulator=true",
SpannerAgent.spannerUrl(params)
);
}
@Test
void defaultsALoopbackEndpointToPlaintextBecauseTheEmulatorRejectsTls() {
ConnectParams params = new ConnectParams();
params.setHost("127.0.0.1");
params.setPort(9010);
params.setDatabase("projects/p/instances/i/databases/d");
// Without this the driver treats the emulator as a production endpoint, retries a TLS
// handshake against a plaintext port for the whole connect timeout, and fails with
// `UNAVAILABLE: io exception`.
assertEquals(
"jdbc:cloudspanner://127.0.0.1:9010/projects/p/instances/i/databases/d?usePlainText=true",
SpannerAgent.spannerUrl(params)
);
}
@Test
void appendsPlaintextAfterTheConfiguredParamsForALoopbackEndpoint() {
ConnectParams params = new ConnectParams();
params.setHost("localhost");
params.setPort(9010);
params.setDatabase("projects/p/instances/i/databases/d");
params.setUrl_params("autocommit=true");
assertEquals(
"jdbc:cloudspanner://localhost:9010/projects/p/instances/i/databases/d"
+ "?autocommit=true;usePlainText=true",
SpannerAgent.spannerUrl(params)
);
}
@Test
void keepsAnExplicitPlaintextChoiceIncludingOptingOut() {
ConnectParams optOut = new ConnectParams();
optOut.setHost("localhost");
optOut.setPort(9010);
optOut.setDatabase("projects/p/instances/i/databases/d");
optOut.setUrl_params("usePlainText=false");
assertEquals(
"jdbc:cloudspanner://localhost:9010/projects/p/instances/i/databases/d?usePlainText=false",
SpannerAgent.spannerUrl(optOut)
);
// autoConfigEmulator already implies plaintext, and it additionally creates a missing
// instance and database, so it must not be silently paired with usePlainText.
ConnectParams autoConfig = new ConnectParams();
autoConfig.setHost("localhost");
autoConfig.setPort(9010);
autoConfig.setDatabase("projects/p/instances/i/databases/d");
autoConfig.setUrl_params("autoConfigEmulator=true");
assertEquals(
"jdbc:cloudspanner://localhost:9010/projects/p/instances/i/databases/d?autoConfigEmulator=true",
SpannerAgent.spannerUrl(autoConfig)
);
}
@Test
void leavesRemoteEndpointsOnTls() {
// A non-loopback host may well be a private or proxied Spanner endpoint, so plaintext stays
// opt-in there. The host-less form used for the real service is covered separately.
ConnectParams params = new ConnectParams();
params.setHost("spanner.internal.example.com");
params.setPort(443);
params.setDatabase("projects/p/instances/i/databases/d");
assertEquals(
"jdbc:cloudspanner://spanner.internal.example.com:443/projects/p/instances/i/databases/d",
SpannerAgent.spannerUrl(params)
);
}
@Test
void usesTheDefaultPortWhenNoneIsConfigured() {
ConnectParams params = new ConnectParams();
params.setHost("spanner.googleapis.com");
params.setDatabase("projects/p/instances/i/databases/d");
assertEquals(
"jdbc:cloudspanner://spanner.googleapis.com:443/projects/p/instances/i/databases/d",
SpannerAgent.spannerUrl(params)
);
}
@Test
void skipsSchemaSwitchingBecauseSpannerHasNoSchemaStatement() {
assertEquals("", new SpannerAgent().setSchemaSQL("public"));
}
private static List<String> names(List<TableInfo> tables) {
List<String> result = new ArrayList<>();
for (TableInfo table : tables) {
result.add(table.getName());
}
return result;
}
private static List<String> databaseNames(List<DatabaseInfo> databases) {
List<String> result = new ArrayList<>();
for (DatabaseInfo database : databases) {
result.add(database.getName());
}
return result;
}
private static void setConfiguredDatabase(SpannerAgent agent, String database) {
for (Class<?> type = agent.getClass(); type != null; type = type.getSuperclass()) {
try {
Field field = type.getDeclaredField("configuredDatabase");
field.setAccessible(true);
field.set(agent, database);
} catch (NoSuchFieldException ignored) {
continue;
} catch (IllegalAccessException e) {
throw new IllegalStateException(e);
}
}
}
private static SpannerAgent connectedAgent(
String product,
String schema,
Map<String, List<Map<String, Object>>> rowsByCall
) {
return connectedAgent(product, schema, rowsByCall, new ArrayList<>());
}
private static SpannerAgent connectedAgent(
String product,
String schema,
Map<String, List<Map<String, Object>>> rowsByCall,
List<String> calls
) {
SpannerAgent agent = new SpannerAgent();
Connection conn = fakeConnection(product, schema, rowsByCall, calls);
agent.detectDialect(conn);
TestSupport.setPrivateConnection(agent, conn);
return agent;
}
private static Connection fakeConnection(
String product,
String schema,
Map<String, List<Map<String, Object>>> rowsByCall
) {
return fakeConnection(product, schema, rowsByCall, new ArrayList<>());
}
private static Connection fakeConnection(
String product,
String schema,
Map<String, List<Map<String, Object>>> rowsByCall,
List<String> calls
) {
DatabaseMetaData metaData = fakeMetaData(product, rowsByCall, calls);
return proxy(Connection.class, (method, args) -> {
String name = method.getName();
if ("getMetaData".equals(name)) {
return metaData;
}
if ("getSchema".equals(name)) {
return schema;
}
return defaultValue(method.getReturnType());
});
}
private static DatabaseMetaData fakeMetaData(
String product,
Map<String, List<Map<String, Object>>> rowsByCall,
List<String> calls
) {
return proxy(DatabaseMetaData.class, (method, args) -> {
String name = method.getName();
if ("getDatabaseProductName".equals(name)) {
return product;
}
if ("getTables".equals(name)) {
calls.add("getTables(catalog=" + args[0] + ", schema=[" + args[1] + "], pattern=" + args[2]
+ ", types=" + Arrays.toString((String[]) args[3]) + ")");
return fakeResultSet(rowsByCall.get("getTables"));
}
if ("getColumns".equals(name) || "getPrimaryKeys".equals(name)
|| "getIndexInfo".equals(name) || "getImportedKeys".equals(name)) {
calls.add(name + "(catalog=" + args[0] + ", schema=[" + args[1] + "], table=" + args[2] + ")");
return fakeResultSet(rowsByCall.get(name));
}
if ("getTypeInfo".equals(name)) {
// Takes no arguments, so it is not recorded alongside the schema-scoped calls.
return fakeResultSet(rowsByCall.get("getTypeInfo"));
}
if ("getSearchStringEscape".equals(name)) {
return "\\";
}
if ("getIdentifierQuoteString".equals(name)) {
// The driver hardcodes a backtick for both dialects; the agent must not trust it.
return "`";
}
return defaultValue(method.getReturnType());
});
}
private static ResultSet fakeResultSet(List<Map<String, Object>> rows) {
List<Map<String, Object>> values = rows == null ? Collections.emptyList() : rows;
int[] index = {-1};
return proxy(ResultSet.class, (method, args) -> {
String name = method.getName();
if ("next".equals(name)) {
index[0] += 1;
return index[0] < values.size();
}
if ("getString".equals(name)) {
Object value = values.get(index[0]).get((String) args[0]);
return value == null ? null : String.valueOf(value);
}
if ("getObject".equals(name)) {
return values.get(index[0]).get((String) args[0]);
}
if ("getInt".equals(name)) {
Object value = values.get(index[0]).get((String) args[0]);
return value instanceof Number ? ((Number) value).intValue() : 0;
}
if ("getBoolean".equals(name)) {
Object value = values.get(index[0]).get((String) args[0]);
return Boolean.TRUE.equals(value);
}
return defaultValue(method.getReturnType());
});
}
private static ResultSet arrayResultSet(Array array, List<String> calls) {
return proxy(ResultSet.class, (method, args) -> {
if ("getArray".equals(method.getName())) {
calls.add("getArray(" + args[0] + ")");
return array;
}
if ("getString".equals(method.getName())) {
throw new AssertionError("ARRAY values must not be read with getString");
}
return defaultValue(method.getReturnType());
});
}
private static Array fakeArray(Object value, List<String> calls, boolean failFree) {
return proxy(Array.class, (method, args) -> {
if ("getArray".equals(method.getName())) {
calls.add("array.getArray()");
return value;
}
if ("free".equals(method.getName())) {
calls.add("array.free()");
if (failFree) {
throw new SQLException("cleanup failed");
}
return null;
}
return defaultValue(method.getReturnType());
});
}
private static Object resultValue(SpannerAgent agent, ResultSet resultSet, int sqlType) {
try {
Method method = SpannerAgent.class.getDeclaredMethod("resultValue", ResultSet.class, int.class, int.class);
method.setAccessible(true);
return method.invoke(agent, resultSet, 1, sqlType);
} catch (InvocationTargetException e) {
Throwable cause = e.getCause();
if (cause instanceof RuntimeException) {
throw (RuntimeException) cause;
}
if (cause instanceof Error) {
throw (Error) cause;
}
throw new IllegalStateException(cause);
} catch (ReflectiveOperationException e) {
throw new IllegalStateException(e);
}
}
private static Map<String, Object> row(Object... keyValues) {
Map<String, Object> result = new LinkedHashMap<>();
for (int i = 0; i < keyValues.length; i += 2) {
result.put((String) keyValues[i], keyValues[i + 1]);
}
return result;
}
private static Map<String, Object> indexRow(String indexName, String column, Integer ordinal, boolean nonUnique) {
return row(
"INDEX_NAME", indexName,
"COLUMN_NAME", column,
"ORDINAL_POSITION", ordinal,
"NON_UNIQUE", nonUnique
);
}
@SuppressWarnings("unchecked")
private static <T> T proxy(Class<T> type, MethodHandler handler) {
InvocationHandler invocationHandler = new InvocationHandler() {
@Override
public Object invoke(Object instance, Method method, Object[] args) throws Throwable {
return handler.handle(method, args == null ? new Object[0] : args);
}
};
return (T) Proxy.newProxyInstance(type.getClassLoader(), new Class<?>[]{type}, invocationHandler);
}
private static Object defaultValue(Class<?> type) {
if (Boolean.TYPE.equals(type)) {
return false;
}
if (Integer.TYPE.equals(type)) {
return 0;
}
if (Short.TYPE.equals(type)) {
return (short) 0;
}
return null;
}
private interface MethodHandler {
Object handle(Method method, Object[] args) throws Throwable;
}
}
+39 -7
View File
@@ -28,6 +28,7 @@ pub struct TdengineDriver {
connection: Option<Taos>,
params: ConnectParams,
server_version: Option<String>,
session_timezone: Option<Tz>,
current_database: String,
query_sessions: HashMap<String, QueryCursor>,
table_cache: Option<TableCache>,
@@ -42,6 +43,7 @@ struct TableCache {
struct QueryCursor {
result_set: ResultSet,
timezone: Option<Tz>,
session_timezone: Option<Tz>,
columns: Vec<String>,
column_types: Vec<String>,
affected_rows: i64,
@@ -59,6 +61,7 @@ impl TdengineDriver {
connection: None,
params: ConnectParams::default(),
server_version: None,
session_timezone: None,
current_database: String::new(),
query_sessions: HashMap::new(),
table_cache: None,
@@ -76,8 +79,13 @@ impl TdengineDriver {
.map_err(|_| anyhow!("TDengine connection timed out after {} seconds", connect_timeout.as_secs()))??;
let token = CancellationToken::new();
let server_version = query_scalar_string(&connection, "SELECT server_version()", &token, 5).await.ok();
let session_timezone = query_scalar_string(&connection, "SELECT timezone()", &token, 5)
.await
.ok()
.and_then(|value| parse_server_timezone(&value));
self.current_database = dsn.database;
self.server_version = server_version;
self.session_timezone = session_timezone;
self.params = params;
self.connection = Some(connection);
Ok(())
@@ -95,6 +103,7 @@ impl TdengineDriver {
self.query_sessions.clear();
self.table_cache = None;
self.server_version = None;
self.session_timezone = None;
self.current_database.clear();
self.connection = None;
}
@@ -469,6 +478,7 @@ impl TdengineDriver {
Ok(QueryCursor {
result_set,
timezone,
session_timezone: self.session_timezone,
columns,
column_types,
affected_rows,
@@ -694,7 +704,7 @@ impl QueryCursor {
loop {
if let Some(block) = &self.pending_block {
if self.pending_row_index < block.nrows() {
let timezone = preferred_timezone(self.timezone, block.timezone());
let timezone = preferred_timezone(self.timezone, block.timezone(), self.session_timezone);
let row_index = self.pending_row_index;
self.pending_row_index += 1;
let row = (0..block.ncols())
@@ -763,15 +773,23 @@ async fn query_scalar_string(
continue;
}
let value = block.get_ref(0, 0).ok_or_else(|| anyhow!("TDengine query returned an empty value"))?;
return match borrowed_value_to_json(value, preferred_timezone(timezone, block.timezone())) {
return match borrowed_value_to_json(value, preferred_timezone(timezone, block.timezone(), None)) {
Value::String(value) => Ok(value),
value => Ok(value.to_string()),
};
}
}
fn preferred_timezone(query_timezone: Option<Tz>, block_timezone: Option<Tz>) -> Option<Tz> {
query_timezone.or(block_timezone)
fn preferred_timezone(
query_timezone: Option<Tz>,
block_timezone: Option<Tz>,
session_timezone: Option<Tz>,
) -> Option<Tz> {
query_timezone.or(block_timezone).or(session_timezone)
}
fn parse_server_timezone(value: &str) -> Option<Tz> {
value.split_whitespace().next()?.parse().ok()
}
async fn cancellable<T, F>(token: &CancellationToken, timeout_secs: u64, future: F) -> Result<T>
@@ -1063,9 +1081,23 @@ mod tests {
#[test]
fn prefers_query_timezone_and_falls_back_to_block_timezone() {
assert_eq!(preferred_timezone(Some(chrono_tz::UTC), Some(chrono_tz::Asia::Shanghai)), Some(chrono_tz::UTC));
assert_eq!(preferred_timezone(None, Some(chrono_tz::Asia::Shanghai)), Some(chrono_tz::Asia::Shanghai));
assert_eq!(preferred_timezone(None, None), None);
assert_eq!(
preferred_timezone(Some(chrono_tz::UTC), Some(chrono_tz::Asia::Tokyo), Some(chrono_tz::Asia::Shanghai)),
Some(chrono_tz::UTC)
);
assert_eq!(
preferred_timezone(None, Some(chrono_tz::Asia::Tokyo), Some(chrono_tz::Asia::Shanghai)),
Some(chrono_tz::Asia::Tokyo)
);
assert_eq!(preferred_timezone(None, None, Some(chrono_tz::Asia::Shanghai)), Some(chrono_tz::Asia::Shanghai));
assert_eq!(preferred_timezone(None, None, None), None);
}
#[test]
fn parses_tdengine_timezone_description() {
assert_eq!(parse_server_timezone("Asia/Shanghai (CST, +0800)"), Some(chrono_tz::Asia::Shanghai));
assert_eq!(parse_server_timezone("Etc/UTC (UTC, +0000)"), Some(chrono_tz::Etc::UTC));
assert_eq!(parse_server_timezone("unknown (+0800)"), None);
}
#[test]
+46 -1
View File
@@ -18,7 +18,7 @@ pub fn borrowed_value_to_json(value: BorrowedValue<'_>, timezone: Option<Tz>) ->
BorrowedValue::USmallInt(value) => json!(value),
BorrowedValue::UInt(value) => json!(value),
BorrowedValue::UBigInt(value) => json!(value),
BorrowedValue::Float(value) => finite_number(value as f64),
BorrowedValue::Float(value) => finite_float_number(value),
BorrowedValue::Double(value) => finite_number(value),
BorrowedValue::VarChar(value) => Value::String(value.to_string()),
BorrowedValue::NChar(value) => Value::String(value.into_owned()),
@@ -38,6 +38,17 @@ fn finite_number(value: f64) -> Value {
Number::from_f64(value).map(Value::Number).unwrap_or_else(|| Value::String(value.to_string()))
}
fn finite_float_number(value: f32) -> Value {
let shortest = value.to_string();
if value.is_finite() {
// Widening f32 directly exposes binary approximation digits that were never significant in FLOAT.
let rounded = shortest.parse::<f64>().expect("a formatted finite f32 is a valid f64");
Value::Number(Number::from_f64(rounded).expect("a finite f32 is a finite f64"))
} else {
Value::String(shortest)
}
}
fn binary_json(value: &[u8]) -> Value {
json!({ "$binary": STANDARD.encode(value) })
}
@@ -86,6 +97,40 @@ mod tests {
use super::*;
use taos::Ty;
#[test]
fn serializes_float_with_shortest_round_trippable_decimal() {
let cases = [
(8.6_f32, "8.6"),
(0.72_f32, "0.72"),
(59.6_f32, "59.6"),
(-13.25_f32, "-13.25"),
(0.0_f32, "0.0"),
(-0.0_f32, "-0.0"),
(f32::from_bits(1), "1e-45"),
(f32::MIN_POSITIVE, "1.1754944e-38"),
(f32::MAX, "3.4028235e+38"),
(1.0e-7_f32, "1e-7"),
(1.0e20_f32, "1e+20"),
];
for (value, expected) in cases {
let encoded = borrowed_value_to_json(BorrowedValue::Float(value), None);
assert!(encoded.is_number(), "finite FLOAT must remain a JSON number");
assert_eq!(encoded.to_string(), expected);
assert_eq!((encoded.as_f64().unwrap() as f32).to_bits(), value.to_bits());
}
}
#[test]
fn keeps_float_fallback_and_other_numeric_types_unchanged() {
assert_eq!(borrowed_value_to_json(BorrowedValue::Float(f32::NAN), None), json!("NaN"));
assert_eq!(borrowed_value_to_json(BorrowedValue::Float(f32::INFINITY), None), json!("inf"));
assert_eq!(borrowed_value_to_json(BorrowedValue::Float(f32::NEG_INFINITY), None), json!("-inf"));
assert_eq!(borrowed_value_to_json(BorrowedValue::Null(Ty::Float), None), Value::Null);
assert_eq!(borrowed_value_to_json(BorrowedValue::Double(8.600000381469727), None), json!(8.600000381469727));
assert_eq!(borrowed_value_to_json(BorrowedValue::Int(42), None), json!(42));
}
#[test]
fn preserves_timestamp_precision_with_at_least_milliseconds() {
assert_eq!(
+4
View File
@@ -142,6 +142,8 @@ fn tdengine_websocket_live_compatibility() {
assert_eq!(query["rows"].as_array().unwrap().len(), 2);
assert_eq!(query["truncated"], true);
assert_eq!(query["rows"][0][0], "2026-08-04 10:00:00.001");
assert_eq!(query["column_types"][1], "FLOAT");
assert_eq!(query["rows"][0][1], 10.1);
let connection_info = agent.call("connection_info", json!({"agentSessionId": session_id}));
assert_eq!(connection_info["databaseInfo"]["currentDatabase"], database);
@@ -158,6 +160,7 @@ fn tdengine_websocket_live_compatibility() {
);
assert_eq!(first_page["has_more"], true);
assert_eq!(first_page["truncated"], false);
assert_eq!(first_page["rows"][0][1], 10.1);
let query_session_id = first_page["session_id"].as_str().expect("paged query session");
let final_page = agent.call(
"fetch_query_page",
@@ -169,6 +172,7 @@ fn tdengine_websocket_live_compatibility() {
}),
);
assert_eq!(final_page["rows"].as_array().unwrap().len(), 1);
assert_eq!(final_page["rows"][0][1], 10.2);
assert_eq!(final_page["has_more"], false);
assert_eq!(final_page["truncated"], true);
+27 -1
View File
@@ -29,8 +29,34 @@ Set `VASTBASE_TEST_HOST`, `VASTBASE_TEST_PORT`, `VASTBASE_TEST_DATABASE`,
`VASTBASE_TEST_USERNAME`, and `VASTBASE_TEST_PASSWORD`, then run:
```bash
go test -run '^TestVastbaseIntegration$' -count=1 ./...
go test -count=1 ./...
```
`TestVastbaseConstraintsIntegration` validates structured primary-key,
foreign-key, unique, check, deferrability, validation, enabled state, and
deparser metadata. To qualify multiple real database compatibility modes,
create one test database per mode and pass all names as a comma-separated list:
```bash
VASTBASE_TEST_DATABASES=vastbase_a,vastbase_b,vastbase_pg go test -run '^TestVastbaseConstraintsIntegration$' -count=1
```
The agent reads each database's `pg_catalog.pg_database.datcompatibility` value
and reports both a normalized `compatibilityMode` (`oracle`, `mysql`,
`postgres`, or `sqlserver`) and the server's original
`compatibilityModeRaw` value. `B`, `M`, and `MYSQL` use MySQL identifier rules.
Catalog availability and SQL Server identity metadata are probed separately
from the normalized mode. Vastbase `A` is normalized to Oracle compatibility,
but the tested A release rejects `DISABLE CONSTRAINT`, so that DDL assertion is
only enabled for server values that explicitly report `O`, `ORA`, or `ORACLE`. For a full compatibility
qualification, run the complete suite separately against A (Oracle), B
(MySQL), PG (PostgreSQL), and MSSQL (SQL Server) instances. Supply credentials
through the `VASTBASE_TEST_*` environment variables; do not commit them.
MySQL-compatible instances intentionally skip the PostgreSQL-only SQL-function
fixture and `search_path` visibility assertion, while still running the table,
column, index, view, transaction, paging, cancellation, and structured
constraint coverage.
The benchmark harness under `bench/` compares this native agent with the
Vastbase JDBC 2.11v and 2.15v agents.
+74 -4
View File
@@ -1,9 +1,11 @@
package main
import (
"context"
"database/sql"
"net/url"
"strings"
"time"
_ "gitcode.com/opengauss/openGauss-connector-go-pq"
)
@@ -29,11 +31,79 @@ func agentDataTypes() []string {
return vastbaseDataTypes
}
func detectAgentMode(_ *sql.DB, configuredMySQL bool) vastbaseMode {
if configuredMySQL {
return vastbaseMode{compatibilityMode: "mysql", mysqlCompat: true, postgresCatalog: true}
type vastbaseCompatibility struct {
mode string
raw string
mysqlCompat bool
sqlServer bool
supportsDisableConstraint bool
}
func normalizeVastbaseCompatibility(raw string) vastbaseCompatibility {
normalizedRaw := strings.ToUpper(strings.TrimSpace(raw))
compact := strings.NewReplacer("_", "", "-", "", " ", "").Replace(normalizedRaw)
result := vastbaseCompatibility{mode: "postgres", raw: normalizedRaw}
switch compact {
case "A":
result.mode = "oracle"
case "O", "ORA", "ORACLE":
result.mode = "oracle"
result.supportsDisableConstraint = true
case "B", "M", "MYSQL":
result.mode = "mysql"
result.mysqlCompat = true
case "MSSQL", "SQLSERVER":
result.mode = "sqlserver"
result.sqlServer = true
case "", "P", "PG", "POSTGRES", "POSTGRESQL":
result.mode = "postgres"
default:
result.mode = strings.ToLower(strings.TrimSpace(raw))
}
return vastbaseMode{compatibilityMode: "postgres", postgresCatalog: true}
return result
}
func detectAgentMode(db *sql.DB, configuredMySQL bool) vastbaseMode {
raw := "PG"
if db != nil {
var detected string
if err := db.QueryRow("SELECT datcompatibility FROM pg_catalog.pg_database WHERE datname = current_database()").Scan(&detected); err == nil && strings.TrimSpace(detected) != "" {
raw = detected
}
}
compatibility := normalizeVastbaseCompatibility(raw)
if configuredMySQL {
compatibility = normalizeVastbaseCompatibility("MYSQL")
}
mode := vastbaseMode{
compatibilityMode: compatibility.mode,
compatibilityModeRaw: compatibility.raw,
mysqlCompat: compatibility.mysqlCompat,
sqlServerIdentity: compatibility.sqlServer,
supportsDisableConstraint: compatibility.supportsDisableConstraint,
}
if db == nil {
mode.postgresCatalog = true
return mode
}
postgresCatalog := catalogExists(db, "pg_catalog.pg_namespace")
systemCatalog := catalogExists(db, "sys_catalog.sys_namespace")
mode.postgresCatalog = postgresCatalog || !systemCatalog
return mode
}
func catalogExists(db *sql.DB, catalog string) bool {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
rows, err := db.QueryContext(ctx, "SELECT 1 FROM "+catalog+" WHERE 1 = 0")
if err != nil {
return false
}
return rows.Close() == nil
}
func vastbaseSupportsDisableConstraint(mode vastbaseMode) bool {
return mode.supportsDisableConstraint
}
func agentSSLModeAttempts(sslMode string) []string {
+136 -10
View File
@@ -65,17 +65,21 @@ func TestVastbaseIntegration(t *testing.T) {
mustExecute(t, server, "CREATE TABLE public."+quoteIdentifier(child)+" (id integer PRIMARY KEY, parent_id integer REFERENCES public."+quoteIdentifier(parent)+"(id))")
mustExecute(t, server, "CREATE INDEX "+quoteIdentifier(child+"_parent_idx")+" ON public."+quoteIdentifier(child)+"(parent_id)")
mustExecute(t, server, "CREATE VIEW public."+quoteIdentifier(view)+" AS SELECT id, name FROM public."+quoteIdentifier(parent))
mustExecute(t, server, "CREATE FUNCTION public."+quoteIdentifier(function)+"() RETURNS text AS $$ SELECT 'dbx'; $$ LANGUAGE SQL")
if !server.mode.mysqlCompat {
mustExecute(t, server, "CREATE FUNCTION public."+quoteIdentifier(function)+"() RETURNS text AS $$ SELECT 'dbx'; $$ LANGUAGE SQL")
}
mustExecute(t, server, "CREATE SCHEMA "+quoteIdentifier(searchFirst))
mustExecute(t, server, "CREATE SCHEMA "+quoteIdentifier(searchSecond))
mustExecute(t, server, "CREATE TABLE "+quoteIdentifier(searchSecond)+"."+quoteIdentifier(searchTable)+" (id integer PRIMARY KEY, name varchar(64))")
mustExecute(t, server, "SET search_path TO "+quoteIdentifier(searchFirst)+", "+quoteIdentifier(searchSecond))
visibleColumns, err := server.getColumns("", searchTable)
if err != nil || len(visibleColumns) != 2 || visibleColumns[0].ResolvedSchema == nil || *visibleColumns[0].ResolvedSchema != searchSecond {
t.Fatalf("unqualified metadata did not resolve the non-first search_path schema: columns=%#v err=%v", visibleColumns, err)
if !server.mode.mysqlCompat {
mustExecute(t, server, "SET search_path TO "+quoteIdentifier(searchFirst)+", "+quoteIdentifier(searchSecond))
visibleColumns, err := server.getColumns("", searchTable)
if err != nil || len(visibleColumns) != 2 || visibleColumns[0].ResolvedSchema == nil || *visibleColumns[0].ResolvedSchema != searchSecond {
t.Fatalf("unqualified metadata did not resolve the non-first search_path schema: columns=%#v err=%v", visibleColumns, err)
}
mustExecute(t, server, "INSERT INTO "+quoteIdentifier(searchTable)+" VALUES (1, 'visible')")
mustExecute(t, server, "SET search_path TO public")
}
mustExecute(t, server, "INSERT INTO "+quoteIdentifier(searchTable)+" VALUES (1, 'visible')")
mustExecute(t, server, "SET search_path TO public")
tables, err := server.listTables("public", metadataListConstraints{Filter: suffix})
if err != nil || len(tables) < 3 {
@@ -111,9 +115,11 @@ func TestVastbaseIntegration(t *testing.T) {
if err != nil || len(foreignKeys) != 1 || foreignKeys[0].RefTable != parent {
t.Fatalf("list foreign keys failed: keys=%v err=%v", foreignKeys, err)
}
source, err := server.getObjectSource("public", function, "FUNCTION")
if err != nil || !strings.Contains(fmt.Sprint(source["source"]), function) {
t.Fatalf("get function source failed: source=%v err=%v", source, err)
if !server.mode.mysqlCompat {
source, err := server.getObjectSource("public", function, "FUNCTION")
if err != nil || !strings.Contains(fmt.Sprint(source["source"]), function) {
t.Fatalf("get function source failed: source=%v err=%v", source, err)
}
}
transactionParams := map[string]json.RawMessage{
@@ -155,6 +161,126 @@ func TestVastbaseIntegration(t *testing.T) {
}
}
func TestVastbaseConstraintsIntegration(t *testing.T) {
host := os.Getenv("VASTBASE_TEST_HOST")
portText := os.Getenv("VASTBASE_TEST_PORT")
username := os.Getenv("VASTBASE_TEST_USERNAME")
password := os.Getenv("VASTBASE_TEST_PASSWORD")
if host == "" || portText == "" || username == "" || password == "" {
t.Skip("Vastbase integration environment is not configured")
}
port, err := strconv.Atoi(portText)
if err != nil {
t.Fatal(err)
}
databases := []string{os.Getenv("VASTBASE_TEST_DATABASE")}
if databases[0] == "" {
databases[0] = "postgres"
}
if raw := os.Getenv("VASTBASE_TEST_DATABASES"); raw != "" {
databases = strings.FieldsFunc(raw, func(r rune) bool { return r == ',' || r == ';' })
}
for _, database := range databases {
database = strings.TrimSpace(database)
if database == "" {
continue
}
t.Run(database, func(t *testing.T) {
suffix := strconv.FormatInt(time.Now().UnixNano(), 36)
schema := "dbx_constraints_" + suffix
parent := schema + ".parent"
child := schema + ".child"
server := newServer()
cp := connectParams{
Host: host, Port: port, Database: database, Username: username, Password: password,
ConnectionString: fmt.Sprintf("jdbc:vastbase://%s:%d/%s", host, port, database),
}
if err := server.connect(cp); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = server.disconnect() })
t.Cleanup(func() { _, _ = server.executeQuery(queryOptions{SQL: "DROP SCHEMA IF EXISTS " + schema + " CASCADE"}) })
expectedModes := map[string]string{"A": "oracle", "B": "mysql", "PG": "postgres", "MSSQL": "sqlserver"}
if expected, ok := expectedModes[server.mode.compatibilityModeRaw]; ok && server.mode.compatibilityMode != expected {
t.Fatalf("compatibility mode %q normalized to %q, want %q", server.mode.compatibilityModeRaw, server.mode.compatibilityMode, expected)
}
mustExecute(t, server, "CREATE SCHEMA "+schema)
mustExecute(t, server, "CREATE TABLE "+parent+" (id integer PRIMARY KEY, region integer, CONSTRAINT parent_pair_unique UNIQUE (id, region))")
childID := "id integer PRIMARY KEY"
if server.mode.sqlServerIdentity {
childID = "id integer IDENTITY(10,2) PRIMARY KEY"
}
mustExecute(t, server, "CREATE TABLE "+child+" ("+childID+", parent_id integer, parent_region integer, amount integer, CONSTRAINT child_parent_fk FOREIGN KEY (parent_id, parent_region) REFERENCES "+parent+" (id, region) DEFERRABLE INITIALLY DEFERRED, CONSTRAINT child_amount_check CHECK (amount > 0))")
constraints, err := server.listConstraints(schema, "child")
if err != nil {
t.Fatalf("list constraints failed in %s mode: %v", server.mode.compatibilityMode, err)
}
byName := map[string]constraintInfo{}
for _, constraint := range constraints {
byName[constraint.Name] = constraint
}
fk := byName["child_parent_fk"]
if fk.ConstraintType != "FOREIGN KEY" || !fk.Enabled || !fk.Valid || !fk.Deferrable || !fk.InitiallyDeferred || !equalVastbaseStrings(fk.Columns, []string{"parent_id", "parent_region"}) || !equalVastbaseStrings(fk.RefColumns, []string{"id", "region"}) {
t.Fatalf("foreign key metadata mismatch in %s mode: %+v", server.mode.compatibilityMode, fk)
}
check := byName["child_amount_check"]
if check.ConstraintType != "CHECK" || !check.Enabled || !check.Valid || !strings.Contains(check.Definition, "amount") {
t.Fatalf("check metadata mismatch in %s mode: %+v", server.mode.compatibilityMode, check)
}
if server.mode.sqlServerIdentity {
columns, err := server.getColumns(schema, "child")
if err != nil || len(columns) == 0 || columns[0].Extra == nil || *columns[0].Extra != "IDENTITY(10,2)" {
t.Fatalf("SQL Server identity metadata mismatch: columns=%+v err=%v", columns, err)
}
}
foreignKeys, err := server.listForeignKeys(schema, "child")
if err != nil || len(foreignKeys) != 2 {
t.Fatalf("foreign key listing failed in %s mode: keys=%v err=%v", server.mode.compatibilityMode, foreignKeys, err)
}
foreignKeyNames := map[string]struct{}{}
for _, foreignKey := range foreignKeys {
foreignKeyNames[foreignKey.Name] = struct{}{}
}
constraintForeignKeyNames := map[string]struct{}{}
for _, constraint := range constraints {
if constraint.ConstraintType == "FOREIGN KEY" {
constraintForeignKeyNames[constraint.Name] = struct{}{}
}
}
if len(foreignKeyNames) != len(constraintForeignKeyNames) {
t.Fatalf("foreign-key metadata sources disagree in %s mode: constraints=%v foreign_keys=%v", server.mode.compatibilityMode, constraintForeignKeyNames, foreignKeyNames)
}
for name := range foreignKeyNames {
if _, ok := constraintForeignKeyNames[name]; !ok {
t.Fatalf("foreign-key %q missing from list_constraints in %s mode", name, server.mode.compatibilityMode)
}
}
if vastbaseSupportsDisableConstraint(server.mode) {
mustExecute(t, server, "ALTER TABLE "+child+" DISABLE CONSTRAINT child_amount_check")
disabled, err := server.listConstraints(schema, "child")
if err != nil {
t.Fatalf("list disabled constraints failed in %s mode: %v", server.mode.compatibilityMode, err)
}
var disabledCheck constraintInfo
found := false
for _, constraint := range disabled {
if constraint.Name == "child_amount_check" {
disabledCheck = constraint
found = true
break
}
}
if !found || disabledCheck.Enabled {
t.Fatalf("disabled constraint state mismatch in %s mode: %+v", server.mode.compatibilityMode, disabledCheck)
}
mustExecute(t, server, "ALTER TABLE "+child+" ENABLE CONSTRAINT child_amount_check")
}
})
}
}
func TestVastbaseCustomTypesIntegration(t *testing.T) {
host := os.Getenv("VASTBASE_TEST_HOST")
portText := os.Getenv("VASTBASE_TEST_PORT")
+34 -19
View File
@@ -136,24 +136,27 @@ type rowScanner struct {
}
type server struct {
db *sql.DB
openDatabase agentDBOpener
params connectParams
mode vastbaseMode
usePgDefaultExpression bool
catalogIdentityUnsupported bool
infoColumnTypeUnsupported bool
infoUdtNameUnsupported bool
listTablesStatement *sql.Stmt
connectionRuntime *connectionRuntime
sessionAffinity bool
currentSchema string
schemaInitialized bool
schemaConnectionID uintptr
sessions map[string]*querySession
nextSessionID uint64
activeCancelMu sync.Mutex
activeCancel context.CancelFunc
db *sql.DB
openDatabase agentDBOpener
params connectParams
mode vastbaseMode
usePgDefaultExpression bool
catalogIdentityUnsupported bool
infoColumnTypeUnsupported bool
infoUdtNameUnsupported bool
constraintDefinitionUnsupported bool
constraintValidatedUnsupported bool
constraintEnabledUnsupported bool
listTablesStatement *sql.Stmt
connectionRuntime *connectionRuntime
sessionAffinity bool
currentSchema string
schemaInitialized bool
schemaConnectionID uintptr
sessions map[string]*querySession
nextSessionID uint64
activeCancelMu sync.Mutex
activeCancel context.CancelFunc
}
type agentSession struct {
@@ -433,6 +436,9 @@ func (s *server) dispatch(method string, params map[string]json.RawMessage) (any
case "list_foreign_keys":
result, err := s.listForeignKeys(stringParam(params, "schema"), stringParam(params, "table"))
return result, false, err
case "list_constraints":
result, err := s.listConstraints(stringParam(params, "schema"), stringParam(params, "table"))
return result, false, err
case "list_triggers":
result, err := s.listTriggers(stringParam(params, "schema"), stringParam(params, "table"))
return result, false, err
@@ -489,6 +495,9 @@ func (s *server) connect(cp connectParams) error {
s.catalogIdentityUnsupported = false
s.infoColumnTypeUnsupported = false
s.infoUdtNameUnsupported = false
s.constraintDefinitionUnsupported = false
s.constraintValidatedUnsupported = false
s.constraintEnabledUnsupported = false
s.sessionAffinity = false
return nil
}
@@ -505,11 +514,14 @@ func (s *server) connectWithRuntime(cp connectParams, connectionRuntime *connect
s.db = db
s.connectionRuntime = connectionRuntime
s.params = cp
s.mode = detectAgentMode(db, cp.MySQLCompatMode)
s.mode = detectAgentMode(connectionRuntime.database(), cp.MySQLCompatMode)
s.usePgDefaultExpression = false
s.catalogIdentityUnsupported = false
s.infoColumnTypeUnsupported = false
s.infoUdtNameUnsupported = false
s.constraintDefinitionUnsupported = false
s.constraintValidatedUnsupported = false
s.constraintEnabledUnsupported = false
s.sessionAffinity = false
return nil
}
@@ -584,6 +596,9 @@ func (s *server) disconnect() error {
s.catalogIdentityUnsupported = false
s.infoColumnTypeUnsupported = false
s.infoUdtNameUnsupported = false
s.constraintDefinitionUnsupported = false
s.constraintValidatedUnsupported = false
s.constraintEnabledUnsupported = false
s.connectionRuntime = nil
s.sessionAffinity = false
s.resetSchemaCache()
+68
View File
@@ -149,6 +149,34 @@ func TestVastbaseMetadataErrorClassificationUsesOpenGaussCodes(t *testing.T) {
t.Fatal("undefined Vastbase function was not recognized")
}
}
func TestVastbaseCompatibilityNormalization(t *testing.T) {
for _, test := range []struct {
raw string
mode string
mysql bool
sqlServer bool
disable bool
}{
{raw: "A", mode: "oracle"},
{raw: "O", mode: "oracle", disable: true},
{raw: "ORA", mode: "oracle", disable: true},
{raw: "ORACLE", mode: "oracle", disable: true},
{raw: "B", mode: "mysql", mysql: true},
{raw: "M", mode: "mysql", mysql: true},
{raw: "MYSQL", mode: "mysql", mysql: true},
{raw: "PG", mode: "postgres"},
{raw: "POSTGRESQL", mode: "postgres"},
{raw: "MSSQL", mode: "sqlserver", sqlServer: true},
{raw: "SQL_SERVER", mode: "sqlserver", sqlServer: true},
} {
t.Run(test.raw, func(t *testing.T) {
actual := normalizeVastbaseCompatibility(test.raw)
if actual.mode != test.mode || actual.mysqlCompat != test.mysql || actual.sqlServer != test.sqlServer || actual.supportsDisableConstraint != test.disable {
t.Fatalf("normalizeVastbaseCompatibility(%q) = %+v", test.raw, actual)
}
})
}
}
func TestVastbaseModeUsesPostgresCatalog(t *testing.T) {
mode := detectAgentMode(nil, false)
@@ -159,6 +187,34 @@ func TestVastbaseModeUsesPostgresCatalog(t *testing.T) {
if mysqlMode.compatibilityMode != "mysql" || !mysqlMode.postgresCatalog || !mysqlMode.mysqlCompat {
t.Fatalf("unexpected MySQL-compatible Vastbase mode: %+v", mysqlMode)
}
for _, compatibility := range []string{"M", "B", "MYSQL"} {
resolved := vastbaseMode{compatibilityMode: strings.ToLower(compatibility), postgresCatalog: true}
resolved.mysqlCompat = resolved.compatibilityMode == "m" || resolved.compatibilityMode == "b" || resolved.compatibilityMode == "mysql"
if !resolved.mysqlCompat {
t.Fatalf("compatibility %q must use MySQL identifier rules: %+v", compatibility, resolved)
}
}
}
func TestVastbaseDisableConstraintMode(t *testing.T) {
for _, mode := range []struct {
raw string
want bool
}{
{raw: "A", want: false},
{raw: "O", want: true},
{raw: "ORA", want: true},
{raw: "oracle", want: true},
{raw: "B", want: false},
{raw: "M", want: false},
{raw: "mysql", want: false},
{raw: "C", want: false},
} {
compatibility := normalizeVastbaseCompatibility(mode.raw)
if actual := vastbaseSupportsDisableConstraint(vastbaseMode{supportsDisableConstraint: compatibility.supportsDisableConstraint}); actual != mode.want {
t.Fatalf("vastbaseSupportsDisableConstraint(%q) = %v, want %v", mode.raw, actual, mode.want)
}
}
}
func TestSchemaConnectionRecoversAfterCanceledDriverConnection(t *testing.T) {
@@ -222,6 +278,18 @@ func TestDisconnectResetsInformationSchemaCapabilityCache(t *testing.T) {
}
}
func TestDisconnectResetsConstraintCapabilityCache(t *testing.T) {
server := newServer()
server.constraintDefinitionUnsupported = true
server.constraintValidatedUnsupported = true
if err := server.disconnect(); err != nil {
t.Fatal(err)
}
if server.constraintDefinitionUnsupported || server.constraintValidatedUnsupported {
t.Fatal("disconnect must reset cached constraint capabilities")
}
}
var (
registerVastbaseSchemaRetryDriver sync.Once
registerVastbasePingRetryDriver sync.Once
+1 -1
View File
@@ -175,7 +175,7 @@ func (s *server) metadataDatabase() (*sql.DB, error) {
func isMetadataOperation(method string) bool {
switch method {
case "connection_info", "list_databases", "list_schemas", "list_tables", "get_table_comment", "list_objects",
"list_data_types", "completion_assistant_search_v1", "get_columns", "list_indexes", "list_foreign_keys",
"list_data_types", "completion_assistant_search_v1", "get_columns", "list_indexes", "list_foreign_keys", "list_constraints",
"list_triggers", "get_object_source", "get_table_ddl", "get_explain_info":
return true
default:
@@ -219,6 +219,25 @@ func (conn *runtimePoolTestConn) Ping(context.Context) error {
return nil
}
func (*runtimePoolTestConn) QueryContext(context.Context, string, []driver.NamedValue) (driver.Rows, error) {
return &runtimePoolTestModeRows{}, nil
}
type runtimePoolTestModeRows struct {
read bool
}
func (*runtimePoolTestModeRows) Columns() []string { return []string{"datcompatibility"} }
func (*runtimePoolTestModeRows) Close() error { return nil }
func (rows *runtimePoolTestModeRows) Next(destination []driver.Value) error {
if rows.read {
return io.EOF
}
rows.read = true
destination[0] = "A"
return nil
}
type runtimePoolTestStmt struct {
state *runtimePoolTestState
}
+394 -64
View File
@@ -28,10 +28,12 @@ var postgresDataTypes = []string{
}
type vastbaseMode struct {
compatibilityMode string
postgresCatalog bool
mysqlCompat bool
sqlServerIdentity bool
compatibilityMode string
compatibilityModeRaw string
postgresCatalog bool
mysqlCompat bool
sqlServerIdentity bool
supportsDisableConstraint bool
}
type databaseInfo struct {
@@ -114,66 +116,29 @@ type foreignKeyInfo struct {
RefColumn string `json:"ref_column"`
}
type constraintInfo struct {
Name string `json:"name"`
ConstraintType string `json:"constraint_type"`
Definition string `json:"definition"`
Columns []string `json:"columns"`
RefSchema *string `json:"ref_schema,omitempty"`
RefTable *string `json:"ref_table,omitempty"`
RefColumns []string `json:"ref_columns"`
MatchType *string `json:"match_type,omitempty"`
OnUpdate *string `json:"on_update,omitempty"`
OnDelete *string `json:"on_delete,omitempty"`
Deferrable bool `json:"deferrable"`
InitiallyDeferred bool `json:"initially_deferred"`
Enabled bool `json:"enabled"`
Valid bool `json:"valid"`
}
type triggerInfo struct {
Name string `json:"name"`
Event string `json:"event"`
Timing string `json:"timing"`
}
func detectVastbaseMode(db *sql.DB, configuredMySQL bool) vastbaseMode {
if configuredMySQL {
return vastbaseMode{compatibilityMode: "mysql", mysqlCompat: true}
}
mode := vastbaseMode{compatibilityMode: detectDatabaseMode(db)}
mode.postgresCatalog = !catalogExists(db, "sys_catalog.sys_namespace") && catalogExists(db, "pg_catalog.pg_namespace")
if !mode.postgresCatalog {
if mode.compatibilityMode != "" {
mode.mysqlCompat = mode.compatibilityMode == "mysql"
} else {
mode.mysqlCompat = supportsBacktickIdentifiers(db)
}
mode.sqlServerIdentity = !mode.mysqlCompat && catalogExists(db, "sys.identity_columns")
}
return mode
}
func catalogExists(db *sql.DB, catalog string) bool {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
rows, err := db.QueryContext(ctx, "SELECT 1 FROM "+catalog+" WHERE 1 = 0")
if err != nil {
return false
}
return rows.Close() == nil
}
func detectMySQLCompatMode(db *sql.DB) bool {
if databaseMode := detectDatabaseMode(db); databaseMode != "" {
return databaseMode == "mysql"
}
return supportsBacktickIdentifiers(db)
}
func detectDatabaseMode(db *sql.DB) string {
var databaseMode string
switch err := db.QueryRow("SELECT setting FROM sys_catalog.sys_settings WHERE LOWER(name) = 'database_mode'").Scan(&databaseMode); {
case err == nil:
// Treat database_mode as authoritative when the server exposes it. This
// avoids misclassifying Oracle-compatible servers that also publish
// sql_mode for MySQL syntax toggles such as ANSI_QUOTES.
return strings.ToLower(strings.TrimSpace(databaseMode))
case errors.Is(err, sql.ErrNoRows):
return ""
default:
return ""
}
}
func supportsBacktickIdentifiers(db *sql.DB) bool {
var value int
return db.QueryRow("SELECT 1 AS `dbx_identifier_probe`").Scan(&value) == nil
}
func (s *server) identifierQuote() string {
// Vastbase MySQL compatibility mode follows MySQL identifier quoting;
// other modes retain the PostgreSQL-compatible double quote.
@@ -195,8 +160,9 @@ func (s *server) connectionInfo() (map[string]any, error) {
}
return map[string]any{
"database": database, "username": username, "version": version, "schema": schema,
"compatibilityMode": s.mode.compatibilityMode, "mysql_compat_mode": s.mode.mysqlCompat,
"identifierQuote": s.identifierQuote(),
"compatibilityMode": s.mode.compatibilityMode, "compatibilityModeRaw": s.mode.compatibilityModeRaw,
"mysql_compat_mode": s.mode.mysqlCompat,
"identifierQuote": s.identifierQuote(),
"databaseInfo": map[string]string{
"productName": "Vastbase",
"productVersion": version,
@@ -1350,13 +1316,17 @@ func (s *server) queryInformationSchemaColumns(schema, table string, primary map
default:
fullDataTypeExpression = "NULL AS column_type"
}
catalog, prefix := "sys_catalog", "sys"
if s.mode.postgresCatalog {
catalog, prefix = "pg_catalog", "pg"
}
query := fmt.Sprintf(`SELECT c.column_name, c.data_type, %s, c.is_nullable, c.column_default,
col_description(a.attrelid, a.attnum), c.numeric_precision, c.numeric_scale, c.character_maximum_length
FROM information_schema.columns c
LEFT JOIN sys_catalog.sys_namespace n ON n.nspname = c.table_schema
LEFT JOIN sys_catalog.sys_class rel ON rel.relnamespace = n.oid AND rel.relname = c.table_name
LEFT JOIN sys_catalog.sys_attribute a ON a.attrelid = rel.oid AND a.attname = c.column_name AND a.attnum > 0 AND NOT a.attisdropped
WHERE c.table_schema = %s AND c.table_name = %s ORDER BY c.ordinal_position`, fullDataTypeExpression, quoteLiteral(schema), quoteLiteral(table))
LEFT JOIN %s.%s_namespace n ON n.nspname = c.table_schema
LEFT JOIN %s.%s_class rel ON rel.relnamespace = n.oid AND rel.relname = c.table_name
LEFT JOIN %s.%s_attribute a ON a.attrelid = rel.oid AND a.attname = c.column_name AND a.attnum > 0 AND NOT a.attisdropped
WHERE c.table_schema = %s AND c.table_name = %s ORDER BY c.ordinal_position`, fullDataTypeExpression, catalog, prefix, catalog, prefix, catalog, prefix, quoteLiteral(schema), quoteLiteral(table))
rows, err := s.metadataQuery(query)
if err != nil {
return nil, err
@@ -1518,6 +1488,314 @@ WHERE tc.table_schema = ` + quoteLiteral(effective) + ` AND tc.table_name = ` +
return result, rows.Err()
}
func vastbaseConstraintFunctionName(catalog string) string {
if catalog == "pg_catalog" {
return "pg_get_constraintdef"
}
return "sys_get_constraintdef"
}
func vastbaseConstraintsQuery(catalog, prefix, schema, table string, definitionUnsupported, validatedUnsupported, enabledUnsupported bool) string {
definitionExpression := fmt.Sprintf("COALESCE(%s(c.oid, true), '')", vastbaseCatalogFunction(catalog, "sys_get_constraintdef", "pg_get_constraintdef"))
validExpression := "COALESCE(CAST(c.convalidated AS text), 'T')"
enabledExpression := "COALESCE(CAST(c.conenable AS text), 'T')"
if definitionUnsupported {
definitionExpression = "''"
}
if validatedUnsupported {
validExpression = "'T'"
}
if enabledUnsupported {
enabledExpression = "'T'"
}
return fmt.Sprintf(`SELECT COALESCE(c.conname, ''), CAST(c.contype AS text), %s,
CAST(c.conkey AS text), rn.nspname, rt.relname, CAST(c.confkey AS text),
CAST(c.confmatchtype AS text), CAST(c.confupdtype AS text), CAST(c.confdeltype AS text),
c.condeferrable, c.condeferred, %s, %s
FROM %s.%s_constraint c
JOIN %s.%s_class t ON t.oid = c.conrelid
JOIN %s.%s_namespace n ON n.oid = t.relnamespace
LEFT JOIN %s.%s_class rt ON rt.oid = c.confrelid
LEFT JOIN %s.%s_namespace rn ON rn.oid = rt.relnamespace
WHERE n.nspname = %s AND t.relname = %s AND t.relkind IN ('r', 'p', 'f')
ORDER BY COALESCE(c.conname, '')`, definitionExpression, validExpression, enabledExpression, catalog, prefix, catalog, prefix, catalog, prefix, catalog, prefix, catalog, prefix, quoteLiteral(schema), quoteLiteral(table))
}
func (s *server) listConstraints(schema, table string) ([]constraintInfo, error) {
effective, err := s.effectiveSchema(schema)
if err != nil {
return nil, err
}
catalog, prefix := "sys_catalog", "sys"
if s.mode.postgresCatalog {
catalog, prefix = "pg_catalog", "pg"
}
effective, table, err = s.resolveConstraintRelation(catalog, prefix, effective, table)
if err != nil {
return nil, err
}
definitionUnsupported := s.constraintDefinitionUnsupported
validatedUnsupported := s.constraintValidatedUnsupported
enabledUnsupported := s.constraintEnabledUnsupported
var rows *sql.Rows
for {
rows, err = s.metadataQuery(vastbaseConstraintsQuery(catalog, prefix, effective, table, definitionUnsupported, validatedUnsupported, enabledUnsupported))
if err == nil {
break
}
changed := false
if !definitionUnsupported && isUndefinedFunction(err, vastbaseConstraintFunctionName(catalog)) {
definitionUnsupported = true
s.constraintDefinitionUnsupported = true
changed = true
}
if !validatedUnsupported && isUndefinedColumn(err, "convalidated") {
validatedUnsupported = true
s.constraintValidatedUnsupported = true
changed = true
}
if !enabledUnsupported && isUndefinedColumn(err, "conenable") {
enabledUnsupported = true
s.constraintEnabledUnsupported = true
changed = true
}
if !changed {
return nil, err
}
}
defer rows.Close()
type rawConstraint struct {
name, kind, definition string
columnNumbers, refColumnNumbers []int
refSchema, refTable sql.NullString
matchType, onUpdate, onDelete sql.NullString
deferrable, initiallyDeferred, valid, enabled bool
}
raw := []rawConstraint{}
for rows.Next() {
var item rawConstraint
var columnsRaw, refColumnsRaw sql.NullString
var validRaw, enabledRaw any
if err := rows.Scan(&item.name, &item.kind, &item.definition, &columnsRaw, &item.refSchema, &item.refTable, &refColumnsRaw, &item.matchType, &item.onUpdate, &item.onDelete, &item.deferrable, &item.initiallyDeferred, &validRaw, &enabledRaw); err != nil {
return nil, err
}
item.columnNumbers, err = parseVastbaseConstraintAttributeNumbers(columnsRaw.String)
if err != nil {
return nil, fmt.Errorf("failed to parse constraint %s columns: %w", item.name, err)
}
item.refColumnNumbers, err = parseVastbaseConstraintAttributeNumbers(refColumnsRaw.String)
if err != nil {
return nil, fmt.Errorf("failed to parse constraint %s referenced columns: %w", item.name, err)
}
item.valid = parseVastbaseConstraintValid(validRaw)
item.enabled = parseVastbaseConstraintEnabled(enabledRaw)
raw = append(raw, item)
}
if err := rows.Err(); err != nil {
return nil, err
}
attributes, err := s.relationAttributesByNumber(catalog, prefix, effective, table)
if err != nil {
return nil, err
}
refAttributes := map[string]map[int]string{}
result := make([]constraintInfo, 0, len(raw))
for _, item := range raw {
constraint := constraintInfo{
Name: item.name, ConstraintType: vastbaseConstraintTypeName(item.kind), Definition: item.definition,
Columns: []string{}, RefColumns: []string{}, Deferrable: item.deferrable,
InitiallyDeferred: item.initiallyDeferred, Enabled: item.enabled, Valid: item.valid,
}
for _, number := range item.columnNumbers {
if name := attributes[number]; name != "" {
constraint.Columns = append(constraint.Columns, name)
}
}
if item.refSchema.Valid {
constraint.RefSchema = stringPtr(item.refSchema.String)
}
if item.refTable.Valid {
constraint.RefTable = stringPtr(item.refTable.String)
}
if strings.EqualFold(strings.TrimSpace(item.kind), "f") && item.refSchema.Valid && item.refTable.Valid {
key := item.refSchema.String + "\x00" + item.refTable.String
ref := refAttributes[key]
if ref == nil {
ref, err = s.relationAttributesByNumber(catalog, prefix, item.refSchema.String, item.refTable.String)
if err != nil {
return nil, err
}
refAttributes[key] = ref
}
for _, number := range item.refColumnNumbers {
if name := ref[number]; name != "" {
constraint.RefColumns = append(constraint.RefColumns, name)
}
}
constraint.MatchType = vastbaseConstraintMatchType(item.matchType)
constraint.OnUpdate = vastbaseConstraintAction(item.onUpdate)
constraint.OnDelete = vastbaseConstraintAction(item.onDelete)
}
result = append(result, constraint)
}
return result, nil
}
func (s *server) resolveConstraintRelation(catalog, prefix, schema, table string) (string, string, error) {
query := fmt.Sprintf(`SELECT n.nspname, c.relname
FROM %s.%s_class c JOIN %s.%s_namespace n ON n.oid = c.relnamespace
WHERE LOWER(n.nspname) = LOWER(%s) AND LOWER(c.relname) = LOWER(%s)
AND c.relkind IN ('r', 'p', 'f')
ORDER BY CASE WHEN n.nspname = %s AND c.relname = %s THEN 0 ELSE 1 END`, catalog, prefix, catalog, prefix, quoteLiteral(schema), quoteLiteral(table), quoteLiteral(schema), quoteLiteral(table))
rows, err := s.metadataQuery(query)
if err != nil {
return "", "", err
}
defer rows.Close()
type relationName struct{ schema, table string }
matches := []relationName{}
for rows.Next() {
var match relationName
if err := rows.Scan(&match.schema, &match.table); err != nil {
return "", "", err
}
matches = append(matches, match)
}
if err := rows.Err(); err != nil {
return "", "", err
}
if len(matches) == 0 {
return "", "", fmt.Errorf("Vastbase relation not found: %s.%s", schema, table)
}
if matches[0].schema == schema && matches[0].table == table {
return schema, table, nil
}
if len(matches) > 1 {
return "", "", fmt.Errorf("ambiguous Vastbase relation name %s.%s under case-insensitive matching", schema, table)
}
return matches[0].schema, matches[0].table, nil
}
func (s *server) relationAttributesByNumber(catalog, prefix, schema, table string) (map[int]string, error) {
query := fmt.Sprintf(`SELECT a.attnum, a.attname
FROM %s.%s_attribute a JOIN %s.%s_class c ON c.oid = a.attrelid
JOIN %s.%s_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = %s AND c.relname = %s AND a.attnum > 0 AND NOT a.attisdropped`, catalog, prefix, catalog, prefix, catalog, prefix, quoteLiteral(schema), quoteLiteral(table))
rows, err := s.metadataQuery(query)
if err != nil {
return nil, err
}
defer rows.Close()
result := map[int]string{}
for rows.Next() {
var number int
var name string
if err := rows.Scan(&number, &name); err != nil {
return nil, err
}
result[number] = name
}
return result, rows.Err()
}
func parseVastbaseConstraintAttributeNumbers(raw string) ([]int, error) {
value := strings.TrimSpace(strings.Trim(raw, "{}[]"))
if value == "" {
return []int{}, nil
}
parts := strings.FieldsFunc(value, func(r rune) bool { return r == ',' || r == ' ' || r == '\t' })
result := make([]int, 0, len(parts))
for _, part := range parts {
number, err := strconv.Atoi(strings.TrimSpace(part))
if err != nil {
return nil, fmt.Errorf("invalid attribute number %q", part)
}
result = append(result, number)
}
return result, nil
}
func parseVastbaseConstraintValue(raw any) string {
if raw == nil {
return ""
}
if bytes, ok := raw.([]byte); ok {
return strings.ToLower(strings.TrimSpace(string(bytes)))
}
return strings.ToLower(strings.TrimSpace(fmt.Sprint(raw)))
}
func parseVastbaseConstraintValid(raw any) bool {
if value, ok := raw.(bool); ok {
return value
}
switch parseVastbaseConstraintValue(raw) {
case "0", "f", "false", "n", "no", "not validated", "not_validated", "invalid":
return false
default:
return true
}
}
func parseVastbaseConstraintEnabled(raw any) bool {
if value, ok := raw.(bool); ok {
return value
}
switch parseVastbaseConstraintValue(raw) {
case "0", "f", "false", "d", "disabled", "disable", "off":
return false
default:
return true
}
}
func vastbaseConstraintTypeName(value string) string {
switch strings.ToLower(strings.TrimSpace(value)) {
case "p":
return "PRIMARY KEY"
case "f":
return "FOREIGN KEY"
case "u":
return "UNIQUE"
case "c":
return "CHECK"
case "t":
return "CONSTRAINT TRIGGER"
case "x":
return "EXCLUDE"
case "n":
return "NOT NULL"
default:
return strings.TrimSpace(value)
}
}
func vastbaseConstraintMatchType(value sql.NullString) *string {
if !value.Valid {
return nil
}
labels := map[string]string{"f": "FULL", "p": "PARTIAL", "s": "SIMPLE"}
result, ok := labels[strings.ToLower(strings.TrimSpace(value.String))]
if !ok {
return nil
}
return &result
}
func vastbaseConstraintAction(value sql.NullString) *string {
if !value.Valid {
return nil
}
labels := map[string]string{"a": "NO ACTION", "r": "RESTRICT", "c": "CASCADE", "n": "SET NULL", "d": "SET DEFAULT"}
result, ok := labels[strings.ToLower(strings.TrimSpace(value.String))]
if !ok {
return nil
}
return &result
}
func (s *server) listTriggers(schema, table string) ([]triggerInfo, error) {
effective, err := s.effectiveSchema(schema)
if err != nil {
@@ -1791,6 +2069,8 @@ func vastbaseIdentityClause(code string) *string {
clause = "GENERATED ALWAYS AS IDENTITY"
case "d":
clause = "GENERATED BY DEFAULT AS IDENTITY"
case "m":
clause = "IDENTITY"
default:
return nil
}
@@ -1869,6 +2149,10 @@ WHERE tc.table_schema=` + quoteLiteral(schema) + ` AND tc.table_name=` + quoteLi
}
func (s *server) applyIdentityMetadata(schema, table string, columns []columnInfo) {
if s.mode.postgresCatalog {
s.applyPostgresIdentityMetadata(schema, table, columns)
return
}
query := `SELECT a.attname, ic.seed_value, ic.increment_value FROM sys.identity_columns ic
JOIN sys_catalog.sys_class c ON c.oid=ic.object_id JOIN sys_catalog.sys_namespace n ON n.oid=c.relnamespace
JOIN sys_catalog.sys_attribute a ON a.attrelid=c.oid AND a.attnum=ic.column_id
@@ -1898,6 +2182,52 @@ WHERE n.nspname=` + quoteLiteral(schema) + ` AND c.relname=` + quoteLiteral(tabl
}
}
func (s *server) applyPostgresIdentityMetadata(schema, table string, columns []columnInfo) {
query := `SELECT a.attname, pg_get_serial_sequence(quote_ident(n.nspname) || '.' || quote_ident(c.relname), a.attname), a.attidentity
FROM pg_catalog.pg_attribute a
JOIN pg_catalog.pg_class c ON c.oid = a.attrelid
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = ` + quoteLiteral(schema) + ` AND c.relname = ` + quoteLiteral(table) + ` AND a.attidentity <> ''`
rows, err := s.metadataQuery(query)
if err != nil {
return
}
type identityColumn struct {
name, sequence, identity string
}
identityColumns := []identityColumn{}
for rows.Next() {
var item identityColumn
if rows.Scan(&item.name, &item.sequence, &item.identity) == nil {
identityColumns = append(identityColumns, item)
}
}
_ = rows.Close()
byName := map[string]*columnInfo{}
for i := range columns {
byName[strings.ToLower(columns[i].Name)] = &columns[i]
}
for _, item := range identityColumns {
column := byName[strings.ToLower(item.name)]
if column == nil {
continue
}
extra := vastbaseIdentityClause(item.identity)
if extra == nil {
extra = stringPtr("IDENTITY")
}
if item.sequence != "" {
var start, increment sql.NullInt64
sequenceQuery := "SELECT start_value, increment_by FROM " + quoteCatalogIdentifier(item.sequence)
if err := s.requireDBQueryRow(sequenceQuery, &start, &increment); err == nil && start.Valid && increment.Valid {
value := fmt.Sprintf("IDENTITY(%d,%d)", start.Int64, increment.Int64)
extra = &value
}
}
column.Extra = extra
}
}
func catalogPrefix(catalog string) string {
if catalog == "pg_catalog" {
return "pg"
@@ -197,6 +197,156 @@ func openVastbaseCustomTypesDB(t *testing.T, state *vastbaseCustomTypesTestState
return db
}
func TestVastbaseConstraintStatusParsing(t *testing.T) {
for _, test := range []struct {
name string
raw any
valid bool
enabled bool
}{
{name: "boolean true", raw: true, valid: true, enabled: true},
{name: "boolean false", raw: false, valid: false, enabled: false},
{name: "text true", raw: "t", valid: true, enabled: true},
{name: "text false", raw: "f", valid: false, enabled: false},
{name: "one", raw: "1", valid: true, enabled: true},
{name: "zero", raw: "0", valid: false, enabled: false},
{name: "oracle enabled", raw: "E", valid: true, enabled: true},
{name: "oracle disabled", raw: "D", valid: true, enabled: false},
{name: "enabled label", raw: "enabled", valid: true, enabled: true},
{name: "disabled label", raw: "disabled", valid: true, enabled: false},
{name: "not validated", raw: "N", valid: false, enabled: true},
{name: "no", raw: "no", valid: false, enabled: true},
{name: "bytes false", raw: []byte("false"), valid: false, enabled: false},
{name: "null defaults true", raw: nil, valid: true, enabled: true},
{name: "unknown defaults true", raw: "future-state", valid: true, enabled: true},
} {
t.Run(test.name, func(t *testing.T) {
if actual := parseVastbaseConstraintValid(test.raw); actual != test.valid {
t.Fatalf("parseVastbaseConstraintValid(%#v) = %v, want %v", test.raw, actual, test.valid)
}
if actual := parseVastbaseConstraintEnabled(test.raw); actual != test.enabled {
t.Fatalf("parseVastbaseConstraintEnabled(%#v) = %v, want %v", test.raw, actual, test.enabled)
}
})
}
}
func TestVastbaseConstraintAttributeNumbersAreStrict(t *testing.T) {
for _, raw := range []string{"1 2", "{3,2}", "[4, 5]", ""} {
if _, err := parseVastbaseConstraintAttributeNumbers(raw); err != nil {
t.Fatalf("valid vector %q failed: %v", raw, err)
}
}
if _, err := parseVastbaseConstraintAttributeNumbers("1 bad 3"); err == nil {
t.Fatal("invalid constraint vector must return an error")
}
}
func TestVastbaseConstraintQueryNormalizesCatalogCapabilities(t *testing.T) {
query := vastbaseConstraintsQuery("pg_catalog", "pg", "public", "orders", false, false, false)
for _, expected := range []string{
"pg_catalog.pg_get_constraintdef",
"COALESCE(CAST(c.convalidated AS text), 'T')",
"CAST(c.conkey AS text)",
"CAST(c.confkey AS text)",
} {
if !strings.Contains(query, expected) {
t.Fatalf("constraint query missing %q: %s", expected, query)
}
}
fallback := vastbaseConstraintsQuery("pg_catalog", "pg", "public", "orders", true, true, true)
if strings.Contains(fallback, "pg_get_constraintdef") || strings.Contains(fallback, "convalidated") || strings.Contains(fallback, "conenable") {
t.Fatalf("fallback constraint query still contains unsupported fields: %s", fallback)
}
}
func TestVastbaseConstraintRelationResolvesFoldedNames(t *testing.T) {
state := &vastbaseCustomTypesTestState{query: func(query string) (driver.Rows, error) {
if !strings.Contains(query, "LOWER(n.nspname) = LOWER('app')") || !strings.Contains(query, "LOWER(c.relname) = LOWER('orders')") {
return nil, fmt.Errorf("unexpected query: %s", query)
}
return &valueRows{columns: []string{"nspname", "relname"}, rows: [][]driver.Value{{"APP", "ORDERS"}}}, nil
}}
server := newServer()
server.db = openVastbaseCustomTypesDB(t, state)
schema, table, err := server.resolveConstraintRelation("pg_catalog", "pg", "app", "orders")
if err != nil {
t.Fatal(err)
}
if schema != "APP" || table != "ORDERS" {
t.Fatalf("folded relation was not resolved: %s.%s", schema, table)
}
}
func TestVastbaseConstraintRelationRejectsAmbiguousFoldedNames(t *testing.T) {
state := &vastbaseCustomTypesTestState{query: func(string) (driver.Rows, error) {
return &valueRows{columns: []string{"nspname", "relname"}, rows: [][]driver.Value{{"APP", "ORDERS"}, {"App", "Orders"}}}, nil
}}
server := newServer()
server.db = openVastbaseCustomTypesDB(t, state)
if _, _, err := server.resolveConstraintRelation("pg_catalog", "pg", "app", "orders"); err == nil || !strings.Contains(err.Error(), "ambiguous") {
t.Fatalf("ambiguous folded relation must fail, got %v", err)
}
}
func TestVastbaseListConstraintsResolvesCatalogVectors(t *testing.T) {
state := &vastbaseCustomTypesTestState{query: func(query string) (driver.Rows, error) {
switch {
case strings.Contains(query, "LOWER(n.nspname) = LOWER('public')") && strings.Contains(query, "LOWER(c.relname) = LOWER('orders')"):
return &valueRows{columns: []string{"nspname", "relname"}, rows: [][]driver.Value{{"public", "orders"}}}, nil
case strings.Contains(query, "FROM pg_catalog.pg_constraint c"):
return &valueRows{
columns: []string{"conname", "contype", "definition", "conkey", "ref_schema", "ref_table", "confkey", "match_type", "on_update", "on_delete", "condeferrable", "condeferred", "valid", "enabled"},
rows: [][]driver.Value{
{"orders_check", "c", "CHECK (amount > 0)", "{4}", nil, nil, nil, nil, nil, nil, false, false, []byte("t"), []byte("D")},
{"orders_customer_fkey", "f", "FOREIGN KEY (customer_id, region_id) REFERENCES customers(id, region_id)", "{2,3}", "public", "customers", "{1,2}", "s", "a", "c", true, true, "f", true},
},
}, nil
case strings.Contains(query, "SELECT a.attnum, a.attname") && strings.Contains(query, "c.relname = 'orders'"):
return &valueRows{columns: []string{"attnum", "attname"}, rows: [][]driver.Value{{int64(2), "customer_id"}, {int64(3), "region_id"}, {int64(4), "amount"}}}, nil
case strings.Contains(query, "SELECT a.attnum, a.attname") && strings.Contains(query, "c.relname = 'customers'"):
return &valueRows{columns: []string{"attnum", "attname"}, rows: [][]driver.Value{{int64(1), "id"}, {int64(2), "region_id"}}}, nil
default:
return nil, fmt.Errorf("unexpected query: %s", query)
}
}}
server := newServer()
server.db = openVastbaseCustomTypesDB(t, state)
server.mode.postgresCatalog = true
constraints, err := server.listConstraints("public", "orders")
if err != nil {
t.Fatal(err)
}
if len(constraints) != 2 {
t.Fatalf("unexpected constraints: %#v", constraints)
}
check, fk := constraints[0], constraints[1]
if check.ConstraintType != "CHECK" || !check.Valid || check.Enabled || !equalVastbaseStrings(check.Columns, []string{"amount"}) {
t.Fatalf("unexpected check constraint: %#v", check)
}
if fk.ConstraintType != "FOREIGN KEY" || fk.Valid || !fk.Enabled || !fk.Deferrable || !fk.InitiallyDeferred || !equalVastbaseStrings(fk.Columns, []string{"customer_id", "region_id"}) || !equalVastbaseStrings(fk.RefColumns, []string{"id", "region_id"}) {
t.Fatalf("unexpected foreign key constraint: %#v", fk)
}
if fk.MatchType == nil || *fk.MatchType != "SIMPLE" || fk.OnUpdate == nil || *fk.OnUpdate != "NO ACTION" || fk.OnDelete == nil || *fk.OnDelete != "CASCADE" {
t.Fatalf("unexpected foreign key actions: %#v", fk)
}
}
func equalVastbaseStrings(left, right []string) bool {
if len(left) != len(right) {
return false
}
for index := range left {
if left[index] != right[index] {
return false
}
}
return true
}
func TestVastbaseGetColumnsResolvesVisibleRelationSchemaInCatalogQuery(t *testing.T) {
queries := []string{}
state := &vastbaseCustomTypesTestState{query: func(query string) (driver.Rows, error) {
@@ -0,0 +1,197 @@
package main
import (
"os"
"strings"
"testing"
)
// TestLiveXuguIndexPartitionDDL verifies the four index cases that are easy to
// conflate in reconstructed DDL: an ordinary index, a LOCAL index, a GLOBAL
// index with explicit partitions, and a GLOBAL index with subpartitions. It is
// opt-in because CI does not provide a XuguDB service.
func TestLiveXuguIndexPartitionDDL(t *testing.T) {
if os.Getenv("XUGU_LIVE_TEST") != "1" {
t.Skip("set XUGU_LIVE_TEST=1 with XUGU_LIVE_* connection settings to run the XuguDB integration test")
}
params := connectParams{
Host: os.Getenv("XUGU_LIVE_HOST"),
Port: parsePort(os.Getenv("XUGU_LIVE_PORT")),
Database: os.Getenv("XUGU_LIVE_DATABASE"),
Username: os.Getenv("XUGU_LIVE_USERNAME"),
Password: os.Getenv("XUGU_LIVE_PASSWORD"),
}
if params.Host == "" || params.Database == "" || params.Username == "" || params.Password == "" {
t.Fatal("XUGU_LIVE_HOST, XUGU_LIVE_DATABASE, XUGU_LIVE_USERNAME, and XUGU_LIVE_PASSWORD are required")
}
s := newServer()
db, err := openDB(params)
if err != nil {
t.Fatal(err)
}
s.db = db
s.params = params
s.currentDatabase = params.Database
defer s.disconnect()
const (
ordinaryTable = "DBX_IDX_SCOPE_LIVE_ORD_T"
localTable = "DBX_IDX_SCOPE_LIVE_LOC_T"
globalTable = "DBX_IDX_SCOPE_LIVE_GLB_T"
subTable = "DBX_IDX_SCOPE_LIVE_SUB_T"
ordinaryIndex = "DBX_IDX_SCOPE_LIVE_ORD_I"
localIndex = "DBX_IDX_SCOPE_LIVE_LOC_I"
globalIndex = "DBX_IDX_SCOPE_LIVE_GLB_I"
subIndex = "DBX_IDX_SCOPE_LIVE_SUB_I"
)
tables := []string{ordinaryTable, localTable, globalTable, subTable}
for _, table := range tables {
_ = s.execWithReconnect("DROP TABLE IF EXISTS " + table)
}
defer func() {
for _, table := range tables {
_ = s.execWithReconnect("DROP TABLE IF EXISTS " + table)
}
}()
statements := []string{
"CREATE TABLE " + ordinaryTable + " (ID INTEGER, VALUE VARCHAR(20))",
"CREATE INDEX " + ordinaryIndex + " ON " + ordinaryTable + " (ID)",
"CREATE TABLE " + localTable + " (ID INTEGER, VALUE VARCHAR(20)) PARTITION BY RANGE (ID) PARTITIONS (P1 VALUES LESS THAN (100), P2 VALUES LESS THAN (MAXVALUES))",
"CREATE INDEX " + localIndex + " ON " + localTable + " (VALUE) LOCAL",
"CREATE TABLE " + globalTable + " (ID INTEGER, REGION VARCHAR(20))",
"CREATE INDEX " + globalIndex + " ON " + globalTable + " (ID) GLOBAL PARTITION BY LIST (REGION) PARTITIONS (P_CN VALUES ('CN'), P_US VALUES ('US'), P_OTHER VALUES (OTHERVALUES))",
"CREATE TABLE " + subTable + " (ID INTEGER, REGION VARCHAR(20))",
"CREATE INDEX " + subIndex + " ON " + subTable + " (ID) GLOBAL PARTITION BY RANGE (ID) PARTITIONS (P1 VALUES LESS THAN (100), P2 VALUES LESS THAN (MAXVALUES)) SUBPARTITION BY HASH (REGION) SUBPARTITIONS 2",
}
for _, statement := range statements {
if err := s.execWithReconnect(statement); err != nil {
t.Fatalf("setup statement failed: %s: %v", statement, err)
}
}
tests := []struct {
name string
table string
index string
wantLocal bool
wantGlobal bool
wantSub bool
}{
{name: "ordinary", table: ordinaryTable, index: ordinaryIndex},
{name: "local", table: localTable, index: localIndex, wantLocal: true},
{name: "global", table: globalTable, index: globalIndex, wantGlobal: true},
{name: "global with subpartitions", table: subTable, index: subIndex, wantGlobal: true, wantSub: true},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
// Exercise the same reconstructed DDL path used by interactive table
// inspection and by the database-export metadata prefetcher. The
// lower-level index fragment assertion below is not sufficient: a
// regression in table/partition assembly could still omit the index.
fullDDL, err := s.getTableDDL(params.Username, tc.table)
if err != nil {
t.Fatalf("getTableDDL failed: %v", err)
}
if !strings.Contains(strings.ToUpper(fullDDL), "CREATE TABLE") {
t.Fatalf("getTableDDL did not return a CREATE TABLE statement: %s", fullDDL)
}
if !strings.Contains(strings.ToUpper(fullDDL), strings.ToUpper(tc.index)) {
t.Fatalf("getTableDDL omitted index %s: %s", tc.index, fullDDL)
}
t.Logf("full reconstructed table DDL: %s", fullDDL)
indexes, err := s.listIndexes(params.Username, tc.table)
if err != nil {
t.Fatal(err)
}
var got *indexInfo
for i := range indexes {
if strings.EqualFold(indexes[i].Name, tc.index) {
got = &indexes[i]
break
}
}
if got == nil {
t.Fatalf("index %s was not listed: %#v", tc.index, indexes)
}
if got.IsLocal != tc.wantLocal {
t.Fatalf("IsLocal=%v, want %v: %#v", got.IsLocal, tc.wantLocal, *got)
}
if tc.wantGlobal && (got.PartitionType == 0 || !got.PartitionRowsLoaded || len(got.IndexPartitions) == 0) {
t.Fatalf("global partition metadata incomplete: %#v", *got)
}
if tc.wantSub && (got.SubpartitionType == 0 || len(got.IndexSubpartitions) == 0) {
t.Fatalf("subpartition metadata incomplete: %#v", *got)
}
// The table already exists; keep a harmless placeholder statement so
// appendDDLStatement produces the same script shape as a real table
// DDL request without attempting to recreate the table during replay.
ddl := s.appendTableIndexDDL(params.Username, tc.table, "/* existing table DDL */")
t.Logf("reconstructed DDL: %s", ddl)
if tc.wantLocal && !strings.Contains(ddl, " LOCAL") {
t.Fatalf("LOCAL was lost from DDL: %s", ddl)
}
if !tc.wantLocal && !tc.wantGlobal && strings.Contains(ddl, " GLOBAL") {
t.Fatalf("ordinary index was incorrectly labeled GLOBAL: %s", ddl)
}
if tc.wantGlobal && !strings.Contains(ddl, " GLOBAL PARTITION BY ") {
t.Fatalf("GLOBAL partition clause was lost from DDL: %s", ddl)
}
if tc.wantSub && !strings.Contains(ddl, " SUBPARTITION BY HASH ") {
t.Fatalf("GLOBAL subpartition clause was lost from DDL: %s", ddl)
}
// Replay the exact reconstructed index DDL after dropping only this
// isolated test index. This catches syntactically plausible output
// that Xugu nevertheless rejects, especially LOCAL/GLOBAL ordering.
if err := s.execWithReconnect("DROP INDEX IF EXISTS " + tc.table + "." + tc.index); err != nil {
t.Fatalf("drop index before replay: %v", err)
}
if err := s.execWithReconnect(ddl); err != nil {
t.Fatalf("replay reconstructed DDL: %v\nDDL: %s", err, ddl)
}
// Replay the complete table DDL as the database-export path writes it:
// one script containing CREATE TABLE followed by any dependent ALTER
// and CREATE INDEX statements. The agent executes one statement per
// request, so split only the generated statement terminators here.
if err := s.execWithReconnect("DROP TABLE IF EXISTS " + tc.table); err != nil {
t.Fatalf("drop table before full DDL replay: %v", err)
}
for _, statement := range splitGeneratedXuguDDL(fullDDL) {
if err := s.execWithReconnect(statement); err != nil {
t.Fatalf("replay full table DDL: %v\nStatement: %s\nFull DDL: %s", err, statement, fullDDL)
}
}
replayedIndexes, err := s.listIndexes(params.Username, tc.table)
if err != nil {
t.Fatalf("list indexes after full DDL replay: %v", err)
}
if !containsXuguIndex(replayedIndexes, tc.index) {
t.Fatalf("full DDL replay did not recreate index %s: %#v", tc.index, replayedIndexes)
}
})
}
}
func splitGeneratedXuguDDL(ddl string) []string {
var statements []string
for _, part := range strings.Split(ddl, ";") {
part = strings.TrimSpace(part)
if part != "" {
statements = append(statements, part+";")
}
}
return statements
}
func containsXuguIndex(indexes []indexInfo, name string) bool {
for _, index := range indexes {
if strings.EqualFold(index.Name, name) {
return true
}
}
return false
}
+314 -34
View File
@@ -107,6 +107,43 @@ WHERE s.DB_ID = CURRENT_DB_ID
AND s.SCHEMA_NAME = ?
AND t.TABLE_NAME = ?
ORDER BY i.INDEX_NAME`
// Index scope and partition metadata are queried separately from the stable
// index listing query. This keeps ordinary index discovery compatible with
// older Xugu catalog versions while allowing newer versions to preserve
// LOCAL/GLOBAL partition semantics in reconstructed DDL.
const xuguIndexPartitionAttributesSQL = `
SELECT i.INDEX_NAME, i.IS_LOCAL, i.PARTI_TYPE, i.PARTI_NUM, i.PARTI_KEY,
i.SUBPARTI_TYPE, i.SUBPARTI_NUM, i.SUBPARTI_KEY
FROM ALL_INDEXES i
JOIN ALL_TABLES t ON t.DB_ID = i.DB_ID AND t.TABLE_ID = i.TABLE_ID
JOIN ALL_SCHEMAS s ON s.DB_ID = t.DB_ID AND s.SCHEMA_ID = t.SCHEMA_ID
WHERE s.DB_ID = CURRENT_DB_ID
AND s.SCHEMA_NAME = ?
AND t.TABLE_NAME = ?
ORDER BY i.INDEX_NAME`
const xuguIndexPartitionsSQL = `
SELECT i.INDEX_NAME, p.PARTI_NO, p.PARTI_NAME, p.PARTI_VAL
FROM ALL_IDX_PARTIS p
JOIN ALL_INDEXES i ON i.DB_ID = p.DB_ID AND i.INDEX_ID = p.INDEX_ID
JOIN ALL_TABLES t ON t.DB_ID = i.DB_ID AND t.TABLE_ID = i.TABLE_ID
JOIN ALL_SCHEMAS s ON s.DB_ID = t.DB_ID AND s.SCHEMA_ID = t.SCHEMA_ID
WHERE s.DB_ID = CURRENT_DB_ID
AND s.SCHEMA_NAME = ?
AND t.TABLE_NAME = ?
AND i.IS_PRIMARY = FALSE
ORDER BY i.INDEX_NAME, p.PARTI_NO`
const xuguIndexSubpartitionsSQL = `
SELECT i.INDEX_NAME, p.SUBPARTI_NO, p.SUBPARTI_NAME, p.SUBPARTI_VAL
FROM ALL_IDX_SUBPARTIS p
JOIN ALL_INDEXES i ON i.DB_ID = p.DB_ID AND i.INDEX_ID = p.INDEX_ID
JOIN ALL_TABLES t ON t.DB_ID = i.DB_ID AND t.TABLE_ID = i.TABLE_ID
JOIN ALL_SCHEMAS s ON s.DB_ID = t.DB_ID AND s.SCHEMA_ID = t.SCHEMA_ID
WHERE s.DB_ID = CURRENT_DB_ID
AND s.SCHEMA_NAME = ?
AND t.TABLE_NAME = ?
AND i.IS_PRIMARY = FALSE
ORDER BY i.INDEX_NAME, p.SUBPARTI_NO`
const xuguTableMetadataSQL = `
SELECT t.TEMP_TYPE, t.ON_COMMIT_DEL, t.PCTFREE, t.COPY_NUM,
t.PARTI_TYPE, t.PARTI_NUM, t.PARTI_KEY,
@@ -201,6 +238,12 @@ var xuguDataTypes = []string{
"DOUBLE[]",
"FLOAT",
"GUID",
"GEOMETRY",
"GEOGRAPHY",
"BOX2D",
"BOX3D",
"SPHEROID",
"RASTER",
"INT",
"INTEGER",
"INTEGER[]",
@@ -274,12 +317,14 @@ type queryOptions struct {
}
type queryResult struct {
Columns []string `json:"columns"`
ColumnTypes []string `json:"column_types"`
Rows [][]any `json:"rows"`
AffectedRows int64 `json:"affected_rows"`
ExecutionTimeMS int64 `json:"execution_time_ms"`
Truncated bool `json:"truncated"`
Columns []string `json:"columns"`
ColumnTypes []string `json:"column_types"`
SpatialColumns []spatialColumn `json:"spatial_columns,omitempty"`
SpatialValues [][]*uint32 `json:"spatial_values,omitempty"`
Rows [][]any `json:"rows"`
AffectedRows int64 `json:"affected_rows"`
ExecutionTimeMS int64 `json:"execution_time_ms"`
Truncated bool `json:"truncated"`
}
func (r queryResult) MarshalJSON() ([]byte, error) {
@@ -298,14 +343,16 @@ func (r queryResult) MarshalJSON() ([]byte, error) {
}
type queryPageResult struct {
Columns []string `json:"columns"`
ColumnTypes []string `json:"column_types"`
Rows [][]any `json:"rows"`
AffectedRows int64 `json:"affected_rows"`
ExecutionTimeMS int64 `json:"execution_time_ms"`
Truncated bool `json:"truncated"`
SessionID *string `json:"session_id"`
HasMore bool `json:"has_more"`
Columns []string `json:"columns"`
ColumnTypes []string `json:"column_types"`
SpatialColumns []spatialColumn `json:"spatial_columns,omitempty"`
SpatialValues [][]*uint32 `json:"spatial_values,omitempty"`
Rows [][]any `json:"rows"`
AffectedRows int64 `json:"affected_rows"`
ExecutionTimeMS int64 `json:"execution_time_ms"`
Truncated bool `json:"truncated"`
SessionID *string `json:"session_id"`
HasMore bool `json:"has_more"`
}
func (r queryPageResult) MarshalJSON() ([]byte, error) {
@@ -324,11 +371,13 @@ func (r queryPageResult) MarshalJSON() ([]byte, error) {
}
type querySession struct {
rows *sql.Rows
columns []string
columnTypes []string
pending []any
remaining int
rows *sql.Rows
columns []string
columnTypes []string
scanner *xuguRowScanner
pending []any
pendingSpatial []*uint32
remaining int
}
type databaseInfo struct {
@@ -386,7 +435,20 @@ type indexInfo struct {
IndexType *string `json:"index_type"`
IncludedColumns []string `json:"included_columns"`
Comment *string `json:"comment"`
keys []xuguIndexKey
// Partition fields are intentionally internal. The generic DBX index
// protocol does not yet model Xugu-specific index partition clauses, but
// the DDL exporter must retain them to avoid changing index semantics.
IsLocal bool `json:"-"`
PartitionType int `json:"-"`
PartitionCount int `json:"-"`
PartitionKey string `json:"-"`
SubpartitionType int `json:"-"`
SubpartitionCount int `json:"-"`
SubpartitionKey string `json:"-"`
PartitionRowsLoaded bool `json:"-"`
IndexPartitions []xuguPartitionInfo `json:"-"`
IndexSubpartitions []xuguPartitionInfo `json:"-"`
keys []xuguIndexKey
}
func (i indexInfo) MarshalJSON() ([]byte, error) {
@@ -1664,7 +1726,7 @@ func isXuguMetadataAccessError(err error) bool {
catalogObject := false
for _, object := range []string{
"DATABASES", "SCHEMAS", "TABLES", "VIEWS", "COLUMNS", "CONSTRAINTS", "INDEXES",
"TRIGGERS", "PARTIS", "SUBPARTIS", "SEQUENCES", "SYNONYMS", "PROCEDURES", "PACKAGES", "TYPES",
"TRIGGERS", "PARTIS", "SUBPARTIS", "IDX_PARTIS", "IDX_SUBPARTIS", "SEQUENCES", "SYNONYMS", "PROCEDURES", "PACKAGES", "TYPES",
} {
if strings.Contains(message, "ALL_"+object) || strings.Contains(message, "SYS_"+object) {
catalogObject = true
@@ -2648,9 +2710,90 @@ func (s *server) listIndexes(schema, table string) ([]indexInfo, error) {
item.IncludedColumns = []string{}
result = append(result, item)
}
// LOCAL/GLOBAL attributes are best-effort metadata. Keep the stable index
// listing usable when an older Xugu catalog does not expose these columns.
s.loadIndexPartitionMetadata(catalogSchema, catalogTable, result)
return emptyIfNil(result), rows.Err()
}
// loadIndexPartitionMetadata enriches the stable index list with Xugu's
// partition scope and partition definitions. The generic DBX index payload
// does not expose these Xugu-specific fields, so they remain internal and are
// consumed by table DDL reconstruction only.
func (s *server) loadIndexPartitionMetadata(schema, table string, indexes []indexInfo) {
if len(indexes) == 0 {
return
}
byName := make(map[string]*indexInfo, len(indexes))
for i := range indexes {
byName[indexes[i].Name] = &indexes[i]
}
rows, err := s.queryRows(xuguTableCatalogQuery(xuguIndexPartitionAttributesSQL, schema, table), nil)
if err != nil {
return
}
func() {
defer s.closeRows(rows)
for rows.Next() {
var name, local, partitionType, partitionCount, partitionKey any
var subpartitionType, subpartitionCount, subpartitionKey any
if err := rows.Scan(&name, &local, &partitionType, &partitionCount, &partitionKey,
&subpartitionType, &subpartitionCount, &subpartitionKey); err != nil {
return
}
item := byName[xuguString(name)]
if item == nil {
continue
}
item.IsLocal = truthy(local)
item.PartitionType = xuguInt(partitionType)
item.PartitionCount = xuguInt(partitionCount)
item.PartitionKey = xuguString(partitionKey)
item.SubpartitionType = xuguInt(subpartitionType)
item.SubpartitionCount = xuguInt(subpartitionCount)
item.SubpartitionKey = xuguString(subpartitionKey)
}
}()
rows, err = s.queryRows(xuguTableCatalogQuery(xuguIndexPartitionsSQL, schema, table), nil)
if err != nil {
return
}
func() {
defer s.closeRows(rows)
for rows.Next() {
var name, position, partitionName, partitionValue any
if err := rows.Scan(&name, &position, &partitionName, &partitionValue); err != nil {
return
}
if item := byName[xuguString(name)]; item != nil {
item.IndexPartitions = append(item.IndexPartitions, xuguPartitionInfo{
Name: xuguString(partitionName), Value: xuguString(partitionValue),
})
item.PartitionRowsLoaded = true
}
}
}()
rows, err = s.queryRows(xuguTableCatalogQuery(xuguIndexSubpartitionsSQL, schema, table), nil)
if err != nil {
return
}
defer s.closeRows(rows)
for rows.Next() {
var name, position, partitionName, partitionValue any
if err := rows.Scan(&name, &position, &partitionName, &partitionValue); err != nil {
return
}
if item := byName[xuguString(name)]; item != nil {
item.IndexSubpartitions = append(item.IndexSubpartitions, xuguPartitionInfo{
Name: xuguString(partitionName), Value: xuguString(partitionValue),
})
}
}
}
func (s *server) listForeignKeys(schema, table string) ([]foreignKeyInfo, error) {
schema, err := s.normalizeSchema(schema)
if err != nil {
@@ -3404,6 +3547,8 @@ func (s *server) executeQueryPage(opts queryOptions, pageSize int) (queryPageRes
return queryPageResult{
Columns: result.Columns,
ColumnTypes: result.ColumnTypes,
SpatialColumns: result.SpatialColumns,
SpatialValues: result.SpatialValues,
Rows: result.Rows,
AffectedRows: result.AffectedRows,
ExecutionTimeMS: result.ExecutionTimeMS,
@@ -3422,11 +3567,19 @@ func (s *server) executeQueryPage(opts queryOptions, pageSize int) (queryPageRes
return queryPageResult{}, err
}
columnTypes := columnTypeNames(rows)
rowScanner := newXuguRowScanner(len(columns), columnTypes)
columnTypes = rowScanner.spatial.columnTypes
maxRows := opts.MaxRows
if maxRows <= 0 {
maxRows = defaultMaxRows
}
session := &querySession{rows: rows, columns: columns, columnTypes: columnTypes, remaining: maxRows}
session := &querySession{
rows: rows,
columns: columns,
columnTypes: columnTypes,
scanner: rowScanner,
remaining: maxRows,
}
result, err := readQuerySessionPage(session, pageSize)
result.ExecutionTimeMS = time.Since(start).Milliseconds()
if err != nil {
@@ -3488,37 +3641,42 @@ func readQuerySessionPage(session *querySession, pageSize int) (queryPageResult,
pageSize = defaultMaxRows
}
result := queryPageResult{Columns: session.columns, ColumnTypes: session.columnTypes, Rows: [][]any{}, SessionID: nil, HasMore: false}
spatialValues := make([][]*uint32, 0, pageSize)
for len(result.Rows) < pageSize && session.remaining > 0 {
if session.pending != nil {
result.Rows = append(result.Rows, session.pending)
spatialValues = append(spatialValues, session.pendingSpatial)
session.pending = nil
session.pendingSpatial = nil
session.remaining--
continue
}
if !session.rows.Next() {
return result, session.rows.Err()
return finishXuguSpatialPage(result, session.scanner, spatialValues), session.rows.Err()
}
row, err := scanRow(session.rows, len(session.columns))
row, rowSpatial, err := session.scanner.scan(session.rows)
if err != nil {
return queryPageResult{}, err
}
result.Rows = append(result.Rows, row)
spatialValues = append(spatialValues, rowSpatial)
session.remaining--
}
if session.remaining <= 0 {
result.Truncated = true
return result, nil
return finishXuguSpatialPage(result, session.scanner, spatialValues), nil
}
if session.rows.Next() {
row, err := scanRow(session.rows, len(session.columns))
row, rowSpatial, err := session.scanner.scan(session.rows)
if err != nil {
return queryPageResult{}, err
}
session.pending = row
session.pendingSpatial = rowSpatial
result.HasMore = true
return result, nil
return finishXuguSpatialPage(result, session.scanner, spatialValues), nil
}
return result, session.rows.Err()
return finishXuguSpatialPage(result, session.scanner, spatialValues), session.rows.Err()
}
func (s *server) executeQuery(opts queryOptions) (queryResult, error) {
@@ -3565,18 +3723,24 @@ func (s *server) executeSelect(sqlText string, maxRows int, timeoutSecs int) (qu
if err != nil {
return queryResult{}, err
}
result := queryResult{Columns: columns, ColumnTypes: columnTypeNames(rows), Rows: [][]any{}}
columnTypes := columnTypeNames(rows)
scanner := newXuguRowScanner(len(columns), columnTypes)
columnTypes = scanner.spatial.columnTypes
result := queryResult{Columns: columns, ColumnTypes: columnTypes, Rows: [][]any{}}
spatialValues := make([][]*uint32, 0)
for rows.Next() {
if len(result.Rows) >= maxRows {
result.Truncated = true
break
}
values, err := scanRow(rows, len(columns))
values, rowSpatial, err := scanner.scan(rows)
if err != nil {
return queryResult{}, err
}
result.Rows = append(result.Rows, values)
spatialValues = append(spatialValues, rowSpatial)
}
result.SpatialColumns, result.SpatialValues = xuguSpatialResultMetadata(scanner, spatialValues)
return result, rows.Err()
}
@@ -4555,10 +4719,7 @@ func (s *server) appendTableIndexDDL(schema, table, ddl string) string {
builder.WriteString(renderXuguIndexKey(key))
}
builder.WriteByte(')')
if index.IndexType != nil && strings.TrimSpace(*index.IndexType) != "" {
builder.WriteString(" INDEXTYPE IS ")
builder.WriteString(strings.TrimSpace(*index.IndexType))
}
appendXuguIndexOptions(&builder, index)
builder.WriteByte(';')
}
if builder.Len() == 0 {
@@ -4567,6 +4728,125 @@ func (s *server) appendTableIndexDDL(schema, table, ddl string) string {
return appendDDLStatement(ddl, builder.String())
}
func appendXuguIndexOptions(builder *strings.Builder, index indexInfo) {
if index.IndexType != nil && strings.TrimSpace(*index.IndexType) != "" {
builder.WriteString(" INDEXTYPE IS ")
builder.WriteString(strings.TrimSpace(*index.IndexType))
}
if index.IsLocal {
builder.WriteString(" LOCAL")
return
}
if index.PartitionType == 0 || !index.PartitionRowsLoaded {
return
}
partitionDDL := renderXuguIndexPartitionDDL(index)
if partitionDDL != "" {
builder.WriteString(" GLOBAL")
builder.WriteString(partitionDDL)
}
}
func renderXuguIndexPartitionDDL(index indexInfo) string {
typeName := xuguPartitionType(index.PartitionType)
key := strings.TrimSpace(index.PartitionKey)
if typeName == "" || key == "" {
return ""
}
var builder strings.Builder
builder.WriteString(" PARTITION BY ")
builder.WriteString(typeName)
builder.WriteString(" (")
builder.WriteString(key)
builder.WriteByte(')')
if index.PartitionType == 3 {
if index.PartitionCount <= 0 {
return ""
}
builder.WriteString(fmt.Sprintf(" PARTITIONS %d", index.PartitionCount))
} else if len(index.IndexPartitions) > 0 {
for _, partition := range index.IndexPartitions {
if strings.TrimSpace(partition.Name) == "" || strings.TrimSpace(partition.Value) == "" {
return ""
}
}
builder.WriteString(" PARTITIONS (\n")
for i, partition := range index.IndexPartitions {
if i > 0 {
builder.WriteString(",\n")
}
builder.WriteString(" ")
builder.WriteString(quoteIdentifier(partition.Name))
builder.WriteByte(' ')
if index.PartitionType == 1 {
builder.WriteString("VALUES LESS THAN (")
} else {
builder.WriteString("VALUES (")
}
builder.WriteString(strings.TrimSpace(partition.Value))
builder.WriteByte(')')
}
builder.WriteString("\n)")
} else {
return ""
}
subType := xuguPartitionType(index.SubpartitionType)
subKey := strings.TrimSpace(index.SubpartitionKey)
if subType == "" || subKey == "" {
return builder.String()
}
builder.WriteString(" SUBPARTITION BY ")
builder.WriteString(subType)
builder.WriteString(" (")
builder.WriteString(subKey)
builder.WriteByte(')')
if index.SubpartitionType == 3 {
if index.SubpartitionCount <= 0 {
return builderWithoutSubpartitionDDL(index, builder)
}
builder.WriteString(fmt.Sprintf(" SUBPARTITIONS %d", index.SubpartitionCount))
} else if len(index.IndexSubpartitions) > 0 {
for _, partition := range index.IndexSubpartitions {
if strings.TrimSpace(partition.Name) == "" || strings.TrimSpace(partition.Value) == "" {
return builderWithoutSubpartitionDDL(index, builder)
}
}
builder.WriteString(" SUBPARTITIONS (\n")
for i, partition := range index.IndexSubpartitions {
if i > 0 {
builder.WriteString(",\n")
}
builder.WriteString(" ")
builder.WriteString(quoteIdentifier(partition.Name))
builder.WriteByte(' ')
if index.SubpartitionType == 1 {
builder.WriteString("VALUES LESS THAN (")
} else {
builder.WriteString("VALUES (")
}
builder.WriteString(strings.TrimSpace(partition.Value))
builder.WriteByte(')')
}
builder.WriteString("\n)")
}
return builder.String()
}
// builderWithoutSubpartitionDDL returns the complete first-level definition
// when Xugu exposes a subpartition marker but not enough detail to replay it.
// Emitting an incomplete SUBPARTITION clause is worse than preserving a valid
// global index DDL without that optional detail.
func builderWithoutSubpartitionDDL(index indexInfo, builder strings.Builder) string {
firstLevel := index
firstLevel.SubpartitionType = 0
firstLevel.SubpartitionKey = ""
firstLevel.SubpartitionCount = 0
firstLevel.IndexSubpartitions = nil
return renderXuguIndexPartitionDDL(firstLevel)
}
// uniqueKeyColumnSets returns column lists for PRIMARY KEY and UNIQUE constraints.
// Xugu stores those definitions as quoted identifiers inside ALL_CONSTRAINTS.DEFINE.
func uniqueKeyColumnSets(constraints []xuguConstraintInfo) [][]string {
+220
View File
@@ -964,6 +964,173 @@ func TestIndexSQLUsesLowPrivilegeDictionary(t *testing.T) {
}
}
func TestIndexPartitionMetadataUsesLowPrivilegeDictionary(t *testing.T) {
for name, query := range map[string]string{
"index attributes": xuguIndexPartitionAttributesSQL,
"index partitions": xuguIndexPartitionsSQL,
"index subpartitions": xuguIndexSubpartitionsSQL,
} {
t.Run(name, func(t *testing.T) {
upper := strings.ToUpper(query)
for _, want := range []string{"ALL_INDEXES", "ALL_TABLES", "ALL_SCHEMAS", "CURRENT_DB_ID"} {
if !strings.Contains(upper, want) {
t.Fatalf("%s query should contain %s: %s", name, want, query)
}
}
if name != "index attributes" && !strings.Contains(upper, "ALL_IDX_") {
t.Fatalf("%s query should use the low-privilege index partition view: %s", name, query)
}
for _, forbidden := range []string{"SYS_INDEXES", "SYS_IDX_PARTIS", "SYS_IDX_SUBPARTIS"} {
if strings.Contains(upper, forbidden) {
t.Fatalf("%s query should not use %s: %s", name, forbidden, query)
}
}
})
}
if !strings.Contains(strings.ToUpper(xuguIndexPartitionAttributesSQL), "IS_LOCAL") {
t.Fatal("index attributes query must preserve LOCAL scope")
}
for _, query := range []string{xuguIndexPartitionAttributesSQL, xuguIndexPartitionsSQL, xuguIndexSubpartitionsSQL} {
upper := strings.ToUpper(query)
if !strings.Contains(upper, "SCHEMA_NAME = ?") || !strings.Contains(upper, "TABLE_NAME = ?") {
t.Fatalf("index metadata query must be scoped to the resolved schema/table: %s", query)
}
}
}
func TestXuguIndexScopeDDL(t *testing.T) {
indexType := "BTREE"
cases := []struct {
name string
index indexInfo
want string
}{
{
name: "ordinary index does not invent GLOBAL",
index: indexInfo{IndexType: &indexType},
want: " INDEXTYPE IS BTREE",
},
{
name: "spatial index preserves Xugu RTREE type",
index: indexInfo{IndexType: indexTypePtr("RTREE")},
want: " INDEXTYPE IS RTREE",
},
{
name: "local partition index",
index: indexInfo{IndexType: &indexType, IsLocal: true},
want: " INDEXTYPE IS BTREE LOCAL",
},
{
name: "global range partition index",
index: indexInfo{
IndexType: indexTypePtr("BTREE"), PartitionType: 1, PartitionKey: `"CREATED_AT"`,
PartitionRowsLoaded: true,
IndexPartitions: []xuguPartitionInfo{
{Name: "P1", Value: "'2025-01-01'"},
{Name: "P2", Value: "'2026-01-01'"},
},
},
want: " GLOBAL PARTITION BY RANGE (\"CREATED_AT\") PARTITIONS (",
},
{
name: "global hash partition index",
index: indexInfo{
IndexType: indexTypePtr("BTREE"), PartitionType: 3, PartitionCount: 4,
PartitionKey: `"CUSTOMER_ID"`, PartitionRowsLoaded: true,
},
want: " GLOBAL PARTITION BY HASH (\"CUSTOMER_ID\") PARTITIONS 4",
},
{
name: "incomplete global metadata is not emitted",
index: indexInfo{IndexType: indexTypePtr("BTREE"), PartitionType: 1, PartitionKey: `"ID"`},
want: " INDEXTYPE IS BTREE",
},
{
name: "global hash without a count is not emitted",
index: indexInfo{IndexType: indexTypePtr("BTREE"), PartitionType: 3, PartitionKey: `"ID"`, PartitionRowsLoaded: true},
want: " INDEXTYPE IS BTREE",
},
{
name: "malformed global partition row is not emitted",
index: indexInfo{
IndexType: indexTypePtr("BTREE"), PartitionType: 2, PartitionKey: `"REGION"`,
PartitionRowsLoaded: true, IndexPartitions: []xuguPartitionInfo{{Name: "P1"}},
},
want: " INDEXTYPE IS BTREE",
},
{
name: "incomplete subpartition keeps valid first level",
index: indexInfo{
IndexType: indexTypePtr("BTREE"), PartitionType: 2, PartitionKey: `"REGION"`,
PartitionRowsLoaded: true, IndexPartitions: []xuguPartitionInfo{{Name: "P1", Value: "'CN'"}},
SubpartitionType: 3, SubpartitionKey: `"ID"`,
},
want: " GLOBAL PARTITION BY LIST (\"REGION\") PARTITIONS (",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
var builder strings.Builder
appendXuguIndexOptions(&builder, tc.index)
got := builder.String()
if got != tc.want && !strings.Contains(got, tc.want) {
t.Fatalf("index option DDL = %q, want %q", got, tc.want)
}
if tc.name == "ordinary index does not invent GLOBAL" && strings.Contains(got, "GLOBAL") {
t.Fatalf("ordinary index must not be labeled GLOBAL: %q", got)
}
if tc.name == "incomplete subpartition keeps valid first level" && strings.Contains(got, "SUBPARTITION") {
t.Fatalf("incomplete subpartition metadata must not produce a partial clause: %q", got)
}
})
}
}
func TestXuguIndexTypeName(t *testing.T) {
for _, tc := range []struct {
value any
want string
}{
{value: int64(0), want: "BTREE"},
{value: int64(1), want: "RTREE"},
{value: int64(2), want: "FULLTEXT"},
{value: int64(3), want: "BITMAP"},
{value: "RTREE", want: "RTREE"},
{value: "vendor-specific", want: "vendor-specific"},
} {
t.Run(fmt.Sprint(tc.value), func(t *testing.T) {
if got := indexTypeName(tc.value); got != tc.want {
t.Fatalf("indexTypeName(%v) = %q, want %q", tc.value, got, tc.want)
}
})
}
}
func indexTypePtr(value string) *string { return &value }
func TestXuguIndexPartitionDetailsStayInternalToTheGenericPayload(t *testing.T) {
data, err := json.Marshal(indexInfo{
Name: "IDX_LOCAL", Columns: []string{"ID"}, IsLocal: true,
PartitionType: 1, PartitionKey: `"ID"`, PartitionRowsLoaded: true,
IndexPartitions: []xuguPartitionInfo{{Name: "P1", Value: "MAXVALUES"}},
})
if err != nil {
t.Fatal(err)
}
text := string(data)
for _, forbidden := range []string{"is_local", "partition_type", "partition_key", "index_partitions"} {
if strings.Contains(text, forbidden) {
t.Fatalf("Xugu-specific index partition field %q leaked into generic metadata: %s", forbidden, text)
}
}
for _, required := range []string{`"name":"IDX_LOCAL"`, `"columns":["ID"]`} {
if !strings.Contains(text, required) {
t.Fatalf("generic index metadata lost %q: %s", required, text)
}
}
}
func TestTableChildMetadataUsesLowPrivilegeDictionary(t *testing.T) {
for name, query := range map[string]string{
"constraints": xuguTableConstraintsSQL,
@@ -2448,6 +2615,7 @@ func init() {
sql.Register("xugu-test-eof", &xuguEOFDriver{})
sql.Register("xugu-test-trigger-details", &xuguTriggerDetailsDriver{})
sql.Register("xugu-test-schema-listing", &xuguSchemaListingDriver{})
sql.Register("xugu-test-index-partition-fallback", &xuguIndexPartitionFallbackDriver{})
}
var xuguSchemaListingState struct {
@@ -2604,6 +2772,24 @@ func TestMetadataPermissionFallbackDoesNotReturnRPCError(t *testing.T) {
}
}
func TestIndexListingSurvivesUnavailablePartitionCatalog(t *testing.T) {
db, err := sql.Open("xugu-test-index-partition-fallback", "")
if err != nil {
t.Fatal(err)
}
defer db.Close()
s := newServer()
s.db = db
indexes, err := s.listIndexes("APP", "T")
if err != nil {
t.Fatal(err)
}
if len(indexes) != 1 || indexes[0].Name != "IDX_T" || indexes[0].IsLocal {
t.Fatalf("stable index listing should survive unavailable partition metadata: %#v", indexes)
}
}
func TestGetColumnsFallsBackToDirectObjectAccessOnMetadataPermission(t *testing.T) {
db, err := sql.Open("xugu-test-permission-metadata", "")
if err != nil {
@@ -2722,6 +2908,40 @@ type xuguShowResultDriver struct{}
type xuguPermissionMetadataDriver struct{}
type xuguIndexPartitionFallbackDriver struct{}
func (d *xuguIndexPartitionFallbackDriver) Open(string) (driver.Conn, error) {
return &xuguIndexPartitionFallbackConn{}, nil
}
type xuguIndexPartitionFallbackConn struct{}
func (c *xuguIndexPartitionFallbackConn) Prepare(string) (driver.Stmt, error) {
return nil, errors.New("not supported")
}
func (c *xuguIndexPartitionFallbackConn) Close() error { return nil }
func (c *xuguIndexPartitionFallbackConn) Begin() (driver.Tx, error) {
return nil, errors.New("not supported")
}
func (c *xuguIndexPartitionFallbackConn) QueryContext(_ context.Context, query string, _ []driver.NamedValue) (driver.Rows, error) {
upper := strings.ToUpper(query)
switch {
case strings.Contains(upper, "SELECT S.SCHEMA_NAME, T.TABLE_NAME"):
return &xuguStaticRows{columns: []string{"SCHEMA_NAME", "TABLE_NAME"}, values: [][]driver.Value{{"APP", "T"}}}, nil
case strings.Contains(upper, "I.IS_LOCAL"):
return nil, errors.New("unknown column IS_LOCAL in older Xugu catalog")
case strings.Contains(upper, "FROM ALL_IDX_PARTIS"), strings.Contains(upper, "FROM ALL_IDX_SUBPARTIS"):
return nil, errors.New("index partition views are unavailable in older Xugu catalog")
case strings.Contains(upper, "SELECT I.INDEX_NAME, I.KEYS"):
return &xuguStaticRows{
columns: []string{"INDEX_NAME", "KEYS", "IS_UNIQUE", "IS_PRIMARY", "INDEX_TYPE", "FILTER"},
values: [][]driver.Value{{"IDX_T", `"ID"`, false, false, int64(0), nil}},
}, nil
default:
return nil, fmt.Errorf("unexpected index fallback query: %s", query)
}
}
type xuguFallbackErrorDriver struct{}
func (d *xuguFallbackErrorDriver) Open(name string) (driver.Conn, error) {
+484
View File
@@ -0,0 +1,484 @@
package main
import (
"database/sql"
"encoding/binary"
"encoding/hex"
"fmt"
"math"
"strconv"
"strings"
"unicode/utf8"
)
// spatialColumn and the corresponding per-cell SRID values are part of the
// common agent query-result protocol consumed by the desktop map preview.
type spatialColumn struct {
ColumnIndex int `json:"column_index"`
SRID *uint32 `json:"srid"`
}
type xuguRowScanner struct {
values []any
targets []any
spatial *xuguSpatialDecoder
}
func newXuguRowScanner(count int, columnTypes []string) *xuguRowScanner {
if len(columnTypes) < count {
expanded := make([]string, count)
copy(expanded, columnTypes)
columnTypes = expanded
}
scanner := &xuguRowScanner{
values: make([]any, count),
targets: make([]any, count),
spatial: newXuguSpatialDecoder(columnTypes),
}
for index := range scanner.targets {
scanner.targets[index] = &scanner.values[index]
}
return scanner
}
func (scanner *xuguRowScanner) scan(rows *sql.Rows) ([]any, []*uint32, error) {
for index := range scanner.values {
scanner.values[index] = nil
}
if err := rows.Scan(scanner.targets...); err != nil {
return nil, nil, err
}
result := make([]any, len(scanner.values))
copy(result, scanner.values)
return scanner.spatial.normalizeRow(result)
}
type xuguSpatialDecoder struct {
columnTypes []string
indices []int
candidates []int
spatial []bool
observed bool
sridByColumn map[int]uint32
}
func newXuguSpatialDecoder(columnTypes []string) *xuguSpatialDecoder {
decoder := &xuguSpatialDecoder{
columnTypes: columnTypes,
spatial: make([]bool, len(columnTypes)),
sridByColumn: map[int]uint32{},
}
for index, columnType := range columnTypes {
if isXuguSpatialColumnType(columnType) {
decoder.indices = append(decoder.indices, index)
decoder.spatial[index] = true
continue
}
// The current Xugu Go driver leaves DatabaseTypeName empty for direct
// GEOMETRY/GEOGRAPHY projections. Keep only those unknown columns as
// candidates; explicit BLOB/BINARY columns are never auto-classified.
if strings.TrimSpace(columnType) == "" {
decoder.candidates = append(decoder.candidates, index)
}
}
return decoder
}
func isXuguSpatialColumnType(columnType string) bool {
normalized := strings.ToLower(strings.TrimSpace(columnType))
if index := strings.LastIndexByte(normalized, '.'); index >= 0 {
normalized = normalized[index+1:]
}
if index := strings.IndexByte(normalized, '('); index >= 0 {
normalized = normalized[:index]
}
return normalized == "geometry" || normalized == "geography"
}
func (decoder *xuguSpatialDecoder) normalizeRow(values []any) ([]any, []*uint32, error) {
rowSRIDs := make([]*uint32, len(values))
for _, index := range decoder.candidates {
if index >= len(values) || decoder.spatial[index] {
continue
}
decoded, srid, recognized := decodeXuguSpatialValue(values[index])
if !recognized {
continue
}
decoder.indices = append(decoder.indices, index)
decoder.spatial[index] = true
decoder.columnTypes[index] = "GEOMETRY"
values[index] = decoded
rowSRIDs[index] = srid
decoder.recordSRID(index, srid)
}
for _, index := range decoder.indices {
if index >= len(values) || rowSRIDs[index] != nil {
continue
}
decoded, srid, _ := decodeXuguSpatialValue(values[index])
values[index] = decoded
rowSRIDs[index] = srid
decoder.recordSRID(index, srid)
}
for index, value := range values {
if !decoder.spatial[index] {
values[index] = normalizeValue(value)
}
}
if len(decoder.indices) > 0 {
decoder.observed = true
}
return values, rowSRIDs, nil
}
func (decoder *xuguSpatialDecoder) recordSRID(index int, srid *uint32) {
if srid == nil {
return
}
if _, exists := decoder.sridByColumn[index]; !exists {
decoder.sridByColumn[index] = *srid
}
}
func (decoder *xuguSpatialDecoder) columns() []spatialColumn {
if !decoder.observed {
return nil
}
columns := make([]spatialColumn, 0, len(decoder.indices))
for _, index := range decoder.indices {
var srid *uint32
if value, found := decoder.sridByColumn[index]; found {
srid = uint32Pointer(value)
}
columns = append(columns, spatialColumn{ColumnIndex: index, SRID: srid})
}
return columns
}
func xuguSpatialResultMetadata(scanner *xuguRowScanner, values [][]*uint32) ([]spatialColumn, [][]*uint32) {
columns := scanner.spatial.columns()
if len(columns) == 0 {
return nil, nil
}
return columns, values
}
func finishXuguSpatialPage(result queryPageResult, scanner *xuguRowScanner, values [][]*uint32) queryPageResult {
result.SpatialColumns, result.SpatialValues = xuguSpatialResultMetadata(scanner, values)
return result
}
func decodeXuguSpatialValue(value any) (any, *uint32, bool) {
if value == nil {
return nil, nil, false
}
switch typed := value.(type) {
case []byte:
if decoded, ok := decodeXuguWKB(typed); ok {
return decoded.WKT, decoded.SRID, true
}
if utf8.Valid(typed) {
return decodeXuguSpatialText(string(typed))
}
return "0x" + hex.EncodeToString(typed), nil, false
case string:
return decodeXuguSpatialText(typed)
default:
return fmt.Sprint(normalizeValue(value)), nil, false
}
}
func decodeXuguSpatialText(value string) (any, *uint32, bool) {
if strings.HasPrefix(strings.ToUpper(value), "SRID=") {
if separator := strings.IndexByte(value, ';'); separator > 5 {
if parsed, err := strconv.ParseUint(value[5:separator], 10, 32); err == nil {
srid := uint32(parsed)
wkt := strings.TrimSpace(value[separator+1:])
if isXuguWKT(wkt) {
return wkt, uint32Pointer(srid), true
}
}
}
}
if raw, ok := parseXuguSpatialHex(value); ok {
if decoded, decodedOK := decodeXuguWKB(raw); decodedOK {
return decoded.WKT, decoded.SRID, true
}
}
// Plain WKT is valid for a declared spatial column. It is intentionally
// not used to promote an unknown column, because free-form text can look
// like WKT and must not change a non-spatial result's representation.
return value, nil, false
}
func isXuguWKT(value string) bool {
upper := strings.ToUpper(strings.TrimSpace(value))
for _, prefix := range []string{"POINT", "LINESTRING", "POLYGON", "MULTIPOINT", "MULTILINESTRING", "MULTIPOLYGON", "GEOMETRYCOLLECTION"} {
if strings.HasPrefix(upper, prefix+"(") || strings.HasPrefix(upper, prefix+" ") {
return true
}
}
return false
}
func parseXuguSpatialHex(value string) ([]byte, bool) {
normalized := strings.TrimSpace(value)
if strings.HasPrefix(normalized, "0x") || strings.HasPrefix(normalized, "0X") || strings.HasPrefix(normalized, `\x`) || strings.HasPrefix(normalized, `\X`) {
normalized = normalized[2:]
}
if len(normalized) < 10 || len(normalized)%2 != 0 || (normalized[:2] != "00" && normalized[:2] != "01") {
return nil, false
}
raw, err := hex.DecodeString(normalized)
return raw, err == nil
}
func uint32Pointer(value uint32) *uint32 {
result := value
return &result
}
type decodedXuguWKB struct {
WKT string
SRID *uint32
}
type xuguWKBDimensions struct{ z, m bool }
func (dimensions xuguWKBDimensions) size() int {
size := 2
if dimensions.z {
size++
}
if dimensions.m {
size++
}
return size
}
func (dimensions xuguWKBDimensions) suffix() string {
if dimensions.z && dimensions.m {
return " ZM"
}
if dimensions.z {
return " Z"
}
if dimensions.m {
return " M"
}
return ""
}
type xuguWKBReader struct {
raw []byte
pos int
}
func (reader *xuguWKBReader) uint32(order binary.ByteOrder) (uint32, bool) {
if reader.pos+4 > len(reader.raw) {
return 0, false
}
value := order.Uint32(reader.raw[reader.pos : reader.pos+4])
reader.pos += 4
return value, true
}
func (reader *xuguWKBReader) float64(order binary.ByteOrder) (float64, bool) {
if reader.pos+8 > len(reader.raw) {
return 0, false
}
value := math.Float64frombits(order.Uint64(reader.raw[reader.pos : reader.pos+8]))
reader.pos += 8
return value, true
}
func (reader *xuguWKBReader) byteOrder() (binary.ByteOrder, bool) {
if reader.pos >= len(reader.raw) {
return nil, false
}
value := reader.raw[reader.pos]
reader.pos++
if value == 0 {
return binary.BigEndian, true
}
if value == 1 {
return binary.LittleEndian, true
}
return nil, false
}
func decodeXuguWKB(raw []byte) (decodedXuguWKB, bool) {
reader := &xuguWKBReader{raw: raw}
wkt, srid, ok := reader.geometry(0)
if !ok || reader.pos != len(raw) {
return decodedXuguWKB{}, false
}
return decodedXuguWKB{WKT: wkt, SRID: srid}, true
}
func (reader *xuguWKBReader) geometry(depth int) (string, *uint32, bool) {
if depth > 64 {
return "", nil, false
}
order, ok := reader.byteOrder()
if !ok {
return "", nil, false
}
typeWord, ok := reader.uint32(order)
if !ok {
return "", nil, false
}
kind, dimensions, hasSRID := parseXuguWKBType(typeWord)
var srid *uint32
if hasSRID {
value, valueOK := reader.uint32(order)
if !valueOK {
return "", nil, false
}
if value != 0 {
srid = uint32Pointer(value)
}
}
name := map[uint32]string{1: "POINT", 2: "LINESTRING", 3: "POLYGON", 4: "MULTIPOINT", 5: "MULTILINESTRING", 6: "MULTIPOLYGON", 7: "GEOMETRYCOLLECTION"}[kind]
if name == "" {
return "", nil, false
}
suffix := dimensions.suffix()
switch kind {
case 1:
point, empty, pointOK := reader.point(order, dimensions)
if !pointOK {
return "", nil, false
}
if empty {
return name + suffix + " EMPTY", srid, true
}
return name + suffix + "(" + point + ")", srid, true
case 2:
points, pointsOK := reader.points(order, dimensions)
if !pointsOK {
return "", nil, false
}
if len(points) == 0 {
return name + suffix + " EMPTY", srid, true
}
return name + suffix + "(" + strings.Join(points, ",") + ")", srid, true
case 3:
rings, ringsOK := reader.rings(order, dimensions)
if !ringsOK {
return "", nil, false
}
if len(rings) == 0 {
return name + suffix + " EMPTY", srid, true
}
return name + suffix + "(" + strings.Join(rings, ",") + ")", srid, true
case 4, 5, 6, 7:
count, countOK := reader.uint32(order)
if !countOK || uint64(count)*5 > uint64(len(reader.raw)-reader.pos) {
return "", nil, false
}
parts := make([]string, int(count))
for index := range parts {
child, _, childOK := reader.geometry(depth + 1)
if !childOK {
return "", nil, false
}
switch kind {
case 4:
if !strings.HasPrefix(child, "POINT"+suffix) {
return "", nil, false
}
rest := strings.TrimPrefix(child, "POINT"+suffix)
if strings.HasPrefix(rest, " EMPTY") {
parts[index] = "EMPTY"
} else if strings.HasPrefix(rest, "(") && strings.HasSuffix(rest, ")") {
parts[index] = "(" + strings.TrimSuffix(strings.TrimPrefix(rest, "("), ")") + ")"
}
case 5:
parts[index] = trimXuguWKBChild(child, "LINESTRING", suffix)
case 6:
parts[index] = trimXuguWKBChild(child, "POLYGON", suffix)
default:
parts[index] = child
}
if parts[index] == "" {
return "", nil, false
}
}
if len(parts) == 0 {
return name + suffix + " EMPTY", srid, true
}
return name + suffix + "(" + strings.Join(parts, ",") + ")", srid, true
}
return "", nil, false
}
func trimXuguWKBChild(value, name, suffix string) string {
prefix := name + suffix
if !strings.HasPrefix(value, prefix+"(") || !strings.HasSuffix(value, ")") {
return ""
}
return "(" + strings.TrimSuffix(strings.TrimPrefix(value, prefix+"("), ")") + ")"
}
func parseXuguWKBType(typeWord uint32) (uint32, xuguWKBDimensions, bool) {
kind := typeWord & 0x1fffffff
dimensions := xuguWKBDimensions{z: typeWord&0x80000000 != 0, m: typeWord&0x40000000 != 0}
hasSRID := typeWord&0x20000000 != 0
switch {
case kind >= 3000:
dimensions.z, dimensions.m, kind = true, true, kind-3000
case kind >= 2000:
dimensions.m, kind = true, kind-2000
case kind >= 1000:
dimensions.z, kind = true, kind-1000
}
return kind, dimensions, hasSRID
}
func (reader *xuguWKBReader) point(order binary.ByteOrder, dimensions xuguWKBDimensions) (string, bool, bool) {
values := make([]string, dimensions.size())
empty := true
for index := range values {
value, ok := reader.float64(order)
if !ok {
return "", false, false
}
empty = empty && math.IsNaN(value)
values[index] = strconv.FormatFloat(value, 'g', -1, 64)
}
return strings.Join(values, " "), empty, true
}
func (reader *xuguWKBReader) points(order binary.ByteOrder, dimensions xuguWKBDimensions) ([]string, bool) {
count, ok := reader.uint32(order)
if !ok || uint64(count)*uint64(dimensions.size())*8 > uint64(len(reader.raw)-reader.pos) {
return nil, false
}
points := make([]string, int(count))
for index := range points {
point, empty, pointOK := reader.point(order, dimensions)
if !pointOK || empty {
return nil, false
}
points[index] = point
}
return points, true
}
func (reader *xuguWKBReader) rings(order binary.ByteOrder, dimensions xuguWKBDimensions) ([]string, bool) {
count, ok := reader.uint32(order)
if !ok || uint64(count)*4 > uint64(len(reader.raw)-reader.pos) {
return nil, false
}
rings := make([]string, int(count))
for index := range rings {
points, pointsOK := reader.points(order, dimensions)
if !pointsOK {
return nil, false
}
rings[index] = "(" + strings.Join(points, ",") + ")"
}
return rings, true
}
@@ -0,0 +1,89 @@
package main
import (
"os"
"strings"
"testing"
)
// TestLiveXuguSpatialIndexDDL verifies the Xugu-specific spatial index
// contract against a real server. Xugu reports spatial indexes as RTREE and
// reconstructed DDL must retain INDEXTYPE IS RTREE; emitting PostgreSQL's
// USING GIST syntax would not be executable on XuguDB.
func TestLiveXuguSpatialIndexDDL(t *testing.T) {
if os.Getenv("XUGU_LIVE_TEST") != "1" {
t.Skip("set XUGU_LIVE_TEST=1 with XUGU_LIVE_* connection settings to run the XuguDB integration test")
}
params := connectParams{
Host: os.Getenv("XUGU_LIVE_HOST"),
Port: parsePort(os.Getenv("XUGU_LIVE_PORT")),
Database: os.Getenv("XUGU_LIVE_DATABASE"),
Username: os.Getenv("XUGU_LIVE_USERNAME"),
Password: os.Getenv("XUGU_LIVE_PASSWORD"),
}
if params.Host == "" || params.Database == "" || params.Username == "" || params.Password == "" {
t.Fatal("XUGU_LIVE_HOST, XUGU_LIVE_DATABASE, XUGU_LIVE_USERNAME, and XUGU_LIVE_PASSWORD are required")
}
s := newServer()
db, err := openDB(params)
if err != nil {
t.Fatal(err)
}
s.db = db
s.params = params
s.currentDatabase = params.Database
defer s.disconnect()
const (
table = "DBX_SPATIAL_INDEX_LIVE_T"
index = "DBX_SPATIAL_INDEX_LIVE_I"
)
_ = s.execWithReconnect("DROP TABLE IF EXISTS " + table)
defer func() { _ = s.execWithReconnect("DROP TABLE IF EXISTS " + table) }()
for _, statement := range []string{
"CREATE TABLE " + table + " (ID INTEGER, GEOM GEOMETRY)",
"CREATE INDEX " + index + " ON " + table + " (GEOM) INDEXTYPE IS RTREE",
} {
if err := s.execWithReconnect(statement); err != nil {
t.Fatalf("setup statement failed: %s: %v", statement, err)
}
}
indexes, err := s.listIndexes(params.Username, table)
if err != nil {
t.Fatal(err)
}
var spatial *indexInfo
for i := range indexes {
if strings.EqualFold(indexes[i].Name, index) {
spatial = &indexes[i]
break
}
}
if spatial == nil {
t.Fatalf("spatial index %s was not listed: %#v", index, indexes)
}
if spatial.IndexType == nil || !strings.EqualFold(strings.TrimSpace(*spatial.IndexType), "RTREE") {
t.Fatalf("spatial index type = %#v, want RTREE: %#v", spatial.IndexType, *spatial)
}
if len(spatial.Columns) != 1 || !strings.EqualFold(spatial.Columns[0], "GEOM") {
t.Fatalf("spatial index columns = %#v, want [GEOM]", spatial.Columns)
}
ddl, err := s.getTableDDL(params.Username, table)
if err != nil {
t.Fatal(err)
}
upperDDL := strings.ToUpper(ddl)
if !strings.Contains(upperDDL, "CREATE INDEX") || !strings.Contains(upperDDL, strings.ToUpper(index)) {
t.Fatalf("table DDL omitted spatial index: %s", ddl)
}
if !strings.Contains(upperDDL, "INDEXTYPE IS RTREE") {
t.Fatalf("table DDL did not preserve Xugu RTREE syntax: %s", ddl)
}
if strings.Contains(upperDDL, "USING GIST") {
t.Fatalf("table DDL emitted PostgreSQL-only GIST syntax: %s", ddl)
}
}
+186
View File
@@ -0,0 +1,186 @@
package main
import (
"fmt"
"os"
"strings"
"testing"
)
// TestLiveXuguSpatialQueryRegression exercises the value path used by the
// desktop map preview against a real Xugu server. It is opt-in because CI does
// not provide a Xugu service. The test deliberately covers both GEOMETRY and
// GEOGRAPHY columns, explicit SRIDs, EWKT input, and the multi-geometry types
// rendered by the shared map preview.
func TestLiveXuguSpatialQueryRegression(t *testing.T) {
if os.Getenv("XUGU_LIVE_TEST") != "1" {
t.Skip("set XUGU_LIVE_TEST=1 with XUGU_LIVE_* connection settings to run the XuguDB integration test")
}
params := liveXuguParams(t)
s := newServer()
db, err := openDB(params)
if err != nil {
t.Fatal(err)
}
s.db = db
s.params = params
s.currentDatabase = params.Database
defer s.disconnect()
const table = "DBX_SPATIAL_VALUE_LIVE_T"
_ = s.execWithReconnect("DROP TABLE IF EXISTS " + table)
defer func() { _ = s.execWithReconnect("DROP TABLE IF EXISTS " + table) }()
for _, statement := range []string{
"CREATE TABLE " + table + " (ID INTEGER, G_POINT GEOMETRY, G_MULTI GEOMETRY, G_MULTI_LINE GEOMETRY, G_COLLECTION GEOMETRY, G_GEO GEOGRAPHY)",
"INSERT INTO " + table + " VALUES (1, ST_GeomFromEWKT('SRID=4326;POINT(116.397 39.908)'), ST_GeomFromEWKT('SRID=3857;MULTIPOINT((1 2),(3 4))'), ST_GeomFromEWKT('SRID=4326;MULTILINESTRING((0 0,1 1),(2 2,3 3))'), ST_GeomFromEWKT('SRID=4326;GEOMETRYCOLLECTION(POINT(1 2),LINESTRING(3 4,5 6))'), ST_GeomFromEWKT('SRID=4326;POINT(116.397 39.908)'))",
"INSERT INTO " + table + " VALUES (2, ST_GeomFromText('LINESTRING(0 0,1 1,2 2)', 0), ST_GeomFromText('MULTIPOLYGON(((0 0,0 1,1 1,0 0)))', 4326), ST_GeomFromText('MULTILINESTRING((4 4,5 5),(6 6,7 7))', 4326), ST_GeomFromText('POLYGON((0 0,0 1,1 1,0 0))', 4326), ST_GeomFromText('POINT(10 20)', 4326))",
} {
if err := s.execWithReconnect(statement); err != nil {
t.Fatalf("setup statement failed: %s: %v", statement, err)
}
}
result, err := s.executeSelect("SELECT ID, G_POINT, G_MULTI, G_MULTI_LINE, G_COLLECTION, G_GEO FROM "+table+" ORDER BY ID", 20, 0)
if err != nil {
t.Fatal(err)
}
if len(result.Rows) != 2 {
t.Fatalf("row count = %d, want 2: %#v", len(result.Rows), result.Rows)
}
// The current Go driver reports direct GEOGRAPHY projections without a
// DatabaseTypeName. The decoder deliberately classifies a valid spatial
// payload as GEOMETRY in that case. This is sufficient for the shared map
// preview (which accepts GEOMETRY and GEOGRAPHY), while catalog metadata
// still exposes the source column as GEOGRAPHY.
wantTypes := []string{"INTEGER", "GEOMETRY", "GEOMETRY", "GEOMETRY", "GEOMETRY", "GEOMETRY"}
for i, want := range wantTypes {
if i >= len(result.ColumnTypes) || !strings.EqualFold(result.ColumnTypes[i], want) {
t.Fatalf("column %d type = %#v, want %s (columns=%#v)", i, result.ColumnTypes, want, result.Columns)
}
}
if len(result.SpatialColumns) != 5 {
t.Fatalf("spatial column metadata = %#v, want five geometry/geography columns", result.SpatialColumns)
}
for rowIndex, row := range result.Rows {
for _, columnIndex := range []int{1, 2, 3, 4, 5} {
if row[columnIndex] == nil {
t.Fatalf("row %d column %d unexpectedly NULL: %#v", rowIndex, columnIndex, row)
}
if !isXuguWKT(fmt.Sprint(row[columnIndex])) {
t.Fatalf("row %d column %d was not decoded as WKT: %#v", rowIndex, columnIndex, row[columnIndex])
}
}
}
if len(result.SpatialValues) != 2 || len(result.SpatialValues[0]) != 6 {
t.Fatalf("spatial cell metadata shape = %#v", result.SpatialValues)
}
if result.SpatialValues[0][1] == nil || *result.SpatialValues[0][1] != 4326 || result.SpatialValues[0][5] == nil || *result.SpatialValues[0][5] != 4326 {
t.Fatalf("row 1 SRIDs = %#v, want geometry/geography SRID 4326", result.SpatialValues[0])
}
if result.SpatialValues[0][2] == nil || *result.SpatialValues[0][2] != 3857 {
t.Fatalf("row 1 multipoint SRID = %#v, want 3857", result.SpatialValues[0][2])
}
raw, err := s.executeSelect("SELECT ST_AsBinary(G_POINT), ST_AsEWKB(G_POINT), ST_AsEWKT(G_GEO), ST_AsEWKB(G_MULTI), ST_AsEWKB(G_COLLECTION) FROM "+table+" WHERE ID = 1", 20, 0)
if err != nil {
t.Fatalf("WKB/EWKB/EWKT query failed: %v", err)
}
if len(raw.Rows) != 1 || len(raw.Rows[0]) != 5 {
t.Fatalf("unexpected WKB/EWKB/EWKT result: %#v", raw)
}
for columnIndex, wantSRID := range map[int]*uint32{0: nil, 1: uint32Pointer(4326)} {
bytes, ok := raw.Rows[0][columnIndex].(string)
if !ok {
t.Fatalf("raw WKB column %d type = %T, want binary string", columnIndex, raw.Rows[0][columnIndex])
}
decoded, decodedOK := decodeXuguWKB([]byte(bytes))
if !decodedOK || !isXuguWKT(decoded.WKT) || (wantSRID == nil && decoded.SRID != nil) || (wantSRID != nil && (decoded.SRID == nil || *decoded.SRID != *wantSRID)) {
t.Fatalf("raw WKB column %d decoded as value=%#v ok=%v", columnIndex, decoded, decodedOK)
}
}
decodedEWKT, sridEWKT, recognizedEWKT := decodeXuguSpatialValue(raw.Rows[0][2])
if !recognizedEWKT || decodedEWKT != "POINT(116.397 39.908)" || sridEWKT == nil || *sridEWKT != 4326 {
t.Fatalf("EWKT column decoded as value=%#v srid=%#v recognized=%v", decodedEWKT, sridEWKT, recognizedEWKT)
}
for columnIndex, want := range map[int]struct {
wkt string
srid uint32
}{
3: {wkt: "MULTIPOINT((1 2),(3 4))", srid: 3857},
4: {wkt: "GEOMETRYCOLLECTION(POINT(1 2),LINESTRING(3 4,5 6))", srid: 4326},
} {
bytes, ok := raw.Rows[0][columnIndex].(string)
if !ok {
t.Fatalf("complex EWKB column %d type = %T, want binary string", columnIndex, raw.Rows[0][columnIndex])
}
decoded, decodedOK := decodeXuguWKB([]byte(bytes))
if !decodedOK || decoded.WKT != want.wkt || decoded.SRID == nil || *decoded.SRID != want.srid {
t.Fatalf("complex EWKB column %d decoded as value=%#v ok=%v", columnIndex, decoded, decodedOK)
}
}
t.Logf("Xugu WKB/EWKB/EWKT result: types=%#v columns=%#v rows=%#v spatialColumns=%#v spatialValues=%#v", raw.ColumnTypes, raw.Columns, raw.Rows, raw.SpatialColumns, raw.SpatialValues)
t.Logf("decoded Xugu spatial result: columns=%#v types=%#v spatialColumns=%#v values=%#v rows=%#v", result.Columns, result.ColumnTypes, result.SpatialColumns, result.SpatialValues, result.Rows)
}
// TestLiveXuguSpatialReplay probes the literal form generated by the generic
// export path. Xugu accepts WKT constructors for replay; the test records
// whether a plain quoted WKT is accepted so export code can remain database
// specific instead of guessing from PostgreSQL syntax.
func TestLiveXuguSpatialReplay(t *testing.T) {
if os.Getenv("XUGU_LIVE_TEST") != "1" {
t.Skip("set XUGU_LIVE_TEST=1 with XUGU_LIVE_* connection settings to run the XuguDB integration test")
}
params := liveXuguParams(t)
s := newServer()
db, err := openDB(params)
if err != nil {
t.Fatal(err)
}
s.db = db
s.params = params
s.currentDatabase = params.Database
defer s.disconnect()
const table = "DBX_SPATIAL_REPLAY_LIVE_T"
_ = s.execWithReconnect("DROP TABLE IF EXISTS " + table)
defer func() { _ = s.execWithReconnect("DROP TABLE IF EXISTS " + table) }()
for _, statement := range []string{
"CREATE TABLE " + table + " (ID INTEGER, G GEOMETRY, GEO GEOGRAPHY)",
"INSERT INTO " + table + " VALUES (1, ST_GeomFromEWKT('SRID=4326;POINT(1 2)'), ST_GeomFromEWKT('SRID=4326;POINT(3 4)'))",
} {
if err := s.execWithReconnect(statement); err != nil {
t.Fatalf("setup statement failed: %s: %v", statement, err)
}
}
if err := s.execWithReconnect("INSERT INTO " + table + " VALUES (2, 'POINT(5 6)', 'POINT(7 8)')"); err != nil {
t.Logf("plain quoted WKT is rejected by Xugu (expected on typed spatial columns): %v", err)
} else {
t.Log("plain quoted WKT is accepted by Xugu typed spatial columns")
}
if err := s.execWithReconnect("INSERT INTO " + table + " VALUES (3, ST_GeomFromEWKT('SRID=3857;POINT(9 10)'), ST_GeomFromEWKT('SRID=4326;POINT(11 12)'))"); err != nil {
t.Fatalf("Xugu spatial constructor replay failed: %v", err)
}
result, err := s.executeSelect("SELECT ID, ST_AsEWKT(G), ST_AsEWKT(GEO) FROM "+table+" ORDER BY ID", 20, 0)
if err != nil {
t.Fatal(err)
}
if len(result.Rows) < 2 {
t.Fatalf("spatial replay returned too few rows: %#v", result.Rows)
}
t.Logf("Xugu spatial replay result: types=%#v rows=%#v", result.ColumnTypes, result.Rows)
}
func liveXuguParams(t *testing.T) connectParams {
t.Helper()
params := connectParams{
Host: os.Getenv("XUGU_LIVE_HOST"),
Port: parsePort(os.Getenv("XUGU_LIVE_PORT")),
Database: os.Getenv("XUGU_LIVE_DATABASE"),
Username: os.Getenv("XUGU_LIVE_USERNAME"),
Password: os.Getenv("XUGU_LIVE_PASSWORD"),
}
if params.Host == "" || params.Database == "" || params.Username == "" || params.Password == "" {
t.Fatal("XUGU_LIVE_HOST, XUGU_LIVE_DATABASE, XUGU_LIVE_USERNAME, and XUGU_LIVE_PASSWORD are required")
}
return params
}
+109
View File
@@ -0,0 +1,109 @@
package main
import (
"encoding/hex"
"reflect"
"testing"
)
func TestDecodeXuguSpatialValues(t *testing.T) {
point := mustDecodeXuguHex(t, "0101000020E6100000C520B07268195D404E62105839F44340")
tests := []struct {
name string
value any
wantValue any
wantSRID *uint32
recognized bool
}{
{"raw EWKB", point, "POINT(116.397 39.908)", uint32Pointer(4326), true},
{"hex EWKB", "0x0101000020E6100000C520B07268195D404E62105839F44340", "POINT(116.397 39.908)", uint32Pointer(4326), true},
{"EWKT", "SRID=3857;POINT(1 2)", "POINT(1 2)", uint32Pointer(3857), true},
{"invalid EWKT is not promoted", "SRID=3857;not a geometry", "SRID=3857;not a geometry", nil, false},
{"WKT remains text for unknown columns", "POINT(1 2)", "POINT(1 2)", nil, false},
{"NULL", nil, nil, nil, false},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
value, srid, recognized := decodeXuguSpatialValue(test.value)
if !reflect.DeepEqual(value, test.wantValue) || !reflect.DeepEqual(srid, test.wantSRID) || recognized != test.recognized {
t.Fatalf("decodeXuguSpatialValue(%v) = (%v, %v, %v), want (%v, %v, %v)", test.value, value, srid, recognized, test.wantValue, test.wantSRID, test.recognized)
}
})
}
}
func TestXuguSpatialDecoderInfersBlankGeometryTypeWithoutPromotingText(t *testing.T) {
columnTypes := []string{"INTEGER", "", "VARCHAR"}
decoder := newXuguSpatialDecoder(columnTypes)
point := mustDecodeXuguHex(t, "0101000020E6100000000000000000F03F0000000000000040")
row, values, err := decoder.normalizeRow([]any{int64(1), point, "POINT(9 9)"})
if err != nil {
t.Fatal(err)
}
if row[1] != "POINT(1 2)" || row[2] != "POINT(9 9)" {
t.Fatalf("unexpected normalized row: %v", row)
}
if columnTypes[1] != "GEOMETRY" || columnTypes[2] != "VARCHAR" {
t.Fatalf("unexpected inferred column types: %v", columnTypes)
}
if len(values) != 3 || values[1] == nil || *values[1] != 4326 || values[2] != nil {
t.Fatalf("unexpected spatial cell metadata: %v", values)
}
columns := decoder.columns()
if len(columns) != 1 || columns[0].ColumnIndex != 1 || columns[0].SRID == nil || *columns[0].SRID != 4326 {
t.Fatalf("unexpected spatial columns: %+v", columns)
}
}
func TestXuguSpatialDecoderKeepsOrdinaryBinaryColumnUntouched(t *testing.T) {
columnTypes := []string{"BLOB", "BINARY"}
decoder := newXuguSpatialDecoder(columnTypes)
value := []byte{0x01, 0x02, 0x03}
row, values, err := decoder.normalizeRow([]any{value, value})
if err != nil {
t.Fatal(err)
}
if len(decoder.indices) != 0 || len(decoder.columns()) != 0 || len(values) != 2 {
t.Fatalf("ordinary binary values were classified as spatial: columns=%+v values=%v", decoder.columns(), values)
}
if row[0] != string(value) || row[1] != string(value) {
t.Fatalf("ordinary binary values changed: %v", row)
}
}
func TestDecodeXuguSpatialComplexEWKB(t *testing.T) {
encoded := "0107000020E61000000200000001010000000000000000005D4000000000000044400102000000020000000000000000405D4000000000008044400000000000805D400000000000004540"
decoded, ok := decodeXuguWKB(mustDecodeXuguHex(t, encoded))
if !ok || decoded.WKT != "GEOMETRYCOLLECTION(POINT(116 40),LINESTRING(117 41,118 42))" || decoded.SRID == nil || *decoded.SRID != 4326 {
t.Fatalf("unexpected complex EWKB: %+v ok=%v", decoded, ok)
}
}
func TestMalformedXuguSpatialBytesFallBack(t *testing.T) {
value, srid, recognized := decodeXuguSpatialValue(mustDecodeXuguHex(t, "0101000020E6100000C520B072"))
if value != "0x0101000020e6100000c520b072" || srid != nil || recognized {
t.Fatalf("unexpected malformed fallback: value=%v srid=%v recognized=%v", value, srid, recognized)
}
}
func TestXuguDataTypesExposeSpatialDomainTypes(t *testing.T) {
want := []string{"GEOMETRY", "GEOGRAPHY", "BOX2D", "BOX3D"}
set := make(map[string]bool, len(xuguDataTypes))
for _, value := range xuguDataTypes {
set[value] = true
}
for _, value := range want {
if !set[value] {
t.Fatalf("spatial data type %q is not exposed", value)
}
}
}
func mustDecodeXuguHex(t *testing.T, value string) []byte {
t.Helper()
decoded, err := hex.DecodeString(value)
if err != nil {
t.Fatal(err)
}
return decoded
}
+15 -1
View File
@@ -13,6 +13,20 @@ import (
"github.com/go-zookeeper/zk"
)
func TestParseServerVersionTrimsBuiltOnSuffix(t *testing.T) {
srvr := "Zookeeper version: 3.6.1--104dcb3dcae0bf1bca4b4a00b2d76f2b5f6d79c, built on 04/21/2020 15:01 GMT\n"
if got := parseServerVersion(srvr); got != "3.6.1--104dcb3dcae0bf1bca4b4a00b2d76f2b5f6d79c" {
t.Fatalf("srvr version = %q", got)
}
envi := "zookeeper.version=3.8.4-9a6b1d5e9c0f, built on 2024-01-01 00:00 GMT\n"
if got := parseServerVersion(envi); got != "3.8.4-9a6b1d5e9c0f" {
t.Fatalf("envi version = %q", got)
}
if got := parseServerVersion("Latency min/avg/max: 0/0/0\n"); got != "" {
t.Fatalf("unrelated line parsed as version %q", got)
}
}
func TestHandshakeAndRPCFailures(t *testing.T) {
service := newServer()
response, shutdown := service.handleRequest([]byte(`{"jsonrpc":"2.0","id":1,"method":"handshake","params":{}}`))
@@ -20,7 +34,7 @@ func TestHandshakeAndRPCFailures(t *testing.T) {
t.Fatalf("handshake response = %#v, shutdown=%v", response, shutdown)
}
handshake, ok := response.Result.(handshakeResult)
if !ok || handshake.ProtocolVersion != 1 || handshake.AgentProtocolVersion != 1 || strings.Join(handshake.Capabilities, ",") != "connect,test_connection,kv" {
if !ok || handshake.ProtocolVersion != 1 || handshake.AgentProtocolVersion != 1 || strings.Join(handshake.Capabilities, ",") != "connect,test_connection,connection_info,kv" {
t.Fatalf("unexpected handshake: %#v", response.Result)
}
+87 -2
View File
@@ -111,13 +111,14 @@ func (service *server) connect(params json.RawMessage) (map[string]bool, error)
}
previousClient := service.activeClient
service.activeClient = nextClient
service.activeConfig = config
if previousClient != nil {
previousClient.Close()
}
return map[string]bool{"ok": true}, nil
}
func (service *server) testConnection(params json.RawMessage) (map[string]bool, error) {
func (service *server) testConnection(params json.RawMessage) (map[string]any, error) {
config, err := decodeConnectionConfig(params)
if err != nil {
return nil, err
@@ -127,7 +128,22 @@ func (service *server) testConnection(params json.RawMessage) (map[string]bool,
return nil, err
}
probe.Close()
return map[string]bool{"ok": true}, nil
result := map[string]any{"ok": true}
if info := databaseInfo(config); info != nil {
result["databaseInfo"] = info
}
return result, nil
}
func (service *server) connectionInfo() (map[string]any, error) {
if _, err := service.requireClient(); err != nil {
return nil, err
}
result := map[string]any{}
if info := databaseInfo(service.activeConfig); info != nil {
result["databaseInfo"] = info
}
return result, nil
}
func openClient(config connectionConfig) (*clientSession, error) {
@@ -272,6 +288,74 @@ func connectionString(config connectionConfig) string {
return net.JoinHostPort(strings.Trim(host, "[]"), strconv.Itoa(port))
}
func databaseInfo(config connectionConfig) map[string]any {
info := map[string]any{"productName": "ZooKeeper"}
target, err := parseConnectTarget(connectionString(config))
if err != nil {
return info
}
if version := detectServerVersion(target.Servers, millisecondsOrDefault(config.ConnectionTimeoutMS, defaultConnectionTimeout)); version != "" {
info["productVersion"] = version
}
return info
}
func detectServerVersion(servers []string, timeout time.Duration) string {
deadline := timeout
if deadline <= 0 || deadline > 2*time.Second {
deadline = 2 * time.Second
}
for _, server := range servers {
address, err := endpointAddress(server)
if err != nil {
continue
}
// ZooKeeper 3.5+ whitelists only "srvr" by default; "envi"/"stat"
// are opt-in, so probe srvr first for default-config clusters.
for _, command := range []string{"srvr", "envi", "stat"} {
connection, err := net.DialTimeout("tcp", address, deadline)
if err != nil {
continue
}
_ = connection.SetDeadline(time.Now().Add(deadline))
if _, err := connection.Write([]byte(command)); err != nil {
connection.Close()
continue
}
buffer := make([]byte, 16*1024)
count, _ := connection.Read(buffer)
if version := parseServerVersion(string(buffer[:count])); version != "" {
connection.Close()
return version
}
connection.Close()
}
}
return ""
}
func parseServerVersion(response string) string {
for _, line := range strings.Split(response, "\n") {
line = strings.TrimSpace(line)
parts := strings.SplitN(line, "=", 2)
if len(parts) != 2 {
parts = strings.SplitN(line, ":", 2)
}
key := strings.ToLower(strings.TrimSpace(parts[0]))
key = strings.NewReplacer(" ", ".", "_", ".").Replace(key)
if len(parts) == 2 && key == "zookeeper.version" {
version := strings.TrimSpace(parts[1])
// Drop the ", built on ..." suffix envi/stat/srvr carry so only
// the version itself is shown.
if idx := strings.Index(version, ","); idx >= 0 {
version = strings.TrimSpace(version[:idx])
}
return version
}
}
return ""
}
func parseConnectTarget(value string) (connectTarget, error) {
connectString := strings.TrimSpace(strings.TrimPrefix(value, "zookeeper://"))
slash := strings.Index(connectString, "/")
@@ -460,6 +544,7 @@ func (service *server) closeClient() {
service.activeClient.Close()
service.activeClient = nil
}
service.activeConfig = connectionConfig{}
}
func (session *clientSession) Close() {
+5 -1
View File
@@ -13,7 +13,7 @@ const (
maxRPCMessageBytes = 32 * 1024 * 1024
)
var capabilities = []string{"connect", "test_connection", "kv"}
var capabilities = []string{"connect", "test_connection", "connection_info", "kv"}
type rpcRequest struct {
ID json.RawMessage `json:"id"`
@@ -41,6 +41,7 @@ type handshakeResult struct {
type server struct {
activeClient znodeClient
activeConfig connectionConfig
statLookupConcurrency int
}
@@ -104,6 +105,9 @@ func (service *server) dispatch(method string, params json.RawMessage) (any, boo
case "test_connection":
result, err := service.testConnection(params)
return result, false, err
case "connection_info":
result, err := service.connectionInfo()
return result, false, err
case "kv_list_prefix":
result, err := service.listPrefix(params)
return result, false, err
+1
View File
@@ -21,6 +21,7 @@ neo4j-go shared-fallback native-go Uses Neo4j schema procedures with stable loca
oceanbase-oracle native-pushdown java-sql Uses Oracle-compatible metadata SQL with type, filter, stable order, and ROWNUM paging.
oracle-go native-pushdown native-go Uses Oracle-compatible metadata SQL with type, filter, stable order, and ROWNUM paging, with legacy profile guard.
snowflake native-pushdown java-sql Uses INFORMATION_SCHEMA with type, filter, stable order, and literal LIMIT/OFFSET.
spanner intentional-fallback java-jdbc-metadata Uses the dialect-aware Spanner JDBC metadata with the schema passed through verbatim because GoogleSQL names the user schema with an empty string; JDBC exposes no portable server-side paging API, common constraints filter locally.
spark intentional-fallback java-show-metadata Uses SHOW TABLES/SHOW DATABASES; stable server-side fuzzy paging is not portable, common constraints filter locally.
sundb native-pushdown java-sql Uses information_schema with type, filter, stable order, and LIMIT/OFFSET.
tdengine shared-fallback native-rust Uses SHOW STABLES/TABLES with information_schema parent enrichment, then applies stable filtering and paging in the native agent.
1 driver strategy scope reason
21 oceanbase-oracle native-pushdown java-sql Uses Oracle-compatible metadata SQL with type, filter, stable order, and ROWNUM paging.
22 oracle-go native-pushdown native-go Uses Oracle-compatible metadata SQL with type, filter, stable order, and ROWNUM paging, with legacy profile guard.
23 snowflake native-pushdown java-sql Uses INFORMATION_SCHEMA with type, filter, stable order, and literal LIMIT/OFFSET.
24 spanner intentional-fallback java-jdbc-metadata Uses the dialect-aware Spanner JDBC metadata with the schema passed through verbatim because GoogleSQL names the user schema with an empty string; JDBC exposes no portable server-side paging API, common constraints filter locally.
25 spark intentional-fallback java-show-metadata Uses SHOW TABLES/SHOW DATABASES; stable server-side fuzzy paging is not portable, common constraints filter locally.
26 sundb native-pushdown java-sql Uses information_schema with type, filter, stable order, and LIMIT/OFFSET.
27 tdengine shared-fallback native-rust Uses SHOW STABLES/TABLES with information_schema parent enrichment, then applies stable filtering and paging in the native agent.
+1 -1
View File
@@ -4,7 +4,7 @@ def infrastructureModules = ['common', 'test-support']
def driverModules = [
'access', 'dameng', 'goldendb', 'databend', 'databricks', 'saphana',
'teradata', 'vertica', 'firebird', 'exasol', 'oceanbase-oracle', 'gbase8a', 'gbase8s',
'bigquery', 'kylin', 'ignite', 'ignite3', 'sundb', 'h2', 'h2-legacy', 'snowflake', 'trino', 'spark',
'bigquery', 'kylin', 'ignite', 'ignite3', 'spanner', 'sundb', 'h2', 'h2-legacy', 'snowflake', 'trino', 'spark',
'db2', 'informix', 'mongodb', 'highgo', 'uxdb', 'yashandb', 'oscar',
'iris', 'etcd', 'kafka', 'sqlserver-legacy'
]
+46 -45
View File
@@ -1,49 +1,50 @@
{
"access": "0.1.43",
"bigquery": "0.1.47",
"cassandra": "0.1.43",
"dameng": "0.1.53",
"databend": "0.1.34",
"databricks": "0.1.41",
"db2": "0.1.46",
"duckdb": "0.1.9",
"exasol": "0.1.41",
"firebird": "0.1.42",
"gbase8a": "0.1.41",
"gbase8s": "0.1.35",
"goldendb": "0.1.46",
"h2": "0.1.46",
"h2-legacy": "0.1.16",
"hive": "0.1.49",
"ignite": "0.1.0",
"ignite3": "0.1.0",
"spark": "0.1.18",
"informix": "0.1.47",
"kingbase": "0.1.49",
"kylin": "0.1.46",
"access": "0.1.47",
"bigquery": "0.1.51",
"cassandra": "0.1.44",
"dameng": "0.1.57",
"databend": "0.1.38",
"databricks": "0.1.45",
"db2": "0.1.50",
"duckdb": "0.1.13",
"exasol": "0.1.45",
"firebird": "0.1.46",
"gbase8a": "0.1.45",
"gbase8s": "0.1.39",
"goldendb": "0.1.50",
"h2": "0.1.50",
"h2-legacy": "0.1.20",
"hive": "0.1.50",
"ignite": "0.1.3",
"ignite3": "0.1.3",
"spark": "0.1.22",
"informix": "0.1.51",
"kingbase": "0.1.52",
"kylin": "0.1.50",
"neo4j": "0.1.45",
"oceanbase-oracle": "0.1.43",
"oracle": "0.1.51",
"snowflake": "0.1.46",
"saphana": "0.1.41",
"sundb": "0.1.46",
"teradata": "0.1.41",
"tdengine": "0.1.45",
"trino": "0.1.46",
"oceanbase-oracle": "0.1.47",
"oracle": "0.1.56",
"snowflake": "0.1.50",
"spanner": "0.1.3",
"saphana": "0.1.46",
"sundb": "0.1.50",
"teradata": "0.1.45",
"tdengine": "0.1.46",
"trino": "0.1.50",
"vastbase": "0.1.46",
"vertica": "0.1.41",
"mongodb": "0.1.48",
"highgo": "0.1.44",
"uxdb": "0.1.13",
"yashandb": "0.1.42",
"oscar": "0.1.21",
"xugu": "0.1.36",
"iris": "0.1.37",
"iotdb": "0.1.35",
"etcd": "0.1.34",
"zookeeper": "0.1.25",
"kafka": "0.1.11",
"rocketmq": "0.1.7",
"rabbitmq": "0.1.6",
"sqlserver-legacy": "0.1.17"
"vertica": "0.1.45",
"mongodb": "0.1.52",
"highgo": "0.1.48",
"uxdb": "0.1.17",
"yashandb": "0.1.46",
"oscar": "0.1.25",
"xugu": "0.1.39",
"iris": "0.1.42",
"iotdb": "0.1.36",
"etcd": "0.1.38",
"zookeeper": "0.1.26",
"kafka": "0.1.13",
"rocketmq": "0.1.9",
"rabbitmq": "0.1.8",
"sqlserver-legacy": "0.1.21"
}

Some files were not shown because too many files have changed in this diff Show More