Compare commits

...
Author SHA1 Message Date
github-actions[bot] 4047686f93 chore(packages): release 0.4.80 [skip node-packages-release] 2026-09-04 10:43:57 +00:00
t8y2 b853d49ac1 fix(databend): skip schema wildcard escaping so databases with underscore list tables 2026-09-04 18:35:45 +08:00
github-actions[bot] 5ca30bb25b chore: bump module versions [skip ci] 2026-09-04 10:20:37 +00:00
t8y2 2c9dcdf3ea feat(docs): add performant animated landing starfield 2026-09-04 18:13:07 +08:00
t8y2 0a1ac3fb14 chore(release): prepare v0.6.3 2026-09-04 17:32:17 +08:00
zipg 6e9e8d1818 feat(ui): progressively shrink data grid toolbar 2026-09-04 17:10:50 +08:00
zipg f861776c3c fix(sidebar): remove row corner radius from tree items 2026-09-04 16:58:51 +08:00
Melody Ruch a559371fc9 fix(editor): disable code editor font ligatures 2026-09-04 16:55:08 +08:00
yanguibao1997 5e03c20821 feat(editor): add naming style conversion for selections 2026-09-04 16:52:47 +08:00
Melody Ruch a2e4d15bbc fix(import): timeout and retry existing-table column load 2026-09-04 16:52:00 +08:00
Melody Ruch 7f7f011f40 fix(editor): stop preloading data grid chunk on tab open 2026-09-04 16:49:05 +08:00
Melody Ruch af32936297 fix(gbase): generate GBase 8s table DDL with Informix dialect 2026-09-04 16:48:35 +08:00
Melody Ruch d430ec9a4a fix(imports): map DBeaver DB2 driver id to native db2 type 2026-09-04 16:48:03 +08:00
Melody Ruch b00821d2c5 fix(grid): preserve pending value editor draft on refresh 2026-09-04 16:47:10 +08:00
Elias 1c972b80c0 test(xugu): add focused live integration baseline 2026-09-04 16:46:36 +08:00
Melody Ruch 795cd13f88 fix(editor): allow case conversion inside string literals 2026-09-04 16:46:24 +08:00
zipg 805ddac61c fix(ai): send Agens-compatible follow-up tool-call requests 2026-09-04 16:45:28 +08:00
t8y2 67f94e1e3b docs: add 1Panel partner badge to readmes 2026-09-04 16:14:09 +08:00
t8y2 392b2bf5b1 fix(i18n): add missing pin/unpin tab context menu keys
EditorGroupTabBar referenced contextMenu.pinTab/unpinTab (introduced in
5a3a6e2bb) but no locale declared them, so the tab context menu rendered
the raw key path in every language. Rename the unused fixTab/unfixTab
entries to pinTab/unpinTab in all nine locales and add a test asserting
every tab-menu key resolves to text in en (the fallback locale) and that
pin/unpin labels resolve in every locale.
2026-09-04 15:41:20 +08:00
zipg d2401896dd feat(redis): fuzzy key search keyspace-aware scan budget 2026-09-04 15:14:42 +08:00
zipg a5c97067ba feat(redis): custom hover scrollbar for key and value views 2026-09-04 15:14:06 +08:00
t8y2 372e5130cd chore: remove tracked .zcode plans directory 2026-09-04 15:10:38 +08:00
jearhe 65abc74590 fix(completion): replace unicode identifier prefixes correctly
Closes #7891
2026-09-04 15:07:30 +08:00
t8y2 7b0a8cc111 docs: split sponsors and partners sections, add 1Panel 2026-09-04 14:56:04 +08:00
Tara 01623ab38d fix(postgres): keep index operator classes in read and generation
Closes #7166
2026-09-04 14:48:41 +08:00
Melody Ruch 09f8d095d2 fix(sql): keep column comments aligned for mixed star selects
Closes #8015
2026-09-04 14:45:53 +08:00
Melody Ruch ca003c5e93 fix(tabs): restore saved sql library content in detached windows 2026-09-04 14:45:04 +08:00
Lan-zk 5a3a6e2bb0 feat(editor): add sql editor split panes with shared results 2026-09-04 14:44:03 +08:00
zipg 24a274bd89 docs(snippets): document snippet placeholder syntax hint 2026-09-04 14:43:43 +08:00
t8y2 a52138129a fix(spanner): keep INT64/FLOAT64 grid filter values as numeric literals
Closes #8140
2026-09-04 14:15:49 +08:00
Abeautifulsnow 543bf66905 feat(mcp): execute sql batches with opt-in rollbackable transactions 2026-09-04 13:43:06 +08:00
ptma 7390ac109c fix(influxdb): keep default time window on refresh and sort rebuilds 2026-09-04 13:42:29 +08:00
t8y2 09de83e64e feat(editor): skip LIKE wildcards when double-clicking a string
Closes #8129
2026-09-04 13:01:13 +08:00
t8y2 eb790e7bfc test(completion): expect inline column comment in completion detail 2026-09-04 12:58:55 +08:00
Melody Ruch 434d44a284 feat(search): match quick-open items by pinyin initials 2026-09-04 12:41:49 +08:00
Guoyu Su 299c342d6d fix(sidebar): restore table list after remote search 2026-09-04 12:41:05 +08:00
t8y2 b6beefc89e fix(xugu): format datetime values as writable literals
Closes #8110
2026-09-04 12:30:33 +08:00
Melody Ruch 057c04d306 fix(grid): scope header generate-value to the clicked column 2026-09-04 12:28:38 +08:00
Melody Ruch 560e1c2c0c fix(mongo): show refresh toolbar for never-loaded empty collections 2026-09-04 12:27:54 +08:00
t8y2 b42ad91fc2 fix(completion): show column comments inline in SQL completions 2026-09-04 12:17:20 +08:00
Henry Pham 84b37d44a6 docs(readme): correct misleading tableplus platform claim 2026-09-04 12:08:41 +08:00
jearhe 2766a5bc49 fix(opengauss): stop doubling backslashes in exported sql literals 2026-09-04 12:08:29 +08:00
t8y2 8cc7393b11 chore(docs): regenerate docs export bundle for diagram changes 2026-09-04 12:07:32 +08:00
Melody Ruch 9eacc94584 fix(jdbc): report object statistics for generic jdbc connections 2026-09-04 11:05:54 +08:00
zipg 28930136d7 feat(sidebar): page full table search results under database 2026-09-04 10:25:57 +08:00
sunzhengbo 6d39eccdbf feat(diagram): support multi-schema and full database relationship views 2026-09-04 10:25:14 +08:00
Sinan Eldem d16645121d docs(i18n): add turkish to the documentation site 2026-09-04 09:51:36 +08:00
zipg d7941572b8 feat(editor): add ignore action to sql parameter dialog 2026-09-04 09:50:46 +08:00
zipg 97f5a37ba8 fix(editor): keep shift-click selection in query editor 2026-09-04 09:49:54 +08:00
Melody Ruch 06dcd0013c fix(mysql): show inherited charset and collation in structure editor 2026-09-04 09:48:41 +08:00
t8y2 48410b70b4 fix(influxdb): support table export query dispatch and pagination
Closes #8022
2026-09-04 02:35:01 +08:00
t8y2 0a511072a7 fix(linux): respect native Wayland backend for AppImage
Closes #7994
2026-09-04 02:35:01 +08:00
t8y2 6a8233b2c8 fix(jdbc): support Oracle table DDL through external driver
Closes #7956
2026-09-04 02:35:01 +08:00
Sinan Eldem 79dbda2a8f feat(i18n): add Turkish locale 2026-09-04 02:22:21 +08:00
zipg 232426ced2 fix(grid): restore copy current row Mod+D shortcut 2026-09-04 02:05:07 +08:00
二丫讲梵 2a4f1a45a9 feat(tabs): add tab group pills with names, colors and collapsible side bar 2026-09-04 01:23:25 +08:00
二丫讲梵 0bdf232487 feat(elasticsearch): add hit document view and field type detection 2026-09-04 01:01:32 +08:00
t8y2 37dc74d9de fix(oracle): refresh table hover metadata from local index
Closes #8056
2026-09-04 00:44:21 +08:00
t8y2 fe955da729 fix(editor): fold Elasticsearch REST requests individually
Closes #8065
2026-09-04 00:44:10 +08:00
t8y2 81587313eb fix(schema-diff): disambiguate duplicate child IDs
Closes #8067
2026-09-04 00:44:10 +08:00
二丫讲梵 0279a0903c feat(nacos): add browser console open, zen mode and metadata view 2026-09-04 00:29:41 +08:00
Mutsumi 39de9499b4 fix(sidebar): use sidebar theme color for sticky database rows 2026-09-04 00:29:29 +08:00
zipg de73c2bd73 fix(kingbase): format bit column values as bit-string literals 2026-09-04 00:29:17 +08:00
jearhe 730a4eca5f fix(kafka): avoid client timeout preempting topic list fallback 2026-09-04 00:29:04 +08:00
二丫讲梵 4dab78ac30 feat(redis): add key directory context menu 2026-09-04 00:28:50 +08:00
t8y2 b1fc9b15b9 fix(oceanbase): show materialized views in object browser
Closes #7984
2026-09-03 23:28:21 +08:00
t8y2 2a53460604 fix(sqlserver): use batch queries for Azure Synapse
Closes #8027
2026-09-03 23:28:17 +08:00
t8y2 b17a87697b fix(dameng): correct routine source fallback
Refs: #7954
2026-09-03 23:28:16 +08:00
Freud 1c170396d3 fix(ai): make custom output token budget configurable 2026-09-03 21:07:37 +08:00
Ghyper9023 56f85a2c77 fix(connection): size metadata gate by real session pool capacity 2026-09-03 21:07:18 +08:00
Jamie 5e4f756a63 fix(editor): keep caret visible at fractional zoom levels 2026-09-03 20:45:26 +08:00
Guoyu Su 2324d84c4e fix(postgres): prevent custom type stream deadlock 2026-09-03 19:28:20 +08:00
二丫讲梵 0448d5bd2c feat(nacos): add config syntax validation with error location 2026-09-03 19:13:33 +08:00
Guoyu Su 7157b04d87 fix(transfer): preserve constraints after deferred foreign keys 2026-09-03 18:50:13 +08:00
Abeautifulsnow dd11e42748 fix(editor): anchor wrapped line numbers via inline style 2026-09-03 18:50:05 +08:00
Guoyu Su 767cc809a1 fix(transfer): preserve PostgreSQL pgvector literals 2026-09-03 18:28:15 +08:00
Weiki c28248cd67 fix(tabs): prevent trackpad taps from starting tab drag 2026-09-03 18:10:26 +08:00
jearhe d765315af4 fix(postgres): bigint cast in sequence sync SQL 2026-09-03 17:52:23 +08:00
zipg 291890f67e feat(export): respect connection query timeout in table export 2026-09-03 17:52:15 +08:00
Melody Ruch aa637b2d90 fix(grid): keep filters when refreshing restored table tabs 2026-09-03 17:52:07 +08:00
Melody Ruch cd8300c3f5 fix(table-editor): clear stale auto-increment flag on PK change 2026-09-03 17:51:58 +08:00
zipg 203eab3dec feat(export): configurable CSV field quoting mode 2026-09-03 17:51:50 +08:00
Abeautifulsnow d421bd33fc fix(ui): scroll long error messages in rename dialogs 2026-09-03 17:51:43 +08:00
jearhe 20f312d650 fix(mongodb): keep plain strings when batch pasting values 2026-09-03 17:51:34 +08:00
zipg 3a52fd2a0b fix(grid): keep caret aligned after filter condition wraps 2026-09-03 17:51:26 +08:00
github-actions[bot] c444bb44b1 chore(packages): release 0.4.79 [skip node-packages-release] 2026-09-03 16:50:27 +08:00
t8y2 719070f2b8 docs: add AstraFlow sponsor card 2026-09-03 16:48:56 +08:00
github-actions[bot] f3c82061c0 chore: bump module versions [skip ci] 2026-09-03 08:27:01 +00:00
t8y2 6088203755 docs(site): add Jalapeño Cloud sponsor 2026-09-03 15:33:16 +08:00
t8y2 486da32e4d fix(ci): pin build matrix job display name 2026-09-03 15:29:22 +08:00
t8y2 178f00d7ff docs: add Jalapeno Cloud sponsor card 2026-09-03 15:25:51 +08:00
t8y2 715261feae chore(release): prepare v0.6.2 2026-09-03 15:18:22 +08:00
t8y2 e98e3157f1 fix(editor): pin completion icon glyph to its box 2026-09-03 15:01:31 +08:00
t8y2 df244627f6 docs: use white-card sponsor logos in readmes 2026-09-03 14:41:29 +08:00
t8y2 4d05a95f5a style(i18n): format backend errors test 2026-09-03 14:17:29 +08:00
zipg 8cfd69c7c9 fix(query): preserve original backend errors with fallback detail 2026-09-03 13:43:55 +08:00
zipg 76c51b5fa6 fix(sqlserver): keep table variable scope in agent batches 2026-09-03 13:43:45 +08:00
二丫讲梵 4b7b72dc65 feat(ai): add kimi model reasoning effort config 2026-09-03 13:04:37 +08:00
zipg c99c0d932c fix(table): hide unsupported identity column for opengauss 2026-09-03 12:34:10 +08:00
Abeautifulsnow 3c4d61339e fix(desktop): let data header name chips flex to content 2026-09-03 12:15:56 +08:00
二丫讲梵 ef13dc024f feat(grid): open cell http urls from hover overlay 2026-09-03 11:56:58 +08:00
onenewcode aa4f503353 feat(redis): search list set and zset members 2026-09-03 11:56:46 +08:00
二丫讲梵 2c7a420ef6 feat(grid): preselect first WHERE field completion suggestion 2026-09-03 11:49:18 +08:00
zipg eec2e63667 fix(windows): stop win7 fixed-runtime installer self-blocking 2026-09-03 11:30:44 +08:00
jearhe 35946427a6 fix(rocketmq): paginate topic consumer groups past 200 2026-09-03 10:27:52 +08:00
zipg e2eb481b8a fix(editor): route grid edits through manual txn session 2026-09-03 10:27:41 +08:00
zipg 321674c4dc chore(release): scope arm64 asset rename and sync references 2026-09-03 03:35:21 +08:00
wangfengxiang 24642a6bd5 feat(argo): dedicated argo-go agent and PL/SQL splitter 2026-09-03 02:09:49 +08:00
Elias 1818387da5 fix(xugu): accept scheduler job object sources 2026-09-03 01:53:54 +08:00
Guoyu Su d6fc1fe52c fix(core): avoid holding pool registry locks during db io 2026-09-03 01:53:42 +08:00
Eve 616c0daf25 fix(sql-file): preserve mysql statement boundaries 2026-09-03 01:53:30 +08:00
zipg 1f396a7d25 fix(sql): guard delimiter detection on empty statement buffer 2026-09-03 01:53:19 +08:00
wangfengxiang aac1d78539 feat(mcp): list stored routines and fetch routine source 2026-09-03 01:53:08 +08:00
Guoyu Su 1ed4b42955 fix(sql): preserve StarRocks materialized view statements 2026-09-03 01:45:20 +08:00
DengQingNian d38be39e03 feat(tabs): detach SQL and data tabs into separate windows 2026-09-03 01:45:08 +08:00
zipg 717dcc1294 chore(ci): tighten win7 loader audit and drop 2012r2 workflow 2026-09-03 01:34:14 +08:00
Elias 7e24f4007c feat(xugu): show tablespace details in sidebar 2026-09-03 01:13:17 +08:00
Allen 2f4a72c4e9 fix(sidebar): collapse the Queries node by default 2026-09-03 01:13:05 +08:00
zipg 6848e78855 fix(iotdb): render timestamps in session zone and type max_by 2026-09-03 01:12:47 +08:00
zipg 45688cafd6 fix(argo): default ArgoDB connections to no-SASL auth 2026-09-03 01:12:36 +08:00
Guoyu Su a13a7302fa feat(sql-file): show all execution failures 2026-09-03 01:12:25 +08:00
t8y2 460b413da8 fix(packages): widen npm visibility wait window to 10 minutes 2026-09-03 00:01:48 +08:00
github-actions[bot] a2aad40190 chore(packages): release 0.4.78 [skip node-packages-release] 2026-09-02 15:03:30 +00:00
github-actions[bot] 0f1bc0a573 chore: bump module versions [skip ci] 2026-09-02 14:50:49 +00:00
github-actions[bot] af29ce44b7 chore(jdbc): bump plugin version [skip ci] 2026-09-02 14:15:13 +00:00
t8y2 41ebb64e95 chore(release): prepare v0.6.1 2026-09-02 22:14:23 +08:00
t8y2 11ff1c29a6 fix(ci): guard win7 webview2 loader link against sccache
v0.6.0 shipped a win7/Server 2012 R2 installer whose dbx.exe statically
linked a WebView2 loader >= 1.0.1054.31 instead of the pinned 1.0.902.49,
failing on kernel 6.3 with ERROR_NOT_SUPPORTED. The loader patch rewrites
a file inside the cargo registry, which no compile cache can observe, so
the win7 release job stops using sccache until the loader is vendored,
and a binary audit now rejects any dbx.exe carrying post-1.0.902.49
loader strings.
2026-09-02 21:08:39 +08:00
Ryan Liu 7d92433689 feat(editor): add occurrence selection actions 2026-09-02 20:38:17 +08:00
Abeautifulsnow 91f309f6c7 feat(ai): default and last-used templates per db type 2026-09-02 20:37:39 +08:00
Guoyu Su d92e119464 fix(mongodb): support shell regex literals 2026-09-02 20:36:59 +08:00
jearhe d6c4457756 fix(editor): substitute sql parameters before explain 2026-09-02 20:36:33 +08:00
zipg 8305d63ea1 perf(editor): single-pass table name semantic coloring 2026-09-02 20:32:59 +08:00
zipg f3258fe08a fix(kingbase): keep legacy empty-database default 2026-09-02 20:29:03 +08:00
zipg b9b9d0bd31 fix(mysql): keep select into user variables declared 2026-09-02 20:24:12 +08:00
二丫讲梵 dd20081316 feat(document): copy document json content 2026-09-02 20:20:25 +08:00
二丫讲梵 0f60b6a655 feat(nacos): restore config editor selection and scroll 2026-09-02 20:16:01 +08:00
zipg fe509e80c7 fix(vastbase): close paged cursors between batch statements 2026-09-02 20:12:26 +08:00
t8y2 cfdee960b4 fix(db-env): add init directory to etcd 3.5 recipe 2026-09-02 18:22:06 +08:00
t8y2 08e6a19c9c fix(db-env): register etcd 3.5 recipe image and docs 2026-09-02 18:18:01 +08:00
t8y2 ac22ada8d0 fix(mongodb): classify shell reads as read-only 2026-09-02 18:17:26 +08:00
二丫讲梵 fdea607efd feat(editor): batch-select completion fields via drag 2026-09-02 18:07:25 +08:00
t8y2 72dcd8355a feat(etcd): rewrite agent in Go and add v2 API support 2026-09-02 18:04:25 +08:00
zipg 41fc755135 fix(iotdb): count tree model loads without derived table 2026-09-02 17:41:58 +08:00
二丫讲梵 c48ebda197 feat(mcp): override execution policy per database 2026-09-02 17:39:07 +08:00
zipg 1706081802 feat(updater): allow switching download source after failure 2026-09-02 17:35:24 +08:00
zipg 0333b3b389 docs: improve legacy windows and linux install guidance 2026-09-02 17:34:23 +08:00
jearhe 2c14016828 fix(grid): preserve sibling results after multi-result edit 2026-09-02 17:08:52 +08:00
zipg c813b76019 fix(hana): show character field length in column types 2026-09-02 17:05:11 +08:00
t8y2 f5113def38 fix(xugu): match primary keys across catalog case 2026-09-02 17:02:03 +08:00
jearhe 061041e1d7 feat(ssh): support ssh agent auth in tunnel manager 2026-09-02 17:00:28 +08:00
Clarence 02430d1b09 fix(influxdb): prefill new-query with rolling time window 2026-09-02 16:56:12 +08:00
Eve e3911a0198 fix(import): pick existing tables from database scope 2026-09-02 16:52:48 +08:00
githubasda 820de91972 fix(webview): keep webview2 copies in clipboard history 2026-09-02 15:45:24 +08:00
zipg 3e4c17bd58 fix(hana): sort results without derived column alias list 2026-09-02 14:31:57 +08:00
onenewcode b45300e9bc feat(structure): support editing table ddl 2026-09-02 14:17:49 +08:00
Mo-Moore 3b2df3814b fix(argo): read inceptor routine and view ddl via system views 2026-09-02 14:15:03 +08:00
hanxuanyu a1f54cf428 fix(diff): use connection tree select in compare dialogs 2026-09-02 14:12:35 +08:00
jearhe f9f61cf397 fix(sidebar): keep object browser scroll after sql generation 2026-09-02 14:11:39 +08:00
Allen e619b70d5d feat(sidebar): add setting to disable hover tooltips 2026-09-02 14:08:06 +08:00
jearhe ee731a3353 test(sql): add prefix replacement regression tests 2026-09-02 14:04:33 +08:00
zipg 8772f10aa4 fix(elasticsearch): add dsl query snippets to completion 2026-09-02 14:01:25 +08:00
t8y2 245ea3b640 fix(mongo): preserve BSON dates during transfer
Closes #7709
2026-09-02 05:37:56 +00:00
t8y2 069b84daf9 fix(settings): render changelog entries as markdown 2026-09-02 13:17:26 +08:00
t8y2 ad98bef167 fix(sqlserver): prevent legacy probe SQL truncation
Closes #7827
2026-09-02 05:01:06 +00:00
t8y2 91b4623278 fix(postgres): handle localized missing prokind errors
Closes #7824
2026-09-02 04:29:09 +00:00
t8y2 032c05d6ce fix(mysql): support PolarDB-X index metadata fallback
Closes #7847
2026-09-02 04:03:09 +00:00
zipg 9b6f3c48e4 fix(sql-file): open and save custom-filtered text files 2026-09-02 11:55:00 +08:00
Ömer Faruk ALKILIÇ d3a6a69796 fix(storage): restrict sqlite database file permissions to owner 2026-09-02 11:52:58 +08:00
Clarence 84669b0d1a fix(influxdb): gate quick-open rolling time window behind opt-in 2026-09-02 11:33:24 +08:00
jearhe 945285ef09 fix(postgres): strip large-value marker columns case-insensitively 2026-09-02 11:30:45 +08:00
jearhe 38b93505c4 fix(mongodb): include TTL index info in MCP getIndexes 2026-09-02 11:27:19 +08:00
zipg a834da7026 feat(ai): add schema context selection to AI chat 2026-09-02 11:24:47 +08:00
zipg 716d2eafd2 fix(query): fall back to autocommit for legacy oracle agents 2026-09-02 11:20:20 +08:00
zipg 10ac216964 fix(grid): allow deleting rows from DISTINCT join results 2026-09-02 11:18:44 +08:00
zipg 103e221cc3 fix(mysql): retry add column if not exists via metadata precheck 2026-09-02 11:17:18 +08:00
Eve a0d868b637 fix(structure): preserve DDL editor content after tab switch 2026-09-02 11:16:03 +08:00
t8y2 55a21f8e9e fix(sqlserver): name paginated metadata columns
Closes #7819
2026-09-02 02:42:34 +00:00
t8y2 8b4b836581 fix(oracle): handle SDE ST_GEOMETRY query results
Closes #7554
2026-09-02 02:04:40 +00:00
StaravenX 2c04c2f1f4 fix(sql): use ROWNUM pagination for Dameng 2026-09-02 09:15:10 +08:00
t8y2 01ce303f46 fix(protocol): accept uppercase completion candidate kinds
Closes #7511
2026-09-01 16:40:42 +00:00
zipg e9f2cc15b7 fix(sidebar): support collapsed-node global search 2026-09-02 00:27:44 +08:00
t8y2 bd7ed63468 fix(iris): avoid quoted table preview identifiers
Closes #7604
2026-09-01 16:15:07 +00:00
Guoyu Su 48f4fac89e fix(transfer): isolate table transfer task 2026-09-01 23:45:33 +08:00
t8y2 51ca40e86e fix(iris): read vendor list values through JDBC objects
Closes #7551
2026-09-01 15:37:39 +00:00
二丫讲梵 3325e97991 feat(ai): support Kimi model configuration deletion 2026-09-01 23:36:15 +08:00
K e79db23e1f fix(sql): preserve positional order by with group by 2026-09-01 23:36:01 +08:00
zipg 07688cac39 fix(backup): delete compressed SQL backups 2026-09-01 23:35:45 +08:00
二丫讲梵 804120c0b1 fix(grid): toggle filter view switch panel 2026-09-01 23:35:31 +08:00
t8y2 877421e596 fix(oracle): load materialized view DDL from metadata
Closes #7538
2026-09-01 15:24:40 +00:00
t8y2 d4d3995bef perf(ci): use concurrent cnb release sync script for resync 2026-09-01 20:23:12 +08:00
t8y2 053f5f6dc5 fix(ci): attribute agent release notes by module path 2026-09-01 20:05:27 +08:00
t8y2 7e3db73ded fix(docs): add argodb and influxdb3 database support entries 2026-09-01 19:57:31 +08:00
github-actions[bot] fac018f169 chore(packages): release 0.4.77 [skip node-packages-release] 2026-09-01 11:20:08 +00:00
github-actions[bot] b31b6845f9 chore: bump module versions [skip ci] 2026-09-01 11:06:04 +00:00
t8y2 0d75456f80 chore(release): prepare v0.6.0 2026-09-01 18:48:49 +08:00
t8y2 c4cdcbc486 improve: clarify SQL save target tooltip 2026-09-01 18:07:19 +08:00
zipg 1f274c57e3 fix(sidebar): support Cmd or Ctrl R for refresh 2026-09-01 18:07:17 +08:00
t8y2 6a7ea4ae95 feat(ai): add Jalapeno Cloud partner preset 2026-09-01 18:07:16 +08:00
zipg 19be194728 fix(sqlserver): preserve CTE prefix in legacy probe 2026-09-01 18:07:10 +08:00
t8y2 5d6f667776 fix(transfer): prevent silent column loss in mysql transfers
The deferred-FK DDL rewrite dropped any SHOW CREATE TABLE line containing
" FOREIGN KEY ", including column definitions whose COMMENT text merely
mentions the words; it now strips only genuine CONSTRAINT ... FOREIGN KEY
constraint lines. Structure-only transfers onto a preexisting target table
skipped the incompatible-structure validation that data transfers perform
and reported success while the target kept missing source columns; the
check now runs on both paths.

Closes #7660
2026-09-01 16:46:31 +08:00
Melody Ruch e9a90173a6 fix(sql): restore SQL file schema for sidebar locate 2026-09-01 16:21:40 +08:00
zipg 440c41f5b9 fix(sqlserver): preserve local temp table count sessions 2026-09-01 16:20:56 +08:00
jearhe a7922b9cbf fix(sidebar): distinguish schema and table icon colors 2026-09-01 16:18:58 +08:00
jearhe cb754cd0cc fix(grid): preserve local result filters 2026-09-01 16:17:54 +08:00
Clarence 264f5c4836 feat(influxdb3): add influxdb 3 engine over http 2026-09-01 15:28:08 +08:00
t8y2 613c4f7ca3 fix(iotdb): align 1.3.x aggregate columns and format max_time
Closes #7306
2026-09-01 15:18:44 +08:00
zipg 6588f27fbc feat(connection): add dedicated ssh tunnel test button 2026-09-01 15:07:51 +08:00
jearhe f19439ad7a fix(grid): mask loading only on executing result run 2026-09-01 14:09:42 +08:00
t8y2 1aa9f06719 fix(ci): update numeric decoder validation 2026-09-01 14:07:44 +08:00
Melody Ruch 1e75a78228 fix(kingbase): keep timezoneless datetimes off utc 2026-09-01 13:57:03 +08:00
zipg 55611706ba fix(redis): bound large string previews to avoid crashes 2026-09-01 13:56:29 +08:00
Eve 70d7a78c66 fix(editor): guard danger dialog focus across reopen 2026-09-01 13:55:38 +08:00
jearhe 7f9e35d1e8 fix(rocketmq): search consumer groups beyond first page 2026-09-01 13:55:02 +08:00
t8y2 4d3d64bd34 fix(postgres): preserve arbitrary-precision numeric values 2026-09-01 13:24:41 +08:00
onenewcode e43e36e7a4 fix(elasticsearch): fall back to from size without shard doc 2026-09-01 13:18:42 +08:00
Mo-Moore 600af87532 feat(editor): extend insert column hints to select 2026-09-01 13:11:58 +08:00
zipg 2e8ed4b485 feat(search): batch scan all tables in database search 2026-09-01 12:55:48 +08:00
jearhe 8f21e7af9f fix(sqlserver): title unnamed aggregate result columns 2026-09-01 12:55:14 +08:00
Ryan Liu 333c3a3f44 feat(editor): preview dml changes as readonly select 2026-09-01 12:33:59 +08:00
jearhe d59b86bd57 fix(grid): auto expand new rows for multi row paste 2026-09-01 12:29:22 +08:00
dadong 9de152cf72 docs: fix step headings in cloud sync pages 2026-09-01 12:18:39 +08:00
zipg 46584ff46e fix(sqlite): accept linux absolute paths over ssh on windows 2026-09-01 12:18:06 +08:00
zipg cbfa8d7a79 fix(objects): hide row and size stats for non tables 2026-09-01 12:17:33 +08:00
jearhe 694611aade fix(editor): keep scroll after canceling dangerous sql 2026-09-01 12:16:59 +08:00
jearhe a751b5690b fix(grid): lift row cap on table data pagination 2026-09-01 12:16:26 +08:00
二丫讲梵 c7cea5a047 feat(grid): inherit physical column formatting in query results 2026-09-01 11:19:46 +08:00
0xTyson 8be503bf40 feat(update): download app updates in the background 2026-09-01 11:09:44 +08:00
Abeautifulsnow d79d357924 feat(sql-library): add collapse all button and collapsed default 2026-09-01 11:00:59 +08:00
jearhe 23f0b1255b fix(compare): let oracle pick schemas in data compare 2026-09-01 11:00:24 +08:00
二丫讲梵 5f9ba51769 feat(tabs): configurable tab bar position, grouping and sorting 2026-09-01 10:59:50 +08:00
jearhe d69cf9da79 feat(grid): arrow key navigation in detail dialogs 2026-09-01 02:12:55 +08:00
jearhe 8cc04f3af4 fix(oracle): create tables with unquoted plain identifiers 2026-09-01 02:09:33 +08:00
zipg c9f0ee89f9 fix(sqlserver): rewrite sql variant queries without panic 2026-09-01 01:59:01 +08:00
jearhe 14ad5ef9bf fix(mysql): copy varbinary text values as raw strings 2026-09-01 01:47:14 +08:00
zipg cd008b2d56 feat(ui): refresh latest ddl on demand 2026-09-01 01:46:39 +08:00
jearhe f9ff9245ce feat(editor): search table structure hover with aligned offsets 2026-09-01 01:46:04 +08:00
知食分子 554be11aea fix(desktop): macos quit flash, first click and press feedback 2026-09-01 01:19:47 +08:00
Tony Xu 23f511de22 fix(connection): hide url row for mq connection types 2026-09-01 01:19:12 +08:00
jearhe c7c420bbd0 fix(sqlserver): honor sidebar probe limits above 1000 2026-09-01 01:18:36 +08:00
onenewcode cb3de0ef49 fix(elasticsearch): complete json keys without stray quote 2026-09-01 01:17:59 +08:00
二丫讲梵 b41dac4617 fix(desktop): clamp view options popover to viewport 2026-09-01 01:17:23 +08:00
zipg 2779a435cf fix(portable): include settings tables in user data check 2026-09-01 01:16:47 +08:00
jearhe 9a5aae49c8 fix(redis): match script unknown command error wording 2026-09-01 01:16:11 +08:00
jearhe 47ddbf6bb4 fix(diagram): stop search box swallowing keystrokes 2026-09-01 01:15:37 +08:00
zipg 49926f25f1 fix(sql-file): import mysql dumps with binary literals 2026-09-01 00:35:01 +08:00
Elias 351a8b699b feat(xugu): expose tablespaces and datafiles in tree 2026-09-01 00:34:12 +08:00
zipg afe3b3f3c2 feat(sqlserver): add sql server 2000 legacy support 2026-09-01 00:33:06 +08:00
Mo-Moore 69a032ed63 feat(sidebar): batch actions for multi selected nodes 2026-09-01 00:22:02 +08:00
wangfengxiang 2f71a1a8aa feat(argo): add argodb connection type with routine browsing 2026-09-01 00:21:10 +08:00
二丫讲梵 5eb071bc22 feat(editor): add table ddl hover preview switch 2026-09-01 00:20:34 +08:00
wojiukankan f958bf4602 fix(sidebar): follow activation mode for object browsing 2026-09-01 00:19:58 +08:00
jearhe ef2fdb941c fix(oracle): detect default after not null columns 2026-08-31 23:49:35 +08:00
Guoyu Su bc41e8d76d fix(export): persist macos destination volume identity 2026-08-31 23:49:01 +08:00
onenewcode d411f04975 fix(elasticsearch): treat rest reads as read-only 2026-08-31 23:48:26 +08:00
Guoyu Su 2fe26052a7 feat(kafka): add message display ordering 2026-08-31 23:47:51 +08:00
二丫讲梵 42c0dc7f89 feat(ai): support custom headers for api services 2026-08-31 23:47:14 +08:00
二丫讲梵 ba11e4be8d feat(editor): pin live preview atop settings panel 2026-08-31 23:46:38 +08:00
guoziyangnb 662037295c feat(tabs): add close tabs to the left action 2026-08-31 23:46:03 +08:00
jearhe 40a806a24a fix(grid): sort paginated table data server side 2026-08-31 23:45:40 +08:00
二丫讲梵 f075caef21 feat(mcp): add streamable http server and fine-grained authorization 2026-08-31 22:56:16 +08:00
andystenhe f725c50b2c feat(sql-file): add configurable file name filter 2026-08-31 22:55:42 +08:00
onenewcode 8f3134dce8 feat(elasticsearch): add mapping, settings, stats and clear actions 2026-08-31 22:31:35 +08:00
二丫讲梵 a84cb637ba feat(grid): auto compact toolbar on overflow 2026-08-31 22:30:57 +08:00
李子凡 b60f04ea61 fix(grid): keep cloned rows next to their source 2026-08-31 22:29:15 +08:00
zipg 4baa46ca88 fix(transfer): allow unquoted gaussdb transfer identifiers 2026-08-31 22:28:40 +08:00
t8y2 0f33449bdf chore(agents): bump kafka version to 0.1.14 2026-08-31 20:44:31 +08:00
t8y2 5e764ebf05 perf(agents): enable s3 sccache for duckdb and tdengine natives 2026-08-31 20:42:21 +08:00
t8y2 1d98eca99c perf(release): cache static browser builds with sccache 2026-08-31 20:08:49 +08:00
t8y2 4605e4c320 chore(agents): bump dameng version to 0.1.58 2026-08-31 20:06:17 +08:00
t8y2 f70fe38a46 perf(release): cache win7 offline build with sccache 2026-08-31 20:02:11 +08:00
github-actions[bot] dc2e16ffaf chore(packages): release 0.4.76 [skip node-packages-release] 2026-08-31 11:09:05 +00:00
github-actions[bot] 9d63e25dee chore: bump module versions [skip ci] 2026-08-31 10:49:42 +00:00
t8y2 1a833662e2 chore(release): prepare v0.5.99 2026-08-31 18:30:17 +08:00
zipg e0746a2dbe fix(sidebar): correct connection tree font weights 2026-08-31 18:09:29 +08:00
t8y2 f489db4f08 fix(windows): reveal items in explorer for paths with spaces
Command::arg wraps the `/select,<path>` argument in MSVC quotes as a
whole once the path contains a space; Explorer cannot parse that form
and silently falls back to the default shell folder (Documents).
Delegate to tauri-plugin-opener, which selects the item through
SHOpenFolderAndSelectItems instead of the Explorer command line.
2026-08-31 17:28:44 +08:00
zipg 0812e57cf1 fix(grid): improve long text cell hover editor 2026-08-31 16:59:20 +08:00
zipg d144e1bc08 fix(grid): decouple where and order by undo histories 2026-08-31 16:58:32 +08:00
李子凡 0e9f92c917 fix(grid): copy NULL cells as empty fields 2026-08-31 16:57:33 +08:00
Guoyu Su 27f62fe6e2 fix(grid): parse ambiguous comma numbers by locale 2026-08-31 16:54:47 +08:00
zipg 99c3bd0f42 feat(ui): animate sidebar and toolbar interaction feedback 2026-08-31 16:37:51 +08:00
dadong bcea953a50 feat(windows): cover server 2012 r2 with the win7 offline installer 2026-08-31 15:57:15 +08:00
t8y2 eba4e95a0c fix(transfer): keep doris-family mysql connections selectable 2026-08-31 15:32:06 +08:00
t8y2 e690298dcf perf(ui): virtualize large JsonTree value views 2026-08-31 14:10:44 +08:00
zipg d96da5dece fix(sqlserver): edit triggers with events and disabled state 2026-08-31 14:04:13 +08:00
zipg d459628141 fix(transfer): support gbase jdbc connections 2026-08-31 14:03:54 +08:00
jearhe 0c8b796c8c feat(sidebar): highlight current login user schema 2026-08-31 14:03:45 +08:00
Elias a849979dd1 feat(xugu): discover and manage scheduler jobs 2026-08-31 14:03:34 +08:00
zipg 1c0e97b52a fix(oracle): quote routine parameter aliases 2026-08-31 12:31:44 +08:00
zipg b7d33e375f fix(influxdb): merge compat query series and quote explicit columns 2026-08-31 12:31:36 +08:00
t8y2 d1046d54f9 perf(export): run export cores on the blocking pool 2026-08-31 09:58:45 +08:00
jearhe 8e2e8c1b8c fix(tabs): flush tab persistence after connection close 2026-08-31 09:03:15 +08:00
jearhe 8da9905ed5 fix(oceanbase-oracle): format view DDL preview 2026-08-31 09:03:07 +08:00
Elias 0a239bddb2 feat(xugu): mark public synonym scope in sidebar 2026-08-31 09:02:29 +08:00
二丫讲梵 b3fcb3a6f1 feat(backup): add gzip compression and history filtering 2026-08-31 09:02:20 +08:00
t8y2 c305325e37 perf(agents): gate download transfer progress by percent or time 2026-08-31 08:45:40 +08:00
t8y2 d424ea38f0 perf(ai): batch streaming delta events to the webview 2026-08-31 08:45:40 +08:00
skyler 9190554e50 fix(sqlite): validate ssh backup and query errors 2026-08-30 13:30:00 +08:00
Abeautifulsnow e6f8b2187a feat(mcp): add global query timeout override 2026-08-30 12:26:54 +08:00
二丫讲梵 7ba30ae761 feat(sql-files): create, rename, and delete SQL files in tree 2026-08-30 12:26:06 +08:00
0xTyson 9491ab08df fix(ai): append generated SQL without duplicates 2026-08-30 12:25:54 +08:00
skyler 143be57d41 fix(editor): stabilize batch column selection 2026-08-30 10:35:34 +08:00
skyler f664918e95 fix(schema-diff): guard deploy script execution 2026-08-30 10:35:08 +08:00
amwps290 469bd88c37 fix(highgo): paginate queries via limit offset for performance 2026-08-30 09:59:24 +08:00
二丫讲梵 5a11b8f148 feat(sidebar): preview and confirm MongoDB database creation 2026-08-30 09:59:12 +08:00
二丫讲梵 eafc0b2b17 feat(grid): move filter column search to dropdown header 2026-08-30 09:58:58 +08:00
skyler de2c4b38cd fix(objects): render MySQL event create editor with fresh state 2026-08-30 09:58:45 +08:00
二丫讲梵 349342efb7 feat(grid): add 12-hour datetime format and timezone aliases 2026-08-30 09:58:34 +08:00
Mae Morella 694c344278 fix(cli): install default rustls crypto provider to prevent panic 2026-08-30 04:36:35 +08:00
Diego Fabricio 4ac362453f fix(postgres): read foreign keys via pg_catalog for read-only users 2026-08-30 04:17:32 +08:00
Lan-zk 73a7b3f2c7 feat(es): paginate past 10k documents via pit and search_after 2026-08-30 03:57:03 +08:00
skyler 1d71a5fc23 feat(diff): show schema comparison progress and next step
Closes #7484
2026-08-30 03:54:15 +08:00
二丫讲梵 d5f4eadf7a fix(editor): preserve SQL selection when executing from toolbar 2026-08-30 03:05:43 +08:00
二丫讲梵 22775f3639 feat(grid): add go-to-column shortcut and keyboard navigation 2026-08-30 03:05:32 +08:00
二丫讲梵 50e9d8bbb1 feat(ddl): make DDL view dialog draggable by title bar 2026-08-30 03:05:20 +08:00
zipg 45fb7fb7e6 fix(grid): move total row count refresh into the total label
Refs: #7520
2026-08-30 03:05:10 +08:00
zipg 1e8c58b3df fix(ssh): stop ssh config overriding explicit connection fields 2026-08-30 03:04:59 +08:00
t8y2 21f8221d9f fix(grid): reduce DataGrid search allocations 2026-08-30 02:10:39 +08:00
t8y2 170e761fd2 docs(readme): switch to trendshift repository badge 2026-08-30 02:01:36 +08:00
知食分子 f4971d509a fix(desktop): restore legacy webview dialog and highlight fixes 2026-08-30 01:33:59 +08:00
hanwn 0fdddfc083 fix(linux): disable dmabuf renderer for AppImage 2026-08-30 01:11:08 +08:00
s1oopX 2529737a64 fix(sqlserver): highlight SET as a keyword 2026-08-30 01:02:01 +08:00
zipg 8a7d0062e3 fix(sqlserver): rewrite sql_variant CTE queries to avoid crash 2026-08-30 00:49:08 +08:00
二丫讲梵 bd3e585628 feat(editor): batch-select and insert SQL completion columns 2026-08-30 00:47:57 +08:00
jearhe 1637948023 fix(schema-diff): keep deploy script preview in sync with selection 2026-08-30 00:47:45 +08:00
jearhe 1302c0a27b fix(grid): unify transpose long-text editor style 2026-08-30 00:47:34 +08:00
jearhe 508369ee39 fix(mysql): open editor when creating a new event 2026-08-30 00:47:21 +08:00
t8y2 c24e4352e8 docs(readme): expand contributor wall 2026-08-29 23:59:03 +08:00
github-actions[bot] 8f54385b97 chore(packages): release 0.4.75 [skip node-packages-release] 2026-08-28 10:27:48 +00:00
github-actions[bot] cb9f87031f chore(jdbc): bump plugin version [skip ci] 2026-08-28 09:35:31 +00:00
t8y2 5c738f5bf2 chore(release): prepare v0.5.98 2026-08-28 17:34:50 +08:00
二丫讲梵 038ff4e773 fix(editor): align wrapped line numbers 2026-08-28 17:19:40 +08:00
t8y2 9c56de2e9f chore(query): gate test-only execution helper 2026-08-28 09:17:46 +00:00
zipg ef87e45cbc fix(windows): recover hidden startup window 2026-08-28 17:16:55 +08:00
t8y2 f83f707fbc fix(kingbase): honor driver identifier quote for schema context
Closes #7505
2026-08-28 09:13:09 +00:00
zipg 7a4edb6c19 fix(postgres): preserve configured search path 2026-08-28 16:55:04 +08:00
二丫讲梵 b4787a63af feat(editor): configure multi-statement result view 2026-08-28 16:52:02 +08:00
Guoyu Su fd17037a2a fix(linux): bundle AppImage input method modules 2026-08-28 16:51:33 +08:00
二丫讲梵 41db375ee6 style(editor): align line number gutter items 2026-08-28 16:51:03 +08:00
t8y2 1498b27314 fix(query): avoid IRIS derived count aliases
Closes #7486
2026-08-28 08:25:31 +00:00
jearhe 6d8e667bed feat(structure): support search in DDL view 2026-08-28 15:30:08 +08:00
StaravenX df88d7baa7 fix(jdbc): fall back to DatabaseMetaData when oracle dictionary views are missing 2026-08-28 15:29:06 +08:00
t8y2 659fb7f330 feat(mcp): expose message queue send tool
Closes #7236
2026-08-28 07:27:51 +00:00
二丫讲梵 c237ecbfca fix(nacos): allow manual scope when namespace directory is missing 2026-08-28 15:10:25 +08:00
t8y2 bcc64274e8 feat(ci): sign arm64 Windows builds and NSIS uninstallers 2026-08-28 15:05:45 +08:00
Mo-Moore 802019daa3 feat(editor): execute custom sql template by shortcut on selection 2026-08-28 14:27:55 +08:00
jearhe 60bfb41f13 feat(editor): add clear-all action for sql parameter values 2026-08-28 14:25:42 +08:00
t8y2 17e2573fbe fix(kafka): fall back to topic names when metadata times out 2026-08-28 06:11:26 +00:00
t8y2 2467d212d5 fix(dameng): normalize legacy DBX connection URLs
Closes #7482
2026-08-28 14:07:10 +08:00
t8y2 80224ab1d9 fix(oracle): support 11g data compare pagination
Closes #7470
2026-08-28 05:42:17 +00:00
t8y2 89086cc22b fix(schema): list HighGo PostgreSQL extensions
Closes #7448
2026-08-28 05:02:43 +00:00
Abeautifulsnow fdfad88590 feat(editor): add global default transaction commit mode 2026-08-28 12:37:11 +08:00
李子凡 d3a97164aa fix(sql): preserve empty raw placeholders 2026-08-28 12:36:37 +08:00
t8y2 d4245f0d58 fix(dameng): open java.net module for multi-NIC hosts 2026-08-28 12:36:00 +08:00
李子凡 2c5f926bd8 fix(grid): pad short timestamp fractions 2026-08-28 12:35:47 +08:00
amwps290 20a2994fa1 feat(structure): show table constraints for postgres-compatible dbs 2026-08-28 12:14:04 +08:00
t8y2 7ad05b178f fix(mysql): support DRDS index expression fallback
Closes #7363
2026-08-28 03:42:29 +00:00
t8y2 be307ba2d7 fix(query): collapse duplicated external-driver query branch 2026-08-28 11:41:24 +08:00
zipg 10bce698ae fix(mcp): install rustls crypto provider for TLS connections 2026-08-28 11:13:14 +08:00
Diego Fabricio c0abc26fe8 fix(grid): allow rerunning exact total row count from status 2026-08-28 11:12:41 +08:00
zipg 425ca055ea fix(mysql): keep pool when health probe times out during export 2026-08-28 11:12:06 +08:00
dadong 99a9961712 fix(export): harden config export passphrase handling 2026-08-28 11:11:35 +08:00
zipg d8ec9bb8d0 feat(connection): choose group when creating a connection 2026-08-28 11:11:00 +08:00
t8y2 4636788f49 fix(export): fall back when Doris rejects read-only snapshot
Closes #7446
2026-08-28 02:36:06 +00:00
t8y2 22305b7d0f fix(jdbc): retry invalid UTF-8 table previews without left wrappers
Closes #7268
2026-08-28 02:05:30 +00:00
zipg 71d07348ad perf(oracle): stop slow index metadata from blocking first query 2026-08-27 22:25:39 +08:00
Melody Ruch b198dd1327 fix(postgres): affected rows for WITH-wrapped DML and CTE completion 2026-08-27 22:25:09 +08:00
Melody Ruch 9290391985 fix(structure): emit NULL keyword for nullable MySQL timestamp columns 2026-08-27 22:24:40 +08:00
Melody Ruch 1016ccc352 fix(backup): match db.schema.table excludes when database equals schema 2026-08-27 22:24:09 +08:00
Abeautifulsnow ec0b6768be feat(grid): add row/column crosshair highlight setting 2026-08-27 22:23:13 +08:00
t8y2 b09c6f4f21 fix(export): stabilize Kingbase large result pagination
Closes #7377
2026-08-27 19:08:05 +08:00
t8y2 74ffab29ca fix(transfer): enable identity insert for SQL Server
Closes #7335
2026-08-27 18:52:15 +08:00
t8y2 e996f8aefb fix(transfer): use rownum pagination for OceanBase Oracle
Closes #7209
2026-08-27 18:47:57 +08:00
zipg 4fed29fba2 fix(structure): improve mysql auto-increment editing and field shortcuts 2026-08-27 18:21:52 +08:00
二丫讲梵 214bf86a6f feat(desktop): add zen mode toggle for data tabs 2026-08-27 17:59:10 +08:00
Farahani 955558c81f feat(safety): temporary write-unlock window for read-only connections 2026-08-27 17:37:54 +08:00
Mo-Moore 7933b7656f feat(jdbc,desktop): browse Hive/Inceptor routines from system views 2026-08-27 17:32:56 +08:00
t8y2 e5572ead7a fix(grid): parse pg timestamps deterministically for local date sort
Closes #7281
2026-08-27 17:32:09 +08:00
t8y2 bd6c780aba fix(core): uppercase dameng column types in structure ddl
Closes #7343
2026-08-27 17:23:40 +08:00
dadong dfd4d3b7ff feat(import): add SQL script import with dialect-aware parsing 2026-08-27 17:17:41 +08:00
Yang Fei ab4e747bf9 feat(editor): drag result grid columns into the SQL editor 2026-08-27 17:16:57 +08:00
Mo-Moore 9e4fe8ce56 fix(jdbc): connect Hive2 JDBC through registered driver 2026-08-27 17:16:32 +08:00
t8y2 6a6e5fbad9 fix(core): use row_number pagination for sqlserver table transfers
Closes #7356
2026-08-27 17:10:10 +08:00
t8y2 e8d6370b1e fix(grid): reset scroll to top on external result replacement
Results replaced without an internal grid action (SQL re-executed from
the editor or a grid-embedded run button) kept the previous viewport, so
DOM-mode grids stayed stranded at the old scroll offset after running an
ORDER BY query. The result watch now resets to the first row unless the
replacement is an in-place refresh (prepareFullReload markers) or an
infinite-scroll append completion. A remounted grid also no longer
adopts a scroll-only pending-changes snapshot; cached scroll is only
replayed alongside unsaved edit state, while the KeepAlive reactivate
path still restores pure scroll.

Closes #7341
2026-08-27 16:33:23 +08:00
t8y2 4eeeafa858 fix(sidebar): expand collapsed groups when locating tab
Closes #7387
2026-08-27 16:33:23 +08:00
t8y2 e4dc5bace8 fix(core): use drop user cascade for oceanbase oracle schema drop
Closes #7351
2026-08-27 16:33:23 +08:00
zipg 920ab62cf1 feat(grid): support deleting rows from JOIN query results 2026-08-27 16:28:14 +08:00
t8y2 36a7b7bd3a feat(ai): localize custom provider label 2026-08-27 16:23:51 +08:00
t8y2 565feb8745 feat(ci): cache native c deps through sccache 2026-08-27 16:14:42 +08:00
Abeautifulsnow 40549c9573 fix(dialog): clamp grid track and wrap footer so dialogs stop clipping 2026-08-27 16:06:25 +08:00
zipg eb0994e980 feat(grid): support composite unique reference keys 2026-08-27 16:00:00 +08:00
t8y2 cf03de122a fix(ci): keep sccache server alive for post-build stats 2026-08-27 15:56:48 +08:00
t8y2 d9beaaaa1c fix(agents): raise gradle daemon heap for parallel builds 2026-08-27 14:38:53 +08:00
1196 changed files with 186894 additions and 12609 deletions
@@ -0,0 +1,75 @@
[CmdletBinding()]
param(
[Parameter(Mandatory = $true)]
[string]$InstallerPath,
[int]$StartupSeconds = 20
)
Set-StrictMode -Version Latest
$ErrorActionPreference = "Stop"
$os = [System.Environment]::OSVersion.Version
if ($os.Major -ne 6 -or ($os.Minor -ne 1 -and $os.Minor -ne 3)) {
throw "The installer smoke test requires Windows 7 or Server 2012 R2 (kernel 6.1/6.3), detected $($os.ToString())."
}
$installerPath = (Resolve-Path -LiteralPath $InstallerPath).Path
$testRoot = Join-Path ([System.IO.Path]::GetTempPath()) "dbx-server2012-installer-smoke"
$resolvedTemp = (Resolve-Path -LiteralPath ([System.IO.Path]::GetTempPath())).Path.TrimEnd('\')
$resolvedTestRoot = [System.IO.Path]::GetFullPath($testRoot).TrimEnd('\')
if (!$resolvedTestRoot.StartsWith("$resolvedTemp\", [System.StringComparison]::OrdinalIgnoreCase)) {
throw "Refusing to use a smoke-test directory outside the system temporary directory: $resolvedTestRoot"
}
if (Test-Path -LiteralPath $resolvedTestRoot) {
Remove-Item -LiteralPath $resolvedTestRoot -Recurse -Force
}
$appProcess = $null
try {
$installer = Start-Process -FilePath $installerPath -ArgumentList @("/S", "/D=$resolvedTestRoot") -Wait -PassThru
if ($installer.ExitCode -ne 0) {
throw "Server 2012 R2 test installer failed with exit code $($installer.ExitCode)."
}
$appPath = Join-Path $resolvedTestRoot "dbx.exe"
$runtimePath = Join-Path $resolvedTestRoot "webview2-fixed-runtime\msedgewebview2.exe"
foreach ($requiredPath in @($appPath, $runtimePath)) {
if (!(Test-Path -LiteralPath $requiredPath -PathType Leaf)) {
throw "Server 2012 R2 installer omitted required file: $requiredPath"
}
}
$env:WEBVIEW2_BROWSER_EXECUTABLE_FOLDER = Join-Path $resolvedTestRoot "webview2-fixed-runtime"
$env:WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS = "--no-sandbox --disable-gpu"
$env:DBX_STARTUP_LOG_DIR = Join-Path $resolvedTestRoot "smoke-logs"
New-Item -ItemType Directory -Path $env:DBX_STARTUP_LOG_DIR -Force | Out-Null
$appProcess = Start-Process -FilePath $appPath -PassThru
Start-Sleep -Seconds $StartupSeconds
$appProcess.Refresh()
if ($appProcess.HasExited) {
$startupLog = Join-Path $env:DBX_STARTUP_LOG_DIR "startup.log"
$logText = if (Test-Path -LiteralPath $startupLog) {
Get-Content -LiteralPath $startupLog -Raw
} else {
"<startup.log was not created>"
}
throw "DBX exited during the Server 2012 R2 startup smoke test with code $($appProcess.ExitCode).`n$logText"
}
Write-Host "Server 2012 R2 installer smoke test passed after $StartupSeconds seconds: $installerPath"
}
finally {
if ($null -ne $appProcess -and !$appProcess.HasExited) {
Stop-Process -Id $appProcess.Id -Force -ErrorAction SilentlyContinue
}
$uninstallerPath = Join-Path $resolvedTestRoot "uninstall.exe"
if (Test-Path -LiteralPath $uninstallerPath -PathType Leaf) {
$uninstaller = Start-Process -FilePath $uninstallerPath -ArgumentList @("/S", "_?=$resolvedTestRoot") -Wait -PassThru
if ($uninstaller.ExitCode -ne 0) {
Write-Warning "Server 2012 R2 test uninstaller returned exit code $($uninstaller.ExitCode)."
}
}
}
@@ -0,0 +1,67 @@
[CmdletBinding()]
param(
[string]$RuntimeDirectory = (Join-Path $PSScriptRoot "..\..\src-tauri\webview2-fixed-runtime"),
[string]$LoaderPath = (Join-Path ([System.IO.Path]::GetTempPath()) "dbx-win7-webview2-loader-probe\WebView2Loader.dll"),
[string]$ExpectedVersion = "109.0.1518.78"
)
Set-StrictMode -Version Latest
$ErrorActionPreference = "Stop"
$os = [System.Environment]::OSVersion.Version
if ($os.Major -ne 6 -or ($os.Minor -ne 1 -and $os.Minor -ne 3)) {
throw "The offline runtime probe requires Windows 7 or Server 2012 R2 (kernel 6.1/6.3), detected $($os.ToString())."
}
$runtimeDirectory = (Resolve-Path -LiteralPath $RuntimeDirectory).Path
$loaderPath = (Resolve-Path -LiteralPath $LoaderPath).Path
$runtimeExecutable = Join-Path $runtimeDirectory "msedgewebview2.exe"
if (!(Test-Path -LiteralPath $runtimeExecutable -PathType Leaf)) {
throw "WebView2 fixed runtime executable does not exist: $runtimeExecutable"
}
$escapedLoaderPath = $loaderPath.Replace('"', '""')
$source = @"
using System;
using System.Runtime.InteropServices;
public static class DbxServer2012WebView2Probe
{
[DllImport(@"$escapedLoaderPath", CharSet = CharSet.Unicode, ExactSpelling = true)]
public static extern int GetAvailableCoreWebView2BrowserVersionString(
string browserExecutableFolder,
out IntPtr versionInfo);
}
"@
Add-Type -TypeDefinition $source -Language CSharp
$versionPointer = [IntPtr]::Zero
$result = [DbxServer2012WebView2Probe]::GetAvailableCoreWebView2BrowserVersionString(
$runtimeDirectory,
[ref]$versionPointer
)
$resultHex = [System.BitConverter]::ToUInt32(
[System.BitConverter]::GetBytes([int32]$result),
0
)
$hresult = "0x{0:X8}" -f $resultHex
if ($result -ne 0) {
throw "Server 2012 R2 WebView2 loader rejected the fixed runtime with HRESULT $hresult."
}
if ($versionPointer -eq [IntPtr]::Zero) {
throw "Server 2012 R2 WebView2 loader returned an empty version pointer."
}
try {
$version = [Runtime.InteropServices.Marshal]::PtrToStringUni($versionPointer)
}
finally {
[Runtime.InteropServices.Marshal]::FreeCoTaskMem($versionPointer)
}
if ([string]::IsNullOrWhiteSpace($version) -or !$version.StartsWith($ExpectedVersion)) {
throw "Expected WebView2 fixed runtime $ExpectedVersion, detected '$version'."
}
Write-Host "Server 2012 R2 WebView2 runtime probe passed: loader=$loaderPath runtime=$runtimeDirectory version=$version"
@@ -0,0 +1,53 @@
[CmdletBinding()]
param(
[Parameter(Mandatory = $true)]
[string]$BinaryPath
)
Set-StrictMode -Version Latest
$ErrorActionPreference = "Stop"
# The Win7 / Server 2012 R2 bundle must statically link the WebView2 loader from
# SDK 1.0.902.49, the last one verified to work on kernel 6.1/6.3. Loader
# 1.0.1054.31 and everything newer fail on Server 2012 R2 with
# ERROR_NOT_SUPPORTED (0x80070032, MicrosoftEdge/WebView2Feedback#2025) and make
# the app show "Could not find the WebView2 Runtime". Those loaders carry error
# strings that 1.0.902.49 does not, so any of the markers below appearing in
# dbx.exe proves the pinned loader was not the one linked. The PE import audit
# cannot catch this: 1.0.1054.31 imports nothing newer than 1.0.902.49.
$loaderMarkers = @(
"WebView2: Failed to find the app exe path.",
"WebView2: Failed to find the WebView2 client dll at:",
"WebView2: Failed to find an installed WebView2 runtime or non-stable Microsoft Edge installation."
)
if (!(Test-Path -LiteralPath $BinaryPath -PathType Leaf)) {
throw "Win7 WebView2 loader audit target does not exist: $BinaryPath"
}
$resolvedPath = (Resolve-Path -LiteralPath $BinaryPath).Path
# Map every byte to one character so the ASCII needle search cannot miss on
# arbitrary binary content.
$latin1 = [System.Text.Encoding]::GetEncoding(28591)
$content = $latin1.GetString([System.IO.File]::ReadAllBytes($resolvedPath))
$violations = @()
foreach ($marker in $loaderMarkers) {
if ($content.Contains($marker)) {
$violations += $marker
}
}
if ($violations.Count -gt 0) {
$summary = $violations -join "`n"
throw @"
dbx.exe contains WebView2 loader strings newer than the pinned 1.0.902.49:
$summary
A loader >= 1.0.1054.31 was linked instead of the prepared one and will fail on
Server 2012 R2 with ERROR_NOT_SUPPORTED. Do not ship this binary: rebuild the
win7 target and investigate how the registry loader patch was bypassed.
"@
}
Write-Host "Win7 WebView2 loader audit passed: no post-1.0.902.49 loader markers in $resolvedPath"
+27 -1
View File
@@ -52,6 +52,7 @@ const nativeDriverDirectories = {
cassandra: "cassandra-go",
duckdb: "duckdb",
hive: "hive-go",
argo: "argo-go",
oracle: "oracle-go",
kingbase: "kingbase-go",
iotdb: "iotdb",
@@ -61,21 +62,46 @@ const nativeDriverDirectories = {
rocketmq: "rocketmq",
zookeeper: "zookeeper",
tdengine: "tdengine",
etcd: "etcd-go",
etcd2: "etcd2-go",
};
const nativeDriverModules = new Set(["cassandra", "duckdb", "hive", "oracle", "xugu", "kingbase", "iotdb", "neo4j", "vastbase", "rabbitmq", "rocketmq", "zookeeper", "tdengine"]);
const crateNativeDriverDirectories = {
"sqlite-worker": "crates/dbx-sqlite-worker",
};
const nativeDriverModules = new Set(["cassandra", "duckdb", "hive", "argo", "oracle", "xugu", "kingbase", "iotdb", "neo4j", "vastbase", "rabbitmq", "rocketmq", "zookeeper", "tdengine", "etcd", "etcd2", "sqlite-worker"]);
const nativeDriverSharedPaths = {
hive: [
"agents/go-common/go-gssapi",
"agents/go-common/gohive",
"agents/go-common/gosasl",
],
argo: [
"agents/go-common/go-gssapi",
"agents/go-common/gohive",
"agents/go-common/gosasl",
],
zookeeper: [
"agents/go-common/go-gssapi",
"agents/go-common/gosasl",
],
etcd: [
"agents/go-common/go-semver",
],
};
function resolveAgentModule(moduleName, { legacyStandaloneModules, moduleExists, readModuleFile }) {
const cratePath = crateNativeDriverDirectories[moduleName];
if (cratePath && moduleExists(cratePath)) {
return {
checkDir: cratePath,
modulePath: cratePath,
sharedPaths: [],
javaBuild: false,
nativeBuild: true,
commonDependent: false,
};
}
let checkDir = null;
const nativeDriverDirectory = nativeDriverDirectories[moduleName];
if (nativeDriverDirectory && moduleExists(`agents/drivers/${nativeDriverDirectory}`)) {
@@ -34,6 +34,20 @@ test("bumps DuckDB after its initial release", () => {
assert.equal(result.versions.duckdb, "0.1.1");
});
test("bumps the SQLite SSH worker from its crate path", () => {
const result = evaluateAgentVersionBump({
versions: { "sqlite-worker": "0.1.0" },
changedFiles: ["crates/dbx-sqlite-worker/src/runtime.rs"],
moduleExists: (path) => path === "crates/dbx-sqlite-worker",
readModuleFile: () => "",
});
assert.equal(result.versions["sqlite-worker"], "0.1.1");
assert.deepEqual(result.changedModules, ["sqlite-worker"]);
assert.deepEqual(result.javaModules, []);
assert.deepEqual(result.nativeModules, ["sqlite-worker"]);
});
test("classifies TDengine Rust changes as native-only", () => {
const result = evaluateAgentVersionBump({
versions: { tdengine: "0.1.39" },
@@ -0,0 +1,93 @@
[CmdletBinding(DefaultParameterSetName = "Runtime")]
param(
[Parameter(Mandatory = $true)]
[string]$ComputerName,
[Parameter(Mandatory = $true)]
[string]$UserName,
[string]$Password = $env:SERVER2012_PASSWORD,
[Parameter(Mandatory = $true)]
[ValidateSet("Runtime", "Installer")]
[string]$Mode,
[Parameter(Mandatory = $true, ParameterSetName = "Runtime")]
[string]$RuntimeDirectory,
[Parameter(Mandatory = $true, ParameterSetName = "Runtime")]
[string]$LoaderPath,
[Parameter(Mandatory = $true, ParameterSetName = "Installer")]
[string]$InstallerPath,
[switch]$UseSsl
)
Set-StrictMode -Version Latest
$ErrorActionPreference = "Stop"
if ([string]::IsNullOrWhiteSpace($Password)) {
throw "SERVER2012_PASSWORD is required."
}
$securePassword = ConvertTo-SecureString $Password -AsPlainText -Force
$credential = New-Object System.Management.Automation.PSCredential($UserName, $securePassword)
$sessionParameters = @{
ComputerName = $ComputerName
Credential = $credential
Authentication = "Negotiate"
ErrorAction = "Stop"
}
$useSslEnabled = $UseSsl -or $env:SERVER2012_USE_SSL -eq "true"
if ($useSslEnabled) {
$sessionParameters.UseSSL = $true
$sessionParameters.SessionOption = New-PSSessionOption -SkipCACheck -SkipCNCheck -SkipRevocationCheck
}
$session = $null
$remoteRoot = "C:\Windows\Temp\dbx-server2012-ci-$([Guid]::NewGuid().ToString('N'))"
try {
$session = New-PSSession @sessionParameters
Invoke-Command -Session $session -ScriptBlock {
param($Path)
New-Item -ItemType Directory -Path $Path -Force | Out-Null
} -ArgumentList $remoteRoot
if ($Mode -eq "Runtime") {
$runtimeDirectory = (Resolve-Path -LiteralPath $RuntimeDirectory).Path
$loaderPath = (Resolve-Path -LiteralPath $LoaderPath).Path
$probeScript = (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot "assert-webview2-server2012-runtime.ps1")).Path
Copy-Item -LiteralPath $runtimeDirectory -Destination "$remoteRoot\webview2-fixed-runtime" -Recurse -ToSession $session
Copy-Item -LiteralPath $loaderPath -Destination "$remoteRoot\WebView2Loader.dll" -ToSession $session
Copy-Item -LiteralPath $probeScript -Destination "$remoteRoot\assert-runtime.ps1" -ToSession $session
Invoke-Command -Session $session -ScriptBlock {
param($Root)
& "$Root\assert-runtime.ps1" `
-RuntimeDirectory "$Root\webview2-fixed-runtime" `
-LoaderPath "$Root\WebView2Loader.dll"
} -ArgumentList $remoteRoot
} else {
$installerPath = (Resolve-Path -LiteralPath $InstallerPath).Path
$smokeScript = (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot "assert-server2012-installer-smoke.ps1")).Path
Copy-Item -LiteralPath $installerPath -Destination "$remoteRoot\installer.exe" -ToSession $session
Copy-Item -LiteralPath $smokeScript -Destination "$remoteRoot\assert-installer.ps1" -ToSession $session
Invoke-Command -Session $session -ScriptBlock {
param($Root)
& "$Root\assert-installer.ps1" -InstallerPath "$Root\installer.exe"
} -ArgumentList $remoteRoot
}
}
finally {
if ($null -ne $session) {
Invoke-Command -Session $session -ScriptBlock {
param($Path)
$windowsTemp = [System.IO.Path]::GetFullPath("C:\Windows\Temp").TrimEnd('\')
$resolvedPath = [System.IO.Path]::GetFullPath($Path).TrimEnd('\')
if (!$resolvedPath.StartsWith("$windowsTemp\dbx-server2012-ci-", [System.StringComparison]::OrdinalIgnoreCase)) {
throw "Refusing to clean an unexpected remote validation directory: $resolvedPath"
}
if (Test-Path -LiteralPath $resolvedPath) {
Remove-Item -LiteralPath $resolvedPath -Recurse -Force
}
} -ArgumentList $remoteRoot
Remove-PSSession $session
}
}
@@ -4,11 +4,16 @@ param()
Set-StrictMode -Version Latest
$ErrorActionPreference = "Stop"
# Newer WebView2 static loaders import EventSetInformation, which does not exist on Windows 7.
# The loader entry points are stable, so the Win7 bundle uses the last verified compatible SDK loader.
$sdkVersion = "1.0.1054.31"
$sdkPackageSha256 = "0afe683aa3d143a5f6330db1ce833c69278b38fe5e1eadec52f26910ad26e22f"
$loaderSha256 = "76314119685bbf4c2b2423a44e81b57beadc914c943d0e772fd6bc78c8e6b0e8"
# WebView2 static loaders newer than SDK 1.0.1054 import EventSetInformation, which does
# not exist on Windows 7, while 1.0.1054.31 itself fails on Server 2012 R2 with
# ERROR_NOT_SUPPORTED (0x80070032, MicrosoftEdge/WebView2Feedback#2025). SDK 1.0.902.49 is
# the loader verified on both Windows 7 SP1 and Server 2012 R2 real machines, so the single
# win7 + server 2012 r2 offline bundle links it instead of the webview2-com-sys default.
$sdkVersion = "1.0.902.49"
$sdkPackageSha256 = "b483c906b03690267108f4304d456eac0e718131ef994e69aaa5a21532b512c6"
$loaderSha256 = "aa5c26670f1b18d0fa2a56ac3f1ae30110c332a8bfbd555a7be3e548d1b0da3d"
$loaderDllSha256 = "fdf978ba706578b05967d7f0181f462147864a5aa74f36016a62cb3d3dbe6909"
$legacy1054LoaderSha256 = "76314119685bbf4c2b2423a44e81b57beadc914c943d0e772fd6bc78c8e6b0e8"
$webView2ComSysVersion = "0.38.2"
$upstreamLoaderSha256 = "0659b741bde6348d4c4a6ec4ceb9af50e3d0048ed9cd3c8659bccbb61fde55ee"
@@ -19,9 +24,10 @@ $extractedPath = Join-Path $temporaryRoot "extracted"
try {
New-Item -ItemType Directory -Path $extractedPath -Force | Out-Null
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
$packageUrl = "https://www.nuget.org/api/v2/package/Microsoft.Web.WebView2/$sdkVersion"
Write-Host "Downloading WebView2 SDK $sdkVersion for the Windows 7 loader..."
Write-Host "Downloading WebView2 SDK $sdkVersion for the Windows 7 / Server 2012 R2 loader..."
Invoke-WebRequest -Uri $packageUrl -OutFile $packagePath -UseBasicParsing
$actualPackageSha256 = (Get-FileHash -LiteralPath $packagePath -Algorithm SHA256).Hash.ToLowerInvariant()
@@ -44,7 +50,12 @@ try {
$actualLoaderSha256 = (Get-FileHash -LiteralPath $legacyLoader -Algorithm SHA256).Hash.ToLowerInvariant()
if ($actualLoaderSha256 -ne $loaderSha256) {
throw "Unexpected Windows 7 WebView2 loader SHA256: $actualLoaderSha256"
throw "Unexpected Windows 7 / Server 2012 R2 WebView2 loader SHA256: $actualLoaderSha256"
}
$actualLoaderDllSha256 = (Get-FileHash -LiteralPath $legacyLoaderDll -Algorithm SHA256).Hash.ToLowerInvariant()
if ($actualLoaderDllSha256 -ne $loaderDllSha256) {
throw "Unexpected Windows 7 / Server 2012 R2 WebView2 loader DLL SHA256: $actualLoaderDllSha256"
}
Push-Location $repositoryRoot
@@ -78,7 +89,7 @@ try {
}
$existingLoaderSha256 = (Get-FileHash -LiteralPath $loaderDestination -Algorithm SHA256).Hash.ToLowerInvariant()
$knownLoaderHashes = @($upstreamLoaderSha256, $loaderSha256)
$knownLoaderHashes = @($upstreamLoaderSha256, $legacy1054LoaderSha256, $loaderSha256)
if ($existingLoaderSha256 -notin $knownLoaderHashes) {
throw "Refusing to replace an unknown webview2-com-sys loader SHA256: $existingLoaderSha256"
}
@@ -96,7 +107,7 @@ try {
$probeLoader = Join-Path $probeDirectory "WebView2Loader.dll"
Copy-Item -LiteralPath $legacyLoaderDll -Destination $probeLoader -Force
Write-Host "Prepared WebView2 SDK $sdkVersion static loader for Windows 7: $loaderDestination"
Write-Host "Prepared WebView2 SDK $sdkVersion static loader for Windows 7 / Server 2012 R2: $loaderDestination"
Write-Host "Prepared WebView2 SDK $sdkVersion loader probe DLL: $probeLoader"
}
finally {
@@ -15,7 +15,7 @@ import { basename, join } from "node:path";
import { tmpdir } from "node:os";
const REGISTRY_ASSET = "agent-registry.json";
const NATIVE_MODULES = new Set(["duckdb", "oracle", "xugu", "kingbase", "iotdb", "neo4j", "vastbase", "rabbitmq", "rocketmq", "zookeeper", "tdengine"]);
const NATIVE_MODULES = new Set(["duckdb", "oracle", "xugu", "kingbase", "iotdb", "neo4j", "vastbase", "rabbitmq", "rocketmq", "zookeeper", "tdengine", "etcd", "etcd2"]);
const PLATFORMS = [
"macos-aarch64",
"macos-x64",
+67
View File
@@ -0,0 +1,67 @@
#!/usr/bin/env bash
set -euo pipefail
die() {
echo "AppImage input-method verification failed: $*" >&2
exit 1
}
if [[ $# -ne 1 ]]; then
die "usage: $0 <AppImage file or bundle directory>"
fi
input_path=$1
if [[ -f "$input_path" ]]; then
appimage=$input_path
elif [[ -d "$input_path" ]]; then
appimages=()
while IFS= read -r -d '' candidate; do
appimages+=("$candidate")
done < <(find "$input_path" -maxdepth 1 -type f -name '*.AppImage' -print0)
[[ ${#appimages[@]} -eq 1 ]] \
|| die "expected exactly one AppImage in $input_path, found ${#appimages[@]}"
appimage=${appimages[0]}
else
die "path does not exist: $input_path"
fi
appimage=$(realpath "$appimage")
work_dir=$(mktemp -d "${TMPDIR:-/tmp}/dbx-appimage-input-methods.XXXXXX")
trap 'rm -rf "$work_dir"' EXIT
(
cd "$work_dir"
"$appimage" --appimage-extract >/dev/null
)
appdir="$work_dir/squashfs-root"
[[ -d "$appdir" ]] || die "AppImage extraction did not create squashfs-root"
appdir=$(realpath "$appdir")
require_artifact() {
local basename=$1
local match
local resolved
match=$(find "$appdir" \( -type f -o -type l \) -name "$basename" -print -quit)
[[ -n "$match" ]] || die "missing bundled artifact: $basename"
[[ -e "$match" ]] || die "bundled artifact is a broken symbolic link: ${match#"$appdir"/}"
resolved=$(realpath "$match")
[[ "$resolved" == "$appdir"/* ]] \
|| die "bundled artifact resolves outside the AppImage: ${match#"$appdir"/}"
echo "Found $basename at ${match#"$appdir"/}"
}
require_artifact im-fcitx5.so
require_artifact im-ibus.so
require_artifact libFcitx5GClient.so.2
require_artifact libibus-1.0.so.5
cache=''
while IFS= read -r -d '' candidate; do
if grep -Fq 'im-fcitx5.so' "$candidate" && grep -Fq 'im-ibus.so' "$candidate"; then
cache=$candidate
break
fi
done < <(find "$appdir" -type f -path '*/gtk-3.0/*/immodules.cache' -print0)
[[ -n "$cache" ]] || die "no private GTK 3 immodules.cache registers both Fcitx 5 and IBus"
echo "Verified GTK input-method cache at ${cache#"$appdir"/}"
+339 -11
View File
@@ -147,7 +147,11 @@ jobs:
for module in "${MODULES[@]}"; do
TASKS+=(":${module}:shadowJar")
done
./gradlew "${TASKS[@]}" --parallel
# agents/common changes trigger a parallel mass rebuild of every
# packaged module; the Gradle daemon default heap is too small for
# ~25 shadowJar fat jars at once on a 16 GB runner.
./gradlew "${TASKS[@]}" --parallel \
-Dorg.gradle.jvmargs="-Xmx4g -XX:MaxMetaspaceSize=1g"
working-directory: agents
- run: python3 scripts/validate_agent_jars.py
working-directory: agents
@@ -343,6 +347,86 @@ jobs:
name: rocketmq-native
path: "release-native/dbx-agent-rocketmq-*"
build-etcd-native:
needs: [bump-versions]
if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'etcd') }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: "1.26.x"
- name: Test etcd native agent
working-directory: agents/drivers/etcd-go
run: go test ./...
- name: Cross-compile etcd native agent
shell: bash
run: |
mkdir -p release-native
cd agents/drivers/etcd-go
declare -A TARGETS=(
["macos-aarch64"]="darwin/arm64"
["macos-x64"]="darwin/amd64"
["linux-aarch64"]="linux/arm64"
["linux-x64"]="linux/amd64"
["windows-aarch64"]="windows/arm64"
["windows-x64"]="windows/amd64"
)
for platform in "${!TARGETS[@]}"; do
IFS=/ read -r goos goarch <<< "${TARGETS[$platform]}"
output="../../../release-native/dbx-agent-etcd-${platform}"
if [[ "$goos" == "windows" ]]; then
output="${output}.exe"
fi
echo "Building $platform ($goos/$goarch)"
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -trimpath -ldflags="-s -w" -o "$output" .
done
ls -lh ../../../release-native
- uses: actions/upload-artifact@v4
with:
name: etcd-native
path: "release-native/dbx-agent-etcd-*"
build-etcd2-native:
needs: [bump-versions]
if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'etcd2') }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: "1.24.x"
- name: Test etcd v2 native agent
working-directory: agents/drivers/etcd2-go
run: go test ./...
- name: Cross-compile etcd v2 native agent
shell: bash
run: |
mkdir -p release-native
cd agents/drivers/etcd2-go
declare -A TARGETS=(
["macos-aarch64"]="darwin/arm64"
["macos-x64"]="darwin/amd64"
["linux-aarch64"]="linux/arm64"
["linux-x64"]="linux/amd64"
["windows-aarch64"]="windows/arm64"
["windows-x64"]="windows/amd64"
)
for platform in "${!TARGETS[@]}"; do
IFS=/ read -r goos goarch <<< "${TARGETS[$platform]}"
output="../../../release-native/dbx-agent-etcd2-${platform}"
if [[ "$goos" == "windows" ]]; then
output="${output}.exe"
fi
echo "Building $platform ($goos/$goarch)"
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -trimpath -ldflags="-s -w" -o "$output" .
done
ls -lh ../../../release-native
- uses: actions/upload-artifact@v4
with:
name: etcd2-native
path: "release-native/dbx-agent-etcd2-*"
build-zookeeper-native:
needs: [bump-versions]
if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'zookeeper') }}
@@ -463,6 +547,46 @@ jobs:
name: hive-native
path: "release-native/dbx-agent-hive-*"
build-argo-native:
needs: [bump-versions]
if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'argo') }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
with:
go-version: "1.23.x"
- name: Test Argo native agent
working-directory: agents/drivers/argo-go
run: go test ./...
- name: Cross-compile Argo native agent
shell: bash
run: |
mkdir -p release-native
cd agents/drivers/argo-go
declare -A TARGETS=(
["macos-aarch64"]="darwin/arm64"
["macos-x64"]="darwin/amd64"
["linux-aarch64"]="linux/arm64"
["linux-x64"]="linux/amd64"
["windows-aarch64"]="windows/arm64"
["windows-x64"]="windows/amd64"
)
for platform in "${!TARGETS[@]}"; do
IFS=/ read -r goos goarch <<< "${TARGETS[$platform]}"
output="../../../release-native/dbx-agent-argo-${platform}"
if [[ "$goos" == "windows" ]]; then
output="${output}.exe"
fi
echo "Building $platform ($goos/$goarch)"
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -trimpath -ldflags="-s -w" -o "$output" .
done
ls -lh ../../../release-native
- uses: actions/upload-artifact@v4
with:
name: argo-native
path: "release-native/dbx-agent-argo-*"
build-kingbase-native:
needs: [bump-versions]
if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'kingbase') }}
@@ -662,6 +786,7 @@ jobs:
CARGO_INCREMENTAL: "0"
CARGO_TARGET_DIR: ${{ github.workspace }}/target/duckdb-driver
RUSTC_WRAPPER: sccache
SCCACHE_GHA_ENABLED: ${{ secrets.SCCACHE_S3_BUCKET == '' && 'true' || 'false' }}
steps:
- uses: actions/checkout@v5
- uses: dtolnay/rust-toolchain@1.97.1
@@ -681,6 +806,36 @@ jobs:
- uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10
with:
version: "v0.10.0"
- name: Configure S3 sccache
if: env.SCCACHE_GHA_ENABLED != 'true'
shell: bash
env:
CACHE_BUCKET: ${{ secrets.SCCACHE_S3_BUCKET }}
CACHE_ENDPOINT: ${{ secrets.SCCACHE_S3_ENDPOINT }}
CACHE_REGION: ${{ secrets.SCCACHE_S3_REGION }}
CACHE_KEY_PREFIX: ${{ secrets.SCCACHE_S3_KEY_PREFIX }}
CACHE_ACCESS_KEY_ID: ${{ secrets.SCCACHE_S3_ACCESS_KEY_ID }}
CACHE_SECRET_ACCESS_KEY: ${{ secrets.SCCACHE_S3_SECRET_ACCESS_KEY }}
run: |
{
echo "SCCACHE_BUCKET=${CACHE_BUCKET}"
echo "SCCACHE_ENDPOINT=${CACHE_ENDPOINT}"
echo "SCCACHE_REGION=${CACHE_REGION}"
echo "SCCACHE_S3_KEY_PREFIX=${CACHE_KEY_PREFIX}"
echo "SCCACHE_S3_USE_SSL=true"
echo "AWS_ACCESS_KEY_ID=${CACHE_ACCESS_KEY_ID}"
echo "AWS_SECRET_ACCESS_KEY=${CACHE_SECRET_ACCESS_KEY}"
echo "SCCACHE_IDLE_TIMEOUT=0"
# libduckdb C++ compilation dominates this build. The manylinux
# legs build inside a container and enable CC/CXX wrapping there;
# MSVC stays unwrapped.
case "${RUNNER_OS}" in
macOS)
echo "CC=${SCCACHE_PATH} clang"
echo "CXX=${SCCACHE_PATH} clang++"
;;
esac
} >> "$GITHUB_ENV"
- name: Build DuckDB native driver
shell: bash
run: |
@@ -693,6 +848,15 @@ jobs:
-e HOME=/tmp/dbx-rust-home \
-e CARGO_INCREMENTAL=0 \
-e CARGO_TARGET_DIR=/workspace/target/duckdb-driver \
-e RUSTC_WRAPPER \
-e SCCACHE_BUCKET \
-e SCCACHE_ENDPOINT \
-e SCCACHE_REGION \
-e SCCACHE_S3_KEY_PREFIX \
-e SCCACHE_S3_USE_SSL \
-e AWS_ACCESS_KEY_ID \
-e AWS_SECRET_ACCESS_KEY \
-e SCCACHE_IDLE_TIMEOUT \
-v "${{ github.workspace }}:/workspace" \
-w /workspace/agents/drivers/duckdb \
"${{ matrix.manylinux_image }}" \
@@ -700,12 +864,27 @@ jobs:
mkdir -p "$HOME"
curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain 1.97.1
source "$HOME/.cargo/env"
if [ -n "${SCCACHE_BUCKET:-}" ]; then
case "$(uname -m)" in
x86_64) SCCACHE_ARCH=x86_64-unknown-linux-musl ;;
aarch64) SCCACHE_ARCH=aarch64-unknown-linux-musl ;;
esac
curl -sSfL "https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-${SCCACHE_ARCH}.tar.gz" \
| tar -xz --strip-components=1 -C "$HOME/.cargo/bin"
export CC="sccache cc" CXX="sccache c++"
sccache --version
else
unset RUSTC_WRAPPER
fi
cargo build \
--manifest-path Cargo.toml \
--locked \
--release \
--bin dbx-duckdb-driver \
--target "${{ matrix.target }}"
if [ -n "${SCCACHE_BUCKET:-}" ]; then
sccache --show-stats || true
fi
'
else
BUILD_STD=()
@@ -737,11 +916,67 @@ jobs:
"${{ matrix.manylinux_image }}" \
"/driver/dbx-agent-duckdb-${{ matrix.platform }}" < /dev/null
fi
- name: Show sccache stats
if: always()
continue-on-error: true
shell: bash
run: ${SCCACHE_PATH} --show-stats
- uses: actions/upload-artifact@v4
with:
name: duckdb-native-${{ matrix.platform }}
path: "release-native/dbx-agent-duckdb-*"
build-sqlite-worker-native:
needs: [bump-versions]
if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'sqlite-worker') }}
strategy:
fail-fast: false
matrix:
include:
- runner: ubuntu-22.04
target: x86_64-unknown-linux-musl
platform: linux-x64
rustflags: "-C target-feature=+crt-static"
- runner: ubuntu-22.04-arm
target: aarch64-unknown-linux-musl
platform: linux-aarch64
rustflags: "-C target-feature=+crt-static -C link-arg=-z -C link-arg=max-page-size=65536"
runs-on: ${{ matrix.runner }}
steps:
- uses: actions/checkout@v5
- name: Setup Rust
uses: dtolnay/rust-toolchain@1.97.1
with:
targets: ${{ matrix.target }}
- name: Install cargo-zigbuild
run: pip3 install cargo-zigbuild
- name: Rust cache
uses: swatinem/rust-cache@v2
with:
workspaces: "./ -> target"
shared-key: sqlite-worker-${{ matrix.target }}
cache-on-failure: true
- name: Build musl SQLite SSH worker
env:
RUSTFLAGS: ${{ matrix.rustflags }}
run: cargo zigbuild --release -p dbx-sqlite-worker --target ${{ matrix.target }}
- name: Stage SQLite SSH worker
run: |
mkdir -p release-native
cp \
"target/${{ matrix.target }}/release/dbx-sqlite-worker" \
"release-native/dbx-agent-sqlite-worker-${{ matrix.platform }}"
- uses: actions/upload-artifact@v4
with:
name: sqlite-worker-native-${{ matrix.platform }}
path: "release-native/dbx-agent-sqlite-worker-*"
build-tdengine-native:
needs: [bump-versions]
if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'tdengine') }}
@@ -784,6 +1019,7 @@ jobs:
CARGO_INCREMENTAL: "0"
CARGO_TARGET_DIR: ${{ github.workspace }}/target/tdengine-driver
RUSTC_WRAPPER: sccache
SCCACHE_GHA_ENABLED: ${{ secrets.SCCACHE_S3_BUCKET == '' && 'true' || 'false' }}
steps:
- uses: actions/checkout@v5
- uses: dtolnay/rust-toolchain@1.97.1
@@ -799,6 +1035,36 @@ jobs:
- uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10
with:
version: "v0.10.0"
- name: Configure S3 sccache
if: env.SCCACHE_GHA_ENABLED != 'true'
shell: bash
env:
CACHE_BUCKET: ${{ secrets.SCCACHE_S3_BUCKET }}
CACHE_ENDPOINT: ${{ secrets.SCCACHE_S3_ENDPOINT }}
CACHE_REGION: ${{ secrets.SCCACHE_S3_REGION }}
CACHE_KEY_PREFIX: ${{ secrets.SCCACHE_S3_KEY_PREFIX }}
CACHE_ACCESS_KEY_ID: ${{ secrets.SCCACHE_S3_ACCESS_KEY_ID }}
CACHE_SECRET_ACCESS_KEY: ${{ secrets.SCCACHE_S3_SECRET_ACCESS_KEY }}
run: |
{
echo "SCCACHE_BUCKET=${CACHE_BUCKET}"
echo "SCCACHE_ENDPOINT=${CACHE_ENDPOINT}"
echo "SCCACHE_REGION=${CACHE_REGION}"
echo "SCCACHE_S3_KEY_PREFIX=${CACHE_KEY_PREFIX}"
echo "SCCACHE_S3_USE_SSL=true"
echo "AWS_ACCESS_KEY_ID=${CACHE_ACCESS_KEY_ID}"
echo "AWS_SECRET_ACCESS_KEY=${CACHE_SECRET_ACCESS_KEY}"
echo "SCCACHE_IDLE_TIMEOUT=0"
# taos client C/C++ compilation dominates this build. The manylinux
# legs build inside a container and enable CC/CXX wrapping there;
# MSVC stays unwrapped.
case "${RUNNER_OS}" in
macOS)
echo "CC=${SCCACHE_PATH} clang"
echo "CXX=${SCCACHE_PATH} clang++"
;;
esac
} >> "$GITHUB_ENV"
- name: Build TDengine native driver
shell: bash
run: |
@@ -808,6 +1074,15 @@ jobs:
-e HOME=/tmp/dbx-rust-home \
-e CARGO_INCREMENTAL=0 \
-e CARGO_TARGET_DIR=/workspace/target/tdengine-driver \
-e RUSTC_WRAPPER \
-e SCCACHE_BUCKET \
-e SCCACHE_ENDPOINT \
-e SCCACHE_REGION \
-e SCCACHE_S3_KEY_PREFIX \
-e SCCACHE_S3_USE_SSL \
-e AWS_ACCESS_KEY_ID \
-e AWS_SECRET_ACCESS_KEY \
-e SCCACHE_IDLE_TIMEOUT \
-v "${{ github.workspace }}:/workspace" \
-w /workspace \
"${{ matrix.manylinux_image }}" \
@@ -815,12 +1090,27 @@ jobs:
mkdir -p "$HOME"
curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain 1.97.1
source "$HOME/.cargo/env"
if [ -n "${SCCACHE_BUCKET:-}" ]; then
case "$(uname -m)" in
x86_64) SCCACHE_ARCH=x86_64-unknown-linux-musl ;;
aarch64) SCCACHE_ARCH=aarch64-unknown-linux-musl ;;
esac
curl -sSfL "https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-${SCCACHE_ARCH}.tar.gz" \
| tar -xz --strip-components=1 -C "$HOME/.cargo/bin"
export CC="sccache cc" CXX="sccache c++"
sccache --version
else
unset RUSTC_WRAPPER
fi
cargo build \
--manifest-path agents/drivers/tdengine/Cargo.toml \
--locked \
--release \
--bin dbx-tdengine-driver \
--target "${{ matrix.target }}"
if [ -n "${SCCACHE_BUCKET:-}" ]; then
sccache --show-stats || true
fi
'
else
BUILD_STD=()
@@ -848,6 +1138,11 @@ jobs:
elif [ "${{ matrix.smoke }}" = "true" ]; then
"release-native/dbx-agent-tdengine-${{ matrix.platform }}${{ matrix.extension }}" < /dev/null
fi
- name: Show sccache stats
if: always()
continue-on-error: true
shell: bash
run: ${SCCACHE_PATH} --show-stats
- uses: actions/upload-artifact@v4
with:
name: tdengine-native-${{ matrix.platform }}
@@ -962,7 +1257,7 @@ jobs:
retention-days: 1
release:
needs: [bump-versions, commit-versions, build-agents, build-oracle-native, build-xugu-native, build-rabbitmq-native, build-rocketmq-native, build-zookeeper-native, build-cassandra-native, build-hive-native, build-kingbase-native, build-vastbase-native, build-neo4j-native, build-iotdb-native, build-duckdb-native, build-tdengine-native, build-jre, reuse-previous-assets]
needs: [bump-versions, commit-versions, build-agents, build-oracle-native, build-xugu-native, build-rabbitmq-native, build-rocketmq-native, build-etcd-native, build-etcd2-native, build-zookeeper-native, build-cassandra-native, build-hive-native, build-argo-native, build-kingbase-native, build-vastbase-native, build-neo4j-native, build-iotdb-native, build-duckdb-native, build-sqlite-worker-native, build-tdengine-native, build-jre, reuse-previous-assets]
if: ${{ always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}
runs-on: ubuntu-latest
steps:
@@ -1087,15 +1382,19 @@ jobs:
kingbase) echo "金仓KingbaseES" ;;
vastbase) echo "Vastbase" ;;
duckdb) echo "DuckDB" ;;
sqlite-worker) echo "SQLite SSH Worker" ;;
xugu) echo "虚谷 XuguDB" ;;
rabbitmq) echo "RabbitMQ" ;;
rocketmq) echo "Apache RocketMQ" ;;
zookeeper) echo "Apache ZooKeeper" ;;
cassandra) echo "Apache Cassandra" ;;
hive) echo "Apache Hive" ;;
argo) echo "星环Argo" ;;
neo4j) echo "Neo4j" ;;
iotdb) echo "Apache IoTDB" ;;
tdengine) echo "TDengine" ;;
etcd) echo "etcd" ;;
etcd2) echo "etcd 2.x (v2 API)" ;;
*) echo "$name" ;;
esac
}
@@ -1159,7 +1458,7 @@ jobs:
[ -n "$DRIVERS" ] && DRIVERS="${DRIVERS},"$'\n'
DRIVERS="${DRIVERS}$(generate_jar_entry "$name" "$label" "$f" "$jre_key" "$version" "$external_driver" "$native_json")"
done
for name in oracle xugu kingbase vastbase neo4j iotdb duckdb rabbitmq rocketmq zookeeper cassandra hive tdengine; do
for name in oracle xugu kingbase vastbase neo4j iotdb duckdb sqlite-worker rabbitmq rocketmq zookeeper cassandra hive argo tdengine etcd etcd2; do
version=$(get_module_version "$name")
[ -f "release/dbx-agent-${name}-${version}.jar" ] && continue
native_json=$(generate_native_platforms "$name" "$version")
@@ -1239,6 +1538,7 @@ jobs:
kingbase) echo "金仓KingbaseES" ;;
vastbase) echo "Vastbase" ;;
duckdb) echo "DuckDB" ;;
sqlite-worker) echo "SQLite SSH Worker" ;;
oracle) echo "Oracle" ;;
xugu) echo "虚谷 XuguDB" ;;
rabbitmq) echo "RabbitMQ" ;;
@@ -1246,13 +1546,33 @@ jobs:
zookeeper) echo "Apache ZooKeeper" ;;
cassandra) echo "Apache Cassandra" ;;
hive) echo "Apache Hive" ;;
argo) echo "星环Argo" ;;
neo4j) echo "Neo4j" ;;
iotdb) echo "Apache IoTDB" ;;
tdengine) echo "TDengine" ;;
etcd) echo "etcd" ;;
etcd2) echo "etcd 2.x (v2 API)" ;;
*) echo "$name" ;;
esac
}
# agents/common rebuilds every packaging driver; report common-only
# commits once in a shared section instead of duplicating them under
# every driver. Commits that also touch driver code stay attributed
# to that driver alone.
COMMON_NOTES=""
while IFS= read -r line; do
sha="${line%% *}"
subject="${line#* }"
if git diff-tree --no-commit-id --name-only -r "$sha" | grep -qv '^agents/common/'; then
continue
fi
COMMON_NOTES="${COMMON_NOTES}- ${subject}"$'\n'
done < <(git log --no-merges --format='%H %s' "$PREV_TAG"..HEAD -- agents/common/build.gradle agents/common/src/main/)
if [ -n "$COMMON_NOTES" ]; then
NOTES="${NOTES}### 公共组件 (agents/common)"$'\n'"${COMMON_NOTES}"$'\n'
fi
for name in "${DRIVER_NAMES[@]}"; do
old_ver=$(echo "$PREV_VERSIONS" | python3 -c "import sys,json; print(json.load(sys.stdin).get('$name',''))")
new_ver=$(echo "$MODULE_VERSIONS" | python3 -c "import sys,json; print(json.load(sys.stdin).get('$name',''))")
@@ -1266,7 +1586,6 @@ jobs:
label=$(native_only_label "$name")
fi
NOTES="${NOTES}### ${label} (${old_ver} → ${new_ver})"$'\n'
if [ "$name" = "oracle" ]; then
LOG_PATH="agents/drivers/oracle-go/"
elif [ "$name" = "kingbase" ]; then
@@ -1277,10 +1596,14 @@ jobs:
LOG_PATH="agents/drivers/cassandra-go/"
elif [ "$name" = "hive" ]; then
LOG_PATH="agents/drivers/hive-go/"
elif [ "$name" = "argo" ]; then
LOG_PATH="agents/drivers/argo-go/"
elif [ "$name" = "neo4j" ]; then
LOG_PATH="agents/drivers/neo4j-go/"
elif [ "$name" = "iotdb" ]; then
LOG_PATH="agents/drivers/iotdb/"
elif [ "$name" = "sqlite-worker" ]; then
LOG_PATH="crates/dbx-sqlite-worker/"
elif [ -d "agents/drivers/$name" ]; then
LOG_PATH="agents/drivers/$name/"
else
@@ -1293,21 +1616,26 @@ jobs:
"agents/go-common/gohive/"
"agents/go-common/gosasl/"
)
elif [ "$name" = "argo" ]; then
LOG_PATHS+=(
"agents/go-common/go-gssapi/"
"agents/go-common/gohive/"
"agents/go-common/gosasl/"
)
elif [ "$name" = "zookeeper" ]; then
LOG_PATHS+=(
"agents/go-common/go-gssapi/"
"agents/go-common/gosasl/"
)
fi
DRIVER_NOTES=""
while IFS= read -r line; do
[ -n "$line" ] && NOTES="${NOTES}- ${line}"$'\n'
[ -n "$line" ] && DRIVER_NOTES="${DRIVER_NOTES}- ${line}"$'\n'
done < <(git log --oneline "$PREV_TAG"..HEAD -- "${LOG_PATHS[@]}" | sed 's/^[0-9a-f]* //')
# include common changes if any
while IFS= read -r line; do
[ -n "$line" ] && NOTES="${NOTES}- ${line}"$'\n'
done < <(git log --oneline "$PREV_TAG"..HEAD -- agents/common/ | sed 's/^[0-9a-f]* //')
NOTES="${NOTES}"$'\n'
if [ -z "$DRIVER_NOTES" ]; then
DRIVER_NOTES="- 无驱动专属变更"$'\n'
fi
NOTES="${NOTES}### ${label} (${old_ver} → ${new_ver})"$'\n'"${DRIVER_NOTES}"$'\n'
done
if [ -z "$NOTES" ]; then
+29 -1
View File
@@ -71,6 +71,13 @@ jobs:
echo "SCCACHE_S3_USE_SSL=true"
echo "AWS_ACCESS_KEY_ID=${CACHE_ACCESS_KEY_ID}"
echo "AWS_SECRET_ACCESS_KEY=${CACHE_SECRET_ACCESS_KEY}"
# Keep the server alive through post-compile test/lint phases so
# "Show sccache stats" reflects the real counters.
echo "SCCACHE_IDLE_TIMEOUT=0"
# Also cache the C/C++ compilation of -sys crates. These jobs are
# native Linux builds, so plain CC/CXX select the target compiler.
echo "CC=${SCCACHE_PATH} cc"
echo "CXX=${SCCACHE_PATH} c++"
} >> "$GITHUB_ENV"
- name: Rust cache
@@ -193,7 +200,7 @@ jobs:
needs: changes
if: needs.changes.outputs.windows_win7_bundle == 'true'
runs-on: windows-2022
timeout-minutes: 90
timeout-minutes: 120
env:
CARGO_INCREMENTAL: "0"
RUSTFLAGS: -C debuginfo=line-tables-only -C target-feature=+crt-static
@@ -251,6 +258,10 @@ jobs:
shell: pwsh
run: ./.github/scripts/assert-win7-pe-compat.ps1 -BinaryPath target/x86_64-win7-windows-msvc/release/dbx.exe
- name: Audit Win7 WebView2 loader markers
shell: pwsh
run: ./.github/scripts/assert-webview2-win7-loader.ps1 -BinaryPath target/x86_64-win7-windows-msvc/release/dbx.exe
- name: Bundle Windows 7 fixed-runtime installer
shell: pwsh
run: |
@@ -378,6 +389,13 @@ jobs:
echo "SCCACHE_S3_USE_SSL=true"
echo "AWS_ACCESS_KEY_ID=${CACHE_ACCESS_KEY_ID}"
echo "AWS_SECRET_ACCESS_KEY=${CACHE_SECRET_ACCESS_KEY}"
# Keep the server alive through post-compile test/lint phases so
# "Show sccache stats" reflects the real counters.
echo "SCCACHE_IDLE_TIMEOUT=0"
# Also cache the C/C++ compilation of -sys crates. These jobs are
# native Linux builds, so plain CC/CXX select the target compiler.
echo "CC=${SCCACHE_PATH} cc"
echo "CXX=${SCCACHE_PATH} c++"
} >> "$GITHUB_ENV"
- name: Rust cache
@@ -413,6 +431,8 @@ jobs:
env:
# sccache cannot reuse Cargo incremental artifacts, so avoid generating them in CI.
CARGO_INCREMENTAL: "0"
# Deep async export tests exceed the Rust test harness's default thread stack on Linux.
RUST_MIN_STACK: 8388608
RUSTC_WRAPPER: sccache
# The fast lane skips only system font discovery while retaining the other default capabilities.
RUST_FEATURE_MODE: ${{ github.event_name == 'pull_request' && needs.changes.outputs.rust_full != 'true' && 'fast' || 'full' }}
@@ -455,6 +475,13 @@ jobs:
echo "SCCACHE_S3_USE_SSL=true"
echo "AWS_ACCESS_KEY_ID=${CACHE_ACCESS_KEY_ID}"
echo "AWS_SECRET_ACCESS_KEY=${CACHE_SECRET_ACCESS_KEY}"
# Keep the server alive through post-compile test/lint phases so
# "Show sccache stats" reflects the real counters.
echo "SCCACHE_IDLE_TIMEOUT=0"
# Also cache the C/C++ compilation of -sys crates. These jobs are
# native Linux builds, so plain CC/CXX select the target compiler.
echo "CC=${SCCACHE_PATH} cc"
echo "CXX=${SCCACHE_PATH} c++"
} >> "$GITHUB_ENV"
- name: Rust cache
@@ -675,6 +702,7 @@ jobs:
- '.github/workflows/update-nix-pnpm-hash.yml'
windows_win7_bundle:
- '.github/scripts/assert-win7-pe-compat.ps1'
- '.github/scripts/assert-webview2-win7-loader.ps1'
- '.github/scripts/assert-win7-installer-content.ps1'
- '.github/scripts/assert-webview2-win7-runtime.ps1'
- '.github/scripts/prepare-webview2-win7-loader.ps1'
+38 -4
View File
@@ -343,6 +343,22 @@ jobs:
echo "SCCACHE_S3_USE_SSL=true"
echo "AWS_ACCESS_KEY_ID=${CACHE_ACCESS_KEY_ID}"
echo "AWS_SECRET_ACCESS_KEY=${CACHE_SECRET_ACCESS_KEY}"
# Keep the server alive through the post-compile packaging tail so
# "Show sccache stats" reflects the real counters.
echo "SCCACHE_IDLE_TIMEOUT=0"
# Also cache the C/C++ compilation of -sys crates. All matrix legs
# are native builds, so plain CC/CXX select the target compiler;
# MSVC stays unwrapped. SCCACHE_PATH avoids depending on PATH.
case "${RUNNER_OS}" in
Linux)
echo "CC=${SCCACHE_PATH} cc"
echo "CXX=${SCCACHE_PATH} c++"
;;
macOS)
echo "CC=${SCCACHE_PATH} clang"
echo "CXX=${SCCACHE_PATH} clang++"
;;
esac
} >> "$GITHUB_ENV"
- name: Rust cache
@@ -458,11 +474,13 @@ jobs:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
VERSION: ${{ needs.prepare.outputs.version }}
run: |
# npm CDN visibility after publish has been observed to lag past
# three minutes; the window must exceed that by a wide margin.
wait_for_package() {
package="$1"
for attempt in $(seq 1 18); do
for attempt in $(seq 1 60); do
if npm view "${package}@${VERSION}" version >/dev/null 2>&1; then return 0; fi
echo "Waiting for ${package}@${VERSION} to become visible on npm (${attempt}/18)."
echo "Waiting for ${package}@${VERSION} to become visible on npm (${attempt}/60)."
sleep 10
done
return 1
@@ -561,6 +579,22 @@ jobs:
echo "SCCACHE_S3_USE_SSL=true"
echo "AWS_ACCESS_KEY_ID=${CACHE_ACCESS_KEY_ID}"
echo "AWS_SECRET_ACCESS_KEY=${CACHE_SECRET_ACCESS_KEY}"
# Keep the server alive through the post-compile packaging tail so
# "Show sccache stats" reflects the real counters.
echo "SCCACHE_IDLE_TIMEOUT=0"
# Also cache the C/C++ compilation of -sys crates. All matrix legs
# are native builds, so plain CC/CXX select the target compiler;
# MSVC stays unwrapped. SCCACHE_PATH avoids depending on PATH.
case "${RUNNER_OS}" in
Linux)
echo "CC=${SCCACHE_PATH} cc"
echo "CXX=${SCCACHE_PATH} c++"
;;
macOS)
echo "CC=${SCCACHE_PATH} clang"
echo "CXX=${SCCACHE_PATH} clang++"
;;
esac
} >> "$GITHUB_ENV"
- name: Rust cache
@@ -685,11 +719,11 @@ jobs:
run: |
wait_for_package() {
package="$1"
for attempt in $(seq 1 18); do
for attempt in $(seq 1 60); do
if npm view "${package}@${VERSION}" version >/dev/null 2>&1; then
return 0
fi
echo "Waiting for ${package}@${VERSION} to become visible on npm (${attempt}/18)."
echo "Waiting for ${package}@${VERSION} to become visible on npm (${attempt}/60)."
sleep 10
done
return 1
+3 -3
View File
@@ -47,7 +47,7 @@ jobs:
TAG="${{ steps.version.outputs.tag }}"
ASSETS=(
"${PRODUCT_NAME}_${VERSION}_aarch64.dmg"
"${PRODUCT_NAME}_${VERSION}_arm64.dmg"
"${PRODUCT_NAME}_${VERSION}_x64.dmg"
"${PRODUCT_NAME}_${VERSION}_x64-setup.exe"
"${PRODUCT_NAME}_${VERSION}_arm64-setup.exe"
@@ -83,7 +83,7 @@ jobs:
}
echo "SHA256 hashes:"
compute_hash "artifacts/${PRODUCT_NAME}_${VERSION}_aarch64.dmg" "sha_dmg_arm64"
compute_hash "artifacts/${PRODUCT_NAME}_${VERSION}_arm64.dmg" "sha_dmg_arm64"
compute_hash "artifacts/${PRODUCT_NAME}_${VERSION}_x64.dmg" "sha_dmg_x64"
compute_hash "artifacts/${PRODUCT_NAME}_${VERSION}_x64-setup.exe" "sha_exe_x64"
compute_hash "artifacts/${PRODUCT_NAME}_${VERSION}_arm64-setup.exe" "sha_exe_arm64"
@@ -104,7 +104,7 @@ jobs:
cat > Casks/dbx.rb <<'RUBY_EOF'
cask "dbx" do
arch arm: "aarch64", intel: "x64"
arch arm: "arm64", intel: "x64"
version "__VERSION__"
sha256 arm: "__SHA_ARM64__",
+133 -46
View File
@@ -117,19 +117,28 @@ jobs:
done
build:
# Explicit name keeps matrix-only fields such as asset_pattern from being
# appended to the generated job display name.
name: build (${{ matrix.platform }}, ${{ matrix.target }})
needs: release-ready
strategy:
fail-fast: false
matrix:
include:
# assetNamePattern renames every bundle artifact of a matrix leg, so
# it is set only on aarch64 legs (aarch64 -> arm64). x64 legs leave
# it empty and keep the default Tauri CLI names (x64.dmg, amd64
# deb/rpm/AppImage) used by downloadLinks and package consumers.
- platform: macos-latest
target: aarch64-apple-darwin
asset_pattern: "DBX_[version]_arm64[setup][ext]"
- platform: macos-15-intel
target: x86_64-apple-darwin
- platform: ubuntu-22.04
target: x86_64-unknown-linux-gnu
- platform: ubuntu-22.04-arm
target: aarch64-unknown-linux-gnu
asset_pattern: "DBX_[version]_arm64[setup][ext]"
runs-on: ${{ matrix.platform }}
env:
@@ -180,6 +189,23 @@ jobs:
echo "SCCACHE_S3_USE_SSL=true"
echo "AWS_ACCESS_KEY_ID=${CACHE_ACCESS_KEY_ID}"
echo "AWS_SECRET_ACCESS_KEY=${CACHE_SECRET_ACCESS_KEY}"
# Linking, bundling, and signing run for 30+ minutes after the last
# rustc call; the server's 10-minute default idle timeout would
# exit first and zero the counters shown by "Show sccache stats".
echo "SCCACHE_IDLE_TIMEOUT=0"
# Also cache the C/C++ compilation of -sys crates. All matrix legs
# are native builds, so plain CC/CXX select the target compiler;
# MSVC stays unwrapped. SCCACHE_PATH avoids depending on PATH.
case "${RUNNER_OS}" in
Linux)
echo "CC=${SCCACHE_PATH} cc"
echo "CXX=${SCCACHE_PATH} c++"
;;
macOS)
echo "CC=${SCCACHE_PATH} clang"
echo "CXX=${SCCACHE_PATH} clang++"
;;
esac
} >> "$GITHUB_ENV"
- name: Compute Rust dependency hash
@@ -226,7 +252,9 @@ jobs:
if: startsWith(matrix.platform, 'ubuntu')
run: |
sudo apt-get update
sudo apt-get install -y libwebkit2gtk-4.1-dev libgtk-3-dev libappindicator3-dev librsvg2-dev patchelf libssl-dev xdg-utils
# Bundle GTK input-method modules from the same Jammy ABI baseline
# instead of loading potentially incompatible modules from the host.
sudo apt-get install -y fcitx5-frontend-gtk3 ibus-gtk3 libwebkit2gtk-4.1-dev libgtk-3-dev libappindicator3-dev librsvg2-dev patchelf libssl-dev xdg-utils
- name: Setup Tauri signing key
shell: bash
@@ -249,8 +277,15 @@ jobs:
with:
tagName: ${{ github.ref_name }}
releaseDraft: true
# Empty on x64 legs: tauri-action then keeps the default Tauri CLI names.
assetNamePattern: ${{ matrix.asset_pattern }}
args: --target ${{ matrix.target }}
- name: Verify AppImage input methods
if: startsWith(matrix.platform, 'ubuntu')
shell: bash
run: ./.github/scripts/verify-appimage-input-methods.sh "target/${{ matrix.target }}/release/bundle/appimage"
- name: Show sccache stats
if: always()
continue-on-error: true
@@ -266,10 +301,8 @@ jobs:
include:
- target: x86_64-pc-windows-msvc
arch: x64
sign_windows: true
- target: aarch64-pc-windows-msvc
arch: arm64
sign_windows: false
runs-on: windows-2022
env:
CARGO_INCREMENTAL: "0"
@@ -322,6 +355,23 @@ jobs:
echo "SCCACHE_S3_USE_SSL=true"
echo "AWS_ACCESS_KEY_ID=${CACHE_ACCESS_KEY_ID}"
echo "AWS_SECRET_ACCESS_KEY=${CACHE_SECRET_ACCESS_KEY}"
# Linking, bundling, and signing run for 30+ minutes after the last
# rustc call; the server's 10-minute default idle timeout would
# exit first and zero the counters shown by "Show sccache stats".
echo "SCCACHE_IDLE_TIMEOUT=0"
# Also cache the C/C++ compilation of -sys crates. All matrix legs
# are native builds, so plain CC/CXX select the target compiler;
# MSVC stays unwrapped. SCCACHE_PATH avoids depending on PATH.
case "${RUNNER_OS}" in
Linux)
echo "CC=${SCCACHE_PATH} cc"
echo "CXX=${SCCACHE_PATH} c++"
;;
macOS)
echo "CC=${SCCACHE_PATH} clang"
echo "CXX=${SCCACHE_PATH} clang++"
;;
esac
} >> "$GITHUB_ENV"
- name: Compute Rust dependency hash
@@ -358,7 +408,6 @@ jobs:
fi
- name: Setup TrustAsia VSign
if: matrix.sign_windows
shell: pwsh
env:
VSIGN_OPERATOR_PFX_B64: ${{ secrets.VSIGN_OPERATOR_PFX_B64 }}
@@ -366,7 +415,6 @@ jobs:
run: ./.github/scripts/setup-vsign.ps1
- name: Build and sign Tauri app
if: matrix.sign_windows
uses: tauri-apps/tauri-action@v0
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -375,18 +423,10 @@ jobs:
with:
tagName: ${{ github.ref_name }}
releaseDraft: true
# No assetNamePattern: the default Tauri CLI names already produce
# DBX_[version]_x64/arm64-setup.exe and keep the _en-US.msi names.
args: --target ${{ matrix.target }} --config src-tauri/tauri.vsign.conf.json
- name: Build unsigned Tauri app
if: ${{ !matrix.sign_windows }}
uses: tauri-apps/tauri-action@v0
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tagName: ${{ github.ref_name }}
releaseDraft: true
args: --target ${{ matrix.target }}
- name: Verify Windows bundles
shell: pwsh
run: |
@@ -403,38 +443,31 @@ jobs:
if ($msiInstallers.Count -eq 0) {
throw "No MSI installer was produced for ${{ matrix.target }}"
}
if ("${{ matrix.sign_windows }}" -eq "true") {
./.github/scripts/assert-authenticode.ps1 -Path @(
$nsisInstallers.FullName
$msiInstallers.FullName
)
}
./.github/scripts/assert-authenticode.ps1 -Path @(
$nsisInstallers.FullName
$msiInstallers.FullName
)
- name: Upload Windows WebView2 offline installer
shell: pwsh
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SSIGNCODE_OPERATOR_PWD: ${{ matrix.sign_windows && secrets.VSIGN_OPERATOR_PWD || '' }}
VSIGN_KEY_PIN: ${{ matrix.sign_windows && secrets.VSIGN_KEY_PIN || '' }}
SSIGNCODE_OPERATOR_PWD: ${{ secrets.VSIGN_OPERATOR_PWD }}
VSIGN_KEY_PIN: ${{ secrets.VSIGN_KEY_PIN }}
run: |
$version = "${env:GITHUB_REF_NAME}".TrimStart("v")
$arch = "${{ matrix.arch }}"
$bundleDir = "target/${{ matrix.target }}/release/bundle/nsis"
$offlineName = "DBX_${version}_${arch}-webview2-offline-setup.exe"
$offlineName = "DBX_${version}_${arch}-offline-setup.exe"
$before = @{}
if (Test-Path $bundleDir) {
Get-ChildItem $bundleDir -Filter "*.exe" | ForEach-Object { $before[$_.FullName] = $_.LastWriteTimeUtc }
}
if ("${{ matrix.sign_windows }}" -eq "true") {
pnpm tauri bundle --bundles nsis --target ${{ matrix.target }} `
--config src-tauri/tauri.webview2-offline.conf.json `
--config src-tauri/tauri.vsign.conf.json
} else {
pnpm tauri bundle --bundles nsis --target ${{ matrix.target }} `
--config src-tauri/tauri.webview2-offline.conf.json
}
pnpm tauri bundle --bundles nsis --target ${{ matrix.target }} `
--config src-tauri/tauri.webview2-offline.conf.json `
--config src-tauri/tauri.vsign.conf.json
$installer = Get-ChildItem $bundleDir -Filter "*.exe" |
Where-Object { !$before.ContainsKey($_.FullName) -or $_.LastWriteTimeUtc -gt $before[$_.FullName] } |
@@ -445,9 +478,7 @@ jobs:
throw "Missing WebView2 offline installer in ${bundleDir}"
}
if ("${{ matrix.sign_windows }}" -eq "true") {
./.github/scripts/assert-authenticode.ps1 -Path $installer.FullName
}
./.github/scripts/assert-authenticode.ps1 -Path $installer.FullName
Copy-Item $installer.FullName $offlineName -Force
gh release upload "${env:GITHUB_REF_NAME}" $offlineName --repo "${env:GITHUB_REPOSITORY}" --clobber
@@ -456,8 +487,8 @@ jobs:
shell: pwsh
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SSIGNCODE_OPERATOR_PWD: ${{ matrix.sign_windows && secrets.VSIGN_OPERATOR_PWD || '' }}
VSIGN_KEY_PIN: ${{ matrix.sign_windows && secrets.VSIGN_KEY_PIN || '' }}
SSIGNCODE_OPERATOR_PWD: ${{ secrets.VSIGN_OPERATOR_PWD }}
VSIGN_KEY_PIN: ${{ secrets.VSIGN_KEY_PIN }}
run: |
$version = "${env:GITHUB_REF_NAME}".TrimStart("v")
$arch = "${{ matrix.arch }}"
@@ -475,10 +506,8 @@ jobs:
throw "Release tag version $version does not match the built DBX package version $appVersion"
}
if ("${{ matrix.sign_windows }}" -eq "true") {
./.github/scripts/vsign-sign.ps1 -FilePath $exePath
./.github/scripts/assert-authenticode.ps1 -Path $exePath
}
./.github/scripts/vsign-sign.ps1 -FilePath $exePath
./.github/scripts/assert-authenticode.ps1 -Path $exePath
New-Item -ItemType Directory -Force -Path $portableDir | Out-Null
Copy-Item $exePath (Join-Path $portableDir "DBX.exe") -Force
@@ -519,7 +548,7 @@ jobs:
run: ${SCCACHE_PATH} --show-stats
- name: Remove temporary VSign files
if: ${{ always() && matrix.sign_windows }}
if: always()
shell: pwsh
run: |
$workDir = if ([string]::IsNullOrWhiteSpace($env:VSIGN_WORK_DIR)) {
@@ -535,6 +564,12 @@ jobs:
needs: release-ready
runs-on: windows-2022
environment: release-signing
# No sccache here: the loader patch rewrites a file inside the cargo
# registry, which is invisible to any compile cache. v0.6.0 shipped a
# binary linked against the wrong (>= 1.0.1054.31) WebView2 loader even
# though the patched 1.0.902.49 loader was verified on disk, so caching is
# disabled for this target until the patch is vendored into the source
# tree.
env:
CARGO_INCREMENTAL: "0"
RUSTFLAGS: -C debuginfo=line-tables-only -C target-feature=+crt-static
@@ -596,6 +631,10 @@ jobs:
shell: pwsh
run: ./.github/scripts/assert-win7-pe-compat.ps1 -BinaryPath target/x86_64-win7-windows-msvc/release/dbx.exe
- name: Audit Win7 WebView2 loader markers
shell: pwsh
run: ./.github/scripts/assert-webview2-win7-loader.ps1 -BinaryPath target/x86_64-win7-windows-msvc/release/dbx.exe
- name: Setup TrustAsia VSign
shell: pwsh
env:
@@ -603,7 +642,7 @@ jobs:
VSIGN_OPERATOR_PWD: ${{ secrets.VSIGN_OPERATOR_PWD }}
run: ./.github/scripts/setup-vsign.ps1
- name: Bundle and upload Windows 7 fixed-runtime installer
- name: Bundle and upload Windows 7 / Server 2012 R2 fixed-runtime installer
shell: pwsh
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -612,7 +651,8 @@ jobs:
run: |
$version = "${env:GITHUB_REF_NAME}".TrimStart("v")
$bundleDir = "target/x86_64-win7-windows-msvc/release/bundle/nsis"
$offlineName = "DBX_${version}_x64-win7-webview2-109-offline-setup.exe"
$offlineName = "DBX_${version}_x64-win7-server2012r2-offline-setup.exe"
$checksumName = "${offlineName}.sha256"
$cargoMetadata = cargo metadata --no-deps --format-version 1 | ConvertFrom-Json
$appVersion = ($cargoMetadata.packages | Where-Object { $_.name -eq "dbx" } | Select-Object -First 1).version
@@ -629,7 +669,7 @@ jobs:
Sort-Object LastWriteTimeUtc -Descending |
Select-Object -First 1
if (!$installer) {
Write-Error "Missing Windows 7 fixed-runtime installer in ${bundleDir}"
Write-Error "Missing Windows 7 / Server 2012 R2 fixed-runtime installer in ${bundleDir}"
exit 1
}
@@ -637,7 +677,10 @@ jobs:
./.github/scripts/assert-authenticode.ps1 -Path $installer.FullName
Copy-Item $installer.FullName $offlineName -Force
gh release upload "${env:GITHUB_REF_NAME}" $offlineName --repo "${env:GITHUB_REPOSITORY}" --clobber
$hash = (Get-FileHash -LiteralPath $offlineName -Algorithm SHA256).Hash.ToLowerInvariant()
"$hash $offlineName" | Set-Content -LiteralPath $checksumName -Encoding ascii
gh release upload "${env:GITHUB_REF_NAME}" $offlineName $checksumName `
--repo "${env:GITHUB_REPOSITORY}" --clobber
- name: Remove temporary VSign files
if: always()
@@ -672,6 +715,10 @@ jobs:
# page-size ARM64 kernels.
rustflags: "-C target-feature=+crt-static -C link-arg=-z -C link-arg=max-page-size=65536"
runs-on: ${{ matrix.platform }}
env:
CARGO_INCREMENTAL: "0"
RUSTC_WRAPPER: sccache
SCCACHE_GHA_ENABLED: ${{ secrets.SCCACHE_S3_BUCKET == '' && 'true' || 'false' }}
steps:
- uses: actions/checkout@v5
@@ -694,6 +741,41 @@ jobs:
- name: Install cargo-zigbuild
run: pip3 install cargo-zigbuild
- name: Setup sccache
uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10
with:
version: "v0.10.0"
# Plain CC/CXX only wrap HOST C compilations (build scripts): for the
# musl targets cargo-zigbuild sets CC_<target> to zig cc, which takes
# precedence and must stay unwrapped.
- name: Configure S3 sccache
if: env.SCCACHE_GHA_ENABLED != 'true'
shell: bash
env:
CACHE_BUCKET: ${{ secrets.SCCACHE_S3_BUCKET }}
CACHE_ENDPOINT: ${{ secrets.SCCACHE_S3_ENDPOINT }}
CACHE_REGION: ${{ secrets.SCCACHE_S3_REGION }}
CACHE_KEY_PREFIX: ${{ secrets.SCCACHE_S3_KEY_PREFIX }}
CACHE_ACCESS_KEY_ID: ${{ secrets.SCCACHE_S3_ACCESS_KEY_ID }}
CACHE_SECRET_ACCESS_KEY: ${{ secrets.SCCACHE_S3_SECRET_ACCESS_KEY }}
run: |
{
echo "SCCACHE_BUCKET=${CACHE_BUCKET}"
echo "SCCACHE_ENDPOINT=${CACHE_ENDPOINT}"
echo "SCCACHE_REGION=${CACHE_REGION}"
echo "SCCACHE_S3_KEY_PREFIX=${CACHE_KEY_PREFIX}"
echo "SCCACHE_S3_USE_SSL=true"
echo "AWS_ACCESS_KEY_ID=${CACHE_ACCESS_KEY_ID}"
echo "AWS_SECRET_ACCESS_KEY=${CACHE_SECRET_ACCESS_KEY}"
# Packaging and container verification run long after the last
# rustc call; the server's 10-minute default idle timeout would
# exit first and zero the counters shown by "Show sccache stats".
echo "SCCACHE_IDLE_TIMEOUT=0"
echo "CC=${SCCACHE_PATH} cc"
echo "CXX=${SCCACHE_PATH} c++"
} >> "$GITHUB_ENV"
- name: Rust cache
uses: swatinem/rust-cache@v2
with:
@@ -754,6 +836,11 @@ jobs:
gh release upload "${GITHUB_REF_NAME}" "$ASSET" "$CHECKSUM" \
--repo "${GITHUB_REPOSITORY}" --clobber
- name: Show sccache stats
if: always()
continue-on-error: true
run: ${SCCACHE_PATH} --show-stats
cleanup-release-signatures:
needs: [build, build-windows, build-windows-7-offline]
runs-on: ubuntu-latest
@@ -0,0 +1,73 @@
name: SQL Server 2000 Windows User Test Build
on:
workflow_dispatch:
permissions:
contents: read
jobs:
build-windows-x64:
runs-on: windows-2022
steps:
- uses: actions/checkout@v5
- name: Setup pnpm
uses: pnpm/action-setup@v6
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version: 22.13.0
cache: pnpm
- name: Setup Java
uses: actions/setup-java@v5
with:
distribution: temurin
java-version: "21"
- name: Setup Gradle
uses: gradle/actions/setup-gradle@v4
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Setup Rust
uses: dtolnay/rust-toolchain@stable
with:
targets: x86_64-pc-windows-msvc
- name: Rust cache
uses: swatinem/rust-cache@v2
with:
workspaces: "./ -> target"
shared-key: sqlserver2000-windows-x64
cache-on-failure: true
- name: Build SQL Server legacy agent
run: .\\gradlew.bat :sqlserver-legacy:shadowJar --no-daemon
working-directory: agents
- name: Stage SQL Server legacy agent
shell: pwsh
run: |
$agent = Get-ChildItem "agents/drivers/sqlserver-legacy/build/libs/dbx-agent-sqlserver-legacy.jar" | Select-Object -First 1
if (!$agent) {
Write-Error "Missing SQL Server legacy agent JAR"
exit 1
}
Copy-Item $agent.FullName "sqlserver-legacy-agent.jar" -Force
- name: Build Windows x64 installer
run: pnpm tauri build --bundles nsis --target x86_64-pc-windows-msvc --ci --no-sign
- name: Upload installer
uses: actions/upload-artifact@v7
with:
name: DBX-SQLServer2000-Windows-x64
path: |
target/x86_64-pc-windows-msvc/release/bundle/nsis/*.exe
sqlserver-legacy-agent.jar
if-no-files-found: error
retention-days: 7
+22 -13
View File
@@ -19,22 +19,31 @@ jobs:
name: Sync release to CNB
runs-on: ubuntu-latest
steps:
- name: Sync release assets to CNB
- name: Checkout repository
uses: actions/checkout@v5
- name: Download GitHub release assets
env:
TAG_NAME: ${{ inputs.tag }}
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_USERNAME: ${{ github.repository_owner }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
CNB_USERNAME: cnb
CNB_TOKEN: ${{ secrets.CNB_TOKEN }}
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set -euo pipefail
mkdir -p "$RUNNER_TEMP/github-release" "$RUNNER_TEMP/release-assets"
gh release view "$TAG_NAME" \
--repo "$GITHUB_REPOSITORY" \
--json tagName,name,body,targetCommitish,isPrerelease,isDraft,assets \
> "$RUNNER_TEMP/github-release/release.json"
gh release download "$TAG_NAME" \
--repo "$GITHUB_REPOSITORY" \
--dir "$RUNNER_TEMP/release-assets" \
--clobber
wget -q https://cnb.cool/znb/mpgrm/-/releases/download/v0.1.0/mpgrm_linux_amd64.tar.gz
tar -xf mpgrm_linux_amd64.tar.gz
chmod +x mpgrm
./mpgrm releases sync \
--repo "https://github.com/${GITHUB_REPOSITORY}.git" \
--target-repo "https://cnb.cool/${CNB_REPOSITORY}.git" \
--tags "${TAG_NAME}"
- name: Sync release assets to CNB
env:
CNB_TOKEN: ${{ secrets.CNB_TOKEN }}
CNB_UPLOAD_CONCURRENCY: "3"
run: |
node .github/scripts/sync-cnb-release.mjs \
--github-release "$RUNNER_TEMP/github-release/release.json" \
--assets-dir "$RUNNER_TEMP/release-assets"
+3
View File
@@ -8,6 +8,7 @@ node_modules
.trellis/
.worktrees/
openspec/
.zcode/
.mcp.json
AGENTS.md
CLAUDE.md
@@ -23,6 +24,8 @@ plugins/jdbc/target/
plugins/jdbc/dependency-reduced-pom.xml
plugins/jdbc/lib/*.jar
agents/drivers/duckdb/target/
agents/drivers/etcd-go/etcd-go
agents/drivers/etcd2-go/etcd2-go
plugins/jdbc/.gradle/
plugins/jdbc/build/
plugins/jdbc/dist/
Generated
+44 -4
View File
@@ -2002,7 +2002,7 @@ dependencies = [
[[package]]
name = "dbx"
version = "0.5.97"
version = "0.6.3"
dependencies = [
"anyhow",
"axum",
@@ -2011,6 +2011,7 @@ dependencies = [
"chrono",
"csv",
"dbx-core",
"dbx-mcp",
"deadpool-postgres",
"font-kit",
"futures",
@@ -2024,6 +2025,7 @@ dependencies = [
"percent-encoding",
"portpicker",
"redis",
"regex",
"reqwest 0.12.28",
"russh",
"rust_decimal",
@@ -2058,11 +2060,12 @@ dependencies = [
[[package]]
name = "dbx-cli"
version = "0.4.74"
version = "0.4.80"
dependencies = [
"async-trait",
"dbx-core",
"dbx-mcp",
"rustls 0.23.43",
"serde",
"serde_json",
"tempfile",
@@ -2086,6 +2089,7 @@ dependencies = [
"cbc 0.1.2",
"chrono",
"csv",
"dbx-sqlite-worker",
"deadpool-postgres",
"encoding_rs",
"flate2",
@@ -2152,11 +2156,13 @@ dependencies = [
[[package]]
name = "dbx-mcp"
version = "0.4.74"
version = "0.4.80"
dependencies = [
"async-trait",
"axum",
"dbx-core",
"dirs",
"log",
"reqwest 0.12.28",
"rmcp",
"rustls 0.23.43",
@@ -2167,13 +2173,25 @@ dependencies = [
"tempfile",
"tokio",
"tokio-rustls 0.26.4",
"tokio-util",
"tower-http",
"url",
"uuid",
]
[[package]]
name = "dbx-sqlite-worker"
version = "0.1.1"
dependencies = [
"rusqlite",
"serde",
"serde_json",
"sqlparser",
]
[[package]]
name = "dbx-web"
version = "0.5.97"
version = "0.6.3"
dependencies = [
"aes-gcm 0.10.3",
"argon2",
@@ -2182,6 +2200,7 @@ dependencies = [
"base64 0.22.1",
"chrono",
"dbx-core",
"dbx-mcp",
"futures",
"log",
"pbkdf2 0.12.2",
@@ -6986,19 +7005,27 @@ checksum = "14db48ee17a9ba61810ab1a9c1beb7d06d8136ae39ac25a1137f10d357af01af"
dependencies = [
"async-trait",
"base64 0.22.1",
"bytes",
"chrono",
"futures",
"http 1.5.0",
"http-body 1.1.0",
"http-body-util",
"pastey",
"pin-project-lite",
"rand 0.10.2",
"rmcp-macros",
"schemars 1.2.2",
"serde",
"serde_json",
"sse-stream",
"thiserror 2.0.19",
"tokio",
"tokio-stream",
"tokio-util",
"tower-service",
"tracing",
"uuid",
]
[[package]]
@@ -8193,6 +8220,19 @@ dependencies = [
"syn 2.0.119",
]
[[package]]
name = "sse-stream"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c123f296ade4ec4b8b0f6162116e6629f5146922ca5ab40ca9d3c2e73ab4761e"
dependencies = [
"bytes",
"futures-util",
"http-body 1.1.0",
"http-body-util",
"pin-project-lite",
]
[[package]]
name = "ssfmt"
version = "0.1.2"
+6 -2
View File
@@ -1,6 +1,6 @@
[workspace]
resolver = "2"
members = ["src-tauri", "crates/dbx-core", "crates/dbx-web", "crates/dbx-mcp", "crates/dbx-cli"]
members = ["src-tauri", "crates/dbx-core", "crates/dbx-web", "crates/dbx-mcp", "crates/dbx-cli", "crates/dbx-sqlite-worker"]
exclude = ["vendor/ctor", "vendor/dirs-sys", "vendor/pageant", "vendor/rumqttc", "vendor/tiberius", "vendor/wry"]
[patch.crates-io]
@@ -22,7 +22,11 @@ pageant = { path = "vendor/pageant" }
# process stays alive (black screen after an overnight idle session, see
# vendor/wry/src/webview2/mod.rs). Recovery is layered by process kind
# (browser exit is fatal, GPU exits are self-healed by WebView2) and bounded
# by a cooldown plus a maximum reload budget.
# by a cooldown plus a maximum reload budget. It also re-owns clipboard
# writes performed by the WebView2 browser process on the host window so
# copies show up in Windows Clipboard History / Win+V
# (MicrosoftEdge/WebView2Feedback#5650, see
# vendor/wry/src/webview2/clipboard_history.rs).
wry = { path = "vendor/wry" }
tokio-postgres = { git = "https://github.com/t8y2/tokio-postgres-gaussdb.git", rev = "ac8ad476de59093d33ab9f294531c956e0c6a724" }
postgres-types = { git = "https://github.com/t8y2/tokio-postgres-gaussdb.git", rev = "ac8ad476de59093d33ab9f294531c956e0c6a724" }
+1 -1
View File
@@ -113,7 +113,7 @@ cargo-check-fast:
cargo check --no-default-features --features sqlite-bundled
cargo-test-fast:
cargo test --no-default-features --features sqlite-bundled
RUST_MIN_STACK=8388608 cargo test --no-default-features --features sqlite-bundled
db-list:
@$(PNPM) db:env -- list
+63 -35
View File
@@ -12,7 +12,7 @@
<a href="https://discord.gg/W7NyVDRt6a"><img src="https://dcbadge.limes.pink/api/server/W7NyVDRt6a" alt="Join Discord" /></a>
</p>
<p>
<a href="https://trendshift.io/repositories/26775?utm_source=trendshift-badge&amp;utm_medium=badge&amp;utm_campaign=badge-trendshift-26775" target="_blank" rel="noopener noreferrer"><img src="https://trendshift.io/api/badge/trendshift/repositories/26775/daily?language=Rust" alt="t8y2%2Fdbx | Trendshift" width="250" height="55"/></a>
<a href="https://trendshift.io/repositories/26775?utm_source=repository-badge&amp;utm_medium=badge&amp;utm_campaign=badge-repository-26775" target="_blank" rel="noopener noreferrer"><img src="https://trendshift.io/api/badge/repositories/26775" alt="t8y2%2Fdbx | Trendshift" width="250" height="55"/></a>
<a href="https://hellogithub.com/repository/t8y2/dbx" target="_blank"><img src="https://api.hellogithub.com/v1/widgets/recommend.svg?rid=7f74ffda697241bf996e17e1b0900a21&claim_uid=p0UjnC1TLtyvWSx" alt="Featured|HelloGitHub" style="width: 250px; height: 54px;" width="250" height="54" /></a>
<a href="https://www.producthunt.com/products/dbx/launches/dbx?embed=true&amp;utm_source=badge-featured&amp;utm_medium=badge&amp;utm_campaign=badge-dbx" target="_blank" rel="noopener noreferrer"><img alt="DBX - Lightweight open-source database manager built with Rust | Product Hunt" width="250" height="54" src="https://api.producthunt.com/widgets/embed-image/v1/featured.svg?post_id=1136811&amp;theme=light&amp;t=1780979717555"></a>
</p>
@@ -46,6 +46,7 @@
<img src="https://img.shields.io/badge/InfluxDB-d30971?logo=influxdb&logoColor=white" />
<img src="https://img.shields.io/badge/JDBC-4B5563?logoColor=white" />
<img src="https://img.shields.io/badge/and%20more...-555555?logoColor=white" />
<a href="https://1panel.cn/" target="_blank" rel="noopener noreferrer"><img src="https://img.shields.io/badge/1Panel-Partner-005EEB?logo=1panel&amp;logoColor=white" alt="1Panel Partner" /></a>
<a href="https://cnb.cool/dbxio.com/dbx"><img src="https://img.shields.io/badge/CNB-dbx-F76945?logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IArs4c6QAAAppJREFUOE9tk01rE1EUhs+5H5lJ0iQT3FRX2Yi4avoLkv4CWxBcNt11oZiuFKGmXQhFN6kuRESSgitXKf6A5g+I6cqFLiKCiptOkraZ5H4cuZOkH9oLw+UO8z7nvGfei3DFohoEQLAKCGWyGJCSbQZsD3ei7r+f45WATV6xmjdISyAlYLaDliveux+ti5ozwLAaFPhIlk6G6f18sxuO7l9bBiWbVoncDGCMWMl++PI/gJ7IohkmD0zkBxT5oRn5LTpN7ULqBByEtFywWvRyHz8Fx3dvN0UwuMPyR0uJFycddH7tKPHZRn7BjnwwkQcUud0HO/K7oOENcTYPRCEAFERmsCpyfZDZfpdnBosOUKGx13BC60QTIdjpmSzrEWDR+UagjsgMciLXAwfhc4MNpKfQtOPE6mVh0gEOaeTXtWXXEdADpJAR/GaZ/rrM9Us82wORPd5zHbRp7JVcyxR5PRP5LTPy6uo0HXIlG9awMgLuEUABkBYI7VLyxs+yyPW3+Nyg4zqoG+UV7TDRlMBbWA/D43u3qlbLGmgRWM2BjFxjQjWQGQBuu4bYYvrm14JMnQSXcjBcny9A5DXIiDJp9/8FWCXAAOaFUEfILTgIcttOvf+2NJnLdI0fBFUyskZKBGTEJEAmBuzr9LCSUDwGQAwwgMzu+m9/VWOAeZSsk+YP48C4li/sRok1nlBd5PogBjgxNw70PfHyqBADyEVXicasZXCVpxYiLfO+HxVZDHCVYwvu2ebPT7fOLNAm65AWLnEAzvsEcOi9/lPU1aACXDfOLdhD9kxNszGdweQGYstqXgItJwDFNxKvwrp57G8h0zVwldHuA0IFt8El83yIs2FSDcpgxDJpUUTwK7gTduN3AK5iG7ehc/E2/gUPD3q3eY4awwAAAABJRU5ErkJggg==" alt="CNB" /></a>
<a href="https://mcptoplist.com/server/io.github.t8y2%2Fdbx"><img src="https://mcptoplist.com/badge/io.github.t8y2%2Fdbx.svg" alt="MCP Toplist" /></a>
</p>
@@ -63,16 +64,13 @@
</p>
</div>
## Sponsors & Partners
## ❤️ Sponsors
<table>
<tr>
<td align="center" valign="middle" width="220">
<td align="center" valign="middle" width="200">
<a href="https://www.rainyun.com/MTE5Mjc4Ng==_" target="_blank">
<picture>
<source media="(prefers-color-scheme: dark)" srcset="docs/public/sponsors/rainyun-dark.png" />
<img src="https://www.rainyun.com/img/logo.d193755d.png" alt="RainYun" height="48" />
</picture>
<img src="docs/public/sponsors/rainyun-card.png" alt="RainYun" width="175" />
</a>
</td>
<td>
@@ -81,37 +79,42 @@
</td>
</tr>
<tr>
<td align="center" valign="middle" width="220">
<a href="https://www.qiniu.com/" target="_blank">
<img src="https://www-static.qbox.me/_next/static/media/logo.0fc18feaa621d2068a7180631f742256.jpg" alt="Qiniu Cloud" height="64" />
<td align="center" valign="middle" width="200">
<a href="https://www.trustasia.com/ssl/trustasia/code-signing" target="_blank">
<img src="docs/public/sponsors/trustasia-card.png" alt="TrustAsia" width="175" />
</a>
</td>
<td>
Qiniu Cloud provides DBX with object storage, CDN, and other cloud infrastructure resources.
<a href="https://www.qiniu.com/" target="_blank">Visit Qiniu Cloud</a>
TrustAsia provides cloud-based code signing service for DBX, enabling trusted software through automated CI/CD builds.
<a href="https://www.trustasia.com/ssl/trustasia/code-signing" target="_blank">Visit TrustAsia</a>
</td>
</tr>
<tr>
<td align="center" valign="middle" width="220">
<a href="https://easysearch.cn" target="_blank">
<picture>
<source media="(prefers-color-scheme: dark)" srcset="docs/public/sponsors/easysearch-dark.png" />
<img src="docs/public/sponsors/easysearch.png" alt="Easysearch" width="150" />
</picture>
<td align="center" valign="middle" width="200">
<a href="https://www.jalapeno-cloud.ai/DBX" target="_blank">
<img src="docs/public/sponsors/jalapeno-card.png" alt="Jalapeño Cloud" width="175" />
</a>
</td>
<td>
Easysearch is an enterprise-grade distributed search engine compatible with Elasticsearch APIs, combining full-text, vector, geospatial search, real-time analytics, and AI capabilities in one platform.
<a href="https://easysearch.cn" target="_blank">Visit Easysearch</a>
Jalapeño Cloud is an AI infrastructure and token compute platform, with an exclusive DBX entry offering free credits and top-up bonuses.
<a href="https://www.jalapeno-cloud.ai/DBX" target="_blank">Visit Jalapeño Cloud</a>
</td>
</tr>
<tr>
<td align="center" valign="middle" width="220">
<td align="center" valign="middle" width="200">
<a href="https://www.ucloud.cn/site/active/kuaijiesale.html?ytag=geo_waituo_github_dbx" target="_blank">
<img src="docs/public/sponsors/astraflow-card.png" alt="AstraFlow" width="175" />
</a>
</td>
<td>
UCloud is the first public cloud provider listed on China's STAR Market, with 28 global regions for cloud hosting, databases, and CDN; its AstraFlow platform offers one-click access to 200+ mainstream LLMs.
<a href="https://www.ucloud.cn/site/active/kuaijiesale.html?ytag=geo_waituo_github_dbx" target="_blank">Visit UCloud</a>
</td>
</tr>
<tr>
<td align="center" valign="middle" width="200">
<a href="https://www.atlascloud.ai/?ref=6YYXWA" target="_blank">
<picture>
<source media="(prefers-color-scheme: dark)" srcset="docs/public/sponsors/atlas-dark.svg" />
<img src="https://www.atlascloud.ai/logo.svg" alt="Atlas Cloud" width="150" />
</picture>
<img src="docs/public/sponsors/atlas-card.png" alt="Atlas Cloud" width="175" />
</a>
</td>
<td>
@@ -120,17 +123,41 @@
</td>
</tr>
<tr>
<td align="center" valign="middle" width="220">
<a href="https://www.trustasia.com/ssl/trustasia/code-signing" target="_blank">
<picture>
<source media="(prefers-color-scheme: dark)" srcset="docs/public/sponsors/trustasia-dark.svg" />
<img src="docs/public/sponsors/trustasia.png" alt="TrustAsia" height="44" />
</picture>
<td align="center" valign="middle" width="200">
<a href="https://www.qiniu.com/" target="_blank">
<img src="docs/public/sponsors/qiniu-card.png" alt="Qiniu Cloud" width="175" />
</a>
</td>
<td>
TrustAsia provides cloud-based code signing service for DBX, enabling trusted software through automated CI/CD builds.
<a href="https://www.trustasia.com/ssl/trustasia/code-signing" target="_blank">Visit TrustAsia</a>
Qiniu Cloud provides DBX with object storage, CDN, and other cloud infrastructure resources.
<a href="https://www.qiniu.com/" target="_blank">Visit Qiniu Cloud</a>
</td>
</tr>
</table>
## 🤝 Partners
<table>
<tr>
<td align="center" valign="middle" width="200">
<a href="https://1panel.cn/" target="_blank">
<img src="docs/public/sponsors/1panel-card.png" alt="1Panel" width="175" />
</a>
</td>
<td>
1Panel is a modern open-source Linux server management panel and lightweight AI management platform, offering an intuitive web interface for one-stop management of AI agents, local LLMs, websites, databases, containers, files, and more.
<a href="https://1panel.cn/" target="_blank">Visit 1Panel</a>
</td>
</tr>
<tr>
<td align="center" valign="middle" width="200">
<a href="https://easysearch.cn" target="_blank">
<img src="docs/public/sponsors/easysearch-card.png" alt="Easysearch" width="175" />
</a>
</td>
<td>
Easysearch is an enterprise-grade distributed search engine compatible with Elasticsearch APIs, combining full-text, vector, geospatial search, real-time analytics, and AI capabilities in one platform.
<a href="https://easysearch.cn" target="_blank">Visit Easysearch</a>
</td>
</tr>
</table>
@@ -141,7 +168,7 @@
<tr>
<td width="50%">
<h3>🪶 20 MB, zero runtime bloat</h3>
<p>No Java JRE. No Python venv. No bundled Chromium. DBX ships as a single small binary — download, install, connect. DBeaver needs Java; TablePlus is macOS-only. DBX runs everywhere with nothing extra.</p>
<p>No Java JRE. No Python venv. No bundled Chromium. DBX ships as a single small binary — download, install, connect. DBeaver needs Java; TablePlus is Freemium. DBX runs everywhere with nothing extra.</p>
</td>
<td width="50%">
<h3>🤖 AI that lives in your editor</h3>
@@ -472,7 +499,7 @@ DBX is free and open source, but ongoing maintenance, database compatibility tes
## Contributors
<a href="https://github.com/t8y2/dbx/graphs/contributors">
<img src="https://contrib.rocks/image?repo=t8y2/dbx" />
<img src="https://contrib.rocks/image?repo=t8y2/dbx&max=300&columns=15" />
</a>
## Community
@@ -481,6 +508,7 @@ DBX is free and open source, but ongoing maintenance, database compatibility tes
<a href="https://qm.qq.com/q/1087880322" target="_blank"><img src="https://img.shields.io/badge/QQ%20群-1087880322-EB1923?logo=tencentqq&logoColor=white" alt="QQ Group" /></a>
<a href="https://docs.qq.com/doc/DVVhMY0h1ekJqc0tz" target="_blank"><img src="https://img.shields.io/badge/微信群-Join-07C160?logo=wechat&logoColor=white" alt="WeChat Group" /></a>
[![LINUX DO](https://img.shields.io/badge/LINUX%20DO-Community-blue)](https://linux.do)
[![1Panel](https://img.shields.io/badge/1Panel-Partner-005EEB?logo=1panel&logoColor=white)](https://1panel.cn)
## Star History
+63 -35
View File
@@ -12,7 +12,7 @@
<a href="https://discord.gg/W7NyVDRt6a"><img src="https://dcbadge.limes.pink/api/server/W7NyVDRt6a" alt="加入 Discord" /></a>
</p>
<p>
<a href="https://trendshift.io/repositories/26775?utm_source=trendshift-badge&amp;utm_medium=badge&amp;utm_campaign=badge-trendshift-26775" target="_blank" rel="noopener noreferrer"><img src="https://trendshift.io/api/badge/trendshift/repositories/26775/daily?language=Rust" alt="t8y2%2Fdbx | Trendshift" width="250" height="55"/></a>
<a href="https://trendshift.io/repositories/26775?utm_source=repository-badge&amp;utm_medium=badge&amp;utm_campaign=badge-repository-26775" target="_blank" rel="noopener noreferrer"><img src="https://trendshift.io/api/badge/repositories/26775" alt="t8y2%2Fdbx | Trendshift" width="250" height="55"/></a>
<a href="https://hellogithub.com/repository/t8y2/dbx" target="_blank"><img src="https://api.hellogithub.com/v1/widgets/recommend.svg?rid=7f74ffda697241bf996e17e1b0900a21&claim_uid=p0UjnC1TLtyvWSx" alt="Featured|HelloGitHub" style="width: 250px; height: 54px;" width="250" height="54" /></a>
<a href="https://www.producthunt.com/products/dbx/launches/dbx?embed=true&amp;utm_source=badge-featured&amp;utm_medium=badge&amp;utm_campaign=badge-dbx" target="_blank" rel="noopener noreferrer"><img alt="DBX - Lightweight open-source database manager built with Rust | Product Hunt" width="250" height="54" src="https://api.producthunt.com/widgets/embed-image/v1/featured.svg?post_id=1136811&amp;theme=light&amp;t=1780979717555"></a>
</p>
@@ -46,6 +46,7 @@
<img src="https://img.shields.io/badge/InfluxDB-d30971?logo=influxdb&logoColor=white" />
<img src="https://img.shields.io/badge/JDBC-4B5563?logoColor=white" />
<img src="https://img.shields.io/badge/and%20more...-555555?logoColor=white" />
<a href="https://1panel.cn/" target="_blank" rel="noopener noreferrer"><img src="https://img.shields.io/badge/1Panel-Partner-005EEB?logo=1panel&amp;logoColor=white" alt="1Panel Partner" /></a>
<a href="https://cnb.cool/dbxio.com/dbx"><img src="https://img.shields.io/badge/CNB-dbx-F76945?logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IArs4c6QAAAppJREFUOE9tk01rE1EUhs+5H5lJ0iQT3FRX2Yi4avoLkv4CWxBcNt11oZiuFKGmXQhFN6kuRESSgitXKf6A5g+I6cqFLiKCiptOkraZ5H4cuZOkH9oLw+UO8z7nvGfei3DFohoEQLAKCGWyGJCSbQZsD3ei7r+f45WATV6xmjdISyAlYLaDliveux+ti5ozwLAaFPhIlk6G6f18sxuO7l9bBiWbVoncDGCMWMl++PI/gJ7IohkmD0zkBxT5oRn5LTpN7ULqBByEtFywWvRyHz8Fx3dvN0UwuMPyR0uJFycddH7tKPHZRn7BjnwwkQcUud0HO/K7oOENcTYPRCEAFERmsCpyfZDZfpdnBosOUKGx13BC60QTIdjpmSzrEWDR+UagjsgMciLXAwfhc4MNpKfQtOPE6mVh0gEOaeTXtWXXEdADpJAR/GaZ/rrM9Us82wORPd5zHbRp7JVcyxR5PRP5LTPy6uo0HXIlG9awMgLuEUABkBYI7VLyxs+yyPW3+Nyg4zqoG+UV7TDRlMBbWA/D43u3qlbLGmgRWM2BjFxjQjWQGQBuu4bYYvrm14JMnQSXcjBcny9A5DXIiDJp9/8FWCXAAOaFUEfILTgIcttOvf+2NJnLdI0fBFUyskZKBGTEJEAmBuzr9LCSUDwGQAwwgMzu+m9/VWOAeZSsk+YP48C4li/sRok1nlBd5PogBjgxNw70PfHyqBADyEVXicasZXCVpxYiLfO+HxVZDHCVYwvu2ebPT7fOLNAm65AWLnEAzvsEcOi9/lPU1aACXDfOLdhD9kxNszGdweQGYstqXgItJwDFNxKvwrp57G8h0zVwldHuA0IFt8El83yIs2FSDcpgxDJpUUTwK7gTduN3AK5iG7ehc/E2/gUPD3q3eY4awwAAAABJRU5ErkJggg==" alt="CNB" /></a>
<a href="https://mcptoplist.com/server/io.github.t8y2%2Fdbx"><img src="https://mcptoplist.com/badge/io.github.t8y2%2Fdbx.svg" alt="MCP Toplist" /></a>
</p>
@@ -63,16 +64,13 @@
</p>
</div>
## 赞助商与合作伙伴
## ❤️ 赞助商
<table>
<tr>
<td align="center" valign="middle" width="220">
<td align="center" valign="middle" width="200">
<a href="https://www.rainyun.com/MTE5Mjc4Ng==_" target="_blank">
<picture>
<source media="(prefers-color-scheme: dark)" srcset="docs/public/sponsors/rainyun-dark.png" />
<img src="https://www.rainyun.com/img/logo.d193755d.png" alt="雨云" height="48" />
</picture>
<img src="docs/public/sponsors/rainyun-card.png" alt="雨云" width="175" />
</a>
</td>
<td>
@@ -81,37 +79,42 @@
</td>
</tr>
<tr>
<td align="center" valign="middle" width="220">
<a href="https://www.qiniu.com/" target="_blank">
<img src="https://www-static.qbox.me/_next/static/media/logo.0fc18feaa621d2068a7180631f742256.jpg" alt="七牛云" height="64" />
<td align="center" valign="middle" width="200">
<a href="https://www.trustasia.com/ssl/trustasia/code-signing" target="_blank">
<img src="docs/public/sponsors/trustasia-card.png" alt="TrustAsia" width="175" />
</a>
</td>
<td>
七牛云为 DBX 提供对象存储、CDN 等云基础设施资源支持。
<a href="https://www.qiniu.com/" target="_blank">访问七牛云</a>
由 TrustAsia 提供代码签名云签服务,实现 CICD 自动化构建可信软件。
<a href="https://www.trustasia.com/ssl/trustasia/code-signing" target="_blank">访问 TrustAsia</a>
</td>
</tr>
<tr>
<td align="center" valign="middle" width="220">
<a href="https://easysearch.cn" target="_blank">
<picture>
<source media="(prefers-color-scheme: dark)" srcset="docs/public/sponsors/easysearch-dark.png" />
<img src="docs/public/sponsors/easysearch.png" alt="Easysearch" width="150" />
</picture>
<td align="center" valign="middle" width="200">
<a href="https://www.jalapeno-cloud.ai/DBX" target="_blank">
<img src="docs/public/sponsors/jalapeno-card.png" alt="Jalapeño Cloud" width="175" />
</a>
</td>
<td>
Easysearch 是一款企业级分布式搜索引擎,兼容 ES API、融合全文检索、向量检索、地理空间位置检索、实时分析与 AI 能力,为企业提供统一的数据检索与智能分析基础设施。
<a href="https://easysearch.cn" target="_blank">访问 Easysearch</a>
Jalapeño Cloud 是 AI 基础设施与 Token 算力平台,通过 DBX 专属入口可享新用户免费额度与充值加赠。
<a href="https://www.jalapeno-cloud.ai/DBX" target="_blank">访问 Jalapeño Cloud</a>
</td>
</tr>
<tr>
<td align="center" valign="middle" width="220">
<td align="center" valign="middle" width="200">
<a href="https://www.ucloud.cn/site/active/kuaijiesale.html?ytag=geo_waituo_github_dbx" target="_blank">
<img src="docs/public/sponsors/astraflow-card.png" alt="AstraFlow" width="175" />
</a>
</td>
<td>
UCloud 优刻得是国内首家公有云科创板上市公司,覆盖国内、亚洲、欧洲、北美等 28 个地域的云主机、数据库、CDN 等服务,注册享新客优惠 0.9 折起;星图 AstraFlow 大模型平台支持主流 200+ 大模型一键调用。
<a href="https://www.ucloud.cn/site/active/kuaijiesale.html?ytag=geo_waituo_github_dbx" target="_blank">访问 UCloud 优刻得</a>
</td>
</tr>
<tr>
<td align="center" valign="middle" width="200">
<a href="https://www.atlascloud.ai/?ref=6YYXWA" target="_blank">
<picture>
<source media="(prefers-color-scheme: dark)" srcset="docs/public/sponsors/atlas-dark.svg" />
<img src="https://www.atlascloud.ai/logo.svg" alt="Atlas Cloud" width="150" />
</picture>
<img src="docs/public/sponsors/atlas-card.png" alt="Atlas Cloud" width="175" />
</a>
</td>
<td>
@@ -120,17 +123,41 @@
</td>
</tr>
<tr>
<td align="center" valign="middle" width="220">
<a href="https://www.trustasia.com/ssl/trustasia/code-signing" target="_blank">
<picture>
<source media="(prefers-color-scheme: dark)" srcset="docs/public/sponsors/trustasia-dark.svg" />
<img src="docs/public/sponsors/trustasia.png" alt="TrustAsia" height="44" />
</picture>
<td align="center" valign="middle" width="200">
<a href="https://www.qiniu.com/" target="_blank">
<img src="docs/public/sponsors/qiniu-card.png" alt="七牛云" width="175" />
</a>
</td>
<td>
由 TrustAsia 提供代码签名云签服务,实现 CICD 自动化构建可信软件。
<a href="https://www.trustasia.com/ssl/trustasia/code-signing" target="_blank">访问 TrustAsia</a>
七牛云为 DBX 提供对象存储、CDN 等云基础设施资源支持。
<a href="https://www.qiniu.com/" target="_blank">访问七牛云</a>
</td>
</tr>
</table>
## 🤝 合作伙伴
<table>
<tr>
<td align="center" valign="middle" width="200">
<a href="https://1panel.cn/" target="_blank">
<img src="docs/public/sponsors/1panel-card.png" alt="1Panel" width="175" />
</a>
</td>
<td>
1Panel 是现代化的开源 Linux 服务器运维管理面板与轻量级 AI 管理平台,提供直观易用的 Web 界面,支持 AI 智能体、本地大模型、网站、数据库、容器、文件等核心场景的一站式管理。
<a href="https://1panel.cn/" target="_blank">访问 1Panel</a>
</td>
</tr>
<tr>
<td align="center" valign="middle" width="200">
<a href="https://easysearch.cn" target="_blank">
<img src="docs/public/sponsors/easysearch-card.png" alt="Easysearch" width="175" />
</a>
</td>
<td>
Easysearch 是一款企业级分布式搜索引擎,兼容 ES API、融合全文检索、向量检索、地理空间位置检索、实时分析与 AI 能力,为企业提供统一的数据检索与智能分析基础设施。
<a href="https://easysearch.cn" target="_blank">访问 Easysearch</a>
</td>
</tr>
</table>
@@ -141,7 +168,7 @@
<tr>
<td width="50%">
<h3>🪶 20 MB,极致轻量</h3>
<p>无需 Java 运行环境,无需 Python 虚拟环境,不内嵌 Chromium。DBX 是单个小巧的二进制文件——下载、安装、连接。DBeaver 依赖 Java;TablePlus 仅限 macOS。DBX 全平台可用,无需额外运行时。</p>
<p>无需 Java 运行环境,无需 Python 虚拟环境,不内嵌 Chromium。DBX 是单个小巧的二进制文件——下载、安装、连接。DBeaver 依赖 Java;TablePlus 是 Freemium。DBX 全平台可用,无需额外运行时。</p>
</td>
<td width="50%">
<h3>🤖 AI 原生集成在编辑器里</h3>
@@ -470,6 +497,7 @@ make package
<a href="https://qm.qq.com/q/1087880322" target="_blank"><img src="https://img.shields.io/badge/QQ%20群-1087880322-EB1923?logo=tencentqq&logoColor=white" alt="QQ 群" /></a>
<a href="https://docs.qq.com/doc/DVVhMY0h1ekJqc0tz" target="_blank"><img src="https://img.shields.io/badge/微信交流群-点击加入-07C160?logo=wechat&logoColor=white" alt="微信交流群" /></a>
[![LINUX DO](https://img.shields.io/badge/LINUX%20DO-社区-blue)](https://linux.do)
[![1Panel](https://img.shields.io/badge/1Panel-合作伙伴-005EEB?logo=1panel&logoColor=white)](https://1panel.cn)
## 赞助与捐助
@@ -514,7 +542,7 @@ MySQL、PostgreSQL、SQLite、Cloudflare D1、Redis、MongoDB、DuckDB、ClickHo
## 贡献者
<a href="https://github.com/t8y2/dbx/graphs/contributors">
<img src="https://contrib.rocks/image?repo=t8y2/dbx" />
<img src="https://contrib.rocks/image?repo=t8y2/dbx&max=300&columns=15" />
</a>
## Star History
+6 -5
View File
@@ -45,7 +45,8 @@ Each agent runs as a standalone process and communicates with DBX via stdin/stdo
| yashandb | 崖山 YashanDB | YashanDB JDBC |
| xugu | 虚谷 XuguDB | XuguDB Go native agent |
| iotdb | Apache IoTDB | Apache IoTDB Go Client native agent |
| etcd | etcd | jetcd |
| etcd | etcd 3.3 degraded, 3.4+ full (v3 API) | etcd client/v3 native Go agent |
| etcd2 | etcd 2.0–2.3, 3.0–3.5 with --enable-v2 (v2 API) | etcd v2 HTTP/JSON native Go agent |
| zookeeper | Apache ZooKeeper | go-zookeeper native agent |
| rabbitmq | RabbitMQ | amqp091-go native agent |
| rocketmq | Apache RocketMQ | rocketmq-admin-go native agent |
@@ -53,7 +54,7 @@ Each agent runs as a standalone process and communicates with DBX via stdin/stdo
## Multi-JRE Support
Most Java agents target JRE 21. Native agents, such as `cassandra`, `duckdb`, `hive`, `iotdb`, `oracle`, `kingbase`, `tdengine`, `xugu`, `rabbitmq`, `rocketmq`, and `zookeeper`, do not require a JRE. DBX downloads and manages the JRE 21 installation automatically for Java agents.
Most Java agents target JRE 21. Native agents, such as `cassandra`, `duckdb`, `hive`, `iotdb`, `oracle`, `kingbase`, `tdengine`, `xugu`, `rabbitmq`, `rocketmq`, `zookeeper`, `etcd`, and `etcd2`, do not require a JRE. DBX downloads and manages the JRE 21 installation automatically for Java agents.
## JDBC Connection Pooling
@@ -79,7 +80,7 @@ Set `DBX_AGENT_JDBC_POOL_ENABLED=false` for a runtime-level compatibility fallba
For new agents, prefer a **native (Go or Rust) driver** over a Java/JDBC agent whenever a mature, license-compatible native driver is available. Native agents ship as a single self-contained executable with no JRE, which significantly reduces memory footprint and startup time — the JVM baseline that every Java agent pays even when idle is avoided entirely.
- **Native (C++/Go/Rust)** — preferred when a usable native driver exists. See `drivers/cassandra-go` (Apache cassandra-gocql-driver), `drivers/duckdb`, `drivers/hive-go` (native HS2), `drivers/iotdb` (Apache IoTDB Go Client), `drivers/oracle-go` (go-ora), `drivers/kingbase-go` (gokb), `drivers/vastbase-go` (openGauss connector), `drivers/tdengine` (taos-connector-rust), `drivers/xugu`, `drivers/rabbitmq` (amqp091-go), `drivers/rocketmq` (rocketmq-admin-go), and `drivers/zookeeper` (go-zookeeper) as reference implementations. No JRE download or management is needed.
- **Native (C++/Go/Rust)** — preferred when a usable native driver exists. See `drivers/cassandra-go` (Apache cassandra-gocql-driver), `drivers/duckdb`, `drivers/hive-go` (native HS2), `drivers/iotdb` (Apache IoTDB Go Client), `drivers/oracle-go` (go-ora), `drivers/kingbase-go` (gokb), `drivers/vastbase-go` (openGauss connector), `drivers/tdengine` (taos-connector-rust), `drivers/xugu`, `drivers/rabbitmq` (amqp091-go), `drivers/rocketmq` (rocketmq-admin-go), and `drivers/zookeeper` (go-zookeeper), `drivers/etcd-go` (etcd client/v3), and `drivers/etcd2-go` (etcd v2 HTTP API) as reference implementations. No JRE download or management is needed.
- **Java/JDBC** — the default fallback when only a JDBC driver exists for the database, or when the native driver is immature or unmaintained. Most agents still fall in this category.
Native agents implement the same JSON-RPC contract and `versions.json` registration as Java agents; they ship an `agent` executable instead of `agent.jar`. If both native and Java source implementations exist for the same database, publish only the native artifact unless the Java variant has a separately registered compatibility profile, such as `oracle-legacy` / `oracle-10g`.
@@ -103,7 +104,7 @@ Requires JDK 21 (Gradle toolchain auto-downloads if needed).
(cd drivers/zookeeper && go build -o agent .)
```
Output JARs are in `drivers/{module}/build/libs/`. Native agents build from `drivers/cassandra-go`, `drivers/duckdb`, `drivers/hive-go`, `drivers/iotdb`, `drivers/oracle-go`, `drivers/kingbase-go`, `drivers/vastbase-go`, `drivers/tdengine`, `drivers/xugu`, `drivers/rabbitmq`, `drivers/rocketmq`, and `drivers/zookeeper`.
Output JARs are in `drivers/{module}/build/libs/`. Native agents build from `drivers/cassandra-go`, `drivers/duckdb`, `drivers/hive-go`, `drivers/iotdb`, `drivers/oracle-go`, `drivers/kingbase-go`, `drivers/vastbase-go`, `drivers/tdengine`, `drivers/xugu`, `drivers/rabbitmq`, `drivers/rocketmq`, `drivers/zookeeper`, `drivers/etcd-go`, and `drivers/etcd2-go`.
### Local DBX Runtime Test
@@ -117,7 +118,7 @@ cp agents/drivers/<db_type>/build/libs/*-all.jar ~/.dbx/agents/drivers/<db_type>
Restart DBX or disconnect and reconnect the database so the new agent process loads the replacement JAR.
Native agents such as `cassandra`, `hive`, `iotdb`, `oracle`, `kingbase`, `tdengine`, `xugu`, `rabbitmq`, `rocketmq`, and `zookeeper` use an `agent` executable instead of `agent.jar`. TDengine builds `target/release/dbx-tdengine-driver` from `drivers/tdengine/Cargo.toml`.
Native agents such as `cassandra`, `hive`, `iotdb`, `oracle`, `kingbase`, `tdengine`, `xugu`, `rabbitmq`, `rocketmq`, `zookeeper`, `etcd`, and `etcd2` use an `agent` executable instead of `agent.jar`. TDengine builds `target/release/dbx-tdengine-driver` from `drivers/tdengine/Cargo.toml`.
## Versioning
@@ -90,14 +90,26 @@ public abstract class AbstractJdbcAgent extends BaseDatabaseAgent {
return unchecked(() -> {
loadDriver(params);
try (Connection conn = openTestConnection(params)) {
String validationQuery = connectionValidationQuery();
boolean valid = isConnectionValid(conn, 5);
Map<String, Object> result = new LinkedHashMap<>();
result.put("ok", valid);
result.put(
"validation",
validationQuery == null || validationQuery.isBlank() ? "jdbc_connection_isValid" : validationQuery
);
if (valid) {
Map<String, String> databaseInfo = JdbcDatabaseInfo.from(conn);
if (!databaseInfo.isEmpty()) {
result.put("databaseInfo", databaseInfo);
}
} else {
result.put(
"error",
"Connection opened, but validation failed (method: "
+ (validationQuery == null || validationQuery.isBlank() ? "Connection.isValid" : validationQuery)
+ ")"
);
}
return result;
}
@@ -184,7 +196,9 @@ public abstract class AbstractJdbcAgent extends BaseDatabaseAgent {
options.getMaxRows(),
options.getFetchSize(),
options.getTimeoutSecs(),
resultValueReader()
resultValueReader(),
JdbcExecutor.StatementMessageReader.NONE,
advancePastUpdateCounts()
);
}
@@ -199,7 +213,8 @@ public abstract class AbstractJdbcAgent extends BaseDatabaseAgent {
this::setSchemaSQL,
this::resetSchemaSQL,
options,
resultValueReader()
resultValueReader(),
advancePastUpdateCounts()
);
}
@@ -517,6 +532,10 @@ public abstract class AbstractJdbcAgent extends BaseDatabaseAgent {
protected void beforeQueryExecution(Connection connection, int timeoutSecs) throws Exception {
}
protected boolean advancePastUpdateCounts() {
return false;
}
protected void beforePooledConnectionReturn(Connection connection) throws Exception {
}
@@ -295,7 +295,9 @@ public final class DdlBuilder {
private static boolean isCharacterType(String normalized) {
return "character varying".equals(normalized)
|| "varchar".equals(normalized)
|| "nvarchar".equals(normalized)
|| "char".equals(normalized)
|| "nchar".equals(normalized)
|| "character".equals(normalized);
}
@@ -18,6 +18,7 @@ public class JdbcAgentProfile {
private final boolean nativeTableDdlSupported;
private final boolean objectSourceSupported;
private final boolean triggersSupported;
private final boolean escapeSchemaWildcards;
public JdbcAgentProfile(String driverClass, String urlTemplate) {
this(driverClass, urlTemplate, 0);
@@ -75,6 +76,40 @@ public class JdbcAgentProfile {
boolean nativeTableDdlSupported,
boolean objectSourceSupported,
boolean triggersSupported
) {
this(
driverClass,
urlTemplate,
defaultPort,
skipExecutionContext,
excludedSchemas,
tableTypes,
identifierQuote,
schemaSwitchPrefix,
catalogFallbackEnabled,
nativeTableDdlSupported,
objectSourceSupported,
triggersSupported,
// 默认对 schema pattern 里的 _ / % 通配符转义(getSearchStringEscape()),
// HANA 的 _SYS_RT 等含下划线 schema 依赖它避免误匹配。
true
);
}
public JdbcAgentProfile(
String driverClass,
String urlTemplate,
int defaultPort,
boolean skipExecutionContext,
Set<String> excludedSchemas,
List<String> tableTypes,
String identifierQuote,
String schemaSwitchPrefix,
boolean catalogFallbackEnabled,
boolean nativeTableDdlSupported,
boolean objectSourceSupported,
boolean triggersSupported,
boolean escapeSchemaWildcards
) {
this.driverClass = driverClass;
this.urlTemplate = urlTemplate;
@@ -88,6 +123,7 @@ public class JdbcAgentProfile {
this.nativeTableDdlSupported = nativeTableDdlSupported;
this.objectSourceSupported = objectSourceSupported;
this.triggersSupported = triggersSupported;
this.escapeSchemaWildcards = escapeSchemaWildcards;
}
public String getDriverClass() {
@@ -138,6 +174,10 @@ public class JdbcAgentProfile {
return triggersSupported;
}
public boolean getEscapeSchemaWildcards() {
return escapeSchemaWildcards;
}
public String buildUrl(ConnectParams params) {
if (!params.getConnection_string().trim().isEmpty()) {
return params.getConnection_string();
@@ -102,6 +102,34 @@ public final class JdbcExecutor {
int timeoutSecs,
ResultValueReader valueReader,
StatementMessageReader statementMessageReader
) {
return execute(
conn,
sql,
schema,
setSchemaSql,
resetSchemaSql,
maxRows,
fetchSize,
timeoutSecs,
valueReader,
statementMessageReader,
false
);
}
public QueryResult execute(
Connection conn,
String sql,
String schema,
Function<String, String> setSchemaSql,
Supplier<String> resetSchemaSql,
int maxRows,
Integer fetchSize,
int timeoutSecs,
ResultValueReader valueReader,
StatementMessageReader statementMessageReader,
boolean advancePastUpdateCounts
) {
return unchecked(() -> {
String trimmedSql = trimSql(sql);
@@ -122,17 +150,31 @@ public final class JdbcExecutor {
// Do not translate them to Connection.commit(), which requires autoCommit=false.
boolean hasResultSet = stmt.execute(trimmedSql);
long elapsed = System.currentTimeMillis() - start;
long affectedRows;
if (advancePastUpdateCounts) {
affectedRows = 0L;
while (!hasResultSet) {
int updateCount = stmt.getUpdateCount();
if (updateCount < 0) {
break;
}
affectedRows += updateCount;
hasResultSet = stmt.getMoreResults();
}
} else {
int updateCount = hasResultSet ? -1 : stmt.getUpdateCount();
affectedRows = updateCount >= 0 ? updateCount : 0L;
}
QueryResult result;
if (hasResultSet) {
try (ResultSet rs = stmt.getResultSet()) {
result = readResultSet(rs, elapsed, effectiveMaxRows, valueReader);
}
} else {
int updateCount = stmt.getUpdateCount();
result = new QueryResult(
Collections.emptyList(),
Collections.emptyList(),
updateCount >= 0 ? updateCount : 0,
affectedRows,
elapsed,
false
);
@@ -240,6 +282,30 @@ public final class JdbcExecutor {
return executePage(conn, sql, schema, setSchemaSql, resetSchemaSql, options, valueReader, StatementMessageReader.NONE, sessions);
}
public QueryPageResult executePage(
Connection conn,
String sql,
String schema,
Function<String, String> setSchemaSql,
Supplier<String> resetSchemaSql,
QueryPageOptions options,
ResultValueReader valueReader,
boolean advancePastUpdateCounts
) {
return executePage(
conn,
sql,
schema,
setSchemaSql,
resetSchemaSql,
options,
valueReader,
StatementMessageReader.NONE,
sessions,
advancePastUpdateCounts
);
}
public QueryPageResult startTableRead(
Connection conn,
String sql,
@@ -273,6 +339,32 @@ public final class JdbcExecutor {
ResultValueReader valueReader,
StatementMessageReader statementMessageReader,
ConcurrentHashMap<String, QuerySession> targetSessions
) {
return executePage(
conn,
sql,
schema,
setSchemaSql,
resetSchemaSql,
options,
valueReader,
statementMessageReader,
targetSessions,
false
);
}
private QueryPageResult executePage(
Connection conn,
String sql,
String schema,
Function<String, String> setSchemaSql,
Supplier<String> resetSchemaSql,
QueryPageOptions options,
ResultValueReader valueReader,
StatementMessageReader statementMessageReader,
ConcurrentHashMap<String, QuerySession> targetSessions,
boolean advancePastUpdateCounts
) {
return unchecked(() -> {
expireIdleSessions(targetSessions, System.currentTimeMillis(), QUERY_SESSION_IDLE_TIMEOUT_MILLIS);
@@ -293,12 +385,26 @@ public final class JdbcExecutor {
// JDBC transaction APIs are reserved for executeTransaction.
boolean hasResultSet = stmt.execute(trimmedSql);
long elapsed = System.currentTimeMillis() - start;
long affectedRows;
if (advancePastUpdateCounts) {
affectedRows = 0L;
while (!hasResultSet) {
int updateCount = stmt.getUpdateCount();
if (updateCount < 0) {
break;
}
affectedRows += updateCount;
hasResultSet = stmt.getMoreResults();
}
} else {
int updateCount = hasResultSet ? -1 : stmt.getUpdateCount();
affectedRows = updateCount >= 0 ? updateCount : 0L;
}
if (!hasResultSet) {
int updateCount = stmt.getUpdateCount();
QueryResult result = new QueryResult(
Collections.emptyList(),
Collections.emptyList(),
updateCount >= 0 ? updateCount : 0,
affectedRows,
elapsed,
false
);
@@ -162,7 +162,8 @@ public final class JsonRpcServer {
if (AgentProtocol.METHOD_TEST_CONNECTION.equals(method)) {
Map<String, Object> result = agent.testConnectionWithInfo(gson.fromJson(params, ConnectParams.class));
if (!Boolean.TRUE.equals(result.get("ok"))) {
throw new RuntimeException("Connection failed");
Object error = result.get("error");
throw new RuntimeException(error == null ? "Connection failed" : String.valueOf(error));
}
return result;
}
@@ -123,12 +123,12 @@ public final class StandardJdbcMetadata {
listObjects(listTables(conn, profile, configuredDatabase, schema, tableConstraints), schema)
);
DatabaseMetaData meta = conn.getMetaData();
appendRoutines(result, meta, null, schema, schema);
appendRoutines(result, meta, profile.getEscapeSchemaWildcards(), null, schema, schema);
if (!containsRoutine(result)
&& profile.getCatalogFallbackEnabled()
&& configuredDatabase != null
&& !configuredDatabase.trim().isEmpty()) {
appendRoutines(result, meta, configuredDatabase, schema, schema);
appendRoutines(result, meta, profile.getEscapeSchemaWildcards(), configuredDatabase, schema, schema);
}
result.sort(Comparator.comparing(ObjectInfo::getName));
return normalized.filterObjects(result);
@@ -350,7 +350,7 @@ public final class StandardJdbcMetadata {
// JDBC has no portable metadata limit/offset. Use table types for safe pushdown and filter/page locally.
// schema 是 DatabaseMetaData.getTables() 的 schemaPattern,_ 和 % 具有通配符语义;
// HANA 存在 _SYS_RT 等含下划线的 schema,需按 getSearchStringEscape() 转义,避免混入其他 schema 的对象。
try (ResultSet rs = meta.getTables(catalog, escapeSchemaPattern(meta, schema), "%", tableTypes)) {
try (ResultSet rs = meta.getTables(catalog, escapeSchemaPattern(meta, schema, profile.getEscapeSchemaWildcards()), "%", tableTypes)) {
while (rs.next()) {
result.add(new TableInfo(
rs.getString("TABLE_NAME"),
@@ -635,12 +635,14 @@ public final class StandardJdbcMetadata {
private static void appendRoutines(
List<ObjectInfo> result,
DatabaseMetaData meta,
boolean escapeSchemaWildcards,
String catalog,
String schema,
String schemaLabel
) {
// schema 作为 getProcedures/getFunctions 的 schemaPattern,同样需要转义 _ 和 % 通配符。
String schemaPattern = escapeSchemaPattern(meta, schema);
// schema 作为 getProcedures/getFunctions 的 schemaPattern,同样需要转义 _ 和 % 通配符
// (databend 等驱动除外,见 escapeSchemaPattern)。
String schemaPattern = escapeSchemaPattern(meta, schema, escapeSchemaWildcards);
Set<String> procedureNames = new LinkedHashSet<>();
try (ResultSet rs = meta.getProcedures(catalog, schemaPattern, "%")) {
while (rs.next()) {
@@ -707,11 +709,17 @@ public final class StandardJdbcMetadata {
*
* <p>当 schema 为空白(将映射为 {@code null},表示不做 schema 过滤)或驱动未提供转义字符时,原样返回。
*/
private static String escapeSchemaPattern(DatabaseMetaData meta, String schema) {
private static String escapeSchemaPattern(DatabaseMetaData meta, String schema, boolean escapeWildcards) {
String normalized = blankToNull(schema);
if (normalized == null) {
return null;
}
// 某些驱动(如 databend)的 getSearchStringEscape() 返回 '\',但其 getTables/getProcedures 把 _ 当字面量
// 且忽略转义,转义反而匹配不到,导致含 _ 的库名(如 my_db)返回 0 行(#8114)。
// 这类驱动通过 profile.escapeSchemaWildcards=false 关闭转义。
if (!escapeWildcards) {
return normalized;
}
String escape;
try {
escape = meta.getSearchStringEscape();
@@ -56,8 +56,9 @@ public final class TransactionExecutor {
return unchecked(() -> {
long start = System.currentTimeMillis();
if (!supportsTransactions(conn)) {
long totalAffected = executeAll(conn, statements, schema, setSchemaSql, resetSchemaSql, runner);
return result(totalAffected, start);
// An explicit transaction request must not degrade to auto-commit:
// callers rely on it to avoid partial writes when a later statement fails.
throw new UnsupportedOperationException("Transactions are not supported by this JDBC driver");
}
boolean savedAutoCommit = conn.getAutoCommit();
@@ -626,17 +626,19 @@ class CommonJavaCompatibilityTest {
}
@Test
void executesTransactionsOneByOneWhenJdbcDriverDoesNotSupportTransactions() {
void rejectsTransactionsWhenJdbcDriverDoesNotSupportTransactions() {
List<String> calls = new ArrayList<>();
DatabaseAgent agent = new TransactionAgent(nonTransactionalConnection(calls));
QueryResult result = agent.executeTransaction(Arrays.asList("UPDATE A SET ID = 1", "UPDATE B SET ID = 2"), "APP");
assertEquals(2L, result.getAffected_rows());
assertEquals(
Arrays.asList("supportsTransactions", "execute:SET SCHEMA \"APP\"", "executeUpdate:UPDATE A SET ID = 1", "executeUpdate:UPDATE B SET ID = 2"),
calls
UnsupportedOperationException error = assertThrows(
UnsupportedOperationException.class,
() -> agent.executeTransaction(Arrays.asList("UPDATE A SET ID = 1", "UPDATE B SET ID = 2"), "APP")
);
assertEquals("Transactions are not supported by this JDBC driver", error.getMessage());
// Capability is checked before schema switching or statements, so a
// failed transaction request cannot leave a partially applied batch.
assertEquals(Collections.singletonList("supportsTransactions"), calls);
}
@Test
@@ -724,6 +726,30 @@ class CommonJavaCompatibilityTest {
);
}
@Test
void buildsTableDdlWithNationalCharacterLength() {
String ddl = DdlBuilder.buildTableDdl(
"DBX_DEMO",
"CUSTOMERS",
Collections.singletonList(new ColumnInfo(
"NAME",
"NVARCHAR",
false,
null,
false,
null,
null,
null,
null,
100
)),
Collections.emptyList(),
Collections.emptyList()
);
assertTrue(ddl.contains("\"NAME\" NVARCHAR(100) NOT NULL"));
}
private static class MinimalAgent implements DatabaseAgent {
@Override
public void connect(ConnectParams params) {
@@ -242,6 +242,47 @@ class JdbcExecutorTest {
assertEquals(Arrays.asList(Arrays.asList(1, "Ada")), result.getRows());
}
@Test
void executeAdvancesPastUpdateCountsToTheFollowingResultSet() {
CountingResultSetFixture fixture = countingResultSet(new Object[][]{{42, "Bill_Record"}});
QueryResult result = JdbcExecutor.INSTANCE.execute(
updateThenResultSetConnection(1, fixture.resultSet()),
"DECLARE @tables TABLE (id INT, name NVARCHAR(128)); INSERT INTO @tables VALUES (42, 'Bill_Record'); SELECT * FROM @tables;",
"",
schema -> "",
() -> "",
100,
null,
0,
JdbcExecutor.INSTANCE::defaultResultValue,
JdbcExecutor.StatementMessageReader.NONE,
true
);
assertEquals(Arrays.asList("id", "name"), result.getColumns());
assertEquals(Arrays.asList(Arrays.asList(42, "Bill_Record")), result.getRows());
}
@Test
void executePageAdvancesPastUpdateCountsToTheFollowingResultSet() {
CountingResultSetFixture fixture = countingResultSet(new Object[][]{{42, "Bill_Record"}});
QueryPageResult result = JdbcExecutor.INSTANCE.executePage(
updateThenResultSetConnection(1, fixture.resultSet()),
"DECLARE @tables TABLE (id INT, name NVARCHAR(128)); INSERT INTO @tables VALUES (42, 'Bill_Record'); SELECT * FROM @tables;",
"",
schema -> "",
() -> "",
new QueryPageOptions(100, null, 100),
JdbcExecutor.INSTANCE::defaultResultValue,
true
);
assertEquals(Arrays.asList("id", "name"), result.getColumns());
assertEquals(Arrays.asList(Arrays.asList(42, "Bill_Record")), result.getRows());
}
@Test
void executeStillPropagatesStatementErrors() {
SQLException failure = new SQLException("permission denied", "42000", 229);
@@ -467,6 +508,42 @@ class JdbcExecutorTest {
);
}
private static Connection updateThenResultSetConnection(int updateCount, ResultSet resultSet) {
AtomicInteger resultIndex = new AtomicInteger();
InvocationHandler statementHandler = (Object unused, Method method, Object[] args) -> {
switch (method.getName()) {
case "execute":
return false;
case "getUpdateCount":
return resultIndex.get() == 0 ? updateCount : -1;
case "getMoreResults":
return resultIndex.incrementAndGet() == 1;
case "getResultSet":
return resultSet;
case "close":
return null;
default:
return defaultValue(method.getReturnType());
}
};
Statement statement = (Statement) Proxy.newProxyInstance(
Statement.class.getClassLoader(),
new Class<?>[]{Statement.class},
statementHandler
);
InvocationHandler connectionHandler = (Object unused, Method method, Object[] args) -> {
if (method.getName().equals("createStatement")) {
return statement;
}
return defaultValue(method.getReturnType());
};
return (Connection) Proxy.newProxyInstance(
Connection.class.getClassLoader(),
new Class<?>[]{Connection.class},
connectionHandler
);
}
private static Object defaultValue(Class<?> type) {
if (type == Boolean.TYPE) {
return false;
@@ -279,6 +279,38 @@ class StandardJdbcMetadataTest {
assertEquals("SALES", capturedArgs.get()[1]);
}
@Test
void listTablesSkipsEscapeWhenProfileDisablesWildcards() {
// databend 等驱动的 getSearchStringEscape() 返回 "\\",但 getTables 把 _ 当字面量且忽略转义,
// 转义含 _ 的库名(如 my_db)会返回 0 行(#8114)。profile.escapeSchemaWildcards=false 时应原样传入。
JdbcAgentProfile noEscapeProfile = new JdbcAgentProfile(
"example.Driver",
"jdbc:example://{host}:{port}/{database}",
0,
false,
Collections.emptySet(),
Arrays.asList("TABLE", "VIEW", "BASE TABLE"),
"\"",
"USE",
true,
false,
false,
false,
false
);
AtomicReference<Object[]> capturedArgs = new AtomicReference<>();
Connection conn = schemaEscapeConnection("\\", rows(
row("TABLE_NAME", "A", "TABLE_TYPE", "TABLE", "REMARKS", null)
), capturedArgs);
List<TableInfo> tables = StandardJdbcMetadata.INSTANCE.listTables(conn, noEscapeProfile, "", "my_db");
assertEquals(1, tables.size());
assertEquals("A", tables.get(0).getName());
// 关闭转义:schemaPattern 应为原始 "my_db",而非 "my\\_db"
assertEquals("my_db", capturedArgs.get()[1]);
}
private static Connection schemaEscapeConnection(String searchEscape, ResultSet tables, AtomicReference<Object[]> capturedArgs) {
DatabaseMetaData meta = proxy(DatabaseMetaData.class, new MethodHandler() {
@Override
+1 -1
View File
@@ -37,7 +37,7 @@ DBX uses unique short-lived logical sessions for independent Agent metadata task
Runtime reuse keys include the Agent driver key, executable or JAR path, launch arguments, working directory, JRE selection, JVM options, classpath-affecting options, and native executable version boundary. Host, account, schema, and credentials belong to sessions and are not part of the runtime key.
Etcd and ZooKeeper retain the legacy path because they use the key-value Agent protocol rather than the SQL session contract. Older Agent binaries and JARs also remain on the legacy path.
ZooKeeper retains the legacy single-session path because its Agent does not advertise the `multi_session` capability; etcd Agents (v3 and v2) run on the shared multi-session path like the SQL Agents. Older Agent binaries and JARs also remain on the legacy path.
## Resource limits and recovery
@@ -0,0 +1,30 @@
# Argo Go agent notes
Status date: 2026-09-03.
This agent is a fork of `agents/drivers/hive-go` created to serve 星环Argo
(Transwarp ArgoDB) connections exclusively: `supportsRoutines()` returns true
unconditionally and connection identity reports `ArgoDB (Transwarp)` /
`DBX ArgoDB Go Agent`. Vanilla Hive / Kyuubi / Impala stay on hive-go.
No Hive 3 / Hive 4 / Kyuubi parity is claimed for this directory. The
validation matrix in `agents/drivers/hive-go/MIGRATION_PARITY.md` applies to
hive-go only.
## Maintenance rule
Most files are kept byte-identical to hive-go. Fixes landing in hive-go that
touch `metadata.go` or `main.go` must be ported here (adjusting only the
argo-specific branding, the unconditional `supportsRoutines()`, and log
prefixes), including their regression tests.
## Validation
Validated against Transwarp ArgoDB by the PR author (#7933):
- `go test ./...` green in this module (routine views asserted queried
unconditionally; ArgoDB identity asserted)
- End-to-end on a local build with an ArgoDB connection: 星环Argo branding
shown, stored-procedure source opens as one whole statement, `CREATE OR
REPLACE PROCEDURE` executes as a single statement, and execute errors
surface readable diagnostics
+101
View File
@@ -0,0 +1,101 @@
# Hive Agent benchmark
This benchmark compares the same DBX JSON-RPC operations through the native
Go Hive Agent and the archived JDBC Hive Agent. Both candidates run on the same
host and connect to the same HiveServer2 instance.
Use `functional_probe.py` before running performance benchmarks. It validates
connection, sessions, query values, metadata, pagination, failed-SQL semantics,
clean shutdown, and Java/Go result parity without concurrency load.
The runner measures:
- process startup and fresh connection latency;
- artifact size, idle RSS, and peak RSS;
- `SELECT 1`-shape lookup and 100/1,000/10,000-row decoding;
- `list_databases`, `list_tables`, and complete paged reads;
- 1, 8, and 32 concurrent DBX Agent sessions;
- mean, p50, p95, p99, throughput, and clean shutdown behavior.
Candidate order rotates between rounds to reduce warm-cache and server-order
bias. Startup and connection samples always use a fresh Agent process.
## Prepare the fixture
The defaults expect `dbx_agent_bench.agent_bench` with exactly 10,000 or more
rows and columns named `id` and `payload`:
```sql
CREATE DATABASE IF NOT EXISTS dbx_agent_bench;
CREATE TABLE IF NOT EXISTS dbx_agent_bench.agent_bench (
id BIGINT,
payload STRING
) STORED AS ORC;
```
Populate deterministic rows before running the benchmark. Keep the fixture,
HiveServer2 configuration, Agent host, and Java runtime unchanged between
candidates.
## Run
Functional parity probe:
```bash
GO_AGENT=/tmp/dbx-hive-bench/hive-agent-linux-amd64 \
JDBC_AGENT_JAR=/tmp/dbx-hive-bench/dbx-agent-hive.jar \
JAVA_BIN=/tmp/dbx-hive-bench/jre21/bin/java \
HIVE_HOST=127.0.0.1 \
HIVE_PORT=10000 \
HIVE_DATABASE=dbx_agent_bench \
HIVE_URL_PARAMS=auth=noSasl \
python3 agents/drivers/hive-go/bench/functional_probe.py \
> /tmp/dbx-hive-bench/functional-result.json
```
The functional probe runs the Go candidate only by default. Set
`BENCH_CANDIDATES=go,jdbc` only when an explicit historical-JDBC comparison is
needed.
Performance benchmark:
```bash
GO_AGENT=/tmp/dbx-hive-bench/hive-agent-linux-amd64 \
JDBC_AGENT_JAR=/tmp/dbx-hive-bench/dbx-agent-hive.jar \
HIVE_HOST=127.0.0.1 \
HIVE_PORT=10000 \
HIVE_DATABASE=dbx_agent_bench \
HIVE_URL_PARAMS=auth=noSasl \
python3 agents/drivers/hive-go/bench/agent_compare.py \
> /tmp/dbx-hive-bench/result.json
```
## Configuration
- `BENCH_CANDIDATES`: performance benchmark default `go,jdbc`; functional probe
default `go`.
- `GO_AGENT_COMMAND`, `JDBC_AGENT_COMMAND`: optional full launch commands.
- `BENCH_STARTUPS`, `BENCH_CONNECTS`: fresh-process sample counts, default `8`.
- `BENCH_ROUNDS`: alternating steady-state rounds, default `3`.
- `BENCH_WARMUPS`: warmups before each workload, default `2`.
- `BENCH_CONCURRENCY`: comma-separated session counts, default `1,8,32`.
- `BENCH_CONCURRENCY_OPS_PER_WORKER`: operations per session, default `8`.
- `HIVE_HOST`, `HIVE_PORT`, `HIVE_DATABASE`, `HIVE_BENCH_TABLE`.
- `HIVE_USERNAME`, `HIVE_PASSWORD`, `HIVE_URL_PARAMS`, `HIVE_CONNECTION_STRING`.
- `HIVE_SSL`, `HIVE_CA_CERT_PATH`, `HIVE_CLIENT_CERT_PATH`, `HIVE_CLIENT_KEY_PATH`.
- `BENCH_*_SQL` and `BENCH_*_COUNT` override individual workloads.
Run the benchmark on the Agent host. Do not compare a local Go process with a
remote JDBC process, use different HS2 endpoints, or mutate the fixture between
candidates.
## Kerberos fixture
`kdc_fixture` starts a test-only in-process KDC and writes a temporary
`krb5.conf` and keytab containing `alice` and `hive/localhost`. Never use these
credentials outside an isolated compatibility environment, and delete the
generated directory after validation.
```bash
go run ./bench/kdc_fixture -dir /tmp/dbx-hive-kerberos
```
+664
View File
@@ -0,0 +1,664 @@
#!/usr/bin/env python3
import json
import os
import queue
import shlex
import statistics
import subprocess
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from pathlib import Path
@dataclass(frozen=True)
class Candidate:
name: str
command: list[str]
artifact: Path
rss_command: str = ""
class AgentProcess:
def __init__(self, candidate: Candidate):
self.candidate = candidate
self.process = subprocess.Popen(
candidate.command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
bufsize=1,
)
self.request_id = 0
self.request_lock = threading.Lock()
self.write_lock = threading.Lock()
self.pending: dict[int, queue.Queue] = {}
self.ready = threading.Event()
self.saw_ready = False
self.exited = threading.Event()
self.stderr_lines: list[str] = []
threading.Thread(target=self._read_stdout, daemon=True).start()
threading.Thread(target=self._drain_stderr, daemon=True).start()
if not self.ready.wait(env_float("BENCH_READY_TIMEOUT", 30.0)) or not self.saw_ready:
raise TimeoutError(self._failure("timed out waiting for agent readiness"))
def _read_stdout(self) -> None:
assert self.process.stdout is not None
for line in self.process.stdout:
try:
response = json.loads(line)
except json.JSONDecodeError:
continue
if response.get("ready") is True:
self.saw_ready = True
self.ready.set()
continue
response_id = response.get("id")
if not isinstance(response_id, int):
continue
with self.request_lock:
response_queue = self.pending.get(response_id)
if response_queue is not None:
response_queue.put(response)
self.exited.set()
self.ready.set()
with self.request_lock:
pending = list(self.pending.values())
for response_queue in pending:
response_queue.put(RuntimeError(self._failure("agent process exited")))
def _drain_stderr(self) -> None:
assert self.process.stderr is not None
for line in self.process.stderr:
self.stderr_lines.append(line.rstrip())
def call(self, method: str, params: dict | None = None) -> object:
if self.exited.is_set():
raise RuntimeError(self._failure(f"agent exited before {method}"))
with self.request_lock:
self.request_id += 1
request_id = self.request_id
response_queue: queue.Queue = queue.Queue(maxsize=1)
self.pending[request_id] = response_queue
request = {
"jsonrpc": "2.0",
"id": request_id,
"method": method,
"params": params or {},
}
try:
assert self.process.stdin is not None
with self.write_lock:
self.process.stdin.write(json.dumps(request, separators=(",", ":")) + "\n")
self.process.stdin.flush()
response = response_queue.get(timeout=env_float("BENCH_RPC_TIMEOUT", 180.0))
except queue.Empty as error:
raise TimeoutError(self._failure(f"timed out during {method}")) from error
finally:
with self.request_lock:
self.pending.pop(request_id, None)
if isinstance(response, Exception):
raise response
if response.get("error") is not None:
raise RuntimeError(
f"{self.candidate.name} {method}: "
f"{json.dumps(response['error'], ensure_ascii=False)}"
)
return response.get("result")
def rss_kib(self) -> int:
if self.candidate.rss_command:
output = subprocess.check_output(
self.candidate.rss_command,
shell=True,
text=True,
).strip()
return int(output or "0")
status_path = Path(f"/proc/{self.process.pid}/status")
if status_path.is_file():
for line in status_path.read_text().splitlines():
if line.startswith("VmRSS:"):
return int(line.split()[1])
output = subprocess.check_output(
["ps", "-o", "rss=", "-p", str(self.process.pid)],
text=True,
).strip()
return int(output or "0")
def close(self) -> bool:
if self.process.poll() is not None:
return True
try:
self.call("shutdown")
except Exception:
pass
try:
self.process.wait(timeout=3)
return True
except subprocess.TimeoutExpired:
self.process.terminate()
try:
self.process.wait(timeout=2)
except subprocess.TimeoutExpired:
self.process.kill()
self.process.wait(timeout=5)
return False
def _failure(self, message: str) -> str:
stderr = "\n".join(self.stderr_lines[-20:])
return f"{self.candidate.name}: {message}\n{stderr}".rstrip()
class RSSMonitor:
def __init__(self, process: AgentProcess):
self.process = process
self.peak_kib = 0
self.stop_event = threading.Event()
self.thread = threading.Thread(target=self._run, daemon=True)
def __enter__(self):
self.peak_kib = self.process.rss_kib()
self.thread.start()
return self
def __exit__(self, exc_type, exc_value, traceback):
self.stop_event.set()
self.thread.join(timeout=1)
try:
self.peak_kib = max(self.peak_kib, self.process.rss_kib())
except (OSError, subprocess.SubprocessError, ValueError):
pass
def _run(self) -> None:
interval = env_float("BENCH_RSS_INTERVAL", 0.02)
while not self.stop_event.wait(interval):
try:
self.peak_kib = max(self.peak_kib, self.process.rss_kib())
except (OSError, subprocess.SubprocessError, ValueError):
return
def main() -> None:
candidates = configured_candidates()
connection = connection_params()
startup_iterations = env_int("BENCH_STARTUPS", 8)
connect_iterations = env_int("BENCH_CONNECTS", 8)
rounds = env_int("BENCH_ROUNDS", 3)
warmups = env_int("BENCH_WARMUPS", 2)
workloads = configured_workloads(connection["database"])
concurrency_levels = env_int_list("BENCH_CONCURRENCY", [1, 8, 32])
startup_samples = {candidate.name: [] for candidate in candidates}
connect_samples = {candidate.name: [] for candidate in candidates}
workload_samples = {
candidate.name: {workload["name"]: [] for workload in workloads}
for candidate in candidates
}
concurrency_samples = {
candidate.name: {str(level): [] for level in concurrency_levels}
for candidate in candidates
}
process_metrics = {candidate.name: [] for candidate in candidates}
for iteration in range(startup_iterations):
for candidate in rotated(candidates, iteration):
startup_samples[candidate.name].append(benchmark_startup(candidate))
for iteration in range(connect_iterations):
for candidate in rotated(candidates, iteration):
connect_samples[candidate.name].append(benchmark_connect(candidate, connection))
for round_index in range(rounds):
for candidate in rotated(candidates, round_index):
process = AgentProcess(candidate)
shutdown_clean = False
metrics = None
try:
process.call("connect", connection)
idle_rss_kib = process.rss_kib()
with RSSMonitor(process) as monitor:
for workload in workloads:
sample = benchmark_workload(process, workload, warmups)
workload_samples[candidate.name][workload["name"]].append(sample)
process.call("disconnect")
metrics = {
"idle_rss_kib": idle_rss_kib,
"peak_rss_kib": monitor.peak_kib,
}
finally:
shutdown_clean = process.close()
if metrics is not None:
metrics["shutdown_exited_within_3s"] = shutdown_clean
process_metrics[candidate.name].append(metrics)
for level in concurrency_levels:
for candidate in rotated(candidates, round_index + level):
concurrency_samples[candidate.name][str(level)].append(
benchmark_concurrency(candidate, connection, level, warmups)
)
results = []
for candidate in candidates:
name = candidate.name
results.append(
{
"candidate": name,
"command": candidate.command,
"artifact_bytes": candidate.artifact.stat().st_size,
"startup": summarize_latencies(startup_samples[name]),
"connect": summarize_latencies(connect_samples[name]),
"process": summarize_process_metrics(process_metrics[name]),
"workloads": [
summarize_rounds(workload["name"], workload_samples[name][workload["name"]])
for workload in workloads
],
"concurrency": [
summarize_rounds(f"concurrency_{level}", concurrency_samples[name][str(level)])
for level in concurrency_levels
],
}
)
output = {
"host": os.uname().nodename,
"server": env_default("HIVE_SERVER", f"{connection['host']}:{connection['port']}"),
"database": connection["database"],
"table": env_default("HIVE_BENCH_TABLE", "agent_bench"),
"startup_iterations": startup_iterations,
"connect_iterations": connect_iterations,
"rounds": rounds,
"warmups": warmups,
"concurrency_levels": concurrency_levels,
"results": results,
}
json.dump(output, sys.stdout, ensure_ascii=False, indent=2)
sys.stdout.write("\n")
def configured_candidates() -> list[Candidate]:
selected = {
item.strip()
for item in env_default("BENCH_CANDIDATES", "go,jdbc").split(",")
if item.strip()
}
candidates = []
if "go" in selected:
artifact = required_path("GO_AGENT")
raw_command = os.getenv("GO_AGENT_COMMAND", "")
command = shlex.split(raw_command) if raw_command else [str(artifact)]
candidates.append(
Candidate("go-native", command, artifact, os.getenv("GO_RSS_COMMAND", ""))
)
if "jdbc" in selected:
artifact = required_path("JDBC_AGENT_JAR")
raw_command = os.getenv("JDBC_AGENT_COMMAND", "")
command = (
shlex.split(raw_command)
if raw_command
else [env_default("JAVA_BIN", "java"), "-jar", str(artifact)]
)
candidates.append(
Candidate("jdbc-java", command, artifact, os.getenv("JDBC_RSS_COMMAND", ""))
)
if not candidates:
raise ValueError("BENCH_CANDIDATES selected no candidates")
return candidates
def connection_params() -> dict:
return {
"host": env_default("HIVE_HOST", "127.0.0.1"),
"port": env_int("HIVE_PORT", 10000),
"database": env_default("HIVE_DATABASE", "dbx_agent_bench"),
"username": os.getenv("HIVE_USERNAME", ""),
"password": os.getenv("HIVE_PASSWORD", ""),
"url_params": env_default("HIVE_URL_PARAMS", "auth=noSasl"),
"connection_string": os.getenv("HIVE_CONNECTION_STRING", ""),
"ssl": env_bool("HIVE_SSL", False),
"ca_cert_path": os.getenv("HIVE_CA_CERT_PATH", ""),
"client_cert_path": os.getenv("HIVE_CLIENT_CERT_PATH", ""),
"client_key_path": os.getenv("HIVE_CLIENT_KEY_PATH", ""),
"connect_timeout_secs": env_int("HIVE_CONNECT_TIMEOUT", 30),
}
def configured_workloads(database: str) -> list[dict]:
table = env_default("HIVE_BENCH_TABLE", "agent_bench")
qualified = f"`{database}`.`{table}`"
workloads = [
query_workload("select_one", "SELECT 1 AS value", 1, 40),
query_workload("rows_100", f"SELECT id, payload FROM {qualified} LIMIT 100", 100, 20),
query_workload("rows_1000", f"SELECT id, payload FROM {qualified} LIMIT 1000", 1000, 10),
query_workload("rows_10000", f"SELECT id, payload FROM {qualified} LIMIT 10000", 10000, 3),
{
"name": "list_databases",
"kind": "rpc",
"method": "list_databases",
"params": {},
"count": env_int("BENCH_LIST_DATABASES_COUNT", 20),
},
{
"name": "list_tables",
"kind": "rpc",
"method": "list_tables",
"params": {"schema": database},
"count": env_int("BENCH_LIST_TABLES_COUNT", 20),
},
{
"name": "page_10000_by_500",
"kind": "paged",
"sql": env_default("BENCH_PAGE_SQL", f"SELECT id, payload FROM {qualified} LIMIT 10000"),
"max_rows": 10000,
"page_size": env_int("BENCH_PAGE_SIZE", 500),
"count": env_int("BENCH_PAGE_COUNT", 3),
},
]
return workloads
def query_workload(name: str, fallback_sql: str, max_rows: int, fallback_count: int) -> dict:
suffix = name.upper()
return {
"name": name,
"kind": "rpc",
"method": "execute_query",
"params": {
"sql": env_default(f"BENCH_{suffix}_SQL", fallback_sql),
"maxRows": max_rows,
"fetchSize": min(max_rows, env_int("BENCH_FETCH_SIZE", 1000)),
},
"count": env_int(f"BENCH_{suffix}_COUNT", fallback_count),
}
def benchmark_startup(candidate: Candidate) -> float:
started = time.perf_counter()
process = AgentProcess(candidate)
elapsed = elapsed_ms(started)
process.close()
return elapsed
def benchmark_connect(candidate: Candidate, connection: dict) -> float:
process = AgentProcess(candidate)
try:
started = time.perf_counter()
process.call("connect", connection)
return elapsed_ms(started)
finally:
process.close()
def benchmark_workload(process: AgentProcess, workload: dict, warmups: int) -> dict:
for _ in range(warmups):
execute_workload(process, workload)
samples = []
started = time.perf_counter()
for _ in range(workload["count"]):
operation_started = time.perf_counter()
execute_workload(process, workload)
samples.append(elapsed_ms(operation_started))
elapsed = time.perf_counter() - started
return sample_result(workload["count"], elapsed, samples)
def benchmark_concurrency(
candidate: Candidate,
connection: dict,
concurrency: int,
warmups: int,
) -> dict:
process = AgentProcess(candidate)
session_ids = [f"bench-{concurrency}-{index}" for index in range(concurrency)]
qualified = (
f"`{connection['database']}`."
f"`{env_default('HIVE_BENCH_TABLE', 'agent_bench')}`"
)
workload = {
"kind": "rpc",
"method": "execute_query",
"params": {
"sql": env_default(
"BENCH_CONCURRENCY_SQL",
"SELECT 1 AS value",
),
"maxRows": 1,
},
}
operations_per_worker = env_int("BENCH_CONCURRENCY_OPS_PER_WORKER", 8)
try:
for session_id in session_ids:
process.call("open_session", {**connection, "agentSessionId": session_id})
for session_id in session_ids:
for _ in range(warmups):
execute_workload(process, workload, session_id)
with RSSMonitor(process) as monitor:
started = time.perf_counter()
with ThreadPoolExecutor(max_workers=concurrency) as executor:
futures = [
executor.submit(
concurrency_worker,
process,
workload,
session_id,
operations_per_worker,
)
for session_id in session_ids
]
samples = [sample for future in futures for sample in future.result()]
elapsed = time.perf_counter() - started
result = sample_result(len(samples), elapsed, samples)
result["concurrency"] = concurrency
result["peak_rss_kib"] = monitor.peak_kib
return result
finally:
for session_id in session_ids:
try:
process.call("close_session", {"agentSessionId": session_id})
except Exception:
pass
process.close()
def concurrency_worker(
process: AgentProcess,
workload: dict,
session_id: str,
operations: int,
) -> list[float]:
samples = []
for _ in range(operations):
started = time.perf_counter()
execute_workload(process, workload, session_id)
samples.append(elapsed_ms(started))
return samples
def execute_workload(
process: AgentProcess,
workload: dict,
agent_session_id: str = "",
) -> object:
params = dict(workload.get("params", {}))
if agent_session_id:
params["agentSessionId"] = agent_session_id
if workload["kind"] == "rpc":
return process.call(workload["method"], params)
if workload["kind"] != "paged":
raise ValueError(f"unknown workload kind: {workload['kind']}")
first = process.call(
"execute_query_page",
{
"sql": workload["sql"],
"maxRows": workload["max_rows"],
"pageSize": workload["page_size"],
**({"agentSessionId": agent_session_id} if agent_session_id else {}),
},
)
rows = len(first.get("rows", []))
session_id = first.get("session_id")
has_more = first.get("has_more", False)
try:
while has_more:
page = process.call(
"fetch_query_page",
{
"sessionId": session_id,
"pageSize": workload["page_size"],
**({"agentSessionId": agent_session_id} if agent_session_id else {}),
},
)
rows += len(page.get("rows", []))
session_id = page.get("session_id")
has_more = page.get("has_more", False)
finally:
if session_id:
process.call(
"close_query_session",
{
"sessionId": session_id,
**({"agentSessionId": agent_session_id} if agent_session_id else {}),
},
)
if rows != workload["max_rows"]:
raise RuntimeError(
f"{process.candidate.name} paged query returned {rows} rows, "
f"expected {workload['max_rows']}"
)
return rows
def sample_result(count: int, elapsed: float, samples: list[float]) -> dict:
summary = summarize_latencies(samples)
summary.update(
{
"count": count,
"elapsed_ms": elapsed * 1000,
"ops_per_sec": count / elapsed,
}
)
return summary
def summarize_latencies(samples: list[float]) -> dict:
ordered = sorted(samples)
return {
"samples_ms": samples,
"mean_ms": statistics.mean(samples),
"p50_ms": percentile(ordered, 0.50),
"p95_ms": percentile(ordered, 0.95),
"p99_ms": percentile(ordered, 0.99),
"min_ms": ordered[0],
"max_ms": ordered[-1],
}
def summarize_rounds(name: str, rounds: list[dict]) -> dict:
latencies = [sample for round_result in rounds for sample in round_result["samples_ms"]]
elapsed = sum(round_result["elapsed_ms"] for round_result in rounds) / 1000
result = summarize_latencies(latencies)
result.update(
{
"name": name,
"rounds": rounds,
"count": len(latencies),
"elapsed_ms": elapsed * 1000,
"ops_per_sec": len(latencies) / elapsed,
}
)
peak_values = [round_result.get("peak_rss_kib", 0) for round_result in rounds]
if any(peak_values):
result["peak_rss_kib"] = max(peak_values)
return result
def summarize_process_metrics(samples: list[dict]) -> dict:
return {
"idle_rss_kib": summarize_numbers([sample["idle_rss_kib"] for sample in samples]),
"peak_rss_kib": summarize_numbers([sample["peak_rss_kib"] for sample in samples]),
"shutdown_exited_within_3s": all(
sample["shutdown_exited_within_3s"] for sample in samples
),
"rounds": samples,
}
def summarize_numbers(values: list[int]) -> dict:
return {
"min": min(values),
"median": statistics.median(values),
"max": max(values),
}
def rotated(values: list[Candidate], offset: int) -> list[Candidate]:
if not values:
return []
shift = offset % len(values)
return values[shift:] + values[:shift]
def percentile(values: list[float], fraction: float) -> float:
if not values:
return 0.0
index = min(len(values) - 1, max(0, round((len(values) - 1) * fraction)))
return values[index]
def elapsed_ms(started: float) -> float:
return (time.perf_counter() - started) * 1000
def required_path(name: str) -> Path:
value = os.getenv(name, "")
if not value:
raise ValueError(f"{name} is required")
path = Path(value).expanduser().resolve()
if not path.is_file():
raise FileNotFoundError(path)
return path
def env_default(name: str, fallback: str) -> str:
return os.getenv(name, "") or fallback
def env_int(name: str, fallback: int) -> int:
value = int(env_default(name, str(fallback)))
if value < 1:
raise ValueError(f"{name} must be positive")
return value
def env_int_list(name: str, fallback: list[int]) -> list[int]:
raw = os.getenv(name, "")
values = fallback if not raw else [int(value.strip()) for value in raw.split(",")]
if not values or any(value < 1 for value in values):
raise ValueError(f"{name} must contain positive integers")
return values
def env_float(name: str, fallback: float) -> float:
value = float(env_default(name, str(fallback)))
if value <= 0:
raise ValueError(f"{name} must be positive")
return value
def env_bool(name: str, fallback: bool) -> bool:
raw = os.getenv(name)
if raw is None or raw == "":
return fallback
normalized = raw.strip().lower()
if normalized in {"1", "true", "yes", "on"}:
return True
if normalized in {"0", "false", "no", "off"}:
return False
raise ValueError(f"{name} must be a boolean")
if __name__ == "__main__":
main()
@@ -0,0 +1,203 @@
#!/usr/bin/env python3
import hashlib
import json
import os
import sys
from pathlib import Path
from agent_compare import AgentProcess, configured_candidates, connection_params, env_default, env_int
def main() -> None:
os.environ.setdefault("BENCH_CANDIDATES", "go")
connection = connection_params()
candidates = configured_candidates()
results = {candidate.name: probe_candidate(candidate, connection) for candidate in candidates}
output = {
"server": env_default("HIVE_SERVER", f"{connection['host']}:{connection['port']}"),
"connection": sanitized_connection(connection),
"artifacts": {
candidate.name: {
"path": str(candidate.artifact),
"sha256": sha256(candidate.artifact),
"size_bytes": candidate.artifact.stat().st_size,
}
for candidate in candidates
},
"results": results,
"parity": compare_results(results),
}
json.dump(output, sys.stdout, ensure_ascii=False, indent=2)
sys.stdout.write("\n")
if any(not result.get("ok") for result in results.values()) or not output["parity"]["ok"]:
raise SystemExit(1)
def probe_candidate(candidate, connection: dict) -> dict:
process = None
session_id = f"functional-probe-{candidate.name}"
result = {"ok": False}
try:
process = AgentProcess(candidate)
result["test_connection"] = process.call("test_connection", connection)
process.call("open_session", {"agentSessionId": session_id, **connection})
result["validate_session"] = process.call("validate_session", {"agentSessionId": session_id})
result["select_one"] = normalized_query(
process.call(
"execute_query",
{
"agentSessionId": session_id,
"sql": env_default("PROBE_SELECT_SQL", "SELECT 1 AS value"),
"maxRows": env_int("PROBE_SELECT_MAX_ROWS", 10),
"fetchSize": env_int("PROBE_FETCH_SIZE", 10),
},
)
)
result["databases"] = sorted(
item.get("name", "")
for item in process.call("list_databases", {"agentSessionId": session_id})
)
schema = env_default("PROBE_SCHEMA", connection["database"])
result["tables"] = sorted(
item.get("name", "")
for item in process.call(
"list_tables",
{"agentSessionId": session_id, "schema": schema},
)
)
result["paging"] = probe_paging(process, session_id)
result["invalid_sql"] = probe_failure_semantics(process, session_id)
result["after_failure"] = normalized_query(
process.call(
"execute_query",
{
"agentSessionId": session_id,
"sql": env_default("PROBE_AFTER_FAILURE_SQL", "SELECT 2 AS value"),
"maxRows": 10,
"fetchSize": env_int("PROBE_FETCH_SIZE", 10),
},
)
)
process.call("close_session", {"agentSessionId": session_id})
result["ok"] = True
except Exception as error:
result["error"] = str(error)
finally:
if process is not None:
result["clean_shutdown"] = process.close()
return result
def probe_paging(process: AgentProcess, agent_session_id: str) -> dict:
page_size = env_int("PROBE_PAGE_SIZE", 2)
first = process.call(
"execute_query_page",
{
"agentSessionId": agent_session_id,
"sql": env_default(
"PROBE_PAGE_SQL",
"SELECT id, payload FROM dbx_agent_bench.agent_bench LIMIT 3",
),
"maxRows": env_int("PROBE_PAGE_MAX_ROWS", 3),
"pageSize": page_size,
},
)
pages = [first]
query_session_id = first.get("session_id")
while pages[-1].get("has_more"):
pages.append(
process.call(
"fetch_query_page",
{
"agentSessionId": agent_session_id,
"sessionId": query_session_id,
"pageSize": page_size,
},
)
)
return {
"columns": first.get("columns", []),
"column_types": first.get("column_types", []),
"rows": [row for page in pages for row in page.get("rows", [])],
"page_count": len(pages),
"has_more_final": pages[-1].get("has_more", False),
"truncated": any(page.get("truncated", False) for page in pages),
}
def probe_failure_semantics(process: AgentProcess, agent_session_id: str) -> dict:
try:
process.call(
"execute_query",
{
"agentSessionId": agent_session_id,
"sql": env_default(
"PROBE_INVALID_SQL",
"SELECT * FROM dbx_missing_table_for_failure_semantics",
),
"maxRows": 10,
"fetchSize": env_int("PROBE_FETCH_SIZE", 10),
},
)
except Exception as error:
return {"failed": True, "error": str(error)}
return {"failed": False, "error": ""}
def normalized_query(result: dict) -> dict:
return {
"columns": result.get("columns", []),
"column_types": result.get("column_types", []),
"rows": result.get("rows", []),
"truncated": result.get("truncated", False),
}
def compare_results(results: dict) -> dict:
successful = [result for result in results.values() if result.get("ok")]
if len(successful) < 2:
return {"ok": len(results) == 1 and len(successful) == 1, "differences": []}
baseline = successful[0]
differences = []
for field in ["select_one", "databases", "tables", "paging", "after_failure"]:
expected = baseline.get(field)
for candidate_name, candidate_result in results.items():
if candidate_result.get("ok") and candidate_result.get(field) != expected:
differences.append(
{
"candidate": candidate_name,
"field": field,
"expected": expected,
"actual": candidate_result.get(field),
}
)
for candidate_name, candidate_result in results.items():
if candidate_result.get("ok") and not candidate_result.get("invalid_sql", {}).get("failed"):
differences.append(
{
"candidate": candidate_name,
"field": "invalid_sql.failed",
"expected": True,
"actual": False,
}
)
return {"ok": not differences, "differences": differences}
def sanitized_connection(connection: dict) -> dict:
result = dict(connection)
if result.get("password"):
result["password"] = "***"
return result
def sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
if __name__ == "__main__":
main()
@@ -0,0 +1,99 @@
package main
import (
"encoding/json"
"flag"
"fmt"
"log"
"os"
"os/signal"
"path/filepath"
"syscall"
"github.com/jcmturner/krb5test"
)
type fixtureInfo struct {
Realm string `json:"realm"`
Address string `json:"address"`
ConfigPath string `json:"config_path"`
KeytabPath string `json:"keytab_path"`
ClientPrincipal string `json:"client_principal"`
ServicePrincipal string `json:"service_principal"`
ZooKeeperPrincipal string `json:"zookeeper_principal"`
}
func main() {
directory := flag.String("dir", "", "directory for generated Kerberos fixture files")
flag.Parse()
if *directory == "" {
log.Fatal("-dir is required")
}
if err := os.MkdirAll(*directory, 0o700); err != nil {
log.Fatal(err)
}
logger := log.New(os.Stderr, "kdc: ", log.LstdFlags)
kdc, err := krb5test.NewKDC(map[string][]string{
"alice": nil,
"hive/localhost": nil,
"zookeeper/localhost": nil,
}, logger)
if err != nil {
log.Fatal(err)
}
kdc.KRB5Conf.LibDefaults.UDPPreferenceLimit = 1
kdc.Start()
defer kdc.Close()
configPath := filepath.Join(*directory, "krb5.conf")
keytabPath := filepath.Join(*directory, "fixture.keytab")
config := fmt.Sprintf(`[libdefaults]
default_realm = %s
dns_lookup_realm = false
dns_lookup_kdc = false
rdns = false
udp_preference_limit = 1
default_tgs_enctypes = aes256-cts-hmac-sha1-96
default_tkt_enctypes = aes256-cts-hmac-sha1-96
permitted_enctypes = aes256-cts-hmac-sha1-96
[realms]
%s = {
kdc = %s
}
[domain_realm]
.localhost = %s
localhost = %s
`, kdc.Realm, kdc.Realm, kdc.TCPListener.Addr().String(), kdc.Realm, kdc.Realm)
if err := os.WriteFile(configPath, []byte(config), 0o644); err != nil {
log.Fatal(err)
}
keytab, err := kdc.Keytab.Marshal()
if err != nil {
log.Fatal(err)
}
if err := os.WriteFile(keytabPath, keytab, 0o600); err != nil {
log.Fatal(err)
}
info := fixtureInfo{
Realm: kdc.Realm,
Address: kdc.TCPListener.Addr().String(),
ConfigPath: configPath,
KeytabPath: keytabPath,
ClientPrincipal: "alice@" + kdc.Realm,
ServicePrincipal: "hive/localhost@" + kdc.Realm,
ZooKeeperPrincipal: "zookeeper/localhost@" + kdc.Realm,
}
if err := json.NewEncoder(os.Stdout).Encode(info); err != nil {
log.Fatal(err)
}
if err := os.Stdout.Sync(); err != nil {
log.Fatal(err)
}
signals := make(chan os.Signal, 1)
signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
<-signals
}
File diff suppressed because it is too large Load Diff
+713
View File
@@ -0,0 +1,713 @@
package main
import (
"bytes"
"crypto/x509"
"encoding/base64"
"net/url"
"os"
"path/filepath"
"reflect"
"runtime"
"strings"
"testing"
"time"
pkcs12 "software.sslmate.com/src/go-pkcs12"
)
func writeHadoopVInt(buffer *bytes.Buffer, value int64) {
if value >= -112 && value <= 127 {
buffer.WriteByte(byte(int8(value)))
return
}
lengthMarker := int8(-112)
encoded := value
if value < 0 {
encoded = ^value
lengthMarker = -120
}
temporary := encoded
for temporary != 0 {
temporary >>= 8
lengthMarker--
}
buffer.WriteByte(byte(lengthMarker))
length := -int(lengthMarker)
if lengthMarker < -120 {
length -= 120
} else {
length -= 112
}
for index := length; index != 0; index-- {
shift := uint((index - 1) * 8)
buffer.WriteByte(byte(encoded >> shift))
}
}
func encodeHadoopToken(identifier, password, kind, service []byte) string {
var buffer bytes.Buffer
for _, value := range [][]byte{identifier, password, kind, service} {
writeHadoopVInt(&buffer, int64(len(value)))
buffer.Write(value)
}
return base64.RawURLEncoding.EncodeToString(buffer.Bytes())
}
func TestParseDirectJDBCConnection(t *testing.T) {
config, err := parseConnectionConfig(connectParams{
ConnectionString: "jdbc:hive2://hive.example.com:10001/analytics;transportMode=http;httpPath=gateway;ssl=true",
Username: "alice",
Password: "secret",
})
if err != nil {
t.Fatal(err)
}
if len(config.Endpoints) != 1 || config.Endpoints[0] != (endpoint{Host: "hive.example.com", Port: 10001}) {
t.Fatalf("unexpected endpoints: %#v", config.Endpoints)
}
if config.Database != "analytics" || config.TransportMode != "http" || config.HTTPPath != "gateway" {
t.Fatalf("unexpected config: %#v", config)
}
if config.TLSConfig == nil {
t.Fatal("expected TLS config")
}
}
func TestParseHTTPKerberosChannelBinding(t *testing.T) {
config, err := parseConnectionConfig(connectParams{
ConnectionString: "jdbc:hive2://hive.example.com:10001/default;transportMode=http;ssl=true;auth=KERBEROS;principal=HTTP/_HOST@EXAMPLE.COM;kerberosChannelBinding=true",
Username: "alice@EXAMPLE.COM",
Password: "secret",
AgentJavaOptions: []string{"-Djava.security.krb5.conf=/etc/krb5.conf"},
})
if err != nil {
t.Fatal(err)
}
if !config.Kerberos.ChannelBinding {
t.Fatal("expected HTTP Kerberos TLS channel binding")
}
}
func TestParseZooKeeperKerberosJDBCConnection(t *testing.T) {
configPath := "/etc/krb5.conf"
if runtime.GOOS == "windows" {
configPath = `C:\ProgramData\MIT\Kerberos5\krb5.ini`
}
config, err := parseConnectionConfig(connectParams{
ConnectionString: "jdbc:hive2://zk1.example.com:2181,zk2.example.com:2181/default;serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=kyuubi;principal=hive/_HOST@EXAMPLE.COM;hive.server2.thrift.sasl.qop=auth-conf",
Username: "alice@EXAMPLE.COM",
Password: "secret",
AgentJavaOptions: []string{"-Djava.security.krb5.conf=" + configPath},
})
if err != nil {
t.Fatal(err)
}
if config.ServiceDiscoveryMode != "zookeeper" || config.ZooKeeperNamespace != "kyuubi" {
t.Fatalf("unexpected ZooKeeper config: %#v", config)
}
if len(config.Endpoints) != 2 || config.Endpoints[1].Host != "zk2.example.com" || config.Endpoints[1].Port != 2181 {
t.Fatalf("unexpected endpoints: %#v", config.Endpoints)
}
if !config.Kerberos.Enabled || config.Kerberos.Service != "hive" || config.Kerberos.QOP != "auth-conf" {
t.Fatalf("unexpected Kerberos config: %#v", config.Kerberos)
}
if config.Kerberos.ConfigPath != configPath {
t.Fatalf("unexpected krb5 config path: %q", config.Kerberos.ConfigPath)
}
if !config.ZooKeeperKerberos.Enabled || config.ZooKeeperKerberos.Service != "zookeeper" || !config.ZooKeeperKerberos.CanonicalHostname {
t.Fatalf("unexpected ZooKeeper Kerberos config: %#v", config.ZooKeeperKerberos)
}
}
func TestParseZooKeeperKerberosCompatibilityProperties(t *testing.T) {
config, err := parseConnectionConfig(connectParams{
ConnectionString: "jdbc:hive2://zk.example.com:2181/default;serviceDiscoveryMode=zooKeeper;principal=hive/_HOST@HIVE.EXAMPLE.COM;zookeeper.sasl.client.username=zkservice;zookeeper.server.realm=ZK.EXAMPLE.COM;zookeeper.sasl.client.canonicalize.hostname=false",
Username: "alice@EXAMPLE.COM",
Password: "secret",
AgentJavaOptions: []string{
"-Djava.security.krb5.conf=/etc/krb5.conf",
"-Dzookeeper.server.principal=zookeeper/zk.example.com@EXPLICIT.EXAMPLE.COM",
},
})
if err != nil {
t.Fatal(err)
}
want := zooKeeperKerberosConfig{
Enabled: true,
Service: "zkservice",
ServerPrincipal: "zookeeper/zk.example.com@EXPLICIT.EXAMPLE.COM",
Realm: "ZK.EXAMPLE.COM",
CanonicalHostname: false,
}
if config.ZooKeeperKerberos != want {
t.Fatalf("ZooKeeper Kerberos config = %#v, want %#v", config.ZooKeeperKerberos, want)
}
}
func TestZooKeeperKerberosCanBeDisabledExplicitly(t *testing.T) {
config, err := parseConnectionConfig(connectParams{
ConnectionString: "jdbc:hive2://zk.example.com:2181/default;serviceDiscoveryMode=zooKeeper;principal=hive/_HOST@EXAMPLE.COM;hive.zookeeper.use.kerberos=true",
Username: "alice@EXAMPLE.COM",
Password: "secret",
AgentJavaOptions: []string{
"-Djava.security.krb5.conf=/etc/krb5.conf",
"-Dzookeeper.sasl.client=false",
},
})
if err != nil {
t.Fatal(err)
}
if config.ZooKeeperKerberos.Enabled {
t.Fatalf("ZooKeeper Kerberos should be disabled: %#v", config.ZooKeeperKerberos)
}
}
func TestURLParamsOverrideConnectionString(t *testing.T) {
config, err := parseConnectionConfig(connectParams{
Host: "hive.example.com",
Port: 10000,
ConnectionString: "jdbc:hive2://old.example.com:10000/default;transportMode=binary",
URLParams: "transportMode=http;httpPath=proxy",
})
if err != nil {
t.Fatal(err)
}
if config.TransportMode != "http" || config.HTTPPath != "proxy" {
t.Fatalf("URL params did not override connection string: %#v", config)
}
if len(config.Endpoints) != 1 || config.Endpoints[0] != (endpoint{Host: "hive.example.com", Port: 10000}) {
t.Fatalf("resolved form endpoint did not override connection string: %#v", config.Endpoints)
}
}
func TestResolvedDirectEndpointAndDatabasePreserveJDBCParameterSections(t *testing.T) {
config, err := parseConnectionConfig(connectParams{
Host: "127.0.0.1",
Port: 18080,
Database: "analytics",
ConnectionString: "jdbc:hive2://old.example.com:10000/default;transportMode=http;httpPath=gateway?hive.exec.dynamic.partition=true#SourceTable=events",
URLParams: "transportMode=http;httpPath=gateway?hive.exec.dynamic.partition=true#SourceTable=events",
})
if err != nil {
t.Fatal(err)
}
if len(config.Endpoints) != 1 || config.Endpoints[0] != (endpoint{Host: "127.0.0.1", Port: 18080}) {
t.Fatalf("unexpected resolved endpoint: %#v", config.Endpoints)
}
if config.Database != "analytics" {
t.Fatalf("resolved database = %q, want analytics", config.Database)
}
if config.TransportMode != "http" || config.HTTPPath != "gateway" {
t.Fatalf("JDBC session parameters were not preserved: %#v", config)
}
if config.HiveConfiguration["set:hiveconf:hive.exec.dynamic.partition"] != "true" {
t.Fatalf("JDBC hiveConfs were not preserved: %#v", config.HiveConfiguration)
}
if config.HiveConfiguration["set:hivevar:SourceTable"] != "events" {
t.Fatalf("JDBC hiveVars were not preserved: %#v", config.HiveConfiguration)
}
}
func TestStructuredFieldsOverridePersistedJDBCValues(t *testing.T) {
config, err := parseConnectionConfig(connectParams{
Host: "127.0.0.1",
Port: 18080,
Username: "edited-user",
Password: "edited-password",
ConnectionString: "jdbc:hive2://old-user:old-password@old.example.com:10000/old_database;transportMode=http;auth=LDAP;ssl=true;serviceDiscoveryMode=zooKeeper?hive.exec.dynamic.partition=true#SourceTable=events",
URLParams: "user=url-user;password=url-password;ssl=true?hive.exec.dynamic.partition=true#SourceTable=events",
})
if err != nil {
t.Fatal(err)
}
if len(config.Endpoints) != 1 || config.Endpoints[0] != (endpoint{Host: "127.0.0.1", Port: 18080}) {
t.Fatalf("unexpected resolved endpoint: %#v", config.Endpoints)
}
if config.Database != defaultHiveDatabase {
t.Fatalf("database = %q, want %q", config.Database, defaultHiveDatabase)
}
if config.Username != "edited-user" || config.Password != "edited-password" {
t.Fatalf("structured credentials were overwritten: username=%q password=%q", config.Username, config.Password)
}
if config.TransportMode != "binary" || config.Auth != "NONE" || config.ServiceDiscoveryMode != "" {
t.Fatalf("removed JDBC parameters remained active: %#v", config)
}
if config.TLSConfig != nil {
t.Fatal("disabled structured SSL was re-enabled by persisted JDBC parameters")
}
if config.HiveConfiguration["set:hiveconf:hive.exec.dynamic.partition"] != "true" {
t.Fatalf("current hiveConfs were not preserved: %#v", config.HiveConfiguration)
}
if config.HiveConfiguration["set:hivevar:SourceTable"] != "events" {
t.Fatalf("current hiveVars were not preserved: %#v", config.HiveConfiguration)
}
}
func TestConnectionStringOnlyKeepsJDBCValues(t *testing.T) {
config, err := parseConnectionConfig(connectParams{
ConnectionString: "jdbc:hive2://raw-user:raw-password@hive.example.com:10001/analytics;transportMode=http;ssl=true",
})
if err != nil {
t.Fatal(err)
}
if config.Database != "analytics" || config.Username != "raw-user" || config.Password != "raw-password" {
t.Fatalf("connection-string-only fields were not preserved: %#v", config)
}
if config.TransportMode != "http" || config.TLSConfig == nil {
t.Fatalf("connection-string-only parameters were not preserved: %#v", config)
}
}
func TestZooKeeperDiscoveryKeepsAllJDBCEndpoints(t *testing.T) {
config, err := parseConnectionConfig(connectParams{
Host: "127.0.0.1",
Port: 12181,
Database: "analytics",
ConnectionString: "jdbc:hive2://zk1.example.com:2181,zk2.example.com:2181/default;serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2",
URLParams: "serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2",
})
if err != nil {
t.Fatal(err)
}
if len(config.Endpoints) != 2 || config.Endpoints[0].Host != "zk1.example.com" || config.Endpoints[1].Host != "zk2.example.com" {
t.Fatalf("ZooKeeper discovery endpoints were not preserved: %#v", config.Endpoints)
}
if config.Database != "analytics" {
t.Fatalf("resolved database = %q, want analytics", config.Database)
}
}
func TestImpalaDefaultsToNoSASL(t *testing.T) {
config, err := parseConnectionConfig(connectParams{
DatabaseType: "impala",
Host: "impala.example.com",
Port: 21050,
Database: "analytics",
})
if err != nil {
t.Fatal(err)
}
if config.Auth != "NOSASL" {
t.Fatalf("unexpected Impala auth mode: %q", config.Auth)
}
if config.Kerberos.Service != "impala" {
t.Fatalf("unexpected Impala Kerberos service: %q", config.Kerberos.Service)
}
}
func TestImpalaExplicitAuthenticationOverridesDefaults(t *testing.T) {
config, err := parseConnectionConfig(connectParams{
DatabaseType: "impala",
Host: "impala.example.com",
Port: 21050,
URLParams: "auth=NONE;service=custom",
})
if err != nil {
t.Fatal(err)
}
if config.Auth != "NONE" || config.Kerberos.Service != "custom" {
t.Fatalf("explicit Impala authentication was not preserved: %#v", config.Kerberos)
}
}
func TestImpalaLDAPHTTPSSLConfiguration(t *testing.T) {
config, err := parseConnectionConfig(connectParams{
DatabaseType: "impala",
Host: "impala.example.com",
Port: 21050,
Username: "alice",
Password: "secret",
URLParams: "auth=LDAP;transportMode=http;httpPath=cliservice",
SSL: true,
})
if err != nil {
t.Fatal(err)
}
if config.Auth != "LDAP" || config.TransportMode != "http" || config.HTTPPath != "cliservice" {
t.Fatalf("unexpected Impala LDAP transport config: %#v", config)
}
if config.Username != "alice" || config.Password != "secret" {
t.Fatalf("Impala LDAP credentials were not preserved: %q / %q", config.Username, config.Password)
}
if config.TLSConfig == nil || config.TLSConfig.ServerName != "impala.example.com" {
t.Fatalf("unexpected Impala LDAP TLS config: %#v", config.TLSConfig)
}
if config.Kerberos.Service != defaultImpalaService {
t.Fatalf("unexpected Impala service default: %q", config.Kerberos.Service)
}
}
func TestParseStandardJDBCURLSectionsAndCredentials(t *testing.T) {
config, err := parseConnectionConfig(connectParams{
ConnectionString: "jdbc:hive2://hs2.example.com:10001/analytics;user=alice;password=p%40ss?hive.server2.transport.mode=http;hive.server2.thrift.http.path=proxy;hive.exec.dynamic.partition=true#SourceTable=events",
URLParams: "user=bob?hive.exec.dynamic.partition=false#SourceTable=override",
})
if err != nil {
t.Fatal(err)
}
if config.Username != "bob" || config.Password != "p@ss" {
t.Fatalf("unexpected credentials: %q / %q", config.Username, config.Password)
}
if config.TransportMode != "http" || config.HTTPPath != "proxy" {
t.Fatalf("deprecated Hive conf transport settings were not applied: %#v", config)
}
want := map[string]string{
"set:hiveconf:hive.resultset.use.unique.column.names": "false",
"set:hiveconf:hive.exec.dynamic.partition": "false",
"set:hivevar:SourceTable": "override",
}
if !reflect.DeepEqual(config.HiveConfiguration, want) {
t.Fatalf("unexpected OpenSession configuration: %#v", config.HiveConfiguration)
}
}
func TestOpenSessionCompatibilityVariablesFromSessionParams(t *testing.T) {
config, err := parseConnectionConfig(connectParams{
Host: "hs2.example.com",
URLParams: "proxyUser=alice;hiveCreateAsExternalLegacy=TRUE;wmPool=etl;hiveconf:hive.exec.compress.output=true;hivevar:source=events",
})
if err != nil {
t.Fatal(err)
}
want := map[string]string{
"hive.server2.proxy.user": "alice",
"set:hiveconf:hive.create.as.external.legacy": "true",
"set:hiveconf:hive.exec.compress.output": "true",
"set:hiveconf:hive.resultset.use.unique.column.names": "false",
"set:hivevar:source": "events",
"set:hivevar:wmpool": "etl",
}
if !reflect.DeepEqual(config.HiveConfiguration, want) {
t.Fatalf("unexpected OpenSession compatibility variables: %#v", config.HiveConfiguration)
}
}
func TestOpenSessionUsesLeafResultLabelsUnlessExplicitlyOverridden(t *testing.T) {
tests := []struct {
name string
connection string
urlParams string
want string
}{
{name: "default", want: "false"},
{
name: "JDBC hiveconf override",
connection: "jdbc:hive2://hs2.example.com:10000/default?hive.resultset.use.unique.column.names=true",
want: "true",
},
{
name: "URL hiveconf override",
urlParams: "?HIVE.RESULTSET.USE.UNIQUE.COLUMN.NAMES=true",
want: "true",
},
{
name: "URL hiveconf wins over JDBC hiveconf",
connection: "jdbc:hive2://hs2.example.com:10000/default?hive.resultset.use.unique.column.names=true",
urlParams: "?hive.resultset.use.unique.column.names=false",
want: "false",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
params := connectParams{Host: "hs2.example.com", ConnectionString: test.connection, URLParams: test.urlParams}
if test.connection != "" {
params.Host = ""
}
config, err := parseConnectionConfig(params)
if err != nil {
t.Fatal(err)
}
const key = "set:hiveconf:hive.resultset.use.unique.column.names"
if got := config.HiveConfiguration[key]; got != test.want {
t.Fatalf("%s = %q, want %q; config=%#v", key, got, test.want, config.HiveConfiguration)
}
matches := 0
for candidate := range config.HiveConfiguration {
if strings.EqualFold(candidate, key) {
matches++
}
}
if matches != 1 {
t.Fatalf("result label hiveconf must appear exactly once: %#v", config.HiveConfiguration)
}
})
}
}
func TestHiveAssignmentMergeCanonicalizesOnlyResultLabelSetting(t *testing.T) {
first := map[string]string{
"CaseSensitive": "first",
"HIVE.RESULTSET.USE.UNIQUE.COLUMN.NAMES": "true",
}
second := map[string]string{
"casesensitive": "second",
"hive.resultset.use.unique.column.names": "false",
}
hiveConfs := mergeHiveConfAssignments(first, second)
if got := hiveConfs[resultSetUniqueColumnNames]; got != "false" {
t.Fatalf("result-label setting override = %q, want false; values=%#v", got, hiveConfs)
}
if got := len(hiveConfs); got != 3 {
t.Fatalf("unrelated case-distinct Hive confs were collapsed: %#v", hiveConfs)
}
hiveVars := mergeHiveAssignments(first, second)
if got := len(hiveVars); got != 4 {
t.Fatalf("case-distinct Hive variables were collapsed: %#v", hiveVars)
}
}
func TestParseHiveJDBCClientCompatibilityOptions(t *testing.T) {
directory := t.TempDir()
initPath := filepath.Join(directory, "hive-init.sql")
if err := os.WriteFile(initPath, []byte("# ignored\nSET hive.exec.dynamic.partition=true;\n-- ignored\nUSE analytics;\n"), 0o600); err != nil {
t.Fatal(err)
}
config, err := parseConnectionConfig(connectParams{
ConnectionString: "jdbc:hive2://hs2.example.com:10001/default;transportMode=http;auth=jwt;jwt=signed-token;fetchSize=77;socketTimeout=9;thrift.client.max.message.size=1048576;retries=3;retryInterval=250;requestTrack=true;cookieAuth=false;cookieName=CustomAuth;http.header.X-Trace-ID=trace-value;http.cookie.SessionID=cookie-value;applicationName=dbx-hive;initFile=" + url.QueryEscape(initPath),
})
if err != nil {
t.Fatal(err)
}
if config.Auth != "JWT" || config.JWT != "signed-token" {
t.Fatalf("unexpected JWT config: %#v", config)
}
if config.FetchSize != 77 || config.SocketTimeout != 9*time.Second || config.MaxMessageSize != 1048576 {
t.Fatalf("unexpected client sizing config: %#v", config)
}
if config.Retries != 3 || config.RetryInterval != 250*time.Millisecond {
t.Fatalf("unexpected retry config: %#v", config)
}
if config.CookieAuth || config.CookieName != "CustomAuth" {
t.Fatalf("unexpected cookie auth config: %#v", config)
}
if config.HTTPHeaders["X-Trace-ID"] != "trace-value" || config.HTTPCookies["SessionID"] != "cookie-value" {
t.Fatalf("HTTP header or cookie case was not preserved: %#v / %#v", config.HTTPHeaders, config.HTTPCookies)
}
if !config.RequestTracking {
t.Fatal("requestTrack was not enabled")
}
if config.HiveConfiguration["set:hivevar:wmapp"] != "dbx-hive" {
t.Fatalf("applicationName was not mapped: %#v", config.HiveConfiguration)
}
if !reflect.DeepEqual(config.InitStatements, []string{"SET hive.exec.dynamic.partition=true", "USE analytics"}) {
t.Fatalf("unexpected init statements: %#v", config.InitStatements)
}
}
func TestJWTCanComeFromEnvironment(t *testing.T) {
t.Setenv("JWT", "environment-token")
config, err := parseConnectionConfig(connectParams{
ConnectionString: "jdbc:hive2://hs2.example.com:10001/default;transportMode=http;auth=jwt",
})
if err != nil {
t.Fatal(err)
}
if config.JWT != "environment-token" {
t.Fatalf("unexpected JWT: %q", config.JWT)
}
}
func TestBrowserAuthSupportsInteractiveSSOParameters(t *testing.T) {
config, err := parseConnectionConfig(connectParams{
ConnectionString: "jdbc:hive2://hs2.example.com:10001/default;transportMode=http;auth=browser;browserResponsePort=18080;browserResponseTimeout=45;browserDisableSslCheck=true",
})
if err != nil {
t.Fatal(err)
}
if config.BrowserToken != "" || config.BrowserResponsePort != 18080 || config.BrowserResponseTimeout != 45*time.Second || !config.BrowserDisableSSLCheck {
t.Fatalf("unexpected browser SSO config: %#v", config)
}
}
func TestInvalidHiveJDBCClientSizesAreRejected(t *testing.T) {
for _, params := range []string{
"fetchSize=zero",
"socketTimeout=soon",
"thrift.client.max.message.size=huge",
} {
_, err := parseConnectionConfig(connectParams{
ConnectionString: "jdbc:hive2://hs2.example.com/default;" + params,
})
if err == nil {
t.Fatalf("expected %q to be rejected", params)
}
}
}
func TestJavaCredentialProviderPasswordPathIsNotSilentlyIgnored(t *testing.T) {
_, err := parseConnectionConfig(connectParams{
ConnectionString: "jdbc:hive2://hs2.example.com/default;ssl=true;sslTrustStore=/tmp/truststore.jks;storePasswordPath=jceks://file/tmp/hive.jceks",
})
if err == nil || !strings.Contains(err.Error(), "configure trustStorePassword explicitly") {
t.Fatalf("unexpected credential-provider error: %v", err)
}
}
func TestKerberosAcceptsSaslQopCompatibilityAlias(t *testing.T) {
configPath := filepath.Join(t.TempDir(), "krb5.conf")
if err := os.WriteFile(configPath, []byte("[libdefaults]\n default_realm = EXAMPLE.COM\n"), 0o600); err != nil {
t.Fatal(err)
}
config, err := parseConnectionConfig(connectParams{
ConnectionString: "jdbc:hive2://hs2.example.com/default;auth=kerberos;principal=hive/_HOST@EXAMPLE.COM;kerberosPrincipal=alice@EXAMPLE.COM;sasl.qop=auth-int",
Password: "secret",
AgentJavaOptions: []string{"-Djava.security.krb5.conf=" + configPath},
})
if err != nil {
t.Fatal(err)
}
if config.Kerberos.QOP != "auth-int" {
t.Fatalf("unexpected SASL QOP: %q", config.Kerberos.QOP)
}
}
func TestDelegationTokenProducesDigestCredentials(t *testing.T) {
identifier := []byte("token-identifier")
password := []byte("token-password")
token := encodeHadoopToken(identifier, password, []byte("HIVE_DELEGATION_TOKEN"), []byte("hs2.example.com:10000"))
config, err := parseConnectionConfig(connectParams{
ConnectionString: "jdbc:hive2://hs2.example.com/default;auth=delegationToken;delegationToken=" + token,
})
if err != nil {
t.Fatal(err)
}
if config.Auth != "DELEGATIONTOKEN" {
t.Fatalf("unexpected auth: %q", config.Auth)
}
if config.DelegationToken != token {
t.Fatalf("raw delegation token was not retained: %q", config.DelegationToken)
}
if config.Username != base64.StdEncoding.EncodeToString(identifier) || config.Password != base64.StdEncoding.EncodeToString(password) {
t.Fatalf("unexpected delegation token credentials: %q / %q", config.Username, config.Password)
}
}
func TestDelegationTokenRejectsMalformedValue(t *testing.T) {
_, err := parseConnectionConfig(connectParams{
ConnectionString: "jdbc:hive2://hs2.example.com/default;auth=delegationToken;delegationToken=not-a-token",
})
if err == nil || !strings.Contains(err.Error(), "decode Hive delegation token") {
t.Fatalf("expected delegation token error, got %v", err)
}
}
func TestParseIPv6Endpoint(t *testing.T) {
value, err := parseEndpoint("[2001:db8::1]:10000", defaultHivePort)
if err != nil {
t.Fatal(err)
}
if value.Host != "2001:db8::1" || value.Port != 10000 {
t.Fatalf("unexpected endpoint: %#v", value)
}
}
func TestKerberosSeparatesServerAndClientPrincipals(t *testing.T) {
configPath := filepath.Join(t.TempDir(), "krb5.conf")
if err := os.WriteFile(configPath, []byte("[libdefaults]\n default_realm = CLIENT.EXAMPLE.COM\n"), 0o600); err != nil {
t.Fatal(err)
}
config, err := parseConnectionConfig(connectParams{
ConnectionString: "jdbc:hive2://alias.example.com:10000/default;auth=kerberos;principal=hive/_HOST@SERVER.EXAMPLE.COM;kerberosPrincipal=alice@CLIENT.EXAMPLE.COM;kerberosEnableCanonicalHostnameCheck=false",
Password: "secret",
AgentJavaOptions: []string{`-Djava.security.krb5.conf="` + configPath + `"`},
})
if err != nil {
t.Fatal(err)
}
if config.Kerberos.ServerPrincipal != "hive/_HOST@SERVER.EXAMPLE.COM" {
t.Fatalf("unexpected server principal: %q", config.Kerberos.ServerPrincipal)
}
if config.Kerberos.ClientPrincipal != "alice@CLIENT.EXAMPLE.COM" {
t.Fatalf("unexpected client principal: %q", config.Kerberos.ClientPrincipal)
}
if config.Kerberos.Realm != "CLIENT.EXAMPLE.COM" || config.Kerberos.CanonicalHostname {
t.Fatalf("unexpected Kerberos options: %#v", config.Kerberos)
}
}
func TestKerberosReadsLegacyJAASKeytabOptions(t *testing.T) {
directory := t.TempDir()
configPath := filepath.Join(directory, "krb5.conf")
jaasPath := filepath.Join(directory, "hive jaas.conf")
keytabPath := filepath.Join(directory, "alice.keytab")
if err := os.WriteFile(configPath, []byte("[libdefaults]\n default_realm = EXAMPLE.COM\n"), 0o600); err != nil {
t.Fatal(err)
}
jaas := `HiveClient {
com.sun.security.auth.module.Krb5LoginModule required
useKeyTab=true
keyTab="` + keytabPath + `"
principal="alice@EXAMPLE.COM"
doNotPrompt=true;
};`
if err := os.WriteFile(jaasPath, []byte(jaas), 0o600); err != nil {
t.Fatal(err)
}
config, err := parseConnectionConfig(connectParams{
ConnectionString: "jdbc:hive2://hs2.example.com/default;auth=kerberos;principal=hive/_HOST@EXAMPLE.COM",
AgentJavaOptions: []string{
"-Djava.security.krb5.conf=" + configPath,
`-Djava.security.auth.login.config="` + jaasPath + `"`,
},
})
if err != nil {
t.Fatal(err)
}
if !config.Kerberos.UseKeytab || config.Kerberos.KeytabPath != keytabPath {
t.Fatalf("JAAS keytab was not preserved: %#v", config.Kerberos)
}
if config.Kerberos.ClientPrincipal != "alice@EXAMPLE.COM" {
t.Fatalf("JAAS principal was not preserved: %#v", config.Kerberos)
}
}
func TestBuildHiveTLSConfigFromPKCS12Stores(t *testing.T) {
privateKey, certificate, _ := testZooKeeperCertificate(t)
password := "changeit"
keyStore, err := pkcs12.Modern.Encode(privateKey, certificate, nil, password)
if err != nil {
t.Fatal(err)
}
trustStore, err := pkcs12.Modern.EncodeTrustStore([]*x509.Certificate{certificate}, password)
if err != nil {
t.Fatal(err)
}
directory := t.TempDir()
keyPath := filepath.Join(directory, "client.p12")
trustPath := filepath.Join(directory, "trust.p12")
if err := os.WriteFile(keyPath, keyStore, 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(trustPath, trustStore, 0o600); err != nil {
t.Fatal(err)
}
config, err := buildTLSConfig(connectParams{}, map[string]string{
"ssl": "true",
"twoway": "true",
"ssltruststore": trustPath,
"truststorepassword": password,
"truststoretype": "PKCS12",
"sslkeystore": keyPath,
"keystorepassword": password,
"keystoretype": "PKCS12",
}, "hs2.example.com")
if err != nil {
t.Fatal(err)
}
if config == nil || config.RootCAs == nil || len(config.Certificates) != 1 {
t.Fatalf("unexpected Hive TLS config: %#v", config)
}
}
func TestBuildHiveTwoWayTLSRequiresTrustAndKeyMaterial(t *testing.T) {
if _, err := buildTLSConfig(connectParams{}, map[string]string{
"ssl": "true",
"twoway": "true",
}, "hs2.example.com"); err == nil {
t.Fatal("expected missing two-way TLS material error")
}
}
+199
View File
@@ -0,0 +1,199 @@
package main
import (
"context"
"crypto/tls"
"database/sql"
"database/sql/driver"
"errors"
"fmt"
"strings"
"time"
"github.com/beltran/gosasl"
gohive "github.com/t8y2/dbx/agents/go-common/gohive"
)
type connectorFactory func(endpoint) driver.Connector
type discoveryConnector struct {
discovery endpointDiscovery
factory connectorFactory
driver driver.Driver
retries int
retryInterval time.Duration
}
func newDiscoveryConnector(config connectionConfig) *discoveryConnector {
return &discoveryConnector{
discovery: newEndpointDiscovery(config),
factory: func(target endpoint) driver.Connector {
tlsConfig := config.TLSConfig
if target.SSL && tlsConfig == nil && !config.TLSExplicit {
tlsConfig = &tls.Config{MinVersion: tls.VersionTLS12, ServerName: target.Host}
}
if tlsConfig != nil {
tlsConfig = tlsConfig.Clone()
if tlsConfig.ServerName == "" || tlsConfig.ServerName == config.Endpoints[0].Host {
tlsConfig.ServerName = target.Host
}
}
hiveConfiguration := make(map[string]string, len(config.HiveConfiguration)+1)
for key, value := range config.HiveConfiguration {
hiveConfiguration[key] = value
}
if config.Kerberos.Enabled {
hiveConfiguration["hive.server2.thrift.sasl.qop"] = config.Kerberos.QOP
}
transportMode := config.TransportMode
if target.TransportMode != "" && !config.TransportModeExplicit {
transportMode = target.TransportMode
}
httpPath := config.HTTPPath
if target.HTTPPath != "" && !config.HTTPPathExplicit {
httpPath = target.HTTPPath
}
auth := config.Auth
if target.Auth != "" && !config.AuthExplicit {
auth = target.Auth
}
service := kerberosServiceForEndpoint(config, target)
return gohive.NewConnector(gohive.Config{
Host: target.Host,
Port: target.Port,
Auth: normalizeHiveAuth(auth),
Username: config.Username,
Password: config.Password,
Database: config.Database,
TransportMode: transportMode,
HTTPPath: httpPath,
Service: service,
HTTPKerberosChannelBinding: config.Kerberos.ChannelBinding,
GSSAPIOptions: gssapiOptionsFromKerberos(config.Kerberos),
TLSConfig: tlsConfig,
HiveConfiguration: hiveConfiguration,
ConnectTimeout: config.ConnectTimeout,
SocketTimeout: config.SocketTimeout,
HTTPTimeout: config.SocketTimeout,
FetchSize: int64(config.FetchSize),
MaxMessageSize: config.MaxMessageSize,
HTTPHeaders: config.HTTPHeaders,
HTTPCookies: config.HTTPCookies,
RequestTracking: config.RequestTracking,
DisableCookieAuth: !config.CookieAuth,
CookieName: config.CookieName,
JWT: config.JWT,
DelegationToken: config.DelegationToken,
BrowserToken: config.BrowserToken,
BrowserClientID: config.BrowserClientID,
BrowserResponsePort: config.BrowserResponsePort,
BrowserResponseTimeout: config.BrowserResponseTimeout,
BrowserDisableSSLCheck: config.BrowserDisableSSLCheck,
WaitForNonQueryCompletion: waitsForNonQueryCompletion(config.DatabaseType),
})
},
driver: &gohive.Driver{},
retries: max(config.Retries, 1),
retryInterval: config.RetryInterval,
}
}
func waitsForNonQueryCompletion(databaseType string) bool {
return strings.EqualFold(databaseType, "impala") || strings.EqualFold(databaseType, "kyuubi")
}
func gssapiOptionsFromKerberos(config kerberosConfig) gosasl.GSSAPIOptions {
return gosasl.GSSAPIOptions{
ConfigPath: config.ConfigPath,
CCachePath: config.CCachePath,
KeytabPath: config.KeytabPath,
Principal: config.ClientPrincipal,
Password: config.Password,
QOP: config.QOP,
AuthorizationID: config.AuthorizationID,
ServiceHost: config.ServerName,
UseCCache: config.UseTicketCache,
UseKeytab: config.UseKeytab,
UseSSPI: config.UseSSPI,
CanonicalizeHost: config.CanonicalHostname,
DisablePAFXFAST: config.DisablePAFXFAST,
}
}
func kerberosServiceForEndpoint(config connectionConfig, target endpoint) string {
service := firstNonEmpty(config.Kerberos.ServerPrincipal, config.Kerberos.Service)
if target.Principal != "" && !config.Kerberos.ServerPrincipalExplicit {
service = target.Principal
}
return service
}
func (connector *discoveryConnector) Connect(ctx context.Context) (driver.Conn, error) {
var failures []string
for attempt := 0; attempt < max(connector.retries, 1); attempt++ {
rejected := map[string]bool{}
for {
endpoints, err := connector.discovery.Endpoints(ctx, rejected)
if err != nil {
failures = append(failures, fmt.Sprintf("discovery attempt %d: %v", attempt+1, err))
break
}
if len(endpoints) == 0 {
break
}
for _, target := range endpoints {
connection, connectErr := connector.factory(target).Connect(ctx)
if connectErr == nil {
return connection, nil
}
rejected[target.address()] = true
failures = append(failures, fmt.Sprintf("attempt %d %s: %v", attempt+1, target.address(), connectErr))
}
break
}
if attempt+1 < connector.retries && connector.retryInterval > 0 {
timer := time.NewTimer(connector.retryInterval)
select {
case <-ctx.Done():
timer.Stop()
return nil, ctx.Err()
case <-timer.C:
}
}
}
if len(failures) == 0 {
return nil, errors.New("Hive discovery returned no endpoints")
}
return nil, fmt.Errorf("all HiveServer2 endpoints failed: %s", strings.Join(failures, "; "))
}
func (connector *discoveryConnector) Driver() driver.Driver {
return connector.driver
}
func openHiveDatabase(config connectionConfig) *sql.DB {
database := sql.OpenDB(newDiscoveryConnector(config))
database.SetMaxOpenConns(1)
database.SetMaxIdleConns(1)
return database
}
func normalizeHiveAuth(value string) string {
normalized := strings.ToUpper(strings.TrimSpace(value))
switch normalized {
case "":
return "NONE"
case "NOSASL", "NO_SASL":
return "NOSASL"
case "KERBEROS", "GSSAPI":
return "KERBEROS"
case "LDAP":
return "LDAP"
case "CUSTOM":
return "CUSTOM"
case "DIGEST-MD5", "DELEGATIONTOKEN", "DELEGATION_TOKEN":
return "DIGEST-MD5"
default:
return normalized
}
}
+164
View File
@@ -0,0 +1,164 @@
package main
import (
"context"
"database/sql/driver"
"errors"
"io"
"reflect"
"testing"
)
type staticDiscovery struct {
values []endpoint
}
func (discovery staticDiscovery) Endpoints(_ context.Context, rejected map[string]bool) ([]endpoint, error) {
result := make([]endpoint, 0, len(discovery.values))
for _, value := range discovery.values {
if !rejected[value.address()] {
result = append(result, value)
}
}
return result, nil
}
type fakeConnector struct {
connection driver.Conn
err error
}
func (connector fakeConnector) Connect(context.Context) (driver.Conn, error) {
return connector.connection, connector.err
}
func (fakeConnector) Driver() driver.Driver { return fakeDriver{} }
type fakeDriver struct{}
func (fakeDriver) Open(string) (driver.Conn, error) { return &fakeConnection{}, nil }
type fakeConnection struct{}
func (*fakeConnection) Prepare(string) (driver.Stmt, error) { return nil, errors.New("unsupported") }
func (*fakeConnection) Close() error { return nil }
func (*fakeConnection) Begin() (driver.Tx, error) { return nil, errors.New("unsupported") }
type emptyRows struct{}
func (emptyRows) Columns() []string { return nil }
func (emptyRows) Close() error { return nil }
func (emptyRows) Next([]driver.Value) error { return io.EOF }
func TestDiscoveryConnectorFailsOver(t *testing.T) {
first := endpoint{Host: "first", Port: 10000}
second := endpoint{Host: "second", Port: 10000}
connected := &fakeConnection{}
var attempts []endpoint
connector := &discoveryConnector{
discovery: staticDiscovery{values: []endpoint{first, second}},
driver: fakeDriver{},
factory: func(value endpoint) driver.Connector {
attempts = append(attempts, value)
if value == first {
return fakeConnector{err: errors.New("unavailable")}
}
return fakeConnector{connection: connected}
},
}
value, err := connector.Connect(context.Background())
if err != nil {
t.Fatal(err)
}
if value != connected || !reflect.DeepEqual(attempts, []endpoint{first, second}) {
t.Fatalf("unexpected failover result: value=%#v attempts=%#v", value, attempts)
}
}
func TestDiscoveryConnectorRetriesAllEndpoints(t *testing.T) {
target := endpoint{Host: "hs2", Port: 10000}
connected := &fakeConnection{}
attempts := 0
connector := &discoveryConnector{
discovery: staticDiscovery{values: []endpoint{target}},
driver: fakeDriver{},
retries: 2,
factory: func(endpoint) driver.Connector {
attempts++
if attempts == 1 {
return fakeConnector{err: errors.New("temporarily unavailable")}
}
return fakeConnector{connection: connected}
},
}
value, err := connector.Connect(context.Background())
if err != nil {
t.Fatal(err)
}
if value != connected || attempts != 2 {
t.Fatalf("unexpected retry result: value=%#v attempts=%d", value, attempts)
}
}
func TestNormalizeHiveAuth(t *testing.T) {
for input, expected := range map[string]string{
"": "NONE",
"noSasl": "NOSASL",
"kerberos": "KERBEROS",
"delegationToken": "DIGEST-MD5",
"vendor-auth-mode": "VENDOR-AUTH-MODE",
} {
if actual := normalizeHiveAuth(input); actual != expected {
t.Fatalf("normalizeHiveAuth(%q) = %q, expected %q", input, actual, expected)
}
}
}
func TestWaitsForKyuubiAndImpalaNonQueryCompletion(t *testing.T) {
if !waitsForNonQueryCompletion("kyuubi") || !waitsForNonQueryCompletion("KYUUBI") || !waitsForNonQueryCompletion("impala") {
t.Fatal("Kyuubi and Impala must wait for asynchronous non-query operations")
}
if waitsForNonQueryCompletion("hive") {
t.Fatal("Hive must retain its existing non-query behavior")
}
}
func TestKerberosServiceForEndpoint(t *testing.T) {
target := endpoint{Host: "hs2.example.com", Port: 10000, Principal: "hive/hs2.example.com@EXAMPLE.COM"}
discovered := connectionConfig{Kerberos: kerberosConfig{Service: "hive"}}
if value := kerberosServiceForEndpoint(discovered, target); value != target.Principal {
t.Fatalf("discovered principal was ignored: %s", value)
}
explicit := connectionConfig{Kerberos: kerberosConfig{
Service: "hive",
ServerPrincipal: "hive/_HOST@USER.EXAMPLE.COM",
ServerPrincipalExplicit: true,
}}
if value := kerberosServiceForEndpoint(explicit, target); value != explicit.Kerberos.ServerPrincipal {
t.Fatalf("explicit principal was overwritten: %s", value)
}
}
func TestKerberosUsesConnectionScopedGSSAPIOptions(t *testing.T) {
config := kerberosConfig{
Enabled: true,
ServerPrincipal: "hive/_HOST@EXAMPLE.COM",
ClientPrincipal: "alice@EXAMPLE.COM",
ServerName: "canonical.example.com",
CanonicalHostname: true,
ConfigPath: "/etc/krb5.conf",
CCachePath: "/tmp/alice.ccache",
KeytabPath: "/tmp/alice.keytab",
Password: "secret",
AuthorizationID: "proxy-user",
QOP: "auth-conf",
UseTicketCache: true,
UseKeytab: true,
UseSSPI: true,
DisablePAFXFAST: true,
}
options := gssapiOptionsFromKerberos(config)
if options.ConfigPath != "/etc/krb5.conf" || options.Principal != "alice@EXAMPLE.COM" || options.Password != "secret" || options.AuthorizationID != "proxy-user" || options.ServiceHost != "canonical.example.com" || !options.CanonicalizeHost || options.CCachePath != "/tmp/alice.ccache" || options.KeytabPath != "/tmp/alice.keytab" || options.QOP != "auth-conf" || !options.UseCCache || !options.UseKeytab || !options.UseSSPI || !options.DisablePAFXFAST {
t.Fatalf("unexpected GSSAPI options: %#v", options)
}
}
+392
View File
@@ -0,0 +1,392 @@
package main
import (
"context"
"crypto/tls"
"encoding/json"
"errors"
"fmt"
"math/rand/v2"
"net"
"net/url"
"strconv"
"strings"
"time"
"github.com/go-zookeeper/zk"
)
type endpointDiscovery interface {
Endpoints(context.Context, map[string]bool) ([]endpoint, error)
}
type directDiscovery struct {
endpoints []endpoint
}
func (discovery directDiscovery) Endpoints(_ context.Context, rejected map[string]bool) ([]endpoint, error) {
return shuffledEndpoints(discovery.endpoints, rejected), nil
}
type zooKeeperDiscovery struct {
servers []endpoint
namespace string
discoveryMode string
authScheme string
auth string
timeout time.Duration
dialer func([]string, time.Duration) (zooKeeperClient, <-chan zk.Event, error)
}
type zooKeeperClient interface {
AddAuth(string, []byte) error
Children(string) ([]string, *zk.Stat, error)
Get(string) ([]byte, *zk.Stat, error)
Close()
}
func newEndpointDiscovery(config connectionConfig) endpointDiscovery {
if strings.EqualFold(config.ServiceDiscoveryMode, "zookeeper") || strings.EqualFold(config.ServiceDiscoveryMode, "zookeeperha") {
return &zooKeeperDiscovery{
servers: append([]endpoint(nil), config.Endpoints...),
namespace: config.ZooKeeperNamespace,
discoveryMode: config.ServiceDiscoveryMode,
authScheme: config.ZooKeeperAuthScheme,
auth: config.ZooKeeperAuth,
timeout: config.ConnectTimeout,
dialer: newZooKeeperDialer(config),
}
}
return directDiscovery{endpoints: append([]endpoint(nil), config.Endpoints...)}
}
func newZooKeeperDialer(config connectionConfig) func([]string, time.Duration) (zooKeeperClient, <-chan zk.Event, error) {
if config.ZooKeeperKerberos.Enabled {
return func(servers []string, timeout time.Duration) (zooKeeperClient, <-chan zk.Event, error) {
return connectKerberosZooKeeper(servers, timeout, config.ZooKeeperTLSConfig, config)
}
}
tlsConfig := config.ZooKeeperTLSConfig
if tlsConfig == nil {
return func(servers []string, timeout time.Duration) (zooKeeperClient, <-chan zk.Event, error) {
return zk.Connect(servers, timeout, zk.WithLogInfo(false))
}
}
return func(servers []string, timeout time.Duration) (zooKeeperClient, <-chan zk.Event, error) {
return zk.Connect(
servers,
timeout,
zk.WithLogInfo(false),
zk.WithDialer(func(network, address string, dialTimeout time.Duration) (net.Conn, error) {
config := tlsConfig.Clone()
if config.ServerName == "" {
host, _, err := net.SplitHostPort(address)
if err == nil {
config.ServerName = host
}
}
dialer := &net.Dialer{Timeout: dialTimeout}
return tls.DialWithDialer(dialer, network, address, config)
}),
)
}
}
func (discovery *zooKeeperDiscovery) Endpoints(ctx context.Context, rejected map[string]bool) ([]endpoint, error) {
addresses := make([]string, 0, len(discovery.servers))
for _, server := range discovery.servers {
addresses = append(addresses, server.address())
}
timeout := discovery.timeout
if timeout <= 0 {
timeout = defaultConnectTimeout
}
connection, events, err := discovery.dialer(addresses, timeout)
if err != nil {
return nil, fmt.Errorf("connect to ZooKeeper: %w", err)
}
defer connection.Close()
if err := waitForZooKeeperSession(ctx, events, timeout); err != nil {
return nil, err
}
if discovery.authScheme != "" || discovery.auth != "" {
if discovery.authScheme == "" || discovery.auth == "" {
return nil, errors.New("ZooKeeper auth scheme and credentials must be configured together")
}
if err := connection.AddAuth(discovery.authScheme, []byte(discovery.auth)); err != nil {
return nil, fmt.Errorf("authenticate to ZooKeeper: %w", err)
}
}
resolved := make([]endpoint, 0)
var listedPath string
var nodeFailures []string
for _, path := range discovery.paths() {
children, _, childrenErr := connection.Children(path)
if errors.Is(childrenErr, zk.ErrNoNode) {
continue
}
if childrenErr != nil {
return nil, fmt.Errorf("list ZooKeeper namespace %s: %w", path, childrenErr)
}
listedPath = path
for _, child := range children {
data, _, dataErr := connection.Get(path + "/" + child)
if dataErr != nil {
if errors.Is(dataErr, zk.ErrNoNode) {
continue
}
nodeFailures = append(nodeFailures, fmt.Sprintf("%s/%s: %v", path, child, dataErr))
continue
}
value, parseErr := parseHiveServerRegistration(child, data)
if parseErr == nil {
resolved = append(resolved, value)
} else {
nodeFailures = append(nodeFailures, fmt.Sprintf("%s/%s: %v", path, child, parseErr))
}
}
if len(resolved) > 0 {
break
}
}
resolved = shuffledEndpoints(uniqueEndpoints(resolved), rejected)
if len(resolved) == 0 {
if listedPath == "" {
return nil, fmt.Errorf("HiveServer2 ZooKeeper namespace not found; tried %s", strings.Join(discovery.paths(), ", "))
}
if len(nodeFailures) > 0 {
return nil, fmt.Errorf("no usable HiveServer2 nodes in ZooKeeper namespace %s: %s", listedPath, strings.Join(nodeFailures, "; "))
}
return nil, fmt.Errorf("no available HiveServer2 nodes in ZooKeeper namespace %s", listedPath)
}
return resolved, nil
}
func (discovery *zooKeeperDiscovery) paths() []string {
namespace := strings.Trim(discovery.namespace, "/")
if strings.EqualFold(discovery.discoveryMode, "zookeeperha") {
return []string{
zooKeeperPath(namespace, "instances"),
zooKeeperPath(namespace+"-unsecure", "instances"),
zooKeeperPath(namespace+"-sasl", "instances"),
}
}
return []string{zooKeeperPath(namespace)}
}
func zooKeeperPath(parts ...string) string {
cleaned := make([]string, 0, len(parts))
for _, part := range parts {
if value := strings.Trim(part, "/"); value != "" {
cleaned = append(cleaned, value)
}
}
if len(cleaned) == 0 {
return "/"
}
return "/" + strings.Join(cleaned, "/")
}
func waitForZooKeeperSession(ctx context.Context, events <-chan zk.Event, timeout time.Duration) error {
timer := time.NewTimer(timeout)
defer timer.Stop()
for {
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
return errors.New("ZooKeeper connection timed out before a session was established")
case event, ok := <-events:
if !ok {
return errors.New("ZooKeeper event stream closed before a session was established")
}
if event.Err != nil {
return fmt.Errorf("ZooKeeper connection event: %w", event.Err)
}
switch event.State {
case zk.StateHasSession:
return nil
case zk.StateAuthFailed:
return errors.New("ZooKeeper authentication failed")
case zk.StateExpired:
return errors.New("ZooKeeper session expired during connection")
}
}
}
}
func parseHiveServerRegistration(child string, data []byte) (endpoint, error) {
candidates := []string{strings.TrimSpace(string(data)), strings.TrimSpace(child)}
for _, candidate := range candidates {
if candidate == "" {
continue
}
if value, err := endpointFromRegistrationJSON(candidate); err == nil {
return value, nil
}
parameters := parseHiveParameters(candidate)
for _, key := range []string{"serveruri", "hiveserver2uri", "server_uri"} {
if raw := parameter(parameters, key); raw != "" {
return parseRegisteredEndpoint(raw)
}
}
if value, err := endpointFromPublishedHiveConfig(parameters); err == nil {
return value, nil
}
if strings.Contains(candidate, "=") {
continue
}
if value, err := parseRegisteredEndpoint(candidate); err == nil {
return value, nil
}
}
return endpoint{}, fmt.Errorf("unsupported HiveServer2 ZooKeeper registration %q", child)
}
func endpointFromRegistrationJSON(value string) (endpoint, error) {
var object map[string]any
if json.Unmarshal([]byte(value), &object) != nil {
return endpoint{}, errors.New("not JSON")
}
for _, key := range []string{"serverUri", "server_uri", "hiveServer2Uri", "uri"} {
if raw, ok := object[key].(string); ok && strings.TrimSpace(raw) != "" {
return parseRegisteredEndpoint(raw)
}
}
if serviceRecordEndpoint, ok := endpointFromServiceRecord(object); ok {
return serviceRecordEndpoint, nil
}
host, _ := object["host"].(string)
if host == "" {
host, _ = object["hostname"].(string)
}
port := 0
switch value := object["port"].(type) {
case float64:
port = int(value)
case string:
port, _ = strconv.Atoi(value)
}
if host != "" && port > 0 {
return endpoint{Host: host, Port: port}, nil
}
return endpoint{}, errors.New("JSON registration has no endpoint")
}
func endpointFromServiceRecord(object map[string]any) (endpoint, bool) {
internal, _ := object["internal"].([]any)
for _, rawEndpoint := range internal {
published, _ := rawEndpoint.(map[string]any)
if !strings.EqualFold(registrationStringValue(published["api"]), "activeEndpoint") {
continue
}
addresses, _ := published["addresses"].([]any)
for _, rawAddress := range addresses {
address, _ := rawAddress.(map[string]any)
host := registrationStringValue(address["host"])
port, _ := strconv.Atoi(registrationStringValue(address["port"]))
if host != "" && port > 0 {
result := endpoint{Host: host, Port: port}
applyPublishedHiveConfig(&result, object)
return result, true
}
}
}
return endpoint{}, false
}
func registrationStringValue(value any) string {
switch typed := value.(type) {
case string:
return strings.TrimSpace(typed)
case float64:
return strconv.FormatInt(int64(typed), 10)
case json.Number:
return typed.String()
default:
return ""
}
}
func applyPublishedHiveConfig(target *endpoint, parameters map[string]any) {
value := func(key string) string {
if nested, ok := parameters["attributes"].(map[string]any); ok {
if result := registrationStringValue(nested[key]); result != "" {
return result
}
}
return registrationStringValue(parameters[key])
}
target.TransportMode = strings.ToLower(value("hive.server2.transport.mode"))
target.HTTPPath = strings.TrimPrefix(value("hive.server2.thrift.http.path"), "/")
target.Auth = strings.ToUpper(value("hive.server2.authentication"))
target.Principal = value("hive.server2.authentication.kerberos.principal")
target.SSL = strings.EqualFold(value("hive.server2.use.ssl"), "true")
}
func endpointFromPublishedHiveConfig(parameters map[string]string) (endpoint, error) {
host := firstNonEmpty(
parameter(parameters, "hive.server2.thrift.bind.host"),
parameter(parameters, "host"),
)
transportMode := strings.ToLower(parameter(parameters, "hive.server2.transport.mode"))
portValue := parameter(parameters, "hive.server2.thrift.port")
if transportMode == "http" {
portValue = firstNonEmpty(parameter(parameters, "hive.server2.thrift.http.port"), portValue)
}
port, err := strconv.Atoi(portValue)
if host == "" || err != nil || port <= 0 {
return endpoint{}, errors.New("published HiveServer2 configuration has no valid host and port")
}
return endpoint{
Host: host,
Port: port,
TransportMode: transportMode,
HTTPPath: strings.TrimPrefix(parameter(parameters, "hive.server2.thrift.http.path"), "/"),
Auth: strings.ToUpper(parameter(parameters, "hive.server2.authentication")),
Principal: parameter(parameters, "hive.server2.authentication.kerberos.principal"),
SSL: parameterBool(parameters, "hive.server2.use.ssl"),
}, nil
}
func parseRegisteredEndpoint(value string) (endpoint, error) {
value = strings.TrimSpace(value)
if parsed, err := url.Parse(value); err == nil && parsed.Hostname() != "" {
port := defaultHivePort
if parsed.Port() != "" {
parsedPort, parseErr := strconv.Atoi(parsed.Port())
if parseErr != nil {
return endpoint{}, parseErr
}
port = parsedPort
}
return endpoint{Host: parsed.Hostname(), Port: port}, nil
}
return parseEndpoint(value, defaultHivePort)
}
func shuffledEndpoints(values []endpoint, rejected map[string]bool) []endpoint {
result := make([]endpoint, 0, len(values))
for _, value := range values {
if !rejected[value.address()] {
result = append(result, value)
}
}
rand.Shuffle(len(result), func(first, second int) {
result[first], result[second] = result[second], result[first]
})
return result
}
func uniqueEndpoints(values []endpoint) []endpoint {
seen := map[string]bool{}
result := make([]endpoint, 0, len(values))
for _, value := range values {
key := value.address()
if !seen[key] {
seen[key] = true
result = append(result, value)
}
}
return result
}
+221
View File
@@ -0,0 +1,221 @@
package main
import (
"context"
"errors"
"testing"
"time"
"github.com/go-zookeeper/zk"
)
func TestParseHiveServerRegistrationFromChildName(t *testing.T) {
value, err := parseHiveServerRegistration("serverUri=hs2.example.com:10000;version=4.0.1", nil)
if err != nil {
t.Fatal(err)
}
if value != (endpoint{Host: "hs2.example.com", Port: 10000}) {
t.Fatalf("unexpected endpoint: %#v", value)
}
}
func TestParseHiveServerRegistrationFromData(t *testing.T) {
value, err := parseHiveServerRegistration("instance-0001", []byte(`{"serverUri":"thrift://kyuubi.example.com:10009"}`))
if err != nil {
t.Fatal(err)
}
if value != (endpoint{Host: "kyuubi.example.com", Port: 10009}) {
t.Fatalf("unexpected endpoint: %#v", value)
}
}
func TestRejectedEndpointsAreExcluded(t *testing.T) {
values := shuffledEndpoints([]endpoint{{Host: "one", Port: 1}, {Host: "two", Port: 2}}, map[string]bool{"one:1": true})
if len(values) != 1 || values[0].Host != "two" {
t.Fatalf("unexpected endpoints: %#v", values)
}
}
func TestParsePublishedHiveServerConfiguration(t *testing.T) {
value, err := parseHiveServerRegistration("instance-1", []byte(
"hive.server2.thrift.bind.host=hs2.example.com;"+
"hive.server2.transport.mode=http;"+
"hive.server2.thrift.http.port=10001;"+
"hive.server2.thrift.http.path=cliservice;"+
"hive.server2.authentication=KERBEROS;"+
"hive.server2.authentication.kerberos.principal=hive/_HOST@EXAMPLE.COM;"+
"hive.server2.use.ssl=true",
))
if err != nil {
t.Fatal(err)
}
if value.Host != "hs2.example.com" || value.Port != 10001 || value.TransportMode != "http" || value.HTTPPath != "cliservice" || value.Auth != "KERBEROS" || !value.SSL {
t.Fatalf("unexpected endpoint: %#v", value)
}
}
func TestParseActivePassiveServiceRecord(t *testing.T) {
value, err := parseHiveServerRegistration("instance-0001", []byte(`{
"hive.server2.transport.mode":"binary",
"hive.server2.authentication":"KERBEROS",
"hive.server2.authentication.kerberos.principal":"hive/_HOST@EXAMPLE.COM",
"internal":[{
"api":"activeEndpoint",
"addresses":[{"host":"active.example.com","port":"10000"}]
}]
}`))
if err != nil {
t.Fatal(err)
}
if value.Host != "active.example.com" || value.Port != 10000 || value.Auth != "KERBEROS" || value.TransportMode != "binary" {
t.Fatalf("unexpected active endpoint: %#v", value)
}
}
func TestZooKeeperHAPaths(t *testing.T) {
discovery := &zooKeeperDiscovery{namespace: "hs2ActivePassiveHA", discoveryMode: "zooKeeperHA"}
want := []string{
"/hs2ActivePassiveHA/instances",
"/hs2ActivePassiveHA-unsecure/instances",
"/hs2ActivePassiveHA-sasl/instances",
}
got := discovery.paths()
if len(got) != len(want) {
t.Fatalf("unexpected paths: %#v", got)
}
for index := range want {
if got[index] != want[index] {
t.Fatalf("path %d = %q, want %q", index, got[index], want[index])
}
}
}
func TestZooKeeperPathNormalizesEmptyNamespace(t *testing.T) {
if value := zooKeeperPath(""); value != "/" {
t.Fatalf("unexpected root path: %q", value)
}
if value := zooKeeperPath("", "instances"); value != "/instances" {
t.Fatalf("unexpected instances path: %q", value)
}
}
func TestWaitForZooKeeperSession(t *testing.T) {
events := make(chan zk.Event, 2)
events <- zk.Event{State: zk.StateConnected}
events <- zk.Event{State: zk.StateHasSession}
if err := waitForZooKeeperSession(context.Background(), events, time.Second); err != nil {
t.Fatal(err)
}
}
func TestWaitForZooKeeperSessionRejectsAuthFailure(t *testing.T) {
events := make(chan zk.Event, 1)
events <- zk.Event{State: zk.StateAuthFailed}
if err := waitForZooKeeperSession(context.Background(), events, time.Second); err == nil {
t.Fatal("expected authentication failure")
}
}
type fakeZooKeeperClient struct {
children map[string][]string
data map[string][]byte
auth []byte
closed bool
}
func (client *fakeZooKeeperClient) AddAuth(_ string, auth []byte) error {
client.auth = append([]byte(nil), auth...)
return nil
}
func (client *fakeZooKeeperClient) Children(path string) ([]string, *zk.Stat, error) {
children, ok := client.children[path]
if !ok {
return nil, nil, zk.ErrNoNode
}
return append([]string(nil), children...), nil, nil
}
func (client *fakeZooKeeperClient) Get(path string) ([]byte, *zk.Stat, error) {
data, ok := client.data[path]
if !ok {
return nil, nil, zk.ErrNoNode
}
return append([]byte(nil), data...), nil, nil
}
func (client *fakeZooKeeperClient) Close() { client.closed = true }
func TestZooKeeperDiscoveryUsesSessionAuthAndSkipsStaleNodes(t *testing.T) {
client := &fakeZooKeeperClient{
children: map[string][]string{"/hiveserver2": {"stale", "live"}},
data: map[string][]byte{
"/hiveserver2/live": []byte("serverUri=live.example.com:10000"),
},
}
discovery := &zooKeeperDiscovery{
servers: []endpoint{{Host: "zk.example.com", Port: 2181}},
namespace: "hiveserver2",
authScheme: "digest",
auth: "user:password",
timeout: time.Second,
discoveryMode: "zookeeper",
dialer: func([]string, time.Duration) (zooKeeperClient, <-chan zk.Event, error) {
events := make(chan zk.Event, 1)
events <- zk.Event{State: zk.StateHasSession}
return client, events, nil
},
}
values, err := discovery.Endpoints(context.Background(), map[string]bool{})
if err != nil {
t.Fatal(err)
}
if len(values) != 1 || values[0].Host != "live.example.com" {
t.Fatalf("unexpected endpoints: %#v", values)
}
if string(client.auth) != "user:password" || !client.closed {
t.Fatalf("auth=%q closed=%v", client.auth, client.closed)
}
}
func TestZooKeeperHAFallsThroughStaleCandidatePath(t *testing.T) {
client := &fakeZooKeeperClient{
children: map[string][]string{
"/hs2ActivePassiveHA/instances": {"stale"},
"/hs2ActivePassiveHA-unsecure/instances": {"live"},
},
data: map[string][]byte{
"/hs2ActivePassiveHA-unsecure/instances/live": []byte("serverUri=active.example.com:10000"),
},
}
discovery := &zooKeeperDiscovery{
servers: []endpoint{{Host: "zk.example.com", Port: 2181}},
namespace: "hs2ActivePassiveHA",
timeout: time.Second,
discoveryMode: "zookeeperHA",
dialer: func([]string, time.Duration) (zooKeeperClient, <-chan zk.Event, error) {
events := make(chan zk.Event, 1)
events <- zk.Event{State: zk.StateHasSession}
return client, events, nil
},
}
values, err := discovery.Endpoints(context.Background(), map[string]bool{})
if err != nil {
t.Fatal(err)
}
if len(values) != 1 || values[0].Host != "active.example.com" {
t.Fatalf("unexpected HA endpoints: %#v", values)
}
}
func TestZooKeeperDiscoveryPropagatesDialError(t *testing.T) {
discovery := &zooKeeperDiscovery{
servers: []endpoint{{Host: "zk.example.com", Port: 2181}},
dialer: func([]string, time.Duration) (zooKeeperClient, <-chan zk.Event, error) {
return nil, nil, errors.New("dial failed")
},
}
if _, err := discovery.Endpoints(context.Background(), nil); err == nil {
t.Fatal("expected dial error")
}
}
+35
View File
@@ -0,0 +1,35 @@
module github.com/t8y2/dbx/agents/drivers/argo-go
go 1.23.0
require (
github.com/beltran/gosasl v1.0.0
github.com/go-zookeeper/zk v1.0.4
github.com/golang-auth/go-gssapi/v2 v2.0.0
github.com/jcmturner/krb5test v0.0.0-20201230140143-102e4b78cdb8
github.com/pavlo-v-chernykh/keystore-go/v4 v4.5.0
github.com/t8y2/dbx/agents/go-common/gohive v0.0.0
software.sslmate.com/src/go-pkcs12 v0.7.3
)
require (
github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e // indirect
github.com/apache/thrift v0.22.0 // indirect
github.com/beltran/gohive/v2 v2.1.0 // indirect
github.com/hashicorp/go-uuid v1.0.2 // indirect
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
github.com/jcmturner/gofork v1.0.0 // indirect
github.com/jcmturner/gokrb5 v8.4.2+incompatible // indirect
github.com/jcmturner/gokrb5/v8 v8.4.2 // indirect
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
github.com/pkg/errors v0.9.1 // indirect
golang.org/x/crypto v0.39.0 // indirect
golang.org/x/net v0.41.0 // indirect
)
replace github.com/beltran/gosasl => ../../go-common/gosasl
replace github.com/golang-auth/go-gssapi/v2 => ../../go-common/go-gssapi
replace github.com/t8y2/dbx/agents/go-common/gohive => ../../go-common/gohive
+68
View File
@@ -0,0 +1,68 @@
github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e h1:4dAU9FXIyQktpoUAgOJK3OTFc/xug0PCXYCqU0FgDKI=
github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
github.com/apache/thrift v0.22.0 h1:r7mTJdj51TMDe6RtcmNdQxgn9XcyfGDOzegMDRg47uc=
github.com/apache/thrift v0.22.0/go.mod h1:1e7J/O1Ae6ZQMTYdy9xa3w9k+XHWPfRvdPyJeynQ+/g=
github.com/beltran/gohive/v2 v2.1.0 h1:+XPfODaLXsZkD8mEnKCYqinC4OmVMx4/oR4ys8IGKeI=
github.com/beltran/gohive/v2 v2.1.0/go.mod h1:MrxyI0sdriG52I+Bo2lJpGR75bxjcI/Hx3+MBnYqR10=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-zookeeper/zk v1.0.4 h1:DPzxraQx7OrPyXq2phlGlNSIyWEsAox0RJmjTseMV6I=
github.com/go-zookeeper/zk v1.0.4/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
github.com/gorilla/sessions v1.2.0/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE=
github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=
github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM=
github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8=
github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o=
github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg=
github.com/jcmturner/gokrb5 v8.4.2+incompatible h1:MQW70Fbazv31g6URAXCjO2bGenIL0wVt3wqcpc0EjHI=
github.com/jcmturner/gokrb5 v8.4.2+incompatible/go.mod h1:0Q5eFyVvYsEsZ8xl1A/jUqhXvxUp/X9ELrJm+zieq5E=
github.com/jcmturner/gokrb5/v8 v8.4.0/go.mod h1:T1hnNppQsBtxW0tCHMHTkAt8n/sABdzZgZdoFrZaZNM=
github.com/jcmturner/gokrb5/v8 v8.4.2 h1:6ZIM6b/JJN0X8UM43ZOM6Z4SJzla+a/u7scXFJzodkA=
github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aWNYyvBVK62bc=
github.com/jcmturner/krb5test v0.0.0-20201230140143-102e4b78cdb8 h1:bWgHpg2hLK3m8zHGSLqa+UV8XkivFFjG5IniW5lWzIA=
github.com/jcmturner/krb5test v0.0.0-20201230140143-102e4b78cdb8/go.mod h1:eUQyrWMo2eMPWPezAYjLLwaiQHzJUD3su+1vAL10dOk=
github.com/jcmturner/rpc/v2 v2.0.2/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
github.com/pavlo-v-chernykh/keystore-go/v4 v4.5.0 h1:2nosf3P75OZv2/ZO/9Px5ZgZ5gbKrzA3joN1QMfOGMQ=
github.com/pavlo-v-chernykh/keystore-go/v4 v4.5.0/go.mod h1:lAVhWwbNaveeJmxrxuSTxMgKpF6DjnuVpn6T8WiBwYQ=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210326220855-61e056675ecf/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k=
golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
software.sslmate.com/src/go-pkcs12 v0.7.3 h1:JBQD3FDqYjTeyDAeZQklj2ar88ykBLtALloPJHyAauU=
software.sslmate.com/src/go-pkcs12 v0.7.3/go.mod h1:Qiz0EyvDRJjjxGyUQa2cCNZn/wMyzrRJ/qcDXOQazLI=
+65
View File
@@ -0,0 +1,65 @@
package main
import (
"context"
"database/sql/driver"
"errors"
"reflect"
"strings"
"testing"
"github.com/t8y2/dbx/agents/go-common/gohive"
)
func TestRunHiveInitStatementsExecutesAndDrainsResults(t *testing.T) {
var resultRows *scriptedRows
behavior := &scriptedBehavior{}
behavior.query = func(ctx context.Context, query string) (driver.Rows, error) {
switch query {
case "SET hive.exec.dynamic.partition=true":
resultRows = newScriptedRows(ctx, []string{"set"}, []string{"STRING"}, [][]driver.Value{
{"hive.exec.dynamic.partition=true"},
})
return resultRows, nil
case "USE analytics":
return nil, &gohive.NonQueryResult{}
default:
return nil, errors.New("unexpected init statement")
}
}
server := newScriptedServer(t, behavior)
err := runHiveInitStatements(
context.Background(),
server.connection,
[]string{"SET hive.exec.dynamic.partition=true", "USE analytics"},
77,
)
if err != nil {
t.Fatal(err)
}
queries, executions, _, _ := behavior.snapshot()
if !reflect.DeepEqual(queries, []string{"SET hive.exec.dynamic.partition=true", "USE analytics"}) || len(executions) != 0 {
t.Fatalf("unexpected init statements: queries=%v executions=%v", queries, executions)
}
if resultRows == nil || !resultRows.isClosed() {
t.Fatal("initFile result set was not drained and closed")
}
}
func TestRunHiveInitStatementsStopsOnFailure(t *testing.T) {
behavior := &scriptedBehavior{}
behavior.query = func(context.Context, string) (driver.Rows, error) {
return nil, errors.New("permission denied")
}
server := newScriptedServer(t, behavior)
err := runHiveInitStatements(context.Background(), server.connection, []string{"USE restricted", "USE skipped"}, 100)
if err == nil || !strings.Contains(err.Error(), "execute Hive initFile statement: permission denied") {
t.Fatalf("unexpected initFile error: %v", err)
}
queries, _, _, _ := behavior.snapshot()
if !reflect.DeepEqual(queries, []string{"USE restricted"}) {
t.Fatalf("initFile continued after failure: %v", queries)
}
}
@@ -0,0 +1,16 @@
//go:build !windows
package main
import (
"fmt"
"os"
)
func defaultKerberosConfigPath() string {
return "/etc/krb5.conf"
}
func defaultKerberosCCachePath() string {
return fmt.Sprintf("/tmp/krb5cc_%d", os.Getuid())
}
@@ -0,0 +1,19 @@
//go:build windows
package main
import (
"os"
"path/filepath"
)
func defaultKerberosConfigPath() string {
if windowsDirectory := os.Getenv("WINDIR"); windowsDirectory != "" {
return filepath.Join(windowsDirectory, "krb5.ini")
}
return `C:\Windows\krb5.ini`
}
func defaultKerberosCCachePath() string {
return ""
}
+620
View File
@@ -0,0 +1,620 @@
package main
import (
"bufio"
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"os"
"runtime"
"strconv"
"strings"
"sync"
"time"
_ "time/tzdata"
)
const (
protocolVersion = 2
defaultMaxRows = 10000
defaultPageSize = 1000
defaultFetchSize = 50
legacyAgentSessionID = "__legacy__"
maxAgentSessions = 256
querySessionIdleTime = 10 * time.Minute
)
type request struct {
ID json.RawMessage `json:"id"`
Method string `json:"method"`
Params map[string]json.RawMessage `json:"params"`
}
type response struct {
JSONRPC string `json:"jsonrpc,omitempty"`
ID json.RawMessage `json:"id,omitempty"`
Result any `json:"result,omitempty"`
Error *rpcError `json:"error,omitempty"`
}
type queryOptions struct {
SQL string `json:"sql"`
Database string `json:"database"`
Schema string `json:"schema"`
MaxRows int `json:"maxRows"`
FetchSize int `json:"fetchSize"`
TimeoutSecs int `json:"timeoutSecs"`
}
type queryResult struct {
Columns []string `json:"columns"`
ColumnTypes []string `json:"column_types"`
Rows [][]any `json:"rows"`
AffectedRows int64 `json:"affected_rows"`
ExecutionTimeMS int64 `json:"execution_time_ms"`
Truncated bool `json:"truncated"`
}
type queryPageResult struct {
Columns []string `json:"columns"`
ColumnTypes []string `json:"column_types"`
Rows [][]any `json:"rows"`
AffectedRows int64 `json:"affected_rows"`
ExecutionTimeMS int64 `json:"execution_time_ms"`
Truncated bool `json:"truncated"`
SessionID *string `json:"session_id"`
HasMore bool `json:"has_more"`
}
type querySession struct {
rows *sql.Rows
columns []string
columnTypes []string
pending []any
remaining int
cancel context.CancelFunc
lastAccessed time.Time
}
type server struct {
params connectParams
config connectionConfig
connectionMu sync.Mutex
database *sql.DB
connection *sql.Conn
querySessions map[string]*querySession
nextSessionID uint64
activeMu sync.Mutex
activeCancel context.CancelFunc
}
type agentSession struct {
server *server
mu sync.Mutex
}
type runtimeServer struct {
mu sync.RWMutex
sessions map[string]*agentSession
}
func main() {
configureRuntimeParallelism()
runtimeServer := newRuntimeServer()
encoder := json.NewEncoder(os.Stdout)
var encoderMu sync.Mutex
var requests sync.WaitGroup
fmt.Fprintln(os.Stdout, `{"ready":true}`)
scanner := bufio.NewScanner(os.Stdin)
scanner.Buffer(make([]byte, 0, 64*1024), 512*1024*1024)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
}
var envelope request
if json.Unmarshal([]byte(line), &envelope) == nil && envelope.Method == "shutdown" {
requests.Wait()
result, _ := runtimeServer.handleLine(line)
encoderMu.Lock()
_ = encoder.Encode(result)
encoderMu.Unlock()
return
}
requests.Add(1)
go func(line string) {
defer requests.Done()
result, _ := runtimeServer.handleLine(line)
encoderMu.Lock()
defer encoderMu.Unlock()
if err := encoder.Encode(result); err != nil {
fmt.Fprintf(os.Stderr, "failed to write response: %v\n", err)
}
}(line)
}
requests.Wait()
}
func configureRuntimeParallelism() {
if raw := strings.TrimSpace(os.Getenv("DBX_AGENT_HIVE_GOMAXPROCS")); raw != "" {
if configured, err := strconv.Atoi(raw); err == nil && configured > 0 {
runtime.GOMAXPROCS(configured)
return
}
}
if strings.TrimSpace(os.Getenv("GOMAXPROCS")) == "" {
runtime.GOMAXPROCS(min(runtime.NumCPU(), 4))
}
}
func newRuntimeServer() *runtimeServer {
return &runtimeServer{sessions: map[string]*agentSession{}}
}
func (runtimeServer *runtimeServer) handleLine(line string) (response, bool) {
var request request
if err := json.Unmarshal([]byte(line), &request); err != nil {
return errorResponse(nil, "", "", err), false
}
if len(request.ID) == 0 {
request.ID = json.RawMessage("1")
}
result, shutdown, err := runtimeServer.dispatch(request.Method, request.Params)
if err != nil {
return errorResponse(request.ID, request.Method, stringParam(request.Params, "agentSessionId"), err), false
}
return response{JSONRPC: "2.0", ID: request.ID, Result: result}, shutdown
}
func (runtimeServer *runtimeServer) dispatch(method string, params map[string]json.RawMessage) (any, bool, error) {
switch method {
case "handshake":
return handshakeResult(true), false, nil
case "open_session":
id := stringParam(params, "agentSessionId")
if id == "" {
return nil, false, errors.New("agentSessionId is required")
}
var connection connectParams
if err := decodeParams(params, &connection); err != nil {
return nil, false, err
}
return map[string]bool{"ok": true}, false, runtimeServer.openSession(id, connection)
case "close_session":
return map[string]bool{"ok": true}, false, runtimeServer.closeSession(stringParam(params, "agentSessionId"))
case "validate_session":
session, err := runtimeServer.session(stringParam(params, "agentSessionId"))
if err != nil {
return nil, false, err
}
session.mu.Lock()
defer session.mu.Unlock()
return map[string]bool{"ok": true}, false, session.server.validateConnection()
case "cancel_session":
session, err := runtimeServer.session(stringParam(params, "agentSessionId"))
if err != nil {
return nil, false, err
}
session.server.cancelActiveQuery()
return map[string]bool{"ok": true}, false, nil
case "test_connection":
var connection connectParams
if err := decodeParams(params, &connection); err != nil {
return nil, false, err
}
result, err := testConnection(connection)
return result, false, err
case "connect":
var connection connectParams
if err := decodeParams(params, &connection); err != nil {
return nil, false, err
}
_ = runtimeServer.closeSession(legacyAgentSessionID)
return map[string]bool{"ok": true}, false, runtimeServer.openSession(legacyAgentSessionID, connection)
case "disconnect":
return map[string]bool{"ok": true}, false, runtimeServer.closeSession(legacyAgentSessionID)
case "shutdown":
return map[string]bool{"ok": true}, true, runtimeServer.closeAllSessions()
}
sessionID := stringParam(params, "agentSessionId")
if sessionID == "" {
sessionID = legacyAgentSessionID
}
session, err := runtimeServer.session(sessionID)
if err != nil {
return nil, false, err
}
session.mu.Lock()
defer session.mu.Unlock()
return session.server.dispatch(method, params)
}
func (runtimeServer *runtimeServer) openSession(id string, params connectParams) error {
runtimeServer.mu.Lock()
if len(runtimeServer.sessions) >= maxAgentSessions {
runtimeServer.mu.Unlock()
return fmt.Errorf("maximum Hive Agent sessions reached (%d)", maxAgentSessions)
}
if _, exists := runtimeServer.sessions[id]; exists {
runtimeServer.mu.Unlock()
return fmt.Errorf("Hive Agent session already exists: %s", id)
}
runtimeServer.mu.Unlock()
server, err := newServer(params)
if err != nil {
return err
}
runtimeServer.mu.Lock()
defer runtimeServer.mu.Unlock()
if _, exists := runtimeServer.sessions[id]; exists {
_ = server.disconnect()
return fmt.Errorf("Hive Agent session already exists: %s", id)
}
if len(runtimeServer.sessions) >= maxAgentSessions {
_ = server.disconnect()
return fmt.Errorf("maximum Hive Agent sessions reached (%d)", maxAgentSessions)
}
runtimeServer.sessions[id] = &agentSession{server: server}
return nil
}
func (runtimeServer *runtimeServer) session(id string) (*agentSession, error) {
runtimeServer.mu.RLock()
session := runtimeServer.sessions[id]
runtimeServer.mu.RUnlock()
if session == nil {
return nil, fmt.Errorf("Hive Agent session not found: %s", id)
}
return session, nil
}
func (runtimeServer *runtimeServer) closeSession(id string) error {
if id == "" {
id = legacyAgentSessionID
}
runtimeServer.mu.Lock()
session := runtimeServer.sessions[id]
delete(runtimeServer.sessions, id)
runtimeServer.mu.Unlock()
if session == nil {
return nil
}
session.mu.Lock()
defer session.mu.Unlock()
return session.server.disconnect()
}
func (runtimeServer *runtimeServer) closeAllSessions() error {
runtimeServer.mu.Lock()
sessions := runtimeServer.sessions
runtimeServer.sessions = map[string]*agentSession{}
runtimeServer.mu.Unlock()
var failures []string
for id, session := range sessions {
session.mu.Lock()
err := session.server.disconnect()
session.mu.Unlock()
if err != nil {
failures = append(failures, fmt.Sprintf("%s: %v", id, err))
}
}
if len(failures) > 0 {
return fmt.Errorf("close Hive Agent sessions: %s", strings.Join(failures, "; "))
}
return nil
}
func newServer(params connectParams) (*server, error) {
config, err := parseConnectionConfig(params)
if err != nil {
return nil, err
}
server := &server{
params: params,
config: config,
querySessions: map[string]*querySession{},
}
if err := server.openConnection(); err != nil {
return nil, err
}
return server, nil
}
func (server *server) openConnection() error {
server.connectionMu.Lock()
defer server.connectionMu.Unlock()
if server.connection != nil {
return nil
}
database := openHiveDatabase(server.config)
ctx, cancel := context.WithTimeout(context.Background(), server.connectionOpenTimeout())
var connection *sql.Conn
connection, err := database.Conn(ctx)
if err == nil {
err = connection.PingContext(ctx)
}
cancel()
if err == nil {
err = runHiveInitStatements(context.Background(), connection, server.config.InitStatements, server.config.FetchSize)
}
if err != nil {
if connection != nil {
_ = connection.Close()
}
_ = database.Close()
return err
}
server.database = database
server.connection = connection
return nil
}
func (server *server) connectionOpenTimeout() time.Duration {
timeout := server.config.ConnectTimeout
if strings.EqualFold(server.config.Auth, "BROWSER") && strings.TrimSpace(server.config.BrowserToken) == "" {
timeout += server.config.BrowserResponseTimeout
}
return timeout
}
func runHiveInitStatements(ctx context.Context, connection *sql.Conn, statements []string, fetchSize int) error {
for _, statement := range statements {
rows, _, hasResultSet, err := executeHiveStatement(ctx, connection, statement, fetchSize)
if err != nil {
return fmt.Errorf("execute Hive initFile statement: %w", err)
}
if !hasResultSet {
continue
}
for rows.Next() {
}
if rowsErr := rows.Err(); rowsErr != nil {
_ = rows.Close()
return fmt.Errorf("read Hive initFile statement result: %w", rowsErr)
}
if closeErr := rows.Close(); closeErr != nil {
return fmt.Errorf("close Hive initFile statement result: %w", closeErr)
}
}
return nil
}
func (server *server) dispatch(method string, params map[string]json.RawMessage) (any, bool, error) {
switch method {
case "validate_connection":
return map[string]bool{"ok": true}, false, server.validateConnection()
case "connection_info":
result, err := server.connectionInfo()
return result, false, err
case "list_databases":
result, err := server.listDatabases()
return result, false, err
case "list_schemas":
result, err := server.listSchemas(stringSliceParam(params, "visible_schemas"))
return result, false, err
case "list_tables":
result, err := server.listTables(stringParam(params, "schema"), metadataListConstraintsFromParams(params))
return result, false, err
case "get_table_comment":
result, err := server.getTableComment(stringParam(params, "schema"), stringParam(params, "table"))
return result, false, err
case "list_objects":
result, err := server.listObjects(
stringParam(params, "database"),
stringParam(params, "schema"),
metadataListConstraintsFromParams(params),
)
return result, false, err
case "list_data_types":
result, err := server.listDataTypes()
return result, false, err
case "completion_assistant_search_v1":
var input completionAssistantRequest
if err := decodeParams(params, &input); err != nil {
return nil, false, err
}
result, err := server.completionAssistantSearch(input)
return result, false, err
case "get_columns":
result, err := server.getColumns(stringParam(params, "schema"), stringParam(params, "table"))
return result, false, err
case "list_indexes":
return []indexInfo{}, false, nil
case "list_foreign_keys":
return []foreignKeyInfo{}, false, nil
case "list_triggers":
return []triggerInfo{}, false, nil
case "list_constraints", "list_partitions", "list_subpartitions":
return []any{}, false, nil
case "get_object_source":
result, err := server.getObjectSource(
stringParam(params, "database"),
stringParam(params, "schema"),
firstNonEmpty(stringParam(params, "name"), stringParam(params, "table")),
stringParam(params, "object_type"),
)
return result, false, err
case "get_table_ddl":
result, err := server.getTableDDL(stringParam(params, "schema"), stringParam(params, "table"))
return result, false, err
case "get_explain_info":
result, err := server.getExplainInfo(stringParam(params, "sql"))
return map[string]any{"plan": result, "has_actual_stats": false}, false, err
case "execute_query":
result, err := server.executeQuery(queryOptionsFromParams(params))
return result, false, err
case "execute_query_page", "start_table_read":
result, err := server.executeQueryPage(queryOptionsFromParams(params), intParam(params, "pageSize"))
return result, false, err
case "fetch_query_page", "fetch_table_read_page":
result, err := server.fetchQueryPage(stringParam(params, "sessionId"), intParam(params, "pageSize"))
return result, false, err
case "close_query_session", "close_table_read_session":
return server.closeQuerySession(stringParam(params, "sessionId")), false, nil
case "execute_transaction":
result, err := server.executeStatements(params, true)
return result, false, err
case "execute_batch":
result, err := server.executeStatements(params, false)
return result, false, err
case "disconnect":
return map[string]bool{"ok": true}, false, server.disconnect()
case "shutdown":
return map[string]bool{"ok": true}, true, server.disconnect()
default:
return nil, false, fmt.Errorf("unknown method: %s", method)
}
}
func handshakeResult(multiSession bool) map[string]any {
capabilities := []string{
"connect", "test_connection", "metadata", "query", "paged_query", "transaction", "ddl", "structured_error_v1",
}
if multiSession {
capabilities = append(capabilities, "multi_session")
}
return map[string]any{
"protocolVersion": protocolVersion,
"agentProtocolVersion": protocolVersion,
"capabilities": capabilities,
}
}
func testConnection(params connectParams) (map[string]any, error) {
server, err := newServer(params)
if err != nil {
return nil, err
}
defer server.disconnect()
if err := server.validateConnection(); err != nil {
return nil, err
}
info, err := server.connectionInfo()
if err != nil {
return nil, err
}
return map[string]any{"ok": true, "info": info}, nil
}
func (server *server) disconnect() error {
server.cancelActiveQuery()
if err := server.closeAllQuerySessions(); err != nil {
return err
}
server.connectionMu.Lock()
connection := server.connection
database := server.database
server.connection = nil
server.database = nil
server.connectionMu.Unlock()
var failures []string
if connection != nil {
if err := connection.Close(); err != nil {
failures = append(failures, err.Error())
}
}
if database != nil {
if err := database.Close(); err != nil {
failures = append(failures, err.Error())
}
}
if len(failures) > 0 {
return errors.New(strings.Join(failures, "; "))
}
return nil
}
func (server *server) requireConnection() (*sql.Conn, error) {
server.connectionMu.Lock()
connection := server.connection
server.connectionMu.Unlock()
if connection == nil {
return nil, errors.New("Hive connection is not open")
}
return connection, nil
}
func (server *server) setActiveOperation(cancel context.CancelFunc) {
server.activeMu.Lock()
server.activeCancel = cancel
server.activeMu.Unlock()
}
func (server *server) clearActiveOperation(cancel context.CancelFunc) {
cancel()
server.activeMu.Lock()
server.activeCancel = nil
server.activeMu.Unlock()
}
func (server *server) cancelActiveQuery() {
server.activeMu.Lock()
cancel := server.activeCancel
server.activeMu.Unlock()
if cancel != nil {
cancel()
}
}
func queryOptionsFromParams(params map[string]json.RawMessage) queryOptions {
return queryOptions{
SQL: stringParam(params, "sql"),
Database: stringParam(params, "database"),
Schema: stringParam(params, "schema"),
MaxRows: intParam(params, "maxRows"),
FetchSize: intParam(params, "fetchSize"),
TimeoutSecs: intParam(params, "timeoutSecs"),
}
}
func decodeParams(params map[string]json.RawMessage, target any) error {
data, err := json.Marshal(params)
if err != nil {
return err
}
return json.Unmarshal(data, target)
}
func stringParam(params map[string]json.RawMessage, key string) string {
if raw, ok := params[key]; ok {
var value string
if json.Unmarshal(raw, &value) == nil {
return value
}
}
return ""
}
func intParam(params map[string]json.RawMessage, key string) int {
if raw, ok := params[key]; ok {
var value int
if json.Unmarshal(raw, &value) == nil {
return value
}
}
return 0
}
func stringSliceParam(params map[string]json.RawMessage, key string) []string {
raw, ok := params[key]
if !ok {
return nil
}
var value []string
if json.Unmarshal(raw, &value) == nil {
return value
}
return nil
}
func errorResponse(id json.RawMessage, method, sessionID string, err error) response {
return response{JSONRPC: "2.0", ID: id, Error: classifyRPCError(method, sessionID, err)}
}
File diff suppressed because it is too large Load Diff
+748
View File
@@ -0,0 +1,748 @@
package main
import (
"context"
"database/sql/driver"
"encoding/json"
"errors"
"reflect"
"strings"
"testing"
"github.com/t8y2/dbx/agents/go-common/gohive"
)
func TestShowTablesRowName(t *testing.T) {
if value := showTablesRowName([]string{"database", "tableName", "isTemporary"}, []any{"default", "events", false}); value != "events" {
t.Fatalf("unexpected table name: %q", value)
}
if value := showTablesRowName([]string{"tab_name"}, []any{"fallback"}); value != "fallback" {
t.Fatalf("unexpected fallback table name: %q", value)
}
}
// The argo agent always reports ArgoDB identity; Kyuubi/Impala/Hive remain on hive-go.
func TestConnectionInfoReportsArgoIdentity(t *testing.T) {
behavior := &scriptedBehavior{
query: func(ctx context.Context, query string) (driver.Rows, error) {
switch query {
case "SELECT VERSION()":
return newScriptedRows(ctx, []string{"version"}, []string{"STRING"}, [][]driver.Value{{"3.5.8"}}), nil
case "SELECT CURRENT_USER()":
return newScriptedRows(ctx, []string{"current_user"}, []string{"STRING"}, [][]driver.Value{{"dbx"}}), nil
default:
return nil, errors.New("unexpected query: " + query)
}
},
}
server := newScriptedServer(t, behavior)
server.params.DatabaseType = "argo"
server.config.Username = "fallback"
info, err := server.connectionInfo()
if err != nil {
t.Fatal(err)
}
if info["compatibilityMode"] != "argo" || info["username"] != "dbx" || info["version"] != "3.5.8" {
t.Fatalf("unexpected Argo connection info: %#v", info)
}
databaseInfo, ok := info["databaseInfo"].(map[string]string)
if !ok || databaseInfo["productName"] != "ArgoDB (Transwarp)" || databaseInfo["driverName"] != "DBX ArgoDB Go Agent" {
t.Fatalf("unexpected Argo database identity: %#v", info["databaseInfo"])
}
}
func TestGetObjectSourceReturnsProtocolObject(t *testing.T) {
behavior := &scriptedBehavior{
query: func(ctx context.Context, query string) (driver.Rows, error) {
if query != "SHOW CREATE TABLE `dbx_kyuubi_demo`.`high_value_orders`" {
t.Fatalf("unexpected query: %q", query)
}
return newScriptedRows(
ctx,
[]string{"createtab_stmt"},
[]string{"STRING"},
[][]driver.Value{
{"CREATE VIEW dbx_kyuubi_demo.high_value_orders"},
{"AS SELECT id, customer, amount FROM dbx_kyuubi_demo.orders WHERE amount >= 50"},
},
), nil
},
}
server := newScriptedServer(t, behavior)
defer server.disconnect()
result, _, err := server.dispatch("get_object_source", map[string]json.RawMessage{
"schema": json.RawMessage(`"dbx_kyuubi_demo"`),
"name": json.RawMessage(`"high_value_orders"`),
"object_type": json.RawMessage(`"VIEW"`),
})
if err != nil {
t.Fatal(err)
}
source, ok := result.(objectSource)
if !ok {
t.Fatalf("get_object_source returned %T instead of objectSource", result)
}
if source.Name != "high_value_orders" || source.ObjectType != "VIEW" || source.Schema == nil || *source.Schema != "dbx_kyuubi_demo" {
t.Fatalf("unexpected object source metadata: %#v", source)
}
expected := "CREATE VIEW dbx_kyuubi_demo.high_value_orders\nAS SELECT id, customer, amount FROM dbx_kyuubi_demo.orders WHERE amount >= 50\n"
if source.Source != expected {
t.Fatalf("unexpected object source DDL: %q", source.Source)
}
}
func TestListDatabasesUsesShowDatabasesBeforeHiveServerMetadata(t *testing.T) {
behavior := &scriptedBehavior{
query: func(ctx context.Context, sql string) (driver.Rows, error) {
if sql != "SHOW DATABASES" {
t.Fatalf("unexpected query: %q", sql)
}
return newScriptedRows(ctx, []string{"database_name"}, []string{"STRING"}, [][]driver.Value{{"warehouse"}, {"default"}, {"default"}}), nil
},
getSchemas: func(_ context.Context, pattern string) (gohive.MetadataResult, error) {
t.Fatalf("metadata fallback must not run after SHOW DATABASES succeeds: %q", pattern)
return gohive.MetadataResult{}, nil
},
}
server := newScriptedServer(t, behavior)
defer server.disconnect()
values, err := server.listDatabases()
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(values, []databaseInfo{{Name: "default"}, {Name: "warehouse"}}) {
t.Fatalf("unexpected databases: %#v", values)
}
}
func TestListDatabasesFallsBackToHiveServerMetadata(t *testing.T) {
behavior := &scriptedBehavior{
query: func(context.Context, string) (driver.Rows, error) {
return nil, errors.New("SHOW DATABASES unsupported")
},
getSchemas: func(_ context.Context, pattern string) (gohive.MetadataResult, error) {
if pattern != "%" {
t.Fatalf("unexpected schema pattern: %q", pattern)
}
return metadataResult([]string{"TABLE_SCHEM", "TABLE_CATALOG"}, []driver.Value{"warehouse", ""}, []driver.Value{"default", ""}, []driver.Value{"default", ""}), nil
},
}
server := newScriptedServer(t, behavior)
defer server.disconnect()
values, err := server.listDatabases()
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(values, []databaseInfo{{Name: "default"}, {Name: "warehouse"}}) {
t.Fatalf("unexpected databases: %#v", values)
}
}
func TestListSchemasHonorsVisibleSchemaFilter(t *testing.T) {
behavior := &scriptedBehavior{
query: func(ctx context.Context, sql string) (driver.Rows, error) {
if sql != "SHOW DATABASES" {
t.Fatalf("unexpected query: %q", sql)
}
return newScriptedRows(
ctx,
[]string{"database_name"},
[]string{"STRING"},
[][]driver.Value{{"default"}, {"analytics"}, {"system"}},
), nil
},
}
server := newScriptedServer(t, behavior)
defer server.disconnect()
values, err := server.listSchemas([]string{"analytics", "missing"})
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(values, []string{"analytics"}) {
t.Fatalf("unexpected visible schemas: %#v", values)
}
values, err = server.listSchemas([]string{})
if err != nil {
t.Fatal(err)
}
if len(values) != 0 {
t.Fatalf("explicit empty visible schema filter must hide all schemas: %#v", values)
}
}
func TestListTablesPreservesViewTypeCommentAndWindow(t *testing.T) {
behavior := &scriptedBehavior{
getTables: func(_ context.Context, schema, table string, tableTypes []string) (gohive.MetadataResult, error) {
if schema != "analytics" || table != "%" || !reflect.DeepEqual(tableTypes, []string{"TABLE", "VIEW", "MATERIALIZED VIEW"}) {
t.Fatalf("unexpected GetTables request: schema=%q table=%q types=%#v", schema, table, tableTypes)
}
return metadataResult(
[]string{"TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "TABLE_TYPE", "REMARKS"},
[]driver.Value{"", "analytics", "events", "TABLE", "event data"},
[]driver.Value{"", "analytics", "events_view", "VIEW", "view data"},
[]driver.Value{"", "analytics", "other", "TABLE", nil},
), nil
},
}
server := newScriptedServer(t, behavior)
defer server.disconnect()
values, err := server.listTables("analytics", metadataListConstraints{Filter: "events", Offset: 1, Limit: 1})
if err != nil {
t.Fatal(err)
}
comment := "view data"
expected := []tableInfo{{Name: "events_view", TableType: "VIEW", Comment: &comment}}
if !reflect.DeepEqual(values, expected) {
t.Fatalf("unexpected tables: %#v", values)
}
}
func TestGetColumnsUsesHiveServerMetadataFields(t *testing.T) {
behavior := &scriptedBehavior{
getColumns: func(_ context.Context, schema, table, column string) (gohive.MetadataResult, error) {
if schema != "analytics" || table != "events" || column != "%" {
t.Fatalf("unexpected GetColumns request: %q %q %q", schema, table, column)
}
return metadataResult(
[]string{"COLUMN_NAME", "TYPE_NAME", "COLUMN_SIZE", "DECIMAL_DIGITS", "NULLABLE", "REMARKS", "COLUMN_DEF"},
[]driver.Value{"name", "string", int64(255), nil, int64(1), "显示名称", "unknown"},
[]driver.Value{"amount", "decimal(18,2)", int64(18), int64(2), int64(0), nil, nil},
), nil
},
}
server := newScriptedServer(t, behavior)
defer server.disconnect()
values, err := server.getColumns("analytics", "events")
if err != nil {
t.Fatal(err)
}
if len(values) != 2 {
t.Fatalf("unexpected columns: %#v", values)
}
if values[0].Name != "name" || !values[0].IsNullable || values[0].CharacterMaximumLength == nil || *values[0].CharacterMaximumLength != 255 || values[0].ColumnDefault == nil || *values[0].ColumnDefault != "unknown" || values[0].Comment == nil || *values[0].Comment != "显示名称" {
t.Fatalf("unexpected string column: %#v", values[0])
}
if values[1].Name != "amount" || values[1].IsNullable || values[1].NumericPrecision == nil || *values[1].NumericPrecision != 18 || values[1].NumericScale == nil || *values[1].NumericScale != 2 || values[1].CharacterMaximumLength != nil {
t.Fatalf("unexpected decimal column: %#v", values[1])
}
}
func TestGetColumnsPreservesChineseDescribeFallbackComments(t *testing.T) {
behavior := &scriptedBehavior{
getColumns: func(context.Context, string, string, string) (gohive.MetadataResult, error) {
return gohive.MetadataResult{}, errors.New("metadata unavailable")
},
query: func(ctx context.Context, query string) (driver.Rows, error) {
if query != "DESCRIBE `analytics`.`events`" {
t.Fatalf("unexpected DESCRIBE query: %q", query)
}
return newScriptedRows(
ctx,
[]string{"col_name", "data_type", "comment"},
[]string{"STRING", "STRING", "STRING"},
[][]driver.Value{{"name", "string", "显示名称"}, {"amount", "decimal(18,2)", "含税金额"}},
), nil
},
}
server := newScriptedServer(t, behavior)
values, err := server.getColumns("analytics", "events")
if err != nil {
t.Fatal(err)
}
if len(values) != 2 || values[0].Comment == nil || *values[0].Comment != "显示名称" || values[1].Comment == nil || *values[1].Comment != "含税金额" {
t.Fatalf("DESCRIBE comments changed: %#v", values)
}
}
func TestTableCommentAndTypeInfoUseHiveServerMetadata(t *testing.T) {
behavior := &scriptedBehavior{
getTables: func(_ context.Context, schema, table string, tableTypes []string) (gohive.MetadataResult, error) {
return metadataResult(
[]string{"TABLE_SCHEM", "TABLE_NAME", "TABLE_TYPE", "REMARKS"},
[]driver.Value{schema, table, "TABLE", "table comment"},
), nil
},
getTypeInfo: func(context.Context) (gohive.MetadataResult, error) {
return metadataResult([]string{"TYPE_NAME"}, []driver.Value{"STRING"}, []driver.Value{"decimal"}, []driver.Value{"STRING"}), nil
},
}
server := newScriptedServer(t, behavior)
defer server.disconnect()
comment, err := server.getTableComment("analytics", "events")
if err != nil || comment == nil || *comment != "table comment" {
t.Fatalf("unexpected table comment: %v, %v", comment, err)
}
types, err := server.listDataTypes()
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(types, []string{"decimal", "string"}) {
t.Fatalf("unexpected data types: %#v", types)
}
}
func TestListTablesFallsBackToShowTablesAndViews(t *testing.T) {
behavior := &scriptedBehavior{
getTables: func(context.Context, string, string, []string) (gohive.MetadataResult, error) {
return gohive.MetadataResult{}, errors.New("metadata unsupported")
},
query: func(ctx context.Context, query string) (driver.Rows, error) {
switch query {
case "SHOW TABLES IN `analytics`":
return newScriptedRows(
ctx,
[]string{"tab_name"},
[]string{"STRING"},
[][]driver.Value{{"events"}, {"shared_name"}},
), nil
case "SHOW VIEWS IN `analytics`":
return newScriptedRows(
ctx,
[]string{"view_name"},
[]string{"STRING"},
[][]driver.Value{{"events_view"}, {"shared_name"}},
), nil
default:
t.Fatalf("unexpected fallback query: %q", query)
return nil, errors.New("unexpected fallback query")
}
},
}
server := newScriptedServer(t, behavior)
defer server.disconnect()
values, err := server.listTables("analytics", metadataListConstraints{})
if err != nil {
t.Fatal(err)
}
expected := []tableInfo{
{Name: "events", TableType: "TABLE"},
{Name: "events_view", TableType: "VIEW"},
{Name: "shared_name", TableType: "VIEW"},
}
if !reflect.DeepEqual(values, expected) {
t.Fatalf("unexpected fallback tables: %#v", values)
}
}
func TestListTablesKeepsShowTablesResultsWhenShowViewsIsUnsupported(t *testing.T) {
behavior := &scriptedBehavior{
getTables: func(context.Context, string, string, []string) (gohive.MetadataResult, error) {
return gohive.MetadataResult{}, errors.New("metadata unsupported")
},
query: func(ctx context.Context, query string) (driver.Rows, error) {
switch query {
case "SHOW TABLES IN `analytics`":
return newScriptedRows(ctx, []string{"tab_name"}, []string{"STRING"}, [][]driver.Value{{"events"}}), nil
case "SHOW VIEWS IN `analytics`":
return nil, errors.New("SHOW VIEWS is unsupported")
default:
t.Fatalf("unexpected fallback query: %q", query)
return nil, errors.New("unexpected fallback query")
}
},
}
server := newScriptedServer(t, behavior)
defer server.disconnect()
values, err := server.listTables("analytics", metadataListConstraints{})
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(values, []tableInfo{{Name: "events", TableType: "TABLE"}}) {
t.Fatalf("unexpected fallback tables: %#v", values)
}
}
func TestListTablesReturnsNonCapabilityShowViewsError(t *testing.T) {
behavior := &scriptedBehavior{
getTables: func(context.Context, string, string, []string) (gohive.MetadataResult, error) {
return gohive.MetadataResult{}, errors.New("metadata unsupported")
},
query: func(ctx context.Context, query string) (driver.Rows, error) {
switch query {
case "SHOW TABLES IN `analytics`":
return newScriptedRows(ctx, []string{"tab_name"}, []string{"STRING"}, [][]driver.Value{{"events"}}), nil
case "SHOW VIEWS IN `analytics`":
return nil, errors.New("permission denied for SHOW VIEWS")
default:
t.Fatalf("unexpected fallback query: %q", query)
return nil, errors.New("unexpected fallback query")
}
},
}
server := newScriptedServer(t, behavior)
defer server.disconnect()
_, err := server.listTables("analytics", metadataListConstraints{})
if err == nil || !strings.Contains(err.Error(), "SHOW VIEWS fallback failed: permission denied") {
t.Fatalf("unexpected mixed fallback error: %v", err)
}
}
func TestShowViewsUnsupported(t *testing.T) {
tests := []struct {
name string
err error
unsupported bool
}{
{name: "explicit unsupported", err: errors.New("SHOW VIEWS is unsupported"), unsupported: true},
{name: "not supported", err: errors.New("SHOW VIEWS is not supported before Hive 2.2"), unsupported: true},
{name: "old parser", err: errors.New("ParseException: syntax error at or near VIEWS"), unsupported: true},
{name: "permission", err: errors.New("permission denied for SHOW VIEWS")},
{name: "timeout", err: context.DeadlineExceeded},
{name: "cancel", err: context.Canceled},
{name: "authentication", err: errors.New("authentication failed")},
{name: "unsupported authentication", err: errors.New("unsupported authentication mechanism")},
{name: "transport", err: errors.New("transport is closed")},
{name: "unsupported transport", err: errors.New("transport does not support SASL")},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if actual := showViewsUnsupported(test.err); actual != test.unsupported {
t.Fatalf("showViewsUnsupported(%v) = %v, want %v", test.err, actual, test.unsupported)
}
})
}
}
func TestListTablesFallbackHonorsExplicitTableType(t *testing.T) {
behavior := &scriptedBehavior{
getTables: func(context.Context, string, string, []string) (gohive.MetadataResult, error) {
return gohive.MetadataResult{}, errors.New("metadata unsupported")
},
query: func(ctx context.Context, query string) (driver.Rows, error) {
if query != "SHOW TABLES IN `analytics`" {
t.Fatalf("unexpected fallback query: %q", query)
}
return newScriptedRows(ctx, []string{"tab_name"}, []string{"STRING"}, [][]driver.Value{{"events"}}), nil
},
}
server := newScriptedServer(t, behavior)
defer server.disconnect()
values, err := server.listTables("analytics", metadataListConstraints{ObjectTypes: []string{"TABLE"}})
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(values, []tableInfo{{Name: "events", TableType: "TABLE"}}) {
t.Fatalf("unexpected fallback tables: %#v", values)
}
}
func TestListViewsFallsBackToShowViews(t *testing.T) {
behavior := &scriptedBehavior{
getTables: func(context.Context, string, string, []string) (gohive.MetadataResult, error) {
return gohive.MetadataResult{}, errors.New("metadata unsupported")
},
query: func(ctx context.Context, query string) (driver.Rows, error) {
if query != "SHOW VIEWS IN `analytics`" {
t.Fatalf("unexpected fallback query: %q", query)
}
return newScriptedRows(ctx, []string{"view_name"}, []string{"STRING"}, [][]driver.Value{{"events_view"}}), nil
},
}
server := newScriptedServer(t, behavior)
defer server.disconnect()
values, err := server.listTables("analytics", metadataListConstraints{ObjectTypes: []string{"VIEW"}})
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(values, []tableInfo{{Name: "events_view", TableType: "VIEW"}}) {
t.Fatalf("unexpected fallback views: %#v", values)
}
}
func TestListViewsReturnsFallbackErrorWhenShowViewsIsUnsupported(t *testing.T) {
behavior := &scriptedBehavior{
getTables: func(context.Context, string, string, []string) (gohive.MetadataResult, error) {
return gohive.MetadataResult{}, errors.New("metadata unsupported")
},
query: func(_ context.Context, query string) (driver.Rows, error) {
if query != "SHOW VIEWS IN `analytics`" {
t.Fatalf("unexpected fallback query: %q", query)
}
return nil, errors.New("SHOW VIEWS is unsupported")
},
}
server := newScriptedServer(t, behavior)
defer server.disconnect()
_, err := server.listTables("analytics", metadataListConstraints{ObjectTypes: []string{"VIEW"}})
if err == nil || !strings.Contains(err.Error(), "SHOW VIEWS fallback failed") {
t.Fatalf("unexpected explicit view fallback error: %v", err)
}
}
func metadataResult(columns []string, rows ...[]driver.Value) gohive.MetadataResult {
return gohive.MetadataResult{Columns: columns, Rows: rows}
}
func TestListObjectsIncludesProceduresAndFunctionsFromSystemViews(t *testing.T) {
proceduresQuery := "SELECT procedure_name FROM system.procedures_v WHERE lower(database_name) = lower('ods') AND lower(procedure_name) LIKE '%sp%' ORDER BY procedure_name"
functionsQuery := "SELECT function_name FROM system.functions_v WHERE lower(database_name) = lower('ods') AND lower(function_name) LIKE '%sp%' ORDER BY function_name"
behavior := &scriptedBehavior{
query: func(_ context.Context, query string) (driver.Rows, error) {
switch query {
case proceduresQuery:
return newScriptedRows(context.Background(), []string{"procedure_name"}, []string{"STRING"}, [][]driver.Value{
{"sp_daily_etl"}, {"sp_hourly_agg"},
}), nil
case functionsQuery:
return newScriptedRows(context.Background(), []string{"function_name"}, []string{"STRING"}, [][]driver.Value{
{"fn_clean"},
}), nil
default:
return nil, errors.New("unexpected query: " + query)
}
},
}
server := newScriptedServer(t, behavior)
server.params.DatabaseType = "argo"
defer server.disconnect()
values, err := server.listObjects("ods", "ods", metadataListConstraints{
ObjectTypes: []string{"PROCEDURE", "FUNCTION"},
Filter: "sp", // "sp" overlaps the procedure filter; both lists get "%sp%" applied
})
if err != nil {
t.Fatalf("listObjects: %v", err)
}
t.Logf("values: %+v", values)
// listObjects applies the same filter pattern to procedures and functions;
// we passed "sp" so procedures match and functions don't (because the function
// list returns rows regardless of filter — that is, listObjects calls each
// listRoutines call with the same Filter). Verify both queries ran and
// the procedure name is present.
queries, _, _, _ := behavior.snapshot()
t.Logf("queries count: %d", len(queries))
for i, q := range queries {
t.Logf("query[%d]: %q", i, q)
}
if len(queries) != 2 {
t.Fatalf("expected exactly 2 routine queries, got %d: %v", len(queries), queries)
}
if values == nil {
t.Fatal("expected non-nil object list")
}
names := make([]string, len(values))
for i, v := range values {
names[i] = v.Name + ":" + v.ObjectType
}
joined := strings.Join(names, ",")
if !strings.Contains(joined, "sp_daily_etl:PROCEDURE") {
t.Fatalf("expected procedures in result, got %v", values)
}
if !strings.Contains(joined, "fn_clean:FUNCTION") {
t.Fatalf("expected functions in result, got %v", values)
}
}
func TestGetObjectSourceRoutesProceduresToSystemProceduresView(t *testing.T) {
expectedSQL := "SELECT full_text FROM system.procedures_v WHERE lower(database_name) = lower('ods') AND procedure_name = 'sp_daily_etl'"
behavior := &scriptedBehavior{
query: func(_ context.Context, query string) (driver.Rows, error) {
if query != expectedSQL {
return nil, errors.New("unexpected query: " + query)
}
return newScriptedRows(context.Background(), []string{"full_text"}, []string{"STRING"}, [][]driver.Value{
{"-- daily ETL pipeline"},
{"INSERT OVERWRITE TABLE ods.daily_summary SELECT * FROM staging.events"},
}), nil
},
}
server := newScriptedServer(t, behavior)
server.params.DatabaseType = "argo"
defer server.disconnect()
result, _, err := server.dispatch("get_object_source", map[string]json.RawMessage{
"schema": json.RawMessage(`"ods"`),
"name": json.RawMessage(`"sp_daily_etl"`),
"object_type": json.RawMessage(`"PROCEDURE"`),
})
if err != nil {
t.Fatal(err)
}
source, ok := result.(objectSource)
if !ok {
t.Fatalf("get_object_source returned %T instead of objectSource", result)
}
if source.Name != "sp_daily_etl" || source.ObjectType != "PROCEDURE" {
t.Fatalf("unexpected object source metadata: %#v", source)
}
expected := "-- daily ETL pipeline\nINSERT OVERWRITE TABLE ods.daily_summary SELECT * FROM staging.events\n"
if source.Source != expected {
t.Fatalf("unexpected procedure source: %q", source.Source)
}
}
func TestGetObjectSourceRoutesFunctionsToSystemFunctionsView(t *testing.T) {
expectedSQL := "SELECT full_text FROM system.functions_v WHERE lower(database_name) = lower('ods') AND function_name = 'fn_clean'"
behavior := &scriptedBehavior{
query: func(_ context.Context, query string) (driver.Rows, error) {
if query != expectedSQL {
return nil, errors.New("unexpected query: " + query)
}
return newScriptedRows(context.Background(), []string{"full_text"}, []string{"STRING"}, [][]driver.Value{
{"-- cleanup helper"},
}), nil
},
}
server := newScriptedServer(t, behavior)
server.params.DatabaseType = "argo"
defer server.disconnect()
result, _, err := server.dispatch("get_object_source", map[string]json.RawMessage{
"schema": json.RawMessage(`"ods"`),
"name": json.RawMessage(`"fn_clean"`),
"object_type": json.RawMessage(`"FUNCTION"`),
})
if err != nil {
t.Fatal(err)
}
source, ok := result.(objectSource)
if !ok {
t.Fatalf("get_object_source returned %T instead of objectSource", result)
}
if source.Source != "-- cleanup helper\n" {
t.Fatalf("unexpected function source: %q", source.Source)
}
}
func TestListRoutinesUsesDatabaseParameterWhenSchemaEmpty(t *testing.T) {
proceduresQuery := "SELECT procedure_name FROM system.procedures_v WHERE lower(database_name) = lower('ods') AND lower(procedure_name) LIKE '%%' ORDER BY procedure_name"
defaultQuery := "SELECT procedure_name FROM system.procedures_v WHERE lower(database_name) = lower('default') AND lower(procedure_name) LIKE '%%' ORDER BY procedure_name"
behavior := &scriptedBehavior{
query: func(_ context.Context, query string) (driver.Rows, error) {
switch query {
case defaultQuery:
return newScriptedRows(context.Background(), []string{"procedure_name"}, []string{"STRING"}, [][]driver.Value{}), nil
case proceduresQuery:
return newScriptedRows(context.Background(), []string{"procedure_name"}, []string{"STRING"}, [][]driver.Value{
{"sp_daily_etl"},
}), nil
default:
return nil, errors.New("unexpected query: " + query)
}
},
}
server := newScriptedServer(t, behavior)
server.params.DatabaseType = "argo"
server.config.Database = "default"
defer server.disconnect()
values, err := server.listObjects("ods", "", metadataListConstraints{
ObjectTypes: []string{"PROCEDURE"},
})
if err != nil {
t.Fatalf("listObjects: %v", err)
}
if len(values) != 1 || values[0].Name != "sp_daily_etl" {
t.Fatalf("expected procedure from database parameter, got %+v", values)
}
}
func TestListRoutinesDoesNotFallbackToConnectionDefaultForExplicitSchema(t *testing.T) {
defaultQuery := "SELECT procedure_name FROM system.procedures_v WHERE lower(database_name) = lower('default') AND lower(procedure_name) LIKE '%%' ORDER BY procedure_name"
behavior := &scriptedBehavior{
query: func(_ context.Context, query string) (driver.Rows, error) {
if query == defaultQuery {
// Serving this row proves the driver fell back to the
// connection default after the explicit schema came back empty.
return newScriptedRows(context.Background(), []string{"procedure_name"}, []string{"STRING"}, [][]driver.Value{
{"sp_should_not_leak"},
}), nil
}
return newScriptedRows(context.Background(), []string{"procedure_name"}, []string{"STRING"}, [][]driver.Value{}), nil
},
}
server := newScriptedServer(t, behavior)
server.params.DatabaseType = "argo"
server.config.Database = "default"
defer server.disconnect()
values, err := server.listObjects("", "ods", metadataListConstraints{
ObjectTypes: []string{"PROCEDURE"},
})
if err != nil {
t.Fatalf("listObjects: %v", err)
}
if len(values) != 0 {
t.Fatalf("expected no routines to leak from the connection default database, got %+v", values)
}
}
func TestGetObjectSourceUsesDatabaseParameterForRoutineSource(t *testing.T) {
expectedSQL := "SELECT full_text FROM system.procedures_v WHERE lower(database_name) = lower('ods') AND procedure_name = 'sp_daily_etl'"
behavior := &scriptedBehavior{
query: func(_ context.Context, query string) (driver.Rows, error) {
if query != expectedSQL {
return nil, errors.New("unexpected query: " + query)
}
return newScriptedRows(context.Background(), []string{"full_text"}, []string{"STRING"}, [][]driver.Value{
{"CREATE PROCEDURE sp_daily_etl() BEGIN SELECT 1; END"},
}), nil
},
}
server := newScriptedServer(t, behavior)
server.params.DatabaseType = "argo"
server.config.Database = "default"
defer server.disconnect()
result, _, err := server.dispatch("get_object_source", map[string]json.RawMessage{
"database": json.RawMessage(`"ods"`),
"schema": json.RawMessage(`""`),
"name": json.RawMessage(`"sp_daily_etl"`),
"object_type": json.RawMessage(`"PROCEDURE"`),
})
if err != nil {
t.Fatal(err)
}
source, ok := result.(objectSource)
if !ok {
t.Fatalf("get_object_source returned %T instead of objectSource", result)
}
if source.Source != "CREATE PROCEDURE sp_daily_etl() BEGIN SELECT 1; END\n" {
t.Fatalf("unexpected procedure source: %q", source.Source)
}
}
// Unlike hive-go (which gates routine listing on the connection's database_type so
// vanilla Apache Hive never fires the catalog-view queries), the argo agent serves
// 星环Argo (Transwarp ArgoDB) exclusively and must always query the routine views.
func TestListObjectsQueriesRoutineViewsUnconditionally(t *testing.T) {
behavior := &scriptedBehavior{
query: func(ctx context.Context, query string) (driver.Rows, error) {
switch {
case strings.Contains(strings.ToUpper(query), "PROCEDURES_V"):
return newScriptedRows(ctx, []string{"procedure_name"}, []string{"STRING"}, [][]driver.Value{{"sp_etl_log"}}), nil
case strings.Contains(strings.ToUpper(query), "FUNCTIONS_V"):
return newScriptedRows(ctx, []string{"function_name"}, []string{"STRING"}, nil), nil
default:
return newScriptedRows(ctx, []string{"tab_name"}, []string{"STRING"}, [][]driver.Value{{"events"}}), nil
}
},
}
server := newScriptedServer(t, behavior)
server.params.DatabaseType = "argo"
defer server.disconnect()
values, err := server.listObjects("ods", "ods", metadataListConstraints{
ObjectTypes: []string{"PROCEDURE", "FUNCTION"},
})
if err != nil {
t.Fatalf("listObjects: %v", err)
}
found := false
for _, v := range values {
if v.ObjectType == "PROCEDURE" && v.Name == "sp_etl_log" {
found = true
}
}
if !found {
t.Fatalf("argo agent must surface routines regardless of database_type, got: %+v", values)
}
}
+145
View File
@@ -0,0 +1,145 @@
package main
import (
"context"
"database/sql"
"errors"
"fmt"
"io"
"net"
"strings"
"github.com/t8y2/dbx/agents/go-common/gohive"
)
type rpcError struct {
Code int `json:"code"`
Message string `json:"message"`
Data *rpcErrorData `json:"data,omitempty"`
}
type rpcErrorData struct {
Category string `json:"category"`
Retryable bool `json:"retryable"`
SessionDisposition string `json:"sessionDisposition"`
Stage string `json:"stage"`
ContractVersion int `json:"contractVersion"`
OperationOutcome string `json:"operationOutcome"`
SQLState string `json:"sqlState,omitempty"`
VendorCode int32 `json:"vendorCode,omitempty"`
ExceptionClass string `json:"exceptionClass,omitempty"`
AgentSessionID string `json:"agentSessionId,omitempty"`
}
func classifyRPCError(method, agentSessionID string, err error) *rpcError {
stage := rpcErrorStage(method)
data := &rpcErrorData{
Category: "protocol",
Retryable: false,
SessionDisposition: "keep",
Stage: stage,
ContractVersion: 1,
OperationOutcome: rpcOperationOutcome(stage),
ExceptionClass: safeRPCDiagnostic(fmt.Sprintf("%T", err), 160),
AgentSessionID: strings.TrimSpace(agentSessionID),
}
var hiveError *gohive.Error
if errors.As(err, &hiveError) {
data.Category = "sql"
data.SQLState = safeRPCDiagnostic(hiveError.SQLState, 160)
data.VendorCode = int32(hiveError.ErrorCode)
data.ExceptionClass = "gohive.Error"
} else if errors.Is(err, context.Canceled) {
data.Category = "canceled"
data.SessionDisposition = "quarantine"
} else if errors.Is(err, context.DeadlineExceeded) || isTimeoutError(err) {
data.Category = "timeout"
data.Retryable = stage == "connect" || stage == "validate"
data.SessionDisposition = "quarantine"
} else if isConnectionError(err) {
data.Category = "connection"
data.Retryable = stage == "connect" || stage == "validate"
if stage != "connect" {
data.SessionDisposition = "quarantine"
}
} else if isHiveSQLError(err) {
data.Category = "sql"
}
return &rpcError{Code: -1, Message: err.Error(), Data: data}
}
func rpcErrorStage(method string) string {
switch method {
case "connect", "open_session", "test_connection":
return "connect"
case "validate_connection", "validate_session":
return "validate"
case "cancel_session":
return "cancel"
case "close_session", "disconnect", "close_query_session", "close_table_read_session", "shutdown":
return "close"
case "fetch_query_page", "fetch_table_read_page":
return "fetch"
case "handshake", "":
return "request"
default:
return "execute"
}
}
func rpcOperationOutcome(stage string) string {
if stage == "request" || stage == "connect" || stage == "validate" {
return "not_started"
}
return "unknown"
}
func isTimeoutError(err error) bool {
var timeout interface{ Timeout() bool }
return errors.As(err, &timeout) && timeout.Timeout()
}
func isConnectionError(err error) bool {
if errors.Is(err, io.EOF) || errors.Is(err, net.ErrClosed) || errors.Is(err, sql.ErrConnDone) {
return true
}
var networkError *net.OpError
if errors.As(err, &networkError) {
return true
}
lower := strings.ToLower(err.Error())
for _, marker := range []string{
"connection refused", "connection reset", "broken pipe", "connection closed", "connection lost",
"unexpected eof", "no route to host", "all hiveserver2 endpoints failed", "transport is not open",
"socket is closed", "not open",
} {
if strings.Contains(lower, marker) {
return true
}
}
return false
}
func isHiveSQLError(err error) bool {
lower := strings.ToLower(err.Error())
for _, marker := range []string{"semanticexception", "parseexception", "hive error", "sqlstate", "error while compiling statement"} {
if strings.Contains(lower, marker) {
return true
}
}
return false
}
func safeRPCDiagnostic(value string, maxLength int) string {
var result strings.Builder
for _, char := range value {
if result.Len() >= maxLength {
break
}
if char >= 0x21 && char <= 0x7e {
result.WriteRune(char)
}
}
return result.String()
}
+529
View File
@@ -0,0 +1,529 @@
package main
import (
"context"
"database/sql"
"database/sql/driver"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/t8y2/dbx/agents/go-common/gohive"
)
func (server *server) validateConnection() error {
connection, err := server.requireConnection()
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), server.config.ConnectTimeout)
defer cancel()
return connection.PingContext(ctx)
}
func (server *server) executeQuery(options queryOptions) (queryResult, error) {
started := time.Now()
if options.FetchSize <= 0 {
options.FetchSize = server.effectiveFetchSize()
}
sqlText := trimStatementSQL(options.SQL)
if sqlText == "" {
return queryResult{}, errors.New("SQL is required")
}
maxRows := options.MaxRows
if maxRows <= 0 {
maxRows = defaultMaxRows
}
connection, err := server.requireConnection()
if err != nil {
return queryResult{}, err
}
ctx, cancel := queryContext(options.TimeoutSecs)
server.setActiveOperation(cancel)
defer server.clearActiveOperation(cancel)
if err := server.applySchemaContext(ctx, connection, effectiveSchema(options)); err != nil {
return queryResult{}, err
}
rows, affected, hasResultSet, err := executeHiveStatement(ctx, connection, sqlText, options.FetchSize)
if err != nil {
return queryResult{}, err
}
if !hasResultSet {
return queryResult{
Columns: []string{},
ColumnTypes: []string{},
Rows: [][]any{},
AffectedRows: affected,
ExecutionTimeMS: time.Since(started).Milliseconds(),
Truncated: false,
}, nil
}
defer rows.Close()
columns, columnTypes, err := queryColumnMetadata(rows)
if err != nil {
return queryResult{}, err
}
values, truncated, err := readSQLRows(rows, columnTypes, maxRows)
if err != nil {
return queryResult{}, err
}
return queryResult{
Columns: columns,
ColumnTypes: columnTypes,
Rows: values,
AffectedRows: 0,
ExecutionTimeMS: time.Since(started).Milliseconds(),
Truncated: truncated,
}, nil
}
func (server *server) executeQueryPage(options queryOptions, requestedPageSize int) (queryPageResult, error) {
started := time.Now()
if options.FetchSize <= 0 {
options.FetchSize = server.effectiveFetchSize()
}
server.expireIdleQuerySessions(started)
sqlText := trimStatementSQL(options.SQL)
if sqlText == "" {
return queryPageResult{}, errors.New("SQL is required")
}
pageSize := requestedPageSize
if pageSize <= 0 {
pageSize = defaultPageSize
}
maxRows := options.MaxRows
if maxRows <= 0 {
maxRows = defaultMaxRows
}
connection, err := server.requireConnection()
if err != nil {
return queryPageResult{}, err
}
ctx, cancel := queryContext(options.TimeoutSecs)
server.setActiveOperation(cancel)
if err := server.applySchemaContext(ctx, connection, effectiveSchema(options)); err != nil {
server.clearActiveOperation(cancel)
return queryPageResult{}, err
}
rows, affected, hasResultSet, err := executeHiveStatement(ctx, connection, sqlText, options.FetchSize)
if err != nil {
server.clearActiveOperation(cancel)
return queryPageResult{}, err
}
if !hasResultSet {
server.clearActiveOperation(cancel)
return queryPageResult{
Columns: []string{},
ColumnTypes: []string{},
Rows: [][]any{},
AffectedRows: affected,
ExecutionTimeMS: time.Since(started).Milliseconds(),
Truncated: false,
SessionID: nil,
HasMore: false,
}, nil
}
columns, columnTypes, err := queryColumnMetadata(rows)
if err != nil {
_ = rows.Close()
server.clearActiveOperation(cancel)
return queryPageResult{}, err
}
server.nextSessionID++
sessionID := fmt.Sprintf("hive-%d", server.nextSessionID)
state := &querySession{
rows: rows,
columns: columns,
columnTypes: columnTypes,
remaining: maxRows,
cancel: cancel,
lastAccessed: started,
}
server.querySessions[sessionID] = state
page, hasMore, truncated, err := server.readQuerySessionPage(ctx, state, pageSize)
server.activeMu.Lock()
if server.activeCancel != nil {
server.activeCancel = nil
}
server.activeMu.Unlock()
if err != nil {
server.closeQuerySession(sessionID)
return queryPageResult{}, err
}
if !hasMore {
server.closeQuerySession(sessionID)
return queryPageResult{
Columns: columns,
ColumnTypes: columnTypes,
Rows: page,
ExecutionTimeMS: time.Since(started).Milliseconds(),
Truncated: truncated,
SessionID: nil,
HasMore: false,
}, nil
}
return queryPageResult{
Columns: columns,
ColumnTypes: columnTypes,
Rows: page,
ExecutionTimeMS: time.Since(started).Milliseconds(),
Truncated: false,
SessionID: &sessionID,
HasMore: true,
}, nil
}
func (server *server) fetchQueryPage(sessionID string, requestedPageSize int) (queryPageResult, error) {
server.expireIdleQuerySessions(time.Now())
state := server.querySessions[sessionID]
if state == nil {
return queryPageResult{}, errors.New("query session not found")
}
pageSize := requestedPageSize
if pageSize <= 0 {
pageSize = defaultPageSize
}
ctx := context.Background()
server.setActiveOperation(state.cancel)
page, hasMore, truncated, err := server.readQuerySessionPage(ctx, state, pageSize)
server.activeMu.Lock()
server.activeCancel = nil
server.activeMu.Unlock()
if err != nil {
server.closeQuerySession(sessionID)
return queryPageResult{}, err
}
var resultSessionID *string
if hasMore {
resultSessionID = &sessionID
} else {
server.closeQuerySession(sessionID)
}
return queryPageResult{
Columns: state.columns,
ColumnTypes: state.columnTypes,
Rows: page,
Truncated: truncated,
SessionID: resultSessionID,
HasMore: hasMore,
}, nil
}
func (server *server) readQuerySessionPage(ctx context.Context, state *querySession, pageSize int) ([][]any, bool, bool, error) {
state.lastAccessed = time.Now()
values := make([][]any, 0, min(pageSize, state.remaining))
if state.pending != nil && state.remaining > 0 {
values = append(values, state.pending)
state.pending = nil
state.remaining--
}
for len(values) < pageSize && state.remaining > 0 {
row, ok, err := nextSQLRow(state.rows, state.columnTypes)
if err != nil {
return nil, false, false, err
}
if !ok {
return values, false, false, nil
}
values = append(values, row)
state.remaining--
select {
case <-ctx.Done():
return nil, false, false, ctx.Err()
default:
}
}
if state.remaining == 0 {
_, ok, err := nextSQLRow(state.rows, state.columnTypes)
if err != nil {
return nil, false, false, err
}
return values, false, ok, nil
}
row, ok, err := nextSQLRow(state.rows, state.columnTypes)
if err != nil {
return nil, false, false, err
}
if !ok {
return values, false, false, nil
}
state.pending = row
return values, true, false, nil
}
func (server *server) closeQuerySession(sessionID string) bool {
state := server.querySessions[sessionID]
if state == nil {
return false
}
delete(server.querySessions, sessionID)
state.cancel()
_ = state.rows.Close()
return true
}
func (server *server) closeAllQuerySessions() error {
var failures []string
for sessionID, state := range server.querySessions {
delete(server.querySessions, sessionID)
state.cancel()
if err := state.rows.Close(); err != nil {
failures = append(failures, fmt.Sprintf("%s: %v", sessionID, err))
}
}
if len(failures) > 0 {
return errors.New(strings.Join(failures, "; "))
}
return nil
}
func (server *server) expireIdleQuerySessions(now time.Time) int {
expired := make([]string, 0)
for sessionID, state := range server.querySessions {
if !state.lastAccessed.IsZero() && now.Sub(state.lastAccessed) >= querySessionIdleTime {
expired = append(expired, sessionID)
}
}
for _, sessionID := range expired {
server.closeQuerySession(sessionID)
}
return len(expired)
}
func (server *server) executeStatements(params map[string]json.RawMessage, transaction bool) (queryResult, error) {
started := time.Now()
statements := stringSliceParam(params, "statements")
if len(statements) == 0 {
return queryResult{}, errors.New("statements are required")
}
connection, err := server.requireConnection()
if err != nil {
return queryResult{}, err
}
ctx, cancel := queryContext(intParam(params, "timeoutSecs"))
server.setActiveOperation(cancel)
defer server.clearActiveOperation(cancel)
if err := server.applySchemaContext(ctx, connection, firstNonEmpty(stringParam(params, "schema"), stringParam(params, "database"))); err != nil {
return queryResult{}, err
}
var affected int64
if transaction {
tx, beginErr := connection.BeginTx(ctx, nil)
if beginErr == nil {
for _, statement := range statements {
trimmed := trimStatementSQL(statement)
if trimmed == "" {
continue
}
result, execErr := tx.ExecContext(ctx, trimmed)
if execErr != nil {
_ = tx.Rollback()
return queryResult{}, execErr
}
count, _ := result.RowsAffected()
affected += max(count, 0)
}
if err := tx.Commit(); err != nil {
return queryResult{}, err
}
return emptyQueryResult(affected, started), nil
}
if !transactionUnsupported(beginErr) {
return queryResult{}, beginErr
}
}
for _, statement := range statements {
trimmed := trimStatementSQL(statement)
if trimmed == "" {
continue
}
result, err := connection.ExecContext(ctx, trimmed)
if err != nil {
return queryResult{}, err
}
count, _ := result.RowsAffected()
affected += max(count, 0)
}
return emptyQueryResult(affected, started), nil
}
func transactionUnsupported(err error) bool {
if err == nil {
return false
}
if errors.Is(err, sql.ErrTxDone) {
return false
}
message := strings.ToLower(err.Error())
return errors.Is(err, driver.ErrSkip) ||
strings.Contains(message, "transactions are not supported") ||
strings.Contains(message, "transaction is not supported") ||
strings.Contains(message, "unsupported transaction") ||
strings.Contains(message, "driver: skip fast-path")
}
func emptyQueryResult(affected int64, started time.Time) queryResult {
return queryResult{
Columns: []string{},
ColumnTypes: []string{},
Rows: [][]any{},
AffectedRows: affected,
ExecutionTimeMS: time.Since(started).Milliseconds(),
Truncated: false,
}
}
func (server *server) applySchemaContext(ctx context.Context, connection *sql.Conn, schema string) error {
schema = strings.TrimSpace(schema)
if schema == "" || strings.EqualFold(schema, server.config.Database) {
return nil
}
_, err := connection.ExecContext(ctx, "USE "+quoteHiveIdentifier(schema))
return err
}
func queryColumnMetadata(rows *sql.Rows) ([]string, []string, error) {
columns, err := rows.Columns()
if err != nil {
return nil, nil, err
}
types, err := rows.ColumnTypes()
if err != nil {
return nil, nil, err
}
columnTypes := make([]string, len(columns))
for index := range columns {
if index < len(types) {
columnTypes[index] = strings.ToLower(strings.TrimSpace(types[index].DatabaseTypeName()))
}
}
return columns, columnTypes, nil
}
func readSQLRows(rows *sql.Rows, columnTypes []string, limit int) ([][]any, bool, error) {
values := make([][]any, 0, min(limit, defaultFetchSize))
for len(values) < limit {
row, ok, err := nextSQLRow(rows, columnTypes)
if err != nil {
return nil, false, err
}
if !ok {
return values, false, nil
}
values = append(values, row)
}
_, ok, err := nextSQLRow(rows, columnTypes)
return values, ok, err
}
func nextSQLRow(rows *sql.Rows, columnTypes []string) ([]any, bool, error) {
if !rows.Next() {
if err := rows.Err(); err != nil {
return nil, false, err
}
return nil, false, nil
}
values := make([]any, len(columnTypes))
targets := make([]any, len(values))
for index := range values {
targets[index] = &values[index]
}
if err := rows.Scan(targets...); err != nil {
return nil, false, err
}
for index, value := range values {
values[index] = normalizeHiveValue(value, columnTypes[index])
}
return values, true, nil
}
func normalizeHiveValue(value any, columnType string) any {
if value == nil {
return nil
}
switch typed := value.(type) {
case []byte:
return bytesToHex(typed)
case time.Time:
return formatHiveJDBCDateTime(typed, columnType)
case fmt.Stringer:
return typed.String()
case string:
return typed
default:
return fmt.Sprint(value)
}
}
func formatHiveJDBCDateTime(value time.Time, columnType string) string {
if strings.EqualFold(strings.TrimSpace(columnType), "DATE") {
return value.Format("2006-01-02")
}
base := value.Format("2006-01-02 15:04:05")
if value.Nanosecond() == 0 {
return base + ".0"
}
fraction := strings.TrimRight(fmt.Sprintf("%09d", value.Nanosecond()), "0")
return base + "." + fraction
}
func executeHiveStatement(
ctx context.Context,
connection *sql.Conn,
sqlText string,
fetchSize int,
) (*sql.Rows, int64, bool, error) {
ctx = gohive.WithFetchSize(ctx, fetchSize)
rows, err := connection.QueryContext(ctx, sqlText)
if err == nil {
return rows, 0, true, nil
}
var nonQuery *gohive.NonQueryResult
if errors.As(err, &nonQuery) {
return nil, max(nonQuery.AffectedRows, 0), false, nil
}
return nil, 0, false, err
}
func bytesToHex(value []byte) string {
const digits = "0123456789abcdef"
result := make([]byte, 2+len(value)*2)
result[0] = '0'
result[1] = 'x'
for index, current := range value {
result[2+index*2] = digits[current>>4]
result[3+index*2] = digits[current&0x0f]
}
return string(result)
}
func queryContext(timeoutSecs int) (context.Context, context.CancelFunc) {
if timeoutSecs > 0 {
return context.WithTimeout(context.Background(), time.Duration(timeoutSecs)*time.Second)
}
return context.WithCancel(context.Background())
}
func (server *server) effectiveFetchSize() int {
if server.config.FetchSize > 0 {
return server.config.FetchSize
}
return defaultFetchSize
}
func effectiveSchema(options queryOptions) string {
return firstNonEmpty(options.Schema, options.Database)
}
func trimStatementSQL(sqlText string) string {
return strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(sqlText), ";"))
}
func quoteHiveIdentifier(value string) string {
return "`" + strings.ReplaceAll(value, "`", "``") + "`"
}
+623
View File
@@ -0,0 +1,623 @@
package main
import (
"context"
"database/sql"
"database/sql/driver"
"encoding/json"
"errors"
"fmt"
"io"
"reflect"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/t8y2/dbx/agents/go-common/gohive"
)
var scriptedDriverSequence atomic.Uint64
type scriptedBehavior struct {
mu sync.Mutex
query func(context.Context, string) (driver.Rows, error)
exec func(context.Context, string) (driver.Result, error)
getSchemas func(context.Context, string) (gohive.MetadataResult, error)
getTables func(context.Context, string, string, []string) (gohive.MetadataResult, error)
getColumns func(context.Context, string, string, string) (gohive.MetadataResult, error)
getTypeInfo func(context.Context) (gohive.MetadataResult, error)
beginErr error
queries []string
executions []string
beginCalls int
closeCalls int
}
func (behavior *scriptedBehavior) queryContext(ctx context.Context, query string) (driver.Rows, error) {
behavior.mu.Lock()
behavior.queries = append(behavior.queries, query)
operation := behavior.query
behavior.mu.Unlock()
if operation == nil {
return nil, fmt.Errorf("unexpected query: %s", query)
}
return operation(ctx, query)
}
func (behavior *scriptedBehavior) execContext(ctx context.Context, query string) (driver.Result, error) {
behavior.mu.Lock()
behavior.executions = append(behavior.executions, query)
operation := behavior.exec
behavior.mu.Unlock()
if operation == nil {
return nil, fmt.Errorf("unexpected execution: %s", query)
}
return operation(ctx, query)
}
func (behavior *scriptedBehavior) snapshot() (queries, executions []string, beginCalls, closeCalls int) {
behavior.mu.Lock()
defer behavior.mu.Unlock()
return append([]string(nil), behavior.queries...), append([]string(nil), behavior.executions...), behavior.beginCalls, behavior.closeCalls
}
type scriptedDriver struct {
behavior *scriptedBehavior
}
func (driverValue *scriptedDriver) Open(string) (driver.Conn, error) {
return &scriptedConnection{behavior: driverValue.behavior}, nil
}
type scriptedConnection struct {
behavior *scriptedBehavior
}
func (connection *scriptedConnection) Prepare(string) (driver.Stmt, error) {
return nil, errors.New("prepared statements are not supported")
}
func (connection *scriptedConnection) Close() error {
connection.behavior.mu.Lock()
connection.behavior.closeCalls++
connection.behavior.mu.Unlock()
return nil
}
func (connection *scriptedConnection) Begin() (driver.Tx, error) {
connection.behavior.mu.Lock()
connection.behavior.beginCalls++
connection.behavior.mu.Unlock()
if connection.behavior.beginErr != nil {
return nil, connection.behavior.beginErr
}
return nil, driver.ErrSkip
}
func (connection *scriptedConnection) BeginTx(context.Context, driver.TxOptions) (driver.Tx, error) {
connection.behavior.mu.Lock()
connection.behavior.beginCalls++
connection.behavior.mu.Unlock()
if connection.behavior.beginErr != nil {
return nil, connection.behavior.beginErr
}
return nil, driver.ErrSkip
}
func (connection *scriptedConnection) Ping(context.Context) error {
return nil
}
func (connection *scriptedConnection) QueryContext(
ctx context.Context,
query string,
_ []driver.NamedValue,
) (driver.Rows, error) {
return connection.behavior.queryContext(ctx, query)
}
func (connection *scriptedConnection) ExecContext(
ctx context.Context,
query string,
_ []driver.NamedValue,
) (driver.Result, error) {
return connection.behavior.execContext(ctx, query)
}
func (connection *scriptedConnection) GetHiveSchemas(ctx context.Context, pattern string) (gohive.MetadataResult, error) {
if connection.behavior.getSchemas == nil {
return gohive.MetadataResult{}, errors.New("GetSchemas unavailable")
}
return connection.behavior.getSchemas(ctx, pattern)
}
func (connection *scriptedConnection) GetHiveTables(ctx context.Context, schema, table string, tableTypes []string) (gohive.MetadataResult, error) {
if connection.behavior.getTables == nil {
return gohive.MetadataResult{}, errors.New("GetTables unavailable")
}
return connection.behavior.getTables(ctx, schema, table, tableTypes)
}
func (connection *scriptedConnection) GetHiveColumns(ctx context.Context, schema, table, column string) (gohive.MetadataResult, error) {
if connection.behavior.getColumns == nil {
return gohive.MetadataResult{}, errors.New("GetColumns unavailable")
}
return connection.behavior.getColumns(ctx, schema, table, column)
}
func (connection *scriptedConnection) GetHiveTypeInfo(ctx context.Context) (gohive.MetadataResult, error) {
if connection.behavior.getTypeInfo == nil {
return gohive.MetadataResult{}, errors.New("GetTypeInfo unavailable")
}
return connection.behavior.getTypeInfo(ctx)
}
type scriptedRows struct {
ctx context.Context
columns []string
types []string
values [][]driver.Value
blockAfter int
blocked chan struct{}
blockOnce sync.Once
mu sync.Mutex
index int
closed bool
}
func newScriptedRows(ctx context.Context, columns, types []string, values [][]driver.Value) *scriptedRows {
return &scriptedRows{
ctx: ctx,
columns: columns,
types: types,
values: values,
blockAfter: -1,
}
}
func (rows *scriptedRows) Columns() []string {
return append([]string(nil), rows.columns...)
}
func (rows *scriptedRows) Close() error {
rows.mu.Lock()
rows.closed = true
rows.mu.Unlock()
return nil
}
func (rows *scriptedRows) Next(destination []driver.Value) error {
rows.mu.Lock()
if rows.closed {
rows.mu.Unlock()
return io.EOF
}
index := rows.index
if rows.blockAfter >= 0 && index >= rows.blockAfter {
blocked := rows.blocked
ctx := rows.ctx
rows.mu.Unlock()
if blocked != nil {
rows.blockOnce.Do(func() { close(blocked) })
}
<-ctx.Done()
return ctx.Err()
}
if index >= len(rows.values) {
rows.mu.Unlock()
return io.EOF
}
current := rows.values[index]
rows.index++
rows.mu.Unlock()
copy(destination, current)
return nil
}
func (rows *scriptedRows) ColumnTypeDatabaseTypeName(index int) string {
if index < 0 || index >= len(rows.types) {
return ""
}
return rows.types[index]
}
func (rows *scriptedRows) isClosed() bool {
rows.mu.Lock()
defer rows.mu.Unlock()
return rows.closed
}
func newScriptedServer(t *testing.T, behavior *scriptedBehavior) *server {
t.Helper()
driverName := fmt.Sprintf("dbx-hive-scripted-%d", scriptedDriverSequence.Add(1))
sql.Register(driverName, &scriptedDriver{behavior: behavior})
database, err := sql.Open(driverName, "")
if err != nil {
t.Fatal(err)
}
connection, err := database.Conn(context.Background())
if err != nil {
database.Close()
t.Fatal(err)
}
server := &server{
config: connectionConfig{Database: defaultHiveDatabase, ConnectTimeout: time.Second},
database: database,
connection: connection,
querySessions: map[string]*querySession{},
}
t.Cleanup(func() { _ = server.disconnect() })
return server
}
func rawParams(values map[string]any) map[string]json.RawMessage {
result := make(map[string]json.RawMessage, len(values))
for key, value := range values {
encoded, err := json.Marshal(value)
if err != nil {
panic(err)
}
result[key] = encoded
}
return result
}
func TestExecuteQueryUsesHiveServerResultSetSignal(t *testing.T) {
behavior := &scriptedBehavior{}
behavior.query = func(ctx context.Context, query string) (driver.Rows, error) {
switch {
case strings.HasPrefix(query, "WITH source AS"):
return nil, &gohive.NonQueryResult{AffectedRows: 4}
case strings.HasPrefix(query, "SET "):
return newScriptedRows(ctx, []string{"set"}, []string{"STRING"}, [][]driver.Value{{"hive.exec.dynamic.partition=true"}}), nil
default:
return nil, fmt.Errorf("unexpected SQL: %s", query)
}
}
server := newScriptedServer(t, behavior)
insertResult, err := server.executeQuery(queryOptions{
SQL: "WITH source AS (SELECT 1) INSERT INTO target SELECT * FROM source",
})
if err != nil {
t.Fatal(err)
}
if insertResult.AffectedRows != 4 || len(insertResult.Columns) != 0 {
t.Fatalf("unexpected non-query result: %#v", insertResult)
}
setResult, err := server.executeQuery(queryOptions{SQL: "SET hive.exec.dynamic.partition"})
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(setResult.Rows, [][]any{{"hive.exec.dynamic.partition=true"}}) {
t.Fatalf("unexpected SET result: %#v", setResult.Rows)
}
queries, executions, _, _ := behavior.snapshot()
if len(queries) != 2 || len(executions) != 0 {
t.Fatalf("statements must use HS2 result-set signaling, queries=%v executions=%v", queries, executions)
}
}
func TestQueryResultsPreserveHiveServerLabelsWithoutDotSplitting(t *testing.T) {
columns := []string{"id", "customer.id", "total + 1", "id"}
columnTypes := []string{"BIGINT", "BIGINT", "DOUBLE", "BIGINT"}
behavior := &scriptedBehavior{}
behavior.query = func(ctx context.Context, _ string) (driver.Rows, error) {
return newScriptedRows(ctx, columns, columnTypes, [][]driver.Value{{int64(1), int64(2), float64(3), int64(4)}, {int64(5), int64(6), float64(7), int64(8)}}), nil
}
server := newScriptedServer(t, behavior)
ordinary, err := server.executeQuery(queryOptions{SQL: "SELECT * FROM labels"})
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(ordinary.Columns, columns) || !reflect.DeepEqual(ordinary.ColumnTypes, []string{"bigint", "bigint", "double", "bigint"}) {
t.Fatalf("ordinary metadata changed: %#v", ordinary)
}
if len(ordinary.Rows) != 2 || ordinary.Rows[0][0] != "1" || ordinary.Rows[0][3] != "4" {
t.Fatalf("ordinary values changed: %#v", ordinary.Rows)
}
first, err := server.executeQueryPage(queryOptions{SQL: "SELECT * FROM labels", MaxRows: 2}, 1)
if err != nil {
t.Fatal(err)
}
if first.SessionID == nil || !first.HasMore || !reflect.DeepEqual(first.Columns, columns) {
t.Fatalf("first page metadata changed: %#v", first)
}
second, err := server.fetchQueryPage(*first.SessionID, 1)
if err != nil {
t.Fatal(err)
}
if second.HasMore || second.SessionID != nil || !reflect.DeepEqual(second.Columns, columns) {
t.Fatalf("cached page metadata changed: %#v", second)
}
}
func TestPagedQueryPreservesDuplicateLeadingValuesAcrossPages(t *testing.T) {
behavior := &scriptedBehavior{}
behavior.query = func(ctx context.Context, _ string) (driver.Rows, error) {
return newScriptedRows(
ctx,
[]string{"group_id", "row_id"},
[]string{"BIGINT", "BIGINT"},
[][]driver.Value{{int64(1), int64(101)}, {int64(1), int64(102)}, {int64(1), int64(103)}},
), nil
}
server := newScriptedServer(t, behavior)
first, err := server.executeQueryPage(queryOptions{SQL: "SELECT group_id, row_id FROM repeated_values", MaxRows: 3}, 2)
if err != nil {
t.Fatal(err)
}
if first.SessionID == nil || !first.HasMore {
t.Fatalf("expected an open cursor after the first page: %#v", first)
}
second, err := server.fetchQueryPage(*first.SessionID, 2)
if err != nil {
t.Fatal(err)
}
allRows := append(append([][]any{}, first.Rows...), second.Rows...)
expected := [][]any{{"1", "101"}, {"1", "102"}, {"1", "103"}}
if !reflect.DeepEqual(allRows, expected) || second.HasMore {
t.Fatalf("duplicate leading values changed across cursor pages: %#v", allRows)
}
}
func TestPagedQueryTruncatesAndPreservesLegacyJDBCValueSemantics(t *testing.T) {
largeValue := strings.Repeat("x", 256*1024)
createdAt := time.Date(2026, time.August, 11, 10, 11, 12, 345000000, time.UTC)
var sourceRows *scriptedRows
behavior := &scriptedBehavior{}
behavior.query = func(ctx context.Context, _ string) (driver.Rows, error) {
sourceRows = newScriptedRows(
ctx,
[]string{"id", "enabled", "payload", "created_at", "complex_value"},
[]string{"BIGINT", "BOOLEAN", "BINARY", "TIMESTAMP", "ARRAY"},
[][]driver.Value{
{int64(1), true, []byte{0x00, 0xff}, createdAt, largeValue},
{int64(2), false, []byte{0x10}, createdAt, "[1,2]"},
{int64(3), true, []byte{}, createdAt, "map('a',1)"},
{int64(4), true, []byte{0x01}, createdAt, "extra"},
},
)
return sourceRows, nil
}
server := newScriptedServer(t, behavior)
first, err := server.executeQueryPage(queryOptions{SQL: "SELECT * FROM values", MaxRows: 3, FetchSize: 2}, 2)
if err != nil {
t.Fatal(err)
}
if !first.HasMore || first.SessionID == nil || len(first.Rows) != 2 {
t.Fatalf("unexpected first page: %#v", first)
}
if !reflect.DeepEqual(first.ColumnTypes, []string{"bigint", "boolean", "binary", "timestamp", "array"}) {
t.Fatalf("unexpected column types: %#v", first.ColumnTypes)
}
if first.Rows[0][0] != "1" || first.Rows[0][1] != "true" || first.Rows[0][2] != "0x00ff" {
t.Fatalf("primitive value types changed: %#v", first.Rows[0])
}
if first.Rows[0][3] != "2026-08-11 10:11:12.345" || first.Rows[0][4] != largeValue {
t.Fatalf("timestamp or large value changed: %#v", first.Rows[0])
}
second, err := server.fetchQueryPage(*first.SessionID, 2)
if err != nil {
t.Fatal(err)
}
if second.HasMore || second.SessionID != nil || !second.Truncated || len(second.Rows) != 1 {
t.Fatalf("unexpected final page: %#v", second)
}
if len(server.querySessions) != 0 || sourceRows == nil || !sourceRows.isClosed() {
t.Fatalf("query session was not closed: sessions=%d rows=%#v", len(server.querySessions), sourceRows)
}
}
func TestCancelPagedFetchQuarantinesWithoutReplayingSQL(t *testing.T) {
blocked := make(chan struct{})
behavior := &scriptedBehavior{}
behavior.query = func(ctx context.Context, _ string) (driver.Rows, error) {
rows := newScriptedRows(ctx, []string{"id"}, []string{"BIGINT"}, [][]driver.Value{{int64(1)}, {int64(2)}, {int64(3)}})
rows.blockAfter = 2
rows.blocked = blocked
return rows, nil
}
server := newScriptedServer(t, behavior)
first, err := server.executeQueryPage(queryOptions{SQL: "SELECT id FROM slow_table", MaxRows: 10}, 1)
if err != nil {
t.Fatal(err)
}
if first.SessionID == nil {
t.Fatal("expected a paged query session")
}
result := make(chan error, 1)
go func() {
_, fetchErr := server.fetchQueryPage(*first.SessionID, 1)
result <- fetchErr
}()
select {
case <-blocked:
case <-time.After(2 * time.Second):
t.Fatal("fetch did not reach the blocking row")
}
server.cancelActiveQuery()
fetchErr := <-result
if !errors.Is(fetchErr, context.Canceled) {
t.Fatalf("expected cancellation, got %v", fetchErr)
}
rpcErr := classifyRPCError("fetch_query_page", "session-a", fetchErr)
if rpcErr.Data.Category != "canceled" || rpcErr.Data.SessionDisposition != "quarantine" {
t.Fatalf("unexpected cancellation classification: %#v", rpcErr)
}
queries, _, _, _ := behavior.snapshot()
if len(queries) != 1 {
t.Fatalf("SQL must never be replayed after cancellation: %v", queries)
}
if len(server.querySessions) != 0 {
t.Fatalf("canceled query session was retained: %#v", server.querySessions)
}
}
func TestPagedFetchHonorsOriginalStatementTimeout(t *testing.T) {
blocked := make(chan struct{})
behavior := &scriptedBehavior{}
behavior.query = func(ctx context.Context, _ string) (driver.Rows, error) {
rows := newScriptedRows(ctx, []string{"id"}, []string{"BIGINT"}, [][]driver.Value{{int64(1)}, {int64(2)}, {int64(3)}})
rows.blockAfter = 2
rows.blocked = blocked
return rows, nil
}
server := newScriptedServer(t, behavior)
first, err := server.executeQueryPage(queryOptions{SQL: "SELECT id FROM slow_table", MaxRows: 10, TimeoutSecs: 1}, 1)
if err != nil {
t.Fatal(err)
}
if first.SessionID == nil {
t.Fatal("expected a paged query session")
}
started := time.Now()
_, fetchErr := server.fetchQueryPage(*first.SessionID, 1)
if !errors.Is(fetchErr, context.DeadlineExceeded) {
t.Fatalf("expected deadline exceeded, got %v", fetchErr)
}
if elapsed := time.Since(started); elapsed > 3*time.Second {
t.Fatalf("statement timeout was not enforced promptly: %s", elapsed)
}
if len(server.querySessions) != 0 {
t.Fatalf("timed out query session was retained: %#v", server.querySessions)
}
}
func TestRuntimeSessionsRemainIsolated(t *testing.T) {
behaviorA := &scriptedBehavior{query: func(ctx context.Context, _ string) (driver.Rows, error) {
return newScriptedRows(ctx, []string{"owner"}, []string{"STRING"}, [][]driver.Value{{"a"}}), nil
}}
behaviorB := &scriptedBehavior{query: func(ctx context.Context, _ string) (driver.Rows, error) {
return newScriptedRows(ctx, []string{"owner"}, []string{"STRING"}, [][]driver.Value{{"b"}}), nil
}}
serverA := newScriptedServer(t, behaviorA)
serverB := newScriptedServer(t, behaviorB)
runtimeServer := newRuntimeServer()
runtimeServer.sessions["a"] = &agentSession{server: serverA}
runtimeServer.sessions["b"] = &agentSession{server: serverB}
resultA, _, err := runtimeServer.dispatch("execute_query", rawParams(map[string]any{
"agentSessionId": "a",
"sql": "SELECT owner",
}))
if err != nil {
t.Fatal(err)
}
resultB, _, err := runtimeServer.dispatch("execute_query", rawParams(map[string]any{
"agentSessionId": "b",
"sql": "SELECT owner",
}))
if err != nil {
t.Fatal(err)
}
if resultA.(queryResult).Rows[0][0] != "a" || resultB.(queryResult).Rows[0][0] != "b" {
t.Fatalf("session results crossed: a=%#v b=%#v", resultA, resultB)
}
if err := runtimeServer.closeSession("a"); err != nil {
t.Fatal(err)
}
if _, err := runtimeServer.session("a"); err == nil {
t.Fatal("closed session a is still registered")
}
if _, err := runtimeServer.session("b"); err != nil {
t.Fatalf("closing session a affected session b: %v", err)
}
}
func TestTransactionFallbackDoesNotReplayFailedStatements(t *testing.T) {
behavior := &scriptedBehavior{}
behavior.exec = func(_ context.Context, query string) (driver.Result, error) {
if strings.Contains(query, "second") {
return nil, io.EOF
}
return driver.RowsAffected(1), nil
}
server := newScriptedServer(t, behavior)
_, err := server.executeStatements(rawParams(map[string]any{
"statements": []string{"INSERT first", "INSERT second", "INSERT third"},
}), true)
if !errors.Is(err, io.EOF) {
t.Fatalf("expected connection failure, got %v", err)
}
_, executions, beginCalls, _ := behavior.snapshot()
if !reflect.DeepEqual(executions, []string{"INSERT first", "INSERT second"}) {
t.Fatalf("failed transaction was replayed or continued: %v", executions)
}
if beginCalls == 0 {
t.Fatal("transaction capability was not attempted before fallback")
}
rpcErr := classifyRPCError("execute_transaction", "session-a", err)
if rpcErr.Data.Category != "connection" || rpcErr.Data.SessionDisposition != "quarantine" {
t.Fatalf("unexpected connection failure classification: %#v", rpcErr)
}
}
func TestTransactionDoesNotFallbackAfterOperationalBeginFailure(t *testing.T) {
behavior := &scriptedBehavior{
beginErr: errors.New("connection reset while beginning transaction"),
exec: func(context.Context, string) (driver.Result, error) {
return driver.RowsAffected(1), nil
},
}
server := newScriptedServer(t, behavior)
defer server.disconnect()
params := map[string]json.RawMessage{
"statements": json.RawMessage(`["INSERT INTO sample VALUES (1)"]`),
}
if _, err := server.executeStatements(params, true); err == nil || !strings.Contains(err.Error(), "connection reset") {
t.Fatalf("expected begin failure, got %v", err)
}
_, executions, _, _ := behavior.snapshot()
if len(executions) != 0 {
t.Fatalf("statements must not execute after begin failure: %#v", executions)
}
}
func TestExpireIdleQuerySessions(t *testing.T) {
behavior := &scriptedBehavior{query: func(ctx context.Context, _ string) (driver.Rows, error) {
return newScriptedRows(ctx, []string{"id"}, []string{"BIGINT"}, [][]driver.Value{{int64(1)}, {int64(2)}}), nil
}}
server := newScriptedServer(t, behavior)
first, err := server.executeQueryPage(queryOptions{SQL: "SELECT id", MaxRows: 10}, 1)
if err != nil {
t.Fatal(err)
}
if first.SessionID == nil {
t.Fatal("expected a query session")
}
server.querySessions[*first.SessionID].lastAccessed = time.Now().Add(-querySessionIdleTime - time.Second)
if expired := server.expireIdleQuerySessions(time.Now()); expired != 1 {
t.Fatalf("unexpected expired session count: %d", expired)
}
if len(server.querySessions) != 0 {
t.Fatalf("idle session was retained: %#v", server.querySessions)
}
}
func TestStructuredHiveErrorIncludesServerDiagnostics(t *testing.T) {
err := &gohive.Error{
Err: errors.New("compile failed"),
Message: "SemanticException",
ErrorCode: 40000,
SQLState: "42000",
}
rpcErr := classifyRPCError("execute_query", "session-a", err)
if rpcErr.Data.Category != "sql" || rpcErr.Data.SQLState != "42000" || rpcErr.Data.VendorCode != 40000 {
t.Fatalf("unexpected Hive diagnostics: %#v", rpcErr)
}
if rpcErr.Data.SessionDisposition != "keep" || rpcErr.Data.OperationOutcome != "unknown" {
t.Fatalf("unexpected SQL failure recovery hints: %#v", rpcErr)
}
}
@@ -0,0 +1,568 @@
package main
import (
"bytes"
"crypto/tls"
"encoding/binary"
"errors"
"fmt"
"io"
"math"
"math/rand/v2"
"net"
"strings"
"sync"
"time"
"github.com/beltran/gosasl"
"github.com/go-zookeeper/zk"
)
const (
zooKeeperProtocolVersion = int32(0)
zooKeeperOpGetData = int32(4)
zooKeeperOpGetChildren2 = int32(12)
zooKeeperOpClose = int32(-11)
zooKeeperOpSetAuth = int32(100)
zooKeeperOpSASL = int32(102)
zooKeeperMaxFrameSize = 16 << 20
zooKeeperMaxSASLRounds = 8
)
var errZooKeeperSessionClosedRequiresSASL = errors.New("ZooKeeper session closed because SASL authentication is required")
type zooKeeperSASLClient interface {
Start() ([]byte, error)
Step([]byte) ([]byte, error)
Complete() bool
Dispose()
}
var newZooKeeperSASLClient = func(host string, config connectionConfig) (zooKeeperSASLClient, error) {
service, options := zooKeeperGSSAPIOptions(config)
mechanism, err := gosasl.NewGSSAPIMechanismWithOptions(service, options)
if err != nil {
return nil, err
}
return gosasl.NewSaslClient(host, mechanism), nil
}
var dialZooKeeperConnection = func(address string, timeout time.Duration, tlsConfig *tls.Config) (net.Conn, error) {
dialer := &net.Dialer{Timeout: timeout}
if tlsConfig == nil {
return dialer.Dial("tcp", address)
}
config := tlsConfig.Clone()
if config.ServerName == "" {
host, _, err := net.SplitHostPort(address)
if err != nil {
return nil, fmt.Errorf("parse ZooKeeper TLS address %q: %w", address, err)
}
config.ServerName = host
}
return tls.DialWithDialer(dialer, "tcp", address, config)
}
var shuffleZooKeeperServers = func(servers []string) {
rand.Shuffle(len(servers), func(first, second int) {
servers[first], servers[second] = servers[second], servers[first]
})
}
func zooKeeperGSSAPIOptions(config connectionConfig) (string, gosasl.GSSAPIOptions) {
service := firstNonEmpty(config.ZooKeeperKerberos.Service, "zookeeper")
options := gssapiOptionsFromKerberos(config.Kerberos)
options.QOP = "auth"
options.AuthorizationID = ""
options.ServiceHost = ""
options.CanonicalizeHost = config.ZooKeeperKerberos.CanonicalHostname
options.ServerName = config.ZooKeeperKerberos.ServerPrincipal
if options.ServerName == "" && config.ZooKeeperKerberos.Realm != "" {
options.ServerName = service + "/_HOST@" + config.ZooKeeperKerberos.Realm
}
return service, options
}
func connectKerberosZooKeeper(
servers []string,
timeout time.Duration,
tlsConfig *tls.Config,
config connectionConfig,
) (zooKeeperClient, <-chan zk.Event, error) {
if len(servers) == 0 {
return nil, nil, errors.New("ZooKeeper server list is empty")
}
if !config.Kerberos.Enabled {
return nil, nil, errors.New("ZooKeeper Kerberos SASL requires Hive Kerberos credentials")
}
ordered := append([]string(nil), servers...)
shuffleZooKeeperServers(ordered)
var failures []string
for _, address := range ordered {
host, _, err := net.SplitHostPort(address)
if err != nil {
failures = append(failures, fmt.Sprintf("%s: %v", address, err))
continue
}
connection, err := dialZooKeeperConnection(address, timeout, tlsConfig)
if err != nil {
failures = append(failures, fmt.Sprintf("%s: %v", address, err))
continue
}
client, err := newProtocolZooKeeperClient(connection, timeout)
if err == nil {
var saslClient zooKeeperSASLClient
saslClient, err = newZooKeeperSASLClient(host, config)
if err == nil {
err = client.authenticateSASL(saslClient)
}
}
if err != nil {
connection.Close()
failures = append(failures, fmt.Sprintf("%s: %v", address, err))
continue
}
events := make(chan zk.Event, 1)
events <- zk.Event{State: zk.StateHasSession, Server: address}
close(events)
return client, events, nil
}
return nil, nil, fmt.Errorf("connect and authenticate to ZooKeeper: %s", strings.Join(failures, "; "))
}
type protocolZooKeeperClient struct {
connection net.Conn
timeout time.Duration
xid int32
mutex sync.Mutex
closed bool
}
func newProtocolZooKeeperClient(connection net.Conn, timeout time.Duration) (*protocolZooKeeperClient, error) {
if connection == nil {
return nil, errors.New("ZooKeeper connection is nil")
}
if timeout <= 0 {
timeout = defaultConnectTimeout
}
client := &protocolZooKeeperClient{connection: connection, timeout: timeout}
request := &zooKeeperEncoder{}
request.int32(zooKeeperProtocolVersion)
request.int64(0)
request.int32(zooKeeperTimeoutMillis(timeout))
request.int64(0)
request.bytes(make([]byte, 16))
if err := client.writeFrame(request.data()); err != nil {
return nil, fmt.Errorf("send ZooKeeper connect request: %w", err)
}
response, err := client.readFrame()
if err != nil {
return nil, fmt.Errorf("read ZooKeeper connect response: %w", err)
}
decoder := newZooKeeperDecoder(response)
if _, err := decoder.int32(); err != nil {
return nil, fmt.Errorf("decode ZooKeeper protocol version: %w", err)
}
if _, err := decoder.int32(); err != nil {
return nil, fmt.Errorf("decode ZooKeeper session timeout: %w", err)
}
sessionID, err := decoder.int64()
if err != nil {
return nil, fmt.Errorf("decode ZooKeeper session ID: %w", err)
}
if _, err := decoder.bytes(); err != nil {
return nil, fmt.Errorf("decode ZooKeeper session password: %w", err)
}
if sessionID == 0 {
return nil, zk.ErrSessionExpired
}
return client, nil
}
func zooKeeperTimeoutMillis(timeout time.Duration) int32 {
milliseconds := timeout.Milliseconds()
if milliseconds < 1 {
return 1
}
if milliseconds > math.MaxInt32 {
return math.MaxInt32
}
return int32(milliseconds)
}
func (client *protocolZooKeeperClient) authenticateSASL(saslClient zooKeeperSASLClient) error {
if saslClient == nil {
return errors.New("ZooKeeper SASL client is nil")
}
defer saslClient.Dispose()
token, err := saslClient.Start()
if err != nil {
return fmt.Errorf("start ZooKeeper GSSAPI negotiation: %w", err)
}
for round := 0; round < zooKeeperMaxSASLRounds; round++ {
response, requestErr := client.request(zooKeeperOpSASL, func(encoder *zooKeeperEncoder) {
if token == nil {
encoder.bytes([]byte{})
return
}
encoder.bytes(token)
})
if requestErr != nil {
return fmt.Errorf("ZooKeeper SASL round %d: %w", round+1, requestErr)
}
decoder := newZooKeeperDecoder(response)
challenge, decodeErr := decoder.bytes()
if decodeErr != nil {
return fmt.Errorf("decode ZooKeeper SASL round %d: %w", round+1, decodeErr)
}
if saslClient.Complete() {
if len(challenge) != 0 {
return errors.New("ZooKeeper sent an unexpected token after GSSAPI completion")
}
return nil
}
token, err = saslClient.Step(challenge)
if err != nil {
return fmt.Errorf("continue ZooKeeper GSSAPI negotiation at round %d: %w", round+1, err)
}
}
return fmt.Errorf("ZooKeeper GSSAPI negotiation exceeded %d rounds", zooKeeperMaxSASLRounds)
}
func (client *protocolZooKeeperClient) AddAuth(scheme string, auth []byte) error {
_, err := client.request(zooKeeperOpSetAuth, func(encoder *zooKeeperEncoder) {
encoder.int32(0)
encoder.string(scheme)
encoder.bytes(auth)
})
return err
}
func (client *protocolZooKeeperClient) Children(path string) ([]string, *zk.Stat, error) {
response, err := client.request(zooKeeperOpGetChildren2, func(encoder *zooKeeperEncoder) {
encoder.string(path)
encoder.boolean(false)
})
if err != nil {
return nil, nil, err
}
decoder := newZooKeeperDecoder(response)
children, err := decoder.strings()
if err != nil {
return nil, nil, err
}
stat, err := decoder.stat()
if err != nil {
return nil, nil, err
}
return children, stat, nil
}
func (client *protocolZooKeeperClient) Get(path string) ([]byte, *zk.Stat, error) {
response, err := client.request(zooKeeperOpGetData, func(encoder *zooKeeperEncoder) {
encoder.string(path)
encoder.boolean(false)
})
if err != nil {
return nil, nil, err
}
decoder := newZooKeeperDecoder(response)
data, err := decoder.bytes()
if err != nil {
return nil, nil, err
}
stat, err := decoder.stat()
if err != nil {
return nil, nil, err
}
return data, stat, nil
}
func (client *protocolZooKeeperClient) Close() {
client.mutex.Lock()
defer client.mutex.Unlock()
if client.closed {
return
}
client.closed = true
_ = client.connection.SetDeadline(time.Now().Add(client.timeout))
client.xid++
request := &zooKeeperEncoder{}
request.int32(client.xid)
request.int32(zooKeeperOpClose)
_ = client.writeFrame(request.data())
_ = client.connection.Close()
}
func (client *protocolZooKeeperClient) request(opcode int32, encodeBody func(*zooKeeperEncoder)) ([]byte, error) {
client.mutex.Lock()
defer client.mutex.Unlock()
if client.closed {
return nil, zk.ErrConnectionClosed
}
client.xid++
request := &zooKeeperEncoder{}
request.int32(client.xid)
request.int32(opcode)
if encodeBody != nil {
encodeBody(request)
}
if err := client.writeFrame(request.data()); err != nil {
return nil, err
}
response, err := client.readFrame()
if err != nil {
return nil, err
}
decoder := newZooKeeperDecoder(response)
xid, err := decoder.int32()
if err != nil {
return nil, err
}
if xid != client.xid {
return nil, fmt.Errorf("ZooKeeper response XID %d does not match request XID %d", xid, client.xid)
}
if _, err := decoder.int64(); err != nil {
return nil, err
}
code, err := decoder.int32()
if err != nil {
return nil, err
}
if err := zooKeeperError(code); err != nil {
return nil, err
}
return decoder.remaining(), nil
}
func (client *protocolZooKeeperClient) writeFrame(payload []byte) error {
if len(payload) > zooKeeperMaxFrameSize {
return fmt.Errorf("ZooKeeper request frame is %d bytes, maximum is %d", len(payload), zooKeeperMaxFrameSize)
}
if err := client.connection.SetWriteDeadline(time.Now().Add(client.timeout)); err != nil {
return err
}
header := make([]byte, 4)
binary.BigEndian.PutUint32(header, uint32(len(payload)))
if err := writeAll(client.connection, header); err != nil {
return err
}
return writeAll(client.connection, payload)
}
func (client *protocolZooKeeperClient) readFrame() ([]byte, error) {
if err := client.connection.SetReadDeadline(time.Now().Add(client.timeout)); err != nil {
return nil, err
}
header := make([]byte, 4)
if _, err := io.ReadFull(client.connection, header); err != nil {
return nil, err
}
length := int(binary.BigEndian.Uint32(header))
if length < 0 || length > zooKeeperMaxFrameSize {
return nil, fmt.Errorf("ZooKeeper response frame is %d bytes, maximum is %d", length, zooKeeperMaxFrameSize)
}
payload := make([]byte, length)
if _, err := io.ReadFull(client.connection, payload); err != nil {
return nil, err
}
return payload, nil
}
func writeAll(writer io.Writer, payload []byte) error {
for len(payload) > 0 {
written, err := writer.Write(payload)
if err != nil {
return err
}
if written <= 0 {
return io.ErrShortWrite
}
payload = payload[written:]
}
return nil
}
func zooKeeperError(code int32) error {
switch code {
case 0:
return nil
case -4:
return zk.ErrConnectionClosed
case -101:
return zk.ErrNoNode
case -102:
return zk.ErrNoAuth
case -112:
return zk.ErrSessionExpired
case -115:
return zk.ErrAuthFailed
case -124:
return errZooKeeperSessionClosedRequiresSASL
default:
return fmt.Errorf("ZooKeeper request failed with error code %d", code)
}
}
type zooKeeperEncoder struct {
buffer bytes.Buffer
}
func (encoder *zooKeeperEncoder) int32(value int32) {
var data [4]byte
binary.BigEndian.PutUint32(data[:], uint32(value))
encoder.buffer.Write(data[:])
}
func (encoder *zooKeeperEncoder) int64(value int64) {
var data [8]byte
binary.BigEndian.PutUint64(data[:], uint64(value))
encoder.buffer.Write(data[:])
}
func (encoder *zooKeeperEncoder) boolean(value bool) {
if value {
encoder.buffer.WriteByte(1)
return
}
encoder.buffer.WriteByte(0)
}
func (encoder *zooKeeperEncoder) string(value string) {
encoder.bytes([]byte(value))
}
func (encoder *zooKeeperEncoder) bytes(value []byte) {
if value == nil {
encoder.int32(-1)
return
}
encoder.int32(int32(len(value)))
encoder.buffer.Write(value)
}
func (encoder *zooKeeperEncoder) data() []byte {
return encoder.buffer.Bytes()
}
type zooKeeperDecoder struct {
data []byte
offset int
}
func newZooKeeperDecoder(data []byte) *zooKeeperDecoder {
return &zooKeeperDecoder{data: data}
}
func (decoder *zooKeeperDecoder) take(length int) ([]byte, error) {
if length < 0 || decoder.offset > len(decoder.data)-length {
return nil, io.ErrUnexpectedEOF
}
value := decoder.data[decoder.offset : decoder.offset+length]
decoder.offset += length
return value, nil
}
func (decoder *zooKeeperDecoder) int32() (int32, error) {
value, err := decoder.take(4)
if err != nil {
return 0, err
}
return int32(binary.BigEndian.Uint32(value)), nil
}
func (decoder *zooKeeperDecoder) int64() (int64, error) {
value, err := decoder.take(8)
if err != nil {
return 0, err
}
return int64(binary.BigEndian.Uint64(value)), nil
}
func (decoder *zooKeeperDecoder) bytes() ([]byte, error) {
length, err := decoder.int32()
if err != nil {
return nil, err
}
if length == -1 {
return nil, nil
}
if length < -1 {
return nil, fmt.Errorf("invalid ZooKeeper buffer length %d", length)
}
value, err := decoder.take(int(length))
if err != nil {
return nil, err
}
return append([]byte(nil), value...), nil
}
func (decoder *zooKeeperDecoder) string() (string, error) {
value, err := decoder.bytes()
return string(value), err
}
func (decoder *zooKeeperDecoder) strings() ([]string, error) {
length, err := decoder.int32()
if err != nil {
return nil, err
}
if length == -1 {
return nil, nil
}
if length < -1 || length > zooKeeperMaxFrameSize/4 {
return nil, fmt.Errorf("invalid ZooKeeper string vector length %d", length)
}
values := make([]string, 0, length)
for index := int32(0); index < length; index++ {
value, valueErr := decoder.string()
if valueErr != nil {
return nil, valueErr
}
values = append(values, value)
}
return values, nil
}
func (decoder *zooKeeperDecoder) stat() (*zk.Stat, error) {
stat := &zk.Stat{}
var err error
if stat.Czxid, err = decoder.int64(); err != nil {
return nil, err
}
if stat.Mzxid, err = decoder.int64(); err != nil {
return nil, err
}
if stat.Ctime, err = decoder.int64(); err != nil {
return nil, err
}
if stat.Mtime, err = decoder.int64(); err != nil {
return nil, err
}
if stat.Version, err = decoder.int32(); err != nil {
return nil, err
}
if stat.Cversion, err = decoder.int32(); err != nil {
return nil, err
}
if stat.Aversion, err = decoder.int32(); err != nil {
return nil, err
}
if stat.EphemeralOwner, err = decoder.int64(); err != nil {
return nil, err
}
if stat.DataLength, err = decoder.int32(); err != nil {
return nil, err
}
if stat.NumChildren, err = decoder.int32(); err != nil {
return nil, err
}
if stat.Pzxid, err = decoder.int64(); err != nil {
return nil, err
}
return stat, nil
}
func (decoder *zooKeeperDecoder) remaining() []byte {
return decoder.data[decoder.offset:]
}
@@ -0,0 +1,562 @@
package main
import (
"bytes"
"crypto/tls"
"errors"
"fmt"
"io"
"log"
"net"
"os"
"path/filepath"
"testing"
"time"
"github.com/go-zookeeper/zk"
gsskrb5 "github.com/golang-auth/go-gssapi/v2/krb5"
"github.com/jcmturner/krb5test"
)
type scriptedZooKeeperSASLClient struct {
complete bool
disposed bool
}
func (client *scriptedZooKeeperSASLClient) Start() ([]byte, error) {
return []byte("client-initial"), nil
}
func (client *scriptedZooKeeperSASLClient) Step(challenge []byte) ([]byte, error) {
if string(challenge) != "server-challenge" {
return nil, fmt.Errorf("unexpected challenge %q", challenge)
}
client.complete = true
return []byte("client-final"), nil
}
func (client *scriptedZooKeeperSASLClient) Complete() bool { return client.complete }
func (client *scriptedZooKeeperSASLClient) Dispose() { client.disposed = true }
func TestProtocolZooKeeperClientAuthenticatesAndReadsDiscoveryData(t *testing.T) {
clientConnection, serverConnection := net.Pipe()
serverErrors := make(chan error, 1)
go func() {
defer serverConnection.Close()
serverErrors <- serveZooKeeperProtocolTest(serverConnection)
}()
client, err := newProtocolZooKeeperClient(clientConnection, time.Second)
if err != nil {
t.Fatal(err)
}
saslClient := &scriptedZooKeeperSASLClient{}
if err := client.authenticateSASL(saslClient); err != nil {
t.Fatal(err)
}
if !saslClient.disposed {
t.Fatal("SASL credentials were not disposed")
}
if err := client.AddAuth("digest", []byte("user:password")); err != nil {
t.Fatal(err)
}
children, stat, err := client.Children("/hiveserver2")
if err != nil {
t.Fatal(err)
}
if len(children) != 1 || children[0] != "server-1" || stat.NumChildren != 1 {
t.Fatalf("children=%#v stat=%#v", children, stat)
}
data, stat, err := client.Get("/hiveserver2/server-1")
if err != nil {
t.Fatal(err)
}
if string(data) != "serverUri=hs2.example.com:10000" || stat.DataLength != int32(len(data)) {
t.Fatalf("data=%q stat=%#v", data, stat)
}
client.Close()
if err := <-serverErrors; err != nil {
t.Fatal(err)
}
}
func TestProtocolZooKeeperClientMapsRequiredSASLError(t *testing.T) {
clientConnection, serverConnection := net.Pipe()
serverErrors := make(chan error, 1)
go func() {
defer serverConnection.Close()
if err := acceptZooKeeperSession(serverConnection); err != nil {
serverErrors <- err
return
}
request, err := readZooKeeperTestFrame(serverConnection)
if err != nil {
serverErrors <- err
return
}
decoder := newZooKeeperDecoder(request)
xid, _ := decoder.int32()
opcode, _ := decoder.int32()
if opcode != zooKeeperOpSASL {
serverErrors <- fmt.Errorf("opcode=%d", opcode)
return
}
serverErrors <- writeZooKeeperTestResponse(serverConnection, xid, -124, nil)
}()
client, err := newProtocolZooKeeperClient(clientConnection, time.Second)
if err != nil {
t.Fatal(err)
}
saslClient := &scriptedZooKeeperSASLClient{}
err = client.authenticateSASL(saslClient)
client.Close()
if !errors.Is(err, errZooKeeperSessionClosedRequiresSASL) {
t.Fatalf("unexpected error %v", err)
}
if serverErr := <-serverErrors; serverErr != nil {
t.Fatal(serverErr)
}
}
func TestZooKeeperGSSAPIOptionsDoNotReuseHiveServerIdentity(t *testing.T) {
config := connectionConfig{
Kerberos: kerberosConfig{
ServerPrincipal: "hive/_HOST@HIVE.EXAMPLE.COM",
ClientPrincipal: "alice@CLIENT.EXAMPLE.COM",
AuthorizationID: "hive-proxy",
QOP: "auth-conf",
ConfigPath: "/etc/krb5.conf",
CCachePath: "/tmp/alice.ccache",
UseTicketCache: true,
CanonicalHostname: true,
},
ZooKeeperKerberos: zooKeeperKerberosConfig{
Enabled: true,
Service: "zookeeper",
Realm: "ZK.EXAMPLE.COM",
CanonicalHostname: false,
},
}
service, options := zooKeeperGSSAPIOptions(config)
if service != "zookeeper" || options.ServerName != "zookeeper/_HOST@ZK.EXAMPLE.COM" {
t.Fatalf("service=%q options=%#v", service, options)
}
if options.QOP != "auth" || options.AuthorizationID != "" || options.CanonicalizeHost || options.Principal != "alice@CLIENT.EXAMPLE.COM" || options.CCachePath != "/tmp/alice.ccache" {
t.Fatalf("unexpected ZooKeeper GSSAPI options: %#v", options)
}
}
func TestConnectKerberosZooKeeperFailsOverAndUsesTargetHost(t *testing.T) {
previousDial := dialZooKeeperConnection
previousFactory := newZooKeeperSASLClient
previousShuffle := shuffleZooKeeperServers
t.Cleanup(func() {
dialZooKeeperConnection = previousDial
newZooKeeperSASLClient = previousFactory
shuffleZooKeeperServers = previousShuffle
})
shuffleZooKeeperServers = func([]string) {}
clientConnection, serverConnection := net.Pipe()
serverErrors := make(chan error, 1)
go func() {
defer serverConnection.Close()
if err := acceptZooKeeperSession(serverConnection); err != nil {
serverErrors <- err
return
}
if err := expectZooKeeperSASLRound(serverConnection, "client-initial", []byte("server-challenge")); err != nil {
serverErrors <- err
return
}
if err := expectZooKeeperSASLRound(serverConnection, "client-final", nil); err != nil {
serverErrors <- err
return
}
request, err := readZooKeeperTestFrame(serverConnection)
if err != nil {
serverErrors <- err
return
}
decoder := newZooKeeperDecoder(request)
_, _ = decoder.int32()
opcode, _ := decoder.int32()
if opcode != zooKeeperOpClose {
serverErrors <- fmt.Errorf("unexpected close opcode %d", opcode)
return
}
serverErrors <- nil
}()
var dialed []string
dialZooKeeperConnection = func(address string, _ time.Duration, _ *tls.Config) (net.Conn, error) {
dialed = append(dialed, address)
if address == "first.example.com:2181" {
return nil, errors.New("first unavailable")
}
return clientConnection, nil
}
var targetHost string
newZooKeeperSASLClient = func(host string, _ connectionConfig) (zooKeeperSASLClient, error) {
targetHost = host
return &scriptedZooKeeperSASLClient{}, nil
}
config := connectionConfig{
Kerberos: kerberosConfig{Enabled: true},
ZooKeeperKerberos: zooKeeperKerberosConfig{Enabled: true},
}
client, events, err := connectKerberosZooKeeper(
[]string{"first.example.com:2181", "second.example.com:2181"},
time.Second,
nil,
config,
)
if err != nil {
t.Fatal(err)
}
if len(dialed) != 2 || dialed[0] != "first.example.com:2181" || dialed[1] != "second.example.com:2181" {
t.Fatalf("dialed = %#v", dialed)
}
if targetHost != "second.example.com" {
t.Fatalf("target host = %q", targetHost)
}
event := <-events
if event.State != zk.StateHasSession || event.Server != "second.example.com:2181" {
t.Fatalf("event = %#v", event)
}
client.Close()
if err := <-serverErrors; err != nil {
t.Fatal(err)
}
}
func TestZooKeeperKerberosSASLWithMiniKDC(t *testing.T) {
logger := log.New(io.Discard, "", 0)
kdc, err := krb5test.NewKDC(map[string][]string{
"alice": nil,
"zookeeper/localhost": nil,
}, logger)
if err != nil {
t.Fatal(err)
}
kdc.KRB5Conf.LibDefaults.UDPPreferenceLimit = 1
kdc.Start()
defer kdc.Close()
directory := t.TempDir()
configPath := filepath.Join(directory, "krb5.conf")
keytabPath := filepath.Join(directory, "zookeeper.keytab")
configContents := fmt.Sprintf(`[libdefaults]
default_realm = %s
dns_lookup_realm = false
dns_lookup_kdc = false
rdns = false
udp_preference_limit = 1
default_tgs_enctypes = aes256-cts-hmac-sha1-96
default_tkt_enctypes = aes256-cts-hmac-sha1-96
permitted_enctypes = aes256-cts-hmac-sha1-96
[realms]
%s = {
kdc = %s
}
`, kdc.Realm, kdc.Realm, kdc.TCPListener.Addr().String())
if err := os.WriteFile(configPath, []byte(configContents), 0o600); err != nil {
t.Fatal(err)
}
keytabBytes, err := kdc.Keytab.Marshal()
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(keytabPath, keytabBytes, 0o600); err != nil {
t.Fatal(err)
}
t.Setenv("KRB5_KTNAME", keytabPath)
t.Setenv("KRB5_CLIENT_KTNAME", keytabPath)
clientConnection, serverConnection := net.Pipe()
serverErrors := make(chan error, 1)
servicePrincipal := "zookeeper/localhost@" + kdc.Realm
go func() {
defer serverConnection.Close()
serverErrors <- serveKerberosZooKeeperSASL(serverConnection, servicePrincipal)
}()
protocolClient, err := newProtocolZooKeeperClient(clientConnection, 5*time.Second)
if err != nil {
t.Fatal(err)
}
config := connectionConfig{
Kerberos: kerberosConfig{
Enabled: true,
ClientPrincipal: "alice@" + kdc.Realm,
ConfigPath: configPath,
Password: kdc.Principals["alice"].Password,
DisablePAFXFAST: true,
},
ZooKeeperKerberos: zooKeeperKerberosConfig{
Enabled: true,
Service: "zookeeper",
ServerPrincipal: servicePrincipal,
CanonicalHostname: false,
},
}
saslClient, err := newZooKeeperSASLClient("localhost", config)
if err != nil {
t.Fatal(err)
}
if err := protocolClient.authenticateSASL(saslClient); err != nil {
t.Fatal(err)
}
protocolClient.Close()
if err := <-serverErrors; err != nil {
t.Fatal(err)
}
}
func serveZooKeeperProtocolTest(connection net.Conn) error {
if err := acceptZooKeeperSession(connection); err != nil {
return err
}
if err := expectZooKeeperSASLRound(connection, "client-initial", []byte("server-challenge")); err != nil {
return err
}
if err := expectZooKeeperSASLRound(connection, "client-final", nil); err != nil {
return err
}
request, err := readZooKeeperTestFrame(connection)
if err != nil {
return err
}
decoder := newZooKeeperDecoder(request)
xid, _ := decoder.int32()
opcode, _ := decoder.int32()
authType, _ := decoder.int32()
scheme, _ := decoder.string()
auth, _ := decoder.bytes()
if opcode != zooKeeperOpSetAuth || authType != 0 || scheme != "digest" || string(auth) != "user:password" {
return fmt.Errorf("unexpected auth request opcode=%d type=%d scheme=%q auth=%q", opcode, authType, scheme, auth)
}
if err := writeZooKeeperTestResponse(connection, xid, 0, nil); err != nil {
return err
}
request, err = readZooKeeperTestFrame(connection)
if err != nil {
return err
}
decoder = newZooKeeperDecoder(request)
xid, _ = decoder.int32()
opcode, _ = decoder.int32()
path, _ := decoder.string()
watch, _ := decoder.take(1)
if opcode != zooKeeperOpGetChildren2 || path != "/hiveserver2" || !bytes.Equal(watch, []byte{0}) {
return fmt.Errorf("unexpected children request opcode=%d path=%q watch=%v", opcode, path, watch)
}
body := &zooKeeperEncoder{}
body.int32(1)
body.string("server-1")
encodeZooKeeperTestStat(body, 0, 1)
if err := writeZooKeeperTestResponse(connection, xid, 0, body.data()); err != nil {
return err
}
request, err = readZooKeeperTestFrame(connection)
if err != nil {
return err
}
decoder = newZooKeeperDecoder(request)
xid, _ = decoder.int32()
opcode, _ = decoder.int32()
path, _ = decoder.string()
watch, _ = decoder.take(1)
if opcode != zooKeeperOpGetData || path != "/hiveserver2/server-1" || !bytes.Equal(watch, []byte{0}) {
return fmt.Errorf("unexpected get request opcode=%d path=%q watch=%v", opcode, path, watch)
}
data := []byte("serverUri=hs2.example.com:10000")
body = &zooKeeperEncoder{}
body.bytes(data)
encodeZooKeeperTestStat(body, int32(len(data)), 0)
if err := writeZooKeeperTestResponse(connection, xid, 0, body.data()); err != nil {
return err
}
request, err = readZooKeeperTestFrame(connection)
if err != nil {
return err
}
decoder = newZooKeeperDecoder(request)
_, _ = decoder.int32()
opcode, _ = decoder.int32()
if opcode != zooKeeperOpClose {
return fmt.Errorf("unexpected close opcode %d", opcode)
}
return nil
}
func serveKerberosZooKeeperSASL(connection net.Conn, servicePrincipal string) error {
if err := acceptZooKeeperSession(connection); err != nil {
return err
}
acceptor := gsskrb5.NewKrb5Mech()
if err := acceptor.Accept(servicePrincipal); err != nil {
return err
}
xid, token, err := readZooKeeperSASLRequest(connection)
if err != nil {
return err
}
apReply, err := acceptor.Continue(token)
if err != nil {
return err
}
if !acceptor.IsEstablished() {
return errors.New("Kerberos acceptor context was not established")
}
if err := writeZooKeeperSASLResponse(connection, xid, apReply); err != nil {
return err
}
xid, token, err = readZooKeeperSASLRequest(connection)
if err != nil {
return err
}
if len(token) != 0 {
return fmt.Errorf("expected empty post-AP-REP token, got %x", token)
}
securityChallenge, err := acceptor.Wrap([]byte{1, 0, 0, 0}, false)
if err != nil {
return err
}
if err := writeZooKeeperSASLResponse(connection, xid, securityChallenge); err != nil {
return err
}
xid, token, err = readZooKeeperSASLRequest(connection)
if err != nil {
return err
}
securityResponse, sealed, err := acceptor.Unwrap(token)
if err != nil {
return err
}
if sealed || len(securityResponse) < 4 || !bytes.Equal(securityResponse[:4], []byte{1, 0, 0, 0}) {
return fmt.Errorf("invalid GSSAPI security-layer response sealed=%v payload=%x", sealed, securityResponse)
}
if len(securityResponse[4:]) == 0 {
return errors.New("GSSAPI authorization identity is empty")
}
if err := writeZooKeeperSASLResponse(connection, xid, nil); err != nil {
return err
}
request, err := readZooKeeperTestFrame(connection)
if err != nil {
return err
}
decoder := newZooKeeperDecoder(request)
_, _ = decoder.int32()
opcode, _ := decoder.int32()
if opcode != zooKeeperOpClose {
return fmt.Errorf("unexpected close opcode %d", opcode)
}
return nil
}
func readZooKeeperSASLRequest(connection net.Conn) (int32, []byte, error) {
request, err := readZooKeeperTestFrame(connection)
if err != nil {
return 0, nil, err
}
decoder := newZooKeeperDecoder(request)
xid, err := decoder.int32()
if err != nil {
return 0, nil, err
}
opcode, err := decoder.int32()
if err != nil {
return 0, nil, err
}
if opcode != zooKeeperOpSASL {
return 0, nil, fmt.Errorf("unexpected SASL opcode %d", opcode)
}
token, err := decoder.bytes()
return xid, token, err
}
func writeZooKeeperSASLResponse(connection net.Conn, xid int32, token []byte) error {
body := &zooKeeperEncoder{}
body.bytes(token)
return writeZooKeeperTestResponse(connection, xid, 0, body.data())
}
func acceptZooKeeperSession(connection net.Conn) error {
request, err := readZooKeeperTestFrame(connection)
if err != nil {
return err
}
decoder := newZooKeeperDecoder(request)
version, _ := decoder.int32()
_, _ = decoder.int64()
timeout, _ := decoder.int32()
sessionID, _ := decoder.int64()
password, _ := decoder.bytes()
if version != 0 || timeout <= 0 || sessionID != 0 || len(password) != 16 {
return fmt.Errorf("invalid connect request version=%d timeout=%d session=%d password=%d", version, timeout, sessionID, len(password))
}
response := &zooKeeperEncoder{}
response.int32(0)
response.int32(timeout)
response.int64(42)
response.bytes(make([]byte, 16))
return writeZooKeeperTestFrame(connection, response.data())
}
func expectZooKeeperSASLRound(connection net.Conn, wantToken string, responseToken []byte) error {
request, err := readZooKeeperTestFrame(connection)
if err != nil {
return err
}
decoder := newZooKeeperDecoder(request)
xid, _ := decoder.int32()
opcode, _ := decoder.int32()
token, _ := decoder.bytes()
if opcode != zooKeeperOpSASL || string(token) != wantToken {
return fmt.Errorf("unexpected SASL request opcode=%d token=%q", opcode, token)
}
body := &zooKeeperEncoder{}
body.bytes(responseToken)
return writeZooKeeperTestResponse(connection, xid, 0, body.data())
}
func encodeZooKeeperTestStat(encoder *zooKeeperEncoder, dataLength, children int32) {
encoder.int64(1)
encoder.int64(2)
encoder.int64(3)
encoder.int64(4)
encoder.int32(5)
encoder.int32(6)
encoder.int32(7)
encoder.int64(8)
encoder.int32(dataLength)
encoder.int32(children)
encoder.int64(9)
}
func writeZooKeeperTestResponse(connection net.Conn, xid, code int32, body []byte) error {
response := &zooKeeperEncoder{}
response.int32(xid)
response.int64(0)
response.int32(code)
response.buffer.Write(body)
return writeZooKeeperTestFrame(connection, response.data())
}
func readZooKeeperTestFrame(connection net.Conn) ([]byte, error) {
client := &protocolZooKeeperClient{connection: connection, timeout: time.Second}
return client.readFrame()
}
func writeZooKeeperTestFrame(connection net.Conn, payload []byte) error {
client := &protocolZooKeeperClient{connection: connection, timeout: time.Second}
return client.writeFrame(payload)
}
var _ zooKeeperClient = (*protocolZooKeeperClient)(nil)
var _ = zk.StateSaslAuthenticated
+240
View File
@@ -0,0 +1,240 @@
package main
import (
"bytes"
"crypto/tls"
"crypto/x509"
"encoding/pem"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
keystore "github.com/pavlo-v-chernykh/keystore-go/v4"
pkcs12 "software.sslmate.com/src/go-pkcs12"
)
func buildZooKeeperTLSConfig(values map[string]string) (*tls.Config, error) {
if !parameterBool(values, "zookeepersslenable") {
return nil, nil
}
config := &tls.Config{
MinVersion: tls.VersionTLS12,
ServerName: parameter(values, "zookeeperservername"),
}
trustStoreLocation := parameter(values, "zookeepertruststorelocation")
if trustStoreLocation != "" {
certificates, err := loadTrustStore(
trustStoreLocation,
parameter(values, "zookeepertruststorepassword"),
parameter(values, "zookeepertruststoretype"),
)
if err != nil {
return nil, fmt.Errorf("load ZooKeeper truststore: %w", err)
}
pool := x509.NewCertPool()
for _, certificate := range certificates {
pool.AddCert(certificate)
}
config.RootCAs = pool
}
keyStoreLocation := parameter(values, "zookeeperkeystorelocation")
if keyStoreLocation != "" {
certificate, err := loadClientKeyStore(
keyStoreLocation,
parameter(values, "zookeeperkeystorepassword"),
parameter(values, "zookeeperkeystoretype"),
)
if err != nil {
return nil, fmt.Errorf("load ZooKeeper keystore: %w", err)
}
config.Certificates = []tls.Certificate{certificate}
}
if parameterBool(values, "zookeepersslinsecureskipverify") {
config.InsecureSkipVerify = true
}
return config, nil
}
func loadTrustStore(path, password, storeType string) ([]*x509.Certificate, error) {
contents, err := os.ReadFile(path)
if err != nil {
return nil, err
}
switch normalizedStoreType(storeType, path) {
case "PEM":
return parsePEMCertificates(contents)
case "PKCS12":
certificates, err := pkcs12.DecodeTrustStore(contents, password)
if err == nil {
return certificates, nil
}
_, certificate, chain, chainErr := pkcs12.DecodeChain(contents, password)
if chainErr != nil {
return nil, err
}
return append([]*x509.Certificate{certificate}, chain...), nil
case "JKS":
store, err := loadJKS(contents, password)
if err != nil {
return nil, err
}
var certificates []*x509.Certificate
for _, alias := range store.Aliases() {
switch {
case store.IsTrustedCertificateEntry(alias):
entry, getErr := store.GetTrustedCertificateEntry(alias)
if getErr != nil {
return nil, getErr
}
certificate, parseErr := x509.ParseCertificate(entry.Certificate.Content)
if parseErr != nil {
return nil, parseErr
}
certificates = append(certificates, certificate)
case store.IsPrivateKeyEntry(alias):
chain, getErr := store.GetPrivateKeyEntryCertificateChain(alias)
if getErr != nil {
return nil, getErr
}
for _, entry := range chain {
certificate, parseErr := x509.ParseCertificate(entry.Content)
if parseErr != nil {
return nil, parseErr
}
certificates = append(certificates, certificate)
}
}
}
if len(certificates) == 0 {
return nil, errors.New("JKS truststore contains no certificates")
}
return certificates, nil
default:
return nil, fmt.Errorf("unsupported store type %q", storeType)
}
}
func loadClientKeyStore(path, password, storeType string) (tls.Certificate, error) {
contents, err := os.ReadFile(path)
if err != nil {
return tls.Certificate{}, err
}
switch normalizedStoreType(storeType, path) {
case "PEM":
return tls.X509KeyPair(contents, contents)
case "PKCS12":
privateKey, certificate, chain, err := pkcs12.DecodeChain(contents, password)
if err != nil {
return tls.Certificate{}, err
}
result := tls.Certificate{PrivateKey: privateKey, Leaf: certificate}
result.Certificate = append(result.Certificate, certificate.Raw)
for _, entry := range chain {
result.Certificate = append(result.Certificate, entry.Raw)
}
return result, nil
case "JKS":
store, err := loadJKS(contents, password)
if err != nil {
return tls.Certificate{}, err
}
passwordBytes := []byte(password)
defer clear(passwordBytes)
for _, alias := range store.Aliases() {
if !store.IsPrivateKeyEntry(alias) {
continue
}
entry, getErr := store.GetPrivateKeyEntry(alias, passwordBytes)
if getErr != nil {
return tls.Certificate{}, getErr
}
privateKey, parseErr := parsePrivateKey(entry.PrivateKey)
if parseErr != nil {
return tls.Certificate{}, parseErr
}
result := tls.Certificate{PrivateKey: privateKey}
for index, certificate := range entry.CertificateChain {
result.Certificate = append(result.Certificate, certificate.Content)
if index == 0 {
result.Leaf, _ = x509.ParseCertificate(certificate.Content)
}
}
if len(result.Certificate) == 0 {
return tls.Certificate{}, errors.New("JKS private key entry has no certificate chain")
}
return result, nil
}
return tls.Certificate{}, errors.New("JKS keystore contains no private key entry")
default:
return tls.Certificate{}, fmt.Errorf("unsupported store type %q", storeType)
}
}
func normalizedStoreType(storeType, path string) string {
value := strings.ToUpper(strings.TrimSpace(storeType))
switch value {
case "P12", "PFX", "PKCS#12":
return "PKCS12"
case "X509", "X.509":
return "PEM"
case "":
switch strings.ToLower(filepath.Ext(path)) {
case ".jks":
return "JKS"
case ".p12", ".pfx", ".pkcs12":
return "PKCS12"
default:
return "PEM"
}
default:
return value
}
}
func loadJKS(contents []byte, password string) (keystore.KeyStore, error) {
store := keystore.New()
passwordBytes := []byte(password)
defer clear(passwordBytes)
if err := store.Load(bytes.NewReader(contents), passwordBytes); err != nil {
return keystore.KeyStore{}, err
}
return store, nil
}
func parsePEMCertificates(contents []byte) ([]*x509.Certificate, error) {
var certificates []*x509.Certificate
for len(contents) > 0 {
block, rest := pem.Decode(contents)
if block == nil {
break
}
contents = rest
if block.Type != "CERTIFICATE" {
continue
}
certificate, err := x509.ParseCertificate(block.Bytes)
if err != nil {
return nil, err
}
certificates = append(certificates, certificate)
}
if len(certificates) == 0 {
return nil, errors.New("PEM truststore contains no certificates")
}
return certificates, nil
}
func parsePrivateKey(contents []byte) (any, error) {
if value, err := x509.ParsePKCS8PrivateKey(contents); err == nil {
return value, nil
}
if value, err := x509.ParsePKCS1PrivateKey(contents); err == nil {
return value, nil
}
if value, err := x509.ParseECPrivateKey(contents); err == nil {
return value, nil
}
return nil, errors.New("unsupported private key encoding")
}
@@ -0,0 +1,162 @@
package main
import (
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"math/big"
"os"
"path/filepath"
"testing"
"time"
keystore "github.com/pavlo-v-chernykh/keystore-go/v4"
pkcs12 "software.sslmate.com/src/go-pkcs12"
)
func testZooKeeperCertificate(t *testing.T) (*rsa.PrivateKey, *x509.Certificate, []byte) {
t.Helper()
privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
t.Fatal(err)
}
template := &x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{CommonName: "zk.example.com"},
DNSNames: []string{"zk.example.com"},
NotBefore: time.Now().Add(-time.Hour),
NotAfter: time.Now().Add(time.Hour),
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
IsCA: true,
}
raw, err := x509.CreateCertificate(rand.Reader, template, template, &privateKey.PublicKey, privateKey)
if err != nil {
t.Fatal(err)
}
certificate, err := x509.ParseCertificate(raw)
if err != nil {
t.Fatal(err)
}
return privateKey, certificate, raw
}
func TestBuildZooKeeperTLSConfigFromPEM(t *testing.T) {
_, _, raw := testZooKeeperCertificate(t)
path := filepath.Join(t.TempDir(), "trust.pem")
if err := os.WriteFile(path, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: raw}), 0o600); err != nil {
t.Fatal(err)
}
config, err := buildZooKeeperTLSConfig(map[string]string{
"zookeepersslenable": "true",
"zookeepertruststorelocation": path,
"zookeepertruststoretype": "PEM",
"zookeeperservername": "zk.example.com",
})
if err != nil {
t.Fatal(err)
}
if config == nil || config.RootCAs == nil || config.ServerName != "zk.example.com" {
t.Fatalf("unexpected TLS config: %#v", config)
}
}
func TestBuildZooKeeperTLSConfigFromJKS(t *testing.T) {
privateKey, certificate, raw := testZooKeeperCertificate(t)
password := []byte("changeit")
store := keystore.New()
if err := store.SetTrustedCertificateEntry("ca", keystore.TrustedCertificateEntry{
CreationTime: time.Now(),
Certificate: keystore.Certificate{Type: "X509", Content: raw},
}); err != nil {
t.Fatal(err)
}
encodedKey, err := x509.MarshalPKCS8PrivateKey(privateKey)
if err != nil {
t.Fatal(err)
}
if err := store.SetPrivateKeyEntry("client", keystore.PrivateKeyEntry{
CreationTime: time.Now(),
PrivateKey: encodedKey,
CertificateChain: []keystore.Certificate{
{Type: "X509", Content: certificate.Raw},
},
}, password); err != nil {
t.Fatal(err)
}
path := filepath.Join(t.TempDir(), "client.jks")
file, err := os.Create(path)
if err != nil {
t.Fatal(err)
}
if err := store.Store(file, password); err != nil {
file.Close()
t.Fatal(err)
}
if err := file.Close(); err != nil {
t.Fatal(err)
}
config, err := buildZooKeeperTLSConfig(map[string]string{
"zookeepersslenable": "true",
"zookeepertruststorelocation": path,
"zookeepertruststorepassword": string(password),
"zookeepertruststoretype": "JKS",
"zookeeperkeystorelocation": path,
"zookeeperkeystorepassword": string(password),
"zookeeperkeystoretype": "JKS",
})
if err != nil {
t.Fatal(err)
}
if config.RootCAs == nil || len(config.Certificates) != 1 || config.Certificates[0].PrivateKey == nil {
t.Fatalf("unexpected JKS TLS config: %#v", config)
}
}
func TestBuildZooKeeperTLSConfigFromPKCS12(t *testing.T) {
privateKey, certificate, _ := testZooKeeperCertificate(t)
password := "changeit"
keyStore, err := pkcs12.Modern.Encode(privateKey, certificate, nil, password)
if err != nil {
t.Fatal(err)
}
trustStore, err := pkcs12.Modern.EncodeTrustStore([]*x509.Certificate{certificate}, password)
if err != nil {
t.Fatal(err)
}
directory := t.TempDir()
keyPath := filepath.Join(directory, "client.p12")
trustPath := filepath.Join(directory, "trust.p12")
if err := os.WriteFile(keyPath, keyStore, 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(trustPath, trustStore, 0o600); err != nil {
t.Fatal(err)
}
config, err := buildZooKeeperTLSConfig(map[string]string{
"zookeepersslenable": "true",
"zookeepertruststorelocation": trustPath,
"zookeepertruststorepassword": password,
"zookeepertruststoretype": "PKCS12",
"zookeeperkeystorelocation": keyPath,
"zookeeperkeystorepassword": password,
"zookeeperkeystoretype": "PKCS12",
})
if err != nil {
t.Fatal(err)
}
if config.RootCAs == nil || len(config.Certificates) != 1 {
t.Fatalf("unexpected PKCS12 TLS config: %#v", config)
}
}
func TestZooKeeperTLSRequiresExistingStore(t *testing.T) {
if _, err := buildZooKeeperTLSConfig(map[string]string{
"zookeepersslenable": "true",
"zookeepertruststorelocation": filepath.Join(t.TempDir(), "missing.jks"),
}); err == nil {
t.Fatal("expected missing truststore error")
}
}
@@ -1274,7 +1274,7 @@ public final class DamengAgent extends AbstractJdbcAgent {
* <li>TRIGGER → ALL_TRIGGERS.TRIGGER_BODY</li>
* <li>SEQUENCE → 由 ALL_SEQUENCES 元数据重建 CREATE SEQUENCE 语句</li>
* <li>PROCEDURE/FUNCTION/PACKAGE/PACKAGE_BODY/TYPE/TYPE_BODY → ALL_SOURCE 按 LINE 拼接,
* 再以 SYS.SYSOBJECTS + SYS.SYSTEXTS 作为最后一层</li>
* 再以 SYS.SYSOBJECTS + SYS.SYSTEXTS(TXT/SEQNO) 作为最后一层</li>
* </ul>
* 所有字典来源都不可用时返回带原因说明的占位源码(不可编辑),避免双击对象查看源码直接报错;
* 连接类错误(断连/超时等)不属于元数据不可用,继续上抛由上层处理会话。
@@ -1418,9 +1418,7 @@ public final class DamengAgent extends AbstractJdbcAgent {
try {
String type = normalizeObjectSourceType(objectType);
List<String> typeCandidates = switch (type) {
// 不同 DM 版本对函数/过程的 TYPE 取值不一致(有的统一为 PROCEDURE),逐一尝试。
case "FUNCTION" -> List.of("FUNCTION", "PROCEDURE");
case "PROCEDURE" -> List.of("PROCEDURE", "FUNCTION");
case "FUNCTION", "PROCEDURE" -> List.of("PROC", type);
case "PACKAGE" -> List.of("PACKAGE");
case "PACKAGE_BODY" -> List.of("PACKAGE BODY", "PACKAGE_BODY");
case "TYPE" -> List.of("TYPE");
@@ -1429,15 +1427,10 @@ public final class DamengAgent extends AbstractJdbcAgent {
};
for (String candidate : typeCandidates) {
String source = readAllSourceLines(schema, name, candidate);
if (notBlank(source)) {
if (notBlank(source) && routineSourceMatchesRequestedType(schema, name, type, candidate, source)) {
return catalogRoutineObjectSource(schema, name, objectType, source);
}
}
// 兜底:忽略 TYPE 过滤按名称读取全部源码行(同一 schema 内对象名唯一)。
String anyTypeSource = readAllSourceLines(schema, name, null);
if (notBlank(anyTypeSource)) {
return catalogRoutineObjectSource(schema, name, objectType, anyTypeSource);
}
return null;
} catch (RuntimeException error) {
return metadataTierError(error);
@@ -1447,16 +1440,12 @@ public final class DamengAgent extends AbstractJdbcAgent {
}
private String readAllSourceLines(String schema, String name, String sourceType) throws Exception {
String sql = sourceType == null
? "SELECT TEXT FROM ALL_SOURCE WHERE OWNER = ? AND NAME = ? ORDER BY LINE"
: "SELECT TEXT FROM ALL_SOURCE WHERE OWNER = ? AND NAME = ? AND TYPE = ? ORDER BY LINE";
String sql = "SELECT TEXT FROM ALL_SOURCE WHERE OWNER = ? AND NAME = ? AND TYPE = ? ORDER BY LINE";
StringBuilder source = new StringBuilder();
try (PreparedStatement stmt = requireConnected().prepareStatement(sql)) {
stmt.setString(1, schema);
stmt.setString(2, name);
if (sourceType != null) {
stmt.setString(3, sourceType);
}
stmt.setString(3, sourceType);
try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) {
String line = coalesce(readTextColumn(rs, 1));
@@ -1470,6 +1459,66 @@ public final class DamengAgent extends AbstractJdbcAgent {
return source.toString();
}
private boolean routineSourceMatchesRequestedType(
String schema,
String name,
String objectType,
String sourceType,
String source
) {
if (!("FUNCTION".equals(objectType) || "PROCEDURE".equals(objectType))
|| !"PROC".equalsIgnoreCase(sourceType)) {
return true;
}
String catalogType = readDamengRoutineObjectType(schema, name);
return catalogType != null
? objectType.equals(catalogType)
: routineSourceStartsWithType(source, objectType);
}
private String readDamengRoutineObjectType(String schema, String name) {
String sql = """
SELECT o.INFO1
FROM SYS.SYSOBJECTS o
JOIN SYS.SYSOBJECTS s ON s.ID = o.SCHID AND s.TYPE$ = 'SCH'
WHERE o.SUBTYPE$ = 'PROC' AND s.NAME = ? AND o.NAME = ?
""".stripIndent().trim();
try (PreparedStatement stmt = requireConnected().prepareStatement(sql)) {
stmt.setString(1, schema);
stmt.setString(2, name);
try (ResultSet rs = stmt.executeQuery()) {
if (!rs.next()) {
return null;
}
String info1 = rs.getString(1);
return switch (info1 == null ? "" : info1.trim()) {
case "0" -> "FUNCTION";
case "1" -> "PROCEDURE";
default -> "PROCEDURE";
};
}
} catch (RuntimeException error) {
if (isDamengConnectionError(error)) {
throw error;
}
return null;
} catch (Exception error) {
if (isDamengConnectionError(error)) {
throw new RuntimeException(error);
}
return null;
}
}
private static boolean routineSourceStartsWithType(String source, String objectType) {
String normalizedSource = source.stripLeading().toUpperCase(Locale.ROOT);
String normalizedType = objectType.toUpperCase(Locale.ROOT);
return startsWithRoutineDeclaration(normalizedSource, normalizedType)
|| startsWithRoutineDeclaration(normalizedSource, "CREATE " + normalizedType)
|| startsWithRoutineDeclaration(normalizedSource, "CREATE OR REPLACE " + normalizedType)
|| startsWithRoutineDeclaration(normalizedSource, "ALTER " + normalizedType);
}
private static ObjectSource catalogRoutineObjectSource(
String schema,
String name,
@@ -1505,12 +1554,12 @@ public final class DamengAgent extends AbstractJdbcAgent {
// DM 将过程/函数/包/类型的定义文本按行存放在 SYS.SYSTEXTS,
// SYS.SYSOBJECTS 的 SCHID 关联所属 schema;适用于 ALL_SOURCE 不可用的实例。
String sql = """
SELECT t.TEXT
SELECT t.TXT
FROM SYS.SYSTEXTS t
JOIN SYS.SYSOBJECTS o ON o.ID = t.ID
JOIN SYS.SYSOBJECTS s ON s.ID = o.SCHID AND s.TYPE$ = 'SCH' AND s.NAME = ?
WHERE o.NAME = ?
ORDER BY t.LINE
ORDER BY t.SEQNO
""".stripIndent().trim();
StringBuilder source = new StringBuilder();
try (PreparedStatement stmt = requireConnected().prepareStatement(sql)) {
@@ -1527,6 +1576,13 @@ public final class DamengAgent extends AbstractJdbcAgent {
}
}
return notBlank(source.toString())
&& routineSourceMatchesRequestedType(
schema,
name,
normalizeObjectSourceType(objectType),
"PROC",
source.toString()
)
? catalogRoutineObjectSource(schema, name, objectType, source.toString())
: null;
} catch (RuntimeException error) {
@@ -2098,7 +2154,8 @@ public final class DamengAgent extends AbstractJdbcAgent {
private static String buildUrl(ConnectParams params) {
String connectionString = params.getConnection_string() == null ? "" : params.getConnection_string().trim();
if (!connectionString.isEmpty()) {
if (!connectionString.isEmpty()
&& !connectionString.regionMatches(true, 0, "dm://", 0, "dm://".length())) {
return connectionString;
}
String database = params.getDatabase() == null ? "" : params.getDatabase().trim();
@@ -562,9 +562,92 @@ class DamengAgentMetadataTest {
Assertions.assertTrue(source.isEditable());
Assertions.assertEquals(1, sqls.stream().filter(sql -> sql.contains("DBMS_METADATA.GET_DDL")).count());
Assertions.assertTrue(sqls.stream().anyMatch(sql -> sql.contains("FROM ALL_SOURCE")), String.join("\n", sqls));
Assertions.assertTrue(sqls.stream().anyMatch(sql -> sql.contains("TYPE = ?")), String.join("\n", sqls));
Assertions.assertTrue(sqls.stream().noneMatch(sql -> sql.contains("FROM ALL_SOURCE") && !sql.contains("TYPE = ?")), String.join("\n", sqls));
Assertions.assertTrue(sqls.stream().noneMatch(sql -> sql.contains("SYS.SYSTEXTS")), String.join("\n", sqls));
}
@Test
void matchesDamengProcedureTypeWhenAllSourceUsesProc() {
DamengAgent agent = new DamengAgent();
List<String> sqls = new ArrayList<>();
TestSupport.setPrivateConnection(
agent,
typedRoutineFallbackConnection(
missingDbmsMetadataPackageError(),
sqls,
List.of(
Arrays.asList("CREATE OR REPLACE PROCEDURE \"APP\".\"REBUILD_CACHE\" AS"),
Arrays.asList("BEGIN NULL; END;")
),
null,
List.of(),
"1"
)
);
ObjectSource source = agent.getObjectSource("APP", "REBUILD_CACHE", "PROCEDURE");
Assertions.assertTrue(source.getSource().contains("CREATE OR REPLACE PROCEDURE"), source.getSource());
Assertions.assertTrue(source.isEditable());
Assertions.assertTrue(sqls.stream().anyMatch(sql -> sql.contains("FROM ALL_SOURCE") && sql.contains("TYPE = ?")), String.join("\n", sqls));
Assertions.assertTrue(sqls.stream().anyMatch(sql -> sql.contains("SELECT o.INFO1")), String.join("\n", sqls));
Assertions.assertTrue(sqls.stream().noneMatch(sql -> sql.contains("FROM ALL_SOURCE") && !sql.contains("TYPE = ?")), String.join("\n", sqls));
}
@Test
void matchesDamengFunctionTypeWhenAllSourceUsesProc() {
DamengAgent agent = new DamengAgent();
List<String> sqls = new ArrayList<>();
TestSupport.setPrivateConnection(
agent,
typedRoutineFallbackConnection(
missingDbmsMetadataPackageError(),
sqls,
List.of(
Arrays.asList("CREATE OR REPLACE FUNCTION \"APP\".\"CALC_SCORE\"(V_ID INT) RETURN INT AS"),
Arrays.asList("BEGIN RETURN V_ID * 2; END;")
),
null,
List.of(),
"0"
)
);
ObjectSource source = agent.getObjectSource("APP", "CALC_SCORE", "FUNCTION");
Assertions.assertTrue(source.getSource().contains("CREATE OR REPLACE FUNCTION"), source.getSource());
Assertions.assertTrue(source.isEditable());
Assertions.assertTrue(sqls.stream().anyMatch(sql -> sql.contains("SELECT o.INFO1")), String.join("\n", sqls));
}
@Test
void doesNotReturnProcedureSourceForFunctionFallback() {
DamengAgent agent = new DamengAgent();
List<String> sqls = new ArrayList<>();
TestSupport.setPrivateConnection(
agent,
typedRoutineFallbackConnection(
missingDbmsMetadataPackageError(),
sqls,
List.of(
Arrays.asList("CREATE OR REPLACE PROCEDURE \"APP\".\"REBUILD_CACHE\" AS"),
Arrays.asList("BEGIN NULL; END;")
),
null,
List.of(),
"1"
)
);
ObjectSource source = agent.getObjectSource("APP", "REBUILD_CACHE", "FUNCTION");
Assertions.assertFalse(source.getSource().contains("CREATE OR REPLACE PROCEDURE"), source.getSource());
Assertions.assertFalse(source.isEditable());
Assertions.assertEquals(2, sqls.stream().filter(sql -> sql.contains("FROM ALL_SOURCE")).count(), String.join("\n", sqls));
Assertions.assertTrue(sqls.stream().noneMatch(sql -> sql.contains("FROM ALL_SOURCE") && !sql.contains("TYPE = ?")), String.join("\n", sqls));
}
@Test
void fallsBackToAllSourceTypeDdlWhenDbmsMetadataPackageIsMissing() {
DamengAgent agent = new DamengAgent();
@@ -731,6 +814,34 @@ class DamengAgentMetadataTest {
Assertions.assertTrue(source.isEditable());
Assertions.assertTrue(sqls.stream().anyMatch(sql -> sql.contains("FROM ALL_SOURCE")), String.join("\n", sqls));
Assertions.assertTrue(sqls.stream().anyMatch(sql -> sql.contains("SYS.SYSTEXTS") && sql.contains("SYS.SYSOBJECTS")), String.join("\n", sqls));
Assertions.assertTrue(sqls.stream().anyMatch(sql -> sql.contains("SELECT t.TXT") && sql.contains("ORDER BY t.SEQNO")), String.join("\n", sqls));
Assertions.assertTrue(sqls.stream().noneMatch(sql -> sql.contains("SYS.SYSTEXTS") && (sql.contains("t.TEXT") || sql.contains("t.LINE"))), String.join("\n", sqls));
}
@Test
void doesNotReturnProcedureSourceForFunctionSysTextsFallback() {
DamengAgent agent = new DamengAgent();
List<String> sqls = new ArrayList<>();
TestSupport.setPrivateConnection(
agent,
typedRoutineFallbackConnection(
missingDbmsMetadataPackageError(),
sqls,
List.of(),
missingCatalogViewError("ALL_SOURCE"),
List.of(
Arrays.asList("CREATE OR REPLACE PROCEDURE \"APP\".\"REBUILD_CACHE\" AS"),
Arrays.asList("BEGIN NULL; END;")
),
"1"
)
);
ObjectSource source = agent.getObjectSource("APP", "REBUILD_CACHE", "FUNCTION");
Assertions.assertFalse(source.getSource().contains("CREATE OR REPLACE PROCEDURE"), source.getSource());
Assertions.assertFalse(source.isEditable());
Assertions.assertTrue(sqls.stream().anyMatch(sql -> sql.contains("SELECT t.TXT") && sql.contains("ORDER BY t.SEQNO")), String.join("\n", sqls));
}
@Test
@@ -1499,6 +1610,47 @@ class DamengAgentMetadataTest {
});
}
private static Connection typedRoutineFallbackConnection(
SQLException dbmsMetadataError,
List<String> sqls,
List<List<Object>> allSourceRows,
SQLException allSourceError,
List<List<Object>> systemTextRows,
String info1
) {
int[] allSourceQueryCount = {0};
return proxy(Connection.class, (method, args) -> {
String name = method.getName();
if ("prepareStatement".equals(name)) {
String sql = (String) args[0];
sqls.add(sql);
if (sql.contains("DBMS_METADATA.GET_DDL")) {
return failingMetadataStatement(dbmsMetadataError);
}
if (sql.contains("SELECT o.INFO1")) {
return metadataStatement(List.of(List.of(info1)));
}
if (sql.contains("FROM ALL_SOURCE")) {
if (allSourceError != null) {
return failingMetadataStatement(allSourceError);
}
return metadataStatement(allSourceQueryCount[0]++ == 0 ? allSourceRows : List.of());
}
if (sql.contains("SYS.SYSTEXTS")) {
return metadataStatement(systemTextRows);
}
return metadataStatement(List.of());
}
if ("close".equals(name)) {
return null;
}
if ("isClosed".equals(name)) {
return false;
}
return defaultValue(method.getReturnType());
});
}
private static Connection restrictedCrossOwnerMetadataConnection(List<String> sqls) {
return proxy(Connection.class, (method, args) -> {
String name = method.getName();
@@ -21,6 +21,24 @@ class DamengAgentUrlTest {
Assertions.assertEquals("jdbc:dm://127.0.0.1:5236/MAIN", url);
}
@Test
void rebuildsLegacyDbxUrlFromConnectionFields() throws Exception {
ConnectParams params = new ConnectParams(
"127.0.0.1",
5236,
"MAIN",
"SYSDBA",
"pwd",
"",
"dm://SYSDBA:SYSDBA@legacy-host:5236/OLD",
false
);
String url = invokeBuildUrl(params);
Assertions.assertEquals("jdbc:dm://127.0.0.1:5236/MAIN", url);
}
@Test
void usesCompleteCustomJdbcUrlWithoutRewritingIt() throws Exception {
ConnectParams params = new ConnectParams(
@@ -31,6 +31,9 @@ public final class DatabendAgent extends ConfiguredJdbcAgent {
true,
false,
false,
false,
// databend 的 getSearchStringEscape() 返回 '\',但 getTables/getProcedures 把 _ 当字面量且忽略转义,
// 对含 _ 的库名(如 my_db)转义后返回 0 行(#8114),故关闭 schema 通配符转义。
false
);
+9 -1
View File
@@ -1243,6 +1243,7 @@ dependencies = [
"cbc 0.1.2",
"chrono",
"csv",
"dbx-sqlite-worker",
"deadpool-postgres",
"encoding_rs",
"flate2",
@@ -1317,6 +1318,14 @@ dependencies = [
"uuid",
]
[[package]]
name = "dbx-sqlite-worker"
version = "0.1.1"
dependencies = [
"serde",
"serde_json",
]
[[package]]
name = "deadpool"
version = "0.12.3"
@@ -4528,7 +4537,6 @@ dependencies = [
"borsh",
"bytes",
"num-traits",
"postgres-types",
"rand 0.8.7",
"rkyv",
"serde",
@@ -0,0 +1,53 @@
# etcd Java to Go migration parity
Status date: 2026-09-02.
The Java `EtcdAgent` (jetcd 0.8.x, 33 protocol methods) was replaced by this
native Go agent. The migration is complete only where a capability is
implemented, covered by automated tests, validated against a real etcd cluster,
and wired into the native build and release path.
## Baseline
- DBX Java baseline: jetcd 0.8.x `EtcdAgent`; all protocol shapes, capability
strings, and error signal texts were copied verbatim.
- Go client: `go.etcd.io/etcd/client/v3` v3.7.1 over gRPC.
- Capabilities (exact Java set, no `structured_error_v1`): `connect`,
`test_connection`, `kv`, `kv_ttl`, `kv_cas`, `kv_list_values`, `kv_status`,
`kv_history`, `etcd_compaction`, `etcd_defrag`, `etcd_watch`, `etcd_lease`,
`etcd_auth`, `multi_session`.
- Error signal strings (`ETCD_CAS_CONFLICT`, `ETCD_NOT_FOUND`,
`ETCD_COMPACTED`, `ETCD_WATCH_LIMIT`, …) are matched verbatim because the
host normalizes agent errors by message text
(`agent_kv.rs::normalize_agent_kv_error`).
- The shared `go-semver` module is vendored under `agents/go-common/go-semver`
because the upstream `v0.3.1` tag declares a broken module path.
## Current matrix
| Capability | Go code | Automated | Live server | Status |
| --- | --- | --- | --- | --- |
| connect / endpoints / TLS / basic auth | yes | yes | etcd 3.7.0, 3.5.21 | parity passed (auth enabled) |
| test_connection probe (PERMISSION_DENIED → limited) | yes | yes | yes | parity passed |
| kv list/get/put/delete/rename with CAS | yes | yes | yes | parity passed |
| TTL + preserveLease (3-attempt retry) | yes | yes | yes | parity passed |
| Lease list/get/grant/keepalive/revoke | yes | yes | yes | parity passed |
| LeaseLeases fallback on 3.3 (UNIMPLEMENTED/timeout → partial) | yes | yes | n/a (3.3 fixture not retained) | coded + unit tested |
| LeaseGrant with custom ID (raw etcdserverpb RPC) | yes | yes | yes | replaces Java reflection wire handling |
| kv_history (temporary watch, prevKV, progress notify) | yes | yes | yes | requires `clientv3.WithCreatedNotify()`; jetcd parity |
| kv_history compacted/future-revision errors | yes | yes | yes | parity passed |
| Watch start/poll/stop with full budget system | yes | yes | yes | 256 batches / 10k events / 8MiB per watch, 16MiB session aggregate |
| Status: members, alarms, latency fan-out, member dedup | yes | yes | yes | parity passed |
| Status raft fields as unsigned decimal strings | yes | yes | yes | bit-level parity with Java `longString(int64(uint64))` |
| Compaction (pre-validation, error propagation) | yes | yes | yes | Java check order preserved (params before connect) |
| Defrag (followers first, leader last, per-endpoint errors) | yes | yes | yes | parity passed |
| Auth users/roles (14 methods) | yes | yes | yes | parity passed |
| Multi-session protocol runtime (256 sessions, cancel_session) | yes | yes | yes | protocol flow live test passed |
| etcd 3.3 degraded mode | partial | partial | no | lease/history fallbacks coded; no retained 3.3 fixture |
| etcd 2.x | n/a | n/a | n/a | served by the dedicated `etcd2` agent, not this one |
## Known divergences
- None intentional. All observable protocol outputs are byte-for-byte
compatible with the Java agent; the raw protobuf RPCs used for lease
operations are an internal implementation change only.
+261
View File
@@ -0,0 +1,261 @@
package main
import (
"encoding/json"
"fmt"
"strings"
clientv3 "go.etcd.io/etcd/client/v3"
)
func (s *etcdSession) authUserList(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
response, err := client.Auth.UserList(ctx)
if err != nil {
return nil, err
}
return map[string]any{"users": response.Users}, nil
}
func (s *etcdSession) authUserGet(params map[string]json.RawMessage) (any, error) {
user, err := requiredString(params, "user")
if err != nil {
return nil, err
}
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
response, err := client.Auth.UserGet(ctx, user)
if err != nil {
return nil, err
}
return map[string]any{"user": user, "roles": response.Roles}, nil
}
func (s *etcdSession) authUserAdd(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
user, err := requiredString(params, "user")
if err != nil {
return nil, err
}
password, err := requiredString(params, "password")
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
if _, err := client.Auth.UserAdd(ctx, user, password); err != nil {
return nil, err
}
return map[string]bool{"created": true}, nil
}
func (s *etcdSession) authUserDelete(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
user, err := requiredString(params, "user")
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
if _, err := client.Auth.UserDelete(ctx, user); err != nil {
return nil, err
}
return map[string]bool{"deleted": true}, nil
}
func (s *etcdSession) authUserChangePassword(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
user, err := requiredString(params, "user")
if err != nil {
return nil, err
}
password, err := requiredString(params, "password")
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
if _, err := client.Auth.UserChangePassword(ctx, user, password); err != nil {
return nil, err
}
return map[string]bool{"changed": true}, nil
}
func (s *etcdSession) authUserGrantRevokeRole(params map[string]json.RawMessage, grant bool) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
user, err := requiredString(params, "user")
if err != nil {
return nil, err
}
role, err := requiredString(params, "role")
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
if grant {
_, err = client.Auth.UserGrantRole(ctx, user, role)
} else {
_, err = client.Auth.UserRevokeRole(ctx, user, role)
}
if err != nil {
return nil, err
}
return map[string]bool{"updated": true}, nil
}
func (s *etcdSession) authRoleList(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
response, err := client.Auth.RoleList(ctx)
if err != nil {
return nil, err
}
return map[string]any{"roles": response.Roles}, nil
}
func (s *etcdSession) authRoleGet(params map[string]json.RawMessage) (any, error) {
role, err := requiredString(params, "role")
if err != nil {
return nil, err
}
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
response, err := client.Auth.RoleGet(ctx, role)
if err != nil {
return nil, err
}
permissions := []map[string]any{}
for _, permission := range response.Perm {
key := []byte(permission.Key)
rangeEnd := []byte(permission.RangeEnd)
resource := "prefix"
if len(key) == 1 && key[0] == 0 && len(rangeEnd) == 1 && rangeEnd[0] == 0 {
resource = "all"
} else if len(rangeEnd) == 0 {
resource = "key"
}
permissions = append(permissions, map[string]any{
"access": strings.ToLower(permission.PermType.String()),
"key": bytesObject(key),
"rangeEnd": bytesObject(rangeEnd),
"resource": resource,
})
}
return map[string]any{"role": role, "permissions": permissions}, nil
}
func (s *etcdSession) authRoleAdd(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
role, err := requiredString(params, "role")
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
if _, err := client.Auth.RoleAdd(ctx, role); err != nil {
return nil, err
}
return map[string]bool{"created": true}, nil
}
func (s *etcdSession) authRoleDelete(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
role, err := requiredString(params, "role")
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
if _, err := client.Auth.RoleDelete(ctx, role); err != nil {
return nil, err
}
return map[string]bool{"deleted": true}, nil
}
func (s *etcdSession) authRolePermission(params map[string]json.RawMessage, grant bool) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
role, err := requiredString(params, "role")
if err != nil {
return nil, err
}
resource := stringOrDefault(params, "resource", "key")
all := resource == "all"
prefix := resource == "prefix"
var key string
var rangeEnd string
if all {
key = "\x00"
rangeEnd = "\x00"
} else {
key, err = keyBytesParam(params)
if err != nil {
return nil, err
}
if prefix {
rangeEnd = prefixEnd(key)
}
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
if grant {
access, err := permissionType(stringOrDefault(params, "access", ""))
if err != nil {
return nil, err
}
if _, err := client.Auth.RoleGrantPermission(ctx, role, key, rangeEnd, access); err != nil {
return nil, err
}
} else {
if _, err := client.Auth.RoleRevokePermission(ctx, role, key, rangeEnd); err != nil {
return nil, err
}
}
return map[string]bool{"updated": true}, nil
}
func permissionType(access string) (clientv3.PermissionType, error) {
permission, err := clientv3.StrToPermissionType(strings.ToUpper(access))
if err != nil {
return 0, fmt.Errorf("ETCD_INVALID_ACCESS: access must be READ, WRITE, or READWRITE, got %s", access)
}
return permission, nil
}
+321
View File
@@ -0,0 +1,321 @@
package main
import (
"context"
"crypto/tls"
"crypto/x509"
"encoding/json"
"errors"
"fmt"
"net/url"
"os"
"strconv"
"strings"
"time"
clientv3 "go.etcd.io/etcd/client/v3"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
const (
defaultGrpcMaxInboundMessageSize = 32 * 1024 * 1024
minGrpcMaxInboundMessageSize = 1024 * 1024
maxGrpcMaxInboundMessageSize = 256 * 1024 * 1024
grpcMaxInboundMessageSizeKey = "grpc_max_inbound_message_size"
defaultPort = 2379
)
// connectionParams mirrors the host-side etcd connection configuration.
type connectionParams struct {
EtcdEndpoints string `json:"etcd_endpoints"`
Endpoints string `json:"endpoints"`
ConnectionString string `json:"connection_string"`
Host string `json:"host"`
Port int `json:"port"`
Username string `json:"username"`
Password string `json:"password"`
SSL bool `json:"ssl"`
CACertPath string `json:"ca_cert_path"`
ClientCertPath string `json:"client_cert_path"`
ClientKeyPath string `json:"client_key_path"`
CertPath string `json:"cert_path"`
KeyPath string `json:"key_path"`
ConnectTimeoutSecs int `json:"connect_timeout_secs"`
GrpcMaxInboundMessageSize int `json:"grpc_max_inbound_message_size"`
URLParams string `json:"url_params"`
}
func connectionObject(params map[string]json.RawMessage) (connectionParams, error) {
var connection connectionParams
if raw, ok := params["connection"]; ok && raw != nil {
if err := json.Unmarshal(raw, &connection); err != nil {
return connection, err
}
return connection, nil
}
if err := decodeParams(params, &connection); err != nil {
return connection, err
}
return connection, nil
}
func (s *etcdSession) connectMap(params map[string]json.RawMessage) (any, error) {
if _, err := s.connect(params); err != nil {
return nil, err
}
return map[string]bool{"ok": true}, nil
}
func (s *etcdSession) connect(params map[string]json.RawMessage) (any, error) {
connection, err := connectionObject(params)
if err != nil {
return nil, err
}
nextClient, err := buildClient(connection)
if err != nil {
return nil, err
}
endpointList := connectionEndpoints(connection)
if _, err := probeClient(nextClient, endpointList); err != nil {
_ = nextClient.Close()
return nil, err
}
s.close()
s.clientMu.Lock()
s.client = nextClient
s.connectedEndpoints = endpointList
s.clientMu.Unlock()
return map[string]bool{"ok": true}, nil
}
func (s *etcdSession) activeClient() (*clientv3.Client, error) {
s.clientMu.Lock()
client := s.client
s.clientMu.Unlock()
if client == nil {
return nil, errors.New("Not connected")
}
return client, nil
}
func (s *etcdSession) connectedEndpointList() []string {
s.clientMu.Lock()
defer s.clientMu.Unlock()
return append([]string(nil), s.connectedEndpoints...)
}
func (s *etcdSession) validateConnection() (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
return probeClient(client, s.connectedEndpointList())
}
func (s *etcdSession) close() error {
s.watchesMu.Lock()
watches := s.watches
s.watches = map[string]*watchState{}
s.knownLeases = map[uint64]struct{}{}
s.watchBufferedBytes = 0
s.watchesMu.Unlock()
for _, watch := range watches {
watch.close()
}
s.clientMu.Lock()
client := s.client
s.client = nil
s.connectedEndpoints = nil
s.clientMu.Unlock()
if client != nil {
_ = client.Close()
}
return nil
}
func buildClient(connection connectionParams) (*clientv3.Client, error) {
endpoints := connectionEndpoints(connection)
config := clientv3.Config{
Endpoints: endpoints,
DialTimeout: time.Duration(connectTimeoutSeconds(connection)) * time.Second,
}
if size := grpcMaxInboundMessageSize(connection); size > 0 {
config.MaxCallSendMsgSize = size
config.MaxCallRecvMsgSize = size
}
if strings.TrimSpace(connection.Username) != "" {
config.Username = connection.Username
config.Password = connection.Password
}
if connection.SSL {
tlsConfig, err := tlsConfigFor(connection)
if err != nil {
return nil, err
}
config.TLS = tlsConfig
}
return clientv3.New(config)
}
func connectTimeoutSeconds(connection connectionParams) int {
seconds := connection.ConnectTimeoutSecs
if seconds == 0 {
seconds = rpcTimeoutSeconds
}
if seconds < 1 {
seconds = 1
}
if seconds > 300 {
seconds = 300
}
return seconds
}
func grpcMaxInboundMessageSize(connection connectionParams) int {
configured := connection.GrpcMaxInboundMessageSize
if configured == 0 {
configured = intURLParamOrDefault(connection.URLParams, grpcMaxInboundMessageSizeKey, defaultGrpcMaxInboundMessageSize)
}
if configured < minGrpcMaxInboundMessageSize {
configured = minGrpcMaxInboundMessageSize
}
if configured > maxGrpcMaxInboundMessageSize {
configured = maxGrpcMaxInboundMessageSize
}
return configured
}
func intURLParamOrDefault(params, key string, fallback int) int {
if strings.TrimSpace(params) == "" {
return fallback
}
for _, entry := range strings.Split(strings.TrimPrefix(params, "?"), "&") {
separator := strings.Index(entry, "=")
entryKey := entry
if separator >= 0 {
entryKey = entry[:separator]
}
if entryKey != key {
continue
}
if separator < 0 {
return fallback
}
value, err := strconv.Atoi(entry[separator+1:])
if err != nil {
return fallback
}
return value
}
return fallback
}
func tlsConfigFor(connection connectionParams) (*tls.Config, error) {
tlsConfig := &tls.Config{}
if ca := strings.TrimSpace(connection.CACertPath); ca != "" {
authorityPEM, err := os.ReadFile(ca)
if err != nil {
return nil, err
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(authorityPEM) {
return nil, fmt.Errorf("failed to parse CA certificate at %s", ca)
}
tlsConfig.RootCAs = pool
}
certPath := firstNonBlank(connection.ClientCertPath, connection.CertPath)
keyPath := firstNonBlank(connection.ClientKeyPath, connection.KeyPath)
if (certPath == "") != (keyPath == "") {
return nil, errors.New("Client certificate and key must be provided together")
}
if certPath != "" {
pair, err := tls.LoadX509KeyPair(certPath, keyPath)
if err != nil {
return nil, err
}
tlsConfig.Certificates = []tls.Certificate{pair}
}
return tlsConfig, nil
}
func probeClient(client *clientv3.Client, endpoints []string) (map[string]any, error) {
var lastFailure error
for _, endpoint := range endpoints {
ctx, cancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
_, err := client.Maintenance.Status(ctx, endpoint)
cancel()
if err == nil {
return map[string]any{"ok": true, "endpoint": endpoint}, nil
}
// A restricted etcd user may not be allowed to call Maintenance.Status.
// PERMISSION_DENIED still proves that the channel reached an etcd server.
if status.Code(err) == codes.PermissionDenied {
return map[string]any{"ok": true, "endpoint": endpoint, "limited": true}, nil
}
lastFailure = err
}
if lastFailure != nil {
return nil, lastFailure
}
return nil, errors.New("No etcd endpoint configured")
}
func connectionEndpoints(connection connectionParams) []string {
configured := firstNonBlank(connection.EtcdEndpoints, connection.Endpoints, connection.ConnectionString)
var result []string
if configured != "" {
for _, endpoint := range strings.FieldsFunc(configured, func(r rune) bool { return r == ',' || r == '\n' }) {
normalized := normalizeEndpoint(strings.TrimSpace(endpoint), connection.SSL)
if normalized != "" {
result = append(result, normalized)
}
}
}
if len(result) == 0 {
host := connection.Host
if host == "" {
host = "127.0.0.1"
}
port := connection.Port
if port == 0 {
port = defaultPort
}
result = append(result, normalizeEndpoint(fmt.Sprintf("%s:%d", host, port), connection.SSL))
}
return result
}
func normalizeEndpoint(endpoint string, tlsEnabled bool) string {
if strings.TrimSpace(endpoint) == "" {
return ""
}
if strings.HasPrefix(endpoint, "http://") || strings.HasPrefix(endpoint, "https://") {
return endpoint
}
if strings.HasPrefix(endpoint, "unix://") || strings.HasPrefix(endpoint, "unixs://") {
return endpoint
}
scheme := "http"
if tlsEnabled {
scheme = "https"
}
return scheme + "://" + endpoint
}
func endpointHost(rawEndpoint string) string {
parsed, err := url.Parse(rawEndpoint)
if err != nil {
return ""
}
return parsed.Hostname()
}
func firstNonBlank(values ...string) string {
for _, value := range values {
if strings.TrimSpace(value) != "" {
return value
}
}
return ""
}
+34
View File
@@ -0,0 +1,34 @@
module github.com/t8y2/dbx/agents/drivers/etcd-go
go 1.26
// The v0.3.1 tag of coreos/go-semver declares the root module without the
// /semver suffix, so the nested module path etcd requires no longer resolves
// upstream. Vendored under go-common from the v0.3.1 tag, package unchanged.
replace github.com/coreos/go-semver/semver => ../../go-common/go-semver
// The pseudo-version recorded in etcd's go.mod predates second-precision
// timestamps and is rejected by modern Go; point at the corrected form.
replace github.com/modern-go/concurrent => github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd
require (
go.etcd.io/etcd/api/v3 v3.7.1
go.etcd.io/etcd/client/v3 v3.7.1
google.golang.org/grpc v1.82.1
)
require (
github.com/coreos/go-semver/semver v0.3.1 // indirect
github.com/coreos/go-systemd/v22 v22.7.0 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.7.1 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.1 // indirect
golang.org/x/net v0.55.0 // indirect
golang.org/x/sys v0.45.0 // indirect
golang.org/x/text v0.37.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect
google.golang.org/protobuf v1.36.11 // indirect
)
+64
View File
@@ -0,0 +1,64 @@
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA=
github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
go.etcd.io/etcd/api/v3 v3.7.1 h1:KJG0/DcWGfe3Y1otDf/fsBf0TSSgpxZ5RO/L8SFt73E=
go.etcd.io/etcd/api/v3 v3.7.1/go.mod h1:8bXIpCMeV7E3/XL0Ix123ATn3dB+0V7d9zklHbB0m78=
go.etcd.io/etcd/client/pkg/v3 v3.7.1 h1:rKYsj3pRkR0eK3yjT3XOgrhqfmIfj9pzNgxjh7mfFv4=
go.etcd.io/etcd/client/pkg/v3 v3.7.1/go.mod h1:cnzZGIUzSfjEwLC6UBVsSXlEK1eepS/JUD7wE6PLRT0=
go.etcd.io/etcd/client/v3 v3.7.1 h1:0PEMMC0KuZmVIN+RAbdqfkZ45pYTgKVtmBEbRCvZFUg=
go.etcd.io/etcd/client/v3 v3.7.1/go.mod h1:ffNqALa8tRCYhYo1F9oR489y23K39Gz+BSR3ApAGYq0=
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM=
go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg=
go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw=
go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A=
go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A=
go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc=
go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 h1:yQugLulqltosq0B/f8l4w9VryjV+N/5gcW0jQ3N8Qec=
google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478/go.mod h1:C6ADNqOxbgdUUeRTU+LCHDPB9ttAMCTff6auwCVa4uc=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw=
google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE=
google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+255
View File
@@ -0,0 +1,255 @@
package main
import (
"context"
"encoding/json"
"errors"
"sort"
"strconv"
"sync"
"time"
"go.etcd.io/etcd/api/v3/mvccpb"
"go.etcd.io/etcd/api/v3/v3rpc/rpctypes"
clientv3 "go.etcd.io/etcd/client/v3"
)
const (
historyDefaultRevisionWindow = int64(10000)
historyLimitMax = 500
)
type historyCollector struct {
mu sync.Mutex
rows []map[string]any
limit int
truncated bool
}
func (c *historyCollector) append(row map[string]any) {
c.mu.Lock()
defer c.mu.Unlock()
if len(c.rows) == c.limit {
c.rows = c.rows[1:]
c.truncated = true
}
c.rows = append(c.rows, row)
}
func (s *etcdSession) history(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
key, err := keyBytesParam(params)
if err != nil {
return nil, err
}
limit := intOrDefault(params, "limit", 100)
if limit < 1 {
limit = 1
}
if limit > historyLimitMax {
limit = historyLimitMax
}
requestedEnd := longOrNull(params, "endRevision")
ctx, cancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
latestOptions := []clientv3.OpOption{}
if requestedEnd != nil && *requestedEnd > 0 {
latestOptions = append(latestOptions, clientv3.WithRev(*requestedEnd))
}
latest, err := client.Get(ctx, key, latestOptions...)
cancel()
if err != nil {
if errors.Is(err, rpctypes.ErrCompacted) {
if revision, ok := compactedRevisionOf(client, key); ok {
return nil, errors.New("ETCD_COMPACTED: requested history was compacted at revision " + longString(revision))
}
return nil, errors.New("ETCD_COMPACTED: requested history was compacted")
}
return nil, err
}
var endRevision int64
if requestedEnd != nil {
endRevision = *requestedEnd
} else {
endRevision = latest.Header.Revision
}
var targetKeyRevision int64
if len(latest.Kvs) == 0 {
targetKeyRevision = endRevision
} else {
targetKeyRevision = latest.Kvs[0].ModRevision
}
requestedStart := longOrNull(params, "startRevision")
startRevision := historyStartRevision(requestedStart, targetKeyRevision)
if startRevision > targetKeyRevision {
return map[string]any{
"events": []any{},
"observedRevision": longString(endRevision),
"truncated": false,
}, nil
}
collector := &historyCollector{limit: limit, truncated: requestedStart == nil && startRevision > 1}
created := make(chan struct{})
var createdOnce sync.Once
completed := make(chan struct{})
var completedOnce sync.Once
var failure error
var failureMu sync.Mutex
watchCtx, watchCancel := context.WithCancel(context.Background())
channel := client.Watch(watchCtx, key,
clientv3.WithRev(startRevision),
clientv3.WithPrevKV(),
clientv3.WithProgressNotify(),
// clientv3 only posts the created response when this option is set;
// the created gate relies on it (jetcd withCreateNotify parity).
clientv3.WithCreatedNotify(),
)
go func() {
for response := range channel {
if response.Err() != nil {
failureMu.Lock()
failure = response.Err()
failureMu.Unlock()
completedOnce.Do(func() { close(completed) })
continue
}
if response.Canceled {
if response.CompactRevision > 0 {
err := errors.New("ETCD_COMPACTED: requested history was compacted at revision " + longString(response.CompactRevision))
failureMu.Lock()
failure = err
failureMu.Unlock()
}
completedOnce.Do(func() { close(completed) })
continue
}
createdOnce.Do(func() { close(created) })
for _, event := range response.Events {
item := event.Kv
revision := item.ModRevision
if revision > endRevision {
continue
}
var previous *mvccpb.KeyValue
if event.PrevKv != nil {
previous = event.PrevKv
}
row := map[string]any{
"eventType": eventType(event),
"revision": longString(revision),
}
if event.Type == mvccpb.DELETE {
row["value"] = nil
} else {
row["value"] = valueObject(item.Value)
}
if previous != nil && previous.Version > 0 {
row["previousValue"] = valueObject(previous.Value)
} else {
row["previousValue"] = nil
}
if event.Type == mvccpb.DELETE && previous != nil {
row["metadata"] = metadataMap(previous)
} else {
row["metadata"] = metadataMap(item)
}
collector.append(row)
if revision >= targetKeyRevision {
completedOnce.Do(func() { close(completed) })
}
}
if response.IsProgressNotify() && response.Header.Revision >= endRevision {
completedOnce.Do(func() { close(completed) })
}
}
completedOnce.Do(func() { close(completed) })
}()
defer watchCancel()
select {
case <-created:
case <-time.After(5 * time.Second):
return nil, errors.New("ETCD_HISTORY_TIMEOUT: watcher was not created")
}
// For an existing exact key, its latest mod revision is an explicit
// replay boundary. This avoids relying on progress notifications,
// which older etcd/jetcd combinations do not consistently emit.
progressCtx, progressCancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
_ = client.RequestProgress(progressCtx)
progressCancel()
select {
case <-completed:
case <-time.After(15 * time.Second):
return nil, errors.New("ETCD_HISTORY_TIMEOUT: history replay did not reach the requested revision")
}
failureMu.Lock()
historyFailure := failure
failureMu.Unlock()
if historyFailure != nil {
if errors.Is(historyFailure, rpctypes.ErrCompacted) {
if revision, ok := compactedRevisionOf(client, key); ok {
return nil, errors.New("ETCD_COMPACTED: requested history was compacted at revision " + longString(revision))
}
return nil, errors.New("ETCD_COMPACTED: requested history was compacted")
}
return nil, errors.New("ETCD_HISTORY_FAILED: " + historyFailure.Error())
}
collector.mu.Lock()
rows := make([]map[string]any, len(collector.rows))
copy(rows, collector.rows)
truncated := collector.truncated
collector.mu.Unlock()
sort.SliceStable(rows, func(i, j int) bool {
a, _ := strconv.ParseInt(rows[i]["revision"].(string), 10, 64)
b, _ := strconv.ParseInt(rows[j]["revision"].(string), 10, 64)
return a > b
})
serialized := make([]any, len(rows))
for i, row := range rows {
serialized[i] = row
}
return map[string]any{
"events": serialized,
"observedRevision": longString(endRevision),
"truncated": truncated,
}, nil
}
func historyStartRevision(requestedStart *int64, targetRevision int64) int64 {
if requestedStart != nil {
if *requestedStart < 1 {
return 1
}
return *requestedStart
}
start := targetRevision - historyDefaultRevisionWindow + 1
if start < 1 {
return 1
}
return start
}
// compactedRevisionOf recovers the compaction boundary that gRPC errors do not
// carry: a watch from revision 1 is cancelled immediately with the compacted
// revision in its response.
func compactedRevisionOf(client *clientv3.Client, key string) (int64, bool) {
watchCtx, watchCancel := context.WithTimeout(context.Background(), 5*time.Second)
defer watchCancel()
channel := client.Watch(watchCtx, key, clientv3.WithRev(1))
select {
case response := <-channel:
if response.CompactRevision > 0 {
return response.CompactRevision, true
}
return 0, false
case <-watchCtx.Done():
return 0, false
}
}
+515
View File
@@ -0,0 +1,515 @@
package main
import (
"encoding/json"
"fmt"
"os"
"strings"
"testing"
"time"
)
// TestLiveEtcdAgent exercises the full agent surface against a real etcd
// server. Enable with DBX_ETCD_LIVE=1; configure via DBX_ETCD_ENDPOINTS,
// DBX_ETCD_USER, and DBX_ETCD_PASSWORD (defaults match the deploy recipes:
// root/123456 on 127.0.0.1:10700 for 3.7).
func TestLiveEtcdAgent(t *testing.T) {
if os.Getenv("DBX_ETCD_LIVE") != "1" {
t.Skip("set DBX_ETCD_LIVE=1 to run the live etcd agent test")
}
endpoints := envOrDefault("DBX_ETCD_ENDPOINTS", "http://127.0.0.1:10700")
user := envOrDefault("DBX_ETCD_USER", "root")
password := envOrDefault("DBX_ETCD_PASSWORD", "123456")
state := newEtcdSession()
connectParams := map[string]json.RawMessage{
"etcd_endpoints": json.RawMessage(`"` + endpoints + `"`),
"username": json.RawMessage(`"` + user + `"`),
"password": json.RawMessage(`"` + password + `"`),
}
if _, err := state.connect(connectParams); err != nil {
t.Fatalf("connect failed: %v", err)
}
defer state.close()
if result, err := state.validateConnection(); err != nil {
t.Fatalf("validate_connection failed: %v", err)
} else {
probe := result.(map[string]any)
if probe["ok"] != true {
t.Fatalf("unexpected probe result: %#v", probe)
}
}
prefix := fmt.Sprintf("dbx/live/%d/", time.Now().UnixNano())
// --- KV basics -----------------------------------------------------
putResult, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "hello"),
"value": jsonValue("world"),
}))
if err != nil {
t.Fatalf("put failed: %v", err)
}
baseRevision := revisionOf(t, putResult)
binaryEncoded := base64Encode([]byte{0x00, 0xff, 0x81})
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "binary"),
"value": json.RawMessage(`{"encoding":"base64","data":"` + binaryEncoded + `"}`),
})); err != nil {
t.Fatalf("binary put failed: %v", err)
}
fetched, err := state.get(paramsWith(map[string]json.RawMessage{"key": jsonString(prefix + "hello")}))
if err != nil {
t.Fatalf("get failed: %v", err)
}
row := fetched.(map[string]any)
if row["found"] != true || row["key"] != prefix+"hello" {
t.Fatalf("unexpected get result: %#v", row)
}
value := row["value"].(map[string]any)
if value["encoding"] != "utf8" || value["data"] != "world" {
t.Fatalf("unexpected value: %#v", value)
}
metadata := row["metadata"].(map[string]any)
if metadata["createRevision"] != metadata["modRevision"] {
t.Fatalf("fresh key revisions should match: %#v", metadata)
}
binaryRow, err := state.get(paramsWith(map[string]json.RawMessage{"key": jsonString(prefix + "binary")}))
if err != nil {
t.Fatalf("binary get failed: %v", err)
}
binaryValue := binaryRow.(map[string]any)["value"].(map[string]any)
if binaryValue["encoding"] != "base64" || binaryValue["data"] != binaryEncoded {
t.Fatalf("binary round trip broken: %#v", binaryValue)
}
// CAS conflict on stale revision.
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "hello"),
"value": jsonValue("v2"),
"expectedModRevision": json.RawMessage(`1`),
})); err == nil || err.Error() != "ETCD_CAS_CONFLICT: key changed after it was loaded" {
t.Fatalf("expected CAS conflict, got %v", err)
}
// CAS success with the fresh revision.
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "hello"),
"value": jsonValue("v3"),
"expectedModRevision": json.RawMessage(metadata["modRevision"].(string)),
})); err != nil {
t.Fatalf("CAS put failed: %v", err)
}
// TTL put, lease metadata, and preserveLease.
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "leased"),
"value": jsonValue("expiring"),
"ttl": json.RawMessage(`300`),
})); err != nil {
t.Fatalf("ttl put failed: %v", err)
}
leased, err := state.get(paramsWith(map[string]json.RawMessage{"key": jsonString(prefix + "leased")}))
if err != nil {
t.Fatalf("leased get failed: %v", err)
}
leasedMetadata := leased.(map[string]any)["metadata"].(map[string]any)
if _, hasTTL := leasedMetadata["ttl"]; !hasTTL {
t.Fatalf("leased key must expose ttl metadata: %#v", leasedMetadata)
}
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "leased"),
"value": jsonValue("renewed"),
"preserveLease": json.RawMessage(`true`),
})); err != nil {
t.Fatalf("preserveLease put failed: %v", err)
}
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "hello"),
"value": jsonValue("nope"),
"preserveLease": json.RawMessage(`true`),
})); err == nil || err.Error() != "Cannot preserve lease: key does not exist or has no lease" {
t.Fatalf("expected preserveLease rejection, got %v", err)
}
// Rename, including conflict and missing-source paths.
if _, err := state.rename(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "old"),
"newKey": jsonString(prefix + "new"),
})); err == nil || err.Error() != "ETCD_NOT_FOUND: source key does not exist" {
t.Fatalf("expected rename not-found, got %v", err)
}
if _, err := state.rename(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "hello"),
"newKey": jsonString(prefix + "renamed"),
})); err != nil {
t.Fatalf("rename failed: %v", err)
}
if _, err := state.rename(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "binary"),
"newKey": jsonString(prefix + "renamed"),
})); err == nil || err.Error() != "ETCD_CAS_CONFLICT: source changed or target already exists" {
t.Fatalf("expected rename conflict, got %v", err)
}
// list_prefix with continuation.
listed, err := state.listPrefix(paramsWith(map[string]json.RawMessage{
"prefix": jsonString(prefix),
"limit": json.RawMessage(`2`),
"includeValues": json.RawMessage(`true`),
}))
if err != nil {
t.Fatalf("list_prefix failed: %v", err)
}
list := listed.(map[string]any)
if len(list["keys"].([]any)) != 2 {
t.Fatalf("expected limited list, got %#v", list)
}
continuation, ok := list["continuation"].(string)
if !ok || continuation == "" {
t.Fatalf("expected continuation, got %#v", list)
}
nextPage, err := state.listPrefix(paramsWith(map[string]json.RawMessage{
"prefix": jsonString(prefix),
"limit": json.RawMessage(`100`),
"includeValues": json.RawMessage(`true`),
"continuation": json.RawMessage(`"` + continuation + `"`),
}))
if err != nil {
t.Fatalf("continuation list failed: %v", err)
}
nextKeys := nextPage.(map[string]any)["keys"].([]any)
if len(nextKeys) == 0 {
t.Fatalf("continuation must resume after the first page")
}
firstContinued := nextKeys[0].(map[string]any)["key"].(string)
if firstContinued == list["keys"].([]any)[1].(map[string]any)["key"].(string) {
t.Fatalf("continuation overlapped the first page")
}
// --- history -------------------------------------------------------
// Explicit startRevision keeps the window above any revision compacted by
// earlier verification runs (the flow itself compacts at the end).
history, err := state.history(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "renamed"),
"startRevision": json.RawMessage(longString(baseRevision)),
}))
if err != nil {
t.Fatalf("history failed: %v", err)
}
historyRow := history.(map[string]any)
events := historyRow["events"].([]any)
// The renamed key only exists from the rename revision onward; the old
// key's earlier revisions belong to the old key.
if len(events) != 1 {
t.Fatalf("expected exactly the rename event, got %#v", historyRow)
}
if historyRow["truncated"] != false {
t.Fatalf("history should not be truncated for a short window: %#v", historyRow)
}
firstEvent := events[0].(map[string]any)
if _, err := strconvParse(firstEvent["revision"].(string)); err != nil {
t.Fatalf("revisions must be strings: %#v", firstEvent)
}
// --- watch ---------------------------------------------------------
watchStarted, err := state.watchStart(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "watched"),
"includePrevKv": json.RawMessage(`true`),
}))
if err != nil {
t.Fatalf("watch_start failed: %v", err)
}
watchID := watchStarted.(map[string]any)["watchId"].(string)
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "watched"),
"value": jsonValue("first"),
})); err != nil {
t.Fatalf("watched put failed: %v", err)
}
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "watched"),
"value": jsonValue("second"),
})); err != nil {
t.Fatalf("watched put 2 failed: %v", err)
}
deadline := time.Now().Add(10 * time.Second)
var watchBatches []any
for time.Now().Before(deadline) {
polled, err := state.watchPoll(paramsWith(map[string]json.RawMessage{"watchId": json.RawMessage(`"` + watchID + `"`)}))
if err != nil {
t.Fatalf("watch_poll failed: %v", err)
}
batches := polled.(map[string]any)["batches"].([]any)
watchBatches = append(watchBatches, batches...)
if len(watchBatches) >= 2 {
break
}
time.Sleep(100 * time.Millisecond)
}
if len(watchBatches) < 2 {
t.Fatalf("watch did not deliver both events: %#v", watchBatches)
}
secondBatch := watchBatches[len(watchBatches)-1].(map[string]any)
secondEvent := secondBatch["events"].([]map[string]any)[0]
if secondEvent["previousValue"].(map[string]any)["data"] != "first" {
t.Fatalf("prevKv missing on watch event: %#v", secondEvent)
}
stopped, err := state.watchStop(paramsWith(map[string]json.RawMessage{"watchId": json.RawMessage(`"` + watchID + `"`)}))
if err != nil || stopped.(map[string]bool)["stopped"] != true {
t.Fatalf("watch_stop failed: %#v %v", stopped, err)
}
if _, err := state.watchPoll(paramsWith(map[string]json.RawMessage{"watchId": json.RawMessage(`"` + watchID + `"`)})); err == nil || err.Error() != "ETCD_WATCH_NOT_FOUND: watch does not exist" {
t.Fatalf("expected watch-not-found after stop, got %v", err)
}
// --- lease ---------------------------------------------------------
granted, err := state.leaseGrant(paramsWith(map[string]json.RawMessage{"ttl": json.RawMessage(`120`)}))
if err != nil {
t.Fatalf("lease_grant failed: %v", err)
}
leaseID := granted.(map[string]any)["id"].(string)
if _, err := state.leaseKeepAlive(paramsWith(map[string]json.RawMessage{"id": json.RawMessage(leaseID)})); err != nil {
t.Fatalf("lease_keepalive_once failed: %v", err)
}
leaseListed, err := state.leaseList(paramsWith(map[string]json.RawMessage{}))
if err != nil {
t.Fatalf("lease_list failed: %v", err)
}
leaseListRow := leaseListed.(map[string]any)
if leaseListRow["partial"] != false {
t.Fatalf("lease list must be complete on etcd >= 3.4: %#v", leaseListRow)
}
foundLease := false
for _, lease := range leaseListRow["leases"].([]map[string]any) {
if lease["id"] == leaseID {
foundLease = true
}
}
if !foundLease {
t.Fatalf("granted lease %s missing from list: %#v", leaseID, leaseListRow)
}
leaseDetail, err := state.leaseGet(paramsWith(map[string]json.RawMessage{
"id": json.RawMessage(leaseID),
"includeKeys": json.RawMessage(`true`),
}))
if err != nil {
t.Fatalf("lease_get failed: %v", err)
}
detail := leaseDetail.(map[string]any)
if detail["grantedTtl"].(int64) != 120 && detail["grantedTtl"].(int64) < 100 {
t.Fatalf("unexpected granted ttl: %#v", detail)
}
if _, err := state.leaseRevoke(paramsWith(map[string]json.RawMessage{"id": json.RawMessage(leaseID)})); err != nil {
t.Fatalf("lease_revoke failed: %v", err)
}
// --- auth ----------------------------------------------------------
userName := fmt.Sprintf("dbx_live_%d", time.Now().UnixNano()%100000)
if _, err := state.authUserAdd(paramsWith(map[string]json.RawMessage{
"user": jsonString(userName),
"password": jsonString("secret123"),
})); err != nil {
t.Fatalf("user add failed: %v", err)
}
if _, err := state.authRoleAdd(paramsWith(map[string]json.RawMessage{"role": jsonString("dbx_live_role")})); err != nil {
t.Fatalf("role add failed: %v", err)
}
if _, err := state.authRolePermission(paramsWith(map[string]json.RawMessage{
"role": jsonString("dbx_live_role"),
"resource": jsonString("prefix"),
"key": jsonString(prefix),
"access": jsonString("READWRITE"),
}), true); err != nil {
t.Fatalf("role grant permission failed: %v", err)
}
roleDetail, err := state.authRoleGet(paramsWith(map[string]json.RawMessage{"role": jsonString("dbx_live_role")}))
if err != nil {
t.Fatalf("role get failed: %v", err)
}
permissions := roleDetail.(map[string]any)["permissions"].([]map[string]any)
if len(permissions) != 1 || permissions[0]["access"] != "readwrite" || permissions[0]["resource"] != "prefix" {
t.Fatalf("unexpected permission shape: %#v", permissions)
}
if _, err := state.authUserGrantRevokeRole(paramsWith(map[string]json.RawMessage{
"user": jsonString(userName),
"role": jsonString("dbx_live_role"),
}), true); err != nil {
t.Fatalf("grant role failed: %v", err)
}
userDetail, err := state.authUserGet(paramsWith(map[string]json.RawMessage{"user": jsonString(userName)}))
if err != nil {
t.Fatalf("user get failed: %v", err)
}
if len(userDetail.(map[string]any)["roles"].([]string)) == 0 {
t.Fatalf("expected granted role: %#v", userDetail)
}
users, err := state.authUserList(paramsWith(map[string]json.RawMessage{}))
if err != nil {
t.Fatalf("user list failed: %v", err)
}
if !containsAny(users.(map[string]any)["users"].([]string), userName) {
t.Fatalf("user list missing created user: %#v", users)
}
if _, err := state.authUserGrantRevokeRole(paramsWith(map[string]json.RawMessage{
"user": jsonString(userName),
"role": jsonString("dbx_live_role"),
}), false); err != nil {
t.Fatalf("revoke role failed: %v", err)
}
if _, err := state.authRolePermission(paramsWith(map[string]json.RawMessage{
"role": jsonString("dbx_live_role"),
"resource": jsonString("prefix"),
"key": jsonString(prefix),
}), false); err != nil {
t.Fatalf("role revoke permission failed: %v", err)
}
if _, err := state.authUserDelete(paramsWith(map[string]json.RawMessage{"user": jsonString(userName)})); err != nil {
t.Fatalf("user delete failed: %v", err)
}
if _, err := state.authRoleDelete(paramsWith(map[string]json.RawMessage{"role": jsonString("dbx_live_role")})); err != nil {
t.Fatalf("role delete failed: %v", err)
}
// --- status / maintenance ------------------------------------------
status, err := state.status(paramsWith(map[string]json.RawMessage{}))
if err != nil {
t.Fatalf("kv_status failed: %v", err)
}
statusRow := status.(map[string]any)
if statusRow["clusterId"] == nil || statusRow["leaderId"] == nil {
t.Fatalf("status missing cluster identity: %#v", statusRow)
}
members := statusRow["members"].([]map[string]any)
if len(members) == 0 || members[0]["reachable"] != true || members[0]["version"] == nil {
t.Fatalf("unexpected member rows: %#v", members)
}
if keyCount, err := strconvParse(statusRow["keyCount"].(string)); err != nil || keyCount <= 0 {
t.Fatalf("keyCount must be a positive numeric string: %#v", statusRow["keyCount"])
}
compactTarget := revisionOf(t, putResult)
if compactTarget < baseRevision {
compactTarget = baseRevision
}
if _, err := state.compact(paramsWith(map[string]json.RawMessage{"revision": json.RawMessage(statusRow["revision"].(string))})); err != nil {
t.Fatalf("compact failed: %v", err)
}
if _, err := state.compact(paramsWith(map[string]json.RawMessage{"revision": json.RawMessage(`999999999999`)})); err == nil || !strings.Contains(err.Error(), "future revision") {
t.Fatalf("expected raw future-revision error (Java parity), got %v", err)
}
defragged, err := state.defrag(paramsWith(map[string]json.RawMessage{
"endpoints": json.RawMessage(`["` + endpoints + `"]`),
}))
if err != nil {
t.Fatalf("defrag failed: %v", err)
}
defragRow := defragged.(map[string]any)["members"].([]map[string]any)[0]
if defragRow["status"] != "succeeded" {
t.Fatalf("defrag should succeed on a live member: %#v", defragRow)
}
// --- delete + cleanup ----------------------------------------------
for _, key := range []string{prefix + "binary", prefix + "leased", prefix + "renamed", prefix + "watched"} {
if _, err := state.delete(paramsWith(map[string]json.RawMessage{"key": jsonString(key)})); err != nil {
t.Fatalf("delete %s failed: %v", key, err)
}
}
if _, err := state.delete(paramsWith(map[string]json.RawMessage{"key": jsonString(prefix + "renamed")})); err != nil {
t.Fatalf("delete missing key failed: %v", err)
}
}
// TestLiveEtcdProtocolFlow drives the multi-session runtime exactly like the
// Rust host does: NDJSON lines with agentSessionId routing.
func TestLiveEtcdProtocolFlow(t *testing.T) {
if os.Getenv("DBX_ETCD_LIVE") != "1" {
t.Skip("set DBX_ETCD_LIVE=1 to run the live etcd agent test")
}
endpoints := envOrDefault("DBX_ETCD_ENDPOINTS", "http://127.0.0.1:10700")
user := envOrDefault("DBX_ETCD_USER", "root")
password := envOrDefault("DBX_ETCD_PASSWORD", "123456")
server := newRuntimeServer()
connection := fmt.Sprintf(`{"etcd_endpoints":%q,"username":%q,"password":%q}`, endpoints, user, password)
handshake, _ := server.handleLine(`{"id":1,"method":"handshake","params":{}}`)
if handshake.Error != nil {
t.Fatalf("handshake failed: %#v", handshake)
}
openLine := fmt.Sprintf(`{"id":2,"method":"open_session","params":{"agentSessionId":"s1","connection":%s}}`, connection)
opened, _ := server.handleLine(openLine)
if opened.Error != nil {
t.Fatalf("open_session failed: %#v", opened)
}
putLine := fmt.Sprintf(`{"id":3,"method":"kv_put","params":{"agentSessionId":"s1","key":"dbx:proto","value":{"encoding":"utf8","data":"flow"}}}`)
put, _ := server.handleLine(putLine)
if put.Error != nil {
t.Fatalf("kv_put failed: %#v", put)
}
unknown, _ := server.handleLine(`{"id":4,"method":"kv_describe","params":{"agentSessionId":"s1"}}`)
if unknown.Error == nil || !strings.Contains(unknown.Error.Message, "unknown method") {
t.Fatalf("expected unknown-method error, got %#v", unknown)
}
closed, _ := server.handleLine(`{"id":5,"method":"close_session","params":{"agentSessionId":"s1"}}`)
if closed.Error != nil {
t.Fatalf("close_session failed: %#v", closed)
}
}
func envOrDefault(key, fallback string) string {
if value := strings.TrimSpace(os.Getenv(key)); value != "" {
return value
}
return fallback
}
func paramsWith(params map[string]json.RawMessage) map[string]json.RawMessage {
return params
}
func jsonString(value string) json.RawMessage {
encoded, _ := json.Marshal(value)
return json.RawMessage(encoded)
}
func jsonValue(value string) json.RawMessage {
encoded, _ := json.Marshal(value)
return json.RawMessage(`{"encoding":"utf8","data":` + string(encoded) + `}`)
}
func revisionOf(t *testing.T, result any) int64 {
t.Helper()
row, ok := result.(map[string]any)
if !ok {
t.Fatalf("expected map result, got %#v", result)
}
revision, err := strconvParse(row["revision"].(string))
if err != nil {
t.Fatalf("revision not numeric: %#v", row["revision"])
}
return revision
}
func strconvParse(value string) (int64, error) {
var parsed int64
_, err := fmt.Sscan(value, &parsed)
return parsed, err
}
func containsAny(values []string, target string) bool {
for _, value := range values {
if value == target {
return true
}
}
return false
}
+524
View File
@@ -0,0 +1,524 @@
package main
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
"time"
"unicode/utf8"
"go.etcd.io/etcd/api/v3/mvccpb"
clientv3 "go.etcd.io/etcd/client/v3"
)
const defaultListLimit = 100
const preserveLeaseMaxAttempts = 3
func (s *etcdSession) listPrefix(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
prefix := stringOrDefault(params, "prefix", "")
limit := intOrDefault(params, "limit", defaultListLimit)
if limit < 1 {
limit = 1
}
revision := longOrNull(params, "revision")
includeValues := boolOrDefault(params, "includeValues", false)
continuation := stringOrNull(params, "continuation")
start := prefixStart(prefix)
if continuation != nil && *continuation != "" {
decoded, err := base64.StdEncoding.DecodeString(*continuation)
if err != nil {
return nil, err
}
start = string(decoded)
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
options := []clientv3.OpOption{
clientv3.WithRange(prefixEnd(prefix)),
clientv3.WithLimit(int64(limit)),
clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend),
}
if revision != nil && *revision > 0 {
options = append(options, clientv3.WithRev(*revision))
}
response, err := client.Get(ctx, start, options...)
if err != nil {
return nil, err
}
keys := make([]any, 0, len(response.Kvs))
for _, item := range response.Kvs {
row := metadataMap(item)
row["key"] = displayBytes(item.Key)
row["keyBytes"] = bytesObject(item.Key)
if includeValues {
row["value"] = valueObject(item.Value)
}
keys = append(keys, row)
}
result := map[string]any{"keys": keys}
if response.More && len(response.Kvs) > 0 {
result["continuation"] = nextContinuation(response.Kvs[len(response.Kvs)-1].Key)
} else {
result["continuation"] = nil
}
result["revision"] = longString(response.Header.Revision)
return result, nil
}
func (s *etcdSession) get(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
key, err := keyBytesParam(params)
if err != nil {
return nil, err
}
revision := longOrNull(params, "revision")
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
options := []clientv3.OpOption{}
if revision != nil && *revision > 0 {
options = append(options, clientv3.WithRev(*revision))
}
response, err := client.Get(ctx, key, options...)
if err != nil {
return nil, err
}
if len(response.Kvs) == 0 {
return map[string]any{"found": false, "key": nil, "value": nil, "metadata": nil}, nil
}
item := response.Kvs[0]
result := map[string]any{
"found": true,
"key": displayBytes(item.Key),
"keyBytes": bytesObject(item.Key),
}
if !boolOrDefault(params, "metadataOnly", false) {
result["value"] = valueObject(item.Value)
} else {
result["value"] = nil
}
metadata, err := s.metadataWithTtl(item)
if err != nil {
return nil, err
}
result["metadata"] = metadata
return result, nil
}
func (s *etcdSession) put(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
key, err := keyBytesParam(params)
if err != nil {
return nil, err
}
value, err := parseValueObject(rawObject(params, "value"))
if err != nil {
return nil, err
}
expectedModRevision := longOrNull(params, "expectedModRevision")
expectedCreateRevision := longOrNull(params, "expectedCreateRevision")
leaseValue := longOrNull(params, "lease")
ttlValue := longOrNull(params, "ttl")
preserveLease := boolOrDefault(params, "preserveLease", false)
hasLease := leaseValue != nil
hasTtl := ttlValue != nil
if (hasLease && hasTtl) || (preserveLease && (hasLease || hasTtl)) {
return nil, errors.New("lease, ttl, and preserveLease cannot be specified together")
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
if preserveLease {
revision, err := putPreservingLease(client, ctx, key, value)
if err != nil {
return nil, err
}
return map[string]any{"revision": longString(revision)}, nil
}
var leaseID clientv3.LeaseID
var grantedLeaseID clientv3.LeaseID
if hasTtl {
if *ttlValue <= 0 {
return nil, errors.New("ttl must be a positive integer")
}
grant, err := client.Lease.Grant(ctx, *ttlValue)
if err != nil {
return nil, err
}
grantedLeaseID = grant.ID
leaseID = grant.ID
} else if hasLease {
leaseID = clientv3.LeaseID(*leaseValue)
}
revision, err := func() (int64, error) {
if expectedModRevision != nil || expectedCreateRevision != nil {
var comparisons []clientv3.Cmp
if expectedModRevision != nil {
comparisons = append(comparisons, clientv3.Compare(clientv3.ModRevision(key), "=", *expectedModRevision))
}
if expectedCreateRevision != nil {
comparisons = append(comparisons, clientv3.Compare(clientv3.CreateRevision(key), "=", *expectedCreateRevision))
}
txn := client.Txn(ctx).If(comparisons...).Then(clientv3.OpPut(key, value, clientv3.WithLease(leaseID)))
response, err := txn.Commit()
if err != nil {
return 0, err
}
if !response.Succeeded {
return 0, errors.New("ETCD_CAS_CONFLICT: key changed after it was loaded")
}
return response.Header.Revision, nil
}
response, err := client.Put(ctx, key, value, clientv3.WithLease(leaseID))
if err != nil {
return 0, err
}
return response.Header.Revision, nil
}()
if err != nil && grantedLeaseID != 0 {
_, _ = client.Lease.Revoke(context.Background(), grantedLeaseID)
}
if err != nil {
return nil, err
}
return map[string]any{"revision": longString(revision)}, nil
}
func putPreservingLease(client *clientv3.Client, ctx context.Context, key, value string) (int64, error) {
for attempt := 0; attempt < preserveLeaseMaxAttempts; attempt++ {
existing, err := client.Get(ctx, key)
if err != nil {
return 0, err
}
if len(existing.Kvs) == 0 || existing.Kvs[0].Lease <= 0 {
return 0, errors.New("Cannot preserve lease: key does not exist or has no lease")
}
current := existing.Kvs[0]
txn := client.Txn(ctx).
If(clientv3.Compare(clientv3.ModRevision(key), "=", current.ModRevision)).
Then(clientv3.OpPut(key, value, clientv3.WithLease(clientv3.LeaseID(current.Lease))))
response, err := txn.Commit()
if err != nil {
return 0, err
}
if response.Succeeded {
return response.Header.Revision, nil
}
}
return 0, errors.New("Cannot preserve lease: key changed concurrently; retry the save")
}
func (s *etcdSession) delete(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
key, err := keyBytesParam(params)
if err != nil {
return nil, err
}
expectedModRevision := longOrNull(params, "expectedModRevision")
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
if expectedModRevision != nil {
txn := client.Txn(ctx).
If(clientv3.Compare(clientv3.ModRevision(key), "=", *expectedModRevision)).
Then(clientv3.OpDelete(key))
response, err := txn.Commit()
if err != nil {
return nil, err
}
if !response.Succeeded {
return nil, errors.New("ETCD_CAS_CONFLICT: key changed after it was loaded")
}
return map[string]any{"deleted": int64(1), "revision": longString(response.Header.Revision)}, nil
}
response, err := client.Delete(ctx, key)
if err != nil {
return nil, err
}
return map[string]any{"deleted": response.Deleted, "revision": longString(response.Header.Revision)}, nil
}
func (s *etcdSession) rename(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
sourceKey, err := keyBytesParam(params)
if err != nil {
return nil, err
}
newKey := stringOrNull(params, "newKey")
if newKey == nil || *newKey == "" {
return nil, errors.New("ETCD_NEWKEY_REQUIRED")
}
targetKey := *newKey
if sourceKey == targetKey {
return map[string]any{"renamed": true, "revision": nil}, nil
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
sourceResponse, err := client.Get(ctx, sourceKey)
if err != nil {
return nil, err
}
if len(sourceResponse.Kvs) == 0 {
return nil, errors.New("ETCD_NOT_FOUND: source key does not exist")
}
source := sourceResponse.Kvs[0]
expected := longOrNull(params, "expectedModRevision")
expectedRevision := source.ModRevision
if expected != nil {
expectedRevision = *expected
}
putOption := []clientv3.OpOption{}
if source.Lease != 0 {
putOption = append(putOption, clientv3.WithLease(clientv3.LeaseID(source.Lease)))
}
txn := client.Txn(ctx).
If(
clientv3.Compare(clientv3.ModRevision(sourceKey), "=", expectedRevision),
clientv3.Compare(clientv3.CreateRevision(targetKey), "=", 0),
).
Then(
clientv3.OpPut(targetKey, string(source.Value), putOption...),
clientv3.OpDelete(sourceKey),
)
response, err := txn.Commit()
if err != nil {
return nil, err
}
if !response.Succeeded {
return nil, errors.New("ETCD_CAS_CONFLICT: source changed or target already exists")
}
return map[string]any{"renamed": true, "revision": longString(response.Header.Revision)}, nil
}
func (s *etcdSession) metadataWithTtl(item *mvccpb.KeyValue) (map[string]any, error) {
metadata := metadataMap(item)
if item.Lease > 0 {
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
defer cancel()
lease, err := client.Lease.TimeToLive(ctx, clientv3.LeaseID(item.Lease))
if err != nil {
return nil, err
}
metadata["ttl"] = lease.TTL
}
return metadata, nil
}
func metadataMap(item *mvccpb.KeyValue) map[string]any {
return map[string]any{
"createRevision": longString(item.CreateRevision),
"modRevision": longString(item.ModRevision),
"version": longString(item.Version),
"lease": longString(item.Lease),
"valueSize": len(item.Value),
}
}
func prefixEnd(prefix string) string {
if prefix == "" {
return "\x00"
}
end := []byte(prefix)
for i := len(end) - 1; i >= 0; i-- {
if end[i] < 0xff {
end[i]++
return string(end[:i+1])
}
}
return "\x00"
}
func prefixStart(prefix string) string {
if prefix == "" {
return "\x00"
}
return prefix
}
func nextContinuation(key []byte) string {
next := make([]byte, len(key)+1)
copy(next, key)
return base64.StdEncoding.EncodeToString(next)
}
func keyBytesParam(params map[string]json.RawMessage) (string, error) {
if encoded := rawObject(params, "keyBytes"); encoded != nil {
value, err := parseValueObject(encoded)
if err != nil {
return "", err
}
return value, nil
}
if raw, ok := params["key"]; ok && raw != nil && string(raw) != "null" {
var key string
if err := json.Unmarshal(raw, &key); err != nil {
return "", err
}
return key, nil
}
return "", errors.New("Key is required")
}
func rawObject(params map[string]json.RawMessage, key string) map[string]json.RawMessage {
raw := params[key]
if len(raw) == 0 || string(raw) == "null" {
return nil
}
var object map[string]json.RawMessage
if err := json.Unmarshal(raw, &object); err != nil || object == nil {
return nil
}
return object
}
func bytesObject(bytes []byte) map[string]any {
return map[string]any{
"encoding": "base64",
"data": base64.StdEncoding.EncodeToString(bytes),
}
}
func valueObject(value []byte) map[string]any {
if utf8.Valid(value) {
return map[string]any{"encoding": "utf8", "data": string(value)}
}
return bytesObject(value)
}
func displayBytes(bytes []byte) string {
if utf8.Valid(bytes) {
return string(bytes)
}
return base64.StdEncoding.EncodeToString(bytes)
}
func parseValueObject(value map[string]json.RawMessage) (string, error) {
if value == nil {
value = map[string]json.RawMessage{}
}
encoding := stringOrDefault(value, "encoding", "utf8")
data := stringOrDefault(value, "data", "")
if encoding == "base64" {
decoded, err := base64.StdEncoding.DecodeString(data)
if err != nil {
return "", err
}
return string(decoded), nil
}
if encoding != "utf8" {
return "", fmt.Errorf("Unsupported value encoding: %s", encoding)
}
return data, nil
}
func longString(value int64) string {
return strconv.FormatInt(value, 10)
}
func unsignedLongString(value int64) string {
return strconv.FormatUint(uint64(value), 10)
}
func stringOrNull(params map[string]json.RawMessage, key string) *string {
raw := params[key]
if len(raw) == 0 || string(raw) == "null" {
return nil
}
var value string
if err := json.Unmarshal(raw, &value); err != nil {
var text *string
return text
}
return &value
}
func stringOrDefault(params map[string]json.RawMessage, key string, fallback string) string {
value := stringOrNull(params, key)
if value == nil {
return fallback
}
return *value
}
func intOrDefault(params map[string]json.RawMessage, key string, fallback int) int {
raw := params[key]
if len(raw) == 0 || string(raw) == "null" {
return fallback
}
var value int
if err := json.Unmarshal(raw, &value); err != nil {
return fallback
}
return value
}
func boolOrDefault(params map[string]json.RawMessage, key string, fallback bool) bool {
raw := params[key]
if len(raw) == 0 || string(raw) == "null" {
return fallback
}
var value bool
if err := json.Unmarshal(raw, &value); err != nil {
return fallback
}
return value
}
func longOrNull(params map[string]json.RawMessage, key string) *int64 {
raw := params[key]
if len(raw) == 0 || string(raw) == "null" {
return nil
}
var value int64
if err := json.Unmarshal(raw, &value); err != nil {
return nil
}
return &value
}
func requiredPositiveLong(params map[string]json.RawMessage, field string) (int64, error) {
value := longOrNull(params, field)
if value == nil || *value <= 0 {
return 0, fmt.Errorf("ETCD_INVALID_%s: a positive integer is required", strings.ToUpper(field))
}
return *value, nil
}
func requiredString(params map[string]json.RawMessage, field string) (string, error) {
value := stringOrNull(params, field)
if value == nil || *value == "" {
return "", fmt.Errorf("ETCD_%s_REQUIRED", strings.ToUpper(field))
}
return *value, nil
}
+339
View File
@@ -0,0 +1,339 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"strconv"
"time"
"go.etcd.io/etcd/api/v3/etcdserverpb"
clientv3 "go.etcd.io/etcd/client/v3"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
const (
maxLeaseAttachedKeys = 256
defaultLeaseListLimit = 100
maxLeaseListLimit = 200
leaseListConcurrency = 8
leaseListDeadlineSeconds = 5
)
type leaseListDeadline struct {
deadline time.Time
}
func (d leaseListDeadline) remaining() (time.Duration, error) {
remaining := time.Until(d.deadline)
if remaining <= 0 {
return 0, errors.New("ETCD_LEASE_LIST_TIMEOUT")
}
return remaining, nil
}
func (s *etcdSession) leaseList(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
deadline := leaseListDeadline{deadline: time.Now().Add(leaseListDeadlineSeconds * time.Second)}
limit := intOrDefault(params, "limit", defaultLeaseListLimit)
if limit < 1 {
limit = 1
}
if limit > maxLeaseListLimit {
limit = maxLeaseListLimit
}
continuation := stringOrNull(params, "continuation")
var afterLeaseID *uint64
if continuation != nil && *continuation != "" {
parsed, err := parseUnsignedLong(*continuation)
if err != nil {
return nil, err
}
afterLeaseID = &parsed
}
partial := false
leaseIDs, err := clusterLeaseIDs(client, deadline)
if err != nil {
if isLeaseListFallbackError(err) {
leaseIDs = s.knownLeaseIDs()
partial = true
} else {
return nil, err
}
} else {
for _, id := range leaseIDs {
s.rememberLease(id)
}
}
leaseIDs = leasePageIDs(leaseIDs, afterLeaseID, limit+1)
hasMore := len(leaseIDs) > limit
if len(leaseIDs) > limit {
leaseIDs = leaseIDs[:limit]
}
leases := []map[string]any{}
var lastProcessedID *uint64
deadlineReached := false
chunkLoop:
for offset := 0; offset < len(leaseIDs); offset += leaseListConcurrency {
end := offset + leaseListConcurrency
if end > len(leaseIDs) {
end = len(leaseIDs)
}
chunkIDs := leaseIDs[offset:end]
type ttlResult struct {
id uint64
ttl int64
granted int64
err error
}
results := make([]chan ttlResult, len(chunkIDs))
for index, id := range chunkIDs {
channel := make(chan ttlResult, 1)
results[index] = channel
go func(id uint64, channel chan ttlResult) {
remaining, err := deadline.remaining()
if err != nil {
channel <- ttlResult{id: id, err: err}
return
}
ctx, cancel := context.WithTimeout(context.Background(), remaining)
defer cancel()
response, err := client.Lease.TimeToLive(ctx, clientv3.LeaseID(int64(id)))
if err != nil {
channel <- ttlResult{id: id, err: err}
return
}
channel <- ttlResult{id: id, ttl: response.TTL, granted: response.GrantedTTL}
}(id, channel)
}
for _, channel := range results {
result := <-channel
id := result.id
if result.err != nil {
if errors.Is(result.err, errLeaseListDeadline) || isDeadlineError(result.err) {
partial = true
deadlineReached = true
break chunkLoop
}
if status.Code(result.err) == codes.NotFound {
s.forgetLease(id)
} else {
partial = true
}
lastProcessedID = &id
continue
}
leases = append(leases, map[string]any{
"id": unsignedLongString(int64(id)),
"ttl": result.ttl,
"grantedTtl": result.granted,
})
processed := id
lastProcessedID = &processed
}
}
var nextContinuation any
if deadlineReached && len(leaseIDs) > 0 {
if lastProcessedID != nil {
nextContinuation = unsignedLongString(int64(*lastProcessedID))
} else if afterLeaseID != nil {
nextContinuation = unsignedLongString(int64(*afterLeaseID))
} else {
nextContinuation = "0"
}
} else if hasMore && len(leaseIDs) > 0 {
nextContinuation = unsignedLongString(int64(leaseIDs[len(leaseIDs)-1]))
}
return map[string]any{
"leases": leases,
"partial": partial,
"nextContinuation": nextContinuation,
}, nil
}
var errLeaseListDeadline = errors.New("ETCD_LEASE_LIST_TIMEOUT")
func isDeadlineError(err error) bool {
return errors.Is(err, context.DeadlineExceeded) || status.Code(err) == codes.DeadlineExceeded
}
func isLeaseListFallbackError(err error) bool {
if errors.Is(err, errLeaseListDeadline) || errors.Is(err, context.DeadlineExceeded) {
return true
}
code := status.Code(err)
return code == codes.Unimplemented || code == codes.DeadlineExceeded
}
func parseUnsignedLong(value string) (uint64, error) {
parsed, err := strconv.ParseUint(value, 10, 64)
if err != nil {
return 0, fmt.Errorf("invalid lease continuation: %s", value)
}
return parsed, nil
}
func leasePageIDs(leaseIDs []uint64, afterLeaseID *uint64, fetchLimit int) []uint64 {
sorted := make([]uint64, len(leaseIDs))
copy(sorted, leaseIDs)
sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] })
result := []uint64{}
for _, id := range sorted {
if afterLeaseID != nil && id <= *afterLeaseID {
continue
}
if len(result) >= fetchLimit {
break
}
result = append(result, id)
}
return result
}
func clusterLeaseIDs(client *clientv3.Client, deadline leaseListDeadline) ([]uint64, error) {
remaining, err := deadline.remaining()
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), remaining)
defer cancel()
leaseClient := etcdserverpb.NewLeaseClient(client.ActiveConnection())
response, err := leaseClient.LeaseLeases(ctx, &etcdserverpb.LeaseLeasesRequest{})
if err != nil {
return nil, err
}
ids := make([]uint64, 0, len(response.Leases))
for _, lease := range response.Leases {
ids = append(ids, uint64(lease.ID))
}
return ids, nil
}
func (s *etcdSession) leaseGet(params map[string]json.RawMessage) (any, error) {
id, err := requiredPositiveLong(params, "id")
if err != nil {
return nil, err
}
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
options := []clientv3.LeaseOption{}
if boolOrDefault(params, "includeKeys", false) {
options = append(options, clientv3.WithAttachedKeys())
}
response, err := client.Lease.TimeToLive(ctx, clientv3.LeaseID(id), options...)
if err != nil {
return nil, err
}
s.rememberLease(uint64(id))
keys := []map[string]any{}
for index, key := range response.Keys {
if index >= maxLeaseAttachedKeys {
break
}
keys = append(keys, bytesObject(key))
}
return map[string]any{
"id": unsignedLongString(int64(response.ID)),
"ttl": response.TTL,
"grantedTtl": response.GrantedTTL,
"keys": keys,
"truncated": len(response.Keys) > maxLeaseAttachedKeys,
}, nil
}
func (s *etcdSession) leaseGrant(params map[string]json.RawMessage) (any, error) {
ttl, err := requiredPositiveLong(params, "ttl")
if err != nil {
return nil, err
}
requestedID := longOrNull(params, "id")
if requestedID != nil && *requestedID < 0 {
return nil, errors.New("id must be a positive integer or 0")
}
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
var grantedID clientv3.LeaseID
var grantedTTL int64
if requestedID == nil || *requestedID == 0 {
response, err := client.Lease.Grant(ctx, ttl)
if err != nil {
return nil, err
}
grantedID = response.ID
grantedTTL = response.TTL
} else {
leaseClient := etcdserverpb.NewLeaseClient(client.ActiveConnection())
response, err := leaseClient.LeaseGrant(ctx, &etcdserverpb.LeaseGrantRequest{TTL: ttl, ID: *requestedID})
if err != nil {
return nil, err
}
grantedID = clientv3.LeaseID(response.ID)
grantedTTL = response.TTL
}
s.rememberLease(uint64(grantedID))
return map[string]any{
"id": unsignedLongString(int64(grantedID)),
"ttl": grantedTTL,
}, nil
}
func (s *etcdSession) leaseKeepAlive(params map[string]json.RawMessage) (any, error) {
id, err := requiredPositiveLong(params, "id")
if err != nil {
return nil, err
}
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
response, err := client.Lease.KeepAliveOnce(ctx, clientv3.LeaseID(id))
if err != nil {
return nil, err
}
s.rememberLease(uint64(id))
return map[string]any{
"id": unsignedLongString(id),
"ttl": response.TTL,
}, nil
}
func (s *etcdSession) leaseRevoke(params map[string]json.RawMessage) (any, error) {
id, err := requiredPositiveLong(params, "id")
if err != nil {
return nil, err
}
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
if _, err := client.Lease.Revoke(ctx, clientv3.LeaseID(id)); err != nil {
return nil, err
}
s.forgetLease(uint64(id))
return map[string]any{
"id": unsignedLongString(id),
"revoked": true,
}, nil
}
+468
View File
@@ -0,0 +1,468 @@
package main
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"os"
"runtime"
"strconv"
"strings"
"sync"
"time"
clientv3 "go.etcd.io/etcd/client/v3"
)
const (
protocolVersion = 2
legacyAgentSessionID = "__legacy__"
maxAgentSessions = 256
rpcTimeoutSeconds = 30
)
var capabilities = []string{
"connect", "test_connection", "kv", "kv_ttl", "kv_cas", "kv_list_values", "kv_status",
"kv_history", "etcd_compaction", "etcd_defrag", "etcd_watch", "etcd_lease", "etcd_auth",
"multi_session",
}
type request struct {
ID json.RawMessage `json:"id"`
Method string `json:"method"`
Params map[string]json.RawMessage `json:"params"`
}
type response struct {
JSONRPC string `json:"jsonrpc,omitempty"`
ID json.RawMessage `json:"id,omitempty"`
Result any `json:"result,omitempty"`
Error *rpcError `json:"error,omitempty"`
}
// etcdSession carries the per-connection etcd client and its derived state.
// Requests for one session are serialized by agentSession.mu; watch callbacks
// reach the maps through watchesMu instead so buffering never blocks dispatch.
type etcdSession struct {
clientMu sync.Mutex
client *clientv3.Client
connectedEndpoints []string
watchesMu sync.Mutex
watches map[string]*watchState
knownLeases map[uint64]struct{}
watchBufferedBytes int64
activeCancelMu sync.Mutex
activeCancel context.CancelFunc
}
func (s *etcdSession) reserveWatchBuffer(bytes int64) bool {
s.watchesMu.Lock()
defer s.watchesMu.Unlock()
if bytes < 0 || s.watchBufferedBytes+bytes > maxSessionWatchBufferBytes {
return false
}
s.watchBufferedBytes += bytes
return true
}
func (s *etcdSession) releaseWatchBuffer(bytes int64) {
s.watchesMu.Lock()
defer s.watchesMu.Unlock()
s.watchBufferedBytes -= bytes
if s.watchBufferedBytes < 0 {
s.watchBufferedBytes = 0
}
}
func (s *etcdSession) rememberLease(id uint64) {
s.watchesMu.Lock()
defer s.watchesMu.Unlock()
s.knownLeases[id] = struct{}{}
}
func (s *etcdSession) forgetLease(id uint64) {
s.watchesMu.Lock()
defer s.watchesMu.Unlock()
delete(s.knownLeases, id)
}
func (s *etcdSession) knownLeaseIDs() []uint64 {
s.watchesMu.Lock()
defer s.watchesMu.Unlock()
ids := make([]uint64, 0, len(s.knownLeases))
for id := range s.knownLeases {
ids = append(ids, id)
}
return ids
}
type agentSession struct {
state *etcdSession
mu sync.Mutex
}
func (a *agentSession) cancelActive() {
a.state.activeCancelMu.Lock()
cancel := a.state.activeCancel
a.state.activeCancelMu.Unlock()
if cancel != nil {
cancel()
}
}
type runtimeServer struct {
mu sync.RWMutex
sessions map[string]*agentSession
}
func main() {
configureRuntimeParallelism()
server := newRuntimeServer()
encoder := json.NewEncoder(os.Stdout)
var encoderMu sync.Mutex
var requests sync.WaitGroup
fmt.Fprintln(os.Stdout, `{"ready":true}`)
scanner := bufio.NewScanner(os.Stdin)
scanner.Buffer(make([]byte, 0, 64*1024), 512*1024*1024)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
}
var envelope request
if json.Unmarshal([]byte(line), &envelope) == nil && envelope.Method == "shutdown" {
requests.Wait()
result, _ := server.handleLine(line)
encoderMu.Lock()
_ = encoder.Encode(result)
encoderMu.Unlock()
server.closeAll()
return
}
requests.Add(1)
go func(line string) {
defer requests.Done()
result, _ := server.handleLine(line)
encoderMu.Lock()
defer encoderMu.Unlock()
if err := encoder.Encode(result); err != nil {
fmt.Fprintf(os.Stderr, "failed to write response: %v\n", err)
}
}(line)
}
requests.Wait()
server.closeAll()
}
func configureRuntimeParallelism() {
if raw := strings.TrimSpace(os.Getenv("DBX_AGENT_ETCD_GOMAXPROCS")); raw != "" {
if configured, err := strconv.Atoi(raw); err == nil && configured > 0 {
runtime.GOMAXPROCS(configured)
return
}
}
if strings.TrimSpace(os.Getenv("GOMAXPROCS")) != "" {
return
}
runtime.GOMAXPROCS(min(runtime.NumCPU(), 4))
}
func newRuntimeServer() *runtimeServer {
return &runtimeServer{sessions: map[string]*agentSession{}}
}
func (r *runtimeServer) handleLine(line string) (response, bool) {
var req request
if err := json.Unmarshal([]byte(line), &req); err != nil {
return errorResponse(nil, "", "", err), false
}
if len(req.ID) == 0 {
req.ID = json.RawMessage("1")
}
result, shutdown, err := r.dispatch(req.Method, req.Params)
if err != nil {
return errorResponse(req.ID, req.Method, stringParam(req.Params, "agentSessionId"), err), false
}
return response{JSONRPC: "2.0", ID: req.ID, Result: result}, shutdown
}
func (r *runtimeServer) dispatch(method string, params map[string]json.RawMessage) (any, bool, error) {
switch method {
case "handshake":
return handshakeResult(), false, nil
case "open_session":
id := requiredSessionID(params)
if id == "" {
return nil, false, errors.New("agentSessionId is required")
}
return r.openSession(id, params)
case "close_session":
return r.closeSession(stringParam(params, "agentSessionId")), false, nil
case "validate_session":
session, err := r.session(requiredSessionID(params))
if err != nil {
return nil, false, err
}
session.mu.Lock()
defer session.mu.Unlock()
result, err := session.state.validateConnection()
return result, false, err
case "cancel_session":
session, err := r.session(requiredSessionID(params))
if err != nil {
return nil, false, err
}
session.cancelActive()
return map[string]bool{"ok": true}, false, nil
case "test_connection":
return r.testConnection(params)
case "connect":
_ = r.closeSession(legacyAgentSessionID)
return r.openSession(legacyAgentSessionID, params)
case "disconnect":
return r.closeSession(legacyAgentSessionID), false, nil
case "shutdown":
return map[string]bool{"ok": true}, true, r.closeAll()
default:
id := stringParam(params, "agentSessionId")
if id == "" {
id = legacyAgentSessionID
}
session, err := r.session(id)
if err != nil {
return nil, false, err
}
session.mu.Lock()
defer session.mu.Unlock()
result, err := session.state.handle(method, params)
return result, false, err
}
}
func requiredSessionID(params map[string]json.RawMessage) string {
return strings.TrimSpace(stringParam(params, "agentSessionId"))
}
func (r *runtimeServer) openSession(id string, params map[string]json.RawMessage) (any, bool, error) {
r.mu.Lock()
if _, exists := r.sessions[id]; exists {
r.mu.Unlock()
return nil, false, fmt.Errorf("Agent session already exists: %s", id)
}
if len(r.sessions) >= maxAgentSessions {
r.mu.Unlock()
return nil, false, fmt.Errorf("Agent session limit reached: %d", maxAgentSessions)
}
session := &agentSession{state: newEtcdSession()}
r.sessions[id] = session
r.mu.Unlock()
if _, err := session.state.connect(params); err != nil {
r.mu.Lock()
delete(r.sessions, id)
r.mu.Unlock()
session.state.close()
return nil, false, err
}
return map[string]bool{"ok": true}, false, nil
}
func (r *runtimeServer) session(id string) (*agentSession, error) {
r.mu.RLock()
session := r.sessions[id]
r.mu.RUnlock()
if session == nil {
return nil, fmt.Errorf("Agent session not found: %s", id)
}
return session, nil
}
func (r *runtimeServer) closeSession(id string) any {
r.mu.Lock()
session := r.sessions[id]
delete(r.sessions, id)
r.mu.Unlock()
if session != nil {
session.cancelActive()
session.mu.Lock()
session.state.close()
session.mu.Unlock()
}
return map[string]bool{"ok": true}
}
func (r *runtimeServer) closeAll() error {
r.mu.Lock()
sessions := r.sessions
r.sessions = map[string]*agentSession{}
r.mu.Unlock()
var firstErr error
for _, session := range sessions {
session.cancelActive()
if err := session.state.close(); err != nil && firstErr == nil {
firstErr = err
}
}
return firstErr
}
func (r *runtimeServer) testConnection(params map[string]json.RawMessage) (any, bool, error) {
state := newEtcdSession()
defer state.close()
if _, err := state.connect(params); err != nil {
return nil, false, err
}
return map[string]bool{"ok": true}, false, nil
}
func newEtcdSession() *etcdSession {
return &etcdSession{
watches: map[string]*watchState{},
knownLeases: map[uint64]struct{}{},
}
}
func handshakeResult() map[string]any {
return map[string]any{
"protocolVersion": protocolVersion,
"agentProtocolVersion": protocolVersion,
"capabilities": capabilities,
}
}
func (s *etcdSession) beginOperation() (context.Context, context.CancelFunc) {
ctx, cancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
s.activeCancelMu.Lock()
s.activeCancel = cancel
s.activeCancelMu.Unlock()
return ctx, cancel
}
func (s *etcdSession) endOperation(cancel context.CancelFunc) {
cancel()
s.activeCancelMu.Lock()
s.activeCancel = nil
s.activeCancelMu.Unlock()
}
func (s *etcdSession) handle(method string, params map[string]json.RawMessage) (any, error) {
switch method {
case "handshake":
return handshakeResult(), nil
case "connect", "test_connection":
return s.connectMap(params)
case "validate_connection":
return s.validateConnection()
case "kv_list_prefix":
return s.listPrefix(params)
case "kv_get":
return s.get(params)
case "kv_put":
return s.put(params)
case "kv_delete":
return s.delete(params)
case "kv_rename":
return s.rename(params)
case "kv_history":
return s.history(params)
case "kv_status":
return s.status(params)
case "etcd_compact":
return s.compact(params)
case "etcd_defrag":
return s.defrag(params)
case "etcd_watch_start":
return s.watchStart(params)
case "etcd_watch_poll":
return s.watchPoll(params)
case "etcd_watch_stop":
return s.watchStop(params)
case "etcd_lease_list":
return s.leaseList(params)
case "etcd_lease_get":
return s.leaseGet(params)
case "etcd_lease_grant":
return s.leaseGrant(params)
case "etcd_lease_keepalive_once":
return s.leaseKeepAlive(params)
case "etcd_lease_revoke":
return s.leaseRevoke(params)
case "etcd_auth_user_list":
return s.authUserList(params)
case "etcd_auth_user_get":
return s.authUserGet(params)
case "etcd_auth_user_add":
return s.authUserAdd(params)
case "etcd_auth_user_delete":
return s.authUserDelete(params)
case "etcd_auth_user_change_password":
return s.authUserChangePassword(params)
case "etcd_auth_user_grant_role":
return s.authUserGrantRevokeRole(params, true)
case "etcd_auth_user_revoke_role":
return s.authUserGrantRevokeRole(params, false)
case "etcd_auth_role_list":
return s.authRoleList(params)
case "etcd_auth_role_get":
return s.authRoleGet(params)
case "etcd_auth_role_add":
return s.authRoleAdd(params)
case "etcd_auth_role_delete":
return s.authRoleDelete(params)
case "etcd_auth_role_grant_permission":
return s.authRolePermission(params, true)
case "etcd_auth_role_revoke_permission":
return s.authRolePermission(params, false)
case "disconnect":
s.close()
return map[string]bool{"ok": true}, nil
case "shutdown":
s.close()
return map[string]bool{"ok": true}, nil
default:
return nil, fmt.Errorf("unknown method: %s", method)
}
}
func decodeParams(params map[string]json.RawMessage, target any) error {
if params == nil {
params = map[string]json.RawMessage{}
}
data, err := json.Marshal(params)
if err != nil {
return err
}
return json.Unmarshal(data, target)
}
func stringParam(params map[string]json.RawMessage, key string) string {
if params == nil {
return ""
}
var result string
_ = json.Unmarshal(params[key], &result)
return result
}
func intParam(params map[string]json.RawMessage, key string) int {
if params == nil {
return 0
}
var result int
_ = json.Unmarshal(params[key], &result)
return result
}
func boolParam(params map[string]json.RawMessage, key string) bool {
if params == nil {
return false
}
var result bool
_ = json.Unmarshal(params[key], &result)
return result
}
+502
View File
@@ -0,0 +1,502 @@
package main
import (
"encoding/base64"
"encoding/json"
"strings"
"testing"
"go.etcd.io/etcd/api/v3/mvccpb"
)
func TestHandshakeAdvertisesCapabilities(t *testing.T) {
server := newRuntimeServer()
result, _, err := server.dispatch("handshake", nil)
if err != nil {
t.Fatalf("handshake failed: %v", err)
}
handshake := result.(map[string]any)
if handshake["protocolVersion"] != 2 || handshake["agentProtocolVersion"] != 2 {
t.Fatalf("unexpected protocol versions: %#v", handshake)
}
want := []string{
"connect", "test_connection", "kv", "kv_ttl", "kv_cas", "kv_list_values", "kv_status",
"kv_history", "etcd_compaction", "etcd_defrag", "etcd_watch", "etcd_lease", "etcd_auth",
"multi_session",
}
got, ok := handshake["capabilities"].([]string)
if !ok {
t.Fatalf("capabilities missing: %#v", handshake)
}
if strings.Join(got, ",") != strings.Join(want, ",") {
t.Fatalf("capability mismatch:\n got %v\nwant %v", got, want)
}
for _, capability := range got {
if capability == "structured_error_v1" {
t.Fatalf("structured_error_v1 must not be advertised (Java parity)")
}
}
}
func TestHandleLineReportsSessionNotFound(t *testing.T) {
server := newRuntimeServer()
response, _ := server.handleLine(`{"id":7,"method":"kv_get","params":{"key":"a"}}`)
if response.Error == nil {
t.Fatalf("expected error response, got %#v", response)
}
if response.Error.Code != -1 {
t.Fatalf("expected code -1, got %d", response.Error.Code)
}
if response.Error.Message != "Agent session not found: __legacy__" {
t.Fatalf("unexpected message: %q", response.Error.Message)
}
var raw map[string]json.RawMessage
if err := json.Unmarshal(response.ID, &raw); err == nil {
_ = raw
}
if string(response.ID) != "7" {
t.Fatalf("expected id 7, got %s", response.ID)
}
}
func TestOpenSessionValidatesAgentSessionID(t *testing.T) {
server := newRuntimeServer()
_, _, err := server.dispatch("open_session", map[string]json.RawMessage{})
if err == nil || !strings.Contains(err.Error(), "agentSessionId is required") {
t.Fatalf("expected agentSessionId requirement, got %v", err)
}
}
func TestOpenSessionRejectsDuplicate(t *testing.T) {
server := newRuntimeServer()
server.sessions["dup"] = &agentSession{state: newEtcdSession()}
_, _, err := server.dispatch("open_session", map[string]json.RawMessage{
"agentSessionId": json.RawMessage(`"dup"`),
})
if err == nil || !strings.Contains(err.Error(), "Agent session already exists") {
t.Fatalf("expected duplicate rejection, got %v", err)
}
}
func TestUnconnectedSessionErrors(t *testing.T) {
state := newEtcdSession()
if _, err := state.get(map[string]json.RawMessage{"key": json.RawMessage(`"k"`)}); err == nil || err.Error() != "Not connected" {
t.Fatalf("kv_get before connect: %v", err)
}
if _, err := state.history(map[string]json.RawMessage{"key": json.RawMessage(`"k"`)}); err == nil || err.Error() != "Not connected" {
t.Fatalf("kv_history before connect: %v", err)
}
if _, err := state.validateConnection(); err == nil || err.Error() != "Not connected" {
t.Fatalf("validate_connection before connect: %v", err)
}
}
func TestConnectTimeoutBounds(t *testing.T) {
cases := []struct {
configured int
want int
}{
{0, 30},
{-5, 1},
{1, 1},
{30, 30},
{300, 300},
{999, 300},
}
for _, testCase := range cases {
got := connectTimeoutSeconds(connectionParams{ConnectTimeoutSecs: testCase.configured})
if got != testCase.want {
t.Fatalf("connectTimeoutSeconds(%d) = %d, want %d", testCase.configured, got, testCase.want)
}
}
}
func TestGrpcMaxInboundMessageSize(t *testing.T) {
cases := []struct {
connection connectionParams
want int
}{
{connectionParams{}, defaultGrpcMaxInboundMessageSize},
{connectionParams{GrpcMaxInboundMessageSize: 1024}, minGrpcMaxInboundMessageSize},
{connectionParams{GrpcMaxInboundMessageSize: 999 * 1024 * 1024}, maxGrpcMaxInboundMessageSize},
{connectionParams{URLParams: "grpc_max_inbound_message_size=104857600"}, 100 * 1024 * 1024},
{connectionParams{URLParams: "?grpc_max_inbound_message_size=64"}, minGrpcMaxInboundMessageSize},
{connectionParams{URLParams: "grpc_max_inbound_message_size=abc"}, defaultGrpcMaxInboundMessageSize},
{connectionParams{GrpcMaxInboundMessageSize: 52428800, URLParams: "grpc_max_inbound_message_size=1024"}, 50 * 1024 * 1024},
}
for _, testCase := range cases {
got := grpcMaxInboundMessageSize(testCase.connection)
if got != testCase.want {
t.Fatalf("grpcMaxInboundMessageSize(%+v) = %d, want %d", testCase.connection, got, testCase.want)
}
}
}
func TestConnectionEndpoints(t *testing.T) {
cases := []struct {
name string
connection connectionParams
want []string
}{
{"host port fallback", connectionParams{Port: 2379}, []string{"http://127.0.0.1:2379"}},
{"ssl scheme", connectionParams{Host: "db", Port: 1, SSL: true}, []string{"https://db:1"}},
{"comma list", connectionParams{EtcdEndpoints: "a:1,b:2"}, []string{"http://a:1", "http://b:2"}},
{"newline list", connectionParams{Endpoints: "a:1\nb:2"}, []string{"http://a:1", "http://b:2"}},
{"precedence", connectionParams{EtcdEndpoints: "a:1", Endpoints: "b:2", ConnectionString: "c:3"}, []string{"http://a:1"}},
{"existing scheme", connectionParams{EtcdEndpoints: "https://a:1,http://b:2"}, []string{"https://a:1", "http://b:2"}},
{"blank entries", connectionParams{EtcdEndpoints: " a:1 , ,b:2"}, []string{"http://a:1", "http://b:2"}},
}
for _, testCase := range cases {
got := connectionEndpoints(testCase.connection)
if strings.Join(got, "|") != strings.Join(testCase.want, "|") {
t.Fatalf("%s: got %v, want %v", testCase.name, got, testCase.want)
}
}
}
func TestTLSConfigRequiresCertAndKeyTogether(t *testing.T) {
_, err := tlsConfigFor(connectionParams{SSL: true, ClientCertPath: "/tmp/cert.pem"})
if err == nil || err.Error() != "Client certificate and key must be provided together" {
t.Fatalf("expected paired cert/key error, got %v", err)
}
_, err = tlsConfigFor(connectionParams{SSL: true, CertPath: "/tmp/cert.pem", KeyPath: "/tmp/key.pem"})
if err == nil {
t.Fatalf("expected key pair load failure for missing files")
}
}
func TestHistoryStartRevision(t *testing.T) {
cases := []struct {
requested *int64
target int64
want int64
}{
{nil, 5000, 1},
{nil, 10000, 1},
{nil, 10001, 2},
{nil, 15000, 5001},
{int64Ptr(200), 15000, 200},
{int64Ptr(-5), 15000, 1},
}
for _, testCase := range cases {
got := historyStartRevision(testCase.requested, testCase.target)
if got != testCase.want {
t.Fatalf("historyStartRevision(%v, %d) = %d, want %d", testCase.requested, testCase.target, got, testCase.want)
}
}
}
func int64Ptr(value int64) *int64 {
return &value
}
func TestHistoryCollectorBounds(t *testing.T) {
collector := &historyCollector{limit: 3}
for i := int64(1); i <= 5; i++ {
collector.append(map[string]any{"revision": longString(i)})
}
if len(collector.rows) != 3 {
t.Fatalf("expected 3 retained rows, got %d", len(collector.rows))
}
if !collector.truncated {
t.Fatalf("expected truncated flag")
}
if collector.rows[0]["revision"] != "3" || collector.rows[2]["revision"] != "5" {
t.Fatalf("expected newest rows retained, got %v", collector.rows)
}
}
func TestPrefixEnd(t *testing.T) {
cases := []struct{ prefix, want string }{
{"", "\x00"},
{"abc", "abd"},
{"a\xff", "b"},
{"\xff\xff", "\x00"},
{"a\xffb", "a\xffc"},
}
for _, testCase := range cases {
if got := prefixEnd(testCase.prefix); got != testCase.want {
t.Fatalf("prefixEnd(%q) = %q, want %q", testCase.prefix, got, testCase.want)
}
}
}
func TestValueEncodings(t *testing.T) {
utf8Value := valueObject([]byte("hello"))
if utf8Value["encoding"] != "utf8" || utf8Value["data"] != "hello" {
t.Fatalf("unexpected utf8 value: %#v", utf8Value)
}
binaryValue := valueObject([]byte{0xff, 0xfe})
if binaryValue["encoding"] != "base64" {
t.Fatalf("expected base64 for binary: %#v", binaryValue)
}
roundTrip, err := parseValueObject(map[string]json.RawMessage{
"encoding": json.RawMessage(`"base64"`),
"data": json.RawMessage(`"` + binaryValue["data"].(string) + `"`),
})
if err != nil || roundTrip != string([]byte{0xff, 0xfe}) {
t.Fatalf("base64 round trip failed: %q %v", roundTrip, err)
}
if _, err := parseValueObject(map[string]json.RawMessage{"encoding": json.RawMessage(`"hex"`)}); err == nil {
t.Fatalf("expected unsupported encoding error")
}
if displayBytes([]byte{0xff}) != valueObject([]byte{0xff})["data"] {
t.Fatalf("displayBytes should fall back to base64")
}
}
func TestNextContinuation(t *testing.T) {
encoded := nextContinuation([]byte("key"))
decoded, err := decodeBase64String(encoded)
if err != nil {
t.Fatalf("invalid continuation: %v", err)
}
if decoded != "key\x00" {
t.Fatalf("unexpected continuation start: %q", decoded)
}
}
func decodeBase64String(value string) (string, error) {
decoded, err := base64Decode(value)
if err != nil {
return "", err
}
return string(decoded), nil
}
func TestWatchEventBufferBytes(t *testing.T) {
item := testKeyValue(3, 10, 20)
previous := testKeyValue(2, 5, 7)
// 512 + 4*3 (key) + 4*10 (value) + 4*5 (previous value) = 584
if got := watchEventBufferBytes(item, previous); got != 584 {
t.Fatalf("watchEventBufferBytes = %d, want 584", got)
}
previous.Version = 0
if got := watchEventBufferBytes(item, previous); got != 564 {
t.Fatalf("zero-version previous ignored: %d, want 564", got)
}
}
func TestWatchOverflowPreservesBufferedBatch(t *testing.T) {
session := newEtcdSession()
state := &watchState{watchID: "w", session: session}
smallBatch := []map[string]any{{"eventType": "put"}}
state.append(10, smallBatch, 128)
state.append(11, []map[string]any{{"eventType": "put"}}, maxWatchBufferBytes+1)
// The buffered batch stays pollable; the terminal rides along once the
// buffer has fully drained within the same poll (Java parity).
polled := state.poll()
if len(polled["batches"].([]any)) != 1 {
t.Fatalf("overflow must keep the buffered batch pollable: %#v", polled)
}
terminal, ok := polled["terminal"].(map[string]any)
if !ok || terminal["reason"] != "overflow" {
t.Fatalf("expected overflow terminal after drain, got %#v", polled)
}
if terminal["message"] != "ETCD_WATCH_OVERFLOW: the event buffer reached its byte or event limit" {
t.Fatalf("unexpected overflow message: %v", terminal["message"])
}
if session.watchBufferedBytesSnapshot() != 0 {
t.Fatalf("session budget not released: %d", session.watchBufferedBytesSnapshot())
}
}
func TestWatchPollReleasesSessionBudget(t *testing.T) {
session := newEtcdSession()
state := &watchState{watchID: "w", session: session}
state.append(1, []map[string]any{{"eventType": "put"}}, 1000)
if session.watchBufferedBytesSnapshot() != 1000 {
t.Fatalf("expected reserved budget 1000, got %d", session.watchBufferedBytesSnapshot())
}
state.poll()
if session.watchBufferedBytesSnapshot() != 0 {
t.Fatalf("expected budget released after poll, got %d", session.watchBufferedBytesSnapshot())
}
}
func TestSessionAggregateBudgetTerminatesOnlyOffendingWatch(t *testing.T) {
session := newEtcdSession()
first := &watchState{watchID: "a", session: session}
second := &watchState{watchID: "b", session: session}
if !session.reserveWatchBuffer(maxSessionWatchBufferBytes / 2) {
t.Fatalf("first reservation should succeed")
}
second.append(1, []map[string]any{{"eventType": "put"}}, maxSessionWatchBufferBytes/2+1)
polled := second.poll()
terminal, ok := polled["terminal"].(map[string]any)
if !ok || terminal["reason"] != "overflow" {
t.Fatalf("expected aggregate overflow on second watch: %#v", polled)
}
if first.hasTerminal() {
t.Fatalf("first watch must stay healthy")
}
session.releaseWatchBuffer(maxSessionWatchBufferBytes / 2)
}
func TestWatchPollRemovesTerminalWatch(t *testing.T) {
session := newEtcdSession()
state := &watchState{watchID: "w", session: session}
session.registerWatch("w", state)
state.fail("closed", "watch closed", nil)
params := map[string]json.RawMessage{"watchId": json.RawMessage(`"w"`)}
result, err := session.watchPoll(params)
if err != nil {
t.Fatalf("watchPoll failed: %v", err)
}
if _, ok := result.(map[string]any)["terminal"]; !ok {
t.Fatalf("expected terminal in poll result: %#v", result)
}
if session.watchCount() != 0 {
t.Fatalf("terminal watch must be removed from the session")
}
}
func TestWatchPollUnknownID(t *testing.T) {
session := newEtcdSession()
_, err := session.watchPoll(map[string]json.RawMessage{"watchId": json.RawMessage(`"missing"`)})
if err == nil || err.Error() != "ETCD_WATCH_NOT_FOUND: watch does not exist" {
t.Fatalf("expected watch-not-found, got %v", err)
}
}
func TestWatchStartLimitReached(t *testing.T) {
session := newEtcdSession()
for i := 0; i < maxWatches; i++ {
session.registerWatch(string(rune('a'+i)), &watchState{watchID: string(rune('a' + i)), session: session})
}
_, err := session.watchStart(map[string]json.RawMessage{"key": json.RawMessage(`"k"`)})
if err == nil || !strings.HasPrefix(err.Error(), "ETCD_WATCH_LIMIT") {
t.Fatalf("expected watch limit error, got %v", err)
}
}
func TestWatchScopeValidation(t *testing.T) {
session := newEtcdSession()
_, err := session.watchStart(map[string]json.RawMessage{"key": json.RawMessage(`"k"`), "scope": json.RawMessage(`"glob"`)})
if err == nil || err.Error() != "ETCD_WATCH_SCOPE_INVALID: scope must be key or prefix" {
t.Fatalf("expected scope error, got %v", err)
}
}
func TestLeasePageIDs(t *testing.T) {
ids := []uint64{30, 10, 20}
page := leasePageIDs(ids, nil, 2)
if page[0] != 10 || page[1] != 20 {
t.Fatalf("expected unsigned ascending order, got %v", page)
}
after := uint64(20)
page = leasePageIDs(ids, &after, 10)
if len(page) != 1 || page[0] != 30 {
t.Fatalf("expected only 30 after cursor 20, got %v", page)
}
// 2^63 boundary must compare unsigned, not signed.
big := []uint64{1, uint64(1) << 63}
page = leasePageIDs(big, nil, 2)
if page[0] != 1 || page[1] != uint64(1)<<63 {
t.Fatalf("unsigned ordering broken: %v", page)
}
}
func TestParseUnsignedLong(t *testing.T) {
parsed, err := parseUnsignedLong("18446744073709551615")
if err != nil || parsed != 18446744073709551615 {
t.Fatalf("uint64 max parse failed: %d %v", parsed, err)
}
if _, err := parseUnsignedLong("nope"); err == nil {
t.Fatalf("expected parse failure")
}
}
func TestAppendUnexecutedDefragMembers(t *testing.T) {
members := []map[string]any{{"endpoint": "a", "status": "failed"}}
appendUnexecutedDefragMembers(&members, []string{"a", "b", "c"}, "a")
if len(members) != 3 {
t.Fatalf("expected 3 rows, got %d", len(members))
}
for _, row := range members[1:] {
if row["status"] != "not_executed" || row["durationMs"] != nil || row["error"] != nil {
t.Fatalf("unexpected not_executed row: %#v", row)
}
}
}
func TestDefragRequiresEndpoints(t *testing.T) {
state := newEtcdSession()
_, err := state.defrag(map[string]json.RawMessage{})
if err == nil || err.Error() != "ETCD_DEFRAG_TARGET_REQUIRED: at least one endpoint is required" {
t.Fatalf("expected defrag target error, got %v", err)
}
}
func TestRequiredPositiveLongAndString(t *testing.T) {
_, err := requiredPositiveLong(map[string]json.RawMessage{"id": json.RawMessage(`0`)}, "id")
if err == nil || err.Error() != "ETCD_INVALID_ID: a positive integer is required" {
t.Fatalf("unexpected error: %v", err)
}
_, err = requiredString(map[string]json.RawMessage{}, "user")
if err == nil || err.Error() != "ETCD_USER_REQUIRED" {
t.Fatalf("unexpected error: %v", err)
}
}
func TestKeyBytesParam(t *testing.T) {
if _, err := keyBytesParam(map[string]json.RawMessage{}); err == nil || err.Error() != "Key is required" {
t.Fatalf("expected Key is required, got %v", err)
}
encoded := base64Encode([]byte{0x01, 0x02})
key, err := keyBytesParam(map[string]json.RawMessage{
"keyBytes": json.RawMessage(`{"encoding":"base64","data":"` + encoded + `"}`),
})
if err != nil || key != string([]byte{0x01, 0x02}) {
t.Fatalf("keyBytes decode failed: %q %v", key, err)
}
}
func TestPutRejectsConflictingLeaseOptions(t *testing.T) {
state := newEtcdSession()
params := map[string]json.RawMessage{
"key": json.RawMessage(`"k"`),
"value": json.RawMessage(`{"encoding":"utf8","data":"v"}`),
"ttl": json.RawMessage(`5`),
"lease": json.RawMessage(`7`),
}
// Java checks the active client before option exclusivity; without a
// connection the error must stay "Not connected".
_, err := state.put(params)
if err == nil || err.Error() != "Not connected" {
t.Fatalf("expected Not connected before option validation, got %v", err)
}
}
func (s *etcdSession) watchBufferedBytesSnapshot() int64 {
s.watchesMu.Lock()
defer s.watchesMu.Unlock()
return s.watchBufferedBytes
}
func (w *watchState) hasTerminal() bool {
w.mu.Lock()
defer w.mu.Unlock()
return w.terminalReason != ""
}
func testKeyValue(keySize, valueSize, version int64) *mvccpb.KeyValue {
return &mvccpb.KeyValue{
Key: bytesOfLength(keySize),
Value: bytesOfLength(valueSize),
Version: version,
}
}
func bytesOfLength(size int64) []byte {
return make([]byte, size)
}
func base64Decode(value string) ([]byte, error) {
return base64.StdEncoding.DecodeString(value)
}
func base64Encode(value []byte) string {
return base64.StdEncoding.EncodeToString(value)
}
+166
View File
@@ -0,0 +1,166 @@
package main
import (
"context"
"encoding/json"
"errors"
"time"
"go.etcd.io/etcd/api/v3/v3rpc/rpctypes"
clientv3 "go.etcd.io/etcd/client/v3"
)
func jsonUnmarshal(raw json.RawMessage, target any) error {
return json.Unmarshal(raw, target)
}
func (s *etcdSession) compact(params map[string]json.RawMessage) (any, error) {
revision, err := requiredPositiveLong(params, "revision")
if err != nil {
return nil, err
}
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
_, err = client.Get(ctx, "\x00", clientv3.WithRange("\x00"), clientv3.WithCountOnly(), clientv3.WithRev(revision))
cancel()
if err != nil {
if errors.Is(err, rpctypes.ErrCompacted) {
if compacted, ok := compactedRevisionOf(client, "\x00"); ok {
return nil, errors.New("ETCD_INVALID_REVISION: revision was already compacted at " + longString(compacted))
}
return nil, errors.New("ETCD_INVALID_REVISION: revision was already compacted")
}
return nil, err
}
currentCtx, currentCancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
current, err := client.Get(currentCtx, "\x00", clientv3.WithRange("\x00"), clientv3.WithCountOnly())
currentCancel()
if err != nil {
return nil, err
}
if revision > current.Header.Revision {
return nil, errors.New("ETCD_INVALID_REVISION: revision is newer than the current revision")
}
compactCtx, compactCancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
defer compactCancel()
if _, err := client.Compact(compactCtx, revision); err != nil {
return nil, err
}
return map[string]any{"revision": longString(revision)}, nil
}
func (s *etcdSession) defrag(params map[string]json.RawMessage) (any, error) {
var targets []string
if raw := params["endpoints"]; len(raw) > 0 && string(raw) != "null" {
var list []string
if jsonUnmarshal(raw, &list) == nil {
for _, endpoint := range list {
if endpoint != "" && !containsString(targets, endpoint) {
targets = append(targets, endpoint)
}
}
}
}
if len(targets) == 0 {
return nil, errors.New("ETCD_DEFRAG_TARGET_REQUIRED: at least one endpoint is required")
}
client, err := s.activeClient()
if err != nil {
return nil, err
}
members := []map[string]any{}
remaining := targets
for len(remaining) > 0 {
endpoint, err := nextDefragEndpoint(client, remaining)
if err != nil {
return nil, err
}
started := time.Now()
ctx, cancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
_, defragErr := client.Maintenance.Defragment(ctx, endpoint)
cancel()
durationMs := time.Since(started).Milliseconds()
if defragErr != nil {
members = append(members, map[string]any{
"endpoint": endpoint,
"status": "failed",
"durationMs": durationMs,
"error": defragErr.Error(),
})
appendUnexecutedDefragMembers(&members, remaining, endpoint)
break
}
members = append(members, map[string]any{
"endpoint": endpoint,
"status": "succeeded",
"durationMs": durationMs,
})
remaining = removeString(remaining, endpoint)
}
return map[string]any{"members": members}, nil
}
func appendUnexecutedDefragMembers(members *[]map[string]any, remaining []string, failedEndpoint string) {
for _, endpoint := range remaining {
if endpoint == failedEndpoint {
continue
}
*members = append(*members, map[string]any{
"endpoint": endpoint,
"status": "not_executed",
"durationMs": nil,
"error": nil,
})
}
}
// nextDefragEndpoint re-evaluates leadership before each member so a leader
// change is not defragmented early.
func nextDefragEndpoint(client *clientv3.Client, remaining []string) (string, error) {
var leaderEndpoint string
for _, endpoint := range remaining {
ctx, cancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
statusResponse, err := client.Maintenance.Status(ctx, endpoint)
cancel()
if err != nil {
// The following defragment call reports this member as the failed step.
continue
}
if statusResponse.Header.MemberId == statusResponse.Leader {
leaderEndpoint = endpoint
break
}
}
for _, endpoint := range remaining {
if endpoint != leaderEndpoint {
return endpoint, nil
}
}
if leaderEndpoint == "" {
return remaining[0], nil
}
return leaderEndpoint, nil
}
func containsString(values []string, target string) bool {
for _, value := range values {
if value == target {
return true
}
}
return false
}
func removeString(values []string, target string) []string {
result := make([]string, 0, len(values))
for _, value := range values {
if value != target {
result = append(result, value)
}
}
return result
}
+143
View File
@@ -0,0 +1,143 @@
package main
import (
"context"
"encoding/json"
"errors"
"io"
"net"
"strings"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
type rpcError struct {
Code int `json:"code"`
Message string `json:"message"`
Data *rpcErrorData `json:"data,omitempty"`
}
type rpcErrorData struct {
Category string `json:"category"`
Retryable bool `json:"retryable"`
SessionDisposition string `json:"sessionDisposition"`
Stage string `json:"stage"`
ContractVersion int `json:"contractVersion"`
OperationOutcome string `json:"operationOutcome"`
ExceptionClass string `json:"exceptionClass,omitempty"`
AgentSessionID string `json:"agentSessionId,omitempty"`
}
func errorResponse(id json.RawMessage, method, agentSessionID string, err error) response {
return response{JSONRPC: "2.0", ID: id, Error: classifyRPCError(method, agentSessionID, err)}
}
func classifyRPCError(method, agentSessionID string, err error) *rpcError {
stage := rpcErrorStage(method)
data := &rpcErrorData{
Category: "protocol",
Retryable: false,
SessionDisposition: "keep",
Stage: stage,
ContractVersion: 1,
OperationOutcome: rpcOperationOutcome(stage),
ExceptionClass: safeRPCDiagnostic(errTypeName(err), 160),
AgentSessionID: strings.TrimSpace(agentSessionID),
}
switch {
case errors.Is(err, context.Canceled):
data.Category = "canceled"
data.SessionDisposition = "quarantine"
case errors.Is(err, context.DeadlineExceeded) || isTimeoutError(err):
data.Category = "timeout"
data.SessionDisposition = "quarantine"
case isUnavailableError(err):
data.Category = "connection"
data.Retryable = stage == "connect" || stage == "validate"
if stage != "connect" {
data.SessionDisposition = "quarantine"
}
}
return &rpcError{Code: -1, Message: err.Error(), Data: data}
}
// isUnavailableError treats gRPC UNAVAILABLE (etcd server unreachable) as a
// connection failure, matching the Java agent's transport classification.
func isUnavailableError(err error) bool {
if status.Code(err) == codes.Unavailable {
return true
}
if errors.Is(err, io.EOF) || errors.Is(err, net.ErrClosed) {
return true
}
var networkError *net.OpError
if errors.As(err, &networkError) {
return true
}
lower := strings.ToLower(err.Error())
for _, marker := range []string{
"connection refused", "connection reset", "broken pipe", "connection closed",
"connection lost", "unexpected eof", "no route to host",
} {
if strings.Contains(lower, marker) {
return true
}
}
return false
}
func errTypeName(err error) string {
if err == nil {
return ""
}
return strings.ReplaceAll(strings.TrimSpace(err.Error()), "\n", " ")
}
func rpcErrorStage(method string) string {
switch method {
case "connect", "open_session", "test_connection":
return "connect"
case "validate_connection", "validate_session":
return "validate"
case "cancel_session":
return "cancel"
case "close_session", "disconnect", "shutdown":
return "close"
case "etcd_watch_poll", "etcd_lease_get":
return "fetch"
case "handshake", "":
return "request"
default:
return "execute"
}
}
func rpcOperationOutcome(stage string) string {
switch stage {
case "request", "connect", "validate":
return "not_started"
default:
return "unknown"
}
}
func isTimeoutError(err error) bool {
var timeout interface{ Timeout() bool }
return errors.As(err, &timeout) && timeout.Timeout()
}
func safeRPCDiagnostic(value string, maxLength int) string {
var result strings.Builder
for _, char := range value {
if result.Len() >= maxLength {
break
}
if char >= 0x21 && char <= 0x7e {
result.WriteRune(char)
}
}
return result.String()
}
+148
View File
@@ -0,0 +1,148 @@
package main
import (
"context"
"encoding/json"
"time"
"go.etcd.io/etcd/api/v3/etcdserverpb"
clientv3 "go.etcd.io/etcd/client/v3"
)
func (s *etcdSession) status(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
maintenance := client.Maintenance
membersByID := map[uint64]*etcdserverpb.Member{}
endpoints := s.connectedEndpointList()
memberCtx, memberCancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
memberList, err := client.Cluster.MemberList(memberCtx)
memberCancel()
if err == nil {
for _, member := range memberList.Members {
membersByID[member.ID] = member
for _, clientURL := range member.ClientURLs {
host := endpointHost(clientURL)
if host == "" || host == "0.0.0.0" || host == "::" {
continue
}
known := false
for _, existing := range endpoints {
if existing == clientURL {
known = true
break
}
}
if !known {
endpoints = append(endpoints, clientURL)
}
}
}
}
alarms := []string{}
alarmCtx, alarmCancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
alarmResponse, err := maintenance.AlarmList(alarmCtx)
alarmCancel()
if err != nil {
alarms = append(alarms, "UNAVAILABLE: "+err.Error())
} else {
for _, alarm := range alarmResponse.Alarms {
alarms = append(alarms, alarm.Alarm.String()+"@"+unsignedLongString(int64(alarm.MemberID)))
}
}
countCtx, countCancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
countResponse, err := client.Get(countCtx, "\x00", clientv3.WithRange("\x00"), clientv3.WithCountOnly())
countCancel()
if err != nil {
return nil, err
}
type statusResult struct {
response *clientv3.StatusResponse
err error
started time.Time
}
statusRequests := make([]chan *statusResult, len(endpoints))
for index, endpoint := range endpoints {
channel := make(chan *statusResult, 1)
statusRequests[index] = channel
go func(endpoint string, channel chan *statusResult) {
started := time.Now()
ctx, cancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
response, err := client.Maintenance.Status(ctx, endpoint)
cancel()
channel <- &statusResult{response: response, err: err, started: started}
}(endpoint, channel)
}
statusMembers := []map[string]any{}
observedMemberIDs := map[string]struct{}{}
var clusterID *string
var leaderID *string
for index, channel := range statusRequests {
result := <-channel
row := map[string]any{"endpoint": endpoints[index]}
if result.err != nil {
row["memberId"] = nil
row["name"] = nil
row["version"] = nil
row["leaderId"] = nil
row["revision"] = nil
row["raftTerm"] = nil
row["raftIndex"] = nil
row["raftAppliedIndex"] = nil
row["dbSize"] = nil
row["dbSizeInUse"] = nil
row["learner"] = false
row["reachable"] = false
row["latencyMs"] = nil
row["errors"] = []string{result.err.Error()}
statusMembers = append(statusMembers, row)
continue
}
memberStatus := result.response
memberID := memberStatus.Header.MemberId
member := membersByID[memberID]
clusterIDValue := unsignedLongString(int64(memberStatus.Header.ClusterId))
leaderIDValue := unsignedLongString(int64(memberStatus.Leader))
clusterID = &clusterIDValue
leaderID = &leaderIDValue
row["memberId"] = unsignedLongString(int64(memberID))
if member == nil {
row["name"] = nil
} else {
row["name"] = member.Name
}
row["version"] = memberStatus.Version
row["leaderId"] = unsignedLongString(int64(memberStatus.Leader))
row["revision"] = longString(memberStatus.Header.Revision)
row["raftTerm"] = longString(int64(memberStatus.RaftTerm))
row["raftIndex"] = longString(int64(memberStatus.RaftIndex))
row["raftAppliedIndex"] = longString(int64(memberStatus.RaftAppliedIndex))
row["dbSize"] = longString(memberStatus.DbSize)
row["dbSizeInUse"] = longString(memberStatus.DbSizeInUse)
row["learner"] = memberStatus.IsLearner
row["reachable"] = true
row["latencyMs"] = time.Since(result.started).Milliseconds()
memberIDKey := unsignedLongString(int64(memberID))
if _, seen := observedMemberIDs[memberIDKey]; seen {
continue
}
observedMemberIDs[memberIDKey] = struct{}{}
statusMembers = append(statusMembers, row)
}
result := map[string]any{
"clusterId": clusterID,
"revision": longString(countResponse.Header.Revision),
"leaderId": leaderID,
"keyCount": longString(countResponse.Count),
"alarms": alarms,
"members": statusMembers,
}
return result, nil
}
+366
View File
@@ -0,0 +1,366 @@
package main
import (
"context"
"crypto/rand"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"sync"
"time"
"go.etcd.io/etcd/api/v3/mvccpb"
clientv3 "go.etcd.io/etcd/client/v3"
)
const (
maxWatches = 4
maxWatchBatches = 256
maxWatchEvents = 10000
maxWatchBufferBytes = int64(8 * 1024 * 1024)
maxSessionWatchBufferBytes = int64(16 * 1024 * 1024)
)
type bufferedWatchBatch struct {
payload map[string]any
bufferedBytes int64
}
// watchState buffers watch events for the polling host. Terminal states are
// only surfaced after the buffer drains, mirroring the Java implementation.
type watchState struct {
watchID string
session *etcdSession
mu sync.Mutex
batches []bufferedWatchBatch
eventCount int
bufferedBytes int64
terminalReason string
terminalMessage string
compactedRevision *int64
watchCancel context.CancelFunc
closed bool
}
func (w *watchState) append(revision int64, events []map[string]any, batchBytes int64) {
w.mu.Lock()
defer w.mu.Unlock()
if w.terminalReason != "" {
return
}
if int64(len(w.batches)) >= maxWatchBatches ||
int64(w.eventCount+len(events)) > maxWatchEvents ||
batchBytes > maxWatchBufferBytes ||
w.bufferedBytes+batchBytes > maxWatchBufferBytes ||
!w.session.reserveWatchBuffer(batchBytes) {
w.overflowLocked()
return
}
w.batches = append(w.batches, bufferedWatchBatch{
payload: map[string]any{"revision": longString(revision), "events": events},
bufferedBytes: batchBytes,
})
w.eventCount += len(events)
w.bufferedBytes += batchBytes
}
func (w *watchState) overflowLocked() {
if w.terminalReason != "" {
return
}
w.terminalReason = "overflow"
w.terminalMessage = "ETCD_WATCH_OVERFLOW: the event buffer reached its byte or event limit"
w.closeWatcherLocked()
}
func (w *watchState) fail(reason, message string, compacted *int64) {
w.mu.Lock()
defer w.mu.Unlock()
if w.terminalReason == "" {
w.terminalReason = reason
w.terminalMessage = message
w.compactedRevision = compacted
}
}
func (w *watchState) poll() map[string]any {
w.mu.Lock()
defer w.mu.Unlock()
page := make([]any, 0, 64)
for len(w.batches) > 0 && len(page) < 64 {
batch := w.batches[0]
w.batches = w.batches[1:]
events := batch.payload["events"].([]map[string]any)
w.eventCount -= len(events)
w.bufferedBytes -= batch.bufferedBytes
w.session.releaseWatchBuffer(batch.bufferedBytes)
page = append(page, batch.payload)
}
result := map[string]any{"watchId": w.watchID, "batches": page}
if w.terminalReason != "" && len(w.batches) == 0 {
terminal := map[string]any{
"reason": w.terminalReason,
"message": w.terminalMessage,
}
if w.compactedRevision != nil {
terminal["compactedRevision"] = longString(*w.compactedRevision)
} else {
terminal["compactedRevision"] = nil
}
result["terminal"] = terminal
}
return result
}
func (w *watchState) close() {
w.mu.Lock()
w.failLocked("stopped", "watch stopped", nil)
w.clearBufferedLocked()
w.closed = true
cancel := w.watchCancel
w.watchCancel = nil
w.mu.Unlock()
if cancel != nil {
cancel()
}
}
func (w *watchState) failLocked(reason, message string, compacted *int64) {
if w.terminalReason == "" {
w.terminalReason = reason
w.terminalMessage = message
w.compactedRevision = compacted
}
}
func (w *watchState) clearBufferedLocked() {
if w.bufferedBytes > 0 {
w.session.releaseWatchBuffer(w.bufferedBytes)
}
w.batches = nil
w.eventCount = 0
w.bufferedBytes = 0
}
func (w *watchState) attachWatcher(cancel context.CancelFunc) {
w.mu.Lock()
defer w.mu.Unlock()
if w.closed || w.terminalReason != "" {
cancel()
return
}
w.watchCancel = cancel
}
func (w *watchState) closeWatcherLocked() {
cancel := w.watchCancel
w.watchCancel = nil
if cancel != nil {
go cancel()
}
}
func watchEventBufferBytes(item, previous *mvccpb.KeyValue) int64 {
bytes := int64(512 + estimatedBufferedBytes(len(item.Key)))
bytes += estimatedBufferedBytes(len(item.Value))
if previous != nil && previous.Version > 0 {
bytes += estimatedBufferedBytes(len(previous.Value))
}
return bytes
}
func estimatedBufferedBytes(sourceBytes int) int64 {
if sourceBytes < 0 {
return 0
}
return int64(sourceBytes) * 4
}
func newWatchID() string {
var bytes [16]byte
if _, err := rand.Read(bytes[:]); err != nil {
return base64.StdEncoding.EncodeToString(bytes[:])
}
bytes[6] = (bytes[6] & 0x0f) | 0x40
bytes[8] = (bytes[8] & 0x3f) | 0x80
hexed := hex.EncodeToString(bytes[:])
return fmt.Sprintf("%s-%s-%s-%s-%s", hexed[0:8], hexed[8:12], hexed[12:16], hexed[16:20], hexed[20:32])
}
func (s *etcdSession) watchCount() int {
s.watchesMu.Lock()
defer s.watchesMu.Unlock()
return len(s.watches)
}
func (s *etcdSession) registerWatch(id string, state *watchState) bool {
s.watchesMu.Lock()
defer s.watchesMu.Unlock()
if _, exists := s.watches[id]; exists {
return false
}
s.watches[id] = state
return true
}
func (s *etcdSession) removeWatch(id string) *watchState {
s.watchesMu.Lock()
defer s.watchesMu.Unlock()
state := s.watches[id]
delete(s.watches, id)
return state
}
func (s *etcdSession) watchStart(params map[string]json.RawMessage) (any, error) {
if s.watchCount() >= maxWatches {
return nil, fmt.Errorf("ETCD_WATCH_LIMIT: at most %d watches are allowed per connection", maxWatches)
}
key, err := keyBytesParam(params)
if err != nil {
return nil, err
}
scope := stringOrDefault(params, "scope", "key")
if scope != "key" && scope != "prefix" {
return nil, errors.New("ETCD_WATCH_SCOPE_INVALID: scope must be key or prefix")
}
client, err := s.activeClient()
if err != nil {
return nil, err
}
requestedRevision := longOrNull(params, "startRevision")
var startedRevision int64
if requestedRevision != nil && *requestedRevision > 0 {
startedRevision = *requestedRevision
} else {
ctx, cancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
response, err := client.Get(ctx, "\x00", clientv3.WithRange("\x00"), clientv3.WithCountOnly())
cancel()
if err != nil {
return nil, err
}
startedRevision = response.Header.Revision + 1
}
watchID := newWatchID()
state := &watchState{watchID: watchID, session: s}
options := []clientv3.OpOption{clientv3.WithRev(startedRevision)}
if boolOrDefault(params, "includePrevKv", false) {
options = append(options, clientv3.WithPrevKV())
}
if scope == "prefix" {
options = append(options, clientv3.WithRange(prefixEnd(key)))
}
watchCtx, watchCancel := context.WithCancel(context.Background())
channel := client.Watch(watchCtx, key, options...)
go consumeWatchChannel(channel, state)
state.attachWatcher(watchCancel)
s.registerWatch(watchID, state)
return map[string]any{"watchId": watchID, "startedRevision": longString(startedRevision)}, nil
}
func consumeWatchChannel(channel clientv3.WatchChan, state *watchState) {
for response := range channel {
if response.Err() != nil {
state.fail("error", response.Err().Error(), nil)
continue
}
if response.Canceled {
if response.CompactRevision > 0 {
compacted := response.CompactRevision
state.fail("compacted", "ETCD_COMPACTED", &compacted)
} else {
state.fail("closed", "watch closed", nil)
}
continue
}
if len(response.Events) == 0 {
continue
}
events := make([]map[string]any, 0, len(response.Events))
bufferedBytes := int64(128)
overflowed := false
for _, event := range response.Events {
item := event.Kv
var previous *mvccpb.KeyValue
if event.PrevKv != nil {
previous = event.PrevKv
}
bufferedBytes += watchEventBufferBytes(item, previous)
if bufferedBytes > maxWatchBufferBytes {
state.failOverflow()
overflowed = true
break
}
events = append(events, watchEventRow(event, item, previous))
}
if overflowed {
continue
}
state.append(response.Header.Revision, events, bufferedBytes)
}
}
func (w *watchState) failOverflow() {
w.mu.Lock()
defer w.mu.Unlock()
w.overflowLocked()
}
func watchEventRow(event *clientv3.Event, item, previous *mvccpb.KeyValue) map[string]any {
row := map[string]any{
"eventType": eventType(event),
"revision": longString(item.ModRevision),
"key": displayBytes(item.Key),
"keyBytes": bytesObject(item.Key),
}
if event.Type == mvccpb.DELETE {
row["value"] = nil
} else {
row["value"] = valueObject(item.Value)
}
if previous != nil && previous.Version > 0 {
row["previousValue"] = valueObject(previous.Value)
} else {
row["previousValue"] = nil
}
if event.Type == mvccpb.DELETE && previous != nil {
row["metadata"] = metadataMap(previous)
} else {
row["metadata"] = metadataMap(item)
}
return row
}
func eventType(event *clientv3.Event) string {
if event.Type == mvccpb.DELETE {
return "delete"
}
return "put"
}
func (s *etcdSession) watchPoll(params map[string]json.RawMessage) (any, error) {
watchID := stringOrDefault(params, "watchId", "")
s.watchesMu.Lock()
state := s.watches[watchID]
s.watchesMu.Unlock()
if state == nil {
return nil, errors.New("ETCD_WATCH_NOT_FOUND: watch does not exist")
}
result := state.poll()
if _, terminal := result["terminal"]; terminal {
if removed := s.removeWatch(watchID); removed != nil {
removed.close()
}
}
return result, nil
}
func (s *etcdSession) watchStop(params map[string]json.RawMessage) (any, error) {
if state := s.removeWatch(stringOrDefault(params, "watchId", "")); state != nil {
state.close()
}
return map[string]bool{"stopped": true}, nil
}
-11
View File
@@ -1,11 +0,0 @@
dependencies {
implementation 'com.google.code.gson:gson:2.12.1'
implementation 'io.etcd:jetcd-core:0.8.6'
}
tasks.named('shadowJar') {
mergeServiceFiles()
manifest {
attributes('Agent-Label': 'etcd', 'Main-Class': 'com.dbx.agent.etcd.EtcdAgent')
}
}
File diff suppressed because it is too large Load Diff
@@ -1,475 +0,0 @@
package com.dbx.agent.etcd;
import com.google.protobuf.ByteString;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import io.etcd.jetcd.ByteSequence;
import io.etcd.jetcd.KV;
import io.etcd.jetcd.Lease;
import io.etcd.jetcd.Txn;
import io.etcd.jetcd.api.KeyValue;
import io.etcd.jetcd.api.RangeResponse;
import io.etcd.jetcd.api.ResponseHeader;
import io.etcd.jetcd.kv.GetResponse;
import io.etcd.jetcd.kv.TxnResponse;
import java.lang.reflect.Proxy;
import java.nio.charset.StandardCharsets;
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Deque;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
final class EtcdAgentTest {
@Test
void handshakeAdvertisesKvCapability() {
String response = EtcdAgent.handleRequest(
"{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"handshake\",\"params\":{}}"
);
JsonObject result = JsonParser.parseString(response).getAsJsonObject().getAsJsonObject("result");
Assertions.assertEquals(2, result.get("protocolVersion").getAsInt());
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"kv\"")));
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"kv_ttl\"")));
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"kv_cas\"")));
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"kv_list_values\"")));
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"kv_status\"")));
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"kv_history\"")));
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"etcd_compaction\"")));
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"etcd_defrag\"")));
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"etcd_watch\"")));
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"etcd_lease\"")));
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"etcd_auth\"")));
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"multi_session\"")));
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"connect\"")));
}
@Test
void endpointsUseConfiguredListAndScheme() {
JsonObject connection = JsonParser.parseString(
"{\"endpoints\":\"etcd-1:2379,https://etcd-2:2379\",\"ssl\":true}"
).getAsJsonObject();
Assertions.assertEquals(
List.of("https://etcd-1:2379", "https://etcd-2:2379"),
EtcdAgent.endpoints(connection)
);
}
@Test
void endpointsFallbackToHostPort() {
JsonObject connection = JsonParser.parseString(
"{\"host\":\"127.0.0.1\",\"port\":2379}"
).getAsJsonObject();
Assertions.assertEquals(List.of("http://127.0.0.1:2379"), EtcdAgent.endpoints(connection));
}
@Test
void connectTimeoutUsesConfiguredValueAndSafeBounds() {
Assertions.assertEquals(
45,
EtcdAgent.connectTimeoutSeconds(JsonParser.parseString("{\"connect_timeout_secs\":45}").getAsJsonObject())
);
Assertions.assertEquals(
1,
EtcdAgent.connectTimeoutSeconds(JsonParser.parseString("{\"connect_timeout_secs\":0}").getAsJsonObject())
);
Assertions.assertEquals(
300,
EtcdAgent.connectTimeoutSeconds(JsonParser.parseString("{\"connect_timeout_secs\":999}").getAsJsonObject())
);
}
@Test
void grpcInboundLimitDefaultsTo32MiBAndUsesSafeBounds() {
Assertions.assertEquals(
32 * 1024 * 1024,
EtcdAgent.grpcMaxInboundMessageSize(new JsonObject())
);
Assertions.assertEquals(
64 * 1024 * 1024,
EtcdAgent.grpcMaxInboundMessageSize(
JsonParser.parseString("{\"grpc_max_inbound_message_size\":67108864}").getAsJsonObject()
)
);
Assertions.assertEquals(
1024 * 1024,
EtcdAgent.grpcMaxInboundMessageSize(
JsonParser.parseString("{\"grpc_max_inbound_message_size\":0}").getAsJsonObject()
)
);
Assertions.assertEquals(
256 * 1024 * 1024,
EtcdAgent.grpcMaxInboundMessageSize(
JsonParser.parseString("{\"grpc_max_inbound_message_size\":536870912}").getAsJsonObject()
)
);
}
@Test
void grpcInboundLimitCanBeConfiguredThroughConnectionUrlParams() {
JsonObject connection = JsonParser.parseString(
"{\"url_params\":\"foo=bar&grpc_max_inbound_message_size=50331648\"}"
).getAsJsonObject();
Assertions.assertEquals(48 * 1024 * 1024, EtcdAgent.grpcMaxInboundMessageSize(connection));
}
@Test
void validateConnectionRequiresAnActiveSession() {
String response = EtcdAgent.handleRequest(
"{\"jsonrpc\":\"2.0\",\"id\":2,\"method\":\"validate_connection\",\"params\":{}}"
);
JsonObject error = JsonParser.parseString(response).getAsJsonObject().getAsJsonObject("error");
Assertions.assertEquals(-1, error.get("code").getAsInt());
Assertions.assertEquals("Not connected", error.get("message").getAsString());
}
@Test
void kvMethodDispatchReturnsJsonRpcErrorWhenDisconnected() {
String response = EtcdAgent.handleRequest(
"{\"jsonrpc\":\"2.0\",\"id\":2,\"method\":\"kv_get\",\"params\":{\"key\":\"/app/name\"}}"
);
JsonObject error = JsonParser.parseString(response).getAsJsonObject().getAsJsonObject("error");
Assertions.assertEquals(-1, error.get("code").getAsInt());
Assertions.assertEquals("Not connected", error.get("message").getAsString());
}
@Test
void historyMethodIsRegistered() {
String response = EtcdAgent.handleRequest(
"{\"jsonrpc\":\"2.0\",\"id\":3,\"method\":\"kv_history\",\"params\":{\"key\":\"/app/name\",\"limit\":100}}"
);
JsonObject error = JsonParser.parseString(response).getAsJsonObject().getAsJsonObject("error");
Assertions.assertEquals(-1, error.get("code").getAsInt());
Assertions.assertEquals("Not connected", error.get("message").getAsString());
}
@Test
void historyDefaultsToABoundedRevisionWindow() {
Assertions.assertEquals(1L, EtcdAgent.historyStartRevision(null, 100L));
Assertions.assertEquals(10_001L, EtcdAgent.historyStartRevision(null, 20_000L));
Assertions.assertEquals(42L, EtcdAgent.historyStartRevision(42L, 20_000L));
}
@Test
void historyRetainsOnlyTheNewestRequestedEvents() {
Deque<Integer> events = new ArrayDeque<>();
AtomicBoolean truncated = new AtomicBoolean();
for (int value = 1; value <= 5; value++) {
EtcdAgent.appendBoundedHistory(events, value, 3, truncated);
}
Assertions.assertEquals(List.of(3, 4, 5), List.copyOf(events));
Assertions.assertTrue(truncated.get());
}
@Test
void preserveLeaseRetriesWithTheLatestLeaseAfterConcurrentChange() throws Exception {
AtomicInteger getCalls = new AtomicInteger();
AtomicInteger txnCalls = new AtomicInteger();
KV active = scriptedKv(
getCalls,
txnCalls,
List.of(getResponse("/app/name", 10, 101), getResponse("/app/name", 11, 202)),
List.of(txnResponse(false, 10), txnResponse(true, 12))
);
long revision = EtcdAgent.putPreservingLease(active, bytes("/app/name"), bytes("updated"));
Assertions.assertEquals(12, revision);
Assertions.assertEquals(2, getCalls.get());
Assertions.assertEquals(2, txnCalls.get());
}
@Test
void preserveLeaseRejectsMissingOrUnleasedKeys() {
KV missing = scriptedKv(
new AtomicInteger(),
new AtomicInteger(),
List.of(emptyGetResponse()),
List.of()
);
KV unleased = scriptedKv(
new AtomicInteger(),
new AtomicInteger(),
List.of(getResponse("/app/name", 10, 0)),
List.of()
);
IllegalStateException missingError = Assertions.assertThrows(
IllegalStateException.class,
() -> EtcdAgent.putPreservingLease(missing, bytes("/app/name"), bytes("updated"))
);
IllegalStateException unleasedError = Assertions.assertThrows(
IllegalStateException.class,
() -> EtcdAgent.putPreservingLease(unleased, bytes("/app/name"), bytes("updated"))
);
Assertions.assertEquals("Cannot preserve lease: key does not exist or has no lease", missingError.getMessage());
Assertions.assertEquals("Cannot preserve lease: key does not exist or has no lease", unleasedError.getMessage());
}
@Test
void preserveLeaseFailsClearlyAfterBoundedConcurrentChanges() {
AtomicInteger getCalls = new AtomicInteger();
AtomicInteger txnCalls = new AtomicInteger();
KV active = scriptedKv(
getCalls,
txnCalls,
List.of(
getResponse("/app/name", 10, 101),
getResponse("/app/name", 11, 202),
getResponse("/app/name", 12, 303)
),
List.of(txnResponse(false, 10), txnResponse(false, 11), txnResponse(false, 12))
);
IllegalStateException error = Assertions.assertThrows(
IllegalStateException.class,
() -> EtcdAgent.putPreservingLease(active, bytes("/app/name"), bytes("updated"))
);
Assertions.assertEquals("Cannot preserve lease: key changed concurrently; retry the save", error.getMessage());
Assertions.assertEquals(3, getCalls.get());
Assertions.assertEquals(3, txnCalls.get());
}
@Test
void grantedLeaseIsRevokedWhenWriteFailsAfterCreation() {
AtomicInteger revokeCalls = new AtomicInteger();
AtomicInteger revokedLeaseId = new AtomicInteger();
Lease lease = (Lease) Proxy.newProxyInstance(
Lease.class.getClassLoader(),
new Class<?>[] { Lease.class },
(proxy, method, args) -> {
if (method.getName().equals("revoke")) {
revokeCalls.incrementAndGet();
revokedLeaseId.set(Math.toIntExact((long) args[0]));
return CompletableFuture.completedFuture(null);
}
if (method.getName().equals("close")) return null;
throw new UnsupportedOperationException(method.getName());
}
);
TimeoutException error = Assertions.assertThrows(
TimeoutException.class,
() -> EtcdAgent.cleanUpGrantedLeaseOnFailure(123, lease, () -> {
throw new TimeoutException("write timed out");
})
);
Assertions.assertEquals("write timed out", error.getMessage());
Assertions.assertEquals(1, revokeCalls.get());
Assertions.assertEquals(123, revokedLeaseId.get());
}
@Test
void defragFailureMarksEveryRemainingEndpointAsNotExecuted() {
List<Map<String, Object>> members = new ArrayList<>();
EtcdAgent.appendUnexecutedDefragMembers(
members,
List.of("http://follower-2:2379", "http://follower-3:2379", "http://leader:2379"),
"http://follower-2:2379"
);
Assertions.assertEquals(
List.of("http://follower-3:2379", "http://leader:2379"),
members.stream().map(member -> member.get("endpoint")).toList()
);
Assertions.assertTrue(members.stream().allMatch(member -> "not_executed".equals(member.get("status"))));
}
@Test
void leaseListResponseParserReadsEveryLeaseId() throws Exception {
byte[] response = new byte[] {
0x12, 0x02, 0x08, 0x7b,
0x12, 0x03, 0x08, (byte) 0xc8, 0x03
};
Assertions.assertEquals(List.of(123L, 456L), EtcdAgent.leaseIdsFromResponse(response));
}
@Test
void leaseListPageUsesAStableCursorAndBoundsTheResult() {
List<Long> leaseIds = new ArrayList<>();
for (long id = 1000; id >= 1; id--) leaseIds.add(id);
List<Long> first = EtcdAgent.leasePageIds(leaseIds, null, 101);
List<Long> second = EtcdAgent.leasePageIds(leaseIds, first.get(99), 101);
Assertions.assertEquals(101, first.size());
Assertions.assertEquals(1L, first.get(0));
Assertions.assertEquals(101L, first.get(100));
Assertions.assertEquals(101L, second.get(0));
Assertions.assertEquals(201L, second.get(100));
}
@Test
void watchBufferRejectsLargeValueAndPreviousValuePayloads() {
long payloadBytes = EtcdAgent.estimatedBufferedBytes(2 * 1024 * 1024)
+ EtcdAgent.estimatedBufferedBytes(2 * 1024 * 1024);
Assertions.assertTrue(payloadBytes > EtcdAgent.MAX_WATCH_BUFFER_BYTES);
}
@Test
void watchOverflowPreservesBufferedPayloadAndReportsTerminalState() {
EtcdAgent.EtcdSessionState session = new EtcdAgent.EtcdSessionState();
EtcdAgent.EtcdWatchState watch = new EtcdAgent.EtcdWatchState("watch-1", session);
List<Map<String, Object>> events = List.of(Map.of("eventType", "put"));
watch.append(1, events, 4L * 1024 * 1024);
Assertions.assertEquals(4L * 1024 * 1024, session.watchBufferedBytes());
watch.append(2, events, 5L * 1024 * 1024);
Map<String, Object> result = watch.poll();
Assertions.assertEquals(0, session.watchBufferedBytes());
List<?> batches = (List<?>) result.get("batches");
Assertions.assertEquals(1, batches.size());
Assertions.assertEquals("1", ((Map<?, ?>) batches.get(0)).get("revision"));
Assertions.assertEquals("overflow", ((Map<?, ?>) result.get("terminal")).get("reason"));
}
@Test
void watchPollReleasesTheSessionByteBudget() {
EtcdAgent.EtcdSessionState session = new EtcdAgent.EtcdSessionState();
EtcdAgent.EtcdWatchState watch = new EtcdAgent.EtcdWatchState("watch-1", session);
watch.append(1, List.of(Map.of("eventType", "put")), 1024);
Assertions.assertEquals(1024, session.watchBufferedBytes());
watch.poll();
Assertions.assertEquals(0, session.watchBufferedBytes());
}
@Test
void aggregateSessionBudgetTerminatesOnlyTheWatchThatExceedsIt() {
EtcdAgent.EtcdSessionState session = new EtcdAgent.EtcdSessionState();
List<Map<String, Object>> events = List.of(Map.of("eventType", "put"));
EtcdAgent.EtcdWatchState first = new EtcdAgent.EtcdWatchState("watch-1", session);
EtcdAgent.EtcdWatchState second = new EtcdAgent.EtcdWatchState("watch-2", session);
EtcdAgent.EtcdWatchState third = new EtcdAgent.EtcdWatchState("watch-3", session);
first.append(1, events, EtcdAgent.MAX_WATCH_BUFFER_BYTES);
second.append(1, events, EtcdAgent.MAX_WATCH_BUFFER_BYTES);
third.append(1, events, 1);
Assertions.assertEquals(EtcdAgent.MAX_SESSION_WATCH_BUFFER_BYTES, session.watchBufferedBytes());
Assertions.assertEquals("overflow", ((Map<?, ?>) third.poll().get("terminal")).get("reason"));
Assertions.assertEquals(EtcdAgent.MAX_SESSION_WATCH_BUFFER_BYTES, session.watchBufferedBytes());
}
@Test
void terminalPollRemovesTheWatchSlot() {
EtcdAgent.EtcdSessionState session = new EtcdAgent.EtcdSessionState();
EtcdAgent.EtcdWatchState watch = new EtcdAgent.EtcdWatchState("watch-1", session);
session.addWatch("watch-1", watch);
watch.overflow();
Map<String, Object> result = EtcdAgent.pollWatchState(session, "watch-1");
Assertions.assertEquals("overflow", ((Map<?, ?>) result.get("terminal")).get("reason"));
Assertions.assertEquals(0, session.watchCount());
}
private static KV scriptedKv(
AtomicInteger getCalls,
AtomicInteger txnCalls,
List<GetResponse> getResponses,
List<TxnResponse> txnResponses
) {
Deque<GetResponse> gets = new ArrayDeque<>(getResponses);
Deque<TxnResponse> txns = new ArrayDeque<>(txnResponses);
return (KV) Proxy.newProxyInstance(
KV.class.getClassLoader(),
new Class<?>[] { KV.class },
(proxy, method, args) -> {
if (method.getName().equals("get")) {
getCalls.incrementAndGet();
return CompletableFuture.completedFuture(gets.removeFirst());
}
if (method.getName().equals("txn")) {
return scriptedTxn(txnCalls, txns);
}
if (method.getName().equals("close")) {
return null;
}
throw new UnsupportedOperationException(method.getName());
}
);
}
private static Txn scriptedTxn(AtomicInteger txnCalls, Deque<TxnResponse> responses) {
return (Txn) Proxy.newProxyInstance(
Txn.class.getClassLoader(),
new Class<?>[] { Txn.class },
(proxy, method, args) -> {
if (Arrays.asList("If", "Then", "Else").contains(method.getName())) {
return proxy;
}
if (method.getName().equals("commit")) {
txnCalls.incrementAndGet();
return CompletableFuture.completedFuture(responses.removeFirst());
}
throw new UnsupportedOperationException(method.getName());
}
);
}
private static ByteSequence bytes(String value) {
return ByteSequence.from(value, StandardCharsets.UTF_8);
}
private static GetResponse getResponse(String key, long modRevision, long lease) {
KeyValue keyValue = KeyValue.newBuilder()
.setKey(ByteString.copyFromUtf8(key))
.setModRevision(modRevision)
.setLease(lease)
.build();
RangeResponse response = RangeResponse.newBuilder()
.setHeader(header(modRevision))
.addKvs(keyValue)
.build();
return new GetResponse(response, ByteSequence.EMPTY);
}
private static GetResponse emptyGetResponse() {
return new GetResponse(
RangeResponse.newBuilder().setHeader(header(1)).build(),
ByteSequence.EMPTY
);
}
private static TxnResponse txnResponse(boolean succeeded, long revision) {
io.etcd.jetcd.api.TxnResponse response = io.etcd.jetcd.api.TxnResponse.newBuilder()
.setHeader(header(revision))
.setSucceeded(succeeded)
.build();
return new TxnResponse(response, ByteSequence.EMPTY);
}
private static ResponseHeader header(long revision) {
return ResponseHeader.newBuilder().setRevision(revision).build();
}
}
+70
View File
@@ -0,0 +1,70 @@
# etcd v2 native agent (`etcd2`)
Native Go agent speaking the etcd **v2 HTTP/JSON API** with the standard DBX
agent protocol (NDJSON JSON-RPC over stdio). Registered as agentKey `etcd2`,
selected through the `etcd-v2` driver profile of the `etcd` connection type.
## Supported servers
| Server | Support |
| --- | --- |
| etcd 2.0–2.3 (pure v2) | fully supported; validated against 2.3.8 |
| etcd 3.0–3.5 started with `--enable-v2` | same code path |
| etcd 3.4–3.5 default configuration | rejected at connect: v2 is disabled by default (`ETCD_V2_API_DISABLED`) |
| etcd 3.6+ | impossible: the v2 API was removed upstream |
## Capabilities
`connect`, `test_connection`, `kv`, `kv_ttl`, `kv_cas`, `kv_list_values`,
`kv_status`, `etcd_watch`, `etcd_auth`, `multi_session`. The v3-only
capabilities (`kv_history`, `etcd_lease`, `etcd_compaction`, `etcd_defrag`)
are deliberately absent; the host blocks those calls before dispatch and the
UI hides the corresponding workspaces for `etcd-v2` connections.
## v2 API semantics and deliberate limits
- **TTL**: v2 attaches TTLs directly to keys (`?ttl=`). Key editing offers
permanent/TTL expiry only; binding a lease is rejected with
`ETCD_V2_LEASE_UNSUPPORTED` because v2 has no lease concept.
- **CAS**: writes use `prevIndex` (modifiedIndex) and `prevExist`, matching
the v3 modRevision contract. `expectedCreateRevision` is only honored when
`0` (key-must-not-exist); a nonzero value is rejected with
`ETCD_V2_CAS_UNSUPPORTED` because v2 does not expose createdIndex for
compare-and-swap.
- **Rename**: check-then-set, **not atomic** — v2 has no transaction
primitive. The source is verified by CAS before the target is created, but
a concurrent writer between the two requests is not serialized.
- **Watch**: long-poll loop (`?wait=true&waitIndex=N`, recursive for prefix
scopes). Idle poll timeouts are retried transparently. There are no
progress notifications; the same batch/event/byte budget system as the v3
agent applies.
- **Hidden directories**: v2 hides `_`-prefixed directory entries from
non-recursive listings; such keys are only reachable when addressed
explicitly, matching `etcdctl` behavior.
- **Auth**: users hold roles; roles hold read/write permissions over exact
keys and `/prefix/*` globs. Grants and revokes use the server's incremental
`grant`/`revoke` role PUT (`{"role":"r","grant":{"kv":{"read":["/p/*"]}}}`),
matching the etcd 2.3 client wire format. Resource mapping: `key` → bare
path, `prefix`/`all` → `/path/*` / `/*` globs.
- **Status**: reduced dashboard. `clusterId` comes from the
`X-Etcd-Cluster-Id` header, `revision` from `X-Etcd-Index`; Prometheus
metrics, alarms, dbSize, and raft details do not exist in v2 and stay null.
Raw `/v2/stats/store` counters are passed through as the `store` object.
## Connect probing
`connect` fetches `/version`, then probes `GET /v2/members`:
- `200` → ok; `403` → ok with `limited: true` (credentials work, root access
missing); `404` → `ETCD_V2_API_DISABLED`; `401` → `ETCD_UNAUTHENTICATED`.
## Testing
```bash
go test ./... # unit tests
DBX_ETCD2_LIVE=1 go test -run Live ./... # live flow (default endpoint below)
```
Live test environment variables: `DBX_ETCD2_ENDPOINTS`, `DBX_ETCD2_USER`,
`DBX_ETCD2_PASSWORD`. The default endpoint points at the team test server's
etcd 2.3.8 instance; see `docs/testing-databases.md` in the workspace root.
+390
View File
@@ -0,0 +1,390 @@
package main
import (
"context"
"encoding/json"
"fmt"
"net/http"
"strings"
)
// v2 auth wire format (etcd 2.3 etcdserver/etcdhttp/client_auth.go):
// a role PUT body without grant/revoke creates the role; a body carrying
// grant/revoke updates it incrementally and must not carry permissions.
// Permission entries are glob patterns: a trailing "*" grants a prefix,
// anything else matches a single key exactly.
type v2RWPermission struct {
Read []string `json:"read"`
Write []string `json:"write"`
}
type v2Permissions struct {
KV v2RWPermission `json:"kv"`
}
type v2Role struct {
Role string `json:"role"`
Permissions *v2Permissions `json:"permissions"`
Grant *v2Permissions `json:"grant,omitempty"`
Revoke *v2Permissions `json:"revoke,omitempty"`
}
// v2UserRecord is the GET shape of /v2/auth/users/<name>: roles come back
// as full role objects, not plain names.
type v2UserRecord struct {
User string `json:"user"`
Roles []struct {
Role string `json:"role"`
} `json:"roles"`
}
// v2UserDocument is the PUT shape: grant/revoke carry role names.
type v2UserDocument struct {
User string `json:"user"`
Password string `json:"password,omitempty"`
Roles []string `json:"roles,omitempty"`
Grant []string `json:"grant,omitempty"`
Revoke []string `json:"revoke,omitempty"`
}
func (s *etcd2Session) authUserList(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
body, _, err := client.do(ctx, http.MethodGet, "/v2/auth/users", "", nil)
if err != nil {
return nil, err
}
var parsed struct {
Users []v2UserRecord `json:"users"`
}
if err := json.Unmarshal(body, &parsed); err != nil {
return nil, err
}
users := make([]string, 0, len(parsed.Users))
for _, user := range parsed.Users {
users = append(users, user.User)
}
return map[string]any{"users": users}, nil
}
func (s *etcd2Session) authUserGet(params map[string]json.RawMessage) (any, error) {
user, err := requiredString(params, "user")
if err != nil {
return nil, err
}
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
body, _, err := client.do(ctx, http.MethodGet, "/v2/auth/users/"+escapePathSegment(user), "", nil)
if err != nil {
return nil, err
}
var parsed v2UserRecord
if err := json.Unmarshal(body, &parsed); err != nil {
return nil, err
}
roles := make([]string, 0, len(parsed.Roles))
for _, role := range parsed.Roles {
roles = append(roles, role.Role)
}
return map[string]any{"user": parsed.User, "roles": roles}, nil
}
// authUserAdd relies on the create-or-update PUT: HTTP 201 means created,
// HTTP 200 means the user existed and only the password would change.
func (s *etcd2Session) authUserAdd(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
user, err := requiredString(params, "user")
if err != nil {
return nil, err
}
password, err := requiredString(params, "password")
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
payload := v2UserDocument{User: user, Password: password}
_, response, err := client.doJSON(ctx, http.MethodPut, "/v2/auth/users/"+escapePathSegment(user), payload)
if err != nil {
return nil, err
}
if response != nil && response.StatusCode == http.StatusOK {
return nil, fmt.Errorf("auth: User %s already exists.", user)
}
return map[string]bool{"created": true}, nil
}
func (s *etcd2Session) authUserDelete(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
user, err := requiredString(params, "user")
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
if _, _, err := client.do(ctx, http.MethodDelete, "/v2/auth/users/"+escapePathSegment(user), "", nil); err != nil {
return nil, err
}
return map[string]bool{"deleted": true}, nil
}
// authUserChangePassword reuses the create-or-update PUT: the server
// replaces the stored password hash when the user exists.
func (s *etcd2Session) authUserChangePassword(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
user, err := requiredString(params, "user")
if err != nil {
return nil, err
}
password, err := requiredString(params, "password")
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
payload := v2UserDocument{User: user, Password: password}
if _, _, err := client.doJSON(ctx, http.MethodPut, "/v2/auth/users/"+escapePathSegment(user), payload); err != nil {
return nil, err
}
return map[string]bool{"changed": true}, nil
}
func (s *etcd2Session) authUserGrantRevokeRole(params map[string]json.RawMessage, grant bool) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
user, err := requiredString(params, "user")
if err != nil {
return nil, err
}
role, err := requiredString(params, "role")
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
payload := v2UserDocument{User: user, Grant: []string{role}}
if !grant {
payload = v2UserDocument{User: user, Revoke: []string{role}}
}
if _, _, err := client.doJSON(ctx, http.MethodPut, "/v2/auth/users/"+escapePathSegment(user), payload); err != nil {
return nil, err
}
return map[string]bool{"updated": true}, nil
}
func (s *etcd2Session) authRoleList(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
body, _, err := client.do(ctx, http.MethodGet, "/v2/auth/roles", "", nil)
if err != nil {
return nil, err
}
var parsed struct {
Roles []v2Role `json:"roles"`
}
if err := json.Unmarshal(body, &parsed); err != nil {
return nil, err
}
roles := make([]string, 0, len(parsed.Roles))
for _, role := range parsed.Roles {
roles = append(roles, role.Role)
}
return map[string]any{"roles": roles}, nil
}
func (s *etcd2Session) authRoleGet(params map[string]json.RawMessage) (any, error) {
role, err := requiredString(params, "role")
if err != nil {
return nil, err
}
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
body, _, err := client.do(ctx, http.MethodGet, "/v2/auth/roles/"+escapePathSegment(role), "", nil)
if err != nil {
return nil, err
}
var parsed v2Role
if err := json.Unmarshal(body, &parsed); err != nil {
return nil, err
}
permissions := []map[string]any{}
if parsed.Permissions != nil {
permissions = append(permissions, v2PermissionRows("read", parsed.Permissions.KV.Read)...)
permissions = append(permissions, v2PermissionRows("write", parsed.Permissions.KV.Write)...)
}
return map[string]any{"role": parsed.Role, "permissions": permissions}, nil
}
// v2PermissionRows converts stored v2 patterns into the
// {access, key, rangeEnd, resource} rows the v3 agent emits: a trailing
// "*" displays as a prefix grant, anything else as an exact key.
func v2PermissionRows(access string, entries []string) []map[string]any {
rows := []map[string]any{}
for _, entry := range entries {
if strings.HasSuffix(entry, "*") {
base := strings.TrimSuffix(entry, "*")
rows = append(rows, map[string]any{
"access": access,
"key": bytesObject([]byte(base)),
"rangeEnd": bytesObject([]byte(prefixEnd(base))),
"resource": "prefix",
})
} else {
rows = append(rows, map[string]any{
"access": access,
"key": bytesObject([]byte(entry)),
"rangeEnd": bytesObject(nil),
"resource": "key",
})
}
}
return rows
}
func (s *etcd2Session) authRoleAdd(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
role, err := requiredString(params, "role")
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
payload := v2Role{Role: role}
if _, _, err := client.doJSON(ctx, http.MethodPut, "/v2/auth/roles/"+escapePathSegment(role), payload); err != nil {
return nil, err
}
return map[string]bool{"created": true}, nil
}
func (s *etcd2Session) authRoleDelete(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
role, err := requiredString(params, "role")
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
if _, _, err := client.do(ctx, http.MethodDelete, "/v2/auth/roles/"+escapePathSegment(role), "", nil); err != nil {
return nil, err
}
return map[string]bool{"deleted": true}, nil
}
// v2PermissionPattern maps the protocol's resource model onto the v2 glob
// convention: exact keys stay bare, prefixes and "all" become "/path/*".
func v2PermissionPattern(resource, key string) string {
switch resource {
case "all":
return "/*"
case "prefix":
return strings.TrimSuffix(key, "/") + "/*"
default:
return key
}
}
// authRolePermission maps grants/revokes onto the server's incremental
// grant/revoke role update (one PUT per access direction).
func (s *etcd2Session) authRolePermission(params map[string]json.RawMessage, grant bool) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
role, err := requiredString(params, "role")
if err != nil {
return nil, err
}
resource := stringOrDefault(params, "resource", "key")
key, err := keyBytesParam(params)
if err != nil {
return nil, err
}
if !strings.HasPrefix(key, "/") {
key = "/" + key
}
pattern := v2PermissionPattern(resource, key)
var accesses []string
if grant {
access := strings.ToUpper(stringOrDefault(params, "access", ""))
switch access {
case "READ":
accesses = []string{"read"}
case "WRITE":
accesses = []string{"write"}
case "READWRITE":
accesses = []string{"read", "write"}
default:
return nil, fmt.Errorf("ETCD_INVALID_ACCESS: access must be READ, WRITE, or READWRITE, got %s", access)
}
} else {
accesses = []string{"read", "write"}
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
for _, access := range accesses {
document := v2Role{Role: role}
direction := v2Permissions{KV: v2RWPermission{}}
if access == "read" {
direction.KV.Read = []string{pattern}
} else {
direction.KV.Write = []string{pattern}
}
if grant {
document.Grant = &direction
} else {
document.Revoke = &direction
}
if _, _, err := client.doJSON(ctx, http.MethodPut, "/v2/auth/roles/"+escapePathSegment(role), document); err != nil {
return nil, err
}
}
return map[string]bool{"updated": true}, nil
}
func (c *authenticatedClient) doJSON(ctx context.Context, method, path string, payload any) ([]byte, *http.Response, error) {
encoded, err := json.Marshal(payload)
if err != nil {
return nil, nil, err
}
return c.do(ctx, method, path, string(encoded), map[string]string{"Content-Type": "application/json"})
}
func escapePathSegment(segment string) string {
return strings.ReplaceAll(segment, "/", "%2F")
}
+457
View File
@@ -0,0 +1,457 @@
package main
import (
"context"
"crypto/tls"
"crypto/x509"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"strings"
"time"
)
const (
defaultPort = 2379
operationTimeout = 30 * time.Second
)
// connectionParams mirrors the host-side etcd connection configuration.
// gRPC-specific options are accepted and ignored: the v2 API is plain HTTP.
type connectionParams struct {
EtcdEndpoints string `json:"etcd_endpoints"`
Endpoints string `json:"endpoints"`
ConnectionString string `json:"connection_string"`
Host string `json:"host"`
Port int `json:"port"`
Username string `json:"username"`
Password string `json:"password"`
SSL bool `json:"ssl"`
CACertPath string `json:"ca_cert_path"`
ClientCertPath string `json:"client_cert_path"`
ClientKeyPath string `json:"client_key_path"`
CertPath string `json:"cert_path"`
KeyPath string `json:"key_path"`
ConnectTimeoutSecs int `json:"connect_timeout_secs"`
GrpcMaxInboundMessageSize int `json:"grpc_max_inbound_message_size"`
URLParams string `json:"url_params"`
}
// authenticatedClient issues v2 API requests against one etcd endpoint with
// optional basic auth and TLS.
type authenticatedClient struct {
endpoint string
http *http.Client
username string
password string
serverVersion string
}
func connectionObject(params map[string]json.RawMessage) (connectionParams, error) {
var connection connectionParams
if raw, ok := params["connection"]; ok && raw != nil {
if err := json.Unmarshal(raw, &connection); err != nil {
return connection, err
}
return connection, nil
}
if err := decodeParams(params, &connection); err != nil {
return connection, err
}
return connection, nil
}
func (s *etcd2Session) connect(params map[string]json.RawMessage) (any, error) {
connection, err := connectionObject(params)
if err != nil {
return nil, err
}
endpointList := connectionEndpoints(connection)
var lastErr error
for _, endpoint := range endpointList {
client, _, err := probeClient(endpoint, connection)
if err != nil {
lastErr = err
continue
}
s.close()
s.clientMu.Lock()
s.httpClient = client
s.connectedEndpoints = endpointList
s.serverVersion = client.serverVersion
s.clientMu.Unlock()
return map[string]bool{"ok": true}, nil
}
if lastErr == nil {
lastErr = errors.New("No etcd endpoint configured")
}
return nil, lastErr
}
func (s *etcd2Session) activeClient() (*authenticatedClient, error) {
s.clientMu.Lock()
client := s.httpClient
s.clientMu.Unlock()
if client == nil {
return nil, errors.New("Not connected")
}
return client, nil
}
func (s *etcd2Session) connectedEndpointList() []string {
s.clientMu.Lock()
defer s.clientMu.Unlock()
return append([]string(nil), s.connectedEndpoints...)
}
func (s *etcd2Session) validateConnection() (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(context.Background(), operationTimeout)
defer cancel()
probe, err := client.probeV2(ctx)
if err != nil {
return nil, err
}
return probe, nil
}
func (s *etcd2Session) close() error {
s.watchesMu.Lock()
watches := s.watches
s.watches = map[string]*watchState{}
s.watchBufferedBytes = 0
s.watchesMu.Unlock()
for _, watch := range watches {
watch.close()
}
s.clientMu.Lock()
s.httpClient = nil
s.connectedEndpoints = nil
s.serverVersion = ""
s.clientMu.Unlock()
return nil
}
func buildHTTPClient(connection connectionParams) (*http.Client, error) {
transport := &http.Transport{
MaxIdleConns: 4,
MaxIdleConnsPerHost: 4,
}
if connection.SSL {
tlsConfig, err := tlsConfigFor(connection)
if err != nil {
return nil, err
}
transport.TLSClientConfig = tlsConfig
}
return &http.Client{
Transport: transport,
// The request context governs per-call timeouts; the client-level
// timeout stays high so long-poll watches are not cut short.
Timeout: 10 * time.Minute,
}, nil
}
func connectTimeoutSeconds(connection connectionParams) time.Duration {
seconds := connection.ConnectTimeoutSecs
if seconds == 0 {
seconds = 30
}
if seconds < 1 {
seconds = 1
}
if seconds > 300 {
seconds = 300
}
return time.Duration(seconds) * time.Second
}
func tlsConfigFor(connection connectionParams) (*tls.Config, error) {
tlsConfig := &tls.Config{}
if ca := strings.TrimSpace(connection.CACertPath); ca != "" {
authorityPEM, err := os.ReadFile(ca)
if err != nil {
return nil, err
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(authorityPEM) {
return nil, fmt.Errorf("failed to parse CA certificate at %s", ca)
}
tlsConfig.RootCAs = pool
}
certPath := firstNonBlank(connection.ClientCertPath, connection.CertPath)
keyPath := firstNonBlank(connection.ClientKeyPath, connection.KeyPath)
if (certPath == "") != (keyPath == "") {
return nil, errors.New("Client certificate and key must be provided together")
}
if certPath != "" {
pair, err := tls.LoadX509KeyPair(certPath, keyPath)
if err != nil {
return nil, err
}
tlsConfig.Certificates = []tls.Certificate{pair}
}
return tlsConfig, nil
}
// probeClient verifies the endpoint speaks the v2 API. It returns the
// connected client and a probe result shaped like the v3 agent's:
// {ok, endpoint, limited?}.
func probeClient(endpoint string, connection connectionParams) (*authenticatedClient, map[string]any, error) {
httpClient, err := buildHTTPClient(connection)
if err != nil {
return nil, nil, err
}
client := &authenticatedClient{
endpoint: strings.TrimSuffix(endpoint, "/"),
http: httpClient,
username: connection.Username,
password: connection.Password,
}
dialCtx, dialCancel := context.WithTimeout(context.Background(), connectTimeoutSeconds(connection))
defer dialCancel()
version, err := client.fetchVersion(dialCtx)
if err != nil {
return nil, nil, err
}
client.serverVersion = version.etcdserver
probe, err := client.probeV2(dialCtx)
if err != nil {
return nil, nil, err
}
return client, probe, nil
}
type etcdVersion struct {
etcdserver string
etcdcluster string
}
func (c *authenticatedClient) fetchVersion(ctx context.Context) (etcdVersion, error) {
body, _, err := c.do(ctx, http.MethodGet, "/version", "", nil)
if err != nil {
return etcdVersion{}, err
}
var parsed struct {
Etcdserver string `json:"etcdserver"`
Etcdcluster string `json:"etcdcluster"`
}
if err := json.Unmarshal(body, &parsed); err != nil {
return etcdVersion{}, fmt.Errorf("unrecognized etcd /version response from %s: %w", c.endpoint, err)
}
return etcdVersion{etcdserver: parsed.Etcdserver, etcdcluster: parsed.Etcdcluster}, nil
}
// probeV2 checks that the v2 keys API is actually served. A 403 proves the
// channel and credentials reached etcd, mirroring the v3 agent's
// PERMISSION_DENIED handling for restricted users.
func (c *authenticatedClient) probeV2(ctx context.Context) (map[string]any, error) {
response, err := c.request(ctx, http.MethodGet, "/v2/members", "", nil)
if err != nil {
return nil, err
}
defer drainClose(response.Body)
switch response.StatusCode {
case http.StatusOK:
return map[string]any{"ok": true, "endpoint": c.endpoint}, nil
case http.StatusForbidden:
return map[string]any{"ok": true, "endpoint": c.endpoint, "limited": true}, nil
case http.StatusNotFound:
return nil, fmt.Errorf("ETCD_V2_API_DISABLED: %s does not expose the etcd v2 API (removed in etcd 3.6+)", c.endpoint)
case http.StatusUnauthorized:
return nil, fmt.Errorf("ETCD_UNAUTHENTICATED: authentication failed against %s", c.endpoint)
default:
body, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
return nil, fmt.Errorf("etcd v2 probe against %s failed: HTTP %d %s", c.endpoint, response.StatusCode, strings.TrimSpace(string(body)))
}
}
// do performs a v2 API request and returns the body. Non-2xx responses are
// converted into etcdError values carrying the server's errorCode/message.
func (c *authenticatedClient) do(ctx context.Context, method, path, body string, header map[string]string) ([]byte, *http.Response, error) {
response, err := c.request(ctx, method, path, body, header)
if err != nil {
return nil, response, err
}
payload, readErr := io.ReadAll(response.Body)
_ = response.Body.Close()
if response.StatusCode < 200 || response.StatusCode >= 300 {
return nil, response, errorFromResponse(response.StatusCode, payload)
}
if readErr != nil {
return nil, response, readErr
}
return payload, response, nil
}
func (c *authenticatedClient) request(ctx context.Context, method, path, body string, header map[string]string) (*http.Response, error) {
return c.requestAt(ctx, method, c.endpoint, path, body, header)
}
// requestAt targets a specific endpoint, used for per-member status fan-out.
func (c *authenticatedClient) requestAt(ctx context.Context, method, endpoint, path, body string, header map[string]string) (*http.Response, error) {
var reader io.Reader
if body != "" {
reader = strings.NewReader(body)
}
req, err := http.NewRequestWithContext(ctx, method, strings.TrimSuffix(endpoint, "/")+path, reader)
if err != nil {
return nil, err
}
if c.username != "" {
req.SetBasicAuth(c.username, c.password)
}
for key, value := range header {
req.Header.Set(key, value)
}
if body != "" && req.Header.Get("Content-Type") == "" {
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
}
response, err := c.http.Do(req)
if err != nil {
// gRPC-style sentinel so the host-side transient-error handling keeps working.
return nil, fmt.Errorf("etcd connection error: %w", err)
}
return response, nil
}
// doAt performs a v2 API request against a specific endpoint.
func (c *authenticatedClient) doAt(ctx context.Context, method, endpoint, path, body string) ([]byte, *http.Response, error) {
response, err := c.requestAt(ctx, method, endpoint, path, body, nil)
if err != nil {
return nil, nil, err
}
payload, readErr := io.ReadAll(response.Body)
_ = response.Body.Close()
if response.StatusCode < 200 || response.StatusCode >= 300 {
return nil, response, errorFromResponse(response.StatusCode, payload)
}
if readErr != nil {
return nil, response, readErr
}
return payload, response, nil
}
// etcdError carries the v2 API error payload (errorCode + message + index).
type etcdError struct {
statusCode int
errorCode int64
message string
cause string
index int64
}
func (e *etcdError) Error() string {
if e.message == "" {
return fmt.Sprintf("etcd v2 error (HTTP %d)", e.statusCode)
}
if e.cause != "" {
return fmt.Sprintf("%s (%s)", e.message, e.cause)
}
return e.message
}
func errorFromResponse(statusCode int, body []byte) *etcdError {
parsed := &etcdError{statusCode: statusCode}
if len(body) > 0 {
var payload struct {
ErrorCode int64 `json:"errorCode"`
Message string `json:"message"`
Cause string `json:"cause"`
Index int64 `json:"index"`
}
if json.Unmarshal(body, &payload) == nil && payload.Message != "" {
parsed.errorCode = payload.ErrorCode
parsed.message = payload.Message
parsed.cause = payload.Cause
parsed.index = payload.Index
return parsed
}
}
parsed.message = strings.TrimSpace(string(body))
return parsed
}
func isEtcdErrorCode(err error, code int64) bool {
var etcdErr *etcdError
if errors.As(err, &etcdErr) {
return etcdErr.errorCode == code
}
return false
}
func isNotFound(err error) bool {
return isEtcdErrorCode(err, 100)
}
func isCompareFailed(err error) bool {
return isEtcdErrorCode(err, 101)
}
func drainClose(body io.ReadCloser) {
_, _ = io.Copy(io.Discard, io.LimitReader(body, 4096))
_ = body.Close()
}
func connectionEndpoints(connection connectionParams) []string {
configured := firstNonBlank(connection.EtcdEndpoints, connection.Endpoints, connection.ConnectionString)
var result []string
if configured != "" {
for _, endpoint := range strings.FieldsFunc(configured, func(r rune) bool { return r == ',' || r == '\n' }) {
normalized := normalizeEndpoint(strings.TrimSpace(endpoint), connection.SSL)
if normalized != "" {
result = append(result, normalized)
}
}
}
if len(result) == 0 {
host := connection.Host
if host == "" {
host = "127.0.0.1"
}
port := connection.Port
if port == 0 {
port = defaultPort
}
result = append(result, normalizeEndpoint(fmt.Sprintf("%s:%d", host, port), connection.SSL))
}
return result
}
func normalizeEndpoint(endpoint string, tlsEnabled bool) string {
if strings.TrimSpace(endpoint) == "" {
return ""
}
if strings.HasPrefix(endpoint, "http://") || strings.HasPrefix(endpoint, "https://") {
return strings.TrimSuffix(endpoint, "/")
}
scheme := "http"
if tlsEnabled {
scheme = "https"
}
return scheme + "://" + endpoint
}
func endpointHost(rawEndpoint string) string {
parsed, err := url.Parse(rawEndpoint)
if err != nil {
return ""
}
return parsed.Hostname()
}
func firstNonBlank(values ...string) string {
for _, value := range values {
if strings.TrimSpace(value) != "" {
return value
}
}
return ""
}
+3
View File
@@ -0,0 +1,3 @@
module github.com/t8y2/dbx/agents/drivers/etcd2-go
go 1.24
+412
View File
@@ -0,0 +1,412 @@
package main
import (
"encoding/json"
"fmt"
"os"
"strings"
"testing"
"time"
)
// TestLiveEtcd2Agent exercises the v2 agent surface against a real etcd 2.x
// server. Enable with DBX_ETCD2_LIVE=1; configure via DBX_ETCD2_ENDPOINTS,
// DBX_ETCD2_USER, and DBX_ETCD2_PASSWORD (defaults match the server
// deployment: root/123456).
func TestLiveEtcd2Agent(t *testing.T) {
if os.Getenv("DBX_ETCD2_LIVE") != "1" {
t.Skip("set DBX_ETCD2_LIVE=1 to run the live etcd v2 agent test")
}
endpoints := envOrDefault("DBX_ETCD2_ENDPOINTS", "http://172.26.129.83:20041")
user := envOrDefault("DBX_ETCD2_USER", "root")
password := envOrDefault("DBX_ETCD2_PASSWORD", "123456")
state := newEtcd2Session()
connectParams := map[string]json.RawMessage{
"etcd_endpoints": json.RawMessage(`"` + endpoints + `"`),
"username": json.RawMessage(`"` + user + `"`),
"password": json.RawMessage(`"` + password + `"`),
}
if _, err := state.connect(connectParams); err != nil {
t.Fatalf("connect failed: %v", err)
}
defer state.close()
if result, err := state.validateConnection(); err != nil {
t.Fatalf("validate_connection failed: %v", err)
} else if probe := result.(map[string]any); probe["ok"] != true {
t.Fatalf("unexpected probe result: %#v", probe)
}
prefix := fmt.Sprintf("/dbx/live/%d/", time.Now().UnixNano())
// --- KV basics -----------------------------------------------------
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "hello"),
"value": jsonValue("world"),
})); err != nil {
t.Fatalf("put failed: %v", err)
}
fetched, err := state.get(paramsWith(map[string]json.RawMessage{"key": jsonString(prefix + "hello")}))
if err != nil {
t.Fatalf("get failed: %v", err)
}
row := fetched.(map[string]any)
if row["found"] != true || row["key"] != prefix+"hello" {
t.Fatalf("unexpected get result: %#v", row)
}
value := row["value"].(map[string]any)
if value["encoding"] != "utf8" || value["data"] != "world" {
t.Fatalf("unexpected value: %#v", value)
}
metadata := row["metadata"].(map[string]any)
if metadata["createRevision"] != metadata["modRevision"] {
t.Fatalf("fresh key indexes should match: %#v", metadata)
}
missing, err := state.get(paramsWith(map[string]json.RawMessage{"key": jsonString(prefix + "missing")}))
if err != nil {
t.Fatalf("missing get failed: %v", err)
}
if missing.(map[string]any)["found"] != false {
t.Fatalf("expected found=false: %#v", missing)
}
// CAS conflict on stale index.
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "hello"),
"value": jsonValue("v2"),
"expectedModRevision": json.RawMessage(`1`),
})); err == nil || err.Error() != "ETCD_CAS_CONFLICT: key changed after it was loaded" {
t.Fatalf("expected CAS conflict, got %v", err)
}
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "hello"),
"value": jsonValue("v3"),
"expectedModRevision": json.RawMessage(metadata["modRevision"].(string)),
})); err != nil {
t.Fatalf("CAS put failed: %v", err)
}
// TTL put and metadata.
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "leased"),
"value": jsonValue("expiring"),
"ttl": json.RawMessage(`300`),
})); err != nil {
t.Fatalf("ttl put failed: %v", err)
}
leased, err := state.get(paramsWith(map[string]json.RawMessage{"key": jsonString(prefix + "leased")}))
if err != nil {
t.Fatalf("leased get failed: %v", err)
}
if _, hasTTL := leased.(map[string]any)["metadata"].(map[string]any)["ttl"]; !hasTTL {
t.Fatalf("leased key must expose ttl metadata: %#v", leased)
}
// Lease objects are a v3 concept and must be rejected clearly.
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "nolease"),
"value": jsonValue("v"),
"lease": json.RawMessage(`5`),
})); err == nil || !strings.HasPrefix(err.Error(), "ETCD_V2_LEASE_UNSUPPORTED") {
t.Fatalf("expected lease rejection, got %v", err)
}
// Rename including conflict and missing-source paths.
if _, err := state.rename(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "old"),
"newKey": jsonString(prefix + "new"),
})); err == nil || err.Error() != "ETCD_NOT_FOUND: source key does not exist" {
t.Fatalf("expected rename not-found, got %v", err)
}
if _, err := state.rename(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "hello"),
"newKey": jsonString(prefix + "renamed"),
})); err != nil {
t.Fatalf("rename failed: %v", err)
}
renamed, err := state.get(paramsWith(map[string]json.RawMessage{"key": jsonString(prefix + "renamed")}))
if err != nil || renamed.(map[string]any)["found"] != true {
t.Fatalf("rename target missing: %#v %v", renamed, err)
}
if renamed.(map[string]any)["value"].(map[string]any)["data"] != "v3" {
t.Fatalf("rename must carry the value: %#v", renamed)
}
source, err := state.get(paramsWith(map[string]json.RawMessage{"key": jsonString(prefix + "hello")}))
if err != nil || source.(map[string]any)["found"] != false {
t.Fatalf("rename must remove the source: %#v %v", source, err)
}
if _, err := state.rename(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "leased"),
"newKey": jsonString(prefix + "renamed"),
})); err == nil || err.Error() != "ETCD_CAS_CONFLICT: source changed or target already exists" {
t.Fatalf("expected rename conflict, got %v", err)
}
// list_prefix with continuation.
for i := 0; i < 3; i++ {
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(fmt.Sprintf("%sbulk%d", prefix, i)),
"value": jsonValue(fmt.Sprintf("v%d", i)),
})); err != nil {
t.Fatalf("bulk put %d failed: %v", i, err)
}
}
listed, err := state.listPrefix(paramsWith(map[string]json.RawMessage{
"prefix": jsonString(prefix),
"limit": json.RawMessage(`2`),
"includeValues": json.RawMessage(`true`),
}))
if err != nil {
t.Fatalf("list_prefix failed: %v", err)
}
list := listed.(map[string]any)
if len(list["keys"].([]any)) != 2 {
t.Fatalf("expected limited list, got %#v", list)
}
continuation, ok := list["continuation"].(string)
if !ok || continuation == "" {
t.Fatalf("expected continuation, got %#v", list)
}
nextPage, err := state.listPrefix(paramsWith(map[string]json.RawMessage{
"prefix": jsonString(prefix),
"limit": json.RawMessage(`100`),
"includeValues": json.RawMessage(`true`),
"continuation": json.RawMessage(`"` + continuation + `"`),
}))
if err != nil {
t.Fatalf("continuation list failed: %v", err)
}
nextKeys := nextPage.(map[string]any)["keys"].([]any)
if len(nextKeys) == 0 {
t.Fatalf("continuation must resume after the first page")
}
if nextKeys[0].(map[string]any)["key"].(string) == list["keys"].([]any)[1].(map[string]any)["key"].(string) {
t.Fatalf("continuation overlapped the first page")
}
// --- watch (long poll) --------------------------------------------
watchStarted, err := state.watchStart(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "watched"),
"includePrevKv": json.RawMessage(`true`),
}))
if err != nil {
t.Fatalf("watch_start failed: %v", err)
}
watchID := watchStarted.(map[string]any)["watchId"].(string)
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "watched"),
"value": jsonValue("first"),
})); err != nil {
t.Fatalf("watched put failed: %v", err)
}
if _, err := state.put(paramsWith(map[string]json.RawMessage{
"key": jsonString(prefix + "watched"),
"value": jsonValue("second"),
})); err != nil {
t.Fatalf("watched put 2 failed: %v", err)
}
deadline := time.Now().Add(20 * time.Second)
var watchBatches []any
for time.Now().Before(deadline) {
polled, err := state.watchPoll(paramsWith(map[string]json.RawMessage{"watchId": json.RawMessage(`"` + watchID + `"`)}))
if err != nil {
t.Fatalf("watch_poll failed: %v", err)
}
batches := polled.(map[string]any)["batches"].([]any)
watchBatches = append(watchBatches, batches...)
if len(watchBatches) >= 2 {
break
}
time.Sleep(150 * time.Millisecond)
}
if len(watchBatches) < 2 {
t.Fatalf("watch did not deliver both events: %#v", watchBatches)
}
lastBatch := watchBatches[len(watchBatches)-1].(map[string]any)
lastEvent := lastBatch["events"].([]map[string]any)[0]
if lastEvent["value"].(map[string]any)["data"] != "second" {
t.Fatalf("unexpected last watch event: %#v", lastEvent)
}
if lastEvent["previousValue"].(map[string]any)["data"] != "first" {
t.Fatalf("prevKv missing on watch event: %#v", lastEvent)
}
stopped, err := state.watchStop(paramsWith(map[string]json.RawMessage{"watchId": json.RawMessage(`"` + watchID + `"`)}))
if err != nil || stopped.(map[string]bool)["stopped"] != true {
t.Fatalf("watch_stop failed: %#v %v", stopped, err)
}
// --- auth ----------------------------------------------------------
userName := fmt.Sprintf("dbxlive%d", time.Now().UnixNano()%100000)
// Idempotent pre-cleanup: an interrupted earlier run may have left the
// fixed-named role behind.
_, _ = state.authRoleDelete(paramsWith(map[string]json.RawMessage{"role": jsonString("dbx_live_role")}))
if _, err := state.authUserAdd(paramsWith(map[string]json.RawMessage{
"user": jsonString(userName),
"password": jsonString("secret123"),
})); err != nil {
t.Fatalf("user add failed: %v", err)
}
if _, err := state.authRoleAdd(paramsWith(map[string]json.RawMessage{"role": jsonString("dbx_live_role")})); err != nil {
t.Fatalf("role add failed: %v", err)
}
if _, err := state.authRolePermission(paramsWith(map[string]json.RawMessage{
"role": jsonString("dbx_live_role"),
"resource": jsonString("prefix"),
"key": jsonString(prefix),
"access": jsonString("READWRITE"),
}), true); err != nil {
t.Fatalf("role grant permission failed: %v", err)
}
roleDetail, err := state.authRoleGet(paramsWith(map[string]json.RawMessage{"role": jsonString("dbx_live_role")}))
if err != nil {
t.Fatalf("role get failed: %v", err)
}
permissions := roleDetail.(map[string]any)["permissions"].([]map[string]any)
if len(permissions) == 0 {
t.Fatalf("expected permission rows after grant: %#v", roleDetail)
}
if _, err := state.authUserGrantRevokeRole(paramsWith(map[string]json.RawMessage{
"user": jsonString(userName),
"role": jsonString("dbx_live_role"),
}), true); err != nil {
t.Fatalf("grant role failed: %v", err)
}
userDetail, err := state.authUserGet(paramsWith(map[string]json.RawMessage{"user": jsonString(userName)}))
if err != nil {
t.Fatalf("user get failed: %v", err)
}
if len(userDetail.(map[string]any)["roles"].([]string)) == 0 {
t.Fatalf("expected granted role: %#v", userDetail)
}
users, err := state.authUserList(paramsWith(map[string]json.RawMessage{}))
if err != nil {
t.Fatalf("user list failed: %v", err)
}
if !containsAny(users.(map[string]any)["users"].([]string), userName) {
t.Fatalf("user list missing created user: %#v", users)
}
roles, err := state.authRoleList(paramsWith(map[string]json.RawMessage{}))
if err != nil || !containsAny(roles.(map[string]any)["roles"].([]string), "dbx_live_role") {
t.Fatalf("role list missing role: %#v %v", roles, err)
}
if _, err := state.authRolePermission(paramsWith(map[string]json.RawMessage{
"role": jsonString("dbx_live_role"),
"resource": jsonString("prefix"),
"key": jsonString(prefix),
}), false); err != nil {
t.Fatalf("role revoke permission failed: %v", err)
}
afterRevoke, err := state.authRoleGet(paramsWith(map[string]json.RawMessage{"role": jsonString("dbx_live_role")}))
if err != nil {
t.Fatalf("role get after revoke failed: %v", err)
}
if len(afterRevoke.(map[string]any)["permissions"].([]map[string]any)) != 0 {
t.Fatalf("permissions must be empty after revoke: %#v", afterRevoke)
}
if _, err := state.authUserGrantRevokeRole(paramsWith(map[string]json.RawMessage{
"user": jsonString(userName),
"role": jsonString("dbx_live_role"),
}), false); err != nil {
t.Fatalf("revoke role failed: %v", err)
}
if _, err := state.authUserChangePassword(paramsWith(map[string]json.RawMessage{
"user": jsonString(userName),
"password": jsonString("rotated456"),
})); err != nil {
t.Fatalf("change password failed: %v", err)
}
if _, err := state.authUserDelete(paramsWith(map[string]json.RawMessage{"user": jsonString(userName)})); err != nil {
t.Fatalf("user delete failed: %v", err)
}
if _, err := state.authRoleDelete(paramsWith(map[string]json.RawMessage{"role": jsonString("dbx_live_role")})); err != nil {
t.Fatalf("role delete failed: %v", err)
}
// --- status --------------------------------------------------------
status, err := state.status(paramsWith(map[string]json.RawMessage{}))
if err != nil {
t.Fatalf("kv_status failed: %v", err)
}
statusRow := status.(map[string]any)
if statusRow["clusterId"] == nil || statusRow["leaderId"] == nil {
t.Fatalf("status missing cluster identity: %#v", statusRow)
}
members := statusRow["members"].([]map[string]any)
if len(members) == 0 || members[0]["reachable"] != true {
t.Fatalf("unexpected member rows: %#v", members)
}
// --- delete + cleanup ----------------------------------------------
for _, key := range []string{prefix + "leased", prefix + "renamed", prefix + "watched", prefix + "bulk0", prefix + "bulk1", prefix + "bulk2"} {
if _, err := state.delete(paramsWith(map[string]json.RawMessage{"key": jsonString(key)})); err != nil {
t.Fatalf("delete %s failed: %v", key, err)
}
}
}
func TestLiveEtcd2ProtocolFlow(t *testing.T) {
if os.Getenv("DBX_ETCD2_LIVE") != "1" {
t.Skip("set DBX_ETCD2_LIVE=1 to run the live etcd v2 agent test")
}
endpoints := envOrDefault("DBX_ETCD2_ENDPOINTS", "http://172.26.129.83:20041")
user := envOrDefault("DBX_ETCD2_USER", "root")
password := envOrDefault("DBX_ETCD2_PASSWORD", "123456")
server := newRuntimeServer()
connection := fmt.Sprintf(`{"etcd_endpoints":%q,"username":%q,"password":%q}`, endpoints, user, password)
handshake, _ := server.handleLine(`{"id":1,"method":"handshake","params":{}}`)
if handshake.Error != nil {
t.Fatalf("handshake failed: %#v", handshake)
}
openLine := fmt.Sprintf(`{"id":2,"method":"open_session","params":{"agentSessionId":"s1","connection":%s}}`, connection)
opened, _ := server.handleLine(openLine)
if opened.Error != nil {
t.Fatalf("open_session failed: %#v", opened)
}
putLine := fmt.Sprintf(`{"id":3,"method":"kv_put","params":{"agentSessionId":"s1","key":"/dbx:proto","value":{"encoding":"utf8","data":"flow"}}}`)
put, _ := server.handleLine(putLine)
if put.Error != nil {
t.Fatalf("kv_put failed: %#v", put)
}
v3Only, _ := server.handleLine(`{"id":4,"method":"kv_history","params":{"agentSessionId":"s1","key":"/dbx:proto"}}`)
if v3Only.Error == nil || !strings.HasPrefix(v3Only.Error.Message, "ETCD_V2_UNSUPPORTED") {
t.Fatalf("expected ETCD_V2_UNSUPPORTED, got %#v", v3Only)
}
closed, _ := server.handleLine(`{"id":5,"method":"close_session","params":{"agentSessionId":"s1"}}`)
if closed.Error != nil {
t.Fatalf("close_session failed: %#v", closed)
}
}
func envOrDefault(key, fallback string) string {
if value := strings.TrimSpace(os.Getenv(key)); value != "" {
return value
}
return fallback
}
func paramsWith(params map[string]json.RawMessage) map[string]json.RawMessage {
return params
}
func jsonString(value string) json.RawMessage {
encoded, _ := json.Marshal(value)
return json.RawMessage(encoded)
}
func jsonValue(value string) json.RawMessage {
encoded, _ := json.Marshal(value)
return json.RawMessage(`{"encoding":"utf8","data":` + string(encoded) + `}`)
}
func containsAny(values []string, target string) bool {
for _, value := range values {
if value == target {
return true
}
}
return false
}
+524
View File
@@ -0,0 +1,524 @@
package main
import (
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"sort"
"strconv"
"strings"
"unicode/utf8"
)
const defaultListLimit = 100
// v2Node is one node of the etcd v2 keys API tree.
type v2Node struct {
Key string `json:"key"`
Value string `json:"value"`
Dir bool `json:"dir"`
Expiration *string `json:"expiration"`
TTL *int64 `json:"ttl"`
CreatedIndex int64 `json:"createdIndex"`
ModifiedIndex int64 `json:"modifiedIndex"`
Nodes []v2Node `json:"nodes"`
}
type v2KeysResponse struct {
Action string `json:"action"`
Node *v2Node `json:"node"`
PrevNode *v2Node `json:"prevNode"`
}
// v2KeyPath normalizes a protocol key into a v2 keys API path. v2 keys are
// slash paths; an empty prefix maps to the root directory.
func v2KeyPath(key string) string {
if key == "" {
return "/v2/keys/"
}
if !strings.HasPrefix(key, "/") {
key = "/" + key
}
return "/v2/keys" + key
}
func (s *etcd2Session) listPrefix(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
prefix := stringOrDefault(params, "prefix", "")
limit := intOrDefault(params, "limit", defaultListLimit)
if limit < 1 {
limit = 1
}
includeValues := boolOrDefault(params, "includeValues", false)
continuation := stringOrNull(params, "continuation")
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
// v2 has no revisioned reads; the revision parameter is accepted and ignored.
query := url.Values{}
query.Set("recursive", "true")
query.Set("sorted", "true")
body, _, err := client.do(ctx, http.MethodGet, v2KeyPath(prefix)+"?"+query.Encode(), "", nil)
if err != nil {
if isNotFound(err) {
return map[string]any{"keys": []any{}, "continuation": nil, "revision": nil}, nil
}
return nil, err
}
var parsed v2KeysResponse
if err := json.Unmarshal(body, &parsed); err != nil {
return nil, err
}
startAfter := ""
if continuation != nil && *continuation != "" {
startAfter = *continuation
}
rows := []map[string]any{}
flattenNodes(parsed.Node, includeValues, func(row map[string]any) {
key := row["key"].(string)
if startAfter != "" && key <= startAfter {
return
}
if len(rows) < limit {
rows = append(rows, row)
}
})
result := map[string]any{"keys": toAnySlice(rows)}
if len(rows) == limit {
result["continuation"] = rows[len(rows)-1]["key"]
} else {
result["continuation"] = nil
}
// etcd v2 exposes the current index on the listing's node headers; the
// modifiedIndex of the returned root is the closest analog of a revision.
var revision any
if parsed.Node != nil {
revision = longString(parsed.Node.ModifiedIndex)
}
result["revision"] = revision
return result, nil
}
func flattenNodes(node *v2Node, includeValues bool, emit func(map[string]any)) {
if node == nil {
return
}
if !node.Dir {
row := nodeMetadata(node)
row["key"] = node.Key
if includeValues {
row["value"] = valueObject([]byte(node.Value))
}
emit(row)
return
}
children := append([]v2Node(nil), node.Nodes...)
sort.SliceStable(children, func(i, j int) bool { return children[i].Key < children[j].Key })
for index := range children {
flattenNodes(&children[index], includeValues, emit)
}
}
func toAnySlice(rows []map[string]any) []any {
result := make([]any, len(rows))
for index, row := range rows {
result[index] = row
}
return result
}
func (s *etcd2Session) get(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
key, err := keyBytesParam(params)
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
body, _, err := client.do(ctx, http.MethodGet, v2KeyPath(key), "", nil)
if err != nil {
if isNotFound(err) {
return map[string]any{"found": false, "key": nil, "value": nil, "metadata": nil}, nil
}
return nil, err
}
var parsed v2KeysResponse
if err := json.Unmarshal(body, &parsed); err != nil {
return nil, err
}
if parsed.Node == nil || parsed.Node.Dir {
return map[string]any{"found": false, "key": nil, "value": nil, "metadata": nil}, nil
}
node := parsed.Node
result := map[string]any{
"found": true,
"key": node.Key,
"keyBytes": bytesObject([]byte(node.Key)),
}
if !boolOrDefault(params, "metadataOnly", false) {
result["value"] = valueObject([]byte(node.Value))
} else {
result["value"] = nil
}
result["metadata"] = nodeMetadata(node)
return result, nil
}
func (s *etcd2Session) put(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
key, err := keyBytesParam(params)
if err != nil {
return nil, err
}
value, err := parseValueObject(rawObject(params, "value"))
if err != nil {
return nil, err
}
expectedModRevision := longOrNull(params, "expectedModRevision")
expectedCreateRevision := longOrNull(params, "expectedCreateRevision")
ttlValue := longOrNull(params, "ttl")
preserveLease := boolOrDefault(params, "preserveLease", false)
leaseValue := longOrNull(params, "lease")
hasLease := leaseValue != nil
hasTtl := ttlValue != nil
if (hasLease && hasTtl) || (preserveLease && (hasLease || hasTtl)) {
return nil, errors.New("lease, ttl, and preserveLease cannot be specified together")
}
if hasLease {
return nil, errors.New("ETCD_V2_LEASE_UNSUPPORTED: the etcd v2 API has no lease objects; use ttl instead")
}
if preserveLease {
return nil, errors.New("ETCD_V2_LEASE_UNSUPPORTED: the etcd v2 API has no lease objects; use ttl instead")
}
if expectedCreateRevision != nil && *expectedCreateRevision != 0 {
return nil, errors.New("ETCD_V2_CAS_UNSUPPORTED: the etcd v2 API cannot compare createdIndex; use expectedModRevision")
}
if hasTtl && *ttlValue <= 0 {
return nil, errors.New("ttl must be a positive integer")
}
form := url.Values{}
form.Set("value", value)
if hasTtl {
form.Set("ttl", strconv.FormatInt(*ttlValue, 10))
}
if expectedModRevision != nil {
form.Set("prevIndex", strconv.FormatInt(*expectedModRevision, 10))
}
if expectedCreateRevision != nil && *expectedCreateRevision == 0 {
form.Set("prevExist", "false")
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
body, _, err := client.do(ctx, http.MethodPut, v2KeyPath(key), form.Encode(), nil)
if err != nil {
if isCompareFailed(err) {
return nil, errors.New("ETCD_CAS_CONFLICT: key changed after it was loaded")
}
return nil, err
}
var parsed v2KeysResponse
if err := json.Unmarshal(body, &parsed); err != nil {
return nil, err
}
var revision int64
if parsed.Node != nil {
revision = parsed.Node.ModifiedIndex
}
return map[string]any{"revision": longString(revision)}, nil
}
func (s *etcd2Session) delete(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
key, err := keyBytesParam(params)
if err != nil {
return nil, err
}
expectedModRevision := longOrNull(params, "expectedModRevision")
path := v2KeyPath(key)
if expectedModRevision != nil {
query := url.Values{}
query.Set("prevIndex", strconv.FormatInt(*expectedModRevision, 10))
path += "?" + query.Encode()
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
body, _, err := client.do(ctx, http.MethodDelete, path, "", nil)
if err != nil {
if isCompareFailed(err) {
return nil, errors.New("ETCD_CAS_CONFLICT: key changed after it was loaded")
}
return nil, err
}
var parsed v2KeysResponse
if err := json.Unmarshal(body, &parsed); err != nil {
return nil, err
}
var revision int64
if parsed.Node != nil {
revision = parsed.Node.ModifiedIndex
}
return map[string]any{"deleted": int64(1), "revision": longString(revision)}, nil
}
// rename is a non-atomic check-then-set: the v2 API has no transactions, so a
// concurrent writer between the target check and the delete can be lost. The
// protocol exposes this honestly instead of pretending atomicity.
func (s *etcd2Session) rename(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
sourceKey, err := keyBytesParam(params)
if err != nil {
return nil, err
}
newKey := stringOrNull(params, "newKey")
if newKey == nil || *newKey == "" {
return nil, errors.New("ETCD_NEWKEY_REQUIRED")
}
targetKey := *newKey
if sourceKey == targetKey {
return map[string]any{"renamed": true, "revision": nil}, nil
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
body, _, err := client.do(ctx, http.MethodGet, v2KeyPath(sourceKey), "", nil)
if err != nil {
if isNotFound(err) {
return nil, errors.New("ETCD_NOT_FOUND: source key does not exist")
}
return nil, err
}
var source v2KeysResponse
if err := json.Unmarshal(body, &source); err != nil {
return nil, err
}
if source.Node == nil || source.Node.Dir {
return nil, errors.New("ETCD_NOT_FOUND: source key does not exist")
}
if _, _, err := client.do(ctx, http.MethodGet, v2KeyPath(targetKey), "", nil); err == nil {
return nil, errors.New("ETCD_CAS_CONFLICT: source changed or target already exists")
} else if !isNotFound(err) {
return nil, err
}
expected := longOrNull(params, "expectedModRevision")
expectedRevision := source.Node.ModifiedIndex
if expected != nil {
expectedRevision = *expected
}
form := url.Values{}
form.Set("value", source.Node.Value)
if source.Node.TTL != nil {
form.Set("ttl", strconv.FormatInt(*source.Node.TTL, 10))
}
form.Set("prevExist", "false")
if _, _, err := client.do(ctx, http.MethodPut, v2KeyPath(targetKey), form.Encode(), nil); err != nil {
if isCompareFailed(err) {
return nil, errors.New("ETCD_CAS_CONFLICT: source changed or target already exists")
}
return nil, err
}
deleteQuery := url.Values{}
deleteQuery.Set("prevIndex", strconv.FormatInt(expectedRevision, 10))
if _, _, err := client.do(ctx, http.MethodDelete, v2KeyPath(sourceKey)+"?"+deleteQuery.Encode(), "", nil); err != nil {
if isCompareFailed(err) {
return nil, errors.New("ETCD_CAS_CONFLICT: source changed or target already exists")
}
return nil, err
}
return map[string]any{"renamed": true, "revision": nil}, nil
}
func nodeMetadata(node *v2Node) map[string]any {
metadata := map[string]any{
"createRevision": longString(node.CreatedIndex),
"modRevision": longString(node.ModifiedIndex),
"version": longString(int64(1)),
"lease": longString(0),
"valueSize": len(node.Value),
}
if node.TTL != nil {
metadata["ttl"] = *node.TTL
}
return metadata
}
func longString(value int64) string {
return strconv.FormatInt(value, 10)
}
func unsignedLongString(value int64) string {
return strconv.FormatUint(uint64(value), 10)
}
// prefixEnd mirrors the v3 agent's byte-increment range end; for v2 rows it
// only feeds UI metadata, where the explicit resource field carries the truth.
func prefixEnd(prefix string) string {
if prefix == "" {
return "\x00"
}
end := []byte(prefix)
for i := len(end) - 1; i >= 0; i-- {
if end[i] < 0xff {
end[i]++
return string(end[:i+1])
}
}
return "\x00"
}
func keyBytesParam(params map[string]json.RawMessage) (string, error) {
if encoded := rawObject(params, "keyBytes"); encoded != nil {
value, err := parseValueObject(encoded)
if err != nil {
return "", err
}
return value, nil
}
if raw, ok := params["key"]; ok && raw != nil && string(raw) != "null" {
var key string
if err := json.Unmarshal(raw, &key); err != nil {
return "", err
}
return key, nil
}
return "", errors.New("Key is required")
}
func rawObject(params map[string]json.RawMessage, key string) map[string]json.RawMessage {
raw := params[key]
if len(raw) == 0 || string(raw) == "null" {
return nil
}
var object map[string]json.RawMessage
if err := json.Unmarshal(raw, &object); err != nil || object == nil {
return nil
}
return object
}
func bytesObject(bytes []byte) map[string]any {
return map[string]any{
"encoding": "base64",
"data": base64.StdEncoding.EncodeToString(bytes),
}
}
func valueObject(value []byte) map[string]any {
if utf8.Valid(value) {
return map[string]any{"encoding": "utf8", "data": string(value)}
}
return bytesObject(value)
}
func displayBytes(bytes []byte) string {
if utf8.Valid(bytes) {
return string(bytes)
}
return base64.StdEncoding.EncodeToString(bytes)
}
func parseValueObject(value map[string]json.RawMessage) (string, error) {
if value == nil {
value = map[string]json.RawMessage{}
}
encoding := stringOrDefault(value, "encoding", "utf8")
data := stringOrDefault(value, "data", "")
if encoding == "base64" {
decoded, err := base64.StdEncoding.DecodeString(data)
if err != nil {
return "", err
}
return string(decoded), nil
}
if encoding != "utf8" {
return "", fmt.Errorf("Unsupported value encoding: %s", encoding)
}
return data, nil
}
func stringOrNull(params map[string]json.RawMessage, key string) *string {
raw := params[key]
if len(raw) == 0 || string(raw) == "null" {
return nil
}
var value string
if err := json.Unmarshal(raw, &value); err != nil {
return nil
}
return &value
}
func stringOrDefault(params map[string]json.RawMessage, key string, fallback string) string {
value := stringOrNull(params, key)
if value == nil {
return fallback
}
return *value
}
func intOrDefault(params map[string]json.RawMessage, key string, fallback int) int {
raw := params[key]
if len(raw) == 0 || string(raw) == "null" {
return fallback
}
var value int
if err := json.Unmarshal(raw, &value); err != nil {
return fallback
}
return value
}
func boolOrDefault(params map[string]json.RawMessage, key string, fallback bool) bool {
raw := params[key]
if len(raw) == 0 || string(raw) == "null" {
return fallback
}
var value bool
if err := json.Unmarshal(raw, &value); err != nil {
return fallback
}
return value
}
func longOrNull(params map[string]json.RawMessage, key string) *int64 {
raw := params[key]
if len(raw) == 0 || string(raw) == "null" {
return nil
}
var value int64
if err := json.Unmarshal(raw, &value); err != nil {
return nil
}
return &value
}
func requiredString(params map[string]json.RawMessage, field string) (string, error) {
value := stringOrNull(params, field)
if value == nil || *value == "" {
return "", fmt.Errorf("ETCD_%s_REQUIRED", strings.ToUpper(field))
}
return *value, nil
}
+422
View File
@@ -0,0 +1,422 @@
package main
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"os"
"runtime"
"strconv"
"strings"
"sync"
)
const (
protocolVersion = 2
legacyAgentSessionID = "__legacy__"
maxAgentSessions = 256
)
// Capabilities advertise the v2 API surface: no history, no lease objects,
// no compaction, and no defrag because etcd v2 semantics do not have them.
var capabilities = []string{
"connect", "test_connection", "kv", "kv_ttl", "kv_cas", "kv_list_values",
"kv_status", "etcd_watch", "etcd_auth", "multi_session",
}
type request struct {
ID json.RawMessage `json:"id"`
Method string `json:"method"`
Params map[string]json.RawMessage `json:"params"`
}
type response struct {
JSONRPC string `json:"jsonrpc,omitempty"`
ID json.RawMessage `json:"id,omitempty"`
Result any `json:"result,omitempty"`
Error *rpcError `json:"error,omitempty"`
}
// etcd2Session carries the per-connection HTTP client and derived state.
// Requests for one session are serialized by agentSession.mu; watch pump
// goroutines reach the maps through watchesMu instead.
type etcd2Session struct {
clientMu sync.Mutex
httpClient *authenticatedClient
connectedEndpoints []string
serverVersion string
watchesMu sync.Mutex
watches map[string]*watchState
watchBufferedBytes int64
activeCancelMu sync.Mutex
activeCancel context.CancelFunc
}
func (s *etcd2Session) reserveWatchBuffer(bytes int64) bool {
s.watchesMu.Lock()
defer s.watchesMu.Unlock()
if bytes < 0 || s.watchBufferedBytes+bytes > maxSessionWatchBufferBytes {
return false
}
s.watchBufferedBytes += bytes
return true
}
func (s *etcd2Session) releaseWatchBuffer(bytes int64) {
s.watchesMu.Lock()
defer s.watchesMu.Unlock()
s.watchBufferedBytes -= bytes
if s.watchBufferedBytes < 0 {
s.watchBufferedBytes = 0
}
}
type agentSession struct {
state *etcd2Session
mu sync.Mutex
}
func (a *agentSession) cancelActive() {
a.state.activeCancelMu.Lock()
cancel := a.state.activeCancel
a.state.activeCancelMu.Unlock()
if cancel != nil {
cancel()
}
}
type runtimeServer struct {
mu sync.RWMutex
sessions map[string]*agentSession
}
func main() {
configureRuntimeParallelism()
server := newRuntimeServer()
encoder := json.NewEncoder(os.Stdout)
var encoderMu sync.Mutex
var requests sync.WaitGroup
fmt.Fprintln(os.Stdout, `{"ready":true}`)
scanner := bufio.NewScanner(os.Stdin)
scanner.Buffer(make([]byte, 0, 64*1024), 512*1024*1024)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
}
var envelope request
if json.Unmarshal([]byte(line), &envelope) == nil && envelope.Method == "shutdown" {
requests.Wait()
result, _ := server.handleLine(line)
encoderMu.Lock()
_ = encoder.Encode(result)
encoderMu.Unlock()
server.closeAll()
return
}
requests.Add(1)
go func(line string) {
defer requests.Done()
result, _ := server.handleLine(line)
encoderMu.Lock()
defer encoderMu.Unlock()
if err := encoder.Encode(result); err != nil {
fmt.Fprintf(os.Stderr, "failed to write response: %v\n", err)
}
}(line)
}
requests.Wait()
server.closeAll()
}
func configureRuntimeParallelism() {
if raw := strings.TrimSpace(os.Getenv("DBX_AGENT_ETCD2_GOMAXPROCS")); raw != "" {
if configured, err := strconv.Atoi(raw); err == nil && configured > 0 {
runtime.GOMAXPROCS(configured)
return
}
}
if strings.TrimSpace(os.Getenv("GOMAXPROCS")) != "" {
return
}
runtime.GOMAXPROCS(min(runtime.NumCPU(), 4))
}
func newRuntimeServer() *runtimeServer {
return &runtimeServer{sessions: map[string]*agentSession{}}
}
func (r *runtimeServer) handleLine(line string) (response, bool) {
var req request
if err := json.Unmarshal([]byte(line), &req); err != nil {
return errorResponse(nil, "", "", err), false
}
if len(req.ID) == 0 {
req.ID = json.RawMessage("1")
}
result, shutdown, err := r.dispatch(req.Method, req.Params)
if err != nil {
return errorResponse(req.ID, req.Method, stringParam(req.Params, "agentSessionId"), err), false
}
return response{JSONRPC: "2.0", ID: req.ID, Result: result}, shutdown
}
func (r *runtimeServer) dispatch(method string, params map[string]json.RawMessage) (any, bool, error) {
switch method {
case "handshake":
return handshakeResult(), false, nil
case "open_session":
id := requiredSessionID(params)
if id == "" {
return nil, false, errors.New("agentSessionId is required")
}
return r.openSession(id, params)
case "close_session":
return r.closeSession(stringParam(params, "agentSessionId")), false, nil
case "validate_session":
session, err := r.session(requiredSessionID(params))
if err != nil {
return nil, false, err
}
session.mu.Lock()
defer session.mu.Unlock()
result, err := session.state.validateConnection()
return result, false, err
case "cancel_session":
session, err := r.session(requiredSessionID(params))
if err != nil {
return nil, false, err
}
session.cancelActive()
return map[string]bool{"ok": true}, false, nil
case "test_connection":
return r.testConnection(params)
case "connect":
_ = r.closeSession(legacyAgentSessionID)
return r.openSession(legacyAgentSessionID, params)
case "disconnect":
return r.closeSession(legacyAgentSessionID), false, nil
case "shutdown":
return map[string]bool{"ok": true}, true, r.closeAll()
default:
id := stringParam(params, "agentSessionId")
if id == "" {
id = legacyAgentSessionID
}
session, err := r.session(id)
if err != nil {
return nil, false, err
}
session.mu.Lock()
defer session.mu.Unlock()
result, err := session.state.handle(method, params)
return result, false, err
}
}
func requiredSessionID(params map[string]json.RawMessage) string {
return strings.TrimSpace(stringParam(params, "agentSessionId"))
}
func (r *runtimeServer) openSession(id string, params map[string]json.RawMessage) (any, bool, error) {
r.mu.Lock()
if _, exists := r.sessions[id]; exists {
r.mu.Unlock()
return nil, false, fmt.Errorf("Agent session already exists: %s", id)
}
if len(r.sessions) >= maxAgentSessions {
r.mu.Unlock()
return nil, false, fmt.Errorf("Agent session limit reached: %d", maxAgentSessions)
}
session := &agentSession{state: newEtcd2Session()}
r.sessions[id] = session
r.mu.Unlock()
if _, err := session.state.connect(params); err != nil {
r.mu.Lock()
delete(r.sessions, id)
r.mu.Unlock()
session.state.close()
return nil, false, err
}
return map[string]bool{"ok": true}, false, nil
}
func (r *runtimeServer) session(id string) (*agentSession, error) {
r.mu.RLock()
session := r.sessions[id]
r.mu.RUnlock()
if session == nil {
return nil, fmt.Errorf("Agent session not found: %s", id)
}
return session, nil
}
func (r *runtimeServer) closeSession(id string) any {
r.mu.Lock()
session := r.sessions[id]
delete(r.sessions, id)
r.mu.Unlock()
if session != nil {
session.cancelActive()
session.mu.Lock()
session.state.close()
session.mu.Unlock()
}
return map[string]bool{"ok": true}
}
func (r *runtimeServer) closeAll() error {
r.mu.Lock()
sessions := r.sessions
r.sessions = map[string]*agentSession{}
r.mu.Unlock()
var firstErr error
for _, session := range sessions {
session.cancelActive()
if err := session.state.close(); err != nil && firstErr == nil {
firstErr = err
}
}
return firstErr
}
func (r *runtimeServer) testConnection(params map[string]json.RawMessage) (any, bool, error) {
state := newEtcd2Session()
defer state.close()
if _, err := state.connect(params); err != nil {
return nil, false, err
}
return map[string]bool{"ok": true}, false, nil
}
func newEtcd2Session() *etcd2Session {
return &etcd2Session{watches: map[string]*watchState{}}
}
func handshakeResult() map[string]any {
return map[string]any{
"protocolVersion": protocolVersion,
"agentProtocolVersion": protocolVersion,
"capabilities": capabilities,
}
}
func (s *etcd2Session) beginOperation() (context.Context, context.CancelFunc) {
ctx, cancel := context.WithTimeout(context.Background(), operationTimeout)
s.activeCancelMu.Lock()
s.activeCancel = cancel
s.activeCancelMu.Unlock()
return ctx, cancel
}
func (s *etcd2Session) endOperation(cancel context.CancelFunc) {
cancel()
s.activeCancelMu.Lock()
s.activeCancel = nil
s.activeCancelMu.Unlock()
}
func (s *etcd2Session) handle(method string, params map[string]json.RawMessage) (any, error) {
switch method {
case "handshake":
return handshakeResult(), nil
case "connect", "test_connection":
if _, err := s.connect(params); err != nil {
return nil, err
}
return map[string]bool{"ok": true}, nil
case "validate_connection":
return s.validateConnection()
case "kv_list_prefix":
return s.listPrefix(params)
case "kv_get":
return s.get(params)
case "kv_put":
return s.put(params)
case "kv_delete":
return s.delete(params)
case "kv_rename":
return s.rename(params)
case "kv_status":
return s.status(params)
case "etcd_watch_start":
return s.watchStart(params)
case "etcd_watch_poll":
return s.watchPoll(params)
case "etcd_watch_stop":
return s.watchStop(params)
case "etcd_auth_user_list":
return s.authUserList(params)
case "etcd_auth_user_get":
return s.authUserGet(params)
case "etcd_auth_user_add":
return s.authUserAdd(params)
case "etcd_auth_user_delete":
return s.authUserDelete(params)
case "etcd_auth_user_change_password":
return s.authUserChangePassword(params)
case "etcd_auth_user_grant_role":
return s.authUserGrantRevokeRole(params, true)
case "etcd_auth_user_revoke_role":
return s.authUserGrantRevokeRole(params, false)
case "etcd_auth_role_list":
return s.authRoleList(params)
case "etcd_auth_role_get":
return s.authRoleGet(params)
case "etcd_auth_role_add":
return s.authRoleAdd(params)
case "etcd_auth_role_delete":
return s.authRoleDelete(params)
case "etcd_auth_role_grant_permission":
return s.authRolePermission(params, true)
case "etcd_auth_role_revoke_permission":
return s.authRolePermission(params, false)
case "disconnect":
s.close()
return map[string]bool{"ok": true}, nil
case "shutdown":
s.close()
return map[string]bool{"ok": true}, nil
default:
if isEtcd3OnlyMethod(method) {
return nil, fmt.Errorf("ETCD_V2_UNSUPPORTED: %s is not available on the etcd v2 API", method)
}
return nil, fmt.Errorf("unknown method: %s", method)
}
}
func isEtcd3OnlyMethod(method string) bool {
switch method {
case "kv_history", "etcd_compact", "etcd_defrag",
"etcd_lease_list", "etcd_lease_get", "etcd_lease_grant",
"etcd_lease_keepalive_once", "etcd_lease_revoke":
return true
}
return false
}
func decodeParams(params map[string]json.RawMessage, target any) error {
if params == nil {
params = map[string]json.RawMessage{}
}
data, err := json.Marshal(params)
if err != nil {
return err
}
return json.Unmarshal(data, target)
}
func stringParam(params map[string]json.RawMessage, key string) string {
if params == nil {
return ""
}
var result string
_ = json.Unmarshal(params[key], &result)
return result
}
+267
View File
@@ -0,0 +1,267 @@
package main
import (
"encoding/json"
"strings"
"testing"
)
func TestHandshakeAdvertisesV2Capabilities(t *testing.T) {
server := newRuntimeServer()
result, _, err := server.dispatch("handshake", nil)
if err != nil {
t.Fatalf("handshake failed: %v", err)
}
handshake := result.(map[string]any)
want := []string{
"connect", "test_connection", "kv", "kv_ttl", "kv_cas", "kv_list_values",
"kv_status", "etcd_watch", "etcd_auth", "multi_session",
}
got := handshake["capabilities"].([]string)
if strings.Join(got, ",") != strings.Join(want, ",") {
t.Fatalf("capability mismatch:\n got %v\nwant %v", got, want)
}
for _, forbidden := range []string{"kv_history", "etcd_lease", "etcd_compaction", "etcd_defrag", "structured_error_v1"} {
for _, capability := range got {
if capability == forbidden {
t.Fatalf("%s must not be advertised on the v2 agent", forbidden)
}
}
}
}
func TestHandleLineReportsSessionNotFound(t *testing.T) {
server := newRuntimeServer()
response, _ := server.handleLine(`{"id":7,"method":"kv_get","params":{"key":"a"}}`)
if response.Error == nil || response.Error.Code != -1 {
t.Fatalf("expected code -1 error, got %#v", response)
}
if response.Error.Message != "Agent session not found: __legacy__" {
t.Fatalf("unexpected message: %q", response.Error.Message)
}
}
func TestUnconnectedSessionErrors(t *testing.T) {
state := newEtcd2Session()
if _, err := state.get(map[string]json.RawMessage{"key": json.RawMessage(`"k"`)}); err == nil || err.Error() != "Not connected" {
t.Fatalf("kv_get before connect: %v", err)
}
if _, err := state.validateConnection(); err == nil || err.Error() != "Not connected" {
t.Fatalf("validate_connection before connect: %v", err)
}
}
func TestEtcd3OnlyMethodsRejected(t *testing.T) {
state := newEtcd2Session()
for _, method := range []string{"kv_history", "etcd_lease_list", "etcd_lease_grant", "etcd_compact", "etcd_defrag"} {
_, err := state.handle(method, map[string]json.RawMessage{})
if err == nil || !strings.HasPrefix(err.Error(), "ETCD_V2_UNSUPPORTED") {
t.Fatalf("%s should report ETCD_V2_UNSUPPORTED, got %v", method, err)
}
}
_, err := state.handle("etcd_nonsense", map[string]json.RawMessage{})
if err == nil || !strings.Contains(err.Error(), "unknown method") {
t.Fatalf("expected unknown method error, got %v", err)
}
}
func TestConnectionEndpoints(t *testing.T) {
cases := []struct {
name string
connection connectionParams
want []string
}{
{"host port fallback", connectionParams{Port: 2379}, []string{"http://127.0.0.1:2379"}},
{"ssl scheme", connectionParams{Host: "db", Port: 1, SSL: true}, []string{"https://db:1"}},
{"comma list", connectionParams{EtcdEndpoints: "a:1,b:2"}, []string{"http://a:1", "http://b:2"}},
{"existing scheme trimmed", connectionParams{EtcdEndpoints: "https://a:1/"}, []string{"https://a:1"}},
}
for _, testCase := range cases {
got := connectionEndpoints(testCase.connection)
if strings.Join(got, "|") != strings.Join(testCase.want, "|") {
t.Fatalf("%s: got %v, want %v", testCase.name, got, testCase.want)
}
}
}
func TestTLSConfigRequiresCertAndKeyTogether(t *testing.T) {
_, err := tlsConfigFor(connectionParams{SSL: true, ClientCertPath: "/tmp/cert.pem"})
if err == nil || err.Error() != "Client certificate and key must be provided together" {
t.Fatalf("expected paired cert/key error, got %v", err)
}
}
func TestV2KeyPath(t *testing.T) {
cases := []struct{ key, want string }{
{"", "/v2/keys/"},
{"foo", "/v2/keys/foo"},
{"/foo/bar", "/v2/keys/foo/bar"},
}
for _, testCase := range cases {
if got := v2KeyPath(testCase.key); got != testCase.want {
t.Fatalf("v2KeyPath(%q) = %q, want %q", testCase.key, got, testCase.want)
}
}
}
func TestErrorFromResponse(t *testing.T) {
err := errorFromResponse(404, []byte(`{"errorCode":100,"message":"Key not found","cause":"/foo","index":37}`))
if !isNotFound(err) {
t.Fatalf("expected key-not-found classification: %#v", err)
}
if isCompareFailed(err) {
t.Fatalf("errorCode 100 is not compare failure")
}
if err.Error() != "Key not found (/foo)" {
t.Fatalf("unexpected message: %q", err.Error())
}
compare := errorFromResponse(412, []byte(`{"errorCode":101,"message":"Compare failed","cause":"[1 != 2]"}`))
if !isCompareFailed(compare) {
t.Fatalf("expected compare failure classification: %#v", compare)
}
plain := errorFromResponse(500, []byte("boom"))
if plain.Error() != "boom" {
t.Fatalf("plain body passthrough broken: %q", plain.Error())
}
}
func TestV2PermissionRows(t *testing.T) {
rows := v2PermissionRows("read", []string{"/a", "/p/*"})
if len(rows) != 2 {
t.Fatalf("expected 2 permission rows, got %d", len(rows))
}
first := rows[0]
if first["access"] != "read" || first["resource"] != "key" {
t.Fatalf("unexpected exact-key row: %#v", first)
}
second := rows[1]
if second["access"] != "read" || second["resource"] != "prefix" {
t.Fatalf("unexpected prefix row: %#v", second)
}
end := second["rangeEnd"].(map[string]any)
if end["data"] != "L3Aw" { // prefixEnd("/p/") == "/p0"
t.Fatalf("prefix rangeEnd wrong: %#v", end)
}
}
func TestV2PermissionPattern(t *testing.T) {
if got := v2PermissionPattern("key", "/foo"); got != "/foo" {
t.Fatalf("key pattern must stay bare: %q", got)
}
if got := v2PermissionPattern("prefix", "/foo/"); got != "/foo/*" {
t.Fatalf("prefix pattern must become glob: %q", got)
}
if got := v2PermissionPattern("all", "/ignored"); got != "/*" {
t.Fatalf("all pattern must be root glob: %q", got)
}
}
func TestV2RoleGrantDocument(t *testing.T) {
encoded, err := json.Marshal(v2Role{
Role: "r1",
Grant: &v2Permissions{KV: v2RWPermission{Read: []string{"/p/*"}}},
})
if err != nil {
t.Fatal(err)
}
var round v2Role
if err := json.Unmarshal(encoded, &round); err != nil {
t.Fatal(err)
}
if round.Role != "r1" || len(round.Grant.KV.Read) != 1 || round.Grant.KV.Read[0] != "/p/*" {
t.Fatalf("grant document round-trip broken: %s", encoded)
}
if round.Permissions != nil {
t.Fatalf("grant documents must not carry permissions: %s", encoded)
}
}
func TestMemberHexToUnsigned(t *testing.T) {
if got := memberHexToUnsigned("272e204152"); got != "168277590354" {
t.Fatalf("unexpected conversion: %v", got)
}
if got := memberHexToUnsigned(""); got != nil {
t.Fatalf("empty id must map to nil, got %v", got)
}
if got := memberHexToUnsigned("zzz"); got != nil {
t.Fatalf("invalid hex must map to nil, got %v", got)
}
}
func TestWatchBudgetState(t *testing.T) {
session := newEtcd2Session()
state := &watchState{watchID: "w", session: session}
state.append(10, []map[string]any{{"eventType": "put"}}, 128)
state.append(11, []map[string]any{{"eventType": "put"}}, maxWatchBufferBytes+1)
polled := state.poll()
if len(polled["batches"].([]any)) != 1 {
t.Fatalf("overflow must keep buffered batch: %#v", polled)
}
terminal, ok := polled["terminal"].(map[string]any)
if !ok || terminal["reason"] != "overflow" {
t.Fatalf("expected overflow terminal: %#v", polled)
}
if session.watchBufferedBytesSnapshot() != 0 {
t.Fatalf("budget not released: %d", session.watchBufferedBytesSnapshot())
}
}
func TestWatchLimits(t *testing.T) {
session := newEtcd2Session()
for i := 0; i < maxWatches; i++ {
session.registerWatch(string(rune('a'+i)), &watchState{watchID: string(rune('a' + i)), session: session})
}
_, err := session.watchStart(map[string]json.RawMessage{"key": json.RawMessage(`"k"`)})
if err == nil || !strings.HasPrefix(err.Error(), "ETCD_WATCH_LIMIT") {
t.Fatalf("expected watch limit error, got %v", err)
}
fresh := newEtcd2Session()
_, err = fresh.watchStart(map[string]json.RawMessage{"key": json.RawMessage(`"k"`), "scope": json.RawMessage(`"glob"`)})
if err == nil || err.Error() != "ETCD_WATCH_SCOPE_INVALID: scope must be key or prefix" {
t.Fatalf("expected scope error, got %v", err)
}
_, err = fresh.watchPoll(map[string]json.RawMessage{"watchId": json.RawMessage(`"none"`)})
if err == nil || err.Error() != "ETCD_WATCH_NOT_FOUND: watch does not exist" {
t.Fatalf("expected watch-not-found, got %v", err)
}
}
func TestPutRejectsLeaseOptions(t *testing.T) {
state := newEtcd2Session()
// Client check comes first (Java parity); without a connection the error
// stays "Not connected".
params := map[string]json.RawMessage{
"key": json.RawMessage(`"k"`),
"value": json.RawMessage(`{"encoding":"utf8","data":"v"}`),
"ttl": json.RawMessage(`5`),
"lease": json.RawMessage(`7`),
}
if _, err := state.put(params); err == nil || err.Error() != "Not connected" {
t.Fatalf("expected Not connected before option validation, got %v", err)
}
}
func TestPrefixEnd(t *testing.T) {
if prefixEnd("") != "\x00" || prefixEnd("ab") != "ac" || prefixEnd("a\xff") != "b" {
t.Fatalf("prefixEnd algorithm broken")
}
}
func TestValueEncodings(t *testing.T) {
utf8Value := valueObject([]byte("hello"))
if utf8Value["encoding"] != "utf8" || utf8Value["data"] != "hello" {
t.Fatalf("unexpected utf8 value: %#v", utf8Value)
}
if valueObject([]byte{0xff})["encoding"] != "base64" {
t.Fatalf("binary must be base64")
}
if displayBytes([]byte{0xff}) == "\xff" {
t.Fatalf("displayBytes must fall back to base64")
}
}
func (s *etcd2Session) watchBufferedBytesSnapshot() int64 {
s.watchesMu.Lock()
defer s.watchesMu.Unlock()
return s.watchBufferedBytes
}
+135
View File
@@ -0,0 +1,135 @@
package main
import (
"context"
"encoding/json"
"errors"
"io"
"net"
"strings"
)
type rpcError struct {
Code int `json:"code"`
Message string `json:"message"`
Data *rpcErrorData `json:"data,omitempty"`
}
type rpcErrorData struct {
Category string `json:"category"`
Retryable bool `json:"retryable"`
SessionDisposition string `json:"sessionDisposition"`
Stage string `json:"stage"`
ContractVersion int `json:"contractVersion"`
OperationOutcome string `json:"operationOutcome"`
ExceptionClass string `json:"exceptionClass,omitempty"`
AgentSessionID string `json:"agentSessionId,omitempty"`
}
func errorResponse(id json.RawMessage, method, agentSessionID string, err error) response {
return response{JSONRPC: "2.0", ID: id, Error: classifyRPCError(method, agentSessionID, err)}
}
func classifyRPCError(method, agentSessionID string, err error) *rpcError {
stage := rpcErrorStage(method)
data := &rpcErrorData{
Category: "protocol",
Retryable: false,
SessionDisposition: "keep",
Stage: stage,
ContractVersion: 1,
OperationOutcome: rpcOperationOutcome(stage),
ExceptionClass: safeRPCDiagnostic(errTypeName(err), 160),
AgentSessionID: strings.TrimSpace(agentSessionID),
}
switch {
case errors.Is(err, context.Canceled):
data.Category = "canceled"
data.SessionDisposition = "quarantine"
case errors.Is(err, context.DeadlineExceeded) || isTimeoutError(err):
data.Category = "timeout"
data.SessionDisposition = "quarantine"
case isConnectionError(err):
data.Category = "connection"
data.Retryable = stage == "connect" || stage == "validate"
if stage != "connect" {
data.SessionDisposition = "quarantine"
}
}
return &rpcError{Code: -1, Message: err.Error(), Data: data}
}
func isConnectionError(err error) bool {
if errors.Is(err, io.EOF) || errors.Is(err, net.ErrClosed) {
return true
}
var networkError *net.OpError
if errors.As(err, &networkError) {
return true
}
lower := strings.ToLower(err.Error())
for _, marker := range []string{
"connection refused", "connection reset", "broken pipe", "connection closed",
"connection lost", "unexpected eof", "no route to host", "etcd connection error",
} {
if strings.Contains(lower, marker) {
return true
}
}
return false
}
func errTypeName(err error) string {
if err == nil {
return ""
}
return strings.ReplaceAll(strings.TrimSpace(err.Error()), "\n", " ")
}
func rpcErrorStage(method string) string {
switch method {
case "connect", "open_session", "test_connection":
return "connect"
case "validate_connection", "validate_session":
return "validate"
case "cancel_session":
return "cancel"
case "close_session", "disconnect", "shutdown":
return "close"
case "etcd_watch_poll":
return "fetch"
case "handshake", "":
return "request"
default:
return "execute"
}
}
func rpcOperationOutcome(stage string) string {
switch stage {
case "request", "connect", "validate":
return "not_started"
default:
return "unknown"
}
}
func isTimeoutError(err error) bool {
var timeout interface{ Timeout() bool }
return errors.As(err, &timeout) && timeout.Timeout()
}
func safeRPCDiagnostic(value string, maxLength int) string {
var result strings.Builder
for _, char := range value {
if result.Len() >= maxLength {
break
}
if char >= 0x21 && char <= 0x7e {
result.WriteRune(char)
}
}
return result.String()
}
+179
View File
@@ -0,0 +1,179 @@
package main
import (
"encoding/json"
"net/http"
"strconv"
"strings"
"time"
)
// status assembles a reduced kv_status document from the v2 endpoints:
// /v2/members, /v2/stats/self, /v2/stats/leader, and /v2/stats/store. Fields
// without a v2 analog (alarms, dbSize, raft detail) stay null.
func (s *etcd2Session) status(params map[string]json.RawMessage) (any, error) {
client, err := s.activeClient()
if err != nil {
return nil, err
}
ctx, cancel := s.beginOperation()
defer s.endOperation(cancel)
type member struct {
ID string `json:"id"`
Name string `json:"name"`
PeerURLs []string `json:"peerURLs"`
ClientURLs []string `json:"clientURLs"`
}
membersByID := map[string]*member{}
var memberList []member
var clusterID *string
if body, response, err := client.do(ctx, http.MethodGet, "/v2/members", "", nil); err == nil {
var parsed struct {
Members []member `json:"members"`
}
if json.Unmarshal(body, &parsed) == nil {
memberList = parsed.Members
for index := range memberList {
membersByID[memberList[index].ID] = &memberList[index]
}
}
if header := response.Header.Get("X-Etcd-Cluster-Id"); header != "" {
clusterID = &header
}
}
// The v2 API exposes the current index only as a response header on the
// keys endpoints, not in any stats document.
var revision *string
if _, response, err := client.do(ctx, http.MethodGet, "/v2/keys/", "", nil); err == nil {
if header := response.Header.Get("X-Etcd-Index"); header != "" {
revision = &header
}
}
type selfStats struct {
Name string `json:"name"`
ID string `json:"id"`
State string `json:"state"`
LeaderInfo struct {
Leader string `json:"leader"`
} `json:"leaderInfo"`
RecvAppendRequestCnt int64 `json:"recvAppendRequestCnt"`
SendAppendRequestCnt int64 `json:"sendAppendRequestCnt"`
}
var leaderID *string
selfByEndpoint := map[string]*selfStats{}
endpoints := s.connectedEndpointList()
for _, endpoint := range endpoints {
self := &selfStats{}
body, _, err := client.doAt(ctx, http.MethodGet, endpoint, "/v2/stats/self", "")
if err != nil {
continue
}
if json.Unmarshal(body, self) != nil {
continue
}
selfByEndpoint[endpoint] = self
if self.LeaderInfo.Leader != "" {
value := self.LeaderInfo.Leader
leaderID = &value
}
}
storeStats := map[string]any{}
if body, _, err := client.do(ctx, http.MethodGet, "/v2/stats/store", "", nil); err == nil {
var parsed map[string]any
if json.Unmarshal(body, &parsed) == nil {
storeStats = parsed
}
}
statusMembers := []map[string]any{}
observedIDs := map[string]struct{}{}
for _, endpoint := range endpoints {
row := map[string]any{"endpoint": endpoint}
started := time.Now()
self := selfByEndpoint[endpoint]
if self == nil {
row["memberId"] = nil
row["name"] = nil
row["version"] = nil
row["leaderId"] = nil
row["revision"] = nil
row["raftTerm"] = nil
row["raftIndex"] = nil
row["raftAppliedIndex"] = nil
row["dbSize"] = nil
row["dbSizeInUse"] = nil
row["learner"] = false
row["reachable"] = false
row["latencyMs"] = nil
row["errors"] = []string{"stats request failed"}
statusMembers = append(statusMembers, row)
continue
}
member := membersByID[self.ID]
row["memberId"] = memberHexToUnsigned(self.ID)
if member == nil {
row["name"] = self.Name
} else {
row["name"] = member.Name
}
row["version"] = client.serverVersion
row["leaderId"] = memberHexToUnsigned(self.LeaderInfo.Leader)
row["revision"] = nil
row["raftTerm"] = nil
row["raftIndex"] = nil
row["raftAppliedIndex"] = nil
row["dbSize"] = nil
row["dbSizeInUse"] = nil
row["learner"] = false
row["reachable"] = true
row["latencyMs"] = time.Since(started).Milliseconds()
row["errors"] = []string{}
row["state"] = strings.TrimPrefix(self.State, "State")
if _, seen := observedIDs[self.ID]; seen {
continue
}
observedIDs[self.ID] = struct{}{}
statusMembers = append(statusMembers, row)
}
var clusterIDValue any
if clusterID != nil {
clusterIDValue = memberHexToUnsigned(*clusterID)
}
var leaderIDValue any
if leaderID != nil {
leaderIDValue = memberHexToUnsigned(*leaderID)
}
var revisionValue any
if revision != nil {
revisionValue = *revision
}
// The v2 API has no key-count equivalent of the v3 count-only range;
// the raw store counters are exposed separately below.
var keyCountValue any
return map[string]any{
"clusterId": clusterIDValue,
"revision": revisionValue,
"leaderId": leaderIDValue,
"keyCount": keyCountValue,
"alarms": []string{},
"members": statusMembers,
"store": storeStats,
}, nil
}
// memberHexToUnsigned converts v2 hex member/cluster IDs (e.g. "272e204152")
// to the unsigned decimal strings the v3 agent emits.
func memberHexToUnsigned(hex string) any {
if hex == "" {
return nil
}
parsed, err := strconv.ParseUint(strings.TrimPrefix(hex, "0x"), 16, 64)
if err != nil {
return nil
}
return unsignedLongString(int64(parsed))
}
+395
View File
@@ -0,0 +1,395 @@
package main
import (
"context"
"crypto/rand"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"sync"
"time"
)
const (
maxWatches = 4
maxWatchBatches = 256
maxWatchEvents = 10000
maxWatchBufferBytes = int64(8 * 1024 * 1024)
maxSessionWatchBufferBytes = int64(16 * 1024 * 1024)
// v2 long polls hold until an event arrives; re-issue the request when a
// poll is cut off so the pump survives idle periods.
watchPollTimeout = 5 * time.Minute
)
type bufferedWatchBatch struct {
payload map[string]any
bufferedBytes int64
}
// watchState buffers watch events for the polling host, mirroring the v3
// agent's budget system so host-side expectations stay identical.
type watchState struct {
watchID string
session *etcd2Session
mu sync.Mutex
batches []bufferedWatchBatch
eventCount int
bufferedBytes int64
terminalReason string
terminalMessage string
pumpCancel context.CancelFunc
closed bool
}
func (w *watchState) append(revision int64, events []map[string]any, batchBytes int64) {
w.mu.Lock()
defer w.mu.Unlock()
if w.terminalReason != "" {
return
}
if int64(len(w.batches)) >= maxWatchBatches ||
int64(w.eventCount+len(events)) > maxWatchEvents ||
batchBytes > maxWatchBufferBytes ||
w.bufferedBytes+batchBytes > maxWatchBufferBytes ||
!w.session.reserveWatchBuffer(batchBytes) {
w.overflowLocked()
return
}
w.batches = append(w.batches, bufferedWatchBatch{
payload: map[string]any{"revision": longString(revision), "events": events},
bufferedBytes: batchBytes,
})
w.eventCount += len(events)
w.bufferedBytes += batchBytes
}
func (w *watchState) overflowLocked() {
if w.terminalReason != "" {
return
}
w.terminalReason = "overflow"
w.terminalMessage = "ETCD_WATCH_OVERFLOW: the event buffer reached its byte or event limit"
w.stopPumpLocked()
}
func (w *watchState) fail(reason, message string) {
w.mu.Lock()
defer w.mu.Unlock()
if w.terminalReason == "" {
w.terminalReason = reason
w.terminalMessage = message
}
}
func (w *watchState) poll() map[string]any {
w.mu.Lock()
defer w.mu.Unlock()
page := make([]any, 0, 64)
for len(w.batches) > 0 && len(page) < 64 {
batch := w.batches[0]
w.batches = w.batches[1:]
events := batch.payload["events"].([]map[string]any)
w.eventCount -= len(events)
w.bufferedBytes -= batch.bufferedBytes
w.session.releaseWatchBuffer(batch.bufferedBytes)
page = append(page, batch.payload)
}
result := map[string]any{"watchId": w.watchID, "batches": page}
if w.terminalReason != "" && len(w.batches) == 0 {
result["terminal"] = map[string]any{
"reason": w.terminalReason,
"message": w.terminalMessage,
}
}
return result
}
func (w *watchState) close() {
w.mu.Lock()
w.failLocked("stopped", "watch stopped")
w.clearBufferedLocked()
w.closed = true
cancel := w.pumpCancel
w.pumpCancel = nil
w.mu.Unlock()
if cancel != nil {
cancel()
}
}
func (w *watchState) failLocked(reason, message string) {
if w.terminalReason == "" {
w.terminalReason = reason
w.terminalMessage = message
}
}
func (w *watchState) clearBufferedLocked() {
if w.bufferedBytes > 0 {
w.session.releaseWatchBuffer(w.bufferedBytes)
}
w.batches = nil
w.eventCount = 0
w.bufferedBytes = 0
}
func (w *watchState) attachPump(cancel context.CancelFunc) {
w.mu.Lock()
defer w.mu.Unlock()
if w.closed || w.terminalReason != "" {
cancel()
return
}
w.pumpCancel = cancel
}
func (w *watchState) stopPumpLocked() {
cancel := w.pumpCancel
w.pumpCancel = nil
if cancel != nil {
go cancel()
}
}
func watchEventBufferBytes(key, value, previousValue string) int64 {
bytes := int64(512 + 4*len(key))
bytes += estimatedBufferedBytes(len(value))
if previousValue != "" {
bytes += estimatedBufferedBytes(len(previousValue))
}
return bytes
}
func estimatedBufferedBytes(sourceBytes int) int64 {
if sourceBytes < 0 {
return 0
}
return int64(sourceBytes) * 4
}
func newWatchID() string {
var bytes [16]byte
if _, err := rand.Read(bytes[:]); err != nil {
return base64.StdEncoding.EncodeToString(bytes[:])
}
bytes[6] = (bytes[6] & 0x0f) | 0x40
bytes[8] = (bytes[8] & 0x3f) | 0x80
hexed := hex.EncodeToString(bytes[:])
return fmt.Sprintf("%s-%s-%s-%s-%s", hexed[0:8], hexed[8:12], hexed[12:16], hexed[16:20], hexed[20:32])
}
func (s *etcd2Session) watchCount() int {
s.watchesMu.Lock()
defer s.watchesMu.Unlock()
return len(s.watches)
}
func (s *etcd2Session) registerWatch(id string, state *watchState) bool {
s.watchesMu.Lock()
defer s.watchesMu.Unlock()
if _, exists := s.watches[id]; exists {
return false
}
s.watches[id] = state
return true
}
func (s *etcd2Session) removeWatch(id string) *watchState {
s.watchesMu.Lock()
defer s.watchesMu.Unlock()
state := s.watches[id]
delete(s.watches, id)
return state
}
func (s *etcd2Session) watchStart(params map[string]json.RawMessage) (any, error) {
if s.watchCount() >= maxWatches {
return nil, fmt.Errorf("ETCD_WATCH_LIMIT: at most %d watches are allowed per connection", maxWatches)
}
key, err := keyBytesParam(params)
if err != nil {
return nil, err
}
scope := stringOrDefault(params, "scope", "key")
if scope != "key" && scope != "prefix" {
return nil, errors.New("ETCD_WATCH_SCOPE_INVALID: scope must be key or prefix")
}
client, err := s.activeClient()
if err != nil {
return nil, err
}
// startRevision maps to the v2 waitIndex: default to the current index+1.
requestedRevision := longOrNull(params, "startRevision")
var waitIndex int64
if requestedRevision != nil && *requestedRevision > 0 {
waitIndex = *requestedRevision
} else {
current, err := client.currentEtcdIndex()
if err != nil {
return nil, err
}
waitIndex = current + 1
}
watchID := newWatchID()
state := &watchState{watchID: watchID, session: s}
pumpCtx, pumpCancel := context.WithCancel(context.Background())
go pumpLongPollWatch(pumpCtx, client, key, scope, waitIndex, boolOrDefault(params, "includePrevKv", false), state)
state.attachPump(pumpCancel)
s.registerWatch(watchID, state)
return map[string]any{"watchId": watchID, "startedRevision": longString(waitIndex)}, nil
}
// currentEtcdIndex reads the X-Etcd-Index header from a lightweight GET.
func (c *authenticatedClient) currentEtcdIndex() (int64, error) {
ctx, cancel := context.WithTimeout(context.Background(), operationTimeout)
defer cancel()
response, err := c.request(ctx, http.MethodGet, "/v2/keys/", "", nil)
if err != nil {
return 0, err
}
defer drainClose(response.Body)
if response.StatusCode < 200 || response.StatusCode >= 300 {
body, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
return 0, errorFromResponse(response.StatusCode, body)
}
index, err := strconv.ParseInt(response.Header.Get("X-Etcd-Index"), 10, 64)
if err != nil {
return 0, fmt.Errorf("etcd v2 server did not report X-Etcd-Index")
}
return index, nil
}
// pumpLongPollWatch drives the v2 wait loop: each response becomes one batch;
// timeouts and connection blips re-issue the same waitIndex.
func pumpLongPollWatch(ctx context.Context, client *authenticatedClient, key, scope string, waitIndex int64, includePrevKv bool, state *watchState) {
for {
if ctx.Err() != nil || state.hasTerminal() {
return
}
query := url.Values{}
query.Set("wait", "true")
query.Set("waitIndex", strconv.FormatInt(waitIndex, 10))
if scope == "prefix" {
query.Set("recursive", "true")
}
pollCtx, pollCancel := context.WithTimeout(ctx, watchPollTimeout)
body, _, err := client.do(pollCtx, http.MethodGet, v2KeyPath(key)+"?"+query.Encode(), "", nil)
pollCancel()
if err != nil {
if errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil {
// Idle long poll cut off: keep waiting from the same index.
continue
}
if ctx.Err() != nil {
return
}
var etcdErr *etcdError
if errors.As(err, &etcdErr) && etcdErr.errorCode == 401 {
state.fail("error", "ETCD_WATCH_EXPIRED: the watched index was cleared from the etcd v2 event history")
return
}
state.fail("error", "watch failed: "+err.Error())
return
}
var parsed v2KeysResponse
if jsonErr := json.Unmarshal(body, &parsed); jsonErr != nil {
state.fail("error", "watch response was not valid etcd v2 JSON: "+jsonErr.Error())
return
}
if parsed.Node == nil {
waitIndex++
continue
}
events := []map[string]any{}
bufferedBytes := int64(128)
node := parsed.Node
previous := parsed.PrevNode
deleted := parsed.Action == "delete" || parsed.Action == "expire"
previousValue := ""
if previous != nil {
previousValue = previous.Value
}
bufferedBytes += watchEventBufferBytes(node.Key, node.Value, previousValue)
if bufferedBytes > maxWatchBufferBytes {
state.mu.Lock()
state.overflowLocked()
state.mu.Unlock()
return
}
row := map[string]any{
"eventType": eventTypeForAction(parsed.Action),
"revision": longString(node.ModifiedIndex),
"key": node.Key,
"keyBytes": bytesObject([]byte(node.Key)),
}
if deleted {
row["value"] = nil
} else {
row["value"] = valueObject([]byte(node.Value))
}
if includePrevKv && previous != nil && previous.Value != "" {
row["previousValue"] = valueObject([]byte(previous.Value))
} else {
row["previousValue"] = nil
}
if deleted && previous != nil {
row["metadata"] = nodeMetadata(previous)
} else {
row["metadata"] = nodeMetadata(node)
}
events = append(events, row)
state.append(node.ModifiedIndex, events, bufferedBytes)
nextIndex := node.ModifiedIndex + 1
if nextIndex > waitIndex {
waitIndex = nextIndex
} else {
waitIndex++
}
}
}
func eventTypeForAction(action string) string {
if action == "delete" || action == "expire" || action == "compareAndDelete" {
return "delete"
}
return "put"
}
func (w *watchState) hasTerminal() bool {
w.mu.Lock()
defer w.mu.Unlock()
return w.terminalReason != ""
}
func (s *etcd2Session) watchPoll(params map[string]json.RawMessage) (any, error) {
watchID := stringOrDefault(params, "watchId", "")
s.watchesMu.Lock()
state := s.watches[watchID]
s.watchesMu.Unlock()
if state == nil {
return nil, errors.New("ETCD_WATCH_NOT_FOUND: watch does not exist")
}
result := state.poll()
if _, terminal := result["terminal"]; terminal {
if removed := s.removeWatch(watchID); removed != nil {
removed.close()
}
}
return result, nil
}
func (s *etcd2Session) watchStop(params map[string]json.RawMessage) (any, error) {
if state := s.removeWatch(stringOrDefault(params, "watchId", "")); state != nil {
state.close()
}
return map[string]bool{"stopped": true}, nil
}
+4 -1
View File
@@ -170,7 +170,10 @@ func parseConnectionConfig(params connectParams) (connectionConfig, error) {
CanonicalHostname: true,
},
}
if strings.EqualFold(params.DatabaseType, "impala") {
if config.DatabaseType == "argo" {
config.Auth = "NOSASL"
}
if config.DatabaseType == "impala" {
config.Auth = "NOSASL"
config.Kerberos.Service = defaultImpalaService
}
+44
View File
@@ -296,6 +296,50 @@ func TestImpalaDefaultsToNoSASL(t *testing.T) {
}
}
func TestArgoDefaultsToNoSASL(t *testing.T) {
config, err := parseConnectionConfig(connectParams{
DatabaseType: "argo",
Host: "argo.example.com",
Port: 10000,
Database: "analytics",
})
if err != nil {
t.Fatal(err)
}
if config.Auth != "NOSASL" {
t.Fatalf("unexpected ArgoDB auth mode: %q", config.Auth)
}
}
func TestArgoExplicitAuthenticationOverridesDefault(t *testing.T) {
config, err := parseConnectionConfig(connectParams{
DatabaseType: "argo",
Host: "argo.example.com",
Port: 10000,
URLParams: "auth=NONE",
})
if err != nil {
t.Fatal(err)
}
if config.Auth != "NONE" || !config.AuthExplicit {
t.Fatalf("explicit ArgoDB authentication was not preserved: %#v", config)
}
}
func TestHiveDefaultAuthenticationRemainsSASL(t *testing.T) {
config, err := parseConnectionConfig(connectParams{
DatabaseType: "hive",
Host: "hive.example.com",
Port: 10000,
})
if err != nil {
t.Fatal(err)
}
if config.Auth != "NONE" {
t.Fatalf("unexpected Hive auth mode: %q", config.Auth)
}
}
func TestImpalaExplicitAuthenticationOverridesDefaults(t *testing.T) {
config, err := parseConnectionConfig(connectParams{
DatabaseType: "impala",
+6 -1
View File
@@ -407,7 +407,11 @@ func (server *server) dispatch(method string, params map[string]json.RawMessage)
result, err := server.getTableComment(stringParam(params, "schema"), stringParam(params, "table"))
return result, false, err
case "list_objects":
result, err := server.listObjects(stringParam(params, "schema"), metadataListConstraintsFromParams(params))
result, err := server.listObjects(
stringParam(params, "database"),
stringParam(params, "schema"),
metadataListConstraintsFromParams(params),
)
return result, false, err
case "list_data_types":
result, err := server.listDataTypes()
@@ -432,6 +436,7 @@ func (server *server) dispatch(method string, params map[string]json.RawMessage)
return []any{}, false, nil
case "get_object_source":
result, err := server.getObjectSource(
stringParam(params, "database"),
stringParam(params, "schema"),
firstNonEmpty(stringParam(params, "name"), stringParam(params, "table")),
stringParam(params, "object_type"),
+221 -12
View File
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"log"
"sort"
"strconv"
"strings"
@@ -437,20 +438,169 @@ func showViewsUnsupported(err error) bool {
return false
}
func (server *server) listObjects(schema string, constraints metadataListConstraints) ([]objectInfo, error) {
if !acceptsHiveTable(constraints.ObjectTypes) {
func (server *server) listObjects(database, schema string, constraints metadataListConstraints) ([]objectInfo, error) {
if !acceptsHiveTable(constraints.ObjectTypes) && !(server.supportsRoutines() && acceptsHiveRoutine(constraints.ObjectTypes)) {
return []objectInfo{}, nil
}
tables, err := server.listTables(schema, constraints)
if err != nil {
return nil, err
}
schema = firstNonEmpty(schema, server.config.Database)
values := make([]objectInfo, 0, len(tables))
for _, table := range tables {
values = append(values, objectInfo{Name: table.Name, ObjectType: table.TableType, Schema: schema, Comment: table.Comment})
values := make([]objectInfo, 0)
if acceptsHiveTable(constraints.ObjectTypes) {
tables, err := server.listTables(schema, constraints)
if err != nil {
return nil, err
}
for _, table := range tables {
values = append(values, objectInfo{Name: table.Name, ObjectType: table.TableType, Schema: schema, Comment: table.Comment})
}
}
return values, nil
if server.supportsRoutines() && acceptsRoutineType(constraints.ObjectTypes, "PROCEDURE") {
procedures, err := server.listRoutines(database, schema, constraints, "PROCEDURE")
if err != nil {
return nil, err
}
values = append(values, procedures...)
}
if server.supportsRoutines() && acceptsRoutineType(constraints.ObjectTypes, "FUNCTION") {
functions, err := server.listRoutines(database, schema, constraints, "FUNCTION")
if err != nil {
return nil, err
}
values = append(values, functions...)
}
sort.Slice(values, func(first, second int) bool { return values[first].Name < values[second].Name })
return applyMetadataWindow(values, constraints.Offset, constraints.Limit), nil
}
// supportsRoutines reports whether the connected server is known to expose the
// Hive procedure / function catalog views (system.procedures_v /
// system.functions_v). ArgoDB / Inceptor / Transwarp forks ship these views;
// vanilla Apache Hive does not, so routine listing is gated on the connection's
// database_type to avoid firing unsupported queries against plain Hive.
func (server *server) supportsRoutines() bool {
switch strings.ToLower(strings.TrimSpace(server.params.DatabaseType)) {
case "argo", "inceptor", "inceptor2", "transwarp":
return true
default:
return false
}
}
// routineDatabaseCandidates returns the database_name filters to try against
// system.procedures_v / system.functions_v. The sidebar may pass the active
// database through either the schema or database RPC parameter, so mirror the
// JDBC plugin and try both. The connection default is only a candidate when
// the caller supplied neither parameter: appending it unconditionally would
// return the default database's routines under an explicit database node that
// has none of its own.
func routineDatabaseCandidates(schema, database, connectionDatabase string) []string {
seen := map[string]bool{}
values := make([]string, 0, 3)
add := func(candidate string) {
candidate = strings.TrimSpace(candidate)
if candidate == "" {
return
}
key := strings.ToLower(candidate)
if seen[key] {
return
}
seen[key] = true
values = append(values, candidate)
}
add(database)
add(schema)
if strings.TrimSpace(database) == "" && strings.TrimSpace(schema) == "" {
add(connectionDatabase)
}
return values
}
// listRoutines queries the server's procedure / function catalog views when the
// driver supports them. Hive and most forks (ArgoDB, Inceptor, Transwarp) expose
// stored procedures / functions through the system.procedures_v / system.functions_v
// views, with columns (procedure_name | function_name, database_name, full_text, ...).
// The full_text column carries the routine source used by getObjectSource.
//
// The query is best-effort: when the view is missing or the server rejects it
// (older Hive without procedure support), the call returns an empty slice and
// nil error so the caller can fall back to listing tables.
func (server *server) listRoutines(database, schema string, constraints metadataListConstraints, routineType string) ([]objectInfo, error) {
nameColumn := "procedure_name"
viewName := "system.procedures_v"
if strings.EqualFold(routineType, "FUNCTION") {
nameColumn = "function_name"
viewName = "system.functions_v"
}
candidates := routineDatabaseCandidates(schema, database, server.config.Database)
if len(candidates) == 0 {
return []objectInfo{}, nil
}
likePattern := buildRoutineLikePattern(constraints.Filter)
resolvedSchema := firstNonEmpty(schema, database, server.config.Database)
for _, targetSchema := range candidates {
// database_name is a string column, so the schema filter must be a single-quoted
// literal — not a backtick-quoted identifier. ArgoDB/Inceptor reject lower(`ods`)
// against system.procedures_v with an ERROR_STATUS, while lower('ods') works.
schemaLiteral := "'" + strings.ReplaceAll(targetSchema, "'", "''") + "'"
sql := "SELECT " + nameColumn + " FROM " + viewName +
" WHERE lower(database_name) = lower(" + schemaLiteral + ")" +
" AND lower(" + nameColumn + ") LIKE " + likePattern +
" ORDER BY " + nameColumn
result, err := server.executeQuery(queryOptions{SQL: sql, MaxRows: metadataQueryLimit})
if err != nil {
log.Printf(
"[hive-go][listRoutines] query failed: database=%q schema=%q routineType=%s sql=%q err=%v",
targetSchema,
resolvedSchema,
routineType,
sql,
err,
)
continue
}
values := make([]objectInfo, 0, len(result.Rows))
for _, row := range result.Rows {
name := rowString(row, 0)
if name == "" {
continue
}
values = append(values, objectInfo{Name: name, ObjectType: strings.ToUpper(routineType), Schema: resolvedSchema, Comment: nil})
}
if len(values) > 0 {
return values, nil
}
}
return []objectInfo{}, nil
}
// buildRoutineLikePattern turns a user-supplied filter into a Hive-safe LIKE
// literal: wraps with %, escapes \, %, _ (the LIKE metacharacters), and quotes
// the whole literal so it can be concatenated directly into SQL.
func buildRoutineLikePattern(filter string) string {
escaped := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(filter)
return "'%" + escaped + "%'"
}
// acceptsHiveRoutine reports whether the requested object types include any
// routine (PROCEDURE / FUNCTION) that hive-go needs to surface through listObjects.
func acceptsHiveRoutine(objectTypes []string) bool {
for _, objectType := range objectTypes {
if strings.EqualFold(objectType, "PROCEDURE") || strings.EqualFold(objectType, "FUNCTION") {
return true
}
}
return false
}
// acceptsRoutineType reports whether the requested object types include the
// given routine kind (case-insensitive).
func acceptsRoutineType(objectTypes []string, routineType string) bool {
for _, objectType := range objectTypes {
if strings.EqualFold(objectType, routineType) {
return true
}
}
return false
}
func (server *server) getColumns(schema, table string) ([]columnInfo, error) {
@@ -584,8 +734,19 @@ func (server *server) getTableDDL(schema, table string) (string, error) {
return strings.Join(lines, "\n") + "\n", nil
}
func (server *server) getObjectSource(schema, name, objectType string) (objectSource, error) {
source, err := server.getTableDDL(schema, name)
func (server *server) getObjectSource(database, schema, name, objectType string) (objectSource, error) {
schema = firstNonEmpty(schema, database, server.config.Database)
var source string
var err error
switch strings.ToUpper(objectType) {
case "PROCEDURE", "FUNCTION":
if !server.supportsRoutines() {
return objectSource{}, fmt.Errorf("routine source is not supported for %s connections", server.params.DatabaseType)
}
source, err = server.getRoutineSource(database, schema, name, strings.ToUpper(objectType))
default:
source, err = server.getTableDDL(schema, name)
}
if err != nil {
return objectSource{}, err
}
@@ -597,6 +758,54 @@ func (server *server) getObjectSource(schema, name, objectType string) (objectSo
}, nil
}
// getRoutineSource fetches a procedure or function's full source from the
// server's system.procedures_v / system.functions_v view. Returns an empty
// string when the view is missing or the routine is not found, so callers can
// fall back to other sources. full_text may span multiple rows (the underlying
// query driver splits long strings), so we join them like getTableDDL does.
func (server *server) getRoutineSource(database, schema, name, routineType string) (string, error) {
nameColumn := "procedure_name"
viewName := "system.procedures_v"
if strings.EqualFold(routineType, "FUNCTION") {
nameColumn = "function_name"
viewName = "system.functions_v"
}
candidates := routineDatabaseCandidates(schema, database, server.config.Database)
if len(candidates) == 0 {
return "", nil
}
escapedName := strings.ReplaceAll(name, "'", "''")
for _, targetSchema := range candidates {
sql := "SELECT full_text FROM " + viewName +
" WHERE lower(database_name) = lower('" + strings.ReplaceAll(targetSchema, "'", "''") + "')" +
" AND " + nameColumn + " = '" + escapedName + "'"
result, err := server.executeQuery(queryOptions{SQL: sql, MaxRows: metadataQueryLimit})
if err != nil {
log.Printf(
"[hive-go][getRoutineSource] query failed: database=%q schema=%q name=%q routineType=%s sql=%q err=%v",
targetSchema,
schema,
name,
routineType,
sql,
err,
)
continue
}
lines := make([]string, 0, len(result.Rows))
for _, row := range result.Rows {
if line := firstRowValue(row); line != "" {
lines = append(lines, line)
}
}
if len(lines) == 0 {
continue
}
return strings.Join(lines, "\n") + "\n", nil
}
return "", nil
}
func (server *server) getExplainInfo(sqlText string) (string, error) {
sqlText = trimStatementSQL(sqlText)
if sqlText == "" {
+255
View File
@@ -479,3 +479,258 @@ func TestListViewsReturnsFallbackErrorWhenShowViewsIsUnsupported(t *testing.T) {
func metadataResult(columns []string, rows ...[]driver.Value) gohive.MetadataResult {
return gohive.MetadataResult{Columns: columns, Rows: rows}
}
func TestListObjectsIncludesProceduresAndFunctionsFromSystemViews(t *testing.T) {
proceduresQuery := "SELECT procedure_name FROM system.procedures_v WHERE lower(database_name) = lower('ods') AND lower(procedure_name) LIKE '%sp%' ORDER BY procedure_name"
functionsQuery := "SELECT function_name FROM system.functions_v WHERE lower(database_name) = lower('ods') AND lower(function_name) LIKE '%sp%' ORDER BY function_name"
behavior := &scriptedBehavior{
query: func(_ context.Context, query string) (driver.Rows, error) {
switch query {
case proceduresQuery:
return newScriptedRows(context.Background(), []string{"procedure_name"}, []string{"STRING"}, [][]driver.Value{
{"sp_daily_etl"}, {"sp_hourly_agg"},
}), nil
case functionsQuery:
return newScriptedRows(context.Background(), []string{"function_name"}, []string{"STRING"}, [][]driver.Value{
{"fn_clean"},
}), nil
default:
return nil, errors.New("unexpected query: " + query)
}
},
}
server := newScriptedServer(t, behavior)
server.params.DatabaseType = "argo"
defer server.disconnect()
values, err := server.listObjects("ods", "ods", metadataListConstraints{
ObjectTypes: []string{"PROCEDURE", "FUNCTION"},
Filter: "sp", // "sp" overlaps the procedure filter; both lists get "%sp%" applied
})
if err != nil {
t.Fatalf("listObjects: %v", err)
}
t.Logf("values: %+v", values)
// listObjects applies the same filter pattern to procedures and functions;
// we passed "sp" so procedures match and functions don't (because the function
// list returns rows regardless of filter — that is, listObjects calls each
// listRoutines call with the same Filter). Verify both queries ran and
// the procedure name is present.
queries, _, _, _ := behavior.snapshot()
t.Logf("queries count: %d", len(queries))
for i, q := range queries {
t.Logf("query[%d]: %q", i, q)
}
if len(queries) != 2 {
t.Fatalf("expected exactly 2 routine queries, got %d: %v", len(queries), queries)
}
if values == nil {
t.Fatal("expected non-nil object list")
}
names := make([]string, len(values))
for i, v := range values {
names[i] = v.Name + ":" + v.ObjectType
}
joined := strings.Join(names, ",")
if !strings.Contains(joined, "sp_daily_etl:PROCEDURE") {
t.Fatalf("expected procedures in result, got %v", values)
}
if !strings.Contains(joined, "fn_clean:FUNCTION") {
t.Fatalf("expected functions in result, got %v", values)
}
}
func TestGetObjectSourceRoutesProceduresToSystemProceduresView(t *testing.T) {
expectedSQL := "SELECT full_text FROM system.procedures_v WHERE lower(database_name) = lower('ods') AND procedure_name = 'sp_daily_etl'"
behavior := &scriptedBehavior{
query: func(_ context.Context, query string) (driver.Rows, error) {
if query != expectedSQL {
return nil, errors.New("unexpected query: " + query)
}
return newScriptedRows(context.Background(), []string{"full_text"}, []string{"STRING"}, [][]driver.Value{
{"-- daily ETL pipeline"},
{"INSERT OVERWRITE TABLE ods.daily_summary SELECT * FROM staging.events"},
}), nil
},
}
server := newScriptedServer(t, behavior)
server.params.DatabaseType = "argo"
defer server.disconnect()
result, _, err := server.dispatch("get_object_source", map[string]json.RawMessage{
"schema": json.RawMessage(`"ods"`),
"name": json.RawMessage(`"sp_daily_etl"`),
"object_type": json.RawMessage(`"PROCEDURE"`),
})
if err != nil {
t.Fatal(err)
}
source, ok := result.(objectSource)
if !ok {
t.Fatalf("get_object_source returned %T instead of objectSource", result)
}
if source.Name != "sp_daily_etl" || source.ObjectType != "PROCEDURE" {
t.Fatalf("unexpected object source metadata: %#v", source)
}
expected := "-- daily ETL pipeline\nINSERT OVERWRITE TABLE ods.daily_summary SELECT * FROM staging.events\n"
if source.Source != expected {
t.Fatalf("unexpected procedure source: %q", source.Source)
}
}
func TestGetObjectSourceRoutesFunctionsToSystemFunctionsView(t *testing.T) {
expectedSQL := "SELECT full_text FROM system.functions_v WHERE lower(database_name) = lower('ods') AND function_name = 'fn_clean'"
behavior := &scriptedBehavior{
query: func(_ context.Context, query string) (driver.Rows, error) {
if query != expectedSQL {
return nil, errors.New("unexpected query: " + query)
}
return newScriptedRows(context.Background(), []string{"full_text"}, []string{"STRING"}, [][]driver.Value{
{"-- cleanup helper"},
}), nil
},
}
server := newScriptedServer(t, behavior)
server.params.DatabaseType = "argo"
defer server.disconnect()
result, _, err := server.dispatch("get_object_source", map[string]json.RawMessage{
"schema": json.RawMessage(`"ods"`),
"name": json.RawMessage(`"fn_clean"`),
"object_type": json.RawMessage(`"FUNCTION"`),
})
if err != nil {
t.Fatal(err)
}
source, ok := result.(objectSource)
if !ok {
t.Fatalf("get_object_source returned %T instead of objectSource", result)
}
if source.Source != "-- cleanup helper\n" {
t.Fatalf("unexpected function source: %q", source.Source)
}
}
func TestListRoutinesUsesDatabaseParameterWhenSchemaEmpty(t *testing.T) {
proceduresQuery := "SELECT procedure_name FROM system.procedures_v WHERE lower(database_name) = lower('ods') AND lower(procedure_name) LIKE '%%' ORDER BY procedure_name"
defaultQuery := "SELECT procedure_name FROM system.procedures_v WHERE lower(database_name) = lower('default') AND lower(procedure_name) LIKE '%%' ORDER BY procedure_name"
behavior := &scriptedBehavior{
query: func(_ context.Context, query string) (driver.Rows, error) {
switch query {
case defaultQuery:
return newScriptedRows(context.Background(), []string{"procedure_name"}, []string{"STRING"}, [][]driver.Value{}), nil
case proceduresQuery:
return newScriptedRows(context.Background(), []string{"procedure_name"}, []string{"STRING"}, [][]driver.Value{
{"sp_daily_etl"},
}), nil
default:
return nil, errors.New("unexpected query: " + query)
}
},
}
server := newScriptedServer(t, behavior)
server.params.DatabaseType = "argo"
server.config.Database = "default"
defer server.disconnect()
values, err := server.listObjects("ods", "", metadataListConstraints{
ObjectTypes: []string{"PROCEDURE"},
})
if err != nil {
t.Fatalf("listObjects: %v", err)
}
if len(values) != 1 || values[0].Name != "sp_daily_etl" {
t.Fatalf("expected procedure from database parameter, got %+v", values)
}
}
func TestListRoutinesDoesNotFallbackToConnectionDefaultForExplicitSchema(t *testing.T) {
defaultQuery := "SELECT procedure_name FROM system.procedures_v WHERE lower(database_name) = lower('default') AND lower(procedure_name) LIKE '%%' ORDER BY procedure_name"
behavior := &scriptedBehavior{
query: func(_ context.Context, query string) (driver.Rows, error) {
if query == defaultQuery {
// Serving this row proves the driver fell back to the
// connection default after the explicit schema came back empty.
return newScriptedRows(context.Background(), []string{"procedure_name"}, []string{"STRING"}, [][]driver.Value{
{"sp_should_not_leak"},
}), nil
}
return newScriptedRows(context.Background(), []string{"procedure_name"}, []string{"STRING"}, [][]driver.Value{}), nil
},
}
server := newScriptedServer(t, behavior)
server.params.DatabaseType = "argo"
server.config.Database = "default"
defer server.disconnect()
values, err := server.listObjects("", "ods", metadataListConstraints{
ObjectTypes: []string{"PROCEDURE"},
})
if err != nil {
t.Fatalf("listObjects: %v", err)
}
if len(values) != 0 {
t.Fatalf("expected no routines to leak from the connection default database, got %+v", values)
}
}
func TestGetObjectSourceUsesDatabaseParameterForRoutineSource(t *testing.T) {
expectedSQL := "SELECT full_text FROM system.procedures_v WHERE lower(database_name) = lower('ods') AND procedure_name = 'sp_daily_etl'"
behavior := &scriptedBehavior{
query: func(_ context.Context, query string) (driver.Rows, error) {
if query != expectedSQL {
return nil, errors.New("unexpected query: " + query)
}
return newScriptedRows(context.Background(), []string{"full_text"}, []string{"STRING"}, [][]driver.Value{
{"CREATE PROCEDURE sp_daily_etl() BEGIN SELECT 1; END"},
}), nil
},
}
server := newScriptedServer(t, behavior)
server.params.DatabaseType = "argo"
server.config.Database = "default"
defer server.disconnect()
result, _, err := server.dispatch("get_object_source", map[string]json.RawMessage{
"database": json.RawMessage(`"ods"`),
"schema": json.RawMessage(`""`),
"name": json.RawMessage(`"sp_daily_etl"`),
"object_type": json.RawMessage(`"PROCEDURE"`),
})
if err != nil {
t.Fatal(err)
}
source, ok := result.(objectSource)
if !ok {
t.Fatalf("get_object_source returned %T instead of objectSource", result)
}
if source.Source != "CREATE PROCEDURE sp_daily_etl() BEGIN SELECT 1; END\n" {
t.Fatalf("unexpected procedure source: %q", source.Source)
}
}
func TestListObjectsDoesNotQueryRoutinesForVanillaHive(t *testing.T) {
behavior := &scriptedBehavior{
query: func(ctx context.Context, query string) (driver.Rows, error) {
if strings.Contains(strings.ToUpper(query), "PROCEDURES_V") || strings.Contains(strings.ToUpper(query), "FUNCTIONS_V") {
t.Fatalf("vanilla Hive must not query routine views, got: %s", query)
}
return newScriptedRows(ctx, []string{"tab_name"}, []string{"STRING"}, [][]driver.Value{{"events"}}), nil
},
}
server := newScriptedServer(t, behavior)
server.params.DatabaseType = "hive"
defer server.disconnect()
values, err := server.listObjects("ods", "ods", metadataListConstraints{
ObjectTypes: []string{"PROCEDURE", "FUNCTION"},
})
if err != nil {
t.Fatalf("listObjects: %v", err)
}
for _, v := range values {
if v.ObjectType == "PROCEDURE" || v.ObjectType == "FUNCTION" {
t.Fatalf("vanilla Hive should not surface routines, got: %+v", v)
}
}
}
+5 -2
View File
@@ -333,9 +333,12 @@ func (server *server) executeStatements(params map[string]json.RawMessage, trans
}
return emptyQueryResult(affected, started), nil
}
if !transactionUnsupported(beginErr) {
return queryResult{}, beginErr
if transactionUnsupported(beginErr) {
// Do not turn an explicit transaction request into auto-commit
// execution: a later error would leave prior statements applied.
return queryResult{}, errors.New("Hive does not support rollbackable transactions")
}
return queryResult{}, beginErr
}
for _, statement := range statements {
trimmed := trimStatementSQL(statement)

Some files were not shown because too many files have changed in this diff Show More