mirror of
https://github.com/t8y2/dbx.git
synced 2026-10-19 16:16:45 +00:00
Compare commits
392
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4047686f93 | ||
|
|
b853d49ac1 | ||
|
|
5ca30bb25b | ||
|
|
2c9dcdf3ea | ||
|
|
0a1ac3fb14 | ||
|
|
6e9e8d1818 | ||
|
|
f861776c3c | ||
|
|
a559371fc9 | ||
|
|
5e03c20821 | ||
|
|
a2e4d15bbc | ||
|
|
7f7f011f40 | ||
|
|
af32936297 | ||
|
|
d430ec9a4a | ||
|
|
b00821d2c5 | ||
|
|
1c972b80c0 | ||
|
|
795cd13f88 | ||
|
|
805ddac61c | ||
|
|
67f94e1e3b | ||
|
|
392b2bf5b1 | ||
|
|
d2401896dd | ||
|
|
a5c97067ba | ||
|
|
372e5130cd | ||
|
|
65abc74590 | ||
|
|
7b0a8cc111 | ||
|
|
01623ab38d | ||
|
|
09f8d095d2 | ||
|
|
ca003c5e93 | ||
|
|
5a3a6e2bb0 | ||
|
|
24a274bd89 | ||
|
|
a52138129a | ||
|
|
543bf66905 | ||
|
|
7390ac109c | ||
|
|
09de83e64e | ||
|
|
eb790e7bfc | ||
|
|
434d44a284 | ||
|
|
299c342d6d | ||
|
|
b6beefc89e | ||
|
|
057c04d306 | ||
|
|
560e1c2c0c | ||
|
|
b42ad91fc2 | ||
|
|
84b37d44a6 | ||
|
|
2766a5bc49 | ||
|
|
8cc7393b11 | ||
|
|
9eacc94584 | ||
|
|
28930136d7 | ||
|
|
6d39eccdbf | ||
|
|
d16645121d | ||
|
|
d7941572b8 | ||
|
|
97f5a37ba8 | ||
|
|
06dcd0013c | ||
|
|
48410b70b4 | ||
|
|
0a511072a7 | ||
|
|
6a8233b2c8 | ||
|
|
79dbda2a8f | ||
|
|
232426ced2 | ||
|
|
2a4f1a45a9 | ||
|
|
0bdf232487 | ||
|
|
37dc74d9de | ||
|
|
fe955da729 | ||
|
|
81587313eb | ||
|
|
0279a0903c | ||
|
|
39de9499b4 | ||
|
|
de73c2bd73 | ||
|
|
730a4eca5f | ||
|
|
4dab78ac30 | ||
|
|
b1fc9b15b9 | ||
|
|
2a53460604 | ||
|
|
b17a87697b | ||
|
|
1c170396d3 | ||
|
|
56f85a2c77 | ||
|
|
5e4f756a63 | ||
|
|
2324d84c4e | ||
|
|
0448d5bd2c | ||
|
|
7157b04d87 | ||
|
|
dd11e42748 | ||
|
|
767cc809a1 | ||
|
|
c28248cd67 | ||
|
|
d765315af4 | ||
|
|
291890f67e | ||
|
|
aa637b2d90 | ||
|
|
cd8300c3f5 | ||
|
|
203eab3dec | ||
|
|
d421bd33fc | ||
|
|
20f312d650 | ||
|
|
3a52fd2a0b | ||
|
|
c444bb44b1 | ||
|
|
719070f2b8 | ||
|
|
f3c82061c0 | ||
|
|
6088203755 | ||
|
|
486da32e4d | ||
|
|
178f00d7ff | ||
|
|
715261feae | ||
|
|
e98e3157f1 | ||
|
|
df244627f6 | ||
|
|
4d05a95f5a | ||
|
|
8cfd69c7c9 | ||
|
|
76c51b5fa6 | ||
|
|
4b7b72dc65 | ||
|
|
c99c0d932c | ||
|
|
3c4d61339e | ||
|
|
ef13dc024f | ||
|
|
aa4f503353 | ||
|
|
2c7a420ef6 | ||
|
|
eec2e63667 | ||
|
|
35946427a6 | ||
|
|
e2eb481b8a | ||
|
|
321674c4dc | ||
|
|
24642a6bd5 | ||
|
|
1818387da5 | ||
|
|
d6fc1fe52c | ||
|
|
616c0daf25 | ||
|
|
1f396a7d25 | ||
|
|
aac1d78539 | ||
|
|
1ed4b42955 | ||
|
|
d38be39e03 | ||
|
|
717dcc1294 | ||
|
|
7e24f4007c | ||
|
|
2f4a72c4e9 | ||
|
|
6848e78855 | ||
|
|
45688cafd6 | ||
|
|
a13a7302fa | ||
|
|
460b413da8 | ||
|
|
a2aad40190 | ||
|
|
0f1bc0a573 | ||
|
|
af29ce44b7 | ||
|
|
41ebb64e95 | ||
|
|
11ff1c29a6 | ||
|
|
7d92433689 | ||
|
|
91f309f6c7 | ||
|
|
d92e119464 | ||
|
|
d6c4457756 | ||
|
|
8305d63ea1 | ||
|
|
f3258fe08a | ||
|
|
b9b9d0bd31 | ||
|
|
dd20081316 | ||
|
|
0f60b6a655 | ||
|
|
fe509e80c7 | ||
|
|
cfdee960b4 | ||
|
|
08e6a19c9c | ||
|
|
ac22ada8d0 | ||
|
|
fdea607efd | ||
|
|
72dcd8355a | ||
|
|
41fc755135 | ||
|
|
c48ebda197 | ||
|
|
1706081802 | ||
|
|
0333b3b389 | ||
|
|
2c14016828 | ||
|
|
c813b76019 | ||
|
|
f5113def38 | ||
|
|
061041e1d7 | ||
|
|
02430d1b09 | ||
|
|
e3911a0198 | ||
|
|
820de91972 | ||
|
|
3e4c17bd58 | ||
|
|
b45300e9bc | ||
|
|
3b2df3814b | ||
|
|
a1f54cf428 | ||
|
|
f9f61cf397 | ||
|
|
e619b70d5d | ||
|
|
ee731a3353 | ||
|
|
8772f10aa4 | ||
|
|
245ea3b640 | ||
|
|
069b84daf9 | ||
|
|
ad98bef167 | ||
|
|
91b4623278 | ||
|
|
032c05d6ce | ||
|
|
9b6f3c48e4 | ||
|
|
d3a6a69796 | ||
|
|
84669b0d1a | ||
|
|
945285ef09 | ||
|
|
38b93505c4 | ||
|
|
a834da7026 | ||
|
|
716d2eafd2 | ||
|
|
10ac216964 | ||
|
|
103e221cc3 | ||
|
|
a0d868b637 | ||
|
|
55a21f8e9e | ||
|
|
8b4b836581 | ||
|
|
2c04c2f1f4 | ||
|
|
01ce303f46 | ||
|
|
e9f2cc15b7 | ||
|
|
bd7ed63468 | ||
|
|
48f4fac89e | ||
|
|
51ca40e86e | ||
|
|
3325e97991 | ||
|
|
e79db23e1f | ||
|
|
07688cac39 | ||
|
|
804120c0b1 | ||
|
|
877421e596 | ||
|
|
d4d3995bef | ||
|
|
053f5f6dc5 | ||
|
|
7e3db73ded | ||
|
|
fac018f169 | ||
|
|
b31b6845f9 | ||
|
|
0d75456f80 | ||
|
|
c4cdcbc486 | ||
|
|
1f274c57e3 | ||
|
|
6a7ea4ae95 | ||
|
|
19be194728 | ||
|
|
5d6f667776 | ||
|
|
e9a90173a6 | ||
|
|
440c41f5b9 | ||
|
|
a7922b9cbf | ||
|
|
cb754cd0cc | ||
|
|
264f5c4836 | ||
|
|
613c4f7ca3 | ||
|
|
6588f27fbc | ||
|
|
f19439ad7a | ||
|
|
1aa9f06719 | ||
|
|
1e75a78228 | ||
|
|
55611706ba | ||
|
|
70d7a78c66 | ||
|
|
7f9e35d1e8 | ||
|
|
4d3d64bd34 | ||
|
|
e43e36e7a4 | ||
|
|
600af87532 | ||
|
|
2e8ed4b485 | ||
|
|
8f21e7af9f | ||
|
|
333c3a3f44 | ||
|
|
d59b86bd57 | ||
|
|
9de152cf72 | ||
|
|
46584ff46e | ||
|
|
cbfa8d7a79 | ||
|
|
694611aade | ||
|
|
a751b5690b | ||
|
|
c7cea5a047 | ||
|
|
8be503bf40 | ||
|
|
d79d357924 | ||
|
|
23f0b1255b | ||
|
|
5f9ba51769 | ||
|
|
d69cf9da79 | ||
|
|
8cc04f3af4 | ||
|
|
c9f0ee89f9 | ||
|
|
14ad5ef9bf | ||
|
|
cd008b2d56 | ||
|
|
f9ff9245ce | ||
|
|
554be11aea | ||
|
|
23f511de22 | ||
|
|
c7c420bbd0 | ||
|
|
cb3de0ef49 | ||
|
|
b41dac4617 | ||
|
|
2779a435cf | ||
|
|
9a5aae49c8 | ||
|
|
47ddbf6bb4 | ||
|
|
49926f25f1 | ||
|
|
351a8b699b | ||
|
|
afe3b3f3c2 | ||
|
|
69a032ed63 | ||
|
|
2f71a1a8aa | ||
|
|
5eb071bc22 | ||
|
|
f958bf4602 | ||
|
|
ef2fdb941c | ||
|
|
bc41e8d76d | ||
|
|
d411f04975 | ||
|
|
2fe26052a7 | ||
|
|
42c0dc7f89 | ||
|
|
ba11e4be8d | ||
|
|
662037295c | ||
|
|
40a806a24a | ||
|
|
f075caef21 | ||
|
|
f725c50b2c | ||
|
|
8f3134dce8 | ||
|
|
a84cb637ba | ||
|
|
b60f04ea61 | ||
|
|
4baa46ca88 | ||
|
|
0f33449bdf | ||
|
|
5e764ebf05 | ||
|
|
1d98eca99c | ||
|
|
4605e4c320 | ||
|
|
f70fe38a46 | ||
|
|
dc2e16ffaf | ||
|
|
9d63e25dee | ||
|
|
1a833662e2 | ||
|
|
e0746a2dbe | ||
|
|
f489db4f08 | ||
|
|
0812e57cf1 | ||
|
|
d144e1bc08 | ||
|
|
0e9f92c917 | ||
|
|
27f62fe6e2 | ||
|
|
99c3bd0f42 | ||
|
|
bcea953a50 | ||
|
|
eba4e95a0c | ||
|
|
e690298dcf | ||
|
|
d96da5dece | ||
|
|
d459628141 | ||
|
|
0c8b796c8c | ||
|
|
a849979dd1 | ||
|
|
1c0e97b52a | ||
|
|
b7d33e375f | ||
|
|
d1046d54f9 | ||
|
|
8e2e8c1b8c | ||
|
|
8da9905ed5 | ||
|
|
0a239bddb2 | ||
|
|
b3fcb3a6f1 | ||
|
|
c305325e37 | ||
|
|
d424ea38f0 | ||
|
|
9190554e50 | ||
|
|
e6f8b2187a | ||
|
|
7ba30ae761 | ||
|
|
9491ab08df | ||
|
|
143be57d41 | ||
|
|
f664918e95 | ||
|
|
469bd88c37 | ||
|
|
5a11b8f148 | ||
|
|
eafc0b2b17 | ||
|
|
de2c4b38cd | ||
|
|
349342efb7 | ||
|
|
694c344278 | ||
|
|
4ac362453f | ||
|
|
73a7b3f2c7 | ||
|
|
1d71a5fc23 | ||
|
|
d5f4eadf7a | ||
|
|
22775f3639 | ||
|
|
50e9d8bbb1 | ||
|
|
45fb7fb7e6 | ||
|
|
1e8c58b3df | ||
|
|
21f8221d9f | ||
|
|
170e761fd2 | ||
|
|
f4971d509a | ||
|
|
0fdddfc083 | ||
|
|
2529737a64 | ||
|
|
8a7d0062e3 | ||
|
|
bd3e585628 | ||
|
|
1637948023 | ||
|
|
1302c0a27b | ||
|
|
508369ee39 | ||
|
|
c24e4352e8 | ||
|
|
8f54385b97 | ||
|
|
cb9f87031f | ||
|
|
5c738f5bf2 | ||
|
|
038ff4e773 | ||
|
|
9c56de2e9f | ||
|
|
ef87e45cbc | ||
|
|
f83f707fbc | ||
|
|
7a4edb6c19 | ||
|
|
b4787a63af | ||
|
|
fd17037a2a | ||
|
|
41db375ee6 | ||
|
|
1498b27314 | ||
|
|
6d8e667bed | ||
|
|
df88d7baa7 | ||
|
|
659fb7f330 | ||
|
|
c237ecbfca | ||
|
|
bcc64274e8 | ||
|
|
802019daa3 | ||
|
|
60bfb41f13 | ||
|
|
17e2573fbe | ||
|
|
2467d212d5 | ||
|
|
80224ab1d9 | ||
|
|
89086cc22b | ||
|
|
fdfad88590 | ||
|
|
d3a97164aa | ||
|
|
d4245f0d58 | ||
|
|
2c5f926bd8 | ||
|
|
20a2994fa1 | ||
|
|
7ad05b178f | ||
|
|
be307ba2d7 | ||
|
|
10bce698ae | ||
|
|
c0abc26fe8 | ||
|
|
425ca055ea | ||
|
|
99a9961712 | ||
|
|
d8ec9bb8d0 | ||
|
|
4636788f49 | ||
|
|
22305b7d0f | ||
|
|
71d07348ad | ||
|
|
b198dd1327 | ||
|
|
9290391985 | ||
|
|
1016ccc352 | ||
|
|
ec0b6768be | ||
|
|
b09c6f4f21 | ||
|
|
74ffab29ca | ||
|
|
e996f8aefb | ||
|
|
4fed29fba2 | ||
|
|
214bf86a6f | ||
|
|
955558c81f | ||
|
|
7933b7656f | ||
|
|
e5572ead7a | ||
|
|
bd6c780aba | ||
|
|
dfd4d3b7ff | ||
|
|
ab4e747bf9 | ||
|
|
9e4fe8ce56 | ||
|
|
6a6e5fbad9 | ||
|
|
e8d6370b1e | ||
|
|
4eeeafa858 | ||
|
|
e4dc5bace8 | ||
|
|
920ab62cf1 | ||
|
|
36a7b7bd3a | ||
|
|
565feb8745 | ||
|
|
40549c9573 | ||
|
|
eb0994e980 | ||
|
|
cf03de122a | ||
|
|
d9beaaaa1c |
@@ -0,0 +1,75 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$InstallerPath,
|
||||
[int]$StartupSeconds = 20
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$os = [System.Environment]::OSVersion.Version
|
||||
if ($os.Major -ne 6 -or ($os.Minor -ne 1 -and $os.Minor -ne 3)) {
|
||||
throw "The installer smoke test requires Windows 7 or Server 2012 R2 (kernel 6.1/6.3), detected $($os.ToString())."
|
||||
}
|
||||
|
||||
$installerPath = (Resolve-Path -LiteralPath $InstallerPath).Path
|
||||
$testRoot = Join-Path ([System.IO.Path]::GetTempPath()) "dbx-server2012-installer-smoke"
|
||||
$resolvedTemp = (Resolve-Path -LiteralPath ([System.IO.Path]::GetTempPath())).Path.TrimEnd('\')
|
||||
$resolvedTestRoot = [System.IO.Path]::GetFullPath($testRoot).TrimEnd('\')
|
||||
if (!$resolvedTestRoot.StartsWith("$resolvedTemp\", [System.StringComparison]::OrdinalIgnoreCase)) {
|
||||
throw "Refusing to use a smoke-test directory outside the system temporary directory: $resolvedTestRoot"
|
||||
}
|
||||
|
||||
if (Test-Path -LiteralPath $resolvedTestRoot) {
|
||||
Remove-Item -LiteralPath $resolvedTestRoot -Recurse -Force
|
||||
}
|
||||
|
||||
$appProcess = $null
|
||||
try {
|
||||
$installer = Start-Process -FilePath $installerPath -ArgumentList @("/S", "/D=$resolvedTestRoot") -Wait -PassThru
|
||||
if ($installer.ExitCode -ne 0) {
|
||||
throw "Server 2012 R2 test installer failed with exit code $($installer.ExitCode)."
|
||||
}
|
||||
|
||||
$appPath = Join-Path $resolvedTestRoot "dbx.exe"
|
||||
$runtimePath = Join-Path $resolvedTestRoot "webview2-fixed-runtime\msedgewebview2.exe"
|
||||
foreach ($requiredPath in @($appPath, $runtimePath)) {
|
||||
if (!(Test-Path -LiteralPath $requiredPath -PathType Leaf)) {
|
||||
throw "Server 2012 R2 installer omitted required file: $requiredPath"
|
||||
}
|
||||
}
|
||||
|
||||
$env:WEBVIEW2_BROWSER_EXECUTABLE_FOLDER = Join-Path $resolvedTestRoot "webview2-fixed-runtime"
|
||||
$env:WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS = "--no-sandbox --disable-gpu"
|
||||
$env:DBX_STARTUP_LOG_DIR = Join-Path $resolvedTestRoot "smoke-logs"
|
||||
New-Item -ItemType Directory -Path $env:DBX_STARTUP_LOG_DIR -Force | Out-Null
|
||||
|
||||
$appProcess = Start-Process -FilePath $appPath -PassThru
|
||||
Start-Sleep -Seconds $StartupSeconds
|
||||
$appProcess.Refresh()
|
||||
if ($appProcess.HasExited) {
|
||||
$startupLog = Join-Path $env:DBX_STARTUP_LOG_DIR "startup.log"
|
||||
$logText = if (Test-Path -LiteralPath $startupLog) {
|
||||
Get-Content -LiteralPath $startupLog -Raw
|
||||
} else {
|
||||
"<startup.log was not created>"
|
||||
}
|
||||
throw "DBX exited during the Server 2012 R2 startup smoke test with code $($appProcess.ExitCode).`n$logText"
|
||||
}
|
||||
|
||||
Write-Host "Server 2012 R2 installer smoke test passed after $StartupSeconds seconds: $installerPath"
|
||||
}
|
||||
finally {
|
||||
if ($null -ne $appProcess -and !$appProcess.HasExited) {
|
||||
Stop-Process -Id $appProcess.Id -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
|
||||
$uninstallerPath = Join-Path $resolvedTestRoot "uninstall.exe"
|
||||
if (Test-Path -LiteralPath $uninstallerPath -PathType Leaf) {
|
||||
$uninstaller = Start-Process -FilePath $uninstallerPath -ArgumentList @("/S", "_?=$resolvedTestRoot") -Wait -PassThru
|
||||
if ($uninstaller.ExitCode -ne 0) {
|
||||
Write-Warning "Server 2012 R2 test uninstaller returned exit code $($uninstaller.ExitCode)."
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$RuntimeDirectory = (Join-Path $PSScriptRoot "..\..\src-tauri\webview2-fixed-runtime"),
|
||||
[string]$LoaderPath = (Join-Path ([System.IO.Path]::GetTempPath()) "dbx-win7-webview2-loader-probe\WebView2Loader.dll"),
|
||||
[string]$ExpectedVersion = "109.0.1518.78"
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$os = [System.Environment]::OSVersion.Version
|
||||
if ($os.Major -ne 6 -or ($os.Minor -ne 1 -and $os.Minor -ne 3)) {
|
||||
throw "The offline runtime probe requires Windows 7 or Server 2012 R2 (kernel 6.1/6.3), detected $($os.ToString())."
|
||||
}
|
||||
|
||||
$runtimeDirectory = (Resolve-Path -LiteralPath $RuntimeDirectory).Path
|
||||
$loaderPath = (Resolve-Path -LiteralPath $LoaderPath).Path
|
||||
$runtimeExecutable = Join-Path $runtimeDirectory "msedgewebview2.exe"
|
||||
if (!(Test-Path -LiteralPath $runtimeExecutable -PathType Leaf)) {
|
||||
throw "WebView2 fixed runtime executable does not exist: $runtimeExecutable"
|
||||
}
|
||||
|
||||
$escapedLoaderPath = $loaderPath.Replace('"', '""')
|
||||
$source = @"
|
||||
using System;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
public static class DbxServer2012WebView2Probe
|
||||
{
|
||||
[DllImport(@"$escapedLoaderPath", CharSet = CharSet.Unicode, ExactSpelling = true)]
|
||||
public static extern int GetAvailableCoreWebView2BrowserVersionString(
|
||||
string browserExecutableFolder,
|
||||
out IntPtr versionInfo);
|
||||
}
|
||||
"@
|
||||
|
||||
Add-Type -TypeDefinition $source -Language CSharp
|
||||
$versionPointer = [IntPtr]::Zero
|
||||
$result = [DbxServer2012WebView2Probe]::GetAvailableCoreWebView2BrowserVersionString(
|
||||
$runtimeDirectory,
|
||||
[ref]$versionPointer
|
||||
)
|
||||
|
||||
$resultHex = [System.BitConverter]::ToUInt32(
|
||||
[System.BitConverter]::GetBytes([int32]$result),
|
||||
0
|
||||
)
|
||||
$hresult = "0x{0:X8}" -f $resultHex
|
||||
if ($result -ne 0) {
|
||||
throw "Server 2012 R2 WebView2 loader rejected the fixed runtime with HRESULT $hresult."
|
||||
}
|
||||
if ($versionPointer -eq [IntPtr]::Zero) {
|
||||
throw "Server 2012 R2 WebView2 loader returned an empty version pointer."
|
||||
}
|
||||
|
||||
try {
|
||||
$version = [Runtime.InteropServices.Marshal]::PtrToStringUni($versionPointer)
|
||||
}
|
||||
finally {
|
||||
[Runtime.InteropServices.Marshal]::FreeCoTaskMem($versionPointer)
|
||||
}
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($version) -or !$version.StartsWith($ExpectedVersion)) {
|
||||
throw "Expected WebView2 fixed runtime $ExpectedVersion, detected '$version'."
|
||||
}
|
||||
|
||||
Write-Host "Server 2012 R2 WebView2 runtime probe passed: loader=$loaderPath runtime=$runtimeDirectory version=$version"
|
||||
@@ -0,0 +1,53 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$BinaryPath
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
# The Win7 / Server 2012 R2 bundle must statically link the WebView2 loader from
|
||||
# SDK 1.0.902.49, the last one verified to work on kernel 6.1/6.3. Loader
|
||||
# 1.0.1054.31 and everything newer fail on Server 2012 R2 with
|
||||
# ERROR_NOT_SUPPORTED (0x80070032, MicrosoftEdge/WebView2Feedback#2025) and make
|
||||
# the app show "Could not find the WebView2 Runtime". Those loaders carry error
|
||||
# strings that 1.0.902.49 does not, so any of the markers below appearing in
|
||||
# dbx.exe proves the pinned loader was not the one linked. The PE import audit
|
||||
# cannot catch this: 1.0.1054.31 imports nothing newer than 1.0.902.49.
|
||||
$loaderMarkers = @(
|
||||
"WebView2: Failed to find the app exe path.",
|
||||
"WebView2: Failed to find the WebView2 client dll at:",
|
||||
"WebView2: Failed to find an installed WebView2 runtime or non-stable Microsoft Edge installation."
|
||||
)
|
||||
|
||||
if (!(Test-Path -LiteralPath $BinaryPath -PathType Leaf)) {
|
||||
throw "Win7 WebView2 loader audit target does not exist: $BinaryPath"
|
||||
}
|
||||
|
||||
$resolvedPath = (Resolve-Path -LiteralPath $BinaryPath).Path
|
||||
# Map every byte to one character so the ASCII needle search cannot miss on
|
||||
# arbitrary binary content.
|
||||
$latin1 = [System.Text.Encoding]::GetEncoding(28591)
|
||||
$content = $latin1.GetString([System.IO.File]::ReadAllBytes($resolvedPath))
|
||||
|
||||
$violations = @()
|
||||
foreach ($marker in $loaderMarkers) {
|
||||
if ($content.Contains($marker)) {
|
||||
$violations += $marker
|
||||
}
|
||||
}
|
||||
|
||||
if ($violations.Count -gt 0) {
|
||||
$summary = $violations -join "`n"
|
||||
throw @"
|
||||
dbx.exe contains WebView2 loader strings newer than the pinned 1.0.902.49:
|
||||
$summary
|
||||
|
||||
A loader >= 1.0.1054.31 was linked instead of the prepared one and will fail on
|
||||
Server 2012 R2 with ERROR_NOT_SUPPORTED. Do not ship this binary: rebuild the
|
||||
win7 target and investigate how the registry loader patch was bypassed.
|
||||
"@
|
||||
}
|
||||
|
||||
Write-Host "Win7 WebView2 loader audit passed: no post-1.0.902.49 loader markers in $resolvedPath"
|
||||
@@ -52,6 +52,7 @@ const nativeDriverDirectories = {
|
||||
cassandra: "cassandra-go",
|
||||
duckdb: "duckdb",
|
||||
hive: "hive-go",
|
||||
argo: "argo-go",
|
||||
oracle: "oracle-go",
|
||||
kingbase: "kingbase-go",
|
||||
iotdb: "iotdb",
|
||||
@@ -61,21 +62,46 @@ const nativeDriverDirectories = {
|
||||
rocketmq: "rocketmq",
|
||||
zookeeper: "zookeeper",
|
||||
tdengine: "tdengine",
|
||||
etcd: "etcd-go",
|
||||
etcd2: "etcd2-go",
|
||||
};
|
||||
const nativeDriverModules = new Set(["cassandra", "duckdb", "hive", "oracle", "xugu", "kingbase", "iotdb", "neo4j", "vastbase", "rabbitmq", "rocketmq", "zookeeper", "tdengine"]);
|
||||
const crateNativeDriverDirectories = {
|
||||
"sqlite-worker": "crates/dbx-sqlite-worker",
|
||||
};
|
||||
const nativeDriverModules = new Set(["cassandra", "duckdb", "hive", "argo", "oracle", "xugu", "kingbase", "iotdb", "neo4j", "vastbase", "rabbitmq", "rocketmq", "zookeeper", "tdengine", "etcd", "etcd2", "sqlite-worker"]);
|
||||
const nativeDriverSharedPaths = {
|
||||
hive: [
|
||||
"agents/go-common/go-gssapi",
|
||||
"agents/go-common/gohive",
|
||||
"agents/go-common/gosasl",
|
||||
],
|
||||
argo: [
|
||||
"agents/go-common/go-gssapi",
|
||||
"agents/go-common/gohive",
|
||||
"agents/go-common/gosasl",
|
||||
],
|
||||
zookeeper: [
|
||||
"agents/go-common/go-gssapi",
|
||||
"agents/go-common/gosasl",
|
||||
],
|
||||
etcd: [
|
||||
"agents/go-common/go-semver",
|
||||
],
|
||||
};
|
||||
|
||||
function resolveAgentModule(moduleName, { legacyStandaloneModules, moduleExists, readModuleFile }) {
|
||||
const cratePath = crateNativeDriverDirectories[moduleName];
|
||||
if (cratePath && moduleExists(cratePath)) {
|
||||
return {
|
||||
checkDir: cratePath,
|
||||
modulePath: cratePath,
|
||||
sharedPaths: [],
|
||||
javaBuild: false,
|
||||
nativeBuild: true,
|
||||
commonDependent: false,
|
||||
};
|
||||
}
|
||||
|
||||
let checkDir = null;
|
||||
const nativeDriverDirectory = nativeDriverDirectories[moduleName];
|
||||
if (nativeDriverDirectory && moduleExists(`agents/drivers/${nativeDriverDirectory}`)) {
|
||||
|
||||
@@ -34,6 +34,20 @@ test("bumps DuckDB after its initial release", () => {
|
||||
assert.equal(result.versions.duckdb, "0.1.1");
|
||||
});
|
||||
|
||||
test("bumps the SQLite SSH worker from its crate path", () => {
|
||||
const result = evaluateAgentVersionBump({
|
||||
versions: { "sqlite-worker": "0.1.0" },
|
||||
changedFiles: ["crates/dbx-sqlite-worker/src/runtime.rs"],
|
||||
moduleExists: (path) => path === "crates/dbx-sqlite-worker",
|
||||
readModuleFile: () => "",
|
||||
});
|
||||
|
||||
assert.equal(result.versions["sqlite-worker"], "0.1.1");
|
||||
assert.deepEqual(result.changedModules, ["sqlite-worker"]);
|
||||
assert.deepEqual(result.javaModules, []);
|
||||
assert.deepEqual(result.nativeModules, ["sqlite-worker"]);
|
||||
});
|
||||
|
||||
test("classifies TDengine Rust changes as native-only", () => {
|
||||
const result = evaluateAgentVersionBump({
|
||||
versions: { tdengine: "0.1.39" },
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
[CmdletBinding(DefaultParameterSetName = "Runtime")]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$ComputerName,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$UserName,
|
||||
[string]$Password = $env:SERVER2012_PASSWORD,
|
||||
[Parameter(Mandatory = $true)]
|
||||
[ValidateSet("Runtime", "Installer")]
|
||||
[string]$Mode,
|
||||
[Parameter(Mandatory = $true, ParameterSetName = "Runtime")]
|
||||
[string]$RuntimeDirectory,
|
||||
[Parameter(Mandatory = $true, ParameterSetName = "Runtime")]
|
||||
[string]$LoaderPath,
|
||||
[Parameter(Mandatory = $true, ParameterSetName = "Installer")]
|
||||
[string]$InstallerPath,
|
||||
[switch]$UseSsl
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($Password)) {
|
||||
throw "SERVER2012_PASSWORD is required."
|
||||
}
|
||||
|
||||
$securePassword = ConvertTo-SecureString $Password -AsPlainText -Force
|
||||
$credential = New-Object System.Management.Automation.PSCredential($UserName, $securePassword)
|
||||
$sessionParameters = @{
|
||||
ComputerName = $ComputerName
|
||||
Credential = $credential
|
||||
Authentication = "Negotiate"
|
||||
ErrorAction = "Stop"
|
||||
}
|
||||
$useSslEnabled = $UseSsl -or $env:SERVER2012_USE_SSL -eq "true"
|
||||
if ($useSslEnabled) {
|
||||
$sessionParameters.UseSSL = $true
|
||||
$sessionParameters.SessionOption = New-PSSessionOption -SkipCACheck -SkipCNCheck -SkipRevocationCheck
|
||||
}
|
||||
|
||||
$session = $null
|
||||
$remoteRoot = "C:\Windows\Temp\dbx-server2012-ci-$([Guid]::NewGuid().ToString('N'))"
|
||||
try {
|
||||
$session = New-PSSession @sessionParameters
|
||||
Invoke-Command -Session $session -ScriptBlock {
|
||||
param($Path)
|
||||
New-Item -ItemType Directory -Path $Path -Force | Out-Null
|
||||
} -ArgumentList $remoteRoot
|
||||
|
||||
if ($Mode -eq "Runtime") {
|
||||
$runtimeDirectory = (Resolve-Path -LiteralPath $RuntimeDirectory).Path
|
||||
$loaderPath = (Resolve-Path -LiteralPath $LoaderPath).Path
|
||||
$probeScript = (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot "assert-webview2-server2012-runtime.ps1")).Path
|
||||
|
||||
Copy-Item -LiteralPath $runtimeDirectory -Destination "$remoteRoot\webview2-fixed-runtime" -Recurse -ToSession $session
|
||||
Copy-Item -LiteralPath $loaderPath -Destination "$remoteRoot\WebView2Loader.dll" -ToSession $session
|
||||
Copy-Item -LiteralPath $probeScript -Destination "$remoteRoot\assert-runtime.ps1" -ToSession $session
|
||||
|
||||
Invoke-Command -Session $session -ScriptBlock {
|
||||
param($Root)
|
||||
& "$Root\assert-runtime.ps1" `
|
||||
-RuntimeDirectory "$Root\webview2-fixed-runtime" `
|
||||
-LoaderPath "$Root\WebView2Loader.dll"
|
||||
} -ArgumentList $remoteRoot
|
||||
} else {
|
||||
$installerPath = (Resolve-Path -LiteralPath $InstallerPath).Path
|
||||
$smokeScript = (Resolve-Path -LiteralPath (Join-Path $PSScriptRoot "assert-server2012-installer-smoke.ps1")).Path
|
||||
|
||||
Copy-Item -LiteralPath $installerPath -Destination "$remoteRoot\installer.exe" -ToSession $session
|
||||
Copy-Item -LiteralPath $smokeScript -Destination "$remoteRoot\assert-installer.ps1" -ToSession $session
|
||||
|
||||
Invoke-Command -Session $session -ScriptBlock {
|
||||
param($Root)
|
||||
& "$Root\assert-installer.ps1" -InstallerPath "$Root\installer.exe"
|
||||
} -ArgumentList $remoteRoot
|
||||
}
|
||||
}
|
||||
finally {
|
||||
if ($null -ne $session) {
|
||||
Invoke-Command -Session $session -ScriptBlock {
|
||||
param($Path)
|
||||
$windowsTemp = [System.IO.Path]::GetFullPath("C:\Windows\Temp").TrimEnd('\')
|
||||
$resolvedPath = [System.IO.Path]::GetFullPath($Path).TrimEnd('\')
|
||||
if (!$resolvedPath.StartsWith("$windowsTemp\dbx-server2012-ci-", [System.StringComparison]::OrdinalIgnoreCase)) {
|
||||
throw "Refusing to clean an unexpected remote validation directory: $resolvedPath"
|
||||
}
|
||||
if (Test-Path -LiteralPath $resolvedPath) {
|
||||
Remove-Item -LiteralPath $resolvedPath -Recurse -Force
|
||||
}
|
||||
} -ArgumentList $remoteRoot
|
||||
Remove-PSSession $session
|
||||
}
|
||||
}
|
||||
@@ -4,11 +4,16 @@ param()
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
# Newer WebView2 static loaders import EventSetInformation, which does not exist on Windows 7.
|
||||
# The loader entry points are stable, so the Win7 bundle uses the last verified compatible SDK loader.
|
||||
$sdkVersion = "1.0.1054.31"
|
||||
$sdkPackageSha256 = "0afe683aa3d143a5f6330db1ce833c69278b38fe5e1eadec52f26910ad26e22f"
|
||||
$loaderSha256 = "76314119685bbf4c2b2423a44e81b57beadc914c943d0e772fd6bc78c8e6b0e8"
|
||||
# WebView2 static loaders newer than SDK 1.0.1054 import EventSetInformation, which does
|
||||
# not exist on Windows 7, while 1.0.1054.31 itself fails on Server 2012 R2 with
|
||||
# ERROR_NOT_SUPPORTED (0x80070032, MicrosoftEdge/WebView2Feedback#2025). SDK 1.0.902.49 is
|
||||
# the loader verified on both Windows 7 SP1 and Server 2012 R2 real machines, so the single
|
||||
# win7 + server 2012 r2 offline bundle links it instead of the webview2-com-sys default.
|
||||
$sdkVersion = "1.0.902.49"
|
||||
$sdkPackageSha256 = "b483c906b03690267108f4304d456eac0e718131ef994e69aaa5a21532b512c6"
|
||||
$loaderSha256 = "aa5c26670f1b18d0fa2a56ac3f1ae30110c332a8bfbd555a7be3e548d1b0da3d"
|
||||
$loaderDllSha256 = "fdf978ba706578b05967d7f0181f462147864a5aa74f36016a62cb3d3dbe6909"
|
||||
$legacy1054LoaderSha256 = "76314119685bbf4c2b2423a44e81b57beadc914c943d0e772fd6bc78c8e6b0e8"
|
||||
$webView2ComSysVersion = "0.38.2"
|
||||
$upstreamLoaderSha256 = "0659b741bde6348d4c4a6ec4ceb9af50e3d0048ed9cd3c8659bccbb61fde55ee"
|
||||
|
||||
@@ -19,9 +24,10 @@ $extractedPath = Join-Path $temporaryRoot "extracted"
|
||||
|
||||
try {
|
||||
New-Item -ItemType Directory -Path $extractedPath -Force | Out-Null
|
||||
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
|
||||
|
||||
$packageUrl = "https://www.nuget.org/api/v2/package/Microsoft.Web.WebView2/$sdkVersion"
|
||||
Write-Host "Downloading WebView2 SDK $sdkVersion for the Windows 7 loader..."
|
||||
Write-Host "Downloading WebView2 SDK $sdkVersion for the Windows 7 / Server 2012 R2 loader..."
|
||||
Invoke-WebRequest -Uri $packageUrl -OutFile $packagePath -UseBasicParsing
|
||||
|
||||
$actualPackageSha256 = (Get-FileHash -LiteralPath $packagePath -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
@@ -44,7 +50,12 @@ try {
|
||||
|
||||
$actualLoaderSha256 = (Get-FileHash -LiteralPath $legacyLoader -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
if ($actualLoaderSha256 -ne $loaderSha256) {
|
||||
throw "Unexpected Windows 7 WebView2 loader SHA256: $actualLoaderSha256"
|
||||
throw "Unexpected Windows 7 / Server 2012 R2 WebView2 loader SHA256: $actualLoaderSha256"
|
||||
}
|
||||
|
||||
$actualLoaderDllSha256 = (Get-FileHash -LiteralPath $legacyLoaderDll -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
if ($actualLoaderDllSha256 -ne $loaderDllSha256) {
|
||||
throw "Unexpected Windows 7 / Server 2012 R2 WebView2 loader DLL SHA256: $actualLoaderDllSha256"
|
||||
}
|
||||
|
||||
Push-Location $repositoryRoot
|
||||
@@ -78,7 +89,7 @@ try {
|
||||
}
|
||||
|
||||
$existingLoaderSha256 = (Get-FileHash -LiteralPath $loaderDestination -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
$knownLoaderHashes = @($upstreamLoaderSha256, $loaderSha256)
|
||||
$knownLoaderHashes = @($upstreamLoaderSha256, $legacy1054LoaderSha256, $loaderSha256)
|
||||
if ($existingLoaderSha256 -notin $knownLoaderHashes) {
|
||||
throw "Refusing to replace an unknown webview2-com-sys loader SHA256: $existingLoaderSha256"
|
||||
}
|
||||
@@ -96,7 +107,7 @@ try {
|
||||
$probeLoader = Join-Path $probeDirectory "WebView2Loader.dll"
|
||||
Copy-Item -LiteralPath $legacyLoaderDll -Destination $probeLoader -Force
|
||||
|
||||
Write-Host "Prepared WebView2 SDK $sdkVersion static loader for Windows 7: $loaderDestination"
|
||||
Write-Host "Prepared WebView2 SDK $sdkVersion static loader for Windows 7 / Server 2012 R2: $loaderDestination"
|
||||
Write-Host "Prepared WebView2 SDK $sdkVersion loader probe DLL: $probeLoader"
|
||||
}
|
||||
finally {
|
||||
|
||||
@@ -15,7 +15,7 @@ import { basename, join } from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
|
||||
const REGISTRY_ASSET = "agent-registry.json";
|
||||
const NATIVE_MODULES = new Set(["duckdb", "oracle", "xugu", "kingbase", "iotdb", "neo4j", "vastbase", "rabbitmq", "rocketmq", "zookeeper", "tdengine"]);
|
||||
const NATIVE_MODULES = new Set(["duckdb", "oracle", "xugu", "kingbase", "iotdb", "neo4j", "vastbase", "rabbitmq", "rocketmq", "zookeeper", "tdengine", "etcd", "etcd2"]);
|
||||
const PLATFORMS = [
|
||||
"macos-aarch64",
|
||||
"macos-x64",
|
||||
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
die() {
|
||||
echo "AppImage input-method verification failed: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
if [[ $# -ne 1 ]]; then
|
||||
die "usage: $0 <AppImage file or bundle directory>"
|
||||
fi
|
||||
|
||||
input_path=$1
|
||||
if [[ -f "$input_path" ]]; then
|
||||
appimage=$input_path
|
||||
elif [[ -d "$input_path" ]]; then
|
||||
appimages=()
|
||||
while IFS= read -r -d '' candidate; do
|
||||
appimages+=("$candidate")
|
||||
done < <(find "$input_path" -maxdepth 1 -type f -name '*.AppImage' -print0)
|
||||
[[ ${#appimages[@]} -eq 1 ]] \
|
||||
|| die "expected exactly one AppImage in $input_path, found ${#appimages[@]}"
|
||||
appimage=${appimages[0]}
|
||||
else
|
||||
die "path does not exist: $input_path"
|
||||
fi
|
||||
appimage=$(realpath "$appimage")
|
||||
|
||||
work_dir=$(mktemp -d "${TMPDIR:-/tmp}/dbx-appimage-input-methods.XXXXXX")
|
||||
trap 'rm -rf "$work_dir"' EXIT
|
||||
|
||||
(
|
||||
cd "$work_dir"
|
||||
"$appimage" --appimage-extract >/dev/null
|
||||
)
|
||||
appdir="$work_dir/squashfs-root"
|
||||
[[ -d "$appdir" ]] || die "AppImage extraction did not create squashfs-root"
|
||||
appdir=$(realpath "$appdir")
|
||||
|
||||
require_artifact() {
|
||||
local basename=$1
|
||||
local match
|
||||
local resolved
|
||||
match=$(find "$appdir" \( -type f -o -type l \) -name "$basename" -print -quit)
|
||||
[[ -n "$match" ]] || die "missing bundled artifact: $basename"
|
||||
[[ -e "$match" ]] || die "bundled artifact is a broken symbolic link: ${match#"$appdir"/}"
|
||||
resolved=$(realpath "$match")
|
||||
[[ "$resolved" == "$appdir"/* ]] \
|
||||
|| die "bundled artifact resolves outside the AppImage: ${match#"$appdir"/}"
|
||||
echo "Found $basename at ${match#"$appdir"/}"
|
||||
}
|
||||
|
||||
require_artifact im-fcitx5.so
|
||||
require_artifact im-ibus.so
|
||||
require_artifact libFcitx5GClient.so.2
|
||||
require_artifact libibus-1.0.so.5
|
||||
|
||||
cache=''
|
||||
while IFS= read -r -d '' candidate; do
|
||||
if grep -Fq 'im-fcitx5.so' "$candidate" && grep -Fq 'im-ibus.so' "$candidate"; then
|
||||
cache=$candidate
|
||||
break
|
||||
fi
|
||||
done < <(find "$appdir" -type f -path '*/gtk-3.0/*/immodules.cache' -print0)
|
||||
|
||||
[[ -n "$cache" ]] || die "no private GTK 3 immodules.cache registers both Fcitx 5 and IBus"
|
||||
echo "Verified GTK input-method cache at ${cache#"$appdir"/}"
|
||||
@@ -147,7 +147,11 @@ jobs:
|
||||
for module in "${MODULES[@]}"; do
|
||||
TASKS+=(":${module}:shadowJar")
|
||||
done
|
||||
./gradlew "${TASKS[@]}" --parallel
|
||||
# agents/common changes trigger a parallel mass rebuild of every
|
||||
# packaged module; the Gradle daemon default heap is too small for
|
||||
# ~25 shadowJar fat jars at once on a 16 GB runner.
|
||||
./gradlew "${TASKS[@]}" --parallel \
|
||||
-Dorg.gradle.jvmargs="-Xmx4g -XX:MaxMetaspaceSize=1g"
|
||||
working-directory: agents
|
||||
- run: python3 scripts/validate_agent_jars.py
|
||||
working-directory: agents
|
||||
@@ -343,6 +347,86 @@ jobs:
|
||||
name: rocketmq-native
|
||||
path: "release-native/dbx-agent-rocketmq-*"
|
||||
|
||||
build-etcd-native:
|
||||
needs: [bump-versions]
|
||||
if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'etcd') }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.26.x"
|
||||
- name: Test etcd native agent
|
||||
working-directory: agents/drivers/etcd-go
|
||||
run: go test ./...
|
||||
- name: Cross-compile etcd native agent
|
||||
shell: bash
|
||||
run: |
|
||||
mkdir -p release-native
|
||||
cd agents/drivers/etcd-go
|
||||
declare -A TARGETS=(
|
||||
["macos-aarch64"]="darwin/arm64"
|
||||
["macos-x64"]="darwin/amd64"
|
||||
["linux-aarch64"]="linux/arm64"
|
||||
["linux-x64"]="linux/amd64"
|
||||
["windows-aarch64"]="windows/arm64"
|
||||
["windows-x64"]="windows/amd64"
|
||||
)
|
||||
for platform in "${!TARGETS[@]}"; do
|
||||
IFS=/ read -r goos goarch <<< "${TARGETS[$platform]}"
|
||||
output="../../../release-native/dbx-agent-etcd-${platform}"
|
||||
if [[ "$goos" == "windows" ]]; then
|
||||
output="${output}.exe"
|
||||
fi
|
||||
echo "Building $platform ($goos/$goarch)"
|
||||
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -trimpath -ldflags="-s -w" -o "$output" .
|
||||
done
|
||||
ls -lh ../../../release-native
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: etcd-native
|
||||
path: "release-native/dbx-agent-etcd-*"
|
||||
|
||||
build-etcd2-native:
|
||||
needs: [bump-versions]
|
||||
if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'etcd2') }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.24.x"
|
||||
- name: Test etcd v2 native agent
|
||||
working-directory: agents/drivers/etcd2-go
|
||||
run: go test ./...
|
||||
- name: Cross-compile etcd v2 native agent
|
||||
shell: bash
|
||||
run: |
|
||||
mkdir -p release-native
|
||||
cd agents/drivers/etcd2-go
|
||||
declare -A TARGETS=(
|
||||
["macos-aarch64"]="darwin/arm64"
|
||||
["macos-x64"]="darwin/amd64"
|
||||
["linux-aarch64"]="linux/arm64"
|
||||
["linux-x64"]="linux/amd64"
|
||||
["windows-aarch64"]="windows/arm64"
|
||||
["windows-x64"]="windows/amd64"
|
||||
)
|
||||
for platform in "${!TARGETS[@]}"; do
|
||||
IFS=/ read -r goos goarch <<< "${TARGETS[$platform]}"
|
||||
output="../../../release-native/dbx-agent-etcd2-${platform}"
|
||||
if [[ "$goos" == "windows" ]]; then
|
||||
output="${output}.exe"
|
||||
fi
|
||||
echo "Building $platform ($goos/$goarch)"
|
||||
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -trimpath -ldflags="-s -w" -o "$output" .
|
||||
done
|
||||
ls -lh ../../../release-native
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: etcd2-native
|
||||
path: "release-native/dbx-agent-etcd2-*"
|
||||
|
||||
build-zookeeper-native:
|
||||
needs: [bump-versions]
|
||||
if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'zookeeper') }}
|
||||
@@ -463,6 +547,46 @@ jobs:
|
||||
name: hive-native
|
||||
path: "release-native/dbx-agent-hive-*"
|
||||
|
||||
build-argo-native:
|
||||
needs: [bump-versions]
|
||||
if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'argo') }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.23.x"
|
||||
- name: Test Argo native agent
|
||||
working-directory: agents/drivers/argo-go
|
||||
run: go test ./...
|
||||
- name: Cross-compile Argo native agent
|
||||
shell: bash
|
||||
run: |
|
||||
mkdir -p release-native
|
||||
cd agents/drivers/argo-go
|
||||
declare -A TARGETS=(
|
||||
["macos-aarch64"]="darwin/arm64"
|
||||
["macos-x64"]="darwin/amd64"
|
||||
["linux-aarch64"]="linux/arm64"
|
||||
["linux-x64"]="linux/amd64"
|
||||
["windows-aarch64"]="windows/arm64"
|
||||
["windows-x64"]="windows/amd64"
|
||||
)
|
||||
for platform in "${!TARGETS[@]}"; do
|
||||
IFS=/ read -r goos goarch <<< "${TARGETS[$platform]}"
|
||||
output="../../../release-native/dbx-agent-argo-${platform}"
|
||||
if [[ "$goos" == "windows" ]]; then
|
||||
output="${output}.exe"
|
||||
fi
|
||||
echo "Building $platform ($goos/$goarch)"
|
||||
CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -trimpath -ldflags="-s -w" -o "$output" .
|
||||
done
|
||||
ls -lh ../../../release-native
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: argo-native
|
||||
path: "release-native/dbx-agent-argo-*"
|
||||
|
||||
build-kingbase-native:
|
||||
needs: [bump-versions]
|
||||
if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'kingbase') }}
|
||||
@@ -662,6 +786,7 @@ jobs:
|
||||
CARGO_INCREMENTAL: "0"
|
||||
CARGO_TARGET_DIR: ${{ github.workspace }}/target/duckdb-driver
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: ${{ secrets.SCCACHE_S3_BUCKET == '' && 'true' || 'false' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: dtolnay/rust-toolchain@1.97.1
|
||||
@@ -681,6 +806,36 @@ jobs:
|
||||
- uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10
|
||||
with:
|
||||
version: "v0.10.0"
|
||||
- name: Configure S3 sccache
|
||||
if: env.SCCACHE_GHA_ENABLED != 'true'
|
||||
shell: bash
|
||||
env:
|
||||
CACHE_BUCKET: ${{ secrets.SCCACHE_S3_BUCKET }}
|
||||
CACHE_ENDPOINT: ${{ secrets.SCCACHE_S3_ENDPOINT }}
|
||||
CACHE_REGION: ${{ secrets.SCCACHE_S3_REGION }}
|
||||
CACHE_KEY_PREFIX: ${{ secrets.SCCACHE_S3_KEY_PREFIX }}
|
||||
CACHE_ACCESS_KEY_ID: ${{ secrets.SCCACHE_S3_ACCESS_KEY_ID }}
|
||||
CACHE_SECRET_ACCESS_KEY: ${{ secrets.SCCACHE_S3_SECRET_ACCESS_KEY }}
|
||||
run: |
|
||||
{
|
||||
echo "SCCACHE_BUCKET=${CACHE_BUCKET}"
|
||||
echo "SCCACHE_ENDPOINT=${CACHE_ENDPOINT}"
|
||||
echo "SCCACHE_REGION=${CACHE_REGION}"
|
||||
echo "SCCACHE_S3_KEY_PREFIX=${CACHE_KEY_PREFIX}"
|
||||
echo "SCCACHE_S3_USE_SSL=true"
|
||||
echo "AWS_ACCESS_KEY_ID=${CACHE_ACCESS_KEY_ID}"
|
||||
echo "AWS_SECRET_ACCESS_KEY=${CACHE_SECRET_ACCESS_KEY}"
|
||||
echo "SCCACHE_IDLE_TIMEOUT=0"
|
||||
# libduckdb C++ compilation dominates this build. The manylinux
|
||||
# legs build inside a container and enable CC/CXX wrapping there;
|
||||
# MSVC stays unwrapped.
|
||||
case "${RUNNER_OS}" in
|
||||
macOS)
|
||||
echo "CC=${SCCACHE_PATH} clang"
|
||||
echo "CXX=${SCCACHE_PATH} clang++"
|
||||
;;
|
||||
esac
|
||||
} >> "$GITHUB_ENV"
|
||||
- name: Build DuckDB native driver
|
||||
shell: bash
|
||||
run: |
|
||||
@@ -693,6 +848,15 @@ jobs:
|
||||
-e HOME=/tmp/dbx-rust-home \
|
||||
-e CARGO_INCREMENTAL=0 \
|
||||
-e CARGO_TARGET_DIR=/workspace/target/duckdb-driver \
|
||||
-e RUSTC_WRAPPER \
|
||||
-e SCCACHE_BUCKET \
|
||||
-e SCCACHE_ENDPOINT \
|
||||
-e SCCACHE_REGION \
|
||||
-e SCCACHE_S3_KEY_PREFIX \
|
||||
-e SCCACHE_S3_USE_SSL \
|
||||
-e AWS_ACCESS_KEY_ID \
|
||||
-e AWS_SECRET_ACCESS_KEY \
|
||||
-e SCCACHE_IDLE_TIMEOUT \
|
||||
-v "${{ github.workspace }}:/workspace" \
|
||||
-w /workspace/agents/drivers/duckdb \
|
||||
"${{ matrix.manylinux_image }}" \
|
||||
@@ -700,12 +864,27 @@ jobs:
|
||||
mkdir -p "$HOME"
|
||||
curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain 1.97.1
|
||||
source "$HOME/.cargo/env"
|
||||
if [ -n "${SCCACHE_BUCKET:-}" ]; then
|
||||
case "$(uname -m)" in
|
||||
x86_64) SCCACHE_ARCH=x86_64-unknown-linux-musl ;;
|
||||
aarch64) SCCACHE_ARCH=aarch64-unknown-linux-musl ;;
|
||||
esac
|
||||
curl -sSfL "https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-${SCCACHE_ARCH}.tar.gz" \
|
||||
| tar -xz --strip-components=1 -C "$HOME/.cargo/bin"
|
||||
export CC="sccache cc" CXX="sccache c++"
|
||||
sccache --version
|
||||
else
|
||||
unset RUSTC_WRAPPER
|
||||
fi
|
||||
cargo build \
|
||||
--manifest-path Cargo.toml \
|
||||
--locked \
|
||||
--release \
|
||||
--bin dbx-duckdb-driver \
|
||||
--target "${{ matrix.target }}"
|
||||
if [ -n "${SCCACHE_BUCKET:-}" ]; then
|
||||
sccache --show-stats || true
|
||||
fi
|
||||
'
|
||||
else
|
||||
BUILD_STD=()
|
||||
@@ -737,11 +916,67 @@ jobs:
|
||||
"${{ matrix.manylinux_image }}" \
|
||||
"/driver/dbx-agent-duckdb-${{ matrix.platform }}" < /dev/null
|
||||
fi
|
||||
- name: Show sccache stats
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
shell: bash
|
||||
run: ${SCCACHE_PATH} --show-stats
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: duckdb-native-${{ matrix.platform }}
|
||||
path: "release-native/dbx-agent-duckdb-*"
|
||||
|
||||
build-sqlite-worker-native:
|
||||
needs: [bump-versions]
|
||||
if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'sqlite-worker') }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- runner: ubuntu-22.04
|
||||
target: x86_64-unknown-linux-musl
|
||||
platform: linux-x64
|
||||
rustflags: "-C target-feature=+crt-static"
|
||||
- runner: ubuntu-22.04-arm
|
||||
target: aarch64-unknown-linux-musl
|
||||
platform: linux-aarch64
|
||||
rustflags: "-C target-feature=+crt-static -C link-arg=-z -C link-arg=max-page-size=65536"
|
||||
runs-on: ${{ matrix.runner }}
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
|
||||
- name: Setup Rust
|
||||
uses: dtolnay/rust-toolchain@1.97.1
|
||||
with:
|
||||
targets: ${{ matrix.target }}
|
||||
|
||||
- name: Install cargo-zigbuild
|
||||
run: pip3 install cargo-zigbuild
|
||||
|
||||
- name: Rust cache
|
||||
uses: swatinem/rust-cache@v2
|
||||
with:
|
||||
workspaces: "./ -> target"
|
||||
shared-key: sqlite-worker-${{ matrix.target }}
|
||||
cache-on-failure: true
|
||||
|
||||
- name: Build musl SQLite SSH worker
|
||||
env:
|
||||
RUSTFLAGS: ${{ matrix.rustflags }}
|
||||
run: cargo zigbuild --release -p dbx-sqlite-worker --target ${{ matrix.target }}
|
||||
|
||||
- name: Stage SQLite SSH worker
|
||||
run: |
|
||||
mkdir -p release-native
|
||||
cp \
|
||||
"target/${{ matrix.target }}/release/dbx-sqlite-worker" \
|
||||
"release-native/dbx-agent-sqlite-worker-${{ matrix.platform }}"
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: sqlite-worker-native-${{ matrix.platform }}
|
||||
path: "release-native/dbx-agent-sqlite-worker-*"
|
||||
|
||||
build-tdengine-native:
|
||||
needs: [bump-versions]
|
||||
if: ${{ contains(fromJSON(needs.bump-versions.outputs.native_modules), 'tdengine') }}
|
||||
@@ -784,6 +1019,7 @@ jobs:
|
||||
CARGO_INCREMENTAL: "0"
|
||||
CARGO_TARGET_DIR: ${{ github.workspace }}/target/tdengine-driver
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: ${{ secrets.SCCACHE_S3_BUCKET == '' && 'true' || 'false' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: dtolnay/rust-toolchain@1.97.1
|
||||
@@ -799,6 +1035,36 @@ jobs:
|
||||
- uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10
|
||||
with:
|
||||
version: "v0.10.0"
|
||||
- name: Configure S3 sccache
|
||||
if: env.SCCACHE_GHA_ENABLED != 'true'
|
||||
shell: bash
|
||||
env:
|
||||
CACHE_BUCKET: ${{ secrets.SCCACHE_S3_BUCKET }}
|
||||
CACHE_ENDPOINT: ${{ secrets.SCCACHE_S3_ENDPOINT }}
|
||||
CACHE_REGION: ${{ secrets.SCCACHE_S3_REGION }}
|
||||
CACHE_KEY_PREFIX: ${{ secrets.SCCACHE_S3_KEY_PREFIX }}
|
||||
CACHE_ACCESS_KEY_ID: ${{ secrets.SCCACHE_S3_ACCESS_KEY_ID }}
|
||||
CACHE_SECRET_ACCESS_KEY: ${{ secrets.SCCACHE_S3_SECRET_ACCESS_KEY }}
|
||||
run: |
|
||||
{
|
||||
echo "SCCACHE_BUCKET=${CACHE_BUCKET}"
|
||||
echo "SCCACHE_ENDPOINT=${CACHE_ENDPOINT}"
|
||||
echo "SCCACHE_REGION=${CACHE_REGION}"
|
||||
echo "SCCACHE_S3_KEY_PREFIX=${CACHE_KEY_PREFIX}"
|
||||
echo "SCCACHE_S3_USE_SSL=true"
|
||||
echo "AWS_ACCESS_KEY_ID=${CACHE_ACCESS_KEY_ID}"
|
||||
echo "AWS_SECRET_ACCESS_KEY=${CACHE_SECRET_ACCESS_KEY}"
|
||||
echo "SCCACHE_IDLE_TIMEOUT=0"
|
||||
# taos client C/C++ compilation dominates this build. The manylinux
|
||||
# legs build inside a container and enable CC/CXX wrapping there;
|
||||
# MSVC stays unwrapped.
|
||||
case "${RUNNER_OS}" in
|
||||
macOS)
|
||||
echo "CC=${SCCACHE_PATH} clang"
|
||||
echo "CXX=${SCCACHE_PATH} clang++"
|
||||
;;
|
||||
esac
|
||||
} >> "$GITHUB_ENV"
|
||||
- name: Build TDengine native driver
|
||||
shell: bash
|
||||
run: |
|
||||
@@ -808,6 +1074,15 @@ jobs:
|
||||
-e HOME=/tmp/dbx-rust-home \
|
||||
-e CARGO_INCREMENTAL=0 \
|
||||
-e CARGO_TARGET_DIR=/workspace/target/tdengine-driver \
|
||||
-e RUSTC_WRAPPER \
|
||||
-e SCCACHE_BUCKET \
|
||||
-e SCCACHE_ENDPOINT \
|
||||
-e SCCACHE_REGION \
|
||||
-e SCCACHE_S3_KEY_PREFIX \
|
||||
-e SCCACHE_S3_USE_SSL \
|
||||
-e AWS_ACCESS_KEY_ID \
|
||||
-e AWS_SECRET_ACCESS_KEY \
|
||||
-e SCCACHE_IDLE_TIMEOUT \
|
||||
-v "${{ github.workspace }}:/workspace" \
|
||||
-w /workspace \
|
||||
"${{ matrix.manylinux_image }}" \
|
||||
@@ -815,12 +1090,27 @@ jobs:
|
||||
mkdir -p "$HOME"
|
||||
curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain 1.97.1
|
||||
source "$HOME/.cargo/env"
|
||||
if [ -n "${SCCACHE_BUCKET:-}" ]; then
|
||||
case "$(uname -m)" in
|
||||
x86_64) SCCACHE_ARCH=x86_64-unknown-linux-musl ;;
|
||||
aarch64) SCCACHE_ARCH=aarch64-unknown-linux-musl ;;
|
||||
esac
|
||||
curl -sSfL "https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-${SCCACHE_ARCH}.tar.gz" \
|
||||
| tar -xz --strip-components=1 -C "$HOME/.cargo/bin"
|
||||
export CC="sccache cc" CXX="sccache c++"
|
||||
sccache --version
|
||||
else
|
||||
unset RUSTC_WRAPPER
|
||||
fi
|
||||
cargo build \
|
||||
--manifest-path agents/drivers/tdengine/Cargo.toml \
|
||||
--locked \
|
||||
--release \
|
||||
--bin dbx-tdengine-driver \
|
||||
--target "${{ matrix.target }}"
|
||||
if [ -n "${SCCACHE_BUCKET:-}" ]; then
|
||||
sccache --show-stats || true
|
||||
fi
|
||||
'
|
||||
else
|
||||
BUILD_STD=()
|
||||
@@ -848,6 +1138,11 @@ jobs:
|
||||
elif [ "${{ matrix.smoke }}" = "true" ]; then
|
||||
"release-native/dbx-agent-tdengine-${{ matrix.platform }}${{ matrix.extension }}" < /dev/null
|
||||
fi
|
||||
- name: Show sccache stats
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
shell: bash
|
||||
run: ${SCCACHE_PATH} --show-stats
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: tdengine-native-${{ matrix.platform }}
|
||||
@@ -962,7 +1257,7 @@ jobs:
|
||||
retention-days: 1
|
||||
|
||||
release:
|
||||
needs: [bump-versions, commit-versions, build-agents, build-oracle-native, build-xugu-native, build-rabbitmq-native, build-rocketmq-native, build-zookeeper-native, build-cassandra-native, build-hive-native, build-kingbase-native, build-vastbase-native, build-neo4j-native, build-iotdb-native, build-duckdb-native, build-tdengine-native, build-jre, reuse-previous-assets]
|
||||
needs: [bump-versions, commit-versions, build-agents, build-oracle-native, build-xugu-native, build-rabbitmq-native, build-rocketmq-native, build-etcd-native, build-etcd2-native, build-zookeeper-native, build-cassandra-native, build-hive-native, build-argo-native, build-kingbase-native, build-vastbase-native, build-neo4j-native, build-iotdb-native, build-duckdb-native, build-sqlite-worker-native, build-tdengine-native, build-jre, reuse-previous-assets]
|
||||
if: ${{ always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
@@ -1087,15 +1382,19 @@ jobs:
|
||||
kingbase) echo "金仓KingbaseES" ;;
|
||||
vastbase) echo "Vastbase" ;;
|
||||
duckdb) echo "DuckDB" ;;
|
||||
sqlite-worker) echo "SQLite SSH Worker" ;;
|
||||
xugu) echo "虚谷 XuguDB" ;;
|
||||
rabbitmq) echo "RabbitMQ" ;;
|
||||
rocketmq) echo "Apache RocketMQ" ;;
|
||||
zookeeper) echo "Apache ZooKeeper" ;;
|
||||
cassandra) echo "Apache Cassandra" ;;
|
||||
hive) echo "Apache Hive" ;;
|
||||
argo) echo "星环Argo" ;;
|
||||
neo4j) echo "Neo4j" ;;
|
||||
iotdb) echo "Apache IoTDB" ;;
|
||||
tdengine) echo "TDengine" ;;
|
||||
etcd) echo "etcd" ;;
|
||||
etcd2) echo "etcd 2.x (v2 API)" ;;
|
||||
*) echo "$name" ;;
|
||||
esac
|
||||
}
|
||||
@@ -1159,7 +1458,7 @@ jobs:
|
||||
[ -n "$DRIVERS" ] && DRIVERS="${DRIVERS},"$'\n'
|
||||
DRIVERS="${DRIVERS}$(generate_jar_entry "$name" "$label" "$f" "$jre_key" "$version" "$external_driver" "$native_json")"
|
||||
done
|
||||
for name in oracle xugu kingbase vastbase neo4j iotdb duckdb rabbitmq rocketmq zookeeper cassandra hive tdengine; do
|
||||
for name in oracle xugu kingbase vastbase neo4j iotdb duckdb sqlite-worker rabbitmq rocketmq zookeeper cassandra hive argo tdengine etcd etcd2; do
|
||||
version=$(get_module_version "$name")
|
||||
[ -f "release/dbx-agent-${name}-${version}.jar" ] && continue
|
||||
native_json=$(generate_native_platforms "$name" "$version")
|
||||
@@ -1239,6 +1538,7 @@ jobs:
|
||||
kingbase) echo "金仓KingbaseES" ;;
|
||||
vastbase) echo "Vastbase" ;;
|
||||
duckdb) echo "DuckDB" ;;
|
||||
sqlite-worker) echo "SQLite SSH Worker" ;;
|
||||
oracle) echo "Oracle" ;;
|
||||
xugu) echo "虚谷 XuguDB" ;;
|
||||
rabbitmq) echo "RabbitMQ" ;;
|
||||
@@ -1246,13 +1546,33 @@ jobs:
|
||||
zookeeper) echo "Apache ZooKeeper" ;;
|
||||
cassandra) echo "Apache Cassandra" ;;
|
||||
hive) echo "Apache Hive" ;;
|
||||
argo) echo "星环Argo" ;;
|
||||
neo4j) echo "Neo4j" ;;
|
||||
iotdb) echo "Apache IoTDB" ;;
|
||||
tdengine) echo "TDengine" ;;
|
||||
etcd) echo "etcd" ;;
|
||||
etcd2) echo "etcd 2.x (v2 API)" ;;
|
||||
*) echo "$name" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# agents/common rebuilds every packaging driver; report common-only
|
||||
# commits once in a shared section instead of duplicating them under
|
||||
# every driver. Commits that also touch driver code stay attributed
|
||||
# to that driver alone.
|
||||
COMMON_NOTES=""
|
||||
while IFS= read -r line; do
|
||||
sha="${line%% *}"
|
||||
subject="${line#* }"
|
||||
if git diff-tree --no-commit-id --name-only -r "$sha" | grep -qv '^agents/common/'; then
|
||||
continue
|
||||
fi
|
||||
COMMON_NOTES="${COMMON_NOTES}- ${subject}"$'\n'
|
||||
done < <(git log --no-merges --format='%H %s' "$PREV_TAG"..HEAD -- agents/common/build.gradle agents/common/src/main/)
|
||||
if [ -n "$COMMON_NOTES" ]; then
|
||||
NOTES="${NOTES}### 公共组件 (agents/common)"$'\n'"${COMMON_NOTES}"$'\n'
|
||||
fi
|
||||
|
||||
for name in "${DRIVER_NAMES[@]}"; do
|
||||
old_ver=$(echo "$PREV_VERSIONS" | python3 -c "import sys,json; print(json.load(sys.stdin).get('$name',''))")
|
||||
new_ver=$(echo "$MODULE_VERSIONS" | python3 -c "import sys,json; print(json.load(sys.stdin).get('$name',''))")
|
||||
@@ -1266,7 +1586,6 @@ jobs:
|
||||
label=$(native_only_label "$name")
|
||||
fi
|
||||
|
||||
NOTES="${NOTES}### ${label} (${old_ver} → ${new_ver})"$'\n'
|
||||
if [ "$name" = "oracle" ]; then
|
||||
LOG_PATH="agents/drivers/oracle-go/"
|
||||
elif [ "$name" = "kingbase" ]; then
|
||||
@@ -1277,10 +1596,14 @@ jobs:
|
||||
LOG_PATH="agents/drivers/cassandra-go/"
|
||||
elif [ "$name" = "hive" ]; then
|
||||
LOG_PATH="agents/drivers/hive-go/"
|
||||
elif [ "$name" = "argo" ]; then
|
||||
LOG_PATH="agents/drivers/argo-go/"
|
||||
elif [ "$name" = "neo4j" ]; then
|
||||
LOG_PATH="agents/drivers/neo4j-go/"
|
||||
elif [ "$name" = "iotdb" ]; then
|
||||
LOG_PATH="agents/drivers/iotdb/"
|
||||
elif [ "$name" = "sqlite-worker" ]; then
|
||||
LOG_PATH="crates/dbx-sqlite-worker/"
|
||||
elif [ -d "agents/drivers/$name" ]; then
|
||||
LOG_PATH="agents/drivers/$name/"
|
||||
else
|
||||
@@ -1293,21 +1616,26 @@ jobs:
|
||||
"agents/go-common/gohive/"
|
||||
"agents/go-common/gosasl/"
|
||||
)
|
||||
elif [ "$name" = "argo" ]; then
|
||||
LOG_PATHS+=(
|
||||
"agents/go-common/go-gssapi/"
|
||||
"agents/go-common/gohive/"
|
||||
"agents/go-common/gosasl/"
|
||||
)
|
||||
elif [ "$name" = "zookeeper" ]; then
|
||||
LOG_PATHS+=(
|
||||
"agents/go-common/go-gssapi/"
|
||||
"agents/go-common/gosasl/"
|
||||
)
|
||||
fi
|
||||
DRIVER_NOTES=""
|
||||
while IFS= read -r line; do
|
||||
[ -n "$line" ] && NOTES="${NOTES}- ${line}"$'\n'
|
||||
[ -n "$line" ] && DRIVER_NOTES="${DRIVER_NOTES}- ${line}"$'\n'
|
||||
done < <(git log --oneline "$PREV_TAG"..HEAD -- "${LOG_PATHS[@]}" | sed 's/^[0-9a-f]* //')
|
||||
|
||||
# include common changes if any
|
||||
while IFS= read -r line; do
|
||||
[ -n "$line" ] && NOTES="${NOTES}- ${line}"$'\n'
|
||||
done < <(git log --oneline "$PREV_TAG"..HEAD -- agents/common/ | sed 's/^[0-9a-f]* //')
|
||||
NOTES="${NOTES}"$'\n'
|
||||
if [ -z "$DRIVER_NOTES" ]; then
|
||||
DRIVER_NOTES="- 无驱动专属变更"$'\n'
|
||||
fi
|
||||
NOTES="${NOTES}### ${label} (${old_ver} → ${new_ver})"$'\n'"${DRIVER_NOTES}"$'\n'
|
||||
done
|
||||
|
||||
if [ -z "$NOTES" ]; then
|
||||
|
||||
@@ -71,6 +71,13 @@ jobs:
|
||||
echo "SCCACHE_S3_USE_SSL=true"
|
||||
echo "AWS_ACCESS_KEY_ID=${CACHE_ACCESS_KEY_ID}"
|
||||
echo "AWS_SECRET_ACCESS_KEY=${CACHE_SECRET_ACCESS_KEY}"
|
||||
# Keep the server alive through post-compile test/lint phases so
|
||||
# "Show sccache stats" reflects the real counters.
|
||||
echo "SCCACHE_IDLE_TIMEOUT=0"
|
||||
# Also cache the C/C++ compilation of -sys crates. These jobs are
|
||||
# native Linux builds, so plain CC/CXX select the target compiler.
|
||||
echo "CC=${SCCACHE_PATH} cc"
|
||||
echo "CXX=${SCCACHE_PATH} c++"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
- name: Rust cache
|
||||
@@ -193,7 +200,7 @@ jobs:
|
||||
needs: changes
|
||||
if: needs.changes.outputs.windows_win7_bundle == 'true'
|
||||
runs-on: windows-2022
|
||||
timeout-minutes: 90
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
CARGO_INCREMENTAL: "0"
|
||||
RUSTFLAGS: -C debuginfo=line-tables-only -C target-feature=+crt-static
|
||||
@@ -251,6 +258,10 @@ jobs:
|
||||
shell: pwsh
|
||||
run: ./.github/scripts/assert-win7-pe-compat.ps1 -BinaryPath target/x86_64-win7-windows-msvc/release/dbx.exe
|
||||
|
||||
- name: Audit Win7 WebView2 loader markers
|
||||
shell: pwsh
|
||||
run: ./.github/scripts/assert-webview2-win7-loader.ps1 -BinaryPath target/x86_64-win7-windows-msvc/release/dbx.exe
|
||||
|
||||
- name: Bundle Windows 7 fixed-runtime installer
|
||||
shell: pwsh
|
||||
run: |
|
||||
@@ -378,6 +389,13 @@ jobs:
|
||||
echo "SCCACHE_S3_USE_SSL=true"
|
||||
echo "AWS_ACCESS_KEY_ID=${CACHE_ACCESS_KEY_ID}"
|
||||
echo "AWS_SECRET_ACCESS_KEY=${CACHE_SECRET_ACCESS_KEY}"
|
||||
# Keep the server alive through post-compile test/lint phases so
|
||||
# "Show sccache stats" reflects the real counters.
|
||||
echo "SCCACHE_IDLE_TIMEOUT=0"
|
||||
# Also cache the C/C++ compilation of -sys crates. These jobs are
|
||||
# native Linux builds, so plain CC/CXX select the target compiler.
|
||||
echo "CC=${SCCACHE_PATH} cc"
|
||||
echo "CXX=${SCCACHE_PATH} c++"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
- name: Rust cache
|
||||
@@ -413,6 +431,8 @@ jobs:
|
||||
env:
|
||||
# sccache cannot reuse Cargo incremental artifacts, so avoid generating them in CI.
|
||||
CARGO_INCREMENTAL: "0"
|
||||
# Deep async export tests exceed the Rust test harness's default thread stack on Linux.
|
||||
RUST_MIN_STACK: 8388608
|
||||
RUSTC_WRAPPER: sccache
|
||||
# The fast lane skips only system font discovery while retaining the other default capabilities.
|
||||
RUST_FEATURE_MODE: ${{ github.event_name == 'pull_request' && needs.changes.outputs.rust_full != 'true' && 'fast' || 'full' }}
|
||||
@@ -455,6 +475,13 @@ jobs:
|
||||
echo "SCCACHE_S3_USE_SSL=true"
|
||||
echo "AWS_ACCESS_KEY_ID=${CACHE_ACCESS_KEY_ID}"
|
||||
echo "AWS_SECRET_ACCESS_KEY=${CACHE_SECRET_ACCESS_KEY}"
|
||||
# Keep the server alive through post-compile test/lint phases so
|
||||
# "Show sccache stats" reflects the real counters.
|
||||
echo "SCCACHE_IDLE_TIMEOUT=0"
|
||||
# Also cache the C/C++ compilation of -sys crates. These jobs are
|
||||
# native Linux builds, so plain CC/CXX select the target compiler.
|
||||
echo "CC=${SCCACHE_PATH} cc"
|
||||
echo "CXX=${SCCACHE_PATH} c++"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
- name: Rust cache
|
||||
@@ -675,6 +702,7 @@ jobs:
|
||||
- '.github/workflows/update-nix-pnpm-hash.yml'
|
||||
windows_win7_bundle:
|
||||
- '.github/scripts/assert-win7-pe-compat.ps1'
|
||||
- '.github/scripts/assert-webview2-win7-loader.ps1'
|
||||
- '.github/scripts/assert-win7-installer-content.ps1'
|
||||
- '.github/scripts/assert-webview2-win7-runtime.ps1'
|
||||
- '.github/scripts/prepare-webview2-win7-loader.ps1'
|
||||
|
||||
@@ -343,6 +343,22 @@ jobs:
|
||||
echo "SCCACHE_S3_USE_SSL=true"
|
||||
echo "AWS_ACCESS_KEY_ID=${CACHE_ACCESS_KEY_ID}"
|
||||
echo "AWS_SECRET_ACCESS_KEY=${CACHE_SECRET_ACCESS_KEY}"
|
||||
# Keep the server alive through the post-compile packaging tail so
|
||||
# "Show sccache stats" reflects the real counters.
|
||||
echo "SCCACHE_IDLE_TIMEOUT=0"
|
||||
# Also cache the C/C++ compilation of -sys crates. All matrix legs
|
||||
# are native builds, so plain CC/CXX select the target compiler;
|
||||
# MSVC stays unwrapped. SCCACHE_PATH avoids depending on PATH.
|
||||
case "${RUNNER_OS}" in
|
||||
Linux)
|
||||
echo "CC=${SCCACHE_PATH} cc"
|
||||
echo "CXX=${SCCACHE_PATH} c++"
|
||||
;;
|
||||
macOS)
|
||||
echo "CC=${SCCACHE_PATH} clang"
|
||||
echo "CXX=${SCCACHE_PATH} clang++"
|
||||
;;
|
||||
esac
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
- name: Rust cache
|
||||
@@ -458,11 +474,13 @@ jobs:
|
||||
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
VERSION: ${{ needs.prepare.outputs.version }}
|
||||
run: |
|
||||
# npm CDN visibility after publish has been observed to lag past
|
||||
# three minutes; the window must exceed that by a wide margin.
|
||||
wait_for_package() {
|
||||
package="$1"
|
||||
for attempt in $(seq 1 18); do
|
||||
for attempt in $(seq 1 60); do
|
||||
if npm view "${package}@${VERSION}" version >/dev/null 2>&1; then return 0; fi
|
||||
echo "Waiting for ${package}@${VERSION} to become visible on npm (${attempt}/18)."
|
||||
echo "Waiting for ${package}@${VERSION} to become visible on npm (${attempt}/60)."
|
||||
sleep 10
|
||||
done
|
||||
return 1
|
||||
@@ -561,6 +579,22 @@ jobs:
|
||||
echo "SCCACHE_S3_USE_SSL=true"
|
||||
echo "AWS_ACCESS_KEY_ID=${CACHE_ACCESS_KEY_ID}"
|
||||
echo "AWS_SECRET_ACCESS_KEY=${CACHE_SECRET_ACCESS_KEY}"
|
||||
# Keep the server alive through the post-compile packaging tail so
|
||||
# "Show sccache stats" reflects the real counters.
|
||||
echo "SCCACHE_IDLE_TIMEOUT=0"
|
||||
# Also cache the C/C++ compilation of -sys crates. All matrix legs
|
||||
# are native builds, so plain CC/CXX select the target compiler;
|
||||
# MSVC stays unwrapped. SCCACHE_PATH avoids depending on PATH.
|
||||
case "${RUNNER_OS}" in
|
||||
Linux)
|
||||
echo "CC=${SCCACHE_PATH} cc"
|
||||
echo "CXX=${SCCACHE_PATH} c++"
|
||||
;;
|
||||
macOS)
|
||||
echo "CC=${SCCACHE_PATH} clang"
|
||||
echo "CXX=${SCCACHE_PATH} clang++"
|
||||
;;
|
||||
esac
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
- name: Rust cache
|
||||
@@ -685,11 +719,11 @@ jobs:
|
||||
run: |
|
||||
wait_for_package() {
|
||||
package="$1"
|
||||
for attempt in $(seq 1 18); do
|
||||
for attempt in $(seq 1 60); do
|
||||
if npm view "${package}@${VERSION}" version >/dev/null 2>&1; then
|
||||
return 0
|
||||
fi
|
||||
echo "Waiting for ${package}@${VERSION} to become visible on npm (${attempt}/18)."
|
||||
echo "Waiting for ${package}@${VERSION} to become visible on npm (${attempt}/60)."
|
||||
sleep 10
|
||||
done
|
||||
return 1
|
||||
|
||||
@@ -47,7 +47,7 @@ jobs:
|
||||
TAG="${{ steps.version.outputs.tag }}"
|
||||
|
||||
ASSETS=(
|
||||
"${PRODUCT_NAME}_${VERSION}_aarch64.dmg"
|
||||
"${PRODUCT_NAME}_${VERSION}_arm64.dmg"
|
||||
"${PRODUCT_NAME}_${VERSION}_x64.dmg"
|
||||
"${PRODUCT_NAME}_${VERSION}_x64-setup.exe"
|
||||
"${PRODUCT_NAME}_${VERSION}_arm64-setup.exe"
|
||||
@@ -83,7 +83,7 @@ jobs:
|
||||
}
|
||||
|
||||
echo "SHA256 hashes:"
|
||||
compute_hash "artifacts/${PRODUCT_NAME}_${VERSION}_aarch64.dmg" "sha_dmg_arm64"
|
||||
compute_hash "artifacts/${PRODUCT_NAME}_${VERSION}_arm64.dmg" "sha_dmg_arm64"
|
||||
compute_hash "artifacts/${PRODUCT_NAME}_${VERSION}_x64.dmg" "sha_dmg_x64"
|
||||
compute_hash "artifacts/${PRODUCT_NAME}_${VERSION}_x64-setup.exe" "sha_exe_x64"
|
||||
compute_hash "artifacts/${PRODUCT_NAME}_${VERSION}_arm64-setup.exe" "sha_exe_arm64"
|
||||
@@ -104,7 +104,7 @@ jobs:
|
||||
|
||||
cat > Casks/dbx.rb <<'RUBY_EOF'
|
||||
cask "dbx" do
|
||||
arch arm: "aarch64", intel: "x64"
|
||||
arch arm: "arm64", intel: "x64"
|
||||
|
||||
version "__VERSION__"
|
||||
sha256 arm: "__SHA_ARM64__",
|
||||
|
||||
+133
-46
@@ -117,19 +117,28 @@ jobs:
|
||||
done
|
||||
|
||||
build:
|
||||
# Explicit name keeps matrix-only fields such as asset_pattern from being
|
||||
# appended to the generated job display name.
|
||||
name: build (${{ matrix.platform }}, ${{ matrix.target }})
|
||||
needs: release-ready
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
# assetNamePattern renames every bundle artifact of a matrix leg, so
|
||||
# it is set only on aarch64 legs (aarch64 -> arm64). x64 legs leave
|
||||
# it empty and keep the default Tauri CLI names (x64.dmg, amd64
|
||||
# deb/rpm/AppImage) used by downloadLinks and package consumers.
|
||||
- platform: macos-latest
|
||||
target: aarch64-apple-darwin
|
||||
asset_pattern: "DBX_[version]_arm64[setup][ext]"
|
||||
- platform: macos-15-intel
|
||||
target: x86_64-apple-darwin
|
||||
- platform: ubuntu-22.04
|
||||
target: x86_64-unknown-linux-gnu
|
||||
- platform: ubuntu-22.04-arm
|
||||
target: aarch64-unknown-linux-gnu
|
||||
asset_pattern: "DBX_[version]_arm64[setup][ext]"
|
||||
|
||||
runs-on: ${{ matrix.platform }}
|
||||
env:
|
||||
@@ -180,6 +189,23 @@ jobs:
|
||||
echo "SCCACHE_S3_USE_SSL=true"
|
||||
echo "AWS_ACCESS_KEY_ID=${CACHE_ACCESS_KEY_ID}"
|
||||
echo "AWS_SECRET_ACCESS_KEY=${CACHE_SECRET_ACCESS_KEY}"
|
||||
# Linking, bundling, and signing run for 30+ minutes after the last
|
||||
# rustc call; the server's 10-minute default idle timeout would
|
||||
# exit first and zero the counters shown by "Show sccache stats".
|
||||
echo "SCCACHE_IDLE_TIMEOUT=0"
|
||||
# Also cache the C/C++ compilation of -sys crates. All matrix legs
|
||||
# are native builds, so plain CC/CXX select the target compiler;
|
||||
# MSVC stays unwrapped. SCCACHE_PATH avoids depending on PATH.
|
||||
case "${RUNNER_OS}" in
|
||||
Linux)
|
||||
echo "CC=${SCCACHE_PATH} cc"
|
||||
echo "CXX=${SCCACHE_PATH} c++"
|
||||
;;
|
||||
macOS)
|
||||
echo "CC=${SCCACHE_PATH} clang"
|
||||
echo "CXX=${SCCACHE_PATH} clang++"
|
||||
;;
|
||||
esac
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
- name: Compute Rust dependency hash
|
||||
@@ -226,7 +252,9 @@ jobs:
|
||||
if: startsWith(matrix.platform, 'ubuntu')
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y libwebkit2gtk-4.1-dev libgtk-3-dev libappindicator3-dev librsvg2-dev patchelf libssl-dev xdg-utils
|
||||
# Bundle GTK input-method modules from the same Jammy ABI baseline
|
||||
# instead of loading potentially incompatible modules from the host.
|
||||
sudo apt-get install -y fcitx5-frontend-gtk3 ibus-gtk3 libwebkit2gtk-4.1-dev libgtk-3-dev libappindicator3-dev librsvg2-dev patchelf libssl-dev xdg-utils
|
||||
|
||||
- name: Setup Tauri signing key
|
||||
shell: bash
|
||||
@@ -249,8 +277,15 @@ jobs:
|
||||
with:
|
||||
tagName: ${{ github.ref_name }}
|
||||
releaseDraft: true
|
||||
# Empty on x64 legs: tauri-action then keeps the default Tauri CLI names.
|
||||
assetNamePattern: ${{ matrix.asset_pattern }}
|
||||
args: --target ${{ matrix.target }}
|
||||
|
||||
- name: Verify AppImage input methods
|
||||
if: startsWith(matrix.platform, 'ubuntu')
|
||||
shell: bash
|
||||
run: ./.github/scripts/verify-appimage-input-methods.sh "target/${{ matrix.target }}/release/bundle/appimage"
|
||||
|
||||
- name: Show sccache stats
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
@@ -266,10 +301,8 @@ jobs:
|
||||
include:
|
||||
- target: x86_64-pc-windows-msvc
|
||||
arch: x64
|
||||
sign_windows: true
|
||||
- target: aarch64-pc-windows-msvc
|
||||
arch: arm64
|
||||
sign_windows: false
|
||||
runs-on: windows-2022
|
||||
env:
|
||||
CARGO_INCREMENTAL: "0"
|
||||
@@ -322,6 +355,23 @@ jobs:
|
||||
echo "SCCACHE_S3_USE_SSL=true"
|
||||
echo "AWS_ACCESS_KEY_ID=${CACHE_ACCESS_KEY_ID}"
|
||||
echo "AWS_SECRET_ACCESS_KEY=${CACHE_SECRET_ACCESS_KEY}"
|
||||
# Linking, bundling, and signing run for 30+ minutes after the last
|
||||
# rustc call; the server's 10-minute default idle timeout would
|
||||
# exit first and zero the counters shown by "Show sccache stats".
|
||||
echo "SCCACHE_IDLE_TIMEOUT=0"
|
||||
# Also cache the C/C++ compilation of -sys crates. All matrix legs
|
||||
# are native builds, so plain CC/CXX select the target compiler;
|
||||
# MSVC stays unwrapped. SCCACHE_PATH avoids depending on PATH.
|
||||
case "${RUNNER_OS}" in
|
||||
Linux)
|
||||
echo "CC=${SCCACHE_PATH} cc"
|
||||
echo "CXX=${SCCACHE_PATH} c++"
|
||||
;;
|
||||
macOS)
|
||||
echo "CC=${SCCACHE_PATH} clang"
|
||||
echo "CXX=${SCCACHE_PATH} clang++"
|
||||
;;
|
||||
esac
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
- name: Compute Rust dependency hash
|
||||
@@ -358,7 +408,6 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Setup TrustAsia VSign
|
||||
if: matrix.sign_windows
|
||||
shell: pwsh
|
||||
env:
|
||||
VSIGN_OPERATOR_PFX_B64: ${{ secrets.VSIGN_OPERATOR_PFX_B64 }}
|
||||
@@ -366,7 +415,6 @@ jobs:
|
||||
run: ./.github/scripts/setup-vsign.ps1
|
||||
|
||||
- name: Build and sign Tauri app
|
||||
if: matrix.sign_windows
|
||||
uses: tauri-apps/tauri-action@v0
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -375,18 +423,10 @@ jobs:
|
||||
with:
|
||||
tagName: ${{ github.ref_name }}
|
||||
releaseDraft: true
|
||||
# No assetNamePattern: the default Tauri CLI names already produce
|
||||
# DBX_[version]_x64/arm64-setup.exe and keep the _en-US.msi names.
|
||||
args: --target ${{ matrix.target }} --config src-tauri/tauri.vsign.conf.json
|
||||
|
||||
- name: Build unsigned Tauri app
|
||||
if: ${{ !matrix.sign_windows }}
|
||||
uses: tauri-apps/tauri-action@v0
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
tagName: ${{ github.ref_name }}
|
||||
releaseDraft: true
|
||||
args: --target ${{ matrix.target }}
|
||||
|
||||
- name: Verify Windows bundles
|
||||
shell: pwsh
|
||||
run: |
|
||||
@@ -403,38 +443,31 @@ jobs:
|
||||
if ($msiInstallers.Count -eq 0) {
|
||||
throw "No MSI installer was produced for ${{ matrix.target }}"
|
||||
}
|
||||
if ("${{ matrix.sign_windows }}" -eq "true") {
|
||||
./.github/scripts/assert-authenticode.ps1 -Path @(
|
||||
$nsisInstallers.FullName
|
||||
$msiInstallers.FullName
|
||||
)
|
||||
}
|
||||
./.github/scripts/assert-authenticode.ps1 -Path @(
|
||||
$nsisInstallers.FullName
|
||||
$msiInstallers.FullName
|
||||
)
|
||||
|
||||
- name: Upload Windows WebView2 offline installer
|
||||
shell: pwsh
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
SSIGNCODE_OPERATOR_PWD: ${{ matrix.sign_windows && secrets.VSIGN_OPERATOR_PWD || '' }}
|
||||
VSIGN_KEY_PIN: ${{ matrix.sign_windows && secrets.VSIGN_KEY_PIN || '' }}
|
||||
SSIGNCODE_OPERATOR_PWD: ${{ secrets.VSIGN_OPERATOR_PWD }}
|
||||
VSIGN_KEY_PIN: ${{ secrets.VSIGN_KEY_PIN }}
|
||||
run: |
|
||||
$version = "${env:GITHUB_REF_NAME}".TrimStart("v")
|
||||
$arch = "${{ matrix.arch }}"
|
||||
$bundleDir = "target/${{ matrix.target }}/release/bundle/nsis"
|
||||
$offlineName = "DBX_${version}_${arch}-webview2-offline-setup.exe"
|
||||
$offlineName = "DBX_${version}_${arch}-offline-setup.exe"
|
||||
$before = @{}
|
||||
|
||||
if (Test-Path $bundleDir) {
|
||||
Get-ChildItem $bundleDir -Filter "*.exe" | ForEach-Object { $before[$_.FullName] = $_.LastWriteTimeUtc }
|
||||
}
|
||||
|
||||
if ("${{ matrix.sign_windows }}" -eq "true") {
|
||||
pnpm tauri bundle --bundles nsis --target ${{ matrix.target }} `
|
||||
--config src-tauri/tauri.webview2-offline.conf.json `
|
||||
--config src-tauri/tauri.vsign.conf.json
|
||||
} else {
|
||||
pnpm tauri bundle --bundles nsis --target ${{ matrix.target }} `
|
||||
--config src-tauri/tauri.webview2-offline.conf.json
|
||||
}
|
||||
pnpm tauri bundle --bundles nsis --target ${{ matrix.target }} `
|
||||
--config src-tauri/tauri.webview2-offline.conf.json `
|
||||
--config src-tauri/tauri.vsign.conf.json
|
||||
|
||||
$installer = Get-ChildItem $bundleDir -Filter "*.exe" |
|
||||
Where-Object { !$before.ContainsKey($_.FullName) -or $_.LastWriteTimeUtc -gt $before[$_.FullName] } |
|
||||
@@ -445,9 +478,7 @@ jobs:
|
||||
throw "Missing WebView2 offline installer in ${bundleDir}"
|
||||
}
|
||||
|
||||
if ("${{ matrix.sign_windows }}" -eq "true") {
|
||||
./.github/scripts/assert-authenticode.ps1 -Path $installer.FullName
|
||||
}
|
||||
./.github/scripts/assert-authenticode.ps1 -Path $installer.FullName
|
||||
Copy-Item $installer.FullName $offlineName -Force
|
||||
gh release upload "${env:GITHUB_REF_NAME}" $offlineName --repo "${env:GITHUB_REPOSITORY}" --clobber
|
||||
|
||||
@@ -456,8 +487,8 @@ jobs:
|
||||
shell: pwsh
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
SSIGNCODE_OPERATOR_PWD: ${{ matrix.sign_windows && secrets.VSIGN_OPERATOR_PWD || '' }}
|
||||
VSIGN_KEY_PIN: ${{ matrix.sign_windows && secrets.VSIGN_KEY_PIN || '' }}
|
||||
SSIGNCODE_OPERATOR_PWD: ${{ secrets.VSIGN_OPERATOR_PWD }}
|
||||
VSIGN_KEY_PIN: ${{ secrets.VSIGN_KEY_PIN }}
|
||||
run: |
|
||||
$version = "${env:GITHUB_REF_NAME}".TrimStart("v")
|
||||
$arch = "${{ matrix.arch }}"
|
||||
@@ -475,10 +506,8 @@ jobs:
|
||||
throw "Release tag version $version does not match the built DBX package version $appVersion"
|
||||
}
|
||||
|
||||
if ("${{ matrix.sign_windows }}" -eq "true") {
|
||||
./.github/scripts/vsign-sign.ps1 -FilePath $exePath
|
||||
./.github/scripts/assert-authenticode.ps1 -Path $exePath
|
||||
}
|
||||
./.github/scripts/vsign-sign.ps1 -FilePath $exePath
|
||||
./.github/scripts/assert-authenticode.ps1 -Path $exePath
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $portableDir | Out-Null
|
||||
Copy-Item $exePath (Join-Path $portableDir "DBX.exe") -Force
|
||||
@@ -519,7 +548,7 @@ jobs:
|
||||
run: ${SCCACHE_PATH} --show-stats
|
||||
|
||||
- name: Remove temporary VSign files
|
||||
if: ${{ always() && matrix.sign_windows }}
|
||||
if: always()
|
||||
shell: pwsh
|
||||
run: |
|
||||
$workDir = if ([string]::IsNullOrWhiteSpace($env:VSIGN_WORK_DIR)) {
|
||||
@@ -535,6 +564,12 @@ jobs:
|
||||
needs: release-ready
|
||||
runs-on: windows-2022
|
||||
environment: release-signing
|
||||
# No sccache here: the loader patch rewrites a file inside the cargo
|
||||
# registry, which is invisible to any compile cache. v0.6.0 shipped a
|
||||
# binary linked against the wrong (>= 1.0.1054.31) WebView2 loader even
|
||||
# though the patched 1.0.902.49 loader was verified on disk, so caching is
|
||||
# disabled for this target until the patch is vendored into the source
|
||||
# tree.
|
||||
env:
|
||||
CARGO_INCREMENTAL: "0"
|
||||
RUSTFLAGS: -C debuginfo=line-tables-only -C target-feature=+crt-static
|
||||
@@ -596,6 +631,10 @@ jobs:
|
||||
shell: pwsh
|
||||
run: ./.github/scripts/assert-win7-pe-compat.ps1 -BinaryPath target/x86_64-win7-windows-msvc/release/dbx.exe
|
||||
|
||||
- name: Audit Win7 WebView2 loader markers
|
||||
shell: pwsh
|
||||
run: ./.github/scripts/assert-webview2-win7-loader.ps1 -BinaryPath target/x86_64-win7-windows-msvc/release/dbx.exe
|
||||
|
||||
- name: Setup TrustAsia VSign
|
||||
shell: pwsh
|
||||
env:
|
||||
@@ -603,7 +642,7 @@ jobs:
|
||||
VSIGN_OPERATOR_PWD: ${{ secrets.VSIGN_OPERATOR_PWD }}
|
||||
run: ./.github/scripts/setup-vsign.ps1
|
||||
|
||||
- name: Bundle and upload Windows 7 fixed-runtime installer
|
||||
- name: Bundle and upload Windows 7 / Server 2012 R2 fixed-runtime installer
|
||||
shell: pwsh
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -612,7 +651,8 @@ jobs:
|
||||
run: |
|
||||
$version = "${env:GITHUB_REF_NAME}".TrimStart("v")
|
||||
$bundleDir = "target/x86_64-win7-windows-msvc/release/bundle/nsis"
|
||||
$offlineName = "DBX_${version}_x64-win7-webview2-109-offline-setup.exe"
|
||||
$offlineName = "DBX_${version}_x64-win7-server2012r2-offline-setup.exe"
|
||||
$checksumName = "${offlineName}.sha256"
|
||||
$cargoMetadata = cargo metadata --no-deps --format-version 1 | ConvertFrom-Json
|
||||
$appVersion = ($cargoMetadata.packages | Where-Object { $_.name -eq "dbx" } | Select-Object -First 1).version
|
||||
|
||||
@@ -629,7 +669,7 @@ jobs:
|
||||
Sort-Object LastWriteTimeUtc -Descending |
|
||||
Select-Object -First 1
|
||||
if (!$installer) {
|
||||
Write-Error "Missing Windows 7 fixed-runtime installer in ${bundleDir}"
|
||||
Write-Error "Missing Windows 7 / Server 2012 R2 fixed-runtime installer in ${bundleDir}"
|
||||
exit 1
|
||||
}
|
||||
|
||||
@@ -637,7 +677,10 @@ jobs:
|
||||
./.github/scripts/assert-authenticode.ps1 -Path $installer.FullName
|
||||
|
||||
Copy-Item $installer.FullName $offlineName -Force
|
||||
gh release upload "${env:GITHUB_REF_NAME}" $offlineName --repo "${env:GITHUB_REPOSITORY}" --clobber
|
||||
$hash = (Get-FileHash -LiteralPath $offlineName -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
"$hash $offlineName" | Set-Content -LiteralPath $checksumName -Encoding ascii
|
||||
gh release upload "${env:GITHUB_REF_NAME}" $offlineName $checksumName `
|
||||
--repo "${env:GITHUB_REPOSITORY}" --clobber
|
||||
|
||||
- name: Remove temporary VSign files
|
||||
if: always()
|
||||
@@ -672,6 +715,10 @@ jobs:
|
||||
# page-size ARM64 kernels.
|
||||
rustflags: "-C target-feature=+crt-static -C link-arg=-z -C link-arg=max-page-size=65536"
|
||||
runs-on: ${{ matrix.platform }}
|
||||
env:
|
||||
CARGO_INCREMENTAL: "0"
|
||||
RUSTC_WRAPPER: sccache
|
||||
SCCACHE_GHA_ENABLED: ${{ secrets.SCCACHE_S3_BUCKET == '' && 'true' || 'false' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
|
||||
@@ -694,6 +741,41 @@ jobs:
|
||||
- name: Install cargo-zigbuild
|
||||
run: pip3 install cargo-zigbuild
|
||||
|
||||
- name: Setup sccache
|
||||
uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10
|
||||
with:
|
||||
version: "v0.10.0"
|
||||
|
||||
# Plain CC/CXX only wrap HOST C compilations (build scripts): for the
|
||||
# musl targets cargo-zigbuild sets CC_<target> to zig cc, which takes
|
||||
# precedence and must stay unwrapped.
|
||||
- name: Configure S3 sccache
|
||||
if: env.SCCACHE_GHA_ENABLED != 'true'
|
||||
shell: bash
|
||||
env:
|
||||
CACHE_BUCKET: ${{ secrets.SCCACHE_S3_BUCKET }}
|
||||
CACHE_ENDPOINT: ${{ secrets.SCCACHE_S3_ENDPOINT }}
|
||||
CACHE_REGION: ${{ secrets.SCCACHE_S3_REGION }}
|
||||
CACHE_KEY_PREFIX: ${{ secrets.SCCACHE_S3_KEY_PREFIX }}
|
||||
CACHE_ACCESS_KEY_ID: ${{ secrets.SCCACHE_S3_ACCESS_KEY_ID }}
|
||||
CACHE_SECRET_ACCESS_KEY: ${{ secrets.SCCACHE_S3_SECRET_ACCESS_KEY }}
|
||||
run: |
|
||||
{
|
||||
echo "SCCACHE_BUCKET=${CACHE_BUCKET}"
|
||||
echo "SCCACHE_ENDPOINT=${CACHE_ENDPOINT}"
|
||||
echo "SCCACHE_REGION=${CACHE_REGION}"
|
||||
echo "SCCACHE_S3_KEY_PREFIX=${CACHE_KEY_PREFIX}"
|
||||
echo "SCCACHE_S3_USE_SSL=true"
|
||||
echo "AWS_ACCESS_KEY_ID=${CACHE_ACCESS_KEY_ID}"
|
||||
echo "AWS_SECRET_ACCESS_KEY=${CACHE_SECRET_ACCESS_KEY}"
|
||||
# Packaging and container verification run long after the last
|
||||
# rustc call; the server's 10-minute default idle timeout would
|
||||
# exit first and zero the counters shown by "Show sccache stats".
|
||||
echo "SCCACHE_IDLE_TIMEOUT=0"
|
||||
echo "CC=${SCCACHE_PATH} cc"
|
||||
echo "CXX=${SCCACHE_PATH} c++"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
- name: Rust cache
|
||||
uses: swatinem/rust-cache@v2
|
||||
with:
|
||||
@@ -754,6 +836,11 @@ jobs:
|
||||
gh release upload "${GITHUB_REF_NAME}" "$ASSET" "$CHECKSUM" \
|
||||
--repo "${GITHUB_REPOSITORY}" --clobber
|
||||
|
||||
- name: Show sccache stats
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
run: ${SCCACHE_PATH} --show-stats
|
||||
|
||||
cleanup-release-signatures:
|
||||
needs: [build, build-windows, build-windows-7-offline]
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
name: SQL Server 2000 Windows User Test Build
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build-windows-x64:
|
||||
runs-on: windows-2022
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
|
||||
- name: Setup pnpm
|
||||
uses: pnpm/action-setup@v6
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: 22.13.0
|
||||
cache: pnpm
|
||||
|
||||
- name: Setup Java
|
||||
uses: actions/setup-java@v5
|
||||
with:
|
||||
distribution: temurin
|
||||
java-version: "21"
|
||||
|
||||
- name: Setup Gradle
|
||||
uses: gradle/actions/setup-gradle@v4
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Setup Rust
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
targets: x86_64-pc-windows-msvc
|
||||
|
||||
- name: Rust cache
|
||||
uses: swatinem/rust-cache@v2
|
||||
with:
|
||||
workspaces: "./ -> target"
|
||||
shared-key: sqlserver2000-windows-x64
|
||||
cache-on-failure: true
|
||||
|
||||
- name: Build SQL Server legacy agent
|
||||
run: .\\gradlew.bat :sqlserver-legacy:shadowJar --no-daemon
|
||||
working-directory: agents
|
||||
|
||||
- name: Stage SQL Server legacy agent
|
||||
shell: pwsh
|
||||
run: |
|
||||
$agent = Get-ChildItem "agents/drivers/sqlserver-legacy/build/libs/dbx-agent-sqlserver-legacy.jar" | Select-Object -First 1
|
||||
if (!$agent) {
|
||||
Write-Error "Missing SQL Server legacy agent JAR"
|
||||
exit 1
|
||||
}
|
||||
Copy-Item $agent.FullName "sqlserver-legacy-agent.jar" -Force
|
||||
|
||||
- name: Build Windows x64 installer
|
||||
run: pnpm tauri build --bundles nsis --target x86_64-pc-windows-msvc --ci --no-sign
|
||||
|
||||
- name: Upload installer
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: DBX-SQLServer2000-Windows-x64
|
||||
path: |
|
||||
target/x86_64-pc-windows-msvc/release/bundle/nsis/*.exe
|
||||
sqlserver-legacy-agent.jar
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
@@ -19,22 +19,31 @@ jobs:
|
||||
name: Sync release to CNB
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Sync release assets to CNB
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Download GitHub release assets
|
||||
env:
|
||||
TAG_NAME: ${{ inputs.tag }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
GITHUB_USERNAME: ${{ github.repository_owner }}
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
CNB_USERNAME: cnb
|
||||
CNB_TOKEN: ${{ secrets.CNB_TOKEN }}
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p "$RUNNER_TEMP/github-release" "$RUNNER_TEMP/release-assets"
|
||||
gh release view "$TAG_NAME" \
|
||||
--repo "$GITHUB_REPOSITORY" \
|
||||
--json tagName,name,body,targetCommitish,isPrerelease,isDraft,assets \
|
||||
> "$RUNNER_TEMP/github-release/release.json"
|
||||
gh release download "$TAG_NAME" \
|
||||
--repo "$GITHUB_REPOSITORY" \
|
||||
--dir "$RUNNER_TEMP/release-assets" \
|
||||
--clobber
|
||||
|
||||
wget -q https://cnb.cool/znb/mpgrm/-/releases/download/v0.1.0/mpgrm_linux_amd64.tar.gz
|
||||
tar -xf mpgrm_linux_amd64.tar.gz
|
||||
chmod +x mpgrm
|
||||
|
||||
./mpgrm releases sync \
|
||||
--repo "https://github.com/${GITHUB_REPOSITORY}.git" \
|
||||
--target-repo "https://cnb.cool/${CNB_REPOSITORY}.git" \
|
||||
--tags "${TAG_NAME}"
|
||||
- name: Sync release assets to CNB
|
||||
env:
|
||||
CNB_TOKEN: ${{ secrets.CNB_TOKEN }}
|
||||
CNB_UPLOAD_CONCURRENCY: "3"
|
||||
run: |
|
||||
node .github/scripts/sync-cnb-release.mjs \
|
||||
--github-release "$RUNNER_TEMP/github-release/release.json" \
|
||||
--assets-dir "$RUNNER_TEMP/release-assets"
|
||||
|
||||
@@ -8,6 +8,7 @@ node_modules
|
||||
.trellis/
|
||||
.worktrees/
|
||||
openspec/
|
||||
.zcode/
|
||||
.mcp.json
|
||||
AGENTS.md
|
||||
CLAUDE.md
|
||||
@@ -23,6 +24,8 @@ plugins/jdbc/target/
|
||||
plugins/jdbc/dependency-reduced-pom.xml
|
||||
plugins/jdbc/lib/*.jar
|
||||
agents/drivers/duckdb/target/
|
||||
agents/drivers/etcd-go/etcd-go
|
||||
agents/drivers/etcd2-go/etcd2-go
|
||||
plugins/jdbc/.gradle/
|
||||
plugins/jdbc/build/
|
||||
plugins/jdbc/dist/
|
||||
|
||||
Generated
+44
-4
@@ -2002,7 +2002,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "dbx"
|
||||
version = "0.5.97"
|
||||
version = "0.6.3"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"axum",
|
||||
@@ -2011,6 +2011,7 @@ dependencies = [
|
||||
"chrono",
|
||||
"csv",
|
||||
"dbx-core",
|
||||
"dbx-mcp",
|
||||
"deadpool-postgres",
|
||||
"font-kit",
|
||||
"futures",
|
||||
@@ -2024,6 +2025,7 @@ dependencies = [
|
||||
"percent-encoding",
|
||||
"portpicker",
|
||||
"redis",
|
||||
"regex",
|
||||
"reqwest 0.12.28",
|
||||
"russh",
|
||||
"rust_decimal",
|
||||
@@ -2058,11 +2060,12 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "dbx-cli"
|
||||
version = "0.4.74"
|
||||
version = "0.4.80"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"dbx-core",
|
||||
"dbx-mcp",
|
||||
"rustls 0.23.43",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
@@ -2086,6 +2089,7 @@ dependencies = [
|
||||
"cbc 0.1.2",
|
||||
"chrono",
|
||||
"csv",
|
||||
"dbx-sqlite-worker",
|
||||
"deadpool-postgres",
|
||||
"encoding_rs",
|
||||
"flate2",
|
||||
@@ -2152,11 +2156,13 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "dbx-mcp"
|
||||
version = "0.4.74"
|
||||
version = "0.4.80"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
"dbx-core",
|
||||
"dirs",
|
||||
"log",
|
||||
"reqwest 0.12.28",
|
||||
"rmcp",
|
||||
"rustls 0.23.43",
|
||||
@@ -2167,13 +2173,25 @@ dependencies = [
|
||||
"tempfile",
|
||||
"tokio",
|
||||
"tokio-rustls 0.26.4",
|
||||
"tokio-util",
|
||||
"tower-http",
|
||||
"url",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dbx-sqlite-worker"
|
||||
version = "0.1.1"
|
||||
dependencies = [
|
||||
"rusqlite",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sqlparser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dbx-web"
|
||||
version = "0.5.97"
|
||||
version = "0.6.3"
|
||||
dependencies = [
|
||||
"aes-gcm 0.10.3",
|
||||
"argon2",
|
||||
@@ -2182,6 +2200,7 @@ dependencies = [
|
||||
"base64 0.22.1",
|
||||
"chrono",
|
||||
"dbx-core",
|
||||
"dbx-mcp",
|
||||
"futures",
|
||||
"log",
|
||||
"pbkdf2 0.12.2",
|
||||
@@ -6986,19 +7005,27 @@ checksum = "14db48ee17a9ba61810ab1a9c1beb7d06d8136ae39ac25a1137f10d357af01af"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"futures",
|
||||
"http 1.5.0",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
"pastey",
|
||||
"pin-project-lite",
|
||||
"rand 0.10.2",
|
||||
"rmcp-macros",
|
||||
"schemars 1.2.2",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sse-stream",
|
||||
"thiserror 2.0.19",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tokio-util",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8193,6 +8220,19 @@ dependencies = [
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sse-stream"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c123f296ade4ec4b8b0f6162116e6629f5146922ca5ab40ca9d3c2e73ab4761e"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http-body 1.1.0",
|
||||
"http-body-util",
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ssfmt"
|
||||
version = "0.1.2"
|
||||
|
||||
+6
-2
@@ -1,6 +1,6 @@
|
||||
[workspace]
|
||||
resolver = "2"
|
||||
members = ["src-tauri", "crates/dbx-core", "crates/dbx-web", "crates/dbx-mcp", "crates/dbx-cli"]
|
||||
members = ["src-tauri", "crates/dbx-core", "crates/dbx-web", "crates/dbx-mcp", "crates/dbx-cli", "crates/dbx-sqlite-worker"]
|
||||
exclude = ["vendor/ctor", "vendor/dirs-sys", "vendor/pageant", "vendor/rumqttc", "vendor/tiberius", "vendor/wry"]
|
||||
|
||||
[patch.crates-io]
|
||||
@@ -22,7 +22,11 @@ pageant = { path = "vendor/pageant" }
|
||||
# process stays alive (black screen after an overnight idle session, see
|
||||
# vendor/wry/src/webview2/mod.rs). Recovery is layered by process kind
|
||||
# (browser exit is fatal, GPU exits are self-healed by WebView2) and bounded
|
||||
# by a cooldown plus a maximum reload budget.
|
||||
# by a cooldown plus a maximum reload budget. It also re-owns clipboard
|
||||
# writes performed by the WebView2 browser process on the host window so
|
||||
# copies show up in Windows Clipboard History / Win+V
|
||||
# (MicrosoftEdge/WebView2Feedback#5650, see
|
||||
# vendor/wry/src/webview2/clipboard_history.rs).
|
||||
wry = { path = "vendor/wry" }
|
||||
tokio-postgres = { git = "https://github.com/t8y2/tokio-postgres-gaussdb.git", rev = "ac8ad476de59093d33ab9f294531c956e0c6a724" }
|
||||
postgres-types = { git = "https://github.com/t8y2/tokio-postgres-gaussdb.git", rev = "ac8ad476de59093d33ab9f294531c956e0c6a724" }
|
||||
|
||||
@@ -113,7 +113,7 @@ cargo-check-fast:
|
||||
cargo check --no-default-features --features sqlite-bundled
|
||||
|
||||
cargo-test-fast:
|
||||
cargo test --no-default-features --features sqlite-bundled
|
||||
RUST_MIN_STACK=8388608 cargo test --no-default-features --features sqlite-bundled
|
||||
|
||||
db-list:
|
||||
@$(PNPM) db:env -- list
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
<a href="https://discord.gg/W7NyVDRt6a"><img src="https://dcbadge.limes.pink/api/server/W7NyVDRt6a" alt="Join Discord" /></a>
|
||||
</p>
|
||||
<p>
|
||||
<a href="https://trendshift.io/repositories/26775?utm_source=trendshift-badge&utm_medium=badge&utm_campaign=badge-trendshift-26775" target="_blank" rel="noopener noreferrer"><img src="https://trendshift.io/api/badge/trendshift/repositories/26775/daily?language=Rust" alt="t8y2%2Fdbx | Trendshift" width="250" height="55"/></a>
|
||||
<a href="https://trendshift.io/repositories/26775?utm_source=repository-badge&utm_medium=badge&utm_campaign=badge-repository-26775" target="_blank" rel="noopener noreferrer"><img src="https://trendshift.io/api/badge/repositories/26775" alt="t8y2%2Fdbx | Trendshift" width="250" height="55"/></a>
|
||||
<a href="https://hellogithub.com/repository/t8y2/dbx" target="_blank"><img src="https://api.hellogithub.com/v1/widgets/recommend.svg?rid=7f74ffda697241bf996e17e1b0900a21&claim_uid=p0UjnC1TLtyvWSx" alt="Featured|HelloGitHub" style="width: 250px; height: 54px;" width="250" height="54" /></a>
|
||||
<a href="https://www.producthunt.com/products/dbx/launches/dbx?embed=true&utm_source=badge-featured&utm_medium=badge&utm_campaign=badge-dbx" target="_blank" rel="noopener noreferrer"><img alt="DBX - Lightweight open-source database manager built with Rust | Product Hunt" width="250" height="54" src="https://api.producthunt.com/widgets/embed-image/v1/featured.svg?post_id=1136811&theme=light&t=1780979717555"></a>
|
||||
</p>
|
||||
@@ -46,6 +46,7 @@
|
||||
<img src="https://img.shields.io/badge/InfluxDB-d30971?logo=influxdb&logoColor=white" />
|
||||
<img src="https://img.shields.io/badge/JDBC-4B5563?logoColor=white" />
|
||||
<img src="https://img.shields.io/badge/and%20more...-555555?logoColor=white" />
|
||||
<a href="https://1panel.cn/" target="_blank" rel="noopener noreferrer"><img src="https://img.shields.io/badge/1Panel-Partner-005EEB?logo=1panel&logoColor=white" alt="1Panel Partner" /></a>
|
||||
<a href="https://cnb.cool/dbxio.com/dbx"><img src="https://img.shields.io/badge/CNB-dbx-F76945?logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IArs4c6QAAAppJREFUOE9tk01rE1EUhs+5H5lJ0iQT3FRX2Yi4avoLkv4CWxBcNt11oZiuFKGmXQhFN6kuRESSgitXKf6A5g+I6cqFLiKCiptOkraZ5H4cuZOkH9oLw+UO8z7nvGfei3DFohoEQLAKCGWyGJCSbQZsD3ei7r+f45WATV6xmjdISyAlYLaDliveux+ti5ozwLAaFPhIlk6G6f18sxuO7l9bBiWbVoncDGCMWMl++PI/gJ7IohkmD0zkBxT5oRn5LTpN7ULqBByEtFywWvRyHz8Fx3dvN0UwuMPyR0uJFycddH7tKPHZRn7BjnwwkQcUud0HO/K7oOENcTYPRCEAFERmsCpyfZDZfpdnBosOUKGx13BC60QTIdjpmSzrEWDR+UagjsgMciLXAwfhc4MNpKfQtOPE6mVh0gEOaeTXtWXXEdADpJAR/GaZ/rrM9Us82wORPd5zHbRp7JVcyxR5PRP5LTPy6uo0HXIlG9awMgLuEUABkBYI7VLyxs+yyPW3+Nyg4zqoG+UV7TDRlMBbWA/D43u3qlbLGmgRWM2BjFxjQjWQGQBuu4bYYvrm14JMnQSXcjBcny9A5DXIiDJp9/8FWCXAAOaFUEfILTgIcttOvf+2NJnLdI0fBFUyskZKBGTEJEAmBuzr9LCSUDwGQAwwgMzu+m9/VWOAeZSsk+YP48C4li/sRok1nlBd5PogBjgxNw70PfHyqBADyEVXicasZXCVpxYiLfO+HxVZDHCVYwvu2ebPT7fOLNAm65AWLnEAzvsEcOi9/lPU1aACXDfOLdhD9kxNszGdweQGYstqXgItJwDFNxKvwrp57G8h0zVwldHuA0IFt8El83yIs2FSDcpgxDJpUUTwK7gTduN3AK5iG7ehc/E2/gUPD3q3eY4awwAAAABJRU5ErkJggg==" alt="CNB" /></a>
|
||||
<a href="https://mcptoplist.com/server/io.github.t8y2%2Fdbx"><img src="https://mcptoplist.com/badge/io.github.t8y2%2Fdbx.svg" alt="MCP Toplist" /></a>
|
||||
</p>
|
||||
@@ -63,16 +64,13 @@
|
||||
</p>
|
||||
</div>
|
||||
|
||||
## Sponsors & Partners
|
||||
## ❤️ Sponsors
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td align="center" valign="middle" width="220">
|
||||
<td align="center" valign="middle" width="200">
|
||||
<a href="https://www.rainyun.com/MTE5Mjc4Ng==_" target="_blank">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="docs/public/sponsors/rainyun-dark.png" />
|
||||
<img src="https://www.rainyun.com/img/logo.d193755d.png" alt="RainYun" height="48" />
|
||||
</picture>
|
||||
<img src="docs/public/sponsors/rainyun-card.png" alt="RainYun" width="175" />
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
@@ -81,37 +79,42 @@
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center" valign="middle" width="220">
|
||||
<a href="https://www.qiniu.com/" target="_blank">
|
||||
<img src="https://www-static.qbox.me/_next/static/media/logo.0fc18feaa621d2068a7180631f742256.jpg" alt="Qiniu Cloud" height="64" />
|
||||
<td align="center" valign="middle" width="200">
|
||||
<a href="https://www.trustasia.com/ssl/trustasia/code-signing" target="_blank">
|
||||
<img src="docs/public/sponsors/trustasia-card.png" alt="TrustAsia" width="175" />
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
Qiniu Cloud provides DBX with object storage, CDN, and other cloud infrastructure resources.
|
||||
<a href="https://www.qiniu.com/" target="_blank">Visit Qiniu Cloud</a>
|
||||
TrustAsia provides cloud-based code signing service for DBX, enabling trusted software through automated CI/CD builds.
|
||||
<a href="https://www.trustasia.com/ssl/trustasia/code-signing" target="_blank">Visit TrustAsia</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center" valign="middle" width="220">
|
||||
<a href="https://easysearch.cn" target="_blank">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="docs/public/sponsors/easysearch-dark.png" />
|
||||
<img src="docs/public/sponsors/easysearch.png" alt="Easysearch" width="150" />
|
||||
</picture>
|
||||
<td align="center" valign="middle" width="200">
|
||||
<a href="https://www.jalapeno-cloud.ai/DBX" target="_blank">
|
||||
<img src="docs/public/sponsors/jalapeno-card.png" alt="Jalapeño Cloud" width="175" />
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
Easysearch is an enterprise-grade distributed search engine compatible with Elasticsearch APIs, combining full-text, vector, geospatial search, real-time analytics, and AI capabilities in one platform.
|
||||
<a href="https://easysearch.cn" target="_blank">Visit Easysearch</a>
|
||||
Jalapeño Cloud is an AI infrastructure and token compute platform, with an exclusive DBX entry offering free credits and top-up bonuses.
|
||||
<a href="https://www.jalapeno-cloud.ai/DBX" target="_blank">Visit Jalapeño Cloud</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center" valign="middle" width="220">
|
||||
<td align="center" valign="middle" width="200">
|
||||
<a href="https://www.ucloud.cn/site/active/kuaijiesale.html?ytag=geo_waituo_github_dbx" target="_blank">
|
||||
<img src="docs/public/sponsors/astraflow-card.png" alt="AstraFlow" width="175" />
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
UCloud is the first public cloud provider listed on China's STAR Market, with 28 global regions for cloud hosting, databases, and CDN; its AstraFlow platform offers one-click access to 200+ mainstream LLMs.
|
||||
<a href="https://www.ucloud.cn/site/active/kuaijiesale.html?ytag=geo_waituo_github_dbx" target="_blank">Visit UCloud</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center" valign="middle" width="200">
|
||||
<a href="https://www.atlascloud.ai/?ref=6YYXWA" target="_blank">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="docs/public/sponsors/atlas-dark.svg" />
|
||||
<img src="https://www.atlascloud.ai/logo.svg" alt="Atlas Cloud" width="150" />
|
||||
</picture>
|
||||
<img src="docs/public/sponsors/atlas-card.png" alt="Atlas Cloud" width="175" />
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
@@ -120,17 +123,41 @@
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center" valign="middle" width="220">
|
||||
<a href="https://www.trustasia.com/ssl/trustasia/code-signing" target="_blank">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="docs/public/sponsors/trustasia-dark.svg" />
|
||||
<img src="docs/public/sponsors/trustasia.png" alt="TrustAsia" height="44" />
|
||||
</picture>
|
||||
<td align="center" valign="middle" width="200">
|
||||
<a href="https://www.qiniu.com/" target="_blank">
|
||||
<img src="docs/public/sponsors/qiniu-card.png" alt="Qiniu Cloud" width="175" />
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
TrustAsia provides cloud-based code signing service for DBX, enabling trusted software through automated CI/CD builds.
|
||||
<a href="https://www.trustasia.com/ssl/trustasia/code-signing" target="_blank">Visit TrustAsia</a>
|
||||
Qiniu Cloud provides DBX with object storage, CDN, and other cloud infrastructure resources.
|
||||
<a href="https://www.qiniu.com/" target="_blank">Visit Qiniu Cloud</a>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
## 🤝 Partners
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td align="center" valign="middle" width="200">
|
||||
<a href="https://1panel.cn/" target="_blank">
|
||||
<img src="docs/public/sponsors/1panel-card.png" alt="1Panel" width="175" />
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
1Panel is a modern open-source Linux server management panel and lightweight AI management platform, offering an intuitive web interface for one-stop management of AI agents, local LLMs, websites, databases, containers, files, and more.
|
||||
<a href="https://1panel.cn/" target="_blank">Visit 1Panel</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center" valign="middle" width="200">
|
||||
<a href="https://easysearch.cn" target="_blank">
|
||||
<img src="docs/public/sponsors/easysearch-card.png" alt="Easysearch" width="175" />
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
Easysearch is an enterprise-grade distributed search engine compatible with Elasticsearch APIs, combining full-text, vector, geospatial search, real-time analytics, and AI capabilities in one platform.
|
||||
<a href="https://easysearch.cn" target="_blank">Visit Easysearch</a>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
@@ -141,7 +168,7 @@
|
||||
<tr>
|
||||
<td width="50%">
|
||||
<h3>🪶 20 MB, zero runtime bloat</h3>
|
||||
<p>No Java JRE. No Python venv. No bundled Chromium. DBX ships as a single small binary — download, install, connect. DBeaver needs Java; TablePlus is macOS-only. DBX runs everywhere with nothing extra.</p>
|
||||
<p>No Java JRE. No Python venv. No bundled Chromium. DBX ships as a single small binary — download, install, connect. DBeaver needs Java; TablePlus is Freemium. DBX runs everywhere with nothing extra.</p>
|
||||
</td>
|
||||
<td width="50%">
|
||||
<h3>🤖 AI that lives in your editor</h3>
|
||||
@@ -472,7 +499,7 @@ DBX is free and open source, but ongoing maintenance, database compatibility tes
|
||||
## Contributors
|
||||
|
||||
<a href="https://github.com/t8y2/dbx/graphs/contributors">
|
||||
<img src="https://contrib.rocks/image?repo=t8y2/dbx" />
|
||||
<img src="https://contrib.rocks/image?repo=t8y2/dbx&max=300&columns=15" />
|
||||
</a>
|
||||
|
||||
## Community
|
||||
@@ -481,6 +508,7 @@ DBX is free and open source, but ongoing maintenance, database compatibility tes
|
||||
<a href="https://qm.qq.com/q/1087880322" target="_blank"><img src="https://img.shields.io/badge/QQ%20群-1087880322-EB1923?logo=tencentqq&logoColor=white" alt="QQ Group" /></a>
|
||||
<a href="https://docs.qq.com/doc/DVVhMY0h1ekJqc0tz" target="_blank"><img src="https://img.shields.io/badge/微信群-Join-07C160?logo=wechat&logoColor=white" alt="WeChat Group" /></a>
|
||||
[](https://linux.do)
|
||||
[](https://1panel.cn)
|
||||
|
||||
## Star History
|
||||
|
||||
|
||||
+63
-35
@@ -12,7 +12,7 @@
|
||||
<a href="https://discord.gg/W7NyVDRt6a"><img src="https://dcbadge.limes.pink/api/server/W7NyVDRt6a" alt="加入 Discord" /></a>
|
||||
</p>
|
||||
<p>
|
||||
<a href="https://trendshift.io/repositories/26775?utm_source=trendshift-badge&utm_medium=badge&utm_campaign=badge-trendshift-26775" target="_blank" rel="noopener noreferrer"><img src="https://trendshift.io/api/badge/trendshift/repositories/26775/daily?language=Rust" alt="t8y2%2Fdbx | Trendshift" width="250" height="55"/></a>
|
||||
<a href="https://trendshift.io/repositories/26775?utm_source=repository-badge&utm_medium=badge&utm_campaign=badge-repository-26775" target="_blank" rel="noopener noreferrer"><img src="https://trendshift.io/api/badge/repositories/26775" alt="t8y2%2Fdbx | Trendshift" width="250" height="55"/></a>
|
||||
<a href="https://hellogithub.com/repository/t8y2/dbx" target="_blank"><img src="https://api.hellogithub.com/v1/widgets/recommend.svg?rid=7f74ffda697241bf996e17e1b0900a21&claim_uid=p0UjnC1TLtyvWSx" alt="Featured|HelloGitHub" style="width: 250px; height: 54px;" width="250" height="54" /></a>
|
||||
<a href="https://www.producthunt.com/products/dbx/launches/dbx?embed=true&utm_source=badge-featured&utm_medium=badge&utm_campaign=badge-dbx" target="_blank" rel="noopener noreferrer"><img alt="DBX - Lightweight open-source database manager built with Rust | Product Hunt" width="250" height="54" src="https://api.producthunt.com/widgets/embed-image/v1/featured.svg?post_id=1136811&theme=light&t=1780979717555"></a>
|
||||
</p>
|
||||
@@ -46,6 +46,7 @@
|
||||
<img src="https://img.shields.io/badge/InfluxDB-d30971?logo=influxdb&logoColor=white" />
|
||||
<img src="https://img.shields.io/badge/JDBC-4B5563?logoColor=white" />
|
||||
<img src="https://img.shields.io/badge/and%20more...-555555?logoColor=white" />
|
||||
<a href="https://1panel.cn/" target="_blank" rel="noopener noreferrer"><img src="https://img.shields.io/badge/1Panel-Partner-005EEB?logo=1panel&logoColor=white" alt="1Panel Partner" /></a>
|
||||
<a href="https://cnb.cool/dbxio.com/dbx"><img src="https://img.shields.io/badge/CNB-dbx-F76945?logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAAAAXNSR0IArs4c6QAAAppJREFUOE9tk01rE1EUhs+5H5lJ0iQT3FRX2Yi4avoLkv4CWxBcNt11oZiuFKGmXQhFN6kuRESSgitXKf6A5g+I6cqFLiKCiptOkraZ5H4cuZOkH9oLw+UO8z7nvGfei3DFohoEQLAKCGWyGJCSbQZsD3ei7r+f45WATV6xmjdISyAlYLaDliveux+ti5ozwLAaFPhIlk6G6f18sxuO7l9bBiWbVoncDGCMWMl++PI/gJ7IohkmD0zkBxT5oRn5LTpN7ULqBByEtFywWvRyHz8Fx3dvN0UwuMPyR0uJFycddH7tKPHZRn7BjnwwkQcUud0HO/K7oOENcTYPRCEAFERmsCpyfZDZfpdnBosOUKGx13BC60QTIdjpmSzrEWDR+UagjsgMciLXAwfhc4MNpKfQtOPE6mVh0gEOaeTXtWXXEdADpJAR/GaZ/rrM9Us82wORPd5zHbRp7JVcyxR5PRP5LTPy6uo0HXIlG9awMgLuEUABkBYI7VLyxs+yyPW3+Nyg4zqoG+UV7TDRlMBbWA/D43u3qlbLGmgRWM2BjFxjQjWQGQBuu4bYYvrm14JMnQSXcjBcny9A5DXIiDJp9/8FWCXAAOaFUEfILTgIcttOvf+2NJnLdI0fBFUyskZKBGTEJEAmBuzr9LCSUDwGQAwwgMzu+m9/VWOAeZSsk+YP48C4li/sRok1nlBd5PogBjgxNw70PfHyqBADyEVXicasZXCVpxYiLfO+HxVZDHCVYwvu2ebPT7fOLNAm65AWLnEAzvsEcOi9/lPU1aACXDfOLdhD9kxNszGdweQGYstqXgItJwDFNxKvwrp57G8h0zVwldHuA0IFt8El83yIs2FSDcpgxDJpUUTwK7gTduN3AK5iG7ehc/E2/gUPD3q3eY4awwAAAABJRU5ErkJggg==" alt="CNB" /></a>
|
||||
<a href="https://mcptoplist.com/server/io.github.t8y2%2Fdbx"><img src="https://mcptoplist.com/badge/io.github.t8y2%2Fdbx.svg" alt="MCP Toplist" /></a>
|
||||
</p>
|
||||
@@ -63,16 +64,13 @@
|
||||
</p>
|
||||
</div>
|
||||
|
||||
## 赞助商与合作伙伴
|
||||
## ❤️ 赞助商
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td align="center" valign="middle" width="220">
|
||||
<td align="center" valign="middle" width="200">
|
||||
<a href="https://www.rainyun.com/MTE5Mjc4Ng==_" target="_blank">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="docs/public/sponsors/rainyun-dark.png" />
|
||||
<img src="https://www.rainyun.com/img/logo.d193755d.png" alt="雨云" height="48" />
|
||||
</picture>
|
||||
<img src="docs/public/sponsors/rainyun-card.png" alt="雨云" width="175" />
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
@@ -81,37 +79,42 @@
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center" valign="middle" width="220">
|
||||
<a href="https://www.qiniu.com/" target="_blank">
|
||||
<img src="https://www-static.qbox.me/_next/static/media/logo.0fc18feaa621d2068a7180631f742256.jpg" alt="七牛云" height="64" />
|
||||
<td align="center" valign="middle" width="200">
|
||||
<a href="https://www.trustasia.com/ssl/trustasia/code-signing" target="_blank">
|
||||
<img src="docs/public/sponsors/trustasia-card.png" alt="TrustAsia" width="175" />
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
七牛云为 DBX 提供对象存储、CDN 等云基础设施资源支持。
|
||||
<a href="https://www.qiniu.com/" target="_blank">访问七牛云</a>
|
||||
由 TrustAsia 提供代码签名云签服务,实现 CICD 自动化构建可信软件。
|
||||
<a href="https://www.trustasia.com/ssl/trustasia/code-signing" target="_blank">访问 TrustAsia</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center" valign="middle" width="220">
|
||||
<a href="https://easysearch.cn" target="_blank">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="docs/public/sponsors/easysearch-dark.png" />
|
||||
<img src="docs/public/sponsors/easysearch.png" alt="Easysearch" width="150" />
|
||||
</picture>
|
||||
<td align="center" valign="middle" width="200">
|
||||
<a href="https://www.jalapeno-cloud.ai/DBX" target="_blank">
|
||||
<img src="docs/public/sponsors/jalapeno-card.png" alt="Jalapeño Cloud" width="175" />
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
Easysearch 是一款企业级分布式搜索引擎,兼容 ES API、融合全文检索、向量检索、地理空间位置检索、实时分析与 AI 能力,为企业提供统一的数据检索与智能分析基础设施。
|
||||
<a href="https://easysearch.cn" target="_blank">访问 Easysearch</a>
|
||||
Jalapeño Cloud 是 AI 基础设施与 Token 算力平台,通过 DBX 专属入口可享新用户免费额度与充值加赠。
|
||||
<a href="https://www.jalapeno-cloud.ai/DBX" target="_blank">访问 Jalapeño Cloud</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center" valign="middle" width="220">
|
||||
<td align="center" valign="middle" width="200">
|
||||
<a href="https://www.ucloud.cn/site/active/kuaijiesale.html?ytag=geo_waituo_github_dbx" target="_blank">
|
||||
<img src="docs/public/sponsors/astraflow-card.png" alt="AstraFlow" width="175" />
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
UCloud 优刻得是国内首家公有云科创板上市公司,覆盖国内、亚洲、欧洲、北美等 28 个地域的云主机、数据库、CDN 等服务,注册享新客优惠 0.9 折起;星图 AstraFlow 大模型平台支持主流 200+ 大模型一键调用。
|
||||
<a href="https://www.ucloud.cn/site/active/kuaijiesale.html?ytag=geo_waituo_github_dbx" target="_blank">访问 UCloud 优刻得</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center" valign="middle" width="200">
|
||||
<a href="https://www.atlascloud.ai/?ref=6YYXWA" target="_blank">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="docs/public/sponsors/atlas-dark.svg" />
|
||||
<img src="https://www.atlascloud.ai/logo.svg" alt="Atlas Cloud" width="150" />
|
||||
</picture>
|
||||
<img src="docs/public/sponsors/atlas-card.png" alt="Atlas Cloud" width="175" />
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
@@ -120,17 +123,41 @@
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center" valign="middle" width="220">
|
||||
<a href="https://www.trustasia.com/ssl/trustasia/code-signing" target="_blank">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="docs/public/sponsors/trustasia-dark.svg" />
|
||||
<img src="docs/public/sponsors/trustasia.png" alt="TrustAsia" height="44" />
|
||||
</picture>
|
||||
<td align="center" valign="middle" width="200">
|
||||
<a href="https://www.qiniu.com/" target="_blank">
|
||||
<img src="docs/public/sponsors/qiniu-card.png" alt="七牛云" width="175" />
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
由 TrustAsia 提供代码签名云签服务,实现 CICD 自动化构建可信软件。
|
||||
<a href="https://www.trustasia.com/ssl/trustasia/code-signing" target="_blank">访问 TrustAsia</a>
|
||||
七牛云为 DBX 提供对象存储、CDN 等云基础设施资源支持。
|
||||
<a href="https://www.qiniu.com/" target="_blank">访问七牛云</a>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
## 🤝 合作伙伴
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td align="center" valign="middle" width="200">
|
||||
<a href="https://1panel.cn/" target="_blank">
|
||||
<img src="docs/public/sponsors/1panel-card.png" alt="1Panel" width="175" />
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
1Panel 是现代化的开源 Linux 服务器运维管理面板与轻量级 AI 管理平台,提供直观易用的 Web 界面,支持 AI 智能体、本地大模型、网站、数据库、容器、文件等核心场景的一站式管理。
|
||||
<a href="https://1panel.cn/" target="_blank">访问 1Panel</a>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td align="center" valign="middle" width="200">
|
||||
<a href="https://easysearch.cn" target="_blank">
|
||||
<img src="docs/public/sponsors/easysearch-card.png" alt="Easysearch" width="175" />
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
Easysearch 是一款企业级分布式搜索引擎,兼容 ES API、融合全文检索、向量检索、地理空间位置检索、实时分析与 AI 能力,为企业提供统一的数据检索与智能分析基础设施。
|
||||
<a href="https://easysearch.cn" target="_blank">访问 Easysearch</a>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
@@ -141,7 +168,7 @@
|
||||
<tr>
|
||||
<td width="50%">
|
||||
<h3>🪶 20 MB,极致轻量</h3>
|
||||
<p>无需 Java 运行环境,无需 Python 虚拟环境,不内嵌 Chromium。DBX 是单个小巧的二进制文件——下载、安装、连接。DBeaver 依赖 Java;TablePlus 仅限 macOS。DBX 全平台可用,无需额外运行时。</p>
|
||||
<p>无需 Java 运行环境,无需 Python 虚拟环境,不内嵌 Chromium。DBX 是单个小巧的二进制文件——下载、安装、连接。DBeaver 依赖 Java;TablePlus 是 Freemium。DBX 全平台可用,无需额外运行时。</p>
|
||||
</td>
|
||||
<td width="50%">
|
||||
<h3>🤖 AI 原生集成在编辑器里</h3>
|
||||
@@ -470,6 +497,7 @@ make package
|
||||
<a href="https://qm.qq.com/q/1087880322" target="_blank"><img src="https://img.shields.io/badge/QQ%20群-1087880322-EB1923?logo=tencentqq&logoColor=white" alt="QQ 群" /></a>
|
||||
<a href="https://docs.qq.com/doc/DVVhMY0h1ekJqc0tz" target="_blank"><img src="https://img.shields.io/badge/微信交流群-点击加入-07C160?logo=wechat&logoColor=white" alt="微信交流群" /></a>
|
||||
[](https://linux.do)
|
||||
[](https://1panel.cn)
|
||||
|
||||
## 赞助与捐助
|
||||
|
||||
@@ -514,7 +542,7 @@ MySQL、PostgreSQL、SQLite、Cloudflare D1、Redis、MongoDB、DuckDB、ClickHo
|
||||
## 贡献者
|
||||
|
||||
<a href="https://github.com/t8y2/dbx/graphs/contributors">
|
||||
<img src="https://contrib.rocks/image?repo=t8y2/dbx" />
|
||||
<img src="https://contrib.rocks/image?repo=t8y2/dbx&max=300&columns=15" />
|
||||
</a>
|
||||
|
||||
## Star History
|
||||
|
||||
+6
-5
@@ -45,7 +45,8 @@ Each agent runs as a standalone process and communicates with DBX via stdin/stdo
|
||||
| yashandb | 崖山 YashanDB | YashanDB JDBC |
|
||||
| xugu | 虚谷 XuguDB | XuguDB Go native agent |
|
||||
| iotdb | Apache IoTDB | Apache IoTDB Go Client native agent |
|
||||
| etcd | etcd | jetcd |
|
||||
| etcd | etcd 3.3 degraded, 3.4+ full (v3 API) | etcd client/v3 native Go agent |
|
||||
| etcd2 | etcd 2.0–2.3, 3.0–3.5 with --enable-v2 (v2 API) | etcd v2 HTTP/JSON native Go agent |
|
||||
| zookeeper | Apache ZooKeeper | go-zookeeper native agent |
|
||||
| rabbitmq | RabbitMQ | amqp091-go native agent |
|
||||
| rocketmq | Apache RocketMQ | rocketmq-admin-go native agent |
|
||||
@@ -53,7 +54,7 @@ Each agent runs as a standalone process and communicates with DBX via stdin/stdo
|
||||
|
||||
## Multi-JRE Support
|
||||
|
||||
Most Java agents target JRE 21. Native agents, such as `cassandra`, `duckdb`, `hive`, `iotdb`, `oracle`, `kingbase`, `tdengine`, `xugu`, `rabbitmq`, `rocketmq`, and `zookeeper`, do not require a JRE. DBX downloads and manages the JRE 21 installation automatically for Java agents.
|
||||
Most Java agents target JRE 21. Native agents, such as `cassandra`, `duckdb`, `hive`, `iotdb`, `oracle`, `kingbase`, `tdengine`, `xugu`, `rabbitmq`, `rocketmq`, `zookeeper`, `etcd`, and `etcd2`, do not require a JRE. DBX downloads and manages the JRE 21 installation automatically for Java agents.
|
||||
|
||||
## JDBC Connection Pooling
|
||||
|
||||
@@ -79,7 +80,7 @@ Set `DBX_AGENT_JDBC_POOL_ENABLED=false` for a runtime-level compatibility fallba
|
||||
|
||||
For new agents, prefer a **native (Go or Rust) driver** over a Java/JDBC agent whenever a mature, license-compatible native driver is available. Native agents ship as a single self-contained executable with no JRE, which significantly reduces memory footprint and startup time — the JVM baseline that every Java agent pays even when idle is avoided entirely.
|
||||
|
||||
- **Native (C++/Go/Rust)** — preferred when a usable native driver exists. See `drivers/cassandra-go` (Apache cassandra-gocql-driver), `drivers/duckdb`, `drivers/hive-go` (native HS2), `drivers/iotdb` (Apache IoTDB Go Client), `drivers/oracle-go` (go-ora), `drivers/kingbase-go` (gokb), `drivers/vastbase-go` (openGauss connector), `drivers/tdengine` (taos-connector-rust), `drivers/xugu`, `drivers/rabbitmq` (amqp091-go), `drivers/rocketmq` (rocketmq-admin-go), and `drivers/zookeeper` (go-zookeeper) as reference implementations. No JRE download or management is needed.
|
||||
- **Native (C++/Go/Rust)** — preferred when a usable native driver exists. See `drivers/cassandra-go` (Apache cassandra-gocql-driver), `drivers/duckdb`, `drivers/hive-go` (native HS2), `drivers/iotdb` (Apache IoTDB Go Client), `drivers/oracle-go` (go-ora), `drivers/kingbase-go` (gokb), `drivers/vastbase-go` (openGauss connector), `drivers/tdengine` (taos-connector-rust), `drivers/xugu`, `drivers/rabbitmq` (amqp091-go), `drivers/rocketmq` (rocketmq-admin-go), and `drivers/zookeeper` (go-zookeeper), `drivers/etcd-go` (etcd client/v3), and `drivers/etcd2-go` (etcd v2 HTTP API) as reference implementations. No JRE download or management is needed.
|
||||
- **Java/JDBC** — the default fallback when only a JDBC driver exists for the database, or when the native driver is immature or unmaintained. Most agents still fall in this category.
|
||||
|
||||
Native agents implement the same JSON-RPC contract and `versions.json` registration as Java agents; they ship an `agent` executable instead of `agent.jar`. If both native and Java source implementations exist for the same database, publish only the native artifact unless the Java variant has a separately registered compatibility profile, such as `oracle-legacy` / `oracle-10g`.
|
||||
@@ -103,7 +104,7 @@ Requires JDK 21 (Gradle toolchain auto-downloads if needed).
|
||||
(cd drivers/zookeeper && go build -o agent .)
|
||||
```
|
||||
|
||||
Output JARs are in `drivers/{module}/build/libs/`. Native agents build from `drivers/cassandra-go`, `drivers/duckdb`, `drivers/hive-go`, `drivers/iotdb`, `drivers/oracle-go`, `drivers/kingbase-go`, `drivers/vastbase-go`, `drivers/tdengine`, `drivers/xugu`, `drivers/rabbitmq`, `drivers/rocketmq`, and `drivers/zookeeper`.
|
||||
Output JARs are in `drivers/{module}/build/libs/`. Native agents build from `drivers/cassandra-go`, `drivers/duckdb`, `drivers/hive-go`, `drivers/iotdb`, `drivers/oracle-go`, `drivers/kingbase-go`, `drivers/vastbase-go`, `drivers/tdengine`, `drivers/xugu`, `drivers/rabbitmq`, `drivers/rocketmq`, `drivers/zookeeper`, `drivers/etcd-go`, and `drivers/etcd2-go`.
|
||||
|
||||
### Local DBX Runtime Test
|
||||
|
||||
@@ -117,7 +118,7 @@ cp agents/drivers/<db_type>/build/libs/*-all.jar ~/.dbx/agents/drivers/<db_type>
|
||||
|
||||
Restart DBX or disconnect and reconnect the database so the new agent process loads the replacement JAR.
|
||||
|
||||
Native agents such as `cassandra`, `hive`, `iotdb`, `oracle`, `kingbase`, `tdengine`, `xugu`, `rabbitmq`, `rocketmq`, and `zookeeper` use an `agent` executable instead of `agent.jar`. TDengine builds `target/release/dbx-tdengine-driver` from `drivers/tdengine/Cargo.toml`.
|
||||
Native agents such as `cassandra`, `hive`, `iotdb`, `oracle`, `kingbase`, `tdengine`, `xugu`, `rabbitmq`, `rocketmq`, `zookeeper`, `etcd`, and `etcd2` use an `agent` executable instead of `agent.jar`. TDengine builds `target/release/dbx-tdengine-driver` from `drivers/tdengine/Cargo.toml`.
|
||||
|
||||
## Versioning
|
||||
|
||||
|
||||
@@ -90,14 +90,26 @@ public abstract class AbstractJdbcAgent extends BaseDatabaseAgent {
|
||||
return unchecked(() -> {
|
||||
loadDriver(params);
|
||||
try (Connection conn = openTestConnection(params)) {
|
||||
String validationQuery = connectionValidationQuery();
|
||||
boolean valid = isConnectionValid(conn, 5);
|
||||
Map<String, Object> result = new LinkedHashMap<>();
|
||||
result.put("ok", valid);
|
||||
result.put(
|
||||
"validation",
|
||||
validationQuery == null || validationQuery.isBlank() ? "jdbc_connection_isValid" : validationQuery
|
||||
);
|
||||
if (valid) {
|
||||
Map<String, String> databaseInfo = JdbcDatabaseInfo.from(conn);
|
||||
if (!databaseInfo.isEmpty()) {
|
||||
result.put("databaseInfo", databaseInfo);
|
||||
}
|
||||
} else {
|
||||
result.put(
|
||||
"error",
|
||||
"Connection opened, but validation failed (method: "
|
||||
+ (validationQuery == null || validationQuery.isBlank() ? "Connection.isValid" : validationQuery)
|
||||
+ ")"
|
||||
);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@@ -184,7 +196,9 @@ public abstract class AbstractJdbcAgent extends BaseDatabaseAgent {
|
||||
options.getMaxRows(),
|
||||
options.getFetchSize(),
|
||||
options.getTimeoutSecs(),
|
||||
resultValueReader()
|
||||
resultValueReader(),
|
||||
JdbcExecutor.StatementMessageReader.NONE,
|
||||
advancePastUpdateCounts()
|
||||
);
|
||||
}
|
||||
|
||||
@@ -199,7 +213,8 @@ public abstract class AbstractJdbcAgent extends BaseDatabaseAgent {
|
||||
this::setSchemaSQL,
|
||||
this::resetSchemaSQL,
|
||||
options,
|
||||
resultValueReader()
|
||||
resultValueReader(),
|
||||
advancePastUpdateCounts()
|
||||
);
|
||||
}
|
||||
|
||||
@@ -517,6 +532,10 @@ public abstract class AbstractJdbcAgent extends BaseDatabaseAgent {
|
||||
protected void beforeQueryExecution(Connection connection, int timeoutSecs) throws Exception {
|
||||
}
|
||||
|
||||
protected boolean advancePastUpdateCounts() {
|
||||
return false;
|
||||
}
|
||||
|
||||
protected void beforePooledConnectionReturn(Connection connection) throws Exception {
|
||||
}
|
||||
|
||||
|
||||
@@ -295,7 +295,9 @@ public final class DdlBuilder {
|
||||
private static boolean isCharacterType(String normalized) {
|
||||
return "character varying".equals(normalized)
|
||||
|| "varchar".equals(normalized)
|
||||
|| "nvarchar".equals(normalized)
|
||||
|| "char".equals(normalized)
|
||||
|| "nchar".equals(normalized)
|
||||
|| "character".equals(normalized);
|
||||
}
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ public class JdbcAgentProfile {
|
||||
private final boolean nativeTableDdlSupported;
|
||||
private final boolean objectSourceSupported;
|
||||
private final boolean triggersSupported;
|
||||
private final boolean escapeSchemaWildcards;
|
||||
|
||||
public JdbcAgentProfile(String driverClass, String urlTemplate) {
|
||||
this(driverClass, urlTemplate, 0);
|
||||
@@ -75,6 +76,40 @@ public class JdbcAgentProfile {
|
||||
boolean nativeTableDdlSupported,
|
||||
boolean objectSourceSupported,
|
||||
boolean triggersSupported
|
||||
) {
|
||||
this(
|
||||
driverClass,
|
||||
urlTemplate,
|
||||
defaultPort,
|
||||
skipExecutionContext,
|
||||
excludedSchemas,
|
||||
tableTypes,
|
||||
identifierQuote,
|
||||
schemaSwitchPrefix,
|
||||
catalogFallbackEnabled,
|
||||
nativeTableDdlSupported,
|
||||
objectSourceSupported,
|
||||
triggersSupported,
|
||||
// 默认对 schema pattern 里的 _ / % 通配符转义(getSearchStringEscape()),
|
||||
// HANA 的 _SYS_RT 等含下划线 schema 依赖它避免误匹配。
|
||||
true
|
||||
);
|
||||
}
|
||||
|
||||
public JdbcAgentProfile(
|
||||
String driverClass,
|
||||
String urlTemplate,
|
||||
int defaultPort,
|
||||
boolean skipExecutionContext,
|
||||
Set<String> excludedSchemas,
|
||||
List<String> tableTypes,
|
||||
String identifierQuote,
|
||||
String schemaSwitchPrefix,
|
||||
boolean catalogFallbackEnabled,
|
||||
boolean nativeTableDdlSupported,
|
||||
boolean objectSourceSupported,
|
||||
boolean triggersSupported,
|
||||
boolean escapeSchemaWildcards
|
||||
) {
|
||||
this.driverClass = driverClass;
|
||||
this.urlTemplate = urlTemplate;
|
||||
@@ -88,6 +123,7 @@ public class JdbcAgentProfile {
|
||||
this.nativeTableDdlSupported = nativeTableDdlSupported;
|
||||
this.objectSourceSupported = objectSourceSupported;
|
||||
this.triggersSupported = triggersSupported;
|
||||
this.escapeSchemaWildcards = escapeSchemaWildcards;
|
||||
}
|
||||
|
||||
public String getDriverClass() {
|
||||
@@ -138,6 +174,10 @@ public class JdbcAgentProfile {
|
||||
return triggersSupported;
|
||||
}
|
||||
|
||||
public boolean getEscapeSchemaWildcards() {
|
||||
return escapeSchemaWildcards;
|
||||
}
|
||||
|
||||
public String buildUrl(ConnectParams params) {
|
||||
if (!params.getConnection_string().trim().isEmpty()) {
|
||||
return params.getConnection_string();
|
||||
|
||||
@@ -102,6 +102,34 @@ public final class JdbcExecutor {
|
||||
int timeoutSecs,
|
||||
ResultValueReader valueReader,
|
||||
StatementMessageReader statementMessageReader
|
||||
) {
|
||||
return execute(
|
||||
conn,
|
||||
sql,
|
||||
schema,
|
||||
setSchemaSql,
|
||||
resetSchemaSql,
|
||||
maxRows,
|
||||
fetchSize,
|
||||
timeoutSecs,
|
||||
valueReader,
|
||||
statementMessageReader,
|
||||
false
|
||||
);
|
||||
}
|
||||
|
||||
public QueryResult execute(
|
||||
Connection conn,
|
||||
String sql,
|
||||
String schema,
|
||||
Function<String, String> setSchemaSql,
|
||||
Supplier<String> resetSchemaSql,
|
||||
int maxRows,
|
||||
Integer fetchSize,
|
||||
int timeoutSecs,
|
||||
ResultValueReader valueReader,
|
||||
StatementMessageReader statementMessageReader,
|
||||
boolean advancePastUpdateCounts
|
||||
) {
|
||||
return unchecked(() -> {
|
||||
String trimmedSql = trimSql(sql);
|
||||
@@ -122,17 +150,31 @@ public final class JdbcExecutor {
|
||||
// Do not translate them to Connection.commit(), which requires autoCommit=false.
|
||||
boolean hasResultSet = stmt.execute(trimmedSql);
|
||||
long elapsed = System.currentTimeMillis() - start;
|
||||
long affectedRows;
|
||||
if (advancePastUpdateCounts) {
|
||||
affectedRows = 0L;
|
||||
while (!hasResultSet) {
|
||||
int updateCount = stmt.getUpdateCount();
|
||||
if (updateCount < 0) {
|
||||
break;
|
||||
}
|
||||
affectedRows += updateCount;
|
||||
hasResultSet = stmt.getMoreResults();
|
||||
}
|
||||
} else {
|
||||
int updateCount = hasResultSet ? -1 : stmt.getUpdateCount();
|
||||
affectedRows = updateCount >= 0 ? updateCount : 0L;
|
||||
}
|
||||
QueryResult result;
|
||||
if (hasResultSet) {
|
||||
try (ResultSet rs = stmt.getResultSet()) {
|
||||
result = readResultSet(rs, elapsed, effectiveMaxRows, valueReader);
|
||||
}
|
||||
} else {
|
||||
int updateCount = stmt.getUpdateCount();
|
||||
result = new QueryResult(
|
||||
Collections.emptyList(),
|
||||
Collections.emptyList(),
|
||||
updateCount >= 0 ? updateCount : 0,
|
||||
affectedRows,
|
||||
elapsed,
|
||||
false
|
||||
);
|
||||
@@ -240,6 +282,30 @@ public final class JdbcExecutor {
|
||||
return executePage(conn, sql, schema, setSchemaSql, resetSchemaSql, options, valueReader, StatementMessageReader.NONE, sessions);
|
||||
}
|
||||
|
||||
public QueryPageResult executePage(
|
||||
Connection conn,
|
||||
String sql,
|
||||
String schema,
|
||||
Function<String, String> setSchemaSql,
|
||||
Supplier<String> resetSchemaSql,
|
||||
QueryPageOptions options,
|
||||
ResultValueReader valueReader,
|
||||
boolean advancePastUpdateCounts
|
||||
) {
|
||||
return executePage(
|
||||
conn,
|
||||
sql,
|
||||
schema,
|
||||
setSchemaSql,
|
||||
resetSchemaSql,
|
||||
options,
|
||||
valueReader,
|
||||
StatementMessageReader.NONE,
|
||||
sessions,
|
||||
advancePastUpdateCounts
|
||||
);
|
||||
}
|
||||
|
||||
public QueryPageResult startTableRead(
|
||||
Connection conn,
|
||||
String sql,
|
||||
@@ -273,6 +339,32 @@ public final class JdbcExecutor {
|
||||
ResultValueReader valueReader,
|
||||
StatementMessageReader statementMessageReader,
|
||||
ConcurrentHashMap<String, QuerySession> targetSessions
|
||||
) {
|
||||
return executePage(
|
||||
conn,
|
||||
sql,
|
||||
schema,
|
||||
setSchemaSql,
|
||||
resetSchemaSql,
|
||||
options,
|
||||
valueReader,
|
||||
statementMessageReader,
|
||||
targetSessions,
|
||||
false
|
||||
);
|
||||
}
|
||||
|
||||
private QueryPageResult executePage(
|
||||
Connection conn,
|
||||
String sql,
|
||||
String schema,
|
||||
Function<String, String> setSchemaSql,
|
||||
Supplier<String> resetSchemaSql,
|
||||
QueryPageOptions options,
|
||||
ResultValueReader valueReader,
|
||||
StatementMessageReader statementMessageReader,
|
||||
ConcurrentHashMap<String, QuerySession> targetSessions,
|
||||
boolean advancePastUpdateCounts
|
||||
) {
|
||||
return unchecked(() -> {
|
||||
expireIdleSessions(targetSessions, System.currentTimeMillis(), QUERY_SESSION_IDLE_TIMEOUT_MILLIS);
|
||||
@@ -293,12 +385,26 @@ public final class JdbcExecutor {
|
||||
// JDBC transaction APIs are reserved for executeTransaction.
|
||||
boolean hasResultSet = stmt.execute(trimmedSql);
|
||||
long elapsed = System.currentTimeMillis() - start;
|
||||
long affectedRows;
|
||||
if (advancePastUpdateCounts) {
|
||||
affectedRows = 0L;
|
||||
while (!hasResultSet) {
|
||||
int updateCount = stmt.getUpdateCount();
|
||||
if (updateCount < 0) {
|
||||
break;
|
||||
}
|
||||
affectedRows += updateCount;
|
||||
hasResultSet = stmt.getMoreResults();
|
||||
}
|
||||
} else {
|
||||
int updateCount = hasResultSet ? -1 : stmt.getUpdateCount();
|
||||
affectedRows = updateCount >= 0 ? updateCount : 0L;
|
||||
}
|
||||
if (!hasResultSet) {
|
||||
int updateCount = stmt.getUpdateCount();
|
||||
QueryResult result = new QueryResult(
|
||||
Collections.emptyList(),
|
||||
Collections.emptyList(),
|
||||
updateCount >= 0 ? updateCount : 0,
|
||||
affectedRows,
|
||||
elapsed,
|
||||
false
|
||||
);
|
||||
|
||||
@@ -162,7 +162,8 @@ public final class JsonRpcServer {
|
||||
if (AgentProtocol.METHOD_TEST_CONNECTION.equals(method)) {
|
||||
Map<String, Object> result = agent.testConnectionWithInfo(gson.fromJson(params, ConnectParams.class));
|
||||
if (!Boolean.TRUE.equals(result.get("ok"))) {
|
||||
throw new RuntimeException("Connection failed");
|
||||
Object error = result.get("error");
|
||||
throw new RuntimeException(error == null ? "Connection failed" : String.valueOf(error));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -123,12 +123,12 @@ public final class StandardJdbcMetadata {
|
||||
listObjects(listTables(conn, profile, configuredDatabase, schema, tableConstraints), schema)
|
||||
);
|
||||
DatabaseMetaData meta = conn.getMetaData();
|
||||
appendRoutines(result, meta, null, schema, schema);
|
||||
appendRoutines(result, meta, profile.getEscapeSchemaWildcards(), null, schema, schema);
|
||||
if (!containsRoutine(result)
|
||||
&& profile.getCatalogFallbackEnabled()
|
||||
&& configuredDatabase != null
|
||||
&& !configuredDatabase.trim().isEmpty()) {
|
||||
appendRoutines(result, meta, configuredDatabase, schema, schema);
|
||||
appendRoutines(result, meta, profile.getEscapeSchemaWildcards(), configuredDatabase, schema, schema);
|
||||
}
|
||||
result.sort(Comparator.comparing(ObjectInfo::getName));
|
||||
return normalized.filterObjects(result);
|
||||
@@ -350,7 +350,7 @@ public final class StandardJdbcMetadata {
|
||||
// JDBC has no portable metadata limit/offset. Use table types for safe pushdown and filter/page locally.
|
||||
// schema 是 DatabaseMetaData.getTables() 的 schemaPattern,_ 和 % 具有通配符语义;
|
||||
// HANA 存在 _SYS_RT 等含下划线的 schema,需按 getSearchStringEscape() 转义,避免混入其他 schema 的对象。
|
||||
try (ResultSet rs = meta.getTables(catalog, escapeSchemaPattern(meta, schema), "%", tableTypes)) {
|
||||
try (ResultSet rs = meta.getTables(catalog, escapeSchemaPattern(meta, schema, profile.getEscapeSchemaWildcards()), "%", tableTypes)) {
|
||||
while (rs.next()) {
|
||||
result.add(new TableInfo(
|
||||
rs.getString("TABLE_NAME"),
|
||||
@@ -635,12 +635,14 @@ public final class StandardJdbcMetadata {
|
||||
private static void appendRoutines(
|
||||
List<ObjectInfo> result,
|
||||
DatabaseMetaData meta,
|
||||
boolean escapeSchemaWildcards,
|
||||
String catalog,
|
||||
String schema,
|
||||
String schemaLabel
|
||||
) {
|
||||
// schema 作为 getProcedures/getFunctions 的 schemaPattern,同样需要转义 _ 和 % 通配符。
|
||||
String schemaPattern = escapeSchemaPattern(meta, schema);
|
||||
// schema 作为 getProcedures/getFunctions 的 schemaPattern,同样需要转义 _ 和 % 通配符
|
||||
// (databend 等驱动除外,见 escapeSchemaPattern)。
|
||||
String schemaPattern = escapeSchemaPattern(meta, schema, escapeSchemaWildcards);
|
||||
Set<String> procedureNames = new LinkedHashSet<>();
|
||||
try (ResultSet rs = meta.getProcedures(catalog, schemaPattern, "%")) {
|
||||
while (rs.next()) {
|
||||
@@ -707,11 +709,17 @@ public final class StandardJdbcMetadata {
|
||||
*
|
||||
* <p>当 schema 为空白(将映射为 {@code null},表示不做 schema 过滤)或驱动未提供转义字符时,原样返回。
|
||||
*/
|
||||
private static String escapeSchemaPattern(DatabaseMetaData meta, String schema) {
|
||||
private static String escapeSchemaPattern(DatabaseMetaData meta, String schema, boolean escapeWildcards) {
|
||||
String normalized = blankToNull(schema);
|
||||
if (normalized == null) {
|
||||
return null;
|
||||
}
|
||||
// 某些驱动(如 databend)的 getSearchStringEscape() 返回 '\',但其 getTables/getProcedures 把 _ 当字面量
|
||||
// 且忽略转义,转义反而匹配不到,导致含 _ 的库名(如 my_db)返回 0 行(#8114)。
|
||||
// 这类驱动通过 profile.escapeSchemaWildcards=false 关闭转义。
|
||||
if (!escapeWildcards) {
|
||||
return normalized;
|
||||
}
|
||||
String escape;
|
||||
try {
|
||||
escape = meta.getSearchStringEscape();
|
||||
|
||||
@@ -56,8 +56,9 @@ public final class TransactionExecutor {
|
||||
return unchecked(() -> {
|
||||
long start = System.currentTimeMillis();
|
||||
if (!supportsTransactions(conn)) {
|
||||
long totalAffected = executeAll(conn, statements, schema, setSchemaSql, resetSchemaSql, runner);
|
||||
return result(totalAffected, start);
|
||||
// An explicit transaction request must not degrade to auto-commit:
|
||||
// callers rely on it to avoid partial writes when a later statement fails.
|
||||
throw new UnsupportedOperationException("Transactions are not supported by this JDBC driver");
|
||||
}
|
||||
|
||||
boolean savedAutoCommit = conn.getAutoCommit();
|
||||
|
||||
@@ -626,17 +626,19 @@ class CommonJavaCompatibilityTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
void executesTransactionsOneByOneWhenJdbcDriverDoesNotSupportTransactions() {
|
||||
void rejectsTransactionsWhenJdbcDriverDoesNotSupportTransactions() {
|
||||
List<String> calls = new ArrayList<>();
|
||||
DatabaseAgent agent = new TransactionAgent(nonTransactionalConnection(calls));
|
||||
|
||||
QueryResult result = agent.executeTransaction(Arrays.asList("UPDATE A SET ID = 1", "UPDATE B SET ID = 2"), "APP");
|
||||
|
||||
assertEquals(2L, result.getAffected_rows());
|
||||
assertEquals(
|
||||
Arrays.asList("supportsTransactions", "execute:SET SCHEMA \"APP\"", "executeUpdate:UPDATE A SET ID = 1", "executeUpdate:UPDATE B SET ID = 2"),
|
||||
calls
|
||||
UnsupportedOperationException error = assertThrows(
|
||||
UnsupportedOperationException.class,
|
||||
() -> agent.executeTransaction(Arrays.asList("UPDATE A SET ID = 1", "UPDATE B SET ID = 2"), "APP")
|
||||
);
|
||||
|
||||
assertEquals("Transactions are not supported by this JDBC driver", error.getMessage());
|
||||
// Capability is checked before schema switching or statements, so a
|
||||
// failed transaction request cannot leave a partially applied batch.
|
||||
assertEquals(Collections.singletonList("supportsTransactions"), calls);
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -724,6 +726,30 @@ class CommonJavaCompatibilityTest {
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void buildsTableDdlWithNationalCharacterLength() {
|
||||
String ddl = DdlBuilder.buildTableDdl(
|
||||
"DBX_DEMO",
|
||||
"CUSTOMERS",
|
||||
Collections.singletonList(new ColumnInfo(
|
||||
"NAME",
|
||||
"NVARCHAR",
|
||||
false,
|
||||
null,
|
||||
false,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
100
|
||||
)),
|
||||
Collections.emptyList(),
|
||||
Collections.emptyList()
|
||||
);
|
||||
|
||||
assertTrue(ddl.contains("\"NAME\" NVARCHAR(100) NOT NULL"));
|
||||
}
|
||||
|
||||
private static class MinimalAgent implements DatabaseAgent {
|
||||
@Override
|
||||
public void connect(ConnectParams params) {
|
||||
|
||||
@@ -242,6 +242,47 @@ class JdbcExecutorTest {
|
||||
assertEquals(Arrays.asList(Arrays.asList(1, "Ada")), result.getRows());
|
||||
}
|
||||
|
||||
@Test
|
||||
void executeAdvancesPastUpdateCountsToTheFollowingResultSet() {
|
||||
CountingResultSetFixture fixture = countingResultSet(new Object[][]{{42, "Bill_Record"}});
|
||||
|
||||
QueryResult result = JdbcExecutor.INSTANCE.execute(
|
||||
updateThenResultSetConnection(1, fixture.resultSet()),
|
||||
"DECLARE @tables TABLE (id INT, name NVARCHAR(128)); INSERT INTO @tables VALUES (42, 'Bill_Record'); SELECT * FROM @tables;",
|
||||
"",
|
||||
schema -> "",
|
||||
() -> "",
|
||||
100,
|
||||
null,
|
||||
0,
|
||||
JdbcExecutor.INSTANCE::defaultResultValue,
|
||||
JdbcExecutor.StatementMessageReader.NONE,
|
||||
true
|
||||
);
|
||||
|
||||
assertEquals(Arrays.asList("id", "name"), result.getColumns());
|
||||
assertEquals(Arrays.asList(Arrays.asList(42, "Bill_Record")), result.getRows());
|
||||
}
|
||||
|
||||
@Test
|
||||
void executePageAdvancesPastUpdateCountsToTheFollowingResultSet() {
|
||||
CountingResultSetFixture fixture = countingResultSet(new Object[][]{{42, "Bill_Record"}});
|
||||
|
||||
QueryPageResult result = JdbcExecutor.INSTANCE.executePage(
|
||||
updateThenResultSetConnection(1, fixture.resultSet()),
|
||||
"DECLARE @tables TABLE (id INT, name NVARCHAR(128)); INSERT INTO @tables VALUES (42, 'Bill_Record'); SELECT * FROM @tables;",
|
||||
"",
|
||||
schema -> "",
|
||||
() -> "",
|
||||
new QueryPageOptions(100, null, 100),
|
||||
JdbcExecutor.INSTANCE::defaultResultValue,
|
||||
true
|
||||
);
|
||||
|
||||
assertEquals(Arrays.asList("id", "name"), result.getColumns());
|
||||
assertEquals(Arrays.asList(Arrays.asList(42, "Bill_Record")), result.getRows());
|
||||
}
|
||||
|
||||
@Test
|
||||
void executeStillPropagatesStatementErrors() {
|
||||
SQLException failure = new SQLException("permission denied", "42000", 229);
|
||||
@@ -467,6 +508,42 @@ class JdbcExecutorTest {
|
||||
);
|
||||
}
|
||||
|
||||
private static Connection updateThenResultSetConnection(int updateCount, ResultSet resultSet) {
|
||||
AtomicInteger resultIndex = new AtomicInteger();
|
||||
InvocationHandler statementHandler = (Object unused, Method method, Object[] args) -> {
|
||||
switch (method.getName()) {
|
||||
case "execute":
|
||||
return false;
|
||||
case "getUpdateCount":
|
||||
return resultIndex.get() == 0 ? updateCount : -1;
|
||||
case "getMoreResults":
|
||||
return resultIndex.incrementAndGet() == 1;
|
||||
case "getResultSet":
|
||||
return resultSet;
|
||||
case "close":
|
||||
return null;
|
||||
default:
|
||||
return defaultValue(method.getReturnType());
|
||||
}
|
||||
};
|
||||
Statement statement = (Statement) Proxy.newProxyInstance(
|
||||
Statement.class.getClassLoader(),
|
||||
new Class<?>[]{Statement.class},
|
||||
statementHandler
|
||||
);
|
||||
InvocationHandler connectionHandler = (Object unused, Method method, Object[] args) -> {
|
||||
if (method.getName().equals("createStatement")) {
|
||||
return statement;
|
||||
}
|
||||
return defaultValue(method.getReturnType());
|
||||
};
|
||||
return (Connection) Proxy.newProxyInstance(
|
||||
Connection.class.getClassLoader(),
|
||||
new Class<?>[]{Connection.class},
|
||||
connectionHandler
|
||||
);
|
||||
}
|
||||
|
||||
private static Object defaultValue(Class<?> type) {
|
||||
if (type == Boolean.TYPE) {
|
||||
return false;
|
||||
|
||||
@@ -279,6 +279,38 @@ class StandardJdbcMetadataTest {
|
||||
assertEquals("SALES", capturedArgs.get()[1]);
|
||||
}
|
||||
|
||||
@Test
|
||||
void listTablesSkipsEscapeWhenProfileDisablesWildcards() {
|
||||
// databend 等驱动的 getSearchStringEscape() 返回 "\\",但 getTables 把 _ 当字面量且忽略转义,
|
||||
// 转义含 _ 的库名(如 my_db)会返回 0 行(#8114)。profile.escapeSchemaWildcards=false 时应原样传入。
|
||||
JdbcAgentProfile noEscapeProfile = new JdbcAgentProfile(
|
||||
"example.Driver",
|
||||
"jdbc:example://{host}:{port}/{database}",
|
||||
0,
|
||||
false,
|
||||
Collections.emptySet(),
|
||||
Arrays.asList("TABLE", "VIEW", "BASE TABLE"),
|
||||
"\"",
|
||||
"USE",
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
false
|
||||
);
|
||||
AtomicReference<Object[]> capturedArgs = new AtomicReference<>();
|
||||
Connection conn = schemaEscapeConnection("\\", rows(
|
||||
row("TABLE_NAME", "A", "TABLE_TYPE", "TABLE", "REMARKS", null)
|
||||
), capturedArgs);
|
||||
|
||||
List<TableInfo> tables = StandardJdbcMetadata.INSTANCE.listTables(conn, noEscapeProfile, "", "my_db");
|
||||
|
||||
assertEquals(1, tables.size());
|
||||
assertEquals("A", tables.get(0).getName());
|
||||
// 关闭转义:schemaPattern 应为原始 "my_db",而非 "my\\_db"
|
||||
assertEquals("my_db", capturedArgs.get()[1]);
|
||||
}
|
||||
|
||||
private static Connection schemaEscapeConnection(String searchEscape, ResultSet tables, AtomicReference<Object[]> capturedArgs) {
|
||||
DatabaseMetaData meta = proxy(DatabaseMetaData.class, new MethodHandler() {
|
||||
@Override
|
||||
|
||||
@@ -37,7 +37,7 @@ DBX uses unique short-lived logical sessions for independent Agent metadata task
|
||||
|
||||
Runtime reuse keys include the Agent driver key, executable or JAR path, launch arguments, working directory, JRE selection, JVM options, classpath-affecting options, and native executable version boundary. Host, account, schema, and credentials belong to sessions and are not part of the runtime key.
|
||||
|
||||
Etcd and ZooKeeper retain the legacy path because they use the key-value Agent protocol rather than the SQL session contract. Older Agent binaries and JARs also remain on the legacy path.
|
||||
ZooKeeper retains the legacy single-session path because its Agent does not advertise the `multi_session` capability; etcd Agents (v3 and v2) run on the shared multi-session path like the SQL Agents. Older Agent binaries and JARs also remain on the legacy path.
|
||||
|
||||
## Resource limits and recovery
|
||||
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
# Argo Go agent notes
|
||||
|
||||
Status date: 2026-09-03.
|
||||
|
||||
This agent is a fork of `agents/drivers/hive-go` created to serve 星环Argo
|
||||
(Transwarp ArgoDB) connections exclusively: `supportsRoutines()` returns true
|
||||
unconditionally and connection identity reports `ArgoDB (Transwarp)` /
|
||||
`DBX ArgoDB Go Agent`. Vanilla Hive / Kyuubi / Impala stay on hive-go.
|
||||
|
||||
No Hive 3 / Hive 4 / Kyuubi parity is claimed for this directory. The
|
||||
validation matrix in `agents/drivers/hive-go/MIGRATION_PARITY.md` applies to
|
||||
hive-go only.
|
||||
|
||||
## Maintenance rule
|
||||
|
||||
Most files are kept byte-identical to hive-go. Fixes landing in hive-go that
|
||||
touch `metadata.go` or `main.go` must be ported here (adjusting only the
|
||||
argo-specific branding, the unconditional `supportsRoutines()`, and log
|
||||
prefixes), including their regression tests.
|
||||
|
||||
## Validation
|
||||
|
||||
Validated against Transwarp ArgoDB by the PR author (#7933):
|
||||
|
||||
- `go test ./...` green in this module (routine views asserted queried
|
||||
unconditionally; ArgoDB identity asserted)
|
||||
- End-to-end on a local build with an ArgoDB connection: 星环Argo branding
|
||||
shown, stored-procedure source opens as one whole statement, `CREATE OR
|
||||
REPLACE PROCEDURE` executes as a single statement, and execute errors
|
||||
surface readable diagnostics
|
||||
@@ -0,0 +1,101 @@
|
||||
# Hive Agent benchmark
|
||||
|
||||
This benchmark compares the same DBX JSON-RPC operations through the native
|
||||
Go Hive Agent and the archived JDBC Hive Agent. Both candidates run on the same
|
||||
host and connect to the same HiveServer2 instance.
|
||||
|
||||
Use `functional_probe.py` before running performance benchmarks. It validates
|
||||
connection, sessions, query values, metadata, pagination, failed-SQL semantics,
|
||||
clean shutdown, and Java/Go result parity without concurrency load.
|
||||
|
||||
The runner measures:
|
||||
|
||||
- process startup and fresh connection latency;
|
||||
- artifact size, idle RSS, and peak RSS;
|
||||
- `SELECT 1`-shape lookup and 100/1,000/10,000-row decoding;
|
||||
- `list_databases`, `list_tables`, and complete paged reads;
|
||||
- 1, 8, and 32 concurrent DBX Agent sessions;
|
||||
- mean, p50, p95, p99, throughput, and clean shutdown behavior.
|
||||
|
||||
Candidate order rotates between rounds to reduce warm-cache and server-order
|
||||
bias. Startup and connection samples always use a fresh Agent process.
|
||||
|
||||
## Prepare the fixture
|
||||
|
||||
The defaults expect `dbx_agent_bench.agent_bench` with exactly 10,000 or more
|
||||
rows and columns named `id` and `payload`:
|
||||
|
||||
```sql
|
||||
CREATE DATABASE IF NOT EXISTS dbx_agent_bench;
|
||||
CREATE TABLE IF NOT EXISTS dbx_agent_bench.agent_bench (
|
||||
id BIGINT,
|
||||
payload STRING
|
||||
) STORED AS ORC;
|
||||
```
|
||||
|
||||
Populate deterministic rows before running the benchmark. Keep the fixture,
|
||||
HiveServer2 configuration, Agent host, and Java runtime unchanged between
|
||||
candidates.
|
||||
|
||||
## Run
|
||||
|
||||
Functional parity probe:
|
||||
|
||||
```bash
|
||||
GO_AGENT=/tmp/dbx-hive-bench/hive-agent-linux-amd64 \
|
||||
JDBC_AGENT_JAR=/tmp/dbx-hive-bench/dbx-agent-hive.jar \
|
||||
JAVA_BIN=/tmp/dbx-hive-bench/jre21/bin/java \
|
||||
HIVE_HOST=127.0.0.1 \
|
||||
HIVE_PORT=10000 \
|
||||
HIVE_DATABASE=dbx_agent_bench \
|
||||
HIVE_URL_PARAMS=auth=noSasl \
|
||||
python3 agents/drivers/hive-go/bench/functional_probe.py \
|
||||
> /tmp/dbx-hive-bench/functional-result.json
|
||||
```
|
||||
|
||||
The functional probe runs the Go candidate only by default. Set
|
||||
`BENCH_CANDIDATES=go,jdbc` only when an explicit historical-JDBC comparison is
|
||||
needed.
|
||||
|
||||
Performance benchmark:
|
||||
|
||||
```bash
|
||||
GO_AGENT=/tmp/dbx-hive-bench/hive-agent-linux-amd64 \
|
||||
JDBC_AGENT_JAR=/tmp/dbx-hive-bench/dbx-agent-hive.jar \
|
||||
HIVE_HOST=127.0.0.1 \
|
||||
HIVE_PORT=10000 \
|
||||
HIVE_DATABASE=dbx_agent_bench \
|
||||
HIVE_URL_PARAMS=auth=noSasl \
|
||||
python3 agents/drivers/hive-go/bench/agent_compare.py \
|
||||
> /tmp/dbx-hive-bench/result.json
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
- `BENCH_CANDIDATES`: performance benchmark default `go,jdbc`; functional probe
|
||||
default `go`.
|
||||
- `GO_AGENT_COMMAND`, `JDBC_AGENT_COMMAND`: optional full launch commands.
|
||||
- `BENCH_STARTUPS`, `BENCH_CONNECTS`: fresh-process sample counts, default `8`.
|
||||
- `BENCH_ROUNDS`: alternating steady-state rounds, default `3`.
|
||||
- `BENCH_WARMUPS`: warmups before each workload, default `2`.
|
||||
- `BENCH_CONCURRENCY`: comma-separated session counts, default `1,8,32`.
|
||||
- `BENCH_CONCURRENCY_OPS_PER_WORKER`: operations per session, default `8`.
|
||||
- `HIVE_HOST`, `HIVE_PORT`, `HIVE_DATABASE`, `HIVE_BENCH_TABLE`.
|
||||
- `HIVE_USERNAME`, `HIVE_PASSWORD`, `HIVE_URL_PARAMS`, `HIVE_CONNECTION_STRING`.
|
||||
- `HIVE_SSL`, `HIVE_CA_CERT_PATH`, `HIVE_CLIENT_CERT_PATH`, `HIVE_CLIENT_KEY_PATH`.
|
||||
- `BENCH_*_SQL` and `BENCH_*_COUNT` override individual workloads.
|
||||
|
||||
Run the benchmark on the Agent host. Do not compare a local Go process with a
|
||||
remote JDBC process, use different HS2 endpoints, or mutate the fixture between
|
||||
candidates.
|
||||
|
||||
## Kerberos fixture
|
||||
|
||||
`kdc_fixture` starts a test-only in-process KDC and writes a temporary
|
||||
`krb5.conf` and keytab containing `alice` and `hive/localhost`. Never use these
|
||||
credentials outside an isolated compatibility environment, and delete the
|
||||
generated directory after validation.
|
||||
|
||||
```bash
|
||||
go run ./bench/kdc_fixture -dir /tmp/dbx-hive-kerberos
|
||||
```
|
||||
+664
@@ -0,0 +1,664 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import os
|
||||
import queue
|
||||
import shlex
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Candidate:
|
||||
name: str
|
||||
command: list[str]
|
||||
artifact: Path
|
||||
rss_command: str = ""
|
||||
|
||||
|
||||
class AgentProcess:
|
||||
def __init__(self, candidate: Candidate):
|
||||
self.candidate = candidate
|
||||
self.process = subprocess.Popen(
|
||||
candidate.command,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
bufsize=1,
|
||||
)
|
||||
self.request_id = 0
|
||||
self.request_lock = threading.Lock()
|
||||
self.write_lock = threading.Lock()
|
||||
self.pending: dict[int, queue.Queue] = {}
|
||||
self.ready = threading.Event()
|
||||
self.saw_ready = False
|
||||
self.exited = threading.Event()
|
||||
self.stderr_lines: list[str] = []
|
||||
threading.Thread(target=self._read_stdout, daemon=True).start()
|
||||
threading.Thread(target=self._drain_stderr, daemon=True).start()
|
||||
if not self.ready.wait(env_float("BENCH_READY_TIMEOUT", 30.0)) or not self.saw_ready:
|
||||
raise TimeoutError(self._failure("timed out waiting for agent readiness"))
|
||||
|
||||
def _read_stdout(self) -> None:
|
||||
assert self.process.stdout is not None
|
||||
for line in self.process.stdout:
|
||||
try:
|
||||
response = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if response.get("ready") is True:
|
||||
self.saw_ready = True
|
||||
self.ready.set()
|
||||
continue
|
||||
response_id = response.get("id")
|
||||
if not isinstance(response_id, int):
|
||||
continue
|
||||
with self.request_lock:
|
||||
response_queue = self.pending.get(response_id)
|
||||
if response_queue is not None:
|
||||
response_queue.put(response)
|
||||
self.exited.set()
|
||||
self.ready.set()
|
||||
with self.request_lock:
|
||||
pending = list(self.pending.values())
|
||||
for response_queue in pending:
|
||||
response_queue.put(RuntimeError(self._failure("agent process exited")))
|
||||
|
||||
def _drain_stderr(self) -> None:
|
||||
assert self.process.stderr is not None
|
||||
for line in self.process.stderr:
|
||||
self.stderr_lines.append(line.rstrip())
|
||||
|
||||
def call(self, method: str, params: dict | None = None) -> object:
|
||||
if self.exited.is_set():
|
||||
raise RuntimeError(self._failure(f"agent exited before {method}"))
|
||||
with self.request_lock:
|
||||
self.request_id += 1
|
||||
request_id = self.request_id
|
||||
response_queue: queue.Queue = queue.Queue(maxsize=1)
|
||||
self.pending[request_id] = response_queue
|
||||
request = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": request_id,
|
||||
"method": method,
|
||||
"params": params or {},
|
||||
}
|
||||
try:
|
||||
assert self.process.stdin is not None
|
||||
with self.write_lock:
|
||||
self.process.stdin.write(json.dumps(request, separators=(",", ":")) + "\n")
|
||||
self.process.stdin.flush()
|
||||
response = response_queue.get(timeout=env_float("BENCH_RPC_TIMEOUT", 180.0))
|
||||
except queue.Empty as error:
|
||||
raise TimeoutError(self._failure(f"timed out during {method}")) from error
|
||||
finally:
|
||||
with self.request_lock:
|
||||
self.pending.pop(request_id, None)
|
||||
if isinstance(response, Exception):
|
||||
raise response
|
||||
if response.get("error") is not None:
|
||||
raise RuntimeError(
|
||||
f"{self.candidate.name} {method}: "
|
||||
f"{json.dumps(response['error'], ensure_ascii=False)}"
|
||||
)
|
||||
return response.get("result")
|
||||
|
||||
def rss_kib(self) -> int:
|
||||
if self.candidate.rss_command:
|
||||
output = subprocess.check_output(
|
||||
self.candidate.rss_command,
|
||||
shell=True,
|
||||
text=True,
|
||||
).strip()
|
||||
return int(output or "0")
|
||||
status_path = Path(f"/proc/{self.process.pid}/status")
|
||||
if status_path.is_file():
|
||||
for line in status_path.read_text().splitlines():
|
||||
if line.startswith("VmRSS:"):
|
||||
return int(line.split()[1])
|
||||
output = subprocess.check_output(
|
||||
["ps", "-o", "rss=", "-p", str(self.process.pid)],
|
||||
text=True,
|
||||
).strip()
|
||||
return int(output or "0")
|
||||
|
||||
def close(self) -> bool:
|
||||
if self.process.poll() is not None:
|
||||
return True
|
||||
try:
|
||||
self.call("shutdown")
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
self.process.wait(timeout=3)
|
||||
return True
|
||||
except subprocess.TimeoutExpired:
|
||||
self.process.terminate()
|
||||
try:
|
||||
self.process.wait(timeout=2)
|
||||
except subprocess.TimeoutExpired:
|
||||
self.process.kill()
|
||||
self.process.wait(timeout=5)
|
||||
return False
|
||||
|
||||
def _failure(self, message: str) -> str:
|
||||
stderr = "\n".join(self.stderr_lines[-20:])
|
||||
return f"{self.candidate.name}: {message}\n{stderr}".rstrip()
|
||||
|
||||
|
||||
class RSSMonitor:
|
||||
def __init__(self, process: AgentProcess):
|
||||
self.process = process
|
||||
self.peak_kib = 0
|
||||
self.stop_event = threading.Event()
|
||||
self.thread = threading.Thread(target=self._run, daemon=True)
|
||||
|
||||
def __enter__(self):
|
||||
self.peak_kib = self.process.rss_kib()
|
||||
self.thread.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
self.stop_event.set()
|
||||
self.thread.join(timeout=1)
|
||||
try:
|
||||
self.peak_kib = max(self.peak_kib, self.process.rss_kib())
|
||||
except (OSError, subprocess.SubprocessError, ValueError):
|
||||
pass
|
||||
|
||||
def _run(self) -> None:
|
||||
interval = env_float("BENCH_RSS_INTERVAL", 0.02)
|
||||
while not self.stop_event.wait(interval):
|
||||
try:
|
||||
self.peak_kib = max(self.peak_kib, self.process.rss_kib())
|
||||
except (OSError, subprocess.SubprocessError, ValueError):
|
||||
return
|
||||
|
||||
|
||||
def main() -> None:
|
||||
candidates = configured_candidates()
|
||||
connection = connection_params()
|
||||
startup_iterations = env_int("BENCH_STARTUPS", 8)
|
||||
connect_iterations = env_int("BENCH_CONNECTS", 8)
|
||||
rounds = env_int("BENCH_ROUNDS", 3)
|
||||
warmups = env_int("BENCH_WARMUPS", 2)
|
||||
workloads = configured_workloads(connection["database"])
|
||||
concurrency_levels = env_int_list("BENCH_CONCURRENCY", [1, 8, 32])
|
||||
|
||||
startup_samples = {candidate.name: [] for candidate in candidates}
|
||||
connect_samples = {candidate.name: [] for candidate in candidates}
|
||||
workload_samples = {
|
||||
candidate.name: {workload["name"]: [] for workload in workloads}
|
||||
for candidate in candidates
|
||||
}
|
||||
concurrency_samples = {
|
||||
candidate.name: {str(level): [] for level in concurrency_levels}
|
||||
for candidate in candidates
|
||||
}
|
||||
process_metrics = {candidate.name: [] for candidate in candidates}
|
||||
|
||||
for iteration in range(startup_iterations):
|
||||
for candidate in rotated(candidates, iteration):
|
||||
startup_samples[candidate.name].append(benchmark_startup(candidate))
|
||||
|
||||
for iteration in range(connect_iterations):
|
||||
for candidate in rotated(candidates, iteration):
|
||||
connect_samples[candidate.name].append(benchmark_connect(candidate, connection))
|
||||
|
||||
for round_index in range(rounds):
|
||||
for candidate in rotated(candidates, round_index):
|
||||
process = AgentProcess(candidate)
|
||||
shutdown_clean = False
|
||||
metrics = None
|
||||
try:
|
||||
process.call("connect", connection)
|
||||
idle_rss_kib = process.rss_kib()
|
||||
with RSSMonitor(process) as monitor:
|
||||
for workload in workloads:
|
||||
sample = benchmark_workload(process, workload, warmups)
|
||||
workload_samples[candidate.name][workload["name"]].append(sample)
|
||||
process.call("disconnect")
|
||||
metrics = {
|
||||
"idle_rss_kib": idle_rss_kib,
|
||||
"peak_rss_kib": monitor.peak_kib,
|
||||
}
|
||||
finally:
|
||||
shutdown_clean = process.close()
|
||||
if metrics is not None:
|
||||
metrics["shutdown_exited_within_3s"] = shutdown_clean
|
||||
process_metrics[candidate.name].append(metrics)
|
||||
|
||||
for level in concurrency_levels:
|
||||
for candidate in rotated(candidates, round_index + level):
|
||||
concurrency_samples[candidate.name][str(level)].append(
|
||||
benchmark_concurrency(candidate, connection, level, warmups)
|
||||
)
|
||||
|
||||
results = []
|
||||
for candidate in candidates:
|
||||
name = candidate.name
|
||||
results.append(
|
||||
{
|
||||
"candidate": name,
|
||||
"command": candidate.command,
|
||||
"artifact_bytes": candidate.artifact.stat().st_size,
|
||||
"startup": summarize_latencies(startup_samples[name]),
|
||||
"connect": summarize_latencies(connect_samples[name]),
|
||||
"process": summarize_process_metrics(process_metrics[name]),
|
||||
"workloads": [
|
||||
summarize_rounds(workload["name"], workload_samples[name][workload["name"]])
|
||||
for workload in workloads
|
||||
],
|
||||
"concurrency": [
|
||||
summarize_rounds(f"concurrency_{level}", concurrency_samples[name][str(level)])
|
||||
for level in concurrency_levels
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
output = {
|
||||
"host": os.uname().nodename,
|
||||
"server": env_default("HIVE_SERVER", f"{connection['host']}:{connection['port']}"),
|
||||
"database": connection["database"],
|
||||
"table": env_default("HIVE_BENCH_TABLE", "agent_bench"),
|
||||
"startup_iterations": startup_iterations,
|
||||
"connect_iterations": connect_iterations,
|
||||
"rounds": rounds,
|
||||
"warmups": warmups,
|
||||
"concurrency_levels": concurrency_levels,
|
||||
"results": results,
|
||||
}
|
||||
json.dump(output, sys.stdout, ensure_ascii=False, indent=2)
|
||||
sys.stdout.write("\n")
|
||||
|
||||
|
||||
def configured_candidates() -> list[Candidate]:
|
||||
selected = {
|
||||
item.strip()
|
||||
for item in env_default("BENCH_CANDIDATES", "go,jdbc").split(",")
|
||||
if item.strip()
|
||||
}
|
||||
candidates = []
|
||||
if "go" in selected:
|
||||
artifact = required_path("GO_AGENT")
|
||||
raw_command = os.getenv("GO_AGENT_COMMAND", "")
|
||||
command = shlex.split(raw_command) if raw_command else [str(artifact)]
|
||||
candidates.append(
|
||||
Candidate("go-native", command, artifact, os.getenv("GO_RSS_COMMAND", ""))
|
||||
)
|
||||
if "jdbc" in selected:
|
||||
artifact = required_path("JDBC_AGENT_JAR")
|
||||
raw_command = os.getenv("JDBC_AGENT_COMMAND", "")
|
||||
command = (
|
||||
shlex.split(raw_command)
|
||||
if raw_command
|
||||
else [env_default("JAVA_BIN", "java"), "-jar", str(artifact)]
|
||||
)
|
||||
candidates.append(
|
||||
Candidate("jdbc-java", command, artifact, os.getenv("JDBC_RSS_COMMAND", ""))
|
||||
)
|
||||
if not candidates:
|
||||
raise ValueError("BENCH_CANDIDATES selected no candidates")
|
||||
return candidates
|
||||
|
||||
|
||||
def connection_params() -> dict:
|
||||
return {
|
||||
"host": env_default("HIVE_HOST", "127.0.0.1"),
|
||||
"port": env_int("HIVE_PORT", 10000),
|
||||
"database": env_default("HIVE_DATABASE", "dbx_agent_bench"),
|
||||
"username": os.getenv("HIVE_USERNAME", ""),
|
||||
"password": os.getenv("HIVE_PASSWORD", ""),
|
||||
"url_params": env_default("HIVE_URL_PARAMS", "auth=noSasl"),
|
||||
"connection_string": os.getenv("HIVE_CONNECTION_STRING", ""),
|
||||
"ssl": env_bool("HIVE_SSL", False),
|
||||
"ca_cert_path": os.getenv("HIVE_CA_CERT_PATH", ""),
|
||||
"client_cert_path": os.getenv("HIVE_CLIENT_CERT_PATH", ""),
|
||||
"client_key_path": os.getenv("HIVE_CLIENT_KEY_PATH", ""),
|
||||
"connect_timeout_secs": env_int("HIVE_CONNECT_TIMEOUT", 30),
|
||||
}
|
||||
|
||||
|
||||
def configured_workloads(database: str) -> list[dict]:
|
||||
table = env_default("HIVE_BENCH_TABLE", "agent_bench")
|
||||
qualified = f"`{database}`.`{table}`"
|
||||
workloads = [
|
||||
query_workload("select_one", "SELECT 1 AS value", 1, 40),
|
||||
query_workload("rows_100", f"SELECT id, payload FROM {qualified} LIMIT 100", 100, 20),
|
||||
query_workload("rows_1000", f"SELECT id, payload FROM {qualified} LIMIT 1000", 1000, 10),
|
||||
query_workload("rows_10000", f"SELECT id, payload FROM {qualified} LIMIT 10000", 10000, 3),
|
||||
{
|
||||
"name": "list_databases",
|
||||
"kind": "rpc",
|
||||
"method": "list_databases",
|
||||
"params": {},
|
||||
"count": env_int("BENCH_LIST_DATABASES_COUNT", 20),
|
||||
},
|
||||
{
|
||||
"name": "list_tables",
|
||||
"kind": "rpc",
|
||||
"method": "list_tables",
|
||||
"params": {"schema": database},
|
||||
"count": env_int("BENCH_LIST_TABLES_COUNT", 20),
|
||||
},
|
||||
{
|
||||
"name": "page_10000_by_500",
|
||||
"kind": "paged",
|
||||
"sql": env_default("BENCH_PAGE_SQL", f"SELECT id, payload FROM {qualified} LIMIT 10000"),
|
||||
"max_rows": 10000,
|
||||
"page_size": env_int("BENCH_PAGE_SIZE", 500),
|
||||
"count": env_int("BENCH_PAGE_COUNT", 3),
|
||||
},
|
||||
]
|
||||
return workloads
|
||||
|
||||
|
||||
def query_workload(name: str, fallback_sql: str, max_rows: int, fallback_count: int) -> dict:
|
||||
suffix = name.upper()
|
||||
return {
|
||||
"name": name,
|
||||
"kind": "rpc",
|
||||
"method": "execute_query",
|
||||
"params": {
|
||||
"sql": env_default(f"BENCH_{suffix}_SQL", fallback_sql),
|
||||
"maxRows": max_rows,
|
||||
"fetchSize": min(max_rows, env_int("BENCH_FETCH_SIZE", 1000)),
|
||||
},
|
||||
"count": env_int(f"BENCH_{suffix}_COUNT", fallback_count),
|
||||
}
|
||||
|
||||
|
||||
def benchmark_startup(candidate: Candidate) -> float:
|
||||
started = time.perf_counter()
|
||||
process = AgentProcess(candidate)
|
||||
elapsed = elapsed_ms(started)
|
||||
process.close()
|
||||
return elapsed
|
||||
|
||||
|
||||
def benchmark_connect(candidate: Candidate, connection: dict) -> float:
|
||||
process = AgentProcess(candidate)
|
||||
try:
|
||||
started = time.perf_counter()
|
||||
process.call("connect", connection)
|
||||
return elapsed_ms(started)
|
||||
finally:
|
||||
process.close()
|
||||
|
||||
|
||||
def benchmark_workload(process: AgentProcess, workload: dict, warmups: int) -> dict:
|
||||
for _ in range(warmups):
|
||||
execute_workload(process, workload)
|
||||
samples = []
|
||||
started = time.perf_counter()
|
||||
for _ in range(workload["count"]):
|
||||
operation_started = time.perf_counter()
|
||||
execute_workload(process, workload)
|
||||
samples.append(elapsed_ms(operation_started))
|
||||
elapsed = time.perf_counter() - started
|
||||
return sample_result(workload["count"], elapsed, samples)
|
||||
|
||||
|
||||
def benchmark_concurrency(
|
||||
candidate: Candidate,
|
||||
connection: dict,
|
||||
concurrency: int,
|
||||
warmups: int,
|
||||
) -> dict:
|
||||
process = AgentProcess(candidate)
|
||||
session_ids = [f"bench-{concurrency}-{index}" for index in range(concurrency)]
|
||||
qualified = (
|
||||
f"`{connection['database']}`."
|
||||
f"`{env_default('HIVE_BENCH_TABLE', 'agent_bench')}`"
|
||||
)
|
||||
workload = {
|
||||
"kind": "rpc",
|
||||
"method": "execute_query",
|
||||
"params": {
|
||||
"sql": env_default(
|
||||
"BENCH_CONCURRENCY_SQL",
|
||||
"SELECT 1 AS value",
|
||||
),
|
||||
"maxRows": 1,
|
||||
},
|
||||
}
|
||||
operations_per_worker = env_int("BENCH_CONCURRENCY_OPS_PER_WORKER", 8)
|
||||
try:
|
||||
for session_id in session_ids:
|
||||
process.call("open_session", {**connection, "agentSessionId": session_id})
|
||||
for session_id in session_ids:
|
||||
for _ in range(warmups):
|
||||
execute_workload(process, workload, session_id)
|
||||
with RSSMonitor(process) as monitor:
|
||||
started = time.perf_counter()
|
||||
with ThreadPoolExecutor(max_workers=concurrency) as executor:
|
||||
futures = [
|
||||
executor.submit(
|
||||
concurrency_worker,
|
||||
process,
|
||||
workload,
|
||||
session_id,
|
||||
operations_per_worker,
|
||||
)
|
||||
for session_id in session_ids
|
||||
]
|
||||
samples = [sample for future in futures for sample in future.result()]
|
||||
elapsed = time.perf_counter() - started
|
||||
result = sample_result(len(samples), elapsed, samples)
|
||||
result["concurrency"] = concurrency
|
||||
result["peak_rss_kib"] = monitor.peak_kib
|
||||
return result
|
||||
finally:
|
||||
for session_id in session_ids:
|
||||
try:
|
||||
process.call("close_session", {"agentSessionId": session_id})
|
||||
except Exception:
|
||||
pass
|
||||
process.close()
|
||||
|
||||
|
||||
def concurrency_worker(
|
||||
process: AgentProcess,
|
||||
workload: dict,
|
||||
session_id: str,
|
||||
operations: int,
|
||||
) -> list[float]:
|
||||
samples = []
|
||||
for _ in range(operations):
|
||||
started = time.perf_counter()
|
||||
execute_workload(process, workload, session_id)
|
||||
samples.append(elapsed_ms(started))
|
||||
return samples
|
||||
|
||||
|
||||
def execute_workload(
|
||||
process: AgentProcess,
|
||||
workload: dict,
|
||||
agent_session_id: str = "",
|
||||
) -> object:
|
||||
params = dict(workload.get("params", {}))
|
||||
if agent_session_id:
|
||||
params["agentSessionId"] = agent_session_id
|
||||
if workload["kind"] == "rpc":
|
||||
return process.call(workload["method"], params)
|
||||
if workload["kind"] != "paged":
|
||||
raise ValueError(f"unknown workload kind: {workload['kind']}")
|
||||
first = process.call(
|
||||
"execute_query_page",
|
||||
{
|
||||
"sql": workload["sql"],
|
||||
"maxRows": workload["max_rows"],
|
||||
"pageSize": workload["page_size"],
|
||||
**({"agentSessionId": agent_session_id} if agent_session_id else {}),
|
||||
},
|
||||
)
|
||||
rows = len(first.get("rows", []))
|
||||
session_id = first.get("session_id")
|
||||
has_more = first.get("has_more", False)
|
||||
try:
|
||||
while has_more:
|
||||
page = process.call(
|
||||
"fetch_query_page",
|
||||
{
|
||||
"sessionId": session_id,
|
||||
"pageSize": workload["page_size"],
|
||||
**({"agentSessionId": agent_session_id} if agent_session_id else {}),
|
||||
},
|
||||
)
|
||||
rows += len(page.get("rows", []))
|
||||
session_id = page.get("session_id")
|
||||
has_more = page.get("has_more", False)
|
||||
finally:
|
||||
if session_id:
|
||||
process.call(
|
||||
"close_query_session",
|
||||
{
|
||||
"sessionId": session_id,
|
||||
**({"agentSessionId": agent_session_id} if agent_session_id else {}),
|
||||
},
|
||||
)
|
||||
if rows != workload["max_rows"]:
|
||||
raise RuntimeError(
|
||||
f"{process.candidate.name} paged query returned {rows} rows, "
|
||||
f"expected {workload['max_rows']}"
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def sample_result(count: int, elapsed: float, samples: list[float]) -> dict:
|
||||
summary = summarize_latencies(samples)
|
||||
summary.update(
|
||||
{
|
||||
"count": count,
|
||||
"elapsed_ms": elapsed * 1000,
|
||||
"ops_per_sec": count / elapsed,
|
||||
}
|
||||
)
|
||||
return summary
|
||||
|
||||
|
||||
def summarize_latencies(samples: list[float]) -> dict:
|
||||
ordered = sorted(samples)
|
||||
return {
|
||||
"samples_ms": samples,
|
||||
"mean_ms": statistics.mean(samples),
|
||||
"p50_ms": percentile(ordered, 0.50),
|
||||
"p95_ms": percentile(ordered, 0.95),
|
||||
"p99_ms": percentile(ordered, 0.99),
|
||||
"min_ms": ordered[0],
|
||||
"max_ms": ordered[-1],
|
||||
}
|
||||
|
||||
|
||||
def summarize_rounds(name: str, rounds: list[dict]) -> dict:
|
||||
latencies = [sample for round_result in rounds for sample in round_result["samples_ms"]]
|
||||
elapsed = sum(round_result["elapsed_ms"] for round_result in rounds) / 1000
|
||||
result = summarize_latencies(latencies)
|
||||
result.update(
|
||||
{
|
||||
"name": name,
|
||||
"rounds": rounds,
|
||||
"count": len(latencies),
|
||||
"elapsed_ms": elapsed * 1000,
|
||||
"ops_per_sec": len(latencies) / elapsed,
|
||||
}
|
||||
)
|
||||
peak_values = [round_result.get("peak_rss_kib", 0) for round_result in rounds]
|
||||
if any(peak_values):
|
||||
result["peak_rss_kib"] = max(peak_values)
|
||||
return result
|
||||
|
||||
|
||||
def summarize_process_metrics(samples: list[dict]) -> dict:
|
||||
return {
|
||||
"idle_rss_kib": summarize_numbers([sample["idle_rss_kib"] for sample in samples]),
|
||||
"peak_rss_kib": summarize_numbers([sample["peak_rss_kib"] for sample in samples]),
|
||||
"shutdown_exited_within_3s": all(
|
||||
sample["shutdown_exited_within_3s"] for sample in samples
|
||||
),
|
||||
"rounds": samples,
|
||||
}
|
||||
|
||||
|
||||
def summarize_numbers(values: list[int]) -> dict:
|
||||
return {
|
||||
"min": min(values),
|
||||
"median": statistics.median(values),
|
||||
"max": max(values),
|
||||
}
|
||||
|
||||
|
||||
def rotated(values: list[Candidate], offset: int) -> list[Candidate]:
|
||||
if not values:
|
||||
return []
|
||||
shift = offset % len(values)
|
||||
return values[shift:] + values[:shift]
|
||||
|
||||
|
||||
def percentile(values: list[float], fraction: float) -> float:
|
||||
if not values:
|
||||
return 0.0
|
||||
index = min(len(values) - 1, max(0, round((len(values) - 1) * fraction)))
|
||||
return values[index]
|
||||
|
||||
|
||||
def elapsed_ms(started: float) -> float:
|
||||
return (time.perf_counter() - started) * 1000
|
||||
|
||||
|
||||
def required_path(name: str) -> Path:
|
||||
value = os.getenv(name, "")
|
||||
if not value:
|
||||
raise ValueError(f"{name} is required")
|
||||
path = Path(value).expanduser().resolve()
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(path)
|
||||
return path
|
||||
|
||||
|
||||
def env_default(name: str, fallback: str) -> str:
|
||||
return os.getenv(name, "") or fallback
|
||||
|
||||
|
||||
def env_int(name: str, fallback: int) -> int:
|
||||
value = int(env_default(name, str(fallback)))
|
||||
if value < 1:
|
||||
raise ValueError(f"{name} must be positive")
|
||||
return value
|
||||
|
||||
|
||||
def env_int_list(name: str, fallback: list[int]) -> list[int]:
|
||||
raw = os.getenv(name, "")
|
||||
values = fallback if not raw else [int(value.strip()) for value in raw.split(",")]
|
||||
if not values or any(value < 1 for value in values):
|
||||
raise ValueError(f"{name} must contain positive integers")
|
||||
return values
|
||||
|
||||
|
||||
def env_float(name: str, fallback: float) -> float:
|
||||
value = float(env_default(name, str(fallback)))
|
||||
if value <= 0:
|
||||
raise ValueError(f"{name} must be positive")
|
||||
return value
|
||||
|
||||
|
||||
def env_bool(name: str, fallback: bool) -> bool:
|
||||
raw = os.getenv(name)
|
||||
if raw is None or raw == "":
|
||||
return fallback
|
||||
normalized = raw.strip().lower()
|
||||
if normalized in {"1", "true", "yes", "on"}:
|
||||
return True
|
||||
if normalized in {"0", "false", "no", "off"}:
|
||||
return False
|
||||
raise ValueError(f"{name} must be a boolean")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,203 @@
|
||||
#!/usr/bin/env python3
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from agent_compare import AgentProcess, configured_candidates, connection_params, env_default, env_int
|
||||
|
||||
|
||||
def main() -> None:
|
||||
os.environ.setdefault("BENCH_CANDIDATES", "go")
|
||||
connection = connection_params()
|
||||
candidates = configured_candidates()
|
||||
results = {candidate.name: probe_candidate(candidate, connection) for candidate in candidates}
|
||||
output = {
|
||||
"server": env_default("HIVE_SERVER", f"{connection['host']}:{connection['port']}"),
|
||||
"connection": sanitized_connection(connection),
|
||||
"artifacts": {
|
||||
candidate.name: {
|
||||
"path": str(candidate.artifact),
|
||||
"sha256": sha256(candidate.artifact),
|
||||
"size_bytes": candidate.artifact.stat().st_size,
|
||||
}
|
||||
for candidate in candidates
|
||||
},
|
||||
"results": results,
|
||||
"parity": compare_results(results),
|
||||
}
|
||||
json.dump(output, sys.stdout, ensure_ascii=False, indent=2)
|
||||
sys.stdout.write("\n")
|
||||
if any(not result.get("ok") for result in results.values()) or not output["parity"]["ok"]:
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def probe_candidate(candidate, connection: dict) -> dict:
|
||||
process = None
|
||||
session_id = f"functional-probe-{candidate.name}"
|
||||
result = {"ok": False}
|
||||
try:
|
||||
process = AgentProcess(candidate)
|
||||
result["test_connection"] = process.call("test_connection", connection)
|
||||
process.call("open_session", {"agentSessionId": session_id, **connection})
|
||||
result["validate_session"] = process.call("validate_session", {"agentSessionId": session_id})
|
||||
result["select_one"] = normalized_query(
|
||||
process.call(
|
||||
"execute_query",
|
||||
{
|
||||
"agentSessionId": session_id,
|
||||
"sql": env_default("PROBE_SELECT_SQL", "SELECT 1 AS value"),
|
||||
"maxRows": env_int("PROBE_SELECT_MAX_ROWS", 10),
|
||||
"fetchSize": env_int("PROBE_FETCH_SIZE", 10),
|
||||
},
|
||||
)
|
||||
)
|
||||
result["databases"] = sorted(
|
||||
item.get("name", "")
|
||||
for item in process.call("list_databases", {"agentSessionId": session_id})
|
||||
)
|
||||
schema = env_default("PROBE_SCHEMA", connection["database"])
|
||||
result["tables"] = sorted(
|
||||
item.get("name", "")
|
||||
for item in process.call(
|
||||
"list_tables",
|
||||
{"agentSessionId": session_id, "schema": schema},
|
||||
)
|
||||
)
|
||||
result["paging"] = probe_paging(process, session_id)
|
||||
result["invalid_sql"] = probe_failure_semantics(process, session_id)
|
||||
result["after_failure"] = normalized_query(
|
||||
process.call(
|
||||
"execute_query",
|
||||
{
|
||||
"agentSessionId": session_id,
|
||||
"sql": env_default("PROBE_AFTER_FAILURE_SQL", "SELECT 2 AS value"),
|
||||
"maxRows": 10,
|
||||
"fetchSize": env_int("PROBE_FETCH_SIZE", 10),
|
||||
},
|
||||
)
|
||||
)
|
||||
process.call("close_session", {"agentSessionId": session_id})
|
||||
result["ok"] = True
|
||||
except Exception as error:
|
||||
result["error"] = str(error)
|
||||
finally:
|
||||
if process is not None:
|
||||
result["clean_shutdown"] = process.close()
|
||||
return result
|
||||
|
||||
|
||||
def probe_paging(process: AgentProcess, agent_session_id: str) -> dict:
|
||||
page_size = env_int("PROBE_PAGE_SIZE", 2)
|
||||
first = process.call(
|
||||
"execute_query_page",
|
||||
{
|
||||
"agentSessionId": agent_session_id,
|
||||
"sql": env_default(
|
||||
"PROBE_PAGE_SQL",
|
||||
"SELECT id, payload FROM dbx_agent_bench.agent_bench LIMIT 3",
|
||||
),
|
||||
"maxRows": env_int("PROBE_PAGE_MAX_ROWS", 3),
|
||||
"pageSize": page_size,
|
||||
},
|
||||
)
|
||||
pages = [first]
|
||||
query_session_id = first.get("session_id")
|
||||
while pages[-1].get("has_more"):
|
||||
pages.append(
|
||||
process.call(
|
||||
"fetch_query_page",
|
||||
{
|
||||
"agentSessionId": agent_session_id,
|
||||
"sessionId": query_session_id,
|
||||
"pageSize": page_size,
|
||||
},
|
||||
)
|
||||
)
|
||||
return {
|
||||
"columns": first.get("columns", []),
|
||||
"column_types": first.get("column_types", []),
|
||||
"rows": [row for page in pages for row in page.get("rows", [])],
|
||||
"page_count": len(pages),
|
||||
"has_more_final": pages[-1].get("has_more", False),
|
||||
"truncated": any(page.get("truncated", False) for page in pages),
|
||||
}
|
||||
|
||||
|
||||
def probe_failure_semantics(process: AgentProcess, agent_session_id: str) -> dict:
|
||||
try:
|
||||
process.call(
|
||||
"execute_query",
|
||||
{
|
||||
"agentSessionId": agent_session_id,
|
||||
"sql": env_default(
|
||||
"PROBE_INVALID_SQL",
|
||||
"SELECT * FROM dbx_missing_table_for_failure_semantics",
|
||||
),
|
||||
"maxRows": 10,
|
||||
"fetchSize": env_int("PROBE_FETCH_SIZE", 10),
|
||||
},
|
||||
)
|
||||
except Exception as error:
|
||||
return {"failed": True, "error": str(error)}
|
||||
return {"failed": False, "error": ""}
|
||||
|
||||
|
||||
def normalized_query(result: dict) -> dict:
|
||||
return {
|
||||
"columns": result.get("columns", []),
|
||||
"column_types": result.get("column_types", []),
|
||||
"rows": result.get("rows", []),
|
||||
"truncated": result.get("truncated", False),
|
||||
}
|
||||
|
||||
|
||||
def compare_results(results: dict) -> dict:
|
||||
successful = [result for result in results.values() if result.get("ok")]
|
||||
if len(successful) < 2:
|
||||
return {"ok": len(results) == 1 and len(successful) == 1, "differences": []}
|
||||
baseline = successful[0]
|
||||
differences = []
|
||||
for field in ["select_one", "databases", "tables", "paging", "after_failure"]:
|
||||
expected = baseline.get(field)
|
||||
for candidate_name, candidate_result in results.items():
|
||||
if candidate_result.get("ok") and candidate_result.get(field) != expected:
|
||||
differences.append(
|
||||
{
|
||||
"candidate": candidate_name,
|
||||
"field": field,
|
||||
"expected": expected,
|
||||
"actual": candidate_result.get(field),
|
||||
}
|
||||
)
|
||||
for candidate_name, candidate_result in results.items():
|
||||
if candidate_result.get("ok") and not candidate_result.get("invalid_sql", {}).get("failed"):
|
||||
differences.append(
|
||||
{
|
||||
"candidate": candidate_name,
|
||||
"field": "invalid_sql.failed",
|
||||
"expected": True,
|
||||
"actual": False,
|
||||
}
|
||||
)
|
||||
return {"ok": not differences, "differences": differences}
|
||||
|
||||
|
||||
def sanitized_connection(connection: dict) -> dict:
|
||||
result = dict(connection)
|
||||
if result.get("password"):
|
||||
result["password"] = "***"
|
||||
return result
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,99 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
|
||||
"github.com/jcmturner/krb5test"
|
||||
)
|
||||
|
||||
type fixtureInfo struct {
|
||||
Realm string `json:"realm"`
|
||||
Address string `json:"address"`
|
||||
ConfigPath string `json:"config_path"`
|
||||
KeytabPath string `json:"keytab_path"`
|
||||
ClientPrincipal string `json:"client_principal"`
|
||||
ServicePrincipal string `json:"service_principal"`
|
||||
ZooKeeperPrincipal string `json:"zookeeper_principal"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
directory := flag.String("dir", "", "directory for generated Kerberos fixture files")
|
||||
flag.Parse()
|
||||
if *directory == "" {
|
||||
log.Fatal("-dir is required")
|
||||
}
|
||||
if err := os.MkdirAll(*directory, 0o700); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
logger := log.New(os.Stderr, "kdc: ", log.LstdFlags)
|
||||
kdc, err := krb5test.NewKDC(map[string][]string{
|
||||
"alice": nil,
|
||||
"hive/localhost": nil,
|
||||
"zookeeper/localhost": nil,
|
||||
}, logger)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
kdc.KRB5Conf.LibDefaults.UDPPreferenceLimit = 1
|
||||
kdc.Start()
|
||||
defer kdc.Close()
|
||||
|
||||
configPath := filepath.Join(*directory, "krb5.conf")
|
||||
keytabPath := filepath.Join(*directory, "fixture.keytab")
|
||||
config := fmt.Sprintf(`[libdefaults]
|
||||
default_realm = %s
|
||||
dns_lookup_realm = false
|
||||
dns_lookup_kdc = false
|
||||
rdns = false
|
||||
udp_preference_limit = 1
|
||||
default_tgs_enctypes = aes256-cts-hmac-sha1-96
|
||||
default_tkt_enctypes = aes256-cts-hmac-sha1-96
|
||||
permitted_enctypes = aes256-cts-hmac-sha1-96
|
||||
|
||||
[realms]
|
||||
%s = {
|
||||
kdc = %s
|
||||
}
|
||||
|
||||
[domain_realm]
|
||||
.localhost = %s
|
||||
localhost = %s
|
||||
`, kdc.Realm, kdc.Realm, kdc.TCPListener.Addr().String(), kdc.Realm, kdc.Realm)
|
||||
if err := os.WriteFile(configPath, []byte(config), 0o644); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
keytab, err := kdc.Keytab.Marshal()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(keytabPath, keytab, 0o600); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
info := fixtureInfo{
|
||||
Realm: kdc.Realm,
|
||||
Address: kdc.TCPListener.Addr().String(),
|
||||
ConfigPath: configPath,
|
||||
KeytabPath: keytabPath,
|
||||
ClientPrincipal: "alice@" + kdc.Realm,
|
||||
ServicePrincipal: "hive/localhost@" + kdc.Realm,
|
||||
ZooKeeperPrincipal: "zookeeper/localhost@" + kdc.Realm,
|
||||
}
|
||||
if err := json.NewEncoder(os.Stdout).Encode(info); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err := os.Stdout.Sync(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
signals := make(chan os.Signal, 1)
|
||||
signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
|
||||
<-signals
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,713 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/x509"
|
||||
"encoding/base64"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
pkcs12 "software.sslmate.com/src/go-pkcs12"
|
||||
)
|
||||
|
||||
func writeHadoopVInt(buffer *bytes.Buffer, value int64) {
|
||||
if value >= -112 && value <= 127 {
|
||||
buffer.WriteByte(byte(int8(value)))
|
||||
return
|
||||
}
|
||||
lengthMarker := int8(-112)
|
||||
encoded := value
|
||||
if value < 0 {
|
||||
encoded = ^value
|
||||
lengthMarker = -120
|
||||
}
|
||||
temporary := encoded
|
||||
for temporary != 0 {
|
||||
temporary >>= 8
|
||||
lengthMarker--
|
||||
}
|
||||
buffer.WriteByte(byte(lengthMarker))
|
||||
length := -int(lengthMarker)
|
||||
if lengthMarker < -120 {
|
||||
length -= 120
|
||||
} else {
|
||||
length -= 112
|
||||
}
|
||||
for index := length; index != 0; index-- {
|
||||
shift := uint((index - 1) * 8)
|
||||
buffer.WriteByte(byte(encoded >> shift))
|
||||
}
|
||||
}
|
||||
|
||||
func encodeHadoopToken(identifier, password, kind, service []byte) string {
|
||||
var buffer bytes.Buffer
|
||||
for _, value := range [][]byte{identifier, password, kind, service} {
|
||||
writeHadoopVInt(&buffer, int64(len(value)))
|
||||
buffer.Write(value)
|
||||
}
|
||||
return base64.RawURLEncoding.EncodeToString(buffer.Bytes())
|
||||
}
|
||||
|
||||
func TestParseDirectJDBCConnection(t *testing.T) {
|
||||
config, err := parseConnectionConfig(connectParams{
|
||||
ConnectionString: "jdbc:hive2://hive.example.com:10001/analytics;transportMode=http;httpPath=gateway;ssl=true",
|
||||
Username: "alice",
|
||||
Password: "secret",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(config.Endpoints) != 1 || config.Endpoints[0] != (endpoint{Host: "hive.example.com", Port: 10001}) {
|
||||
t.Fatalf("unexpected endpoints: %#v", config.Endpoints)
|
||||
}
|
||||
if config.Database != "analytics" || config.TransportMode != "http" || config.HTTPPath != "gateway" {
|
||||
t.Fatalf("unexpected config: %#v", config)
|
||||
}
|
||||
if config.TLSConfig == nil {
|
||||
t.Fatal("expected TLS config")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseHTTPKerberosChannelBinding(t *testing.T) {
|
||||
config, err := parseConnectionConfig(connectParams{
|
||||
ConnectionString: "jdbc:hive2://hive.example.com:10001/default;transportMode=http;ssl=true;auth=KERBEROS;principal=HTTP/_HOST@EXAMPLE.COM;kerberosChannelBinding=true",
|
||||
Username: "alice@EXAMPLE.COM",
|
||||
Password: "secret",
|
||||
AgentJavaOptions: []string{"-Djava.security.krb5.conf=/etc/krb5.conf"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !config.Kerberos.ChannelBinding {
|
||||
t.Fatal("expected HTTP Kerberos TLS channel binding")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseZooKeeperKerberosJDBCConnection(t *testing.T) {
|
||||
configPath := "/etc/krb5.conf"
|
||||
if runtime.GOOS == "windows" {
|
||||
configPath = `C:\ProgramData\MIT\Kerberos5\krb5.ini`
|
||||
}
|
||||
config, err := parseConnectionConfig(connectParams{
|
||||
ConnectionString: "jdbc:hive2://zk1.example.com:2181,zk2.example.com:2181/default;serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=kyuubi;principal=hive/_HOST@EXAMPLE.COM;hive.server2.thrift.sasl.qop=auth-conf",
|
||||
Username: "alice@EXAMPLE.COM",
|
||||
Password: "secret",
|
||||
AgentJavaOptions: []string{"-Djava.security.krb5.conf=" + configPath},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if config.ServiceDiscoveryMode != "zookeeper" || config.ZooKeeperNamespace != "kyuubi" {
|
||||
t.Fatalf("unexpected ZooKeeper config: %#v", config)
|
||||
}
|
||||
if len(config.Endpoints) != 2 || config.Endpoints[1].Host != "zk2.example.com" || config.Endpoints[1].Port != 2181 {
|
||||
t.Fatalf("unexpected endpoints: %#v", config.Endpoints)
|
||||
}
|
||||
if !config.Kerberos.Enabled || config.Kerberos.Service != "hive" || config.Kerberos.QOP != "auth-conf" {
|
||||
t.Fatalf("unexpected Kerberos config: %#v", config.Kerberos)
|
||||
}
|
||||
if config.Kerberos.ConfigPath != configPath {
|
||||
t.Fatalf("unexpected krb5 config path: %q", config.Kerberos.ConfigPath)
|
||||
}
|
||||
if !config.ZooKeeperKerberos.Enabled || config.ZooKeeperKerberos.Service != "zookeeper" || !config.ZooKeeperKerberos.CanonicalHostname {
|
||||
t.Fatalf("unexpected ZooKeeper Kerberos config: %#v", config.ZooKeeperKerberos)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseZooKeeperKerberosCompatibilityProperties(t *testing.T) {
|
||||
config, err := parseConnectionConfig(connectParams{
|
||||
ConnectionString: "jdbc:hive2://zk.example.com:2181/default;serviceDiscoveryMode=zooKeeper;principal=hive/_HOST@HIVE.EXAMPLE.COM;zookeeper.sasl.client.username=zkservice;zookeeper.server.realm=ZK.EXAMPLE.COM;zookeeper.sasl.client.canonicalize.hostname=false",
|
||||
Username: "alice@EXAMPLE.COM",
|
||||
Password: "secret",
|
||||
AgentJavaOptions: []string{
|
||||
"-Djava.security.krb5.conf=/etc/krb5.conf",
|
||||
"-Dzookeeper.server.principal=zookeeper/zk.example.com@EXPLICIT.EXAMPLE.COM",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := zooKeeperKerberosConfig{
|
||||
Enabled: true,
|
||||
Service: "zkservice",
|
||||
ServerPrincipal: "zookeeper/zk.example.com@EXPLICIT.EXAMPLE.COM",
|
||||
Realm: "ZK.EXAMPLE.COM",
|
||||
CanonicalHostname: false,
|
||||
}
|
||||
if config.ZooKeeperKerberos != want {
|
||||
t.Fatalf("ZooKeeper Kerberos config = %#v, want %#v", config.ZooKeeperKerberos, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestZooKeeperKerberosCanBeDisabledExplicitly(t *testing.T) {
|
||||
config, err := parseConnectionConfig(connectParams{
|
||||
ConnectionString: "jdbc:hive2://zk.example.com:2181/default;serviceDiscoveryMode=zooKeeper;principal=hive/_HOST@EXAMPLE.COM;hive.zookeeper.use.kerberos=true",
|
||||
Username: "alice@EXAMPLE.COM",
|
||||
Password: "secret",
|
||||
AgentJavaOptions: []string{
|
||||
"-Djava.security.krb5.conf=/etc/krb5.conf",
|
||||
"-Dzookeeper.sasl.client=false",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if config.ZooKeeperKerberos.Enabled {
|
||||
t.Fatalf("ZooKeeper Kerberos should be disabled: %#v", config.ZooKeeperKerberos)
|
||||
}
|
||||
}
|
||||
|
||||
func TestURLParamsOverrideConnectionString(t *testing.T) {
|
||||
config, err := parseConnectionConfig(connectParams{
|
||||
Host: "hive.example.com",
|
||||
Port: 10000,
|
||||
ConnectionString: "jdbc:hive2://old.example.com:10000/default;transportMode=binary",
|
||||
URLParams: "transportMode=http;httpPath=proxy",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if config.TransportMode != "http" || config.HTTPPath != "proxy" {
|
||||
t.Fatalf("URL params did not override connection string: %#v", config)
|
||||
}
|
||||
if len(config.Endpoints) != 1 || config.Endpoints[0] != (endpoint{Host: "hive.example.com", Port: 10000}) {
|
||||
t.Fatalf("resolved form endpoint did not override connection string: %#v", config.Endpoints)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvedDirectEndpointAndDatabasePreserveJDBCParameterSections(t *testing.T) {
|
||||
config, err := parseConnectionConfig(connectParams{
|
||||
Host: "127.0.0.1",
|
||||
Port: 18080,
|
||||
Database: "analytics",
|
||||
ConnectionString: "jdbc:hive2://old.example.com:10000/default;transportMode=http;httpPath=gateway?hive.exec.dynamic.partition=true#SourceTable=events",
|
||||
URLParams: "transportMode=http;httpPath=gateway?hive.exec.dynamic.partition=true#SourceTable=events",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(config.Endpoints) != 1 || config.Endpoints[0] != (endpoint{Host: "127.0.0.1", Port: 18080}) {
|
||||
t.Fatalf("unexpected resolved endpoint: %#v", config.Endpoints)
|
||||
}
|
||||
if config.Database != "analytics" {
|
||||
t.Fatalf("resolved database = %q, want analytics", config.Database)
|
||||
}
|
||||
if config.TransportMode != "http" || config.HTTPPath != "gateway" {
|
||||
t.Fatalf("JDBC session parameters were not preserved: %#v", config)
|
||||
}
|
||||
if config.HiveConfiguration["set:hiveconf:hive.exec.dynamic.partition"] != "true" {
|
||||
t.Fatalf("JDBC hiveConfs were not preserved: %#v", config.HiveConfiguration)
|
||||
}
|
||||
if config.HiveConfiguration["set:hivevar:SourceTable"] != "events" {
|
||||
t.Fatalf("JDBC hiveVars were not preserved: %#v", config.HiveConfiguration)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStructuredFieldsOverridePersistedJDBCValues(t *testing.T) {
|
||||
config, err := parseConnectionConfig(connectParams{
|
||||
Host: "127.0.0.1",
|
||||
Port: 18080,
|
||||
Username: "edited-user",
|
||||
Password: "edited-password",
|
||||
ConnectionString: "jdbc:hive2://old-user:old-password@old.example.com:10000/old_database;transportMode=http;auth=LDAP;ssl=true;serviceDiscoveryMode=zooKeeper?hive.exec.dynamic.partition=true#SourceTable=events",
|
||||
URLParams: "user=url-user;password=url-password;ssl=true?hive.exec.dynamic.partition=true#SourceTable=events",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(config.Endpoints) != 1 || config.Endpoints[0] != (endpoint{Host: "127.0.0.1", Port: 18080}) {
|
||||
t.Fatalf("unexpected resolved endpoint: %#v", config.Endpoints)
|
||||
}
|
||||
if config.Database != defaultHiveDatabase {
|
||||
t.Fatalf("database = %q, want %q", config.Database, defaultHiveDatabase)
|
||||
}
|
||||
if config.Username != "edited-user" || config.Password != "edited-password" {
|
||||
t.Fatalf("structured credentials were overwritten: username=%q password=%q", config.Username, config.Password)
|
||||
}
|
||||
if config.TransportMode != "binary" || config.Auth != "NONE" || config.ServiceDiscoveryMode != "" {
|
||||
t.Fatalf("removed JDBC parameters remained active: %#v", config)
|
||||
}
|
||||
if config.TLSConfig != nil {
|
||||
t.Fatal("disabled structured SSL was re-enabled by persisted JDBC parameters")
|
||||
}
|
||||
if config.HiveConfiguration["set:hiveconf:hive.exec.dynamic.partition"] != "true" {
|
||||
t.Fatalf("current hiveConfs were not preserved: %#v", config.HiveConfiguration)
|
||||
}
|
||||
if config.HiveConfiguration["set:hivevar:SourceTable"] != "events" {
|
||||
t.Fatalf("current hiveVars were not preserved: %#v", config.HiveConfiguration)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnectionStringOnlyKeepsJDBCValues(t *testing.T) {
|
||||
config, err := parseConnectionConfig(connectParams{
|
||||
ConnectionString: "jdbc:hive2://raw-user:raw-password@hive.example.com:10001/analytics;transportMode=http;ssl=true",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if config.Database != "analytics" || config.Username != "raw-user" || config.Password != "raw-password" {
|
||||
t.Fatalf("connection-string-only fields were not preserved: %#v", config)
|
||||
}
|
||||
if config.TransportMode != "http" || config.TLSConfig == nil {
|
||||
t.Fatalf("connection-string-only parameters were not preserved: %#v", config)
|
||||
}
|
||||
}
|
||||
|
||||
func TestZooKeeperDiscoveryKeepsAllJDBCEndpoints(t *testing.T) {
|
||||
config, err := parseConnectionConfig(connectParams{
|
||||
Host: "127.0.0.1",
|
||||
Port: 12181,
|
||||
Database: "analytics",
|
||||
ConnectionString: "jdbc:hive2://zk1.example.com:2181,zk2.example.com:2181/default;serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2",
|
||||
URLParams: "serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(config.Endpoints) != 2 || config.Endpoints[0].Host != "zk1.example.com" || config.Endpoints[1].Host != "zk2.example.com" {
|
||||
t.Fatalf("ZooKeeper discovery endpoints were not preserved: %#v", config.Endpoints)
|
||||
}
|
||||
if config.Database != "analytics" {
|
||||
t.Fatalf("resolved database = %q, want analytics", config.Database)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImpalaDefaultsToNoSASL(t *testing.T) {
|
||||
config, err := parseConnectionConfig(connectParams{
|
||||
DatabaseType: "impala",
|
||||
Host: "impala.example.com",
|
||||
Port: 21050,
|
||||
Database: "analytics",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if config.Auth != "NOSASL" {
|
||||
t.Fatalf("unexpected Impala auth mode: %q", config.Auth)
|
||||
}
|
||||
if config.Kerberos.Service != "impala" {
|
||||
t.Fatalf("unexpected Impala Kerberos service: %q", config.Kerberos.Service)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImpalaExplicitAuthenticationOverridesDefaults(t *testing.T) {
|
||||
config, err := parseConnectionConfig(connectParams{
|
||||
DatabaseType: "impala",
|
||||
Host: "impala.example.com",
|
||||
Port: 21050,
|
||||
URLParams: "auth=NONE;service=custom",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if config.Auth != "NONE" || config.Kerberos.Service != "custom" {
|
||||
t.Fatalf("explicit Impala authentication was not preserved: %#v", config.Kerberos)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImpalaLDAPHTTPSSLConfiguration(t *testing.T) {
|
||||
config, err := parseConnectionConfig(connectParams{
|
||||
DatabaseType: "impala",
|
||||
Host: "impala.example.com",
|
||||
Port: 21050,
|
||||
Username: "alice",
|
||||
Password: "secret",
|
||||
URLParams: "auth=LDAP;transportMode=http;httpPath=cliservice",
|
||||
SSL: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if config.Auth != "LDAP" || config.TransportMode != "http" || config.HTTPPath != "cliservice" {
|
||||
t.Fatalf("unexpected Impala LDAP transport config: %#v", config)
|
||||
}
|
||||
if config.Username != "alice" || config.Password != "secret" {
|
||||
t.Fatalf("Impala LDAP credentials were not preserved: %q / %q", config.Username, config.Password)
|
||||
}
|
||||
if config.TLSConfig == nil || config.TLSConfig.ServerName != "impala.example.com" {
|
||||
t.Fatalf("unexpected Impala LDAP TLS config: %#v", config.TLSConfig)
|
||||
}
|
||||
if config.Kerberos.Service != defaultImpalaService {
|
||||
t.Fatalf("unexpected Impala service default: %q", config.Kerberos.Service)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseStandardJDBCURLSectionsAndCredentials(t *testing.T) {
|
||||
config, err := parseConnectionConfig(connectParams{
|
||||
ConnectionString: "jdbc:hive2://hs2.example.com:10001/analytics;user=alice;password=p%40ss?hive.server2.transport.mode=http;hive.server2.thrift.http.path=proxy;hive.exec.dynamic.partition=true#SourceTable=events",
|
||||
URLParams: "user=bob?hive.exec.dynamic.partition=false#SourceTable=override",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if config.Username != "bob" || config.Password != "p@ss" {
|
||||
t.Fatalf("unexpected credentials: %q / %q", config.Username, config.Password)
|
||||
}
|
||||
if config.TransportMode != "http" || config.HTTPPath != "proxy" {
|
||||
t.Fatalf("deprecated Hive conf transport settings were not applied: %#v", config)
|
||||
}
|
||||
want := map[string]string{
|
||||
"set:hiveconf:hive.resultset.use.unique.column.names": "false",
|
||||
"set:hiveconf:hive.exec.dynamic.partition": "false",
|
||||
"set:hivevar:SourceTable": "override",
|
||||
}
|
||||
if !reflect.DeepEqual(config.HiveConfiguration, want) {
|
||||
t.Fatalf("unexpected OpenSession configuration: %#v", config.HiveConfiguration)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenSessionCompatibilityVariablesFromSessionParams(t *testing.T) {
|
||||
config, err := parseConnectionConfig(connectParams{
|
||||
Host: "hs2.example.com",
|
||||
URLParams: "proxyUser=alice;hiveCreateAsExternalLegacy=TRUE;wmPool=etl;hiveconf:hive.exec.compress.output=true;hivevar:source=events",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]string{
|
||||
"hive.server2.proxy.user": "alice",
|
||||
"set:hiveconf:hive.create.as.external.legacy": "true",
|
||||
"set:hiveconf:hive.exec.compress.output": "true",
|
||||
"set:hiveconf:hive.resultset.use.unique.column.names": "false",
|
||||
"set:hivevar:source": "events",
|
||||
"set:hivevar:wmpool": "etl",
|
||||
}
|
||||
if !reflect.DeepEqual(config.HiveConfiguration, want) {
|
||||
t.Fatalf("unexpected OpenSession compatibility variables: %#v", config.HiveConfiguration)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenSessionUsesLeafResultLabelsUnlessExplicitlyOverridden(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
connection string
|
||||
urlParams string
|
||||
want string
|
||||
}{
|
||||
{name: "default", want: "false"},
|
||||
{
|
||||
name: "JDBC hiveconf override",
|
||||
connection: "jdbc:hive2://hs2.example.com:10000/default?hive.resultset.use.unique.column.names=true",
|
||||
want: "true",
|
||||
},
|
||||
{
|
||||
name: "URL hiveconf override",
|
||||
urlParams: "?HIVE.RESULTSET.USE.UNIQUE.COLUMN.NAMES=true",
|
||||
want: "true",
|
||||
},
|
||||
{
|
||||
name: "URL hiveconf wins over JDBC hiveconf",
|
||||
connection: "jdbc:hive2://hs2.example.com:10000/default?hive.resultset.use.unique.column.names=true",
|
||||
urlParams: "?hive.resultset.use.unique.column.names=false",
|
||||
want: "false",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
params := connectParams{Host: "hs2.example.com", ConnectionString: test.connection, URLParams: test.urlParams}
|
||||
if test.connection != "" {
|
||||
params.Host = ""
|
||||
}
|
||||
config, err := parseConnectionConfig(params)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const key = "set:hiveconf:hive.resultset.use.unique.column.names"
|
||||
if got := config.HiveConfiguration[key]; got != test.want {
|
||||
t.Fatalf("%s = %q, want %q; config=%#v", key, got, test.want, config.HiveConfiguration)
|
||||
}
|
||||
matches := 0
|
||||
for candidate := range config.HiveConfiguration {
|
||||
if strings.EqualFold(candidate, key) {
|
||||
matches++
|
||||
}
|
||||
}
|
||||
if matches != 1 {
|
||||
t.Fatalf("result label hiveconf must appear exactly once: %#v", config.HiveConfiguration)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHiveAssignmentMergeCanonicalizesOnlyResultLabelSetting(t *testing.T) {
|
||||
first := map[string]string{
|
||||
"CaseSensitive": "first",
|
||||
"HIVE.RESULTSET.USE.UNIQUE.COLUMN.NAMES": "true",
|
||||
}
|
||||
second := map[string]string{
|
||||
"casesensitive": "second",
|
||||
"hive.resultset.use.unique.column.names": "false",
|
||||
}
|
||||
|
||||
hiveConfs := mergeHiveConfAssignments(first, second)
|
||||
if got := hiveConfs[resultSetUniqueColumnNames]; got != "false" {
|
||||
t.Fatalf("result-label setting override = %q, want false; values=%#v", got, hiveConfs)
|
||||
}
|
||||
if got := len(hiveConfs); got != 3 {
|
||||
t.Fatalf("unrelated case-distinct Hive confs were collapsed: %#v", hiveConfs)
|
||||
}
|
||||
|
||||
hiveVars := mergeHiveAssignments(first, second)
|
||||
if got := len(hiveVars); got != 4 {
|
||||
t.Fatalf("case-distinct Hive variables were collapsed: %#v", hiveVars)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseHiveJDBCClientCompatibilityOptions(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
initPath := filepath.Join(directory, "hive-init.sql")
|
||||
if err := os.WriteFile(initPath, []byte("# ignored\nSET hive.exec.dynamic.partition=true;\n-- ignored\nUSE analytics;\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config, err := parseConnectionConfig(connectParams{
|
||||
ConnectionString: "jdbc:hive2://hs2.example.com:10001/default;transportMode=http;auth=jwt;jwt=signed-token;fetchSize=77;socketTimeout=9;thrift.client.max.message.size=1048576;retries=3;retryInterval=250;requestTrack=true;cookieAuth=false;cookieName=CustomAuth;http.header.X-Trace-ID=trace-value;http.cookie.SessionID=cookie-value;applicationName=dbx-hive;initFile=" + url.QueryEscape(initPath),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if config.Auth != "JWT" || config.JWT != "signed-token" {
|
||||
t.Fatalf("unexpected JWT config: %#v", config)
|
||||
}
|
||||
if config.FetchSize != 77 || config.SocketTimeout != 9*time.Second || config.MaxMessageSize != 1048576 {
|
||||
t.Fatalf("unexpected client sizing config: %#v", config)
|
||||
}
|
||||
if config.Retries != 3 || config.RetryInterval != 250*time.Millisecond {
|
||||
t.Fatalf("unexpected retry config: %#v", config)
|
||||
}
|
||||
if config.CookieAuth || config.CookieName != "CustomAuth" {
|
||||
t.Fatalf("unexpected cookie auth config: %#v", config)
|
||||
}
|
||||
if config.HTTPHeaders["X-Trace-ID"] != "trace-value" || config.HTTPCookies["SessionID"] != "cookie-value" {
|
||||
t.Fatalf("HTTP header or cookie case was not preserved: %#v / %#v", config.HTTPHeaders, config.HTTPCookies)
|
||||
}
|
||||
if !config.RequestTracking {
|
||||
t.Fatal("requestTrack was not enabled")
|
||||
}
|
||||
if config.HiveConfiguration["set:hivevar:wmapp"] != "dbx-hive" {
|
||||
t.Fatalf("applicationName was not mapped: %#v", config.HiveConfiguration)
|
||||
}
|
||||
if !reflect.DeepEqual(config.InitStatements, []string{"SET hive.exec.dynamic.partition=true", "USE analytics"}) {
|
||||
t.Fatalf("unexpected init statements: %#v", config.InitStatements)
|
||||
}
|
||||
}
|
||||
|
||||
func TestJWTCanComeFromEnvironment(t *testing.T) {
|
||||
t.Setenv("JWT", "environment-token")
|
||||
config, err := parseConnectionConfig(connectParams{
|
||||
ConnectionString: "jdbc:hive2://hs2.example.com:10001/default;transportMode=http;auth=jwt",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if config.JWT != "environment-token" {
|
||||
t.Fatalf("unexpected JWT: %q", config.JWT)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBrowserAuthSupportsInteractiveSSOParameters(t *testing.T) {
|
||||
config, err := parseConnectionConfig(connectParams{
|
||||
ConnectionString: "jdbc:hive2://hs2.example.com:10001/default;transportMode=http;auth=browser;browserResponsePort=18080;browserResponseTimeout=45;browserDisableSslCheck=true",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if config.BrowserToken != "" || config.BrowserResponsePort != 18080 || config.BrowserResponseTimeout != 45*time.Second || !config.BrowserDisableSSLCheck {
|
||||
t.Fatalf("unexpected browser SSO config: %#v", config)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidHiveJDBCClientSizesAreRejected(t *testing.T) {
|
||||
for _, params := range []string{
|
||||
"fetchSize=zero",
|
||||
"socketTimeout=soon",
|
||||
"thrift.client.max.message.size=huge",
|
||||
} {
|
||||
_, err := parseConnectionConfig(connectParams{
|
||||
ConnectionString: "jdbc:hive2://hs2.example.com/default;" + params,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatalf("expected %q to be rejected", params)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestJavaCredentialProviderPasswordPathIsNotSilentlyIgnored(t *testing.T) {
|
||||
_, err := parseConnectionConfig(connectParams{
|
||||
ConnectionString: "jdbc:hive2://hs2.example.com/default;ssl=true;sslTrustStore=/tmp/truststore.jks;storePasswordPath=jceks://file/tmp/hive.jceks",
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "configure trustStorePassword explicitly") {
|
||||
t.Fatalf("unexpected credential-provider error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKerberosAcceptsSaslQopCompatibilityAlias(t *testing.T) {
|
||||
configPath := filepath.Join(t.TempDir(), "krb5.conf")
|
||||
if err := os.WriteFile(configPath, []byte("[libdefaults]\n default_realm = EXAMPLE.COM\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config, err := parseConnectionConfig(connectParams{
|
||||
ConnectionString: "jdbc:hive2://hs2.example.com/default;auth=kerberos;principal=hive/_HOST@EXAMPLE.COM;kerberosPrincipal=alice@EXAMPLE.COM;sasl.qop=auth-int",
|
||||
Password: "secret",
|
||||
AgentJavaOptions: []string{"-Djava.security.krb5.conf=" + configPath},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if config.Kerberos.QOP != "auth-int" {
|
||||
t.Fatalf("unexpected SASL QOP: %q", config.Kerberos.QOP)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelegationTokenProducesDigestCredentials(t *testing.T) {
|
||||
identifier := []byte("token-identifier")
|
||||
password := []byte("token-password")
|
||||
token := encodeHadoopToken(identifier, password, []byte("HIVE_DELEGATION_TOKEN"), []byte("hs2.example.com:10000"))
|
||||
config, err := parseConnectionConfig(connectParams{
|
||||
ConnectionString: "jdbc:hive2://hs2.example.com/default;auth=delegationToken;delegationToken=" + token,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if config.Auth != "DELEGATIONTOKEN" {
|
||||
t.Fatalf("unexpected auth: %q", config.Auth)
|
||||
}
|
||||
if config.DelegationToken != token {
|
||||
t.Fatalf("raw delegation token was not retained: %q", config.DelegationToken)
|
||||
}
|
||||
if config.Username != base64.StdEncoding.EncodeToString(identifier) || config.Password != base64.StdEncoding.EncodeToString(password) {
|
||||
t.Fatalf("unexpected delegation token credentials: %q / %q", config.Username, config.Password)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelegationTokenRejectsMalformedValue(t *testing.T) {
|
||||
_, err := parseConnectionConfig(connectParams{
|
||||
ConnectionString: "jdbc:hive2://hs2.example.com/default;auth=delegationToken;delegationToken=not-a-token",
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "decode Hive delegation token") {
|
||||
t.Fatalf("expected delegation token error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseIPv6Endpoint(t *testing.T) {
|
||||
value, err := parseEndpoint("[2001:db8::1]:10000", defaultHivePort)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if value.Host != "2001:db8::1" || value.Port != 10000 {
|
||||
t.Fatalf("unexpected endpoint: %#v", value)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKerberosSeparatesServerAndClientPrincipals(t *testing.T) {
|
||||
configPath := filepath.Join(t.TempDir(), "krb5.conf")
|
||||
if err := os.WriteFile(configPath, []byte("[libdefaults]\n default_realm = CLIENT.EXAMPLE.COM\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config, err := parseConnectionConfig(connectParams{
|
||||
ConnectionString: "jdbc:hive2://alias.example.com:10000/default;auth=kerberos;principal=hive/_HOST@SERVER.EXAMPLE.COM;kerberosPrincipal=alice@CLIENT.EXAMPLE.COM;kerberosEnableCanonicalHostnameCheck=false",
|
||||
Password: "secret",
|
||||
AgentJavaOptions: []string{`-Djava.security.krb5.conf="` + configPath + `"`},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if config.Kerberos.ServerPrincipal != "hive/_HOST@SERVER.EXAMPLE.COM" {
|
||||
t.Fatalf("unexpected server principal: %q", config.Kerberos.ServerPrincipal)
|
||||
}
|
||||
if config.Kerberos.ClientPrincipal != "alice@CLIENT.EXAMPLE.COM" {
|
||||
t.Fatalf("unexpected client principal: %q", config.Kerberos.ClientPrincipal)
|
||||
}
|
||||
if config.Kerberos.Realm != "CLIENT.EXAMPLE.COM" || config.Kerberos.CanonicalHostname {
|
||||
t.Fatalf("unexpected Kerberos options: %#v", config.Kerberos)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKerberosReadsLegacyJAASKeytabOptions(t *testing.T) {
|
||||
directory := t.TempDir()
|
||||
configPath := filepath.Join(directory, "krb5.conf")
|
||||
jaasPath := filepath.Join(directory, "hive jaas.conf")
|
||||
keytabPath := filepath.Join(directory, "alice.keytab")
|
||||
if err := os.WriteFile(configPath, []byte("[libdefaults]\n default_realm = EXAMPLE.COM\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
jaas := `HiveClient {
|
||||
com.sun.security.auth.module.Krb5LoginModule required
|
||||
useKeyTab=true
|
||||
keyTab="` + keytabPath + `"
|
||||
principal="alice@EXAMPLE.COM"
|
||||
doNotPrompt=true;
|
||||
};`
|
||||
if err := os.WriteFile(jaasPath, []byte(jaas), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config, err := parseConnectionConfig(connectParams{
|
||||
ConnectionString: "jdbc:hive2://hs2.example.com/default;auth=kerberos;principal=hive/_HOST@EXAMPLE.COM",
|
||||
AgentJavaOptions: []string{
|
||||
"-Djava.security.krb5.conf=" + configPath,
|
||||
`-Djava.security.auth.login.config="` + jaasPath + `"`,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !config.Kerberos.UseKeytab || config.Kerberos.KeytabPath != keytabPath {
|
||||
t.Fatalf("JAAS keytab was not preserved: %#v", config.Kerberos)
|
||||
}
|
||||
if config.Kerberos.ClientPrincipal != "alice@EXAMPLE.COM" {
|
||||
t.Fatalf("JAAS principal was not preserved: %#v", config.Kerberos)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildHiveTLSConfigFromPKCS12Stores(t *testing.T) {
|
||||
privateKey, certificate, _ := testZooKeeperCertificate(t)
|
||||
password := "changeit"
|
||||
keyStore, err := pkcs12.Modern.Encode(privateKey, certificate, nil, password)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
trustStore, err := pkcs12.Modern.EncodeTrustStore([]*x509.Certificate{certificate}, password)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
directory := t.TempDir()
|
||||
keyPath := filepath.Join(directory, "client.p12")
|
||||
trustPath := filepath.Join(directory, "trust.p12")
|
||||
if err := os.WriteFile(keyPath, keyStore, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(trustPath, trustStore, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config, err := buildTLSConfig(connectParams{}, map[string]string{
|
||||
"ssl": "true",
|
||||
"twoway": "true",
|
||||
"ssltruststore": trustPath,
|
||||
"truststorepassword": password,
|
||||
"truststoretype": "PKCS12",
|
||||
"sslkeystore": keyPath,
|
||||
"keystorepassword": password,
|
||||
"keystoretype": "PKCS12",
|
||||
}, "hs2.example.com")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if config == nil || config.RootCAs == nil || len(config.Certificates) != 1 {
|
||||
t.Fatalf("unexpected Hive TLS config: %#v", config)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildHiveTwoWayTLSRequiresTrustAndKeyMaterial(t *testing.T) {
|
||||
if _, err := buildTLSConfig(connectParams{}, map[string]string{
|
||||
"ssl": "true",
|
||||
"twoway": "true",
|
||||
}, "hs2.example.com"); err == nil {
|
||||
t.Fatal("expected missing two-way TLS material error")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"database/sql"
|
||||
"database/sql/driver"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/beltran/gosasl"
|
||||
gohive "github.com/t8y2/dbx/agents/go-common/gohive"
|
||||
)
|
||||
|
||||
type connectorFactory func(endpoint) driver.Connector
|
||||
|
||||
type discoveryConnector struct {
|
||||
discovery endpointDiscovery
|
||||
factory connectorFactory
|
||||
driver driver.Driver
|
||||
retries int
|
||||
retryInterval time.Duration
|
||||
}
|
||||
|
||||
func newDiscoveryConnector(config connectionConfig) *discoveryConnector {
|
||||
return &discoveryConnector{
|
||||
discovery: newEndpointDiscovery(config),
|
||||
factory: func(target endpoint) driver.Connector {
|
||||
tlsConfig := config.TLSConfig
|
||||
if target.SSL && tlsConfig == nil && !config.TLSExplicit {
|
||||
tlsConfig = &tls.Config{MinVersion: tls.VersionTLS12, ServerName: target.Host}
|
||||
}
|
||||
if tlsConfig != nil {
|
||||
tlsConfig = tlsConfig.Clone()
|
||||
if tlsConfig.ServerName == "" || tlsConfig.ServerName == config.Endpoints[0].Host {
|
||||
tlsConfig.ServerName = target.Host
|
||||
}
|
||||
}
|
||||
hiveConfiguration := make(map[string]string, len(config.HiveConfiguration)+1)
|
||||
for key, value := range config.HiveConfiguration {
|
||||
hiveConfiguration[key] = value
|
||||
}
|
||||
if config.Kerberos.Enabled {
|
||||
hiveConfiguration["hive.server2.thrift.sasl.qop"] = config.Kerberos.QOP
|
||||
}
|
||||
transportMode := config.TransportMode
|
||||
if target.TransportMode != "" && !config.TransportModeExplicit {
|
||||
transportMode = target.TransportMode
|
||||
}
|
||||
httpPath := config.HTTPPath
|
||||
if target.HTTPPath != "" && !config.HTTPPathExplicit {
|
||||
httpPath = target.HTTPPath
|
||||
}
|
||||
auth := config.Auth
|
||||
if target.Auth != "" && !config.AuthExplicit {
|
||||
auth = target.Auth
|
||||
}
|
||||
service := kerberosServiceForEndpoint(config, target)
|
||||
return gohive.NewConnector(gohive.Config{
|
||||
Host: target.Host,
|
||||
Port: target.Port,
|
||||
Auth: normalizeHiveAuth(auth),
|
||||
Username: config.Username,
|
||||
Password: config.Password,
|
||||
Database: config.Database,
|
||||
TransportMode: transportMode,
|
||||
HTTPPath: httpPath,
|
||||
Service: service,
|
||||
HTTPKerberosChannelBinding: config.Kerberos.ChannelBinding,
|
||||
GSSAPIOptions: gssapiOptionsFromKerberos(config.Kerberos),
|
||||
TLSConfig: tlsConfig,
|
||||
HiveConfiguration: hiveConfiguration,
|
||||
ConnectTimeout: config.ConnectTimeout,
|
||||
SocketTimeout: config.SocketTimeout,
|
||||
HTTPTimeout: config.SocketTimeout,
|
||||
FetchSize: int64(config.FetchSize),
|
||||
MaxMessageSize: config.MaxMessageSize,
|
||||
HTTPHeaders: config.HTTPHeaders,
|
||||
HTTPCookies: config.HTTPCookies,
|
||||
RequestTracking: config.RequestTracking,
|
||||
DisableCookieAuth: !config.CookieAuth,
|
||||
CookieName: config.CookieName,
|
||||
JWT: config.JWT,
|
||||
DelegationToken: config.DelegationToken,
|
||||
BrowserToken: config.BrowserToken,
|
||||
BrowserClientID: config.BrowserClientID,
|
||||
BrowserResponsePort: config.BrowserResponsePort,
|
||||
BrowserResponseTimeout: config.BrowserResponseTimeout,
|
||||
BrowserDisableSSLCheck: config.BrowserDisableSSLCheck,
|
||||
WaitForNonQueryCompletion: waitsForNonQueryCompletion(config.DatabaseType),
|
||||
})
|
||||
},
|
||||
driver: &gohive.Driver{},
|
||||
retries: max(config.Retries, 1),
|
||||
retryInterval: config.RetryInterval,
|
||||
}
|
||||
}
|
||||
|
||||
func waitsForNonQueryCompletion(databaseType string) bool {
|
||||
return strings.EqualFold(databaseType, "impala") || strings.EqualFold(databaseType, "kyuubi")
|
||||
}
|
||||
|
||||
func gssapiOptionsFromKerberos(config kerberosConfig) gosasl.GSSAPIOptions {
|
||||
return gosasl.GSSAPIOptions{
|
||||
ConfigPath: config.ConfigPath,
|
||||
CCachePath: config.CCachePath,
|
||||
KeytabPath: config.KeytabPath,
|
||||
Principal: config.ClientPrincipal,
|
||||
Password: config.Password,
|
||||
QOP: config.QOP,
|
||||
AuthorizationID: config.AuthorizationID,
|
||||
ServiceHost: config.ServerName,
|
||||
UseCCache: config.UseTicketCache,
|
||||
UseKeytab: config.UseKeytab,
|
||||
UseSSPI: config.UseSSPI,
|
||||
CanonicalizeHost: config.CanonicalHostname,
|
||||
DisablePAFXFAST: config.DisablePAFXFAST,
|
||||
}
|
||||
}
|
||||
|
||||
func kerberosServiceForEndpoint(config connectionConfig, target endpoint) string {
|
||||
service := firstNonEmpty(config.Kerberos.ServerPrincipal, config.Kerberos.Service)
|
||||
if target.Principal != "" && !config.Kerberos.ServerPrincipalExplicit {
|
||||
service = target.Principal
|
||||
}
|
||||
return service
|
||||
}
|
||||
|
||||
func (connector *discoveryConnector) Connect(ctx context.Context) (driver.Conn, error) {
|
||||
var failures []string
|
||||
for attempt := 0; attempt < max(connector.retries, 1); attempt++ {
|
||||
rejected := map[string]bool{}
|
||||
for {
|
||||
endpoints, err := connector.discovery.Endpoints(ctx, rejected)
|
||||
if err != nil {
|
||||
failures = append(failures, fmt.Sprintf("discovery attempt %d: %v", attempt+1, err))
|
||||
break
|
||||
}
|
||||
if len(endpoints) == 0 {
|
||||
break
|
||||
}
|
||||
for _, target := range endpoints {
|
||||
connection, connectErr := connector.factory(target).Connect(ctx)
|
||||
if connectErr == nil {
|
||||
return connection, nil
|
||||
}
|
||||
rejected[target.address()] = true
|
||||
failures = append(failures, fmt.Sprintf("attempt %d %s: %v", attempt+1, target.address(), connectErr))
|
||||
}
|
||||
break
|
||||
}
|
||||
if attempt+1 < connector.retries && connector.retryInterval > 0 {
|
||||
timer := time.NewTimer(connector.retryInterval)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return nil, ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(failures) == 0 {
|
||||
return nil, errors.New("Hive discovery returned no endpoints")
|
||||
}
|
||||
return nil, fmt.Errorf("all HiveServer2 endpoints failed: %s", strings.Join(failures, "; "))
|
||||
}
|
||||
|
||||
func (connector *discoveryConnector) Driver() driver.Driver {
|
||||
return connector.driver
|
||||
}
|
||||
|
||||
func openHiveDatabase(config connectionConfig) *sql.DB {
|
||||
database := sql.OpenDB(newDiscoveryConnector(config))
|
||||
database.SetMaxOpenConns(1)
|
||||
database.SetMaxIdleConns(1)
|
||||
return database
|
||||
}
|
||||
|
||||
func normalizeHiveAuth(value string) string {
|
||||
normalized := strings.ToUpper(strings.TrimSpace(value))
|
||||
switch normalized {
|
||||
case "":
|
||||
return "NONE"
|
||||
case "NOSASL", "NO_SASL":
|
||||
return "NOSASL"
|
||||
case "KERBEROS", "GSSAPI":
|
||||
return "KERBEROS"
|
||||
case "LDAP":
|
||||
return "LDAP"
|
||||
case "CUSTOM":
|
||||
return "CUSTOM"
|
||||
case "DIGEST-MD5", "DELEGATIONTOKEN", "DELEGATION_TOKEN":
|
||||
return "DIGEST-MD5"
|
||||
default:
|
||||
return normalized
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql/driver"
|
||||
"errors"
|
||||
"io"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type staticDiscovery struct {
|
||||
values []endpoint
|
||||
}
|
||||
|
||||
func (discovery staticDiscovery) Endpoints(_ context.Context, rejected map[string]bool) ([]endpoint, error) {
|
||||
result := make([]endpoint, 0, len(discovery.values))
|
||||
for _, value := range discovery.values {
|
||||
if !rejected[value.address()] {
|
||||
result = append(result, value)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
type fakeConnector struct {
|
||||
connection driver.Conn
|
||||
err error
|
||||
}
|
||||
|
||||
func (connector fakeConnector) Connect(context.Context) (driver.Conn, error) {
|
||||
return connector.connection, connector.err
|
||||
}
|
||||
|
||||
func (fakeConnector) Driver() driver.Driver { return fakeDriver{} }
|
||||
|
||||
type fakeDriver struct{}
|
||||
|
||||
func (fakeDriver) Open(string) (driver.Conn, error) { return &fakeConnection{}, nil }
|
||||
|
||||
type fakeConnection struct{}
|
||||
|
||||
func (*fakeConnection) Prepare(string) (driver.Stmt, error) { return nil, errors.New("unsupported") }
|
||||
func (*fakeConnection) Close() error { return nil }
|
||||
func (*fakeConnection) Begin() (driver.Tx, error) { return nil, errors.New("unsupported") }
|
||||
|
||||
type emptyRows struct{}
|
||||
|
||||
func (emptyRows) Columns() []string { return nil }
|
||||
func (emptyRows) Close() error { return nil }
|
||||
func (emptyRows) Next([]driver.Value) error { return io.EOF }
|
||||
|
||||
func TestDiscoveryConnectorFailsOver(t *testing.T) {
|
||||
first := endpoint{Host: "first", Port: 10000}
|
||||
second := endpoint{Host: "second", Port: 10000}
|
||||
connected := &fakeConnection{}
|
||||
var attempts []endpoint
|
||||
connector := &discoveryConnector{
|
||||
discovery: staticDiscovery{values: []endpoint{first, second}},
|
||||
driver: fakeDriver{},
|
||||
factory: func(value endpoint) driver.Connector {
|
||||
attempts = append(attempts, value)
|
||||
if value == first {
|
||||
return fakeConnector{err: errors.New("unavailable")}
|
||||
}
|
||||
return fakeConnector{connection: connected}
|
||||
},
|
||||
}
|
||||
value, err := connector.Connect(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if value != connected || !reflect.DeepEqual(attempts, []endpoint{first, second}) {
|
||||
t.Fatalf("unexpected failover result: value=%#v attempts=%#v", value, attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoveryConnectorRetriesAllEndpoints(t *testing.T) {
|
||||
target := endpoint{Host: "hs2", Port: 10000}
|
||||
connected := &fakeConnection{}
|
||||
attempts := 0
|
||||
connector := &discoveryConnector{
|
||||
discovery: staticDiscovery{values: []endpoint{target}},
|
||||
driver: fakeDriver{},
|
||||
retries: 2,
|
||||
factory: func(endpoint) driver.Connector {
|
||||
attempts++
|
||||
if attempts == 1 {
|
||||
return fakeConnector{err: errors.New("temporarily unavailable")}
|
||||
}
|
||||
return fakeConnector{connection: connected}
|
||||
},
|
||||
}
|
||||
value, err := connector.Connect(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if value != connected || attempts != 2 {
|
||||
t.Fatalf("unexpected retry result: value=%#v attempts=%d", value, attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeHiveAuth(t *testing.T) {
|
||||
for input, expected := range map[string]string{
|
||||
"": "NONE",
|
||||
"noSasl": "NOSASL",
|
||||
"kerberos": "KERBEROS",
|
||||
"delegationToken": "DIGEST-MD5",
|
||||
"vendor-auth-mode": "VENDOR-AUTH-MODE",
|
||||
} {
|
||||
if actual := normalizeHiveAuth(input); actual != expected {
|
||||
t.Fatalf("normalizeHiveAuth(%q) = %q, expected %q", input, actual, expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitsForKyuubiAndImpalaNonQueryCompletion(t *testing.T) {
|
||||
if !waitsForNonQueryCompletion("kyuubi") || !waitsForNonQueryCompletion("KYUUBI") || !waitsForNonQueryCompletion("impala") {
|
||||
t.Fatal("Kyuubi and Impala must wait for asynchronous non-query operations")
|
||||
}
|
||||
if waitsForNonQueryCompletion("hive") {
|
||||
t.Fatal("Hive must retain its existing non-query behavior")
|
||||
}
|
||||
}
|
||||
|
||||
func TestKerberosServiceForEndpoint(t *testing.T) {
|
||||
target := endpoint{Host: "hs2.example.com", Port: 10000, Principal: "hive/hs2.example.com@EXAMPLE.COM"}
|
||||
discovered := connectionConfig{Kerberos: kerberosConfig{Service: "hive"}}
|
||||
if value := kerberosServiceForEndpoint(discovered, target); value != target.Principal {
|
||||
t.Fatalf("discovered principal was ignored: %s", value)
|
||||
}
|
||||
explicit := connectionConfig{Kerberos: kerberosConfig{
|
||||
Service: "hive",
|
||||
ServerPrincipal: "hive/_HOST@USER.EXAMPLE.COM",
|
||||
ServerPrincipalExplicit: true,
|
||||
}}
|
||||
if value := kerberosServiceForEndpoint(explicit, target); value != explicit.Kerberos.ServerPrincipal {
|
||||
t.Fatalf("explicit principal was overwritten: %s", value)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKerberosUsesConnectionScopedGSSAPIOptions(t *testing.T) {
|
||||
config := kerberosConfig{
|
||||
Enabled: true,
|
||||
ServerPrincipal: "hive/_HOST@EXAMPLE.COM",
|
||||
ClientPrincipal: "alice@EXAMPLE.COM",
|
||||
ServerName: "canonical.example.com",
|
||||
CanonicalHostname: true,
|
||||
ConfigPath: "/etc/krb5.conf",
|
||||
CCachePath: "/tmp/alice.ccache",
|
||||
KeytabPath: "/tmp/alice.keytab",
|
||||
Password: "secret",
|
||||
AuthorizationID: "proxy-user",
|
||||
QOP: "auth-conf",
|
||||
UseTicketCache: true,
|
||||
UseKeytab: true,
|
||||
UseSSPI: true,
|
||||
DisablePAFXFAST: true,
|
||||
}
|
||||
options := gssapiOptionsFromKerberos(config)
|
||||
if options.ConfigPath != "/etc/krb5.conf" || options.Principal != "alice@EXAMPLE.COM" || options.Password != "secret" || options.AuthorizationID != "proxy-user" || options.ServiceHost != "canonical.example.com" || !options.CanonicalizeHost || options.CCachePath != "/tmp/alice.ccache" || options.KeytabPath != "/tmp/alice.keytab" || options.QOP != "auth-conf" || !options.UseCCache || !options.UseKeytab || !options.UseSSPI || !options.DisablePAFXFAST {
|
||||
t.Fatalf("unexpected GSSAPI options: %#v", options)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,392 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
"net"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-zookeeper/zk"
|
||||
)
|
||||
|
||||
type endpointDiscovery interface {
|
||||
Endpoints(context.Context, map[string]bool) ([]endpoint, error)
|
||||
}
|
||||
|
||||
type directDiscovery struct {
|
||||
endpoints []endpoint
|
||||
}
|
||||
|
||||
func (discovery directDiscovery) Endpoints(_ context.Context, rejected map[string]bool) ([]endpoint, error) {
|
||||
return shuffledEndpoints(discovery.endpoints, rejected), nil
|
||||
}
|
||||
|
||||
type zooKeeperDiscovery struct {
|
||||
servers []endpoint
|
||||
namespace string
|
||||
discoveryMode string
|
||||
authScheme string
|
||||
auth string
|
||||
timeout time.Duration
|
||||
dialer func([]string, time.Duration) (zooKeeperClient, <-chan zk.Event, error)
|
||||
}
|
||||
|
||||
type zooKeeperClient interface {
|
||||
AddAuth(string, []byte) error
|
||||
Children(string) ([]string, *zk.Stat, error)
|
||||
Get(string) ([]byte, *zk.Stat, error)
|
||||
Close()
|
||||
}
|
||||
|
||||
func newEndpointDiscovery(config connectionConfig) endpointDiscovery {
|
||||
if strings.EqualFold(config.ServiceDiscoveryMode, "zookeeper") || strings.EqualFold(config.ServiceDiscoveryMode, "zookeeperha") {
|
||||
return &zooKeeperDiscovery{
|
||||
servers: append([]endpoint(nil), config.Endpoints...),
|
||||
namespace: config.ZooKeeperNamespace,
|
||||
discoveryMode: config.ServiceDiscoveryMode,
|
||||
authScheme: config.ZooKeeperAuthScheme,
|
||||
auth: config.ZooKeeperAuth,
|
||||
timeout: config.ConnectTimeout,
|
||||
dialer: newZooKeeperDialer(config),
|
||||
}
|
||||
}
|
||||
return directDiscovery{endpoints: append([]endpoint(nil), config.Endpoints...)}
|
||||
}
|
||||
|
||||
func newZooKeeperDialer(config connectionConfig) func([]string, time.Duration) (zooKeeperClient, <-chan zk.Event, error) {
|
||||
if config.ZooKeeperKerberos.Enabled {
|
||||
return func(servers []string, timeout time.Duration) (zooKeeperClient, <-chan zk.Event, error) {
|
||||
return connectKerberosZooKeeper(servers, timeout, config.ZooKeeperTLSConfig, config)
|
||||
}
|
||||
}
|
||||
tlsConfig := config.ZooKeeperTLSConfig
|
||||
if tlsConfig == nil {
|
||||
return func(servers []string, timeout time.Duration) (zooKeeperClient, <-chan zk.Event, error) {
|
||||
return zk.Connect(servers, timeout, zk.WithLogInfo(false))
|
||||
}
|
||||
}
|
||||
return func(servers []string, timeout time.Duration) (zooKeeperClient, <-chan zk.Event, error) {
|
||||
return zk.Connect(
|
||||
servers,
|
||||
timeout,
|
||||
zk.WithLogInfo(false),
|
||||
zk.WithDialer(func(network, address string, dialTimeout time.Duration) (net.Conn, error) {
|
||||
config := tlsConfig.Clone()
|
||||
if config.ServerName == "" {
|
||||
host, _, err := net.SplitHostPort(address)
|
||||
if err == nil {
|
||||
config.ServerName = host
|
||||
}
|
||||
}
|
||||
dialer := &net.Dialer{Timeout: dialTimeout}
|
||||
return tls.DialWithDialer(dialer, network, address, config)
|
||||
}),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
func (discovery *zooKeeperDiscovery) Endpoints(ctx context.Context, rejected map[string]bool) ([]endpoint, error) {
|
||||
addresses := make([]string, 0, len(discovery.servers))
|
||||
for _, server := range discovery.servers {
|
||||
addresses = append(addresses, server.address())
|
||||
}
|
||||
timeout := discovery.timeout
|
||||
if timeout <= 0 {
|
||||
timeout = defaultConnectTimeout
|
||||
}
|
||||
connection, events, err := discovery.dialer(addresses, timeout)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("connect to ZooKeeper: %w", err)
|
||||
}
|
||||
defer connection.Close()
|
||||
if err := waitForZooKeeperSession(ctx, events, timeout); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if discovery.authScheme != "" || discovery.auth != "" {
|
||||
if discovery.authScheme == "" || discovery.auth == "" {
|
||||
return nil, errors.New("ZooKeeper auth scheme and credentials must be configured together")
|
||||
}
|
||||
if err := connection.AddAuth(discovery.authScheme, []byte(discovery.auth)); err != nil {
|
||||
return nil, fmt.Errorf("authenticate to ZooKeeper: %w", err)
|
||||
}
|
||||
}
|
||||
resolved := make([]endpoint, 0)
|
||||
var listedPath string
|
||||
var nodeFailures []string
|
||||
for _, path := range discovery.paths() {
|
||||
children, _, childrenErr := connection.Children(path)
|
||||
if errors.Is(childrenErr, zk.ErrNoNode) {
|
||||
continue
|
||||
}
|
||||
if childrenErr != nil {
|
||||
return nil, fmt.Errorf("list ZooKeeper namespace %s: %w", path, childrenErr)
|
||||
}
|
||||
listedPath = path
|
||||
for _, child := range children {
|
||||
data, _, dataErr := connection.Get(path + "/" + child)
|
||||
if dataErr != nil {
|
||||
if errors.Is(dataErr, zk.ErrNoNode) {
|
||||
continue
|
||||
}
|
||||
nodeFailures = append(nodeFailures, fmt.Sprintf("%s/%s: %v", path, child, dataErr))
|
||||
continue
|
||||
}
|
||||
value, parseErr := parseHiveServerRegistration(child, data)
|
||||
if parseErr == nil {
|
||||
resolved = append(resolved, value)
|
||||
} else {
|
||||
nodeFailures = append(nodeFailures, fmt.Sprintf("%s/%s: %v", path, child, parseErr))
|
||||
}
|
||||
}
|
||||
if len(resolved) > 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
resolved = shuffledEndpoints(uniqueEndpoints(resolved), rejected)
|
||||
if len(resolved) == 0 {
|
||||
if listedPath == "" {
|
||||
return nil, fmt.Errorf("HiveServer2 ZooKeeper namespace not found; tried %s", strings.Join(discovery.paths(), ", "))
|
||||
}
|
||||
if len(nodeFailures) > 0 {
|
||||
return nil, fmt.Errorf("no usable HiveServer2 nodes in ZooKeeper namespace %s: %s", listedPath, strings.Join(nodeFailures, "; "))
|
||||
}
|
||||
return nil, fmt.Errorf("no available HiveServer2 nodes in ZooKeeper namespace %s", listedPath)
|
||||
}
|
||||
return resolved, nil
|
||||
}
|
||||
|
||||
func (discovery *zooKeeperDiscovery) paths() []string {
|
||||
namespace := strings.Trim(discovery.namespace, "/")
|
||||
if strings.EqualFold(discovery.discoveryMode, "zookeeperha") {
|
||||
return []string{
|
||||
zooKeeperPath(namespace, "instances"),
|
||||
zooKeeperPath(namespace+"-unsecure", "instances"),
|
||||
zooKeeperPath(namespace+"-sasl", "instances"),
|
||||
}
|
||||
}
|
||||
return []string{zooKeeperPath(namespace)}
|
||||
}
|
||||
|
||||
func zooKeeperPath(parts ...string) string {
|
||||
cleaned := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
if value := strings.Trim(part, "/"); value != "" {
|
||||
cleaned = append(cleaned, value)
|
||||
}
|
||||
}
|
||||
if len(cleaned) == 0 {
|
||||
return "/"
|
||||
}
|
||||
return "/" + strings.Join(cleaned, "/")
|
||||
}
|
||||
|
||||
func waitForZooKeeperSession(ctx context.Context, events <-chan zk.Event, timeout time.Duration) error {
|
||||
timer := time.NewTimer(timeout)
|
||||
defer timer.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return errors.New("ZooKeeper connection timed out before a session was established")
|
||||
case event, ok := <-events:
|
||||
if !ok {
|
||||
return errors.New("ZooKeeper event stream closed before a session was established")
|
||||
}
|
||||
if event.Err != nil {
|
||||
return fmt.Errorf("ZooKeeper connection event: %w", event.Err)
|
||||
}
|
||||
switch event.State {
|
||||
case zk.StateHasSession:
|
||||
return nil
|
||||
case zk.StateAuthFailed:
|
||||
return errors.New("ZooKeeper authentication failed")
|
||||
case zk.StateExpired:
|
||||
return errors.New("ZooKeeper session expired during connection")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func parseHiveServerRegistration(child string, data []byte) (endpoint, error) {
|
||||
candidates := []string{strings.TrimSpace(string(data)), strings.TrimSpace(child)}
|
||||
for _, candidate := range candidates {
|
||||
if candidate == "" {
|
||||
continue
|
||||
}
|
||||
if value, err := endpointFromRegistrationJSON(candidate); err == nil {
|
||||
return value, nil
|
||||
}
|
||||
parameters := parseHiveParameters(candidate)
|
||||
for _, key := range []string{"serveruri", "hiveserver2uri", "server_uri"} {
|
||||
if raw := parameter(parameters, key); raw != "" {
|
||||
return parseRegisteredEndpoint(raw)
|
||||
}
|
||||
}
|
||||
if value, err := endpointFromPublishedHiveConfig(parameters); err == nil {
|
||||
return value, nil
|
||||
}
|
||||
if strings.Contains(candidate, "=") {
|
||||
continue
|
||||
}
|
||||
if value, err := parseRegisteredEndpoint(candidate); err == nil {
|
||||
return value, nil
|
||||
}
|
||||
}
|
||||
return endpoint{}, fmt.Errorf("unsupported HiveServer2 ZooKeeper registration %q", child)
|
||||
}
|
||||
|
||||
func endpointFromRegistrationJSON(value string) (endpoint, error) {
|
||||
var object map[string]any
|
||||
if json.Unmarshal([]byte(value), &object) != nil {
|
||||
return endpoint{}, errors.New("not JSON")
|
||||
}
|
||||
for _, key := range []string{"serverUri", "server_uri", "hiveServer2Uri", "uri"} {
|
||||
if raw, ok := object[key].(string); ok && strings.TrimSpace(raw) != "" {
|
||||
return parseRegisteredEndpoint(raw)
|
||||
}
|
||||
}
|
||||
if serviceRecordEndpoint, ok := endpointFromServiceRecord(object); ok {
|
||||
return serviceRecordEndpoint, nil
|
||||
}
|
||||
host, _ := object["host"].(string)
|
||||
if host == "" {
|
||||
host, _ = object["hostname"].(string)
|
||||
}
|
||||
port := 0
|
||||
switch value := object["port"].(type) {
|
||||
case float64:
|
||||
port = int(value)
|
||||
case string:
|
||||
port, _ = strconv.Atoi(value)
|
||||
}
|
||||
if host != "" && port > 0 {
|
||||
return endpoint{Host: host, Port: port}, nil
|
||||
}
|
||||
return endpoint{}, errors.New("JSON registration has no endpoint")
|
||||
}
|
||||
|
||||
func endpointFromServiceRecord(object map[string]any) (endpoint, bool) {
|
||||
internal, _ := object["internal"].([]any)
|
||||
for _, rawEndpoint := range internal {
|
||||
published, _ := rawEndpoint.(map[string]any)
|
||||
if !strings.EqualFold(registrationStringValue(published["api"]), "activeEndpoint") {
|
||||
continue
|
||||
}
|
||||
addresses, _ := published["addresses"].([]any)
|
||||
for _, rawAddress := range addresses {
|
||||
address, _ := rawAddress.(map[string]any)
|
||||
host := registrationStringValue(address["host"])
|
||||
port, _ := strconv.Atoi(registrationStringValue(address["port"]))
|
||||
if host != "" && port > 0 {
|
||||
result := endpoint{Host: host, Port: port}
|
||||
applyPublishedHiveConfig(&result, object)
|
||||
return result, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return endpoint{}, false
|
||||
}
|
||||
|
||||
func registrationStringValue(value any) string {
|
||||
switch typed := value.(type) {
|
||||
case string:
|
||||
return strings.TrimSpace(typed)
|
||||
case float64:
|
||||
return strconv.FormatInt(int64(typed), 10)
|
||||
case json.Number:
|
||||
return typed.String()
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func applyPublishedHiveConfig(target *endpoint, parameters map[string]any) {
|
||||
value := func(key string) string {
|
||||
if nested, ok := parameters["attributes"].(map[string]any); ok {
|
||||
if result := registrationStringValue(nested[key]); result != "" {
|
||||
return result
|
||||
}
|
||||
}
|
||||
return registrationStringValue(parameters[key])
|
||||
}
|
||||
target.TransportMode = strings.ToLower(value("hive.server2.transport.mode"))
|
||||
target.HTTPPath = strings.TrimPrefix(value("hive.server2.thrift.http.path"), "/")
|
||||
target.Auth = strings.ToUpper(value("hive.server2.authentication"))
|
||||
target.Principal = value("hive.server2.authentication.kerberos.principal")
|
||||
target.SSL = strings.EqualFold(value("hive.server2.use.ssl"), "true")
|
||||
}
|
||||
|
||||
func endpointFromPublishedHiveConfig(parameters map[string]string) (endpoint, error) {
|
||||
host := firstNonEmpty(
|
||||
parameter(parameters, "hive.server2.thrift.bind.host"),
|
||||
parameter(parameters, "host"),
|
||||
)
|
||||
transportMode := strings.ToLower(parameter(parameters, "hive.server2.transport.mode"))
|
||||
portValue := parameter(parameters, "hive.server2.thrift.port")
|
||||
if transportMode == "http" {
|
||||
portValue = firstNonEmpty(parameter(parameters, "hive.server2.thrift.http.port"), portValue)
|
||||
}
|
||||
port, err := strconv.Atoi(portValue)
|
||||
if host == "" || err != nil || port <= 0 {
|
||||
return endpoint{}, errors.New("published HiveServer2 configuration has no valid host and port")
|
||||
}
|
||||
return endpoint{
|
||||
Host: host,
|
||||
Port: port,
|
||||
TransportMode: transportMode,
|
||||
HTTPPath: strings.TrimPrefix(parameter(parameters, "hive.server2.thrift.http.path"), "/"),
|
||||
Auth: strings.ToUpper(parameter(parameters, "hive.server2.authentication")),
|
||||
Principal: parameter(parameters, "hive.server2.authentication.kerberos.principal"),
|
||||
SSL: parameterBool(parameters, "hive.server2.use.ssl"),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func parseRegisteredEndpoint(value string) (endpoint, error) {
|
||||
value = strings.TrimSpace(value)
|
||||
if parsed, err := url.Parse(value); err == nil && parsed.Hostname() != "" {
|
||||
port := defaultHivePort
|
||||
if parsed.Port() != "" {
|
||||
parsedPort, parseErr := strconv.Atoi(parsed.Port())
|
||||
if parseErr != nil {
|
||||
return endpoint{}, parseErr
|
||||
}
|
||||
port = parsedPort
|
||||
}
|
||||
return endpoint{Host: parsed.Hostname(), Port: port}, nil
|
||||
}
|
||||
return parseEndpoint(value, defaultHivePort)
|
||||
}
|
||||
|
||||
func shuffledEndpoints(values []endpoint, rejected map[string]bool) []endpoint {
|
||||
result := make([]endpoint, 0, len(values))
|
||||
for _, value := range values {
|
||||
if !rejected[value.address()] {
|
||||
result = append(result, value)
|
||||
}
|
||||
}
|
||||
rand.Shuffle(len(result), func(first, second int) {
|
||||
result[first], result[second] = result[second], result[first]
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
func uniqueEndpoints(values []endpoint) []endpoint {
|
||||
seen := map[string]bool{}
|
||||
result := make([]endpoint, 0, len(values))
|
||||
for _, value := range values {
|
||||
key := value.address()
|
||||
if !seen[key] {
|
||||
seen[key] = true
|
||||
result = append(result, value)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-zookeeper/zk"
|
||||
)
|
||||
|
||||
func TestParseHiveServerRegistrationFromChildName(t *testing.T) {
|
||||
value, err := parseHiveServerRegistration("serverUri=hs2.example.com:10000;version=4.0.1", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if value != (endpoint{Host: "hs2.example.com", Port: 10000}) {
|
||||
t.Fatalf("unexpected endpoint: %#v", value)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseHiveServerRegistrationFromData(t *testing.T) {
|
||||
value, err := parseHiveServerRegistration("instance-0001", []byte(`{"serverUri":"thrift://kyuubi.example.com:10009"}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if value != (endpoint{Host: "kyuubi.example.com", Port: 10009}) {
|
||||
t.Fatalf("unexpected endpoint: %#v", value)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRejectedEndpointsAreExcluded(t *testing.T) {
|
||||
values := shuffledEndpoints([]endpoint{{Host: "one", Port: 1}, {Host: "two", Port: 2}}, map[string]bool{"one:1": true})
|
||||
if len(values) != 1 || values[0].Host != "two" {
|
||||
t.Fatalf("unexpected endpoints: %#v", values)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsePublishedHiveServerConfiguration(t *testing.T) {
|
||||
value, err := parseHiveServerRegistration("instance-1", []byte(
|
||||
"hive.server2.thrift.bind.host=hs2.example.com;"+
|
||||
"hive.server2.transport.mode=http;"+
|
||||
"hive.server2.thrift.http.port=10001;"+
|
||||
"hive.server2.thrift.http.path=cliservice;"+
|
||||
"hive.server2.authentication=KERBEROS;"+
|
||||
"hive.server2.authentication.kerberos.principal=hive/_HOST@EXAMPLE.COM;"+
|
||||
"hive.server2.use.ssl=true",
|
||||
))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if value.Host != "hs2.example.com" || value.Port != 10001 || value.TransportMode != "http" || value.HTTPPath != "cliservice" || value.Auth != "KERBEROS" || !value.SSL {
|
||||
t.Fatalf("unexpected endpoint: %#v", value)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseActivePassiveServiceRecord(t *testing.T) {
|
||||
value, err := parseHiveServerRegistration("instance-0001", []byte(`{
|
||||
"hive.server2.transport.mode":"binary",
|
||||
"hive.server2.authentication":"KERBEROS",
|
||||
"hive.server2.authentication.kerberos.principal":"hive/_HOST@EXAMPLE.COM",
|
||||
"internal":[{
|
||||
"api":"activeEndpoint",
|
||||
"addresses":[{"host":"active.example.com","port":"10000"}]
|
||||
}]
|
||||
}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if value.Host != "active.example.com" || value.Port != 10000 || value.Auth != "KERBEROS" || value.TransportMode != "binary" {
|
||||
t.Fatalf("unexpected active endpoint: %#v", value)
|
||||
}
|
||||
}
|
||||
|
||||
func TestZooKeeperHAPaths(t *testing.T) {
|
||||
discovery := &zooKeeperDiscovery{namespace: "hs2ActivePassiveHA", discoveryMode: "zooKeeperHA"}
|
||||
want := []string{
|
||||
"/hs2ActivePassiveHA/instances",
|
||||
"/hs2ActivePassiveHA-unsecure/instances",
|
||||
"/hs2ActivePassiveHA-sasl/instances",
|
||||
}
|
||||
got := discovery.paths()
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("unexpected paths: %#v", got)
|
||||
}
|
||||
for index := range want {
|
||||
if got[index] != want[index] {
|
||||
t.Fatalf("path %d = %q, want %q", index, got[index], want[index])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestZooKeeperPathNormalizesEmptyNamespace(t *testing.T) {
|
||||
if value := zooKeeperPath(""); value != "/" {
|
||||
t.Fatalf("unexpected root path: %q", value)
|
||||
}
|
||||
if value := zooKeeperPath("", "instances"); value != "/instances" {
|
||||
t.Fatalf("unexpected instances path: %q", value)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitForZooKeeperSession(t *testing.T) {
|
||||
events := make(chan zk.Event, 2)
|
||||
events <- zk.Event{State: zk.StateConnected}
|
||||
events <- zk.Event{State: zk.StateHasSession}
|
||||
if err := waitForZooKeeperSession(context.Background(), events, time.Second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWaitForZooKeeperSessionRejectsAuthFailure(t *testing.T) {
|
||||
events := make(chan zk.Event, 1)
|
||||
events <- zk.Event{State: zk.StateAuthFailed}
|
||||
if err := waitForZooKeeperSession(context.Background(), events, time.Second); err == nil {
|
||||
t.Fatal("expected authentication failure")
|
||||
}
|
||||
}
|
||||
|
||||
type fakeZooKeeperClient struct {
|
||||
children map[string][]string
|
||||
data map[string][]byte
|
||||
auth []byte
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (client *fakeZooKeeperClient) AddAuth(_ string, auth []byte) error {
|
||||
client.auth = append([]byte(nil), auth...)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (client *fakeZooKeeperClient) Children(path string) ([]string, *zk.Stat, error) {
|
||||
children, ok := client.children[path]
|
||||
if !ok {
|
||||
return nil, nil, zk.ErrNoNode
|
||||
}
|
||||
return append([]string(nil), children...), nil, nil
|
||||
}
|
||||
|
||||
func (client *fakeZooKeeperClient) Get(path string) ([]byte, *zk.Stat, error) {
|
||||
data, ok := client.data[path]
|
||||
if !ok {
|
||||
return nil, nil, zk.ErrNoNode
|
||||
}
|
||||
return append([]byte(nil), data...), nil, nil
|
||||
}
|
||||
|
||||
func (client *fakeZooKeeperClient) Close() { client.closed = true }
|
||||
|
||||
func TestZooKeeperDiscoveryUsesSessionAuthAndSkipsStaleNodes(t *testing.T) {
|
||||
client := &fakeZooKeeperClient{
|
||||
children: map[string][]string{"/hiveserver2": {"stale", "live"}},
|
||||
data: map[string][]byte{
|
||||
"/hiveserver2/live": []byte("serverUri=live.example.com:10000"),
|
||||
},
|
||||
}
|
||||
discovery := &zooKeeperDiscovery{
|
||||
servers: []endpoint{{Host: "zk.example.com", Port: 2181}},
|
||||
namespace: "hiveserver2",
|
||||
authScheme: "digest",
|
||||
auth: "user:password",
|
||||
timeout: time.Second,
|
||||
discoveryMode: "zookeeper",
|
||||
dialer: func([]string, time.Duration) (zooKeeperClient, <-chan zk.Event, error) {
|
||||
events := make(chan zk.Event, 1)
|
||||
events <- zk.Event{State: zk.StateHasSession}
|
||||
return client, events, nil
|
||||
},
|
||||
}
|
||||
values, err := discovery.Endpoints(context.Background(), map[string]bool{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(values) != 1 || values[0].Host != "live.example.com" {
|
||||
t.Fatalf("unexpected endpoints: %#v", values)
|
||||
}
|
||||
if string(client.auth) != "user:password" || !client.closed {
|
||||
t.Fatalf("auth=%q closed=%v", client.auth, client.closed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestZooKeeperHAFallsThroughStaleCandidatePath(t *testing.T) {
|
||||
client := &fakeZooKeeperClient{
|
||||
children: map[string][]string{
|
||||
"/hs2ActivePassiveHA/instances": {"stale"},
|
||||
"/hs2ActivePassiveHA-unsecure/instances": {"live"},
|
||||
},
|
||||
data: map[string][]byte{
|
||||
"/hs2ActivePassiveHA-unsecure/instances/live": []byte("serverUri=active.example.com:10000"),
|
||||
},
|
||||
}
|
||||
discovery := &zooKeeperDiscovery{
|
||||
servers: []endpoint{{Host: "zk.example.com", Port: 2181}},
|
||||
namespace: "hs2ActivePassiveHA",
|
||||
timeout: time.Second,
|
||||
discoveryMode: "zookeeperHA",
|
||||
dialer: func([]string, time.Duration) (zooKeeperClient, <-chan zk.Event, error) {
|
||||
events := make(chan zk.Event, 1)
|
||||
events <- zk.Event{State: zk.StateHasSession}
|
||||
return client, events, nil
|
||||
},
|
||||
}
|
||||
values, err := discovery.Endpoints(context.Background(), map[string]bool{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(values) != 1 || values[0].Host != "active.example.com" {
|
||||
t.Fatalf("unexpected HA endpoints: %#v", values)
|
||||
}
|
||||
}
|
||||
|
||||
func TestZooKeeperDiscoveryPropagatesDialError(t *testing.T) {
|
||||
discovery := &zooKeeperDiscovery{
|
||||
servers: []endpoint{{Host: "zk.example.com", Port: 2181}},
|
||||
dialer: func([]string, time.Duration) (zooKeeperClient, <-chan zk.Event, error) {
|
||||
return nil, nil, errors.New("dial failed")
|
||||
},
|
||||
}
|
||||
if _, err := discovery.Endpoints(context.Background(), nil); err == nil {
|
||||
t.Fatal("expected dial error")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
module github.com/t8y2/dbx/agents/drivers/argo-go
|
||||
|
||||
go 1.23.0
|
||||
|
||||
require (
|
||||
github.com/beltran/gosasl v1.0.0
|
||||
github.com/go-zookeeper/zk v1.0.4
|
||||
github.com/golang-auth/go-gssapi/v2 v2.0.0
|
||||
github.com/jcmturner/krb5test v0.0.0-20201230140143-102e4b78cdb8
|
||||
github.com/pavlo-v-chernykh/keystore-go/v4 v4.5.0
|
||||
github.com/t8y2/dbx/agents/go-common/gohive v0.0.0
|
||||
software.sslmate.com/src/go-pkcs12 v0.7.3
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e // indirect
|
||||
github.com/apache/thrift v0.22.0 // indirect
|
||||
github.com/beltran/gohive/v2 v2.1.0 // indirect
|
||||
github.com/hashicorp/go-uuid v1.0.2 // indirect
|
||||
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
|
||||
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
|
||||
github.com/jcmturner/gofork v1.0.0 // indirect
|
||||
github.com/jcmturner/gokrb5 v8.4.2+incompatible // indirect
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.2 // indirect
|
||||
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
golang.org/x/crypto v0.39.0 // indirect
|
||||
golang.org/x/net v0.41.0 // indirect
|
||||
)
|
||||
|
||||
replace github.com/beltran/gosasl => ../../go-common/gosasl
|
||||
|
||||
replace github.com/golang-auth/go-gssapi/v2 => ../../go-common/go-gssapi
|
||||
|
||||
replace github.com/t8y2/dbx/agents/go-common/gohive => ../../go-common/gohive
|
||||
@@ -0,0 +1,68 @@
|
||||
github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e h1:4dAU9FXIyQktpoUAgOJK3OTFc/xug0PCXYCqU0FgDKI=
|
||||
github.com/alexbrainman/sspi v0.0.0-20250919150558-7d374ff0d59e/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
|
||||
github.com/apache/thrift v0.22.0 h1:r7mTJdj51TMDe6RtcmNdQxgn9XcyfGDOzegMDRg47uc=
|
||||
github.com/apache/thrift v0.22.0/go.mod h1:1e7J/O1Ae6ZQMTYdy9xa3w9k+XHWPfRvdPyJeynQ+/g=
|
||||
github.com/beltran/gohive/v2 v2.1.0 h1:+XPfODaLXsZkD8mEnKCYqinC4OmVMx4/oR4ys8IGKeI=
|
||||
github.com/beltran/gohive/v2 v2.1.0/go.mod h1:MrxyI0sdriG52I+Bo2lJpGR75bxjcI/Hx3+MBnYqR10=
|
||||
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/go-zookeeper/zk v1.0.4 h1:DPzxraQx7OrPyXq2phlGlNSIyWEsAox0RJmjTseMV6I=
|
||||
github.com/go-zookeeper/zk v1.0.4/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw=
|
||||
github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
|
||||
github.com/gorilla/sessions v1.2.0/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
|
||||
github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
|
||||
github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE=
|
||||
github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=
|
||||
github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
|
||||
github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=
|
||||
github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM=
|
||||
github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8=
|
||||
github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
|
||||
github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o=
|
||||
github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg=
|
||||
github.com/jcmturner/gokrb5 v8.4.2+incompatible h1:MQW70Fbazv31g6URAXCjO2bGenIL0wVt3wqcpc0EjHI=
|
||||
github.com/jcmturner/gokrb5 v8.4.2+incompatible/go.mod h1:0Q5eFyVvYsEsZ8xl1A/jUqhXvxUp/X9ELrJm+zieq5E=
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.0/go.mod h1:T1hnNppQsBtxW0tCHMHTkAt8n/sABdzZgZdoFrZaZNM=
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.2 h1:6ZIM6b/JJN0X8UM43ZOM6Z4SJzla+a/u7scXFJzodkA=
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aWNYyvBVK62bc=
|
||||
github.com/jcmturner/krb5test v0.0.0-20201230140143-102e4b78cdb8 h1:bWgHpg2hLK3m8zHGSLqa+UV8XkivFFjG5IniW5lWzIA=
|
||||
github.com/jcmturner/krb5test v0.0.0-20201230140143-102e4b78cdb8/go.mod h1:eUQyrWMo2eMPWPezAYjLLwaiQHzJUD3su+1vAL10dOk=
|
||||
github.com/jcmturner/rpc/v2 v2.0.2/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
|
||||
github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
|
||||
github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
|
||||
github.com/pavlo-v-chernykh/keystore-go/v4 v4.5.0 h1:2nosf3P75OZv2/ZO/9Px5ZgZ5gbKrzA3joN1QMfOGMQ=
|
||||
github.com/pavlo-v-chernykh/keystore-go/v4 v4.5.0/go.mod h1:lAVhWwbNaveeJmxrxuSTxMgKpF6DjnuVpn6T8WiBwYQ=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
|
||||
golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210326220855-61e056675ecf/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k=
|
||||
golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
|
||||
golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
software.sslmate.com/src/go-pkcs12 v0.7.3 h1:JBQD3FDqYjTeyDAeZQklj2ar88ykBLtALloPJHyAauU=
|
||||
software.sslmate.com/src/go-pkcs12 v0.7.3/go.mod h1:Qiz0EyvDRJjjxGyUQa2cCNZn/wMyzrRJ/qcDXOQazLI=
|
||||
@@ -0,0 +1,65 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql/driver"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/t8y2/dbx/agents/go-common/gohive"
|
||||
)
|
||||
|
||||
func TestRunHiveInitStatementsExecutesAndDrainsResults(t *testing.T) {
|
||||
var resultRows *scriptedRows
|
||||
behavior := &scriptedBehavior{}
|
||||
behavior.query = func(ctx context.Context, query string) (driver.Rows, error) {
|
||||
switch query {
|
||||
case "SET hive.exec.dynamic.partition=true":
|
||||
resultRows = newScriptedRows(ctx, []string{"set"}, []string{"STRING"}, [][]driver.Value{
|
||||
{"hive.exec.dynamic.partition=true"},
|
||||
})
|
||||
return resultRows, nil
|
||||
case "USE analytics":
|
||||
return nil, &gohive.NonQueryResult{}
|
||||
default:
|
||||
return nil, errors.New("unexpected init statement")
|
||||
}
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
|
||||
err := runHiveInitStatements(
|
||||
context.Background(),
|
||||
server.connection,
|
||||
[]string{"SET hive.exec.dynamic.partition=true", "USE analytics"},
|
||||
77,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
queries, executions, _, _ := behavior.snapshot()
|
||||
if !reflect.DeepEqual(queries, []string{"SET hive.exec.dynamic.partition=true", "USE analytics"}) || len(executions) != 0 {
|
||||
t.Fatalf("unexpected init statements: queries=%v executions=%v", queries, executions)
|
||||
}
|
||||
if resultRows == nil || !resultRows.isClosed() {
|
||||
t.Fatal("initFile result set was not drained and closed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunHiveInitStatementsStopsOnFailure(t *testing.T) {
|
||||
behavior := &scriptedBehavior{}
|
||||
behavior.query = func(context.Context, string) (driver.Rows, error) {
|
||||
return nil, errors.New("permission denied")
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
|
||||
err := runHiveInitStatements(context.Background(), server.connection, []string{"USE restricted", "USE skipped"}, 100)
|
||||
if err == nil || !strings.Contains(err.Error(), "execute Hive initFile statement: permission denied") {
|
||||
t.Fatalf("unexpected initFile error: %v", err)
|
||||
}
|
||||
queries, _, _, _ := behavior.snapshot()
|
||||
if !reflect.DeepEqual(queries, []string{"USE restricted"}) {
|
||||
t.Fatalf("initFile continued after failure: %v", queries)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
//go:build !windows
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
func defaultKerberosConfigPath() string {
|
||||
return "/etc/krb5.conf"
|
||||
}
|
||||
|
||||
func defaultKerberosCCachePath() string {
|
||||
return fmt.Sprintf("/tmp/krb5cc_%d", os.Getuid())
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
//go:build windows
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
func defaultKerberosConfigPath() string {
|
||||
if windowsDirectory := os.Getenv("WINDIR"); windowsDirectory != "" {
|
||||
return filepath.Join(windowsDirectory, "krb5.ini")
|
||||
}
|
||||
return `C:\Windows\krb5.ini`
|
||||
}
|
||||
|
||||
func defaultKerberosCCachePath() string {
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,620 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
_ "time/tzdata"
|
||||
)
|
||||
|
||||
const (
|
||||
protocolVersion = 2
|
||||
defaultMaxRows = 10000
|
||||
defaultPageSize = 1000
|
||||
defaultFetchSize = 50
|
||||
legacyAgentSessionID = "__legacy__"
|
||||
maxAgentSessions = 256
|
||||
querySessionIdleTime = 10 * time.Minute
|
||||
)
|
||||
|
||||
type request struct {
|
||||
ID json.RawMessage `json:"id"`
|
||||
Method string `json:"method"`
|
||||
Params map[string]json.RawMessage `json:"params"`
|
||||
}
|
||||
|
||||
type response struct {
|
||||
JSONRPC string `json:"jsonrpc,omitempty"`
|
||||
ID json.RawMessage `json:"id,omitempty"`
|
||||
Result any `json:"result,omitempty"`
|
||||
Error *rpcError `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type queryOptions struct {
|
||||
SQL string `json:"sql"`
|
||||
Database string `json:"database"`
|
||||
Schema string `json:"schema"`
|
||||
MaxRows int `json:"maxRows"`
|
||||
FetchSize int `json:"fetchSize"`
|
||||
TimeoutSecs int `json:"timeoutSecs"`
|
||||
}
|
||||
|
||||
type queryResult struct {
|
||||
Columns []string `json:"columns"`
|
||||
ColumnTypes []string `json:"column_types"`
|
||||
Rows [][]any `json:"rows"`
|
||||
AffectedRows int64 `json:"affected_rows"`
|
||||
ExecutionTimeMS int64 `json:"execution_time_ms"`
|
||||
Truncated bool `json:"truncated"`
|
||||
}
|
||||
|
||||
type queryPageResult struct {
|
||||
Columns []string `json:"columns"`
|
||||
ColumnTypes []string `json:"column_types"`
|
||||
Rows [][]any `json:"rows"`
|
||||
AffectedRows int64 `json:"affected_rows"`
|
||||
ExecutionTimeMS int64 `json:"execution_time_ms"`
|
||||
Truncated bool `json:"truncated"`
|
||||
SessionID *string `json:"session_id"`
|
||||
HasMore bool `json:"has_more"`
|
||||
}
|
||||
|
||||
type querySession struct {
|
||||
rows *sql.Rows
|
||||
columns []string
|
||||
columnTypes []string
|
||||
pending []any
|
||||
remaining int
|
||||
cancel context.CancelFunc
|
||||
lastAccessed time.Time
|
||||
}
|
||||
|
||||
type server struct {
|
||||
params connectParams
|
||||
config connectionConfig
|
||||
|
||||
connectionMu sync.Mutex
|
||||
database *sql.DB
|
||||
connection *sql.Conn
|
||||
|
||||
querySessions map[string]*querySession
|
||||
nextSessionID uint64
|
||||
|
||||
activeMu sync.Mutex
|
||||
activeCancel context.CancelFunc
|
||||
}
|
||||
|
||||
type agentSession struct {
|
||||
server *server
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
type runtimeServer struct {
|
||||
mu sync.RWMutex
|
||||
sessions map[string]*agentSession
|
||||
}
|
||||
|
||||
func main() {
|
||||
configureRuntimeParallelism()
|
||||
runtimeServer := newRuntimeServer()
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
var encoderMu sync.Mutex
|
||||
var requests sync.WaitGroup
|
||||
fmt.Fprintln(os.Stdout, `{"ready":true}`)
|
||||
|
||||
scanner := bufio.NewScanner(os.Stdin)
|
||||
scanner.Buffer(make([]byte, 0, 64*1024), 512*1024*1024)
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
var envelope request
|
||||
if json.Unmarshal([]byte(line), &envelope) == nil && envelope.Method == "shutdown" {
|
||||
requests.Wait()
|
||||
result, _ := runtimeServer.handleLine(line)
|
||||
encoderMu.Lock()
|
||||
_ = encoder.Encode(result)
|
||||
encoderMu.Unlock()
|
||||
return
|
||||
}
|
||||
requests.Add(1)
|
||||
go func(line string) {
|
||||
defer requests.Done()
|
||||
result, _ := runtimeServer.handleLine(line)
|
||||
encoderMu.Lock()
|
||||
defer encoderMu.Unlock()
|
||||
if err := encoder.Encode(result); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "failed to write response: %v\n", err)
|
||||
}
|
||||
}(line)
|
||||
}
|
||||
requests.Wait()
|
||||
}
|
||||
|
||||
func configureRuntimeParallelism() {
|
||||
if raw := strings.TrimSpace(os.Getenv("DBX_AGENT_HIVE_GOMAXPROCS")); raw != "" {
|
||||
if configured, err := strconv.Atoi(raw); err == nil && configured > 0 {
|
||||
runtime.GOMAXPROCS(configured)
|
||||
return
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(os.Getenv("GOMAXPROCS")) == "" {
|
||||
runtime.GOMAXPROCS(min(runtime.NumCPU(), 4))
|
||||
}
|
||||
}
|
||||
|
||||
func newRuntimeServer() *runtimeServer {
|
||||
return &runtimeServer{sessions: map[string]*agentSession{}}
|
||||
}
|
||||
|
||||
func (runtimeServer *runtimeServer) handleLine(line string) (response, bool) {
|
||||
var request request
|
||||
if err := json.Unmarshal([]byte(line), &request); err != nil {
|
||||
return errorResponse(nil, "", "", err), false
|
||||
}
|
||||
if len(request.ID) == 0 {
|
||||
request.ID = json.RawMessage("1")
|
||||
}
|
||||
result, shutdown, err := runtimeServer.dispatch(request.Method, request.Params)
|
||||
if err != nil {
|
||||
return errorResponse(request.ID, request.Method, stringParam(request.Params, "agentSessionId"), err), false
|
||||
}
|
||||
return response{JSONRPC: "2.0", ID: request.ID, Result: result}, shutdown
|
||||
}
|
||||
|
||||
func (runtimeServer *runtimeServer) dispatch(method string, params map[string]json.RawMessage) (any, bool, error) {
|
||||
switch method {
|
||||
case "handshake":
|
||||
return handshakeResult(true), false, nil
|
||||
case "open_session":
|
||||
id := stringParam(params, "agentSessionId")
|
||||
if id == "" {
|
||||
return nil, false, errors.New("agentSessionId is required")
|
||||
}
|
||||
var connection connectParams
|
||||
if err := decodeParams(params, &connection); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return map[string]bool{"ok": true}, false, runtimeServer.openSession(id, connection)
|
||||
case "close_session":
|
||||
return map[string]bool{"ok": true}, false, runtimeServer.closeSession(stringParam(params, "agentSessionId"))
|
||||
case "validate_session":
|
||||
session, err := runtimeServer.session(stringParam(params, "agentSessionId"))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
session.mu.Lock()
|
||||
defer session.mu.Unlock()
|
||||
return map[string]bool{"ok": true}, false, session.server.validateConnection()
|
||||
case "cancel_session":
|
||||
session, err := runtimeServer.session(stringParam(params, "agentSessionId"))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
session.server.cancelActiveQuery()
|
||||
return map[string]bool{"ok": true}, false, nil
|
||||
case "test_connection":
|
||||
var connection connectParams
|
||||
if err := decodeParams(params, &connection); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
result, err := testConnection(connection)
|
||||
return result, false, err
|
||||
case "connect":
|
||||
var connection connectParams
|
||||
if err := decodeParams(params, &connection); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
_ = runtimeServer.closeSession(legacyAgentSessionID)
|
||||
return map[string]bool{"ok": true}, false, runtimeServer.openSession(legacyAgentSessionID, connection)
|
||||
case "disconnect":
|
||||
return map[string]bool{"ok": true}, false, runtimeServer.closeSession(legacyAgentSessionID)
|
||||
case "shutdown":
|
||||
return map[string]bool{"ok": true}, true, runtimeServer.closeAllSessions()
|
||||
}
|
||||
|
||||
sessionID := stringParam(params, "agentSessionId")
|
||||
if sessionID == "" {
|
||||
sessionID = legacyAgentSessionID
|
||||
}
|
||||
session, err := runtimeServer.session(sessionID)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
session.mu.Lock()
|
||||
defer session.mu.Unlock()
|
||||
return session.server.dispatch(method, params)
|
||||
}
|
||||
|
||||
func (runtimeServer *runtimeServer) openSession(id string, params connectParams) error {
|
||||
runtimeServer.mu.Lock()
|
||||
if len(runtimeServer.sessions) >= maxAgentSessions {
|
||||
runtimeServer.mu.Unlock()
|
||||
return fmt.Errorf("maximum Hive Agent sessions reached (%d)", maxAgentSessions)
|
||||
}
|
||||
if _, exists := runtimeServer.sessions[id]; exists {
|
||||
runtimeServer.mu.Unlock()
|
||||
return fmt.Errorf("Hive Agent session already exists: %s", id)
|
||||
}
|
||||
runtimeServer.mu.Unlock()
|
||||
|
||||
server, err := newServer(params)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
runtimeServer.mu.Lock()
|
||||
defer runtimeServer.mu.Unlock()
|
||||
if _, exists := runtimeServer.sessions[id]; exists {
|
||||
_ = server.disconnect()
|
||||
return fmt.Errorf("Hive Agent session already exists: %s", id)
|
||||
}
|
||||
if len(runtimeServer.sessions) >= maxAgentSessions {
|
||||
_ = server.disconnect()
|
||||
return fmt.Errorf("maximum Hive Agent sessions reached (%d)", maxAgentSessions)
|
||||
}
|
||||
runtimeServer.sessions[id] = &agentSession{server: server}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (runtimeServer *runtimeServer) session(id string) (*agentSession, error) {
|
||||
runtimeServer.mu.RLock()
|
||||
session := runtimeServer.sessions[id]
|
||||
runtimeServer.mu.RUnlock()
|
||||
if session == nil {
|
||||
return nil, fmt.Errorf("Hive Agent session not found: %s", id)
|
||||
}
|
||||
return session, nil
|
||||
}
|
||||
|
||||
func (runtimeServer *runtimeServer) closeSession(id string) error {
|
||||
if id == "" {
|
||||
id = legacyAgentSessionID
|
||||
}
|
||||
runtimeServer.mu.Lock()
|
||||
session := runtimeServer.sessions[id]
|
||||
delete(runtimeServer.sessions, id)
|
||||
runtimeServer.mu.Unlock()
|
||||
if session == nil {
|
||||
return nil
|
||||
}
|
||||
session.mu.Lock()
|
||||
defer session.mu.Unlock()
|
||||
return session.server.disconnect()
|
||||
}
|
||||
|
||||
func (runtimeServer *runtimeServer) closeAllSessions() error {
|
||||
runtimeServer.mu.Lock()
|
||||
sessions := runtimeServer.sessions
|
||||
runtimeServer.sessions = map[string]*agentSession{}
|
||||
runtimeServer.mu.Unlock()
|
||||
var failures []string
|
||||
for id, session := range sessions {
|
||||
session.mu.Lock()
|
||||
err := session.server.disconnect()
|
||||
session.mu.Unlock()
|
||||
if err != nil {
|
||||
failures = append(failures, fmt.Sprintf("%s: %v", id, err))
|
||||
}
|
||||
}
|
||||
if len(failures) > 0 {
|
||||
return fmt.Errorf("close Hive Agent sessions: %s", strings.Join(failures, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func newServer(params connectParams) (*server, error) {
|
||||
config, err := parseConnectionConfig(params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
server := &server{
|
||||
params: params,
|
||||
config: config,
|
||||
querySessions: map[string]*querySession{},
|
||||
}
|
||||
if err := server.openConnection(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return server, nil
|
||||
}
|
||||
|
||||
func (server *server) openConnection() error {
|
||||
server.connectionMu.Lock()
|
||||
defer server.connectionMu.Unlock()
|
||||
if server.connection != nil {
|
||||
return nil
|
||||
}
|
||||
database := openHiveDatabase(server.config)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), server.connectionOpenTimeout())
|
||||
var connection *sql.Conn
|
||||
connection, err := database.Conn(ctx)
|
||||
if err == nil {
|
||||
err = connection.PingContext(ctx)
|
||||
}
|
||||
cancel()
|
||||
if err == nil {
|
||||
err = runHiveInitStatements(context.Background(), connection, server.config.InitStatements, server.config.FetchSize)
|
||||
}
|
||||
if err != nil {
|
||||
if connection != nil {
|
||||
_ = connection.Close()
|
||||
}
|
||||
_ = database.Close()
|
||||
return err
|
||||
}
|
||||
server.database = database
|
||||
server.connection = connection
|
||||
return nil
|
||||
}
|
||||
|
||||
func (server *server) connectionOpenTimeout() time.Duration {
|
||||
timeout := server.config.ConnectTimeout
|
||||
if strings.EqualFold(server.config.Auth, "BROWSER") && strings.TrimSpace(server.config.BrowserToken) == "" {
|
||||
timeout += server.config.BrowserResponseTimeout
|
||||
}
|
||||
return timeout
|
||||
}
|
||||
|
||||
func runHiveInitStatements(ctx context.Context, connection *sql.Conn, statements []string, fetchSize int) error {
|
||||
for _, statement := range statements {
|
||||
rows, _, hasResultSet, err := executeHiveStatement(ctx, connection, statement, fetchSize)
|
||||
if err != nil {
|
||||
return fmt.Errorf("execute Hive initFile statement: %w", err)
|
||||
}
|
||||
if !hasResultSet {
|
||||
continue
|
||||
}
|
||||
for rows.Next() {
|
||||
}
|
||||
if rowsErr := rows.Err(); rowsErr != nil {
|
||||
_ = rows.Close()
|
||||
return fmt.Errorf("read Hive initFile statement result: %w", rowsErr)
|
||||
}
|
||||
if closeErr := rows.Close(); closeErr != nil {
|
||||
return fmt.Errorf("close Hive initFile statement result: %w", closeErr)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (server *server) dispatch(method string, params map[string]json.RawMessage) (any, bool, error) {
|
||||
switch method {
|
||||
case "validate_connection":
|
||||
return map[string]bool{"ok": true}, false, server.validateConnection()
|
||||
case "connection_info":
|
||||
result, err := server.connectionInfo()
|
||||
return result, false, err
|
||||
case "list_databases":
|
||||
result, err := server.listDatabases()
|
||||
return result, false, err
|
||||
case "list_schemas":
|
||||
result, err := server.listSchemas(stringSliceParam(params, "visible_schemas"))
|
||||
return result, false, err
|
||||
case "list_tables":
|
||||
result, err := server.listTables(stringParam(params, "schema"), metadataListConstraintsFromParams(params))
|
||||
return result, false, err
|
||||
case "get_table_comment":
|
||||
result, err := server.getTableComment(stringParam(params, "schema"), stringParam(params, "table"))
|
||||
return result, false, err
|
||||
case "list_objects":
|
||||
result, err := server.listObjects(
|
||||
stringParam(params, "database"),
|
||||
stringParam(params, "schema"),
|
||||
metadataListConstraintsFromParams(params),
|
||||
)
|
||||
return result, false, err
|
||||
case "list_data_types":
|
||||
result, err := server.listDataTypes()
|
||||
return result, false, err
|
||||
case "completion_assistant_search_v1":
|
||||
var input completionAssistantRequest
|
||||
if err := decodeParams(params, &input); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
result, err := server.completionAssistantSearch(input)
|
||||
return result, false, err
|
||||
case "get_columns":
|
||||
result, err := server.getColumns(stringParam(params, "schema"), stringParam(params, "table"))
|
||||
return result, false, err
|
||||
case "list_indexes":
|
||||
return []indexInfo{}, false, nil
|
||||
case "list_foreign_keys":
|
||||
return []foreignKeyInfo{}, false, nil
|
||||
case "list_triggers":
|
||||
return []triggerInfo{}, false, nil
|
||||
case "list_constraints", "list_partitions", "list_subpartitions":
|
||||
return []any{}, false, nil
|
||||
case "get_object_source":
|
||||
result, err := server.getObjectSource(
|
||||
stringParam(params, "database"),
|
||||
stringParam(params, "schema"),
|
||||
firstNonEmpty(stringParam(params, "name"), stringParam(params, "table")),
|
||||
stringParam(params, "object_type"),
|
||||
)
|
||||
return result, false, err
|
||||
case "get_table_ddl":
|
||||
result, err := server.getTableDDL(stringParam(params, "schema"), stringParam(params, "table"))
|
||||
return result, false, err
|
||||
case "get_explain_info":
|
||||
result, err := server.getExplainInfo(stringParam(params, "sql"))
|
||||
return map[string]any{"plan": result, "has_actual_stats": false}, false, err
|
||||
case "execute_query":
|
||||
result, err := server.executeQuery(queryOptionsFromParams(params))
|
||||
return result, false, err
|
||||
case "execute_query_page", "start_table_read":
|
||||
result, err := server.executeQueryPage(queryOptionsFromParams(params), intParam(params, "pageSize"))
|
||||
return result, false, err
|
||||
case "fetch_query_page", "fetch_table_read_page":
|
||||
result, err := server.fetchQueryPage(stringParam(params, "sessionId"), intParam(params, "pageSize"))
|
||||
return result, false, err
|
||||
case "close_query_session", "close_table_read_session":
|
||||
return server.closeQuerySession(stringParam(params, "sessionId")), false, nil
|
||||
case "execute_transaction":
|
||||
result, err := server.executeStatements(params, true)
|
||||
return result, false, err
|
||||
case "execute_batch":
|
||||
result, err := server.executeStatements(params, false)
|
||||
return result, false, err
|
||||
case "disconnect":
|
||||
return map[string]bool{"ok": true}, false, server.disconnect()
|
||||
case "shutdown":
|
||||
return map[string]bool{"ok": true}, true, server.disconnect()
|
||||
default:
|
||||
return nil, false, fmt.Errorf("unknown method: %s", method)
|
||||
}
|
||||
}
|
||||
|
||||
func handshakeResult(multiSession bool) map[string]any {
|
||||
capabilities := []string{
|
||||
"connect", "test_connection", "metadata", "query", "paged_query", "transaction", "ddl", "structured_error_v1",
|
||||
}
|
||||
if multiSession {
|
||||
capabilities = append(capabilities, "multi_session")
|
||||
}
|
||||
return map[string]any{
|
||||
"protocolVersion": protocolVersion,
|
||||
"agentProtocolVersion": protocolVersion,
|
||||
"capabilities": capabilities,
|
||||
}
|
||||
}
|
||||
|
||||
func testConnection(params connectParams) (map[string]any, error) {
|
||||
server, err := newServer(params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer server.disconnect()
|
||||
if err := server.validateConnection(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := server.connectionInfo()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"ok": true, "info": info}, nil
|
||||
}
|
||||
|
||||
func (server *server) disconnect() error {
|
||||
server.cancelActiveQuery()
|
||||
if err := server.closeAllQuerySessions(); err != nil {
|
||||
return err
|
||||
}
|
||||
server.connectionMu.Lock()
|
||||
connection := server.connection
|
||||
database := server.database
|
||||
server.connection = nil
|
||||
server.database = nil
|
||||
server.connectionMu.Unlock()
|
||||
var failures []string
|
||||
if connection != nil {
|
||||
if err := connection.Close(); err != nil {
|
||||
failures = append(failures, err.Error())
|
||||
}
|
||||
}
|
||||
if database != nil {
|
||||
if err := database.Close(); err != nil {
|
||||
failures = append(failures, err.Error())
|
||||
}
|
||||
}
|
||||
if len(failures) > 0 {
|
||||
return errors.New(strings.Join(failures, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (server *server) requireConnection() (*sql.Conn, error) {
|
||||
server.connectionMu.Lock()
|
||||
connection := server.connection
|
||||
server.connectionMu.Unlock()
|
||||
if connection == nil {
|
||||
return nil, errors.New("Hive connection is not open")
|
||||
}
|
||||
return connection, nil
|
||||
}
|
||||
|
||||
func (server *server) setActiveOperation(cancel context.CancelFunc) {
|
||||
server.activeMu.Lock()
|
||||
server.activeCancel = cancel
|
||||
server.activeMu.Unlock()
|
||||
}
|
||||
|
||||
func (server *server) clearActiveOperation(cancel context.CancelFunc) {
|
||||
cancel()
|
||||
server.activeMu.Lock()
|
||||
server.activeCancel = nil
|
||||
server.activeMu.Unlock()
|
||||
}
|
||||
|
||||
func (server *server) cancelActiveQuery() {
|
||||
server.activeMu.Lock()
|
||||
cancel := server.activeCancel
|
||||
server.activeMu.Unlock()
|
||||
if cancel != nil {
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
|
||||
func queryOptionsFromParams(params map[string]json.RawMessage) queryOptions {
|
||||
return queryOptions{
|
||||
SQL: stringParam(params, "sql"),
|
||||
Database: stringParam(params, "database"),
|
||||
Schema: stringParam(params, "schema"),
|
||||
MaxRows: intParam(params, "maxRows"),
|
||||
FetchSize: intParam(params, "fetchSize"),
|
||||
TimeoutSecs: intParam(params, "timeoutSecs"),
|
||||
}
|
||||
}
|
||||
|
||||
func decodeParams(params map[string]json.RawMessage, target any) error {
|
||||
data, err := json.Marshal(params)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.Unmarshal(data, target)
|
||||
}
|
||||
|
||||
func stringParam(params map[string]json.RawMessage, key string) string {
|
||||
if raw, ok := params[key]; ok {
|
||||
var value string
|
||||
if json.Unmarshal(raw, &value) == nil {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func intParam(params map[string]json.RawMessage, key string) int {
|
||||
if raw, ok := params[key]; ok {
|
||||
var value int
|
||||
if json.Unmarshal(raw, &value) == nil {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func stringSliceParam(params map[string]json.RawMessage, key string) []string {
|
||||
raw, ok := params[key]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
var value []string
|
||||
if json.Unmarshal(raw, &value) == nil {
|
||||
return value
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func errorResponse(id json.RawMessage, method, sessionID string, err error) response {
|
||||
return response{JSONRPC: "2.0", ID: id, Error: classifyRPCError(method, sessionID, err)}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,748 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql/driver"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/t8y2/dbx/agents/go-common/gohive"
|
||||
)
|
||||
|
||||
func TestShowTablesRowName(t *testing.T) {
|
||||
if value := showTablesRowName([]string{"database", "tableName", "isTemporary"}, []any{"default", "events", false}); value != "events" {
|
||||
t.Fatalf("unexpected table name: %q", value)
|
||||
}
|
||||
if value := showTablesRowName([]string{"tab_name"}, []any{"fallback"}); value != "fallback" {
|
||||
t.Fatalf("unexpected fallback table name: %q", value)
|
||||
}
|
||||
}
|
||||
|
||||
// The argo agent always reports ArgoDB identity; Kyuubi/Impala/Hive remain on hive-go.
|
||||
func TestConnectionInfoReportsArgoIdentity(t *testing.T) {
|
||||
behavior := &scriptedBehavior{
|
||||
query: func(ctx context.Context, query string) (driver.Rows, error) {
|
||||
switch query {
|
||||
case "SELECT VERSION()":
|
||||
return newScriptedRows(ctx, []string{"version"}, []string{"STRING"}, [][]driver.Value{{"3.5.8"}}), nil
|
||||
case "SELECT CURRENT_USER()":
|
||||
return newScriptedRows(ctx, []string{"current_user"}, []string{"STRING"}, [][]driver.Value{{"dbx"}}), nil
|
||||
default:
|
||||
return nil, errors.New("unexpected query: " + query)
|
||||
}
|
||||
},
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
server.params.DatabaseType = "argo"
|
||||
server.config.Username = "fallback"
|
||||
|
||||
info, err := server.connectionInfo()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info["compatibilityMode"] != "argo" || info["username"] != "dbx" || info["version"] != "3.5.8" {
|
||||
t.Fatalf("unexpected Argo connection info: %#v", info)
|
||||
}
|
||||
databaseInfo, ok := info["databaseInfo"].(map[string]string)
|
||||
if !ok || databaseInfo["productName"] != "ArgoDB (Transwarp)" || databaseInfo["driverName"] != "DBX ArgoDB Go Agent" {
|
||||
t.Fatalf("unexpected Argo database identity: %#v", info["databaseInfo"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetObjectSourceReturnsProtocolObject(t *testing.T) {
|
||||
behavior := &scriptedBehavior{
|
||||
query: func(ctx context.Context, query string) (driver.Rows, error) {
|
||||
if query != "SHOW CREATE TABLE `dbx_kyuubi_demo`.`high_value_orders`" {
|
||||
t.Fatalf("unexpected query: %q", query)
|
||||
}
|
||||
return newScriptedRows(
|
||||
ctx,
|
||||
[]string{"createtab_stmt"},
|
||||
[]string{"STRING"},
|
||||
[][]driver.Value{
|
||||
{"CREATE VIEW dbx_kyuubi_demo.high_value_orders"},
|
||||
{"AS SELECT id, customer, amount FROM dbx_kyuubi_demo.orders WHERE amount >= 50"},
|
||||
},
|
||||
), nil
|
||||
},
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
defer server.disconnect()
|
||||
|
||||
result, _, err := server.dispatch("get_object_source", map[string]json.RawMessage{
|
||||
"schema": json.RawMessage(`"dbx_kyuubi_demo"`),
|
||||
"name": json.RawMessage(`"high_value_orders"`),
|
||||
"object_type": json.RawMessage(`"VIEW"`),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
source, ok := result.(objectSource)
|
||||
if !ok {
|
||||
t.Fatalf("get_object_source returned %T instead of objectSource", result)
|
||||
}
|
||||
if source.Name != "high_value_orders" || source.ObjectType != "VIEW" || source.Schema == nil || *source.Schema != "dbx_kyuubi_demo" {
|
||||
t.Fatalf("unexpected object source metadata: %#v", source)
|
||||
}
|
||||
expected := "CREATE VIEW dbx_kyuubi_demo.high_value_orders\nAS SELECT id, customer, amount FROM dbx_kyuubi_demo.orders WHERE amount >= 50\n"
|
||||
if source.Source != expected {
|
||||
t.Fatalf("unexpected object source DDL: %q", source.Source)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListDatabasesUsesShowDatabasesBeforeHiveServerMetadata(t *testing.T) {
|
||||
behavior := &scriptedBehavior{
|
||||
query: func(ctx context.Context, sql string) (driver.Rows, error) {
|
||||
if sql != "SHOW DATABASES" {
|
||||
t.Fatalf("unexpected query: %q", sql)
|
||||
}
|
||||
return newScriptedRows(ctx, []string{"database_name"}, []string{"STRING"}, [][]driver.Value{{"warehouse"}, {"default"}, {"default"}}), nil
|
||||
},
|
||||
getSchemas: func(_ context.Context, pattern string) (gohive.MetadataResult, error) {
|
||||
t.Fatalf("metadata fallback must not run after SHOW DATABASES succeeds: %q", pattern)
|
||||
return gohive.MetadataResult{}, nil
|
||||
},
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
defer server.disconnect()
|
||||
values, err := server.listDatabases()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(values, []databaseInfo{{Name: "default"}, {Name: "warehouse"}}) {
|
||||
t.Fatalf("unexpected databases: %#v", values)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListDatabasesFallsBackToHiveServerMetadata(t *testing.T) {
|
||||
behavior := &scriptedBehavior{
|
||||
query: func(context.Context, string) (driver.Rows, error) {
|
||||
return nil, errors.New("SHOW DATABASES unsupported")
|
||||
},
|
||||
getSchemas: func(_ context.Context, pattern string) (gohive.MetadataResult, error) {
|
||||
if pattern != "%" {
|
||||
t.Fatalf("unexpected schema pattern: %q", pattern)
|
||||
}
|
||||
return metadataResult([]string{"TABLE_SCHEM", "TABLE_CATALOG"}, []driver.Value{"warehouse", ""}, []driver.Value{"default", ""}, []driver.Value{"default", ""}), nil
|
||||
},
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
defer server.disconnect()
|
||||
values, err := server.listDatabases()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(values, []databaseInfo{{Name: "default"}, {Name: "warehouse"}}) {
|
||||
t.Fatalf("unexpected databases: %#v", values)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListSchemasHonorsVisibleSchemaFilter(t *testing.T) {
|
||||
behavior := &scriptedBehavior{
|
||||
query: func(ctx context.Context, sql string) (driver.Rows, error) {
|
||||
if sql != "SHOW DATABASES" {
|
||||
t.Fatalf("unexpected query: %q", sql)
|
||||
}
|
||||
return newScriptedRows(
|
||||
ctx,
|
||||
[]string{"database_name"},
|
||||
[]string{"STRING"},
|
||||
[][]driver.Value{{"default"}, {"analytics"}, {"system"}},
|
||||
), nil
|
||||
},
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
defer server.disconnect()
|
||||
|
||||
values, err := server.listSchemas([]string{"analytics", "missing"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(values, []string{"analytics"}) {
|
||||
t.Fatalf("unexpected visible schemas: %#v", values)
|
||||
}
|
||||
|
||||
values, err = server.listSchemas([]string{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(values) != 0 {
|
||||
t.Fatalf("explicit empty visible schema filter must hide all schemas: %#v", values)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListTablesPreservesViewTypeCommentAndWindow(t *testing.T) {
|
||||
behavior := &scriptedBehavior{
|
||||
getTables: func(_ context.Context, schema, table string, tableTypes []string) (gohive.MetadataResult, error) {
|
||||
if schema != "analytics" || table != "%" || !reflect.DeepEqual(tableTypes, []string{"TABLE", "VIEW", "MATERIALIZED VIEW"}) {
|
||||
t.Fatalf("unexpected GetTables request: schema=%q table=%q types=%#v", schema, table, tableTypes)
|
||||
}
|
||||
return metadataResult(
|
||||
[]string{"TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "TABLE_TYPE", "REMARKS"},
|
||||
[]driver.Value{"", "analytics", "events", "TABLE", "event data"},
|
||||
[]driver.Value{"", "analytics", "events_view", "VIEW", "view data"},
|
||||
[]driver.Value{"", "analytics", "other", "TABLE", nil},
|
||||
), nil
|
||||
},
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
defer server.disconnect()
|
||||
values, err := server.listTables("analytics", metadataListConstraints{Filter: "events", Offset: 1, Limit: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
comment := "view data"
|
||||
expected := []tableInfo{{Name: "events_view", TableType: "VIEW", Comment: &comment}}
|
||||
if !reflect.DeepEqual(values, expected) {
|
||||
t.Fatalf("unexpected tables: %#v", values)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetColumnsUsesHiveServerMetadataFields(t *testing.T) {
|
||||
behavior := &scriptedBehavior{
|
||||
getColumns: func(_ context.Context, schema, table, column string) (gohive.MetadataResult, error) {
|
||||
if schema != "analytics" || table != "events" || column != "%" {
|
||||
t.Fatalf("unexpected GetColumns request: %q %q %q", schema, table, column)
|
||||
}
|
||||
return metadataResult(
|
||||
[]string{"COLUMN_NAME", "TYPE_NAME", "COLUMN_SIZE", "DECIMAL_DIGITS", "NULLABLE", "REMARKS", "COLUMN_DEF"},
|
||||
[]driver.Value{"name", "string", int64(255), nil, int64(1), "显示名称", "unknown"},
|
||||
[]driver.Value{"amount", "decimal(18,2)", int64(18), int64(2), int64(0), nil, nil},
|
||||
), nil
|
||||
},
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
defer server.disconnect()
|
||||
values, err := server.getColumns("analytics", "events")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(values) != 2 {
|
||||
t.Fatalf("unexpected columns: %#v", values)
|
||||
}
|
||||
if values[0].Name != "name" || !values[0].IsNullable || values[0].CharacterMaximumLength == nil || *values[0].CharacterMaximumLength != 255 || values[0].ColumnDefault == nil || *values[0].ColumnDefault != "unknown" || values[0].Comment == nil || *values[0].Comment != "显示名称" {
|
||||
t.Fatalf("unexpected string column: %#v", values[0])
|
||||
}
|
||||
if values[1].Name != "amount" || values[1].IsNullable || values[1].NumericPrecision == nil || *values[1].NumericPrecision != 18 || values[1].NumericScale == nil || *values[1].NumericScale != 2 || values[1].CharacterMaximumLength != nil {
|
||||
t.Fatalf("unexpected decimal column: %#v", values[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetColumnsPreservesChineseDescribeFallbackComments(t *testing.T) {
|
||||
behavior := &scriptedBehavior{
|
||||
getColumns: func(context.Context, string, string, string) (gohive.MetadataResult, error) {
|
||||
return gohive.MetadataResult{}, errors.New("metadata unavailable")
|
||||
},
|
||||
query: func(ctx context.Context, query string) (driver.Rows, error) {
|
||||
if query != "DESCRIBE `analytics`.`events`" {
|
||||
t.Fatalf("unexpected DESCRIBE query: %q", query)
|
||||
}
|
||||
return newScriptedRows(
|
||||
ctx,
|
||||
[]string{"col_name", "data_type", "comment"},
|
||||
[]string{"STRING", "STRING", "STRING"},
|
||||
[][]driver.Value{{"name", "string", "显示名称"}, {"amount", "decimal(18,2)", "含税金额"}},
|
||||
), nil
|
||||
},
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
values, err := server.getColumns("analytics", "events")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(values) != 2 || values[0].Comment == nil || *values[0].Comment != "显示名称" || values[1].Comment == nil || *values[1].Comment != "含税金额" {
|
||||
t.Fatalf("DESCRIBE comments changed: %#v", values)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTableCommentAndTypeInfoUseHiveServerMetadata(t *testing.T) {
|
||||
behavior := &scriptedBehavior{
|
||||
getTables: func(_ context.Context, schema, table string, tableTypes []string) (gohive.MetadataResult, error) {
|
||||
return metadataResult(
|
||||
[]string{"TABLE_SCHEM", "TABLE_NAME", "TABLE_TYPE", "REMARKS"},
|
||||
[]driver.Value{schema, table, "TABLE", "table comment"},
|
||||
), nil
|
||||
},
|
||||
getTypeInfo: func(context.Context) (gohive.MetadataResult, error) {
|
||||
return metadataResult([]string{"TYPE_NAME"}, []driver.Value{"STRING"}, []driver.Value{"decimal"}, []driver.Value{"STRING"}), nil
|
||||
},
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
defer server.disconnect()
|
||||
comment, err := server.getTableComment("analytics", "events")
|
||||
if err != nil || comment == nil || *comment != "table comment" {
|
||||
t.Fatalf("unexpected table comment: %v, %v", comment, err)
|
||||
}
|
||||
types, err := server.listDataTypes()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(types, []string{"decimal", "string"}) {
|
||||
t.Fatalf("unexpected data types: %#v", types)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListTablesFallsBackToShowTablesAndViews(t *testing.T) {
|
||||
behavior := &scriptedBehavior{
|
||||
getTables: func(context.Context, string, string, []string) (gohive.MetadataResult, error) {
|
||||
return gohive.MetadataResult{}, errors.New("metadata unsupported")
|
||||
},
|
||||
query: func(ctx context.Context, query string) (driver.Rows, error) {
|
||||
switch query {
|
||||
case "SHOW TABLES IN `analytics`":
|
||||
return newScriptedRows(
|
||||
ctx,
|
||||
[]string{"tab_name"},
|
||||
[]string{"STRING"},
|
||||
[][]driver.Value{{"events"}, {"shared_name"}},
|
||||
), nil
|
||||
case "SHOW VIEWS IN `analytics`":
|
||||
return newScriptedRows(
|
||||
ctx,
|
||||
[]string{"view_name"},
|
||||
[]string{"STRING"},
|
||||
[][]driver.Value{{"events_view"}, {"shared_name"}},
|
||||
), nil
|
||||
default:
|
||||
t.Fatalf("unexpected fallback query: %q", query)
|
||||
return nil, errors.New("unexpected fallback query")
|
||||
}
|
||||
},
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
defer server.disconnect()
|
||||
values, err := server.listTables("analytics", metadataListConstraints{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expected := []tableInfo{
|
||||
{Name: "events", TableType: "TABLE"},
|
||||
{Name: "events_view", TableType: "VIEW"},
|
||||
{Name: "shared_name", TableType: "VIEW"},
|
||||
}
|
||||
if !reflect.DeepEqual(values, expected) {
|
||||
t.Fatalf("unexpected fallback tables: %#v", values)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListTablesKeepsShowTablesResultsWhenShowViewsIsUnsupported(t *testing.T) {
|
||||
behavior := &scriptedBehavior{
|
||||
getTables: func(context.Context, string, string, []string) (gohive.MetadataResult, error) {
|
||||
return gohive.MetadataResult{}, errors.New("metadata unsupported")
|
||||
},
|
||||
query: func(ctx context.Context, query string) (driver.Rows, error) {
|
||||
switch query {
|
||||
case "SHOW TABLES IN `analytics`":
|
||||
return newScriptedRows(ctx, []string{"tab_name"}, []string{"STRING"}, [][]driver.Value{{"events"}}), nil
|
||||
case "SHOW VIEWS IN `analytics`":
|
||||
return nil, errors.New("SHOW VIEWS is unsupported")
|
||||
default:
|
||||
t.Fatalf("unexpected fallback query: %q", query)
|
||||
return nil, errors.New("unexpected fallback query")
|
||||
}
|
||||
},
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
defer server.disconnect()
|
||||
|
||||
values, err := server.listTables("analytics", metadataListConstraints{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(values, []tableInfo{{Name: "events", TableType: "TABLE"}}) {
|
||||
t.Fatalf("unexpected fallback tables: %#v", values)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListTablesReturnsNonCapabilityShowViewsError(t *testing.T) {
|
||||
behavior := &scriptedBehavior{
|
||||
getTables: func(context.Context, string, string, []string) (gohive.MetadataResult, error) {
|
||||
return gohive.MetadataResult{}, errors.New("metadata unsupported")
|
||||
},
|
||||
query: func(ctx context.Context, query string) (driver.Rows, error) {
|
||||
switch query {
|
||||
case "SHOW TABLES IN `analytics`":
|
||||
return newScriptedRows(ctx, []string{"tab_name"}, []string{"STRING"}, [][]driver.Value{{"events"}}), nil
|
||||
case "SHOW VIEWS IN `analytics`":
|
||||
return nil, errors.New("permission denied for SHOW VIEWS")
|
||||
default:
|
||||
t.Fatalf("unexpected fallback query: %q", query)
|
||||
return nil, errors.New("unexpected fallback query")
|
||||
}
|
||||
},
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
defer server.disconnect()
|
||||
|
||||
_, err := server.listTables("analytics", metadataListConstraints{})
|
||||
if err == nil || !strings.Contains(err.Error(), "SHOW VIEWS fallback failed: permission denied") {
|
||||
t.Fatalf("unexpected mixed fallback error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShowViewsUnsupported(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
unsupported bool
|
||||
}{
|
||||
{name: "explicit unsupported", err: errors.New("SHOW VIEWS is unsupported"), unsupported: true},
|
||||
{name: "not supported", err: errors.New("SHOW VIEWS is not supported before Hive 2.2"), unsupported: true},
|
||||
{name: "old parser", err: errors.New("ParseException: syntax error at or near VIEWS"), unsupported: true},
|
||||
{name: "permission", err: errors.New("permission denied for SHOW VIEWS")},
|
||||
{name: "timeout", err: context.DeadlineExceeded},
|
||||
{name: "cancel", err: context.Canceled},
|
||||
{name: "authentication", err: errors.New("authentication failed")},
|
||||
{name: "unsupported authentication", err: errors.New("unsupported authentication mechanism")},
|
||||
{name: "transport", err: errors.New("transport is closed")},
|
||||
{name: "unsupported transport", err: errors.New("transport does not support SASL")},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if actual := showViewsUnsupported(test.err); actual != test.unsupported {
|
||||
t.Fatalf("showViewsUnsupported(%v) = %v, want %v", test.err, actual, test.unsupported)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestListTablesFallbackHonorsExplicitTableType(t *testing.T) {
|
||||
behavior := &scriptedBehavior{
|
||||
getTables: func(context.Context, string, string, []string) (gohive.MetadataResult, error) {
|
||||
return gohive.MetadataResult{}, errors.New("metadata unsupported")
|
||||
},
|
||||
query: func(ctx context.Context, query string) (driver.Rows, error) {
|
||||
if query != "SHOW TABLES IN `analytics`" {
|
||||
t.Fatalf("unexpected fallback query: %q", query)
|
||||
}
|
||||
return newScriptedRows(ctx, []string{"tab_name"}, []string{"STRING"}, [][]driver.Value{{"events"}}), nil
|
||||
},
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
defer server.disconnect()
|
||||
|
||||
values, err := server.listTables("analytics", metadataListConstraints{ObjectTypes: []string{"TABLE"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(values, []tableInfo{{Name: "events", TableType: "TABLE"}}) {
|
||||
t.Fatalf("unexpected fallback tables: %#v", values)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListViewsFallsBackToShowViews(t *testing.T) {
|
||||
behavior := &scriptedBehavior{
|
||||
getTables: func(context.Context, string, string, []string) (gohive.MetadataResult, error) {
|
||||
return gohive.MetadataResult{}, errors.New("metadata unsupported")
|
||||
},
|
||||
query: func(ctx context.Context, query string) (driver.Rows, error) {
|
||||
if query != "SHOW VIEWS IN `analytics`" {
|
||||
t.Fatalf("unexpected fallback query: %q", query)
|
||||
}
|
||||
return newScriptedRows(ctx, []string{"view_name"}, []string{"STRING"}, [][]driver.Value{{"events_view"}}), nil
|
||||
},
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
defer server.disconnect()
|
||||
values, err := server.listTables("analytics", metadataListConstraints{ObjectTypes: []string{"VIEW"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(values, []tableInfo{{Name: "events_view", TableType: "VIEW"}}) {
|
||||
t.Fatalf("unexpected fallback views: %#v", values)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListViewsReturnsFallbackErrorWhenShowViewsIsUnsupported(t *testing.T) {
|
||||
behavior := &scriptedBehavior{
|
||||
getTables: func(context.Context, string, string, []string) (gohive.MetadataResult, error) {
|
||||
return gohive.MetadataResult{}, errors.New("metadata unsupported")
|
||||
},
|
||||
query: func(_ context.Context, query string) (driver.Rows, error) {
|
||||
if query != "SHOW VIEWS IN `analytics`" {
|
||||
t.Fatalf("unexpected fallback query: %q", query)
|
||||
}
|
||||
return nil, errors.New("SHOW VIEWS is unsupported")
|
||||
},
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
defer server.disconnect()
|
||||
|
||||
_, err := server.listTables("analytics", metadataListConstraints{ObjectTypes: []string{"VIEW"}})
|
||||
if err == nil || !strings.Contains(err.Error(), "SHOW VIEWS fallback failed") {
|
||||
t.Fatalf("unexpected explicit view fallback error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func metadataResult(columns []string, rows ...[]driver.Value) gohive.MetadataResult {
|
||||
return gohive.MetadataResult{Columns: columns, Rows: rows}
|
||||
}
|
||||
|
||||
func TestListObjectsIncludesProceduresAndFunctionsFromSystemViews(t *testing.T) {
|
||||
proceduresQuery := "SELECT procedure_name FROM system.procedures_v WHERE lower(database_name) = lower('ods') AND lower(procedure_name) LIKE '%sp%' ORDER BY procedure_name"
|
||||
functionsQuery := "SELECT function_name FROM system.functions_v WHERE lower(database_name) = lower('ods') AND lower(function_name) LIKE '%sp%' ORDER BY function_name"
|
||||
behavior := &scriptedBehavior{
|
||||
query: func(_ context.Context, query string) (driver.Rows, error) {
|
||||
switch query {
|
||||
case proceduresQuery:
|
||||
return newScriptedRows(context.Background(), []string{"procedure_name"}, []string{"STRING"}, [][]driver.Value{
|
||||
{"sp_daily_etl"}, {"sp_hourly_agg"},
|
||||
}), nil
|
||||
case functionsQuery:
|
||||
return newScriptedRows(context.Background(), []string{"function_name"}, []string{"STRING"}, [][]driver.Value{
|
||||
{"fn_clean"},
|
||||
}), nil
|
||||
default:
|
||||
return nil, errors.New("unexpected query: " + query)
|
||||
}
|
||||
},
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
server.params.DatabaseType = "argo"
|
||||
defer server.disconnect()
|
||||
|
||||
values, err := server.listObjects("ods", "ods", metadataListConstraints{
|
||||
ObjectTypes: []string{"PROCEDURE", "FUNCTION"},
|
||||
Filter: "sp", // "sp" overlaps the procedure filter; both lists get "%sp%" applied
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("listObjects: %v", err)
|
||||
}
|
||||
t.Logf("values: %+v", values)
|
||||
// listObjects applies the same filter pattern to procedures and functions;
|
||||
// we passed "sp" so procedures match and functions don't (because the function
|
||||
// list returns rows regardless of filter — that is, listObjects calls each
|
||||
// listRoutines call with the same Filter). Verify both queries ran and
|
||||
// the procedure name is present.
|
||||
queries, _, _, _ := behavior.snapshot()
|
||||
t.Logf("queries count: %d", len(queries))
|
||||
for i, q := range queries {
|
||||
t.Logf("query[%d]: %q", i, q)
|
||||
}
|
||||
if len(queries) != 2 {
|
||||
t.Fatalf("expected exactly 2 routine queries, got %d: %v", len(queries), queries)
|
||||
}
|
||||
if values == nil {
|
||||
t.Fatal("expected non-nil object list")
|
||||
}
|
||||
names := make([]string, len(values))
|
||||
for i, v := range values {
|
||||
names[i] = v.Name + ":" + v.ObjectType
|
||||
}
|
||||
joined := strings.Join(names, ",")
|
||||
if !strings.Contains(joined, "sp_daily_etl:PROCEDURE") {
|
||||
t.Fatalf("expected procedures in result, got %v", values)
|
||||
}
|
||||
if !strings.Contains(joined, "fn_clean:FUNCTION") {
|
||||
t.Fatalf("expected functions in result, got %v", values)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetObjectSourceRoutesProceduresToSystemProceduresView(t *testing.T) {
|
||||
expectedSQL := "SELECT full_text FROM system.procedures_v WHERE lower(database_name) = lower('ods') AND procedure_name = 'sp_daily_etl'"
|
||||
behavior := &scriptedBehavior{
|
||||
query: func(_ context.Context, query string) (driver.Rows, error) {
|
||||
if query != expectedSQL {
|
||||
return nil, errors.New("unexpected query: " + query)
|
||||
}
|
||||
return newScriptedRows(context.Background(), []string{"full_text"}, []string{"STRING"}, [][]driver.Value{
|
||||
{"-- daily ETL pipeline"},
|
||||
{"INSERT OVERWRITE TABLE ods.daily_summary SELECT * FROM staging.events"},
|
||||
}), nil
|
||||
},
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
server.params.DatabaseType = "argo"
|
||||
defer server.disconnect()
|
||||
|
||||
result, _, err := server.dispatch("get_object_source", map[string]json.RawMessage{
|
||||
"schema": json.RawMessage(`"ods"`),
|
||||
"name": json.RawMessage(`"sp_daily_etl"`),
|
||||
"object_type": json.RawMessage(`"PROCEDURE"`),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
source, ok := result.(objectSource)
|
||||
if !ok {
|
||||
t.Fatalf("get_object_source returned %T instead of objectSource", result)
|
||||
}
|
||||
if source.Name != "sp_daily_etl" || source.ObjectType != "PROCEDURE" {
|
||||
t.Fatalf("unexpected object source metadata: %#v", source)
|
||||
}
|
||||
expected := "-- daily ETL pipeline\nINSERT OVERWRITE TABLE ods.daily_summary SELECT * FROM staging.events\n"
|
||||
if source.Source != expected {
|
||||
t.Fatalf("unexpected procedure source: %q", source.Source)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetObjectSourceRoutesFunctionsToSystemFunctionsView(t *testing.T) {
|
||||
expectedSQL := "SELECT full_text FROM system.functions_v WHERE lower(database_name) = lower('ods') AND function_name = 'fn_clean'"
|
||||
behavior := &scriptedBehavior{
|
||||
query: func(_ context.Context, query string) (driver.Rows, error) {
|
||||
if query != expectedSQL {
|
||||
return nil, errors.New("unexpected query: " + query)
|
||||
}
|
||||
return newScriptedRows(context.Background(), []string{"full_text"}, []string{"STRING"}, [][]driver.Value{
|
||||
{"-- cleanup helper"},
|
||||
}), nil
|
||||
},
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
server.params.DatabaseType = "argo"
|
||||
defer server.disconnect()
|
||||
|
||||
result, _, err := server.dispatch("get_object_source", map[string]json.RawMessage{
|
||||
"schema": json.RawMessage(`"ods"`),
|
||||
"name": json.RawMessage(`"fn_clean"`),
|
||||
"object_type": json.RawMessage(`"FUNCTION"`),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
source, ok := result.(objectSource)
|
||||
if !ok {
|
||||
t.Fatalf("get_object_source returned %T instead of objectSource", result)
|
||||
}
|
||||
if source.Source != "-- cleanup helper\n" {
|
||||
t.Fatalf("unexpected function source: %q", source.Source)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListRoutinesUsesDatabaseParameterWhenSchemaEmpty(t *testing.T) {
|
||||
proceduresQuery := "SELECT procedure_name FROM system.procedures_v WHERE lower(database_name) = lower('ods') AND lower(procedure_name) LIKE '%%' ORDER BY procedure_name"
|
||||
defaultQuery := "SELECT procedure_name FROM system.procedures_v WHERE lower(database_name) = lower('default') AND lower(procedure_name) LIKE '%%' ORDER BY procedure_name"
|
||||
behavior := &scriptedBehavior{
|
||||
query: func(_ context.Context, query string) (driver.Rows, error) {
|
||||
switch query {
|
||||
case defaultQuery:
|
||||
return newScriptedRows(context.Background(), []string{"procedure_name"}, []string{"STRING"}, [][]driver.Value{}), nil
|
||||
case proceduresQuery:
|
||||
return newScriptedRows(context.Background(), []string{"procedure_name"}, []string{"STRING"}, [][]driver.Value{
|
||||
{"sp_daily_etl"},
|
||||
}), nil
|
||||
default:
|
||||
return nil, errors.New("unexpected query: " + query)
|
||||
}
|
||||
},
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
server.params.DatabaseType = "argo"
|
||||
server.config.Database = "default"
|
||||
defer server.disconnect()
|
||||
|
||||
values, err := server.listObjects("ods", "", metadataListConstraints{
|
||||
ObjectTypes: []string{"PROCEDURE"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("listObjects: %v", err)
|
||||
}
|
||||
if len(values) != 1 || values[0].Name != "sp_daily_etl" {
|
||||
t.Fatalf("expected procedure from database parameter, got %+v", values)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListRoutinesDoesNotFallbackToConnectionDefaultForExplicitSchema(t *testing.T) {
|
||||
defaultQuery := "SELECT procedure_name FROM system.procedures_v WHERE lower(database_name) = lower('default') AND lower(procedure_name) LIKE '%%' ORDER BY procedure_name"
|
||||
behavior := &scriptedBehavior{
|
||||
query: func(_ context.Context, query string) (driver.Rows, error) {
|
||||
if query == defaultQuery {
|
||||
// Serving this row proves the driver fell back to the
|
||||
// connection default after the explicit schema came back empty.
|
||||
return newScriptedRows(context.Background(), []string{"procedure_name"}, []string{"STRING"}, [][]driver.Value{
|
||||
{"sp_should_not_leak"},
|
||||
}), nil
|
||||
}
|
||||
return newScriptedRows(context.Background(), []string{"procedure_name"}, []string{"STRING"}, [][]driver.Value{}), nil
|
||||
},
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
server.params.DatabaseType = "argo"
|
||||
server.config.Database = "default"
|
||||
defer server.disconnect()
|
||||
|
||||
values, err := server.listObjects("", "ods", metadataListConstraints{
|
||||
ObjectTypes: []string{"PROCEDURE"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("listObjects: %v", err)
|
||||
}
|
||||
if len(values) != 0 {
|
||||
t.Fatalf("expected no routines to leak from the connection default database, got %+v", values)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetObjectSourceUsesDatabaseParameterForRoutineSource(t *testing.T) {
|
||||
expectedSQL := "SELECT full_text FROM system.procedures_v WHERE lower(database_name) = lower('ods') AND procedure_name = 'sp_daily_etl'"
|
||||
behavior := &scriptedBehavior{
|
||||
query: func(_ context.Context, query string) (driver.Rows, error) {
|
||||
if query != expectedSQL {
|
||||
return nil, errors.New("unexpected query: " + query)
|
||||
}
|
||||
return newScriptedRows(context.Background(), []string{"full_text"}, []string{"STRING"}, [][]driver.Value{
|
||||
{"CREATE PROCEDURE sp_daily_etl() BEGIN SELECT 1; END"},
|
||||
}), nil
|
||||
},
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
server.params.DatabaseType = "argo"
|
||||
server.config.Database = "default"
|
||||
defer server.disconnect()
|
||||
|
||||
result, _, err := server.dispatch("get_object_source", map[string]json.RawMessage{
|
||||
"database": json.RawMessage(`"ods"`),
|
||||
"schema": json.RawMessage(`""`),
|
||||
"name": json.RawMessage(`"sp_daily_etl"`),
|
||||
"object_type": json.RawMessage(`"PROCEDURE"`),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
source, ok := result.(objectSource)
|
||||
if !ok {
|
||||
t.Fatalf("get_object_source returned %T instead of objectSource", result)
|
||||
}
|
||||
if source.Source != "CREATE PROCEDURE sp_daily_etl() BEGIN SELECT 1; END\n" {
|
||||
t.Fatalf("unexpected procedure source: %q", source.Source)
|
||||
}
|
||||
}
|
||||
|
||||
// Unlike hive-go (which gates routine listing on the connection's database_type so
|
||||
// vanilla Apache Hive never fires the catalog-view queries), the argo agent serves
|
||||
// 星环Argo (Transwarp ArgoDB) exclusively and must always query the routine views.
|
||||
func TestListObjectsQueriesRoutineViewsUnconditionally(t *testing.T) {
|
||||
behavior := &scriptedBehavior{
|
||||
query: func(ctx context.Context, query string) (driver.Rows, error) {
|
||||
switch {
|
||||
case strings.Contains(strings.ToUpper(query), "PROCEDURES_V"):
|
||||
return newScriptedRows(ctx, []string{"procedure_name"}, []string{"STRING"}, [][]driver.Value{{"sp_etl_log"}}), nil
|
||||
case strings.Contains(strings.ToUpper(query), "FUNCTIONS_V"):
|
||||
return newScriptedRows(ctx, []string{"function_name"}, []string{"STRING"}, nil), nil
|
||||
default:
|
||||
return newScriptedRows(ctx, []string{"tab_name"}, []string{"STRING"}, [][]driver.Value{{"events"}}), nil
|
||||
}
|
||||
},
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
server.params.DatabaseType = "argo"
|
||||
defer server.disconnect()
|
||||
|
||||
values, err := server.listObjects("ods", "ods", metadataListConstraints{
|
||||
ObjectTypes: []string{"PROCEDURE", "FUNCTION"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("listObjects: %v", err)
|
||||
}
|
||||
found := false
|
||||
for _, v := range values {
|
||||
if v.ObjectType == "PROCEDURE" && v.Name == "sp_etl_log" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("argo agent must surface routines regardless of database_type, got: %+v", values)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
|
||||
"github.com/t8y2/dbx/agents/go-common/gohive"
|
||||
)
|
||||
|
||||
type rpcError struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data *rpcErrorData `json:"data,omitempty"`
|
||||
}
|
||||
|
||||
type rpcErrorData struct {
|
||||
Category string `json:"category"`
|
||||
Retryable bool `json:"retryable"`
|
||||
SessionDisposition string `json:"sessionDisposition"`
|
||||
Stage string `json:"stage"`
|
||||
ContractVersion int `json:"contractVersion"`
|
||||
OperationOutcome string `json:"operationOutcome"`
|
||||
SQLState string `json:"sqlState,omitempty"`
|
||||
VendorCode int32 `json:"vendorCode,omitempty"`
|
||||
ExceptionClass string `json:"exceptionClass,omitempty"`
|
||||
AgentSessionID string `json:"agentSessionId,omitempty"`
|
||||
}
|
||||
|
||||
func classifyRPCError(method, agentSessionID string, err error) *rpcError {
|
||||
stage := rpcErrorStage(method)
|
||||
data := &rpcErrorData{
|
||||
Category: "protocol",
|
||||
Retryable: false,
|
||||
SessionDisposition: "keep",
|
||||
Stage: stage,
|
||||
ContractVersion: 1,
|
||||
OperationOutcome: rpcOperationOutcome(stage),
|
||||
ExceptionClass: safeRPCDiagnostic(fmt.Sprintf("%T", err), 160),
|
||||
AgentSessionID: strings.TrimSpace(agentSessionID),
|
||||
}
|
||||
|
||||
var hiveError *gohive.Error
|
||||
if errors.As(err, &hiveError) {
|
||||
data.Category = "sql"
|
||||
data.SQLState = safeRPCDiagnostic(hiveError.SQLState, 160)
|
||||
data.VendorCode = int32(hiveError.ErrorCode)
|
||||
data.ExceptionClass = "gohive.Error"
|
||||
} else if errors.Is(err, context.Canceled) {
|
||||
data.Category = "canceled"
|
||||
data.SessionDisposition = "quarantine"
|
||||
} else if errors.Is(err, context.DeadlineExceeded) || isTimeoutError(err) {
|
||||
data.Category = "timeout"
|
||||
data.Retryable = stage == "connect" || stage == "validate"
|
||||
data.SessionDisposition = "quarantine"
|
||||
} else if isConnectionError(err) {
|
||||
data.Category = "connection"
|
||||
data.Retryable = stage == "connect" || stage == "validate"
|
||||
if stage != "connect" {
|
||||
data.SessionDisposition = "quarantine"
|
||||
}
|
||||
} else if isHiveSQLError(err) {
|
||||
data.Category = "sql"
|
||||
}
|
||||
return &rpcError{Code: -1, Message: err.Error(), Data: data}
|
||||
}
|
||||
|
||||
func rpcErrorStage(method string) string {
|
||||
switch method {
|
||||
case "connect", "open_session", "test_connection":
|
||||
return "connect"
|
||||
case "validate_connection", "validate_session":
|
||||
return "validate"
|
||||
case "cancel_session":
|
||||
return "cancel"
|
||||
case "close_session", "disconnect", "close_query_session", "close_table_read_session", "shutdown":
|
||||
return "close"
|
||||
case "fetch_query_page", "fetch_table_read_page":
|
||||
return "fetch"
|
||||
case "handshake", "":
|
||||
return "request"
|
||||
default:
|
||||
return "execute"
|
||||
}
|
||||
}
|
||||
|
||||
func rpcOperationOutcome(stage string) string {
|
||||
if stage == "request" || stage == "connect" || stage == "validate" {
|
||||
return "not_started"
|
||||
}
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
func isTimeoutError(err error) bool {
|
||||
var timeout interface{ Timeout() bool }
|
||||
return errors.As(err, &timeout) && timeout.Timeout()
|
||||
}
|
||||
|
||||
func isConnectionError(err error) bool {
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, net.ErrClosed) || errors.Is(err, sql.ErrConnDone) {
|
||||
return true
|
||||
}
|
||||
var networkError *net.OpError
|
||||
if errors.As(err, &networkError) {
|
||||
return true
|
||||
}
|
||||
lower := strings.ToLower(err.Error())
|
||||
for _, marker := range []string{
|
||||
"connection refused", "connection reset", "broken pipe", "connection closed", "connection lost",
|
||||
"unexpected eof", "no route to host", "all hiveserver2 endpoints failed", "transport is not open",
|
||||
"socket is closed", "not open",
|
||||
} {
|
||||
if strings.Contains(lower, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isHiveSQLError(err error) bool {
|
||||
lower := strings.ToLower(err.Error())
|
||||
for _, marker := range []string{"semanticexception", "parseexception", "hive error", "sqlstate", "error while compiling statement"} {
|
||||
if strings.Contains(lower, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func safeRPCDiagnostic(value string, maxLength int) string {
|
||||
var result strings.Builder
|
||||
for _, char := range value {
|
||||
if result.Len() >= maxLength {
|
||||
break
|
||||
}
|
||||
if char >= 0x21 && char <= 0x7e {
|
||||
result.WriteRune(char)
|
||||
}
|
||||
}
|
||||
return result.String()
|
||||
}
|
||||
@@ -0,0 +1,529 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"database/sql/driver"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/t8y2/dbx/agents/go-common/gohive"
|
||||
)
|
||||
|
||||
func (server *server) validateConnection() error {
|
||||
connection, err := server.requireConnection()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), server.config.ConnectTimeout)
|
||||
defer cancel()
|
||||
return connection.PingContext(ctx)
|
||||
}
|
||||
|
||||
func (server *server) executeQuery(options queryOptions) (queryResult, error) {
|
||||
started := time.Now()
|
||||
if options.FetchSize <= 0 {
|
||||
options.FetchSize = server.effectiveFetchSize()
|
||||
}
|
||||
sqlText := trimStatementSQL(options.SQL)
|
||||
if sqlText == "" {
|
||||
return queryResult{}, errors.New("SQL is required")
|
||||
}
|
||||
maxRows := options.MaxRows
|
||||
if maxRows <= 0 {
|
||||
maxRows = defaultMaxRows
|
||||
}
|
||||
connection, err := server.requireConnection()
|
||||
if err != nil {
|
||||
return queryResult{}, err
|
||||
}
|
||||
|
||||
ctx, cancel := queryContext(options.TimeoutSecs)
|
||||
server.setActiveOperation(cancel)
|
||||
defer server.clearActiveOperation(cancel)
|
||||
if err := server.applySchemaContext(ctx, connection, effectiveSchema(options)); err != nil {
|
||||
return queryResult{}, err
|
||||
}
|
||||
rows, affected, hasResultSet, err := executeHiveStatement(ctx, connection, sqlText, options.FetchSize)
|
||||
if err != nil {
|
||||
return queryResult{}, err
|
||||
}
|
||||
if !hasResultSet {
|
||||
return queryResult{
|
||||
Columns: []string{},
|
||||
ColumnTypes: []string{},
|
||||
Rows: [][]any{},
|
||||
AffectedRows: affected,
|
||||
ExecutionTimeMS: time.Since(started).Milliseconds(),
|
||||
Truncated: false,
|
||||
}, nil
|
||||
}
|
||||
defer rows.Close()
|
||||
columns, columnTypes, err := queryColumnMetadata(rows)
|
||||
if err != nil {
|
||||
return queryResult{}, err
|
||||
}
|
||||
values, truncated, err := readSQLRows(rows, columnTypes, maxRows)
|
||||
if err != nil {
|
||||
return queryResult{}, err
|
||||
}
|
||||
return queryResult{
|
||||
Columns: columns,
|
||||
ColumnTypes: columnTypes,
|
||||
Rows: values,
|
||||
AffectedRows: 0,
|
||||
ExecutionTimeMS: time.Since(started).Milliseconds(),
|
||||
Truncated: truncated,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (server *server) executeQueryPage(options queryOptions, requestedPageSize int) (queryPageResult, error) {
|
||||
started := time.Now()
|
||||
if options.FetchSize <= 0 {
|
||||
options.FetchSize = server.effectiveFetchSize()
|
||||
}
|
||||
server.expireIdleQuerySessions(started)
|
||||
sqlText := trimStatementSQL(options.SQL)
|
||||
if sqlText == "" {
|
||||
return queryPageResult{}, errors.New("SQL is required")
|
||||
}
|
||||
pageSize := requestedPageSize
|
||||
if pageSize <= 0 {
|
||||
pageSize = defaultPageSize
|
||||
}
|
||||
maxRows := options.MaxRows
|
||||
if maxRows <= 0 {
|
||||
maxRows = defaultMaxRows
|
||||
}
|
||||
connection, err := server.requireConnection()
|
||||
if err != nil {
|
||||
return queryPageResult{}, err
|
||||
}
|
||||
ctx, cancel := queryContext(options.TimeoutSecs)
|
||||
server.setActiveOperation(cancel)
|
||||
if err := server.applySchemaContext(ctx, connection, effectiveSchema(options)); err != nil {
|
||||
server.clearActiveOperation(cancel)
|
||||
return queryPageResult{}, err
|
||||
}
|
||||
rows, affected, hasResultSet, err := executeHiveStatement(ctx, connection, sqlText, options.FetchSize)
|
||||
if err != nil {
|
||||
server.clearActiveOperation(cancel)
|
||||
return queryPageResult{}, err
|
||||
}
|
||||
if !hasResultSet {
|
||||
server.clearActiveOperation(cancel)
|
||||
return queryPageResult{
|
||||
Columns: []string{},
|
||||
ColumnTypes: []string{},
|
||||
Rows: [][]any{},
|
||||
AffectedRows: affected,
|
||||
ExecutionTimeMS: time.Since(started).Milliseconds(),
|
||||
Truncated: false,
|
||||
SessionID: nil,
|
||||
HasMore: false,
|
||||
}, nil
|
||||
}
|
||||
|
||||
columns, columnTypes, err := queryColumnMetadata(rows)
|
||||
if err != nil {
|
||||
_ = rows.Close()
|
||||
server.clearActiveOperation(cancel)
|
||||
return queryPageResult{}, err
|
||||
}
|
||||
server.nextSessionID++
|
||||
sessionID := fmt.Sprintf("hive-%d", server.nextSessionID)
|
||||
state := &querySession{
|
||||
rows: rows,
|
||||
columns: columns,
|
||||
columnTypes: columnTypes,
|
||||
remaining: maxRows,
|
||||
cancel: cancel,
|
||||
lastAccessed: started,
|
||||
}
|
||||
server.querySessions[sessionID] = state
|
||||
page, hasMore, truncated, err := server.readQuerySessionPage(ctx, state, pageSize)
|
||||
server.activeMu.Lock()
|
||||
if server.activeCancel != nil {
|
||||
server.activeCancel = nil
|
||||
}
|
||||
server.activeMu.Unlock()
|
||||
if err != nil {
|
||||
server.closeQuerySession(sessionID)
|
||||
return queryPageResult{}, err
|
||||
}
|
||||
if !hasMore {
|
||||
server.closeQuerySession(sessionID)
|
||||
return queryPageResult{
|
||||
Columns: columns,
|
||||
ColumnTypes: columnTypes,
|
||||
Rows: page,
|
||||
ExecutionTimeMS: time.Since(started).Milliseconds(),
|
||||
Truncated: truncated,
|
||||
SessionID: nil,
|
||||
HasMore: false,
|
||||
}, nil
|
||||
}
|
||||
return queryPageResult{
|
||||
Columns: columns,
|
||||
ColumnTypes: columnTypes,
|
||||
Rows: page,
|
||||
ExecutionTimeMS: time.Since(started).Milliseconds(),
|
||||
Truncated: false,
|
||||
SessionID: &sessionID,
|
||||
HasMore: true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (server *server) fetchQueryPage(sessionID string, requestedPageSize int) (queryPageResult, error) {
|
||||
server.expireIdleQuerySessions(time.Now())
|
||||
state := server.querySessions[sessionID]
|
||||
if state == nil {
|
||||
return queryPageResult{}, errors.New("query session not found")
|
||||
}
|
||||
pageSize := requestedPageSize
|
||||
if pageSize <= 0 {
|
||||
pageSize = defaultPageSize
|
||||
}
|
||||
ctx := context.Background()
|
||||
server.setActiveOperation(state.cancel)
|
||||
page, hasMore, truncated, err := server.readQuerySessionPage(ctx, state, pageSize)
|
||||
server.activeMu.Lock()
|
||||
server.activeCancel = nil
|
||||
server.activeMu.Unlock()
|
||||
if err != nil {
|
||||
server.closeQuerySession(sessionID)
|
||||
return queryPageResult{}, err
|
||||
}
|
||||
var resultSessionID *string
|
||||
if hasMore {
|
||||
resultSessionID = &sessionID
|
||||
} else {
|
||||
server.closeQuerySession(sessionID)
|
||||
}
|
||||
return queryPageResult{
|
||||
Columns: state.columns,
|
||||
ColumnTypes: state.columnTypes,
|
||||
Rows: page,
|
||||
Truncated: truncated,
|
||||
SessionID: resultSessionID,
|
||||
HasMore: hasMore,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (server *server) readQuerySessionPage(ctx context.Context, state *querySession, pageSize int) ([][]any, bool, bool, error) {
|
||||
state.lastAccessed = time.Now()
|
||||
values := make([][]any, 0, min(pageSize, state.remaining))
|
||||
if state.pending != nil && state.remaining > 0 {
|
||||
values = append(values, state.pending)
|
||||
state.pending = nil
|
||||
state.remaining--
|
||||
}
|
||||
for len(values) < pageSize && state.remaining > 0 {
|
||||
row, ok, err := nextSQLRow(state.rows, state.columnTypes)
|
||||
if err != nil {
|
||||
return nil, false, false, err
|
||||
}
|
||||
if !ok {
|
||||
return values, false, false, nil
|
||||
}
|
||||
values = append(values, row)
|
||||
state.remaining--
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, false, false, ctx.Err()
|
||||
default:
|
||||
}
|
||||
}
|
||||
if state.remaining == 0 {
|
||||
_, ok, err := nextSQLRow(state.rows, state.columnTypes)
|
||||
if err != nil {
|
||||
return nil, false, false, err
|
||||
}
|
||||
return values, false, ok, nil
|
||||
}
|
||||
row, ok, err := nextSQLRow(state.rows, state.columnTypes)
|
||||
if err != nil {
|
||||
return nil, false, false, err
|
||||
}
|
||||
if !ok {
|
||||
return values, false, false, nil
|
||||
}
|
||||
state.pending = row
|
||||
return values, true, false, nil
|
||||
}
|
||||
|
||||
func (server *server) closeQuerySession(sessionID string) bool {
|
||||
state := server.querySessions[sessionID]
|
||||
if state == nil {
|
||||
return false
|
||||
}
|
||||
delete(server.querySessions, sessionID)
|
||||
state.cancel()
|
||||
_ = state.rows.Close()
|
||||
return true
|
||||
}
|
||||
|
||||
func (server *server) closeAllQuerySessions() error {
|
||||
var failures []string
|
||||
for sessionID, state := range server.querySessions {
|
||||
delete(server.querySessions, sessionID)
|
||||
state.cancel()
|
||||
if err := state.rows.Close(); err != nil {
|
||||
failures = append(failures, fmt.Sprintf("%s: %v", sessionID, err))
|
||||
}
|
||||
}
|
||||
if len(failures) > 0 {
|
||||
return errors.New(strings.Join(failures, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (server *server) expireIdleQuerySessions(now time.Time) int {
|
||||
expired := make([]string, 0)
|
||||
for sessionID, state := range server.querySessions {
|
||||
if !state.lastAccessed.IsZero() && now.Sub(state.lastAccessed) >= querySessionIdleTime {
|
||||
expired = append(expired, sessionID)
|
||||
}
|
||||
}
|
||||
for _, sessionID := range expired {
|
||||
server.closeQuerySession(sessionID)
|
||||
}
|
||||
return len(expired)
|
||||
}
|
||||
|
||||
func (server *server) executeStatements(params map[string]json.RawMessage, transaction bool) (queryResult, error) {
|
||||
started := time.Now()
|
||||
statements := stringSliceParam(params, "statements")
|
||||
if len(statements) == 0 {
|
||||
return queryResult{}, errors.New("statements are required")
|
||||
}
|
||||
connection, err := server.requireConnection()
|
||||
if err != nil {
|
||||
return queryResult{}, err
|
||||
}
|
||||
ctx, cancel := queryContext(intParam(params, "timeoutSecs"))
|
||||
server.setActiveOperation(cancel)
|
||||
defer server.clearActiveOperation(cancel)
|
||||
if err := server.applySchemaContext(ctx, connection, firstNonEmpty(stringParam(params, "schema"), stringParam(params, "database"))); err != nil {
|
||||
return queryResult{}, err
|
||||
}
|
||||
|
||||
var affected int64
|
||||
if transaction {
|
||||
tx, beginErr := connection.BeginTx(ctx, nil)
|
||||
if beginErr == nil {
|
||||
for _, statement := range statements {
|
||||
trimmed := trimStatementSQL(statement)
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
result, execErr := tx.ExecContext(ctx, trimmed)
|
||||
if execErr != nil {
|
||||
_ = tx.Rollback()
|
||||
return queryResult{}, execErr
|
||||
}
|
||||
count, _ := result.RowsAffected()
|
||||
affected += max(count, 0)
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return queryResult{}, err
|
||||
}
|
||||
return emptyQueryResult(affected, started), nil
|
||||
}
|
||||
if !transactionUnsupported(beginErr) {
|
||||
return queryResult{}, beginErr
|
||||
}
|
||||
}
|
||||
for _, statement := range statements {
|
||||
trimmed := trimStatementSQL(statement)
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
result, err := connection.ExecContext(ctx, trimmed)
|
||||
if err != nil {
|
||||
return queryResult{}, err
|
||||
}
|
||||
count, _ := result.RowsAffected()
|
||||
affected += max(count, 0)
|
||||
}
|
||||
return emptyQueryResult(affected, started), nil
|
||||
}
|
||||
|
||||
func transactionUnsupported(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, sql.ErrTxDone) {
|
||||
return false
|
||||
}
|
||||
message := strings.ToLower(err.Error())
|
||||
return errors.Is(err, driver.ErrSkip) ||
|
||||
strings.Contains(message, "transactions are not supported") ||
|
||||
strings.Contains(message, "transaction is not supported") ||
|
||||
strings.Contains(message, "unsupported transaction") ||
|
||||
strings.Contains(message, "driver: skip fast-path")
|
||||
}
|
||||
|
||||
func emptyQueryResult(affected int64, started time.Time) queryResult {
|
||||
return queryResult{
|
||||
Columns: []string{},
|
||||
ColumnTypes: []string{},
|
||||
Rows: [][]any{},
|
||||
AffectedRows: affected,
|
||||
ExecutionTimeMS: time.Since(started).Milliseconds(),
|
||||
Truncated: false,
|
||||
}
|
||||
}
|
||||
|
||||
func (server *server) applySchemaContext(ctx context.Context, connection *sql.Conn, schema string) error {
|
||||
schema = strings.TrimSpace(schema)
|
||||
if schema == "" || strings.EqualFold(schema, server.config.Database) {
|
||||
return nil
|
||||
}
|
||||
_, err := connection.ExecContext(ctx, "USE "+quoteHiveIdentifier(schema))
|
||||
return err
|
||||
}
|
||||
|
||||
func queryColumnMetadata(rows *sql.Rows) ([]string, []string, error) {
|
||||
columns, err := rows.Columns()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
types, err := rows.ColumnTypes()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
columnTypes := make([]string, len(columns))
|
||||
for index := range columns {
|
||||
if index < len(types) {
|
||||
columnTypes[index] = strings.ToLower(strings.TrimSpace(types[index].DatabaseTypeName()))
|
||||
}
|
||||
}
|
||||
return columns, columnTypes, nil
|
||||
}
|
||||
|
||||
func readSQLRows(rows *sql.Rows, columnTypes []string, limit int) ([][]any, bool, error) {
|
||||
values := make([][]any, 0, min(limit, defaultFetchSize))
|
||||
for len(values) < limit {
|
||||
row, ok, err := nextSQLRow(rows, columnTypes)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if !ok {
|
||||
return values, false, nil
|
||||
}
|
||||
values = append(values, row)
|
||||
}
|
||||
_, ok, err := nextSQLRow(rows, columnTypes)
|
||||
return values, ok, err
|
||||
}
|
||||
|
||||
func nextSQLRow(rows *sql.Rows, columnTypes []string) ([]any, bool, error) {
|
||||
if !rows.Next() {
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return nil, false, nil
|
||||
}
|
||||
values := make([]any, len(columnTypes))
|
||||
targets := make([]any, len(values))
|
||||
for index := range values {
|
||||
targets[index] = &values[index]
|
||||
}
|
||||
if err := rows.Scan(targets...); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
for index, value := range values {
|
||||
values[index] = normalizeHiveValue(value, columnTypes[index])
|
||||
}
|
||||
return values, true, nil
|
||||
}
|
||||
|
||||
func normalizeHiveValue(value any, columnType string) any {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
switch typed := value.(type) {
|
||||
case []byte:
|
||||
return bytesToHex(typed)
|
||||
case time.Time:
|
||||
return formatHiveJDBCDateTime(typed, columnType)
|
||||
case fmt.Stringer:
|
||||
return typed.String()
|
||||
case string:
|
||||
return typed
|
||||
default:
|
||||
return fmt.Sprint(value)
|
||||
}
|
||||
}
|
||||
|
||||
func formatHiveJDBCDateTime(value time.Time, columnType string) string {
|
||||
if strings.EqualFold(strings.TrimSpace(columnType), "DATE") {
|
||||
return value.Format("2006-01-02")
|
||||
}
|
||||
base := value.Format("2006-01-02 15:04:05")
|
||||
if value.Nanosecond() == 0 {
|
||||
return base + ".0"
|
||||
}
|
||||
fraction := strings.TrimRight(fmt.Sprintf("%09d", value.Nanosecond()), "0")
|
||||
return base + "." + fraction
|
||||
}
|
||||
|
||||
func executeHiveStatement(
|
||||
ctx context.Context,
|
||||
connection *sql.Conn,
|
||||
sqlText string,
|
||||
fetchSize int,
|
||||
) (*sql.Rows, int64, bool, error) {
|
||||
ctx = gohive.WithFetchSize(ctx, fetchSize)
|
||||
rows, err := connection.QueryContext(ctx, sqlText)
|
||||
if err == nil {
|
||||
return rows, 0, true, nil
|
||||
}
|
||||
var nonQuery *gohive.NonQueryResult
|
||||
if errors.As(err, &nonQuery) {
|
||||
return nil, max(nonQuery.AffectedRows, 0), false, nil
|
||||
}
|
||||
return nil, 0, false, err
|
||||
}
|
||||
|
||||
func bytesToHex(value []byte) string {
|
||||
const digits = "0123456789abcdef"
|
||||
result := make([]byte, 2+len(value)*2)
|
||||
result[0] = '0'
|
||||
result[1] = 'x'
|
||||
for index, current := range value {
|
||||
result[2+index*2] = digits[current>>4]
|
||||
result[3+index*2] = digits[current&0x0f]
|
||||
}
|
||||
return string(result)
|
||||
}
|
||||
|
||||
func queryContext(timeoutSecs int) (context.Context, context.CancelFunc) {
|
||||
if timeoutSecs > 0 {
|
||||
return context.WithTimeout(context.Background(), time.Duration(timeoutSecs)*time.Second)
|
||||
}
|
||||
return context.WithCancel(context.Background())
|
||||
}
|
||||
|
||||
func (server *server) effectiveFetchSize() int {
|
||||
if server.config.FetchSize > 0 {
|
||||
return server.config.FetchSize
|
||||
}
|
||||
return defaultFetchSize
|
||||
}
|
||||
|
||||
func effectiveSchema(options queryOptions) string {
|
||||
return firstNonEmpty(options.Schema, options.Database)
|
||||
}
|
||||
|
||||
func trimStatementSQL(sqlText string) string {
|
||||
return strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(sqlText), ";"))
|
||||
}
|
||||
|
||||
func quoteHiveIdentifier(value string) string {
|
||||
return "`" + strings.ReplaceAll(value, "`", "``") + "`"
|
||||
}
|
||||
@@ -0,0 +1,623 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"database/sql/driver"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/t8y2/dbx/agents/go-common/gohive"
|
||||
)
|
||||
|
||||
var scriptedDriverSequence atomic.Uint64
|
||||
|
||||
type scriptedBehavior struct {
|
||||
mu sync.Mutex
|
||||
query func(context.Context, string) (driver.Rows, error)
|
||||
exec func(context.Context, string) (driver.Result, error)
|
||||
getSchemas func(context.Context, string) (gohive.MetadataResult, error)
|
||||
getTables func(context.Context, string, string, []string) (gohive.MetadataResult, error)
|
||||
getColumns func(context.Context, string, string, string) (gohive.MetadataResult, error)
|
||||
getTypeInfo func(context.Context) (gohive.MetadataResult, error)
|
||||
beginErr error
|
||||
queries []string
|
||||
executions []string
|
||||
beginCalls int
|
||||
closeCalls int
|
||||
}
|
||||
|
||||
func (behavior *scriptedBehavior) queryContext(ctx context.Context, query string) (driver.Rows, error) {
|
||||
behavior.mu.Lock()
|
||||
behavior.queries = append(behavior.queries, query)
|
||||
operation := behavior.query
|
||||
behavior.mu.Unlock()
|
||||
if operation == nil {
|
||||
return nil, fmt.Errorf("unexpected query: %s", query)
|
||||
}
|
||||
return operation(ctx, query)
|
||||
}
|
||||
|
||||
func (behavior *scriptedBehavior) execContext(ctx context.Context, query string) (driver.Result, error) {
|
||||
behavior.mu.Lock()
|
||||
behavior.executions = append(behavior.executions, query)
|
||||
operation := behavior.exec
|
||||
behavior.mu.Unlock()
|
||||
if operation == nil {
|
||||
return nil, fmt.Errorf("unexpected execution: %s", query)
|
||||
}
|
||||
return operation(ctx, query)
|
||||
}
|
||||
|
||||
func (behavior *scriptedBehavior) snapshot() (queries, executions []string, beginCalls, closeCalls int) {
|
||||
behavior.mu.Lock()
|
||||
defer behavior.mu.Unlock()
|
||||
return append([]string(nil), behavior.queries...), append([]string(nil), behavior.executions...), behavior.beginCalls, behavior.closeCalls
|
||||
}
|
||||
|
||||
type scriptedDriver struct {
|
||||
behavior *scriptedBehavior
|
||||
}
|
||||
|
||||
func (driverValue *scriptedDriver) Open(string) (driver.Conn, error) {
|
||||
return &scriptedConnection{behavior: driverValue.behavior}, nil
|
||||
}
|
||||
|
||||
type scriptedConnection struct {
|
||||
behavior *scriptedBehavior
|
||||
}
|
||||
|
||||
func (connection *scriptedConnection) Prepare(string) (driver.Stmt, error) {
|
||||
return nil, errors.New("prepared statements are not supported")
|
||||
}
|
||||
|
||||
func (connection *scriptedConnection) Close() error {
|
||||
connection.behavior.mu.Lock()
|
||||
connection.behavior.closeCalls++
|
||||
connection.behavior.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (connection *scriptedConnection) Begin() (driver.Tx, error) {
|
||||
connection.behavior.mu.Lock()
|
||||
connection.behavior.beginCalls++
|
||||
connection.behavior.mu.Unlock()
|
||||
if connection.behavior.beginErr != nil {
|
||||
return nil, connection.behavior.beginErr
|
||||
}
|
||||
return nil, driver.ErrSkip
|
||||
}
|
||||
|
||||
func (connection *scriptedConnection) BeginTx(context.Context, driver.TxOptions) (driver.Tx, error) {
|
||||
connection.behavior.mu.Lock()
|
||||
connection.behavior.beginCalls++
|
||||
connection.behavior.mu.Unlock()
|
||||
if connection.behavior.beginErr != nil {
|
||||
return nil, connection.behavior.beginErr
|
||||
}
|
||||
return nil, driver.ErrSkip
|
||||
}
|
||||
|
||||
func (connection *scriptedConnection) Ping(context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (connection *scriptedConnection) QueryContext(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
_ []driver.NamedValue,
|
||||
) (driver.Rows, error) {
|
||||
return connection.behavior.queryContext(ctx, query)
|
||||
}
|
||||
|
||||
func (connection *scriptedConnection) ExecContext(
|
||||
ctx context.Context,
|
||||
query string,
|
||||
_ []driver.NamedValue,
|
||||
) (driver.Result, error) {
|
||||
return connection.behavior.execContext(ctx, query)
|
||||
}
|
||||
|
||||
func (connection *scriptedConnection) GetHiveSchemas(ctx context.Context, pattern string) (gohive.MetadataResult, error) {
|
||||
if connection.behavior.getSchemas == nil {
|
||||
return gohive.MetadataResult{}, errors.New("GetSchemas unavailable")
|
||||
}
|
||||
return connection.behavior.getSchemas(ctx, pattern)
|
||||
}
|
||||
|
||||
func (connection *scriptedConnection) GetHiveTables(ctx context.Context, schema, table string, tableTypes []string) (gohive.MetadataResult, error) {
|
||||
if connection.behavior.getTables == nil {
|
||||
return gohive.MetadataResult{}, errors.New("GetTables unavailable")
|
||||
}
|
||||
return connection.behavior.getTables(ctx, schema, table, tableTypes)
|
||||
}
|
||||
|
||||
func (connection *scriptedConnection) GetHiveColumns(ctx context.Context, schema, table, column string) (gohive.MetadataResult, error) {
|
||||
if connection.behavior.getColumns == nil {
|
||||
return gohive.MetadataResult{}, errors.New("GetColumns unavailable")
|
||||
}
|
||||
return connection.behavior.getColumns(ctx, schema, table, column)
|
||||
}
|
||||
|
||||
func (connection *scriptedConnection) GetHiveTypeInfo(ctx context.Context) (gohive.MetadataResult, error) {
|
||||
if connection.behavior.getTypeInfo == nil {
|
||||
return gohive.MetadataResult{}, errors.New("GetTypeInfo unavailable")
|
||||
}
|
||||
return connection.behavior.getTypeInfo(ctx)
|
||||
}
|
||||
|
||||
type scriptedRows struct {
|
||||
ctx context.Context
|
||||
columns []string
|
||||
types []string
|
||||
values [][]driver.Value
|
||||
blockAfter int
|
||||
blocked chan struct{}
|
||||
blockOnce sync.Once
|
||||
|
||||
mu sync.Mutex
|
||||
index int
|
||||
closed bool
|
||||
}
|
||||
|
||||
func newScriptedRows(ctx context.Context, columns, types []string, values [][]driver.Value) *scriptedRows {
|
||||
return &scriptedRows{
|
||||
ctx: ctx,
|
||||
columns: columns,
|
||||
types: types,
|
||||
values: values,
|
||||
blockAfter: -1,
|
||||
}
|
||||
}
|
||||
|
||||
func (rows *scriptedRows) Columns() []string {
|
||||
return append([]string(nil), rows.columns...)
|
||||
}
|
||||
|
||||
func (rows *scriptedRows) Close() error {
|
||||
rows.mu.Lock()
|
||||
rows.closed = true
|
||||
rows.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (rows *scriptedRows) Next(destination []driver.Value) error {
|
||||
rows.mu.Lock()
|
||||
if rows.closed {
|
||||
rows.mu.Unlock()
|
||||
return io.EOF
|
||||
}
|
||||
index := rows.index
|
||||
if rows.blockAfter >= 0 && index >= rows.blockAfter {
|
||||
blocked := rows.blocked
|
||||
ctx := rows.ctx
|
||||
rows.mu.Unlock()
|
||||
if blocked != nil {
|
||||
rows.blockOnce.Do(func() { close(blocked) })
|
||||
}
|
||||
<-ctx.Done()
|
||||
return ctx.Err()
|
||||
}
|
||||
if index >= len(rows.values) {
|
||||
rows.mu.Unlock()
|
||||
return io.EOF
|
||||
}
|
||||
current := rows.values[index]
|
||||
rows.index++
|
||||
rows.mu.Unlock()
|
||||
copy(destination, current)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (rows *scriptedRows) ColumnTypeDatabaseTypeName(index int) string {
|
||||
if index < 0 || index >= len(rows.types) {
|
||||
return ""
|
||||
}
|
||||
return rows.types[index]
|
||||
}
|
||||
|
||||
func (rows *scriptedRows) isClosed() bool {
|
||||
rows.mu.Lock()
|
||||
defer rows.mu.Unlock()
|
||||
return rows.closed
|
||||
}
|
||||
|
||||
func newScriptedServer(t *testing.T, behavior *scriptedBehavior) *server {
|
||||
t.Helper()
|
||||
driverName := fmt.Sprintf("dbx-hive-scripted-%d", scriptedDriverSequence.Add(1))
|
||||
sql.Register(driverName, &scriptedDriver{behavior: behavior})
|
||||
database, err := sql.Open(driverName, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
connection, err := database.Conn(context.Background())
|
||||
if err != nil {
|
||||
database.Close()
|
||||
t.Fatal(err)
|
||||
}
|
||||
server := &server{
|
||||
config: connectionConfig{Database: defaultHiveDatabase, ConnectTimeout: time.Second},
|
||||
database: database,
|
||||
connection: connection,
|
||||
querySessions: map[string]*querySession{},
|
||||
}
|
||||
t.Cleanup(func() { _ = server.disconnect() })
|
||||
return server
|
||||
}
|
||||
|
||||
func rawParams(values map[string]any) map[string]json.RawMessage {
|
||||
result := make(map[string]json.RawMessage, len(values))
|
||||
for key, value := range values {
|
||||
encoded, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
result[key] = encoded
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func TestExecuteQueryUsesHiveServerResultSetSignal(t *testing.T) {
|
||||
behavior := &scriptedBehavior{}
|
||||
behavior.query = func(ctx context.Context, query string) (driver.Rows, error) {
|
||||
switch {
|
||||
case strings.HasPrefix(query, "WITH source AS"):
|
||||
return nil, &gohive.NonQueryResult{AffectedRows: 4}
|
||||
case strings.HasPrefix(query, "SET "):
|
||||
return newScriptedRows(ctx, []string{"set"}, []string{"STRING"}, [][]driver.Value{{"hive.exec.dynamic.partition=true"}}), nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected SQL: %s", query)
|
||||
}
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
|
||||
insertResult, err := server.executeQuery(queryOptions{
|
||||
SQL: "WITH source AS (SELECT 1) INSERT INTO target SELECT * FROM source",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if insertResult.AffectedRows != 4 || len(insertResult.Columns) != 0 {
|
||||
t.Fatalf("unexpected non-query result: %#v", insertResult)
|
||||
}
|
||||
|
||||
setResult, err := server.executeQuery(queryOptions{SQL: "SET hive.exec.dynamic.partition"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(setResult.Rows, [][]any{{"hive.exec.dynamic.partition=true"}}) {
|
||||
t.Fatalf("unexpected SET result: %#v", setResult.Rows)
|
||||
}
|
||||
queries, executions, _, _ := behavior.snapshot()
|
||||
if len(queries) != 2 || len(executions) != 0 {
|
||||
t.Fatalf("statements must use HS2 result-set signaling, queries=%v executions=%v", queries, executions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryResultsPreserveHiveServerLabelsWithoutDotSplitting(t *testing.T) {
|
||||
columns := []string{"id", "customer.id", "total + 1", "id"}
|
||||
columnTypes := []string{"BIGINT", "BIGINT", "DOUBLE", "BIGINT"}
|
||||
behavior := &scriptedBehavior{}
|
||||
behavior.query = func(ctx context.Context, _ string) (driver.Rows, error) {
|
||||
return newScriptedRows(ctx, columns, columnTypes, [][]driver.Value{{int64(1), int64(2), float64(3), int64(4)}, {int64(5), int64(6), float64(7), int64(8)}}), nil
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
|
||||
ordinary, err := server.executeQuery(queryOptions{SQL: "SELECT * FROM labels"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(ordinary.Columns, columns) || !reflect.DeepEqual(ordinary.ColumnTypes, []string{"bigint", "bigint", "double", "bigint"}) {
|
||||
t.Fatalf("ordinary metadata changed: %#v", ordinary)
|
||||
}
|
||||
if len(ordinary.Rows) != 2 || ordinary.Rows[0][0] != "1" || ordinary.Rows[0][3] != "4" {
|
||||
t.Fatalf("ordinary values changed: %#v", ordinary.Rows)
|
||||
}
|
||||
|
||||
first, err := server.executeQueryPage(queryOptions{SQL: "SELECT * FROM labels", MaxRows: 2}, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first.SessionID == nil || !first.HasMore || !reflect.DeepEqual(first.Columns, columns) {
|
||||
t.Fatalf("first page metadata changed: %#v", first)
|
||||
}
|
||||
second, err := server.fetchQueryPage(*first.SessionID, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if second.HasMore || second.SessionID != nil || !reflect.DeepEqual(second.Columns, columns) {
|
||||
t.Fatalf("cached page metadata changed: %#v", second)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPagedQueryPreservesDuplicateLeadingValuesAcrossPages(t *testing.T) {
|
||||
behavior := &scriptedBehavior{}
|
||||
behavior.query = func(ctx context.Context, _ string) (driver.Rows, error) {
|
||||
return newScriptedRows(
|
||||
ctx,
|
||||
[]string{"group_id", "row_id"},
|
||||
[]string{"BIGINT", "BIGINT"},
|
||||
[][]driver.Value{{int64(1), int64(101)}, {int64(1), int64(102)}, {int64(1), int64(103)}},
|
||||
), nil
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
|
||||
first, err := server.executeQueryPage(queryOptions{SQL: "SELECT group_id, row_id FROM repeated_values", MaxRows: 3}, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first.SessionID == nil || !first.HasMore {
|
||||
t.Fatalf("expected an open cursor after the first page: %#v", first)
|
||||
}
|
||||
second, err := server.fetchQueryPage(*first.SessionID, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
allRows := append(append([][]any{}, first.Rows...), second.Rows...)
|
||||
expected := [][]any{{"1", "101"}, {"1", "102"}, {"1", "103"}}
|
||||
if !reflect.DeepEqual(allRows, expected) || second.HasMore {
|
||||
t.Fatalf("duplicate leading values changed across cursor pages: %#v", allRows)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPagedQueryTruncatesAndPreservesLegacyJDBCValueSemantics(t *testing.T) {
|
||||
largeValue := strings.Repeat("x", 256*1024)
|
||||
createdAt := time.Date(2026, time.August, 11, 10, 11, 12, 345000000, time.UTC)
|
||||
var sourceRows *scriptedRows
|
||||
behavior := &scriptedBehavior{}
|
||||
behavior.query = func(ctx context.Context, _ string) (driver.Rows, error) {
|
||||
sourceRows = newScriptedRows(
|
||||
ctx,
|
||||
[]string{"id", "enabled", "payload", "created_at", "complex_value"},
|
||||
[]string{"BIGINT", "BOOLEAN", "BINARY", "TIMESTAMP", "ARRAY"},
|
||||
[][]driver.Value{
|
||||
{int64(1), true, []byte{0x00, 0xff}, createdAt, largeValue},
|
||||
{int64(2), false, []byte{0x10}, createdAt, "[1,2]"},
|
||||
{int64(3), true, []byte{}, createdAt, "map('a',1)"},
|
||||
{int64(4), true, []byte{0x01}, createdAt, "extra"},
|
||||
},
|
||||
)
|
||||
return sourceRows, nil
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
|
||||
first, err := server.executeQueryPage(queryOptions{SQL: "SELECT * FROM values", MaxRows: 3, FetchSize: 2}, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !first.HasMore || first.SessionID == nil || len(first.Rows) != 2 {
|
||||
t.Fatalf("unexpected first page: %#v", first)
|
||||
}
|
||||
if !reflect.DeepEqual(first.ColumnTypes, []string{"bigint", "boolean", "binary", "timestamp", "array"}) {
|
||||
t.Fatalf("unexpected column types: %#v", first.ColumnTypes)
|
||||
}
|
||||
if first.Rows[0][0] != "1" || first.Rows[0][1] != "true" || first.Rows[0][2] != "0x00ff" {
|
||||
t.Fatalf("primitive value types changed: %#v", first.Rows[0])
|
||||
}
|
||||
if first.Rows[0][3] != "2026-08-11 10:11:12.345" || first.Rows[0][4] != largeValue {
|
||||
t.Fatalf("timestamp or large value changed: %#v", first.Rows[0])
|
||||
}
|
||||
|
||||
second, err := server.fetchQueryPage(*first.SessionID, 2)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if second.HasMore || second.SessionID != nil || !second.Truncated || len(second.Rows) != 1 {
|
||||
t.Fatalf("unexpected final page: %#v", second)
|
||||
}
|
||||
if len(server.querySessions) != 0 || sourceRows == nil || !sourceRows.isClosed() {
|
||||
t.Fatalf("query session was not closed: sessions=%d rows=%#v", len(server.querySessions), sourceRows)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancelPagedFetchQuarantinesWithoutReplayingSQL(t *testing.T) {
|
||||
blocked := make(chan struct{})
|
||||
behavior := &scriptedBehavior{}
|
||||
behavior.query = func(ctx context.Context, _ string) (driver.Rows, error) {
|
||||
rows := newScriptedRows(ctx, []string{"id"}, []string{"BIGINT"}, [][]driver.Value{{int64(1)}, {int64(2)}, {int64(3)}})
|
||||
rows.blockAfter = 2
|
||||
rows.blocked = blocked
|
||||
return rows, nil
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
first, err := server.executeQueryPage(queryOptions{SQL: "SELECT id FROM slow_table", MaxRows: 10}, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first.SessionID == nil {
|
||||
t.Fatal("expected a paged query session")
|
||||
}
|
||||
|
||||
result := make(chan error, 1)
|
||||
go func() {
|
||||
_, fetchErr := server.fetchQueryPage(*first.SessionID, 1)
|
||||
result <- fetchErr
|
||||
}()
|
||||
select {
|
||||
case <-blocked:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("fetch did not reach the blocking row")
|
||||
}
|
||||
server.cancelActiveQuery()
|
||||
fetchErr := <-result
|
||||
if !errors.Is(fetchErr, context.Canceled) {
|
||||
t.Fatalf("expected cancellation, got %v", fetchErr)
|
||||
}
|
||||
rpcErr := classifyRPCError("fetch_query_page", "session-a", fetchErr)
|
||||
if rpcErr.Data.Category != "canceled" || rpcErr.Data.SessionDisposition != "quarantine" {
|
||||
t.Fatalf("unexpected cancellation classification: %#v", rpcErr)
|
||||
}
|
||||
queries, _, _, _ := behavior.snapshot()
|
||||
if len(queries) != 1 {
|
||||
t.Fatalf("SQL must never be replayed after cancellation: %v", queries)
|
||||
}
|
||||
if len(server.querySessions) != 0 {
|
||||
t.Fatalf("canceled query session was retained: %#v", server.querySessions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPagedFetchHonorsOriginalStatementTimeout(t *testing.T) {
|
||||
blocked := make(chan struct{})
|
||||
behavior := &scriptedBehavior{}
|
||||
behavior.query = func(ctx context.Context, _ string) (driver.Rows, error) {
|
||||
rows := newScriptedRows(ctx, []string{"id"}, []string{"BIGINT"}, [][]driver.Value{{int64(1)}, {int64(2)}, {int64(3)}})
|
||||
rows.blockAfter = 2
|
||||
rows.blocked = blocked
|
||||
return rows, nil
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
first, err := server.executeQueryPage(queryOptions{SQL: "SELECT id FROM slow_table", MaxRows: 10, TimeoutSecs: 1}, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first.SessionID == nil {
|
||||
t.Fatal("expected a paged query session")
|
||||
}
|
||||
|
||||
started := time.Now()
|
||||
_, fetchErr := server.fetchQueryPage(*first.SessionID, 1)
|
||||
if !errors.Is(fetchErr, context.DeadlineExceeded) {
|
||||
t.Fatalf("expected deadline exceeded, got %v", fetchErr)
|
||||
}
|
||||
if elapsed := time.Since(started); elapsed > 3*time.Second {
|
||||
t.Fatalf("statement timeout was not enforced promptly: %s", elapsed)
|
||||
}
|
||||
if len(server.querySessions) != 0 {
|
||||
t.Fatalf("timed out query session was retained: %#v", server.querySessions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRuntimeSessionsRemainIsolated(t *testing.T) {
|
||||
behaviorA := &scriptedBehavior{query: func(ctx context.Context, _ string) (driver.Rows, error) {
|
||||
return newScriptedRows(ctx, []string{"owner"}, []string{"STRING"}, [][]driver.Value{{"a"}}), nil
|
||||
}}
|
||||
behaviorB := &scriptedBehavior{query: func(ctx context.Context, _ string) (driver.Rows, error) {
|
||||
return newScriptedRows(ctx, []string{"owner"}, []string{"STRING"}, [][]driver.Value{{"b"}}), nil
|
||||
}}
|
||||
serverA := newScriptedServer(t, behaviorA)
|
||||
serverB := newScriptedServer(t, behaviorB)
|
||||
runtimeServer := newRuntimeServer()
|
||||
runtimeServer.sessions["a"] = &agentSession{server: serverA}
|
||||
runtimeServer.sessions["b"] = &agentSession{server: serverB}
|
||||
|
||||
resultA, _, err := runtimeServer.dispatch("execute_query", rawParams(map[string]any{
|
||||
"agentSessionId": "a",
|
||||
"sql": "SELECT owner",
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resultB, _, err := runtimeServer.dispatch("execute_query", rawParams(map[string]any{
|
||||
"agentSessionId": "b",
|
||||
"sql": "SELECT owner",
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resultA.(queryResult).Rows[0][0] != "a" || resultB.(queryResult).Rows[0][0] != "b" {
|
||||
t.Fatalf("session results crossed: a=%#v b=%#v", resultA, resultB)
|
||||
}
|
||||
if err := runtimeServer.closeSession("a"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := runtimeServer.session("a"); err == nil {
|
||||
t.Fatal("closed session a is still registered")
|
||||
}
|
||||
if _, err := runtimeServer.session("b"); err != nil {
|
||||
t.Fatalf("closing session a affected session b: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTransactionFallbackDoesNotReplayFailedStatements(t *testing.T) {
|
||||
behavior := &scriptedBehavior{}
|
||||
behavior.exec = func(_ context.Context, query string) (driver.Result, error) {
|
||||
if strings.Contains(query, "second") {
|
||||
return nil, io.EOF
|
||||
}
|
||||
return driver.RowsAffected(1), nil
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
_, err := server.executeStatements(rawParams(map[string]any{
|
||||
"statements": []string{"INSERT first", "INSERT second", "INSERT third"},
|
||||
}), true)
|
||||
if !errors.Is(err, io.EOF) {
|
||||
t.Fatalf("expected connection failure, got %v", err)
|
||||
}
|
||||
_, executions, beginCalls, _ := behavior.snapshot()
|
||||
if !reflect.DeepEqual(executions, []string{"INSERT first", "INSERT second"}) {
|
||||
t.Fatalf("failed transaction was replayed or continued: %v", executions)
|
||||
}
|
||||
if beginCalls == 0 {
|
||||
t.Fatal("transaction capability was not attempted before fallback")
|
||||
}
|
||||
rpcErr := classifyRPCError("execute_transaction", "session-a", err)
|
||||
if rpcErr.Data.Category != "connection" || rpcErr.Data.SessionDisposition != "quarantine" {
|
||||
t.Fatalf("unexpected connection failure classification: %#v", rpcErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTransactionDoesNotFallbackAfterOperationalBeginFailure(t *testing.T) {
|
||||
behavior := &scriptedBehavior{
|
||||
beginErr: errors.New("connection reset while beginning transaction"),
|
||||
exec: func(context.Context, string) (driver.Result, error) {
|
||||
return driver.RowsAffected(1), nil
|
||||
},
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
defer server.disconnect()
|
||||
params := map[string]json.RawMessage{
|
||||
"statements": json.RawMessage(`["INSERT INTO sample VALUES (1)"]`),
|
||||
}
|
||||
if _, err := server.executeStatements(params, true); err == nil || !strings.Contains(err.Error(), "connection reset") {
|
||||
t.Fatalf("expected begin failure, got %v", err)
|
||||
}
|
||||
_, executions, _, _ := behavior.snapshot()
|
||||
if len(executions) != 0 {
|
||||
t.Fatalf("statements must not execute after begin failure: %#v", executions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpireIdleQuerySessions(t *testing.T) {
|
||||
behavior := &scriptedBehavior{query: func(ctx context.Context, _ string) (driver.Rows, error) {
|
||||
return newScriptedRows(ctx, []string{"id"}, []string{"BIGINT"}, [][]driver.Value{{int64(1)}, {int64(2)}}), nil
|
||||
}}
|
||||
server := newScriptedServer(t, behavior)
|
||||
first, err := server.executeQueryPage(queryOptions{SQL: "SELECT id", MaxRows: 10}, 1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first.SessionID == nil {
|
||||
t.Fatal("expected a query session")
|
||||
}
|
||||
server.querySessions[*first.SessionID].lastAccessed = time.Now().Add(-querySessionIdleTime - time.Second)
|
||||
if expired := server.expireIdleQuerySessions(time.Now()); expired != 1 {
|
||||
t.Fatalf("unexpected expired session count: %d", expired)
|
||||
}
|
||||
if len(server.querySessions) != 0 {
|
||||
t.Fatalf("idle session was retained: %#v", server.querySessions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStructuredHiveErrorIncludesServerDiagnostics(t *testing.T) {
|
||||
err := &gohive.Error{
|
||||
Err: errors.New("compile failed"),
|
||||
Message: "SemanticException",
|
||||
ErrorCode: 40000,
|
||||
SQLState: "42000",
|
||||
}
|
||||
rpcErr := classifyRPCError("execute_query", "session-a", err)
|
||||
if rpcErr.Data.Category != "sql" || rpcErr.Data.SQLState != "42000" || rpcErr.Data.VendorCode != 40000 {
|
||||
t.Fatalf("unexpected Hive diagnostics: %#v", rpcErr)
|
||||
}
|
||||
if rpcErr.Data.SessionDisposition != "keep" || rpcErr.Data.OperationOutcome != "unknown" {
|
||||
t.Fatalf("unexpected SQL failure recovery hints: %#v", rpcErr)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,568 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"math/rand/v2"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/beltran/gosasl"
|
||||
"github.com/go-zookeeper/zk"
|
||||
)
|
||||
|
||||
const (
|
||||
zooKeeperProtocolVersion = int32(0)
|
||||
zooKeeperOpGetData = int32(4)
|
||||
zooKeeperOpGetChildren2 = int32(12)
|
||||
zooKeeperOpClose = int32(-11)
|
||||
zooKeeperOpSetAuth = int32(100)
|
||||
zooKeeperOpSASL = int32(102)
|
||||
zooKeeperMaxFrameSize = 16 << 20
|
||||
zooKeeperMaxSASLRounds = 8
|
||||
)
|
||||
|
||||
var errZooKeeperSessionClosedRequiresSASL = errors.New("ZooKeeper session closed because SASL authentication is required")
|
||||
|
||||
type zooKeeperSASLClient interface {
|
||||
Start() ([]byte, error)
|
||||
Step([]byte) ([]byte, error)
|
||||
Complete() bool
|
||||
Dispose()
|
||||
}
|
||||
|
||||
var newZooKeeperSASLClient = func(host string, config connectionConfig) (zooKeeperSASLClient, error) {
|
||||
service, options := zooKeeperGSSAPIOptions(config)
|
||||
mechanism, err := gosasl.NewGSSAPIMechanismWithOptions(service, options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return gosasl.NewSaslClient(host, mechanism), nil
|
||||
}
|
||||
|
||||
var dialZooKeeperConnection = func(address string, timeout time.Duration, tlsConfig *tls.Config) (net.Conn, error) {
|
||||
dialer := &net.Dialer{Timeout: timeout}
|
||||
if tlsConfig == nil {
|
||||
return dialer.Dial("tcp", address)
|
||||
}
|
||||
config := tlsConfig.Clone()
|
||||
if config.ServerName == "" {
|
||||
host, _, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse ZooKeeper TLS address %q: %w", address, err)
|
||||
}
|
||||
config.ServerName = host
|
||||
}
|
||||
return tls.DialWithDialer(dialer, "tcp", address, config)
|
||||
}
|
||||
|
||||
var shuffleZooKeeperServers = func(servers []string) {
|
||||
rand.Shuffle(len(servers), func(first, second int) {
|
||||
servers[first], servers[second] = servers[second], servers[first]
|
||||
})
|
||||
}
|
||||
|
||||
func zooKeeperGSSAPIOptions(config connectionConfig) (string, gosasl.GSSAPIOptions) {
|
||||
service := firstNonEmpty(config.ZooKeeperKerberos.Service, "zookeeper")
|
||||
options := gssapiOptionsFromKerberos(config.Kerberos)
|
||||
options.QOP = "auth"
|
||||
options.AuthorizationID = ""
|
||||
options.ServiceHost = ""
|
||||
options.CanonicalizeHost = config.ZooKeeperKerberos.CanonicalHostname
|
||||
options.ServerName = config.ZooKeeperKerberos.ServerPrincipal
|
||||
if options.ServerName == "" && config.ZooKeeperKerberos.Realm != "" {
|
||||
options.ServerName = service + "/_HOST@" + config.ZooKeeperKerberos.Realm
|
||||
}
|
||||
return service, options
|
||||
}
|
||||
|
||||
func connectKerberosZooKeeper(
|
||||
servers []string,
|
||||
timeout time.Duration,
|
||||
tlsConfig *tls.Config,
|
||||
config connectionConfig,
|
||||
) (zooKeeperClient, <-chan zk.Event, error) {
|
||||
if len(servers) == 0 {
|
||||
return nil, nil, errors.New("ZooKeeper server list is empty")
|
||||
}
|
||||
if !config.Kerberos.Enabled {
|
||||
return nil, nil, errors.New("ZooKeeper Kerberos SASL requires Hive Kerberos credentials")
|
||||
}
|
||||
ordered := append([]string(nil), servers...)
|
||||
shuffleZooKeeperServers(ordered)
|
||||
var failures []string
|
||||
for _, address := range ordered {
|
||||
host, _, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
failures = append(failures, fmt.Sprintf("%s: %v", address, err))
|
||||
continue
|
||||
}
|
||||
connection, err := dialZooKeeperConnection(address, timeout, tlsConfig)
|
||||
if err != nil {
|
||||
failures = append(failures, fmt.Sprintf("%s: %v", address, err))
|
||||
continue
|
||||
}
|
||||
client, err := newProtocolZooKeeperClient(connection, timeout)
|
||||
if err == nil {
|
||||
var saslClient zooKeeperSASLClient
|
||||
saslClient, err = newZooKeeperSASLClient(host, config)
|
||||
if err == nil {
|
||||
err = client.authenticateSASL(saslClient)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
connection.Close()
|
||||
failures = append(failures, fmt.Sprintf("%s: %v", address, err))
|
||||
continue
|
||||
}
|
||||
events := make(chan zk.Event, 1)
|
||||
events <- zk.Event{State: zk.StateHasSession, Server: address}
|
||||
close(events)
|
||||
return client, events, nil
|
||||
}
|
||||
return nil, nil, fmt.Errorf("connect and authenticate to ZooKeeper: %s", strings.Join(failures, "; "))
|
||||
}
|
||||
|
||||
type protocolZooKeeperClient struct {
|
||||
connection net.Conn
|
||||
timeout time.Duration
|
||||
xid int32
|
||||
mutex sync.Mutex
|
||||
closed bool
|
||||
}
|
||||
|
||||
func newProtocolZooKeeperClient(connection net.Conn, timeout time.Duration) (*protocolZooKeeperClient, error) {
|
||||
if connection == nil {
|
||||
return nil, errors.New("ZooKeeper connection is nil")
|
||||
}
|
||||
if timeout <= 0 {
|
||||
timeout = defaultConnectTimeout
|
||||
}
|
||||
client := &protocolZooKeeperClient{connection: connection, timeout: timeout}
|
||||
request := &zooKeeperEncoder{}
|
||||
request.int32(zooKeeperProtocolVersion)
|
||||
request.int64(0)
|
||||
request.int32(zooKeeperTimeoutMillis(timeout))
|
||||
request.int64(0)
|
||||
request.bytes(make([]byte, 16))
|
||||
if err := client.writeFrame(request.data()); err != nil {
|
||||
return nil, fmt.Errorf("send ZooKeeper connect request: %w", err)
|
||||
}
|
||||
response, err := client.readFrame()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read ZooKeeper connect response: %w", err)
|
||||
}
|
||||
decoder := newZooKeeperDecoder(response)
|
||||
if _, err := decoder.int32(); err != nil {
|
||||
return nil, fmt.Errorf("decode ZooKeeper protocol version: %w", err)
|
||||
}
|
||||
if _, err := decoder.int32(); err != nil {
|
||||
return nil, fmt.Errorf("decode ZooKeeper session timeout: %w", err)
|
||||
}
|
||||
sessionID, err := decoder.int64()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decode ZooKeeper session ID: %w", err)
|
||||
}
|
||||
if _, err := decoder.bytes(); err != nil {
|
||||
return nil, fmt.Errorf("decode ZooKeeper session password: %w", err)
|
||||
}
|
||||
if sessionID == 0 {
|
||||
return nil, zk.ErrSessionExpired
|
||||
}
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func zooKeeperTimeoutMillis(timeout time.Duration) int32 {
|
||||
milliseconds := timeout.Milliseconds()
|
||||
if milliseconds < 1 {
|
||||
return 1
|
||||
}
|
||||
if milliseconds > math.MaxInt32 {
|
||||
return math.MaxInt32
|
||||
}
|
||||
return int32(milliseconds)
|
||||
}
|
||||
|
||||
func (client *protocolZooKeeperClient) authenticateSASL(saslClient zooKeeperSASLClient) error {
|
||||
if saslClient == nil {
|
||||
return errors.New("ZooKeeper SASL client is nil")
|
||||
}
|
||||
defer saslClient.Dispose()
|
||||
token, err := saslClient.Start()
|
||||
if err != nil {
|
||||
return fmt.Errorf("start ZooKeeper GSSAPI negotiation: %w", err)
|
||||
}
|
||||
for round := 0; round < zooKeeperMaxSASLRounds; round++ {
|
||||
response, requestErr := client.request(zooKeeperOpSASL, func(encoder *zooKeeperEncoder) {
|
||||
if token == nil {
|
||||
encoder.bytes([]byte{})
|
||||
return
|
||||
}
|
||||
encoder.bytes(token)
|
||||
})
|
||||
if requestErr != nil {
|
||||
return fmt.Errorf("ZooKeeper SASL round %d: %w", round+1, requestErr)
|
||||
}
|
||||
decoder := newZooKeeperDecoder(response)
|
||||
challenge, decodeErr := decoder.bytes()
|
||||
if decodeErr != nil {
|
||||
return fmt.Errorf("decode ZooKeeper SASL round %d: %w", round+1, decodeErr)
|
||||
}
|
||||
if saslClient.Complete() {
|
||||
if len(challenge) != 0 {
|
||||
return errors.New("ZooKeeper sent an unexpected token after GSSAPI completion")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
token, err = saslClient.Step(challenge)
|
||||
if err != nil {
|
||||
return fmt.Errorf("continue ZooKeeper GSSAPI negotiation at round %d: %w", round+1, err)
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("ZooKeeper GSSAPI negotiation exceeded %d rounds", zooKeeperMaxSASLRounds)
|
||||
}
|
||||
|
||||
func (client *protocolZooKeeperClient) AddAuth(scheme string, auth []byte) error {
|
||||
_, err := client.request(zooKeeperOpSetAuth, func(encoder *zooKeeperEncoder) {
|
||||
encoder.int32(0)
|
||||
encoder.string(scheme)
|
||||
encoder.bytes(auth)
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func (client *protocolZooKeeperClient) Children(path string) ([]string, *zk.Stat, error) {
|
||||
response, err := client.request(zooKeeperOpGetChildren2, func(encoder *zooKeeperEncoder) {
|
||||
encoder.string(path)
|
||||
encoder.boolean(false)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
decoder := newZooKeeperDecoder(response)
|
||||
children, err := decoder.strings()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
stat, err := decoder.stat()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return children, stat, nil
|
||||
}
|
||||
|
||||
func (client *protocolZooKeeperClient) Get(path string) ([]byte, *zk.Stat, error) {
|
||||
response, err := client.request(zooKeeperOpGetData, func(encoder *zooKeeperEncoder) {
|
||||
encoder.string(path)
|
||||
encoder.boolean(false)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
decoder := newZooKeeperDecoder(response)
|
||||
data, err := decoder.bytes()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
stat, err := decoder.stat()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return data, stat, nil
|
||||
}
|
||||
|
||||
func (client *protocolZooKeeperClient) Close() {
|
||||
client.mutex.Lock()
|
||||
defer client.mutex.Unlock()
|
||||
if client.closed {
|
||||
return
|
||||
}
|
||||
client.closed = true
|
||||
_ = client.connection.SetDeadline(time.Now().Add(client.timeout))
|
||||
client.xid++
|
||||
request := &zooKeeperEncoder{}
|
||||
request.int32(client.xid)
|
||||
request.int32(zooKeeperOpClose)
|
||||
_ = client.writeFrame(request.data())
|
||||
_ = client.connection.Close()
|
||||
}
|
||||
|
||||
func (client *protocolZooKeeperClient) request(opcode int32, encodeBody func(*zooKeeperEncoder)) ([]byte, error) {
|
||||
client.mutex.Lock()
|
||||
defer client.mutex.Unlock()
|
||||
if client.closed {
|
||||
return nil, zk.ErrConnectionClosed
|
||||
}
|
||||
client.xid++
|
||||
request := &zooKeeperEncoder{}
|
||||
request.int32(client.xid)
|
||||
request.int32(opcode)
|
||||
if encodeBody != nil {
|
||||
encodeBody(request)
|
||||
}
|
||||
if err := client.writeFrame(request.data()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
response, err := client.readFrame()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
decoder := newZooKeeperDecoder(response)
|
||||
xid, err := decoder.int32()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if xid != client.xid {
|
||||
return nil, fmt.Errorf("ZooKeeper response XID %d does not match request XID %d", xid, client.xid)
|
||||
}
|
||||
if _, err := decoder.int64(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
code, err := decoder.int32()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := zooKeeperError(code); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return decoder.remaining(), nil
|
||||
}
|
||||
|
||||
func (client *protocolZooKeeperClient) writeFrame(payload []byte) error {
|
||||
if len(payload) > zooKeeperMaxFrameSize {
|
||||
return fmt.Errorf("ZooKeeper request frame is %d bytes, maximum is %d", len(payload), zooKeeperMaxFrameSize)
|
||||
}
|
||||
if err := client.connection.SetWriteDeadline(time.Now().Add(client.timeout)); err != nil {
|
||||
return err
|
||||
}
|
||||
header := make([]byte, 4)
|
||||
binary.BigEndian.PutUint32(header, uint32(len(payload)))
|
||||
if err := writeAll(client.connection, header); err != nil {
|
||||
return err
|
||||
}
|
||||
return writeAll(client.connection, payload)
|
||||
}
|
||||
|
||||
func (client *protocolZooKeeperClient) readFrame() ([]byte, error) {
|
||||
if err := client.connection.SetReadDeadline(time.Now().Add(client.timeout)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
header := make([]byte, 4)
|
||||
if _, err := io.ReadFull(client.connection, header); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
length := int(binary.BigEndian.Uint32(header))
|
||||
if length < 0 || length > zooKeeperMaxFrameSize {
|
||||
return nil, fmt.Errorf("ZooKeeper response frame is %d bytes, maximum is %d", length, zooKeeperMaxFrameSize)
|
||||
}
|
||||
payload := make([]byte, length)
|
||||
if _, err := io.ReadFull(client.connection, payload); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return payload, nil
|
||||
}
|
||||
|
||||
func writeAll(writer io.Writer, payload []byte) error {
|
||||
for len(payload) > 0 {
|
||||
written, err := writer.Write(payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if written <= 0 {
|
||||
return io.ErrShortWrite
|
||||
}
|
||||
payload = payload[written:]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func zooKeeperError(code int32) error {
|
||||
switch code {
|
||||
case 0:
|
||||
return nil
|
||||
case -4:
|
||||
return zk.ErrConnectionClosed
|
||||
case -101:
|
||||
return zk.ErrNoNode
|
||||
case -102:
|
||||
return zk.ErrNoAuth
|
||||
case -112:
|
||||
return zk.ErrSessionExpired
|
||||
case -115:
|
||||
return zk.ErrAuthFailed
|
||||
case -124:
|
||||
return errZooKeeperSessionClosedRequiresSASL
|
||||
default:
|
||||
return fmt.Errorf("ZooKeeper request failed with error code %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
type zooKeeperEncoder struct {
|
||||
buffer bytes.Buffer
|
||||
}
|
||||
|
||||
func (encoder *zooKeeperEncoder) int32(value int32) {
|
||||
var data [4]byte
|
||||
binary.BigEndian.PutUint32(data[:], uint32(value))
|
||||
encoder.buffer.Write(data[:])
|
||||
}
|
||||
|
||||
func (encoder *zooKeeperEncoder) int64(value int64) {
|
||||
var data [8]byte
|
||||
binary.BigEndian.PutUint64(data[:], uint64(value))
|
||||
encoder.buffer.Write(data[:])
|
||||
}
|
||||
|
||||
func (encoder *zooKeeperEncoder) boolean(value bool) {
|
||||
if value {
|
||||
encoder.buffer.WriteByte(1)
|
||||
return
|
||||
}
|
||||
encoder.buffer.WriteByte(0)
|
||||
}
|
||||
|
||||
func (encoder *zooKeeperEncoder) string(value string) {
|
||||
encoder.bytes([]byte(value))
|
||||
}
|
||||
|
||||
func (encoder *zooKeeperEncoder) bytes(value []byte) {
|
||||
if value == nil {
|
||||
encoder.int32(-1)
|
||||
return
|
||||
}
|
||||
encoder.int32(int32(len(value)))
|
||||
encoder.buffer.Write(value)
|
||||
}
|
||||
|
||||
func (encoder *zooKeeperEncoder) data() []byte {
|
||||
return encoder.buffer.Bytes()
|
||||
}
|
||||
|
||||
type zooKeeperDecoder struct {
|
||||
data []byte
|
||||
offset int
|
||||
}
|
||||
|
||||
func newZooKeeperDecoder(data []byte) *zooKeeperDecoder {
|
||||
return &zooKeeperDecoder{data: data}
|
||||
}
|
||||
|
||||
func (decoder *zooKeeperDecoder) take(length int) ([]byte, error) {
|
||||
if length < 0 || decoder.offset > len(decoder.data)-length {
|
||||
return nil, io.ErrUnexpectedEOF
|
||||
}
|
||||
value := decoder.data[decoder.offset : decoder.offset+length]
|
||||
decoder.offset += length
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func (decoder *zooKeeperDecoder) int32() (int32, error) {
|
||||
value, err := decoder.take(4)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return int32(binary.BigEndian.Uint32(value)), nil
|
||||
}
|
||||
|
||||
func (decoder *zooKeeperDecoder) int64() (int64, error) {
|
||||
value, err := decoder.take(8)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return int64(binary.BigEndian.Uint64(value)), nil
|
||||
}
|
||||
|
||||
func (decoder *zooKeeperDecoder) bytes() ([]byte, error) {
|
||||
length, err := decoder.int32()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if length == -1 {
|
||||
return nil, nil
|
||||
}
|
||||
if length < -1 {
|
||||
return nil, fmt.Errorf("invalid ZooKeeper buffer length %d", length)
|
||||
}
|
||||
value, err := decoder.take(int(length))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append([]byte(nil), value...), nil
|
||||
}
|
||||
|
||||
func (decoder *zooKeeperDecoder) string() (string, error) {
|
||||
value, err := decoder.bytes()
|
||||
return string(value), err
|
||||
}
|
||||
|
||||
func (decoder *zooKeeperDecoder) strings() ([]string, error) {
|
||||
length, err := decoder.int32()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if length == -1 {
|
||||
return nil, nil
|
||||
}
|
||||
if length < -1 || length > zooKeeperMaxFrameSize/4 {
|
||||
return nil, fmt.Errorf("invalid ZooKeeper string vector length %d", length)
|
||||
}
|
||||
values := make([]string, 0, length)
|
||||
for index := int32(0); index < length; index++ {
|
||||
value, valueErr := decoder.string()
|
||||
if valueErr != nil {
|
||||
return nil, valueErr
|
||||
}
|
||||
values = append(values, value)
|
||||
}
|
||||
return values, nil
|
||||
}
|
||||
|
||||
func (decoder *zooKeeperDecoder) stat() (*zk.Stat, error) {
|
||||
stat := &zk.Stat{}
|
||||
var err error
|
||||
if stat.Czxid, err = decoder.int64(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if stat.Mzxid, err = decoder.int64(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if stat.Ctime, err = decoder.int64(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if stat.Mtime, err = decoder.int64(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if stat.Version, err = decoder.int32(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if stat.Cversion, err = decoder.int32(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if stat.Aversion, err = decoder.int32(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if stat.EphemeralOwner, err = decoder.int64(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if stat.DataLength, err = decoder.int32(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if stat.NumChildren, err = decoder.int32(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if stat.Pzxid, err = decoder.int64(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return stat, nil
|
||||
}
|
||||
|
||||
func (decoder *zooKeeperDecoder) remaining() []byte {
|
||||
return decoder.data[decoder.offset:]
|
||||
}
|
||||
@@ -0,0 +1,562 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-zookeeper/zk"
|
||||
gsskrb5 "github.com/golang-auth/go-gssapi/v2/krb5"
|
||||
"github.com/jcmturner/krb5test"
|
||||
)
|
||||
|
||||
type scriptedZooKeeperSASLClient struct {
|
||||
complete bool
|
||||
disposed bool
|
||||
}
|
||||
|
||||
func (client *scriptedZooKeeperSASLClient) Start() ([]byte, error) {
|
||||
return []byte("client-initial"), nil
|
||||
}
|
||||
|
||||
func (client *scriptedZooKeeperSASLClient) Step(challenge []byte) ([]byte, error) {
|
||||
if string(challenge) != "server-challenge" {
|
||||
return nil, fmt.Errorf("unexpected challenge %q", challenge)
|
||||
}
|
||||
client.complete = true
|
||||
return []byte("client-final"), nil
|
||||
}
|
||||
|
||||
func (client *scriptedZooKeeperSASLClient) Complete() bool { return client.complete }
|
||||
func (client *scriptedZooKeeperSASLClient) Dispose() { client.disposed = true }
|
||||
|
||||
func TestProtocolZooKeeperClientAuthenticatesAndReadsDiscoveryData(t *testing.T) {
|
||||
clientConnection, serverConnection := net.Pipe()
|
||||
serverErrors := make(chan error, 1)
|
||||
go func() {
|
||||
defer serverConnection.Close()
|
||||
serverErrors <- serveZooKeeperProtocolTest(serverConnection)
|
||||
}()
|
||||
|
||||
client, err := newProtocolZooKeeperClient(clientConnection, time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saslClient := &scriptedZooKeeperSASLClient{}
|
||||
if err := client.authenticateSASL(saslClient); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !saslClient.disposed {
|
||||
t.Fatal("SASL credentials were not disposed")
|
||||
}
|
||||
if err := client.AddAuth("digest", []byte("user:password")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
children, stat, err := client.Children("/hiveserver2")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(children) != 1 || children[0] != "server-1" || stat.NumChildren != 1 {
|
||||
t.Fatalf("children=%#v stat=%#v", children, stat)
|
||||
}
|
||||
data, stat, err := client.Get("/hiveserver2/server-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(data) != "serverUri=hs2.example.com:10000" || stat.DataLength != int32(len(data)) {
|
||||
t.Fatalf("data=%q stat=%#v", data, stat)
|
||||
}
|
||||
client.Close()
|
||||
if err := <-serverErrors; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProtocolZooKeeperClientMapsRequiredSASLError(t *testing.T) {
|
||||
clientConnection, serverConnection := net.Pipe()
|
||||
serverErrors := make(chan error, 1)
|
||||
go func() {
|
||||
defer serverConnection.Close()
|
||||
if err := acceptZooKeeperSession(serverConnection); err != nil {
|
||||
serverErrors <- err
|
||||
return
|
||||
}
|
||||
request, err := readZooKeeperTestFrame(serverConnection)
|
||||
if err != nil {
|
||||
serverErrors <- err
|
||||
return
|
||||
}
|
||||
decoder := newZooKeeperDecoder(request)
|
||||
xid, _ := decoder.int32()
|
||||
opcode, _ := decoder.int32()
|
||||
if opcode != zooKeeperOpSASL {
|
||||
serverErrors <- fmt.Errorf("opcode=%d", opcode)
|
||||
return
|
||||
}
|
||||
serverErrors <- writeZooKeeperTestResponse(serverConnection, xid, -124, nil)
|
||||
}()
|
||||
|
||||
client, err := newProtocolZooKeeperClient(clientConnection, time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saslClient := &scriptedZooKeeperSASLClient{}
|
||||
err = client.authenticateSASL(saslClient)
|
||||
client.Close()
|
||||
if !errors.Is(err, errZooKeeperSessionClosedRequiresSASL) {
|
||||
t.Fatalf("unexpected error %v", err)
|
||||
}
|
||||
if serverErr := <-serverErrors; serverErr != nil {
|
||||
t.Fatal(serverErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestZooKeeperGSSAPIOptionsDoNotReuseHiveServerIdentity(t *testing.T) {
|
||||
config := connectionConfig{
|
||||
Kerberos: kerberosConfig{
|
||||
ServerPrincipal: "hive/_HOST@HIVE.EXAMPLE.COM",
|
||||
ClientPrincipal: "alice@CLIENT.EXAMPLE.COM",
|
||||
AuthorizationID: "hive-proxy",
|
||||
QOP: "auth-conf",
|
||||
ConfigPath: "/etc/krb5.conf",
|
||||
CCachePath: "/tmp/alice.ccache",
|
||||
UseTicketCache: true,
|
||||
CanonicalHostname: true,
|
||||
},
|
||||
ZooKeeperKerberos: zooKeeperKerberosConfig{
|
||||
Enabled: true,
|
||||
Service: "zookeeper",
|
||||
Realm: "ZK.EXAMPLE.COM",
|
||||
CanonicalHostname: false,
|
||||
},
|
||||
}
|
||||
service, options := zooKeeperGSSAPIOptions(config)
|
||||
if service != "zookeeper" || options.ServerName != "zookeeper/_HOST@ZK.EXAMPLE.COM" {
|
||||
t.Fatalf("service=%q options=%#v", service, options)
|
||||
}
|
||||
if options.QOP != "auth" || options.AuthorizationID != "" || options.CanonicalizeHost || options.Principal != "alice@CLIENT.EXAMPLE.COM" || options.CCachePath != "/tmp/alice.ccache" {
|
||||
t.Fatalf("unexpected ZooKeeper GSSAPI options: %#v", options)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnectKerberosZooKeeperFailsOverAndUsesTargetHost(t *testing.T) {
|
||||
previousDial := dialZooKeeperConnection
|
||||
previousFactory := newZooKeeperSASLClient
|
||||
previousShuffle := shuffleZooKeeperServers
|
||||
t.Cleanup(func() {
|
||||
dialZooKeeperConnection = previousDial
|
||||
newZooKeeperSASLClient = previousFactory
|
||||
shuffleZooKeeperServers = previousShuffle
|
||||
})
|
||||
shuffleZooKeeperServers = func([]string) {}
|
||||
|
||||
clientConnection, serverConnection := net.Pipe()
|
||||
serverErrors := make(chan error, 1)
|
||||
go func() {
|
||||
defer serverConnection.Close()
|
||||
if err := acceptZooKeeperSession(serverConnection); err != nil {
|
||||
serverErrors <- err
|
||||
return
|
||||
}
|
||||
if err := expectZooKeeperSASLRound(serverConnection, "client-initial", []byte("server-challenge")); err != nil {
|
||||
serverErrors <- err
|
||||
return
|
||||
}
|
||||
if err := expectZooKeeperSASLRound(serverConnection, "client-final", nil); err != nil {
|
||||
serverErrors <- err
|
||||
return
|
||||
}
|
||||
request, err := readZooKeeperTestFrame(serverConnection)
|
||||
if err != nil {
|
||||
serverErrors <- err
|
||||
return
|
||||
}
|
||||
decoder := newZooKeeperDecoder(request)
|
||||
_, _ = decoder.int32()
|
||||
opcode, _ := decoder.int32()
|
||||
if opcode != zooKeeperOpClose {
|
||||
serverErrors <- fmt.Errorf("unexpected close opcode %d", opcode)
|
||||
return
|
||||
}
|
||||
serverErrors <- nil
|
||||
}()
|
||||
|
||||
var dialed []string
|
||||
dialZooKeeperConnection = func(address string, _ time.Duration, _ *tls.Config) (net.Conn, error) {
|
||||
dialed = append(dialed, address)
|
||||
if address == "first.example.com:2181" {
|
||||
return nil, errors.New("first unavailable")
|
||||
}
|
||||
return clientConnection, nil
|
||||
}
|
||||
var targetHost string
|
||||
newZooKeeperSASLClient = func(host string, _ connectionConfig) (zooKeeperSASLClient, error) {
|
||||
targetHost = host
|
||||
return &scriptedZooKeeperSASLClient{}, nil
|
||||
}
|
||||
|
||||
config := connectionConfig{
|
||||
Kerberos: kerberosConfig{Enabled: true},
|
||||
ZooKeeperKerberos: zooKeeperKerberosConfig{Enabled: true},
|
||||
}
|
||||
client, events, err := connectKerberosZooKeeper(
|
||||
[]string{"first.example.com:2181", "second.example.com:2181"},
|
||||
time.Second,
|
||||
nil,
|
||||
config,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(dialed) != 2 || dialed[0] != "first.example.com:2181" || dialed[1] != "second.example.com:2181" {
|
||||
t.Fatalf("dialed = %#v", dialed)
|
||||
}
|
||||
if targetHost != "second.example.com" {
|
||||
t.Fatalf("target host = %q", targetHost)
|
||||
}
|
||||
event := <-events
|
||||
if event.State != zk.StateHasSession || event.Server != "second.example.com:2181" {
|
||||
t.Fatalf("event = %#v", event)
|
||||
}
|
||||
client.Close()
|
||||
if err := <-serverErrors; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestZooKeeperKerberosSASLWithMiniKDC(t *testing.T) {
|
||||
logger := log.New(io.Discard, "", 0)
|
||||
kdc, err := krb5test.NewKDC(map[string][]string{
|
||||
"alice": nil,
|
||||
"zookeeper/localhost": nil,
|
||||
}, logger)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kdc.KRB5Conf.LibDefaults.UDPPreferenceLimit = 1
|
||||
kdc.Start()
|
||||
defer kdc.Close()
|
||||
|
||||
directory := t.TempDir()
|
||||
configPath := filepath.Join(directory, "krb5.conf")
|
||||
keytabPath := filepath.Join(directory, "zookeeper.keytab")
|
||||
configContents := fmt.Sprintf(`[libdefaults]
|
||||
default_realm = %s
|
||||
dns_lookup_realm = false
|
||||
dns_lookup_kdc = false
|
||||
rdns = false
|
||||
udp_preference_limit = 1
|
||||
default_tgs_enctypes = aes256-cts-hmac-sha1-96
|
||||
default_tkt_enctypes = aes256-cts-hmac-sha1-96
|
||||
permitted_enctypes = aes256-cts-hmac-sha1-96
|
||||
|
||||
[realms]
|
||||
%s = {
|
||||
kdc = %s
|
||||
}
|
||||
`, kdc.Realm, kdc.Realm, kdc.TCPListener.Addr().String())
|
||||
if err := os.WriteFile(configPath, []byte(configContents), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
keytabBytes, err := kdc.Keytab.Marshal()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(keytabPath, keytabBytes, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("KRB5_KTNAME", keytabPath)
|
||||
t.Setenv("KRB5_CLIENT_KTNAME", keytabPath)
|
||||
|
||||
clientConnection, serverConnection := net.Pipe()
|
||||
serverErrors := make(chan error, 1)
|
||||
servicePrincipal := "zookeeper/localhost@" + kdc.Realm
|
||||
go func() {
|
||||
defer serverConnection.Close()
|
||||
serverErrors <- serveKerberosZooKeeperSASL(serverConnection, servicePrincipal)
|
||||
}()
|
||||
|
||||
protocolClient, err := newProtocolZooKeeperClient(clientConnection, 5*time.Second)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config := connectionConfig{
|
||||
Kerberos: kerberosConfig{
|
||||
Enabled: true,
|
||||
ClientPrincipal: "alice@" + kdc.Realm,
|
||||
ConfigPath: configPath,
|
||||
Password: kdc.Principals["alice"].Password,
|
||||
DisablePAFXFAST: true,
|
||||
},
|
||||
ZooKeeperKerberos: zooKeeperKerberosConfig{
|
||||
Enabled: true,
|
||||
Service: "zookeeper",
|
||||
ServerPrincipal: servicePrincipal,
|
||||
CanonicalHostname: false,
|
||||
},
|
||||
}
|
||||
saslClient, err := newZooKeeperSASLClient("localhost", config)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := protocolClient.authenticateSASL(saslClient); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
protocolClient.Close()
|
||||
if err := <-serverErrors; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func serveZooKeeperProtocolTest(connection net.Conn) error {
|
||||
if err := acceptZooKeeperSession(connection); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := expectZooKeeperSASLRound(connection, "client-initial", []byte("server-challenge")); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := expectZooKeeperSASLRound(connection, "client-final", nil); err != nil {
|
||||
return err
|
||||
}
|
||||
request, err := readZooKeeperTestFrame(connection)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
decoder := newZooKeeperDecoder(request)
|
||||
xid, _ := decoder.int32()
|
||||
opcode, _ := decoder.int32()
|
||||
authType, _ := decoder.int32()
|
||||
scheme, _ := decoder.string()
|
||||
auth, _ := decoder.bytes()
|
||||
if opcode != zooKeeperOpSetAuth || authType != 0 || scheme != "digest" || string(auth) != "user:password" {
|
||||
return fmt.Errorf("unexpected auth request opcode=%d type=%d scheme=%q auth=%q", opcode, authType, scheme, auth)
|
||||
}
|
||||
if err := writeZooKeeperTestResponse(connection, xid, 0, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
request, err = readZooKeeperTestFrame(connection)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
decoder = newZooKeeperDecoder(request)
|
||||
xid, _ = decoder.int32()
|
||||
opcode, _ = decoder.int32()
|
||||
path, _ := decoder.string()
|
||||
watch, _ := decoder.take(1)
|
||||
if opcode != zooKeeperOpGetChildren2 || path != "/hiveserver2" || !bytes.Equal(watch, []byte{0}) {
|
||||
return fmt.Errorf("unexpected children request opcode=%d path=%q watch=%v", opcode, path, watch)
|
||||
}
|
||||
body := &zooKeeperEncoder{}
|
||||
body.int32(1)
|
||||
body.string("server-1")
|
||||
encodeZooKeeperTestStat(body, 0, 1)
|
||||
if err := writeZooKeeperTestResponse(connection, xid, 0, body.data()); err != nil {
|
||||
return err
|
||||
}
|
||||
request, err = readZooKeeperTestFrame(connection)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
decoder = newZooKeeperDecoder(request)
|
||||
xid, _ = decoder.int32()
|
||||
opcode, _ = decoder.int32()
|
||||
path, _ = decoder.string()
|
||||
watch, _ = decoder.take(1)
|
||||
if opcode != zooKeeperOpGetData || path != "/hiveserver2/server-1" || !bytes.Equal(watch, []byte{0}) {
|
||||
return fmt.Errorf("unexpected get request opcode=%d path=%q watch=%v", opcode, path, watch)
|
||||
}
|
||||
data := []byte("serverUri=hs2.example.com:10000")
|
||||
body = &zooKeeperEncoder{}
|
||||
body.bytes(data)
|
||||
encodeZooKeeperTestStat(body, int32(len(data)), 0)
|
||||
if err := writeZooKeeperTestResponse(connection, xid, 0, body.data()); err != nil {
|
||||
return err
|
||||
}
|
||||
request, err = readZooKeeperTestFrame(connection)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
decoder = newZooKeeperDecoder(request)
|
||||
_, _ = decoder.int32()
|
||||
opcode, _ = decoder.int32()
|
||||
if opcode != zooKeeperOpClose {
|
||||
return fmt.Errorf("unexpected close opcode %d", opcode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func serveKerberosZooKeeperSASL(connection net.Conn, servicePrincipal string) error {
|
||||
if err := acceptZooKeeperSession(connection); err != nil {
|
||||
return err
|
||||
}
|
||||
acceptor := gsskrb5.NewKrb5Mech()
|
||||
if err := acceptor.Accept(servicePrincipal); err != nil {
|
||||
return err
|
||||
}
|
||||
xid, token, err := readZooKeeperSASLRequest(connection)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
apReply, err := acceptor.Continue(token)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !acceptor.IsEstablished() {
|
||||
return errors.New("Kerberos acceptor context was not established")
|
||||
}
|
||||
if err := writeZooKeeperSASLResponse(connection, xid, apReply); err != nil {
|
||||
return err
|
||||
}
|
||||
xid, token, err = readZooKeeperSASLRequest(connection)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(token) != 0 {
|
||||
return fmt.Errorf("expected empty post-AP-REP token, got %x", token)
|
||||
}
|
||||
securityChallenge, err := acceptor.Wrap([]byte{1, 0, 0, 0}, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := writeZooKeeperSASLResponse(connection, xid, securityChallenge); err != nil {
|
||||
return err
|
||||
}
|
||||
xid, token, err = readZooKeeperSASLRequest(connection)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
securityResponse, sealed, err := acceptor.Unwrap(token)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if sealed || len(securityResponse) < 4 || !bytes.Equal(securityResponse[:4], []byte{1, 0, 0, 0}) {
|
||||
return fmt.Errorf("invalid GSSAPI security-layer response sealed=%v payload=%x", sealed, securityResponse)
|
||||
}
|
||||
if len(securityResponse[4:]) == 0 {
|
||||
return errors.New("GSSAPI authorization identity is empty")
|
||||
}
|
||||
if err := writeZooKeeperSASLResponse(connection, xid, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
request, err := readZooKeeperTestFrame(connection)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
decoder := newZooKeeperDecoder(request)
|
||||
_, _ = decoder.int32()
|
||||
opcode, _ := decoder.int32()
|
||||
if opcode != zooKeeperOpClose {
|
||||
return fmt.Errorf("unexpected close opcode %d", opcode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func readZooKeeperSASLRequest(connection net.Conn) (int32, []byte, error) {
|
||||
request, err := readZooKeeperTestFrame(connection)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
decoder := newZooKeeperDecoder(request)
|
||||
xid, err := decoder.int32()
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
opcode, err := decoder.int32()
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
if opcode != zooKeeperOpSASL {
|
||||
return 0, nil, fmt.Errorf("unexpected SASL opcode %d", opcode)
|
||||
}
|
||||
token, err := decoder.bytes()
|
||||
return xid, token, err
|
||||
}
|
||||
|
||||
func writeZooKeeperSASLResponse(connection net.Conn, xid int32, token []byte) error {
|
||||
body := &zooKeeperEncoder{}
|
||||
body.bytes(token)
|
||||
return writeZooKeeperTestResponse(connection, xid, 0, body.data())
|
||||
}
|
||||
|
||||
func acceptZooKeeperSession(connection net.Conn) error {
|
||||
request, err := readZooKeeperTestFrame(connection)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
decoder := newZooKeeperDecoder(request)
|
||||
version, _ := decoder.int32()
|
||||
_, _ = decoder.int64()
|
||||
timeout, _ := decoder.int32()
|
||||
sessionID, _ := decoder.int64()
|
||||
password, _ := decoder.bytes()
|
||||
if version != 0 || timeout <= 0 || sessionID != 0 || len(password) != 16 {
|
||||
return fmt.Errorf("invalid connect request version=%d timeout=%d session=%d password=%d", version, timeout, sessionID, len(password))
|
||||
}
|
||||
response := &zooKeeperEncoder{}
|
||||
response.int32(0)
|
||||
response.int32(timeout)
|
||||
response.int64(42)
|
||||
response.bytes(make([]byte, 16))
|
||||
return writeZooKeeperTestFrame(connection, response.data())
|
||||
}
|
||||
|
||||
func expectZooKeeperSASLRound(connection net.Conn, wantToken string, responseToken []byte) error {
|
||||
request, err := readZooKeeperTestFrame(connection)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
decoder := newZooKeeperDecoder(request)
|
||||
xid, _ := decoder.int32()
|
||||
opcode, _ := decoder.int32()
|
||||
token, _ := decoder.bytes()
|
||||
if opcode != zooKeeperOpSASL || string(token) != wantToken {
|
||||
return fmt.Errorf("unexpected SASL request opcode=%d token=%q", opcode, token)
|
||||
}
|
||||
body := &zooKeeperEncoder{}
|
||||
body.bytes(responseToken)
|
||||
return writeZooKeeperTestResponse(connection, xid, 0, body.data())
|
||||
}
|
||||
|
||||
func encodeZooKeeperTestStat(encoder *zooKeeperEncoder, dataLength, children int32) {
|
||||
encoder.int64(1)
|
||||
encoder.int64(2)
|
||||
encoder.int64(3)
|
||||
encoder.int64(4)
|
||||
encoder.int32(5)
|
||||
encoder.int32(6)
|
||||
encoder.int32(7)
|
||||
encoder.int64(8)
|
||||
encoder.int32(dataLength)
|
||||
encoder.int32(children)
|
||||
encoder.int64(9)
|
||||
}
|
||||
|
||||
func writeZooKeeperTestResponse(connection net.Conn, xid, code int32, body []byte) error {
|
||||
response := &zooKeeperEncoder{}
|
||||
response.int32(xid)
|
||||
response.int64(0)
|
||||
response.int32(code)
|
||||
response.buffer.Write(body)
|
||||
return writeZooKeeperTestFrame(connection, response.data())
|
||||
}
|
||||
|
||||
func readZooKeeperTestFrame(connection net.Conn) ([]byte, error) {
|
||||
client := &protocolZooKeeperClient{connection: connection, timeout: time.Second}
|
||||
return client.readFrame()
|
||||
}
|
||||
|
||||
func writeZooKeeperTestFrame(connection net.Conn, payload []byte) error {
|
||||
client := &protocolZooKeeperClient{connection: connection, timeout: time.Second}
|
||||
return client.writeFrame(payload)
|
||||
}
|
||||
|
||||
var _ zooKeeperClient = (*protocolZooKeeperClient)(nil)
|
||||
var _ = zk.StateSaslAuthenticated
|
||||
@@ -0,0 +1,240 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
keystore "github.com/pavlo-v-chernykh/keystore-go/v4"
|
||||
pkcs12 "software.sslmate.com/src/go-pkcs12"
|
||||
)
|
||||
|
||||
func buildZooKeeperTLSConfig(values map[string]string) (*tls.Config, error) {
|
||||
if !parameterBool(values, "zookeepersslenable") {
|
||||
return nil, nil
|
||||
}
|
||||
config := &tls.Config{
|
||||
MinVersion: tls.VersionTLS12,
|
||||
ServerName: parameter(values, "zookeeperservername"),
|
||||
}
|
||||
trustStoreLocation := parameter(values, "zookeepertruststorelocation")
|
||||
if trustStoreLocation != "" {
|
||||
certificates, err := loadTrustStore(
|
||||
trustStoreLocation,
|
||||
parameter(values, "zookeepertruststorepassword"),
|
||||
parameter(values, "zookeepertruststoretype"),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load ZooKeeper truststore: %w", err)
|
||||
}
|
||||
pool := x509.NewCertPool()
|
||||
for _, certificate := range certificates {
|
||||
pool.AddCert(certificate)
|
||||
}
|
||||
config.RootCAs = pool
|
||||
}
|
||||
keyStoreLocation := parameter(values, "zookeeperkeystorelocation")
|
||||
if keyStoreLocation != "" {
|
||||
certificate, err := loadClientKeyStore(
|
||||
keyStoreLocation,
|
||||
parameter(values, "zookeeperkeystorepassword"),
|
||||
parameter(values, "zookeeperkeystoretype"),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load ZooKeeper keystore: %w", err)
|
||||
}
|
||||
config.Certificates = []tls.Certificate{certificate}
|
||||
}
|
||||
if parameterBool(values, "zookeepersslinsecureskipverify") {
|
||||
config.InsecureSkipVerify = true
|
||||
}
|
||||
return config, nil
|
||||
}
|
||||
|
||||
func loadTrustStore(path, password, storeType string) ([]*x509.Certificate, error) {
|
||||
contents, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch normalizedStoreType(storeType, path) {
|
||||
case "PEM":
|
||||
return parsePEMCertificates(contents)
|
||||
case "PKCS12":
|
||||
certificates, err := pkcs12.DecodeTrustStore(contents, password)
|
||||
if err == nil {
|
||||
return certificates, nil
|
||||
}
|
||||
_, certificate, chain, chainErr := pkcs12.DecodeChain(contents, password)
|
||||
if chainErr != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append([]*x509.Certificate{certificate}, chain...), nil
|
||||
case "JKS":
|
||||
store, err := loadJKS(contents, password)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var certificates []*x509.Certificate
|
||||
for _, alias := range store.Aliases() {
|
||||
switch {
|
||||
case store.IsTrustedCertificateEntry(alias):
|
||||
entry, getErr := store.GetTrustedCertificateEntry(alias)
|
||||
if getErr != nil {
|
||||
return nil, getErr
|
||||
}
|
||||
certificate, parseErr := x509.ParseCertificate(entry.Certificate.Content)
|
||||
if parseErr != nil {
|
||||
return nil, parseErr
|
||||
}
|
||||
certificates = append(certificates, certificate)
|
||||
case store.IsPrivateKeyEntry(alias):
|
||||
chain, getErr := store.GetPrivateKeyEntryCertificateChain(alias)
|
||||
if getErr != nil {
|
||||
return nil, getErr
|
||||
}
|
||||
for _, entry := range chain {
|
||||
certificate, parseErr := x509.ParseCertificate(entry.Content)
|
||||
if parseErr != nil {
|
||||
return nil, parseErr
|
||||
}
|
||||
certificates = append(certificates, certificate)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(certificates) == 0 {
|
||||
return nil, errors.New("JKS truststore contains no certificates")
|
||||
}
|
||||
return certificates, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported store type %q", storeType)
|
||||
}
|
||||
}
|
||||
|
||||
func loadClientKeyStore(path, password, storeType string) (tls.Certificate, error) {
|
||||
contents, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return tls.Certificate{}, err
|
||||
}
|
||||
switch normalizedStoreType(storeType, path) {
|
||||
case "PEM":
|
||||
return tls.X509KeyPair(contents, contents)
|
||||
case "PKCS12":
|
||||
privateKey, certificate, chain, err := pkcs12.DecodeChain(contents, password)
|
||||
if err != nil {
|
||||
return tls.Certificate{}, err
|
||||
}
|
||||
result := tls.Certificate{PrivateKey: privateKey, Leaf: certificate}
|
||||
result.Certificate = append(result.Certificate, certificate.Raw)
|
||||
for _, entry := range chain {
|
||||
result.Certificate = append(result.Certificate, entry.Raw)
|
||||
}
|
||||
return result, nil
|
||||
case "JKS":
|
||||
store, err := loadJKS(contents, password)
|
||||
if err != nil {
|
||||
return tls.Certificate{}, err
|
||||
}
|
||||
passwordBytes := []byte(password)
|
||||
defer clear(passwordBytes)
|
||||
for _, alias := range store.Aliases() {
|
||||
if !store.IsPrivateKeyEntry(alias) {
|
||||
continue
|
||||
}
|
||||
entry, getErr := store.GetPrivateKeyEntry(alias, passwordBytes)
|
||||
if getErr != nil {
|
||||
return tls.Certificate{}, getErr
|
||||
}
|
||||
privateKey, parseErr := parsePrivateKey(entry.PrivateKey)
|
||||
if parseErr != nil {
|
||||
return tls.Certificate{}, parseErr
|
||||
}
|
||||
result := tls.Certificate{PrivateKey: privateKey}
|
||||
for index, certificate := range entry.CertificateChain {
|
||||
result.Certificate = append(result.Certificate, certificate.Content)
|
||||
if index == 0 {
|
||||
result.Leaf, _ = x509.ParseCertificate(certificate.Content)
|
||||
}
|
||||
}
|
||||
if len(result.Certificate) == 0 {
|
||||
return tls.Certificate{}, errors.New("JKS private key entry has no certificate chain")
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
return tls.Certificate{}, errors.New("JKS keystore contains no private key entry")
|
||||
default:
|
||||
return tls.Certificate{}, fmt.Errorf("unsupported store type %q", storeType)
|
||||
}
|
||||
}
|
||||
|
||||
func normalizedStoreType(storeType, path string) string {
|
||||
value := strings.ToUpper(strings.TrimSpace(storeType))
|
||||
switch value {
|
||||
case "P12", "PFX", "PKCS#12":
|
||||
return "PKCS12"
|
||||
case "X509", "X.509":
|
||||
return "PEM"
|
||||
case "":
|
||||
switch strings.ToLower(filepath.Ext(path)) {
|
||||
case ".jks":
|
||||
return "JKS"
|
||||
case ".p12", ".pfx", ".pkcs12":
|
||||
return "PKCS12"
|
||||
default:
|
||||
return "PEM"
|
||||
}
|
||||
default:
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
func loadJKS(contents []byte, password string) (keystore.KeyStore, error) {
|
||||
store := keystore.New()
|
||||
passwordBytes := []byte(password)
|
||||
defer clear(passwordBytes)
|
||||
if err := store.Load(bytes.NewReader(contents), passwordBytes); err != nil {
|
||||
return keystore.KeyStore{}, err
|
||||
}
|
||||
return store, nil
|
||||
}
|
||||
|
||||
func parsePEMCertificates(contents []byte) ([]*x509.Certificate, error) {
|
||||
var certificates []*x509.Certificate
|
||||
for len(contents) > 0 {
|
||||
block, rest := pem.Decode(contents)
|
||||
if block == nil {
|
||||
break
|
||||
}
|
||||
contents = rest
|
||||
if block.Type != "CERTIFICATE" {
|
||||
continue
|
||||
}
|
||||
certificate, err := x509.ParseCertificate(block.Bytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
certificates = append(certificates, certificate)
|
||||
}
|
||||
if len(certificates) == 0 {
|
||||
return nil, errors.New("PEM truststore contains no certificates")
|
||||
}
|
||||
return certificates, nil
|
||||
}
|
||||
|
||||
func parsePrivateKey(contents []byte) (any, error) {
|
||||
if value, err := x509.ParsePKCS8PrivateKey(contents); err == nil {
|
||||
return value, nil
|
||||
}
|
||||
if value, err := x509.ParsePKCS1PrivateKey(contents); err == nil {
|
||||
return value, nil
|
||||
}
|
||||
if value, err := x509.ParseECPrivateKey(contents); err == nil {
|
||||
return value, nil
|
||||
}
|
||||
return nil, errors.New("unsupported private key encoding")
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/pem"
|
||||
"math/big"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
keystore "github.com/pavlo-v-chernykh/keystore-go/v4"
|
||||
pkcs12 "software.sslmate.com/src/go-pkcs12"
|
||||
)
|
||||
|
||||
func testZooKeeperCertificate(t *testing.T) (*rsa.PrivateKey, *x509.Certificate, []byte) {
|
||||
t.Helper()
|
||||
privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
template := &x509.Certificate{
|
||||
SerialNumber: big.NewInt(1),
|
||||
Subject: pkix.Name{CommonName: "zk.example.com"},
|
||||
DNSNames: []string{"zk.example.com"},
|
||||
NotBefore: time.Now().Add(-time.Hour),
|
||||
NotAfter: time.Now().Add(time.Hour),
|
||||
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
|
||||
IsCA: true,
|
||||
}
|
||||
raw, err := x509.CreateCertificate(rand.Reader, template, template, &privateKey.PublicKey, privateKey)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
certificate, err := x509.ParseCertificate(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return privateKey, certificate, raw
|
||||
}
|
||||
|
||||
func TestBuildZooKeeperTLSConfigFromPEM(t *testing.T) {
|
||||
_, _, raw := testZooKeeperCertificate(t)
|
||||
path := filepath.Join(t.TempDir(), "trust.pem")
|
||||
if err := os.WriteFile(path, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: raw}), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config, err := buildZooKeeperTLSConfig(map[string]string{
|
||||
"zookeepersslenable": "true",
|
||||
"zookeepertruststorelocation": path,
|
||||
"zookeepertruststoretype": "PEM",
|
||||
"zookeeperservername": "zk.example.com",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if config == nil || config.RootCAs == nil || config.ServerName != "zk.example.com" {
|
||||
t.Fatalf("unexpected TLS config: %#v", config)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildZooKeeperTLSConfigFromJKS(t *testing.T) {
|
||||
privateKey, certificate, raw := testZooKeeperCertificate(t)
|
||||
password := []byte("changeit")
|
||||
store := keystore.New()
|
||||
if err := store.SetTrustedCertificateEntry("ca", keystore.TrustedCertificateEntry{
|
||||
CreationTime: time.Now(),
|
||||
Certificate: keystore.Certificate{Type: "X509", Content: raw},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
encodedKey, err := x509.MarshalPKCS8PrivateKey(privateKey)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.SetPrivateKeyEntry("client", keystore.PrivateKeyEntry{
|
||||
CreationTime: time.Now(),
|
||||
PrivateKey: encodedKey,
|
||||
CertificateChain: []keystore.Certificate{
|
||||
{Type: "X509", Content: certificate.Raw},
|
||||
},
|
||||
}, password); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := filepath.Join(t.TempDir(), "client.jks")
|
||||
file, err := os.Create(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.Store(file, password); err != nil {
|
||||
file.Close()
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config, err := buildZooKeeperTLSConfig(map[string]string{
|
||||
"zookeepersslenable": "true",
|
||||
"zookeepertruststorelocation": path,
|
||||
"zookeepertruststorepassword": string(password),
|
||||
"zookeepertruststoretype": "JKS",
|
||||
"zookeeperkeystorelocation": path,
|
||||
"zookeeperkeystorepassword": string(password),
|
||||
"zookeeperkeystoretype": "JKS",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if config.RootCAs == nil || len(config.Certificates) != 1 || config.Certificates[0].PrivateKey == nil {
|
||||
t.Fatalf("unexpected JKS TLS config: %#v", config)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildZooKeeperTLSConfigFromPKCS12(t *testing.T) {
|
||||
privateKey, certificate, _ := testZooKeeperCertificate(t)
|
||||
password := "changeit"
|
||||
keyStore, err := pkcs12.Modern.Encode(privateKey, certificate, nil, password)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
trustStore, err := pkcs12.Modern.EncodeTrustStore([]*x509.Certificate{certificate}, password)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
directory := t.TempDir()
|
||||
keyPath := filepath.Join(directory, "client.p12")
|
||||
trustPath := filepath.Join(directory, "trust.p12")
|
||||
if err := os.WriteFile(keyPath, keyStore, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(trustPath, trustStore, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
config, err := buildZooKeeperTLSConfig(map[string]string{
|
||||
"zookeepersslenable": "true",
|
||||
"zookeepertruststorelocation": trustPath,
|
||||
"zookeepertruststorepassword": password,
|
||||
"zookeepertruststoretype": "PKCS12",
|
||||
"zookeeperkeystorelocation": keyPath,
|
||||
"zookeeperkeystorepassword": password,
|
||||
"zookeeperkeystoretype": "PKCS12",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if config.RootCAs == nil || len(config.Certificates) != 1 {
|
||||
t.Fatalf("unexpected PKCS12 TLS config: %#v", config)
|
||||
}
|
||||
}
|
||||
|
||||
func TestZooKeeperTLSRequiresExistingStore(t *testing.T) {
|
||||
if _, err := buildZooKeeperTLSConfig(map[string]string{
|
||||
"zookeepersslenable": "true",
|
||||
"zookeepertruststorelocation": filepath.Join(t.TempDir(), "missing.jks"),
|
||||
}); err == nil {
|
||||
t.Fatal("expected missing truststore error")
|
||||
}
|
||||
}
|
||||
@@ -1274,7 +1274,7 @@ public final class DamengAgent extends AbstractJdbcAgent {
|
||||
* <li>TRIGGER → ALL_TRIGGERS.TRIGGER_BODY</li>
|
||||
* <li>SEQUENCE → 由 ALL_SEQUENCES 元数据重建 CREATE SEQUENCE 语句</li>
|
||||
* <li>PROCEDURE/FUNCTION/PACKAGE/PACKAGE_BODY/TYPE/TYPE_BODY → ALL_SOURCE 按 LINE 拼接,
|
||||
* 再以 SYS.SYSOBJECTS + SYS.SYSTEXTS 作为最后一层</li>
|
||||
* 再以 SYS.SYSOBJECTS + SYS.SYSTEXTS(TXT/SEQNO) 作为最后一层</li>
|
||||
* </ul>
|
||||
* 所有字典来源都不可用时返回带原因说明的占位源码(不可编辑),避免双击对象查看源码直接报错;
|
||||
* 连接类错误(断连/超时等)不属于元数据不可用,继续上抛由上层处理会话。
|
||||
@@ -1418,9 +1418,7 @@ public final class DamengAgent extends AbstractJdbcAgent {
|
||||
try {
|
||||
String type = normalizeObjectSourceType(objectType);
|
||||
List<String> typeCandidates = switch (type) {
|
||||
// 不同 DM 版本对函数/过程的 TYPE 取值不一致(有的统一为 PROCEDURE),逐一尝试。
|
||||
case "FUNCTION" -> List.of("FUNCTION", "PROCEDURE");
|
||||
case "PROCEDURE" -> List.of("PROCEDURE", "FUNCTION");
|
||||
case "FUNCTION", "PROCEDURE" -> List.of("PROC", type);
|
||||
case "PACKAGE" -> List.of("PACKAGE");
|
||||
case "PACKAGE_BODY" -> List.of("PACKAGE BODY", "PACKAGE_BODY");
|
||||
case "TYPE" -> List.of("TYPE");
|
||||
@@ -1429,15 +1427,10 @@ public final class DamengAgent extends AbstractJdbcAgent {
|
||||
};
|
||||
for (String candidate : typeCandidates) {
|
||||
String source = readAllSourceLines(schema, name, candidate);
|
||||
if (notBlank(source)) {
|
||||
if (notBlank(source) && routineSourceMatchesRequestedType(schema, name, type, candidate, source)) {
|
||||
return catalogRoutineObjectSource(schema, name, objectType, source);
|
||||
}
|
||||
}
|
||||
// 兜底:忽略 TYPE 过滤按名称读取全部源码行(同一 schema 内对象名唯一)。
|
||||
String anyTypeSource = readAllSourceLines(schema, name, null);
|
||||
if (notBlank(anyTypeSource)) {
|
||||
return catalogRoutineObjectSource(schema, name, objectType, anyTypeSource);
|
||||
}
|
||||
return null;
|
||||
} catch (RuntimeException error) {
|
||||
return metadataTierError(error);
|
||||
@@ -1447,16 +1440,12 @@ public final class DamengAgent extends AbstractJdbcAgent {
|
||||
}
|
||||
|
||||
private String readAllSourceLines(String schema, String name, String sourceType) throws Exception {
|
||||
String sql = sourceType == null
|
||||
? "SELECT TEXT FROM ALL_SOURCE WHERE OWNER = ? AND NAME = ? ORDER BY LINE"
|
||||
: "SELECT TEXT FROM ALL_SOURCE WHERE OWNER = ? AND NAME = ? AND TYPE = ? ORDER BY LINE";
|
||||
String sql = "SELECT TEXT FROM ALL_SOURCE WHERE OWNER = ? AND NAME = ? AND TYPE = ? ORDER BY LINE";
|
||||
StringBuilder source = new StringBuilder();
|
||||
try (PreparedStatement stmt = requireConnected().prepareStatement(sql)) {
|
||||
stmt.setString(1, schema);
|
||||
stmt.setString(2, name);
|
||||
if (sourceType != null) {
|
||||
stmt.setString(3, sourceType);
|
||||
}
|
||||
stmt.setString(3, sourceType);
|
||||
try (ResultSet rs = stmt.executeQuery()) {
|
||||
while (rs.next()) {
|
||||
String line = coalesce(readTextColumn(rs, 1));
|
||||
@@ -1470,6 +1459,66 @@ public final class DamengAgent extends AbstractJdbcAgent {
|
||||
return source.toString();
|
||||
}
|
||||
|
||||
private boolean routineSourceMatchesRequestedType(
|
||||
String schema,
|
||||
String name,
|
||||
String objectType,
|
||||
String sourceType,
|
||||
String source
|
||||
) {
|
||||
if (!("FUNCTION".equals(objectType) || "PROCEDURE".equals(objectType))
|
||||
|| !"PROC".equalsIgnoreCase(sourceType)) {
|
||||
return true;
|
||||
}
|
||||
String catalogType = readDamengRoutineObjectType(schema, name);
|
||||
return catalogType != null
|
||||
? objectType.equals(catalogType)
|
||||
: routineSourceStartsWithType(source, objectType);
|
||||
}
|
||||
|
||||
private String readDamengRoutineObjectType(String schema, String name) {
|
||||
String sql = """
|
||||
SELECT o.INFO1
|
||||
FROM SYS.SYSOBJECTS o
|
||||
JOIN SYS.SYSOBJECTS s ON s.ID = o.SCHID AND s.TYPE$ = 'SCH'
|
||||
WHERE o.SUBTYPE$ = 'PROC' AND s.NAME = ? AND o.NAME = ?
|
||||
""".stripIndent().trim();
|
||||
try (PreparedStatement stmt = requireConnected().prepareStatement(sql)) {
|
||||
stmt.setString(1, schema);
|
||||
stmt.setString(2, name);
|
||||
try (ResultSet rs = stmt.executeQuery()) {
|
||||
if (!rs.next()) {
|
||||
return null;
|
||||
}
|
||||
String info1 = rs.getString(1);
|
||||
return switch (info1 == null ? "" : info1.trim()) {
|
||||
case "0" -> "FUNCTION";
|
||||
case "1" -> "PROCEDURE";
|
||||
default -> "PROCEDURE";
|
||||
};
|
||||
}
|
||||
} catch (RuntimeException error) {
|
||||
if (isDamengConnectionError(error)) {
|
||||
throw error;
|
||||
}
|
||||
return null;
|
||||
} catch (Exception error) {
|
||||
if (isDamengConnectionError(error)) {
|
||||
throw new RuntimeException(error);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static boolean routineSourceStartsWithType(String source, String objectType) {
|
||||
String normalizedSource = source.stripLeading().toUpperCase(Locale.ROOT);
|
||||
String normalizedType = objectType.toUpperCase(Locale.ROOT);
|
||||
return startsWithRoutineDeclaration(normalizedSource, normalizedType)
|
||||
|| startsWithRoutineDeclaration(normalizedSource, "CREATE " + normalizedType)
|
||||
|| startsWithRoutineDeclaration(normalizedSource, "CREATE OR REPLACE " + normalizedType)
|
||||
|| startsWithRoutineDeclaration(normalizedSource, "ALTER " + normalizedType);
|
||||
}
|
||||
|
||||
private static ObjectSource catalogRoutineObjectSource(
|
||||
String schema,
|
||||
String name,
|
||||
@@ -1505,12 +1554,12 @@ public final class DamengAgent extends AbstractJdbcAgent {
|
||||
// DM 将过程/函数/包/类型的定义文本按行存放在 SYS.SYSTEXTS,
|
||||
// SYS.SYSOBJECTS 的 SCHID 关联所属 schema;适用于 ALL_SOURCE 不可用的实例。
|
||||
String sql = """
|
||||
SELECT t.TEXT
|
||||
SELECT t.TXT
|
||||
FROM SYS.SYSTEXTS t
|
||||
JOIN SYS.SYSOBJECTS o ON o.ID = t.ID
|
||||
JOIN SYS.SYSOBJECTS s ON s.ID = o.SCHID AND s.TYPE$ = 'SCH' AND s.NAME = ?
|
||||
WHERE o.NAME = ?
|
||||
ORDER BY t.LINE
|
||||
ORDER BY t.SEQNO
|
||||
""".stripIndent().trim();
|
||||
StringBuilder source = new StringBuilder();
|
||||
try (PreparedStatement stmt = requireConnected().prepareStatement(sql)) {
|
||||
@@ -1527,6 +1576,13 @@ public final class DamengAgent extends AbstractJdbcAgent {
|
||||
}
|
||||
}
|
||||
return notBlank(source.toString())
|
||||
&& routineSourceMatchesRequestedType(
|
||||
schema,
|
||||
name,
|
||||
normalizeObjectSourceType(objectType),
|
||||
"PROC",
|
||||
source.toString()
|
||||
)
|
||||
? catalogRoutineObjectSource(schema, name, objectType, source.toString())
|
||||
: null;
|
||||
} catch (RuntimeException error) {
|
||||
@@ -2098,7 +2154,8 @@ public final class DamengAgent extends AbstractJdbcAgent {
|
||||
|
||||
private static String buildUrl(ConnectParams params) {
|
||||
String connectionString = params.getConnection_string() == null ? "" : params.getConnection_string().trim();
|
||||
if (!connectionString.isEmpty()) {
|
||||
if (!connectionString.isEmpty()
|
||||
&& !connectionString.regionMatches(true, 0, "dm://", 0, "dm://".length())) {
|
||||
return connectionString;
|
||||
}
|
||||
String database = params.getDatabase() == null ? "" : params.getDatabase().trim();
|
||||
|
||||
+152
@@ -562,9 +562,92 @@ class DamengAgentMetadataTest {
|
||||
Assertions.assertTrue(source.isEditable());
|
||||
Assertions.assertEquals(1, sqls.stream().filter(sql -> sql.contains("DBMS_METADATA.GET_DDL")).count());
|
||||
Assertions.assertTrue(sqls.stream().anyMatch(sql -> sql.contains("FROM ALL_SOURCE")), String.join("\n", sqls));
|
||||
Assertions.assertTrue(sqls.stream().anyMatch(sql -> sql.contains("TYPE = ?")), String.join("\n", sqls));
|
||||
Assertions.assertTrue(sqls.stream().noneMatch(sql -> sql.contains("FROM ALL_SOURCE") && !sql.contains("TYPE = ?")), String.join("\n", sqls));
|
||||
Assertions.assertTrue(sqls.stream().noneMatch(sql -> sql.contains("SYS.SYSTEXTS")), String.join("\n", sqls));
|
||||
}
|
||||
|
||||
@Test
|
||||
void matchesDamengProcedureTypeWhenAllSourceUsesProc() {
|
||||
DamengAgent agent = new DamengAgent();
|
||||
List<String> sqls = new ArrayList<>();
|
||||
TestSupport.setPrivateConnection(
|
||||
agent,
|
||||
typedRoutineFallbackConnection(
|
||||
missingDbmsMetadataPackageError(),
|
||||
sqls,
|
||||
List.of(
|
||||
Arrays.asList("CREATE OR REPLACE PROCEDURE \"APP\".\"REBUILD_CACHE\" AS"),
|
||||
Arrays.asList("BEGIN NULL; END;")
|
||||
),
|
||||
null,
|
||||
List.of(),
|
||||
"1"
|
||||
)
|
||||
);
|
||||
|
||||
ObjectSource source = agent.getObjectSource("APP", "REBUILD_CACHE", "PROCEDURE");
|
||||
|
||||
Assertions.assertTrue(source.getSource().contains("CREATE OR REPLACE PROCEDURE"), source.getSource());
|
||||
Assertions.assertTrue(source.isEditable());
|
||||
Assertions.assertTrue(sqls.stream().anyMatch(sql -> sql.contains("FROM ALL_SOURCE") && sql.contains("TYPE = ?")), String.join("\n", sqls));
|
||||
Assertions.assertTrue(sqls.stream().anyMatch(sql -> sql.contains("SELECT o.INFO1")), String.join("\n", sqls));
|
||||
Assertions.assertTrue(sqls.stream().noneMatch(sql -> sql.contains("FROM ALL_SOURCE") && !sql.contains("TYPE = ?")), String.join("\n", sqls));
|
||||
}
|
||||
|
||||
@Test
|
||||
void matchesDamengFunctionTypeWhenAllSourceUsesProc() {
|
||||
DamengAgent agent = new DamengAgent();
|
||||
List<String> sqls = new ArrayList<>();
|
||||
TestSupport.setPrivateConnection(
|
||||
agent,
|
||||
typedRoutineFallbackConnection(
|
||||
missingDbmsMetadataPackageError(),
|
||||
sqls,
|
||||
List.of(
|
||||
Arrays.asList("CREATE OR REPLACE FUNCTION \"APP\".\"CALC_SCORE\"(V_ID INT) RETURN INT AS"),
|
||||
Arrays.asList("BEGIN RETURN V_ID * 2; END;")
|
||||
),
|
||||
null,
|
||||
List.of(),
|
||||
"0"
|
||||
)
|
||||
);
|
||||
|
||||
ObjectSource source = agent.getObjectSource("APP", "CALC_SCORE", "FUNCTION");
|
||||
|
||||
Assertions.assertTrue(source.getSource().contains("CREATE OR REPLACE FUNCTION"), source.getSource());
|
||||
Assertions.assertTrue(source.isEditable());
|
||||
Assertions.assertTrue(sqls.stream().anyMatch(sql -> sql.contains("SELECT o.INFO1")), String.join("\n", sqls));
|
||||
}
|
||||
|
||||
@Test
|
||||
void doesNotReturnProcedureSourceForFunctionFallback() {
|
||||
DamengAgent agent = new DamengAgent();
|
||||
List<String> sqls = new ArrayList<>();
|
||||
TestSupport.setPrivateConnection(
|
||||
agent,
|
||||
typedRoutineFallbackConnection(
|
||||
missingDbmsMetadataPackageError(),
|
||||
sqls,
|
||||
List.of(
|
||||
Arrays.asList("CREATE OR REPLACE PROCEDURE \"APP\".\"REBUILD_CACHE\" AS"),
|
||||
Arrays.asList("BEGIN NULL; END;")
|
||||
),
|
||||
null,
|
||||
List.of(),
|
||||
"1"
|
||||
)
|
||||
);
|
||||
|
||||
ObjectSource source = agent.getObjectSource("APP", "REBUILD_CACHE", "FUNCTION");
|
||||
|
||||
Assertions.assertFalse(source.getSource().contains("CREATE OR REPLACE PROCEDURE"), source.getSource());
|
||||
Assertions.assertFalse(source.isEditable());
|
||||
Assertions.assertEquals(2, sqls.stream().filter(sql -> sql.contains("FROM ALL_SOURCE")).count(), String.join("\n", sqls));
|
||||
Assertions.assertTrue(sqls.stream().noneMatch(sql -> sql.contains("FROM ALL_SOURCE") && !sql.contains("TYPE = ?")), String.join("\n", sqls));
|
||||
}
|
||||
|
||||
@Test
|
||||
void fallsBackToAllSourceTypeDdlWhenDbmsMetadataPackageIsMissing() {
|
||||
DamengAgent agent = new DamengAgent();
|
||||
@@ -731,6 +814,34 @@ class DamengAgentMetadataTest {
|
||||
Assertions.assertTrue(source.isEditable());
|
||||
Assertions.assertTrue(sqls.stream().anyMatch(sql -> sql.contains("FROM ALL_SOURCE")), String.join("\n", sqls));
|
||||
Assertions.assertTrue(sqls.stream().anyMatch(sql -> sql.contains("SYS.SYSTEXTS") && sql.contains("SYS.SYSOBJECTS")), String.join("\n", sqls));
|
||||
Assertions.assertTrue(sqls.stream().anyMatch(sql -> sql.contains("SELECT t.TXT") && sql.contains("ORDER BY t.SEQNO")), String.join("\n", sqls));
|
||||
Assertions.assertTrue(sqls.stream().noneMatch(sql -> sql.contains("SYS.SYSTEXTS") && (sql.contains("t.TEXT") || sql.contains("t.LINE"))), String.join("\n", sqls));
|
||||
}
|
||||
|
||||
@Test
|
||||
void doesNotReturnProcedureSourceForFunctionSysTextsFallback() {
|
||||
DamengAgent agent = new DamengAgent();
|
||||
List<String> sqls = new ArrayList<>();
|
||||
TestSupport.setPrivateConnection(
|
||||
agent,
|
||||
typedRoutineFallbackConnection(
|
||||
missingDbmsMetadataPackageError(),
|
||||
sqls,
|
||||
List.of(),
|
||||
missingCatalogViewError("ALL_SOURCE"),
|
||||
List.of(
|
||||
Arrays.asList("CREATE OR REPLACE PROCEDURE \"APP\".\"REBUILD_CACHE\" AS"),
|
||||
Arrays.asList("BEGIN NULL; END;")
|
||||
),
|
||||
"1"
|
||||
)
|
||||
);
|
||||
|
||||
ObjectSource source = agent.getObjectSource("APP", "REBUILD_CACHE", "FUNCTION");
|
||||
|
||||
Assertions.assertFalse(source.getSource().contains("CREATE OR REPLACE PROCEDURE"), source.getSource());
|
||||
Assertions.assertFalse(source.isEditable());
|
||||
Assertions.assertTrue(sqls.stream().anyMatch(sql -> sql.contains("SELECT t.TXT") && sql.contains("ORDER BY t.SEQNO")), String.join("\n", sqls));
|
||||
}
|
||||
|
||||
@Test
|
||||
@@ -1499,6 +1610,47 @@ class DamengAgentMetadataTest {
|
||||
});
|
||||
}
|
||||
|
||||
private static Connection typedRoutineFallbackConnection(
|
||||
SQLException dbmsMetadataError,
|
||||
List<String> sqls,
|
||||
List<List<Object>> allSourceRows,
|
||||
SQLException allSourceError,
|
||||
List<List<Object>> systemTextRows,
|
||||
String info1
|
||||
) {
|
||||
int[] allSourceQueryCount = {0};
|
||||
return proxy(Connection.class, (method, args) -> {
|
||||
String name = method.getName();
|
||||
if ("prepareStatement".equals(name)) {
|
||||
String sql = (String) args[0];
|
||||
sqls.add(sql);
|
||||
if (sql.contains("DBMS_METADATA.GET_DDL")) {
|
||||
return failingMetadataStatement(dbmsMetadataError);
|
||||
}
|
||||
if (sql.contains("SELECT o.INFO1")) {
|
||||
return metadataStatement(List.of(List.of(info1)));
|
||||
}
|
||||
if (sql.contains("FROM ALL_SOURCE")) {
|
||||
if (allSourceError != null) {
|
||||
return failingMetadataStatement(allSourceError);
|
||||
}
|
||||
return metadataStatement(allSourceQueryCount[0]++ == 0 ? allSourceRows : List.of());
|
||||
}
|
||||
if (sql.contains("SYS.SYSTEXTS")) {
|
||||
return metadataStatement(systemTextRows);
|
||||
}
|
||||
return metadataStatement(List.of());
|
||||
}
|
||||
if ("close".equals(name)) {
|
||||
return null;
|
||||
}
|
||||
if ("isClosed".equals(name)) {
|
||||
return false;
|
||||
}
|
||||
return defaultValue(method.getReturnType());
|
||||
});
|
||||
}
|
||||
|
||||
private static Connection restrictedCrossOwnerMetadataConnection(List<String> sqls) {
|
||||
return proxy(Connection.class, (method, args) -> {
|
||||
String name = method.getName();
|
||||
|
||||
@@ -21,6 +21,24 @@ class DamengAgentUrlTest {
|
||||
Assertions.assertEquals("jdbc:dm://127.0.0.1:5236/MAIN", url);
|
||||
}
|
||||
|
||||
@Test
|
||||
void rebuildsLegacyDbxUrlFromConnectionFields() throws Exception {
|
||||
ConnectParams params = new ConnectParams(
|
||||
"127.0.0.1",
|
||||
5236,
|
||||
"MAIN",
|
||||
"SYSDBA",
|
||||
"pwd",
|
||||
"",
|
||||
"dm://SYSDBA:SYSDBA@legacy-host:5236/OLD",
|
||||
false
|
||||
);
|
||||
|
||||
String url = invokeBuildUrl(params);
|
||||
|
||||
Assertions.assertEquals("jdbc:dm://127.0.0.1:5236/MAIN", url);
|
||||
}
|
||||
|
||||
@Test
|
||||
void usesCompleteCustomJdbcUrlWithoutRewritingIt() throws Exception {
|
||||
ConnectParams params = new ConnectParams(
|
||||
|
||||
@@ -31,6 +31,9 @@ public final class DatabendAgent extends ConfiguredJdbcAgent {
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
// databend 的 getSearchStringEscape() 返回 '\',但 getTables/getProcedures 把 _ 当字面量且忽略转义,
|
||||
// 对含 _ 的库名(如 my_db)转义后返回 0 行(#8114),故关闭 schema 通配符转义。
|
||||
false
|
||||
);
|
||||
|
||||
|
||||
Generated
+9
-1
@@ -1243,6 +1243,7 @@ dependencies = [
|
||||
"cbc 0.1.2",
|
||||
"chrono",
|
||||
"csv",
|
||||
"dbx-sqlite-worker",
|
||||
"deadpool-postgres",
|
||||
"encoding_rs",
|
||||
"flate2",
|
||||
@@ -1317,6 +1318,14 @@ dependencies = [
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dbx-sqlite-worker"
|
||||
version = "0.1.1"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "deadpool"
|
||||
version = "0.12.3"
|
||||
@@ -4528,7 +4537,6 @@ dependencies = [
|
||||
"borsh",
|
||||
"bytes",
|
||||
"num-traits",
|
||||
"postgres-types",
|
||||
"rand 0.8.7",
|
||||
"rkyv",
|
||||
"serde",
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
# etcd Java to Go migration parity
|
||||
|
||||
Status date: 2026-09-02.
|
||||
|
||||
The Java `EtcdAgent` (jetcd 0.8.x, 33 protocol methods) was replaced by this
|
||||
native Go agent. The migration is complete only where a capability is
|
||||
implemented, covered by automated tests, validated against a real etcd cluster,
|
||||
and wired into the native build and release path.
|
||||
|
||||
## Baseline
|
||||
|
||||
- DBX Java baseline: jetcd 0.8.x `EtcdAgent`; all protocol shapes, capability
|
||||
strings, and error signal texts were copied verbatim.
|
||||
- Go client: `go.etcd.io/etcd/client/v3` v3.7.1 over gRPC.
|
||||
- Capabilities (exact Java set, no `structured_error_v1`): `connect`,
|
||||
`test_connection`, `kv`, `kv_ttl`, `kv_cas`, `kv_list_values`, `kv_status`,
|
||||
`kv_history`, `etcd_compaction`, `etcd_defrag`, `etcd_watch`, `etcd_lease`,
|
||||
`etcd_auth`, `multi_session`.
|
||||
- Error signal strings (`ETCD_CAS_CONFLICT`, `ETCD_NOT_FOUND`,
|
||||
`ETCD_COMPACTED`, `ETCD_WATCH_LIMIT`, …) are matched verbatim because the
|
||||
host normalizes agent errors by message text
|
||||
(`agent_kv.rs::normalize_agent_kv_error`).
|
||||
- The shared `go-semver` module is vendored under `agents/go-common/go-semver`
|
||||
because the upstream `v0.3.1` tag declares a broken module path.
|
||||
|
||||
## Current matrix
|
||||
|
||||
| Capability | Go code | Automated | Live server | Status |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| connect / endpoints / TLS / basic auth | yes | yes | etcd 3.7.0, 3.5.21 | parity passed (auth enabled) |
|
||||
| test_connection probe (PERMISSION_DENIED → limited) | yes | yes | yes | parity passed |
|
||||
| kv list/get/put/delete/rename with CAS | yes | yes | yes | parity passed |
|
||||
| TTL + preserveLease (3-attempt retry) | yes | yes | yes | parity passed |
|
||||
| Lease list/get/grant/keepalive/revoke | yes | yes | yes | parity passed |
|
||||
| LeaseLeases fallback on 3.3 (UNIMPLEMENTED/timeout → partial) | yes | yes | n/a (3.3 fixture not retained) | coded + unit tested |
|
||||
| LeaseGrant with custom ID (raw etcdserverpb RPC) | yes | yes | yes | replaces Java reflection wire handling |
|
||||
| kv_history (temporary watch, prevKV, progress notify) | yes | yes | yes | requires `clientv3.WithCreatedNotify()`; jetcd parity |
|
||||
| kv_history compacted/future-revision errors | yes | yes | yes | parity passed |
|
||||
| Watch start/poll/stop with full budget system | yes | yes | yes | 256 batches / 10k events / 8MiB per watch, 16MiB session aggregate |
|
||||
| Status: members, alarms, latency fan-out, member dedup | yes | yes | yes | parity passed |
|
||||
| Status raft fields as unsigned decimal strings | yes | yes | yes | bit-level parity with Java `longString(int64(uint64))` |
|
||||
| Compaction (pre-validation, error propagation) | yes | yes | yes | Java check order preserved (params before connect) |
|
||||
| Defrag (followers first, leader last, per-endpoint errors) | yes | yes | yes | parity passed |
|
||||
| Auth users/roles (14 methods) | yes | yes | yes | parity passed |
|
||||
| Multi-session protocol runtime (256 sessions, cancel_session) | yes | yes | yes | protocol flow live test passed |
|
||||
| etcd 3.3 degraded mode | partial | partial | no | lease/history fallbacks coded; no retained 3.3 fixture |
|
||||
| etcd 2.x | n/a | n/a | n/a | served by the dedicated `etcd2` agent, not this one |
|
||||
|
||||
## Known divergences
|
||||
|
||||
- None intentional. All observable protocol outputs are byte-for-byte
|
||||
compatible with the Java agent; the raw protobuf RPCs used for lease
|
||||
operations are an internal implementation change only.
|
||||
@@ -0,0 +1,261 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
clientv3 "go.etcd.io/etcd/client/v3"
|
||||
)
|
||||
|
||||
func (s *etcdSession) authUserList(params map[string]json.RawMessage) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
response, err := client.Auth.UserList(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"users": response.Users}, nil
|
||||
}
|
||||
|
||||
func (s *etcdSession) authUserGet(params map[string]json.RawMessage) (any, error) {
|
||||
user, err := requiredString(params, "user")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
response, err := client.Auth.UserGet(ctx, user)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"user": user, "roles": response.Roles}, nil
|
||||
}
|
||||
|
||||
func (s *etcdSession) authUserAdd(params map[string]json.RawMessage) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
user, err := requiredString(params, "user")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
password, err := requiredString(params, "password")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
if _, err := client.Auth.UserAdd(ctx, user, password); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]bool{"created": true}, nil
|
||||
}
|
||||
|
||||
func (s *etcdSession) authUserDelete(params map[string]json.RawMessage) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
user, err := requiredString(params, "user")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
if _, err := client.Auth.UserDelete(ctx, user); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]bool{"deleted": true}, nil
|
||||
}
|
||||
|
||||
func (s *etcdSession) authUserChangePassword(params map[string]json.RawMessage) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
user, err := requiredString(params, "user")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
password, err := requiredString(params, "password")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
if _, err := client.Auth.UserChangePassword(ctx, user, password); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]bool{"changed": true}, nil
|
||||
}
|
||||
|
||||
func (s *etcdSession) authUserGrantRevokeRole(params map[string]json.RawMessage, grant bool) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
user, err := requiredString(params, "user")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
role, err := requiredString(params, "role")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
if grant {
|
||||
_, err = client.Auth.UserGrantRole(ctx, user, role)
|
||||
} else {
|
||||
_, err = client.Auth.UserRevokeRole(ctx, user, role)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]bool{"updated": true}, nil
|
||||
}
|
||||
|
||||
func (s *etcdSession) authRoleList(params map[string]json.RawMessage) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
response, err := client.Auth.RoleList(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"roles": response.Roles}, nil
|
||||
}
|
||||
|
||||
func (s *etcdSession) authRoleGet(params map[string]json.RawMessage) (any, error) {
|
||||
role, err := requiredString(params, "role")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
response, err := client.Auth.RoleGet(ctx, role)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
permissions := []map[string]any{}
|
||||
for _, permission := range response.Perm {
|
||||
key := []byte(permission.Key)
|
||||
rangeEnd := []byte(permission.RangeEnd)
|
||||
resource := "prefix"
|
||||
if len(key) == 1 && key[0] == 0 && len(rangeEnd) == 1 && rangeEnd[0] == 0 {
|
||||
resource = "all"
|
||||
} else if len(rangeEnd) == 0 {
|
||||
resource = "key"
|
||||
}
|
||||
permissions = append(permissions, map[string]any{
|
||||
"access": strings.ToLower(permission.PermType.String()),
|
||||
"key": bytesObject(key),
|
||||
"rangeEnd": bytesObject(rangeEnd),
|
||||
"resource": resource,
|
||||
})
|
||||
}
|
||||
return map[string]any{"role": role, "permissions": permissions}, nil
|
||||
}
|
||||
|
||||
func (s *etcdSession) authRoleAdd(params map[string]json.RawMessage) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
role, err := requiredString(params, "role")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
if _, err := client.Auth.RoleAdd(ctx, role); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]bool{"created": true}, nil
|
||||
}
|
||||
|
||||
func (s *etcdSession) authRoleDelete(params map[string]json.RawMessage) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
role, err := requiredString(params, "role")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
if _, err := client.Auth.RoleDelete(ctx, role); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]bool{"deleted": true}, nil
|
||||
}
|
||||
|
||||
func (s *etcdSession) authRolePermission(params map[string]json.RawMessage, grant bool) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
role, err := requiredString(params, "role")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resource := stringOrDefault(params, "resource", "key")
|
||||
all := resource == "all"
|
||||
prefix := resource == "prefix"
|
||||
var key string
|
||||
var rangeEnd string
|
||||
if all {
|
||||
key = "\x00"
|
||||
rangeEnd = "\x00"
|
||||
} else {
|
||||
key, err = keyBytesParam(params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if prefix {
|
||||
rangeEnd = prefixEnd(key)
|
||||
}
|
||||
}
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
if grant {
|
||||
access, err := permissionType(stringOrDefault(params, "access", ""))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := client.Auth.RoleGrantPermission(ctx, role, key, rangeEnd, access); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
if _, err := client.Auth.RoleRevokePermission(ctx, role, key, rangeEnd); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return map[string]bool{"updated": true}, nil
|
||||
}
|
||||
|
||||
func permissionType(access string) (clientv3.PermissionType, error) {
|
||||
permission, err := clientv3.StrToPermissionType(strings.ToUpper(access))
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("ETCD_INVALID_ACCESS: access must be READ, WRITE, or READWRITE, got %s", access)
|
||||
}
|
||||
return permission, nil
|
||||
}
|
||||
@@ -0,0 +1,321 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
clientv3 "go.etcd.io/etcd/client/v3"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultGrpcMaxInboundMessageSize = 32 * 1024 * 1024
|
||||
minGrpcMaxInboundMessageSize = 1024 * 1024
|
||||
maxGrpcMaxInboundMessageSize = 256 * 1024 * 1024
|
||||
grpcMaxInboundMessageSizeKey = "grpc_max_inbound_message_size"
|
||||
defaultPort = 2379
|
||||
)
|
||||
|
||||
// connectionParams mirrors the host-side etcd connection configuration.
|
||||
type connectionParams struct {
|
||||
EtcdEndpoints string `json:"etcd_endpoints"`
|
||||
Endpoints string `json:"endpoints"`
|
||||
ConnectionString string `json:"connection_string"`
|
||||
Host string `json:"host"`
|
||||
Port int `json:"port"`
|
||||
Username string `json:"username"`
|
||||
Password string `json:"password"`
|
||||
SSL bool `json:"ssl"`
|
||||
CACertPath string `json:"ca_cert_path"`
|
||||
ClientCertPath string `json:"client_cert_path"`
|
||||
ClientKeyPath string `json:"client_key_path"`
|
||||
CertPath string `json:"cert_path"`
|
||||
KeyPath string `json:"key_path"`
|
||||
ConnectTimeoutSecs int `json:"connect_timeout_secs"`
|
||||
GrpcMaxInboundMessageSize int `json:"grpc_max_inbound_message_size"`
|
||||
URLParams string `json:"url_params"`
|
||||
}
|
||||
|
||||
func connectionObject(params map[string]json.RawMessage) (connectionParams, error) {
|
||||
var connection connectionParams
|
||||
if raw, ok := params["connection"]; ok && raw != nil {
|
||||
if err := json.Unmarshal(raw, &connection); err != nil {
|
||||
return connection, err
|
||||
}
|
||||
return connection, nil
|
||||
}
|
||||
if err := decodeParams(params, &connection); err != nil {
|
||||
return connection, err
|
||||
}
|
||||
return connection, nil
|
||||
}
|
||||
|
||||
func (s *etcdSession) connectMap(params map[string]json.RawMessage) (any, error) {
|
||||
if _, err := s.connect(params); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]bool{"ok": true}, nil
|
||||
}
|
||||
|
||||
func (s *etcdSession) connect(params map[string]json.RawMessage) (any, error) {
|
||||
connection, err := connectionObject(params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nextClient, err := buildClient(connection)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
endpointList := connectionEndpoints(connection)
|
||||
if _, err := probeClient(nextClient, endpointList); err != nil {
|
||||
_ = nextClient.Close()
|
||||
return nil, err
|
||||
}
|
||||
s.close()
|
||||
s.clientMu.Lock()
|
||||
s.client = nextClient
|
||||
s.connectedEndpoints = endpointList
|
||||
s.clientMu.Unlock()
|
||||
return map[string]bool{"ok": true}, nil
|
||||
}
|
||||
|
||||
func (s *etcdSession) activeClient() (*clientv3.Client, error) {
|
||||
s.clientMu.Lock()
|
||||
client := s.client
|
||||
s.clientMu.Unlock()
|
||||
if client == nil {
|
||||
return nil, errors.New("Not connected")
|
||||
}
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func (s *etcdSession) connectedEndpointList() []string {
|
||||
s.clientMu.Lock()
|
||||
defer s.clientMu.Unlock()
|
||||
return append([]string(nil), s.connectedEndpoints...)
|
||||
}
|
||||
|
||||
func (s *etcdSession) validateConnection() (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return probeClient(client, s.connectedEndpointList())
|
||||
}
|
||||
|
||||
func (s *etcdSession) close() error {
|
||||
s.watchesMu.Lock()
|
||||
watches := s.watches
|
||||
s.watches = map[string]*watchState{}
|
||||
s.knownLeases = map[uint64]struct{}{}
|
||||
s.watchBufferedBytes = 0
|
||||
s.watchesMu.Unlock()
|
||||
for _, watch := range watches {
|
||||
watch.close()
|
||||
}
|
||||
s.clientMu.Lock()
|
||||
client := s.client
|
||||
s.client = nil
|
||||
s.connectedEndpoints = nil
|
||||
s.clientMu.Unlock()
|
||||
if client != nil {
|
||||
_ = client.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func buildClient(connection connectionParams) (*clientv3.Client, error) {
|
||||
endpoints := connectionEndpoints(connection)
|
||||
config := clientv3.Config{
|
||||
Endpoints: endpoints,
|
||||
DialTimeout: time.Duration(connectTimeoutSeconds(connection)) * time.Second,
|
||||
}
|
||||
if size := grpcMaxInboundMessageSize(connection); size > 0 {
|
||||
config.MaxCallSendMsgSize = size
|
||||
config.MaxCallRecvMsgSize = size
|
||||
}
|
||||
if strings.TrimSpace(connection.Username) != "" {
|
||||
config.Username = connection.Username
|
||||
config.Password = connection.Password
|
||||
}
|
||||
if connection.SSL {
|
||||
tlsConfig, err := tlsConfigFor(connection)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
config.TLS = tlsConfig
|
||||
}
|
||||
return clientv3.New(config)
|
||||
}
|
||||
|
||||
func connectTimeoutSeconds(connection connectionParams) int {
|
||||
seconds := connection.ConnectTimeoutSecs
|
||||
if seconds == 0 {
|
||||
seconds = rpcTimeoutSeconds
|
||||
}
|
||||
if seconds < 1 {
|
||||
seconds = 1
|
||||
}
|
||||
if seconds > 300 {
|
||||
seconds = 300
|
||||
}
|
||||
return seconds
|
||||
}
|
||||
|
||||
func grpcMaxInboundMessageSize(connection connectionParams) int {
|
||||
configured := connection.GrpcMaxInboundMessageSize
|
||||
if configured == 0 {
|
||||
configured = intURLParamOrDefault(connection.URLParams, grpcMaxInboundMessageSizeKey, defaultGrpcMaxInboundMessageSize)
|
||||
}
|
||||
if configured < minGrpcMaxInboundMessageSize {
|
||||
configured = minGrpcMaxInboundMessageSize
|
||||
}
|
||||
if configured > maxGrpcMaxInboundMessageSize {
|
||||
configured = maxGrpcMaxInboundMessageSize
|
||||
}
|
||||
return configured
|
||||
}
|
||||
|
||||
func intURLParamOrDefault(params, key string, fallback int) int {
|
||||
if strings.TrimSpace(params) == "" {
|
||||
return fallback
|
||||
}
|
||||
for _, entry := range strings.Split(strings.TrimPrefix(params, "?"), "&") {
|
||||
separator := strings.Index(entry, "=")
|
||||
entryKey := entry
|
||||
if separator >= 0 {
|
||||
entryKey = entry[:separator]
|
||||
}
|
||||
if entryKey != key {
|
||||
continue
|
||||
}
|
||||
if separator < 0 {
|
||||
return fallback
|
||||
}
|
||||
value, err := strconv.Atoi(entry[separator+1:])
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func tlsConfigFor(connection connectionParams) (*tls.Config, error) {
|
||||
tlsConfig := &tls.Config{}
|
||||
if ca := strings.TrimSpace(connection.CACertPath); ca != "" {
|
||||
authorityPEM, err := os.ReadFile(ca)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pool := x509.NewCertPool()
|
||||
if !pool.AppendCertsFromPEM(authorityPEM) {
|
||||
return nil, fmt.Errorf("failed to parse CA certificate at %s", ca)
|
||||
}
|
||||
tlsConfig.RootCAs = pool
|
||||
}
|
||||
certPath := firstNonBlank(connection.ClientCertPath, connection.CertPath)
|
||||
keyPath := firstNonBlank(connection.ClientKeyPath, connection.KeyPath)
|
||||
if (certPath == "") != (keyPath == "") {
|
||||
return nil, errors.New("Client certificate and key must be provided together")
|
||||
}
|
||||
if certPath != "" {
|
||||
pair, err := tls.LoadX509KeyPair(certPath, keyPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tlsConfig.Certificates = []tls.Certificate{pair}
|
||||
}
|
||||
return tlsConfig, nil
|
||||
}
|
||||
|
||||
func probeClient(client *clientv3.Client, endpoints []string) (map[string]any, error) {
|
||||
var lastFailure error
|
||||
for _, endpoint := range endpoints {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
|
||||
_, err := client.Maintenance.Status(ctx, endpoint)
|
||||
cancel()
|
||||
if err == nil {
|
||||
return map[string]any{"ok": true, "endpoint": endpoint}, nil
|
||||
}
|
||||
// A restricted etcd user may not be allowed to call Maintenance.Status.
|
||||
// PERMISSION_DENIED still proves that the channel reached an etcd server.
|
||||
if status.Code(err) == codes.PermissionDenied {
|
||||
return map[string]any{"ok": true, "endpoint": endpoint, "limited": true}, nil
|
||||
}
|
||||
lastFailure = err
|
||||
}
|
||||
if lastFailure != nil {
|
||||
return nil, lastFailure
|
||||
}
|
||||
return nil, errors.New("No etcd endpoint configured")
|
||||
}
|
||||
|
||||
func connectionEndpoints(connection connectionParams) []string {
|
||||
configured := firstNonBlank(connection.EtcdEndpoints, connection.Endpoints, connection.ConnectionString)
|
||||
var result []string
|
||||
if configured != "" {
|
||||
for _, endpoint := range strings.FieldsFunc(configured, func(r rune) bool { return r == ',' || r == '\n' }) {
|
||||
normalized := normalizeEndpoint(strings.TrimSpace(endpoint), connection.SSL)
|
||||
if normalized != "" {
|
||||
result = append(result, normalized)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(result) == 0 {
|
||||
host := connection.Host
|
||||
if host == "" {
|
||||
host = "127.0.0.1"
|
||||
}
|
||||
port := connection.Port
|
||||
if port == 0 {
|
||||
port = defaultPort
|
||||
}
|
||||
result = append(result, normalizeEndpoint(fmt.Sprintf("%s:%d", host, port), connection.SSL))
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func normalizeEndpoint(endpoint string, tlsEnabled bool) string {
|
||||
if strings.TrimSpace(endpoint) == "" {
|
||||
return ""
|
||||
}
|
||||
if strings.HasPrefix(endpoint, "http://") || strings.HasPrefix(endpoint, "https://") {
|
||||
return endpoint
|
||||
}
|
||||
if strings.HasPrefix(endpoint, "unix://") || strings.HasPrefix(endpoint, "unixs://") {
|
||||
return endpoint
|
||||
}
|
||||
scheme := "http"
|
||||
if tlsEnabled {
|
||||
scheme = "https"
|
||||
}
|
||||
return scheme + "://" + endpoint
|
||||
}
|
||||
|
||||
func endpointHost(rawEndpoint string) string {
|
||||
parsed, err := url.Parse(rawEndpoint)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return parsed.Hostname()
|
||||
}
|
||||
|
||||
func firstNonBlank(values ...string) string {
|
||||
for _, value := range values {
|
||||
if strings.TrimSpace(value) != "" {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
module github.com/t8y2/dbx/agents/drivers/etcd-go
|
||||
|
||||
go 1.26
|
||||
|
||||
// The v0.3.1 tag of coreos/go-semver declares the root module without the
|
||||
// /semver suffix, so the nested module path etcd requires no longer resolves
|
||||
// upstream. Vendored under go-common from the v0.3.1 tag, package unchanged.
|
||||
replace github.com/coreos/go-semver/semver => ../../go-common/go-semver
|
||||
|
||||
// The pseudo-version recorded in etcd's go.mod predates second-precision
|
||||
// timestamps and is rejected by modern Go; point at the corrected form.
|
||||
replace github.com/modern-go/concurrent => github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd
|
||||
|
||||
require (
|
||||
go.etcd.io/etcd/api/v3 v3.7.1
|
||||
go.etcd.io/etcd/client/v3 v3.7.1
|
||||
google.golang.org/grpc v1.82.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/coreos/go-semver/semver v0.3.1 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.7.0 // indirect
|
||||
github.com/golang/protobuf v1.5.4 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.7.1 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.27.1 // indirect
|
||||
golang.org/x/net v0.55.0 // indirect
|
||||
golang.org/x/sys v0.45.0 // indirect
|
||||
golang.org/x/text v0.37.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect
|
||||
google.golang.org/protobuf v1.36.11 // indirect
|
||||
)
|
||||
@@ -0,0 +1,64 @@
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA=
|
||||
github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
go.etcd.io/etcd/api/v3 v3.7.1 h1:KJG0/DcWGfe3Y1otDf/fsBf0TSSgpxZ5RO/L8SFt73E=
|
||||
go.etcd.io/etcd/api/v3 v3.7.1/go.mod h1:8bXIpCMeV7E3/XL0Ix123ATn3dB+0V7d9zklHbB0m78=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.7.1 h1:rKYsj3pRkR0eK3yjT3XOgrhqfmIfj9pzNgxjh7mfFv4=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.7.1/go.mod h1:cnzZGIUzSfjEwLC6UBVsSXlEK1eepS/JUD7wE6PLRT0=
|
||||
go.etcd.io/etcd/client/v3 v3.7.1 h1:0PEMMC0KuZmVIN+RAbdqfkZ45pYTgKVtmBEbRCvZFUg=
|
||||
go.etcd.io/etcd/client/v3 v3.7.1/go.mod h1:ffNqALa8tRCYhYo1F9oR489y23K39Gz+BSR3ApAGYq0=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
|
||||
go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
|
||||
go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM=
|
||||
go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
|
||||
go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg=
|
||||
go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A=
|
||||
go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A=
|
||||
go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc=
|
||||
go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
|
||||
golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
|
||||
golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
|
||||
golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
|
||||
golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
|
||||
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
|
||||
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
|
||||
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 h1:yQugLulqltosq0B/f8l4w9VryjV+N/5gcW0jQ3N8Qec=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478/go.mod h1:C6ADNqOxbgdUUeRTU+LCHDPB9ttAMCTff6auwCVa4uc=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE=
|
||||
google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA=
|
||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -0,0 +1,255 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sort"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.etcd.io/etcd/api/v3/mvccpb"
|
||||
"go.etcd.io/etcd/api/v3/v3rpc/rpctypes"
|
||||
clientv3 "go.etcd.io/etcd/client/v3"
|
||||
)
|
||||
|
||||
const (
|
||||
historyDefaultRevisionWindow = int64(10000)
|
||||
historyLimitMax = 500
|
||||
)
|
||||
|
||||
type historyCollector struct {
|
||||
mu sync.Mutex
|
||||
rows []map[string]any
|
||||
limit int
|
||||
truncated bool
|
||||
}
|
||||
|
||||
func (c *historyCollector) append(row map[string]any) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if len(c.rows) == c.limit {
|
||||
c.rows = c.rows[1:]
|
||||
c.truncated = true
|
||||
}
|
||||
c.rows = append(c.rows, row)
|
||||
}
|
||||
|
||||
func (s *etcdSession) history(params map[string]json.RawMessage) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key, err := keyBytesParam(params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
limit := intOrDefault(params, "limit", 100)
|
||||
if limit < 1 {
|
||||
limit = 1
|
||||
}
|
||||
if limit > historyLimitMax {
|
||||
limit = historyLimitMax
|
||||
}
|
||||
requestedEnd := longOrNull(params, "endRevision")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
|
||||
latestOptions := []clientv3.OpOption{}
|
||||
if requestedEnd != nil && *requestedEnd > 0 {
|
||||
latestOptions = append(latestOptions, clientv3.WithRev(*requestedEnd))
|
||||
}
|
||||
latest, err := client.Get(ctx, key, latestOptions...)
|
||||
cancel()
|
||||
if err != nil {
|
||||
if errors.Is(err, rpctypes.ErrCompacted) {
|
||||
if revision, ok := compactedRevisionOf(client, key); ok {
|
||||
return nil, errors.New("ETCD_COMPACTED: requested history was compacted at revision " + longString(revision))
|
||||
}
|
||||
return nil, errors.New("ETCD_COMPACTED: requested history was compacted")
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
var endRevision int64
|
||||
if requestedEnd != nil {
|
||||
endRevision = *requestedEnd
|
||||
} else {
|
||||
endRevision = latest.Header.Revision
|
||||
}
|
||||
var targetKeyRevision int64
|
||||
if len(latest.Kvs) == 0 {
|
||||
targetKeyRevision = endRevision
|
||||
} else {
|
||||
targetKeyRevision = latest.Kvs[0].ModRevision
|
||||
}
|
||||
requestedStart := longOrNull(params, "startRevision")
|
||||
startRevision := historyStartRevision(requestedStart, targetKeyRevision)
|
||||
if startRevision > targetKeyRevision {
|
||||
return map[string]any{
|
||||
"events": []any{},
|
||||
"observedRevision": longString(endRevision),
|
||||
"truncated": false,
|
||||
}, nil
|
||||
}
|
||||
|
||||
collector := &historyCollector{limit: limit, truncated: requestedStart == nil && startRevision > 1}
|
||||
created := make(chan struct{})
|
||||
var createdOnce sync.Once
|
||||
completed := make(chan struct{})
|
||||
var completedOnce sync.Once
|
||||
var failure error
|
||||
var failureMu sync.Mutex
|
||||
|
||||
watchCtx, watchCancel := context.WithCancel(context.Background())
|
||||
channel := client.Watch(watchCtx, key,
|
||||
clientv3.WithRev(startRevision),
|
||||
clientv3.WithPrevKV(),
|
||||
clientv3.WithProgressNotify(),
|
||||
// clientv3 only posts the created response when this option is set;
|
||||
// the created gate relies on it (jetcd withCreateNotify parity).
|
||||
clientv3.WithCreatedNotify(),
|
||||
)
|
||||
go func() {
|
||||
for response := range channel {
|
||||
if response.Err() != nil {
|
||||
failureMu.Lock()
|
||||
failure = response.Err()
|
||||
failureMu.Unlock()
|
||||
completedOnce.Do(func() { close(completed) })
|
||||
continue
|
||||
}
|
||||
if response.Canceled {
|
||||
if response.CompactRevision > 0 {
|
||||
err := errors.New("ETCD_COMPACTED: requested history was compacted at revision " + longString(response.CompactRevision))
|
||||
failureMu.Lock()
|
||||
failure = err
|
||||
failureMu.Unlock()
|
||||
}
|
||||
completedOnce.Do(func() { close(completed) })
|
||||
continue
|
||||
}
|
||||
createdOnce.Do(func() { close(created) })
|
||||
for _, event := range response.Events {
|
||||
item := event.Kv
|
||||
revision := item.ModRevision
|
||||
if revision > endRevision {
|
||||
continue
|
||||
}
|
||||
var previous *mvccpb.KeyValue
|
||||
if event.PrevKv != nil {
|
||||
previous = event.PrevKv
|
||||
}
|
||||
row := map[string]any{
|
||||
"eventType": eventType(event),
|
||||
"revision": longString(revision),
|
||||
}
|
||||
if event.Type == mvccpb.DELETE {
|
||||
row["value"] = nil
|
||||
} else {
|
||||
row["value"] = valueObject(item.Value)
|
||||
}
|
||||
if previous != nil && previous.Version > 0 {
|
||||
row["previousValue"] = valueObject(previous.Value)
|
||||
} else {
|
||||
row["previousValue"] = nil
|
||||
}
|
||||
if event.Type == mvccpb.DELETE && previous != nil {
|
||||
row["metadata"] = metadataMap(previous)
|
||||
} else {
|
||||
row["metadata"] = metadataMap(item)
|
||||
}
|
||||
collector.append(row)
|
||||
if revision >= targetKeyRevision {
|
||||
completedOnce.Do(func() { close(completed) })
|
||||
}
|
||||
}
|
||||
if response.IsProgressNotify() && response.Header.Revision >= endRevision {
|
||||
completedOnce.Do(func() { close(completed) })
|
||||
}
|
||||
}
|
||||
completedOnce.Do(func() { close(completed) })
|
||||
}()
|
||||
defer watchCancel()
|
||||
|
||||
select {
|
||||
case <-created:
|
||||
case <-time.After(5 * time.Second):
|
||||
return nil, errors.New("ETCD_HISTORY_TIMEOUT: watcher was not created")
|
||||
}
|
||||
// For an existing exact key, its latest mod revision is an explicit
|
||||
// replay boundary. This avoids relying on progress notifications,
|
||||
// which older etcd/jetcd combinations do not consistently emit.
|
||||
progressCtx, progressCancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
|
||||
_ = client.RequestProgress(progressCtx)
|
||||
progressCancel()
|
||||
select {
|
||||
case <-completed:
|
||||
case <-time.After(15 * time.Second):
|
||||
return nil, errors.New("ETCD_HISTORY_TIMEOUT: history replay did not reach the requested revision")
|
||||
}
|
||||
|
||||
failureMu.Lock()
|
||||
historyFailure := failure
|
||||
failureMu.Unlock()
|
||||
if historyFailure != nil {
|
||||
if errors.Is(historyFailure, rpctypes.ErrCompacted) {
|
||||
if revision, ok := compactedRevisionOf(client, key); ok {
|
||||
return nil, errors.New("ETCD_COMPACTED: requested history was compacted at revision " + longString(revision))
|
||||
}
|
||||
return nil, errors.New("ETCD_COMPACTED: requested history was compacted")
|
||||
}
|
||||
return nil, errors.New("ETCD_HISTORY_FAILED: " + historyFailure.Error())
|
||||
}
|
||||
|
||||
collector.mu.Lock()
|
||||
rows := make([]map[string]any, len(collector.rows))
|
||||
copy(rows, collector.rows)
|
||||
truncated := collector.truncated
|
||||
collector.mu.Unlock()
|
||||
sort.SliceStable(rows, func(i, j int) bool {
|
||||
a, _ := strconv.ParseInt(rows[i]["revision"].(string), 10, 64)
|
||||
b, _ := strconv.ParseInt(rows[j]["revision"].(string), 10, 64)
|
||||
return a > b
|
||||
})
|
||||
serialized := make([]any, len(rows))
|
||||
for i, row := range rows {
|
||||
serialized[i] = row
|
||||
}
|
||||
return map[string]any{
|
||||
"events": serialized,
|
||||
"observedRevision": longString(endRevision),
|
||||
"truncated": truncated,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func historyStartRevision(requestedStart *int64, targetRevision int64) int64 {
|
||||
if requestedStart != nil {
|
||||
if *requestedStart < 1 {
|
||||
return 1
|
||||
}
|
||||
return *requestedStart
|
||||
}
|
||||
start := targetRevision - historyDefaultRevisionWindow + 1
|
||||
if start < 1 {
|
||||
return 1
|
||||
}
|
||||
return start
|
||||
}
|
||||
|
||||
// compactedRevisionOf recovers the compaction boundary that gRPC errors do not
|
||||
// carry: a watch from revision 1 is cancelled immediately with the compacted
|
||||
// revision in its response.
|
||||
func compactedRevisionOf(client *clientv3.Client, key string) (int64, bool) {
|
||||
watchCtx, watchCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer watchCancel()
|
||||
channel := client.Watch(watchCtx, key, clientv3.WithRev(1))
|
||||
select {
|
||||
case response := <-channel:
|
||||
if response.CompactRevision > 0 {
|
||||
return response.CompactRevision, true
|
||||
}
|
||||
return 0, false
|
||||
case <-watchCtx.Done():
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,515 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestLiveEtcdAgent exercises the full agent surface against a real etcd
|
||||
// server. Enable with DBX_ETCD_LIVE=1; configure via DBX_ETCD_ENDPOINTS,
|
||||
// DBX_ETCD_USER, and DBX_ETCD_PASSWORD (defaults match the deploy recipes:
|
||||
// root/123456 on 127.0.0.1:10700 for 3.7).
|
||||
func TestLiveEtcdAgent(t *testing.T) {
|
||||
if os.Getenv("DBX_ETCD_LIVE") != "1" {
|
||||
t.Skip("set DBX_ETCD_LIVE=1 to run the live etcd agent test")
|
||||
}
|
||||
endpoints := envOrDefault("DBX_ETCD_ENDPOINTS", "http://127.0.0.1:10700")
|
||||
user := envOrDefault("DBX_ETCD_USER", "root")
|
||||
password := envOrDefault("DBX_ETCD_PASSWORD", "123456")
|
||||
|
||||
state := newEtcdSession()
|
||||
connectParams := map[string]json.RawMessage{
|
||||
"etcd_endpoints": json.RawMessage(`"` + endpoints + `"`),
|
||||
"username": json.RawMessage(`"` + user + `"`),
|
||||
"password": json.RawMessage(`"` + password + `"`),
|
||||
}
|
||||
if _, err := state.connect(connectParams); err != nil {
|
||||
t.Fatalf("connect failed: %v", err)
|
||||
}
|
||||
defer state.close()
|
||||
|
||||
if result, err := state.validateConnection(); err != nil {
|
||||
t.Fatalf("validate_connection failed: %v", err)
|
||||
} else {
|
||||
probe := result.(map[string]any)
|
||||
if probe["ok"] != true {
|
||||
t.Fatalf("unexpected probe result: %#v", probe)
|
||||
}
|
||||
}
|
||||
|
||||
prefix := fmt.Sprintf("dbx/live/%d/", time.Now().UnixNano())
|
||||
|
||||
// --- KV basics -----------------------------------------------------
|
||||
putResult, err := state.put(paramsWith(map[string]json.RawMessage{
|
||||
"key": jsonString(prefix + "hello"),
|
||||
"value": jsonValue("world"),
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("put failed: %v", err)
|
||||
}
|
||||
baseRevision := revisionOf(t, putResult)
|
||||
|
||||
binaryEncoded := base64Encode([]byte{0x00, 0xff, 0x81})
|
||||
if _, err := state.put(paramsWith(map[string]json.RawMessage{
|
||||
"key": jsonString(prefix + "binary"),
|
||||
"value": json.RawMessage(`{"encoding":"base64","data":"` + binaryEncoded + `"}`),
|
||||
})); err != nil {
|
||||
t.Fatalf("binary put failed: %v", err)
|
||||
}
|
||||
|
||||
fetched, err := state.get(paramsWith(map[string]json.RawMessage{"key": jsonString(prefix + "hello")}))
|
||||
if err != nil {
|
||||
t.Fatalf("get failed: %v", err)
|
||||
}
|
||||
row := fetched.(map[string]any)
|
||||
if row["found"] != true || row["key"] != prefix+"hello" {
|
||||
t.Fatalf("unexpected get result: %#v", row)
|
||||
}
|
||||
value := row["value"].(map[string]any)
|
||||
if value["encoding"] != "utf8" || value["data"] != "world" {
|
||||
t.Fatalf("unexpected value: %#v", value)
|
||||
}
|
||||
metadata := row["metadata"].(map[string]any)
|
||||
if metadata["createRevision"] != metadata["modRevision"] {
|
||||
t.Fatalf("fresh key revisions should match: %#v", metadata)
|
||||
}
|
||||
|
||||
binaryRow, err := state.get(paramsWith(map[string]json.RawMessage{"key": jsonString(prefix + "binary")}))
|
||||
if err != nil {
|
||||
t.Fatalf("binary get failed: %v", err)
|
||||
}
|
||||
binaryValue := binaryRow.(map[string]any)["value"].(map[string]any)
|
||||
if binaryValue["encoding"] != "base64" || binaryValue["data"] != binaryEncoded {
|
||||
t.Fatalf("binary round trip broken: %#v", binaryValue)
|
||||
}
|
||||
|
||||
// CAS conflict on stale revision.
|
||||
if _, err := state.put(paramsWith(map[string]json.RawMessage{
|
||||
"key": jsonString(prefix + "hello"),
|
||||
"value": jsonValue("v2"),
|
||||
"expectedModRevision": json.RawMessage(`1`),
|
||||
})); err == nil || err.Error() != "ETCD_CAS_CONFLICT: key changed after it was loaded" {
|
||||
t.Fatalf("expected CAS conflict, got %v", err)
|
||||
}
|
||||
|
||||
// CAS success with the fresh revision.
|
||||
if _, err := state.put(paramsWith(map[string]json.RawMessage{
|
||||
"key": jsonString(prefix + "hello"),
|
||||
"value": jsonValue("v3"),
|
||||
"expectedModRevision": json.RawMessage(metadata["modRevision"].(string)),
|
||||
})); err != nil {
|
||||
t.Fatalf("CAS put failed: %v", err)
|
||||
}
|
||||
|
||||
// TTL put, lease metadata, and preserveLease.
|
||||
if _, err := state.put(paramsWith(map[string]json.RawMessage{
|
||||
"key": jsonString(prefix + "leased"),
|
||||
"value": jsonValue("expiring"),
|
||||
"ttl": json.RawMessage(`300`),
|
||||
})); err != nil {
|
||||
t.Fatalf("ttl put failed: %v", err)
|
||||
}
|
||||
leased, err := state.get(paramsWith(map[string]json.RawMessage{"key": jsonString(prefix + "leased")}))
|
||||
if err != nil {
|
||||
t.Fatalf("leased get failed: %v", err)
|
||||
}
|
||||
leasedMetadata := leased.(map[string]any)["metadata"].(map[string]any)
|
||||
if _, hasTTL := leasedMetadata["ttl"]; !hasTTL {
|
||||
t.Fatalf("leased key must expose ttl metadata: %#v", leasedMetadata)
|
||||
}
|
||||
if _, err := state.put(paramsWith(map[string]json.RawMessage{
|
||||
"key": jsonString(prefix + "leased"),
|
||||
"value": jsonValue("renewed"),
|
||||
"preserveLease": json.RawMessage(`true`),
|
||||
})); err != nil {
|
||||
t.Fatalf("preserveLease put failed: %v", err)
|
||||
}
|
||||
if _, err := state.put(paramsWith(map[string]json.RawMessage{
|
||||
"key": jsonString(prefix + "hello"),
|
||||
"value": jsonValue("nope"),
|
||||
"preserveLease": json.RawMessage(`true`),
|
||||
})); err == nil || err.Error() != "Cannot preserve lease: key does not exist or has no lease" {
|
||||
t.Fatalf("expected preserveLease rejection, got %v", err)
|
||||
}
|
||||
|
||||
// Rename, including conflict and missing-source paths.
|
||||
if _, err := state.rename(paramsWith(map[string]json.RawMessage{
|
||||
"key": jsonString(prefix + "old"),
|
||||
"newKey": jsonString(prefix + "new"),
|
||||
})); err == nil || err.Error() != "ETCD_NOT_FOUND: source key does not exist" {
|
||||
t.Fatalf("expected rename not-found, got %v", err)
|
||||
}
|
||||
if _, err := state.rename(paramsWith(map[string]json.RawMessage{
|
||||
"key": jsonString(prefix + "hello"),
|
||||
"newKey": jsonString(prefix + "renamed"),
|
||||
})); err != nil {
|
||||
t.Fatalf("rename failed: %v", err)
|
||||
}
|
||||
if _, err := state.rename(paramsWith(map[string]json.RawMessage{
|
||||
"key": jsonString(prefix + "binary"),
|
||||
"newKey": jsonString(prefix + "renamed"),
|
||||
})); err == nil || err.Error() != "ETCD_CAS_CONFLICT: source changed or target already exists" {
|
||||
t.Fatalf("expected rename conflict, got %v", err)
|
||||
}
|
||||
|
||||
// list_prefix with continuation.
|
||||
listed, err := state.listPrefix(paramsWith(map[string]json.RawMessage{
|
||||
"prefix": jsonString(prefix),
|
||||
"limit": json.RawMessage(`2`),
|
||||
"includeValues": json.RawMessage(`true`),
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("list_prefix failed: %v", err)
|
||||
}
|
||||
list := listed.(map[string]any)
|
||||
if len(list["keys"].([]any)) != 2 {
|
||||
t.Fatalf("expected limited list, got %#v", list)
|
||||
}
|
||||
continuation, ok := list["continuation"].(string)
|
||||
if !ok || continuation == "" {
|
||||
t.Fatalf("expected continuation, got %#v", list)
|
||||
}
|
||||
nextPage, err := state.listPrefix(paramsWith(map[string]json.RawMessage{
|
||||
"prefix": jsonString(prefix),
|
||||
"limit": json.RawMessage(`100`),
|
||||
"includeValues": json.RawMessage(`true`),
|
||||
"continuation": json.RawMessage(`"` + continuation + `"`),
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("continuation list failed: %v", err)
|
||||
}
|
||||
nextKeys := nextPage.(map[string]any)["keys"].([]any)
|
||||
if len(nextKeys) == 0 {
|
||||
t.Fatalf("continuation must resume after the first page")
|
||||
}
|
||||
firstContinued := nextKeys[0].(map[string]any)["key"].(string)
|
||||
if firstContinued == list["keys"].([]any)[1].(map[string]any)["key"].(string) {
|
||||
t.Fatalf("continuation overlapped the first page")
|
||||
}
|
||||
|
||||
// --- history -------------------------------------------------------
|
||||
// Explicit startRevision keeps the window above any revision compacted by
|
||||
// earlier verification runs (the flow itself compacts at the end).
|
||||
history, err := state.history(paramsWith(map[string]json.RawMessage{
|
||||
"key": jsonString(prefix + "renamed"),
|
||||
"startRevision": json.RawMessage(longString(baseRevision)),
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("history failed: %v", err)
|
||||
}
|
||||
historyRow := history.(map[string]any)
|
||||
events := historyRow["events"].([]any)
|
||||
// The renamed key only exists from the rename revision onward; the old
|
||||
// key's earlier revisions belong to the old key.
|
||||
if len(events) != 1 {
|
||||
t.Fatalf("expected exactly the rename event, got %#v", historyRow)
|
||||
}
|
||||
if historyRow["truncated"] != false {
|
||||
t.Fatalf("history should not be truncated for a short window: %#v", historyRow)
|
||||
}
|
||||
firstEvent := events[0].(map[string]any)
|
||||
if _, err := strconvParse(firstEvent["revision"].(string)); err != nil {
|
||||
t.Fatalf("revisions must be strings: %#v", firstEvent)
|
||||
}
|
||||
|
||||
// --- watch ---------------------------------------------------------
|
||||
watchStarted, err := state.watchStart(paramsWith(map[string]json.RawMessage{
|
||||
"key": jsonString(prefix + "watched"),
|
||||
"includePrevKv": json.RawMessage(`true`),
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("watch_start failed: %v", err)
|
||||
}
|
||||
watchID := watchStarted.(map[string]any)["watchId"].(string)
|
||||
if _, err := state.put(paramsWith(map[string]json.RawMessage{
|
||||
"key": jsonString(prefix + "watched"),
|
||||
"value": jsonValue("first"),
|
||||
})); err != nil {
|
||||
t.Fatalf("watched put failed: %v", err)
|
||||
}
|
||||
if _, err := state.put(paramsWith(map[string]json.RawMessage{
|
||||
"key": jsonString(prefix + "watched"),
|
||||
"value": jsonValue("second"),
|
||||
})); err != nil {
|
||||
t.Fatalf("watched put 2 failed: %v", err)
|
||||
}
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
var watchBatches []any
|
||||
for time.Now().Before(deadline) {
|
||||
polled, err := state.watchPoll(paramsWith(map[string]json.RawMessage{"watchId": json.RawMessage(`"` + watchID + `"`)}))
|
||||
if err != nil {
|
||||
t.Fatalf("watch_poll failed: %v", err)
|
||||
}
|
||||
batches := polled.(map[string]any)["batches"].([]any)
|
||||
watchBatches = append(watchBatches, batches...)
|
||||
if len(watchBatches) >= 2 {
|
||||
break
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
if len(watchBatches) < 2 {
|
||||
t.Fatalf("watch did not deliver both events: %#v", watchBatches)
|
||||
}
|
||||
secondBatch := watchBatches[len(watchBatches)-1].(map[string]any)
|
||||
secondEvent := secondBatch["events"].([]map[string]any)[0]
|
||||
if secondEvent["previousValue"].(map[string]any)["data"] != "first" {
|
||||
t.Fatalf("prevKv missing on watch event: %#v", secondEvent)
|
||||
}
|
||||
stopped, err := state.watchStop(paramsWith(map[string]json.RawMessage{"watchId": json.RawMessage(`"` + watchID + `"`)}))
|
||||
if err != nil || stopped.(map[string]bool)["stopped"] != true {
|
||||
t.Fatalf("watch_stop failed: %#v %v", stopped, err)
|
||||
}
|
||||
if _, err := state.watchPoll(paramsWith(map[string]json.RawMessage{"watchId": json.RawMessage(`"` + watchID + `"`)})); err == nil || err.Error() != "ETCD_WATCH_NOT_FOUND: watch does not exist" {
|
||||
t.Fatalf("expected watch-not-found after stop, got %v", err)
|
||||
}
|
||||
|
||||
// --- lease ---------------------------------------------------------
|
||||
granted, err := state.leaseGrant(paramsWith(map[string]json.RawMessage{"ttl": json.RawMessage(`120`)}))
|
||||
if err != nil {
|
||||
t.Fatalf("lease_grant failed: %v", err)
|
||||
}
|
||||
leaseID := granted.(map[string]any)["id"].(string)
|
||||
if _, err := state.leaseKeepAlive(paramsWith(map[string]json.RawMessage{"id": json.RawMessage(leaseID)})); err != nil {
|
||||
t.Fatalf("lease_keepalive_once failed: %v", err)
|
||||
}
|
||||
leaseListed, err := state.leaseList(paramsWith(map[string]json.RawMessage{}))
|
||||
if err != nil {
|
||||
t.Fatalf("lease_list failed: %v", err)
|
||||
}
|
||||
leaseListRow := leaseListed.(map[string]any)
|
||||
if leaseListRow["partial"] != false {
|
||||
t.Fatalf("lease list must be complete on etcd >= 3.4: %#v", leaseListRow)
|
||||
}
|
||||
foundLease := false
|
||||
for _, lease := range leaseListRow["leases"].([]map[string]any) {
|
||||
if lease["id"] == leaseID {
|
||||
foundLease = true
|
||||
}
|
||||
}
|
||||
if !foundLease {
|
||||
t.Fatalf("granted lease %s missing from list: %#v", leaseID, leaseListRow)
|
||||
}
|
||||
leaseDetail, err := state.leaseGet(paramsWith(map[string]json.RawMessage{
|
||||
"id": json.RawMessage(leaseID),
|
||||
"includeKeys": json.RawMessage(`true`),
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("lease_get failed: %v", err)
|
||||
}
|
||||
detail := leaseDetail.(map[string]any)
|
||||
if detail["grantedTtl"].(int64) != 120 && detail["grantedTtl"].(int64) < 100 {
|
||||
t.Fatalf("unexpected granted ttl: %#v", detail)
|
||||
}
|
||||
if _, err := state.leaseRevoke(paramsWith(map[string]json.RawMessage{"id": json.RawMessage(leaseID)})); err != nil {
|
||||
t.Fatalf("lease_revoke failed: %v", err)
|
||||
}
|
||||
|
||||
// --- auth ----------------------------------------------------------
|
||||
userName := fmt.Sprintf("dbx_live_%d", time.Now().UnixNano()%100000)
|
||||
if _, err := state.authUserAdd(paramsWith(map[string]json.RawMessage{
|
||||
"user": jsonString(userName),
|
||||
"password": jsonString("secret123"),
|
||||
})); err != nil {
|
||||
t.Fatalf("user add failed: %v", err)
|
||||
}
|
||||
if _, err := state.authRoleAdd(paramsWith(map[string]json.RawMessage{"role": jsonString("dbx_live_role")})); err != nil {
|
||||
t.Fatalf("role add failed: %v", err)
|
||||
}
|
||||
if _, err := state.authRolePermission(paramsWith(map[string]json.RawMessage{
|
||||
"role": jsonString("dbx_live_role"),
|
||||
"resource": jsonString("prefix"),
|
||||
"key": jsonString(prefix),
|
||||
"access": jsonString("READWRITE"),
|
||||
}), true); err != nil {
|
||||
t.Fatalf("role grant permission failed: %v", err)
|
||||
}
|
||||
roleDetail, err := state.authRoleGet(paramsWith(map[string]json.RawMessage{"role": jsonString("dbx_live_role")}))
|
||||
if err != nil {
|
||||
t.Fatalf("role get failed: %v", err)
|
||||
}
|
||||
permissions := roleDetail.(map[string]any)["permissions"].([]map[string]any)
|
||||
if len(permissions) != 1 || permissions[0]["access"] != "readwrite" || permissions[0]["resource"] != "prefix" {
|
||||
t.Fatalf("unexpected permission shape: %#v", permissions)
|
||||
}
|
||||
if _, err := state.authUserGrantRevokeRole(paramsWith(map[string]json.RawMessage{
|
||||
"user": jsonString(userName),
|
||||
"role": jsonString("dbx_live_role"),
|
||||
}), true); err != nil {
|
||||
t.Fatalf("grant role failed: %v", err)
|
||||
}
|
||||
userDetail, err := state.authUserGet(paramsWith(map[string]json.RawMessage{"user": jsonString(userName)}))
|
||||
if err != nil {
|
||||
t.Fatalf("user get failed: %v", err)
|
||||
}
|
||||
if len(userDetail.(map[string]any)["roles"].([]string)) == 0 {
|
||||
t.Fatalf("expected granted role: %#v", userDetail)
|
||||
}
|
||||
users, err := state.authUserList(paramsWith(map[string]json.RawMessage{}))
|
||||
if err != nil {
|
||||
t.Fatalf("user list failed: %v", err)
|
||||
}
|
||||
if !containsAny(users.(map[string]any)["users"].([]string), userName) {
|
||||
t.Fatalf("user list missing created user: %#v", users)
|
||||
}
|
||||
if _, err := state.authUserGrantRevokeRole(paramsWith(map[string]json.RawMessage{
|
||||
"user": jsonString(userName),
|
||||
"role": jsonString("dbx_live_role"),
|
||||
}), false); err != nil {
|
||||
t.Fatalf("revoke role failed: %v", err)
|
||||
}
|
||||
if _, err := state.authRolePermission(paramsWith(map[string]json.RawMessage{
|
||||
"role": jsonString("dbx_live_role"),
|
||||
"resource": jsonString("prefix"),
|
||||
"key": jsonString(prefix),
|
||||
}), false); err != nil {
|
||||
t.Fatalf("role revoke permission failed: %v", err)
|
||||
}
|
||||
if _, err := state.authUserDelete(paramsWith(map[string]json.RawMessage{"user": jsonString(userName)})); err != nil {
|
||||
t.Fatalf("user delete failed: %v", err)
|
||||
}
|
||||
if _, err := state.authRoleDelete(paramsWith(map[string]json.RawMessage{"role": jsonString("dbx_live_role")})); err != nil {
|
||||
t.Fatalf("role delete failed: %v", err)
|
||||
}
|
||||
|
||||
// --- status / maintenance ------------------------------------------
|
||||
status, err := state.status(paramsWith(map[string]json.RawMessage{}))
|
||||
if err != nil {
|
||||
t.Fatalf("kv_status failed: %v", err)
|
||||
}
|
||||
statusRow := status.(map[string]any)
|
||||
if statusRow["clusterId"] == nil || statusRow["leaderId"] == nil {
|
||||
t.Fatalf("status missing cluster identity: %#v", statusRow)
|
||||
}
|
||||
members := statusRow["members"].([]map[string]any)
|
||||
if len(members) == 0 || members[0]["reachable"] != true || members[0]["version"] == nil {
|
||||
t.Fatalf("unexpected member rows: %#v", members)
|
||||
}
|
||||
if keyCount, err := strconvParse(statusRow["keyCount"].(string)); err != nil || keyCount <= 0 {
|
||||
t.Fatalf("keyCount must be a positive numeric string: %#v", statusRow["keyCount"])
|
||||
}
|
||||
|
||||
compactTarget := revisionOf(t, putResult)
|
||||
if compactTarget < baseRevision {
|
||||
compactTarget = baseRevision
|
||||
}
|
||||
if _, err := state.compact(paramsWith(map[string]json.RawMessage{"revision": json.RawMessage(statusRow["revision"].(string))})); err != nil {
|
||||
t.Fatalf("compact failed: %v", err)
|
||||
}
|
||||
if _, err := state.compact(paramsWith(map[string]json.RawMessage{"revision": json.RawMessage(`999999999999`)})); err == nil || !strings.Contains(err.Error(), "future revision") {
|
||||
t.Fatalf("expected raw future-revision error (Java parity), got %v", err)
|
||||
}
|
||||
defragged, err := state.defrag(paramsWith(map[string]json.RawMessage{
|
||||
"endpoints": json.RawMessage(`["` + endpoints + `"]`),
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("defrag failed: %v", err)
|
||||
}
|
||||
defragRow := defragged.(map[string]any)["members"].([]map[string]any)[0]
|
||||
if defragRow["status"] != "succeeded" {
|
||||
t.Fatalf("defrag should succeed on a live member: %#v", defragRow)
|
||||
}
|
||||
|
||||
// --- delete + cleanup ----------------------------------------------
|
||||
for _, key := range []string{prefix + "binary", prefix + "leased", prefix + "renamed", prefix + "watched"} {
|
||||
if _, err := state.delete(paramsWith(map[string]json.RawMessage{"key": jsonString(key)})); err != nil {
|
||||
t.Fatalf("delete %s failed: %v", key, err)
|
||||
}
|
||||
}
|
||||
if _, err := state.delete(paramsWith(map[string]json.RawMessage{"key": jsonString(prefix + "renamed")})); err != nil {
|
||||
t.Fatalf("delete missing key failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLiveEtcdProtocolFlow drives the multi-session runtime exactly like the
|
||||
// Rust host does: NDJSON lines with agentSessionId routing.
|
||||
func TestLiveEtcdProtocolFlow(t *testing.T) {
|
||||
if os.Getenv("DBX_ETCD_LIVE") != "1" {
|
||||
t.Skip("set DBX_ETCD_LIVE=1 to run the live etcd agent test")
|
||||
}
|
||||
endpoints := envOrDefault("DBX_ETCD_ENDPOINTS", "http://127.0.0.1:10700")
|
||||
user := envOrDefault("DBX_ETCD_USER", "root")
|
||||
password := envOrDefault("DBX_ETCD_PASSWORD", "123456")
|
||||
|
||||
server := newRuntimeServer()
|
||||
connection := fmt.Sprintf(`{"etcd_endpoints":%q,"username":%q,"password":%q}`, endpoints, user, password)
|
||||
|
||||
handshake, _ := server.handleLine(`{"id":1,"method":"handshake","params":{}}`)
|
||||
if handshake.Error != nil {
|
||||
t.Fatalf("handshake failed: %#v", handshake)
|
||||
}
|
||||
|
||||
openLine := fmt.Sprintf(`{"id":2,"method":"open_session","params":{"agentSessionId":"s1","connection":%s}}`, connection)
|
||||
opened, _ := server.handleLine(openLine)
|
||||
if opened.Error != nil {
|
||||
t.Fatalf("open_session failed: %#v", opened)
|
||||
}
|
||||
|
||||
putLine := fmt.Sprintf(`{"id":3,"method":"kv_put","params":{"agentSessionId":"s1","key":"dbx:proto","value":{"encoding":"utf8","data":"flow"}}}`)
|
||||
put, _ := server.handleLine(putLine)
|
||||
if put.Error != nil {
|
||||
t.Fatalf("kv_put failed: %#v", put)
|
||||
}
|
||||
|
||||
unknown, _ := server.handleLine(`{"id":4,"method":"kv_describe","params":{"agentSessionId":"s1"}}`)
|
||||
if unknown.Error == nil || !strings.Contains(unknown.Error.Message, "unknown method") {
|
||||
t.Fatalf("expected unknown-method error, got %#v", unknown)
|
||||
}
|
||||
|
||||
closed, _ := server.handleLine(`{"id":5,"method":"close_session","params":{"agentSessionId":"s1"}}`)
|
||||
if closed.Error != nil {
|
||||
t.Fatalf("close_session failed: %#v", closed)
|
||||
}
|
||||
}
|
||||
|
||||
func envOrDefault(key, fallback string) string {
|
||||
if value := strings.TrimSpace(os.Getenv(key)); value != "" {
|
||||
return value
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func paramsWith(params map[string]json.RawMessage) map[string]json.RawMessage {
|
||||
return params
|
||||
}
|
||||
|
||||
func jsonString(value string) json.RawMessage {
|
||||
encoded, _ := json.Marshal(value)
|
||||
return json.RawMessage(encoded)
|
||||
}
|
||||
|
||||
func jsonValue(value string) json.RawMessage {
|
||||
encoded, _ := json.Marshal(value)
|
||||
return json.RawMessage(`{"encoding":"utf8","data":` + string(encoded) + `}`)
|
||||
}
|
||||
|
||||
func revisionOf(t *testing.T, result any) int64 {
|
||||
t.Helper()
|
||||
row, ok := result.(map[string]any)
|
||||
if !ok {
|
||||
t.Fatalf("expected map result, got %#v", result)
|
||||
}
|
||||
revision, err := strconvParse(row["revision"].(string))
|
||||
if err != nil {
|
||||
t.Fatalf("revision not numeric: %#v", row["revision"])
|
||||
}
|
||||
return revision
|
||||
}
|
||||
|
||||
func strconvParse(value string) (int64, error) {
|
||||
var parsed int64
|
||||
_, err := fmt.Sscan(value, &parsed)
|
||||
return parsed, err
|
||||
}
|
||||
|
||||
func containsAny(values []string, target string) bool {
|
||||
for _, value := range values {
|
||||
if value == target {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,524 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"go.etcd.io/etcd/api/v3/mvccpb"
|
||||
clientv3 "go.etcd.io/etcd/client/v3"
|
||||
)
|
||||
|
||||
const defaultListLimit = 100
|
||||
const preserveLeaseMaxAttempts = 3
|
||||
|
||||
func (s *etcdSession) listPrefix(params map[string]json.RawMessage) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
prefix := stringOrDefault(params, "prefix", "")
|
||||
limit := intOrDefault(params, "limit", defaultListLimit)
|
||||
if limit < 1 {
|
||||
limit = 1
|
||||
}
|
||||
revision := longOrNull(params, "revision")
|
||||
includeValues := boolOrDefault(params, "includeValues", false)
|
||||
continuation := stringOrNull(params, "continuation")
|
||||
|
||||
start := prefixStart(prefix)
|
||||
if continuation != nil && *continuation != "" {
|
||||
decoded, err := base64.StdEncoding.DecodeString(*continuation)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
start = string(decoded)
|
||||
}
|
||||
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
options := []clientv3.OpOption{
|
||||
clientv3.WithRange(prefixEnd(prefix)),
|
||||
clientv3.WithLimit(int64(limit)),
|
||||
clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend),
|
||||
}
|
||||
if revision != nil && *revision > 0 {
|
||||
options = append(options, clientv3.WithRev(*revision))
|
||||
}
|
||||
response, err := client.Get(ctx, start, options...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
keys := make([]any, 0, len(response.Kvs))
|
||||
for _, item := range response.Kvs {
|
||||
row := metadataMap(item)
|
||||
row["key"] = displayBytes(item.Key)
|
||||
row["keyBytes"] = bytesObject(item.Key)
|
||||
if includeValues {
|
||||
row["value"] = valueObject(item.Value)
|
||||
}
|
||||
keys = append(keys, row)
|
||||
}
|
||||
|
||||
result := map[string]any{"keys": keys}
|
||||
if response.More && len(response.Kvs) > 0 {
|
||||
result["continuation"] = nextContinuation(response.Kvs[len(response.Kvs)-1].Key)
|
||||
} else {
|
||||
result["continuation"] = nil
|
||||
}
|
||||
result["revision"] = longString(response.Header.Revision)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *etcdSession) get(params map[string]json.RawMessage) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key, err := keyBytesParam(params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
revision := longOrNull(params, "revision")
|
||||
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
options := []clientv3.OpOption{}
|
||||
if revision != nil && *revision > 0 {
|
||||
options = append(options, clientv3.WithRev(*revision))
|
||||
}
|
||||
response, err := client.Get(ctx, key, options...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(response.Kvs) == 0 {
|
||||
return map[string]any{"found": false, "key": nil, "value": nil, "metadata": nil}, nil
|
||||
}
|
||||
item := response.Kvs[0]
|
||||
result := map[string]any{
|
||||
"found": true,
|
||||
"key": displayBytes(item.Key),
|
||||
"keyBytes": bytesObject(item.Key),
|
||||
}
|
||||
if !boolOrDefault(params, "metadataOnly", false) {
|
||||
result["value"] = valueObject(item.Value)
|
||||
} else {
|
||||
result["value"] = nil
|
||||
}
|
||||
metadata, err := s.metadataWithTtl(item)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result["metadata"] = metadata
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *etcdSession) put(params map[string]json.RawMessage) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key, err := keyBytesParam(params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
value, err := parseValueObject(rawObject(params, "value"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
expectedModRevision := longOrNull(params, "expectedModRevision")
|
||||
expectedCreateRevision := longOrNull(params, "expectedCreateRevision")
|
||||
leaseValue := longOrNull(params, "lease")
|
||||
ttlValue := longOrNull(params, "ttl")
|
||||
preserveLease := boolOrDefault(params, "preserveLease", false)
|
||||
hasLease := leaseValue != nil
|
||||
hasTtl := ttlValue != nil
|
||||
if (hasLease && hasTtl) || (preserveLease && (hasLease || hasTtl)) {
|
||||
return nil, errors.New("lease, ttl, and preserveLease cannot be specified together")
|
||||
}
|
||||
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
if preserveLease {
|
||||
revision, err := putPreservingLease(client, ctx, key, value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"revision": longString(revision)}, nil
|
||||
}
|
||||
|
||||
var leaseID clientv3.LeaseID
|
||||
var grantedLeaseID clientv3.LeaseID
|
||||
if hasTtl {
|
||||
if *ttlValue <= 0 {
|
||||
return nil, errors.New("ttl must be a positive integer")
|
||||
}
|
||||
grant, err := client.Lease.Grant(ctx, *ttlValue)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
grantedLeaseID = grant.ID
|
||||
leaseID = grant.ID
|
||||
} else if hasLease {
|
||||
leaseID = clientv3.LeaseID(*leaseValue)
|
||||
}
|
||||
|
||||
revision, err := func() (int64, error) {
|
||||
if expectedModRevision != nil || expectedCreateRevision != nil {
|
||||
var comparisons []clientv3.Cmp
|
||||
if expectedModRevision != nil {
|
||||
comparisons = append(comparisons, clientv3.Compare(clientv3.ModRevision(key), "=", *expectedModRevision))
|
||||
}
|
||||
if expectedCreateRevision != nil {
|
||||
comparisons = append(comparisons, clientv3.Compare(clientv3.CreateRevision(key), "=", *expectedCreateRevision))
|
||||
}
|
||||
txn := client.Txn(ctx).If(comparisons...).Then(clientv3.OpPut(key, value, clientv3.WithLease(leaseID)))
|
||||
response, err := txn.Commit()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if !response.Succeeded {
|
||||
return 0, errors.New("ETCD_CAS_CONFLICT: key changed after it was loaded")
|
||||
}
|
||||
return response.Header.Revision, nil
|
||||
}
|
||||
response, err := client.Put(ctx, key, value, clientv3.WithLease(leaseID))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return response.Header.Revision, nil
|
||||
}()
|
||||
if err != nil && grantedLeaseID != 0 {
|
||||
_, _ = client.Lease.Revoke(context.Background(), grantedLeaseID)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"revision": longString(revision)}, nil
|
||||
}
|
||||
|
||||
func putPreservingLease(client *clientv3.Client, ctx context.Context, key, value string) (int64, error) {
|
||||
for attempt := 0; attempt < preserveLeaseMaxAttempts; attempt++ {
|
||||
existing, err := client.Get(ctx, key)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(existing.Kvs) == 0 || existing.Kvs[0].Lease <= 0 {
|
||||
return 0, errors.New("Cannot preserve lease: key does not exist or has no lease")
|
||||
}
|
||||
current := existing.Kvs[0]
|
||||
txn := client.Txn(ctx).
|
||||
If(clientv3.Compare(clientv3.ModRevision(key), "=", current.ModRevision)).
|
||||
Then(clientv3.OpPut(key, value, clientv3.WithLease(clientv3.LeaseID(current.Lease))))
|
||||
response, err := txn.Commit()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if response.Succeeded {
|
||||
return response.Header.Revision, nil
|
||||
}
|
||||
}
|
||||
return 0, errors.New("Cannot preserve lease: key changed concurrently; retry the save")
|
||||
}
|
||||
|
||||
func (s *etcdSession) delete(params map[string]json.RawMessage) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key, err := keyBytesParam(params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
expectedModRevision := longOrNull(params, "expectedModRevision")
|
||||
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
if expectedModRevision != nil {
|
||||
txn := client.Txn(ctx).
|
||||
If(clientv3.Compare(clientv3.ModRevision(key), "=", *expectedModRevision)).
|
||||
Then(clientv3.OpDelete(key))
|
||||
response, err := txn.Commit()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !response.Succeeded {
|
||||
return nil, errors.New("ETCD_CAS_CONFLICT: key changed after it was loaded")
|
||||
}
|
||||
return map[string]any{"deleted": int64(1), "revision": longString(response.Header.Revision)}, nil
|
||||
}
|
||||
response, err := client.Delete(ctx, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"deleted": response.Deleted, "revision": longString(response.Header.Revision)}, nil
|
||||
}
|
||||
|
||||
func (s *etcdSession) rename(params map[string]json.RawMessage) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sourceKey, err := keyBytesParam(params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
newKey := stringOrNull(params, "newKey")
|
||||
if newKey == nil || *newKey == "" {
|
||||
return nil, errors.New("ETCD_NEWKEY_REQUIRED")
|
||||
}
|
||||
targetKey := *newKey
|
||||
if sourceKey == targetKey {
|
||||
return map[string]any{"renamed": true, "revision": nil}, nil
|
||||
}
|
||||
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
sourceResponse, err := client.Get(ctx, sourceKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(sourceResponse.Kvs) == 0 {
|
||||
return nil, errors.New("ETCD_NOT_FOUND: source key does not exist")
|
||||
}
|
||||
source := sourceResponse.Kvs[0]
|
||||
expected := longOrNull(params, "expectedModRevision")
|
||||
expectedRevision := source.ModRevision
|
||||
if expected != nil {
|
||||
expectedRevision = *expected
|
||||
}
|
||||
putOption := []clientv3.OpOption{}
|
||||
if source.Lease != 0 {
|
||||
putOption = append(putOption, clientv3.WithLease(clientv3.LeaseID(source.Lease)))
|
||||
}
|
||||
txn := client.Txn(ctx).
|
||||
If(
|
||||
clientv3.Compare(clientv3.ModRevision(sourceKey), "=", expectedRevision),
|
||||
clientv3.Compare(clientv3.CreateRevision(targetKey), "=", 0),
|
||||
).
|
||||
Then(
|
||||
clientv3.OpPut(targetKey, string(source.Value), putOption...),
|
||||
clientv3.OpDelete(sourceKey),
|
||||
)
|
||||
response, err := txn.Commit()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !response.Succeeded {
|
||||
return nil, errors.New("ETCD_CAS_CONFLICT: source changed or target already exists")
|
||||
}
|
||||
return map[string]any{"renamed": true, "revision": longString(response.Header.Revision)}, nil
|
||||
}
|
||||
|
||||
func (s *etcdSession) metadataWithTtl(item *mvccpb.KeyValue) (map[string]any, error) {
|
||||
metadata := metadataMap(item)
|
||||
if item.Lease > 0 {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
|
||||
defer cancel()
|
||||
lease, err := client.Lease.TimeToLive(ctx, clientv3.LeaseID(item.Lease))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
metadata["ttl"] = lease.TTL
|
||||
}
|
||||
return metadata, nil
|
||||
}
|
||||
|
||||
func metadataMap(item *mvccpb.KeyValue) map[string]any {
|
||||
return map[string]any{
|
||||
"createRevision": longString(item.CreateRevision),
|
||||
"modRevision": longString(item.ModRevision),
|
||||
"version": longString(item.Version),
|
||||
"lease": longString(item.Lease),
|
||||
"valueSize": len(item.Value),
|
||||
}
|
||||
}
|
||||
|
||||
func prefixEnd(prefix string) string {
|
||||
if prefix == "" {
|
||||
return "\x00"
|
||||
}
|
||||
end := []byte(prefix)
|
||||
for i := len(end) - 1; i >= 0; i-- {
|
||||
if end[i] < 0xff {
|
||||
end[i]++
|
||||
return string(end[:i+1])
|
||||
}
|
||||
}
|
||||
return "\x00"
|
||||
}
|
||||
|
||||
func prefixStart(prefix string) string {
|
||||
if prefix == "" {
|
||||
return "\x00"
|
||||
}
|
||||
return prefix
|
||||
}
|
||||
|
||||
func nextContinuation(key []byte) string {
|
||||
next := make([]byte, len(key)+1)
|
||||
copy(next, key)
|
||||
return base64.StdEncoding.EncodeToString(next)
|
||||
}
|
||||
|
||||
func keyBytesParam(params map[string]json.RawMessage) (string, error) {
|
||||
if encoded := rawObject(params, "keyBytes"); encoded != nil {
|
||||
value, err := parseValueObject(encoded)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
if raw, ok := params["key"]; ok && raw != nil && string(raw) != "null" {
|
||||
var key string
|
||||
if err := json.Unmarshal(raw, &key); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
return "", errors.New("Key is required")
|
||||
}
|
||||
|
||||
func rawObject(params map[string]json.RawMessage, key string) map[string]json.RawMessage {
|
||||
raw := params[key]
|
||||
if len(raw) == 0 || string(raw) == "null" {
|
||||
return nil
|
||||
}
|
||||
var object map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &object); err != nil || object == nil {
|
||||
return nil
|
||||
}
|
||||
return object
|
||||
}
|
||||
|
||||
func bytesObject(bytes []byte) map[string]any {
|
||||
return map[string]any{
|
||||
"encoding": "base64",
|
||||
"data": base64.StdEncoding.EncodeToString(bytes),
|
||||
}
|
||||
}
|
||||
|
||||
func valueObject(value []byte) map[string]any {
|
||||
if utf8.Valid(value) {
|
||||
return map[string]any{"encoding": "utf8", "data": string(value)}
|
||||
}
|
||||
return bytesObject(value)
|
||||
}
|
||||
|
||||
func displayBytes(bytes []byte) string {
|
||||
if utf8.Valid(bytes) {
|
||||
return string(bytes)
|
||||
}
|
||||
return base64.StdEncoding.EncodeToString(bytes)
|
||||
}
|
||||
|
||||
func parseValueObject(value map[string]json.RawMessage) (string, error) {
|
||||
if value == nil {
|
||||
value = map[string]json.RawMessage{}
|
||||
}
|
||||
encoding := stringOrDefault(value, "encoding", "utf8")
|
||||
data := stringOrDefault(value, "data", "")
|
||||
if encoding == "base64" {
|
||||
decoded, err := base64.StdEncoding.DecodeString(data)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(decoded), nil
|
||||
}
|
||||
if encoding != "utf8" {
|
||||
return "", fmt.Errorf("Unsupported value encoding: %s", encoding)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func longString(value int64) string {
|
||||
return strconv.FormatInt(value, 10)
|
||||
}
|
||||
|
||||
func unsignedLongString(value int64) string {
|
||||
return strconv.FormatUint(uint64(value), 10)
|
||||
}
|
||||
|
||||
func stringOrNull(params map[string]json.RawMessage, key string) *string {
|
||||
raw := params[key]
|
||||
if len(raw) == 0 || string(raw) == "null" {
|
||||
return nil
|
||||
}
|
||||
var value string
|
||||
if err := json.Unmarshal(raw, &value); err != nil {
|
||||
var text *string
|
||||
return text
|
||||
}
|
||||
return &value
|
||||
}
|
||||
|
||||
func stringOrDefault(params map[string]json.RawMessage, key string, fallback string) string {
|
||||
value := stringOrNull(params, key)
|
||||
if value == nil {
|
||||
return fallback
|
||||
}
|
||||
return *value
|
||||
}
|
||||
|
||||
func intOrDefault(params map[string]json.RawMessage, key string, fallback int) int {
|
||||
raw := params[key]
|
||||
if len(raw) == 0 || string(raw) == "null" {
|
||||
return fallback
|
||||
}
|
||||
var value int
|
||||
if err := json.Unmarshal(raw, &value); err != nil {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func boolOrDefault(params map[string]json.RawMessage, key string, fallback bool) bool {
|
||||
raw := params[key]
|
||||
if len(raw) == 0 || string(raw) == "null" {
|
||||
return fallback
|
||||
}
|
||||
var value bool
|
||||
if err := json.Unmarshal(raw, &value); err != nil {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func longOrNull(params map[string]json.RawMessage, key string) *int64 {
|
||||
raw := params[key]
|
||||
if len(raw) == 0 || string(raw) == "null" {
|
||||
return nil
|
||||
}
|
||||
var value int64
|
||||
if err := json.Unmarshal(raw, &value); err != nil {
|
||||
return nil
|
||||
}
|
||||
return &value
|
||||
}
|
||||
|
||||
func requiredPositiveLong(params map[string]json.RawMessage, field string) (int64, error) {
|
||||
value := longOrNull(params, field)
|
||||
if value == nil || *value <= 0 {
|
||||
return 0, fmt.Errorf("ETCD_INVALID_%s: a positive integer is required", strings.ToUpper(field))
|
||||
}
|
||||
return *value, nil
|
||||
}
|
||||
|
||||
func requiredString(params map[string]json.RawMessage, field string) (string, error) {
|
||||
value := stringOrNull(params, field)
|
||||
if value == nil || *value == "" {
|
||||
return "", fmt.Errorf("ETCD_%s_REQUIRED", strings.ToUpper(field))
|
||||
}
|
||||
return *value, nil
|
||||
}
|
||||
@@ -0,0 +1,339 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"go.etcd.io/etcd/api/v3/etcdserverpb"
|
||||
clientv3 "go.etcd.io/etcd/client/v3"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
const (
|
||||
maxLeaseAttachedKeys = 256
|
||||
defaultLeaseListLimit = 100
|
||||
maxLeaseListLimit = 200
|
||||
leaseListConcurrency = 8
|
||||
leaseListDeadlineSeconds = 5
|
||||
)
|
||||
|
||||
type leaseListDeadline struct {
|
||||
deadline time.Time
|
||||
}
|
||||
|
||||
func (d leaseListDeadline) remaining() (time.Duration, error) {
|
||||
remaining := time.Until(d.deadline)
|
||||
if remaining <= 0 {
|
||||
return 0, errors.New("ETCD_LEASE_LIST_TIMEOUT")
|
||||
}
|
||||
return remaining, nil
|
||||
}
|
||||
|
||||
func (s *etcdSession) leaseList(params map[string]json.RawMessage) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
deadline := leaseListDeadline{deadline: time.Now().Add(leaseListDeadlineSeconds * time.Second)}
|
||||
limit := intOrDefault(params, "limit", defaultLeaseListLimit)
|
||||
if limit < 1 {
|
||||
limit = 1
|
||||
}
|
||||
if limit > maxLeaseListLimit {
|
||||
limit = maxLeaseListLimit
|
||||
}
|
||||
continuation := stringOrNull(params, "continuation")
|
||||
var afterLeaseID *uint64
|
||||
if continuation != nil && *continuation != "" {
|
||||
parsed, err := parseUnsignedLong(*continuation)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
afterLeaseID = &parsed
|
||||
}
|
||||
|
||||
partial := false
|
||||
leaseIDs, err := clusterLeaseIDs(client, deadline)
|
||||
if err != nil {
|
||||
if isLeaseListFallbackError(err) {
|
||||
leaseIDs = s.knownLeaseIDs()
|
||||
partial = true
|
||||
} else {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
for _, id := range leaseIDs {
|
||||
s.rememberLease(id)
|
||||
}
|
||||
}
|
||||
leaseIDs = leasePageIDs(leaseIDs, afterLeaseID, limit+1)
|
||||
hasMore := len(leaseIDs) > limit
|
||||
if len(leaseIDs) > limit {
|
||||
leaseIDs = leaseIDs[:limit]
|
||||
}
|
||||
|
||||
leases := []map[string]any{}
|
||||
var lastProcessedID *uint64
|
||||
deadlineReached := false
|
||||
chunkLoop:
|
||||
for offset := 0; offset < len(leaseIDs); offset += leaseListConcurrency {
|
||||
end := offset + leaseListConcurrency
|
||||
if end > len(leaseIDs) {
|
||||
end = len(leaseIDs)
|
||||
}
|
||||
chunkIDs := leaseIDs[offset:end]
|
||||
|
||||
type ttlResult struct {
|
||||
id uint64
|
||||
ttl int64
|
||||
granted int64
|
||||
err error
|
||||
}
|
||||
results := make([]chan ttlResult, len(chunkIDs))
|
||||
for index, id := range chunkIDs {
|
||||
channel := make(chan ttlResult, 1)
|
||||
results[index] = channel
|
||||
go func(id uint64, channel chan ttlResult) {
|
||||
remaining, err := deadline.remaining()
|
||||
if err != nil {
|
||||
channel <- ttlResult{id: id, err: err}
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), remaining)
|
||||
defer cancel()
|
||||
response, err := client.Lease.TimeToLive(ctx, clientv3.LeaseID(int64(id)))
|
||||
if err != nil {
|
||||
channel <- ttlResult{id: id, err: err}
|
||||
return
|
||||
}
|
||||
channel <- ttlResult{id: id, ttl: response.TTL, granted: response.GrantedTTL}
|
||||
}(id, channel)
|
||||
}
|
||||
for _, channel := range results {
|
||||
result := <-channel
|
||||
id := result.id
|
||||
if result.err != nil {
|
||||
if errors.Is(result.err, errLeaseListDeadline) || isDeadlineError(result.err) {
|
||||
partial = true
|
||||
deadlineReached = true
|
||||
break chunkLoop
|
||||
}
|
||||
if status.Code(result.err) == codes.NotFound {
|
||||
s.forgetLease(id)
|
||||
} else {
|
||||
partial = true
|
||||
}
|
||||
lastProcessedID = &id
|
||||
continue
|
||||
}
|
||||
leases = append(leases, map[string]any{
|
||||
"id": unsignedLongString(int64(id)),
|
||||
"ttl": result.ttl,
|
||||
"grantedTtl": result.granted,
|
||||
})
|
||||
processed := id
|
||||
lastProcessedID = &processed
|
||||
}
|
||||
}
|
||||
|
||||
var nextContinuation any
|
||||
if deadlineReached && len(leaseIDs) > 0 {
|
||||
if lastProcessedID != nil {
|
||||
nextContinuation = unsignedLongString(int64(*lastProcessedID))
|
||||
} else if afterLeaseID != nil {
|
||||
nextContinuation = unsignedLongString(int64(*afterLeaseID))
|
||||
} else {
|
||||
nextContinuation = "0"
|
||||
}
|
||||
} else if hasMore && len(leaseIDs) > 0 {
|
||||
nextContinuation = unsignedLongString(int64(leaseIDs[len(leaseIDs)-1]))
|
||||
}
|
||||
return map[string]any{
|
||||
"leases": leases,
|
||||
"partial": partial,
|
||||
"nextContinuation": nextContinuation,
|
||||
}, nil
|
||||
}
|
||||
|
||||
var errLeaseListDeadline = errors.New("ETCD_LEASE_LIST_TIMEOUT")
|
||||
|
||||
func isDeadlineError(err error) bool {
|
||||
return errors.Is(err, context.DeadlineExceeded) || status.Code(err) == codes.DeadlineExceeded
|
||||
}
|
||||
|
||||
func isLeaseListFallbackError(err error) bool {
|
||||
if errors.Is(err, errLeaseListDeadline) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return true
|
||||
}
|
||||
code := status.Code(err)
|
||||
return code == codes.Unimplemented || code == codes.DeadlineExceeded
|
||||
}
|
||||
|
||||
func parseUnsignedLong(value string) (uint64, error) {
|
||||
parsed, err := strconv.ParseUint(value, 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid lease continuation: %s", value)
|
||||
}
|
||||
return parsed, nil
|
||||
}
|
||||
|
||||
func leasePageIDs(leaseIDs []uint64, afterLeaseID *uint64, fetchLimit int) []uint64 {
|
||||
sorted := make([]uint64, len(leaseIDs))
|
||||
copy(sorted, leaseIDs)
|
||||
sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] })
|
||||
result := []uint64{}
|
||||
for _, id := range sorted {
|
||||
if afterLeaseID != nil && id <= *afterLeaseID {
|
||||
continue
|
||||
}
|
||||
if len(result) >= fetchLimit {
|
||||
break
|
||||
}
|
||||
result = append(result, id)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func clusterLeaseIDs(client *clientv3.Client, deadline leaseListDeadline) ([]uint64, error) {
|
||||
remaining, err := deadline.remaining()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), remaining)
|
||||
defer cancel()
|
||||
leaseClient := etcdserverpb.NewLeaseClient(client.ActiveConnection())
|
||||
response, err := leaseClient.LeaseLeases(ctx, &etcdserverpb.LeaseLeasesRequest{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ids := make([]uint64, 0, len(response.Leases))
|
||||
for _, lease := range response.Leases {
|
||||
ids = append(ids, uint64(lease.ID))
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
func (s *etcdSession) leaseGet(params map[string]json.RawMessage) (any, error) {
|
||||
id, err := requiredPositiveLong(params, "id")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
options := []clientv3.LeaseOption{}
|
||||
if boolOrDefault(params, "includeKeys", false) {
|
||||
options = append(options, clientv3.WithAttachedKeys())
|
||||
}
|
||||
response, err := client.Lease.TimeToLive(ctx, clientv3.LeaseID(id), options...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.rememberLease(uint64(id))
|
||||
keys := []map[string]any{}
|
||||
for index, key := range response.Keys {
|
||||
if index >= maxLeaseAttachedKeys {
|
||||
break
|
||||
}
|
||||
keys = append(keys, bytesObject(key))
|
||||
}
|
||||
return map[string]any{
|
||||
"id": unsignedLongString(int64(response.ID)),
|
||||
"ttl": response.TTL,
|
||||
"grantedTtl": response.GrantedTTL,
|
||||
"keys": keys,
|
||||
"truncated": len(response.Keys) > maxLeaseAttachedKeys,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *etcdSession) leaseGrant(params map[string]json.RawMessage) (any, error) {
|
||||
ttl, err := requiredPositiveLong(params, "ttl")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
requestedID := longOrNull(params, "id")
|
||||
if requestedID != nil && *requestedID < 0 {
|
||||
return nil, errors.New("id must be a positive integer or 0")
|
||||
}
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
var grantedID clientv3.LeaseID
|
||||
var grantedTTL int64
|
||||
if requestedID == nil || *requestedID == 0 {
|
||||
response, err := client.Lease.Grant(ctx, ttl)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
grantedID = response.ID
|
||||
grantedTTL = response.TTL
|
||||
} else {
|
||||
leaseClient := etcdserverpb.NewLeaseClient(client.ActiveConnection())
|
||||
response, err := leaseClient.LeaseGrant(ctx, &etcdserverpb.LeaseGrantRequest{TTL: ttl, ID: *requestedID})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
grantedID = clientv3.LeaseID(response.ID)
|
||||
grantedTTL = response.TTL
|
||||
}
|
||||
s.rememberLease(uint64(grantedID))
|
||||
return map[string]any{
|
||||
"id": unsignedLongString(int64(grantedID)),
|
||||
"ttl": grantedTTL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *etcdSession) leaseKeepAlive(params map[string]json.RawMessage) (any, error) {
|
||||
id, err := requiredPositiveLong(params, "id")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
response, err := client.Lease.KeepAliveOnce(ctx, clientv3.LeaseID(id))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.rememberLease(uint64(id))
|
||||
return map[string]any{
|
||||
"id": unsignedLongString(id),
|
||||
"ttl": response.TTL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *etcdSession) leaseRevoke(params map[string]json.RawMessage) (any, error) {
|
||||
id, err := requiredPositiveLong(params, "id")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
if _, err := client.Lease.Revoke(ctx, clientv3.LeaseID(id)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.forgetLease(uint64(id))
|
||||
return map[string]any{
|
||||
"id": unsignedLongString(id),
|
||||
"revoked": true,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,468 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
clientv3 "go.etcd.io/etcd/client/v3"
|
||||
)
|
||||
|
||||
const (
|
||||
protocolVersion = 2
|
||||
legacyAgentSessionID = "__legacy__"
|
||||
maxAgentSessions = 256
|
||||
rpcTimeoutSeconds = 30
|
||||
)
|
||||
|
||||
var capabilities = []string{
|
||||
"connect", "test_connection", "kv", "kv_ttl", "kv_cas", "kv_list_values", "kv_status",
|
||||
"kv_history", "etcd_compaction", "etcd_defrag", "etcd_watch", "etcd_lease", "etcd_auth",
|
||||
"multi_session",
|
||||
}
|
||||
|
||||
type request struct {
|
||||
ID json.RawMessage `json:"id"`
|
||||
Method string `json:"method"`
|
||||
Params map[string]json.RawMessage `json:"params"`
|
||||
}
|
||||
|
||||
type response struct {
|
||||
JSONRPC string `json:"jsonrpc,omitempty"`
|
||||
ID json.RawMessage `json:"id,omitempty"`
|
||||
Result any `json:"result,omitempty"`
|
||||
Error *rpcError `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// etcdSession carries the per-connection etcd client and its derived state.
|
||||
// Requests for one session are serialized by agentSession.mu; watch callbacks
|
||||
// reach the maps through watchesMu instead so buffering never blocks dispatch.
|
||||
type etcdSession struct {
|
||||
clientMu sync.Mutex
|
||||
client *clientv3.Client
|
||||
connectedEndpoints []string
|
||||
|
||||
watchesMu sync.Mutex
|
||||
watches map[string]*watchState
|
||||
knownLeases map[uint64]struct{}
|
||||
watchBufferedBytes int64
|
||||
|
||||
activeCancelMu sync.Mutex
|
||||
activeCancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (s *etcdSession) reserveWatchBuffer(bytes int64) bool {
|
||||
s.watchesMu.Lock()
|
||||
defer s.watchesMu.Unlock()
|
||||
if bytes < 0 || s.watchBufferedBytes+bytes > maxSessionWatchBufferBytes {
|
||||
return false
|
||||
}
|
||||
s.watchBufferedBytes += bytes
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *etcdSession) releaseWatchBuffer(bytes int64) {
|
||||
s.watchesMu.Lock()
|
||||
defer s.watchesMu.Unlock()
|
||||
s.watchBufferedBytes -= bytes
|
||||
if s.watchBufferedBytes < 0 {
|
||||
s.watchBufferedBytes = 0
|
||||
}
|
||||
}
|
||||
|
||||
func (s *etcdSession) rememberLease(id uint64) {
|
||||
s.watchesMu.Lock()
|
||||
defer s.watchesMu.Unlock()
|
||||
s.knownLeases[id] = struct{}{}
|
||||
}
|
||||
|
||||
func (s *etcdSession) forgetLease(id uint64) {
|
||||
s.watchesMu.Lock()
|
||||
defer s.watchesMu.Unlock()
|
||||
delete(s.knownLeases, id)
|
||||
}
|
||||
|
||||
func (s *etcdSession) knownLeaseIDs() []uint64 {
|
||||
s.watchesMu.Lock()
|
||||
defer s.watchesMu.Unlock()
|
||||
ids := make([]uint64, 0, len(s.knownLeases))
|
||||
for id := range s.knownLeases {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
type agentSession struct {
|
||||
state *etcdSession
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func (a *agentSession) cancelActive() {
|
||||
a.state.activeCancelMu.Lock()
|
||||
cancel := a.state.activeCancel
|
||||
a.state.activeCancelMu.Unlock()
|
||||
if cancel != nil {
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
|
||||
type runtimeServer struct {
|
||||
mu sync.RWMutex
|
||||
sessions map[string]*agentSession
|
||||
}
|
||||
|
||||
func main() {
|
||||
configureRuntimeParallelism()
|
||||
server := newRuntimeServer()
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
var encoderMu sync.Mutex
|
||||
var requests sync.WaitGroup
|
||||
fmt.Fprintln(os.Stdout, `{"ready":true}`)
|
||||
|
||||
scanner := bufio.NewScanner(os.Stdin)
|
||||
scanner.Buffer(make([]byte, 0, 64*1024), 512*1024*1024)
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
var envelope request
|
||||
if json.Unmarshal([]byte(line), &envelope) == nil && envelope.Method == "shutdown" {
|
||||
requests.Wait()
|
||||
result, _ := server.handleLine(line)
|
||||
encoderMu.Lock()
|
||||
_ = encoder.Encode(result)
|
||||
encoderMu.Unlock()
|
||||
server.closeAll()
|
||||
return
|
||||
}
|
||||
requests.Add(1)
|
||||
go func(line string) {
|
||||
defer requests.Done()
|
||||
result, _ := server.handleLine(line)
|
||||
encoderMu.Lock()
|
||||
defer encoderMu.Unlock()
|
||||
if err := encoder.Encode(result); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "failed to write response: %v\n", err)
|
||||
}
|
||||
}(line)
|
||||
}
|
||||
requests.Wait()
|
||||
server.closeAll()
|
||||
}
|
||||
|
||||
func configureRuntimeParallelism() {
|
||||
if raw := strings.TrimSpace(os.Getenv("DBX_AGENT_ETCD_GOMAXPROCS")); raw != "" {
|
||||
if configured, err := strconv.Atoi(raw); err == nil && configured > 0 {
|
||||
runtime.GOMAXPROCS(configured)
|
||||
return
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(os.Getenv("GOMAXPROCS")) != "" {
|
||||
return
|
||||
}
|
||||
runtime.GOMAXPROCS(min(runtime.NumCPU(), 4))
|
||||
}
|
||||
|
||||
func newRuntimeServer() *runtimeServer {
|
||||
return &runtimeServer{sessions: map[string]*agentSession{}}
|
||||
}
|
||||
|
||||
func (r *runtimeServer) handleLine(line string) (response, bool) {
|
||||
var req request
|
||||
if err := json.Unmarshal([]byte(line), &req); err != nil {
|
||||
return errorResponse(nil, "", "", err), false
|
||||
}
|
||||
if len(req.ID) == 0 {
|
||||
req.ID = json.RawMessage("1")
|
||||
}
|
||||
result, shutdown, err := r.dispatch(req.Method, req.Params)
|
||||
if err != nil {
|
||||
return errorResponse(req.ID, req.Method, stringParam(req.Params, "agentSessionId"), err), false
|
||||
}
|
||||
return response{JSONRPC: "2.0", ID: req.ID, Result: result}, shutdown
|
||||
}
|
||||
|
||||
func (r *runtimeServer) dispatch(method string, params map[string]json.RawMessage) (any, bool, error) {
|
||||
switch method {
|
||||
case "handshake":
|
||||
return handshakeResult(), false, nil
|
||||
case "open_session":
|
||||
id := requiredSessionID(params)
|
||||
if id == "" {
|
||||
return nil, false, errors.New("agentSessionId is required")
|
||||
}
|
||||
return r.openSession(id, params)
|
||||
case "close_session":
|
||||
return r.closeSession(stringParam(params, "agentSessionId")), false, nil
|
||||
case "validate_session":
|
||||
session, err := r.session(requiredSessionID(params))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
session.mu.Lock()
|
||||
defer session.mu.Unlock()
|
||||
result, err := session.state.validateConnection()
|
||||
return result, false, err
|
||||
case "cancel_session":
|
||||
session, err := r.session(requiredSessionID(params))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
session.cancelActive()
|
||||
return map[string]bool{"ok": true}, false, nil
|
||||
case "test_connection":
|
||||
return r.testConnection(params)
|
||||
case "connect":
|
||||
_ = r.closeSession(legacyAgentSessionID)
|
||||
return r.openSession(legacyAgentSessionID, params)
|
||||
case "disconnect":
|
||||
return r.closeSession(legacyAgentSessionID), false, nil
|
||||
case "shutdown":
|
||||
return map[string]bool{"ok": true}, true, r.closeAll()
|
||||
default:
|
||||
id := stringParam(params, "agentSessionId")
|
||||
if id == "" {
|
||||
id = legacyAgentSessionID
|
||||
}
|
||||
session, err := r.session(id)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
session.mu.Lock()
|
||||
defer session.mu.Unlock()
|
||||
result, err := session.state.handle(method, params)
|
||||
return result, false, err
|
||||
}
|
||||
}
|
||||
|
||||
func requiredSessionID(params map[string]json.RawMessage) string {
|
||||
return strings.TrimSpace(stringParam(params, "agentSessionId"))
|
||||
}
|
||||
|
||||
func (r *runtimeServer) openSession(id string, params map[string]json.RawMessage) (any, bool, error) {
|
||||
r.mu.Lock()
|
||||
if _, exists := r.sessions[id]; exists {
|
||||
r.mu.Unlock()
|
||||
return nil, false, fmt.Errorf("Agent session already exists: %s", id)
|
||||
}
|
||||
if len(r.sessions) >= maxAgentSessions {
|
||||
r.mu.Unlock()
|
||||
return nil, false, fmt.Errorf("Agent session limit reached: %d", maxAgentSessions)
|
||||
}
|
||||
session := &agentSession{state: newEtcdSession()}
|
||||
r.sessions[id] = session
|
||||
r.mu.Unlock()
|
||||
|
||||
if _, err := session.state.connect(params); err != nil {
|
||||
r.mu.Lock()
|
||||
delete(r.sessions, id)
|
||||
r.mu.Unlock()
|
||||
session.state.close()
|
||||
return nil, false, err
|
||||
}
|
||||
return map[string]bool{"ok": true}, false, nil
|
||||
}
|
||||
|
||||
func (r *runtimeServer) session(id string) (*agentSession, error) {
|
||||
r.mu.RLock()
|
||||
session := r.sessions[id]
|
||||
r.mu.RUnlock()
|
||||
if session == nil {
|
||||
return nil, fmt.Errorf("Agent session not found: %s", id)
|
||||
}
|
||||
return session, nil
|
||||
}
|
||||
|
||||
func (r *runtimeServer) closeSession(id string) any {
|
||||
r.mu.Lock()
|
||||
session := r.sessions[id]
|
||||
delete(r.sessions, id)
|
||||
r.mu.Unlock()
|
||||
if session != nil {
|
||||
session.cancelActive()
|
||||
session.mu.Lock()
|
||||
session.state.close()
|
||||
session.mu.Unlock()
|
||||
}
|
||||
return map[string]bool{"ok": true}
|
||||
}
|
||||
|
||||
func (r *runtimeServer) closeAll() error {
|
||||
r.mu.Lock()
|
||||
sessions := r.sessions
|
||||
r.sessions = map[string]*agentSession{}
|
||||
r.mu.Unlock()
|
||||
var firstErr error
|
||||
for _, session := range sessions {
|
||||
session.cancelActive()
|
||||
if err := session.state.close(); err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
}
|
||||
return firstErr
|
||||
}
|
||||
|
||||
func (r *runtimeServer) testConnection(params map[string]json.RawMessage) (any, bool, error) {
|
||||
state := newEtcdSession()
|
||||
defer state.close()
|
||||
if _, err := state.connect(params); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return map[string]bool{"ok": true}, false, nil
|
||||
}
|
||||
|
||||
func newEtcdSession() *etcdSession {
|
||||
return &etcdSession{
|
||||
watches: map[string]*watchState{},
|
||||
knownLeases: map[uint64]struct{}{},
|
||||
}
|
||||
}
|
||||
|
||||
func handshakeResult() map[string]any {
|
||||
return map[string]any{
|
||||
"protocolVersion": protocolVersion,
|
||||
"agentProtocolVersion": protocolVersion,
|
||||
"capabilities": capabilities,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *etcdSession) beginOperation() (context.Context, context.CancelFunc) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
|
||||
s.activeCancelMu.Lock()
|
||||
s.activeCancel = cancel
|
||||
s.activeCancelMu.Unlock()
|
||||
return ctx, cancel
|
||||
}
|
||||
|
||||
func (s *etcdSession) endOperation(cancel context.CancelFunc) {
|
||||
cancel()
|
||||
s.activeCancelMu.Lock()
|
||||
s.activeCancel = nil
|
||||
s.activeCancelMu.Unlock()
|
||||
}
|
||||
|
||||
func (s *etcdSession) handle(method string, params map[string]json.RawMessage) (any, error) {
|
||||
switch method {
|
||||
case "handshake":
|
||||
return handshakeResult(), nil
|
||||
case "connect", "test_connection":
|
||||
return s.connectMap(params)
|
||||
case "validate_connection":
|
||||
return s.validateConnection()
|
||||
case "kv_list_prefix":
|
||||
return s.listPrefix(params)
|
||||
case "kv_get":
|
||||
return s.get(params)
|
||||
case "kv_put":
|
||||
return s.put(params)
|
||||
case "kv_delete":
|
||||
return s.delete(params)
|
||||
case "kv_rename":
|
||||
return s.rename(params)
|
||||
case "kv_history":
|
||||
return s.history(params)
|
||||
case "kv_status":
|
||||
return s.status(params)
|
||||
case "etcd_compact":
|
||||
return s.compact(params)
|
||||
case "etcd_defrag":
|
||||
return s.defrag(params)
|
||||
case "etcd_watch_start":
|
||||
return s.watchStart(params)
|
||||
case "etcd_watch_poll":
|
||||
return s.watchPoll(params)
|
||||
case "etcd_watch_stop":
|
||||
return s.watchStop(params)
|
||||
case "etcd_lease_list":
|
||||
return s.leaseList(params)
|
||||
case "etcd_lease_get":
|
||||
return s.leaseGet(params)
|
||||
case "etcd_lease_grant":
|
||||
return s.leaseGrant(params)
|
||||
case "etcd_lease_keepalive_once":
|
||||
return s.leaseKeepAlive(params)
|
||||
case "etcd_lease_revoke":
|
||||
return s.leaseRevoke(params)
|
||||
case "etcd_auth_user_list":
|
||||
return s.authUserList(params)
|
||||
case "etcd_auth_user_get":
|
||||
return s.authUserGet(params)
|
||||
case "etcd_auth_user_add":
|
||||
return s.authUserAdd(params)
|
||||
case "etcd_auth_user_delete":
|
||||
return s.authUserDelete(params)
|
||||
case "etcd_auth_user_change_password":
|
||||
return s.authUserChangePassword(params)
|
||||
case "etcd_auth_user_grant_role":
|
||||
return s.authUserGrantRevokeRole(params, true)
|
||||
case "etcd_auth_user_revoke_role":
|
||||
return s.authUserGrantRevokeRole(params, false)
|
||||
case "etcd_auth_role_list":
|
||||
return s.authRoleList(params)
|
||||
case "etcd_auth_role_get":
|
||||
return s.authRoleGet(params)
|
||||
case "etcd_auth_role_add":
|
||||
return s.authRoleAdd(params)
|
||||
case "etcd_auth_role_delete":
|
||||
return s.authRoleDelete(params)
|
||||
case "etcd_auth_role_grant_permission":
|
||||
return s.authRolePermission(params, true)
|
||||
case "etcd_auth_role_revoke_permission":
|
||||
return s.authRolePermission(params, false)
|
||||
case "disconnect":
|
||||
s.close()
|
||||
return map[string]bool{"ok": true}, nil
|
||||
case "shutdown":
|
||||
s.close()
|
||||
return map[string]bool{"ok": true}, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown method: %s", method)
|
||||
}
|
||||
}
|
||||
|
||||
func decodeParams(params map[string]json.RawMessage, target any) error {
|
||||
if params == nil {
|
||||
params = map[string]json.RawMessage{}
|
||||
}
|
||||
data, err := json.Marshal(params)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.Unmarshal(data, target)
|
||||
}
|
||||
|
||||
func stringParam(params map[string]json.RawMessage, key string) string {
|
||||
if params == nil {
|
||||
return ""
|
||||
}
|
||||
var result string
|
||||
_ = json.Unmarshal(params[key], &result)
|
||||
return result
|
||||
}
|
||||
|
||||
func intParam(params map[string]json.RawMessage, key string) int {
|
||||
if params == nil {
|
||||
return 0
|
||||
}
|
||||
var result int
|
||||
_ = json.Unmarshal(params[key], &result)
|
||||
return result
|
||||
}
|
||||
|
||||
func boolParam(params map[string]json.RawMessage, key string) bool {
|
||||
if params == nil {
|
||||
return false
|
||||
}
|
||||
var result bool
|
||||
_ = json.Unmarshal(params[key], &result)
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,502 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.etcd.io/etcd/api/v3/mvccpb"
|
||||
)
|
||||
|
||||
func TestHandshakeAdvertisesCapabilities(t *testing.T) {
|
||||
server := newRuntimeServer()
|
||||
result, _, err := server.dispatch("handshake", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("handshake failed: %v", err)
|
||||
}
|
||||
handshake := result.(map[string]any)
|
||||
if handshake["protocolVersion"] != 2 || handshake["agentProtocolVersion"] != 2 {
|
||||
t.Fatalf("unexpected protocol versions: %#v", handshake)
|
||||
}
|
||||
want := []string{
|
||||
"connect", "test_connection", "kv", "kv_ttl", "kv_cas", "kv_list_values", "kv_status",
|
||||
"kv_history", "etcd_compaction", "etcd_defrag", "etcd_watch", "etcd_lease", "etcd_auth",
|
||||
"multi_session",
|
||||
}
|
||||
got, ok := handshake["capabilities"].([]string)
|
||||
if !ok {
|
||||
t.Fatalf("capabilities missing: %#v", handshake)
|
||||
}
|
||||
if strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("capability mismatch:\n got %v\nwant %v", got, want)
|
||||
}
|
||||
for _, capability := range got {
|
||||
if capability == "structured_error_v1" {
|
||||
t.Fatalf("structured_error_v1 must not be advertised (Java parity)")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleLineReportsSessionNotFound(t *testing.T) {
|
||||
server := newRuntimeServer()
|
||||
response, _ := server.handleLine(`{"id":7,"method":"kv_get","params":{"key":"a"}}`)
|
||||
if response.Error == nil {
|
||||
t.Fatalf("expected error response, got %#v", response)
|
||||
}
|
||||
if response.Error.Code != -1 {
|
||||
t.Fatalf("expected code -1, got %d", response.Error.Code)
|
||||
}
|
||||
if response.Error.Message != "Agent session not found: __legacy__" {
|
||||
t.Fatalf("unexpected message: %q", response.Error.Message)
|
||||
}
|
||||
var raw map[string]json.RawMessage
|
||||
if err := json.Unmarshal(response.ID, &raw); err == nil {
|
||||
_ = raw
|
||||
}
|
||||
if string(response.ID) != "7" {
|
||||
t.Fatalf("expected id 7, got %s", response.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenSessionValidatesAgentSessionID(t *testing.T) {
|
||||
server := newRuntimeServer()
|
||||
_, _, err := server.dispatch("open_session", map[string]json.RawMessage{})
|
||||
if err == nil || !strings.Contains(err.Error(), "agentSessionId is required") {
|
||||
t.Fatalf("expected agentSessionId requirement, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOpenSessionRejectsDuplicate(t *testing.T) {
|
||||
server := newRuntimeServer()
|
||||
server.sessions["dup"] = &agentSession{state: newEtcdSession()}
|
||||
_, _, err := server.dispatch("open_session", map[string]json.RawMessage{
|
||||
"agentSessionId": json.RawMessage(`"dup"`),
|
||||
})
|
||||
if err == nil || !strings.Contains(err.Error(), "Agent session already exists") {
|
||||
t.Fatalf("expected duplicate rejection, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnconnectedSessionErrors(t *testing.T) {
|
||||
state := newEtcdSession()
|
||||
if _, err := state.get(map[string]json.RawMessage{"key": json.RawMessage(`"k"`)}); err == nil || err.Error() != "Not connected" {
|
||||
t.Fatalf("kv_get before connect: %v", err)
|
||||
}
|
||||
if _, err := state.history(map[string]json.RawMessage{"key": json.RawMessage(`"k"`)}); err == nil || err.Error() != "Not connected" {
|
||||
t.Fatalf("kv_history before connect: %v", err)
|
||||
}
|
||||
if _, err := state.validateConnection(); err == nil || err.Error() != "Not connected" {
|
||||
t.Fatalf("validate_connection before connect: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnectTimeoutBounds(t *testing.T) {
|
||||
cases := []struct {
|
||||
configured int
|
||||
want int
|
||||
}{
|
||||
{0, 30},
|
||||
{-5, 1},
|
||||
{1, 1},
|
||||
{30, 30},
|
||||
{300, 300},
|
||||
{999, 300},
|
||||
}
|
||||
for _, testCase := range cases {
|
||||
got := connectTimeoutSeconds(connectionParams{ConnectTimeoutSecs: testCase.configured})
|
||||
if got != testCase.want {
|
||||
t.Fatalf("connectTimeoutSeconds(%d) = %d, want %d", testCase.configured, got, testCase.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGrpcMaxInboundMessageSize(t *testing.T) {
|
||||
cases := []struct {
|
||||
connection connectionParams
|
||||
want int
|
||||
}{
|
||||
{connectionParams{}, defaultGrpcMaxInboundMessageSize},
|
||||
{connectionParams{GrpcMaxInboundMessageSize: 1024}, minGrpcMaxInboundMessageSize},
|
||||
{connectionParams{GrpcMaxInboundMessageSize: 999 * 1024 * 1024}, maxGrpcMaxInboundMessageSize},
|
||||
{connectionParams{URLParams: "grpc_max_inbound_message_size=104857600"}, 100 * 1024 * 1024},
|
||||
{connectionParams{URLParams: "?grpc_max_inbound_message_size=64"}, minGrpcMaxInboundMessageSize},
|
||||
{connectionParams{URLParams: "grpc_max_inbound_message_size=abc"}, defaultGrpcMaxInboundMessageSize},
|
||||
{connectionParams{GrpcMaxInboundMessageSize: 52428800, URLParams: "grpc_max_inbound_message_size=1024"}, 50 * 1024 * 1024},
|
||||
}
|
||||
for _, testCase := range cases {
|
||||
got := grpcMaxInboundMessageSize(testCase.connection)
|
||||
if got != testCase.want {
|
||||
t.Fatalf("grpcMaxInboundMessageSize(%+v) = %d, want %d", testCase.connection, got, testCase.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnectionEndpoints(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
connection connectionParams
|
||||
want []string
|
||||
}{
|
||||
{"host port fallback", connectionParams{Port: 2379}, []string{"http://127.0.0.1:2379"}},
|
||||
{"ssl scheme", connectionParams{Host: "db", Port: 1, SSL: true}, []string{"https://db:1"}},
|
||||
{"comma list", connectionParams{EtcdEndpoints: "a:1,b:2"}, []string{"http://a:1", "http://b:2"}},
|
||||
{"newline list", connectionParams{Endpoints: "a:1\nb:2"}, []string{"http://a:1", "http://b:2"}},
|
||||
{"precedence", connectionParams{EtcdEndpoints: "a:1", Endpoints: "b:2", ConnectionString: "c:3"}, []string{"http://a:1"}},
|
||||
{"existing scheme", connectionParams{EtcdEndpoints: "https://a:1,http://b:2"}, []string{"https://a:1", "http://b:2"}},
|
||||
{"blank entries", connectionParams{EtcdEndpoints: " a:1 , ,b:2"}, []string{"http://a:1", "http://b:2"}},
|
||||
}
|
||||
for _, testCase := range cases {
|
||||
got := connectionEndpoints(testCase.connection)
|
||||
if strings.Join(got, "|") != strings.Join(testCase.want, "|") {
|
||||
t.Fatalf("%s: got %v, want %v", testCase.name, got, testCase.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTLSConfigRequiresCertAndKeyTogether(t *testing.T) {
|
||||
_, err := tlsConfigFor(connectionParams{SSL: true, ClientCertPath: "/tmp/cert.pem"})
|
||||
if err == nil || err.Error() != "Client certificate and key must be provided together" {
|
||||
t.Fatalf("expected paired cert/key error, got %v", err)
|
||||
}
|
||||
_, err = tlsConfigFor(connectionParams{SSL: true, CertPath: "/tmp/cert.pem", KeyPath: "/tmp/key.pem"})
|
||||
if err == nil {
|
||||
t.Fatalf("expected key pair load failure for missing files")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHistoryStartRevision(t *testing.T) {
|
||||
cases := []struct {
|
||||
requested *int64
|
||||
target int64
|
||||
want int64
|
||||
}{
|
||||
{nil, 5000, 1},
|
||||
{nil, 10000, 1},
|
||||
{nil, 10001, 2},
|
||||
{nil, 15000, 5001},
|
||||
{int64Ptr(200), 15000, 200},
|
||||
{int64Ptr(-5), 15000, 1},
|
||||
}
|
||||
for _, testCase := range cases {
|
||||
got := historyStartRevision(testCase.requested, testCase.target)
|
||||
if got != testCase.want {
|
||||
t.Fatalf("historyStartRevision(%v, %d) = %d, want %d", testCase.requested, testCase.target, got, testCase.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func int64Ptr(value int64) *int64 {
|
||||
return &value
|
||||
}
|
||||
|
||||
func TestHistoryCollectorBounds(t *testing.T) {
|
||||
collector := &historyCollector{limit: 3}
|
||||
for i := int64(1); i <= 5; i++ {
|
||||
collector.append(map[string]any{"revision": longString(i)})
|
||||
}
|
||||
if len(collector.rows) != 3 {
|
||||
t.Fatalf("expected 3 retained rows, got %d", len(collector.rows))
|
||||
}
|
||||
if !collector.truncated {
|
||||
t.Fatalf("expected truncated flag")
|
||||
}
|
||||
if collector.rows[0]["revision"] != "3" || collector.rows[2]["revision"] != "5" {
|
||||
t.Fatalf("expected newest rows retained, got %v", collector.rows)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrefixEnd(t *testing.T) {
|
||||
cases := []struct{ prefix, want string }{
|
||||
{"", "\x00"},
|
||||
{"abc", "abd"},
|
||||
{"a\xff", "b"},
|
||||
{"\xff\xff", "\x00"},
|
||||
{"a\xffb", "a\xffc"},
|
||||
}
|
||||
for _, testCase := range cases {
|
||||
if got := prefixEnd(testCase.prefix); got != testCase.want {
|
||||
t.Fatalf("prefixEnd(%q) = %q, want %q", testCase.prefix, got, testCase.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValueEncodings(t *testing.T) {
|
||||
utf8Value := valueObject([]byte("hello"))
|
||||
if utf8Value["encoding"] != "utf8" || utf8Value["data"] != "hello" {
|
||||
t.Fatalf("unexpected utf8 value: %#v", utf8Value)
|
||||
}
|
||||
binaryValue := valueObject([]byte{0xff, 0xfe})
|
||||
if binaryValue["encoding"] != "base64" {
|
||||
t.Fatalf("expected base64 for binary: %#v", binaryValue)
|
||||
}
|
||||
roundTrip, err := parseValueObject(map[string]json.RawMessage{
|
||||
"encoding": json.RawMessage(`"base64"`),
|
||||
"data": json.RawMessage(`"` + binaryValue["data"].(string) + `"`),
|
||||
})
|
||||
if err != nil || roundTrip != string([]byte{0xff, 0xfe}) {
|
||||
t.Fatalf("base64 round trip failed: %q %v", roundTrip, err)
|
||||
}
|
||||
if _, err := parseValueObject(map[string]json.RawMessage{"encoding": json.RawMessage(`"hex"`)}); err == nil {
|
||||
t.Fatalf("expected unsupported encoding error")
|
||||
}
|
||||
if displayBytes([]byte{0xff}) != valueObject([]byte{0xff})["data"] {
|
||||
t.Fatalf("displayBytes should fall back to base64")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNextContinuation(t *testing.T) {
|
||||
encoded := nextContinuation([]byte("key"))
|
||||
decoded, err := decodeBase64String(encoded)
|
||||
if err != nil {
|
||||
t.Fatalf("invalid continuation: %v", err)
|
||||
}
|
||||
if decoded != "key\x00" {
|
||||
t.Fatalf("unexpected continuation start: %q", decoded)
|
||||
}
|
||||
}
|
||||
|
||||
func decodeBase64String(value string) (string, error) {
|
||||
decoded, err := base64Decode(value)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(decoded), nil
|
||||
}
|
||||
|
||||
func TestWatchEventBufferBytes(t *testing.T) {
|
||||
item := testKeyValue(3, 10, 20)
|
||||
previous := testKeyValue(2, 5, 7)
|
||||
// 512 + 4*3 (key) + 4*10 (value) + 4*5 (previous value) = 584
|
||||
if got := watchEventBufferBytes(item, previous); got != 584 {
|
||||
t.Fatalf("watchEventBufferBytes = %d, want 584", got)
|
||||
}
|
||||
previous.Version = 0
|
||||
if got := watchEventBufferBytes(item, previous); got != 564 {
|
||||
t.Fatalf("zero-version previous ignored: %d, want 564", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchOverflowPreservesBufferedBatch(t *testing.T) {
|
||||
session := newEtcdSession()
|
||||
state := &watchState{watchID: "w", session: session}
|
||||
smallBatch := []map[string]any{{"eventType": "put"}}
|
||||
state.append(10, smallBatch, 128)
|
||||
state.append(11, []map[string]any{{"eventType": "put"}}, maxWatchBufferBytes+1)
|
||||
|
||||
// The buffered batch stays pollable; the terminal rides along once the
|
||||
// buffer has fully drained within the same poll (Java parity).
|
||||
polled := state.poll()
|
||||
if len(polled["batches"].([]any)) != 1 {
|
||||
t.Fatalf("overflow must keep the buffered batch pollable: %#v", polled)
|
||||
}
|
||||
terminal, ok := polled["terminal"].(map[string]any)
|
||||
if !ok || terminal["reason"] != "overflow" {
|
||||
t.Fatalf("expected overflow terminal after drain, got %#v", polled)
|
||||
}
|
||||
if terminal["message"] != "ETCD_WATCH_OVERFLOW: the event buffer reached its byte or event limit" {
|
||||
t.Fatalf("unexpected overflow message: %v", terminal["message"])
|
||||
}
|
||||
if session.watchBufferedBytesSnapshot() != 0 {
|
||||
t.Fatalf("session budget not released: %d", session.watchBufferedBytesSnapshot())
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchPollReleasesSessionBudget(t *testing.T) {
|
||||
session := newEtcdSession()
|
||||
state := &watchState{watchID: "w", session: session}
|
||||
state.append(1, []map[string]any{{"eventType": "put"}}, 1000)
|
||||
if session.watchBufferedBytesSnapshot() != 1000 {
|
||||
t.Fatalf("expected reserved budget 1000, got %d", session.watchBufferedBytesSnapshot())
|
||||
}
|
||||
state.poll()
|
||||
if session.watchBufferedBytesSnapshot() != 0 {
|
||||
t.Fatalf("expected budget released after poll, got %d", session.watchBufferedBytesSnapshot())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionAggregateBudgetTerminatesOnlyOffendingWatch(t *testing.T) {
|
||||
session := newEtcdSession()
|
||||
first := &watchState{watchID: "a", session: session}
|
||||
second := &watchState{watchID: "b", session: session}
|
||||
if !session.reserveWatchBuffer(maxSessionWatchBufferBytes / 2) {
|
||||
t.Fatalf("first reservation should succeed")
|
||||
}
|
||||
second.append(1, []map[string]any{{"eventType": "put"}}, maxSessionWatchBufferBytes/2+1)
|
||||
polled := second.poll()
|
||||
terminal, ok := polled["terminal"].(map[string]any)
|
||||
if !ok || terminal["reason"] != "overflow" {
|
||||
t.Fatalf("expected aggregate overflow on second watch: %#v", polled)
|
||||
}
|
||||
if first.hasTerminal() {
|
||||
t.Fatalf("first watch must stay healthy")
|
||||
}
|
||||
session.releaseWatchBuffer(maxSessionWatchBufferBytes / 2)
|
||||
}
|
||||
|
||||
func TestWatchPollRemovesTerminalWatch(t *testing.T) {
|
||||
session := newEtcdSession()
|
||||
state := &watchState{watchID: "w", session: session}
|
||||
session.registerWatch("w", state)
|
||||
state.fail("closed", "watch closed", nil)
|
||||
params := map[string]json.RawMessage{"watchId": json.RawMessage(`"w"`)}
|
||||
result, err := session.watchPoll(params)
|
||||
if err != nil {
|
||||
t.Fatalf("watchPoll failed: %v", err)
|
||||
}
|
||||
if _, ok := result.(map[string]any)["terminal"]; !ok {
|
||||
t.Fatalf("expected terminal in poll result: %#v", result)
|
||||
}
|
||||
if session.watchCount() != 0 {
|
||||
t.Fatalf("terminal watch must be removed from the session")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchPollUnknownID(t *testing.T) {
|
||||
session := newEtcdSession()
|
||||
_, err := session.watchPoll(map[string]json.RawMessage{"watchId": json.RawMessage(`"missing"`)})
|
||||
if err == nil || err.Error() != "ETCD_WATCH_NOT_FOUND: watch does not exist" {
|
||||
t.Fatalf("expected watch-not-found, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchStartLimitReached(t *testing.T) {
|
||||
session := newEtcdSession()
|
||||
for i := 0; i < maxWatches; i++ {
|
||||
session.registerWatch(string(rune('a'+i)), &watchState{watchID: string(rune('a' + i)), session: session})
|
||||
}
|
||||
_, err := session.watchStart(map[string]json.RawMessage{"key": json.RawMessage(`"k"`)})
|
||||
if err == nil || !strings.HasPrefix(err.Error(), "ETCD_WATCH_LIMIT") {
|
||||
t.Fatalf("expected watch limit error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchScopeValidation(t *testing.T) {
|
||||
session := newEtcdSession()
|
||||
_, err := session.watchStart(map[string]json.RawMessage{"key": json.RawMessage(`"k"`), "scope": json.RawMessage(`"glob"`)})
|
||||
if err == nil || err.Error() != "ETCD_WATCH_SCOPE_INVALID: scope must be key or prefix" {
|
||||
t.Fatalf("expected scope error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLeasePageIDs(t *testing.T) {
|
||||
ids := []uint64{30, 10, 20}
|
||||
page := leasePageIDs(ids, nil, 2)
|
||||
if page[0] != 10 || page[1] != 20 {
|
||||
t.Fatalf("expected unsigned ascending order, got %v", page)
|
||||
}
|
||||
after := uint64(20)
|
||||
page = leasePageIDs(ids, &after, 10)
|
||||
if len(page) != 1 || page[0] != 30 {
|
||||
t.Fatalf("expected only 30 after cursor 20, got %v", page)
|
||||
}
|
||||
// 2^63 boundary must compare unsigned, not signed.
|
||||
big := []uint64{1, uint64(1) << 63}
|
||||
page = leasePageIDs(big, nil, 2)
|
||||
if page[0] != 1 || page[1] != uint64(1)<<63 {
|
||||
t.Fatalf("unsigned ordering broken: %v", page)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseUnsignedLong(t *testing.T) {
|
||||
parsed, err := parseUnsignedLong("18446744073709551615")
|
||||
if err != nil || parsed != 18446744073709551615 {
|
||||
t.Fatalf("uint64 max parse failed: %d %v", parsed, err)
|
||||
}
|
||||
if _, err := parseUnsignedLong("nope"); err == nil {
|
||||
t.Fatalf("expected parse failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppendUnexecutedDefragMembers(t *testing.T) {
|
||||
members := []map[string]any{{"endpoint": "a", "status": "failed"}}
|
||||
appendUnexecutedDefragMembers(&members, []string{"a", "b", "c"}, "a")
|
||||
if len(members) != 3 {
|
||||
t.Fatalf("expected 3 rows, got %d", len(members))
|
||||
}
|
||||
for _, row := range members[1:] {
|
||||
if row["status"] != "not_executed" || row["durationMs"] != nil || row["error"] != nil {
|
||||
t.Fatalf("unexpected not_executed row: %#v", row)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefragRequiresEndpoints(t *testing.T) {
|
||||
state := newEtcdSession()
|
||||
_, err := state.defrag(map[string]json.RawMessage{})
|
||||
if err == nil || err.Error() != "ETCD_DEFRAG_TARGET_REQUIRED: at least one endpoint is required" {
|
||||
t.Fatalf("expected defrag target error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequiredPositiveLongAndString(t *testing.T) {
|
||||
_, err := requiredPositiveLong(map[string]json.RawMessage{"id": json.RawMessage(`0`)}, "id")
|
||||
if err == nil || err.Error() != "ETCD_INVALID_ID: a positive integer is required" {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
_, err = requiredString(map[string]json.RawMessage{}, "user")
|
||||
if err == nil || err.Error() != "ETCD_USER_REQUIRED" {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestKeyBytesParam(t *testing.T) {
|
||||
if _, err := keyBytesParam(map[string]json.RawMessage{}); err == nil || err.Error() != "Key is required" {
|
||||
t.Fatalf("expected Key is required, got %v", err)
|
||||
}
|
||||
encoded := base64Encode([]byte{0x01, 0x02})
|
||||
key, err := keyBytesParam(map[string]json.RawMessage{
|
||||
"keyBytes": json.RawMessage(`{"encoding":"base64","data":"` + encoded + `"}`),
|
||||
})
|
||||
if err != nil || key != string([]byte{0x01, 0x02}) {
|
||||
t.Fatalf("keyBytes decode failed: %q %v", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPutRejectsConflictingLeaseOptions(t *testing.T) {
|
||||
state := newEtcdSession()
|
||||
params := map[string]json.RawMessage{
|
||||
"key": json.RawMessage(`"k"`),
|
||||
"value": json.RawMessage(`{"encoding":"utf8","data":"v"}`),
|
||||
"ttl": json.RawMessage(`5`),
|
||||
"lease": json.RawMessage(`7`),
|
||||
}
|
||||
// Java checks the active client before option exclusivity; without a
|
||||
// connection the error must stay "Not connected".
|
||||
_, err := state.put(params)
|
||||
if err == nil || err.Error() != "Not connected" {
|
||||
t.Fatalf("expected Not connected before option validation, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *etcdSession) watchBufferedBytesSnapshot() int64 {
|
||||
s.watchesMu.Lock()
|
||||
defer s.watchesMu.Unlock()
|
||||
return s.watchBufferedBytes
|
||||
}
|
||||
|
||||
func (w *watchState) hasTerminal() bool {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.terminalReason != ""
|
||||
}
|
||||
|
||||
func testKeyValue(keySize, valueSize, version int64) *mvccpb.KeyValue {
|
||||
return &mvccpb.KeyValue{
|
||||
Key: bytesOfLength(keySize),
|
||||
Value: bytesOfLength(valueSize),
|
||||
Version: version,
|
||||
}
|
||||
}
|
||||
|
||||
func bytesOfLength(size int64) []byte {
|
||||
return make([]byte, size)
|
||||
}
|
||||
|
||||
func base64Decode(value string) ([]byte, error) {
|
||||
return base64.StdEncoding.DecodeString(value)
|
||||
}
|
||||
|
||||
func base64Encode(value []byte) string {
|
||||
return base64.StdEncoding.EncodeToString(value)
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"go.etcd.io/etcd/api/v3/v3rpc/rpctypes"
|
||||
clientv3 "go.etcd.io/etcd/client/v3"
|
||||
)
|
||||
|
||||
func jsonUnmarshal(raw json.RawMessage, target any) error {
|
||||
return json.Unmarshal(raw, target)
|
||||
}
|
||||
|
||||
func (s *etcdSession) compact(params map[string]json.RawMessage) (any, error) {
|
||||
revision, err := requiredPositiveLong(params, "revision")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
|
||||
_, err = client.Get(ctx, "\x00", clientv3.WithRange("\x00"), clientv3.WithCountOnly(), clientv3.WithRev(revision))
|
||||
cancel()
|
||||
if err != nil {
|
||||
if errors.Is(err, rpctypes.ErrCompacted) {
|
||||
if compacted, ok := compactedRevisionOf(client, "\x00"); ok {
|
||||
return nil, errors.New("ETCD_INVALID_REVISION: revision was already compacted at " + longString(compacted))
|
||||
}
|
||||
return nil, errors.New("ETCD_INVALID_REVISION: revision was already compacted")
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
currentCtx, currentCancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
|
||||
current, err := client.Get(currentCtx, "\x00", clientv3.WithRange("\x00"), clientv3.WithCountOnly())
|
||||
currentCancel()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if revision > current.Header.Revision {
|
||||
return nil, errors.New("ETCD_INVALID_REVISION: revision is newer than the current revision")
|
||||
}
|
||||
compactCtx, compactCancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
|
||||
defer compactCancel()
|
||||
if _, err := client.Compact(compactCtx, revision); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"revision": longString(revision)}, nil
|
||||
}
|
||||
|
||||
func (s *etcdSession) defrag(params map[string]json.RawMessage) (any, error) {
|
||||
var targets []string
|
||||
if raw := params["endpoints"]; len(raw) > 0 && string(raw) != "null" {
|
||||
var list []string
|
||||
if jsonUnmarshal(raw, &list) == nil {
|
||||
for _, endpoint := range list {
|
||||
if endpoint != "" && !containsString(targets, endpoint) {
|
||||
targets = append(targets, endpoint)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(targets) == 0 {
|
||||
return nil, errors.New("ETCD_DEFRAG_TARGET_REQUIRED: at least one endpoint is required")
|
||||
}
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
members := []map[string]any{}
|
||||
remaining := targets
|
||||
for len(remaining) > 0 {
|
||||
endpoint, err := nextDefragEndpoint(client, remaining)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
started := time.Now()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
|
||||
_, defragErr := client.Maintenance.Defragment(ctx, endpoint)
|
||||
cancel()
|
||||
durationMs := time.Since(started).Milliseconds()
|
||||
if defragErr != nil {
|
||||
members = append(members, map[string]any{
|
||||
"endpoint": endpoint,
|
||||
"status": "failed",
|
||||
"durationMs": durationMs,
|
||||
"error": defragErr.Error(),
|
||||
})
|
||||
appendUnexecutedDefragMembers(&members, remaining, endpoint)
|
||||
break
|
||||
}
|
||||
members = append(members, map[string]any{
|
||||
"endpoint": endpoint,
|
||||
"status": "succeeded",
|
||||
"durationMs": durationMs,
|
||||
})
|
||||
remaining = removeString(remaining, endpoint)
|
||||
}
|
||||
return map[string]any{"members": members}, nil
|
||||
}
|
||||
|
||||
func appendUnexecutedDefragMembers(members *[]map[string]any, remaining []string, failedEndpoint string) {
|
||||
for _, endpoint := range remaining {
|
||||
if endpoint == failedEndpoint {
|
||||
continue
|
||||
}
|
||||
*members = append(*members, map[string]any{
|
||||
"endpoint": endpoint,
|
||||
"status": "not_executed",
|
||||
"durationMs": nil,
|
||||
"error": nil,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// nextDefragEndpoint re-evaluates leadership before each member so a leader
|
||||
// change is not defragmented early.
|
||||
func nextDefragEndpoint(client *clientv3.Client, remaining []string) (string, error) {
|
||||
var leaderEndpoint string
|
||||
for _, endpoint := range remaining {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
|
||||
statusResponse, err := client.Maintenance.Status(ctx, endpoint)
|
||||
cancel()
|
||||
if err != nil {
|
||||
// The following defragment call reports this member as the failed step.
|
||||
continue
|
||||
}
|
||||
if statusResponse.Header.MemberId == statusResponse.Leader {
|
||||
leaderEndpoint = endpoint
|
||||
break
|
||||
}
|
||||
}
|
||||
for _, endpoint := range remaining {
|
||||
if endpoint != leaderEndpoint {
|
||||
return endpoint, nil
|
||||
}
|
||||
}
|
||||
if leaderEndpoint == "" {
|
||||
return remaining[0], nil
|
||||
}
|
||||
return leaderEndpoint, nil
|
||||
}
|
||||
|
||||
func containsString(values []string, target string) bool {
|
||||
for _, value := range values {
|
||||
if value == target {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func removeString(values []string, target string) []string {
|
||||
result := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
if value != target {
|
||||
result = append(result, value)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
type rpcError struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data *rpcErrorData `json:"data,omitempty"`
|
||||
}
|
||||
|
||||
type rpcErrorData struct {
|
||||
Category string `json:"category"`
|
||||
Retryable bool `json:"retryable"`
|
||||
SessionDisposition string `json:"sessionDisposition"`
|
||||
Stage string `json:"stage"`
|
||||
ContractVersion int `json:"contractVersion"`
|
||||
OperationOutcome string `json:"operationOutcome"`
|
||||
ExceptionClass string `json:"exceptionClass,omitempty"`
|
||||
AgentSessionID string `json:"agentSessionId,omitempty"`
|
||||
}
|
||||
|
||||
func errorResponse(id json.RawMessage, method, agentSessionID string, err error) response {
|
||||
return response{JSONRPC: "2.0", ID: id, Error: classifyRPCError(method, agentSessionID, err)}
|
||||
}
|
||||
|
||||
func classifyRPCError(method, agentSessionID string, err error) *rpcError {
|
||||
stage := rpcErrorStage(method)
|
||||
data := &rpcErrorData{
|
||||
Category: "protocol",
|
||||
Retryable: false,
|
||||
SessionDisposition: "keep",
|
||||
Stage: stage,
|
||||
ContractVersion: 1,
|
||||
OperationOutcome: rpcOperationOutcome(stage),
|
||||
ExceptionClass: safeRPCDiagnostic(errTypeName(err), 160),
|
||||
AgentSessionID: strings.TrimSpace(agentSessionID),
|
||||
}
|
||||
|
||||
switch {
|
||||
case errors.Is(err, context.Canceled):
|
||||
data.Category = "canceled"
|
||||
data.SessionDisposition = "quarantine"
|
||||
case errors.Is(err, context.DeadlineExceeded) || isTimeoutError(err):
|
||||
data.Category = "timeout"
|
||||
data.SessionDisposition = "quarantine"
|
||||
case isUnavailableError(err):
|
||||
data.Category = "connection"
|
||||
data.Retryable = stage == "connect" || stage == "validate"
|
||||
if stage != "connect" {
|
||||
data.SessionDisposition = "quarantine"
|
||||
}
|
||||
}
|
||||
|
||||
return &rpcError{Code: -1, Message: err.Error(), Data: data}
|
||||
}
|
||||
|
||||
// isUnavailableError treats gRPC UNAVAILABLE (etcd server unreachable) as a
|
||||
// connection failure, matching the Java agent's transport classification.
|
||||
func isUnavailableError(err error) bool {
|
||||
if status.Code(err) == codes.Unavailable {
|
||||
return true
|
||||
}
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, net.ErrClosed) {
|
||||
return true
|
||||
}
|
||||
var networkError *net.OpError
|
||||
if errors.As(err, &networkError) {
|
||||
return true
|
||||
}
|
||||
lower := strings.ToLower(err.Error())
|
||||
for _, marker := range []string{
|
||||
"connection refused", "connection reset", "broken pipe", "connection closed",
|
||||
"connection lost", "unexpected eof", "no route to host",
|
||||
} {
|
||||
if strings.Contains(lower, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func errTypeName(err error) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.ReplaceAll(strings.TrimSpace(err.Error()), "\n", " ")
|
||||
}
|
||||
|
||||
func rpcErrorStage(method string) string {
|
||||
switch method {
|
||||
case "connect", "open_session", "test_connection":
|
||||
return "connect"
|
||||
case "validate_connection", "validate_session":
|
||||
return "validate"
|
||||
case "cancel_session":
|
||||
return "cancel"
|
||||
case "close_session", "disconnect", "shutdown":
|
||||
return "close"
|
||||
case "etcd_watch_poll", "etcd_lease_get":
|
||||
return "fetch"
|
||||
case "handshake", "":
|
||||
return "request"
|
||||
default:
|
||||
return "execute"
|
||||
}
|
||||
}
|
||||
|
||||
func rpcOperationOutcome(stage string) string {
|
||||
switch stage {
|
||||
case "request", "connect", "validate":
|
||||
return "not_started"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
func isTimeoutError(err error) bool {
|
||||
var timeout interface{ Timeout() bool }
|
||||
return errors.As(err, &timeout) && timeout.Timeout()
|
||||
}
|
||||
|
||||
func safeRPCDiagnostic(value string, maxLength int) string {
|
||||
var result strings.Builder
|
||||
for _, char := range value {
|
||||
if result.Len() >= maxLength {
|
||||
break
|
||||
}
|
||||
if char >= 0x21 && char <= 0x7e {
|
||||
result.WriteRune(char)
|
||||
}
|
||||
}
|
||||
return result.String()
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"go.etcd.io/etcd/api/v3/etcdserverpb"
|
||||
clientv3 "go.etcd.io/etcd/client/v3"
|
||||
)
|
||||
|
||||
func (s *etcdSession) status(params map[string]json.RawMessage) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
maintenance := client.Maintenance
|
||||
membersByID := map[uint64]*etcdserverpb.Member{}
|
||||
endpoints := s.connectedEndpointList()
|
||||
memberCtx, memberCancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
|
||||
memberList, err := client.Cluster.MemberList(memberCtx)
|
||||
memberCancel()
|
||||
if err == nil {
|
||||
for _, member := range memberList.Members {
|
||||
membersByID[member.ID] = member
|
||||
for _, clientURL := range member.ClientURLs {
|
||||
host := endpointHost(clientURL)
|
||||
if host == "" || host == "0.0.0.0" || host == "::" {
|
||||
continue
|
||||
}
|
||||
known := false
|
||||
for _, existing := range endpoints {
|
||||
if existing == clientURL {
|
||||
known = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !known {
|
||||
endpoints = append(endpoints, clientURL)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
alarms := []string{}
|
||||
alarmCtx, alarmCancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
|
||||
alarmResponse, err := maintenance.AlarmList(alarmCtx)
|
||||
alarmCancel()
|
||||
if err != nil {
|
||||
alarms = append(alarms, "UNAVAILABLE: "+err.Error())
|
||||
} else {
|
||||
for _, alarm := range alarmResponse.Alarms {
|
||||
alarms = append(alarms, alarm.Alarm.String()+"@"+unsignedLongString(int64(alarm.MemberID)))
|
||||
}
|
||||
}
|
||||
|
||||
countCtx, countCancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
|
||||
countResponse, err := client.Get(countCtx, "\x00", clientv3.WithRange("\x00"), clientv3.WithCountOnly())
|
||||
countCancel()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
type statusResult struct {
|
||||
response *clientv3.StatusResponse
|
||||
err error
|
||||
started time.Time
|
||||
}
|
||||
statusRequests := make([]chan *statusResult, len(endpoints))
|
||||
for index, endpoint := range endpoints {
|
||||
channel := make(chan *statusResult, 1)
|
||||
statusRequests[index] = channel
|
||||
go func(endpoint string, channel chan *statusResult) {
|
||||
started := time.Now()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
|
||||
response, err := client.Maintenance.Status(ctx, endpoint)
|
||||
cancel()
|
||||
channel <- &statusResult{response: response, err: err, started: started}
|
||||
}(endpoint, channel)
|
||||
}
|
||||
|
||||
statusMembers := []map[string]any{}
|
||||
observedMemberIDs := map[string]struct{}{}
|
||||
var clusterID *string
|
||||
var leaderID *string
|
||||
for index, channel := range statusRequests {
|
||||
result := <-channel
|
||||
row := map[string]any{"endpoint": endpoints[index]}
|
||||
if result.err != nil {
|
||||
row["memberId"] = nil
|
||||
row["name"] = nil
|
||||
row["version"] = nil
|
||||
row["leaderId"] = nil
|
||||
row["revision"] = nil
|
||||
row["raftTerm"] = nil
|
||||
row["raftIndex"] = nil
|
||||
row["raftAppliedIndex"] = nil
|
||||
row["dbSize"] = nil
|
||||
row["dbSizeInUse"] = nil
|
||||
row["learner"] = false
|
||||
row["reachable"] = false
|
||||
row["latencyMs"] = nil
|
||||
row["errors"] = []string{result.err.Error()}
|
||||
statusMembers = append(statusMembers, row)
|
||||
continue
|
||||
}
|
||||
memberStatus := result.response
|
||||
memberID := memberStatus.Header.MemberId
|
||||
member := membersByID[memberID]
|
||||
clusterIDValue := unsignedLongString(int64(memberStatus.Header.ClusterId))
|
||||
leaderIDValue := unsignedLongString(int64(memberStatus.Leader))
|
||||
clusterID = &clusterIDValue
|
||||
leaderID = &leaderIDValue
|
||||
row["memberId"] = unsignedLongString(int64(memberID))
|
||||
if member == nil {
|
||||
row["name"] = nil
|
||||
} else {
|
||||
row["name"] = member.Name
|
||||
}
|
||||
row["version"] = memberStatus.Version
|
||||
row["leaderId"] = unsignedLongString(int64(memberStatus.Leader))
|
||||
row["revision"] = longString(memberStatus.Header.Revision)
|
||||
row["raftTerm"] = longString(int64(memberStatus.RaftTerm))
|
||||
row["raftIndex"] = longString(int64(memberStatus.RaftIndex))
|
||||
row["raftAppliedIndex"] = longString(int64(memberStatus.RaftAppliedIndex))
|
||||
row["dbSize"] = longString(memberStatus.DbSize)
|
||||
row["dbSizeInUse"] = longString(memberStatus.DbSizeInUse)
|
||||
row["learner"] = memberStatus.IsLearner
|
||||
row["reachable"] = true
|
||||
row["latencyMs"] = time.Since(result.started).Milliseconds()
|
||||
memberIDKey := unsignedLongString(int64(memberID))
|
||||
if _, seen := observedMemberIDs[memberIDKey]; seen {
|
||||
continue
|
||||
}
|
||||
observedMemberIDs[memberIDKey] = struct{}{}
|
||||
statusMembers = append(statusMembers, row)
|
||||
}
|
||||
|
||||
result := map[string]any{
|
||||
"clusterId": clusterID,
|
||||
"revision": longString(countResponse.Header.Revision),
|
||||
"leaderId": leaderID,
|
||||
"keyCount": longString(countResponse.Count),
|
||||
"alarms": alarms,
|
||||
"members": statusMembers,
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
@@ -0,0 +1,366 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.etcd.io/etcd/api/v3/mvccpb"
|
||||
clientv3 "go.etcd.io/etcd/client/v3"
|
||||
)
|
||||
|
||||
const (
|
||||
maxWatches = 4
|
||||
maxWatchBatches = 256
|
||||
maxWatchEvents = 10000
|
||||
maxWatchBufferBytes = int64(8 * 1024 * 1024)
|
||||
maxSessionWatchBufferBytes = int64(16 * 1024 * 1024)
|
||||
)
|
||||
|
||||
type bufferedWatchBatch struct {
|
||||
payload map[string]any
|
||||
bufferedBytes int64
|
||||
}
|
||||
|
||||
// watchState buffers watch events for the polling host. Terminal states are
|
||||
// only surfaced after the buffer drains, mirroring the Java implementation.
|
||||
type watchState struct {
|
||||
watchID string
|
||||
session *etcdSession
|
||||
|
||||
mu sync.Mutex
|
||||
batches []bufferedWatchBatch
|
||||
eventCount int
|
||||
bufferedBytes int64
|
||||
terminalReason string
|
||||
terminalMessage string
|
||||
compactedRevision *int64
|
||||
watchCancel context.CancelFunc
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (w *watchState) append(revision int64, events []map[string]any, batchBytes int64) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.terminalReason != "" {
|
||||
return
|
||||
}
|
||||
if int64(len(w.batches)) >= maxWatchBatches ||
|
||||
int64(w.eventCount+len(events)) > maxWatchEvents ||
|
||||
batchBytes > maxWatchBufferBytes ||
|
||||
w.bufferedBytes+batchBytes > maxWatchBufferBytes ||
|
||||
!w.session.reserveWatchBuffer(batchBytes) {
|
||||
w.overflowLocked()
|
||||
return
|
||||
}
|
||||
w.batches = append(w.batches, bufferedWatchBatch{
|
||||
payload: map[string]any{"revision": longString(revision), "events": events},
|
||||
bufferedBytes: batchBytes,
|
||||
})
|
||||
w.eventCount += len(events)
|
||||
w.bufferedBytes += batchBytes
|
||||
}
|
||||
|
||||
func (w *watchState) overflowLocked() {
|
||||
if w.terminalReason != "" {
|
||||
return
|
||||
}
|
||||
w.terminalReason = "overflow"
|
||||
w.terminalMessage = "ETCD_WATCH_OVERFLOW: the event buffer reached its byte or event limit"
|
||||
w.closeWatcherLocked()
|
||||
}
|
||||
|
||||
func (w *watchState) fail(reason, message string, compacted *int64) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.terminalReason == "" {
|
||||
w.terminalReason = reason
|
||||
w.terminalMessage = message
|
||||
w.compactedRevision = compacted
|
||||
}
|
||||
}
|
||||
|
||||
func (w *watchState) poll() map[string]any {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
page := make([]any, 0, 64)
|
||||
for len(w.batches) > 0 && len(page) < 64 {
|
||||
batch := w.batches[0]
|
||||
w.batches = w.batches[1:]
|
||||
events := batch.payload["events"].([]map[string]any)
|
||||
w.eventCount -= len(events)
|
||||
w.bufferedBytes -= batch.bufferedBytes
|
||||
w.session.releaseWatchBuffer(batch.bufferedBytes)
|
||||
page = append(page, batch.payload)
|
||||
}
|
||||
result := map[string]any{"watchId": w.watchID, "batches": page}
|
||||
if w.terminalReason != "" && len(w.batches) == 0 {
|
||||
terminal := map[string]any{
|
||||
"reason": w.terminalReason,
|
||||
"message": w.terminalMessage,
|
||||
}
|
||||
if w.compactedRevision != nil {
|
||||
terminal["compactedRevision"] = longString(*w.compactedRevision)
|
||||
} else {
|
||||
terminal["compactedRevision"] = nil
|
||||
}
|
||||
result["terminal"] = terminal
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (w *watchState) close() {
|
||||
w.mu.Lock()
|
||||
w.failLocked("stopped", "watch stopped", nil)
|
||||
w.clearBufferedLocked()
|
||||
w.closed = true
|
||||
cancel := w.watchCancel
|
||||
w.watchCancel = nil
|
||||
w.mu.Unlock()
|
||||
if cancel != nil {
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
|
||||
func (w *watchState) failLocked(reason, message string, compacted *int64) {
|
||||
if w.terminalReason == "" {
|
||||
w.terminalReason = reason
|
||||
w.terminalMessage = message
|
||||
w.compactedRevision = compacted
|
||||
}
|
||||
}
|
||||
|
||||
func (w *watchState) clearBufferedLocked() {
|
||||
if w.bufferedBytes > 0 {
|
||||
w.session.releaseWatchBuffer(w.bufferedBytes)
|
||||
}
|
||||
w.batches = nil
|
||||
w.eventCount = 0
|
||||
w.bufferedBytes = 0
|
||||
}
|
||||
|
||||
func (w *watchState) attachWatcher(cancel context.CancelFunc) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.closed || w.terminalReason != "" {
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
w.watchCancel = cancel
|
||||
}
|
||||
|
||||
func (w *watchState) closeWatcherLocked() {
|
||||
cancel := w.watchCancel
|
||||
w.watchCancel = nil
|
||||
if cancel != nil {
|
||||
go cancel()
|
||||
}
|
||||
}
|
||||
|
||||
func watchEventBufferBytes(item, previous *mvccpb.KeyValue) int64 {
|
||||
bytes := int64(512 + estimatedBufferedBytes(len(item.Key)))
|
||||
bytes += estimatedBufferedBytes(len(item.Value))
|
||||
if previous != nil && previous.Version > 0 {
|
||||
bytes += estimatedBufferedBytes(len(previous.Value))
|
||||
}
|
||||
return bytes
|
||||
}
|
||||
|
||||
func estimatedBufferedBytes(sourceBytes int) int64 {
|
||||
if sourceBytes < 0 {
|
||||
return 0
|
||||
}
|
||||
return int64(sourceBytes) * 4
|
||||
}
|
||||
|
||||
func newWatchID() string {
|
||||
var bytes [16]byte
|
||||
if _, err := rand.Read(bytes[:]); err != nil {
|
||||
return base64.StdEncoding.EncodeToString(bytes[:])
|
||||
}
|
||||
bytes[6] = (bytes[6] & 0x0f) | 0x40
|
||||
bytes[8] = (bytes[8] & 0x3f) | 0x80
|
||||
hexed := hex.EncodeToString(bytes[:])
|
||||
return fmt.Sprintf("%s-%s-%s-%s-%s", hexed[0:8], hexed[8:12], hexed[12:16], hexed[16:20], hexed[20:32])
|
||||
}
|
||||
|
||||
func (s *etcdSession) watchCount() int {
|
||||
s.watchesMu.Lock()
|
||||
defer s.watchesMu.Unlock()
|
||||
return len(s.watches)
|
||||
}
|
||||
|
||||
func (s *etcdSession) registerWatch(id string, state *watchState) bool {
|
||||
s.watchesMu.Lock()
|
||||
defer s.watchesMu.Unlock()
|
||||
if _, exists := s.watches[id]; exists {
|
||||
return false
|
||||
}
|
||||
s.watches[id] = state
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *etcdSession) removeWatch(id string) *watchState {
|
||||
s.watchesMu.Lock()
|
||||
defer s.watchesMu.Unlock()
|
||||
state := s.watches[id]
|
||||
delete(s.watches, id)
|
||||
return state
|
||||
}
|
||||
|
||||
func (s *etcdSession) watchStart(params map[string]json.RawMessage) (any, error) {
|
||||
if s.watchCount() >= maxWatches {
|
||||
return nil, fmt.Errorf("ETCD_WATCH_LIMIT: at most %d watches are allowed per connection", maxWatches)
|
||||
}
|
||||
key, err := keyBytesParam(params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
scope := stringOrDefault(params, "scope", "key")
|
||||
if scope != "key" && scope != "prefix" {
|
||||
return nil, errors.New("ETCD_WATCH_SCOPE_INVALID: scope must be key or prefix")
|
||||
}
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
requestedRevision := longOrNull(params, "startRevision")
|
||||
var startedRevision int64
|
||||
if requestedRevision != nil && *requestedRevision > 0 {
|
||||
startedRevision = *requestedRevision
|
||||
} else {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), rpcTimeoutSeconds*time.Second)
|
||||
response, err := client.Get(ctx, "\x00", clientv3.WithRange("\x00"), clientv3.WithCountOnly())
|
||||
cancel()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
startedRevision = response.Header.Revision + 1
|
||||
}
|
||||
|
||||
watchID := newWatchID()
|
||||
state := &watchState{watchID: watchID, session: s}
|
||||
options := []clientv3.OpOption{clientv3.WithRev(startedRevision)}
|
||||
if boolOrDefault(params, "includePrevKv", false) {
|
||||
options = append(options, clientv3.WithPrevKV())
|
||||
}
|
||||
if scope == "prefix" {
|
||||
options = append(options, clientv3.WithRange(prefixEnd(key)))
|
||||
}
|
||||
watchCtx, watchCancel := context.WithCancel(context.Background())
|
||||
channel := client.Watch(watchCtx, key, options...)
|
||||
go consumeWatchChannel(channel, state)
|
||||
state.attachWatcher(watchCancel)
|
||||
s.registerWatch(watchID, state)
|
||||
return map[string]any{"watchId": watchID, "startedRevision": longString(startedRevision)}, nil
|
||||
}
|
||||
|
||||
func consumeWatchChannel(channel clientv3.WatchChan, state *watchState) {
|
||||
for response := range channel {
|
||||
if response.Err() != nil {
|
||||
state.fail("error", response.Err().Error(), nil)
|
||||
continue
|
||||
}
|
||||
if response.Canceled {
|
||||
if response.CompactRevision > 0 {
|
||||
compacted := response.CompactRevision
|
||||
state.fail("compacted", "ETCD_COMPACTED", &compacted)
|
||||
} else {
|
||||
state.fail("closed", "watch closed", nil)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if len(response.Events) == 0 {
|
||||
continue
|
||||
}
|
||||
events := make([]map[string]any, 0, len(response.Events))
|
||||
bufferedBytes := int64(128)
|
||||
overflowed := false
|
||||
for _, event := range response.Events {
|
||||
item := event.Kv
|
||||
var previous *mvccpb.KeyValue
|
||||
if event.PrevKv != nil {
|
||||
previous = event.PrevKv
|
||||
}
|
||||
bufferedBytes += watchEventBufferBytes(item, previous)
|
||||
if bufferedBytes > maxWatchBufferBytes {
|
||||
state.failOverflow()
|
||||
overflowed = true
|
||||
break
|
||||
}
|
||||
events = append(events, watchEventRow(event, item, previous))
|
||||
}
|
||||
if overflowed {
|
||||
continue
|
||||
}
|
||||
state.append(response.Header.Revision, events, bufferedBytes)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *watchState) failOverflow() {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.overflowLocked()
|
||||
}
|
||||
|
||||
func watchEventRow(event *clientv3.Event, item, previous *mvccpb.KeyValue) map[string]any {
|
||||
row := map[string]any{
|
||||
"eventType": eventType(event),
|
||||
"revision": longString(item.ModRevision),
|
||||
"key": displayBytes(item.Key),
|
||||
"keyBytes": bytesObject(item.Key),
|
||||
}
|
||||
if event.Type == mvccpb.DELETE {
|
||||
row["value"] = nil
|
||||
} else {
|
||||
row["value"] = valueObject(item.Value)
|
||||
}
|
||||
if previous != nil && previous.Version > 0 {
|
||||
row["previousValue"] = valueObject(previous.Value)
|
||||
} else {
|
||||
row["previousValue"] = nil
|
||||
}
|
||||
if event.Type == mvccpb.DELETE && previous != nil {
|
||||
row["metadata"] = metadataMap(previous)
|
||||
} else {
|
||||
row["metadata"] = metadataMap(item)
|
||||
}
|
||||
return row
|
||||
}
|
||||
|
||||
func eventType(event *clientv3.Event) string {
|
||||
if event.Type == mvccpb.DELETE {
|
||||
return "delete"
|
||||
}
|
||||
return "put"
|
||||
}
|
||||
|
||||
func (s *etcdSession) watchPoll(params map[string]json.RawMessage) (any, error) {
|
||||
watchID := stringOrDefault(params, "watchId", "")
|
||||
s.watchesMu.Lock()
|
||||
state := s.watches[watchID]
|
||||
s.watchesMu.Unlock()
|
||||
if state == nil {
|
||||
return nil, errors.New("ETCD_WATCH_NOT_FOUND: watch does not exist")
|
||||
}
|
||||
result := state.poll()
|
||||
if _, terminal := result["terminal"]; terminal {
|
||||
if removed := s.removeWatch(watchID); removed != nil {
|
||||
removed.close()
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *etcdSession) watchStop(params map[string]json.RawMessage) (any, error) {
|
||||
if state := s.removeWatch(stringOrDefault(params, "watchId", "")); state != nil {
|
||||
state.close()
|
||||
}
|
||||
return map[string]bool{"stopped": true}, nil
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
dependencies {
|
||||
implementation 'com.google.code.gson:gson:2.12.1'
|
||||
implementation 'io.etcd:jetcd-core:0.8.6'
|
||||
}
|
||||
|
||||
tasks.named('shadowJar') {
|
||||
mergeServiceFiles()
|
||||
manifest {
|
||||
attributes('Agent-Label': 'etcd', 'Main-Class': 'com.dbx.agent.etcd.EtcdAgent')
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,475 +0,0 @@
|
||||
package com.dbx.agent.etcd;
|
||||
|
||||
import com.google.protobuf.ByteString;
|
||||
import com.google.gson.JsonObject;
|
||||
import com.google.gson.JsonParser;
|
||||
import io.etcd.jetcd.ByteSequence;
|
||||
import io.etcd.jetcd.KV;
|
||||
import io.etcd.jetcd.Lease;
|
||||
import io.etcd.jetcd.Txn;
|
||||
import io.etcd.jetcd.api.KeyValue;
|
||||
import io.etcd.jetcd.api.RangeResponse;
|
||||
import io.etcd.jetcd.api.ResponseHeader;
|
||||
import io.etcd.jetcd.kv.GetResponse;
|
||||
import io.etcd.jetcd.kv.TxnResponse;
|
||||
import java.lang.reflect.Proxy;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Deque;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
final class EtcdAgentTest {
|
||||
@Test
|
||||
void handshakeAdvertisesKvCapability() {
|
||||
String response = EtcdAgent.handleRequest(
|
||||
"{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"handshake\",\"params\":{}}"
|
||||
);
|
||||
|
||||
JsonObject result = JsonParser.parseString(response).getAsJsonObject().getAsJsonObject("result");
|
||||
|
||||
Assertions.assertEquals(2, result.get("protocolVersion").getAsInt());
|
||||
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"kv\"")));
|
||||
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"kv_ttl\"")));
|
||||
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"kv_cas\"")));
|
||||
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"kv_list_values\"")));
|
||||
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"kv_status\"")));
|
||||
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"kv_history\"")));
|
||||
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"etcd_compaction\"")));
|
||||
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"etcd_defrag\"")));
|
||||
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"etcd_watch\"")));
|
||||
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"etcd_lease\"")));
|
||||
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"etcd_auth\"")));
|
||||
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"multi_session\"")));
|
||||
Assertions.assertTrue(result.getAsJsonArray("capabilities").contains(JsonParser.parseString("\"connect\"")));
|
||||
}
|
||||
|
||||
@Test
|
||||
void endpointsUseConfiguredListAndScheme() {
|
||||
JsonObject connection = JsonParser.parseString(
|
||||
"{\"endpoints\":\"etcd-1:2379,https://etcd-2:2379\",\"ssl\":true}"
|
||||
).getAsJsonObject();
|
||||
|
||||
Assertions.assertEquals(
|
||||
List.of("https://etcd-1:2379", "https://etcd-2:2379"),
|
||||
EtcdAgent.endpoints(connection)
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void endpointsFallbackToHostPort() {
|
||||
JsonObject connection = JsonParser.parseString(
|
||||
"{\"host\":\"127.0.0.1\",\"port\":2379}"
|
||||
).getAsJsonObject();
|
||||
|
||||
Assertions.assertEquals(List.of("http://127.0.0.1:2379"), EtcdAgent.endpoints(connection));
|
||||
}
|
||||
|
||||
@Test
|
||||
void connectTimeoutUsesConfiguredValueAndSafeBounds() {
|
||||
Assertions.assertEquals(
|
||||
45,
|
||||
EtcdAgent.connectTimeoutSeconds(JsonParser.parseString("{\"connect_timeout_secs\":45}").getAsJsonObject())
|
||||
);
|
||||
Assertions.assertEquals(
|
||||
1,
|
||||
EtcdAgent.connectTimeoutSeconds(JsonParser.parseString("{\"connect_timeout_secs\":0}").getAsJsonObject())
|
||||
);
|
||||
Assertions.assertEquals(
|
||||
300,
|
||||
EtcdAgent.connectTimeoutSeconds(JsonParser.parseString("{\"connect_timeout_secs\":999}").getAsJsonObject())
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void grpcInboundLimitDefaultsTo32MiBAndUsesSafeBounds() {
|
||||
Assertions.assertEquals(
|
||||
32 * 1024 * 1024,
|
||||
EtcdAgent.grpcMaxInboundMessageSize(new JsonObject())
|
||||
);
|
||||
Assertions.assertEquals(
|
||||
64 * 1024 * 1024,
|
||||
EtcdAgent.grpcMaxInboundMessageSize(
|
||||
JsonParser.parseString("{\"grpc_max_inbound_message_size\":67108864}").getAsJsonObject()
|
||||
)
|
||||
);
|
||||
Assertions.assertEquals(
|
||||
1024 * 1024,
|
||||
EtcdAgent.grpcMaxInboundMessageSize(
|
||||
JsonParser.parseString("{\"grpc_max_inbound_message_size\":0}").getAsJsonObject()
|
||||
)
|
||||
);
|
||||
Assertions.assertEquals(
|
||||
256 * 1024 * 1024,
|
||||
EtcdAgent.grpcMaxInboundMessageSize(
|
||||
JsonParser.parseString("{\"grpc_max_inbound_message_size\":536870912}").getAsJsonObject()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
void grpcInboundLimitCanBeConfiguredThroughConnectionUrlParams() {
|
||||
JsonObject connection = JsonParser.parseString(
|
||||
"{\"url_params\":\"foo=bar&grpc_max_inbound_message_size=50331648\"}"
|
||||
).getAsJsonObject();
|
||||
|
||||
Assertions.assertEquals(48 * 1024 * 1024, EtcdAgent.grpcMaxInboundMessageSize(connection));
|
||||
}
|
||||
|
||||
@Test
|
||||
void validateConnectionRequiresAnActiveSession() {
|
||||
String response = EtcdAgent.handleRequest(
|
||||
"{\"jsonrpc\":\"2.0\",\"id\":2,\"method\":\"validate_connection\",\"params\":{}}"
|
||||
);
|
||||
|
||||
JsonObject error = JsonParser.parseString(response).getAsJsonObject().getAsJsonObject("error");
|
||||
|
||||
Assertions.assertEquals(-1, error.get("code").getAsInt());
|
||||
Assertions.assertEquals("Not connected", error.get("message").getAsString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void kvMethodDispatchReturnsJsonRpcErrorWhenDisconnected() {
|
||||
String response = EtcdAgent.handleRequest(
|
||||
"{\"jsonrpc\":\"2.0\",\"id\":2,\"method\":\"kv_get\",\"params\":{\"key\":\"/app/name\"}}"
|
||||
);
|
||||
|
||||
JsonObject error = JsonParser.parseString(response).getAsJsonObject().getAsJsonObject("error");
|
||||
|
||||
Assertions.assertEquals(-1, error.get("code").getAsInt());
|
||||
Assertions.assertEquals("Not connected", error.get("message").getAsString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void historyMethodIsRegistered() {
|
||||
String response = EtcdAgent.handleRequest(
|
||||
"{\"jsonrpc\":\"2.0\",\"id\":3,\"method\":\"kv_history\",\"params\":{\"key\":\"/app/name\",\"limit\":100}}"
|
||||
);
|
||||
|
||||
JsonObject error = JsonParser.parseString(response).getAsJsonObject().getAsJsonObject("error");
|
||||
|
||||
Assertions.assertEquals(-1, error.get("code").getAsInt());
|
||||
Assertions.assertEquals("Not connected", error.get("message").getAsString());
|
||||
}
|
||||
|
||||
@Test
|
||||
void historyDefaultsToABoundedRevisionWindow() {
|
||||
Assertions.assertEquals(1L, EtcdAgent.historyStartRevision(null, 100L));
|
||||
Assertions.assertEquals(10_001L, EtcdAgent.historyStartRevision(null, 20_000L));
|
||||
Assertions.assertEquals(42L, EtcdAgent.historyStartRevision(42L, 20_000L));
|
||||
}
|
||||
|
||||
@Test
|
||||
void historyRetainsOnlyTheNewestRequestedEvents() {
|
||||
Deque<Integer> events = new ArrayDeque<>();
|
||||
AtomicBoolean truncated = new AtomicBoolean();
|
||||
|
||||
for (int value = 1; value <= 5; value++) {
|
||||
EtcdAgent.appendBoundedHistory(events, value, 3, truncated);
|
||||
}
|
||||
|
||||
Assertions.assertEquals(List.of(3, 4, 5), List.copyOf(events));
|
||||
Assertions.assertTrue(truncated.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void preserveLeaseRetriesWithTheLatestLeaseAfterConcurrentChange() throws Exception {
|
||||
AtomicInteger getCalls = new AtomicInteger();
|
||||
AtomicInteger txnCalls = new AtomicInteger();
|
||||
KV active = scriptedKv(
|
||||
getCalls,
|
||||
txnCalls,
|
||||
List.of(getResponse("/app/name", 10, 101), getResponse("/app/name", 11, 202)),
|
||||
List.of(txnResponse(false, 10), txnResponse(true, 12))
|
||||
);
|
||||
|
||||
long revision = EtcdAgent.putPreservingLease(active, bytes("/app/name"), bytes("updated"));
|
||||
|
||||
Assertions.assertEquals(12, revision);
|
||||
Assertions.assertEquals(2, getCalls.get());
|
||||
Assertions.assertEquals(2, txnCalls.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void preserveLeaseRejectsMissingOrUnleasedKeys() {
|
||||
KV missing = scriptedKv(
|
||||
new AtomicInteger(),
|
||||
new AtomicInteger(),
|
||||
List.of(emptyGetResponse()),
|
||||
List.of()
|
||||
);
|
||||
KV unleased = scriptedKv(
|
||||
new AtomicInteger(),
|
||||
new AtomicInteger(),
|
||||
List.of(getResponse("/app/name", 10, 0)),
|
||||
List.of()
|
||||
);
|
||||
|
||||
IllegalStateException missingError = Assertions.assertThrows(
|
||||
IllegalStateException.class,
|
||||
() -> EtcdAgent.putPreservingLease(missing, bytes("/app/name"), bytes("updated"))
|
||||
);
|
||||
IllegalStateException unleasedError = Assertions.assertThrows(
|
||||
IllegalStateException.class,
|
||||
() -> EtcdAgent.putPreservingLease(unleased, bytes("/app/name"), bytes("updated"))
|
||||
);
|
||||
|
||||
Assertions.assertEquals("Cannot preserve lease: key does not exist or has no lease", missingError.getMessage());
|
||||
Assertions.assertEquals("Cannot preserve lease: key does not exist or has no lease", unleasedError.getMessage());
|
||||
}
|
||||
|
||||
@Test
|
||||
void preserveLeaseFailsClearlyAfterBoundedConcurrentChanges() {
|
||||
AtomicInteger getCalls = new AtomicInteger();
|
||||
AtomicInteger txnCalls = new AtomicInteger();
|
||||
KV active = scriptedKv(
|
||||
getCalls,
|
||||
txnCalls,
|
||||
List.of(
|
||||
getResponse("/app/name", 10, 101),
|
||||
getResponse("/app/name", 11, 202),
|
||||
getResponse("/app/name", 12, 303)
|
||||
),
|
||||
List.of(txnResponse(false, 10), txnResponse(false, 11), txnResponse(false, 12))
|
||||
);
|
||||
|
||||
IllegalStateException error = Assertions.assertThrows(
|
||||
IllegalStateException.class,
|
||||
() -> EtcdAgent.putPreservingLease(active, bytes("/app/name"), bytes("updated"))
|
||||
);
|
||||
|
||||
Assertions.assertEquals("Cannot preserve lease: key changed concurrently; retry the save", error.getMessage());
|
||||
Assertions.assertEquals(3, getCalls.get());
|
||||
Assertions.assertEquals(3, txnCalls.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void grantedLeaseIsRevokedWhenWriteFailsAfterCreation() {
|
||||
AtomicInteger revokeCalls = new AtomicInteger();
|
||||
AtomicInteger revokedLeaseId = new AtomicInteger();
|
||||
Lease lease = (Lease) Proxy.newProxyInstance(
|
||||
Lease.class.getClassLoader(),
|
||||
new Class<?>[] { Lease.class },
|
||||
(proxy, method, args) -> {
|
||||
if (method.getName().equals("revoke")) {
|
||||
revokeCalls.incrementAndGet();
|
||||
revokedLeaseId.set(Math.toIntExact((long) args[0]));
|
||||
return CompletableFuture.completedFuture(null);
|
||||
}
|
||||
if (method.getName().equals("close")) return null;
|
||||
throw new UnsupportedOperationException(method.getName());
|
||||
}
|
||||
);
|
||||
|
||||
TimeoutException error = Assertions.assertThrows(
|
||||
TimeoutException.class,
|
||||
() -> EtcdAgent.cleanUpGrantedLeaseOnFailure(123, lease, () -> {
|
||||
throw new TimeoutException("write timed out");
|
||||
})
|
||||
);
|
||||
|
||||
Assertions.assertEquals("write timed out", error.getMessage());
|
||||
Assertions.assertEquals(1, revokeCalls.get());
|
||||
Assertions.assertEquals(123, revokedLeaseId.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void defragFailureMarksEveryRemainingEndpointAsNotExecuted() {
|
||||
List<Map<String, Object>> members = new ArrayList<>();
|
||||
|
||||
EtcdAgent.appendUnexecutedDefragMembers(
|
||||
members,
|
||||
List.of("http://follower-2:2379", "http://follower-3:2379", "http://leader:2379"),
|
||||
"http://follower-2:2379"
|
||||
);
|
||||
|
||||
Assertions.assertEquals(
|
||||
List.of("http://follower-3:2379", "http://leader:2379"),
|
||||
members.stream().map(member -> member.get("endpoint")).toList()
|
||||
);
|
||||
Assertions.assertTrue(members.stream().allMatch(member -> "not_executed".equals(member.get("status"))));
|
||||
}
|
||||
|
||||
@Test
|
||||
void leaseListResponseParserReadsEveryLeaseId() throws Exception {
|
||||
byte[] response = new byte[] {
|
||||
0x12, 0x02, 0x08, 0x7b,
|
||||
0x12, 0x03, 0x08, (byte) 0xc8, 0x03
|
||||
};
|
||||
|
||||
Assertions.assertEquals(List.of(123L, 456L), EtcdAgent.leaseIdsFromResponse(response));
|
||||
}
|
||||
|
||||
@Test
|
||||
void leaseListPageUsesAStableCursorAndBoundsTheResult() {
|
||||
List<Long> leaseIds = new ArrayList<>();
|
||||
for (long id = 1000; id >= 1; id--) leaseIds.add(id);
|
||||
|
||||
List<Long> first = EtcdAgent.leasePageIds(leaseIds, null, 101);
|
||||
List<Long> second = EtcdAgent.leasePageIds(leaseIds, first.get(99), 101);
|
||||
|
||||
Assertions.assertEquals(101, first.size());
|
||||
Assertions.assertEquals(1L, first.get(0));
|
||||
Assertions.assertEquals(101L, first.get(100));
|
||||
Assertions.assertEquals(101L, second.get(0));
|
||||
Assertions.assertEquals(201L, second.get(100));
|
||||
}
|
||||
|
||||
@Test
|
||||
void watchBufferRejectsLargeValueAndPreviousValuePayloads() {
|
||||
long payloadBytes = EtcdAgent.estimatedBufferedBytes(2 * 1024 * 1024)
|
||||
+ EtcdAgent.estimatedBufferedBytes(2 * 1024 * 1024);
|
||||
|
||||
Assertions.assertTrue(payloadBytes > EtcdAgent.MAX_WATCH_BUFFER_BYTES);
|
||||
}
|
||||
|
||||
@Test
|
||||
void watchOverflowPreservesBufferedPayloadAndReportsTerminalState() {
|
||||
EtcdAgent.EtcdSessionState session = new EtcdAgent.EtcdSessionState();
|
||||
EtcdAgent.EtcdWatchState watch = new EtcdAgent.EtcdWatchState("watch-1", session);
|
||||
List<Map<String, Object>> events = List.of(Map.of("eventType", "put"));
|
||||
|
||||
watch.append(1, events, 4L * 1024 * 1024);
|
||||
Assertions.assertEquals(4L * 1024 * 1024, session.watchBufferedBytes());
|
||||
|
||||
watch.append(2, events, 5L * 1024 * 1024);
|
||||
Map<String, Object> result = watch.poll();
|
||||
|
||||
Assertions.assertEquals(0, session.watchBufferedBytes());
|
||||
List<?> batches = (List<?>) result.get("batches");
|
||||
Assertions.assertEquals(1, batches.size());
|
||||
Assertions.assertEquals("1", ((Map<?, ?>) batches.get(0)).get("revision"));
|
||||
Assertions.assertEquals("overflow", ((Map<?, ?>) result.get("terminal")).get("reason"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void watchPollReleasesTheSessionByteBudget() {
|
||||
EtcdAgent.EtcdSessionState session = new EtcdAgent.EtcdSessionState();
|
||||
EtcdAgent.EtcdWatchState watch = new EtcdAgent.EtcdWatchState("watch-1", session);
|
||||
|
||||
watch.append(1, List.of(Map.of("eventType", "put")), 1024);
|
||||
Assertions.assertEquals(1024, session.watchBufferedBytes());
|
||||
|
||||
watch.poll();
|
||||
|
||||
Assertions.assertEquals(0, session.watchBufferedBytes());
|
||||
}
|
||||
|
||||
@Test
|
||||
void aggregateSessionBudgetTerminatesOnlyTheWatchThatExceedsIt() {
|
||||
EtcdAgent.EtcdSessionState session = new EtcdAgent.EtcdSessionState();
|
||||
List<Map<String, Object>> events = List.of(Map.of("eventType", "put"));
|
||||
EtcdAgent.EtcdWatchState first = new EtcdAgent.EtcdWatchState("watch-1", session);
|
||||
EtcdAgent.EtcdWatchState second = new EtcdAgent.EtcdWatchState("watch-2", session);
|
||||
EtcdAgent.EtcdWatchState third = new EtcdAgent.EtcdWatchState("watch-3", session);
|
||||
|
||||
first.append(1, events, EtcdAgent.MAX_WATCH_BUFFER_BYTES);
|
||||
second.append(1, events, EtcdAgent.MAX_WATCH_BUFFER_BYTES);
|
||||
third.append(1, events, 1);
|
||||
|
||||
Assertions.assertEquals(EtcdAgent.MAX_SESSION_WATCH_BUFFER_BYTES, session.watchBufferedBytes());
|
||||
Assertions.assertEquals("overflow", ((Map<?, ?>) third.poll().get("terminal")).get("reason"));
|
||||
Assertions.assertEquals(EtcdAgent.MAX_SESSION_WATCH_BUFFER_BYTES, session.watchBufferedBytes());
|
||||
}
|
||||
|
||||
@Test
|
||||
void terminalPollRemovesTheWatchSlot() {
|
||||
EtcdAgent.EtcdSessionState session = new EtcdAgent.EtcdSessionState();
|
||||
EtcdAgent.EtcdWatchState watch = new EtcdAgent.EtcdWatchState("watch-1", session);
|
||||
session.addWatch("watch-1", watch);
|
||||
watch.overflow();
|
||||
|
||||
Map<String, Object> result = EtcdAgent.pollWatchState(session, "watch-1");
|
||||
|
||||
Assertions.assertEquals("overflow", ((Map<?, ?>) result.get("terminal")).get("reason"));
|
||||
Assertions.assertEquals(0, session.watchCount());
|
||||
}
|
||||
|
||||
private static KV scriptedKv(
|
||||
AtomicInteger getCalls,
|
||||
AtomicInteger txnCalls,
|
||||
List<GetResponse> getResponses,
|
||||
List<TxnResponse> txnResponses
|
||||
) {
|
||||
Deque<GetResponse> gets = new ArrayDeque<>(getResponses);
|
||||
Deque<TxnResponse> txns = new ArrayDeque<>(txnResponses);
|
||||
return (KV) Proxy.newProxyInstance(
|
||||
KV.class.getClassLoader(),
|
||||
new Class<?>[] { KV.class },
|
||||
(proxy, method, args) -> {
|
||||
if (method.getName().equals("get")) {
|
||||
getCalls.incrementAndGet();
|
||||
return CompletableFuture.completedFuture(gets.removeFirst());
|
||||
}
|
||||
if (method.getName().equals("txn")) {
|
||||
return scriptedTxn(txnCalls, txns);
|
||||
}
|
||||
if (method.getName().equals("close")) {
|
||||
return null;
|
||||
}
|
||||
throw new UnsupportedOperationException(method.getName());
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
private static Txn scriptedTxn(AtomicInteger txnCalls, Deque<TxnResponse> responses) {
|
||||
return (Txn) Proxy.newProxyInstance(
|
||||
Txn.class.getClassLoader(),
|
||||
new Class<?>[] { Txn.class },
|
||||
(proxy, method, args) -> {
|
||||
if (Arrays.asList("If", "Then", "Else").contains(method.getName())) {
|
||||
return proxy;
|
||||
}
|
||||
if (method.getName().equals("commit")) {
|
||||
txnCalls.incrementAndGet();
|
||||
return CompletableFuture.completedFuture(responses.removeFirst());
|
||||
}
|
||||
throw new UnsupportedOperationException(method.getName());
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
private static ByteSequence bytes(String value) {
|
||||
return ByteSequence.from(value, StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
private static GetResponse getResponse(String key, long modRevision, long lease) {
|
||||
KeyValue keyValue = KeyValue.newBuilder()
|
||||
.setKey(ByteString.copyFromUtf8(key))
|
||||
.setModRevision(modRevision)
|
||||
.setLease(lease)
|
||||
.build();
|
||||
RangeResponse response = RangeResponse.newBuilder()
|
||||
.setHeader(header(modRevision))
|
||||
.addKvs(keyValue)
|
||||
.build();
|
||||
return new GetResponse(response, ByteSequence.EMPTY);
|
||||
}
|
||||
|
||||
private static GetResponse emptyGetResponse() {
|
||||
return new GetResponse(
|
||||
RangeResponse.newBuilder().setHeader(header(1)).build(),
|
||||
ByteSequence.EMPTY
|
||||
);
|
||||
}
|
||||
|
||||
private static TxnResponse txnResponse(boolean succeeded, long revision) {
|
||||
io.etcd.jetcd.api.TxnResponse response = io.etcd.jetcd.api.TxnResponse.newBuilder()
|
||||
.setHeader(header(revision))
|
||||
.setSucceeded(succeeded)
|
||||
.build();
|
||||
return new TxnResponse(response, ByteSequence.EMPTY);
|
||||
}
|
||||
|
||||
private static ResponseHeader header(long revision) {
|
||||
return ResponseHeader.newBuilder().setRevision(revision).build();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
# etcd v2 native agent (`etcd2`)
|
||||
|
||||
Native Go agent speaking the etcd **v2 HTTP/JSON API** with the standard DBX
|
||||
agent protocol (NDJSON JSON-RPC over stdio). Registered as agentKey `etcd2`,
|
||||
selected through the `etcd-v2` driver profile of the `etcd` connection type.
|
||||
|
||||
## Supported servers
|
||||
|
||||
| Server | Support |
|
||||
| --- | --- |
|
||||
| etcd 2.0–2.3 (pure v2) | fully supported; validated against 2.3.8 |
|
||||
| etcd 3.0–3.5 started with `--enable-v2` | same code path |
|
||||
| etcd 3.4–3.5 default configuration | rejected at connect: v2 is disabled by default (`ETCD_V2_API_DISABLED`) |
|
||||
| etcd 3.6+ | impossible: the v2 API was removed upstream |
|
||||
|
||||
## Capabilities
|
||||
|
||||
`connect`, `test_connection`, `kv`, `kv_ttl`, `kv_cas`, `kv_list_values`,
|
||||
`kv_status`, `etcd_watch`, `etcd_auth`, `multi_session`. The v3-only
|
||||
capabilities (`kv_history`, `etcd_lease`, `etcd_compaction`, `etcd_defrag`)
|
||||
are deliberately absent; the host blocks those calls before dispatch and the
|
||||
UI hides the corresponding workspaces for `etcd-v2` connections.
|
||||
|
||||
## v2 API semantics and deliberate limits
|
||||
|
||||
- **TTL**: v2 attaches TTLs directly to keys (`?ttl=`). Key editing offers
|
||||
permanent/TTL expiry only; binding a lease is rejected with
|
||||
`ETCD_V2_LEASE_UNSUPPORTED` because v2 has no lease concept.
|
||||
- **CAS**: writes use `prevIndex` (modifiedIndex) and `prevExist`, matching
|
||||
the v3 modRevision contract. `expectedCreateRevision` is only honored when
|
||||
`0` (key-must-not-exist); a nonzero value is rejected with
|
||||
`ETCD_V2_CAS_UNSUPPORTED` because v2 does not expose createdIndex for
|
||||
compare-and-swap.
|
||||
- **Rename**: check-then-set, **not atomic** — v2 has no transaction
|
||||
primitive. The source is verified by CAS before the target is created, but
|
||||
a concurrent writer between the two requests is not serialized.
|
||||
- **Watch**: long-poll loop (`?wait=true&waitIndex=N`, recursive for prefix
|
||||
scopes). Idle poll timeouts are retried transparently. There are no
|
||||
progress notifications; the same batch/event/byte budget system as the v3
|
||||
agent applies.
|
||||
- **Hidden directories**: v2 hides `_`-prefixed directory entries from
|
||||
non-recursive listings; such keys are only reachable when addressed
|
||||
explicitly, matching `etcdctl` behavior.
|
||||
- **Auth**: users hold roles; roles hold read/write permissions over exact
|
||||
keys and `/prefix/*` globs. Grants and revokes use the server's incremental
|
||||
`grant`/`revoke` role PUT (`{"role":"r","grant":{"kv":{"read":["/p/*"]}}}`),
|
||||
matching the etcd 2.3 client wire format. Resource mapping: `key` → bare
|
||||
path, `prefix`/`all` → `/path/*` / `/*` globs.
|
||||
- **Status**: reduced dashboard. `clusterId` comes from the
|
||||
`X-Etcd-Cluster-Id` header, `revision` from `X-Etcd-Index`; Prometheus
|
||||
metrics, alarms, dbSize, and raft details do not exist in v2 and stay null.
|
||||
Raw `/v2/stats/store` counters are passed through as the `store` object.
|
||||
|
||||
## Connect probing
|
||||
|
||||
`connect` fetches `/version`, then probes `GET /v2/members`:
|
||||
|
||||
- `200` → ok; `403` → ok with `limited: true` (credentials work, root access
|
||||
missing); `404` → `ETCD_V2_API_DISABLED`; `401` → `ETCD_UNAUTHENTICATED`.
|
||||
|
||||
## Testing
|
||||
|
||||
```bash
|
||||
go test ./... # unit tests
|
||||
DBX_ETCD2_LIVE=1 go test -run Live ./... # live flow (default endpoint below)
|
||||
```
|
||||
|
||||
Live test environment variables: `DBX_ETCD2_ENDPOINTS`, `DBX_ETCD2_USER`,
|
||||
`DBX_ETCD2_PASSWORD`. The default endpoint points at the team test server's
|
||||
etcd 2.3.8 instance; see `docs/testing-databases.md` in the workspace root.
|
||||
@@ -0,0 +1,390 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// v2 auth wire format (etcd 2.3 etcdserver/etcdhttp/client_auth.go):
|
||||
// a role PUT body without grant/revoke creates the role; a body carrying
|
||||
// grant/revoke updates it incrementally and must not carry permissions.
|
||||
// Permission entries are glob patterns: a trailing "*" grants a prefix,
|
||||
// anything else matches a single key exactly.
|
||||
|
||||
type v2RWPermission struct {
|
||||
Read []string `json:"read"`
|
||||
Write []string `json:"write"`
|
||||
}
|
||||
|
||||
type v2Permissions struct {
|
||||
KV v2RWPermission `json:"kv"`
|
||||
}
|
||||
|
||||
type v2Role struct {
|
||||
Role string `json:"role"`
|
||||
Permissions *v2Permissions `json:"permissions"`
|
||||
Grant *v2Permissions `json:"grant,omitempty"`
|
||||
Revoke *v2Permissions `json:"revoke,omitempty"`
|
||||
}
|
||||
|
||||
// v2UserRecord is the GET shape of /v2/auth/users/<name>: roles come back
|
||||
// as full role objects, not plain names.
|
||||
type v2UserRecord struct {
|
||||
User string `json:"user"`
|
||||
Roles []struct {
|
||||
Role string `json:"role"`
|
||||
} `json:"roles"`
|
||||
}
|
||||
|
||||
// v2UserDocument is the PUT shape: grant/revoke carry role names.
|
||||
type v2UserDocument struct {
|
||||
User string `json:"user"`
|
||||
Password string `json:"password,omitempty"`
|
||||
Roles []string `json:"roles,omitempty"`
|
||||
Grant []string `json:"grant,omitempty"`
|
||||
Revoke []string `json:"revoke,omitempty"`
|
||||
}
|
||||
|
||||
func (s *etcd2Session) authUserList(params map[string]json.RawMessage) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
body, _, err := client.do(ctx, http.MethodGet, "/v2/auth/users", "", nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var parsed struct {
|
||||
Users []v2UserRecord `json:"users"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &parsed); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
users := make([]string, 0, len(parsed.Users))
|
||||
for _, user := range parsed.Users {
|
||||
users = append(users, user.User)
|
||||
}
|
||||
return map[string]any{"users": users}, nil
|
||||
}
|
||||
|
||||
func (s *etcd2Session) authUserGet(params map[string]json.RawMessage) (any, error) {
|
||||
user, err := requiredString(params, "user")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
body, _, err := client.do(ctx, http.MethodGet, "/v2/auth/users/"+escapePathSegment(user), "", nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var parsed v2UserRecord
|
||||
if err := json.Unmarshal(body, &parsed); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
roles := make([]string, 0, len(parsed.Roles))
|
||||
for _, role := range parsed.Roles {
|
||||
roles = append(roles, role.Role)
|
||||
}
|
||||
return map[string]any{"user": parsed.User, "roles": roles}, nil
|
||||
}
|
||||
|
||||
// authUserAdd relies on the create-or-update PUT: HTTP 201 means created,
|
||||
// HTTP 200 means the user existed and only the password would change.
|
||||
func (s *etcd2Session) authUserAdd(params map[string]json.RawMessage) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
user, err := requiredString(params, "user")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
password, err := requiredString(params, "password")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
payload := v2UserDocument{User: user, Password: password}
|
||||
_, response, err := client.doJSON(ctx, http.MethodPut, "/v2/auth/users/"+escapePathSegment(user), payload)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if response != nil && response.StatusCode == http.StatusOK {
|
||||
return nil, fmt.Errorf("auth: User %s already exists.", user)
|
||||
}
|
||||
return map[string]bool{"created": true}, nil
|
||||
}
|
||||
|
||||
func (s *etcd2Session) authUserDelete(params map[string]json.RawMessage) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
user, err := requiredString(params, "user")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
if _, _, err := client.do(ctx, http.MethodDelete, "/v2/auth/users/"+escapePathSegment(user), "", nil); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]bool{"deleted": true}, nil
|
||||
}
|
||||
|
||||
// authUserChangePassword reuses the create-or-update PUT: the server
|
||||
// replaces the stored password hash when the user exists.
|
||||
func (s *etcd2Session) authUserChangePassword(params map[string]json.RawMessage) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
user, err := requiredString(params, "user")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
password, err := requiredString(params, "password")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
payload := v2UserDocument{User: user, Password: password}
|
||||
if _, _, err := client.doJSON(ctx, http.MethodPut, "/v2/auth/users/"+escapePathSegment(user), payload); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]bool{"changed": true}, nil
|
||||
}
|
||||
|
||||
func (s *etcd2Session) authUserGrantRevokeRole(params map[string]json.RawMessage, grant bool) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
user, err := requiredString(params, "user")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
role, err := requiredString(params, "role")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
payload := v2UserDocument{User: user, Grant: []string{role}}
|
||||
if !grant {
|
||||
payload = v2UserDocument{User: user, Revoke: []string{role}}
|
||||
}
|
||||
if _, _, err := client.doJSON(ctx, http.MethodPut, "/v2/auth/users/"+escapePathSegment(user), payload); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]bool{"updated": true}, nil
|
||||
}
|
||||
|
||||
func (s *etcd2Session) authRoleList(params map[string]json.RawMessage) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
body, _, err := client.do(ctx, http.MethodGet, "/v2/auth/roles", "", nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var parsed struct {
|
||||
Roles []v2Role `json:"roles"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &parsed); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
roles := make([]string, 0, len(parsed.Roles))
|
||||
for _, role := range parsed.Roles {
|
||||
roles = append(roles, role.Role)
|
||||
}
|
||||
return map[string]any{"roles": roles}, nil
|
||||
}
|
||||
|
||||
func (s *etcd2Session) authRoleGet(params map[string]json.RawMessage) (any, error) {
|
||||
role, err := requiredString(params, "role")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
body, _, err := client.do(ctx, http.MethodGet, "/v2/auth/roles/"+escapePathSegment(role), "", nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var parsed v2Role
|
||||
if err := json.Unmarshal(body, &parsed); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
permissions := []map[string]any{}
|
||||
if parsed.Permissions != nil {
|
||||
permissions = append(permissions, v2PermissionRows("read", parsed.Permissions.KV.Read)...)
|
||||
permissions = append(permissions, v2PermissionRows("write", parsed.Permissions.KV.Write)...)
|
||||
}
|
||||
return map[string]any{"role": parsed.Role, "permissions": permissions}, nil
|
||||
}
|
||||
|
||||
// v2PermissionRows converts stored v2 patterns into the
|
||||
// {access, key, rangeEnd, resource} rows the v3 agent emits: a trailing
|
||||
// "*" displays as a prefix grant, anything else as an exact key.
|
||||
func v2PermissionRows(access string, entries []string) []map[string]any {
|
||||
rows := []map[string]any{}
|
||||
for _, entry := range entries {
|
||||
if strings.HasSuffix(entry, "*") {
|
||||
base := strings.TrimSuffix(entry, "*")
|
||||
rows = append(rows, map[string]any{
|
||||
"access": access,
|
||||
"key": bytesObject([]byte(base)),
|
||||
"rangeEnd": bytesObject([]byte(prefixEnd(base))),
|
||||
"resource": "prefix",
|
||||
})
|
||||
} else {
|
||||
rows = append(rows, map[string]any{
|
||||
"access": access,
|
||||
"key": bytesObject([]byte(entry)),
|
||||
"rangeEnd": bytesObject(nil),
|
||||
"resource": "key",
|
||||
})
|
||||
}
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
func (s *etcd2Session) authRoleAdd(params map[string]json.RawMessage) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
role, err := requiredString(params, "role")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
payload := v2Role{Role: role}
|
||||
if _, _, err := client.doJSON(ctx, http.MethodPut, "/v2/auth/roles/"+escapePathSegment(role), payload); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]bool{"created": true}, nil
|
||||
}
|
||||
|
||||
func (s *etcd2Session) authRoleDelete(params map[string]json.RawMessage) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
role, err := requiredString(params, "role")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
if _, _, err := client.do(ctx, http.MethodDelete, "/v2/auth/roles/"+escapePathSegment(role), "", nil); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]bool{"deleted": true}, nil
|
||||
}
|
||||
|
||||
// v2PermissionPattern maps the protocol's resource model onto the v2 glob
|
||||
// convention: exact keys stay bare, prefixes and "all" become "/path/*".
|
||||
func v2PermissionPattern(resource, key string) string {
|
||||
switch resource {
|
||||
case "all":
|
||||
return "/*"
|
||||
case "prefix":
|
||||
return strings.TrimSuffix(key, "/") + "/*"
|
||||
default:
|
||||
return key
|
||||
}
|
||||
}
|
||||
|
||||
// authRolePermission maps grants/revokes onto the server's incremental
|
||||
// grant/revoke role update (one PUT per access direction).
|
||||
func (s *etcd2Session) authRolePermission(params map[string]json.RawMessage, grant bool) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
role, err := requiredString(params, "role")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resource := stringOrDefault(params, "resource", "key")
|
||||
key, err := keyBytesParam(params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !strings.HasPrefix(key, "/") {
|
||||
key = "/" + key
|
||||
}
|
||||
pattern := v2PermissionPattern(resource, key)
|
||||
|
||||
var accesses []string
|
||||
if grant {
|
||||
access := strings.ToUpper(stringOrDefault(params, "access", ""))
|
||||
switch access {
|
||||
case "READ":
|
||||
accesses = []string{"read"}
|
||||
case "WRITE":
|
||||
accesses = []string{"write"}
|
||||
case "READWRITE":
|
||||
accesses = []string{"read", "write"}
|
||||
default:
|
||||
return nil, fmt.Errorf("ETCD_INVALID_ACCESS: access must be READ, WRITE, or READWRITE, got %s", access)
|
||||
}
|
||||
} else {
|
||||
accesses = []string{"read", "write"}
|
||||
}
|
||||
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
for _, access := range accesses {
|
||||
document := v2Role{Role: role}
|
||||
direction := v2Permissions{KV: v2RWPermission{}}
|
||||
if access == "read" {
|
||||
direction.KV.Read = []string{pattern}
|
||||
} else {
|
||||
direction.KV.Write = []string{pattern}
|
||||
}
|
||||
if grant {
|
||||
document.Grant = &direction
|
||||
} else {
|
||||
document.Revoke = &direction
|
||||
}
|
||||
if _, _, err := client.doJSON(ctx, http.MethodPut, "/v2/auth/roles/"+escapePathSegment(role), document); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return map[string]bool{"updated": true}, nil
|
||||
}
|
||||
|
||||
func (c *authenticatedClient) doJSON(ctx context.Context, method, path string, payload any) ([]byte, *http.Response, error) {
|
||||
encoded, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return c.do(ctx, method, path, string(encoded), map[string]string{"Content-Type": "application/json"})
|
||||
}
|
||||
|
||||
func escapePathSegment(segment string) string {
|
||||
return strings.ReplaceAll(segment, "/", "%2F")
|
||||
}
|
||||
@@ -0,0 +1,457 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultPort = 2379
|
||||
operationTimeout = 30 * time.Second
|
||||
)
|
||||
|
||||
// connectionParams mirrors the host-side etcd connection configuration.
|
||||
// gRPC-specific options are accepted and ignored: the v2 API is plain HTTP.
|
||||
type connectionParams struct {
|
||||
EtcdEndpoints string `json:"etcd_endpoints"`
|
||||
Endpoints string `json:"endpoints"`
|
||||
ConnectionString string `json:"connection_string"`
|
||||
Host string `json:"host"`
|
||||
Port int `json:"port"`
|
||||
Username string `json:"username"`
|
||||
Password string `json:"password"`
|
||||
SSL bool `json:"ssl"`
|
||||
CACertPath string `json:"ca_cert_path"`
|
||||
ClientCertPath string `json:"client_cert_path"`
|
||||
ClientKeyPath string `json:"client_key_path"`
|
||||
CertPath string `json:"cert_path"`
|
||||
KeyPath string `json:"key_path"`
|
||||
ConnectTimeoutSecs int `json:"connect_timeout_secs"`
|
||||
GrpcMaxInboundMessageSize int `json:"grpc_max_inbound_message_size"`
|
||||
URLParams string `json:"url_params"`
|
||||
}
|
||||
|
||||
// authenticatedClient issues v2 API requests against one etcd endpoint with
|
||||
// optional basic auth and TLS.
|
||||
type authenticatedClient struct {
|
||||
endpoint string
|
||||
http *http.Client
|
||||
username string
|
||||
password string
|
||||
serverVersion string
|
||||
}
|
||||
|
||||
func connectionObject(params map[string]json.RawMessage) (connectionParams, error) {
|
||||
var connection connectionParams
|
||||
if raw, ok := params["connection"]; ok && raw != nil {
|
||||
if err := json.Unmarshal(raw, &connection); err != nil {
|
||||
return connection, err
|
||||
}
|
||||
return connection, nil
|
||||
}
|
||||
if err := decodeParams(params, &connection); err != nil {
|
||||
return connection, err
|
||||
}
|
||||
return connection, nil
|
||||
}
|
||||
|
||||
func (s *etcd2Session) connect(params map[string]json.RawMessage) (any, error) {
|
||||
connection, err := connectionObject(params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
endpointList := connectionEndpoints(connection)
|
||||
var lastErr error
|
||||
for _, endpoint := range endpointList {
|
||||
client, _, err := probeClient(endpoint, connection)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
s.close()
|
||||
s.clientMu.Lock()
|
||||
s.httpClient = client
|
||||
s.connectedEndpoints = endpointList
|
||||
s.serverVersion = client.serverVersion
|
||||
s.clientMu.Unlock()
|
||||
return map[string]bool{"ok": true}, nil
|
||||
}
|
||||
if lastErr == nil {
|
||||
lastErr = errors.New("No etcd endpoint configured")
|
||||
}
|
||||
return nil, lastErr
|
||||
}
|
||||
|
||||
func (s *etcd2Session) activeClient() (*authenticatedClient, error) {
|
||||
s.clientMu.Lock()
|
||||
client := s.httpClient
|
||||
s.clientMu.Unlock()
|
||||
if client == nil {
|
||||
return nil, errors.New("Not connected")
|
||||
}
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func (s *etcd2Session) connectedEndpointList() []string {
|
||||
s.clientMu.Lock()
|
||||
defer s.clientMu.Unlock()
|
||||
return append([]string(nil), s.connectedEndpoints...)
|
||||
}
|
||||
|
||||
func (s *etcd2Session) validateConnection() (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), operationTimeout)
|
||||
defer cancel()
|
||||
probe, err := client.probeV2(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return probe, nil
|
||||
}
|
||||
|
||||
func (s *etcd2Session) close() error {
|
||||
s.watchesMu.Lock()
|
||||
watches := s.watches
|
||||
s.watches = map[string]*watchState{}
|
||||
s.watchBufferedBytes = 0
|
||||
s.watchesMu.Unlock()
|
||||
for _, watch := range watches {
|
||||
watch.close()
|
||||
}
|
||||
s.clientMu.Lock()
|
||||
s.httpClient = nil
|
||||
s.connectedEndpoints = nil
|
||||
s.serverVersion = ""
|
||||
s.clientMu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func buildHTTPClient(connection connectionParams) (*http.Client, error) {
|
||||
transport := &http.Transport{
|
||||
MaxIdleConns: 4,
|
||||
MaxIdleConnsPerHost: 4,
|
||||
}
|
||||
if connection.SSL {
|
||||
tlsConfig, err := tlsConfigFor(connection)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
transport.TLSClientConfig = tlsConfig
|
||||
}
|
||||
return &http.Client{
|
||||
Transport: transport,
|
||||
// The request context governs per-call timeouts; the client-level
|
||||
// timeout stays high so long-poll watches are not cut short.
|
||||
Timeout: 10 * time.Minute,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func connectTimeoutSeconds(connection connectionParams) time.Duration {
|
||||
seconds := connection.ConnectTimeoutSecs
|
||||
if seconds == 0 {
|
||||
seconds = 30
|
||||
}
|
||||
if seconds < 1 {
|
||||
seconds = 1
|
||||
}
|
||||
if seconds > 300 {
|
||||
seconds = 300
|
||||
}
|
||||
return time.Duration(seconds) * time.Second
|
||||
}
|
||||
|
||||
func tlsConfigFor(connection connectionParams) (*tls.Config, error) {
|
||||
tlsConfig := &tls.Config{}
|
||||
if ca := strings.TrimSpace(connection.CACertPath); ca != "" {
|
||||
authorityPEM, err := os.ReadFile(ca)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pool := x509.NewCertPool()
|
||||
if !pool.AppendCertsFromPEM(authorityPEM) {
|
||||
return nil, fmt.Errorf("failed to parse CA certificate at %s", ca)
|
||||
}
|
||||
tlsConfig.RootCAs = pool
|
||||
}
|
||||
certPath := firstNonBlank(connection.ClientCertPath, connection.CertPath)
|
||||
keyPath := firstNonBlank(connection.ClientKeyPath, connection.KeyPath)
|
||||
if (certPath == "") != (keyPath == "") {
|
||||
return nil, errors.New("Client certificate and key must be provided together")
|
||||
}
|
||||
if certPath != "" {
|
||||
pair, err := tls.LoadX509KeyPair(certPath, keyPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tlsConfig.Certificates = []tls.Certificate{pair}
|
||||
}
|
||||
return tlsConfig, nil
|
||||
}
|
||||
|
||||
// probeClient verifies the endpoint speaks the v2 API. It returns the
|
||||
// connected client and a probe result shaped like the v3 agent's:
|
||||
// {ok, endpoint, limited?}.
|
||||
func probeClient(endpoint string, connection connectionParams) (*authenticatedClient, map[string]any, error) {
|
||||
httpClient, err := buildHTTPClient(connection)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
client := &authenticatedClient{
|
||||
endpoint: strings.TrimSuffix(endpoint, "/"),
|
||||
http: httpClient,
|
||||
username: connection.Username,
|
||||
password: connection.Password,
|
||||
}
|
||||
dialCtx, dialCancel := context.WithTimeout(context.Background(), connectTimeoutSeconds(connection))
|
||||
defer dialCancel()
|
||||
version, err := client.fetchVersion(dialCtx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
client.serverVersion = version.etcdserver
|
||||
probe, err := client.probeV2(dialCtx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return client, probe, nil
|
||||
}
|
||||
|
||||
type etcdVersion struct {
|
||||
etcdserver string
|
||||
etcdcluster string
|
||||
}
|
||||
|
||||
func (c *authenticatedClient) fetchVersion(ctx context.Context) (etcdVersion, error) {
|
||||
body, _, err := c.do(ctx, http.MethodGet, "/version", "", nil)
|
||||
if err != nil {
|
||||
return etcdVersion{}, err
|
||||
}
|
||||
var parsed struct {
|
||||
Etcdserver string `json:"etcdserver"`
|
||||
Etcdcluster string `json:"etcdcluster"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &parsed); err != nil {
|
||||
return etcdVersion{}, fmt.Errorf("unrecognized etcd /version response from %s: %w", c.endpoint, err)
|
||||
}
|
||||
return etcdVersion{etcdserver: parsed.Etcdserver, etcdcluster: parsed.Etcdcluster}, nil
|
||||
}
|
||||
|
||||
// probeV2 checks that the v2 keys API is actually served. A 403 proves the
|
||||
// channel and credentials reached etcd, mirroring the v3 agent's
|
||||
// PERMISSION_DENIED handling for restricted users.
|
||||
func (c *authenticatedClient) probeV2(ctx context.Context) (map[string]any, error) {
|
||||
response, err := c.request(ctx, http.MethodGet, "/v2/members", "", nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer drainClose(response.Body)
|
||||
switch response.StatusCode {
|
||||
case http.StatusOK:
|
||||
return map[string]any{"ok": true, "endpoint": c.endpoint}, nil
|
||||
case http.StatusForbidden:
|
||||
return map[string]any{"ok": true, "endpoint": c.endpoint, "limited": true}, nil
|
||||
case http.StatusNotFound:
|
||||
return nil, fmt.Errorf("ETCD_V2_API_DISABLED: %s does not expose the etcd v2 API (removed in etcd 3.6+)", c.endpoint)
|
||||
case http.StatusUnauthorized:
|
||||
return nil, fmt.Errorf("ETCD_UNAUTHENTICATED: authentication failed against %s", c.endpoint)
|
||||
default:
|
||||
body, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
|
||||
return nil, fmt.Errorf("etcd v2 probe against %s failed: HTTP %d %s", c.endpoint, response.StatusCode, strings.TrimSpace(string(body)))
|
||||
}
|
||||
}
|
||||
|
||||
// do performs a v2 API request and returns the body. Non-2xx responses are
|
||||
// converted into etcdError values carrying the server's errorCode/message.
|
||||
func (c *authenticatedClient) do(ctx context.Context, method, path, body string, header map[string]string) ([]byte, *http.Response, error) {
|
||||
response, err := c.request(ctx, method, path, body, header)
|
||||
if err != nil {
|
||||
return nil, response, err
|
||||
}
|
||||
payload, readErr := io.ReadAll(response.Body)
|
||||
_ = response.Body.Close()
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
return nil, response, errorFromResponse(response.StatusCode, payload)
|
||||
}
|
||||
if readErr != nil {
|
||||
return nil, response, readErr
|
||||
}
|
||||
return payload, response, nil
|
||||
}
|
||||
|
||||
func (c *authenticatedClient) request(ctx context.Context, method, path, body string, header map[string]string) (*http.Response, error) {
|
||||
return c.requestAt(ctx, method, c.endpoint, path, body, header)
|
||||
}
|
||||
|
||||
// requestAt targets a specific endpoint, used for per-member status fan-out.
|
||||
func (c *authenticatedClient) requestAt(ctx context.Context, method, endpoint, path, body string, header map[string]string) (*http.Response, error) {
|
||||
var reader io.Reader
|
||||
if body != "" {
|
||||
reader = strings.NewReader(body)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, strings.TrimSuffix(endpoint, "/")+path, reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c.username != "" {
|
||||
req.SetBasicAuth(c.username, c.password)
|
||||
}
|
||||
for key, value := range header {
|
||||
req.Header.Set(key, value)
|
||||
}
|
||||
if body != "" && req.Header.Get("Content-Type") == "" {
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
}
|
||||
response, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
// gRPC-style sentinel so the host-side transient-error handling keeps working.
|
||||
return nil, fmt.Errorf("etcd connection error: %w", err)
|
||||
}
|
||||
return response, nil
|
||||
}
|
||||
|
||||
// doAt performs a v2 API request against a specific endpoint.
|
||||
func (c *authenticatedClient) doAt(ctx context.Context, method, endpoint, path, body string) ([]byte, *http.Response, error) {
|
||||
response, err := c.requestAt(ctx, method, endpoint, path, body, nil)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
payload, readErr := io.ReadAll(response.Body)
|
||||
_ = response.Body.Close()
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
return nil, response, errorFromResponse(response.StatusCode, payload)
|
||||
}
|
||||
if readErr != nil {
|
||||
return nil, response, readErr
|
||||
}
|
||||
return payload, response, nil
|
||||
}
|
||||
|
||||
// etcdError carries the v2 API error payload (errorCode + message + index).
|
||||
type etcdError struct {
|
||||
statusCode int
|
||||
errorCode int64
|
||||
message string
|
||||
cause string
|
||||
index int64
|
||||
}
|
||||
|
||||
func (e *etcdError) Error() string {
|
||||
if e.message == "" {
|
||||
return fmt.Sprintf("etcd v2 error (HTTP %d)", e.statusCode)
|
||||
}
|
||||
if e.cause != "" {
|
||||
return fmt.Sprintf("%s (%s)", e.message, e.cause)
|
||||
}
|
||||
return e.message
|
||||
}
|
||||
|
||||
func errorFromResponse(statusCode int, body []byte) *etcdError {
|
||||
parsed := &etcdError{statusCode: statusCode}
|
||||
if len(body) > 0 {
|
||||
var payload struct {
|
||||
ErrorCode int64 `json:"errorCode"`
|
||||
Message string `json:"message"`
|
||||
Cause string `json:"cause"`
|
||||
Index int64 `json:"index"`
|
||||
}
|
||||
if json.Unmarshal(body, &payload) == nil && payload.Message != "" {
|
||||
parsed.errorCode = payload.ErrorCode
|
||||
parsed.message = payload.Message
|
||||
parsed.cause = payload.Cause
|
||||
parsed.index = payload.Index
|
||||
return parsed
|
||||
}
|
||||
}
|
||||
parsed.message = strings.TrimSpace(string(body))
|
||||
return parsed
|
||||
}
|
||||
|
||||
func isEtcdErrorCode(err error, code int64) bool {
|
||||
var etcdErr *etcdError
|
||||
if errors.As(err, &etcdErr) {
|
||||
return etcdErr.errorCode == code
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isNotFound(err error) bool {
|
||||
return isEtcdErrorCode(err, 100)
|
||||
}
|
||||
|
||||
func isCompareFailed(err error) bool {
|
||||
return isEtcdErrorCode(err, 101)
|
||||
}
|
||||
|
||||
func drainClose(body io.ReadCloser) {
|
||||
_, _ = io.Copy(io.Discard, io.LimitReader(body, 4096))
|
||||
_ = body.Close()
|
||||
}
|
||||
|
||||
func connectionEndpoints(connection connectionParams) []string {
|
||||
configured := firstNonBlank(connection.EtcdEndpoints, connection.Endpoints, connection.ConnectionString)
|
||||
var result []string
|
||||
if configured != "" {
|
||||
for _, endpoint := range strings.FieldsFunc(configured, func(r rune) bool { return r == ',' || r == '\n' }) {
|
||||
normalized := normalizeEndpoint(strings.TrimSpace(endpoint), connection.SSL)
|
||||
if normalized != "" {
|
||||
result = append(result, normalized)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(result) == 0 {
|
||||
host := connection.Host
|
||||
if host == "" {
|
||||
host = "127.0.0.1"
|
||||
}
|
||||
port := connection.Port
|
||||
if port == 0 {
|
||||
port = defaultPort
|
||||
}
|
||||
result = append(result, normalizeEndpoint(fmt.Sprintf("%s:%d", host, port), connection.SSL))
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func normalizeEndpoint(endpoint string, tlsEnabled bool) string {
|
||||
if strings.TrimSpace(endpoint) == "" {
|
||||
return ""
|
||||
}
|
||||
if strings.HasPrefix(endpoint, "http://") || strings.HasPrefix(endpoint, "https://") {
|
||||
return strings.TrimSuffix(endpoint, "/")
|
||||
}
|
||||
scheme := "http"
|
||||
if tlsEnabled {
|
||||
scheme = "https"
|
||||
}
|
||||
return scheme + "://" + endpoint
|
||||
}
|
||||
|
||||
func endpointHost(rawEndpoint string) string {
|
||||
parsed, err := url.Parse(rawEndpoint)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return parsed.Hostname()
|
||||
}
|
||||
|
||||
func firstNonBlank(values ...string) string {
|
||||
for _, value := range values {
|
||||
if strings.TrimSpace(value) != "" {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
module github.com/t8y2/dbx/agents/drivers/etcd2-go
|
||||
|
||||
go 1.24
|
||||
@@ -0,0 +1,412 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestLiveEtcd2Agent exercises the v2 agent surface against a real etcd 2.x
|
||||
// server. Enable with DBX_ETCD2_LIVE=1; configure via DBX_ETCD2_ENDPOINTS,
|
||||
// DBX_ETCD2_USER, and DBX_ETCD2_PASSWORD (defaults match the server
|
||||
// deployment: root/123456).
|
||||
func TestLiveEtcd2Agent(t *testing.T) {
|
||||
if os.Getenv("DBX_ETCD2_LIVE") != "1" {
|
||||
t.Skip("set DBX_ETCD2_LIVE=1 to run the live etcd v2 agent test")
|
||||
}
|
||||
endpoints := envOrDefault("DBX_ETCD2_ENDPOINTS", "http://172.26.129.83:20041")
|
||||
user := envOrDefault("DBX_ETCD2_USER", "root")
|
||||
password := envOrDefault("DBX_ETCD2_PASSWORD", "123456")
|
||||
|
||||
state := newEtcd2Session()
|
||||
connectParams := map[string]json.RawMessage{
|
||||
"etcd_endpoints": json.RawMessage(`"` + endpoints + `"`),
|
||||
"username": json.RawMessage(`"` + user + `"`),
|
||||
"password": json.RawMessage(`"` + password + `"`),
|
||||
}
|
||||
if _, err := state.connect(connectParams); err != nil {
|
||||
t.Fatalf("connect failed: %v", err)
|
||||
}
|
||||
defer state.close()
|
||||
|
||||
if result, err := state.validateConnection(); err != nil {
|
||||
t.Fatalf("validate_connection failed: %v", err)
|
||||
} else if probe := result.(map[string]any); probe["ok"] != true {
|
||||
t.Fatalf("unexpected probe result: %#v", probe)
|
||||
}
|
||||
|
||||
prefix := fmt.Sprintf("/dbx/live/%d/", time.Now().UnixNano())
|
||||
|
||||
// --- KV basics -----------------------------------------------------
|
||||
if _, err := state.put(paramsWith(map[string]json.RawMessage{
|
||||
"key": jsonString(prefix + "hello"),
|
||||
"value": jsonValue("world"),
|
||||
})); err != nil {
|
||||
t.Fatalf("put failed: %v", err)
|
||||
}
|
||||
fetched, err := state.get(paramsWith(map[string]json.RawMessage{"key": jsonString(prefix + "hello")}))
|
||||
if err != nil {
|
||||
t.Fatalf("get failed: %v", err)
|
||||
}
|
||||
row := fetched.(map[string]any)
|
||||
if row["found"] != true || row["key"] != prefix+"hello" {
|
||||
t.Fatalf("unexpected get result: %#v", row)
|
||||
}
|
||||
value := row["value"].(map[string]any)
|
||||
if value["encoding"] != "utf8" || value["data"] != "world" {
|
||||
t.Fatalf("unexpected value: %#v", value)
|
||||
}
|
||||
metadata := row["metadata"].(map[string]any)
|
||||
if metadata["createRevision"] != metadata["modRevision"] {
|
||||
t.Fatalf("fresh key indexes should match: %#v", metadata)
|
||||
}
|
||||
|
||||
missing, err := state.get(paramsWith(map[string]json.RawMessage{"key": jsonString(prefix + "missing")}))
|
||||
if err != nil {
|
||||
t.Fatalf("missing get failed: %v", err)
|
||||
}
|
||||
if missing.(map[string]any)["found"] != false {
|
||||
t.Fatalf("expected found=false: %#v", missing)
|
||||
}
|
||||
|
||||
// CAS conflict on stale index.
|
||||
if _, err := state.put(paramsWith(map[string]json.RawMessage{
|
||||
"key": jsonString(prefix + "hello"),
|
||||
"value": jsonValue("v2"),
|
||||
"expectedModRevision": json.RawMessage(`1`),
|
||||
})); err == nil || err.Error() != "ETCD_CAS_CONFLICT: key changed after it was loaded" {
|
||||
t.Fatalf("expected CAS conflict, got %v", err)
|
||||
}
|
||||
if _, err := state.put(paramsWith(map[string]json.RawMessage{
|
||||
"key": jsonString(prefix + "hello"),
|
||||
"value": jsonValue("v3"),
|
||||
"expectedModRevision": json.RawMessage(metadata["modRevision"].(string)),
|
||||
})); err != nil {
|
||||
t.Fatalf("CAS put failed: %v", err)
|
||||
}
|
||||
|
||||
// TTL put and metadata.
|
||||
if _, err := state.put(paramsWith(map[string]json.RawMessage{
|
||||
"key": jsonString(prefix + "leased"),
|
||||
"value": jsonValue("expiring"),
|
||||
"ttl": json.RawMessage(`300`),
|
||||
})); err != nil {
|
||||
t.Fatalf("ttl put failed: %v", err)
|
||||
}
|
||||
leased, err := state.get(paramsWith(map[string]json.RawMessage{"key": jsonString(prefix + "leased")}))
|
||||
if err != nil {
|
||||
t.Fatalf("leased get failed: %v", err)
|
||||
}
|
||||
if _, hasTTL := leased.(map[string]any)["metadata"].(map[string]any)["ttl"]; !hasTTL {
|
||||
t.Fatalf("leased key must expose ttl metadata: %#v", leased)
|
||||
}
|
||||
|
||||
// Lease objects are a v3 concept and must be rejected clearly.
|
||||
if _, err := state.put(paramsWith(map[string]json.RawMessage{
|
||||
"key": jsonString(prefix + "nolease"),
|
||||
"value": jsonValue("v"),
|
||||
"lease": json.RawMessage(`5`),
|
||||
})); err == nil || !strings.HasPrefix(err.Error(), "ETCD_V2_LEASE_UNSUPPORTED") {
|
||||
t.Fatalf("expected lease rejection, got %v", err)
|
||||
}
|
||||
|
||||
// Rename including conflict and missing-source paths.
|
||||
if _, err := state.rename(paramsWith(map[string]json.RawMessage{
|
||||
"key": jsonString(prefix + "old"),
|
||||
"newKey": jsonString(prefix + "new"),
|
||||
})); err == nil || err.Error() != "ETCD_NOT_FOUND: source key does not exist" {
|
||||
t.Fatalf("expected rename not-found, got %v", err)
|
||||
}
|
||||
if _, err := state.rename(paramsWith(map[string]json.RawMessage{
|
||||
"key": jsonString(prefix + "hello"),
|
||||
"newKey": jsonString(prefix + "renamed"),
|
||||
})); err != nil {
|
||||
t.Fatalf("rename failed: %v", err)
|
||||
}
|
||||
renamed, err := state.get(paramsWith(map[string]json.RawMessage{"key": jsonString(prefix + "renamed")}))
|
||||
if err != nil || renamed.(map[string]any)["found"] != true {
|
||||
t.Fatalf("rename target missing: %#v %v", renamed, err)
|
||||
}
|
||||
if renamed.(map[string]any)["value"].(map[string]any)["data"] != "v3" {
|
||||
t.Fatalf("rename must carry the value: %#v", renamed)
|
||||
}
|
||||
source, err := state.get(paramsWith(map[string]json.RawMessage{"key": jsonString(prefix + "hello")}))
|
||||
if err != nil || source.(map[string]any)["found"] != false {
|
||||
t.Fatalf("rename must remove the source: %#v %v", source, err)
|
||||
}
|
||||
if _, err := state.rename(paramsWith(map[string]json.RawMessage{
|
||||
"key": jsonString(prefix + "leased"),
|
||||
"newKey": jsonString(prefix + "renamed"),
|
||||
})); err == nil || err.Error() != "ETCD_CAS_CONFLICT: source changed or target already exists" {
|
||||
t.Fatalf("expected rename conflict, got %v", err)
|
||||
}
|
||||
|
||||
// list_prefix with continuation.
|
||||
for i := 0; i < 3; i++ {
|
||||
if _, err := state.put(paramsWith(map[string]json.RawMessage{
|
||||
"key": jsonString(fmt.Sprintf("%sbulk%d", prefix, i)),
|
||||
"value": jsonValue(fmt.Sprintf("v%d", i)),
|
||||
})); err != nil {
|
||||
t.Fatalf("bulk put %d failed: %v", i, err)
|
||||
}
|
||||
}
|
||||
listed, err := state.listPrefix(paramsWith(map[string]json.RawMessage{
|
||||
"prefix": jsonString(prefix),
|
||||
"limit": json.RawMessage(`2`),
|
||||
"includeValues": json.RawMessage(`true`),
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("list_prefix failed: %v", err)
|
||||
}
|
||||
list := listed.(map[string]any)
|
||||
if len(list["keys"].([]any)) != 2 {
|
||||
t.Fatalf("expected limited list, got %#v", list)
|
||||
}
|
||||
continuation, ok := list["continuation"].(string)
|
||||
if !ok || continuation == "" {
|
||||
t.Fatalf("expected continuation, got %#v", list)
|
||||
}
|
||||
nextPage, err := state.listPrefix(paramsWith(map[string]json.RawMessage{
|
||||
"prefix": jsonString(prefix),
|
||||
"limit": json.RawMessage(`100`),
|
||||
"includeValues": json.RawMessage(`true`),
|
||||
"continuation": json.RawMessage(`"` + continuation + `"`),
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("continuation list failed: %v", err)
|
||||
}
|
||||
nextKeys := nextPage.(map[string]any)["keys"].([]any)
|
||||
if len(nextKeys) == 0 {
|
||||
t.Fatalf("continuation must resume after the first page")
|
||||
}
|
||||
if nextKeys[0].(map[string]any)["key"].(string) == list["keys"].([]any)[1].(map[string]any)["key"].(string) {
|
||||
t.Fatalf("continuation overlapped the first page")
|
||||
}
|
||||
|
||||
// --- watch (long poll) --------------------------------------------
|
||||
watchStarted, err := state.watchStart(paramsWith(map[string]json.RawMessage{
|
||||
"key": jsonString(prefix + "watched"),
|
||||
"includePrevKv": json.RawMessage(`true`),
|
||||
}))
|
||||
if err != nil {
|
||||
t.Fatalf("watch_start failed: %v", err)
|
||||
}
|
||||
watchID := watchStarted.(map[string]any)["watchId"].(string)
|
||||
if _, err := state.put(paramsWith(map[string]json.RawMessage{
|
||||
"key": jsonString(prefix + "watched"),
|
||||
"value": jsonValue("first"),
|
||||
})); err != nil {
|
||||
t.Fatalf("watched put failed: %v", err)
|
||||
}
|
||||
if _, err := state.put(paramsWith(map[string]json.RawMessage{
|
||||
"key": jsonString(prefix + "watched"),
|
||||
"value": jsonValue("second"),
|
||||
})); err != nil {
|
||||
t.Fatalf("watched put 2 failed: %v", err)
|
||||
}
|
||||
deadline := time.Now().Add(20 * time.Second)
|
||||
var watchBatches []any
|
||||
for time.Now().Before(deadline) {
|
||||
polled, err := state.watchPoll(paramsWith(map[string]json.RawMessage{"watchId": json.RawMessage(`"` + watchID + `"`)}))
|
||||
if err != nil {
|
||||
t.Fatalf("watch_poll failed: %v", err)
|
||||
}
|
||||
batches := polled.(map[string]any)["batches"].([]any)
|
||||
watchBatches = append(watchBatches, batches...)
|
||||
if len(watchBatches) >= 2 {
|
||||
break
|
||||
}
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
}
|
||||
if len(watchBatches) < 2 {
|
||||
t.Fatalf("watch did not deliver both events: %#v", watchBatches)
|
||||
}
|
||||
lastBatch := watchBatches[len(watchBatches)-1].(map[string]any)
|
||||
lastEvent := lastBatch["events"].([]map[string]any)[0]
|
||||
if lastEvent["value"].(map[string]any)["data"] != "second" {
|
||||
t.Fatalf("unexpected last watch event: %#v", lastEvent)
|
||||
}
|
||||
if lastEvent["previousValue"].(map[string]any)["data"] != "first" {
|
||||
t.Fatalf("prevKv missing on watch event: %#v", lastEvent)
|
||||
}
|
||||
stopped, err := state.watchStop(paramsWith(map[string]json.RawMessage{"watchId": json.RawMessage(`"` + watchID + `"`)}))
|
||||
if err != nil || stopped.(map[string]bool)["stopped"] != true {
|
||||
t.Fatalf("watch_stop failed: %#v %v", stopped, err)
|
||||
}
|
||||
|
||||
// --- auth ----------------------------------------------------------
|
||||
userName := fmt.Sprintf("dbxlive%d", time.Now().UnixNano()%100000)
|
||||
// Idempotent pre-cleanup: an interrupted earlier run may have left the
|
||||
// fixed-named role behind.
|
||||
_, _ = state.authRoleDelete(paramsWith(map[string]json.RawMessage{"role": jsonString("dbx_live_role")}))
|
||||
if _, err := state.authUserAdd(paramsWith(map[string]json.RawMessage{
|
||||
"user": jsonString(userName),
|
||||
"password": jsonString("secret123"),
|
||||
})); err != nil {
|
||||
t.Fatalf("user add failed: %v", err)
|
||||
}
|
||||
if _, err := state.authRoleAdd(paramsWith(map[string]json.RawMessage{"role": jsonString("dbx_live_role")})); err != nil {
|
||||
t.Fatalf("role add failed: %v", err)
|
||||
}
|
||||
if _, err := state.authRolePermission(paramsWith(map[string]json.RawMessage{
|
||||
"role": jsonString("dbx_live_role"),
|
||||
"resource": jsonString("prefix"),
|
||||
"key": jsonString(prefix),
|
||||
"access": jsonString("READWRITE"),
|
||||
}), true); err != nil {
|
||||
t.Fatalf("role grant permission failed: %v", err)
|
||||
}
|
||||
roleDetail, err := state.authRoleGet(paramsWith(map[string]json.RawMessage{"role": jsonString("dbx_live_role")}))
|
||||
if err != nil {
|
||||
t.Fatalf("role get failed: %v", err)
|
||||
}
|
||||
permissions := roleDetail.(map[string]any)["permissions"].([]map[string]any)
|
||||
if len(permissions) == 0 {
|
||||
t.Fatalf("expected permission rows after grant: %#v", roleDetail)
|
||||
}
|
||||
if _, err := state.authUserGrantRevokeRole(paramsWith(map[string]json.RawMessage{
|
||||
"user": jsonString(userName),
|
||||
"role": jsonString("dbx_live_role"),
|
||||
}), true); err != nil {
|
||||
t.Fatalf("grant role failed: %v", err)
|
||||
}
|
||||
userDetail, err := state.authUserGet(paramsWith(map[string]json.RawMessage{"user": jsonString(userName)}))
|
||||
if err != nil {
|
||||
t.Fatalf("user get failed: %v", err)
|
||||
}
|
||||
if len(userDetail.(map[string]any)["roles"].([]string)) == 0 {
|
||||
t.Fatalf("expected granted role: %#v", userDetail)
|
||||
}
|
||||
users, err := state.authUserList(paramsWith(map[string]json.RawMessage{}))
|
||||
if err != nil {
|
||||
t.Fatalf("user list failed: %v", err)
|
||||
}
|
||||
if !containsAny(users.(map[string]any)["users"].([]string), userName) {
|
||||
t.Fatalf("user list missing created user: %#v", users)
|
||||
}
|
||||
roles, err := state.authRoleList(paramsWith(map[string]json.RawMessage{}))
|
||||
if err != nil || !containsAny(roles.(map[string]any)["roles"].([]string), "dbx_live_role") {
|
||||
t.Fatalf("role list missing role: %#v %v", roles, err)
|
||||
}
|
||||
if _, err := state.authRolePermission(paramsWith(map[string]json.RawMessage{
|
||||
"role": jsonString("dbx_live_role"),
|
||||
"resource": jsonString("prefix"),
|
||||
"key": jsonString(prefix),
|
||||
}), false); err != nil {
|
||||
t.Fatalf("role revoke permission failed: %v", err)
|
||||
}
|
||||
afterRevoke, err := state.authRoleGet(paramsWith(map[string]json.RawMessage{"role": jsonString("dbx_live_role")}))
|
||||
if err != nil {
|
||||
t.Fatalf("role get after revoke failed: %v", err)
|
||||
}
|
||||
if len(afterRevoke.(map[string]any)["permissions"].([]map[string]any)) != 0 {
|
||||
t.Fatalf("permissions must be empty after revoke: %#v", afterRevoke)
|
||||
}
|
||||
if _, err := state.authUserGrantRevokeRole(paramsWith(map[string]json.RawMessage{
|
||||
"user": jsonString(userName),
|
||||
"role": jsonString("dbx_live_role"),
|
||||
}), false); err != nil {
|
||||
t.Fatalf("revoke role failed: %v", err)
|
||||
}
|
||||
if _, err := state.authUserChangePassword(paramsWith(map[string]json.RawMessage{
|
||||
"user": jsonString(userName),
|
||||
"password": jsonString("rotated456"),
|
||||
})); err != nil {
|
||||
t.Fatalf("change password failed: %v", err)
|
||||
}
|
||||
if _, err := state.authUserDelete(paramsWith(map[string]json.RawMessage{"user": jsonString(userName)})); err != nil {
|
||||
t.Fatalf("user delete failed: %v", err)
|
||||
}
|
||||
if _, err := state.authRoleDelete(paramsWith(map[string]json.RawMessage{"role": jsonString("dbx_live_role")})); err != nil {
|
||||
t.Fatalf("role delete failed: %v", err)
|
||||
}
|
||||
|
||||
// --- status --------------------------------------------------------
|
||||
status, err := state.status(paramsWith(map[string]json.RawMessage{}))
|
||||
if err != nil {
|
||||
t.Fatalf("kv_status failed: %v", err)
|
||||
}
|
||||
statusRow := status.(map[string]any)
|
||||
if statusRow["clusterId"] == nil || statusRow["leaderId"] == nil {
|
||||
t.Fatalf("status missing cluster identity: %#v", statusRow)
|
||||
}
|
||||
members := statusRow["members"].([]map[string]any)
|
||||
if len(members) == 0 || members[0]["reachable"] != true {
|
||||
t.Fatalf("unexpected member rows: %#v", members)
|
||||
}
|
||||
|
||||
// --- delete + cleanup ----------------------------------------------
|
||||
for _, key := range []string{prefix + "leased", prefix + "renamed", prefix + "watched", prefix + "bulk0", prefix + "bulk1", prefix + "bulk2"} {
|
||||
if _, err := state.delete(paramsWith(map[string]json.RawMessage{"key": jsonString(key)})); err != nil {
|
||||
t.Fatalf("delete %s failed: %v", key, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLiveEtcd2ProtocolFlow(t *testing.T) {
|
||||
if os.Getenv("DBX_ETCD2_LIVE") != "1" {
|
||||
t.Skip("set DBX_ETCD2_LIVE=1 to run the live etcd v2 agent test")
|
||||
}
|
||||
endpoints := envOrDefault("DBX_ETCD2_ENDPOINTS", "http://172.26.129.83:20041")
|
||||
user := envOrDefault("DBX_ETCD2_USER", "root")
|
||||
password := envOrDefault("DBX_ETCD2_PASSWORD", "123456")
|
||||
|
||||
server := newRuntimeServer()
|
||||
connection := fmt.Sprintf(`{"etcd_endpoints":%q,"username":%q,"password":%q}`, endpoints, user, password)
|
||||
|
||||
handshake, _ := server.handleLine(`{"id":1,"method":"handshake","params":{}}`)
|
||||
if handshake.Error != nil {
|
||||
t.Fatalf("handshake failed: %#v", handshake)
|
||||
}
|
||||
openLine := fmt.Sprintf(`{"id":2,"method":"open_session","params":{"agentSessionId":"s1","connection":%s}}`, connection)
|
||||
opened, _ := server.handleLine(openLine)
|
||||
if opened.Error != nil {
|
||||
t.Fatalf("open_session failed: %#v", opened)
|
||||
}
|
||||
putLine := fmt.Sprintf(`{"id":3,"method":"kv_put","params":{"agentSessionId":"s1","key":"/dbx:proto","value":{"encoding":"utf8","data":"flow"}}}`)
|
||||
put, _ := server.handleLine(putLine)
|
||||
if put.Error != nil {
|
||||
t.Fatalf("kv_put failed: %#v", put)
|
||||
}
|
||||
v3Only, _ := server.handleLine(`{"id":4,"method":"kv_history","params":{"agentSessionId":"s1","key":"/dbx:proto"}}`)
|
||||
if v3Only.Error == nil || !strings.HasPrefix(v3Only.Error.Message, "ETCD_V2_UNSUPPORTED") {
|
||||
t.Fatalf("expected ETCD_V2_UNSUPPORTED, got %#v", v3Only)
|
||||
}
|
||||
closed, _ := server.handleLine(`{"id":5,"method":"close_session","params":{"agentSessionId":"s1"}}`)
|
||||
if closed.Error != nil {
|
||||
t.Fatalf("close_session failed: %#v", closed)
|
||||
}
|
||||
}
|
||||
|
||||
func envOrDefault(key, fallback string) string {
|
||||
if value := strings.TrimSpace(os.Getenv(key)); value != "" {
|
||||
return value
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func paramsWith(params map[string]json.RawMessage) map[string]json.RawMessage {
|
||||
return params
|
||||
}
|
||||
|
||||
func jsonString(value string) json.RawMessage {
|
||||
encoded, _ := json.Marshal(value)
|
||||
return json.RawMessage(encoded)
|
||||
}
|
||||
|
||||
func jsonValue(value string) json.RawMessage {
|
||||
encoded, _ := json.Marshal(value)
|
||||
return json.RawMessage(`{"encoding":"utf8","data":` + string(encoded) + `}`)
|
||||
}
|
||||
|
||||
func containsAny(values []string, target string) bool {
|
||||
for _, value := range values {
|
||||
if value == target {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,524 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const defaultListLimit = 100
|
||||
|
||||
// v2Node is one node of the etcd v2 keys API tree.
|
||||
type v2Node struct {
|
||||
Key string `json:"key"`
|
||||
Value string `json:"value"`
|
||||
Dir bool `json:"dir"`
|
||||
Expiration *string `json:"expiration"`
|
||||
TTL *int64 `json:"ttl"`
|
||||
CreatedIndex int64 `json:"createdIndex"`
|
||||
ModifiedIndex int64 `json:"modifiedIndex"`
|
||||
Nodes []v2Node `json:"nodes"`
|
||||
}
|
||||
|
||||
type v2KeysResponse struct {
|
||||
Action string `json:"action"`
|
||||
Node *v2Node `json:"node"`
|
||||
PrevNode *v2Node `json:"prevNode"`
|
||||
}
|
||||
|
||||
// v2KeyPath normalizes a protocol key into a v2 keys API path. v2 keys are
|
||||
// slash paths; an empty prefix maps to the root directory.
|
||||
func v2KeyPath(key string) string {
|
||||
if key == "" {
|
||||
return "/v2/keys/"
|
||||
}
|
||||
if !strings.HasPrefix(key, "/") {
|
||||
key = "/" + key
|
||||
}
|
||||
return "/v2/keys" + key
|
||||
}
|
||||
|
||||
func (s *etcd2Session) listPrefix(params map[string]json.RawMessage) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
prefix := stringOrDefault(params, "prefix", "")
|
||||
limit := intOrDefault(params, "limit", defaultListLimit)
|
||||
if limit < 1 {
|
||||
limit = 1
|
||||
}
|
||||
includeValues := boolOrDefault(params, "includeValues", false)
|
||||
continuation := stringOrNull(params, "continuation")
|
||||
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
// v2 has no revisioned reads; the revision parameter is accepted and ignored.
|
||||
query := url.Values{}
|
||||
query.Set("recursive", "true")
|
||||
query.Set("sorted", "true")
|
||||
body, _, err := client.do(ctx, http.MethodGet, v2KeyPath(prefix)+"?"+query.Encode(), "", nil)
|
||||
if err != nil {
|
||||
if isNotFound(err) {
|
||||
return map[string]any{"keys": []any{}, "continuation": nil, "revision": nil}, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
var parsed v2KeysResponse
|
||||
if err := json.Unmarshal(body, &parsed); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
startAfter := ""
|
||||
if continuation != nil && *continuation != "" {
|
||||
startAfter = *continuation
|
||||
}
|
||||
rows := []map[string]any{}
|
||||
flattenNodes(parsed.Node, includeValues, func(row map[string]any) {
|
||||
key := row["key"].(string)
|
||||
if startAfter != "" && key <= startAfter {
|
||||
return
|
||||
}
|
||||
if len(rows) < limit {
|
||||
rows = append(rows, row)
|
||||
}
|
||||
})
|
||||
|
||||
result := map[string]any{"keys": toAnySlice(rows)}
|
||||
if len(rows) == limit {
|
||||
result["continuation"] = rows[len(rows)-1]["key"]
|
||||
} else {
|
||||
result["continuation"] = nil
|
||||
}
|
||||
// etcd v2 exposes the current index on the listing's node headers; the
|
||||
// modifiedIndex of the returned root is the closest analog of a revision.
|
||||
var revision any
|
||||
if parsed.Node != nil {
|
||||
revision = longString(parsed.Node.ModifiedIndex)
|
||||
}
|
||||
result["revision"] = revision
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func flattenNodes(node *v2Node, includeValues bool, emit func(map[string]any)) {
|
||||
if node == nil {
|
||||
return
|
||||
}
|
||||
if !node.Dir {
|
||||
row := nodeMetadata(node)
|
||||
row["key"] = node.Key
|
||||
if includeValues {
|
||||
row["value"] = valueObject([]byte(node.Value))
|
||||
}
|
||||
emit(row)
|
||||
return
|
||||
}
|
||||
children := append([]v2Node(nil), node.Nodes...)
|
||||
sort.SliceStable(children, func(i, j int) bool { return children[i].Key < children[j].Key })
|
||||
for index := range children {
|
||||
flattenNodes(&children[index], includeValues, emit)
|
||||
}
|
||||
}
|
||||
|
||||
func toAnySlice(rows []map[string]any) []any {
|
||||
result := make([]any, len(rows))
|
||||
for index, row := range rows {
|
||||
result[index] = row
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (s *etcd2Session) get(params map[string]json.RawMessage) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key, err := keyBytesParam(params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
body, _, err := client.do(ctx, http.MethodGet, v2KeyPath(key), "", nil)
|
||||
if err != nil {
|
||||
if isNotFound(err) {
|
||||
return map[string]any{"found": false, "key": nil, "value": nil, "metadata": nil}, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
var parsed v2KeysResponse
|
||||
if err := json.Unmarshal(body, &parsed); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if parsed.Node == nil || parsed.Node.Dir {
|
||||
return map[string]any{"found": false, "key": nil, "value": nil, "metadata": nil}, nil
|
||||
}
|
||||
node := parsed.Node
|
||||
result := map[string]any{
|
||||
"found": true,
|
||||
"key": node.Key,
|
||||
"keyBytes": bytesObject([]byte(node.Key)),
|
||||
}
|
||||
if !boolOrDefault(params, "metadataOnly", false) {
|
||||
result["value"] = valueObject([]byte(node.Value))
|
||||
} else {
|
||||
result["value"] = nil
|
||||
}
|
||||
result["metadata"] = nodeMetadata(node)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *etcd2Session) put(params map[string]json.RawMessage) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key, err := keyBytesParam(params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
value, err := parseValueObject(rawObject(params, "value"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
expectedModRevision := longOrNull(params, "expectedModRevision")
|
||||
expectedCreateRevision := longOrNull(params, "expectedCreateRevision")
|
||||
ttlValue := longOrNull(params, "ttl")
|
||||
preserveLease := boolOrDefault(params, "preserveLease", false)
|
||||
|
||||
leaseValue := longOrNull(params, "lease")
|
||||
hasLease := leaseValue != nil
|
||||
hasTtl := ttlValue != nil
|
||||
if (hasLease && hasTtl) || (preserveLease && (hasLease || hasTtl)) {
|
||||
return nil, errors.New("lease, ttl, and preserveLease cannot be specified together")
|
||||
}
|
||||
if hasLease {
|
||||
return nil, errors.New("ETCD_V2_LEASE_UNSUPPORTED: the etcd v2 API has no lease objects; use ttl instead")
|
||||
}
|
||||
if preserveLease {
|
||||
return nil, errors.New("ETCD_V2_LEASE_UNSUPPORTED: the etcd v2 API has no lease objects; use ttl instead")
|
||||
}
|
||||
if expectedCreateRevision != nil && *expectedCreateRevision != 0 {
|
||||
return nil, errors.New("ETCD_V2_CAS_UNSUPPORTED: the etcd v2 API cannot compare createdIndex; use expectedModRevision")
|
||||
}
|
||||
if hasTtl && *ttlValue <= 0 {
|
||||
return nil, errors.New("ttl must be a positive integer")
|
||||
}
|
||||
|
||||
form := url.Values{}
|
||||
form.Set("value", value)
|
||||
if hasTtl {
|
||||
form.Set("ttl", strconv.FormatInt(*ttlValue, 10))
|
||||
}
|
||||
if expectedModRevision != nil {
|
||||
form.Set("prevIndex", strconv.FormatInt(*expectedModRevision, 10))
|
||||
}
|
||||
if expectedCreateRevision != nil && *expectedCreateRevision == 0 {
|
||||
form.Set("prevExist", "false")
|
||||
}
|
||||
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
body, _, err := client.do(ctx, http.MethodPut, v2KeyPath(key), form.Encode(), nil)
|
||||
if err != nil {
|
||||
if isCompareFailed(err) {
|
||||
return nil, errors.New("ETCD_CAS_CONFLICT: key changed after it was loaded")
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
var parsed v2KeysResponse
|
||||
if err := json.Unmarshal(body, &parsed); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var revision int64
|
||||
if parsed.Node != nil {
|
||||
revision = parsed.Node.ModifiedIndex
|
||||
}
|
||||
return map[string]any{"revision": longString(revision)}, nil
|
||||
}
|
||||
|
||||
func (s *etcd2Session) delete(params map[string]json.RawMessage) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
key, err := keyBytesParam(params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
expectedModRevision := longOrNull(params, "expectedModRevision")
|
||||
path := v2KeyPath(key)
|
||||
if expectedModRevision != nil {
|
||||
query := url.Values{}
|
||||
query.Set("prevIndex", strconv.FormatInt(*expectedModRevision, 10))
|
||||
path += "?" + query.Encode()
|
||||
}
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
body, _, err := client.do(ctx, http.MethodDelete, path, "", nil)
|
||||
if err != nil {
|
||||
if isCompareFailed(err) {
|
||||
return nil, errors.New("ETCD_CAS_CONFLICT: key changed after it was loaded")
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
var parsed v2KeysResponse
|
||||
if err := json.Unmarshal(body, &parsed); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var revision int64
|
||||
if parsed.Node != nil {
|
||||
revision = parsed.Node.ModifiedIndex
|
||||
}
|
||||
return map[string]any{"deleted": int64(1), "revision": longString(revision)}, nil
|
||||
}
|
||||
|
||||
// rename is a non-atomic check-then-set: the v2 API has no transactions, so a
|
||||
// concurrent writer between the target check and the delete can be lost. The
|
||||
// protocol exposes this honestly instead of pretending atomicity.
|
||||
func (s *etcd2Session) rename(params map[string]json.RawMessage) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sourceKey, err := keyBytesParam(params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
newKey := stringOrNull(params, "newKey")
|
||||
if newKey == nil || *newKey == "" {
|
||||
return nil, errors.New("ETCD_NEWKEY_REQUIRED")
|
||||
}
|
||||
targetKey := *newKey
|
||||
if sourceKey == targetKey {
|
||||
return map[string]any{"renamed": true, "revision": nil}, nil
|
||||
}
|
||||
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
body, _, err := client.do(ctx, http.MethodGet, v2KeyPath(sourceKey), "", nil)
|
||||
if err != nil {
|
||||
if isNotFound(err) {
|
||||
return nil, errors.New("ETCD_NOT_FOUND: source key does not exist")
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
var source v2KeysResponse
|
||||
if err := json.Unmarshal(body, &source); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if source.Node == nil || source.Node.Dir {
|
||||
return nil, errors.New("ETCD_NOT_FOUND: source key does not exist")
|
||||
}
|
||||
if _, _, err := client.do(ctx, http.MethodGet, v2KeyPath(targetKey), "", nil); err == nil {
|
||||
return nil, errors.New("ETCD_CAS_CONFLICT: source changed or target already exists")
|
||||
} else if !isNotFound(err) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
expected := longOrNull(params, "expectedModRevision")
|
||||
expectedRevision := source.Node.ModifiedIndex
|
||||
if expected != nil {
|
||||
expectedRevision = *expected
|
||||
}
|
||||
form := url.Values{}
|
||||
form.Set("value", source.Node.Value)
|
||||
if source.Node.TTL != nil {
|
||||
form.Set("ttl", strconv.FormatInt(*source.Node.TTL, 10))
|
||||
}
|
||||
form.Set("prevExist", "false")
|
||||
if _, _, err := client.do(ctx, http.MethodPut, v2KeyPath(targetKey), form.Encode(), nil); err != nil {
|
||||
if isCompareFailed(err) {
|
||||
return nil, errors.New("ETCD_CAS_CONFLICT: source changed or target already exists")
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
deleteQuery := url.Values{}
|
||||
deleteQuery.Set("prevIndex", strconv.FormatInt(expectedRevision, 10))
|
||||
if _, _, err := client.do(ctx, http.MethodDelete, v2KeyPath(sourceKey)+"?"+deleteQuery.Encode(), "", nil); err != nil {
|
||||
if isCompareFailed(err) {
|
||||
return nil, errors.New("ETCD_CAS_CONFLICT: source changed or target already exists")
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{"renamed": true, "revision": nil}, nil
|
||||
}
|
||||
|
||||
func nodeMetadata(node *v2Node) map[string]any {
|
||||
metadata := map[string]any{
|
||||
"createRevision": longString(node.CreatedIndex),
|
||||
"modRevision": longString(node.ModifiedIndex),
|
||||
"version": longString(int64(1)),
|
||||
"lease": longString(0),
|
||||
"valueSize": len(node.Value),
|
||||
}
|
||||
if node.TTL != nil {
|
||||
metadata["ttl"] = *node.TTL
|
||||
}
|
||||
return metadata
|
||||
}
|
||||
|
||||
func longString(value int64) string {
|
||||
return strconv.FormatInt(value, 10)
|
||||
}
|
||||
|
||||
func unsignedLongString(value int64) string {
|
||||
return strconv.FormatUint(uint64(value), 10)
|
||||
}
|
||||
|
||||
// prefixEnd mirrors the v3 agent's byte-increment range end; for v2 rows it
|
||||
// only feeds UI metadata, where the explicit resource field carries the truth.
|
||||
func prefixEnd(prefix string) string {
|
||||
if prefix == "" {
|
||||
return "\x00"
|
||||
}
|
||||
end := []byte(prefix)
|
||||
for i := len(end) - 1; i >= 0; i-- {
|
||||
if end[i] < 0xff {
|
||||
end[i]++
|
||||
return string(end[:i+1])
|
||||
}
|
||||
}
|
||||
return "\x00"
|
||||
}
|
||||
|
||||
func keyBytesParam(params map[string]json.RawMessage) (string, error) {
|
||||
if encoded := rawObject(params, "keyBytes"); encoded != nil {
|
||||
value, err := parseValueObject(encoded)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
if raw, ok := params["key"]; ok && raw != nil && string(raw) != "null" {
|
||||
var key string
|
||||
if err := json.Unmarshal(raw, &key); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
return "", errors.New("Key is required")
|
||||
}
|
||||
|
||||
func rawObject(params map[string]json.RawMessage, key string) map[string]json.RawMessage {
|
||||
raw := params[key]
|
||||
if len(raw) == 0 || string(raw) == "null" {
|
||||
return nil
|
||||
}
|
||||
var object map[string]json.RawMessage
|
||||
if err := json.Unmarshal(raw, &object); err != nil || object == nil {
|
||||
return nil
|
||||
}
|
||||
return object
|
||||
}
|
||||
|
||||
func bytesObject(bytes []byte) map[string]any {
|
||||
return map[string]any{
|
||||
"encoding": "base64",
|
||||
"data": base64.StdEncoding.EncodeToString(bytes),
|
||||
}
|
||||
}
|
||||
|
||||
func valueObject(value []byte) map[string]any {
|
||||
if utf8.Valid(value) {
|
||||
return map[string]any{"encoding": "utf8", "data": string(value)}
|
||||
}
|
||||
return bytesObject(value)
|
||||
}
|
||||
|
||||
func displayBytes(bytes []byte) string {
|
||||
if utf8.Valid(bytes) {
|
||||
return string(bytes)
|
||||
}
|
||||
return base64.StdEncoding.EncodeToString(bytes)
|
||||
}
|
||||
|
||||
func parseValueObject(value map[string]json.RawMessage) (string, error) {
|
||||
if value == nil {
|
||||
value = map[string]json.RawMessage{}
|
||||
}
|
||||
encoding := stringOrDefault(value, "encoding", "utf8")
|
||||
data := stringOrDefault(value, "data", "")
|
||||
if encoding == "base64" {
|
||||
decoded, err := base64.StdEncoding.DecodeString(data)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(decoded), nil
|
||||
}
|
||||
if encoding != "utf8" {
|
||||
return "", fmt.Errorf("Unsupported value encoding: %s", encoding)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func stringOrNull(params map[string]json.RawMessage, key string) *string {
|
||||
raw := params[key]
|
||||
if len(raw) == 0 || string(raw) == "null" {
|
||||
return nil
|
||||
}
|
||||
var value string
|
||||
if err := json.Unmarshal(raw, &value); err != nil {
|
||||
return nil
|
||||
}
|
||||
return &value
|
||||
}
|
||||
|
||||
func stringOrDefault(params map[string]json.RawMessage, key string, fallback string) string {
|
||||
value := stringOrNull(params, key)
|
||||
if value == nil {
|
||||
return fallback
|
||||
}
|
||||
return *value
|
||||
}
|
||||
|
||||
func intOrDefault(params map[string]json.RawMessage, key string, fallback int) int {
|
||||
raw := params[key]
|
||||
if len(raw) == 0 || string(raw) == "null" {
|
||||
return fallback
|
||||
}
|
||||
var value int
|
||||
if err := json.Unmarshal(raw, &value); err != nil {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func boolOrDefault(params map[string]json.RawMessage, key string, fallback bool) bool {
|
||||
raw := params[key]
|
||||
if len(raw) == 0 || string(raw) == "null" {
|
||||
return fallback
|
||||
}
|
||||
var value bool
|
||||
if err := json.Unmarshal(raw, &value); err != nil {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func longOrNull(params map[string]json.RawMessage, key string) *int64 {
|
||||
raw := params[key]
|
||||
if len(raw) == 0 || string(raw) == "null" {
|
||||
return nil
|
||||
}
|
||||
var value int64
|
||||
if err := json.Unmarshal(raw, &value); err != nil {
|
||||
return nil
|
||||
}
|
||||
return &value
|
||||
}
|
||||
|
||||
func requiredString(params map[string]json.RawMessage, field string) (string, error) {
|
||||
value := stringOrNull(params, field)
|
||||
if value == nil || *value == "" {
|
||||
return "", fmt.Errorf("ETCD_%s_REQUIRED", strings.ToUpper(field))
|
||||
}
|
||||
return *value, nil
|
||||
}
|
||||
@@ -0,0 +1,422 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
const (
|
||||
protocolVersion = 2
|
||||
legacyAgentSessionID = "__legacy__"
|
||||
maxAgentSessions = 256
|
||||
)
|
||||
|
||||
// Capabilities advertise the v2 API surface: no history, no lease objects,
|
||||
// no compaction, and no defrag because etcd v2 semantics do not have them.
|
||||
var capabilities = []string{
|
||||
"connect", "test_connection", "kv", "kv_ttl", "kv_cas", "kv_list_values",
|
||||
"kv_status", "etcd_watch", "etcd_auth", "multi_session",
|
||||
}
|
||||
|
||||
type request struct {
|
||||
ID json.RawMessage `json:"id"`
|
||||
Method string `json:"method"`
|
||||
Params map[string]json.RawMessage `json:"params"`
|
||||
}
|
||||
|
||||
type response struct {
|
||||
JSONRPC string `json:"jsonrpc,omitempty"`
|
||||
ID json.RawMessage `json:"id,omitempty"`
|
||||
Result any `json:"result,omitempty"`
|
||||
Error *rpcError `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// etcd2Session carries the per-connection HTTP client and derived state.
|
||||
// Requests for one session are serialized by agentSession.mu; watch pump
|
||||
// goroutines reach the maps through watchesMu instead.
|
||||
type etcd2Session struct {
|
||||
clientMu sync.Mutex
|
||||
httpClient *authenticatedClient
|
||||
connectedEndpoints []string
|
||||
serverVersion string
|
||||
|
||||
watchesMu sync.Mutex
|
||||
watches map[string]*watchState
|
||||
watchBufferedBytes int64
|
||||
|
||||
activeCancelMu sync.Mutex
|
||||
activeCancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (s *etcd2Session) reserveWatchBuffer(bytes int64) bool {
|
||||
s.watchesMu.Lock()
|
||||
defer s.watchesMu.Unlock()
|
||||
if bytes < 0 || s.watchBufferedBytes+bytes > maxSessionWatchBufferBytes {
|
||||
return false
|
||||
}
|
||||
s.watchBufferedBytes += bytes
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *etcd2Session) releaseWatchBuffer(bytes int64) {
|
||||
s.watchesMu.Lock()
|
||||
defer s.watchesMu.Unlock()
|
||||
s.watchBufferedBytes -= bytes
|
||||
if s.watchBufferedBytes < 0 {
|
||||
s.watchBufferedBytes = 0
|
||||
}
|
||||
}
|
||||
|
||||
type agentSession struct {
|
||||
state *etcd2Session
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func (a *agentSession) cancelActive() {
|
||||
a.state.activeCancelMu.Lock()
|
||||
cancel := a.state.activeCancel
|
||||
a.state.activeCancelMu.Unlock()
|
||||
if cancel != nil {
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
|
||||
type runtimeServer struct {
|
||||
mu sync.RWMutex
|
||||
sessions map[string]*agentSession
|
||||
}
|
||||
|
||||
func main() {
|
||||
configureRuntimeParallelism()
|
||||
server := newRuntimeServer()
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
var encoderMu sync.Mutex
|
||||
var requests sync.WaitGroup
|
||||
fmt.Fprintln(os.Stdout, `{"ready":true}`)
|
||||
|
||||
scanner := bufio.NewScanner(os.Stdin)
|
||||
scanner.Buffer(make([]byte, 0, 64*1024), 512*1024*1024)
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
var envelope request
|
||||
if json.Unmarshal([]byte(line), &envelope) == nil && envelope.Method == "shutdown" {
|
||||
requests.Wait()
|
||||
result, _ := server.handleLine(line)
|
||||
encoderMu.Lock()
|
||||
_ = encoder.Encode(result)
|
||||
encoderMu.Unlock()
|
||||
server.closeAll()
|
||||
return
|
||||
}
|
||||
requests.Add(1)
|
||||
go func(line string) {
|
||||
defer requests.Done()
|
||||
result, _ := server.handleLine(line)
|
||||
encoderMu.Lock()
|
||||
defer encoderMu.Unlock()
|
||||
if err := encoder.Encode(result); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "failed to write response: %v\n", err)
|
||||
}
|
||||
}(line)
|
||||
}
|
||||
requests.Wait()
|
||||
server.closeAll()
|
||||
}
|
||||
|
||||
func configureRuntimeParallelism() {
|
||||
if raw := strings.TrimSpace(os.Getenv("DBX_AGENT_ETCD2_GOMAXPROCS")); raw != "" {
|
||||
if configured, err := strconv.Atoi(raw); err == nil && configured > 0 {
|
||||
runtime.GOMAXPROCS(configured)
|
||||
return
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(os.Getenv("GOMAXPROCS")) != "" {
|
||||
return
|
||||
}
|
||||
runtime.GOMAXPROCS(min(runtime.NumCPU(), 4))
|
||||
}
|
||||
|
||||
func newRuntimeServer() *runtimeServer {
|
||||
return &runtimeServer{sessions: map[string]*agentSession{}}
|
||||
}
|
||||
|
||||
func (r *runtimeServer) handleLine(line string) (response, bool) {
|
||||
var req request
|
||||
if err := json.Unmarshal([]byte(line), &req); err != nil {
|
||||
return errorResponse(nil, "", "", err), false
|
||||
}
|
||||
if len(req.ID) == 0 {
|
||||
req.ID = json.RawMessage("1")
|
||||
}
|
||||
result, shutdown, err := r.dispatch(req.Method, req.Params)
|
||||
if err != nil {
|
||||
return errorResponse(req.ID, req.Method, stringParam(req.Params, "agentSessionId"), err), false
|
||||
}
|
||||
return response{JSONRPC: "2.0", ID: req.ID, Result: result}, shutdown
|
||||
}
|
||||
|
||||
func (r *runtimeServer) dispatch(method string, params map[string]json.RawMessage) (any, bool, error) {
|
||||
switch method {
|
||||
case "handshake":
|
||||
return handshakeResult(), false, nil
|
||||
case "open_session":
|
||||
id := requiredSessionID(params)
|
||||
if id == "" {
|
||||
return nil, false, errors.New("agentSessionId is required")
|
||||
}
|
||||
return r.openSession(id, params)
|
||||
case "close_session":
|
||||
return r.closeSession(stringParam(params, "agentSessionId")), false, nil
|
||||
case "validate_session":
|
||||
session, err := r.session(requiredSessionID(params))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
session.mu.Lock()
|
||||
defer session.mu.Unlock()
|
||||
result, err := session.state.validateConnection()
|
||||
return result, false, err
|
||||
case "cancel_session":
|
||||
session, err := r.session(requiredSessionID(params))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
session.cancelActive()
|
||||
return map[string]bool{"ok": true}, false, nil
|
||||
case "test_connection":
|
||||
return r.testConnection(params)
|
||||
case "connect":
|
||||
_ = r.closeSession(legacyAgentSessionID)
|
||||
return r.openSession(legacyAgentSessionID, params)
|
||||
case "disconnect":
|
||||
return r.closeSession(legacyAgentSessionID), false, nil
|
||||
case "shutdown":
|
||||
return map[string]bool{"ok": true}, true, r.closeAll()
|
||||
default:
|
||||
id := stringParam(params, "agentSessionId")
|
||||
if id == "" {
|
||||
id = legacyAgentSessionID
|
||||
}
|
||||
session, err := r.session(id)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
session.mu.Lock()
|
||||
defer session.mu.Unlock()
|
||||
result, err := session.state.handle(method, params)
|
||||
return result, false, err
|
||||
}
|
||||
}
|
||||
|
||||
func requiredSessionID(params map[string]json.RawMessage) string {
|
||||
return strings.TrimSpace(stringParam(params, "agentSessionId"))
|
||||
}
|
||||
|
||||
func (r *runtimeServer) openSession(id string, params map[string]json.RawMessage) (any, bool, error) {
|
||||
r.mu.Lock()
|
||||
if _, exists := r.sessions[id]; exists {
|
||||
r.mu.Unlock()
|
||||
return nil, false, fmt.Errorf("Agent session already exists: %s", id)
|
||||
}
|
||||
if len(r.sessions) >= maxAgentSessions {
|
||||
r.mu.Unlock()
|
||||
return nil, false, fmt.Errorf("Agent session limit reached: %d", maxAgentSessions)
|
||||
}
|
||||
session := &agentSession{state: newEtcd2Session()}
|
||||
r.sessions[id] = session
|
||||
r.mu.Unlock()
|
||||
|
||||
if _, err := session.state.connect(params); err != nil {
|
||||
r.mu.Lock()
|
||||
delete(r.sessions, id)
|
||||
r.mu.Unlock()
|
||||
session.state.close()
|
||||
return nil, false, err
|
||||
}
|
||||
return map[string]bool{"ok": true}, false, nil
|
||||
}
|
||||
|
||||
func (r *runtimeServer) session(id string) (*agentSession, error) {
|
||||
r.mu.RLock()
|
||||
session := r.sessions[id]
|
||||
r.mu.RUnlock()
|
||||
if session == nil {
|
||||
return nil, fmt.Errorf("Agent session not found: %s", id)
|
||||
}
|
||||
return session, nil
|
||||
}
|
||||
|
||||
func (r *runtimeServer) closeSession(id string) any {
|
||||
r.mu.Lock()
|
||||
session := r.sessions[id]
|
||||
delete(r.sessions, id)
|
||||
r.mu.Unlock()
|
||||
if session != nil {
|
||||
session.cancelActive()
|
||||
session.mu.Lock()
|
||||
session.state.close()
|
||||
session.mu.Unlock()
|
||||
}
|
||||
return map[string]bool{"ok": true}
|
||||
}
|
||||
|
||||
func (r *runtimeServer) closeAll() error {
|
||||
r.mu.Lock()
|
||||
sessions := r.sessions
|
||||
r.sessions = map[string]*agentSession{}
|
||||
r.mu.Unlock()
|
||||
var firstErr error
|
||||
for _, session := range sessions {
|
||||
session.cancelActive()
|
||||
if err := session.state.close(); err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
}
|
||||
return firstErr
|
||||
}
|
||||
|
||||
func (r *runtimeServer) testConnection(params map[string]json.RawMessage) (any, bool, error) {
|
||||
state := newEtcd2Session()
|
||||
defer state.close()
|
||||
if _, err := state.connect(params); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return map[string]bool{"ok": true}, false, nil
|
||||
}
|
||||
|
||||
func newEtcd2Session() *etcd2Session {
|
||||
return &etcd2Session{watches: map[string]*watchState{}}
|
||||
}
|
||||
|
||||
func handshakeResult() map[string]any {
|
||||
return map[string]any{
|
||||
"protocolVersion": protocolVersion,
|
||||
"agentProtocolVersion": protocolVersion,
|
||||
"capabilities": capabilities,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *etcd2Session) beginOperation() (context.Context, context.CancelFunc) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), operationTimeout)
|
||||
s.activeCancelMu.Lock()
|
||||
s.activeCancel = cancel
|
||||
s.activeCancelMu.Unlock()
|
||||
return ctx, cancel
|
||||
}
|
||||
|
||||
func (s *etcd2Session) endOperation(cancel context.CancelFunc) {
|
||||
cancel()
|
||||
s.activeCancelMu.Lock()
|
||||
s.activeCancel = nil
|
||||
s.activeCancelMu.Unlock()
|
||||
}
|
||||
|
||||
func (s *etcd2Session) handle(method string, params map[string]json.RawMessage) (any, error) {
|
||||
switch method {
|
||||
case "handshake":
|
||||
return handshakeResult(), nil
|
||||
case "connect", "test_connection":
|
||||
if _, err := s.connect(params); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]bool{"ok": true}, nil
|
||||
case "validate_connection":
|
||||
return s.validateConnection()
|
||||
case "kv_list_prefix":
|
||||
return s.listPrefix(params)
|
||||
case "kv_get":
|
||||
return s.get(params)
|
||||
case "kv_put":
|
||||
return s.put(params)
|
||||
case "kv_delete":
|
||||
return s.delete(params)
|
||||
case "kv_rename":
|
||||
return s.rename(params)
|
||||
case "kv_status":
|
||||
return s.status(params)
|
||||
case "etcd_watch_start":
|
||||
return s.watchStart(params)
|
||||
case "etcd_watch_poll":
|
||||
return s.watchPoll(params)
|
||||
case "etcd_watch_stop":
|
||||
return s.watchStop(params)
|
||||
case "etcd_auth_user_list":
|
||||
return s.authUserList(params)
|
||||
case "etcd_auth_user_get":
|
||||
return s.authUserGet(params)
|
||||
case "etcd_auth_user_add":
|
||||
return s.authUserAdd(params)
|
||||
case "etcd_auth_user_delete":
|
||||
return s.authUserDelete(params)
|
||||
case "etcd_auth_user_change_password":
|
||||
return s.authUserChangePassword(params)
|
||||
case "etcd_auth_user_grant_role":
|
||||
return s.authUserGrantRevokeRole(params, true)
|
||||
case "etcd_auth_user_revoke_role":
|
||||
return s.authUserGrantRevokeRole(params, false)
|
||||
case "etcd_auth_role_list":
|
||||
return s.authRoleList(params)
|
||||
case "etcd_auth_role_get":
|
||||
return s.authRoleGet(params)
|
||||
case "etcd_auth_role_add":
|
||||
return s.authRoleAdd(params)
|
||||
case "etcd_auth_role_delete":
|
||||
return s.authRoleDelete(params)
|
||||
case "etcd_auth_role_grant_permission":
|
||||
return s.authRolePermission(params, true)
|
||||
case "etcd_auth_role_revoke_permission":
|
||||
return s.authRolePermission(params, false)
|
||||
case "disconnect":
|
||||
s.close()
|
||||
return map[string]bool{"ok": true}, nil
|
||||
case "shutdown":
|
||||
s.close()
|
||||
return map[string]bool{"ok": true}, nil
|
||||
default:
|
||||
if isEtcd3OnlyMethod(method) {
|
||||
return nil, fmt.Errorf("ETCD_V2_UNSUPPORTED: %s is not available on the etcd v2 API", method)
|
||||
}
|
||||
return nil, fmt.Errorf("unknown method: %s", method)
|
||||
}
|
||||
}
|
||||
|
||||
func isEtcd3OnlyMethod(method string) bool {
|
||||
switch method {
|
||||
case "kv_history", "etcd_compact", "etcd_defrag",
|
||||
"etcd_lease_list", "etcd_lease_get", "etcd_lease_grant",
|
||||
"etcd_lease_keepalive_once", "etcd_lease_revoke":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func decodeParams(params map[string]json.RawMessage, target any) error {
|
||||
if params == nil {
|
||||
params = map[string]json.RawMessage{}
|
||||
}
|
||||
data, err := json.Marshal(params)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.Unmarshal(data, target)
|
||||
}
|
||||
|
||||
func stringParam(params map[string]json.RawMessage, key string) string {
|
||||
if params == nil {
|
||||
return ""
|
||||
}
|
||||
var result string
|
||||
_ = json.Unmarshal(params[key], &result)
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,267 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHandshakeAdvertisesV2Capabilities(t *testing.T) {
|
||||
server := newRuntimeServer()
|
||||
result, _, err := server.dispatch("handshake", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("handshake failed: %v", err)
|
||||
}
|
||||
handshake := result.(map[string]any)
|
||||
want := []string{
|
||||
"connect", "test_connection", "kv", "kv_ttl", "kv_cas", "kv_list_values",
|
||||
"kv_status", "etcd_watch", "etcd_auth", "multi_session",
|
||||
}
|
||||
got := handshake["capabilities"].([]string)
|
||||
if strings.Join(got, ",") != strings.Join(want, ",") {
|
||||
t.Fatalf("capability mismatch:\n got %v\nwant %v", got, want)
|
||||
}
|
||||
for _, forbidden := range []string{"kv_history", "etcd_lease", "etcd_compaction", "etcd_defrag", "structured_error_v1"} {
|
||||
for _, capability := range got {
|
||||
if capability == forbidden {
|
||||
t.Fatalf("%s must not be advertised on the v2 agent", forbidden)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleLineReportsSessionNotFound(t *testing.T) {
|
||||
server := newRuntimeServer()
|
||||
response, _ := server.handleLine(`{"id":7,"method":"kv_get","params":{"key":"a"}}`)
|
||||
if response.Error == nil || response.Error.Code != -1 {
|
||||
t.Fatalf("expected code -1 error, got %#v", response)
|
||||
}
|
||||
if response.Error.Message != "Agent session not found: __legacy__" {
|
||||
t.Fatalf("unexpected message: %q", response.Error.Message)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnconnectedSessionErrors(t *testing.T) {
|
||||
state := newEtcd2Session()
|
||||
if _, err := state.get(map[string]json.RawMessage{"key": json.RawMessage(`"k"`)}); err == nil || err.Error() != "Not connected" {
|
||||
t.Fatalf("kv_get before connect: %v", err)
|
||||
}
|
||||
if _, err := state.validateConnection(); err == nil || err.Error() != "Not connected" {
|
||||
t.Fatalf("validate_connection before connect: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEtcd3OnlyMethodsRejected(t *testing.T) {
|
||||
state := newEtcd2Session()
|
||||
for _, method := range []string{"kv_history", "etcd_lease_list", "etcd_lease_grant", "etcd_compact", "etcd_defrag"} {
|
||||
_, err := state.handle(method, map[string]json.RawMessage{})
|
||||
if err == nil || !strings.HasPrefix(err.Error(), "ETCD_V2_UNSUPPORTED") {
|
||||
t.Fatalf("%s should report ETCD_V2_UNSUPPORTED, got %v", method, err)
|
||||
}
|
||||
}
|
||||
_, err := state.handle("etcd_nonsense", map[string]json.RawMessage{})
|
||||
if err == nil || !strings.Contains(err.Error(), "unknown method") {
|
||||
t.Fatalf("expected unknown method error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConnectionEndpoints(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
connection connectionParams
|
||||
want []string
|
||||
}{
|
||||
{"host port fallback", connectionParams{Port: 2379}, []string{"http://127.0.0.1:2379"}},
|
||||
{"ssl scheme", connectionParams{Host: "db", Port: 1, SSL: true}, []string{"https://db:1"}},
|
||||
{"comma list", connectionParams{EtcdEndpoints: "a:1,b:2"}, []string{"http://a:1", "http://b:2"}},
|
||||
{"existing scheme trimmed", connectionParams{EtcdEndpoints: "https://a:1/"}, []string{"https://a:1"}},
|
||||
}
|
||||
for _, testCase := range cases {
|
||||
got := connectionEndpoints(testCase.connection)
|
||||
if strings.Join(got, "|") != strings.Join(testCase.want, "|") {
|
||||
t.Fatalf("%s: got %v, want %v", testCase.name, got, testCase.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTLSConfigRequiresCertAndKeyTogether(t *testing.T) {
|
||||
_, err := tlsConfigFor(connectionParams{SSL: true, ClientCertPath: "/tmp/cert.pem"})
|
||||
if err == nil || err.Error() != "Client certificate and key must be provided together" {
|
||||
t.Fatalf("expected paired cert/key error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestV2KeyPath(t *testing.T) {
|
||||
cases := []struct{ key, want string }{
|
||||
{"", "/v2/keys/"},
|
||||
{"foo", "/v2/keys/foo"},
|
||||
{"/foo/bar", "/v2/keys/foo/bar"},
|
||||
}
|
||||
for _, testCase := range cases {
|
||||
if got := v2KeyPath(testCase.key); got != testCase.want {
|
||||
t.Fatalf("v2KeyPath(%q) = %q, want %q", testCase.key, got, testCase.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestErrorFromResponse(t *testing.T) {
|
||||
err := errorFromResponse(404, []byte(`{"errorCode":100,"message":"Key not found","cause":"/foo","index":37}`))
|
||||
if !isNotFound(err) {
|
||||
t.Fatalf("expected key-not-found classification: %#v", err)
|
||||
}
|
||||
if isCompareFailed(err) {
|
||||
t.Fatalf("errorCode 100 is not compare failure")
|
||||
}
|
||||
if err.Error() != "Key not found (/foo)" {
|
||||
t.Fatalf("unexpected message: %q", err.Error())
|
||||
}
|
||||
compare := errorFromResponse(412, []byte(`{"errorCode":101,"message":"Compare failed","cause":"[1 != 2]"}`))
|
||||
if !isCompareFailed(compare) {
|
||||
t.Fatalf("expected compare failure classification: %#v", compare)
|
||||
}
|
||||
plain := errorFromResponse(500, []byte("boom"))
|
||||
if plain.Error() != "boom" {
|
||||
t.Fatalf("plain body passthrough broken: %q", plain.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestV2PermissionRows(t *testing.T) {
|
||||
rows := v2PermissionRows("read", []string{"/a", "/p/*"})
|
||||
if len(rows) != 2 {
|
||||
t.Fatalf("expected 2 permission rows, got %d", len(rows))
|
||||
}
|
||||
first := rows[0]
|
||||
if first["access"] != "read" || first["resource"] != "key" {
|
||||
t.Fatalf("unexpected exact-key row: %#v", first)
|
||||
}
|
||||
second := rows[1]
|
||||
if second["access"] != "read" || second["resource"] != "prefix" {
|
||||
t.Fatalf("unexpected prefix row: %#v", second)
|
||||
}
|
||||
end := second["rangeEnd"].(map[string]any)
|
||||
if end["data"] != "L3Aw" { // prefixEnd("/p/") == "/p0"
|
||||
t.Fatalf("prefix rangeEnd wrong: %#v", end)
|
||||
}
|
||||
}
|
||||
|
||||
func TestV2PermissionPattern(t *testing.T) {
|
||||
if got := v2PermissionPattern("key", "/foo"); got != "/foo" {
|
||||
t.Fatalf("key pattern must stay bare: %q", got)
|
||||
}
|
||||
if got := v2PermissionPattern("prefix", "/foo/"); got != "/foo/*" {
|
||||
t.Fatalf("prefix pattern must become glob: %q", got)
|
||||
}
|
||||
if got := v2PermissionPattern("all", "/ignored"); got != "/*" {
|
||||
t.Fatalf("all pattern must be root glob: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestV2RoleGrantDocument(t *testing.T) {
|
||||
encoded, err := json.Marshal(v2Role{
|
||||
Role: "r1",
|
||||
Grant: &v2Permissions{KV: v2RWPermission{Read: []string{"/p/*"}}},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var round v2Role
|
||||
if err := json.Unmarshal(encoded, &round); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if round.Role != "r1" || len(round.Grant.KV.Read) != 1 || round.Grant.KV.Read[0] != "/p/*" {
|
||||
t.Fatalf("grant document round-trip broken: %s", encoded)
|
||||
}
|
||||
if round.Permissions != nil {
|
||||
t.Fatalf("grant documents must not carry permissions: %s", encoded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMemberHexToUnsigned(t *testing.T) {
|
||||
if got := memberHexToUnsigned("272e204152"); got != "168277590354" {
|
||||
t.Fatalf("unexpected conversion: %v", got)
|
||||
}
|
||||
if got := memberHexToUnsigned(""); got != nil {
|
||||
t.Fatalf("empty id must map to nil, got %v", got)
|
||||
}
|
||||
if got := memberHexToUnsigned("zzz"); got != nil {
|
||||
t.Fatalf("invalid hex must map to nil, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchBudgetState(t *testing.T) {
|
||||
session := newEtcd2Session()
|
||||
state := &watchState{watchID: "w", session: session}
|
||||
state.append(10, []map[string]any{{"eventType": "put"}}, 128)
|
||||
state.append(11, []map[string]any{{"eventType": "put"}}, maxWatchBufferBytes+1)
|
||||
polled := state.poll()
|
||||
if len(polled["batches"].([]any)) != 1 {
|
||||
t.Fatalf("overflow must keep buffered batch: %#v", polled)
|
||||
}
|
||||
terminal, ok := polled["terminal"].(map[string]any)
|
||||
if !ok || terminal["reason"] != "overflow" {
|
||||
t.Fatalf("expected overflow terminal: %#v", polled)
|
||||
}
|
||||
if session.watchBufferedBytesSnapshot() != 0 {
|
||||
t.Fatalf("budget not released: %d", session.watchBufferedBytesSnapshot())
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchLimits(t *testing.T) {
|
||||
session := newEtcd2Session()
|
||||
for i := 0; i < maxWatches; i++ {
|
||||
session.registerWatch(string(rune('a'+i)), &watchState{watchID: string(rune('a' + i)), session: session})
|
||||
}
|
||||
_, err := session.watchStart(map[string]json.RawMessage{"key": json.RawMessage(`"k"`)})
|
||||
if err == nil || !strings.HasPrefix(err.Error(), "ETCD_WATCH_LIMIT") {
|
||||
t.Fatalf("expected watch limit error, got %v", err)
|
||||
}
|
||||
fresh := newEtcd2Session()
|
||||
_, err = fresh.watchStart(map[string]json.RawMessage{"key": json.RawMessage(`"k"`), "scope": json.RawMessage(`"glob"`)})
|
||||
if err == nil || err.Error() != "ETCD_WATCH_SCOPE_INVALID: scope must be key or prefix" {
|
||||
t.Fatalf("expected scope error, got %v", err)
|
||||
}
|
||||
_, err = fresh.watchPoll(map[string]json.RawMessage{"watchId": json.RawMessage(`"none"`)})
|
||||
if err == nil || err.Error() != "ETCD_WATCH_NOT_FOUND: watch does not exist" {
|
||||
t.Fatalf("expected watch-not-found, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPutRejectsLeaseOptions(t *testing.T) {
|
||||
state := newEtcd2Session()
|
||||
// Client check comes first (Java parity); without a connection the error
|
||||
// stays "Not connected".
|
||||
params := map[string]json.RawMessage{
|
||||
"key": json.RawMessage(`"k"`),
|
||||
"value": json.RawMessage(`{"encoding":"utf8","data":"v"}`),
|
||||
"ttl": json.RawMessage(`5`),
|
||||
"lease": json.RawMessage(`7`),
|
||||
}
|
||||
if _, err := state.put(params); err == nil || err.Error() != "Not connected" {
|
||||
t.Fatalf("expected Not connected before option validation, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrefixEnd(t *testing.T) {
|
||||
if prefixEnd("") != "\x00" || prefixEnd("ab") != "ac" || prefixEnd("a\xff") != "b" {
|
||||
t.Fatalf("prefixEnd algorithm broken")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValueEncodings(t *testing.T) {
|
||||
utf8Value := valueObject([]byte("hello"))
|
||||
if utf8Value["encoding"] != "utf8" || utf8Value["data"] != "hello" {
|
||||
t.Fatalf("unexpected utf8 value: %#v", utf8Value)
|
||||
}
|
||||
if valueObject([]byte{0xff})["encoding"] != "base64" {
|
||||
t.Fatalf("binary must be base64")
|
||||
}
|
||||
if displayBytes([]byte{0xff}) == "\xff" {
|
||||
t.Fatalf("displayBytes must fall back to base64")
|
||||
}
|
||||
}
|
||||
|
||||
func (s *etcd2Session) watchBufferedBytesSnapshot() int64 {
|
||||
s.watchesMu.Lock()
|
||||
defer s.watchesMu.Unlock()
|
||||
return s.watchBufferedBytes
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type rpcError struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data *rpcErrorData `json:"data,omitempty"`
|
||||
}
|
||||
|
||||
type rpcErrorData struct {
|
||||
Category string `json:"category"`
|
||||
Retryable bool `json:"retryable"`
|
||||
SessionDisposition string `json:"sessionDisposition"`
|
||||
Stage string `json:"stage"`
|
||||
ContractVersion int `json:"contractVersion"`
|
||||
OperationOutcome string `json:"operationOutcome"`
|
||||
ExceptionClass string `json:"exceptionClass,omitempty"`
|
||||
AgentSessionID string `json:"agentSessionId,omitempty"`
|
||||
}
|
||||
|
||||
func errorResponse(id json.RawMessage, method, agentSessionID string, err error) response {
|
||||
return response{JSONRPC: "2.0", ID: id, Error: classifyRPCError(method, agentSessionID, err)}
|
||||
}
|
||||
|
||||
func classifyRPCError(method, agentSessionID string, err error) *rpcError {
|
||||
stage := rpcErrorStage(method)
|
||||
data := &rpcErrorData{
|
||||
Category: "protocol",
|
||||
Retryable: false,
|
||||
SessionDisposition: "keep",
|
||||
Stage: stage,
|
||||
ContractVersion: 1,
|
||||
OperationOutcome: rpcOperationOutcome(stage),
|
||||
ExceptionClass: safeRPCDiagnostic(errTypeName(err), 160),
|
||||
AgentSessionID: strings.TrimSpace(agentSessionID),
|
||||
}
|
||||
|
||||
switch {
|
||||
case errors.Is(err, context.Canceled):
|
||||
data.Category = "canceled"
|
||||
data.SessionDisposition = "quarantine"
|
||||
case errors.Is(err, context.DeadlineExceeded) || isTimeoutError(err):
|
||||
data.Category = "timeout"
|
||||
data.SessionDisposition = "quarantine"
|
||||
case isConnectionError(err):
|
||||
data.Category = "connection"
|
||||
data.Retryable = stage == "connect" || stage == "validate"
|
||||
if stage != "connect" {
|
||||
data.SessionDisposition = "quarantine"
|
||||
}
|
||||
}
|
||||
|
||||
return &rpcError{Code: -1, Message: err.Error(), Data: data}
|
||||
}
|
||||
|
||||
func isConnectionError(err error) bool {
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, net.ErrClosed) {
|
||||
return true
|
||||
}
|
||||
var networkError *net.OpError
|
||||
if errors.As(err, &networkError) {
|
||||
return true
|
||||
}
|
||||
lower := strings.ToLower(err.Error())
|
||||
for _, marker := range []string{
|
||||
"connection refused", "connection reset", "broken pipe", "connection closed",
|
||||
"connection lost", "unexpected eof", "no route to host", "etcd connection error",
|
||||
} {
|
||||
if strings.Contains(lower, marker) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func errTypeName(err error) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.ReplaceAll(strings.TrimSpace(err.Error()), "\n", " ")
|
||||
}
|
||||
|
||||
func rpcErrorStage(method string) string {
|
||||
switch method {
|
||||
case "connect", "open_session", "test_connection":
|
||||
return "connect"
|
||||
case "validate_connection", "validate_session":
|
||||
return "validate"
|
||||
case "cancel_session":
|
||||
return "cancel"
|
||||
case "close_session", "disconnect", "shutdown":
|
||||
return "close"
|
||||
case "etcd_watch_poll":
|
||||
return "fetch"
|
||||
case "handshake", "":
|
||||
return "request"
|
||||
default:
|
||||
return "execute"
|
||||
}
|
||||
}
|
||||
|
||||
func rpcOperationOutcome(stage string) string {
|
||||
switch stage {
|
||||
case "request", "connect", "validate":
|
||||
return "not_started"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
func isTimeoutError(err error) bool {
|
||||
var timeout interface{ Timeout() bool }
|
||||
return errors.As(err, &timeout) && timeout.Timeout()
|
||||
}
|
||||
|
||||
func safeRPCDiagnostic(value string, maxLength int) string {
|
||||
var result strings.Builder
|
||||
for _, char := range value {
|
||||
if result.Len() >= maxLength {
|
||||
break
|
||||
}
|
||||
if char >= 0x21 && char <= 0x7e {
|
||||
result.WriteRune(char)
|
||||
}
|
||||
}
|
||||
return result.String()
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// status assembles a reduced kv_status document from the v2 endpoints:
|
||||
// /v2/members, /v2/stats/self, /v2/stats/leader, and /v2/stats/store. Fields
|
||||
// without a v2 analog (alarms, dbSize, raft detail) stay null.
|
||||
func (s *etcd2Session) status(params map[string]json.RawMessage) (any, error) {
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx, cancel := s.beginOperation()
|
||||
defer s.endOperation(cancel)
|
||||
|
||||
type member struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
PeerURLs []string `json:"peerURLs"`
|
||||
ClientURLs []string `json:"clientURLs"`
|
||||
}
|
||||
membersByID := map[string]*member{}
|
||||
var memberList []member
|
||||
var clusterID *string
|
||||
if body, response, err := client.do(ctx, http.MethodGet, "/v2/members", "", nil); err == nil {
|
||||
var parsed struct {
|
||||
Members []member `json:"members"`
|
||||
}
|
||||
if json.Unmarshal(body, &parsed) == nil {
|
||||
memberList = parsed.Members
|
||||
for index := range memberList {
|
||||
membersByID[memberList[index].ID] = &memberList[index]
|
||||
}
|
||||
}
|
||||
if header := response.Header.Get("X-Etcd-Cluster-Id"); header != "" {
|
||||
clusterID = &header
|
||||
}
|
||||
}
|
||||
// The v2 API exposes the current index only as a response header on the
|
||||
// keys endpoints, not in any stats document.
|
||||
var revision *string
|
||||
if _, response, err := client.do(ctx, http.MethodGet, "/v2/keys/", "", nil); err == nil {
|
||||
if header := response.Header.Get("X-Etcd-Index"); header != "" {
|
||||
revision = &header
|
||||
}
|
||||
}
|
||||
|
||||
type selfStats struct {
|
||||
Name string `json:"name"`
|
||||
ID string `json:"id"`
|
||||
State string `json:"state"`
|
||||
LeaderInfo struct {
|
||||
Leader string `json:"leader"`
|
||||
} `json:"leaderInfo"`
|
||||
RecvAppendRequestCnt int64 `json:"recvAppendRequestCnt"`
|
||||
SendAppendRequestCnt int64 `json:"sendAppendRequestCnt"`
|
||||
}
|
||||
var leaderID *string
|
||||
selfByEndpoint := map[string]*selfStats{}
|
||||
endpoints := s.connectedEndpointList()
|
||||
for _, endpoint := range endpoints {
|
||||
self := &selfStats{}
|
||||
body, _, err := client.doAt(ctx, http.MethodGet, endpoint, "/v2/stats/self", "")
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if json.Unmarshal(body, self) != nil {
|
||||
continue
|
||||
}
|
||||
selfByEndpoint[endpoint] = self
|
||||
if self.LeaderInfo.Leader != "" {
|
||||
value := self.LeaderInfo.Leader
|
||||
leaderID = &value
|
||||
}
|
||||
}
|
||||
|
||||
storeStats := map[string]any{}
|
||||
if body, _, err := client.do(ctx, http.MethodGet, "/v2/stats/store", "", nil); err == nil {
|
||||
var parsed map[string]any
|
||||
if json.Unmarshal(body, &parsed) == nil {
|
||||
storeStats = parsed
|
||||
}
|
||||
}
|
||||
|
||||
statusMembers := []map[string]any{}
|
||||
observedIDs := map[string]struct{}{}
|
||||
for _, endpoint := range endpoints {
|
||||
row := map[string]any{"endpoint": endpoint}
|
||||
started := time.Now()
|
||||
self := selfByEndpoint[endpoint]
|
||||
if self == nil {
|
||||
row["memberId"] = nil
|
||||
row["name"] = nil
|
||||
row["version"] = nil
|
||||
row["leaderId"] = nil
|
||||
row["revision"] = nil
|
||||
row["raftTerm"] = nil
|
||||
row["raftIndex"] = nil
|
||||
row["raftAppliedIndex"] = nil
|
||||
row["dbSize"] = nil
|
||||
row["dbSizeInUse"] = nil
|
||||
row["learner"] = false
|
||||
row["reachable"] = false
|
||||
row["latencyMs"] = nil
|
||||
row["errors"] = []string{"stats request failed"}
|
||||
statusMembers = append(statusMembers, row)
|
||||
continue
|
||||
}
|
||||
member := membersByID[self.ID]
|
||||
row["memberId"] = memberHexToUnsigned(self.ID)
|
||||
if member == nil {
|
||||
row["name"] = self.Name
|
||||
} else {
|
||||
row["name"] = member.Name
|
||||
}
|
||||
row["version"] = client.serverVersion
|
||||
row["leaderId"] = memberHexToUnsigned(self.LeaderInfo.Leader)
|
||||
row["revision"] = nil
|
||||
row["raftTerm"] = nil
|
||||
row["raftIndex"] = nil
|
||||
row["raftAppliedIndex"] = nil
|
||||
row["dbSize"] = nil
|
||||
row["dbSizeInUse"] = nil
|
||||
row["learner"] = false
|
||||
row["reachable"] = true
|
||||
row["latencyMs"] = time.Since(started).Milliseconds()
|
||||
row["errors"] = []string{}
|
||||
row["state"] = strings.TrimPrefix(self.State, "State")
|
||||
if _, seen := observedIDs[self.ID]; seen {
|
||||
continue
|
||||
}
|
||||
observedIDs[self.ID] = struct{}{}
|
||||
statusMembers = append(statusMembers, row)
|
||||
}
|
||||
|
||||
var clusterIDValue any
|
||||
if clusterID != nil {
|
||||
clusterIDValue = memberHexToUnsigned(*clusterID)
|
||||
}
|
||||
var leaderIDValue any
|
||||
if leaderID != nil {
|
||||
leaderIDValue = memberHexToUnsigned(*leaderID)
|
||||
}
|
||||
var revisionValue any
|
||||
if revision != nil {
|
||||
revisionValue = *revision
|
||||
}
|
||||
// The v2 API has no key-count equivalent of the v3 count-only range;
|
||||
// the raw store counters are exposed separately below.
|
||||
var keyCountValue any
|
||||
return map[string]any{
|
||||
"clusterId": clusterIDValue,
|
||||
"revision": revisionValue,
|
||||
"leaderId": leaderIDValue,
|
||||
"keyCount": keyCountValue,
|
||||
"alarms": []string{},
|
||||
"members": statusMembers,
|
||||
"store": storeStats,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// memberHexToUnsigned converts v2 hex member/cluster IDs (e.g. "272e204152")
|
||||
// to the unsigned decimal strings the v3 agent emits.
|
||||
func memberHexToUnsigned(hex string) any {
|
||||
if hex == "" {
|
||||
return nil
|
||||
}
|
||||
parsed, err := strconv.ParseUint(strings.TrimPrefix(hex, "0x"), 16, 64)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return unsignedLongString(int64(parsed))
|
||||
}
|
||||
@@ -0,0 +1,395 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
maxWatches = 4
|
||||
maxWatchBatches = 256
|
||||
maxWatchEvents = 10000
|
||||
maxWatchBufferBytes = int64(8 * 1024 * 1024)
|
||||
maxSessionWatchBufferBytes = int64(16 * 1024 * 1024)
|
||||
// v2 long polls hold until an event arrives; re-issue the request when a
|
||||
// poll is cut off so the pump survives idle periods.
|
||||
watchPollTimeout = 5 * time.Minute
|
||||
)
|
||||
|
||||
type bufferedWatchBatch struct {
|
||||
payload map[string]any
|
||||
bufferedBytes int64
|
||||
}
|
||||
|
||||
// watchState buffers watch events for the polling host, mirroring the v3
|
||||
// agent's budget system so host-side expectations stay identical.
|
||||
type watchState struct {
|
||||
watchID string
|
||||
session *etcd2Session
|
||||
|
||||
mu sync.Mutex
|
||||
batches []bufferedWatchBatch
|
||||
eventCount int
|
||||
bufferedBytes int64
|
||||
terminalReason string
|
||||
terminalMessage string
|
||||
pumpCancel context.CancelFunc
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (w *watchState) append(revision int64, events []map[string]any, batchBytes int64) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.terminalReason != "" {
|
||||
return
|
||||
}
|
||||
if int64(len(w.batches)) >= maxWatchBatches ||
|
||||
int64(w.eventCount+len(events)) > maxWatchEvents ||
|
||||
batchBytes > maxWatchBufferBytes ||
|
||||
w.bufferedBytes+batchBytes > maxWatchBufferBytes ||
|
||||
!w.session.reserveWatchBuffer(batchBytes) {
|
||||
w.overflowLocked()
|
||||
return
|
||||
}
|
||||
w.batches = append(w.batches, bufferedWatchBatch{
|
||||
payload: map[string]any{"revision": longString(revision), "events": events},
|
||||
bufferedBytes: batchBytes,
|
||||
})
|
||||
w.eventCount += len(events)
|
||||
w.bufferedBytes += batchBytes
|
||||
}
|
||||
|
||||
func (w *watchState) overflowLocked() {
|
||||
if w.terminalReason != "" {
|
||||
return
|
||||
}
|
||||
w.terminalReason = "overflow"
|
||||
w.terminalMessage = "ETCD_WATCH_OVERFLOW: the event buffer reached its byte or event limit"
|
||||
w.stopPumpLocked()
|
||||
}
|
||||
|
||||
func (w *watchState) fail(reason, message string) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.terminalReason == "" {
|
||||
w.terminalReason = reason
|
||||
w.terminalMessage = message
|
||||
}
|
||||
}
|
||||
|
||||
func (w *watchState) poll() map[string]any {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
page := make([]any, 0, 64)
|
||||
for len(w.batches) > 0 && len(page) < 64 {
|
||||
batch := w.batches[0]
|
||||
w.batches = w.batches[1:]
|
||||
events := batch.payload["events"].([]map[string]any)
|
||||
w.eventCount -= len(events)
|
||||
w.bufferedBytes -= batch.bufferedBytes
|
||||
w.session.releaseWatchBuffer(batch.bufferedBytes)
|
||||
page = append(page, batch.payload)
|
||||
}
|
||||
result := map[string]any{"watchId": w.watchID, "batches": page}
|
||||
if w.terminalReason != "" && len(w.batches) == 0 {
|
||||
result["terminal"] = map[string]any{
|
||||
"reason": w.terminalReason,
|
||||
"message": w.terminalMessage,
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (w *watchState) close() {
|
||||
w.mu.Lock()
|
||||
w.failLocked("stopped", "watch stopped")
|
||||
w.clearBufferedLocked()
|
||||
w.closed = true
|
||||
cancel := w.pumpCancel
|
||||
w.pumpCancel = nil
|
||||
w.mu.Unlock()
|
||||
if cancel != nil {
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
|
||||
func (w *watchState) failLocked(reason, message string) {
|
||||
if w.terminalReason == "" {
|
||||
w.terminalReason = reason
|
||||
w.terminalMessage = message
|
||||
}
|
||||
}
|
||||
|
||||
func (w *watchState) clearBufferedLocked() {
|
||||
if w.bufferedBytes > 0 {
|
||||
w.session.releaseWatchBuffer(w.bufferedBytes)
|
||||
}
|
||||
w.batches = nil
|
||||
w.eventCount = 0
|
||||
w.bufferedBytes = 0
|
||||
}
|
||||
|
||||
func (w *watchState) attachPump(cancel context.CancelFunc) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.closed || w.terminalReason != "" {
|
||||
cancel()
|
||||
return
|
||||
}
|
||||
w.pumpCancel = cancel
|
||||
}
|
||||
|
||||
func (w *watchState) stopPumpLocked() {
|
||||
cancel := w.pumpCancel
|
||||
w.pumpCancel = nil
|
||||
if cancel != nil {
|
||||
go cancel()
|
||||
}
|
||||
}
|
||||
|
||||
func watchEventBufferBytes(key, value, previousValue string) int64 {
|
||||
bytes := int64(512 + 4*len(key))
|
||||
bytes += estimatedBufferedBytes(len(value))
|
||||
if previousValue != "" {
|
||||
bytes += estimatedBufferedBytes(len(previousValue))
|
||||
}
|
||||
return bytes
|
||||
}
|
||||
|
||||
func estimatedBufferedBytes(sourceBytes int) int64 {
|
||||
if sourceBytes < 0 {
|
||||
return 0
|
||||
}
|
||||
return int64(sourceBytes) * 4
|
||||
}
|
||||
|
||||
func newWatchID() string {
|
||||
var bytes [16]byte
|
||||
if _, err := rand.Read(bytes[:]); err != nil {
|
||||
return base64.StdEncoding.EncodeToString(bytes[:])
|
||||
}
|
||||
bytes[6] = (bytes[6] & 0x0f) | 0x40
|
||||
bytes[8] = (bytes[8] & 0x3f) | 0x80
|
||||
hexed := hex.EncodeToString(bytes[:])
|
||||
return fmt.Sprintf("%s-%s-%s-%s-%s", hexed[0:8], hexed[8:12], hexed[12:16], hexed[16:20], hexed[20:32])
|
||||
}
|
||||
|
||||
func (s *etcd2Session) watchCount() int {
|
||||
s.watchesMu.Lock()
|
||||
defer s.watchesMu.Unlock()
|
||||
return len(s.watches)
|
||||
}
|
||||
|
||||
func (s *etcd2Session) registerWatch(id string, state *watchState) bool {
|
||||
s.watchesMu.Lock()
|
||||
defer s.watchesMu.Unlock()
|
||||
if _, exists := s.watches[id]; exists {
|
||||
return false
|
||||
}
|
||||
s.watches[id] = state
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *etcd2Session) removeWatch(id string) *watchState {
|
||||
s.watchesMu.Lock()
|
||||
defer s.watchesMu.Unlock()
|
||||
state := s.watches[id]
|
||||
delete(s.watches, id)
|
||||
return state
|
||||
}
|
||||
|
||||
func (s *etcd2Session) watchStart(params map[string]json.RawMessage) (any, error) {
|
||||
if s.watchCount() >= maxWatches {
|
||||
return nil, fmt.Errorf("ETCD_WATCH_LIMIT: at most %d watches are allowed per connection", maxWatches)
|
||||
}
|
||||
key, err := keyBytesParam(params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
scope := stringOrDefault(params, "scope", "key")
|
||||
if scope != "key" && scope != "prefix" {
|
||||
return nil, errors.New("ETCD_WATCH_SCOPE_INVALID: scope must be key or prefix")
|
||||
}
|
||||
client, err := s.activeClient()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// startRevision maps to the v2 waitIndex: default to the current index+1.
|
||||
requestedRevision := longOrNull(params, "startRevision")
|
||||
var waitIndex int64
|
||||
if requestedRevision != nil && *requestedRevision > 0 {
|
||||
waitIndex = *requestedRevision
|
||||
} else {
|
||||
current, err := client.currentEtcdIndex()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
waitIndex = current + 1
|
||||
}
|
||||
|
||||
watchID := newWatchID()
|
||||
state := &watchState{watchID: watchID, session: s}
|
||||
pumpCtx, pumpCancel := context.WithCancel(context.Background())
|
||||
go pumpLongPollWatch(pumpCtx, client, key, scope, waitIndex, boolOrDefault(params, "includePrevKv", false), state)
|
||||
state.attachPump(pumpCancel)
|
||||
s.registerWatch(watchID, state)
|
||||
return map[string]any{"watchId": watchID, "startedRevision": longString(waitIndex)}, nil
|
||||
}
|
||||
|
||||
// currentEtcdIndex reads the X-Etcd-Index header from a lightweight GET.
|
||||
func (c *authenticatedClient) currentEtcdIndex() (int64, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), operationTimeout)
|
||||
defer cancel()
|
||||
response, err := c.request(ctx, http.MethodGet, "/v2/keys/", "", nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer drainClose(response.Body)
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
body, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
|
||||
return 0, errorFromResponse(response.StatusCode, body)
|
||||
}
|
||||
index, err := strconv.ParseInt(response.Header.Get("X-Etcd-Index"), 10, 64)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("etcd v2 server did not report X-Etcd-Index")
|
||||
}
|
||||
return index, nil
|
||||
}
|
||||
|
||||
// pumpLongPollWatch drives the v2 wait loop: each response becomes one batch;
|
||||
// timeouts and connection blips re-issue the same waitIndex.
|
||||
func pumpLongPollWatch(ctx context.Context, client *authenticatedClient, key, scope string, waitIndex int64, includePrevKv bool, state *watchState) {
|
||||
for {
|
||||
if ctx.Err() != nil || state.hasTerminal() {
|
||||
return
|
||||
}
|
||||
query := url.Values{}
|
||||
query.Set("wait", "true")
|
||||
query.Set("waitIndex", strconv.FormatInt(waitIndex, 10))
|
||||
if scope == "prefix" {
|
||||
query.Set("recursive", "true")
|
||||
}
|
||||
pollCtx, pollCancel := context.WithTimeout(ctx, watchPollTimeout)
|
||||
body, _, err := client.do(pollCtx, http.MethodGet, v2KeyPath(key)+"?"+query.Encode(), "", nil)
|
||||
pollCancel()
|
||||
if err != nil {
|
||||
if errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil {
|
||||
// Idle long poll cut off: keep waiting from the same index.
|
||||
continue
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
var etcdErr *etcdError
|
||||
if errors.As(err, &etcdErr) && etcdErr.errorCode == 401 {
|
||||
state.fail("error", "ETCD_WATCH_EXPIRED: the watched index was cleared from the etcd v2 event history")
|
||||
return
|
||||
}
|
||||
state.fail("error", "watch failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
var parsed v2KeysResponse
|
||||
if jsonErr := json.Unmarshal(body, &parsed); jsonErr != nil {
|
||||
state.fail("error", "watch response was not valid etcd v2 JSON: "+jsonErr.Error())
|
||||
return
|
||||
}
|
||||
if parsed.Node == nil {
|
||||
waitIndex++
|
||||
continue
|
||||
}
|
||||
events := []map[string]any{}
|
||||
bufferedBytes := int64(128)
|
||||
node := parsed.Node
|
||||
previous := parsed.PrevNode
|
||||
deleted := parsed.Action == "delete" || parsed.Action == "expire"
|
||||
previousValue := ""
|
||||
if previous != nil {
|
||||
previousValue = previous.Value
|
||||
}
|
||||
bufferedBytes += watchEventBufferBytes(node.Key, node.Value, previousValue)
|
||||
if bufferedBytes > maxWatchBufferBytes {
|
||||
state.mu.Lock()
|
||||
state.overflowLocked()
|
||||
state.mu.Unlock()
|
||||
return
|
||||
}
|
||||
row := map[string]any{
|
||||
"eventType": eventTypeForAction(parsed.Action),
|
||||
"revision": longString(node.ModifiedIndex),
|
||||
"key": node.Key,
|
||||
"keyBytes": bytesObject([]byte(node.Key)),
|
||||
}
|
||||
if deleted {
|
||||
row["value"] = nil
|
||||
} else {
|
||||
row["value"] = valueObject([]byte(node.Value))
|
||||
}
|
||||
if includePrevKv && previous != nil && previous.Value != "" {
|
||||
row["previousValue"] = valueObject([]byte(previous.Value))
|
||||
} else {
|
||||
row["previousValue"] = nil
|
||||
}
|
||||
if deleted && previous != nil {
|
||||
row["metadata"] = nodeMetadata(previous)
|
||||
} else {
|
||||
row["metadata"] = nodeMetadata(node)
|
||||
}
|
||||
events = append(events, row)
|
||||
state.append(node.ModifiedIndex, events, bufferedBytes)
|
||||
nextIndex := node.ModifiedIndex + 1
|
||||
if nextIndex > waitIndex {
|
||||
waitIndex = nextIndex
|
||||
} else {
|
||||
waitIndex++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func eventTypeForAction(action string) string {
|
||||
if action == "delete" || action == "expire" || action == "compareAndDelete" {
|
||||
return "delete"
|
||||
}
|
||||
return "put"
|
||||
}
|
||||
|
||||
func (w *watchState) hasTerminal() bool {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.terminalReason != ""
|
||||
}
|
||||
|
||||
func (s *etcd2Session) watchPoll(params map[string]json.RawMessage) (any, error) {
|
||||
watchID := stringOrDefault(params, "watchId", "")
|
||||
s.watchesMu.Lock()
|
||||
state := s.watches[watchID]
|
||||
s.watchesMu.Unlock()
|
||||
if state == nil {
|
||||
return nil, errors.New("ETCD_WATCH_NOT_FOUND: watch does not exist")
|
||||
}
|
||||
result := state.poll()
|
||||
if _, terminal := result["terminal"]; terminal {
|
||||
if removed := s.removeWatch(watchID); removed != nil {
|
||||
removed.close()
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *etcd2Session) watchStop(params map[string]json.RawMessage) (any, error) {
|
||||
if state := s.removeWatch(stringOrDefault(params, "watchId", "")); state != nil {
|
||||
state.close()
|
||||
}
|
||||
return map[string]bool{"stopped": true}, nil
|
||||
}
|
||||
@@ -170,7 +170,10 @@ func parseConnectionConfig(params connectParams) (connectionConfig, error) {
|
||||
CanonicalHostname: true,
|
||||
},
|
||||
}
|
||||
if strings.EqualFold(params.DatabaseType, "impala") {
|
||||
if config.DatabaseType == "argo" {
|
||||
config.Auth = "NOSASL"
|
||||
}
|
||||
if config.DatabaseType == "impala" {
|
||||
config.Auth = "NOSASL"
|
||||
config.Kerberos.Service = defaultImpalaService
|
||||
}
|
||||
|
||||
@@ -296,6 +296,50 @@ func TestImpalaDefaultsToNoSASL(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestArgoDefaultsToNoSASL(t *testing.T) {
|
||||
config, err := parseConnectionConfig(connectParams{
|
||||
DatabaseType: "argo",
|
||||
Host: "argo.example.com",
|
||||
Port: 10000,
|
||||
Database: "analytics",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if config.Auth != "NOSASL" {
|
||||
t.Fatalf("unexpected ArgoDB auth mode: %q", config.Auth)
|
||||
}
|
||||
}
|
||||
|
||||
func TestArgoExplicitAuthenticationOverridesDefault(t *testing.T) {
|
||||
config, err := parseConnectionConfig(connectParams{
|
||||
DatabaseType: "argo",
|
||||
Host: "argo.example.com",
|
||||
Port: 10000,
|
||||
URLParams: "auth=NONE",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if config.Auth != "NONE" || !config.AuthExplicit {
|
||||
t.Fatalf("explicit ArgoDB authentication was not preserved: %#v", config)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHiveDefaultAuthenticationRemainsSASL(t *testing.T) {
|
||||
config, err := parseConnectionConfig(connectParams{
|
||||
DatabaseType: "hive",
|
||||
Host: "hive.example.com",
|
||||
Port: 10000,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if config.Auth != "NONE" {
|
||||
t.Fatalf("unexpected Hive auth mode: %q", config.Auth)
|
||||
}
|
||||
}
|
||||
|
||||
func TestImpalaExplicitAuthenticationOverridesDefaults(t *testing.T) {
|
||||
config, err := parseConnectionConfig(connectParams{
|
||||
DatabaseType: "impala",
|
||||
|
||||
@@ -407,7 +407,11 @@ func (server *server) dispatch(method string, params map[string]json.RawMessage)
|
||||
result, err := server.getTableComment(stringParam(params, "schema"), stringParam(params, "table"))
|
||||
return result, false, err
|
||||
case "list_objects":
|
||||
result, err := server.listObjects(stringParam(params, "schema"), metadataListConstraintsFromParams(params))
|
||||
result, err := server.listObjects(
|
||||
stringParam(params, "database"),
|
||||
stringParam(params, "schema"),
|
||||
metadataListConstraintsFromParams(params),
|
||||
)
|
||||
return result, false, err
|
||||
case "list_data_types":
|
||||
result, err := server.listDataTypes()
|
||||
@@ -432,6 +436,7 @@ func (server *server) dispatch(method string, params map[string]json.RawMessage)
|
||||
return []any{}, false, nil
|
||||
case "get_object_source":
|
||||
result, err := server.getObjectSource(
|
||||
stringParam(params, "database"),
|
||||
stringParam(params, "schema"),
|
||||
firstNonEmpty(stringParam(params, "name"), stringParam(params, "table")),
|
||||
stringParam(params, "object_type"),
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -437,20 +438,169 @@ func showViewsUnsupported(err error) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (server *server) listObjects(schema string, constraints metadataListConstraints) ([]objectInfo, error) {
|
||||
if !acceptsHiveTable(constraints.ObjectTypes) {
|
||||
func (server *server) listObjects(database, schema string, constraints metadataListConstraints) ([]objectInfo, error) {
|
||||
if !acceptsHiveTable(constraints.ObjectTypes) && !(server.supportsRoutines() && acceptsHiveRoutine(constraints.ObjectTypes)) {
|
||||
return []objectInfo{}, nil
|
||||
}
|
||||
tables, err := server.listTables(schema, constraints)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
schema = firstNonEmpty(schema, server.config.Database)
|
||||
values := make([]objectInfo, 0, len(tables))
|
||||
for _, table := range tables {
|
||||
values = append(values, objectInfo{Name: table.Name, ObjectType: table.TableType, Schema: schema, Comment: table.Comment})
|
||||
values := make([]objectInfo, 0)
|
||||
if acceptsHiveTable(constraints.ObjectTypes) {
|
||||
tables, err := server.listTables(schema, constraints)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, table := range tables {
|
||||
values = append(values, objectInfo{Name: table.Name, ObjectType: table.TableType, Schema: schema, Comment: table.Comment})
|
||||
}
|
||||
}
|
||||
return values, nil
|
||||
if server.supportsRoutines() && acceptsRoutineType(constraints.ObjectTypes, "PROCEDURE") {
|
||||
procedures, err := server.listRoutines(database, schema, constraints, "PROCEDURE")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
values = append(values, procedures...)
|
||||
}
|
||||
if server.supportsRoutines() && acceptsRoutineType(constraints.ObjectTypes, "FUNCTION") {
|
||||
functions, err := server.listRoutines(database, schema, constraints, "FUNCTION")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
values = append(values, functions...)
|
||||
}
|
||||
sort.Slice(values, func(first, second int) bool { return values[first].Name < values[second].Name })
|
||||
return applyMetadataWindow(values, constraints.Offset, constraints.Limit), nil
|
||||
}
|
||||
|
||||
// supportsRoutines reports whether the connected server is known to expose the
|
||||
// Hive procedure / function catalog views (system.procedures_v /
|
||||
// system.functions_v). ArgoDB / Inceptor / Transwarp forks ship these views;
|
||||
// vanilla Apache Hive does not, so routine listing is gated on the connection's
|
||||
// database_type to avoid firing unsupported queries against plain Hive.
|
||||
func (server *server) supportsRoutines() bool {
|
||||
switch strings.ToLower(strings.TrimSpace(server.params.DatabaseType)) {
|
||||
case "argo", "inceptor", "inceptor2", "transwarp":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// routineDatabaseCandidates returns the database_name filters to try against
|
||||
// system.procedures_v / system.functions_v. The sidebar may pass the active
|
||||
// database through either the schema or database RPC parameter, so mirror the
|
||||
// JDBC plugin and try both. The connection default is only a candidate when
|
||||
// the caller supplied neither parameter: appending it unconditionally would
|
||||
// return the default database's routines under an explicit database node that
|
||||
// has none of its own.
|
||||
func routineDatabaseCandidates(schema, database, connectionDatabase string) []string {
|
||||
seen := map[string]bool{}
|
||||
values := make([]string, 0, 3)
|
||||
add := func(candidate string) {
|
||||
candidate = strings.TrimSpace(candidate)
|
||||
if candidate == "" {
|
||||
return
|
||||
}
|
||||
key := strings.ToLower(candidate)
|
||||
if seen[key] {
|
||||
return
|
||||
}
|
||||
seen[key] = true
|
||||
values = append(values, candidate)
|
||||
}
|
||||
add(database)
|
||||
add(schema)
|
||||
if strings.TrimSpace(database) == "" && strings.TrimSpace(schema) == "" {
|
||||
add(connectionDatabase)
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
// listRoutines queries the server's procedure / function catalog views when the
|
||||
// driver supports them. Hive and most forks (ArgoDB, Inceptor, Transwarp) expose
|
||||
// stored procedures / functions through the system.procedures_v / system.functions_v
|
||||
// views, with columns (procedure_name | function_name, database_name, full_text, ...).
|
||||
// The full_text column carries the routine source used by getObjectSource.
|
||||
//
|
||||
// The query is best-effort: when the view is missing or the server rejects it
|
||||
// (older Hive without procedure support), the call returns an empty slice and
|
||||
// nil error so the caller can fall back to listing tables.
|
||||
func (server *server) listRoutines(database, schema string, constraints metadataListConstraints, routineType string) ([]objectInfo, error) {
|
||||
nameColumn := "procedure_name"
|
||||
viewName := "system.procedures_v"
|
||||
if strings.EqualFold(routineType, "FUNCTION") {
|
||||
nameColumn = "function_name"
|
||||
viewName = "system.functions_v"
|
||||
}
|
||||
candidates := routineDatabaseCandidates(schema, database, server.config.Database)
|
||||
if len(candidates) == 0 {
|
||||
return []objectInfo{}, nil
|
||||
}
|
||||
likePattern := buildRoutineLikePattern(constraints.Filter)
|
||||
resolvedSchema := firstNonEmpty(schema, database, server.config.Database)
|
||||
for _, targetSchema := range candidates {
|
||||
// database_name is a string column, so the schema filter must be a single-quoted
|
||||
// literal — not a backtick-quoted identifier. ArgoDB/Inceptor reject lower(`ods`)
|
||||
// against system.procedures_v with an ERROR_STATUS, while lower('ods') works.
|
||||
schemaLiteral := "'" + strings.ReplaceAll(targetSchema, "'", "''") + "'"
|
||||
sql := "SELECT " + nameColumn + " FROM " + viewName +
|
||||
" WHERE lower(database_name) = lower(" + schemaLiteral + ")" +
|
||||
" AND lower(" + nameColumn + ") LIKE " + likePattern +
|
||||
" ORDER BY " + nameColumn
|
||||
result, err := server.executeQuery(queryOptions{SQL: sql, MaxRows: metadataQueryLimit})
|
||||
if err != nil {
|
||||
log.Printf(
|
||||
"[hive-go][listRoutines] query failed: database=%q schema=%q routineType=%s sql=%q err=%v",
|
||||
targetSchema,
|
||||
resolvedSchema,
|
||||
routineType,
|
||||
sql,
|
||||
err,
|
||||
)
|
||||
continue
|
||||
}
|
||||
values := make([]objectInfo, 0, len(result.Rows))
|
||||
for _, row := range result.Rows {
|
||||
name := rowString(row, 0)
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
values = append(values, objectInfo{Name: name, ObjectType: strings.ToUpper(routineType), Schema: resolvedSchema, Comment: nil})
|
||||
}
|
||||
if len(values) > 0 {
|
||||
return values, nil
|
||||
}
|
||||
}
|
||||
return []objectInfo{}, nil
|
||||
}
|
||||
|
||||
// buildRoutineLikePattern turns a user-supplied filter into a Hive-safe LIKE
|
||||
// literal: wraps with %, escapes \, %, _ (the LIKE metacharacters), and quotes
|
||||
// the whole literal so it can be concatenated directly into SQL.
|
||||
func buildRoutineLikePattern(filter string) string {
|
||||
escaped := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(filter)
|
||||
return "'%" + escaped + "%'"
|
||||
}
|
||||
|
||||
// acceptsHiveRoutine reports whether the requested object types include any
|
||||
// routine (PROCEDURE / FUNCTION) that hive-go needs to surface through listObjects.
|
||||
func acceptsHiveRoutine(objectTypes []string) bool {
|
||||
for _, objectType := range objectTypes {
|
||||
if strings.EqualFold(objectType, "PROCEDURE") || strings.EqualFold(objectType, "FUNCTION") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// acceptsRoutineType reports whether the requested object types include the
|
||||
// given routine kind (case-insensitive).
|
||||
func acceptsRoutineType(objectTypes []string, routineType string) bool {
|
||||
for _, objectType := range objectTypes {
|
||||
if strings.EqualFold(objectType, routineType) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (server *server) getColumns(schema, table string) ([]columnInfo, error) {
|
||||
@@ -584,8 +734,19 @@ func (server *server) getTableDDL(schema, table string) (string, error) {
|
||||
return strings.Join(lines, "\n") + "\n", nil
|
||||
}
|
||||
|
||||
func (server *server) getObjectSource(schema, name, objectType string) (objectSource, error) {
|
||||
source, err := server.getTableDDL(schema, name)
|
||||
func (server *server) getObjectSource(database, schema, name, objectType string) (objectSource, error) {
|
||||
schema = firstNonEmpty(schema, database, server.config.Database)
|
||||
var source string
|
||||
var err error
|
||||
switch strings.ToUpper(objectType) {
|
||||
case "PROCEDURE", "FUNCTION":
|
||||
if !server.supportsRoutines() {
|
||||
return objectSource{}, fmt.Errorf("routine source is not supported for %s connections", server.params.DatabaseType)
|
||||
}
|
||||
source, err = server.getRoutineSource(database, schema, name, strings.ToUpper(objectType))
|
||||
default:
|
||||
source, err = server.getTableDDL(schema, name)
|
||||
}
|
||||
if err != nil {
|
||||
return objectSource{}, err
|
||||
}
|
||||
@@ -597,6 +758,54 @@ func (server *server) getObjectSource(schema, name, objectType string) (objectSo
|
||||
}, nil
|
||||
}
|
||||
|
||||
// getRoutineSource fetches a procedure or function's full source from the
|
||||
// server's system.procedures_v / system.functions_v view. Returns an empty
|
||||
// string when the view is missing or the routine is not found, so callers can
|
||||
// fall back to other sources. full_text may span multiple rows (the underlying
|
||||
// query driver splits long strings), so we join them like getTableDDL does.
|
||||
func (server *server) getRoutineSource(database, schema, name, routineType string) (string, error) {
|
||||
nameColumn := "procedure_name"
|
||||
viewName := "system.procedures_v"
|
||||
if strings.EqualFold(routineType, "FUNCTION") {
|
||||
nameColumn = "function_name"
|
||||
viewName = "system.functions_v"
|
||||
}
|
||||
candidates := routineDatabaseCandidates(schema, database, server.config.Database)
|
||||
if len(candidates) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
escapedName := strings.ReplaceAll(name, "'", "''")
|
||||
for _, targetSchema := range candidates {
|
||||
sql := "SELECT full_text FROM " + viewName +
|
||||
" WHERE lower(database_name) = lower('" + strings.ReplaceAll(targetSchema, "'", "''") + "')" +
|
||||
" AND " + nameColumn + " = '" + escapedName + "'"
|
||||
result, err := server.executeQuery(queryOptions{SQL: sql, MaxRows: metadataQueryLimit})
|
||||
if err != nil {
|
||||
log.Printf(
|
||||
"[hive-go][getRoutineSource] query failed: database=%q schema=%q name=%q routineType=%s sql=%q err=%v",
|
||||
targetSchema,
|
||||
schema,
|
||||
name,
|
||||
routineType,
|
||||
sql,
|
||||
err,
|
||||
)
|
||||
continue
|
||||
}
|
||||
lines := make([]string, 0, len(result.Rows))
|
||||
for _, row := range result.Rows {
|
||||
if line := firstRowValue(row); line != "" {
|
||||
lines = append(lines, line)
|
||||
}
|
||||
}
|
||||
if len(lines) == 0 {
|
||||
continue
|
||||
}
|
||||
return strings.Join(lines, "\n") + "\n", nil
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (server *server) getExplainInfo(sqlText string) (string, error) {
|
||||
sqlText = trimStatementSQL(sqlText)
|
||||
if sqlText == "" {
|
||||
|
||||
@@ -479,3 +479,258 @@ func TestListViewsReturnsFallbackErrorWhenShowViewsIsUnsupported(t *testing.T) {
|
||||
func metadataResult(columns []string, rows ...[]driver.Value) gohive.MetadataResult {
|
||||
return gohive.MetadataResult{Columns: columns, Rows: rows}
|
||||
}
|
||||
|
||||
func TestListObjectsIncludesProceduresAndFunctionsFromSystemViews(t *testing.T) {
|
||||
proceduresQuery := "SELECT procedure_name FROM system.procedures_v WHERE lower(database_name) = lower('ods') AND lower(procedure_name) LIKE '%sp%' ORDER BY procedure_name"
|
||||
functionsQuery := "SELECT function_name FROM system.functions_v WHERE lower(database_name) = lower('ods') AND lower(function_name) LIKE '%sp%' ORDER BY function_name"
|
||||
behavior := &scriptedBehavior{
|
||||
query: func(_ context.Context, query string) (driver.Rows, error) {
|
||||
switch query {
|
||||
case proceduresQuery:
|
||||
return newScriptedRows(context.Background(), []string{"procedure_name"}, []string{"STRING"}, [][]driver.Value{
|
||||
{"sp_daily_etl"}, {"sp_hourly_agg"},
|
||||
}), nil
|
||||
case functionsQuery:
|
||||
return newScriptedRows(context.Background(), []string{"function_name"}, []string{"STRING"}, [][]driver.Value{
|
||||
{"fn_clean"},
|
||||
}), nil
|
||||
default:
|
||||
return nil, errors.New("unexpected query: " + query)
|
||||
}
|
||||
},
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
server.params.DatabaseType = "argo"
|
||||
defer server.disconnect()
|
||||
|
||||
values, err := server.listObjects("ods", "ods", metadataListConstraints{
|
||||
ObjectTypes: []string{"PROCEDURE", "FUNCTION"},
|
||||
Filter: "sp", // "sp" overlaps the procedure filter; both lists get "%sp%" applied
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("listObjects: %v", err)
|
||||
}
|
||||
t.Logf("values: %+v", values)
|
||||
// listObjects applies the same filter pattern to procedures and functions;
|
||||
// we passed "sp" so procedures match and functions don't (because the function
|
||||
// list returns rows regardless of filter — that is, listObjects calls each
|
||||
// listRoutines call with the same Filter). Verify both queries ran and
|
||||
// the procedure name is present.
|
||||
queries, _, _, _ := behavior.snapshot()
|
||||
t.Logf("queries count: %d", len(queries))
|
||||
for i, q := range queries {
|
||||
t.Logf("query[%d]: %q", i, q)
|
||||
}
|
||||
if len(queries) != 2 {
|
||||
t.Fatalf("expected exactly 2 routine queries, got %d: %v", len(queries), queries)
|
||||
}
|
||||
if values == nil {
|
||||
t.Fatal("expected non-nil object list")
|
||||
}
|
||||
names := make([]string, len(values))
|
||||
for i, v := range values {
|
||||
names[i] = v.Name + ":" + v.ObjectType
|
||||
}
|
||||
joined := strings.Join(names, ",")
|
||||
if !strings.Contains(joined, "sp_daily_etl:PROCEDURE") {
|
||||
t.Fatalf("expected procedures in result, got %v", values)
|
||||
}
|
||||
if !strings.Contains(joined, "fn_clean:FUNCTION") {
|
||||
t.Fatalf("expected functions in result, got %v", values)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetObjectSourceRoutesProceduresToSystemProceduresView(t *testing.T) {
|
||||
expectedSQL := "SELECT full_text FROM system.procedures_v WHERE lower(database_name) = lower('ods') AND procedure_name = 'sp_daily_etl'"
|
||||
behavior := &scriptedBehavior{
|
||||
query: func(_ context.Context, query string) (driver.Rows, error) {
|
||||
if query != expectedSQL {
|
||||
return nil, errors.New("unexpected query: " + query)
|
||||
}
|
||||
return newScriptedRows(context.Background(), []string{"full_text"}, []string{"STRING"}, [][]driver.Value{
|
||||
{"-- daily ETL pipeline"},
|
||||
{"INSERT OVERWRITE TABLE ods.daily_summary SELECT * FROM staging.events"},
|
||||
}), nil
|
||||
},
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
server.params.DatabaseType = "argo"
|
||||
defer server.disconnect()
|
||||
|
||||
result, _, err := server.dispatch("get_object_source", map[string]json.RawMessage{
|
||||
"schema": json.RawMessage(`"ods"`),
|
||||
"name": json.RawMessage(`"sp_daily_etl"`),
|
||||
"object_type": json.RawMessage(`"PROCEDURE"`),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
source, ok := result.(objectSource)
|
||||
if !ok {
|
||||
t.Fatalf("get_object_source returned %T instead of objectSource", result)
|
||||
}
|
||||
if source.Name != "sp_daily_etl" || source.ObjectType != "PROCEDURE" {
|
||||
t.Fatalf("unexpected object source metadata: %#v", source)
|
||||
}
|
||||
expected := "-- daily ETL pipeline\nINSERT OVERWRITE TABLE ods.daily_summary SELECT * FROM staging.events\n"
|
||||
if source.Source != expected {
|
||||
t.Fatalf("unexpected procedure source: %q", source.Source)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetObjectSourceRoutesFunctionsToSystemFunctionsView(t *testing.T) {
|
||||
expectedSQL := "SELECT full_text FROM system.functions_v WHERE lower(database_name) = lower('ods') AND function_name = 'fn_clean'"
|
||||
behavior := &scriptedBehavior{
|
||||
query: func(_ context.Context, query string) (driver.Rows, error) {
|
||||
if query != expectedSQL {
|
||||
return nil, errors.New("unexpected query: " + query)
|
||||
}
|
||||
return newScriptedRows(context.Background(), []string{"full_text"}, []string{"STRING"}, [][]driver.Value{
|
||||
{"-- cleanup helper"},
|
||||
}), nil
|
||||
},
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
server.params.DatabaseType = "argo"
|
||||
defer server.disconnect()
|
||||
|
||||
result, _, err := server.dispatch("get_object_source", map[string]json.RawMessage{
|
||||
"schema": json.RawMessage(`"ods"`),
|
||||
"name": json.RawMessage(`"fn_clean"`),
|
||||
"object_type": json.RawMessage(`"FUNCTION"`),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
source, ok := result.(objectSource)
|
||||
if !ok {
|
||||
t.Fatalf("get_object_source returned %T instead of objectSource", result)
|
||||
}
|
||||
if source.Source != "-- cleanup helper\n" {
|
||||
t.Fatalf("unexpected function source: %q", source.Source)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListRoutinesUsesDatabaseParameterWhenSchemaEmpty(t *testing.T) {
|
||||
proceduresQuery := "SELECT procedure_name FROM system.procedures_v WHERE lower(database_name) = lower('ods') AND lower(procedure_name) LIKE '%%' ORDER BY procedure_name"
|
||||
defaultQuery := "SELECT procedure_name FROM system.procedures_v WHERE lower(database_name) = lower('default') AND lower(procedure_name) LIKE '%%' ORDER BY procedure_name"
|
||||
behavior := &scriptedBehavior{
|
||||
query: func(_ context.Context, query string) (driver.Rows, error) {
|
||||
switch query {
|
||||
case defaultQuery:
|
||||
return newScriptedRows(context.Background(), []string{"procedure_name"}, []string{"STRING"}, [][]driver.Value{}), nil
|
||||
case proceduresQuery:
|
||||
return newScriptedRows(context.Background(), []string{"procedure_name"}, []string{"STRING"}, [][]driver.Value{
|
||||
{"sp_daily_etl"},
|
||||
}), nil
|
||||
default:
|
||||
return nil, errors.New("unexpected query: " + query)
|
||||
}
|
||||
},
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
server.params.DatabaseType = "argo"
|
||||
server.config.Database = "default"
|
||||
defer server.disconnect()
|
||||
|
||||
values, err := server.listObjects("ods", "", metadataListConstraints{
|
||||
ObjectTypes: []string{"PROCEDURE"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("listObjects: %v", err)
|
||||
}
|
||||
if len(values) != 1 || values[0].Name != "sp_daily_etl" {
|
||||
t.Fatalf("expected procedure from database parameter, got %+v", values)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListRoutinesDoesNotFallbackToConnectionDefaultForExplicitSchema(t *testing.T) {
|
||||
defaultQuery := "SELECT procedure_name FROM system.procedures_v WHERE lower(database_name) = lower('default') AND lower(procedure_name) LIKE '%%' ORDER BY procedure_name"
|
||||
behavior := &scriptedBehavior{
|
||||
query: func(_ context.Context, query string) (driver.Rows, error) {
|
||||
if query == defaultQuery {
|
||||
// Serving this row proves the driver fell back to the
|
||||
// connection default after the explicit schema came back empty.
|
||||
return newScriptedRows(context.Background(), []string{"procedure_name"}, []string{"STRING"}, [][]driver.Value{
|
||||
{"sp_should_not_leak"},
|
||||
}), nil
|
||||
}
|
||||
return newScriptedRows(context.Background(), []string{"procedure_name"}, []string{"STRING"}, [][]driver.Value{}), nil
|
||||
},
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
server.params.DatabaseType = "argo"
|
||||
server.config.Database = "default"
|
||||
defer server.disconnect()
|
||||
|
||||
values, err := server.listObjects("", "ods", metadataListConstraints{
|
||||
ObjectTypes: []string{"PROCEDURE"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("listObjects: %v", err)
|
||||
}
|
||||
if len(values) != 0 {
|
||||
t.Fatalf("expected no routines to leak from the connection default database, got %+v", values)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetObjectSourceUsesDatabaseParameterForRoutineSource(t *testing.T) {
|
||||
expectedSQL := "SELECT full_text FROM system.procedures_v WHERE lower(database_name) = lower('ods') AND procedure_name = 'sp_daily_etl'"
|
||||
behavior := &scriptedBehavior{
|
||||
query: func(_ context.Context, query string) (driver.Rows, error) {
|
||||
if query != expectedSQL {
|
||||
return nil, errors.New("unexpected query: " + query)
|
||||
}
|
||||
return newScriptedRows(context.Background(), []string{"full_text"}, []string{"STRING"}, [][]driver.Value{
|
||||
{"CREATE PROCEDURE sp_daily_etl() BEGIN SELECT 1; END"},
|
||||
}), nil
|
||||
},
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
server.params.DatabaseType = "argo"
|
||||
server.config.Database = "default"
|
||||
defer server.disconnect()
|
||||
|
||||
result, _, err := server.dispatch("get_object_source", map[string]json.RawMessage{
|
||||
"database": json.RawMessage(`"ods"`),
|
||||
"schema": json.RawMessage(`""`),
|
||||
"name": json.RawMessage(`"sp_daily_etl"`),
|
||||
"object_type": json.RawMessage(`"PROCEDURE"`),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
source, ok := result.(objectSource)
|
||||
if !ok {
|
||||
t.Fatalf("get_object_source returned %T instead of objectSource", result)
|
||||
}
|
||||
if source.Source != "CREATE PROCEDURE sp_daily_etl() BEGIN SELECT 1; END\n" {
|
||||
t.Fatalf("unexpected procedure source: %q", source.Source)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListObjectsDoesNotQueryRoutinesForVanillaHive(t *testing.T) {
|
||||
behavior := &scriptedBehavior{
|
||||
query: func(ctx context.Context, query string) (driver.Rows, error) {
|
||||
if strings.Contains(strings.ToUpper(query), "PROCEDURES_V") || strings.Contains(strings.ToUpper(query), "FUNCTIONS_V") {
|
||||
t.Fatalf("vanilla Hive must not query routine views, got: %s", query)
|
||||
}
|
||||
return newScriptedRows(ctx, []string{"tab_name"}, []string{"STRING"}, [][]driver.Value{{"events"}}), nil
|
||||
},
|
||||
}
|
||||
server := newScriptedServer(t, behavior)
|
||||
server.params.DatabaseType = "hive"
|
||||
defer server.disconnect()
|
||||
|
||||
values, err := server.listObjects("ods", "ods", metadataListConstraints{
|
||||
ObjectTypes: []string{"PROCEDURE", "FUNCTION"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("listObjects: %v", err)
|
||||
}
|
||||
for _, v := range values {
|
||||
if v.ObjectType == "PROCEDURE" || v.ObjectType == "FUNCTION" {
|
||||
t.Fatalf("vanilla Hive should not surface routines, got: %+v", v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -333,9 +333,12 @@ func (server *server) executeStatements(params map[string]json.RawMessage, trans
|
||||
}
|
||||
return emptyQueryResult(affected, started), nil
|
||||
}
|
||||
if !transactionUnsupported(beginErr) {
|
||||
return queryResult{}, beginErr
|
||||
if transactionUnsupported(beginErr) {
|
||||
// Do not turn an explicit transaction request into auto-commit
|
||||
// execution: a later error would leave prior statements applied.
|
||||
return queryResult{}, errors.New("Hive does not support rollbackable transactions")
|
||||
}
|
||||
return queryResult{}, beginErr
|
||||
}
|
||||
for _, statement := range statements {
|
||||
trimmed := trimStatementSQL(statement)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user