feat(hive): add Kerberos connection options

This commit is contained in:
t8y2 committed 2026-07-08 13:23:21 +08:00
1 parent 5b70626cf7
commit 2929b1c035
30 files changed
+639 -23

No files matched your search

+1
View File
@@ -581,6 +581,7 @@ mod tests {
driver_profile: None,
driver_label: None,
url_params: None,
agent_java_options: Vec::new(),
host: "127.0.0.1".to_string(),
port: 3306,
username: "user".to_string(),
+21 -2
View File
@@ -578,6 +578,16 @@ impl AgentManager {
state: &AgentState,
driver_key: &str,
jre_key: &str,
) -> Result<AgentLaunchSpec, String> {
self.resolve_agent_launch_spec_with_extra_args(state, driver_key, jre_key, &[])
}
pub fn resolve_agent_launch_spec_with_extra_args(
&self,
state: &AgentState,
driver_key: &str,
jre_key: &str,
extra_java_args: &[String],
) -> Result<AgentLaunchSpec, String> {
let driver_dir = self.driver_dir(driver_key);
let config_path = self.driver_launch_config_path(driver_key);
@@ -598,7 +608,7 @@ impl AgentManager {
"{driver_key} driver jar is invalid or corrupt. Please reinstall it from the Driver Manager."
));
}
return Ok(AgentLaunchSpec::java_jar(java, jar_path));
return Ok(AgentLaunchSpec::java_jar_with_extra_args(java, jar_path, extra_java_args));
}
Err(format!("{driver_key} driver is not installed. Please install it from the Driver Manager."))
@@ -766,7 +776,16 @@ impl AgentManager {
db_type: &DatabaseType,
driver_profile: Option<&str>,
) -> Result<AgentDriverClient, String> {
crate::agent_runtime::spawn_connection_client(self, db_type, driver_profile).await
self.spawn_with_extra_java_args(db_type, driver_profile, &[]).await
}
pub async fn spawn_with_extra_java_args(
&self,
db_type: &DatabaseType,
driver_profile: Option<&str>,
extra_java_args: &[String],
) -> Result<AgentDriverClient, String> {
crate::agent_runtime::spawn_connection_client(self, db_type, driver_profile, extra_java_args).await
}
pub async fn call_daemon<T: serde::de::DeserializeOwned + Send + 'static>(
+15 -9
View File
@@ -24,7 +24,7 @@ pub async fn stop_daemon_by_key(manager: &AgentManager, agent_key: &str) {
pub async fn restart_daemon_by_key(manager: &AgentManager, agent_key: &str) -> Result<(), String> {
manager.daemons.lock().await.remove(agent_key);
let client = spawn_client_for_key(manager, agent_key).await?;
let client = spawn_client_for_key(manager, agent_key, &[]).await?;
manager.daemons.lock().await.insert(agent_key.to_string(), client);
Ok(())
}
@@ -33,10 +33,11 @@ pub async fn spawn_connection_client(
manager: &AgentManager,
db_type: &DatabaseType,
driver_profile: Option<&str>,
extra_java_args: &[String],
) -> Result<AgentDriverClient, String> {
let keys = runtime_agent_key_candidates(db_type, driver_profile)
.ok_or_else(|| format!("{:?} is not an agent-driven database type", db_type))?;
spawn_first_available_client(manager, &keys).await
spawn_first_available_client(manager, &keys, extra_java_args).await
}
pub async fn call_daemon<T: DeserializeOwned + Send + 'static>(
@@ -53,7 +54,7 @@ pub async fn call_daemon<T: DeserializeOwned + Send + 'static>(
let mut daemons = manager.daemons.lock().await;
if !daemons.contains_key(&key) {
let client = spawn_client_for_key(manager, &key).await?;
let client = spawn_client_for_key(manager, &key, &[]).await?;
daemons.insert(key.clone(), client);
}
@@ -63,7 +64,7 @@ pub async fn call_daemon<T: DeserializeOwned + Send + 'static>(
Err(err) => {
log::warn!("[agent] daemon call failed, respawning: {err}");
daemons.remove(&key);
let mut new_client = spawn_client_for_key(manager, &key).await?;
let mut new_client = spawn_client_for_key(manager, &key, &[]).await?;
let result = new_client.call::<T>(method, params).await?;
daemons.insert(key, new_client);
Ok(result)
@@ -86,7 +87,7 @@ pub async fn call_daemon_with_timeout<T: DeserializeOwned + Send + 'static>(
let mut daemons = manager.daemons.lock().await;
if !daemons.contains_key(&key) {
let client = spawn_client_for_key(manager, &key).await?;
let client = spawn_client_for_key(manager, &key, &[]).await?;
daemons.insert(key.clone(), client);
}
@@ -96,7 +97,7 @@ pub async fn call_daemon_with_timeout<T: DeserializeOwned + Send + 'static>(
Err(err) => {
log::warn!("[agent] daemon call failed, respawning: {err}");
daemons.remove(&key);
let mut new_client = spawn_client_for_key(manager, &key).await?;
let mut new_client = spawn_client_for_key(manager, &key, &[]).await?;
let result = new_client.call_with_timeout::<T>(method, params, timeout_duration).await?;
daemons.insert(key, new_client);
Ok(result)
@@ -137,10 +138,11 @@ fn first_installed_agent_key<'a>(manager: &AgentManager, keys: &'a [&'static str
async fn spawn_first_available_client(
manager: &AgentManager,
keys: &[&'static str],
extra_java_args: &[String],
) -> Result<AgentDriverClient, String> {
let mut last_error = None;
for key in keys {
match spawn_client_for_key(manager, key).await {
match spawn_client_for_key(manager, key, extra_java_args).await {
Ok(client) => return Ok(client),
Err(err) => last_error = Some(err),
}
@@ -148,11 +150,15 @@ async fn spawn_first_available_client(
Err(last_error.unwrap_or_else(|| "No agent driver candidates available".to_string()))
}
async fn spawn_client_for_key(manager: &AgentManager, key: &str) -> Result<AgentDriverClient, String> {
async fn spawn_client_for_key(
manager: &AgentManager,
key: &str,
extra_java_args: &[String],
) -> Result<AgentDriverClient, String> {
let state = manager.load_state();
let jre_key = state.installed_drivers.get(key).map(|driver| driver.jre.as_str()).unwrap_or(DEFAULT_JRE_KEY);
let launch = manager.resolve_agent_launch_spec(&state, key, jre_key)?;
let launch = manager.resolve_agent_launch_spec_with_extra_args(&state, key, jre_key, extra_java_args)?;
let mut client = AgentDriverClient::spawn(launch).await?;
client.try_optional_handshake(manager.agent_app_version()).await;
Ok(client)
+3
View File
@@ -752,6 +752,7 @@ mod tests {
driver_profile: None,
driver_label: None,
url_params: None,
agent_java_options: Vec::new(),
host: "127.0.0.1".to_string(),
port: 5432,
username: "app".to_string(),
@@ -801,6 +802,7 @@ mod tests {
driver_profile: None,
driver_label: None,
url_params: None,
agent_java_options: Vec::new(),
host: "127.0.0.1".to_string(),
port: 8848,
username: "nacos".to_string(),
@@ -887,6 +889,7 @@ mod tests {
driver_profile: None,
driver_label: None,
url_params: None,
agent_java_options: Vec::new(),
host: "localhost".to_string(),
port: 5432,
username: "user".to_string(),
+10 -2
View File
@@ -1187,8 +1187,15 @@ impl AppState {
agent_connection_pool_database_type!() => {
let connect_params =
agent_connect_params(&db_config, &host, port, db_config.effective_database().unwrap_or(""));
let mut client =
self.agent_manager.spawn(&db_config.db_type, db_config.driver_profile.as_deref()).await?;
// Kerberos JVM properties are connection-scoped; shared agent daemons must not inherit them.
let mut client = self
.agent_manager
.spawn_with_extra_java_args(
&db_config.db_type,
db_config.driver_profile.as_deref(),
&db_config.agent_java_options,
)
.await?;
let connect_result = client
.call_method_with_timeout::<serde_json::Value>(
AgentMethod::Connect,
@@ -3005,6 +3012,7 @@ mod tests {
driver_profile: None,
driver_label: None,
url_params: None,
agent_java_options: Vec::new(),
host: "127.0.0.1".to_string(),
port: 3306,
username: "root".to_string(),
@@ -724,6 +724,7 @@ mod tests {
driver_profile: None,
driver_label: None,
url_params: None,
agent_java_options: Vec::new(),
host: "localhost".to_string(),
port: 5432,
username: "postgres".to_string(),
+133 -8
View File
@@ -15,6 +15,7 @@ const RPC_TIMEOUT_SECS: u64 = 30;
const STARTUP_TIMEOUT_SECS: u64 = 15;
const STDERR_TAIL_LINES: usize = 20;
const AGENT_EXIT_DIAGNOSTIC_WAIT_MS: u64 = 200;
const AGENT_JAVA_OPTS_ENV: &str = "DBX_AGENT_JAVA_OPTS";
const AGENT_JAVA_TOO_OLD_MESSAGE: &str =
"Agent requires Java 21, but DBX started it with an older Java runtime. Use DBX managed JRE 21 or select a Java 21 executable in Driver Manager.";
#[cfg(windows)]
@@ -42,10 +43,18 @@ impl AgentLaunchSpec {
}
pub fn java_jar(java_path: impl Into<PathBuf>, jar_path: impl AsRef<Path>) -> Self {
Self::java_jar_with_extra_args(java_path, jar_path, &[])
}
pub fn java_jar_with_extra_args(
java_path: impl Into<PathBuf>,
jar_path: impl AsRef<Path>,
extra_java_args: &[String],
) -> Self {
let jar_path = jar_path.as_ref();
Self {
program: java_path.into(),
args: agent_java_args(&jar_path.to_string_lossy()),
args: agent_java_args_with_extra_args(&jar_path.to_string_lossy(), extra_java_args),
working_dir: jar_path.parent().map(Path::to_path_buf),
}
}
@@ -1172,7 +1181,25 @@ pub fn mongo_document_id_params(database: &str, collection: &str, id: &str) -> V
serde_json::json!({ "database": database, "collection": collection, "id": id })
}
#[cfg(test)]
fn agent_java_args(jar_path: &str) -> Vec<String> {
agent_java_args_with_extra(jar_path, std::env::var(AGENT_JAVA_OPTS_ENV).ok().as_deref())
}
#[cfg(test)]
fn agent_java_args_with_extra(jar_path: &str, extra_opts: Option<&str>) -> Vec<String> {
agent_java_args_with_extra_opts(jar_path, extra_opts, &[])
}
fn agent_java_args_with_extra_args(jar_path: &str, extra_java_args: &[String]) -> Vec<String> {
agent_java_args_with_extra_opts(jar_path, std::env::var(AGENT_JAVA_OPTS_ENV).ok().as_deref(), extra_java_args)
}
fn agent_java_args_with_extra_opts(
jar_path: &str,
extra_opts: Option<&str>,
extra_java_args: &[String],
) -> Vec<String> {
let mut args = vec![
"-Dfile.encoding=UTF-8",
"-Dsun.stdout.encoding=UTF-8",
@@ -1192,6 +1219,13 @@ fn agent_java_args(jar_path: &str) -> Vec<String> {
args.push("-Djava.net.preferIPv4Stack=true".to_string());
}
// Hive/Kerberos JDBC drivers read JAAS and krb5 settings during JVM startup,
// so users need a process-level escape hatch before the agent jar is loaded.
if let Some(extra) = extra_opts {
args.extend(parse_agent_java_opts(extra));
}
args.extend(extra_java_args.iter().map(|arg| arg.trim()).filter(|arg| !arg.is_empty()).map(str::to_string));
args.push("--add-opens=java.sql/java.sql=ALL-UNNAMED".to_string());
args.extend(["-XX:TieredStopAtLevel=1", "-XX:+UseSerialGC", "-jar", jar_path].into_iter().map(str::to_string));
@@ -1199,6 +1233,43 @@ fn agent_java_args(jar_path: &str) -> Vec<String> {
args
}
fn parse_agent_java_opts(opts: &str) -> Vec<String> {
let mut args = Vec::new();
let mut current = String::new();
let mut quote = None;
let mut chars = opts.chars().peekable();
while let Some(ch) = chars.next() {
match (quote, ch) {
(None, ch) if ch.is_whitespace() => {
if !current.is_empty() {
args.push(std::mem::take(&mut current));
}
}
(None, '\'' | '"') => quote = Some(ch),
(Some(q), ch) if ch == q => quote = None,
(Some('"'), '\\') => {
if let Some(&next) = chars.peek() {
if next == '"' || next == '\\' {
current.push(chars.next().unwrap());
} else {
current.push(ch);
}
} else {
current.push(ch);
}
}
_ => current.push(ch),
}
}
if !current.is_empty() {
args.push(current);
}
args
}
fn agent_jar_path_matches_key(jar_path: &str, key: &str) -> bool {
Path::new(jar_path).components().any(|component| component.as_os_str().to_string_lossy() == key)
}
@@ -1345,13 +1416,13 @@ impl Drop for AgentDriverClient {
#[cfg(test)]
mod tests {
use super::{
agent_close_query_session_params, agent_handshake_params, agent_java_args, agent_object_source_params,
agent_proxy_env_vars, agent_schema_params, agent_schema_table_params, agent_supports_capability,
agent_transaction_params, format_agent_process_error, format_agent_startup_error,
is_unsupported_handshake_error, mongo_collection_params, mongo_database_params, mongo_document_id_params,
read_agent_line, start_stderr_collector, AgentCapability, AgentDriverClient, AgentHandshake, AgentKvMethod,
AgentMethod, AgentTableReadCloseParams, AgentTableReadPageParams, AgentTableReadStartParams, MongoAgentMethod,
StderrTail, AGENT_PROTOCOL_VERSION,
agent_close_query_session_params, agent_handshake_params, agent_java_args, agent_java_args_with_extra,
agent_java_args_with_extra_opts, agent_object_source_params, agent_proxy_env_vars, agent_schema_params,
agent_schema_table_params, agent_supports_capability, agent_transaction_params, format_agent_process_error,
format_agent_startup_error, is_unsupported_handshake_error, mongo_collection_params, mongo_database_params,
mongo_document_id_params, parse_agent_java_opts, read_agent_line, start_stderr_collector, AgentCapability,
AgentDriverClient, AgentHandshake, AgentKvMethod, AgentMethod, AgentTableReadCloseParams,
AgentTableReadPageParams, AgentTableReadStartParams, MongoAgentMethod, StderrTail, AGENT_PROTOCOL_VERSION,
};
use std::io::Cursor;
use std::process::{Command, Stdio};
@@ -1403,6 +1474,60 @@ mod tests {
assert!(!args.iter().any(|arg| arg == "-Djava.net.preferIPv4Stack=true"));
}
#[test]
fn agent_java_args_include_custom_jvm_options_before_jar() {
let args = agent_java_args_with_extra(
"/tmp/dbx/drivers/hive/agent.jar",
Some("-Djava.security.auth.login.config=C:\\jaas.conf -Djavax.security.auth.useSubjectCredsOnly=false"),
);
let login_config = args
.iter()
.position(|arg| arg == "-Djava.security.auth.login.config=C:\\jaas.conf")
.expect("custom JAAS option should be present");
let jar = args.iter().position(|arg| arg == "-jar").expect("agent jar marker should be present");
assert!(login_config < jar);
assert!(args.iter().any(|arg| arg == "-Djavax.security.auth.useSubjectCredsOnly=false"));
}
#[test]
fn agent_java_args_include_connection_jvm_options_after_env_options() {
let args = agent_java_args_with_extra_opts(
"/tmp/dbx/drivers/hive/agent.jar",
Some("-Djava.security.krb5.conf=/etc/global-krb5.conf"),
&["-Djava.security.krb5.conf=/etc/connection-krb5.conf".to_string()],
);
let global = args
.iter()
.position(|arg| arg == "-Djava.security.krb5.conf=/etc/global-krb5.conf")
.expect("global krb5 option should be present");
let connection = args
.iter()
.position(|arg| arg == "-Djava.security.krb5.conf=/etc/connection-krb5.conf")
.expect("connection krb5 option should be present");
let jar = args.iter().position(|arg| arg == "-jar").expect("agent jar marker should be present");
assert!(global < connection);
assert!(connection < jar);
}
#[test]
fn agent_java_opts_parser_preserves_quoted_windows_paths() {
let args = parse_agent_java_opts(
r#"-Djava.security.krb5.conf="C:\Program Files\MIT\Kerberos5\krb5.ini" -Dsun.security.krb5.debug=true"#,
);
assert_eq!(
args,
vec![
"-Djava.security.krb5.conf=C:\\Program Files\\MIT\\Kerberos5\\krb5.ini",
"-Dsun.security.krb5.debug=true"
]
);
}
#[test]
fn agent_process_environment_removes_common_proxy_variables() {
let proxy_env_vars = agent_proxy_env_vars();
+1
View File
@@ -3029,6 +3029,7 @@ mod tests {
driver_profile: None,
driver_label: None,
url_params: None,
agent_java_options: Vec::new(),
host: "cache.example.com".to_string(),
port: 6379,
username: String::new(),
+23
View File
@@ -14,6 +14,8 @@ pub struct ConnectionConfig {
pub driver_label: Option<String>,
#[serde(default)]
pub url_params: Option<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub agent_java_options: Vec<String>,
pub host: String,
pub port: u16,
pub username: String,
@@ -382,6 +384,8 @@ struct ConnectionConfigData {
pub driver_label: Option<String>,
#[serde(default)]
pub url_params: Option<String>,
#[serde(default)]
pub agent_java_options: Vec<String>,
pub host: String,
pub port: u16,
pub username: String,
@@ -464,6 +468,7 @@ impl From<ConnectionConfigData> for ConnectionConfig {
driver_profile: data.driver_profile,
driver_label: data.driver_label,
url_params: data.url_params,
agent_java_options: data.agent_java_options,
host: data.host,
port: data.port,
username: data.username,
@@ -1790,6 +1795,7 @@ mod tests {
driver_profile: None,
driver_label: None,
url_params: None,
agent_java_options: Vec::new(),
host: "10.1.2.3".to_string(),
port: 2883,
username: username.to_string(),
@@ -1844,6 +1850,23 @@ mod tests {
assert_eq!(serde_json::from_str::<DatabaseType>("\"zookeeper\"").unwrap(), DatabaseType::ZooKeeper);
}
#[test]
fn connection_config_defaults_missing_agent_java_options_to_empty() {
let config: ConnectionConfig = serde_json::from_value(serde_json::json!({
"id": "id",
"name": "Hive",
"db_type": "hive",
"host": "hive.local",
"port": 10000,
"username": "",
"password": "",
"database": "default"
}))
.unwrap();
assert!(config.agent_java_options.is_empty());
}
#[test]
fn zookeeper_connection_url_uses_zookeeper_scheme() {
let mut config = mysql_config("", "", None);
+1
View File
@@ -105,6 +105,7 @@ mod tests {
driver_profile: None,
driver_label: None,
url_params: None,
agent_java_options: Vec::new(),
host: String::new(),
port: 0,
username: String::new(),
+1
View File
@@ -553,6 +553,7 @@ mod tests {
driver_profile: Some("pulsar".to_string()),
driver_label: Some("Apache Pulsar".to_string()),
url_params: None,
agent_java_options: Vec::new(),
host: "127.0.0.1".to_string(),
port: 8080,
username: String::new(),
+1
View File
@@ -130,6 +130,7 @@ mod tests {
driver_profile: None,
driver_label: None,
url_params: None,
agent_java_options: Vec::new(),
host: "127.0.0.1".to_string(),
port: 8848,
username: String::new(),
+2
View File
@@ -174,6 +174,7 @@ mod tests {
driver_profile: None,
driver_label: None,
url_params: None,
agent_java_options: Vec::new(),
host: "127.0.0.1".to_string(),
port: 8848,
username: String::new(),
@@ -238,6 +239,7 @@ mod tests {
driver_profile: None,
driver_label: None,
url_params: None,
agent_java_options: Vec::new(),
host: "127.0.0.1".to_string(),
port: 8848,
username: String::new(),
+2
View File
@@ -2890,6 +2890,7 @@ mod tests {
driver_profile: None,
driver_label: None,
url_params: None,
agent_java_options: Vec::new(),
host: "localhost".to_string(),
port: 0,
username: String::new(),
@@ -3519,6 +3520,7 @@ mod tests {
driver_profile: None,
driver_label: None,
url_params: None,
agent_java_options: Vec::new(),
host: "localhost".to_string(),
port: 0,
username: String::new(),
+1
View File
@@ -2091,6 +2091,7 @@ mod tests {
driver_profile: None,
driver_label: None,
url_params: None,
agent_java_options: Vec::new(),
host: "127.0.0.1".to_string(),
port: 5432,
username: "user".to_string(),
@@ -151,6 +151,7 @@ mod tests {
driver_profile: None,
driver_label: None,
url_params: None,
agent_java_options: Vec::new(),
host: "127.0.0.1".to_string(),
port: 5432,
username: "user".to_string(),
+2
View File
@@ -2415,6 +2415,7 @@ mod tests {
driver_profile: Some("pulsar".to_string()),
driver_label: Some("Apache Pulsar".to_string()),
url_params: None,
agent_java_options: Vec::new(),
host: "127.0.0.1".to_string(),
port: 8080,
username: String::new(),
@@ -2471,6 +2472,7 @@ mod tests {
driver_profile: None,
driver_label: None,
url_params: None,
agent_java_options: Vec::new(),
host: "127.0.0.1".to_string(),
port: 8848,
username: "nacos".to_string(),
+1
View File
@@ -4298,6 +4298,7 @@ mod tests {
driver_profile: None,
driver_label: None,
url_params: None,
agent_java_options: Vec::new(),
host: ":memory:".to_string(),
port: 0,
username: String::new(),