fix(agents): share runtimes across connection sessions

This commit is contained in:
t8y2 committed 2026-07-13 15:13:48 +08:00
1 parent 1280450b98
commit 67541232a2
67 files changed
+2904 -354

No files matched your search

+38 -4
View File
@@ -6,7 +6,7 @@ use std::path::{Path, PathBuf};
use serde::{Deserialize, Serialize};
use tokio::sync::Mutex;
use crate::db::agent_driver::{AgentDriverClient, AgentLaunchSpec, AgentMethod};
use crate::db::agent_driver::{AgentDriverClient, AgentLaunchSpec, AgentMethod, AgentRuntimeClient};
use crate::models::connection::DatabaseType;
pub const DEFAULT_JRE_KEY: &str = "21";
@@ -394,6 +394,9 @@ pub struct AgentManager {
base_dir: PathBuf,
app_version: String,
pub(crate) daemons: Mutex<std::collections::HashMap<String, AgentDriverClient>>,
pub(crate) connection_runtimes: Mutex<
std::collections::HashMap<String, std::sync::Arc<tokio::sync::OnceCell<std::sync::Arc<AgentRuntimeClient>>>>,
>,
}
impl Default for AgentManager {
@@ -414,8 +417,12 @@ impl AgentManager {
}
pub fn new_with_base_dir_and_app_version(base_dir: PathBuf, app_version: impl Into<String>) -> Self {
let mgr =
Self { base_dir, app_version: app_version.into(), daemons: Mutex::new(std::collections::HashMap::new()) };
let mgr = Self {
base_dir,
app_version: app_version.into(),
daemons: Mutex::new(std::collections::HashMap::new()),
connection_runtimes: Mutex::new(std::collections::HashMap::new()),
};
mgr.migrate_legacy_jre();
mgr.cleanup_pending_jre_dirs();
mgr.cleanup_orphan_jre_dirs();
@@ -760,7 +767,13 @@ impl AgentManager {
}
pub async fn active_daemon_keys(&self) -> Vec<String> {
self.daemons.lock().await.keys().cloned().collect()
let mut keys = self.daemons.lock().await.keys().cloned().collect::<std::collections::HashSet<_>>();
for runtime_key in self.connection_runtimes.lock().await.keys() {
if let Some((agent_key, _)) = runtime_key.split_once('|') {
keys.insert(agent_key.to_string());
}
}
keys.into_iter().collect()
}
pub fn db_type_to_agent_key(db_type: &DatabaseType, driver_profile: Option<&str>) -> Option<&'static str> {
@@ -788,6 +801,27 @@ impl AgentManager {
crate::agent_runtime::spawn_connection_client(self, db_type, driver_profile, extra_java_args).await
}
pub async fn spawn_shared_connection_client(
&self,
db_type: &DatabaseType,
driver_profile: Option<&str>,
extra_java_args: &[String],
agent_session_id: String,
connect_params: serde_json::Value,
connect_timeout: std::time::Duration,
) -> Result<AgentDriverClient, String> {
crate::agent_runtime::spawn_shared_connection_client(
self,
db_type,
driver_profile,
extra_java_args,
agent_session_id,
connect_params,
connect_timeout,
)
.await
}
pub async fn call_daemon<T: serde::de::DeserializeOwned + Send + 'static>(
&self,
db_type: &DatabaseType,
+74 -1
View File
@@ -3,7 +3,7 @@ use std::time::Duration;
use crate::agent_manager::{AgentManager, DEFAULT_JRE_KEY};
use crate::database_capabilities;
use crate::db::agent_driver::{AgentDriverClient, AgentMethod};
use crate::db::agent_driver::{AgentDriverClient, AgentMethod, AgentRuntimeClient};
use crate::models::connection::DatabaseType;
pub fn db_type_to_agent_key(db_type: &DatabaseType, driver_profile: Option<&str>) -> Option<&'static str> {
@@ -16,10 +16,21 @@ pub fn is_agent_type(db_type: &DatabaseType) -> bool {
pub async fn stop_daemons(manager: &AgentManager) {
manager.daemons.lock().await.clear();
let runtimes = std::mem::take(&mut *manager.connection_runtimes.lock().await);
for runtime in runtimes.into_values().filter_map(|cell| cell.get().cloned()) {
runtime.kill();
}
}
pub async fn stop_daemon_by_key(manager: &AgentManager, agent_key: &str) {
manager.daemons.lock().await.remove(agent_key);
let mut runtimes = manager.connection_runtimes.lock().await;
let matching = runtimes.keys().filter(|key| key.starts_with(&format!("{agent_key}|"))).cloned().collect::<Vec<_>>();
for key in matching {
if let Some(runtime) = runtimes.remove(&key).and_then(|cell| cell.get().cloned()) {
runtime.kill();
}
}
}
pub async fn restart_daemon_by_key(manager: &AgentManager, agent_key: &str) -> Result<(), String> {
@@ -40,6 +51,54 @@ pub async fn spawn_connection_client(
spawn_first_available_client(manager, &keys, extra_java_args).await
}
pub async fn spawn_shared_connection_client(
manager: &AgentManager,
db_type: &DatabaseType,
driver_profile: Option<&str>,
extra_java_args: &[String],
agent_session_id: String,
connect_params: serde_json::Value,
connect_timeout: Duration,
) -> Result<AgentDriverClient, String> {
let keys = runtime_agent_key_candidates(db_type, driver_profile)
.ok_or_else(|| format!("{:?} is not an agent-driven database type", db_type))?;
let key = first_installed_agent_key(manager, &keys).unwrap_or(keys[0]);
let state = manager.load_state();
let jre_key = state.installed_drivers.get(key).map(|driver| driver.jre.as_str()).unwrap_or(DEFAULT_JRE_KEY);
let launch = manager.resolve_agent_launch_spec_with_extra_args(&state, key, jre_key, extra_java_args)?;
let runtime_key = shared_runtime_key(key, &launch);
let runtime_cell = {
let mut runtimes = manager.connection_runtimes.lock().await;
if runtimes.get(&runtime_key).and_then(|cell| cell.get()).is_some_and(|runtime| runtime.is_failed()) {
runtimes.remove(&runtime_key);
}
runtimes.entry(runtime_key).or_insert_with(|| std::sync::Arc::new(tokio::sync::OnceCell::new())).clone()
};
let runtime =
runtime_cell.get_or_try_init(|| AgentRuntimeClient::spawn(launch, manager.agent_app_version())).await?.clone();
let mut session_params = connect_params;
session_params
.as_object_mut()
.ok_or_else(|| "Agent connect parameters must be an object".to_string())?
.insert("agentSessionId".to_string(), serde_json::Value::String(agent_session_id.clone()));
runtime
.call::<serde_json::Value>(AgentMethod::OpenSession.as_str(), session_params, Some(connect_timeout), None)
.await?;
runtime.increment_session_count();
Ok(AgentDriverClient::shared_session(runtime, agent_session_id))
}
fn shared_runtime_key(agent_key: &str, launch: &crate::db::agent_driver::AgentLaunchSpec) -> String {
format!(
"{}|{}|{}|{}",
agent_key,
launch.program.display(),
launch.args.join("\u{1f}"),
launch.working_dir.as_ref().map(|path| path.display().to_string()).unwrap_or_default()
)
}
pub async fn call_daemon<T: DeserializeOwned + Send + 'static>(
manager: &AgentManager,
db_type: &DatabaseType,
@@ -167,6 +226,7 @@ async fn spawn_client_for_key(
#[cfg(test)]
mod tests {
use super::*;
use std::path::PathBuf;
#[test]
fn prestosql_does_not_use_agent_driver() {
@@ -177,4 +237,17 @@ mod tests {
fn trino_uses_only_trino_agent_driver() {
assert_eq!(runtime_agent_key_candidates(&DatabaseType::Trino, None).unwrap(), vec!["trino"]);
}
#[test]
fn shared_runtime_key_includes_launch_fingerprint() {
let base = crate::db::agent_driver::AgentLaunchSpec::new(PathBuf::from("oracle-agent"))
.with_args(["--mode", "stdio"])
.with_working_dir(PathBuf::from("/tmp/oracle"));
let different_args = crate::db::agent_driver::AgentLaunchSpec::new(PathBuf::from("oracle-agent"))
.with_args(["--mode", "debug"])
.with_working_dir(PathBuf::from("/tmp/oracle"));
assert_eq!(shared_runtime_key("oracle", &base), shared_runtime_key("oracle", &base));
assert_ne!(shared_runtime_key("oracle", &base), shared_runtime_key("oracle", &different_args));
}
}
+175 -55
View File
@@ -1264,71 +1264,161 @@ impl AppState {
agent_connection_pool_database_type!() => {
let connect_params =
agent_connect_params(&db_config, &host, port, db_config.effective_database().unwrap_or(""));
// Kerberos JVM properties are connection-scoped; shared agent daemons must not inherit them.
let mut client = self
.agent_manager
.spawn_with_extra_java_args(
&db_config.db_type,
db_config.driver_profile.as_deref(),
&db_config.agent_java_options,
)
.await?;
let connect_result = client
.call_method_with_timeout::<serde_json::Value>(
AgentMethod::Connect,
connect_params,
Some(agent_connect_timeout(&db_config)),
)
.await;
if let Err(err) = connect_result {
let alternate_configs = oracle_alternate_connect_configs(&db_config, &err);
if !alternate_configs.is_empty() {
log::warn!(
if db_config.db_type != DatabaseType::Etcd && db_config.db_type != DatabaseType::ZooKeeper {
let agent_session_id = uuid::Uuid::new_v4().simple().to_string();
let initial_result = self
.agent_manager
.spawn_shared_connection_client(
&db_config.db_type,
db_config.driver_profile.as_deref(),
&db_config.agent_java_options,
agent_session_id.clone(),
connect_params,
agent_connect_timeout(&db_config),
)
.await;
let client = match initial_result {
Ok(client) => client,
Err(err) => {
let alternate_configs = oracle_alternate_connect_configs(&db_config, &err);
if alternate_configs.is_empty() {
if err.contains("does not support multi_session protocol v2") {
let mut client = self
.agent_manager
.spawn_with_extra_java_args(
&db_config.db_type,
db_config.driver_profile.as_deref(),
&db_config.agent_java_options,
)
.await?;
client
.call_method_with_timeout::<serde_json::Value>(
AgentMethod::Connect,
agent_connect_params(
&db_config,
&host,
port,
db_config.effective_database().unwrap_or(""),
),
Some(agent_connect_timeout(&db_config)),
)
.await?;
client
} else {
return Err(oracle_error_with_driver_hint(&db_config, &err));
}
} else {
let mut fallback_errors = Vec::new();
let mut connected = None;
for alternate_config in alternate_configs {
let label =
oracle_alternate_connect_config_labels(std::slice::from_ref(&alternate_config))
.into_iter()
.next()
.unwrap_or_else(|| "alternate".to_string());
let alternate_params = agent_connect_params(
&alternate_config,
&host,
port,
alternate_config.effective_database().unwrap_or(""),
);
match self
.agent_manager
.spawn_shared_connection_client(
&alternate_config.db_type,
alternate_config.driver_profile.as_deref(),
&alternate_config.agent_java_options,
agent_session_id.clone(),
alternate_params,
agent_connect_timeout(&alternate_config),
)
.await
{
Ok(client) => {
connected = Some(client);
break;
}
Err(alternate_err) => fallback_errors.push(format!("{label}: {alternate_err}")),
}
}
connected.ok_or_else(|| {
format!(
"{err}\n\nFallback with alternate Oracle connection descriptors failed: {}",
fallback_errors.join("\n")
)
})?
}
}
};
PoolKind::Agent(Arc::new(tokio::sync::Mutex::new(client)))
} else {
// Kerberos JVM properties are connection-scoped; shared agent daemons must not inherit them.
let mut client = self
.agent_manager
.spawn_with_extra_java_args(
&db_config.db_type,
db_config.driver_profile.as_deref(),
&db_config.agent_java_options,
)
.await?;
let connect_result = client
.call_method_with_timeout::<serde_json::Value>(
AgentMethod::Connect,
connect_params,
Some(agent_connect_timeout(&db_config)),
)
.await;
if let Err(err) = connect_result {
let alternate_configs = oracle_alternate_connect_configs(&db_config, &err);
if !alternate_configs.is_empty() {
log::warn!(
"Oracle connect failed with {:?} descriptor: {}. Retrying with Oracle JDBC URL variants: {:?}.",
db_config.oracle_connection_type,
err,
oracle_alternate_connect_config_labels(&alternate_configs)
);
let mut fallback_errors = Vec::new();
let mut connected = false;
for alternate_config in alternate_configs {
let label = oracle_alternate_connect_config_labels(std::slice::from_ref(&alternate_config))
.into_iter()
.next()
.unwrap_or_else(|| "alternate".to_string());
match client
.call_method_with_timeout::<serde_json::Value>(
AgentMethod::Connect,
agent_connect_params(
&alternate_config,
&host,
port,
alternate_config.effective_database().unwrap_or(""),
),
Some(agent_connect_timeout(&alternate_config)),
)
.await
{
Ok(_) => {
connected = true;
break;
}
Err(alternate_err) => {
fallback_errors.push(format!("{label}: {alternate_err}"));
let mut fallback_errors = Vec::new();
let mut connected = false;
for alternate_config in alternate_configs {
let label =
oracle_alternate_connect_config_labels(std::slice::from_ref(&alternate_config))
.into_iter()
.next()
.unwrap_or_else(|| "alternate".to_string());
match client
.call_method_with_timeout::<serde_json::Value>(
AgentMethod::Connect,
agent_connect_params(
&alternate_config,
&host,
port,
alternate_config.effective_database().unwrap_or(""),
),
Some(agent_connect_timeout(&alternate_config)),
)
.await
{
Ok(_) => {
connected = true;
break;
}
Err(alternate_err) => {
fallback_errors.push(format!("{label}: {alternate_err}"));
}
}
}
if !connected {
return Err(format!(
"{err}\n\nFallback with alternate Oracle JDBC URLs failed: {}",
fallback_errors.join("\n")
));
}
} else {
return Err(oracle_error_with_driver_hint(&db_config, &err));
}
if !connected {
return Err(format!(
"{err}\n\nFallback with alternate Oracle JDBC URLs failed: {}",
fallback_errors.join("\n")
));
}
} else {
return Err(oracle_error_with_driver_hint(&db_config, &err));
}
PoolKind::Agent(Arc::new(tokio::sync::Mutex::new(client)))
}
PoolKind::Agent(Arc::new(tokio::sync::Mutex::new(client)))
}
DatabaseType::PrestoSql => {
let jdbc_config = prestosql_jdbc_config_for_endpoint(&db_config, &host, port);
@@ -4003,6 +4093,17 @@ mod tests {
);
}
#[test]
fn oracle_uses_isolated_pool_keys_for_tab_sessions() {
let mut config = mysql_config(Some("ORCLPDB1"));
config.db_type = DatabaseType::Oracle;
assert_eq!(
super::session_scoped_pool_key_for(Some(&config), "oracle-conn".to_string(), Some("table-tab-1")),
"oracle-conn:session:table-tab-1"
);
}
#[test]
fn other_agent_single_connection_types_keep_database_scoped_pool_keys() {
assert_eq!(
@@ -4338,6 +4439,25 @@ mod tests {
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test]
async fn closing_oracle_table_tab_keeps_connection_scoped_pool() {
let (state, dir) = test_app_state().await;
let mut config = mysql_config(Some("ORCLPDB1"));
config.id = "oracle-conn".to_string();
config.db_type = DatabaseType::Oracle;
state.configs.write().await.insert(config.id.clone(), config);
let pool = crate::db::sqlite::connect_path(":memory:").await.unwrap();
state.connections.write().await.insert("oracle-conn".to_string(), PoolKind::Sqlite(pool));
state.pool_activity.write().await.insert("oracle-conn".to_string(), super::PoolActivity::now());
assert!(!state.close_client_session_pool("oracle-conn", Some("APP"), "table-tab-1").await.unwrap());
assert!(state.connections.read().await.contains_key("oracle-conn"));
assert!(state.pool_activity.read().await.contains_key("oracle-conn"));
let _ = std::fs::remove_dir_all(dir);
}
#[test]
fn agent_validate_connection_unknown_method_is_not_stale() {
assert!(super::is_agent_validate_connection_unsupported(
+510 -18
View File
@@ -1,7 +1,8 @@
use std::collections::VecDeque;
use std::collections::{HashMap, VecDeque};
use std::io::{BufRead, BufReader, BufWriter, Write};
use std::path::{Path, PathBuf};
use std::process::{Child, ChildStderr, ChildStdin, ChildStdout, Command, Stdio};
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
@@ -10,23 +11,261 @@ use serde::{Deserialize, Serialize};
use serde_json::Value;
use tokio_util::sync::CancellationToken;
pub const AGENT_PROTOCOL_VERSION: u32 = 1;
type PendingAgentResponse = tokio::sync::oneshot::Sender<Result<Value, String>>;
pub struct AgentRuntimeClient {
child: Arc<Mutex<Child>>,
stdin: Arc<Mutex<BufWriter<ChildStdin>>>,
pending: Arc<Mutex<HashMap<u64, PendingAgentResponse>>>,
stderr_tail: Arc<Mutex<StderrTail>>,
next_id: AtomicU64,
active_sessions: AtomicU64,
failed: Arc<AtomicBool>,
handshake: AgentHandshake,
}
impl AgentRuntimeClient {
pub async fn spawn(launch: AgentLaunchSpec, app_version: &str) -> Result<Arc<Self>, String> {
let mut command = crate::process::new_std_command(&launch.program);
command.args(&launch.args).stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::piped());
if let Some(working_dir) = &launch.working_dir {
command.current_dir(working_dir);
}
remove_agent_proxy_env(&mut command);
let mut child =
command.spawn().map_err(|e| format!("Failed to spawn agent process {}: {e}", launch_display(&launch)))?;
let child_stdin = child.stdin.take().ok_or("Failed to capture agent stdin")?;
let child_stdout = child.stdout.take().ok_or("Failed to capture agent stdout")?;
let child_stderr = child.stderr.take().ok_or("Failed to capture agent stderr")?;
let stderr_tail = Arc::new(Mutex::new(StderrTail::default()));
start_stderr_collector(child_stderr, stderr_tail.clone());
let mut stdout = BufReader::new(child_stdout);
let stdout = tokio::time::timeout(
Duration::from_secs(STARTUP_TIMEOUT_SECS),
tokio::task::spawn_blocking(move || loop {
let line = read_agent_line(&mut stdout, "startup line")?;
let trimmed = line.trim();
if trimmed.is_empty() {
continue;
}
match serde_json::from_str::<Value>(trimmed) {
Ok(value) if value.get("ready") == Some(&Value::Bool(true)) => return Ok(stdout),
Ok(_) => return Err(format!("Agent did not send ready signal, got: {line}")),
Err(_) => log::warn!("[agent:stdout] ignoring non-JSON line during startup: {trimmed}"),
}
}),
)
.await
.map_err(|_| format!("Agent startup timed out ({STARTUP_TIMEOUT_SECS}s)"))?
.map_err(|e| format!("Agent startup task failed: {e}"))??;
let runtime = Arc::new(Self {
child: Arc::new(Mutex::new(child)),
stdin: Arc::new(Mutex::new(BufWriter::new(child_stdin))),
pending: Arc::new(Mutex::new(HashMap::new())),
stderr_tail,
next_id: AtomicU64::new(0),
active_sessions: AtomicU64::new(0),
failed: Arc::new(AtomicBool::new(false)),
handshake: AgentHandshake { protocol_version: 0, agent_protocol_version: 0, capabilities: Vec::new() },
});
runtime.start_response_reader(stdout);
let handshake = runtime
.call::<AgentHandshake>(
AgentMethod::Handshake.as_str(),
agent_handshake_params(app_version),
Some(Duration::from_secs(RPC_TIMEOUT_SECS)),
None,
)
.await?;
if handshake.protocol_version < 2 || !handshake.supports(AgentCapability::MultiSession) {
runtime.kill();
return Err("Agent runtime does not support multi_session protocol v2".to_string());
}
let runtime =
Arc::try_unwrap(runtime).map_err(|_| "Agent runtime initialization is still referenced".to_string())?;
Ok(Arc::new(Self { handshake, ..runtime }))
}
fn start_response_reader(self: &Arc<Self>, mut stdout: BufReader<ChildStdout>) {
let pending = self.pending.clone();
let failed = self.failed.clone();
std::thread::spawn(move || loop {
let line = match read_agent_line(&mut stdout, "response") {
Ok(line) => line,
Err(err) => {
failed.store(true, Ordering::Release);
fail_pending_requests(&pending, err);
return;
}
};
let response: Value = match serde_json::from_str(line.trim()) {
Ok(response) => response,
Err(err) => {
failed.store(true, Ordering::Release);
fail_pending_requests(&pending, format!("Invalid JSON response from agent: {err}"));
return;
}
};
let Some(id) = response.get("id").and_then(Value::as_u64) else {
continue;
};
if let Some(sender) = pending.lock().expect("agent pending response lock poisoned").remove(&id) {
let _ = sender.send(Ok(response));
}
});
}
pub async fn call<T: DeserializeOwned>(
&self,
method: &str,
params: Value,
timeout_duration: Option<Duration>,
cancel_token: Option<CancellationToken>,
) -> Result<T, String> {
if self.failed.load(Ordering::Acquire) {
return Err("Agent runtime is unavailable".to_string());
}
let id = self.next_id.fetch_add(1, Ordering::Relaxed) + 1;
let request = serde_json::json!({ "jsonrpc": "2.0", "id": id, "method": method, "params": params });
let request_line =
serde_json::to_string(&request).map_err(|e| format!("Failed to serialize JSON-RPC request: {e}"))?;
let (sender, receiver) = tokio::sync::oneshot::channel();
self.pending.lock().map_err(|_| "Agent pending response lock poisoned".to_string())?.insert(id, sender);
let write_result =
self.stdin.lock().map_err(|_| "Agent stdin lock poisoned".to_string()).and_then(|mut writer| {
writer
.write_all(request_line.as_bytes())
.and_then(|_| writer.write_all(b"\n"))
.and_then(|_| writer.flush())
.map_err(|e| format!("Failed to write agent request: {e}"))
});
if let Err(err) = write_result {
self.pending.lock().expect("agent pending response lock poisoned").remove(&id);
return Err(err);
}
let receive = async { receiver.await.map_err(|_| "Agent response channel closed".to_string())? };
let response = match (timeout_duration, cancel_token) {
(Some(duration), Some(token)) => tokio::select! {
_ = token.cancelled() => {
self.cancel_session_request(&params).await;
Err("Query canceled".to_string())
},
result = tokio::time::timeout(duration, receive) => result.map_err(|_| format!("Agent RPC call timed out ({}s)", duration.as_secs()))?,
},
(Some(duration), None) => tokio::time::timeout(duration, receive)
.await
.map_err(|_| format!("Agent RPC call timed out ({}s)", duration.as_secs()))?,
(None, Some(token)) => tokio::select! {
_ = token.cancelled() => {
self.cancel_session_request(&params).await;
Err("Query canceled".to_string())
},
result = receive => result,
},
(None, None) => receive.await,
};
if response.is_err() {
self.pending.lock().expect("agent pending response lock poisoned").remove(&id);
}
decode_agent_response(response?)
}
async fn cancel_session_request(&self, params: &Value) {
let Some(agent_session_id) = params.get("agentSessionId").and_then(Value::as_str) else {
return;
};
let _ = Box::pin(self.call::<Value>(
AgentMethod::CancelSession.as_str(),
serde_json::json!({ "agentSessionId": agent_session_id }),
Some(Duration::from_secs(5)),
None,
))
.await;
}
pub fn handshake(&self) -> &AgentHandshake {
&self.handshake
}
pub fn is_failed(&self) -> bool {
self.failed.load(Ordering::Acquire)
}
pub fn active_session_count(&self) -> u64 {
self.active_sessions.load(Ordering::Acquire)
}
pub fn increment_session_count(&self) {
self.active_sessions.fetch_add(1, Ordering::AcqRel);
}
pub fn decrement_session_count(runtime: &Arc<Self>) {
let previous = runtime
.active_sessions
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |value| Some(value.saturating_sub(1)))
.unwrap_or_default();
if previous <= 1 {
let runtime = runtime.clone();
tokio::spawn(async move {
tokio::time::sleep(Duration::from_secs(SHARED_RUNTIME_IDLE_GRACE_SECS)).await;
if runtime.active_session_count() == 0 {
runtime.kill();
}
});
}
}
pub fn kill(&self) {
self.failed.store(true, Ordering::Release);
if let Ok(mut child) = self.child.lock() {
let _ = child.kill();
}
fail_pending_requests(&self.pending, "Agent runtime terminated".to_string());
}
}
fn decode_agent_response<T: DeserializeOwned>(response: Value) -> Result<T, String> {
if let Some(err) = response.get("error") {
let message = err.get("message").and_then(Value::as_str).unwrap_or("Unknown agent error");
let code = err.get("code").and_then(Value::as_i64).unwrap_or(-1);
return Err(format!("Agent RPC error ({code}): {message}"));
}
let result =
response.get("result").ok_or_else(|| "Agent response missing both 'result' and 'error'".to_string())?;
serde_json::from_value(result.clone()).map_err(|e| format!("Failed to deserialize agent result: {e}"))
}
fn fail_pending_requests(pending: &Arc<Mutex<HashMap<u64, PendingAgentResponse>>>, error: String) {
let requests = std::mem::take(&mut *pending.lock().expect("agent pending response lock poisoned"));
for (_, sender) in requests {
let _ = sender.send(Err(error.clone()));
}
}
pub const AGENT_PROTOCOL_VERSION: u32 = 2;
const RPC_TIMEOUT_SECS: u64 = 30;
const STARTUP_TIMEOUT_SECS: u64 = 15;
const STDERR_TAIL_LINES: usize = 20;
const AGENT_EXIT_DIAGNOSTIC_WAIT_MS: u64 = 1_000;
const AGENT_EXIT_DIAGNOSTIC_POLL_MS: u64 = 10;
const SHARED_RUNTIME_IDLE_GRACE_SECS: u64 = 30;
const AGENT_JAVA_OPTS_ENV: &str = "DBX_AGENT_JAVA_OPTS";
const AGENT_JAVA_TOO_OLD_MESSAGE: &str =
"Agent requires Java 21, but DBX started it with an older Java runtime. Use DBX managed JRE 21 or select a Java 21 executable in Driver Manager.";
pub struct AgentDriverClient {
child: Child,
child: Option<Child>,
stdin: Option<BufWriter<ChildStdin>>,
stdout: Option<BufReader<ChildStdout>>,
stderr_tail: Arc<Mutex<StderrTail>>,
handshake: Option<AgentHandshake>,
next_id: u64,
shared_runtime: Option<Arc<AgentRuntimeClient>>,
agent_session_id: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -93,10 +332,11 @@ pub enum AgentCapability {
Transaction,
Ddl,
Kv,
MultiSession,
}
impl AgentCapability {
pub const ALL: [Self; 8] = [
pub const ALL: [Self; 9] = [
Self::Connect,
Self::TestConnection,
Self::Metadata,
@@ -105,6 +345,7 @@ impl AgentCapability {
Self::Transaction,
Self::Ddl,
Self::Kv,
Self::MultiSession,
];
pub fn as_str(self) -> &'static str {
@@ -117,6 +358,7 @@ impl AgentCapability {
Self::Transaction => "transaction",
Self::Ddl => "ddl",
Self::Kv => "kv",
Self::MultiSession => "multi_session",
}
}
}
@@ -125,6 +367,10 @@ impl AgentCapability {
pub enum AgentMethod {
Handshake,
Connect,
OpenSession,
CloseSession,
ValidateSession,
CancelSession,
TestConnection,
ValidateConnection,
ListDatabases,
@@ -154,9 +400,13 @@ pub enum AgentMethod {
}
impl AgentMethod {
pub const ALL: [Self; 28] = [
pub const ALL: [Self; 32] = [
Self::Handshake,
Self::Connect,
Self::OpenSession,
Self::CloseSession,
Self::ValidateSession,
Self::CancelSession,
Self::TestConnection,
Self::ValidateConnection,
Self::ListDatabases,
@@ -189,6 +439,10 @@ impl AgentMethod {
match self {
Self::Handshake => "handshake",
Self::Connect => "connect",
Self::OpenSession => "open_session",
Self::CloseSession => "close_session",
Self::ValidateSession => "validate_session",
Self::CancelSession => "cancel_session",
Self::TestConnection => "test_connection",
Self::ValidateConnection => "validate_connection",
Self::ListDatabases => "list_databases",
@@ -425,7 +679,29 @@ impl AgentDriverClient {
}
};
Ok(Self { child, stdin: Some(stdin), stdout: Some(ready_stdout), stderr_tail, handshake: None, next_id: 0 })
Ok(Self {
child: Some(child),
stdin: Some(stdin),
stdout: Some(ready_stdout),
stderr_tail,
handshake: None,
next_id: 0,
shared_runtime: None,
agent_session_id: None,
})
}
pub fn shared_session(runtime: Arc<AgentRuntimeClient>, agent_session_id: String) -> Self {
Self {
child: None,
stdin: None,
stdout: None,
stderr_tail: runtime.stderr_tail.clone(),
handshake: Some(runtime.handshake().clone()),
next_id: 0,
shared_runtime: Some(runtime),
agent_session_id: Some(agent_session_id),
}
}
/// Send a JSON-RPC 2.0 request and wait for the response.
@@ -458,6 +734,20 @@ impl AgentDriverClient {
timeout_duration: Option<Duration>,
cancel_token: Option<CancellationToken>,
) -> Result<T, String> {
if let Some(runtime) = &self.shared_runtime {
let mut params = params;
if method != AgentMethod::Handshake.as_str()
&& method != AgentMethod::TestConnection.as_str()
&& method != AgentMethod::Shutdown.as_str()
{
let session_id = self.agent_session_id.as_ref().ok_or("Shared Agent session id is missing")?;
params
.as_object_mut()
.ok_or_else(|| "Agent RPC parameters must be an object".to_string())?
.insert("agentSessionId".to_string(), Value::String(session_id.clone()));
}
return runtime.call(method, params, timeout_duration, cancel_token).await;
}
self.next_id += 1;
let id = self.next_id;
@@ -587,6 +877,31 @@ impl AgentDriverClient {
self.call_method(AgentMethod::Connect, params).await
}
pub async fn open_session(&mut self, agent_session_id: &str, mut params: Value) -> Result<Value, String> {
params
.as_object_mut()
.ok_or_else(|| "Agent session parameters must be an object".to_string())?
.insert("agentSessionId".to_string(), Value::String(agent_session_id.to_string()));
self.call_method(AgentMethod::OpenSession, params).await
}
pub async fn close_session(&mut self, agent_session_id: &str) -> Result<Value, String> {
self.call_method(AgentMethod::CloseSession, serde_json::json!({ "agentSessionId": agent_session_id })).await
}
pub async fn validate_session(
&mut self,
agent_session_id: &str,
timeout_duration: Option<Duration>,
) -> Result<Value, String> {
self.call_method_with_timeout(
AgentMethod::ValidateSession,
serde_json::json!({ "agentSessionId": agent_session_id }),
timeout_duration,
)
.await
}
pub async fn test_connection(&mut self, params: Value) -> Result<Value, String> {
self.call_method(AgentMethod::TestConnection, params).await
}
@@ -596,6 +911,18 @@ impl AgentDriverClient {
}
pub async fn disconnect(&mut self) -> Result<Value, String> {
if self.shared_runtime.is_some() {
let session_id = self.agent_session_id.as_ref().ok_or("Shared Agent session id is missing")?.clone();
let result =
self.call_method(AgentMethod::CloseSession, serde_json::json!({ "agentSessionId": session_id })).await;
if result.is_ok() {
if let Some(runtime) = &self.shared_runtime {
AgentRuntimeClient::decrement_session_count(runtime);
}
self.agent_session_id = None;
}
return result;
}
self.call_method(AgentMethod::Disconnect, serde_json::json!({})).await
}
@@ -1105,6 +1432,10 @@ impl AgentDriverClient {
/// Send a shutdown message to the agent and wait for the process to exit.
pub async fn shutdown(&mut self) {
if self.shared_runtime.is_some() {
let _ = self.disconnect().await;
return;
}
// Try to send a shutdown RPC; ignore errors if the agent is already gone
let shutdown_result: Result<Value, String> = self.call_method(AgentMethod::Shutdown, Value::Null).await;
if let Err(e) = &shutdown_result {
@@ -1115,7 +1446,8 @@ impl AgentDriverClient {
self.stdin.take();
// Wait for the child to exit
match self.child.wait() {
let Some(child) = self.child.as_mut() else { return };
match child.wait() {
Ok(status) => log::info!("Agent process exited with {status}"),
Err(e) => log::warn!("Failed to wait for agent process: {e}"),
}
@@ -1123,9 +1455,14 @@ impl AgentDriverClient {
/// Forcefully kill the agent process.
pub fn kill(&mut self) {
if let Some(runtime) = &self.shared_runtime {
runtime.kill();
return;
}
self.stdin.take();
self.stdout.take();
if let Err(e) = self.child.kill() {
let Some(child) = self.child.as_mut() else { return };
if let Err(e) = child.kill() {
log::warn!("Failed to kill agent process: {e}");
}
// Reap the child to avoid zombie processes.
@@ -1133,7 +1470,7 @@ impl AgentDriverClient {
// hanging in Drop during async cleanup. Poll up to 100ms for the
// process to exit after kill().
for _ in 0..10 {
match self.child.try_wait() {
match child.try_wait() {
Ok(Some(_status)) => return,
Ok(None) => std::thread::sleep(Duration::from_millis(10)),
Err(e) => {
@@ -1143,13 +1480,28 @@ impl AgentDriverClient {
}
}
// Final blocking wait as a last resort
if let Err(e) = self.child.wait() {
if let Err(e) = child.wait() {
log::warn!("Final wait failed for agent process: {e}");
}
}
pub fn pid(&self) -> u32 {
self.child.id()
if let Some(runtime) = &self.shared_runtime {
return runtime.child.lock().map(|child| child.id()).unwrap_or_default();
}
self.child.as_ref().map(Child::id).unwrap_or_default()
}
pub fn protocol_mode(&self) -> &'static str {
if self.shared_runtime.is_some() {
"multi_session"
} else {
"legacy"
}
}
pub fn active_session_count(&self) -> u64 {
self.shared_runtime.as_ref().map(|runtime| runtime.active_session_count()).unwrap_or(1)
}
pub fn stderr_tail_snapshot(&self) -> String {
@@ -1438,13 +1790,22 @@ impl AgentDriverClient {
fn format_agent_process_error(&mut self, base: &str) -> String {
// Runtime RPC errors are common SQL/driver paths. Do not wait for the
// child to exit unless startup diagnostics already expect the process to die.
format_agent_process_error(base, child_exit_status(&mut self.child), &stderr_tail_snapshot(&self.stderr_tail))
let exit_status = self.child.as_mut().and_then(child_exit_status);
format_agent_process_error(base, exit_status, &stderr_tail_snapshot(&self.stderr_tail))
}
}
impl Drop for AgentDriverClient {
fn drop(&mut self) {
self.kill();
// Shared-session clients do not own the runtime process. Session closure
// is explicit in pool cleanup; dropping one tab must not kill other tabs.
if let Some(runtime) = &self.shared_runtime {
if self.agent_session_id.take().is_some() {
AgentRuntimeClient::decrement_session_count(runtime);
}
} else {
self.kill();
}
}
}
@@ -1456,12 +1817,16 @@ mod tests {
agent_schema_table_params, agent_supports_capability, agent_transaction_params, format_agent_process_error,
format_agent_startup_error, is_unsupported_handshake_error, mongo_collection_params, mongo_database_params,
mongo_document_id_params, parse_agent_java_opts, read_agent_line, start_stderr_collector, AgentCapability,
AgentDriverClient, AgentHandshake, AgentKvMethod, AgentMethod, AgentTableReadCloseParams,
AgentTableReadPageParams, AgentTableReadStartParams, MongoAgentMethod, StderrTail, AGENT_PROTOCOL_VERSION,
AgentDriverClient, AgentHandshake, AgentKvMethod, AgentLaunchSpec, AgentMethod, AgentRuntimeClient,
AgentTableReadCloseParams, AgentTableReadPageParams, AgentTableReadStartParams, MongoAgentMethod, StderrTail,
AGENT_PROTOCOL_VERSION,
};
use std::io::Cursor;
use std::io::Write;
use std::process::{Command, Stdio};
use std::sync::{Arc, Mutex};
use std::time::Duration;
use tokio_util::sync::CancellationToken;
#[test]
fn agent_java_args_include_oracle_network_compatibility_flags() {
@@ -1638,12 +2003,14 @@ mod tests {
.spawn()
.expect("child should start");
let mut client = AgentDriverClient {
child,
child: Some(child),
stdin: None,
stdout: None,
stderr_tail: Arc::new(Mutex::new(StderrTail::default())),
handshake: None,
next_id: 0,
shared_runtime: None,
agent_session_id: None,
};
let started_at = std::time::Instant::now();
@@ -1654,6 +2021,129 @@ mod tests {
assert!(!message.contains("agent process exited"));
}
#[tokio::test]
async fn multiplexed_runtime_correlates_out_of_order_responses() {
let script_path = std::env::temp_dir().join(format!("dbx-agent-runtime-test-{}.py", uuid::Uuid::new_v4()));
let mut script = std::fs::File::create(&script_path).unwrap();
script
.write_all(
br#"import json, sys, threading, time
print(json.dumps({'ready': True}), flush=True)
def respond(req):
if req['method'] == 'handshake':
result = {'protocolVersion': 2, 'agentProtocolVersion': 2, 'capabilities': ['multi_session']}
else:
time.sleep(0.05 if req['params']['value'] == 1 else 0.0)
result = req['params']['value']
print(json.dumps({'jsonrpc': '2.0', 'id': req['id'], 'result': result}), flush=True)
for line in sys.stdin:
threading.Thread(target=respond, args=(json.loads(line),), daemon=True).start()
"#,
)
.unwrap();
drop(script);
let runtime = AgentRuntimeClient::spawn(
AgentLaunchSpec::new("python3").with_args([script_path.to_string_lossy().to_string()]),
"test",
)
.await
.unwrap();
let first = runtime.call::<u64>("echo", serde_json::json!({"value": 1}), Some(Duration::from_secs(2)), None);
let second = runtime.call::<u64>("echo", serde_json::json!({"value": 2}), Some(Duration::from_secs(2)), None);
let (first, second) = tokio::join!(first, second);
assert_eq!(first.unwrap(), 1);
assert_eq!(second.unwrap(), 2);
runtime.kill();
let _ = std::fs::remove_file(script_path);
}
#[tokio::test]
async fn canceling_one_runtime_request_keeps_other_requests_alive() {
let script_path = std::env::temp_dir().join(format!("dbx-agent-cancel-test-{}.py", uuid::Uuid::new_v4()));
std::fs::write(
&script_path,
r#"import json, sys, threading, time
print(json.dumps({'ready': True}), flush=True)
def respond(req):
if req['method'] == 'handshake':
result = {'protocolVersion': 2, 'agentProtocolVersion': 2, 'capabilities': ['multi_session']}
else:
time.sleep(req['params'].get('delay', 0))
result = req['params'].get('value')
print(json.dumps({'jsonrpc': '2.0', 'id': req['id'], 'result': result}), flush=True)
for line in sys.stdin:
threading.Thread(target=respond, args=(json.loads(line),), daemon=True).start()
"#,
)
.unwrap();
let runtime = AgentRuntimeClient::spawn(
AgentLaunchSpec::new("python3").with_args([script_path.to_string_lossy().to_string()]),
"test",
)
.await
.unwrap();
let token = CancellationToken::new();
let canceled = runtime.call::<u64>(
"echo",
serde_json::json!({"value": 1, "delay": 0.2}),
Some(Duration::from_secs(2)),
Some(token.clone()),
);
let healthy = runtime.call::<u64>(
"echo",
serde_json::json!({"value": 2, "delay": 0}),
Some(Duration::from_secs(2)),
None,
);
token.cancel();
let (canceled, healthy) = tokio::join!(canceled, healthy);
assert_eq!(canceled.unwrap_err(), "Query canceled");
assert_eq!(healthy.unwrap(), 2);
assert_eq!(
runtime
.call::<u64>("echo", serde_json::json!({"value": 3}), Some(Duration::from_secs(2)), None)
.await
.unwrap(),
3
);
runtime.kill();
let _ = std::fs::remove_file(script_path);
}
#[tokio::test]
async fn runtime_eof_fails_pending_requests_and_marks_runtime_failed() {
let script_path = std::env::temp_dir().join(format!("dbx-agent-eof-test-{}.py", uuid::Uuid::new_v4()));
std::fs::write(
&script_path,
r#"import json, sys
print(json.dumps({'ready': True}), flush=True)
for line in sys.stdin:
req = json.loads(line)
if req['method'] == 'handshake':
print(json.dumps({'jsonrpc':'2.0','id':req['id'],'result':{'protocolVersion':2,'agentProtocolVersion':2,'capabilities':['multi_session']}}), flush=True)
else:
sys.exit(0)
"#,
)
.unwrap();
let runtime = AgentRuntimeClient::spawn(
AgentLaunchSpec::new("python3").with_args([script_path.to_string_lossy().to_string()]),
"test",
)
.await
.unwrap();
let error = runtime
.call::<serde_json::Value>("crash", serde_json::json!({}), Some(Duration::from_secs(2)), None)
.await
.unwrap_err();
assert!(error.contains("end of stream") || error.contains("response channel closed"));
assert!(runtime.is_failed());
let _ = std::fs::remove_file(script_path);
}
#[test]
fn stderr_tail_keeps_recent_lines_only() {
let mut stderr_tail = StderrTail::with_capacity(3);
@@ -1697,7 +2187,8 @@ mod tests {
assert_eq!(AgentCapability::Transaction.as_str(), "transaction");
assert_eq!(AgentCapability::Ddl.as_str(), "ddl");
assert_eq!(AgentCapability::Kv.as_str(), "kv");
assert_eq!(AgentCapability::ALL.len(), 8);
assert_eq!(AgentCapability::MultiSession.as_str(), "multi_session");
assert_eq!(AgentCapability::ALL.len(), 9);
}
#[test]
@@ -1915,7 +2406,7 @@ mod tests {
#[test]
fn agent_protocol_matches_contract_file() {
let contract: serde_json::Value =
serde_json::from_str(include_str!("../../assets/agent-protocol-v1.json")).unwrap();
serde_json::from_str(include_str!("../../assets/agent-protocol-v2.json")).unwrap();
assert_eq!(contract["protocolVersion"], AGENT_PROTOCOL_VERSION);
assert_eq!(contract["handshakeMethod"], AgentMethod::Handshake.as_str());
@@ -1991,6 +2482,7 @@ mod tests {
AgentCapability::PagedQuery,
AgentCapability::Transaction,
AgentCapability::Ddl,
AgentCapability::MultiSession,
]
.iter()
.map(|capability| capability.as_str())
+56 -12
View File
@@ -23,6 +23,10 @@ pub struct DriverRuntimeInfo {
pub can_stop: bool,
pub can_restart: bool,
pub control_unavailable_reason: Option<String>,
#[serde(default)]
pub protocol_mode: Option<String>,
#[serde(default)]
pub active_sessions: Option<u64>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
@@ -48,6 +52,8 @@ struct RuntimeSeed {
can_stop: bool,
can_restart: bool,
control_unavailable_reason: Option<String>,
protocol_mode: Option<String>,
active_sessions: Option<u64>,
}
#[derive(Debug, Clone, Default)]
@@ -65,16 +71,27 @@ pub async fn collect_driver_runtime_summary(state: &AppState) -> DriverRuntimeSu
}
pub async fn stop_driver_runtime(state: &AppState, runtime_id: &str) -> Result<(), String> {
let agent_key =
runtime_id.strip_prefix("agent:").ok_or_else(|| "Only agent daemon runtimes can be stopped".to_string())?;
let agent_key = runtime_agent_key(runtime_id).ok_or_else(|| "Only agent runtimes can be stopped".to_string())?;
state.agent_manager.stop_daemon_by_key(agent_key).await;
Ok(())
}
pub async fn restart_driver_runtime(state: &AppState, runtime_id: &str) -> Result<(), String> {
let agent_key =
runtime_id.strip_prefix("agent:").ok_or_else(|| "Only agent daemon runtimes can be restarted".to_string())?;
state.agent_manager.restart_daemon_by_key(agent_key).await
let agent_key = runtime_agent_key(runtime_id).ok_or_else(|| "Only agent runtimes can be restarted".to_string())?;
if runtime_id.starts_with("agent-runtime:") {
// Shared connection runtimes are recreated lazily with their exact launch
// fingerprint; eagerly spawning the generic daemon could use incompatible options.
state.agent_manager.stop_daemon_by_key(agent_key).await;
Ok(())
} else {
state.agent_manager.restart_daemon_by_key(agent_key).await
}
}
fn runtime_agent_key(runtime_id: &str) -> Option<&str> {
runtime_id.strip_prefix("agent:").or_else(|| {
runtime_id.strip_prefix("agent-runtime:").and_then(|value| value.rsplit_once(':').map(|(key, _)| key))
})
}
async fn collect_runtime_seeds(state: &AppState) -> Vec<RuntimeSeed> {
@@ -97,6 +114,8 @@ async fn collect_runtime_seeds(state: &AppState) -> Vec<RuntimeSeed> {
can_stop: false,
can_restart: state.agent_manager.is_driver_installed(key),
control_unavailable_reason: None,
protocol_mode: None,
active_sessions: None,
},
);
}
@@ -120,6 +139,8 @@ async fn collect_runtime_seeds(state: &AppState) -> Vec<RuntimeSeed> {
can_stop: true,
can_restart: true,
control_unavailable_reason: None,
protocol_mode: Some("legacy".to_string()),
active_sessions: None,
},
);
}
@@ -140,21 +161,31 @@ async fn collect_runtime_seeds(state: &AppState) -> Vec<RuntimeSeed> {
.unwrap_or("agent");
let client = client.lock().await;
let last_error = non_empty(client.stderr_tail_snapshot());
let protocol_mode = client.protocol_mode();
let runtime_id = if protocol_mode == "multi_session" {
format!("agent-runtime:{key}:{}", client.pid())
} else {
format!("agent-connection:{pool_key}")
};
seeds.insert(
format!("agent-connection:{pool_key}"),
runtime_id.clone(),
RuntimeSeed {
id: format!("agent-connection:{pool_key}"),
id: runtime_id,
driver_key: key.to_string(),
label: agent_catalog::label_for_key(key).unwrap_or(key).to_string(),
kind: "agent".to_string(),
source: "connection".to_string(),
source: if protocol_mode == "multi_session" { "shared-runtime" } else { "connection" }
.to_string(),
status: "running".to_string(),
pid: Some(client.pid()),
version: agent_state.installed_drivers.get(key).map(|driver| driver.version.clone()),
last_error,
can_stop: false,
can_restart: false,
control_unavailable_reason: Some("connection-owned".to_string()),
can_stop: protocol_mode == "multi_session",
can_restart: protocol_mode == "multi_session",
control_unavailable_reason: (protocol_mode != "multi_session")
.then(|| "connection-owned".to_string()),
protocol_mode: Some(protocol_mode.to_string()),
active_sessions: Some(client.active_session_count()),
},
);
}
@@ -174,6 +205,8 @@ async fn collect_runtime_seeds(state: &AppState) -> Vec<RuntimeSeed> {
can_stop: false,
can_restart: false,
control_unavailable_reason: Some("connection-owned".to_string()),
protocol_mode: None,
active_sessions: None,
},
);
}
@@ -248,6 +281,8 @@ fn build_summary(seeds: Vec<RuntimeSeed>, stats: &HashMap<u32, ProcessStats>) ->
can_stop: seed.can_stop,
can_restart: seed.can_restart,
control_unavailable_reason: seed.control_unavailable_reason,
protocol_mode: seed.protocol_mode,
active_sessions: seed.active_sessions,
}
})
.collect::<Vec<_>>();
@@ -274,7 +309,7 @@ fn non_empty(value: String) -> Option<String> {
#[cfg(test)]
mod tests {
use super::{build_summary, ProcessStats, RuntimeSeed};
use super::{build_summary, runtime_agent_key, ProcessStats, RuntimeSeed};
use std::collections::HashMap;
fn seed(id: &str, status: &str, pid: Option<u32>) -> RuntimeSeed {
@@ -291,6 +326,8 @@ mod tests {
can_stop: true,
can_restart: true,
control_unavailable_reason: None,
protocol_mode: None,
active_sessions: None,
}
}
@@ -306,4 +343,11 @@ mod tests {
assert_eq!(summary.health, "healthy");
assert_eq!(summary.runtimes[0].memory_bytes, Some(1024));
}
#[test]
fn parses_legacy_and_shared_agent_runtime_ids() {
assert_eq!(runtime_agent_key("agent:oracle"), Some("oracle"));
assert_eq!(runtime_agent_key("agent-runtime:oracle:123"), Some("oracle"));
assert_eq!(runtime_agent_key("plugin-connection:test"), None);
}
}