fix(agent): allow driver upgrades after closing connections

This commit is contained in:
zipg authored and GitHub committed 2026-07-16 17:31:05 +08:00
1 parent c4f4ce2460
commit 0f335119e3
5 files changed
+164 -16

No files matched your search

+28 -7
View File
@@ -18,18 +18,23 @@ pub async fn stop_daemons(manager: &AgentManager) {
manager.daemons.lock().await.clear();
let runtimes = std::mem::take(&mut *manager.connection_runtimes.lock().await);
for runtime in runtimes.into_values().filter_map(|cell| cell.get().cloned()) {
runtime.kill();
runtime.kill_and_wait().await;
}
}
pub async fn stop_daemon_by_key(manager: &AgentManager, agent_key: &str) {
manager.daemons.lock().await.remove(agent_key);
let mut runtimes = manager.connection_runtimes.lock().await;
let matching = runtimes.keys().filter(|key| key.starts_with(&format!("{agent_key}|"))).cloned().collect::<Vec<_>>();
for key in matching {
if let Some(runtime) = runtimes.remove(&key).and_then(|cell| cell.get().cloned()) {
runtime.kill();
}
let runtimes = {
let mut runtimes = manager.connection_runtimes.lock().await;
let matching =
runtimes.keys().filter(|key| key.starts_with(&format!("{agent_key}|"))).cloned().collect::<Vec<_>>();
matching
.into_iter()
.filter_map(|key| runtimes.remove(&key).and_then(|cell| cell.get().cloned()))
.collect::<Vec<_>>()
};
for runtime in runtimes {
runtime.kill_and_wait().await;
}
}
@@ -400,4 +405,20 @@ for line in sys.stdin:
runtime.kill();
let _ = std::fs::remove_file(script_path);
}
#[tokio::test]
async fn stopping_driver_runtime_removes_and_terminates_shared_process() {
let (manager, cell, runtime, script_path) = test_shared_runtime("stop-driver-runtime").await;
let runtime_key = "dameng|test";
manager.connection_runtimes.lock().await.insert(runtime_key.to_string(), cell);
stop_daemon_by_key(&manager, "dameng").await;
assert!(!manager.connection_runtimes.lock().await.contains_key(runtime_key));
assert!(runtime.is_failed());
let call_result =
runtime.call::<serde_json::Value>("ping", serde_json::json!({}), Some(Duration::from_secs(1)), None).await;
assert_eq!(call_result.unwrap_err(), "Agent runtime is unavailable");
let _ = std::fs::remove_file(script_path);
}
}
+96 -6
View File
@@ -2326,7 +2326,7 @@ impl AppState {
Ok(closed)
}
pub async fn active_agent_driver_keys(&self) -> HashSet<String> {
pub async fn active_agent_connection_driver_keys(&self) -> HashSet<String> {
let configs = self.configs.read().await;
let connections = self.connections.read().await;
let mut keys = HashSet::new();
@@ -2346,14 +2346,31 @@ impl AppState {
}
}
drop(connections);
drop(configs);
keys
}
for key in self.agent_manager.active_daemon_keys().await {
keys.insert(key);
pub async fn prepare_agent_driver_updates(&self, driver_keys: &[String]) -> HashSet<String> {
let candidates = driver_keys.iter().cloned().collect::<HashSet<_>>();
if candidates.is_empty() {
return HashSet::new();
}
keys
let blockers = self
.active_agent_connection_driver_keys()
.await
.into_iter()
.filter(|key| candidates.contains(key))
.collect::<HashSet<_>>();
if !blockers.is_empty() {
return blockers;
}
for key in &candidates {
self.agent_manager.stop_daemon_by_key(key).await;
}
// A connection may have started while idle runtimes were stopping.
self.active_agent_connection_driver_keys().await.into_iter().filter(|key| candidates.contains(key)).collect()
}
pub async fn connection_identifier_quote(
@@ -3848,6 +3865,79 @@ mod tests {
(AppState::new(storage), dir)
}
fn agent_pool_stub() -> PoolKind {
PoolKind::Agent(std::sync::Arc::new(tokio::sync::Mutex::new(
crate::db::agent_driver::AgentDriverClient::test_stub(),
)))
}
#[tokio::test]
async fn agent_update_blockers_only_include_open_connections() {
let (state, dir) = test_app_state().await;
let mut config = mysql_config(None);
config.id = "dameng-conn".to_string();
config.db_type = DatabaseType::Dameng;
state.configs.write().await.insert(config.id.clone(), config);
state
.agent_manager
.daemons
.lock()
.await
.insert("oracle".to_string(), crate::db::agent_driver::AgentDriverClient::test_stub());
state.connections.write().await.insert("dameng-conn".to_string(), agent_pool_stub());
assert_eq!(
state.active_agent_connection_driver_keys().await,
std::collections::HashSet::from(["dameng".to_string()])
);
state.connections.write().await.remove("dameng-conn");
assert!(state.active_agent_connection_driver_keys().await.is_empty());
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test]
async fn preparing_agent_update_stops_idle_runtime() {
let (state, dir) = test_app_state().await;
state
.agent_manager
.daemons
.lock()
.await
.insert("dameng".to_string(), crate::db::agent_driver::AgentDriverClient::test_stub());
let blockers = state.prepare_agent_driver_updates(&["dameng".to_string()]).await;
assert!(blockers.is_empty());
assert!(state.agent_manager.active_daemon_keys().await.is_empty());
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test]
async fn preparing_agent_update_keeps_runtime_when_connection_is_open() {
let (state, dir) = test_app_state().await;
let mut config = mysql_config(None);
config.id = "dameng-conn".to_string();
config.db_type = DatabaseType::Dameng;
state.configs.write().await.insert(config.id.clone(), config);
state.connections.write().await.insert("dameng-conn".to_string(), agent_pool_stub());
state
.agent_manager
.daemons
.lock()
.await
.insert("dameng".to_string(), crate::db::agent_driver::AgentDriverClient::test_stub());
let blockers = state.prepare_agent_driver_updates(&["dameng".to_string()]).await;
assert_eq!(blockers, std::collections::HashSet::from(["dameng".to_string()]));
assert_eq!(state.agent_manager.active_daemon_keys().await, vec!["dameng".to_string()]);
let _ = std::fs::remove_dir_all(dir);
}
fn touch_executable(path: &std::path::Path) {
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent).unwrap();
+31
View File
@@ -230,6 +230,23 @@ impl AgentRuntimeClient {
}
fail_pending_requests(&self.pending, "Agent runtime terminated".to_string());
}
pub async fn kill_and_wait(&self) {
self.failed.store(true, Ordering::Release);
fail_pending_requests(&self.pending, "Agent runtime terminated".to_string());
let child = self.child.clone();
match tokio::task::spawn_blocking(move || {
let mut child = child.lock().map_err(|_| "Shared agent process lock poisoned".to_string())?;
let _ = child.kill();
child.wait().map(|_| ()).map_err(|err| format!("Failed to wait for shared agent runtime: {err}"))
})
.await
{
Ok(Ok(())) => {}
Ok(Err(err)) => log::warn!("{err}"),
Err(err) => log::warn!("Failed to join shared agent shutdown task: {err}"),
}
}
}
fn decode_agent_response<T: DeserializeOwned>(response: Value) -> Result<T, String> {
@@ -1807,6 +1824,20 @@ fn format_agent_startup_error(base: &str, child: &mut Child, stderr_tail: &Arc<M
}
impl AgentDriverClient {
#[cfg(test)]
pub(crate) fn test_stub() -> Self {
Self {
child: None,
stdin: None,
stdout: None,
stderr_tail: Arc::new(Mutex::new(StderrTail::default())),
handshake: None,
next_id: 0,
shared_runtime: None,
agent_session_id: None,
}
}
fn format_agent_process_error(&mut self, base: &str) -> String {
// Runtime RPC errors are common SQL/driver paths. Do not wait for the
// child to exit unless startup diagnostics already expect the process to die.