fix(agent): allow driver upgrades after closing connections
This commit is contained in:
5 files changed
+164
-16
No files matched your search
@@ -18,18 +18,23 @@ pub async fn stop_daemons(manager: &AgentManager) {
|
||||
manager.daemons.lock().await.clear();
|
||||
let runtimes = std::mem::take(&mut *manager.connection_runtimes.lock().await);
|
||||
for runtime in runtimes.into_values().filter_map(|cell| cell.get().cloned()) {
|
||||
runtime.kill();
|
||||
runtime.kill_and_wait().await;
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn stop_daemon_by_key(manager: &AgentManager, agent_key: &str) {
|
||||
manager.daemons.lock().await.remove(agent_key);
|
||||
let mut runtimes = manager.connection_runtimes.lock().await;
|
||||
let matching = runtimes.keys().filter(|key| key.starts_with(&format!("{agent_key}|"))).cloned().collect::<Vec<_>>();
|
||||
for key in matching {
|
||||
if let Some(runtime) = runtimes.remove(&key).and_then(|cell| cell.get().cloned()) {
|
||||
runtime.kill();
|
||||
}
|
||||
let runtimes = {
|
||||
let mut runtimes = manager.connection_runtimes.lock().await;
|
||||
let matching =
|
||||
runtimes.keys().filter(|key| key.starts_with(&format!("{agent_key}|"))).cloned().collect::<Vec<_>>();
|
||||
matching
|
||||
.into_iter()
|
||||
.filter_map(|key| runtimes.remove(&key).and_then(|cell| cell.get().cloned()))
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
for runtime in runtimes {
|
||||
runtime.kill_and_wait().await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -400,4 +405,20 @@ for line in sys.stdin:
|
||||
runtime.kill();
|
||||
let _ = std::fs::remove_file(script_path);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stopping_driver_runtime_removes_and_terminates_shared_process() {
|
||||
let (manager, cell, runtime, script_path) = test_shared_runtime("stop-driver-runtime").await;
|
||||
let runtime_key = "dameng|test";
|
||||
manager.connection_runtimes.lock().await.insert(runtime_key.to_string(), cell);
|
||||
|
||||
stop_daemon_by_key(&manager, "dameng").await;
|
||||
|
||||
assert!(!manager.connection_runtimes.lock().await.contains_key(runtime_key));
|
||||
assert!(runtime.is_failed());
|
||||
let call_result =
|
||||
runtime.call::<serde_json::Value>("ping", serde_json::json!({}), Some(Duration::from_secs(1)), None).await;
|
||||
assert_eq!(call_result.unwrap_err(), "Agent runtime is unavailable");
|
||||
let _ = std::fs::remove_file(script_path);
|
||||
}
|
||||
}
|
||||
@@ -2326,7 +2326,7 @@ impl AppState {
|
||||
Ok(closed)
|
||||
}
|
||||
|
||||
pub async fn active_agent_driver_keys(&self) -> HashSet<String> {
|
||||
pub async fn active_agent_connection_driver_keys(&self) -> HashSet<String> {
|
||||
let configs = self.configs.read().await;
|
||||
let connections = self.connections.read().await;
|
||||
let mut keys = HashSet::new();
|
||||
@@ -2346,14 +2346,31 @@ impl AppState {
|
||||
}
|
||||
}
|
||||
|
||||
drop(connections);
|
||||
drop(configs);
|
||||
keys
|
||||
}
|
||||
|
||||
for key in self.agent_manager.active_daemon_keys().await {
|
||||
keys.insert(key);
|
||||
pub async fn prepare_agent_driver_updates(&self, driver_keys: &[String]) -> HashSet<String> {
|
||||
let candidates = driver_keys.iter().cloned().collect::<HashSet<_>>();
|
||||
if candidates.is_empty() {
|
||||
return HashSet::new();
|
||||
}
|
||||
|
||||
keys
|
||||
let blockers = self
|
||||
.active_agent_connection_driver_keys()
|
||||
.await
|
||||
.into_iter()
|
||||
.filter(|key| candidates.contains(key))
|
||||
.collect::<HashSet<_>>();
|
||||
if !blockers.is_empty() {
|
||||
return blockers;
|
||||
}
|
||||
|
||||
for key in &candidates {
|
||||
self.agent_manager.stop_daemon_by_key(key).await;
|
||||
}
|
||||
|
||||
// A connection may have started while idle runtimes were stopping.
|
||||
self.active_agent_connection_driver_keys().await.into_iter().filter(|key| candidates.contains(key)).collect()
|
||||
}
|
||||
|
||||
pub async fn connection_identifier_quote(
|
||||
@@ -3848,6 +3865,79 @@ mod tests {
|
||||
(AppState::new(storage), dir)
|
||||
}
|
||||
|
||||
fn agent_pool_stub() -> PoolKind {
|
||||
PoolKind::Agent(std::sync::Arc::new(tokio::sync::Mutex::new(
|
||||
crate::db::agent_driver::AgentDriverClient::test_stub(),
|
||||
)))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn agent_update_blockers_only_include_open_connections() {
|
||||
let (state, dir) = test_app_state().await;
|
||||
let mut config = mysql_config(None);
|
||||
config.id = "dameng-conn".to_string();
|
||||
config.db_type = DatabaseType::Dameng;
|
||||
state.configs.write().await.insert(config.id.clone(), config);
|
||||
state
|
||||
.agent_manager
|
||||
.daemons
|
||||
.lock()
|
||||
.await
|
||||
.insert("oracle".to_string(), crate::db::agent_driver::AgentDriverClient::test_stub());
|
||||
state.connections.write().await.insert("dameng-conn".to_string(), agent_pool_stub());
|
||||
|
||||
assert_eq!(
|
||||
state.active_agent_connection_driver_keys().await,
|
||||
std::collections::HashSet::from(["dameng".to_string()])
|
||||
);
|
||||
|
||||
state.connections.write().await.remove("dameng-conn");
|
||||
assert!(state.active_agent_connection_driver_keys().await.is_empty());
|
||||
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn preparing_agent_update_stops_idle_runtime() {
|
||||
let (state, dir) = test_app_state().await;
|
||||
state
|
||||
.agent_manager
|
||||
.daemons
|
||||
.lock()
|
||||
.await
|
||||
.insert("dameng".to_string(), crate::db::agent_driver::AgentDriverClient::test_stub());
|
||||
|
||||
let blockers = state.prepare_agent_driver_updates(&["dameng".to_string()]).await;
|
||||
|
||||
assert!(blockers.is_empty());
|
||||
assert!(state.agent_manager.active_daemon_keys().await.is_empty());
|
||||
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn preparing_agent_update_keeps_runtime_when_connection_is_open() {
|
||||
let (state, dir) = test_app_state().await;
|
||||
let mut config = mysql_config(None);
|
||||
config.id = "dameng-conn".to_string();
|
||||
config.db_type = DatabaseType::Dameng;
|
||||
state.configs.write().await.insert(config.id.clone(), config);
|
||||
state.connections.write().await.insert("dameng-conn".to_string(), agent_pool_stub());
|
||||
state
|
||||
.agent_manager
|
||||
.daemons
|
||||
.lock()
|
||||
.await
|
||||
.insert("dameng".to_string(), crate::db::agent_driver::AgentDriverClient::test_stub());
|
||||
|
||||
let blockers = state.prepare_agent_driver_updates(&["dameng".to_string()]).await;
|
||||
|
||||
assert_eq!(blockers, std::collections::HashSet::from(["dameng".to_string()]));
|
||||
assert_eq!(state.agent_manager.active_daemon_keys().await, vec!["dameng".to_string()]);
|
||||
|
||||
let _ = std::fs::remove_dir_all(dir);
|
||||
}
|
||||
|
||||
fn touch_executable(path: &std::path::Path) {
|
||||
if let Some(parent) = path.parent() {
|
||||
std::fs::create_dir_all(parent).unwrap();
|
||||
|
||||
@@ -230,6 +230,23 @@ impl AgentRuntimeClient {
|
||||
}
|
||||
fail_pending_requests(&self.pending, "Agent runtime terminated".to_string());
|
||||
}
|
||||
|
||||
pub async fn kill_and_wait(&self) {
|
||||
self.failed.store(true, Ordering::Release);
|
||||
fail_pending_requests(&self.pending, "Agent runtime terminated".to_string());
|
||||
let child = self.child.clone();
|
||||
match tokio::task::spawn_blocking(move || {
|
||||
let mut child = child.lock().map_err(|_| "Shared agent process lock poisoned".to_string())?;
|
||||
let _ = child.kill();
|
||||
child.wait().map(|_| ()).map_err(|err| format!("Failed to wait for shared agent runtime: {err}"))
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(err)) => log::warn!("{err}"),
|
||||
Err(err) => log::warn!("Failed to join shared agent shutdown task: {err}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_agent_response<T: DeserializeOwned>(response: Value) -> Result<T, String> {
|
||||
@@ -1807,6 +1824,20 @@ fn format_agent_startup_error(base: &str, child: &mut Child, stderr_tail: &Arc<M
|
||||
}
|
||||
|
||||
impl AgentDriverClient {
|
||||
#[cfg(test)]
|
||||
pub(crate) fn test_stub() -> Self {
|
||||
Self {
|
||||
child: None,
|
||||
stdin: None,
|
||||
stdout: None,
|
||||
stderr_tail: Arc::new(Mutex::new(StderrTail::default())),
|
||||
handshake: None,
|
||||
next_id: 0,
|
||||
shared_runtime: None,
|
||||
agent_session_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn format_agent_process_error(&mut self, base: &str) -> String {
|
||||
// Runtime RPC errors are common SQL/driver paths. Do not wait for the
|
||||
// child to exit unless startup diagnostics already expect the process to die.
|
||||
|
||||
Reference in new issue
Block a user