fix(agent): recover JDBC connections after query timeouts

This commit is contained in:
miracle authored and GitHub committed 2026-08-01 01:17:49 +08:00
1 parent 0d484f5c74
commit 0e414e638e
24 files changed
+5983 -473

No files matched your search

+223 -13
View File
@@ -285,15 +285,78 @@ impl AgentRuntimeClient {
fn decode_agent_response<T: DeserializeOwned>(response: Value) -> Result<T, String> {
if let Some(err) = response.get("error") {
let message = err.get("message").and_then(Value::as_str).unwrap_or("Unknown agent error");
let code = err.get("code").and_then(Value::as_i64).unwrap_or(-1);
return Err(format!("Agent RPC error ({code}): {message}"));
return Err(format_agent_rpc_error(err));
}
let result =
response.get("result").ok_or_else(|| "Agent response missing both 'result' and 'error'".to_string())?;
serde_json::from_value(result.clone()).map_err(|e| format!("Failed to deserialize agent result: {e}"))
}
const AGENT_RPC_ERROR_DATA_MARKER: &str = "\nDBX_AGENT_ERROR_DATA:";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AgentSessionDisposition {
Keep,
Quarantine,
ReplaceRuntime,
}
pub fn agent_rpc_error_category(error: &str) -> Option<String> {
agent_rpc_error_data(error)?.get("category")?.as_str().map(str::to_string)
}
pub fn agent_rpc_error_session_id(error: &str) -> Option<String> {
agent_rpc_error_data(error)?.get("agentSessionId")?.as_str().map(str::to_string)
}
pub fn agent_session_disposition(error: &str) -> Option<AgentSessionDisposition> {
match agent_rpc_error_data(error)?.get("sessionDisposition")?.as_str()? {
"keep" => Some(AgentSessionDisposition::Keep),
"quarantine" => Some(AgentSessionDisposition::Quarantine),
"replace_runtime" => Some(AgentSessionDisposition::ReplaceRuntime),
_ => None,
}
}
fn format_agent_rpc_error(error: &Value) -> String {
let message = error.get("message").and_then(Value::as_str).unwrap_or("Unknown agent error");
let code = error.get("code").and_then(Value::as_i64).unwrap_or(-1);
let mut formatted = format!("Agent RPC error ({code}): {message}");
if let Some(data) = error.get("data").filter(|data| data.is_object()) {
formatted.push_str(AGENT_RPC_ERROR_DATA_MARKER);
formatted.push_str(&data.to_string());
}
formatted
}
fn agent_rpc_error_data(error: &str) -> Option<Value> {
let (_, data) = error.rsplit_once(AGENT_RPC_ERROR_DATA_MARKER)?;
serde_json::from_str(data).ok()
}
fn with_agent_rpc_error_session_id(error: String, agent_session_id: Option<&str>) -> String {
let Some(agent_session_id) = agent_session_id else {
return error;
};
let Some((message, data)) = error.rsplit_once(AGENT_RPC_ERROR_DATA_MARKER) else {
return format!(
"{error}{AGENT_RPC_ERROR_DATA_MARKER}{}",
serde_json::json!({ "agentSessionId": agent_session_id })
);
};
let Ok(mut data) = serde_json::from_str::<Value>(data) else {
return format!(
"{error}{AGENT_RPC_ERROR_DATA_MARKER}{}",
serde_json::json!({ "agentSessionId": agent_session_id })
);
};
let Some(data) = data.as_object_mut() else {
return error;
};
data.insert("agentSessionId".to_string(), Value::String(agent_session_id.to_string()));
format!("{message}{AGENT_RPC_ERROR_DATA_MARKER}{}", Value::Object(data.clone()))
}
fn deserialize_cached_agent_result<T: DeserializeOwned>(result: Result<Value, String>) -> Result<T, String> {
result
.and_then(|value| serde_json::from_value(value).map_err(|e| format!("Failed to deserialize agent result: {e}")))
@@ -329,6 +392,66 @@ pub struct AgentDriverClient {
cached_query: Option<CachedAgentQuery>,
}
/// Keeps serialized session RPC access separate from the process-level fail-stop handle.
/// A stuck RPC may hold `client` indefinitely, but it must never prevent terminating the
/// shared Agent runtime after the pool has been removed from routing.
pub struct PooledAgentClient {
client: tokio::sync::Mutex<AgentDriverClient>,
shared_runtime: Option<Arc<AgentRuntimeClient>>,
agent_session_id: Option<String>,
}
impl PooledAgentClient {
pub fn new(client: AgentDriverClient) -> Self {
let shared_runtime = client.shared_runtime.clone();
let agent_session_id = client.agent_session_id.clone();
Self { client: tokio::sync::Mutex::new(client), shared_runtime, agent_session_id }
}
pub async fn lock(&self) -> tokio::sync::MutexGuard<'_, AgentDriverClient> {
self.client.lock().await
}
pub fn try_lock(&self) -> Result<tokio::sync::MutexGuard<'_, AgentDriverClient>, tokio::sync::TryLockError> {
self.client.try_lock()
}
pub fn shares_runtime_with(&self, other: &Self) -> bool {
match (&self.shared_runtime, &other.shared_runtime) {
(Some(runtime), Some(other_runtime)) => Arc::ptr_eq(runtime, other_runtime),
_ => false,
}
}
pub fn uses_runtime(&self, runtime: &Arc<AgentRuntimeClient>) -> bool {
self.shared_runtime.as_ref().is_some_and(|current| Arc::ptr_eq(current, runtime))
}
pub fn matches_session_id(&self, session_id: &str) -> bool {
self.agent_session_id.as_deref() == Some(session_id)
}
pub fn is_runtime_available(&self) -> bool {
self.shared_runtime.as_ref().is_none_or(|runtime| !runtime.is_failed())
}
/// Terminates a protocol-v2 shared runtime without waiting for the logical session lock.
/// Legacy single-session clients can only be killed immediately when they are not busy.
pub fn fail_stop(&self) -> bool {
if let Some(runtime) = &self.shared_runtime {
runtime.kill();
return true;
}
match self.client.try_lock() {
Ok(mut client) => {
client.kill();
true
}
Err(_) => false,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AgentLaunchSpec {
pub program: PathBuf,
@@ -936,18 +1059,22 @@ impl AgentDriverClient {
cancel_token: Option<CancellationToken>,
) -> Result<T, String> {
if let Some(runtime) = &self.shared_runtime {
let agent_session_id = self.agent_session_id.clone();
let mut params = params;
if method != AgentMethod::Handshake.as_str()
&& method != AgentMethod::TestConnection.as_str()
&& method != AgentMethod::Shutdown.as_str()
{
let session_id = self.agent_session_id.as_ref().ok_or("Shared Agent session id is missing")?;
let session_id = agent_session_id.as_ref().ok_or("Shared Agent session id is missing")?;
params
.as_object_mut()
.ok_or_else(|| "Agent RPC parameters must be an object".to_string())?
.insert("agentSessionId".to_string(), Value::String(session_id.clone()));
}
return runtime.call(method, params, timeout_duration, cancel_token).await;
return runtime
.call(method, params, timeout_duration, cancel_token)
.await
.map_err(|error| with_agent_rpc_error_session_id(error, agent_session_id.as_deref()));
}
self.next_id += 1;
let id = self.next_id;
@@ -993,9 +1120,7 @@ impl AgentDriverClient {
};
let result = if let Some(err) = resp.get("error") {
let msg = err.get("message").and_then(|m| m.as_str()).unwrap_or("Unknown agent error");
let code = err.get("code").and_then(|c| c.as_i64()).unwrap_or(-1);
Err(format!("Agent RPC error ({code}): {msg}"))
Err(format_agent_rpc_error(err))
} else if let Some(result_val) = resp.get("result") {
serde_json::from_value::<T>(result_val.clone())
.map_err(|e| format!("Failed to deserialize agent result: {e}"))
@@ -2178,14 +2303,15 @@ impl Drop for AgentDriverClient {
mod tests {
use super::{
agent_close_query_session_params, agent_handshake_params, agent_java_args, agent_java_args_with_extra,
agent_java_args_with_extra_opts, agent_object_source_params, agent_proxy_env_vars, agent_schema_params,
agent_schema_table_params, agent_supports_capability, agent_transaction_params, format_agent_process_error,
agent_java_args_with_extra_opts, agent_object_source_params, agent_proxy_env_vars, agent_rpc_error_category,
agent_rpc_error_session_id, agent_schema_params, agent_schema_table_params, agent_session_disposition,
agent_supports_capability, agent_transaction_params, decode_agent_response, format_agent_process_error,
format_agent_startup_error, is_agent_rpc_response_error, is_unsupported_handshake_error,
mongo_collection_params, mongo_database_params, mongo_document_id_params, parse_agent_java_opts,
read_agent_line, start_stderr_collector, validate_dameng_java_system_properties, AgentCapability,
AgentDriverClient, AgentHandshake, AgentKvMethod, AgentLaunchSpec, AgentMethod, AgentRuntimeClient,
AgentTableReadCloseParams, AgentTableReadPageParams, AgentTableReadStartParams, MongoAgentMethod, StderrTail,
AGENT_PROTOCOL_VERSION,
AgentSessionDisposition, AgentTableReadCloseParams, AgentTableReadPageParams, AgentTableReadStartParams,
MongoAgentMethod, StderrTail, AGENT_PROTOCOL_VERSION,
};
use std::io::Cursor;
use std::io::Write;
@@ -2194,6 +2320,30 @@ mod tests {
use std::time::{Duration, Instant};
use tokio_util::sync::CancellationToken;
#[test]
fn structured_agent_error_data_survives_legacy_string_boundary() {
let response = serde_json::json!({
"error": {
"code": -1,
"message": "connection lost",
"data": {
"category": "connection",
"retryable": true,
"sessionDisposition": "quarantine",
"agentSessionId": "session-generation-1",
"stage": "execute"
}
}
});
let error = decode_agent_response::<serde_json::Value>(response).unwrap_err();
assert!(error.starts_with("Agent RPC error (-1): connection lost"));
assert_eq!(agent_rpc_error_category(&error).as_deref(), Some("connection"));
assert_eq!(agent_rpc_error_session_id(&error).as_deref(), Some("session-generation-1"));
assert_eq!(agent_session_disposition(&error), Some(AgentSessionDisposition::Quarantine));
}
#[test]
fn agent_java_args_include_oracle_network_compatibility_flags() {
let args = agent_java_args("/tmp/dbx-agent-oracle.jar");
@@ -2494,8 +2644,9 @@ for line in sys.stdin:
"#,
)
.unwrap();
let python = if cfg!(windows) { "python" } else { "python3" };
let runtime = AgentRuntimeClient::spawn(
AgentLaunchSpec::new("python3").with_args([script_path.to_string_lossy().to_string()]),
AgentLaunchSpec::new(python).with_args([script_path.to_string_lossy().to_string()]),
"test",
)
.await
@@ -2520,6 +2671,7 @@ for line in sys.stdin:
.await
.unwrap_err();
assert!(error.contains("Agent RPC call timed out"));
assert_eq!(agent_rpc_error_session_id(&error).as_deref(), Some("timeout-session"));
let started = Instant::now();
client
.call_with_timeout::<serde_json::Value>("probe", serde_json::json!({}), Some(Duration::from_millis(500)))
@@ -2536,6 +2688,7 @@ for line in sys.stdin:
.await
.unwrap_err();
assert!(error.contains("Agent RPC call timed out"));
assert_eq!(agent_rpc_error_session_id(&error).as_deref(), Some("timeout-session"));
let started = Instant::now();
client
.call_with_timeout::<serde_json::Value>("probe", serde_json::json!({}), Some(Duration::from_millis(500)))
@@ -2776,6 +2929,63 @@ for line in sys.stdin:
let _ = std::fs::remove_file(script_path);
}
#[tokio::test]
async fn disconnect_reports_runtime_replacement_without_owning_runtime_routing() {
let script_path =
std::env::temp_dir().join(format!("dbx-agent-runtime-cleanup-saturation-{}.py", uuid::Uuid::new_v4()));
std::fs::write(
&script_path,
r#"import json, sys
print(json.dumps({'ready': True}), flush=True)
for line in sys.stdin:
req = json.loads(line)
if req['method'] == 'handshake':
response = {
'jsonrpc': '2.0',
'id': req['id'],
'result': {'protocolVersion': 2, 'agentProtocolVersion': 2, 'capabilities': ['multi_session']}
}
elif req['method'] == 'close_session':
response = {
'jsonrpc': '2.0',
'id': req['id'],
'error': {
'code': -1,
'message': 'Agent runtime resource limit reached',
'data': {
'category': 'resource',
'retryable': False,
'sessionDisposition': 'replace_runtime',
'stage': 'close'
}
}
}
else:
response = {'jsonrpc': '2.0', 'id': req['id'], 'result': {}}
print(json.dumps(response), flush=True)
"#,
)
.unwrap();
let python = if cfg!(windows) { "python" } else { "python3" };
let runtime = AgentRuntimeClient::spawn(
AgentLaunchSpec::new(python).with_args([script_path.to_string_lossy().to_string()]),
"test",
)
.await
.unwrap();
runtime.increment_session_count();
let mut client = AgentDriverClient::shared_session(runtime.clone(), "session-1".to_string());
let error = client.disconnect().await.unwrap_err();
assert_eq!(agent_session_disposition(&error), Some(AgentSessionDisposition::ReplaceRuntime));
assert!(!runtime.is_failed());
runtime.kill();
drop(client);
let _ = std::fs::remove_file(script_path);
}
#[tokio::test]
async fn canceling_one_runtime_request_keeps_other_requests_alive() {
let script_path = std::env::temp_dir().join(format!("dbx-agent-cancel-test-{}.py", uuid::Uuid::new_v4()));