fix(agent): recover JDBC connections after query timeouts

This commit is contained in:
miracle authored and GitHub committed 2026-08-01 01:17:49 +08:00
1 parent 0d484f5c74
commit 0e414e638e
24 files changed
+5983 -473

No files matched your search

+276 -39
View File
@@ -545,6 +545,9 @@ pub fn agent_close_query_session_params(session_id: &str) -> serde_json::Value {
}
pub fn is_connection_error(err: &str) -> bool {
if crate::db::agent_driver::agent_rpc_error_category(err).as_deref() == Some("connection") {
return true;
}
let lower = err.to_lowercase();
if is_dbx_query_timeout_error(&lower) || is_agent_rpc_timeout_error(&lower) {
return false;
@@ -571,6 +574,9 @@ pub fn is_connection_error(err: &str) -> bool {
|| lower.contains("idle")
|| lower.contains("agent stdin not available")
|| lower.contains("agent stdout not available")
|| lower.contains("agent runtime terminated")
|| lower.contains("agent runtime is unavailable")
|| lower.contains("agent runtime unavailable")
|| lower.contains("failed to write to agent stdin")
|| lower.contains("failed to flush agent stdin")
|| lower.contains("communicating with the server")
@@ -590,6 +596,15 @@ fn is_schema_reset_cleanup_error(lower: &str) -> bool {
}
fn should_discard_agent_pool_after_error(err: &str) -> bool {
if matches!(
crate::db::agent_driver::agent_session_disposition(err),
Some(
crate::db::agent_driver::AgentSessionDisposition::Quarantine
| crate::db::agent_driver::AgentSessionDisposition::ReplaceRuntime
)
) {
return true;
}
let lower = err.to_lowercase();
is_dbx_query_timeout_error(&lower)
|| is_agent_rpc_timeout_error(&lower)
@@ -601,6 +616,19 @@ fn should_discard_agent_pool_after_error(err: &str) -> bool {
}
pub fn pool_error_action(db_type: Option<DatabaseType>, err: &str) -> PoolErrorAction {
if db_type.is_some_and(|db_type| database_capabilities::is_agent_type(&db_type))
&& matches!(
crate::db::agent_driver::agent_session_disposition(err),
Some(
crate::db::agent_driver::AgentSessionDisposition::Quarantine
| crate::db::agent_driver::AgentSessionDisposition::ReplaceRuntime
)
)
{
// The connection may be replaced, but the result of the user operation is unknown.
// Discard the session without replaying SQL, DDL, writes, or transactions.
return PoolErrorAction::Discard;
}
let lower = err.to_lowercase();
if db::sqlserver::is_driver_panic_error(err)
|| (is_dbx_query_timeout_error(&lower) && should_discard_pool_after_query_timeout(db_type))
@@ -680,6 +708,41 @@ pub fn should_discard_pool_after_error(db_type: Option<DatabaseType>, err: &str)
matches!(pool_error_action(db_type, err), PoolErrorAction::Discard | PoolErrorAction::ReconnectAndRetry)
}
async fn discard_pool_after_error(state: &AppState, pool_key: &str, db_type: Option<DatabaseType>, error: &str) {
let action = pool_error_action(db_type, error);
if !matches!(action, PoolErrorAction::Discard | PoolErrorAction::ReconnectAndRetry) {
return;
}
let replace_agent_runtime = matches!(
crate::db::agent_driver::agent_session_disposition(error),
Some(crate::db::agent_driver::AgentSessionDisposition::ReplaceRuntime)
);
if replace_agent_runtime {
state.detach_pool_by_key(pool_key, true).await;
} else {
state.remove_pool_by_key(pool_key).await;
}
}
async fn discard_agent_pool_after_error(
state: &AppState,
pool_key: &str,
client: &Arc<crate::db::agent_driver::PooledAgentClient>,
db_type: Option<DatabaseType>,
error: &str,
) {
let action = pool_error_action(db_type, error);
if !matches!(action, PoolErrorAction::Discard | PoolErrorAction::ReconnectAndRetry) {
return;
}
let replace_agent_runtime = matches!(
crate::db::agent_driver::agent_session_disposition(error),
Some(crate::db::agent_driver::AgentSessionDisposition::ReplaceRuntime)
);
state.detach_agent_pool_if_current(pool_key, client, replace_agent_runtime).await;
}
fn query_pool_error_action(db_type: Option<DatabaseType>, sql: &str, err: &str) -> PoolErrorAction {
match pool_error_action(db_type, err) {
// A connection error does not prove that the database did not receive
@@ -1264,6 +1327,7 @@ pub async fn do_execute(
}
PoolKind::Agent(client) => {
let client = client.clone();
let source_client = client.clone();
let sql = sql_for_execution_context(pool_db_type, sql, schema);
let database = database.map(|s| s.to_string());
let schema = schema_for_execution_context(pool_db_type, schema).map(|s| s.to_string());
@@ -1301,10 +1365,12 @@ pub async fn do_execute(
.await
.map(|result| truncate_result_with_max_rows(result, max_rows));
if matches!(result.as_ref(), Err(err) if err == QUERY_CANCELED) {
state.remove_pool_by_key(pool_key).await;
state.detach_agent_pool_if_current(pool_key, &source_client, false).await;
}
if matches!(result.as_ref(), Err(err) if should_discard_pool_after_error(pool_db_type, err)) {
state.remove_pool_by_key(pool_key).await;
if let Err(err) = result.as_ref() {
if err != QUERY_CANCELED {
discard_agent_pool_after_error(state, pool_key, &source_client, pool_db_type, err).await;
}
}
result
}
@@ -1493,7 +1559,11 @@ pub async fn execute_sql_statement_with_options(
)
}
Some(PoolErrorAction::Discard) => {
state.remove_pool_by_key(&pool_key).await;
// Agent execution owns structured quarantine/runtime replacement before
// returning. Native drivers retain the existing caller-side cleanup.
if !db_type.is_some_and(|db_type| database_capabilities::is_agent_type(&db_type)) {
state.remove_pool_by_key(&pool_key).await;
}
with_sql_context(result)
}
_ => with_sql_context(result),
@@ -2169,6 +2239,7 @@ pub async fn execute_statements(
}
};
if let Some(client) = agent_client {
let source_client = client.clone();
check_read_only_for_connection_multi(state, &pool_key, statements).await?;
let db_type = connection_database_type_for_pool_key(state, &pool_key).await;
let execution_schema = schema_for_execution_context(db_type, schema);
@@ -2183,6 +2254,7 @@ pub async fn execute_statements(
let mut client = client.lock().await;
let database = if database.trim().is_empty() { None } else { Some(database) };
let result = execute_multi_agent(&mut client, database, statements, execution_schema, timeout_secs).await;
drop(client);
match result {
Ok(result) => return Ok(db::QueryResult { execution_time_ms: start.elapsed().as_millis(), ..result }),
Err(err) => {
@@ -2191,12 +2263,8 @@ pub async fn execute_statements(
"Agent does not support execute_batch; falling back to statement-by-statement execution"
);
} else {
match pool_error_action(connection_database_type(state, connection_id).await, &err) {
PoolErrorAction::ReconnectAndRetry | PoolErrorAction::Discard => {
let _ = state.remove_pool_by_key(&pool_key).await;
}
PoolErrorAction::Keep => {}
}
let db_type = connection_database_type(state, connection_id).await;
discard_agent_pool_after_error(state, &pool_key, &source_client, db_type, &err).await;
return Err(query_error_with_omitted_sql_context(&err, sql_ctx));
}
}
@@ -2220,15 +2288,18 @@ pub async fn execute_statements(
total_affected += result.affected_rows;
}
Err(e) => {
match pool_error_action(connection_database_type(state, connection_id).await, &e) {
let db_type = connection_database_type(state, connection_id).await;
match pool_error_action(db_type, &e) {
PoolErrorAction::ReconnectAndRetry => {
let db_opt = if database.is_empty() { None } else { Some(database) };
let _ = state.reconnect_pool(connection_id, db_opt).await;
}
PoolErrorAction::Discard => {
PoolErrorAction::Discard
if !db_type.is_some_and(|db_type| database_capabilities::is_agent_type(&db_type)) =>
{
let _ = state.remove_pool_by_key(&pool_key).await;
}
PoolErrorAction::Keep => {}
PoolErrorAction::Discard | PoolErrorAction::Keep => {}
}
return Err(query_error_with_omitted_sql_context(
&format!("Statement {} failed: {}. Previous {} statement(s) may have been committed.", i + 1, e, i),
@@ -2560,11 +2631,10 @@ pub async fn execute_statements_in_transaction_on_pool(
PoolKind::Mysql(mp, _mode) => TxPath::Mysql(mp.clone(), false),
PoolKind::Sqlite(sq) => TxPath::Sqlite(sq.clone()),
PoolKind::CloudflareD1(client) => TxPath::CloudflareD1(client.clone()),
PoolKind::ClickHouse(_)
| PoolKind::Rqlite(_)
| PoolKind::Turso(_)
| PoolKind::SqlServer(_)
| PoolKind::Agent(_) => TxPath::Explicit,
PoolKind::ClickHouse(_) | PoolKind::Rqlite(_) | PoolKind::Turso(_) | PoolKind::SqlServer(_) => {
TxPath::Explicit
}
PoolKind::Agent(client) => TxPath::Agent(client.clone()),
PoolKind::MessageQueue | PoolKind::Nacos | PoolKind::HBase(_) => TxPath::None,
PoolKind::DuckDbWorker(_)
| PoolKind::Redis(_)
@@ -2606,6 +2676,13 @@ pub async fn execute_statements_in_transaction_on_pool(
)
.await
}
Some(TxPath::Agent(client)) => {
let result = exec_tx_agent_inner(client.clone(), db_type, Some(database), statements, schema, start).await;
if let Err(error) = result.as_ref() {
discard_agent_pool_after_error(state, pool_key, &client, db_type, error).await;
}
return result;
}
Some(TxPath::Explicit) => {
let mysql_dialect = connection_mysql_query_dialect(state, connection_id).await;
exec_tx_explicit_inner(state, pool_key, mysql_dialect, Some(database), statements, schema, start).await
@@ -2618,9 +2695,7 @@ pub async fn execute_statements_in_transaction_on_pool(
};
if let Err(err) = result.as_ref() {
if matches!(pool_error_action(db_type, err), PoolErrorAction::Discard | PoolErrorAction::ReconnectAndRetry) {
state.remove_pool_by_key(pool_key).await;
}
discard_pool_after_error(state, pool_key, db_type, err).await;
}
result
@@ -2632,6 +2707,7 @@ enum TxPath {
Mysql(mysql_async::Pool, bool),
Sqlite(db::sqlite::SqliteHandle),
CloudflareD1(db::cloudflare_d1_driver::CloudflareD1Client),
Agent(Arc<crate::db::agent_driver::PooledAgentClient>),
Explicit,
None,
}
@@ -2859,24 +2935,6 @@ async fn exec_tx_explicit_inner(
schema: Option<&str>,
start: std::time::Instant,
) -> Result<db::QueryResult, String> {
let conns = state.connections.read().await;
if let Some(crate::connection::PoolKind::Agent(client)) = conns.get(pool_key) {
let db_type = connection_database_type_for_pool_key(state, pool_key).await;
let execution_schema = schema_for_execution_context(db_type, schema);
let rewritten_statements;
let statements = if qualifies_unqualified_agent_relations(db_type) {
rewritten_statements =
statements.iter().map(|sql| sql_for_execution_context(db_type, sql, schema)).collect::<Vec<_>>();
rewritten_statements.as_slice()
} else {
statements
};
let mut client = client.lock().await;
let result: db::QueryResult = client.execute_transaction(database, statements, execution_schema).await?;
return Ok(db::QueryResult { execution_time_ms: start.elapsed().as_millis(), ..result });
}
drop(conns);
do_execute(
state,
pool_key,
@@ -2938,6 +2996,28 @@ async fn exec_tx_explicit_inner(
})
}
async fn exec_tx_agent_inner(
client: Arc<crate::db::agent_driver::PooledAgentClient>,
db_type: Option<DatabaseType>,
database: Option<&str>,
statements: &[String],
schema: Option<&str>,
start: std::time::Instant,
) -> Result<db::QueryResult, String> {
let execution_schema = schema_for_execution_context(db_type, schema);
let rewritten_statements;
let statements = if qualifies_unqualified_agent_relations(db_type) {
rewritten_statements =
statements.iter().map(|sql| sql_for_execution_context(db_type, sql, schema)).collect::<Vec<_>>();
rewritten_statements.as_slice()
} else {
statements
};
let mut client = client.lock().await;
let result: db::QueryResult = client.execute_transaction(database, statements, execution_schema).await?;
Ok(db::QueryResult { execution_time_ms: start.elapsed().as_millis(), ..result })
}
async fn exec_tx_none_inner(
state: &AppState,
pool_key: &str,
@@ -3884,6 +3964,141 @@ for line in sys.stdin:
}
}
async fn agent_error_state(
disposition: &str,
) -> (AppState, std::path::PathBuf, std::sync::Arc<crate::db::agent_driver::AgentRuntimeClient>) {
let dir = std::env::temp_dir().join(format!("dbx-query-agent-error-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
let script_path = dir.join("agent.py");
std::fs::write(
&script_path,
format!(
r#"import json, sys
print(json.dumps({{'ready': True}}), flush=True)
for line in sys.stdin:
req = json.loads(line)
if req['method'] == 'handshake':
response = {{
'jsonrpc': '2.0',
'id': req['id'],
'result': {{'protocolVersion': 2, 'agentProtocolVersion': 2, 'capabilities': ['multi_session']}}
}}
elif req['method'] in ('execute_query', 'execute_batch', 'execute_transaction'):
response = {{
'jsonrpc': '2.0',
'id': req['id'],
'error': {{
'code': -1,
'message': 'injected Agent failure',
'data': {{
'category': 'resource',
'retryable': False,
'sessionDisposition': '{disposition}',
'stage': 'execute'
}}
}}
}}
else:
response = {{'jsonrpc': '2.0', 'id': req['id'], 'result': {{}}}}
print(json.dumps(response), flush=True)
"#
),
)
.unwrap();
let python = if cfg!(windows) { "python" } else { "python3" };
let runtime = crate::db::agent_driver::AgentRuntimeClient::spawn(
crate::db::agent_driver::AgentLaunchSpec::new(python)
.with_args([script_path.to_string_lossy().to_string()]),
"test",
)
.await
.unwrap();
runtime.increment_session_count();
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
let state = AppState::new(storage);
state.configs.write().await.insert("conn-1".to_string(), test_connection_config(DatabaseType::Dameng));
state.connections.write().await.insert(
"conn-1".to_string(),
PoolKind::agent(crate::db::agent_driver::AgentDriverClient::shared_session(
runtime.clone(),
"session-1".to_string(),
)),
);
(state, dir, runtime)
}
#[tokio::test]
async fn agent_query_replace_runtime_error_detaches_pool_and_stops_runtime() {
let (state, dir, runtime) = agent_error_state("replace_runtime").await;
let error = execute_sql_statement(&state, "conn-1", "", "SELECT 1", None, None).await.unwrap_err();
assert!(error.contains("injected Agent failure"));
assert!(!state.connections.read().await.contains_key("conn-1"));
assert!(runtime.is_failed());
runtime.kill();
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test]
async fn agent_transaction_replace_runtime_error_detaches_pool_and_stops_runtime() {
let (state, dir, runtime) = agent_error_state("replace_runtime").await;
let error = execute_statements_in_transaction_on_pool(
&state,
"conn-1",
"conn-1",
"",
&["UPDATE test_table SET value = 1".to_string()],
None,
None,
)
.await
.unwrap_err();
assert!(error.contains("injected Agent failure"));
assert!(!state.connections.read().await.contains_key("conn-1"));
assert!(runtime.is_failed());
runtime.kill();
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test]
async fn agent_batch_replace_runtime_error_detaches_pool_and_stops_runtime() {
let (state, dir, runtime) = agent_error_state("replace_runtime").await;
let error =
execute_statements(&state, "conn-1", "", &["UPDATE test_table SET value = 1".to_string()], None, None)
.await
.unwrap_err();
assert!(error.contains("injected Agent failure"));
assert!(!state.connections.read().await.contains_key("conn-1"));
assert!(runtime.is_failed());
runtime.kill();
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test]
async fn agent_quarantine_error_removes_only_target_pool() {
let (state, dir, runtime) = agent_error_state("quarantine").await;
let error = execute_sql_statement(&state, "conn-1", "", "SELECT 1", None, None).await.unwrap_err();
assert!(error.contains("injected Agent failure"));
assert!(!state.connections.read().await.contains_key("conn-1"));
assert!(!runtime.is_failed());
runtime.kill();
let _ = std::fs::remove_dir_all(dir);
}
struct FakeMysqlBatchExecutor {
outcomes: std::collections::VecDeque<Result<db::QueryResult, String>>,
executed: Vec<String>,
@@ -5010,16 +5225,38 @@ for line in sys.stdin:
);
}
#[test]
fn structured_agent_disposition_controls_pool_recovery() {
let quarantined = "Agent RPC error (-1): lost\nDBX_AGENT_ERROR_DATA:{\"category\":\"connection\",\"sessionDisposition\":\"quarantine\"}";
let replace_runtime = "Agent RPC error (-1): saturated\nDBX_AGENT_ERROR_DATA:{\"category\":\"resource\",\"sessionDisposition\":\"replace_runtime\"}";
assert!(should_discard_agent_pool_after_error(quarantined));
assert!(should_discard_agent_pool_after_error(replace_runtime));
assert!(is_connection_error(quarantined));
assert_eq!(pool_error_action(Some(DatabaseType::Oracle), quarantined), PoolErrorAction::Discard);
assert_eq!(pool_error_action(Some(DatabaseType::Oracle), replace_runtime), PoolErrorAction::Discard);
}
#[test]
fn unavailable_agent_pipes_are_reconnectable_errors() {
assert!(should_discard_agent_pool_after_error("Agent stdin not available"));
assert!(should_discard_agent_pool_after_error("Agent stdout not available"));
assert!(is_connection_error("Agent stdin not available"));
assert!(is_connection_error("Agent stdout not available"));
assert!(is_connection_error("Agent runtime terminated"));
assert!(is_connection_error("Agent runtime is unavailable"));
assert_eq!(
pool_error_action(Some(DatabaseType::Oracle), "Agent stdin not available"),
PoolErrorAction::ReconnectAndRetry
);
assert_eq!(
pool_error_action(Some(DatabaseType::Oracle), "Agent runtime terminated"),
PoolErrorAction::ReconnectAndRetry
);
assert_eq!(
pool_error_action(Some(DatabaseType::Oracle), "Agent runtime is unavailable"),
PoolErrorAction::ReconnectAndRetry
);
}
#[test]