feat(agent): add execution context recovery

This commit is contained in:
t8y2 committed 2026-06-22 22:15:16 +08:00
1 parent e55dbe2acc
commit 0b6e1579ca
7 files changed
+381 -60

No files matched your search

+92 -4
View File
@@ -8,6 +8,7 @@ use std::time::Duration;
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use tokio_util::sync::CancellationToken;
pub const AGENT_PROTOCOL_VERSION: u32 = 1;
const RPC_TIMEOUT_SECS: u64 = 30;
@@ -114,6 +115,7 @@ pub enum AgentMethod {
Handshake,
Connect,
TestConnection,
ValidateConnection,
ListDatabases,
ListSchemas,
ListTables,
@@ -135,10 +137,11 @@ pub enum AgentMethod {
}
impl AgentMethod {
pub const ALL: [Self; 21] = [
pub const ALL: [Self; 22] = [
Self::Handshake,
Self::Connect,
Self::TestConnection,
Self::ValidateConnection,
Self::ListDatabases,
Self::ListSchemas,
Self::ListTables,
@@ -164,6 +167,7 @@ impl AgentMethod {
Self::Handshake => "handshake",
Self::Connect => "connect",
Self::TestConnection => "test_connection",
Self::ValidateConnection => "validate_connection",
Self::ListDatabases => "list_databases",
Self::ListSchemas => "list_schemas",
Self::ListTables => "list_tables",
@@ -370,6 +374,19 @@ impl AgentDriverClient {
method: &str,
params: Value,
timeout_duration: Option<Duration>,
) -> Result<T, String> {
self.call_with_timeout_and_cancel(method, params, timeout_duration, None).await
}
/// Send a JSON-RPC 2.0 request and wait for the response.
/// If cancellation happens while a response is pending, kill the agent
/// process because the stdio stream cannot safely skip that response.
pub async fn call_with_timeout_and_cancel<T: DeserializeOwned + Send + 'static>(
&mut self,
method: &str,
params: Value,
timeout_duration: Option<Duration>,
cancel_token: Option<CancellationToken>,
) -> Result<T, String> {
self.next_id += 1;
let id = self.next_id;
@@ -427,15 +444,41 @@ impl AgentDriverClient {
(reader, result)
});
let (returned_reader, result) = match timeout_duration {
Some(duration) => match tokio::time::timeout(duration, response_task).await {
let (returned_reader, result) = match (timeout_duration, cancel_token) {
(Some(duration), Some(token)) => {
tokio::select! {
biased;
_ = token.cancelled() => {
self.kill();
return Err("Query canceled".to_string());
}
result = tokio::time::timeout(duration, response_task) => match result {
Ok(result) => result,
Err(_) => {
self.kill();
return Err(format!("Agent RPC call timed out ({}s)", duration.as_secs()));
}
},
}
}
(Some(duration), None) => match tokio::time::timeout(duration, response_task).await {
Ok(result) => result,
Err(_) => {
self.kill();
return Err(format!("Agent RPC call timed out ({}s)", duration.as_secs()));
}
},
None => response_task.await,
(None, Some(token)) => {
tokio::select! {
biased;
_ = token.cancelled() => {
self.kill();
return Err("Query canceled".to_string());
}
result = response_task => result,
}
}
(None, None) => response_task.await,
}
.map_err(|e| format!("Agent RPC task failed: {e}"))?;
@@ -460,6 +503,16 @@ impl AgentDriverClient {
self.call_with_timeout(method.as_str(), params, timeout_duration).await
}
pub async fn call_method_with_timeout_and_cancel<T: DeserializeOwned + Send + 'static>(
&mut self,
method: AgentMethod,
params: Value,
timeout_duration: Option<Duration>,
cancel_token: Option<CancellationToken>,
) -> Result<T, String> {
self.call_with_timeout_and_cancel(method.as_str(), params, timeout_duration, cancel_token).await
}
pub async fn connect(&mut self, params: Value) -> Result<Value, String> {
self.call_method(AgentMethod::Connect, params).await
}
@@ -468,6 +521,10 @@ impl AgentDriverClient {
self.call_method(AgentMethod::TestConnection, params).await
}
pub async fn validate_connection(&mut self, timeout_duration: Option<Duration>) -> Result<Value, String> {
self.call_method_with_timeout(AgentMethod::ValidateConnection, serde_json::json!({}), timeout_duration).await
}
pub async fn disconnect(&mut self) -> Result<Value, String> {
self.call_method(AgentMethod::Disconnect, serde_json::json!({})).await
}
@@ -615,6 +672,16 @@ impl AgentDriverClient {
self.call_method_with_timeout(AgentMethod::ExecuteQuery, params, timeout_duration).await
}
pub async fn execute_query_with_timeout_and_cancel<T: DeserializeOwned + Send + 'static>(
&mut self,
params: Value,
timeout_duration: Option<Duration>,
cancel_token: Option<CancellationToken>,
) -> Result<T, String> {
self.call_method_with_timeout_and_cancel(AgentMethod::ExecuteQuery, params, timeout_duration, cancel_token)
.await
}
pub async fn execute_query_page<T: DeserializeOwned + Send + 'static>(
&mut self,
params: Value,
@@ -630,6 +697,16 @@ impl AgentDriverClient {
self.call_method_with_timeout(AgentMethod::ExecuteQueryPage, params, timeout_duration).await
}
pub async fn execute_query_page_with_timeout_and_cancel<T: DeserializeOwned + Send + 'static>(
&mut self,
params: Value,
timeout_duration: Option<Duration>,
cancel_token: Option<CancellationToken>,
) -> Result<T, String> {
self.call_method_with_timeout_and_cancel(AgentMethod::ExecuteQueryPage, params, timeout_duration, cancel_token)
.await
}
pub async fn fetch_query_page<T: DeserializeOwned + Send + 'static>(&mut self, params: Value) -> Result<T, String> {
self.call_method(AgentMethod::FetchQueryPage, params).await
}
@@ -642,6 +719,16 @@ impl AgentDriverClient {
self.call_method_with_timeout(AgentMethod::FetchQueryPage, params, timeout_duration).await
}
pub async fn fetch_query_page_with_timeout_and_cancel<T: DeserializeOwned + Send + 'static>(
&mut self,
params: Value,
timeout_duration: Option<Duration>,
cancel_token: Option<CancellationToken>,
) -> Result<T, String> {
self.call_method_with_timeout_and_cancel(AgentMethod::FetchQueryPage, params, timeout_duration, cancel_token)
.await
}
pub async fn get_explain_info<T: DeserializeOwned + Send + 'static>(&mut self, params: Value) -> Result<T, String> {
self.call_method(AgentMethod::GetExplainInfo, params).await
}
@@ -1153,6 +1240,7 @@ mod tests {
assert_eq!(AgentMethod::Handshake.as_str(), "handshake");
assert_eq!(AgentMethod::Connect.as_str(), "connect");
assert_eq!(AgentMethod::TestConnection.as_str(), "test_connection");
assert_eq!(AgentMethod::ValidateConnection.as_str(), "validate_connection");
assert_eq!(AgentMethod::ListDatabases.as_str(), "list_databases");
assert_eq!(AgentMethod::ListSchemas.as_str(), "list_schemas");
assert_eq!(AgentMethod::ListTables.as_str(), "list_tables");
+146 -1
View File
@@ -9,13 +9,15 @@ use rustls::crypto::{verify_tls12_signature, verify_tls13_signature, CryptoProvi
use rustls::pki_types::{CertificateDer, PrivateKeyDer, ServerName, UnixTime};
use rustls::server::ParsedCertificate;
use std::fs::File;
use std::future::Future;
use std::io::BufReader;
use std::str::FromStr;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio_postgres::config::SslMode;
use tokio_postgres::types::{FromSql, Type};
use tokio_postgres::{Row, SimpleQueryMessage};
use tokio_postgres::{NoTls, Row, SimpleQueryMessage};
use tokio_util::sync::CancellationToken;
use super::file_validator::validate_file_path;
use crate::sql::starts_with_executable_sql_keyword;
@@ -1330,6 +1332,24 @@ pub async fn execute_query_with_max_rows(
}
}
pub async fn execute_query_with_max_rows_and_cancel(
pool: &Pool,
sql: &str,
max_rows: Option<usize>,
cancel_token: Option<CancellationToken>,
timeout_duration: Option<Duration>,
) -> Result<QueryResult, String> {
let client = pool.get().await.map_err(|e| e.to_string())?;
let pg_cancel_token = client.cancel_token();
wait_postgres_query(
pg_cancel_token,
cancel_token,
timeout_duration,
execute_query_with_max_rows_inner(&client, sql, max_rows),
)
.await
}
pub async fn execute_query_with_schema(pool: &Pool, schema: &str, sql: &str) -> Result<QueryResult, String> {
execute_query_with_schema_and_max_rows(pool, schema, sql, None).await
}
@@ -1389,6 +1409,131 @@ pub async fn execute_query_with_schema_and_max_rows(
result
}
pub async fn execute_query_with_schema_and_max_rows_and_cancel(
pool: &Pool,
schema: &str,
sql: &str,
max_rows: Option<usize>,
cancel_token: Option<CancellationToken>,
timeout_duration: Option<Duration>,
) -> Result<QueryResult, String> {
let start = Instant::now();
let checkout_start = Instant::now();
let client = pool.get().await.map_err(|e| e.to_string())?;
log::info!(
"[postgres][execute_with_schema:pool:done] elapsed_ms={} total_ms={} schema={}",
checkout_start.elapsed().as_millis(),
start.elapsed().as_millis(),
schema
);
if is_transaction_recovery_statement(sql) {
log::info!(
"[postgres][execute_with_schema:skip-search-path] total_ms={} reason=transaction-recovery",
start.elapsed().as_millis()
);
let pg_cancel_token = client.cancel_token();
return wait_postgres_query(
pg_cancel_token,
cancel_token,
timeout_duration,
execute_query_with_max_rows_inner(&client, sql, max_rows),
)
.await;
}
let set_schema_start = Instant::now();
client.execute(&format!("SET search_path TO {}", pg_quote_ident(schema)), &[]).await.map_err(pg_error_to_string)?;
log::info!(
"[postgres][execute_with_schema:set-search-path:done] elapsed_ms={} total_ms={}",
set_schema_start.elapsed().as_millis(),
start.elapsed().as_millis()
);
let query_start = Instant::now();
let pg_cancel_token = client.cancel_token();
let result = wait_postgres_query(
pg_cancel_token,
cancel_token,
timeout_duration,
execute_query_with_max_rows_inner(&client, sql, max_rows),
)
.await;
if result.is_ok() {
clear_postgres_caches_after_ddl(pool, Some(&client), sql);
}
log::info!(
"[postgres][execute_with_schema:query:done] elapsed_ms={} total_ms={} ok={}",
query_start.elapsed().as_millis(),
start.elapsed().as_millis(),
result.is_ok()
);
let reset_start = Instant::now();
let _ = client.execute("RESET search_path", &[]).await;
log::info!(
"[postgres][execute_with_schema:reset-search-path:done] elapsed_ms={} total_ms={}",
reset_start.elapsed().as_millis(),
start.elapsed().as_millis()
);
result
}
async fn wait_postgres_query<T, F>(
pg_cancel_token: tokio_postgres::CancelToken,
cancel_token: Option<CancellationToken>,
timeout_duration: Option<Duration>,
future: F,
) -> Result<T, String>
where
F: Future<Output = Result<T, String>>,
{
match (cancel_token, timeout_duration) {
(Some(token), Some(duration)) => {
tokio::select! {
biased;
_ = token.cancelled() => {
cancel_postgres_query(pg_cancel_token).await;
Err(crate::query::canceled_error())
}
result = tokio::time::timeout(duration, future) => match result {
Ok(result) => result,
Err(_) => {
cancel_postgres_query(pg_cancel_token).await;
Err(format!("Query timed out after {} seconds", duration.as_secs()))
}
},
}
}
(None, Some(duration)) => match tokio::time::timeout(duration, future).await {
Ok(result) => result,
Err(_) => {
cancel_postgres_query(pg_cancel_token).await;
Err(format!("Query timed out after {} seconds", duration.as_secs()))
}
},
(Some(token), None) => {
tokio::select! {
biased;
_ = token.cancelled() => {
cancel_postgres_query(pg_cancel_token).await;
Err(crate::query::canceled_error())
}
result = future => result,
}
}
(None, None) => future.await,
}
}
async fn cancel_postgres_query(pg_cancel_token: tokio_postgres::CancelToken) {
match tokio::time::timeout(Duration::from_secs(2), pg_cancel_token.cancel_query(NoTls)).await {
Ok(Ok(())) => {}
Ok(Err(err)) => log::warn!("Failed to send PostgreSQL cancel request: {err}"),
Err(_) => log::warn!("Timed out sending PostgreSQL cancel request"),
}
}
fn is_transaction_recovery_statement(sql: &str) -> bool {
starts_with_executable_sql_keyword(sql, &["ROLLBACK", "ABORT", "COMMIT", "END"])
}