feat(agent): add execution context recovery
This commit is contained in:
1 parent
e55dbe2acc
commit
0b6e1579ca
7 files changed
+381
-60
No files matched your search
@@ -82,8 +82,8 @@ pub enum PoolKind {
|
||||
}
|
||||
|
||||
pub struct AppState {
|
||||
pub connections: RwLock<HashMap<String, PoolKind>>,
|
||||
keepalive_tasks: RwLock<HashMap<String, JoinHandle<()>>>,
|
||||
pub connections: Arc<RwLock<HashMap<String, PoolKind>>>,
|
||||
keepalive_tasks: Arc<RwLock<HashMap<String, JoinHandle<()>>>>,
|
||||
pub configs: RwLock<HashMap<String, ConnectionConfig>>,
|
||||
pub running_queries: RunningQueries,
|
||||
pub tunnels: TunnelManager,
|
||||
@@ -261,8 +261,8 @@ impl AppState {
|
||||
app_version: impl Into<String>,
|
||||
) -> Self {
|
||||
Self {
|
||||
connections: RwLock::new(HashMap::new()),
|
||||
keepalive_tasks: RwLock::new(HashMap::new()),
|
||||
connections: Arc::new(RwLock::new(HashMap::new())),
|
||||
keepalive_tasks: Arc::new(RwLock::new(HashMap::new())),
|
||||
configs: RwLock::new(HashMap::new()),
|
||||
running_queries: RunningQueries::default(),
|
||||
tunnels: TunnelManager::new(),
|
||||
@@ -348,13 +348,23 @@ impl AppState {
|
||||
let key = pool_key.to_string();
|
||||
let interval = Duration::from_secs(interval_secs.max(1));
|
||||
let timeout = Duration::from_secs(config.effective_connect_timeout_secs().max(1));
|
||||
let connections = self.connections.clone();
|
||||
let keepalive_tasks = self.keepalive_tasks.clone();
|
||||
let handle = tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::time::sleep(interval).await;
|
||||
let result = tokio::time::timeout(timeout, ping_keepalive_target(&mut target, timeout)).await;
|
||||
match result {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(err)) => log::warn!("Connection keepalive failed for '{key}': {err}"),
|
||||
Ok(Err(err)) => {
|
||||
log::warn!("Connection keepalive failed for '{key}': {err}; invalidating pool");
|
||||
keepalive_tasks.write().await.remove(&key);
|
||||
let removed = connections.write().await.remove(&key);
|
||||
if let Some(pool) = removed {
|
||||
close_pool_kind_with_timeout(key, pool).await;
|
||||
}
|
||||
break;
|
||||
}
|
||||
Err(_) => log::warn!("Connection keepalive timed out for '{key}' after {}s", timeout.as_secs()),
|
||||
}
|
||||
}
|
||||
@@ -1053,7 +1063,8 @@ impl AppState {
|
||||
let client = client.clone();
|
||||
drop(connections);
|
||||
let mut agent = client.lock().await;
|
||||
match agent.test_connection(serde_json::json!({})).await {
|
||||
let timeout = crate::db::connection_timeout();
|
||||
match agent.validate_connection(Some(timeout)).await {
|
||||
Ok(_) => false,
|
||||
Err(err) => {
|
||||
log::warn!("Agent connection pool '{pool_key}' is stale: {err}");
|
||||
@@ -1535,6 +1546,7 @@ enum KeepaliveTarget {
|
||||
Elasticsearch(db::elasticsearch_driver::EsClient),
|
||||
VectorDb(db::vector_driver::VectorClient),
|
||||
InfluxDb(db::influxdb_driver::InfluxdbClient),
|
||||
Agent(Arc<tokio::sync::Mutex<db::agent_driver::AgentDriverClient>>),
|
||||
}
|
||||
|
||||
fn keepalive_target_from_pool(pool: &PoolKind, config: &ConnectionConfig) -> Option<KeepaliveTarget> {
|
||||
@@ -1552,6 +1564,7 @@ fn keepalive_target_from_pool(pool: &PoolKind, config: &ConnectionConfig) -> Opt
|
||||
PoolKind::Elasticsearch(client) => Some(KeepaliveTarget::Elasticsearch(client.clone())),
|
||||
PoolKind::VectorDb(client) => Some(KeepaliveTarget::VectorDb(client.clone())),
|
||||
PoolKind::InfluxDb(client) => Some(KeepaliveTarget::InfluxDb(client.clone())),
|
||||
PoolKind::Agent(client) => Some(KeepaliveTarget::Agent(client.clone())),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -1573,12 +1586,26 @@ async fn ping_keepalive_target(target: &mut KeepaliveTarget, timeout: Duration)
|
||||
}
|
||||
KeepaliveTarget::ClickHouse(client) => db::clickhouse_driver::test_connection(client, timeout).await,
|
||||
KeepaliveTarget::SqlServer(client) => {
|
||||
let mut client = client.lock().await;
|
||||
let Ok(mut client) = client.try_lock() else {
|
||||
return Ok(());
|
||||
};
|
||||
db::sqlserver::test_connection(&mut client).await
|
||||
}
|
||||
KeepaliveTarget::Elasticsearch(client) => db::elasticsearch_driver::test_connection(client, timeout).await,
|
||||
KeepaliveTarget::VectorDb(client) => db::vector_driver::test_connection(client, timeout).await,
|
||||
KeepaliveTarget::InfluxDb(client) => db::influxdb_driver::test_connection(client, timeout).await,
|
||||
KeepaliveTarget::Agent(client) => {
|
||||
let Ok(mut client) = client.try_lock() else {
|
||||
return Ok(());
|
||||
};
|
||||
match client.validate_connection(Some(timeout)).await {
|
||||
Ok(_) => Ok(()),
|
||||
Err(err) => {
|
||||
client.kill();
|
||||
Err(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ use std::time::Duration;
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
pub const AGENT_PROTOCOL_VERSION: u32 = 1;
|
||||
const RPC_TIMEOUT_SECS: u64 = 30;
|
||||
@@ -114,6 +115,7 @@ pub enum AgentMethod {
|
||||
Handshake,
|
||||
Connect,
|
||||
TestConnection,
|
||||
ValidateConnection,
|
||||
ListDatabases,
|
||||
ListSchemas,
|
||||
ListTables,
|
||||
@@ -135,10 +137,11 @@ pub enum AgentMethod {
|
||||
}
|
||||
|
||||
impl AgentMethod {
|
||||
pub const ALL: [Self; 21] = [
|
||||
pub const ALL: [Self; 22] = [
|
||||
Self::Handshake,
|
||||
Self::Connect,
|
||||
Self::TestConnection,
|
||||
Self::ValidateConnection,
|
||||
Self::ListDatabases,
|
||||
Self::ListSchemas,
|
||||
Self::ListTables,
|
||||
@@ -164,6 +167,7 @@ impl AgentMethod {
|
||||
Self::Handshake => "handshake",
|
||||
Self::Connect => "connect",
|
||||
Self::TestConnection => "test_connection",
|
||||
Self::ValidateConnection => "validate_connection",
|
||||
Self::ListDatabases => "list_databases",
|
||||
Self::ListSchemas => "list_schemas",
|
||||
Self::ListTables => "list_tables",
|
||||
@@ -370,6 +374,19 @@ impl AgentDriverClient {
|
||||
method: &str,
|
||||
params: Value,
|
||||
timeout_duration: Option<Duration>,
|
||||
) -> Result<T, String> {
|
||||
self.call_with_timeout_and_cancel(method, params, timeout_duration, None).await
|
||||
}
|
||||
|
||||
/// Send a JSON-RPC 2.0 request and wait for the response.
|
||||
/// If cancellation happens while a response is pending, kill the agent
|
||||
/// process because the stdio stream cannot safely skip that response.
|
||||
pub async fn call_with_timeout_and_cancel<T: DeserializeOwned + Send + 'static>(
|
||||
&mut self,
|
||||
method: &str,
|
||||
params: Value,
|
||||
timeout_duration: Option<Duration>,
|
||||
cancel_token: Option<CancellationToken>,
|
||||
) -> Result<T, String> {
|
||||
self.next_id += 1;
|
||||
let id = self.next_id;
|
||||
@@ -427,15 +444,41 @@ impl AgentDriverClient {
|
||||
|
||||
(reader, result)
|
||||
});
|
||||
let (returned_reader, result) = match timeout_duration {
|
||||
Some(duration) => match tokio::time::timeout(duration, response_task).await {
|
||||
let (returned_reader, result) = match (timeout_duration, cancel_token) {
|
||||
(Some(duration), Some(token)) => {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = token.cancelled() => {
|
||||
self.kill();
|
||||
return Err("Query canceled".to_string());
|
||||
}
|
||||
result = tokio::time::timeout(duration, response_task) => match result {
|
||||
Ok(result) => result,
|
||||
Err(_) => {
|
||||
self.kill();
|
||||
return Err(format!("Agent RPC call timed out ({}s)", duration.as_secs()));
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
(Some(duration), None) => match tokio::time::timeout(duration, response_task).await {
|
||||
Ok(result) => result,
|
||||
Err(_) => {
|
||||
self.kill();
|
||||
return Err(format!("Agent RPC call timed out ({}s)", duration.as_secs()));
|
||||
}
|
||||
},
|
||||
None => response_task.await,
|
||||
(None, Some(token)) => {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = token.cancelled() => {
|
||||
self.kill();
|
||||
return Err("Query canceled".to_string());
|
||||
}
|
||||
result = response_task => result,
|
||||
}
|
||||
}
|
||||
(None, None) => response_task.await,
|
||||
}
|
||||
.map_err(|e| format!("Agent RPC task failed: {e}"))?;
|
||||
|
||||
@@ -460,6 +503,16 @@ impl AgentDriverClient {
|
||||
self.call_with_timeout(method.as_str(), params, timeout_duration).await
|
||||
}
|
||||
|
||||
pub async fn call_method_with_timeout_and_cancel<T: DeserializeOwned + Send + 'static>(
|
||||
&mut self,
|
||||
method: AgentMethod,
|
||||
params: Value,
|
||||
timeout_duration: Option<Duration>,
|
||||
cancel_token: Option<CancellationToken>,
|
||||
) -> Result<T, String> {
|
||||
self.call_with_timeout_and_cancel(method.as_str(), params, timeout_duration, cancel_token).await
|
||||
}
|
||||
|
||||
pub async fn connect(&mut self, params: Value) -> Result<Value, String> {
|
||||
self.call_method(AgentMethod::Connect, params).await
|
||||
}
|
||||
@@ -468,6 +521,10 @@ impl AgentDriverClient {
|
||||
self.call_method(AgentMethod::TestConnection, params).await
|
||||
}
|
||||
|
||||
pub async fn validate_connection(&mut self, timeout_duration: Option<Duration>) -> Result<Value, String> {
|
||||
self.call_method_with_timeout(AgentMethod::ValidateConnection, serde_json::json!({}), timeout_duration).await
|
||||
}
|
||||
|
||||
pub async fn disconnect(&mut self) -> Result<Value, String> {
|
||||
self.call_method(AgentMethod::Disconnect, serde_json::json!({})).await
|
||||
}
|
||||
@@ -615,6 +672,16 @@ impl AgentDriverClient {
|
||||
self.call_method_with_timeout(AgentMethod::ExecuteQuery, params, timeout_duration).await
|
||||
}
|
||||
|
||||
pub async fn execute_query_with_timeout_and_cancel<T: DeserializeOwned + Send + 'static>(
|
||||
&mut self,
|
||||
params: Value,
|
||||
timeout_duration: Option<Duration>,
|
||||
cancel_token: Option<CancellationToken>,
|
||||
) -> Result<T, String> {
|
||||
self.call_method_with_timeout_and_cancel(AgentMethod::ExecuteQuery, params, timeout_duration, cancel_token)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn execute_query_page<T: DeserializeOwned + Send + 'static>(
|
||||
&mut self,
|
||||
params: Value,
|
||||
@@ -630,6 +697,16 @@ impl AgentDriverClient {
|
||||
self.call_method_with_timeout(AgentMethod::ExecuteQueryPage, params, timeout_duration).await
|
||||
}
|
||||
|
||||
pub async fn execute_query_page_with_timeout_and_cancel<T: DeserializeOwned + Send + 'static>(
|
||||
&mut self,
|
||||
params: Value,
|
||||
timeout_duration: Option<Duration>,
|
||||
cancel_token: Option<CancellationToken>,
|
||||
) -> Result<T, String> {
|
||||
self.call_method_with_timeout_and_cancel(AgentMethod::ExecuteQueryPage, params, timeout_duration, cancel_token)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn fetch_query_page<T: DeserializeOwned + Send + 'static>(&mut self, params: Value) -> Result<T, String> {
|
||||
self.call_method(AgentMethod::FetchQueryPage, params).await
|
||||
}
|
||||
@@ -642,6 +719,16 @@ impl AgentDriverClient {
|
||||
self.call_method_with_timeout(AgentMethod::FetchQueryPage, params, timeout_duration).await
|
||||
}
|
||||
|
||||
pub async fn fetch_query_page_with_timeout_and_cancel<T: DeserializeOwned + Send + 'static>(
|
||||
&mut self,
|
||||
params: Value,
|
||||
timeout_duration: Option<Duration>,
|
||||
cancel_token: Option<CancellationToken>,
|
||||
) -> Result<T, String> {
|
||||
self.call_method_with_timeout_and_cancel(AgentMethod::FetchQueryPage, params, timeout_duration, cancel_token)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn get_explain_info<T: DeserializeOwned + Send + 'static>(&mut self, params: Value) -> Result<T, String> {
|
||||
self.call_method(AgentMethod::GetExplainInfo, params).await
|
||||
}
|
||||
@@ -1153,6 +1240,7 @@ mod tests {
|
||||
assert_eq!(AgentMethod::Handshake.as_str(), "handshake");
|
||||
assert_eq!(AgentMethod::Connect.as_str(), "connect");
|
||||
assert_eq!(AgentMethod::TestConnection.as_str(), "test_connection");
|
||||
assert_eq!(AgentMethod::ValidateConnection.as_str(), "validate_connection");
|
||||
assert_eq!(AgentMethod::ListDatabases.as_str(), "list_databases");
|
||||
assert_eq!(AgentMethod::ListSchemas.as_str(), "list_schemas");
|
||||
assert_eq!(AgentMethod::ListTables.as_str(), "list_tables");
|
||||
|
||||
@@ -9,13 +9,15 @@ use rustls::crypto::{verify_tls12_signature, verify_tls13_signature, CryptoProvi
|
||||
use rustls::pki_types::{CertificateDer, PrivateKeyDer, ServerName, UnixTime};
|
||||
use rustls::server::ParsedCertificate;
|
||||
use std::fs::File;
|
||||
use std::future::Future;
|
||||
use std::io::BufReader;
|
||||
use std::str::FromStr;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio_postgres::config::SslMode;
|
||||
use tokio_postgres::types::{FromSql, Type};
|
||||
use tokio_postgres::{Row, SimpleQueryMessage};
|
||||
use tokio_postgres::{NoTls, Row, SimpleQueryMessage};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use super::file_validator::validate_file_path;
|
||||
use crate::sql::starts_with_executable_sql_keyword;
|
||||
@@ -1330,6 +1332,24 @@ pub async fn execute_query_with_max_rows(
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn execute_query_with_max_rows_and_cancel(
|
||||
pool: &Pool,
|
||||
sql: &str,
|
||||
max_rows: Option<usize>,
|
||||
cancel_token: Option<CancellationToken>,
|
||||
timeout_duration: Option<Duration>,
|
||||
) -> Result<QueryResult, String> {
|
||||
let client = pool.get().await.map_err(|e| e.to_string())?;
|
||||
let pg_cancel_token = client.cancel_token();
|
||||
wait_postgres_query(
|
||||
pg_cancel_token,
|
||||
cancel_token,
|
||||
timeout_duration,
|
||||
execute_query_with_max_rows_inner(&client, sql, max_rows),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn execute_query_with_schema(pool: &Pool, schema: &str, sql: &str) -> Result<QueryResult, String> {
|
||||
execute_query_with_schema_and_max_rows(pool, schema, sql, None).await
|
||||
}
|
||||
@@ -1389,6 +1409,131 @@ pub async fn execute_query_with_schema_and_max_rows(
|
||||
result
|
||||
}
|
||||
|
||||
pub async fn execute_query_with_schema_and_max_rows_and_cancel(
|
||||
pool: &Pool,
|
||||
schema: &str,
|
||||
sql: &str,
|
||||
max_rows: Option<usize>,
|
||||
cancel_token: Option<CancellationToken>,
|
||||
timeout_duration: Option<Duration>,
|
||||
) -> Result<QueryResult, String> {
|
||||
let start = Instant::now();
|
||||
let checkout_start = Instant::now();
|
||||
let client = pool.get().await.map_err(|e| e.to_string())?;
|
||||
log::info!(
|
||||
"[postgres][execute_with_schema:pool:done] elapsed_ms={} total_ms={} schema={}",
|
||||
checkout_start.elapsed().as_millis(),
|
||||
start.elapsed().as_millis(),
|
||||
schema
|
||||
);
|
||||
if is_transaction_recovery_statement(sql) {
|
||||
log::info!(
|
||||
"[postgres][execute_with_schema:skip-search-path] total_ms={} reason=transaction-recovery",
|
||||
start.elapsed().as_millis()
|
||||
);
|
||||
let pg_cancel_token = client.cancel_token();
|
||||
return wait_postgres_query(
|
||||
pg_cancel_token,
|
||||
cancel_token,
|
||||
timeout_duration,
|
||||
execute_query_with_max_rows_inner(&client, sql, max_rows),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
let set_schema_start = Instant::now();
|
||||
client.execute(&format!("SET search_path TO {}", pg_quote_ident(schema)), &[]).await.map_err(pg_error_to_string)?;
|
||||
log::info!(
|
||||
"[postgres][execute_with_schema:set-search-path:done] elapsed_ms={} total_ms={}",
|
||||
set_schema_start.elapsed().as_millis(),
|
||||
start.elapsed().as_millis()
|
||||
);
|
||||
|
||||
let query_start = Instant::now();
|
||||
let pg_cancel_token = client.cancel_token();
|
||||
let result = wait_postgres_query(
|
||||
pg_cancel_token,
|
||||
cancel_token,
|
||||
timeout_duration,
|
||||
execute_query_with_max_rows_inner(&client, sql, max_rows),
|
||||
)
|
||||
.await;
|
||||
if result.is_ok() {
|
||||
clear_postgres_caches_after_ddl(pool, Some(&client), sql);
|
||||
}
|
||||
log::info!(
|
||||
"[postgres][execute_with_schema:query:done] elapsed_ms={} total_ms={} ok={}",
|
||||
query_start.elapsed().as_millis(),
|
||||
start.elapsed().as_millis(),
|
||||
result.is_ok()
|
||||
);
|
||||
|
||||
let reset_start = Instant::now();
|
||||
let _ = client.execute("RESET search_path", &[]).await;
|
||||
log::info!(
|
||||
"[postgres][execute_with_schema:reset-search-path:done] elapsed_ms={} total_ms={}",
|
||||
reset_start.elapsed().as_millis(),
|
||||
start.elapsed().as_millis()
|
||||
);
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
async fn wait_postgres_query<T, F>(
|
||||
pg_cancel_token: tokio_postgres::CancelToken,
|
||||
cancel_token: Option<CancellationToken>,
|
||||
timeout_duration: Option<Duration>,
|
||||
future: F,
|
||||
) -> Result<T, String>
|
||||
where
|
||||
F: Future<Output = Result<T, String>>,
|
||||
{
|
||||
match (cancel_token, timeout_duration) {
|
||||
(Some(token), Some(duration)) => {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = token.cancelled() => {
|
||||
cancel_postgres_query(pg_cancel_token).await;
|
||||
Err(crate::query::canceled_error())
|
||||
}
|
||||
result = tokio::time::timeout(duration, future) => match result {
|
||||
Ok(result) => result,
|
||||
Err(_) => {
|
||||
cancel_postgres_query(pg_cancel_token).await;
|
||||
Err(format!("Query timed out after {} seconds", duration.as_secs()))
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
(None, Some(duration)) => match tokio::time::timeout(duration, future).await {
|
||||
Ok(result) => result,
|
||||
Err(_) => {
|
||||
cancel_postgres_query(pg_cancel_token).await;
|
||||
Err(format!("Query timed out after {} seconds", duration.as_secs()))
|
||||
}
|
||||
},
|
||||
(Some(token), None) => {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = token.cancelled() => {
|
||||
cancel_postgres_query(pg_cancel_token).await;
|
||||
Err(crate::query::canceled_error())
|
||||
}
|
||||
result = future => result,
|
||||
}
|
||||
}
|
||||
(None, None) => future.await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn cancel_postgres_query(pg_cancel_token: tokio_postgres::CancelToken) {
|
||||
match tokio::time::timeout(Duration::from_secs(2), pg_cancel_token.cancel_query(NoTls)).await {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(err)) => log::warn!("Failed to send PostgreSQL cancel request: {err}"),
|
||||
Err(_) => log::warn!("Timed out sending PostgreSQL cancel request"),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_transaction_recovery_statement(sql: &str) -> bool {
|
||||
starts_with_executable_sql_keyword(sql, &["ROLLBACK", "ABORT", "COMMIT", "END"])
|
||||
}
|
||||
|
||||
@@ -877,21 +877,21 @@ pub async fn do_execute(
|
||||
let p = p.clone();
|
||||
let schema = schema.map(|s| s.to_string());
|
||||
let max_rows = options.max_rows;
|
||||
let query_timeout = query_timeout;
|
||||
drop(connections);
|
||||
if let Some(schema) = schema {
|
||||
wait_for_query_opt(
|
||||
db::postgres::execute_query_with_schema_and_max_rows_and_cancel(
|
||||
&p,
|
||||
&schema,
|
||||
sql,
|
||||
max_rows,
|
||||
cancel_token,
|
||||
query_timeout,
|
||||
db::postgres::execute_query_with_schema_and_max_rows(&p, &schema, sql, max_rows),
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
wait_for_query_opt(
|
||||
cancel_token,
|
||||
query_timeout,
|
||||
db::postgres::execute_query_with_max_rows(&p, sql, max_rows),
|
||||
)
|
||||
.await
|
||||
db::postgres::execute_query_with_max_rows_and_cancel(&p, sql, max_rows, cancel_token, query_timeout)
|
||||
.await
|
||||
}
|
||||
}
|
||||
PoolKind::Sqlite(p) => {
|
||||
@@ -1024,21 +1024,39 @@ pub async fn do_execute(
|
||||
let max_rows = options.max_rows;
|
||||
let rpc_timeout = query_timeout;
|
||||
drop(connections);
|
||||
let result = wait_for_query_opt(cancel_token, query_timeout, async move {
|
||||
let mut client = client.lock().await;
|
||||
if is_canceled(&cancel_token) {
|
||||
return Err(canceled_error());
|
||||
}
|
||||
let cancel_for_agent = cancel_token.clone();
|
||||
let result = async move {
|
||||
let mut client = match cancel_for_agent.as_ref() {
|
||||
Some(token) => {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = token.cancelled() => return Err(canceled_error()),
|
||||
guard = client.lock() => guard,
|
||||
}
|
||||
}
|
||||
None => client.lock().await,
|
||||
};
|
||||
if let Some(session_id) = options.result_session_id.as_deref() {
|
||||
let params = agent_fetch_query_page_params(session_id, options.page_size.unwrap_or(MAX_ROWS));
|
||||
client.fetch_query_page_with_timeout(params, rpc_timeout).await
|
||||
client.fetch_query_page_with_timeout_and_cancel(params, rpc_timeout, cancel_for_agent.clone()).await
|
||||
} else if options.page_size.is_some() {
|
||||
let params = agent_execute_query_page_params(&sql, database.as_deref(), schema.as_deref(), options);
|
||||
client.execute_query_page_with_timeout(params, rpc_timeout).await
|
||||
client
|
||||
.execute_query_page_with_timeout_and_cancel(params, rpc_timeout, cancel_for_agent.clone())
|
||||
.await
|
||||
} else {
|
||||
let params = agent_execute_query_params(&sql, database.as_deref(), schema.as_deref(), options);
|
||||
client.execute_query_with_timeout(params, rpc_timeout).await
|
||||
client.execute_query_with_timeout_and_cancel(params, rpc_timeout, cancel_for_agent.clone()).await
|
||||
}
|
||||
})
|
||||
}
|
||||
.await
|
||||
.map(|result| normalize_query_result_for_js(truncate_result_with_max_rows(result, max_rows)));
|
||||
if matches!(result.as_ref(), Err(err) if err == QUERY_CANCELED) {
|
||||
state.remove_pool_by_key(pool_key).await;
|
||||
}
|
||||
if matches!(result.as_ref(), Err(err) if should_discard_pool_after_error(pool_db_type, err)) {
|
||||
state.remove_pool_by_key(pool_key).await;
|
||||
}
|
||||
|
||||
@@ -2112,6 +2112,20 @@ pub async fn get_object_source_core(
|
||||
schema: &str,
|
||||
name: &str,
|
||||
object_type: db::ObjectSourceKind,
|
||||
) -> Result<db::ObjectSource, String> {
|
||||
retry_metadata_connection(state, connection_id, Some(database), || {
|
||||
get_object_source_once(state, connection_id, database, schema, name, object_type.clone())
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_object_source_once(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
database: &str,
|
||||
schema: &str,
|
||||
name: &str,
|
||||
object_type: db::ObjectSourceKind,
|
||||
) -> Result<db::ObjectSource, String> {
|
||||
let pool_key = state.get_or_create_pool(connection_id, Some(database)).await?;
|
||||
let db_config = connection_config(state, connection_id).await;
|
||||
|
||||
Reference in new issue
Block a user