feat(query): add manual transaction controls

This commit is contained in:
Abeautifulsnow authored and GitHub committed 2026-07-02 01:30:31 +08:00
1 parent f871e00330
commit d55a708d6a
27 files changed
+927 -40

No files matched your search

+19 -1
View File
@@ -2,7 +2,7 @@ use std::collections::{HashMap, HashSet};
use std::path::PathBuf;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::RwLock;
use tokio::sync::{Mutex, RwLock};
use tokio::task::JoinHandle;
use mysql_async::prelude::Queryable;
@@ -96,6 +96,22 @@ pub enum PoolKind {
Nacos,
}
/// Held connection for a manual transaction session
pub enum TxnConnection {
Postgres(Box<deadpool_postgres::Object>),
Mysql(mysql_async::Conn),
}
pub struct TransactionSession {
pub connection: Arc<Mutex<TxnConnection>>,
pub pool_key: String,
pub last_activity: std::time::Instant,
pub busy: bool,
pub connection_id: String,
pub database: String,
pub schema: Option<String>,
}
macro_rules! agent_connection_pool_database_type {
() => {
DatabaseType::Dameng
@@ -152,6 +168,7 @@ pub struct AppState {
/// PostgreSQL TLS cancel context, keyed by pool_key.
/// Used to reconstruct a TLS connector compatible with the original connection when cancelling.
postgres_cancel_contexts: Arc<RwLock<HashMap<String, db::postgres::PostgresCancelContext>>>,
pub transaction_sessions: Arc<RwLock<HashMap<String, TransactionSession>>>,
#[cfg(feature = "mq-admin")]
pub mq_registry: crate::mq::MqAdminRegistry,
}
@@ -419,6 +436,7 @@ impl AppState {
),
nacos_registry: crate::nacos::NacosAdminRegistry::new(),
postgres_cancel_contexts: Arc::new(RwLock::new(HashMap::new())),
transaction_sessions: Arc::new(RwLock::new(HashMap::new())),
#[cfg(feature = "mq-admin")]
mq_registry: crate::mq::MqAdminRegistry::new(),
}
+2 -2
View File
@@ -218,7 +218,7 @@ fn mysql_bytes_to_json(bytes: Vec<u8>, column: &mysql_async::Column) -> serde_js
/// Map a MySQL column to a user-facing type name for the result-grid header.
/// Returns the bare lowercase type name (no length/precision/signedness), which
/// is enough for display; unknown variants fall back to a lowercased debug name.
fn mysql_column_type_name(ty: ColumnType) -> String {
pub(crate) fn mysql_column_type_name(ty: ColumnType) -> String {
use mysql_async::consts::ColumnType::*;
match ty {
MYSQL_TYPE_TINY => "tinyint",
@@ -251,7 +251,7 @@ fn mysql_column_type_name(ty: ColumnType) -> String {
.to_string()
}
fn mysql_value_to_json(row: &mysql_async::Row, idx: usize) -> serde_json::Value {
pub(crate) fn mysql_value_to_json(row: &mysql_async::Row, idx: usize) -> serde_json::Value {
let Some(column) = row.columns_ref().get(idx) else {
return serde_json::Value::Null;
};
+1 -1
View File
@@ -304,7 +304,7 @@ fn format_pg_timestamptz(value: DateTime<Local>) -> String {
value.to_rfc3339()
}
fn pg_value_to_json(row: &Row, idx: usize, type_name: &str) -> serde_json::Value {
pub(crate) fn pg_value_to_json(row: &Row, idx: usize, type_name: &str) -> serde_json::Value {
let upper = type_name.to_uppercase();
if upper == "BYTEA" {
+428 -2
View File
@@ -2,6 +2,7 @@
use chrono::{DateTime, Duration as ChronoDuration, NaiveDate, NaiveDateTime, NaiveTime, Utc};
#[cfg(feature = "duckdb-bundled")]
use duckdb::types::{TimeUnit, Value, ValueRef};
use futures::StreamExt;
use mysql_async::prelude::Queryable;
use sqlparser::ast::{visit_relations_mut, Ident, ObjectName, ObjectNamePart, ObjectType, Statement};
use sqlparser::dialect::{GenericDialect, PostgreSqlDialect};
@@ -9,6 +10,7 @@ use sqlparser::parser::Parser;
use std::collections::HashSet;
use std::future::Future;
use std::ops::ControlFlow;
use std::sync::Arc;
use std::time::Duration;
#[cfg(feature = "duckdb-bundled")]
use tokio::task::JoinHandle;
@@ -17,13 +19,13 @@ use tokio::time::sleep;
use tokio::time::timeout;
use tokio_util::sync::CancellationToken;
use crate::connection::{AppState, PoolKind};
use crate::connection::{AppState, PoolKind, TransactionSession, TxnConnection};
use crate::database_capabilities;
use crate::db;
use crate::models::connection::{ConnectionConfig, DatabaseType};
#[cfg(feature = "duckdb-bundled")]
use crate::sql::starts_with_duckdb_result_sql_keyword;
use crate::sql::{split_sql_batches, split_sql_statements};
use crate::sql::{split_sql_batches, split_sql_statements, starts_with_executable_sql_keyword};
pub const QUERY_TIMEOUT: Duration = Duration::from_secs(30);
pub const MAX_ROWS: usize = 10000;
@@ -259,6 +261,10 @@ pub struct QueryExecutionOptions {
/// `Some(0)` disables the timeout entirely.
pub timeout_secs: Option<u64>,
pub execution_id: Option<String>,
/// When `Some(true)`, multiple statements are executed within a single transaction
/// (BEGIN … COMMIT) instead of auto-commit mode. `None` and `Some(false)` behave
/// identically — auto-commit for each statement.
pub use_transaction: Option<bool>,
}
fn query_result_row_limit(max_rows: Option<usize>) -> usize {
@@ -1567,6 +1573,13 @@ pub async fn execute_multi_core_with_options(
return Ok(vec![empty_query_result(0)]);
}
// When use_transaction is explicitly true and we have multiple statements,
// route through the transaction wrapper instead of the sequential auto-commit loop.
if options.use_transaction == Some(true) && statements.len() > 1 {
let result = execute_statements_in_transaction(state, connection_id, database, &statements, schema).await?;
return Ok(vec![result]);
}
let mysql_pool = {
let connections = state.connections.read().await;
match connections.get(&pool_key) {
@@ -2355,6 +2368,401 @@ async fn exec_tx_none_inner(
})
}
/// Start a manual transaction session, holding a connection from the pool.
/// Returns a transaction session ID that must be passed to subsequent calls.
pub async fn begin_manual_transaction(
state: &AppState,
connection_id: &str,
database: &str,
schema: Option<&str>,
) -> Result<String, String> {
let pool_key = if database.is_empty() {
connection_id.to_string()
} else {
state.get_or_create_pool(connection_id, Some(database)).await?
};
// Clone the pool handle under a brief read lock, then drop the lock before
// any async I/O — same pattern as do_execute throughout this file.
enum TxnPoolHandle {
Postgres(deadpool_postgres::Pool),
Mysql(db::mysql::MySqlPool),
}
let pool_handle = {
let connections = state.connections.read().await;
match connections.get(&pool_key).ok_or("Connection not found")? {
PoolKind::Postgres(pg) => TxnPoolHandle::Postgres(pg.clone()),
PoolKind::Mysql(mp, _) => TxnPoolHandle::Mysql(mp.clone()),
_ => return Err("Manual transaction is not supported for this database type".to_string()),
}
}; // connections lock released here
let txn_conn = match pool_handle {
TxnPoolHandle::Postgres(pg_pool) => {
let conn = pg_pool.get().await.map_err(|e| format!("Failed to get Postgres connection: {e}"))?;
conn.execute("BEGIN", &[]).await.map_err(|e| format!("BEGIN failed: {e}"))?;
if let Some(schema) = schema {
conn.execute(&format!("SET LOCAL search_path TO {}", db::postgres::pg_quote_ident(schema)), &[])
.await
.map_err(|e| format!("SET search_path failed: {e}"))?;
}
TxnConnection::Postgres(Box::new(conn))
}
TxnPoolHandle::Mysql(mysql_pool) => {
let mut conn = mysql_pool.get_conn().await.map_err(|e| format!("Failed to get MySQL connection: {e}"))?;
conn.query_drop("START TRANSACTION").await.map_err(|e| format!("START TRANSACTION failed: {e}"))?;
TxnConnection::Mysql(conn)
}
};
let txn_session_id = uuid::Uuid::new_v4().to_string();
let session = TransactionSession {
connection: Arc::new(tokio::sync::Mutex::new(txn_conn)),
pool_key: pool_key.clone(),
last_activity: std::time::Instant::now(),
busy: false,
connection_id: connection_id.to_string(),
database: database.to_string(),
schema: schema.map(|s| s.to_string()),
};
{
let mut sessions = state.transaction_sessions.write().await;
sessions.insert(txn_session_id.clone(), session);
}
// Schedule idle timeout watcher
spawn_txn_idle_watcher(state, txn_session_id.clone());
log::info!("[query][manual_txn:begin] session_id={}", txn_session_id);
Ok(txn_session_id)
}
/// Execute SQL within an existing manual transaction session.
pub async fn execute_in_manual_transaction(
state: &AppState,
txn_session_id: &str,
sql: &str,
_database: &str,
_schema: Option<&str>,
max_rows: Option<usize>,
) -> Result<Vec<db::QueryResult>, String> {
const TXN_IDLE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(300);
// Resolve statements and validate before taking the per-session connection
// lock. The session stays visible in the map so close/disconnect cleanup can
// remove it and roll back once the current DB operation releases the lock.
let (pool_key, connection_id) = {
let sessions = state.transaction_sessions.read().await;
let session = sessions
.get(txn_session_id)
.ok_or("Transaction session not found or expired; it may have been auto-rolled back due to inactivity")?;
(session.pool_key.clone(), session.connection_id.clone())
};
let db_type = connection_database_type(state, &connection_id).await;
let statements = db_type.map_or_else(
|| split_sql_statements(sql),
|db_type| crate::sql::split_sql_statements_for_database(sql, db_type),
);
if statements.is_empty() {
return Ok(vec![empty_query_result(0)]);
}
// Read-only check while the session is still in the map. If this fails the
// session remains intact.
check_read_only_for_connection_multi(state, &pool_key, &statements).await?;
let connection = {
let mut sessions = state.transaction_sessions.write().await;
let Some(session) = sessions.get_mut(txn_session_id) else {
return Err(
"Transaction session not found or expired; it may have been auto-rolled back due to inactivity"
.to_string(),
);
};
if session.busy {
return Err("Transaction session is already executing".to_string());
}
if session.last_activity.elapsed() > TXN_IDLE_TIMEOUT {
let session = sessions.remove(txn_session_id).expect("session exists");
Some(session.connection)
} else {
session.busy = true;
session.last_activity = std::time::Instant::now();
None
}
};
if let Some(connection) = connection {
let mut conn = connection.lock().await;
let _ = rollback_manual_txn_connection(&mut conn).await;
return Err("Transaction was auto-rolled back due to 5 minutes of inactivity".to_string());
}
let connection = {
let sessions = state.transaction_sessions.read().await;
sessions
.get(txn_session_id)
.map(|session| Arc::clone(&session.connection))
.ok_or("Transaction session not found or expired; it may have been auto-rolled back due to inactivity")?
};
let row_limit = max_rows.unwrap_or(MAX_ROWS).max(1);
let mut results = Vec::with_capacity(statements.len());
let mut conn = connection.lock().await;
for (i, statement) in statements.iter().enumerate() {
let result = match &mut *conn {
TxnConnection::Postgres(conn) => {
execute_manual_txn_postgres_statement(conn.as_ref(), statement, row_limit).await
}
TxnConnection::Mysql(conn) => execute_manual_txn_mysql_statement(conn, statement, row_limit).await,
};
match result {
Ok(query_result) => results.push(query_result),
Err(e) => {
// Statement failure ends the transaction. If another cleanup path
// already removed the session, it owns the final rollback.
let should_rollback = {
let mut sessions = state.transaction_sessions.write().await;
sessions.remove(txn_session_id).is_some()
};
if should_rollback {
let _ = rollback_manual_txn_connection(&mut conn).await;
}
return Err(format!("Statement {} failed: {}. Transaction was auto-rolled back.", i + 1, e));
}
}
}
drop(conn);
let should_watch = {
let mut sessions = state.transaction_sessions.write().await;
if let Some(session) = sessions.get_mut(txn_session_id) {
session.busy = false;
session.last_activity = std::time::Instant::now();
true
} else {
false
}
};
if should_watch {
spawn_txn_idle_watcher(state, txn_session_id.to_string());
}
Ok(results)
}
async fn rollback_manual_txn_connection(conn: &mut TxnConnection) -> Result<(), String> {
match conn {
TxnConnection::Postgres(conn) => {
conn.execute("ROLLBACK", &[]).await.map_err(|e| format!("ROLLBACK failed: {e}"))?;
}
TxnConnection::Mysql(conn) => {
conn.query_drop("ROLLBACK").await.map_err(|e| format!("ROLLBACK failed: {e}"))?;
}
}
Ok(())
}
/// Spawn a background task that removes and rolls back a transaction session
/// after 5 minutes of inactivity. The task does not hold the global lock across
/// I/O: it briefly checks the map, and if the session exists and is expired,
/// removes it, drops the lock, then rolls back the held connection.
///
/// Safety: if multiple watchers exist for the same session ID (e.g. due to
/// a race), only the one that actually finds the session in the map and
/// observes an elapsed time >= timeout will remove and roll back. Others
/// will see a missing session or a non-expired one and exit harmlessly.
fn spawn_txn_idle_watcher(state: &AppState, txn_session_id: String) {
let sessions = Arc::clone(&state.transaction_sessions);
tokio::spawn(async move {
const TXN_IDLE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(300);
tokio::time::sleep(TXN_IDLE_TIMEOUT).await;
let removed: Option<TransactionSession> = {
let mut guard = sessions.write().await;
match guard.get(&txn_session_id) {
Some(session) if !session.busy && session.last_activity.elapsed() >= TXN_IDLE_TIMEOUT => {
guard.remove(&txn_session_id)
}
_ => None,
}
};
if let Some(session) = removed {
let mut conn = session.connection.lock().await;
let _ = rollback_manual_txn_connection(&mut conn).await;
log::info!(
"[query][manual_txn:idle_timeout] session_id={} auto-rolled back after 5 minutes of inactivity",
txn_session_id
);
}
});
}
async fn execute_manual_txn_postgres_statement(
conn: &deadpool_postgres::Object,
sql: &str,
row_limit: usize,
) -> Result<db::QueryResult, String> {
if starts_with_executable_sql_keyword(sql, &["SELECT", "SHOW", "EXPLAIN", "WITH", "TABLE"]) {
let start = std::time::Instant::now();
let stmt = conn.prepare_cached(sql).await.map_err(|e| format!("Prepare failed: {e}"))?;
let columns: Vec<String> = stmt.columns().iter().map(|c| c.name().to_string()).collect();
let column_types: Vec<String> = stmt.columns().iter().map(|c| c.type_().name().to_string()).collect();
let params: Vec<&(dyn tokio_postgres::types::ToSql + Sync)> = Vec::new();
let stream = conn.query_raw(&stmt, params).await.map_err(|e| format!("Query failed: {e}"))?;
tokio::pin!(stream);
let mut data: Vec<Vec<serde_json::Value>> = Vec::with_capacity(row_limit.min(1024));
let mut truncated = false;
while let Some(row_result) = stream.next().await {
if data.len() >= row_limit {
truncated = true;
break;
}
let row = row_result.map_err(|e| format!("Query failed: {e}"))?;
let values: Vec<serde_json::Value> = (0..row.columns().len())
.map(|i| db::postgres::pg_value_to_json(&row, i, column_types.get(i).map(String::as_str).unwrap_or("")))
.collect();
data.push(values);
}
Ok(db::QueryResult {
columns,
column_types,
column_sortables: vec![],
rows: data,
affected_rows: 0,
execution_time_ms: start.elapsed().as_millis(),
truncated,
session_id: None,
has_more: false,
})
} else {
let affected = conn.execute(sql, &[]).await.map_err(|e| format!("Query failed: {e}"))?;
Ok(db::QueryResult {
columns: vec![],
column_types: Vec::new(),
column_sortables: vec![],
rows: vec![],
affected_rows: affected,
execution_time_ms: 0,
truncated: false,
session_id: None,
has_more: false,
})
}
}
async fn execute_manual_txn_mysql_statement(
conn: &mut mysql_async::Conn,
sql: &str,
row_limit: usize,
) -> Result<db::QueryResult, String> {
if starts_with_executable_sql_keyword(sql, &["SELECT", "SHOW", "DESCRIBE", "EXPLAIN", "WITH"]) {
let start = std::time::Instant::now();
let mut result = conn.query_iter(sql).await.map_err(|e| format!("Query failed: {e}"))?;
let columns: Vec<String> = result.columns_ref().iter().map(|c| c.name_str().to_string()).collect();
let column_types: Vec<String> =
result.columns_ref().iter().map(|c| db::mysql::mysql_column_type_name(c.column_type())).collect();
let mut data: Vec<Vec<serde_json::Value>> = Vec::with_capacity(row_limit.min(1024));
let mut stream = result
.stream::<mysql_async::Row>()
.await
.map_err(|e| format!("Query failed: {e}"))?
.ok_or_else(|| "Empty result set stream".to_string())?;
let mut truncated = false;
while let Some(row) = stream.next().await {
if data.len() >= row_limit {
truncated = true;
break;
}
let row = row.map_err(|e| format!("Query failed: {e}"))?;
data.push((0..row.len()).map(|i| db::mysql::mysql_value_to_json(&row, i)).collect());
}
Ok(db::QueryResult {
columns,
column_types,
column_sortables: vec![],
rows: data,
affected_rows: 0,
execution_time_ms: start.elapsed().as_millis(),
truncated,
session_id: None,
has_more: false,
})
} else {
let result = conn.query_iter(sql).await.map_err(|e| format!("Query failed: {e}"))?;
let affected_rows = result.affected_rows();
result.drop_result().await.map_err(|e| format!("Query failed: {e}"))?;
Ok(db::QueryResult {
columns: vec![],
column_types: Vec::new(),
column_sortables: vec![],
rows: vec![],
affected_rows,
execution_time_ms: 0,
truncated: false,
session_id: None,
has_more: false,
})
}
}
/// Commit an existing manual transaction session.
pub async fn commit_manual_transaction(state: &AppState, txn_session_id: &str) -> Result<db::QueryResult, String> {
let session = {
let mut sessions = state.transaction_sessions.write().await;
sessions.remove(txn_session_id).ok_or("Transaction session not found")?
};
let mut conn = session.connection.lock().await;
match &mut *conn {
TxnConnection::Postgres(conn) => {
conn.execute("COMMIT", &[]).await.map_err(|e| format!("COMMIT failed: {e}"))?;
}
TxnConnection::Mysql(conn) => {
conn.query_drop("COMMIT").await.map_err(|e| format!("COMMIT failed: {e}"))?;
}
}
log::info!("[query][manual_txn:commit] session_id={}", txn_session_id);
Ok(db::QueryResult {
columns: vec![],
column_types: Vec::new(),
column_sortables: vec![],
rows: vec![],
affected_rows: 0,
execution_time_ms: 0,
truncated: false,
session_id: None,
has_more: false,
})
}
/// Rollback an existing manual transaction session.
pub async fn rollback_manual_transaction(state: &AppState, txn_session_id: &str) -> Result<db::QueryResult, String> {
let session = {
let mut sessions = state.transaction_sessions.write().await;
sessions.remove(txn_session_id).ok_or("Transaction session not found")?
};
let mut conn = session.connection.lock().await;
rollback_manual_txn_connection(&mut conn).await?;
log::info!("[query][manual_txn:rollback] session_id={}", txn_session_id);
Ok(db::QueryResult {
columns: vec![],
column_types: Vec::new(),
column_sortables: vec![],
rows: vec![],
affected_rows: 0,
execution_time_ms: 0,
truncated: false,
session_id: None,
has_more: false,
})
}
#[cfg(test)]
mod tests {
use super::*;
@@ -3091,4 +3499,22 @@ mod tests {
assert_eq!(normalized.rows[0][0], serde_json::json!("2041797190226354178"));
assert_eq!(normalized.rows[0][1], serde_json::json!([1, "2041797190226354178"]));
}
#[test]
fn query_execution_options_default_use_transaction_is_none() {
let opts = QueryExecutionOptions::default();
assert_eq!(opts.use_transaction, None);
}
#[test]
fn query_execution_options_use_transaction_some_true_is_preserved() {
let opts = QueryExecutionOptions { use_transaction: Some(true), ..Default::default() };
assert_eq!(opts.use_transaction, Some(true));
}
#[test]
fn query_execution_options_use_transaction_some_false_is_preserved() {
let opts = QueryExecutionOptions { use_transaction: Some(false), ..Default::default() };
assert_eq!(opts.use_transaction, Some(false));
}
}
@@ -399,6 +399,7 @@ async fn export_query_result_core_inner(
timeout_secs: request.timeout_secs,
client_session_id: request.client_session_id.clone(),
execution_id: request.execution_id.clone(),
..Default::default()
}
} else {
QueryExecutionOptions {
+1 -9
View File
@@ -876,15 +876,7 @@ pub async fn get_table_comment_core(
&sql,
Some(database),
Some(schema),
QueryExecutionOptions {
max_rows: Some(1),
fetch_size: None,
page_size: None,
result_session_id: None,
client_session_id: None,
timeout_secs: None,
execution_id: None,
},
QueryExecutionOptions { max_rows: Some(1), ..Default::default() },
),
timeout,
)