feat(export): prefetch table metadata during database export

This commit is contained in:
vrustx authored and GitHub committed 2026-07-19 08:50:27 +08:00
1 parent 2891167015
commit 81f787e8d1
2 files changed
+363 -18

No files matched your search

+153 -18
View File
@@ -1057,6 +1057,18 @@ pub async fn begin_database_backup_snapshot_core(
Ok(DatabaseBackupSnapshot { session_id, schemas })
}
/// 只有确认底层支持并发请求的多连接池才允许并发预取导出元数据。
/// SqlServer(Arc<Mutex> 串行客户端)、Agent/ExternalDriver(插件请求超时覆盖
/// 排队时间且超时会终止 sidecar)、SQLite/DuckDB 等单连接类型都必须回退串行。
fn concurrent_metadata_prefetch_allowed(pool_kind: Option<&crate::connection::PoolKind>) -> bool {
matches!(
pool_kind,
Some(crate::connection::PoolKind::Postgres(_))
| Some(crate::connection::PoolKind::Mysql(..))
| Some(crate::connection::PoolKind::ClickHouse(_))
)
}
fn record_export_error(file: &mut std::fs::File, fail_on_error: bool, message: String) -> Result<(), String> {
if fail_on_error {
Err(message)
@@ -1254,7 +1266,89 @@ pub async fn export_database_sql_core(
object_index += 1;
}
for table_info in &tables {
// 预取各表的 DDL 与列元数据:逐表串行往返在多表数据库上是整库导出耗时的
// 主要来源(每表 1-2 次网络往返 × 表数)。有界并发预取后,下方写出循环仍按
// 原顺序消费,文件内容与逐表查询完全一致。
struct PrefetchedTableMetadata {
ddl: Option<Result<String, String>>,
columns: Option<Result<Vec<crate::db::ColumnInfo>, String>>,
}
const EXPORT_METADATA_PREFETCH_CONCURRENCY: usize = 8;
let mut prefetched_table_metadata: Vec<Option<PrefetchedTableMetadata>> = Vec::new();
prefetched_table_metadata.resize_with(tables.len(), || None);
// 防护门按「实际连接池种类」放行,而非数据库类型的能力标记:只有确认底层
// 支持并发请求的多连接池(Postgres/MySQL/ClickHouse)才启用并发预取。反例:
// 原生 SQL Server 是 Arc<Mutex<..>> 串行客户端;Agent/外部 JDBC(含 SQL Server
// legacy profile、PrestoSQL 等路由结果)的插件请求超时覆盖排队时间且超时会
// 终止 sidecar;SQLite/DuckDB 等为单连接。被挡住的场景预取 Vec 保持全 None,
// 写出循环内的 None 回退路径即原有的逐表串行直查行为。
let concurrent_prefetch_is_safe =
match state.get_or_create_pool(&request.connection_id, Some(&request.database)).await {
Ok(metadata_pool_key) => {
concurrent_metadata_prefetch_allowed(state.connections.read().await.get(&metadata_pool_key))
}
// 建池失败时不预取,让写出循环的直查路径按原有方式报告错误
Err(_) => false,
};
if concurrent_prefetch_is_safe
&& (request.include_structure || request.include_data)
&& !tables.is_empty()
&& !is_export_cancelled(&request.export_id).await
{
use futures::StreamExt;
let prefetch_targets: Vec<(usize, String)> =
tables.iter().enumerate().map(|(index, table_info)| (index, table_info.name.clone())).collect();
let mut prefetch_stream =
futures::stream::iter(prefetch_targets.into_iter().map(|(index, table_name)| async move {
if is_export_cancelled_now(&request.export_id) {
return (index, PrefetchedTableMetadata { ddl: None, columns: None });
}
let ddl = if request.include_structure {
Some(
crate::schema::get_table_ddl_core(
state,
&request.connection_id,
&request.database,
&request.schema,
&table_name,
None,
)
.await,
)
} else {
None
};
if is_export_cancelled_now(&request.export_id) {
return (index, PrefetchedTableMetadata { ddl, columns: None });
}
let columns = if request.include_data {
Some(
crate::schema::get_columns_core(
state,
&request.connection_id,
&request.database,
&request.schema,
&table_name,
)
.await,
)
} else {
None
};
(index, PrefetchedTableMetadata { ddl, columns })
}))
.buffer_unordered(EXPORT_METADATA_PREFETCH_CONCURRENCY);
while let Some((index, metadata)) = prefetch_stream.next().await {
prefetched_table_metadata[index] = Some(metadata);
// 取消后不再调度新的预取任务(已在途的任务随 stream 释放而中止),
// 写出循环入口的取消检查负责最终收尾
if is_export_cancelled_now(&request.export_id) {
break;
}
}
}
for (table_index, table_info) in tables.iter().enumerate() {
// Check cancellation
if is_export_cancelled(&request.export_id).await {
on_progress(ExportProgress {
@@ -1309,16 +1403,25 @@ pub async fn export_database_sql_core(
.map_err(|e| format!("Failed to write file: {e}"))?;
object_index += 1;
}
match crate::schema::get_table_ddl_core(
state,
&request.connection_id,
&request.database,
&request.schema,
table_name,
None,
)
.await
let ddl_result = match prefetched_table_metadata
.get_mut(table_index)
.and_then(|m| m.as_mut())
.and_then(|m| m.ddl.take())
{
Some(result) => result,
None => {
crate::schema::get_table_ddl_core(
state,
&request.connection_id,
&request.database,
&request.schema,
table_name,
None,
)
.await
}
};
match ddl_result {
Ok(ddl) => {
let ddl = normalize_export_table_ddl(&ddl, Some(db_type));
writeln!(file, "{};\n", ddl).map_err(|e| format!("Failed to write file: {e}"))?;
@@ -1336,15 +1439,24 @@ pub async fn export_database_sql_core(
// Export data
if request.include_data {
// Get columns
let columns = match crate::schema::get_columns_core(
state,
&request.connection_id,
&request.database,
&request.schema,
table_name,
)
.await
let columns_result = match prefetched_table_metadata
.get_mut(table_index)
.and_then(|m| m.as_mut())
.and_then(|m| m.columns.take())
{
Some(result) => result,
None => {
crate::schema::get_columns_core(
state,
&request.connection_id,
&request.database,
&request.schema,
table_name,
)
.await
}
};
let columns = match columns_result {
Ok(cols) => cols,
Err(e) => {
record_export_error(
@@ -1688,6 +1800,7 @@ fn drop_table_if_exists_sql(table_name: &str, schema: &str, db_type: &DatabaseTy
#[cfg(test)]
mod tests {
use super::concurrent_metadata_prefetch_allowed;
use super::{
build_database_sql_export, build_export_insert_statements, drop_table_if_exists_sql,
filter_selected_table_infos, format_export_sql_literal, generate_postgres_sequence_create_ddl,
@@ -1709,6 +1822,28 @@ mod tests {
}
}
#[test]
fn concurrent_prefetch_only_allowed_for_multi_connection_pools() {
use crate::connection::PoolKind;
use std::sync::Arc;
// ChClient::new 只构造 HTTP 客户端,不发起连接
let clickhouse = PoolKind::ClickHouse(crate::db::clickhouse_driver::ChClient::new(
"http://127.0.0.1:1",
None,
None,
std::time::Duration::from_secs(1),
));
assert!(concurrent_metadata_prefetch_allowed(Some(&clickhouse)));
// Agent(JDBC sidecar)请求超时覆盖排队时间,必须回退串行
let agent =
PoolKind::Agent(Arc::new(tokio::sync::Mutex::new(crate::db::agent_driver::AgentDriverClient::test_stub())));
assert!(!concurrent_metadata_prefetch_allowed(Some(&agent)));
assert!(!concurrent_metadata_prefetch_allowed(None));
}
#[test]
fn filters_export_tables_by_selected_names() {
let tables = vec![table("users", "TABLE"), table("orders", "TABLE"), table("active_users", "VIEW")];