feat(export): run data dictionary PDF exports in background

This commit is contained in:
ccq authored and GitHub committed 2026-09-26 00:04:52 +08:00
1 parent 4baec21fb7
commit 705ea7464d
18 files changed
+702 -104

No files matched your search

+91 -74
View File
@@ -1,5 +1,5 @@
use std::collections::{BTreeMap, HashSet};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use futures::stream::{self, StreamExt};
@@ -144,6 +144,18 @@ pub async fn collect_snapshot(
options: &CollectOptions,
progress: &(dyn Fn(CollectProgress) + Send + Sync),
cancel: &AtomicBool,
) -> Result<SchemaSnapshot, String> {
collect_snapshot_with_concurrency(state, connection, options, progress, cancel, MAX_CONCURRENT_TABLES).await
}
/// Dictionary exports use a smaller limit to leave pool capacity for interactive queries.
pub async fn collect_snapshot_with_concurrency(
state: &AppState,
connection: &ConnectionConfig,
options: &CollectOptions,
progress: &(dyn Fn(CollectProgress) + Send + Sync),
cancel: &AtomicBool,
max_concurrent_tables: usize,
) -> Result<SchemaSnapshot, String> {
let mut warnings: Vec<SnapshotWarning> = Vec::new();
let engine = database_type_label(connection.db_type);
@@ -199,64 +211,64 @@ pub async fn collect_snapshot(
let total = targets.len();
progress(CollectProgress { completed: 0, total, current: String::new() });
let completed = AtomicUsize::new(0);
let collected: Vec<Result<(DocTable, Vec<SnapshotWarning>), SnapshotWarning>> =
stream::iter(targets.into_iter().enumerate())
.map(|(index, (schema_name, info))| {
.map(|(_index, (schema_name, info))| {
let database = options.database.clone();
let completed = &completed;
async move {
if cancelled(cancel) {
return Err(SnapshotWarning::TableSkipped {
table: info.name.clone(),
reason: "cancelled".to_string(),
});
}
progress(CollectProgress {
completed: index,
total,
current: format!("{schema_name}.{}", info.name),
});
let columns = schema::get_columns_core(state, connection_id, &database, &schema_name, &info.name)
.await
.map_err(|error| SnapshotWarning::TableSkipped {
table: format!("{schema_name}.{}", info.name),
reason: error,
})?;
let column_notes = database_column_notes(&columns);
let mut table_warnings = Vec::new();
// Indexes feed `relations.rs`'s uniqueness check, so a
// failure here must not look identical to "this table
// genuinely has no indexes" — that would silently
// downgrade a OneToOne relationship to ManyToOne.
let indexes = match schema::list_indexes_core(
state,
connection_id,
&database,
&schema_name,
&info.name,
)
.await
{
Ok(indexes) => indexes,
Err(error) => {
table_warnings.push(SnapshotWarning::TableSkipped {
table: format!("{schema_name}.{}", info.name),
reason: format!("indexes unavailable: {error}"),
let current = format!("{schema_name}.{}", info.name);
let outcome = async {
if cancelled(cancel) {
return Err(SnapshotWarning::TableSkipped {
table: info.name.clone(),
reason: "cancelled".to_string(),
});
Vec::new()
}
};
// Foreign keys also degrade to empty rather than failing the
// table, but a real query failure must not look identical to
// "this table genuinely has no foreign keys" — it is reported
// as its own warning instead of being silently discarded.
let foreign_keys =
match schema::list_foreign_keys_core(state, connection_id, &database, &schema_name, &info.name)
.await
let columns =
schema::get_columns_core(state, connection_id, &database, &schema_name, &info.name)
.await
.map_err(|error| SnapshotWarning::TableSkipped {
table: format!("{schema_name}.{}", info.name),
reason: error,
})?;
let column_notes = database_column_notes(&columns);
let mut table_warnings = Vec::new();
// Indexes feed `relations.rs`'s uniqueness check, so a
// failure here must not look identical to "this table
// genuinely has no indexes" — that would silently
// downgrade a OneToOne relationship to ManyToOne.
let indexes =
match schema::list_indexes_core(state, connection_id, &database, &schema_name, &info.name)
.await
{
Ok(indexes) => indexes,
Err(error) => {
table_warnings.push(SnapshotWarning::TableSkipped {
table: format!("{schema_name}.{}", info.name),
reason: format!("indexes unavailable: {error}"),
});
Vec::new()
}
};
// Foreign keys also degrade to empty rather than failing the
// table, but a real query failure must not look identical to
// "this table genuinely has no foreign keys" — it is reported
// as its own warning instead of being silently discarded.
let foreign_keys = match schema::list_foreign_keys_core(
state,
connection_id,
&database,
&schema_name,
&info.name,
)
.await
{
Ok(keys) => keys,
Err(error) => {
@@ -268,31 +280,36 @@ pub async fn collect_snapshot(
}
};
Ok((
DocTable {
schema: (!schema_name.is_empty()).then(|| schema_name.clone()),
name: info.name.clone(),
kind: table_kind_from(&info.table_type),
columns,
indexes,
foreign_keys,
group_id: None,
note: info.comment.clone().filter(|value| !value.trim().is_empty()),
note_source: if info.comment.as_deref().is_some_and(|v| !v.trim().is_empty()) {
NoteSource::Database
} else {
NoteSource::None
Ok((
DocTable {
schema: (!schema_name.is_empty()).then(|| schema_name.clone()),
name: info.name.clone(),
kind: table_kind_from(&info.table_type),
columns,
indexes,
foreign_keys,
group_id: None,
note: info.comment.clone().filter(|value| !value.trim().is_empty()),
note_source: if info.comment.as_deref().is_some_and(|v| !v.trim().is_empty()) {
NoteSource::Database
} else {
NoteSource::None
},
shadowed_note: None,
column_notes,
estimated_rows: None,
view_definition: None,
},
shadowed_note: None,
column_notes,
estimated_rows: None,
view_definition: None,
},
table_warnings,
))
table_warnings,
))
}
.await;
let done = completed.fetch_add(1, Ordering::SeqCst) + 1;
progress(CollectProgress { completed: done, total, current });
outcome
}
})
.buffer_unordered(MAX_CONCURRENT_TABLES)
.buffer_unordered(max_concurrent_tables.max(1))
.collect()
.await;
+1 -1
View File
@@ -7,7 +7,7 @@ pub mod keys;
pub mod relations;
pub mod snapshot;
pub use collector::{collect_snapshot, CollectOptions, CollectProgress};
pub use collector::{collect_snapshot, collect_snapshot_with_concurrency, CollectOptions, CollectProgress};
pub use color::hue_to_hex;
pub use dbml::{to_dbml, DbmlOutput};
pub use export::{to_standalone_html, EXPORT_LANGUAGES};
+12 -4
View File
@@ -21,6 +21,8 @@ pub struct DocsSnapshotRequest {
pub tables: Vec<String>,
#[serde(default)]
pub project_name: Option<String>,
#[serde(default)]
pub max_concurrent_tables: Option<usize>,
}
async fn load_connection(state: &Arc<WebState>, connection_id: &str) -> Result<ConnectionConfig, AppError> {
@@ -48,10 +50,16 @@ pub async fn collect_snapshot(
project_name: request.project_name.clone().unwrap_or_else(|| connection.name.clone()),
};
let snapshot =
dbx_core::docs::collect_snapshot(&state.app, &connection, &options, &|_progress| {}, &AtomicBool::new(false))
.await
.map_err(AppError::from)?;
let snapshot = dbx_core::docs::collect_snapshot_with_concurrency(
&state.app,
&connection,
&options,
&|_progress| {},
&AtomicBool::new(false),
request.max_concurrent_tables.unwrap_or(8).clamp(1, 8),
)
.await
.map_err(AppError::from)?;
Ok(Json(snapshot))
}