feat(backup): support exact table selection by database and schema

This commit is contained in:
zipg authored and GitHub committed 2026-10-02 01:37:41 +08:00
1 parent 738160523a
commit 312673cadf
27 files changed
+741 -17

No files matched your search

+31 -1
View File
@@ -233,6 +233,7 @@ impl BackupService {
}
// Metadata discovery may be slow; finish it before starting the snapshot idle timeout.
let mut targets = Vec::new();
let mut found_selected_tables = std::collections::HashSet::new();
for schema_name in &schemas {
check_cancel(stop)?;
let mut selected_tables = Vec::new();
@@ -250,7 +251,21 @@ impl BackupService {
None,
)
.await?;
let exact_names = job
.config
.selected_tables
.iter()
.filter(|target| target.database == *database && target.schema == *schema_name)
.map(|target| target.table.as_str())
.collect::<std::collections::HashSet<_>>();
for table in &tables {
if job.config.table_filter_mode == "selected" {
if exact_names.contains(table.name.as_str()) {
selected_tables.push(table.name.clone());
found_selected_tables.insert((schema_name.as_str(), table.name.clone()));
}
continue;
}
let matched = job
.config
.table_patterns
@@ -264,7 +279,7 @@ impl BackupService {
}
}
}
let included = if job.config.table_filter_mode == "include" {
let included = if matches!(job.config.table_filter_mode.as_str(), "include" | "selected") {
selected_tables.len()
} else {
tables.len() - excluded_tables.len()
@@ -276,6 +291,21 @@ impl BackupService {
}
targets.push((schema_name.clone(), selected_tables, excluded_tables));
}
if job.config.table_filter_mode == "selected" {
let missing = job
.config
.selected_tables
.iter()
.filter(|target| {
target.database == *database
&& !found_selected_tables.contains(&(target.schema.as_str(), target.table.clone()))
})
.map(|target| format!("{}.{}.{}", target.database, target.schema, target.table))
.collect::<Vec<_>>();
if !missing.is_empty() {
return Err(format!("Selected backup tables are unavailable: {}", missing.join(", ")));
}
}
if targets.is_empty() {
continue;
}
+1 -1
View File
@@ -6,7 +6,7 @@ mod store;
#[cfg(test)]
mod tests;
pub use models::{BackupConfig, BackupFile, BackupRun, BackupSchedule, Migration, RunRequest};
pub use models::{BackupConfig, BackupFile, BackupRun, BackupSchedule, BackupTableTarget, Migration, RunRequest};
pub use service::{BackupCommand, BackupService};
pub use store::{BackupSnapshot, BackupStore};
+28 -2
View File
@@ -5,6 +5,14 @@ use serde::{Deserialize, Serialize};
pub const DEFAULT_DIRECTORY: &str = "dbx-backup__{schedule}__{timestamp}__{runId}";
pub const DEFAULT_FILE: &str = "dbx-backup__{schedule}__{timestamp}__{database}__{runId}";
#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct BackupTableTarget {
pub database: String,
pub schema: String,
pub table: String,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct BackupConfig {
@@ -15,6 +23,8 @@ pub struct BackupConfig {
pub table_filter_mode: String,
#[serde(default)]
pub table_patterns: Vec<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub selected_tables: Vec<BackupTableTarget>,
pub destination_directory: String,
pub include_structure: bool,
pub include_data: bool,
@@ -145,7 +155,7 @@ impl BackupConfig {
if !matches!(self.output_compression.as_str(), "none" | "gzip") {
return Err("Unsupported backup compression".into());
}
if !matches!(self.table_filter_mode.as_str(), "all" | "include" | "exclude") {
if !matches!(self.table_filter_mode.as_str(), "all" | "include" | "exclude" | "selected") {
return Err("Unsupported backup table filter".into());
}
if self.databases.len() > 10_000 || self.table_patterns.len() > 1000 {
@@ -154,9 +164,25 @@ impl BackupConfig {
for value in self.databases.iter().chain(self.table_patterns.iter()) {
bounded_text(value, 1024, "Backup target")?;
}
if self.table_filter_mode != "all" && self.table_patterns.is_empty() {
if matches!(self.table_filter_mode.as_str(), "include" | "exclude") && self.table_patterns.is_empty() {
return Err("Select backup table patterns".into());
}
if self.table_filter_mode == "selected" {
if self.databases.is_empty() || self.selected_tables.is_empty() {
return Err("Select backup databases and tables".into());
}
if self.selected_tables.len() > 10_000 {
return Err("Too many selected backup tables".into());
}
for target in &self.selected_tables {
bounded_text(&target.database, 1024, "Backup table database")?;
bounded_text(&target.schema, 1024, "Backup table schema")?;
bounded_text(&target.table, 1024, "Backup table name")?;
if !self.databases.contains(&target.database) {
return Err("Selected backup table is outside the database scope".into());
}
}
}
validate_template(self.file_name_pattern.as_deref().unwrap_or(DEFAULT_FILE), false)
}
}
@@ -56,6 +56,29 @@ fn invalid_schedule_and_path_templates_are_rejected() {
assert!(super::models::matches_pattern("PUBLIC.*", "users", "app", "public", false));
}
#[test]
fn exact_backup_scope_preserves_literal_names_and_rejects_empty_or_foreign_scope() {
let dir = tempfile::tempdir().unwrap();
let mut config = schedule(dir.path()).config;
let legacy = serde_json::to_value(&config).unwrap();
assert!(legacy.get("selectedTables").is_none());
assert!(serde_json::from_value::<BackupConfig>(legacy).unwrap().selected_tables.is_empty());
config.table_filter_mode = "selected".into();
assert!(config.validate().is_err());
config.databases = vec!["app".into()];
assert!(config.validate().is_err());
config.selected_tables =
vec![BackupTableTarget { database: "app".into(), schema: "a.b".into(), table: " odd*,?;客户'表 ".into() }];
assert!(config.validate().is_ok());
let roundtrip: BackupConfig = serde_json::from_value(serde_json::to_value(&config).unwrap()).unwrap();
assert_eq!(roundtrip.selected_tables, config.selected_tables);
config.selected_tables[0].database = "other".into();
assert!(config.validate().unwrap_err().contains("outside"));
config.selected_tables[0].database = "app".into();
config.selected_tables[0].schema = String::new();
assert!(config.validate().is_err());
}
#[tokio::test]
async fn schedule_survives_reopen_and_rejects_stale_edit() {
let dir = tempfile::tempdir().unwrap();
@@ -438,6 +461,42 @@ async fn live_mysql_worker_scenario() {
assert!(completed.error.as_deref().unwrap().contains("No tables matched"));
assert!(completed.files.is_empty());
assert!(previous.unwrap().exists());
execute_sql_statement(
&service.state,
"mysql",
&database,
"CREATE TABLE `odd*,name.with.dot` (id INT PRIMARY KEY, value TEXT)",
None,
None,
)
.await?;
execute_sql_statement(
&service.state,
"mysql",
&database,
"INSERT INTO `odd*,name.with.dot` VALUES (17, 'mysql-exact-scope-sentinel')",
None,
None,
)
.await?;
let mut exact = plan_with_id(dir.path(), "mysql-exact");
exact.config.connection_id = "mysql".into();
exact.config.databases = vec![database.clone()];
exact.config.table_filter_mode = "selected".into();
exact.config.selected_tables = vec![BackupTableTarget {
database: database.clone(),
schema: database.clone(),
table: "odd*,name.with.dot".into(),
}];
service.command(BackupCommand::Save { schedule: exact }).await?;
let exact_run = enqueue_and_run(&service, "mysql-exact").await;
assert_eq!(exact_run.status, "success", "{:?}", exact_run.error);
assert_eq!(exact_run.files.len(), 1);
let exact_sql = read_backup(&exact_run.files[0].file_path, false);
assert!(exact_sql.contains("mysql-exact-scope-sentinel"));
assert!(exact_sql.contains("odd*,name.with.dot"));
assert!(!exact_sql.contains("CREATE TABLE `chosen`"));
assert!(!exact_sql.contains("CREATE TABLE `skipped`"));
Ok::<_, String>(())
})
.catch_unwind()
@@ -533,6 +592,7 @@ fn live_postgres_worker_exports_selected_tables_across_schemas() {
service.state.storage.save_connections(std::slice::from_ref(&admin)).await.unwrap();
service.state.configs.write().await.insert(admin.id.clone(), admin);
let database = format!("dbx_pg_backup_{}", uuid::Uuid::new_v4().simple());
eprintln!("Backup source test database: {database}");
execute_sql_statement(
&service.state,
admin_id,
@@ -591,6 +651,129 @@ fn live_postgres_worker_exports_selected_tables_across_schemas() {
// A view inside the same schema is not part of a table-name scope.
assert!(!sql.contains("big_orders"), "{sql}");
execute_sql_statement(
&service.state,
"postgres",
&database,
"CREATE TABLE app.\"odd*,name.with.dot\" (id integer PRIMARY KEY, note text)",
None,
None,
)
.await
.unwrap();
execute_sql_statement(
&service.state,
"postgres",
&database,
"CREATE TABLE audit.\"odd*,name.with.dot\" (id integer PRIMARY KEY, note text)",
None,
None,
)
.await
.unwrap();
execute_sql_statement(
&service.state,
"postgres",
&database,
"INSERT INTO app.\"odd*,name.with.dot\" VALUES (17, 'exact-scope-sentinel')",
None,
None,
)
.await
.unwrap();
let mut exact = plan_with_id(dir.path(), "exact");
exact.config.connection_id = "postgres".into();
exact.config.databases = vec![database.clone()];
exact.config.table_filter_mode = "selected".into();
exact.config.selected_tables = vec![BackupTableTarget {
database: database.clone(),
schema: "app".into(),
table: "odd*,name.with.dot".into(),
}];
service.command(BackupCommand::Save { schedule: exact.clone() }).await.unwrap();
let exact_run = enqueue_and_run(&service, "exact").await;
assert_eq!(exact_run.status, "success", "{:?}", exact_run.error);
assert_eq!(exact_run.files.len(), 1);
assert_eq!(exact_run.files[0].schema, "app");
let exact_sql = read_backup(&exact_run.files[0].file_path, false);
assert!(exact_sql.contains("odd*,name.with.dot"));
assert!(exact_sql.contains("exact-scope-sentinel"));
assert!(!exact_sql.contains("orders"));
assert!(!exact_sql.contains("accounts"));
assert!(!exact_sql.contains("audit"));
// Replay the real exported SQL into another disposable database.
let restore_database = format!("dbx_pg_backup_restore_{}", uuid::Uuid::new_v4().simple());
eprintln!("Backup replay test database: {restore_database}");
execute_sql_statement(
&service.state,
admin_id,
"postgres",
&format!("CREATE DATABASE \"{restore_database}\""),
None,
None,
)
.await
.unwrap();
let restore_config = live_postgres_config("postgres-restore", &restore_database);
service.state.storage.save_connections(std::slice::from_ref(&restore_config)).await.unwrap();
service.state.configs.write().await.insert(restore_config.id.clone(), restore_config);
// Selected-table dumps retain their source schema. Prepare the matching
// namespace, as a restore into an existing application database does.
execute_sql_statement(&service.state, "postgres-restore", &restore_database, "CREATE SCHEMA app", None, None)
.await
.unwrap();
for statement in
crate::sql::split_sql_statements_for_database(&exact_sql, crate::models::connection::DatabaseType::Postgres)
{
execute_sql_statement(&service.state, "postgres-restore", &restore_database, &statement, None, None)
.await
.unwrap();
}
let restored = execute_sql_statement(
&service.state,
"postgres-restore",
&restore_database,
"SELECT id, note FROM app.\"odd*,name.with.dot\"",
None,
None,
)
.await
.unwrap();
assert_eq!(restored.rows, vec![vec![json!(17), json!("exact-scope-sentinel")]]);
let absent = execute_sql_statement(
&service.state,
"postgres-restore",
&restore_database,
"SELECT to_regclass('app.orders'), to_regclass('public.accounts')",
None,
None,
)
.await
.unwrap();
assert_eq!(absent.rows, vec![vec![serde_json::Value::Null, serde_json::Value::Null]]);
execute_sql_statement(
&service.state,
admin_id,
"postgres",
&format!("DROP DATABASE \"{restore_database}\" WITH (FORCE)"),
None,
None,
)
.await
.unwrap();
exact.id = "exact-missing".into();
exact.config.selected_tables.push(BackupTableTarget {
database: database.clone(),
schema: "missing_schema".into(),
table: "not_there".into(),
});
service.command(BackupCommand::Save { schedule: exact }).await.unwrap();
let missing_run = enqueue_and_run(&service, "exact-missing").await;
assert_eq!(missing_run.status, "failed");
assert!(missing_run.error.as_deref().unwrap().contains("unavailable"));
assert!(missing_run.files.is_empty());
// Patterns are case-sensitive on PostgreSQL, and a scope that matches
// nothing must fail without leaving a partial file behind.
let mut empty = plan_with_id(dir.path(), "empty");