feat(backup): support exact table selection by database and schema
This commit is contained in:
27 files changed
+741
-17
No files matched your search
@@ -233,6 +233,7 @@ impl BackupService {
|
||||
}
|
||||
// Metadata discovery may be slow; finish it before starting the snapshot idle timeout.
|
||||
let mut targets = Vec::new();
|
||||
let mut found_selected_tables = std::collections::HashSet::new();
|
||||
for schema_name in &schemas {
|
||||
check_cancel(stop)?;
|
||||
let mut selected_tables = Vec::new();
|
||||
@@ -250,7 +251,21 @@ impl BackupService {
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
let exact_names = job
|
||||
.config
|
||||
.selected_tables
|
||||
.iter()
|
||||
.filter(|target| target.database == *database && target.schema == *schema_name)
|
||||
.map(|target| target.table.as_str())
|
||||
.collect::<std::collections::HashSet<_>>();
|
||||
for table in &tables {
|
||||
if job.config.table_filter_mode == "selected" {
|
||||
if exact_names.contains(table.name.as_str()) {
|
||||
selected_tables.push(table.name.clone());
|
||||
found_selected_tables.insert((schema_name.as_str(), table.name.clone()));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let matched = job
|
||||
.config
|
||||
.table_patterns
|
||||
@@ -264,7 +279,7 @@ impl BackupService {
|
||||
}
|
||||
}
|
||||
}
|
||||
let included = if job.config.table_filter_mode == "include" {
|
||||
let included = if matches!(job.config.table_filter_mode.as_str(), "include" | "selected") {
|
||||
selected_tables.len()
|
||||
} else {
|
||||
tables.len() - excluded_tables.len()
|
||||
@@ -276,6 +291,21 @@ impl BackupService {
|
||||
}
|
||||
targets.push((schema_name.clone(), selected_tables, excluded_tables));
|
||||
}
|
||||
if job.config.table_filter_mode == "selected" {
|
||||
let missing = job
|
||||
.config
|
||||
.selected_tables
|
||||
.iter()
|
||||
.filter(|target| {
|
||||
target.database == *database
|
||||
&& !found_selected_tables.contains(&(target.schema.as_str(), target.table.clone()))
|
||||
})
|
||||
.map(|target| format!("{}.{}.{}", target.database, target.schema, target.table))
|
||||
.collect::<Vec<_>>();
|
||||
if !missing.is_empty() {
|
||||
return Err(format!("Selected backup tables are unavailable: {}", missing.join(", ")));
|
||||
}
|
||||
}
|
||||
if targets.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ mod store;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
pub use models::{BackupConfig, BackupFile, BackupRun, BackupSchedule, Migration, RunRequest};
|
||||
pub use models::{BackupConfig, BackupFile, BackupRun, BackupSchedule, BackupTableTarget, Migration, RunRequest};
|
||||
pub use service::{BackupCommand, BackupService};
|
||||
pub use store::{BackupSnapshot, BackupStore};
|
||||
|
||||
|
||||
@@ -5,6 +5,14 @@ use serde::{Deserialize, Serialize};
|
||||
pub const DEFAULT_DIRECTORY: &str = "dbx-backup__{schedule}__{timestamp}__{runId}";
|
||||
pub const DEFAULT_FILE: &str = "dbx-backup__{schedule}__{timestamp}__{database}__{runId}";
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct BackupTableTarget {
|
||||
pub database: String,
|
||||
pub schema: String,
|
||||
pub table: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct BackupConfig {
|
||||
@@ -15,6 +23,8 @@ pub struct BackupConfig {
|
||||
pub table_filter_mode: String,
|
||||
#[serde(default)]
|
||||
pub table_patterns: Vec<String>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub selected_tables: Vec<BackupTableTarget>,
|
||||
pub destination_directory: String,
|
||||
pub include_structure: bool,
|
||||
pub include_data: bool,
|
||||
@@ -145,7 +155,7 @@ impl BackupConfig {
|
||||
if !matches!(self.output_compression.as_str(), "none" | "gzip") {
|
||||
return Err("Unsupported backup compression".into());
|
||||
}
|
||||
if !matches!(self.table_filter_mode.as_str(), "all" | "include" | "exclude") {
|
||||
if !matches!(self.table_filter_mode.as_str(), "all" | "include" | "exclude" | "selected") {
|
||||
return Err("Unsupported backup table filter".into());
|
||||
}
|
||||
if self.databases.len() > 10_000 || self.table_patterns.len() > 1000 {
|
||||
@@ -154,9 +164,25 @@ impl BackupConfig {
|
||||
for value in self.databases.iter().chain(self.table_patterns.iter()) {
|
||||
bounded_text(value, 1024, "Backup target")?;
|
||||
}
|
||||
if self.table_filter_mode != "all" && self.table_patterns.is_empty() {
|
||||
if matches!(self.table_filter_mode.as_str(), "include" | "exclude") && self.table_patterns.is_empty() {
|
||||
return Err("Select backup table patterns".into());
|
||||
}
|
||||
if self.table_filter_mode == "selected" {
|
||||
if self.databases.is_empty() || self.selected_tables.is_empty() {
|
||||
return Err("Select backup databases and tables".into());
|
||||
}
|
||||
if self.selected_tables.len() > 10_000 {
|
||||
return Err("Too many selected backup tables".into());
|
||||
}
|
||||
for target in &self.selected_tables {
|
||||
bounded_text(&target.database, 1024, "Backup table database")?;
|
||||
bounded_text(&target.schema, 1024, "Backup table schema")?;
|
||||
bounded_text(&target.table, 1024, "Backup table name")?;
|
||||
if !self.databases.contains(&target.database) {
|
||||
return Err("Selected backup table is outside the database scope".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
validate_template(self.file_name_pattern.as_deref().unwrap_or(DEFAULT_FILE), false)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -56,6 +56,29 @@ fn invalid_schedule_and_path_templates_are_rejected() {
|
||||
assert!(super::models::matches_pattern("PUBLIC.*", "users", "app", "public", false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exact_backup_scope_preserves_literal_names_and_rejects_empty_or_foreign_scope() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut config = schedule(dir.path()).config;
|
||||
let legacy = serde_json::to_value(&config).unwrap();
|
||||
assert!(legacy.get("selectedTables").is_none());
|
||||
assert!(serde_json::from_value::<BackupConfig>(legacy).unwrap().selected_tables.is_empty());
|
||||
config.table_filter_mode = "selected".into();
|
||||
assert!(config.validate().is_err());
|
||||
config.databases = vec!["app".into()];
|
||||
assert!(config.validate().is_err());
|
||||
config.selected_tables =
|
||||
vec![BackupTableTarget { database: "app".into(), schema: "a.b".into(), table: " odd*,?;客户'表 ".into() }];
|
||||
assert!(config.validate().is_ok());
|
||||
let roundtrip: BackupConfig = serde_json::from_value(serde_json::to_value(&config).unwrap()).unwrap();
|
||||
assert_eq!(roundtrip.selected_tables, config.selected_tables);
|
||||
config.selected_tables[0].database = "other".into();
|
||||
assert!(config.validate().unwrap_err().contains("outside"));
|
||||
config.selected_tables[0].database = "app".into();
|
||||
config.selected_tables[0].schema = String::new();
|
||||
assert!(config.validate().is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn schedule_survives_reopen_and_rejects_stale_edit() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
@@ -438,6 +461,42 @@ async fn live_mysql_worker_scenario() {
|
||||
assert!(completed.error.as_deref().unwrap().contains("No tables matched"));
|
||||
assert!(completed.files.is_empty());
|
||||
assert!(previous.unwrap().exists());
|
||||
execute_sql_statement(
|
||||
&service.state,
|
||||
"mysql",
|
||||
&database,
|
||||
"CREATE TABLE `odd*,name.with.dot` (id INT PRIMARY KEY, value TEXT)",
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
execute_sql_statement(
|
||||
&service.state,
|
||||
"mysql",
|
||||
&database,
|
||||
"INSERT INTO `odd*,name.with.dot` VALUES (17, 'mysql-exact-scope-sentinel')",
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
let mut exact = plan_with_id(dir.path(), "mysql-exact");
|
||||
exact.config.connection_id = "mysql".into();
|
||||
exact.config.databases = vec![database.clone()];
|
||||
exact.config.table_filter_mode = "selected".into();
|
||||
exact.config.selected_tables = vec![BackupTableTarget {
|
||||
database: database.clone(),
|
||||
schema: database.clone(),
|
||||
table: "odd*,name.with.dot".into(),
|
||||
}];
|
||||
service.command(BackupCommand::Save { schedule: exact }).await?;
|
||||
let exact_run = enqueue_and_run(&service, "mysql-exact").await;
|
||||
assert_eq!(exact_run.status, "success", "{:?}", exact_run.error);
|
||||
assert_eq!(exact_run.files.len(), 1);
|
||||
let exact_sql = read_backup(&exact_run.files[0].file_path, false);
|
||||
assert!(exact_sql.contains("mysql-exact-scope-sentinel"));
|
||||
assert!(exact_sql.contains("odd*,name.with.dot"));
|
||||
assert!(!exact_sql.contains("CREATE TABLE `chosen`"));
|
||||
assert!(!exact_sql.contains("CREATE TABLE `skipped`"));
|
||||
Ok::<_, String>(())
|
||||
})
|
||||
.catch_unwind()
|
||||
@@ -533,6 +592,7 @@ fn live_postgres_worker_exports_selected_tables_across_schemas() {
|
||||
service.state.storage.save_connections(std::slice::from_ref(&admin)).await.unwrap();
|
||||
service.state.configs.write().await.insert(admin.id.clone(), admin);
|
||||
let database = format!("dbx_pg_backup_{}", uuid::Uuid::new_v4().simple());
|
||||
eprintln!("Backup source test database: {database}");
|
||||
execute_sql_statement(
|
||||
&service.state,
|
||||
admin_id,
|
||||
@@ -591,6 +651,129 @@ fn live_postgres_worker_exports_selected_tables_across_schemas() {
|
||||
// A view inside the same schema is not part of a table-name scope.
|
||||
assert!(!sql.contains("big_orders"), "{sql}");
|
||||
|
||||
execute_sql_statement(
|
||||
&service.state,
|
||||
"postgres",
|
||||
&database,
|
||||
"CREATE TABLE app.\"odd*,name.with.dot\" (id integer PRIMARY KEY, note text)",
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
execute_sql_statement(
|
||||
&service.state,
|
||||
"postgres",
|
||||
&database,
|
||||
"CREATE TABLE audit.\"odd*,name.with.dot\" (id integer PRIMARY KEY, note text)",
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
execute_sql_statement(
|
||||
&service.state,
|
||||
"postgres",
|
||||
&database,
|
||||
"INSERT INTO app.\"odd*,name.with.dot\" VALUES (17, 'exact-scope-sentinel')",
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut exact = plan_with_id(dir.path(), "exact");
|
||||
exact.config.connection_id = "postgres".into();
|
||||
exact.config.databases = vec![database.clone()];
|
||||
exact.config.table_filter_mode = "selected".into();
|
||||
exact.config.selected_tables = vec![BackupTableTarget {
|
||||
database: database.clone(),
|
||||
schema: "app".into(),
|
||||
table: "odd*,name.with.dot".into(),
|
||||
}];
|
||||
service.command(BackupCommand::Save { schedule: exact.clone() }).await.unwrap();
|
||||
let exact_run = enqueue_and_run(&service, "exact").await;
|
||||
assert_eq!(exact_run.status, "success", "{:?}", exact_run.error);
|
||||
assert_eq!(exact_run.files.len(), 1);
|
||||
assert_eq!(exact_run.files[0].schema, "app");
|
||||
let exact_sql = read_backup(&exact_run.files[0].file_path, false);
|
||||
assert!(exact_sql.contains("odd*,name.with.dot"));
|
||||
assert!(exact_sql.contains("exact-scope-sentinel"));
|
||||
assert!(!exact_sql.contains("orders"));
|
||||
assert!(!exact_sql.contains("accounts"));
|
||||
assert!(!exact_sql.contains("audit"));
|
||||
// Replay the real exported SQL into another disposable database.
|
||||
let restore_database = format!("dbx_pg_backup_restore_{}", uuid::Uuid::new_v4().simple());
|
||||
eprintln!("Backup replay test database: {restore_database}");
|
||||
execute_sql_statement(
|
||||
&service.state,
|
||||
admin_id,
|
||||
"postgres",
|
||||
&format!("CREATE DATABASE \"{restore_database}\""),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let restore_config = live_postgres_config("postgres-restore", &restore_database);
|
||||
service.state.storage.save_connections(std::slice::from_ref(&restore_config)).await.unwrap();
|
||||
service.state.configs.write().await.insert(restore_config.id.clone(), restore_config);
|
||||
// Selected-table dumps retain their source schema. Prepare the matching
|
||||
// namespace, as a restore into an existing application database does.
|
||||
execute_sql_statement(&service.state, "postgres-restore", &restore_database, "CREATE SCHEMA app", None, None)
|
||||
.await
|
||||
.unwrap();
|
||||
for statement in
|
||||
crate::sql::split_sql_statements_for_database(&exact_sql, crate::models::connection::DatabaseType::Postgres)
|
||||
{
|
||||
execute_sql_statement(&service.state, "postgres-restore", &restore_database, &statement, None, None)
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let restored = execute_sql_statement(
|
||||
&service.state,
|
||||
"postgres-restore",
|
||||
&restore_database,
|
||||
"SELECT id, note FROM app.\"odd*,name.with.dot\"",
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(restored.rows, vec![vec![json!(17), json!("exact-scope-sentinel")]]);
|
||||
let absent = execute_sql_statement(
|
||||
&service.state,
|
||||
"postgres-restore",
|
||||
&restore_database,
|
||||
"SELECT to_regclass('app.orders'), to_regclass('public.accounts')",
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(absent.rows, vec![vec![serde_json::Value::Null, serde_json::Value::Null]]);
|
||||
execute_sql_statement(
|
||||
&service.state,
|
||||
admin_id,
|
||||
"postgres",
|
||||
&format!("DROP DATABASE \"{restore_database}\" WITH (FORCE)"),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
exact.id = "exact-missing".into();
|
||||
exact.config.selected_tables.push(BackupTableTarget {
|
||||
database: database.clone(),
|
||||
schema: "missing_schema".into(),
|
||||
table: "not_there".into(),
|
||||
});
|
||||
service.command(BackupCommand::Save { schedule: exact }).await.unwrap();
|
||||
let missing_run = enqueue_and_run(&service, "exact-missing").await;
|
||||
assert_eq!(missing_run.status, "failed");
|
||||
assert!(missing_run.error.as_deref().unwrap().contains("unavailable"));
|
||||
assert!(missing_run.files.is_empty());
|
||||
|
||||
// Patterns are case-sensitive on PostgreSQL, and a scope that matches
|
||||
// nothing must fail without leaving a partial file behind.
|
||||
let mut empty = plan_with_id(dir.path(), "empty");
|
||||
|
||||
Reference in new issue
Block a user