fix(postgres): preserve identity and sequence state in SQL exports
This commit is contained in:
1 parent
25e1c6d25d
commit
de265287e0
4 files changed
+429
-46
No files matched your search
@@ -2,8 +2,10 @@ use dbx_core::connection::AppState;
|
||||
use dbx_core::database_export::{export_database_sql_core, DatabaseExportRequest, ExportStatus};
|
||||
use dbx_core::models::connection::{ConnectionConfig, DatabaseType};
|
||||
use dbx_core::query::execute_sql_statement;
|
||||
use dbx_core::query_result_export::{export_query_result_core, QueryResultExportRequest};
|
||||
use dbx_core::sql::SqlFileRequest;
|
||||
use dbx_core::sql_file_import::execute_sql_file_path;
|
||||
use dbx_core::table_export::{export_table_data_core, TableExportRequest};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
@@ -45,7 +47,12 @@ async fn live_postgres_all_schema_export_restores_one_sql_file() {
|
||||
let dir = tempfile::tempdir().expect("create export temp directory");
|
||||
let storage =
|
||||
dbx_core::persistence::test_storage::open(&dir.path().join("storage.db")).await.expect("open temp storage");
|
||||
let state = Arc::new(AppState::new(storage));
|
||||
let state = Arc::new(AppState::new_with_plugin_and_agent_dir_and_app_version(
|
||||
storage,
|
||||
dir.path().join("plugins"),
|
||||
dir.path().join("agents"),
|
||||
env!("CARGO_PKG_VERSION"),
|
||||
));
|
||||
state
|
||||
.configs
|
||||
.write()
|
||||
@@ -78,12 +85,41 @@ async fn live_postgres_all_schema_export_restores_one_sql_file() {
|
||||
"CREATE SCHEMA inventory",
|
||||
"CREATE SCHEMA reporting",
|
||||
"CREATE TABLE public.accounts (id integer PRIMARY KEY, name text NOT NULL)",
|
||||
"CREATE TABLE public.identity_always (id integer GENERATED ALWAYS AS IDENTITY PRIMARY KEY, name text NOT NULL)",
|
||||
"CREATE TABLE public.identity_default (id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, name text NOT NULL)",
|
||||
"CREATE TABLE public.serial_ids (id serial PRIMARY KEY, name text NOT NULL)",
|
||||
"CREATE TABLE public.identity_desc (id integer GENERATED ALWAYS AS IDENTITY (START WITH 0 INCREMENT BY -1 MINVALUE -100 MAXVALUE 0) PRIMARY KEY, name text NOT NULL)",
|
||||
"CREATE TABLE public.identity_unused (id integer GENERATED ALWAYS AS IDENTITY PRIMARY KEY, name text NOT NULL)",
|
||||
"CREATE TABLE public.serial_unused (id serial PRIMARY KEY, name text NOT NULL)",
|
||||
"CREATE TABLE public.identity_empty (id integer GENERATED ALWAYS AS IDENTITY PRIMARY KEY)",
|
||||
"CREATE TABLE public.identity_cycle (id integer GENERATED ALWAYS AS IDENTITY (MINVALUE 1 MAXVALUE 5 CYCLE) PRIMARY KEY, name text NOT NULL)",
|
||||
"CREATE TABLE inventory.products (id integer PRIMARY KEY, sku text NOT NULL)",
|
||||
"CREATE TABLE reporting.daily_totals (day date PRIMARY KEY, amount numeric(12,2) NOT NULL)",
|
||||
"CREATE VIEW reporting.positive_totals AS SELECT day, amount FROM reporting.daily_totals WHERE amount > 0",
|
||||
"INSERT INTO public.accounts VALUES (1, 'Alice'), (2, 'Bob')",
|
||||
"INSERT INTO inventory.products VALUES (10, 'SKU-10')",
|
||||
"INSERT INTO reporting.daily_totals VALUES ('2026-09-07', 42.50)",
|
||||
"INSERT INTO public.identity_always (name) VALUES ('one'), ('two')",
|
||||
"INSERT INTO public.identity_default (name) VALUES ('one'), ('two')",
|
||||
"INSERT INTO public.serial_ids (name) VALUES ('one'), ('two')",
|
||||
"INSERT INTO public.identity_always OVERRIDING SYSTEM VALUE VALUES (42, 'explicit')",
|
||||
"INSERT INTO public.identity_default VALUES (42, 'explicit')",
|
||||
"INSERT INTO public.serial_ids VALUES (42, 'explicit')",
|
||||
"ALTER SEQUENCE public.identity_always_id_seq RENAME TO renamed_identity_sequence",
|
||||
"ALTER SEQUENCE public.renamed_identity_sequence MINVALUE -100 MAXVALUE 1000 CACHE 3 CYCLE",
|
||||
"ALTER SEQUENCE public.serial_ids_id_seq START WITH 10 INCREMENT BY 5 MINVALUE 1 MAXVALUE 1000 CACHE 3 CYCLE",
|
||||
"SELECT setval('public.renamed_identity_sequence', 80)",
|
||||
"SELECT setval('public.identity_default_id_seq', 80)",
|
||||
"SELECT setval('public.serial_ids_id_seq', 80)",
|
||||
"INSERT INTO public.identity_desc (name) VALUES ('zero'), ('minus one')",
|
||||
"INSERT INTO public.identity_desc OVERRIDING SYSTEM VALUE VALUES (-42, 'explicit')",
|
||||
"SELECT setval('public.identity_desc_id_seq', -80)",
|
||||
"INSERT INTO public.identity_unused OVERRIDING SYSTEM VALUE VALUES (1, 'explicit')",
|
||||
"INSERT INTO public.serial_unused VALUES (1, 'explicit')",
|
||||
"INSERT INTO public.identity_cycle (name) VALUES ('one')",
|
||||
"INSERT INTO public.identity_cycle OVERRIDING SYSTEM VALUE VALUES (5, 'explicit')",
|
||||
"SELECT setval('public.identity_cycle_id_seq', 5)",
|
||||
"SELECT nextval('public.identity_cycle_id_seq')",
|
||||
] {
|
||||
Box::pin(execute_sql_statement(&state, &source_connection_id, &source_database, statement, None, None))
|
||||
.await
|
||||
@@ -97,7 +133,7 @@ async fn live_postgres_all_schema_export_restores_one_sql_file() {
|
||||
&state,
|
||||
&DatabaseExportRequest {
|
||||
export_id: format!("postgres-all-schema-export-{suffix}"),
|
||||
connection_id: source_connection_id,
|
||||
connection_id: source_connection_id.clone(),
|
||||
database: source_database.clone(),
|
||||
schema: String::new(),
|
||||
file_path: export_path.display().to_string(),
|
||||
@@ -181,6 +217,219 @@ async fn live_postgres_all_schema_export_restores_one_sql_file() {
|
||||
.expect("query restored schemas");
|
||||
assert_eq!(restored.rows, vec![vec![serde_json::json!(2), serde_json::json!(1), serde_json::json!("42.50")]]);
|
||||
|
||||
for (table, next_id, increment, duplicate_id) in [
|
||||
("identity_always", 81, 1, 42),
|
||||
("identity_default", 81, 1, 42),
|
||||
("serial_ids", 85, 5, 42),
|
||||
("identity_desc", -81, -1, -42),
|
||||
("identity_unused", 2, 1, 1),
|
||||
("serial_unused", 2, 1, 1),
|
||||
("identity_cycle", 2, 1, 5),
|
||||
] {
|
||||
let select = format!("SELECT id, name FROM public.{table} ORDER BY id");
|
||||
let expected =
|
||||
Box::pin(execute_sql_statement(&state, &source_connection_id, &source_database, &select, None, None))
|
||||
.await
|
||||
.unwrap()
|
||||
.rows;
|
||||
let restored =
|
||||
Box::pin(execute_sql_statement(&state, &target_connection_id, &target_database, &select, None, None))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(restored.rows, expected, "database export must preserve {table} IDs");
|
||||
let parameters_sql = format!(
|
||||
"SELECT seqstart, seqmin, seqmax, seqincrement, seqcache, seqcycle FROM pg_sequence \
|
||||
WHERE seqrelid = pg_get_serial_sequence('public.{table}', 'id')::regclass"
|
||||
);
|
||||
let source_parameters = Box::pin(execute_sql_statement(
|
||||
&state,
|
||||
&source_connection_id,
|
||||
&source_database,
|
||||
¶meters_sql,
|
||||
None,
|
||||
None,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let target_parameters = Box::pin(execute_sql_statement(
|
||||
&state,
|
||||
&target_connection_id,
|
||||
&target_database,
|
||||
¶meters_sql,
|
||||
None,
|
||||
None,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(target_parameters.rows, source_parameters.rows, "restore {table} sequence parameters");
|
||||
let next = Box::pin(execute_sql_statement(
|
||||
&state,
|
||||
&target_connection_id,
|
||||
&target_database,
|
||||
&format!("INSERT INTO public.{table} (name) VALUES ('next') RETURNING id"),
|
||||
None,
|
||||
None,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(next.rows, vec![vec![serde_json::json!(next_id)]], "restore {table} sequence state");
|
||||
let sequence_state_sql =
|
||||
format!("SELECT pg_sequence_last_value(pg_get_serial_sequence('public.{table}', 'id')::regclass)::text");
|
||||
let before_data_import = Box::pin(execute_sql_statement(
|
||||
&state,
|
||||
&target_connection_id,
|
||||
&target_database,
|
||||
&sequence_state_sql,
|
||||
None,
|
||||
None,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let columns =
|
||||
dbx_core::schema::get_columns_core(&state, &source_connection_id, &source_database, "public", table)
|
||||
.await
|
||||
.unwrap();
|
||||
for mode in ["single", "batch"] {
|
||||
for entry in ["table", "query"] {
|
||||
let path = dir.path().join(format!("{table}-{mode}-{entry}.sql"));
|
||||
if entry == "table" {
|
||||
// No supplied columns: exercise the table exporter's metadata query.
|
||||
let request: TableExportRequest = serde_json::from_value(serde_json::json!({
|
||||
"exportId": format!("{suffix}-{table}-{mode}-{entry}"),
|
||||
"connectionId": source_connection_id, "database": source_database,
|
||||
"schema": "public", "tableName": table, "filePath": path,
|
||||
"format": "sql", "insertMode": mode, "batchSize": 2
|
||||
}))
|
||||
.unwrap();
|
||||
export_table_data_core(&state, &request, |_| {}).await.unwrap();
|
||||
} else {
|
||||
// Query export receives extras from the result's table metadata.
|
||||
let request: QueryResultExportRequest = serde_json::from_value(serde_json::json!({
|
||||
"exportId": format!("{suffix}-{table}-{mode}-{entry}"),
|
||||
"connectionId": source_connection_id, "database": source_database,
|
||||
"schema": "public", "databaseType": "postgres", "sql": select,
|
||||
"queryBaseSql": select, "exportTableName": table, "filePath": path,
|
||||
"format": "sql", "insertMode": mode, "pageSize": 2,
|
||||
"exportColumnExtras": columns.iter().map(|c| c.extra.clone()).collect::<Vec<_>>()
|
||||
}))
|
||||
.unwrap();
|
||||
export_query_result_core(&state, &request, None, |_| {}).await.unwrap();
|
||||
}
|
||||
Box::pin(execute_sql_statement(
|
||||
&state,
|
||||
&target_connection_id,
|
||||
&target_database,
|
||||
&format!("TRUNCATE public.{table} CONTINUE IDENTITY"),
|
||||
None,
|
||||
None,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
execute_sql_file_path(
|
||||
&state,
|
||||
&SqlFileRequest {
|
||||
txn_session_id: None,
|
||||
execution_id: uuid::Uuid::new_v4().to_string(),
|
||||
connection_id: target_connection_id.clone(),
|
||||
database: target_database.clone(),
|
||||
schema: None,
|
||||
file_path: path.display().to_string(),
|
||||
continue_on_error: false,
|
||||
selected_tables: None,
|
||||
part_cooldown_ms: 0,
|
||||
skip_relational_constraints: false,
|
||||
},
|
||||
&path,
|
||||
CancellationToken::new(),
|
||||
std::time::Instant::now(),
|
||||
|_| {},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let restored = Box::pin(execute_sql_statement(
|
||||
&state,
|
||||
&target_connection_id,
|
||||
&target_database,
|
||||
&select,
|
||||
None,
|
||||
None,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(restored.rows, expected, "{entry}/{mode} must preserve {table} IDs");
|
||||
}
|
||||
}
|
||||
// Data-only exports preserve values, but deliberately leave the target sequence alone.
|
||||
let after_data_import = Box::pin(execute_sql_statement(
|
||||
&state,
|
||||
&target_connection_id,
|
||||
&target_database,
|
||||
&sequence_state_sql,
|
||||
None,
|
||||
None,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(after_data_import.rows, before_data_import.rows, "data import must not reset {table} sequence");
|
||||
let next = Box::pin(execute_sql_statement(
|
||||
&state,
|
||||
&target_connection_id,
|
||||
&target_database,
|
||||
&format!("INSERT INTO public.{table} (name) VALUES ('after data import') RETURNING id"),
|
||||
None,
|
||||
None,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
let after_id = next.rows[0][0].as_i64().unwrap();
|
||||
assert!(
|
||||
(after_id - next_id) * increment > 0 && (after_id - next_id) % increment == 0,
|
||||
"{table} must continue in the configured direction/step: {after_id}"
|
||||
);
|
||||
assert!(
|
||||
Box::pin(execute_sql_statement(
|
||||
&state,
|
||||
&target_connection_id,
|
||||
&target_database,
|
||||
&format!("INSERT INTO public.{table} OVERRIDING SYSTEM VALUE VALUES ({duplicate_id}, 'duplicate')"),
|
||||
None,
|
||||
None,
|
||||
))
|
||||
.await
|
||||
.is_err(),
|
||||
"primary key remains enforced for {table}"
|
||||
);
|
||||
}
|
||||
let empty_next = Box::pin(execute_sql_statement(
|
||||
&state,
|
||||
&target_connection_id,
|
||||
&target_database,
|
||||
"INSERT INTO public.identity_empty DEFAULT VALUES RETURNING id",
|
||||
None,
|
||||
None,
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
empty_next.rows,
|
||||
vec![vec![serde_json::json!(1)]],
|
||||
"an empty uncalled sequence must retain its first value"
|
||||
);
|
||||
assert!(
|
||||
Box::pin(execute_sql_statement(
|
||||
&state,
|
||||
&target_connection_id,
|
||||
&target_database,
|
||||
"INSERT INTO public.identity_always VALUES (99, 'forbidden')",
|
||||
None,
|
||||
None,
|
||||
))
|
||||
.await
|
||||
.is_err(),
|
||||
"ALWAYS identity remains enforced after restore"
|
||||
);
|
||||
|
||||
for database in [&source_database, &target_database] {
|
||||
Box::pin(execute_sql_statement(
|
||||
&state,
|
||||
|
||||
Reference in new issue
Block a user