From de265287e0399e4cdd6462f8a522bd37bb35f51f Mon Sep 17 00:00:00 2001 From: Ethereal49 Date: Mon, 5 Oct 2026 14:20:43 +0800 Subject: [PATCH] fix(postgres): preserve identity and sequence state in SQL exports --- crates/dbx-core/src/data/database_export.rs | 204 +++++++++++--- crates/dbx-core/src/data/transfer.rs | 10 +- .../tests/live_postgres_all_schema_export.rs | 253 +++++++++++++++++- crates/dbx-driver-postgres/src/postgres.rs | 8 +- 4 files changed, 429 insertions(+), 46 deletions(-) diff --git a/crates/dbx-core/src/data/database_export.rs b/crates/dbx-core/src/data/database_export.rs index 1c43cfa9c1..14b174fd29 100644 --- a/crates/dbx-core/src/data/database_export.rs +++ b/crates/dbx-core/src/data/database_export.rs @@ -19,9 +19,9 @@ use crate::sql::{SqlParsingOptions, SqlStatementSplitter}; use crate::sql_dialect::{qualified_table_name, uses_single_row_insert_statements}; use crate::transfer::{ format_ch_array_sql_literal, format_pg_array_sql_literal, format_postgres_vector_sql_literal, - is_identity_column_extra, is_mysql_generated_column_extra, is_postgres_vector_type, - keyset_pagination_sql_with_identifier_quote, quote_identifier, quote_postgres_string_literal, - wrap_dameng_identity_insert_sql_for_table, + is_identity_column_extra, is_mysql_generated_column_extra, is_postgres_generated_always_identity_extra, + is_postgres_vector_type, keyset_pagination_sql_with_identifier_quote, quote_identifier, + quote_postgres_string_literal, wrap_dameng_identity_insert_sql_for_table, }; use crate::types::{ObjectSourceKind, SpatialColumn, SqlExportColumnSelection}; @@ -465,6 +465,7 @@ struct PostgresExportSequence { last_value: Option, owner_table: Option, owner_column: Option, + created_by_table: bool, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -1523,7 +1524,17 @@ pub(crate) fn build_export_insert_statements_excluding_with_dialect( // `.sql` files stay readable in plain text editors. Statements that hold a // single tuple keep the compact `VALUES (..);` form, which also preserves // the "one INSERT per row" output of the single-row insert mode. - let statement_head = format!("INSERT INTO {table} ({columns}) VALUES"); + let overriding = if options.database_type == Some(DatabaseType::Postgres) + && insert_columns.iter().any(|(index, _, _)| { + is_postgres_generated_always_identity_extra( + options.column_extras.get(*index).and_then(|value| value.as_deref()), + ) + }) { + " OVERRIDING SYSTEM VALUE" + } else { + "" + }; + let statement_head = format!("INSERT INTO {table} ({columns}){overriding} VALUES"); let statement_prefix = format!("{statement_head} "); let output_database_type = if insert_dialect == SqlInsertDialect::Source { options.database_type } else { None }; let statement_overhead_bytes = export_sql_statement_bytes(output_database_type, &statement_prefix) + 1; @@ -2040,11 +2051,10 @@ fn postgres_sequence_qualified_name(schema: &str, sequence_name: &str) -> String } } -fn generate_postgres_sequence_create_ddl(sequence: &PostgresExportSequence, schema: &str) -> String { - let qualified_name = postgres_sequence_qualified_name(schema, &sequence.name); +fn postgres_sequence_options(sequence: &PostgresExportSequence) -> String { let cycle = if sequence.cycle { "CYCLE" } else { "NO CYCLE" }; format!( - "CREATE SEQUENCE IF NOT EXISTS {qualified_name}\n AS {data_type}\n START WITH {start_value}\n INCREMENT BY {increment}\n MINVALUE {min_value}\n MAXVALUE {max_value}\n CACHE {cache_value}\n {cycle}", + " AS {data_type}\n START WITH {start_value}\n INCREMENT BY {increment}\n MINVALUE {min_value}\n MAXVALUE {max_value}\n CACHE {cache_value}\n {cycle}", data_type = sequence.data_type, start_value = sequence.start_value, increment = sequence.increment, @@ -2054,33 +2064,73 @@ fn generate_postgres_sequence_create_ddl(sequence: &PostgresExportSequence, sche ) } -fn generate_postgres_sequence_owner_ddl(sequence: &PostgresExportSequence, schema: &str) -> Option { +fn generate_postgres_sequence_create_ddl(sequence: &PostgresExportSequence, schema: &str) -> String { + format!( + "CREATE SEQUENCE IF NOT EXISTS {}\n{}", + postgres_sequence_qualified_name(schema, &sequence.name), + postgres_sequence_options(sequence), + ) +} + +fn generate_postgres_sequence_post_table_ddl(sequence: &PostgresExportSequence, schema: &str) -> Option { let owner_table = sequence.owner_table.as_deref()?; let owner_column = sequence.owner_column.as_deref()?; + let owner_table = crate::transfer::qualified_table(owner_table, schema, &DatabaseType::Postgres, None); + if sequence.created_by_table { + // CREATE TABLE owns the sequence. Configure that actual binding without + // relying on its name, and quote the whole DO body (identifiers may contain dollar tags). + let body = format!( + "BEGIN EXECUTE 'ALTER SEQUENCE ' || pg_get_serial_sequence({}, {}) || {}; END;", + quote_postgres_string_literal(&owner_table), + quote_postgres_string_literal(owner_column), + quote_postgres_string_literal(&format!( + "\n{} RESTART WITH {}", + postgres_sequence_options(sequence), + sequence.start_value + )), + ); + return Some(format!("DO {}", quote_postgres_string_literal(&body))); + } Some(format!( "ALTER SEQUENCE {} OWNED BY {}.{}", postgres_sequence_qualified_name(schema, &sequence.name), - crate::transfer::qualified_table(owner_table, schema, &DatabaseType::Postgres, None), + owner_table, quote_identifier(owner_column, &DatabaseType::Postgres) )) } fn generate_postgres_sequence_setval_sql(sequence: &PostgresExportSequence, schema: &str) -> Option { - let last_value = sequence.last_value.as_deref()?.trim(); - if last_value.is_empty() { - return None; - } - - let sequence_literal = quote_postgres_string_literal(&postgres_sequence_qualified_name(schema, &sequence.name)); + let last_value = sequence.last_value.as_deref().map(str::trim).filter(|value| !value.is_empty()); + let mut sequence_literal = quote_postgres_string_literal(&postgres_sequence_qualified_name(schema, &sequence.name)); match (sequence.owner_table.as_deref(), sequence.owner_column.as_deref()) { (Some(owner_table), Some(owner_column)) => { let owner_table = crate::transfer::qualified_table(owner_table, schema, &DatabaseType::Postgres, None); + if sequence.created_by_table { + sequence_literal = format!( + "pg_get_serial_sequence({}, {})", + quote_postgres_string_literal(&owner_table), + quote_postgres_string_literal(owner_column), + ); + } + if let Some(value) = last_value.filter(|_| sequence.cycle) { + // A cycled counter may be below/above the table's extrema after wrapping. + return Some(format!("SELECT setval({sequence_literal}, {value}, true)")); + } let owner_column = quote_identifier(owner_column, &DatabaseType::Postgres); + let descending = sequence.increment.trim().starts_with('-'); + let (aggregate, extreme, comparison) = + if descending { ("MIN", "LEAST", "<=") } else { ("MAX", "GREATEST", ">=") }; + let fallback = last_value.unwrap_or(&sequence.start_value); + let called = if last_value.is_some() { + "true".to_string() + } else { + format!("COALESCE({aggregate}({owner_column}) {comparison} {fallback}, false)") + }; Some(format!( - "SELECT setval({sequence_literal}, GREATEST(COALESCE(MAX({owner_column}), {last_value}), {last_value}), true) FROM {owner_table}" + "SELECT setval({sequence_literal}, {extreme}(COALESCE({aggregate}({owner_column}), {fallback}), {fallback}), {called}) FROM {owner_table}" )) } - _ => Some(format!("SELECT setval({sequence_literal}, {last_value}, true)")), + _ => last_value.map(|value| format!("SELECT setval({sequence_literal}, {value}, true)")), } } @@ -2175,7 +2225,10 @@ const POSTGRES_EXPORT_SEQUENCES_SQL: &str = "SELECT c.relname, \ COALESCE(s.seqcycle, false), \ COALESCE(s.seqcache::text, '1'), \ t.relname, \ - a.attname \ + a.attname, \ + COALESCE(d.deptype = 'i', false) OR \ + COALESCE(a.atttypid IN (20, 21, 23) AND \ + pg_get_expr(ad.adbin, ad.adrelid) = format('nextval(%L::regclass)', c.oid::regclass::text), false) \ FROM pg_class c \ JOIN pg_namespace n ON n.oid = c.relnamespace \ LEFT JOIN pg_sequence s ON s.seqrelid = c.oid \ @@ -2186,6 +2239,7 @@ const POSTGRES_EXPORT_SEQUENCES_SQL: &str = "SELECT c.relname, \ LEFT JOIN pg_class t ON t.oid = d.refobjid \ LEFT JOIN pg_namespace tn ON tn.oid = t.relnamespace AND tn.nspname = n.nspname \ LEFT JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = d.refobjsubid \ + LEFT JOIN pg_attrdef ad ON ad.adrelid = a.attrelid AND ad.adnum = a.attnum \ WHERE c.relkind = 'S' AND n.nspname = $1 \ ORDER BY c.relname"; @@ -2203,7 +2257,9 @@ const POSTGRES_EXPORT_SEQUENCES_COMPAT_SQL: &str = "SELECT c.relname, \ false, \ '1', \ t.relname, \ - a.attname \ + a.attname, \ + COALESCE(a.atttypid IN (20, 21, 23) AND \ + pg_get_expr(ad.adbin, ad.adrelid) = format('nextval(%L::regclass)', c.oid::regclass::text), false) \ FROM pg_class c \ JOIN pg_namespace n ON n.oid = c.relnamespace \ LEFT JOIN pg_depend d ON d.classid = 'pg_class'::regclass \ @@ -2213,6 +2269,7 @@ const POSTGRES_EXPORT_SEQUENCES_COMPAT_SQL: &str = "SELECT c.relname, \ LEFT JOIN pg_class t ON t.oid = d.refobjid \ LEFT JOIN pg_namespace tn ON tn.oid = t.relnamespace AND tn.nspname = n.nspname \ LEFT JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = d.refobjsubid \ + LEFT JOIN pg_attrdef ad ON ad.adrelid = a.attrelid AND ad.adnum = a.attnum \ WHERE c.relkind = 'S' AND n.nspname = $1 \ ORDER BY c.relname"; @@ -2312,6 +2369,7 @@ async fn list_postgres_export_sequences( last_value: None, owner_table: row.get::<_, Option>(8), owner_column: row.get::<_, Option>(9), + created_by_table: row.get::<_, bool>(10), }) .filter(|sequence| { sequence.owner_table.as_deref().is_none_or(|owner_table| !excluded.contains(owner_table)) @@ -3908,7 +3966,7 @@ async fn export_database_sql_core_inner( &DatabaseExportObjectCounts { tables: tables.len(), views: views.len(), - sequences: postgres_sequences.len(), + sequences: postgres_sequences.iter().filter(|sequence| !sequence.created_by_table).count(), extensions: postgres_extensions.len(), enums: postgres_enums.len(), procedures: procedures.len(), @@ -4150,10 +4208,9 @@ async fn export_database_sql_core_inner( writeln!(file, "{}\n", drop_table_if_exists_sql(table_name, &request.schema, &db_type)) .map_err(|e| format!("Failed to write file: {e}"))?; } - for sequence in postgres_sequences - .iter() - .filter(|sequence| sequence.owner_table.as_deref() == Some(table_name.as_str())) - { + for sequence in postgres_sequences.iter().filter(|sequence| { + !sequence.created_by_table && sequence.owner_table.as_deref() == Some(table_name.as_str()) + }) { on_progress(ExportProgress { export_id: request.export_id.clone(), current_object: sequence.name.clone(), @@ -4458,7 +4515,7 @@ async fn export_database_sql_core_inner( if request.include_structure && !postgres_sequences.is_empty() { for sequence in &postgres_sequences { - if let Some(sql) = generate_postgres_sequence_owner_ddl(sequence, &request.schema) { + if let Some(sql) = generate_postgres_sequence_post_table_ddl(sequence, &request.schema) { writeln!(file, "{};\n", sql).map_err(|e| format!("Failed to write file: {e}"))?; } } @@ -4814,16 +4871,17 @@ mod tests { ensure_export_destination_dir, export_destination_identity_mismatch, extract_postgres_deferred_foreign_keys, filter_export_table_infos, format_export_sql_literal, format_export_table_ddl, format_mysql_spatial_export_literal, format_xugu_spatial_export_literal, generate_postgres_enum_ddl, - generate_postgres_extension_ddl, generate_postgres_sequence_create_ddl, generate_postgres_sequence_owner_ddl, - generate_postgres_sequence_setval_sql, is_postgres_extension_member_routine, mysql_database_export_preamble, - mysql_view_dependencies_from_rows, mysql_view_dependencies_sql, normalize_export_table_ddl, - record_export_destination_identity, record_export_error, replace_database_export_select_list, - sort_export_views_by_dependencies, split_postgres_export_table_triggers, write_database_export_rows, - write_database_export_rows_with_mode, BuildDatabaseSqlExportOptions, BuildExportInsertStatementsOptions, - BuildExportSqlInsertOptions, DatabaseExportObjectCounts, DatabaseExportRequest, DatabaseExportWriter, - DdlNormalizeOptions, ExportedTableSql, PostgresExportEnum, PostgresExportExtension, PostgresExportSequence, - PostgresExtensionMembers, SqlInsertDialect, SqlInsertMode, DATABASE_EXPORT_INSERT_BATCH_SIZE, - DATABASE_EXPORT_ROW_LIMIT, POSTGRES_EXPORT_SEQUENCES_COMPAT_SQL, POSTGRES_EXPORT_SEQUENCES_SQL, + generate_postgres_extension_ddl, generate_postgres_sequence_create_ddl, + generate_postgres_sequence_post_table_ddl, generate_postgres_sequence_setval_sql, + is_postgres_extension_member_routine, mysql_database_export_preamble, mysql_view_dependencies_from_rows, + mysql_view_dependencies_sql, normalize_export_table_ddl, record_export_destination_identity, + record_export_error, replace_database_export_select_list, sort_export_views_by_dependencies, + split_postgres_export_table_triggers, write_database_export_rows, write_database_export_rows_with_mode, + BuildDatabaseSqlExportOptions, BuildExportInsertStatementsOptions, BuildExportSqlInsertOptions, + DatabaseExportObjectCounts, DatabaseExportRequest, DatabaseExportWriter, DdlNormalizeOptions, ExportedTableSql, + PostgresExportEnum, PostgresExportExtension, PostgresExportSequence, PostgresExtensionMembers, + SqlInsertDialect, SqlInsertMode, DATABASE_EXPORT_INSERT_BATCH_SIZE, DATABASE_EXPORT_ROW_LIMIT, + POSTGRES_EXPORT_SEQUENCES_COMPAT_SQL, POSTGRES_EXPORT_SEQUENCES_SQL, }; use super::{ExportProgress, LenientExportErrors}; use crate::connection::AppState; @@ -7528,6 +7586,56 @@ mod tests { assert_eq!(combined.matches("INSERT INTO").count(), 20); } + #[test] + fn postgres_identity_export_preserves_explicit_values_in_each_batch() { + for (extra, overrides) in [ + (Some("generated always as identity (start with 1 increment by 1)"), true), + (Some(" GENERATED\tALWAYS AS IDENTITY "), true), + (Some("generated by default as identity"), false), + (Some("serial"), false), + (None, false), + ] { + for dialect in ["source", "standard"] { + for (batch_size, tuples) in [ + (1, vec!["(1, 'Ada');", "(2, 'Grace');", "(3, 'Hopper');"]), + (2, vec!["(1, 'Ada'),\n(2, 'Grace');", "(3, 'Hopper');"]), + (10, vec!["(1, 'Ada'),\n(2, 'Grace'),\n(3, 'Hopper');"]), + ] { + let options = serde_json::from_value(json!({ + "databaseType": "postgres", "schema": "public", "tableName": "users", + "columns": ["id", "name"], "columnTypes": ["integer", "text"], + "columnExtras": [extra, null], "rows": [[1, "Ada"], [2, "Grace"], [3, "Hopper"]], + "batchSize": batch_size, "insertDialect": dialect + })) + .unwrap(); + let sql = build_export_sql_insert(options).unwrap(); + let overriding = if overrides { " OVERRIDING SYSTEM VALUE" } else { "" }; + let head = format!("INSERT INTO \"public\".\"users\" (\"id\", \"name\"){overriding} VALUES"); + let expected = tuples + .iter() + .map(|tuple| format!("{head}{}{tuple}", if tuple.contains('\n') { "\n" } else { " " })) + .collect::>() + .join("\n"); + assert_eq!(sql, expected, "extra={extra:?}, dialect={dialect}, batch={batch_size}"); + } + } + } + } + + #[test] + fn postgres_identity_export_does_not_override_excluded_identity_columns() { + let options = serde_json::from_value(json!({ + "databaseType": "postgres", "schema": "public", "tableName": "users", + "columns": ["id", "name"], "columnExtras": ["generated always as identity", null], + "rows": [[42, "Ada"]], "excludeColumns": ["ID"] + })) + .unwrap(); + assert_eq!( + build_export_sql_insert(options).unwrap(), + "INSERT INTO \"public\".\"users\" (\"name\") VALUES ('Ada');" + ); + } + #[test] fn dameng_identity_export_inserts_enable_identity_insert() { let statements = build_export_insert_statements(BuildExportInsertStatementsOptions { @@ -7762,7 +7870,7 @@ mod tests { assert!(!POSTGRES_EXPORT_SEQUENCES_COMPAT_SQL.contains("pg_sequence")); assert!(!POSTGRES_EXPORT_SEQUENCES_COMPAT_SQL.contains("pg_sequence_last_value")); - // Both tiers must describe the same ten columns in the same order so + // Both tiers must describe the same eleven columns in the same order so // the row decoding below stays identical. for sql in [POSTGRES_EXPORT_SEQUENCES_SQL, POSTGRES_EXPORT_SEQUENCES_COMPAT_SQL] { for fragment in [ @@ -7795,6 +7903,7 @@ mod tests { last_value: Some("42".to_string()), owner_table: Some("permissions".to_string()), owner_column: Some("id".to_string()), + created_by_table: false, } } @@ -7823,7 +7932,7 @@ mod tests { let sequence = postgres_sequence("permissions_id_seq"); assert_eq!( - generate_postgres_sequence_owner_ddl(&sequence, "public").as_deref(), + generate_postgres_sequence_post_table_ddl(&sequence, "public").as_deref(), Some("ALTER SEQUENCE \"public\".\"permissions_id_seq\" OWNED BY \"public\".\"permissions\".\"id\"") ); assert_eq!( @@ -7834,6 +7943,20 @@ mod tests { ); } + #[test] + fn postgres_table_generated_sequence_restores_the_table_bound_sequence() { + let mut sequence = postgres_sequence("renamed_identity_sequence"); + sequence.created_by_table = true; + let ddl = generate_postgres_sequence_post_table_ddl(&sequence, "public").unwrap(); + assert!(ddl.starts_with("DO ")); + assert!(ddl.contains("ALTER SEQUENCE ")); + assert!(!ddl.contains("OWNED BY")); + assert_eq!( + generate_postgres_sequence_setval_sql(&sequence, "public").as_deref(), + Some("SELECT setval(pg_get_serial_sequence('\"public\".\"permissions\"', 'id'), GREATEST(COALESCE(MAX(\"id\"), 42), 42), true) FROM \"public\".\"permissions\"") + ); + } + #[test] fn strict_exports_return_object_errors_instead_of_writing_error_comments() { let path = std::env::temp_dir().join(format!("dbx-strict-export-{}.sql", uuid::Uuid::new_v4())); @@ -7908,7 +8031,12 @@ mod tests { async fn test_app_state(scratch_dir: &std::path::Path) -> AppState { let storage = crate::persistence::test_storage::open(&scratch_dir.join("storage.db")).await.unwrap(); - AppState::new(storage) + AppState::new_with_plugin_and_agent_dir_and_app_version( + storage, + scratch_dir.join("plugins"), + scratch_dir.join("agents"), + env!("CARGO_PKG_VERSION"), + ) } // Regression tests for #6327: create_dir_all on every export run is unsafe diff --git a/crates/dbx-core/src/data/transfer.rs b/crates/dbx-core/src/data/transfer.rs index aabdff766a..8e2b1d6942 100644 --- a/crates/dbx-core/src/data/transfer.rs +++ b/crates/dbx-core/src/data/transfer.rs @@ -1634,7 +1634,7 @@ fn is_postgres_identity_extra(extra: Option<&str>) -> bool { }) } -fn is_postgres_generated_always_identity_extra(extra: Option<&str>) -> bool { +pub(crate) fn is_postgres_generated_always_identity_extra(extra: Option<&str>) -> bool { extra.is_some_and(|value| { let mut parts = value.split_whitespace(); parts.next().is_some_and(|part| part.eq_ignore_ascii_case("generated")) @@ -12415,7 +12415,13 @@ CREATE TABLE "Other"."prefix""Source"."NAME" ("ID" INT);"#; let dir = std::env::temp_dir().join(format!("dbx-transfer-test-{}", uuid::Uuid::new_v4())); std::fs::create_dir_all(&dir).unwrap(); let storage = crate::persistence::test_storage::open(&dir.join("storage.db")).await.unwrap(); - (AppState::new(storage), dir) + let state = AppState::new_with_plugin_and_agent_dir_and_app_version( + storage, + dir.join("plugins"), + dir.join("agents"), + env!("CARGO_PKG_VERSION"), + ); + (state, dir) } async fn spawn_influxdb3_transfer_server() -> (String, tokio::task::JoinHandle) { diff --git a/crates/dbx-core/tests/live_postgres_all_schema_export.rs b/crates/dbx-core/tests/live_postgres_all_schema_export.rs index dccd8bce48..b319cf8d0d 100644 --- a/crates/dbx-core/tests/live_postgres_all_schema_export.rs +++ b/crates/dbx-core/tests/live_postgres_all_schema_export.rs @@ -2,8 +2,10 @@ use dbx_core::connection::AppState; use dbx_core::database_export::{export_database_sql_core, DatabaseExportRequest, ExportStatus}; use dbx_core::models::connection::{ConnectionConfig, DatabaseType}; use dbx_core::query::execute_sql_statement; +use dbx_core::query_result_export::{export_query_result_core, QueryResultExportRequest}; use dbx_core::sql::SqlFileRequest; use dbx_core::sql_file_import::execute_sql_file_path; +use dbx_core::table_export::{export_table_data_core, TableExportRequest}; use std::sync::{Arc, Mutex}; use tokio_util::sync::CancellationToken; @@ -45,7 +47,12 @@ async fn live_postgres_all_schema_export_restores_one_sql_file() { let dir = tempfile::tempdir().expect("create export temp directory"); let storage = dbx_core::persistence::test_storage::open(&dir.path().join("storage.db")).await.expect("open temp storage"); - let state = Arc::new(AppState::new(storage)); + let state = Arc::new(AppState::new_with_plugin_and_agent_dir_and_app_version( + storage, + dir.path().join("plugins"), + dir.path().join("agents"), + env!("CARGO_PKG_VERSION"), + )); state .configs .write() @@ -78,12 +85,41 @@ async fn live_postgres_all_schema_export_restores_one_sql_file() { "CREATE SCHEMA inventory", "CREATE SCHEMA reporting", "CREATE TABLE public.accounts (id integer PRIMARY KEY, name text NOT NULL)", + "CREATE TABLE public.identity_always (id integer GENERATED ALWAYS AS IDENTITY PRIMARY KEY, name text NOT NULL)", + "CREATE TABLE public.identity_default (id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, name text NOT NULL)", + "CREATE TABLE public.serial_ids (id serial PRIMARY KEY, name text NOT NULL)", + "CREATE TABLE public.identity_desc (id integer GENERATED ALWAYS AS IDENTITY (START WITH 0 INCREMENT BY -1 MINVALUE -100 MAXVALUE 0) PRIMARY KEY, name text NOT NULL)", + "CREATE TABLE public.identity_unused (id integer GENERATED ALWAYS AS IDENTITY PRIMARY KEY, name text NOT NULL)", + "CREATE TABLE public.serial_unused (id serial PRIMARY KEY, name text NOT NULL)", + "CREATE TABLE public.identity_empty (id integer GENERATED ALWAYS AS IDENTITY PRIMARY KEY)", + "CREATE TABLE public.identity_cycle (id integer GENERATED ALWAYS AS IDENTITY (MINVALUE 1 MAXVALUE 5 CYCLE) PRIMARY KEY, name text NOT NULL)", "CREATE TABLE inventory.products (id integer PRIMARY KEY, sku text NOT NULL)", "CREATE TABLE reporting.daily_totals (day date PRIMARY KEY, amount numeric(12,2) NOT NULL)", "CREATE VIEW reporting.positive_totals AS SELECT day, amount FROM reporting.daily_totals WHERE amount > 0", "INSERT INTO public.accounts VALUES (1, 'Alice'), (2, 'Bob')", "INSERT INTO inventory.products VALUES (10, 'SKU-10')", "INSERT INTO reporting.daily_totals VALUES ('2026-09-07', 42.50)", + "INSERT INTO public.identity_always (name) VALUES ('one'), ('two')", + "INSERT INTO public.identity_default (name) VALUES ('one'), ('two')", + "INSERT INTO public.serial_ids (name) VALUES ('one'), ('two')", + "INSERT INTO public.identity_always OVERRIDING SYSTEM VALUE VALUES (42, 'explicit')", + "INSERT INTO public.identity_default VALUES (42, 'explicit')", + "INSERT INTO public.serial_ids VALUES (42, 'explicit')", + "ALTER SEQUENCE public.identity_always_id_seq RENAME TO renamed_identity_sequence", + "ALTER SEQUENCE public.renamed_identity_sequence MINVALUE -100 MAXVALUE 1000 CACHE 3 CYCLE", + "ALTER SEQUENCE public.serial_ids_id_seq START WITH 10 INCREMENT BY 5 MINVALUE 1 MAXVALUE 1000 CACHE 3 CYCLE", + "SELECT setval('public.renamed_identity_sequence', 80)", + "SELECT setval('public.identity_default_id_seq', 80)", + "SELECT setval('public.serial_ids_id_seq', 80)", + "INSERT INTO public.identity_desc (name) VALUES ('zero'), ('minus one')", + "INSERT INTO public.identity_desc OVERRIDING SYSTEM VALUE VALUES (-42, 'explicit')", + "SELECT setval('public.identity_desc_id_seq', -80)", + "INSERT INTO public.identity_unused OVERRIDING SYSTEM VALUE VALUES (1, 'explicit')", + "INSERT INTO public.serial_unused VALUES (1, 'explicit')", + "INSERT INTO public.identity_cycle (name) VALUES ('one')", + "INSERT INTO public.identity_cycle OVERRIDING SYSTEM VALUE VALUES (5, 'explicit')", + "SELECT setval('public.identity_cycle_id_seq', 5)", + "SELECT nextval('public.identity_cycle_id_seq')", ] { Box::pin(execute_sql_statement(&state, &source_connection_id, &source_database, statement, None, None)) .await @@ -97,7 +133,7 @@ async fn live_postgres_all_schema_export_restores_one_sql_file() { &state, &DatabaseExportRequest { export_id: format!("postgres-all-schema-export-{suffix}"), - connection_id: source_connection_id, + connection_id: source_connection_id.clone(), database: source_database.clone(), schema: String::new(), file_path: export_path.display().to_string(), @@ -181,6 +217,219 @@ async fn live_postgres_all_schema_export_restores_one_sql_file() { .expect("query restored schemas"); assert_eq!(restored.rows, vec![vec![serde_json::json!(2), serde_json::json!(1), serde_json::json!("42.50")]]); + for (table, next_id, increment, duplicate_id) in [ + ("identity_always", 81, 1, 42), + ("identity_default", 81, 1, 42), + ("serial_ids", 85, 5, 42), + ("identity_desc", -81, -1, -42), + ("identity_unused", 2, 1, 1), + ("serial_unused", 2, 1, 1), + ("identity_cycle", 2, 1, 5), + ] { + let select = format!("SELECT id, name FROM public.{table} ORDER BY id"); + let expected = + Box::pin(execute_sql_statement(&state, &source_connection_id, &source_database, &select, None, None)) + .await + .unwrap() + .rows; + let restored = + Box::pin(execute_sql_statement(&state, &target_connection_id, &target_database, &select, None, None)) + .await + .unwrap(); + assert_eq!(restored.rows, expected, "database export must preserve {table} IDs"); + let parameters_sql = format!( + "SELECT seqstart, seqmin, seqmax, seqincrement, seqcache, seqcycle FROM pg_sequence \ + WHERE seqrelid = pg_get_serial_sequence('public.{table}', 'id')::regclass" + ); + let source_parameters = Box::pin(execute_sql_statement( + &state, + &source_connection_id, + &source_database, + ¶meters_sql, + None, + None, + )) + .await + .unwrap(); + let target_parameters = Box::pin(execute_sql_statement( + &state, + &target_connection_id, + &target_database, + ¶meters_sql, + None, + None, + )) + .await + .unwrap(); + assert_eq!(target_parameters.rows, source_parameters.rows, "restore {table} sequence parameters"); + let next = Box::pin(execute_sql_statement( + &state, + &target_connection_id, + &target_database, + &format!("INSERT INTO public.{table} (name) VALUES ('next') RETURNING id"), + None, + None, + )) + .await + .unwrap(); + assert_eq!(next.rows, vec![vec![serde_json::json!(next_id)]], "restore {table} sequence state"); + let sequence_state_sql = + format!("SELECT pg_sequence_last_value(pg_get_serial_sequence('public.{table}', 'id')::regclass)::text"); + let before_data_import = Box::pin(execute_sql_statement( + &state, + &target_connection_id, + &target_database, + &sequence_state_sql, + None, + None, + )) + .await + .unwrap(); + + let columns = + dbx_core::schema::get_columns_core(&state, &source_connection_id, &source_database, "public", table) + .await + .unwrap(); + for mode in ["single", "batch"] { + for entry in ["table", "query"] { + let path = dir.path().join(format!("{table}-{mode}-{entry}.sql")); + if entry == "table" { + // No supplied columns: exercise the table exporter's metadata query. + let request: TableExportRequest = serde_json::from_value(serde_json::json!({ + "exportId": format!("{suffix}-{table}-{mode}-{entry}"), + "connectionId": source_connection_id, "database": source_database, + "schema": "public", "tableName": table, "filePath": path, + "format": "sql", "insertMode": mode, "batchSize": 2 + })) + .unwrap(); + export_table_data_core(&state, &request, |_| {}).await.unwrap(); + } else { + // Query export receives extras from the result's table metadata. + let request: QueryResultExportRequest = serde_json::from_value(serde_json::json!({ + "exportId": format!("{suffix}-{table}-{mode}-{entry}"), + "connectionId": source_connection_id, "database": source_database, + "schema": "public", "databaseType": "postgres", "sql": select, + "queryBaseSql": select, "exportTableName": table, "filePath": path, + "format": "sql", "insertMode": mode, "pageSize": 2, + "exportColumnExtras": columns.iter().map(|c| c.extra.clone()).collect::>() + })) + .unwrap(); + export_query_result_core(&state, &request, None, |_| {}).await.unwrap(); + } + Box::pin(execute_sql_statement( + &state, + &target_connection_id, + &target_database, + &format!("TRUNCATE public.{table} CONTINUE IDENTITY"), + None, + None, + )) + .await + .unwrap(); + execute_sql_file_path( + &state, + &SqlFileRequest { + txn_session_id: None, + execution_id: uuid::Uuid::new_v4().to_string(), + connection_id: target_connection_id.clone(), + database: target_database.clone(), + schema: None, + file_path: path.display().to_string(), + continue_on_error: false, + selected_tables: None, + part_cooldown_ms: 0, + skip_relational_constraints: false, + }, + &path, + CancellationToken::new(), + std::time::Instant::now(), + |_| {}, + ) + .await + .unwrap(); + let restored = Box::pin(execute_sql_statement( + &state, + &target_connection_id, + &target_database, + &select, + None, + None, + )) + .await + .unwrap(); + assert_eq!(restored.rows, expected, "{entry}/{mode} must preserve {table} IDs"); + } + } + // Data-only exports preserve values, but deliberately leave the target sequence alone. + let after_data_import = Box::pin(execute_sql_statement( + &state, + &target_connection_id, + &target_database, + &sequence_state_sql, + None, + None, + )) + .await + .unwrap(); + assert_eq!(after_data_import.rows, before_data_import.rows, "data import must not reset {table} sequence"); + let next = Box::pin(execute_sql_statement( + &state, + &target_connection_id, + &target_database, + &format!("INSERT INTO public.{table} (name) VALUES ('after data import') RETURNING id"), + None, + None, + )) + .await + .unwrap(); + let after_id = next.rows[0][0].as_i64().unwrap(); + assert!( + (after_id - next_id) * increment > 0 && (after_id - next_id) % increment == 0, + "{table} must continue in the configured direction/step: {after_id}" + ); + assert!( + Box::pin(execute_sql_statement( + &state, + &target_connection_id, + &target_database, + &format!("INSERT INTO public.{table} OVERRIDING SYSTEM VALUE VALUES ({duplicate_id}, 'duplicate')"), + None, + None, + )) + .await + .is_err(), + "primary key remains enforced for {table}" + ); + } + let empty_next = Box::pin(execute_sql_statement( + &state, + &target_connection_id, + &target_database, + "INSERT INTO public.identity_empty DEFAULT VALUES RETURNING id", + None, + None, + )) + .await + .unwrap(); + assert_eq!( + empty_next.rows, + vec![vec![serde_json::json!(1)]], + "an empty uncalled sequence must retain its first value" + ); + assert!( + Box::pin(execute_sql_statement( + &state, + &target_connection_id, + &target_database, + "INSERT INTO public.identity_always VALUES (99, 'forbidden')", + None, + None, + )) + .await + .is_err(), + "ALWAYS identity remains enforced after restore" + ); + for database in [&source_database, &target_database] { Box::pin(execute_sql_statement( &state, diff --git a/crates/dbx-driver-postgres/src/postgres.rs b/crates/dbx-driver-postgres/src/postgres.rs index 782cf67586..c33e2af3fd 100644 --- a/crates/dbx-driver-postgres/src/postgres.rs +++ b/crates/dbx-driver-postgres/src/postgres.rs @@ -4736,8 +4736,8 @@ fn postgres_columns_for_relations_sql() -> &'static str { ) AS is_pk, \ col_description(a.attrelid, a.attnum) AS column_comment, \ CASE a.attidentity \ - WHEN 'd' THEN 'generated by default as identity' || CASE WHEN pseq.seqstart IS NOT NULL THEN format(' (start with %s increment by %s)', pseq.seqstart, pseq.seqincrement) ELSE '' END \ - WHEN 'a' THEN 'generated always as identity' || CASE WHEN pseq.seqstart IS NOT NULL THEN format(' (start with %s increment by %s)', pseq.seqstart, pseq.seqincrement) ELSE '' END \ + WHEN 'd' THEN 'generated by default as identity' || CASE WHEN pseq.seqstart IS NOT NULL THEN format(' (start with %s increment by %s minvalue %s maxvalue %s cache %s %s)', pseq.seqstart, pseq.seqincrement, pseq.seqmin, pseq.seqmax, pseq.seqcache, CASE WHEN pseq.seqcycle THEN 'cycle' ELSE 'no cycle' END) ELSE '' END \ + WHEN 'a' THEN 'generated always as identity' || CASE WHEN pseq.seqstart IS NOT NULL THEN format(' (start with %s increment by %s minvalue %s maxvalue %s cache %s %s)', pseq.seqstart, pseq.seqincrement, pseq.seqmin, pseq.seqmax, pseq.seqcache, CASE WHEN pseq.seqcycle THEN 'cycle' ELSE 'no cycle' END) ELSE '' END \ ELSE CASE a.attgenerated \ WHEN 's' THEN 'generated always as (' || pg_get_expr(ad.adbin, ad.adrelid) || ') stored' \ WHEN 'v' THEN 'generated always as (' || pg_get_expr(ad.adbin, ad.adrelid) || ') virtual' \ @@ -7594,8 +7594,8 @@ const POSTGRES_COLUMNS_SQL: &str = "SELECT a.attname AS column_name, \ ) AS is_pk, \ col_description(a.attrelid, a.attnum) AS column_comment, \ CASE a.attidentity \ - WHEN 'd' THEN 'generated by default as identity' || CASE WHEN pseq.seqstart IS NOT NULL THEN format(' (start with %s increment by %s)', pseq.seqstart, pseq.seqincrement) ELSE '' END \ - WHEN 'a' THEN 'generated always as identity' || CASE WHEN pseq.seqstart IS NOT NULL THEN format(' (start with %s increment by %s)', pseq.seqstart, pseq.seqincrement) ELSE '' END \ + WHEN 'd' THEN 'generated by default as identity' || CASE WHEN pseq.seqstart IS NOT NULL THEN format(' (start with %s increment by %s minvalue %s maxvalue %s cache %s %s)', pseq.seqstart, pseq.seqincrement, pseq.seqmin, pseq.seqmax, pseq.seqcache, CASE WHEN pseq.seqcycle THEN 'cycle' ELSE 'no cycle' END) ELSE '' END \ + WHEN 'a' THEN 'generated always as identity' || CASE WHEN pseq.seqstart IS NOT NULL THEN format(' (start with %s increment by %s minvalue %s maxvalue %s cache %s %s)', pseq.seqstart, pseq.seqincrement, pseq.seqmin, pseq.seqmax, pseq.seqcache, CASE WHEN pseq.seqcycle THEN 'cycle' ELSE 'no cycle' END) ELSE '' END \ ELSE CASE a.attgenerated \ WHEN 's' THEN 'generated always as (' || pg_get_expr(ad.adbin, ad.adrelid) || ') stored' \ WHEN 'v' THEN 'generated always as (' || pg_get_expr(ad.adbin, ad.adrelid) || ') virtual' \