fix(postgres): preserve identity and sequence state in SQL exports

This commit is contained in:
Ethereal49 authored and GitHub committed 2026-10-05 14:20:43 +08:00
1 parent 25e1c6d25d
commit de265287e0
4 files changed
+429 -46

No files matched your search

+166 -38
View File
@@ -19,9 +19,9 @@ use crate::sql::{SqlParsingOptions, SqlStatementSplitter};
use crate::sql_dialect::{qualified_table_name, uses_single_row_insert_statements};
use crate::transfer::{
format_ch_array_sql_literal, format_pg_array_sql_literal, format_postgres_vector_sql_literal,
is_identity_column_extra, is_mysql_generated_column_extra, is_postgres_vector_type,
keyset_pagination_sql_with_identifier_quote, quote_identifier, quote_postgres_string_literal,
wrap_dameng_identity_insert_sql_for_table,
is_identity_column_extra, is_mysql_generated_column_extra, is_postgres_generated_always_identity_extra,
is_postgres_vector_type, keyset_pagination_sql_with_identifier_quote, quote_identifier,
quote_postgres_string_literal, wrap_dameng_identity_insert_sql_for_table,
};
use crate::types::{ObjectSourceKind, SpatialColumn, SqlExportColumnSelection};
@@ -465,6 +465,7 @@ struct PostgresExportSequence {
last_value: Option<String>,
owner_table: Option<String>,
owner_column: Option<String>,
created_by_table: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -1523,7 +1524,17 @@ pub(crate) fn build_export_insert_statements_excluding_with_dialect(
// `.sql` files stay readable in plain text editors. Statements that hold a
// single tuple keep the compact `VALUES (..);` form, which also preserves
// the "one INSERT per row" output of the single-row insert mode.
let statement_head = format!("INSERT INTO {table} ({columns}) VALUES");
let overriding = if options.database_type == Some(DatabaseType::Postgres)
&& insert_columns.iter().any(|(index, _, _)| {
is_postgres_generated_always_identity_extra(
options.column_extras.get(*index).and_then(|value| value.as_deref()),
)
}) {
" OVERRIDING SYSTEM VALUE"
} else {
""
};
let statement_head = format!("INSERT INTO {table} ({columns}){overriding} VALUES");
let statement_prefix = format!("{statement_head} ");
let output_database_type = if insert_dialect == SqlInsertDialect::Source { options.database_type } else { None };
let statement_overhead_bytes = export_sql_statement_bytes(output_database_type, &statement_prefix) + 1;
@@ -2040,11 +2051,10 @@ fn postgres_sequence_qualified_name(schema: &str, sequence_name: &str) -> String
}
}
fn generate_postgres_sequence_create_ddl(sequence: &PostgresExportSequence, schema: &str) -> String {
let qualified_name = postgres_sequence_qualified_name(schema, &sequence.name);
fn postgres_sequence_options(sequence: &PostgresExportSequence) -> String {
let cycle = if sequence.cycle { "CYCLE" } else { "NO CYCLE" };
format!(
"CREATE SEQUENCE IF NOT EXISTS {qualified_name}\n AS {data_type}\n START WITH {start_value}\n INCREMENT BY {increment}\n MINVALUE {min_value}\n MAXVALUE {max_value}\n CACHE {cache_value}\n {cycle}",
" AS {data_type}\n START WITH {start_value}\n INCREMENT BY {increment}\n MINVALUE {min_value}\n MAXVALUE {max_value}\n CACHE {cache_value}\n {cycle}",
data_type = sequence.data_type,
start_value = sequence.start_value,
increment = sequence.increment,
@@ -2054,33 +2064,73 @@ fn generate_postgres_sequence_create_ddl(sequence: &PostgresExportSequence, sche
)
}
fn generate_postgres_sequence_owner_ddl(sequence: &PostgresExportSequence, schema: &str) -> Option<String> {
fn generate_postgres_sequence_create_ddl(sequence: &PostgresExportSequence, schema: &str) -> String {
format!(
"CREATE SEQUENCE IF NOT EXISTS {}\n{}",
postgres_sequence_qualified_name(schema, &sequence.name),
postgres_sequence_options(sequence),
)
}
fn generate_postgres_sequence_post_table_ddl(sequence: &PostgresExportSequence, schema: &str) -> Option<String> {
let owner_table = sequence.owner_table.as_deref()?;
let owner_column = sequence.owner_column.as_deref()?;
let owner_table = crate::transfer::qualified_table(owner_table, schema, &DatabaseType::Postgres, None);
if sequence.created_by_table {
// CREATE TABLE owns the sequence. Configure that actual binding without
// relying on its name, and quote the whole DO body (identifiers may contain dollar tags).
let body = format!(
"BEGIN EXECUTE 'ALTER SEQUENCE ' || pg_get_serial_sequence({}, {}) || {}; END;",
quote_postgres_string_literal(&owner_table),
quote_postgres_string_literal(owner_column),
quote_postgres_string_literal(&format!(
"\n{} RESTART WITH {}",
postgres_sequence_options(sequence),
sequence.start_value
)),
);
return Some(format!("DO {}", quote_postgres_string_literal(&body)));
}
Some(format!(
"ALTER SEQUENCE {} OWNED BY {}.{}",
postgres_sequence_qualified_name(schema, &sequence.name),
crate::transfer::qualified_table(owner_table, schema, &DatabaseType::Postgres, None),
owner_table,
quote_identifier(owner_column, &DatabaseType::Postgres)
))
}
fn generate_postgres_sequence_setval_sql(sequence: &PostgresExportSequence, schema: &str) -> Option<String> {
let last_value = sequence.last_value.as_deref()?.trim();
if last_value.is_empty() {
return None;
}
let sequence_literal = quote_postgres_string_literal(&postgres_sequence_qualified_name(schema, &sequence.name));
let last_value = sequence.last_value.as_deref().map(str::trim).filter(|value| !value.is_empty());
let mut sequence_literal = quote_postgres_string_literal(&postgres_sequence_qualified_name(schema, &sequence.name));
match (sequence.owner_table.as_deref(), sequence.owner_column.as_deref()) {
(Some(owner_table), Some(owner_column)) => {
let owner_table = crate::transfer::qualified_table(owner_table, schema, &DatabaseType::Postgres, None);
if sequence.created_by_table {
sequence_literal = format!(
"pg_get_serial_sequence({}, {})",
quote_postgres_string_literal(&owner_table),
quote_postgres_string_literal(owner_column),
);
}
if let Some(value) = last_value.filter(|_| sequence.cycle) {
// A cycled counter may be below/above the table's extrema after wrapping.
return Some(format!("SELECT setval({sequence_literal}, {value}, true)"));
}
let owner_column = quote_identifier(owner_column, &DatabaseType::Postgres);
let descending = sequence.increment.trim().starts_with('-');
let (aggregate, extreme, comparison) =
if descending { ("MIN", "LEAST", "<=") } else { ("MAX", "GREATEST", ">=") };
let fallback = last_value.unwrap_or(&sequence.start_value);
let called = if last_value.is_some() {
"true".to_string()
} else {
format!("COALESCE({aggregate}({owner_column}) {comparison} {fallback}, false)")
};
Some(format!(
"SELECT setval({sequence_literal}, GREATEST(COALESCE(MAX({owner_column}), {last_value}), {last_value}), true) FROM {owner_table}"
"SELECT setval({sequence_literal}, {extreme}(COALESCE({aggregate}({owner_column}), {fallback}), {fallback}), {called}) FROM {owner_table}"
))
}
_ => Some(format!("SELECT setval({sequence_literal}, {last_value}, true)")),
_ => last_value.map(|value| format!("SELECT setval({sequence_literal}, {value}, true)")),
}
}
@@ -2175,7 +2225,10 @@ const POSTGRES_EXPORT_SEQUENCES_SQL: &str = "SELECT c.relname, \
COALESCE(s.seqcycle, false), \
COALESCE(s.seqcache::text, '1'), \
t.relname, \
a.attname \
a.attname, \
COALESCE(d.deptype = 'i', false) OR \
COALESCE(a.atttypid IN (20, 21, 23) AND \
pg_get_expr(ad.adbin, ad.adrelid) = format('nextval(%L::regclass)', c.oid::regclass::text), false) \
FROM pg_class c \
JOIN pg_namespace n ON n.oid = c.relnamespace \
LEFT JOIN pg_sequence s ON s.seqrelid = c.oid \
@@ -2186,6 +2239,7 @@ const POSTGRES_EXPORT_SEQUENCES_SQL: &str = "SELECT c.relname, \
LEFT JOIN pg_class t ON t.oid = d.refobjid \
LEFT JOIN pg_namespace tn ON tn.oid = t.relnamespace AND tn.nspname = n.nspname \
LEFT JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = d.refobjsubid \
LEFT JOIN pg_attrdef ad ON ad.adrelid = a.attrelid AND ad.adnum = a.attnum \
WHERE c.relkind = 'S' AND n.nspname = $1 \
ORDER BY c.relname";
@@ -2203,7 +2257,9 @@ const POSTGRES_EXPORT_SEQUENCES_COMPAT_SQL: &str = "SELECT c.relname, \
false, \
'1', \
t.relname, \
a.attname \
a.attname, \
COALESCE(a.atttypid IN (20, 21, 23) AND \
pg_get_expr(ad.adbin, ad.adrelid) = format('nextval(%L::regclass)', c.oid::regclass::text), false) \
FROM pg_class c \
JOIN pg_namespace n ON n.oid = c.relnamespace \
LEFT JOIN pg_depend d ON d.classid = 'pg_class'::regclass \
@@ -2213,6 +2269,7 @@ const POSTGRES_EXPORT_SEQUENCES_COMPAT_SQL: &str = "SELECT c.relname, \
LEFT JOIN pg_class t ON t.oid = d.refobjid \
LEFT JOIN pg_namespace tn ON tn.oid = t.relnamespace AND tn.nspname = n.nspname \
LEFT JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = d.refobjsubid \
LEFT JOIN pg_attrdef ad ON ad.adrelid = a.attrelid AND ad.adnum = a.attnum \
WHERE c.relkind = 'S' AND n.nspname = $1 \
ORDER BY c.relname";
@@ -2312,6 +2369,7 @@ async fn list_postgres_export_sequences(
last_value: None,
owner_table: row.get::<_, Option<String>>(8),
owner_column: row.get::<_, Option<String>>(9),
created_by_table: row.get::<_, bool>(10),
})
.filter(|sequence| {
sequence.owner_table.as_deref().is_none_or(|owner_table| !excluded.contains(owner_table))
@@ -3908,7 +3966,7 @@ async fn export_database_sql_core_inner(
&DatabaseExportObjectCounts {
tables: tables.len(),
views: views.len(),
sequences: postgres_sequences.len(),
sequences: postgres_sequences.iter().filter(|sequence| !sequence.created_by_table).count(),
extensions: postgres_extensions.len(),
enums: postgres_enums.len(),
procedures: procedures.len(),
@@ -4150,10 +4208,9 @@ async fn export_database_sql_core_inner(
writeln!(file, "{}\n", drop_table_if_exists_sql(table_name, &request.schema, &db_type))
.map_err(|e| format!("Failed to write file: {e}"))?;
}
for sequence in postgres_sequences
.iter()
.filter(|sequence| sequence.owner_table.as_deref() == Some(table_name.as_str()))
{
for sequence in postgres_sequences.iter().filter(|sequence| {
!sequence.created_by_table && sequence.owner_table.as_deref() == Some(table_name.as_str())
}) {
on_progress(ExportProgress {
export_id: request.export_id.clone(),
current_object: sequence.name.clone(),
@@ -4458,7 +4515,7 @@ async fn export_database_sql_core_inner(
if request.include_structure && !postgres_sequences.is_empty() {
for sequence in &postgres_sequences {
if let Some(sql) = generate_postgres_sequence_owner_ddl(sequence, &request.schema) {
if let Some(sql) = generate_postgres_sequence_post_table_ddl(sequence, &request.schema) {
writeln!(file, "{};\n", sql).map_err(|e| format!("Failed to write file: {e}"))?;
}
}
@@ -4814,16 +4871,17 @@ mod tests {
ensure_export_destination_dir, export_destination_identity_mismatch, extract_postgres_deferred_foreign_keys,
filter_export_table_infos, format_export_sql_literal, format_export_table_ddl,
format_mysql_spatial_export_literal, format_xugu_spatial_export_literal, generate_postgres_enum_ddl,
generate_postgres_extension_ddl, generate_postgres_sequence_create_ddl, generate_postgres_sequence_owner_ddl,
generate_postgres_sequence_setval_sql, is_postgres_extension_member_routine, mysql_database_export_preamble,
mysql_view_dependencies_from_rows, mysql_view_dependencies_sql, normalize_export_table_ddl,
record_export_destination_identity, record_export_error, replace_database_export_select_list,
sort_export_views_by_dependencies, split_postgres_export_table_triggers, write_database_export_rows,
write_database_export_rows_with_mode, BuildDatabaseSqlExportOptions, BuildExportInsertStatementsOptions,
BuildExportSqlInsertOptions, DatabaseExportObjectCounts, DatabaseExportRequest, DatabaseExportWriter,
DdlNormalizeOptions, ExportedTableSql, PostgresExportEnum, PostgresExportExtension, PostgresExportSequence,
PostgresExtensionMembers, SqlInsertDialect, SqlInsertMode, DATABASE_EXPORT_INSERT_BATCH_SIZE,
DATABASE_EXPORT_ROW_LIMIT, POSTGRES_EXPORT_SEQUENCES_COMPAT_SQL, POSTGRES_EXPORT_SEQUENCES_SQL,
generate_postgres_extension_ddl, generate_postgres_sequence_create_ddl,
generate_postgres_sequence_post_table_ddl, generate_postgres_sequence_setval_sql,
is_postgres_extension_member_routine, mysql_database_export_preamble, mysql_view_dependencies_from_rows,
mysql_view_dependencies_sql, normalize_export_table_ddl, record_export_destination_identity,
record_export_error, replace_database_export_select_list, sort_export_views_by_dependencies,
split_postgres_export_table_triggers, write_database_export_rows, write_database_export_rows_with_mode,
BuildDatabaseSqlExportOptions, BuildExportInsertStatementsOptions, BuildExportSqlInsertOptions,
DatabaseExportObjectCounts, DatabaseExportRequest, DatabaseExportWriter, DdlNormalizeOptions, ExportedTableSql,
PostgresExportEnum, PostgresExportExtension, PostgresExportSequence, PostgresExtensionMembers,
SqlInsertDialect, SqlInsertMode, DATABASE_EXPORT_INSERT_BATCH_SIZE, DATABASE_EXPORT_ROW_LIMIT,
POSTGRES_EXPORT_SEQUENCES_COMPAT_SQL, POSTGRES_EXPORT_SEQUENCES_SQL,
};
use super::{ExportProgress, LenientExportErrors};
use crate::connection::AppState;
@@ -7528,6 +7586,56 @@ mod tests {
assert_eq!(combined.matches("INSERT INTO").count(), 20);
}
#[test]
fn postgres_identity_export_preserves_explicit_values_in_each_batch() {
for (extra, overrides) in [
(Some("generated always as identity (start with 1 increment by 1)"), true),
(Some(" GENERATED\tALWAYS AS IDENTITY "), true),
(Some("generated by default as identity"), false),
(Some("serial"), false),
(None, false),
] {
for dialect in ["source", "standard"] {
for (batch_size, tuples) in [
(1, vec!["(1, 'Ada');", "(2, 'Grace');", "(3, 'Hopper');"]),
(2, vec!["(1, 'Ada'),\n(2, 'Grace');", "(3, 'Hopper');"]),
(10, vec!["(1, 'Ada'),\n(2, 'Grace'),\n(3, 'Hopper');"]),
] {
let options = serde_json::from_value(json!({
"databaseType": "postgres", "schema": "public", "tableName": "users",
"columns": ["id", "name"], "columnTypes": ["integer", "text"],
"columnExtras": [extra, null], "rows": [[1, "Ada"], [2, "Grace"], [3, "Hopper"]],
"batchSize": batch_size, "insertDialect": dialect
}))
.unwrap();
let sql = build_export_sql_insert(options).unwrap();
let overriding = if overrides { " OVERRIDING SYSTEM VALUE" } else { "" };
let head = format!("INSERT INTO \"public\".\"users\" (\"id\", \"name\"){overriding} VALUES");
let expected = tuples
.iter()
.map(|tuple| format!("{head}{}{tuple}", if tuple.contains('\n') { "\n" } else { " " }))
.collect::<Vec<_>>()
.join("\n");
assert_eq!(sql, expected, "extra={extra:?}, dialect={dialect}, batch={batch_size}");
}
}
}
}
#[test]
fn postgres_identity_export_does_not_override_excluded_identity_columns() {
let options = serde_json::from_value(json!({
"databaseType": "postgres", "schema": "public", "tableName": "users",
"columns": ["id", "name"], "columnExtras": ["generated always as identity", null],
"rows": [[42, "Ada"]], "excludeColumns": ["ID"]
}))
.unwrap();
assert_eq!(
build_export_sql_insert(options).unwrap(),
"INSERT INTO \"public\".\"users\" (\"name\") VALUES ('Ada');"
);
}
#[test]
fn dameng_identity_export_inserts_enable_identity_insert() {
let statements = build_export_insert_statements(BuildExportInsertStatementsOptions {
@@ -7762,7 +7870,7 @@ mod tests {
assert!(!POSTGRES_EXPORT_SEQUENCES_COMPAT_SQL.contains("pg_sequence"));
assert!(!POSTGRES_EXPORT_SEQUENCES_COMPAT_SQL.contains("pg_sequence_last_value"));
// Both tiers must describe the same ten columns in the same order so
// Both tiers must describe the same eleven columns in the same order so
// the row decoding below stays identical.
for sql in [POSTGRES_EXPORT_SEQUENCES_SQL, POSTGRES_EXPORT_SEQUENCES_COMPAT_SQL] {
for fragment in [
@@ -7795,6 +7903,7 @@ mod tests {
last_value: Some("42".to_string()),
owner_table: Some("permissions".to_string()),
owner_column: Some("id".to_string()),
created_by_table: false,
}
}
@@ -7823,7 +7932,7 @@ mod tests {
let sequence = postgres_sequence("permissions_id_seq");
assert_eq!(
generate_postgres_sequence_owner_ddl(&sequence, "public").as_deref(),
generate_postgres_sequence_post_table_ddl(&sequence, "public").as_deref(),
Some("ALTER SEQUENCE \"public\".\"permissions_id_seq\" OWNED BY \"public\".\"permissions\".\"id\"")
);
assert_eq!(
@@ -7834,6 +7943,20 @@ mod tests {
);
}
#[test]
fn postgres_table_generated_sequence_restores_the_table_bound_sequence() {
let mut sequence = postgres_sequence("renamed_identity_sequence");
sequence.created_by_table = true;
let ddl = generate_postgres_sequence_post_table_ddl(&sequence, "public").unwrap();
assert!(ddl.starts_with("DO "));
assert!(ddl.contains("ALTER SEQUENCE "));
assert!(!ddl.contains("OWNED BY"));
assert_eq!(
generate_postgres_sequence_setval_sql(&sequence, "public").as_deref(),
Some("SELECT setval(pg_get_serial_sequence('\"public\".\"permissions\"', 'id'), GREATEST(COALESCE(MAX(\"id\"), 42), 42), true) FROM \"public\".\"permissions\"")
);
}
#[test]
fn strict_exports_return_object_errors_instead_of_writing_error_comments() {
let path = std::env::temp_dir().join(format!("dbx-strict-export-{}.sql", uuid::Uuid::new_v4()));
@@ -7908,7 +8031,12 @@ mod tests {
async fn test_app_state(scratch_dir: &std::path::Path) -> AppState {
let storage = crate::persistence::test_storage::open(&scratch_dir.join("storage.db")).await.unwrap();
AppState::new(storage)
AppState::new_with_plugin_and_agent_dir_and_app_version(
storage,
scratch_dir.join("plugins"),
scratch_dir.join("agents"),
env!("CARGO_PKG_VERSION"),
)
}
// Regression tests for #6327: create_dir_all on every export run is unsafe
+8 -2
View File
@@ -1634,7 +1634,7 @@ fn is_postgres_identity_extra(extra: Option<&str>) -> bool {
})
}
fn is_postgres_generated_always_identity_extra(extra: Option<&str>) -> bool {
pub(crate) fn is_postgres_generated_always_identity_extra(extra: Option<&str>) -> bool {
extra.is_some_and(|value| {
let mut parts = value.split_whitespace();
parts.next().is_some_and(|part| part.eq_ignore_ascii_case("generated"))
@@ -12415,7 +12415,13 @@ CREATE TABLE "Other"."prefix""Source"."NAME" ("ID" INT);"#;
let dir = std::env::temp_dir().join(format!("dbx-transfer-test-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
let storage = crate::persistence::test_storage::open(&dir.join("storage.db")).await.unwrap();
(AppState::new(storage), dir)
let state = AppState::new_with_plugin_and_agent_dir_and_app_version(
storage,
dir.join("plugins"),
dir.join("agents"),
env!("CARGO_PKG_VERSION"),
);
(state, dir)
}
async fn spawn_influxdb3_transfer_server() -> (String, tokio::task::JoinHandle<String>) {