fix(postgres): preserve identity and sequence state in SQL exports

This commit is contained in:
Ethereal49 authored and GitHub committed 2026-10-05 14:20:43 +08:00
1 parent 25e1c6d25d
commit de265287e0
4 files changed
+429 -46

No files matched your search

+166 -38
View File
@@ -19,9 +19,9 @@ use crate::sql::{SqlParsingOptions, SqlStatementSplitter};
use crate::sql_dialect::{qualified_table_name, uses_single_row_insert_statements}; use crate::sql_dialect::{qualified_table_name, uses_single_row_insert_statements};
use crate::transfer::{ use crate::transfer::{
format_ch_array_sql_literal, format_pg_array_sql_literal, format_postgres_vector_sql_literal, format_ch_array_sql_literal, format_pg_array_sql_literal, format_postgres_vector_sql_literal,
is_identity_column_extra, is_mysql_generated_column_extra, is_postgres_vector_type, is_identity_column_extra, is_mysql_generated_column_extra, is_postgres_generated_always_identity_extra,
keyset_pagination_sql_with_identifier_quote, quote_identifier, quote_postgres_string_literal, is_postgres_vector_type, keyset_pagination_sql_with_identifier_quote, quote_identifier,
wrap_dameng_identity_insert_sql_for_table, quote_postgres_string_literal, wrap_dameng_identity_insert_sql_for_table,
}; };
use crate::types::{ObjectSourceKind, SpatialColumn, SqlExportColumnSelection}; use crate::types::{ObjectSourceKind, SpatialColumn, SqlExportColumnSelection};
@@ -465,6 +465,7 @@ struct PostgresExportSequence {
last_value: Option<String>, last_value: Option<String>,
owner_table: Option<String>, owner_table: Option<String>,
owner_column: Option<String>, owner_column: Option<String>,
created_by_table: bool,
} }
#[derive(Debug, Clone, PartialEq, Eq)] #[derive(Debug, Clone, PartialEq, Eq)]
@@ -1523,7 +1524,17 @@ pub(crate) fn build_export_insert_statements_excluding_with_dialect(
// `.sql` files stay readable in plain text editors. Statements that hold a // `.sql` files stay readable in plain text editors. Statements that hold a
// single tuple keep the compact `VALUES (..);` form, which also preserves // single tuple keep the compact `VALUES (..);` form, which also preserves
// the "one INSERT per row" output of the single-row insert mode. // the "one INSERT per row" output of the single-row insert mode.
let statement_head = format!("INSERT INTO {table} ({columns}) VALUES"); let overriding = if options.database_type == Some(DatabaseType::Postgres)
&& insert_columns.iter().any(|(index, _, _)| {
is_postgres_generated_always_identity_extra(
options.column_extras.get(*index).and_then(|value| value.as_deref()),
)
}) {
" OVERRIDING SYSTEM VALUE"
} else {
""
};
let statement_head = format!("INSERT INTO {table} ({columns}){overriding} VALUES");
let statement_prefix = format!("{statement_head} "); let statement_prefix = format!("{statement_head} ");
let output_database_type = if insert_dialect == SqlInsertDialect::Source { options.database_type } else { None }; let output_database_type = if insert_dialect == SqlInsertDialect::Source { options.database_type } else { None };
let statement_overhead_bytes = export_sql_statement_bytes(output_database_type, &statement_prefix) + 1; let statement_overhead_bytes = export_sql_statement_bytes(output_database_type, &statement_prefix) + 1;
@@ -2040,11 +2051,10 @@ fn postgres_sequence_qualified_name(schema: &str, sequence_name: &str) -> String
} }
} }
fn generate_postgres_sequence_create_ddl(sequence: &PostgresExportSequence, schema: &str) -> String { fn postgres_sequence_options(sequence: &PostgresExportSequence) -> String {
let qualified_name = postgres_sequence_qualified_name(schema, &sequence.name);
let cycle = if sequence.cycle { "CYCLE" } else { "NO CYCLE" }; let cycle = if sequence.cycle { "CYCLE" } else { "NO CYCLE" };
format!( format!(
"CREATE SEQUENCE IF NOT EXISTS {qualified_name}\n AS {data_type}\n START WITH {start_value}\n INCREMENT BY {increment}\n MINVALUE {min_value}\n MAXVALUE {max_value}\n CACHE {cache_value}\n {cycle}", " AS {data_type}\n START WITH {start_value}\n INCREMENT BY {increment}\n MINVALUE {min_value}\n MAXVALUE {max_value}\n CACHE {cache_value}\n {cycle}",
data_type = sequence.data_type, data_type = sequence.data_type,
start_value = sequence.start_value, start_value = sequence.start_value,
increment = sequence.increment, increment = sequence.increment,
@@ -2054,33 +2064,73 @@ fn generate_postgres_sequence_create_ddl(sequence: &PostgresExportSequence, sche
) )
} }
fn generate_postgres_sequence_owner_ddl(sequence: &PostgresExportSequence, schema: &str) -> Option<String> { fn generate_postgres_sequence_create_ddl(sequence: &PostgresExportSequence, schema: &str) -> String {
format!(
"CREATE SEQUENCE IF NOT EXISTS {}\n{}",
postgres_sequence_qualified_name(schema, &sequence.name),
postgres_sequence_options(sequence),
)
}
fn generate_postgres_sequence_post_table_ddl(sequence: &PostgresExportSequence, schema: &str) -> Option<String> {
let owner_table = sequence.owner_table.as_deref()?; let owner_table = sequence.owner_table.as_deref()?;
let owner_column = sequence.owner_column.as_deref()?; let owner_column = sequence.owner_column.as_deref()?;
let owner_table = crate::transfer::qualified_table(owner_table, schema, &DatabaseType::Postgres, None);
if sequence.created_by_table {
// CREATE TABLE owns the sequence. Configure that actual binding without
// relying on its name, and quote the whole DO body (identifiers may contain dollar tags).
let body = format!(
"BEGIN EXECUTE 'ALTER SEQUENCE ' || pg_get_serial_sequence({}, {}) || {}; END;",
quote_postgres_string_literal(&owner_table),
quote_postgres_string_literal(owner_column),
quote_postgres_string_literal(&format!(
"\n{} RESTART WITH {}",
postgres_sequence_options(sequence),
sequence.start_value
)),
);
return Some(format!("DO {}", quote_postgres_string_literal(&body)));
}
Some(format!( Some(format!(
"ALTER SEQUENCE {} OWNED BY {}.{}", "ALTER SEQUENCE {} OWNED BY {}.{}",
postgres_sequence_qualified_name(schema, &sequence.name), postgres_sequence_qualified_name(schema, &sequence.name),
crate::transfer::qualified_table(owner_table, schema, &DatabaseType::Postgres, None), owner_table,
quote_identifier(owner_column, &DatabaseType::Postgres) quote_identifier(owner_column, &DatabaseType::Postgres)
)) ))
} }
fn generate_postgres_sequence_setval_sql(sequence: &PostgresExportSequence, schema: &str) -> Option<String> { fn generate_postgres_sequence_setval_sql(sequence: &PostgresExportSequence, schema: &str) -> Option<String> {
let last_value = sequence.last_value.as_deref()?.trim(); let last_value = sequence.last_value.as_deref().map(str::trim).filter(|value| !value.is_empty());
if last_value.is_empty() { let mut sequence_literal = quote_postgres_string_literal(&postgres_sequence_qualified_name(schema, &sequence.name));
return None;
}
let sequence_literal = quote_postgres_string_literal(&postgres_sequence_qualified_name(schema, &sequence.name));
match (sequence.owner_table.as_deref(), sequence.owner_column.as_deref()) { match (sequence.owner_table.as_deref(), sequence.owner_column.as_deref()) {
(Some(owner_table), Some(owner_column)) => { (Some(owner_table), Some(owner_column)) => {
let owner_table = crate::transfer::qualified_table(owner_table, schema, &DatabaseType::Postgres, None); let owner_table = crate::transfer::qualified_table(owner_table, schema, &DatabaseType::Postgres, None);
if sequence.created_by_table {
sequence_literal = format!(
"pg_get_serial_sequence({}, {})",
quote_postgres_string_literal(&owner_table),
quote_postgres_string_literal(owner_column),
);
}
if let Some(value) = last_value.filter(|_| sequence.cycle) {
// A cycled counter may be below/above the table's extrema after wrapping.
return Some(format!("SELECT setval({sequence_literal}, {value}, true)"));
}
let owner_column = quote_identifier(owner_column, &DatabaseType::Postgres); let owner_column = quote_identifier(owner_column, &DatabaseType::Postgres);
let descending = sequence.increment.trim().starts_with('-');
let (aggregate, extreme, comparison) =
if descending { ("MIN", "LEAST", "<=") } else { ("MAX", "GREATEST", ">=") };
let fallback = last_value.unwrap_or(&sequence.start_value);
let called = if last_value.is_some() {
"true".to_string()
} else {
format!("COALESCE({aggregate}({owner_column}) {comparison} {fallback}, false)")
};
Some(format!( Some(format!(
"SELECT setval({sequence_literal}, GREATEST(COALESCE(MAX({owner_column}), {last_value}), {last_value}), true) FROM {owner_table}" "SELECT setval({sequence_literal}, {extreme}(COALESCE({aggregate}({owner_column}), {fallback}), {fallback}), {called}) FROM {owner_table}"
)) ))
} }
_ => Some(format!("SELECT setval({sequence_literal}, {last_value}, true)")), _ => last_value.map(|value| format!("SELECT setval({sequence_literal}, {value}, true)")),
} }
} }
@@ -2175,7 +2225,10 @@ const POSTGRES_EXPORT_SEQUENCES_SQL: &str = "SELECT c.relname, \
COALESCE(s.seqcycle, false), \ COALESCE(s.seqcycle, false), \
COALESCE(s.seqcache::text, '1'), \ COALESCE(s.seqcache::text, '1'), \
t.relname, \ t.relname, \
a.attname \ a.attname, \
COALESCE(d.deptype = 'i', false) OR \
COALESCE(a.atttypid IN (20, 21, 23) AND \
pg_get_expr(ad.adbin, ad.adrelid) = format('nextval(%L::regclass)', c.oid::regclass::text), false) \
FROM pg_class c \ FROM pg_class c \
JOIN pg_namespace n ON n.oid = c.relnamespace \ JOIN pg_namespace n ON n.oid = c.relnamespace \
LEFT JOIN pg_sequence s ON s.seqrelid = c.oid \ LEFT JOIN pg_sequence s ON s.seqrelid = c.oid \
@@ -2186,6 +2239,7 @@ const POSTGRES_EXPORT_SEQUENCES_SQL: &str = "SELECT c.relname, \
LEFT JOIN pg_class t ON t.oid = d.refobjid \ LEFT JOIN pg_class t ON t.oid = d.refobjid \
LEFT JOIN pg_namespace tn ON tn.oid = t.relnamespace AND tn.nspname = n.nspname \ LEFT JOIN pg_namespace tn ON tn.oid = t.relnamespace AND tn.nspname = n.nspname \
LEFT JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = d.refobjsubid \ LEFT JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = d.refobjsubid \
LEFT JOIN pg_attrdef ad ON ad.adrelid = a.attrelid AND ad.adnum = a.attnum \
WHERE c.relkind = 'S' AND n.nspname = $1 \ WHERE c.relkind = 'S' AND n.nspname = $1 \
ORDER BY c.relname"; ORDER BY c.relname";
@@ -2203,7 +2257,9 @@ const POSTGRES_EXPORT_SEQUENCES_COMPAT_SQL: &str = "SELECT c.relname, \
false, \ false, \
'1', \ '1', \
t.relname, \ t.relname, \
a.attname \ a.attname, \
COALESCE(a.atttypid IN (20, 21, 23) AND \
pg_get_expr(ad.adbin, ad.adrelid) = format('nextval(%L::regclass)', c.oid::regclass::text), false) \
FROM pg_class c \ FROM pg_class c \
JOIN pg_namespace n ON n.oid = c.relnamespace \ JOIN pg_namespace n ON n.oid = c.relnamespace \
LEFT JOIN pg_depend d ON d.classid = 'pg_class'::regclass \ LEFT JOIN pg_depend d ON d.classid = 'pg_class'::regclass \
@@ -2213,6 +2269,7 @@ const POSTGRES_EXPORT_SEQUENCES_COMPAT_SQL: &str = "SELECT c.relname, \
LEFT JOIN pg_class t ON t.oid = d.refobjid \ LEFT JOIN pg_class t ON t.oid = d.refobjid \
LEFT JOIN pg_namespace tn ON tn.oid = t.relnamespace AND tn.nspname = n.nspname \ LEFT JOIN pg_namespace tn ON tn.oid = t.relnamespace AND tn.nspname = n.nspname \
LEFT JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = d.refobjsubid \ LEFT JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = d.refobjsubid \
LEFT JOIN pg_attrdef ad ON ad.adrelid = a.attrelid AND ad.adnum = a.attnum \
WHERE c.relkind = 'S' AND n.nspname = $1 \ WHERE c.relkind = 'S' AND n.nspname = $1 \
ORDER BY c.relname"; ORDER BY c.relname";
@@ -2312,6 +2369,7 @@ async fn list_postgres_export_sequences(
last_value: None, last_value: None,
owner_table: row.get::<_, Option<String>>(8), owner_table: row.get::<_, Option<String>>(8),
owner_column: row.get::<_, Option<String>>(9), owner_column: row.get::<_, Option<String>>(9),
created_by_table: row.get::<_, bool>(10),
}) })
.filter(|sequence| { .filter(|sequence| {
sequence.owner_table.as_deref().is_none_or(|owner_table| !excluded.contains(owner_table)) sequence.owner_table.as_deref().is_none_or(|owner_table| !excluded.contains(owner_table))
@@ -3908,7 +3966,7 @@ async fn export_database_sql_core_inner(
&DatabaseExportObjectCounts { &DatabaseExportObjectCounts {
tables: tables.len(), tables: tables.len(),
views: views.len(), views: views.len(),
sequences: postgres_sequences.len(), sequences: postgres_sequences.iter().filter(|sequence| !sequence.created_by_table).count(),
extensions: postgres_extensions.len(), extensions: postgres_extensions.len(),
enums: postgres_enums.len(), enums: postgres_enums.len(),
procedures: procedures.len(), procedures: procedures.len(),
@@ -4150,10 +4208,9 @@ async fn export_database_sql_core_inner(
writeln!(file, "{}\n", drop_table_if_exists_sql(table_name, &request.schema, &db_type)) writeln!(file, "{}\n", drop_table_if_exists_sql(table_name, &request.schema, &db_type))
.map_err(|e| format!("Failed to write file: {e}"))?; .map_err(|e| format!("Failed to write file: {e}"))?;
} }
for sequence in postgres_sequences for sequence in postgres_sequences.iter().filter(|sequence| {
.iter() !sequence.created_by_table && sequence.owner_table.as_deref() == Some(table_name.as_str())
.filter(|sequence| sequence.owner_table.as_deref() == Some(table_name.as_str())) }) {
{
on_progress(ExportProgress { on_progress(ExportProgress {
export_id: request.export_id.clone(), export_id: request.export_id.clone(),
current_object: sequence.name.clone(), current_object: sequence.name.clone(),
@@ -4458,7 +4515,7 @@ async fn export_database_sql_core_inner(
if request.include_structure && !postgres_sequences.is_empty() { if request.include_structure && !postgres_sequences.is_empty() {
for sequence in &postgres_sequences { for sequence in &postgres_sequences {
if let Some(sql) = generate_postgres_sequence_owner_ddl(sequence, &request.schema) { if let Some(sql) = generate_postgres_sequence_post_table_ddl(sequence, &request.schema) {
writeln!(file, "{};\n", sql).map_err(|e| format!("Failed to write file: {e}"))?; writeln!(file, "{};\n", sql).map_err(|e| format!("Failed to write file: {e}"))?;
} }
} }
@@ -4814,16 +4871,17 @@ mod tests {
ensure_export_destination_dir, export_destination_identity_mismatch, extract_postgres_deferred_foreign_keys, ensure_export_destination_dir, export_destination_identity_mismatch, extract_postgres_deferred_foreign_keys,
filter_export_table_infos, format_export_sql_literal, format_export_table_ddl, filter_export_table_infos, format_export_sql_literal, format_export_table_ddl,
format_mysql_spatial_export_literal, format_xugu_spatial_export_literal, generate_postgres_enum_ddl, format_mysql_spatial_export_literal, format_xugu_spatial_export_literal, generate_postgres_enum_ddl,
generate_postgres_extension_ddl, generate_postgres_sequence_create_ddl, generate_postgres_sequence_owner_ddl, generate_postgres_extension_ddl, generate_postgres_sequence_create_ddl,
generate_postgres_sequence_setval_sql, is_postgres_extension_member_routine, mysql_database_export_preamble, generate_postgres_sequence_post_table_ddl, generate_postgres_sequence_setval_sql,
mysql_view_dependencies_from_rows, mysql_view_dependencies_sql, normalize_export_table_ddl, is_postgres_extension_member_routine, mysql_database_export_preamble, mysql_view_dependencies_from_rows,
record_export_destination_identity, record_export_error, replace_database_export_select_list, mysql_view_dependencies_sql, normalize_export_table_ddl, record_export_destination_identity,
sort_export_views_by_dependencies, split_postgres_export_table_triggers, write_database_export_rows, record_export_error, replace_database_export_select_list, sort_export_views_by_dependencies,
write_database_export_rows_with_mode, BuildDatabaseSqlExportOptions, BuildExportInsertStatementsOptions, split_postgres_export_table_triggers, write_database_export_rows, write_database_export_rows_with_mode,
BuildExportSqlInsertOptions, DatabaseExportObjectCounts, DatabaseExportRequest, DatabaseExportWriter, BuildDatabaseSqlExportOptions, BuildExportInsertStatementsOptions, BuildExportSqlInsertOptions,
DdlNormalizeOptions, ExportedTableSql, PostgresExportEnum, PostgresExportExtension, PostgresExportSequence, DatabaseExportObjectCounts, DatabaseExportRequest, DatabaseExportWriter, DdlNormalizeOptions, ExportedTableSql,
PostgresExtensionMembers, SqlInsertDialect, SqlInsertMode, DATABASE_EXPORT_INSERT_BATCH_SIZE, PostgresExportEnum, PostgresExportExtension, PostgresExportSequence, PostgresExtensionMembers,
DATABASE_EXPORT_ROW_LIMIT, POSTGRES_EXPORT_SEQUENCES_COMPAT_SQL, POSTGRES_EXPORT_SEQUENCES_SQL, SqlInsertDialect, SqlInsertMode, DATABASE_EXPORT_INSERT_BATCH_SIZE, DATABASE_EXPORT_ROW_LIMIT,
POSTGRES_EXPORT_SEQUENCES_COMPAT_SQL, POSTGRES_EXPORT_SEQUENCES_SQL,
}; };
use super::{ExportProgress, LenientExportErrors}; use super::{ExportProgress, LenientExportErrors};
use crate::connection::AppState; use crate::connection::AppState;
@@ -7528,6 +7586,56 @@ mod tests {
assert_eq!(combined.matches("INSERT INTO").count(), 20); assert_eq!(combined.matches("INSERT INTO").count(), 20);
} }
#[test]
fn postgres_identity_export_preserves_explicit_values_in_each_batch() {
for (extra, overrides) in [
(Some("generated always as identity (start with 1 increment by 1)"), true),
(Some(" GENERATED\tALWAYS AS IDENTITY "), true),
(Some("generated by default as identity"), false),
(Some("serial"), false),
(None, false),
] {
for dialect in ["source", "standard"] {
for (batch_size, tuples) in [
(1, vec!["(1, 'Ada');", "(2, 'Grace');", "(3, 'Hopper');"]),
(2, vec!["(1, 'Ada'),\n(2, 'Grace');", "(3, 'Hopper');"]),
(10, vec!["(1, 'Ada'),\n(2, 'Grace'),\n(3, 'Hopper');"]),
] {
let options = serde_json::from_value(json!({
"databaseType": "postgres", "schema": "public", "tableName": "users",
"columns": ["id", "name"], "columnTypes": ["integer", "text"],
"columnExtras": [extra, null], "rows": [[1, "Ada"], [2, "Grace"], [3, "Hopper"]],
"batchSize": batch_size, "insertDialect": dialect
}))
.unwrap();
let sql = build_export_sql_insert(options).unwrap();
let overriding = if overrides { " OVERRIDING SYSTEM VALUE" } else { "" };
let head = format!("INSERT INTO \"public\".\"users\" (\"id\", \"name\"){overriding} VALUES");
let expected = tuples
.iter()
.map(|tuple| format!("{head}{}{tuple}", if tuple.contains('\n') { "\n" } else { " " }))
.collect::<Vec<_>>()
.join("\n");
assert_eq!(sql, expected, "extra={extra:?}, dialect={dialect}, batch={batch_size}");
}
}
}
}
#[test]
fn postgres_identity_export_does_not_override_excluded_identity_columns() {
let options = serde_json::from_value(json!({
"databaseType": "postgres", "schema": "public", "tableName": "users",
"columns": ["id", "name"], "columnExtras": ["generated always as identity", null],
"rows": [[42, "Ada"]], "excludeColumns": ["ID"]
}))
.unwrap();
assert_eq!(
build_export_sql_insert(options).unwrap(),
"INSERT INTO \"public\".\"users\" (\"name\") VALUES ('Ada');"
);
}
#[test] #[test]
fn dameng_identity_export_inserts_enable_identity_insert() { fn dameng_identity_export_inserts_enable_identity_insert() {
let statements = build_export_insert_statements(BuildExportInsertStatementsOptions { let statements = build_export_insert_statements(BuildExportInsertStatementsOptions {
@@ -7762,7 +7870,7 @@ mod tests {
assert!(!POSTGRES_EXPORT_SEQUENCES_COMPAT_SQL.contains("pg_sequence")); assert!(!POSTGRES_EXPORT_SEQUENCES_COMPAT_SQL.contains("pg_sequence"));
assert!(!POSTGRES_EXPORT_SEQUENCES_COMPAT_SQL.contains("pg_sequence_last_value")); assert!(!POSTGRES_EXPORT_SEQUENCES_COMPAT_SQL.contains("pg_sequence_last_value"));
// Both tiers must describe the same ten columns in the same order so // Both tiers must describe the same eleven columns in the same order so
// the row decoding below stays identical. // the row decoding below stays identical.
for sql in [POSTGRES_EXPORT_SEQUENCES_SQL, POSTGRES_EXPORT_SEQUENCES_COMPAT_SQL] { for sql in [POSTGRES_EXPORT_SEQUENCES_SQL, POSTGRES_EXPORT_SEQUENCES_COMPAT_SQL] {
for fragment in [ for fragment in [
@@ -7795,6 +7903,7 @@ mod tests {
last_value: Some("42".to_string()), last_value: Some("42".to_string()),
owner_table: Some("permissions".to_string()), owner_table: Some("permissions".to_string()),
owner_column: Some("id".to_string()), owner_column: Some("id".to_string()),
created_by_table: false,
} }
} }
@@ -7823,7 +7932,7 @@ mod tests {
let sequence = postgres_sequence("permissions_id_seq"); let sequence = postgres_sequence("permissions_id_seq");
assert_eq!( assert_eq!(
generate_postgres_sequence_owner_ddl(&sequence, "public").as_deref(), generate_postgres_sequence_post_table_ddl(&sequence, "public").as_deref(),
Some("ALTER SEQUENCE \"public\".\"permissions_id_seq\" OWNED BY \"public\".\"permissions\".\"id\"") Some("ALTER SEQUENCE \"public\".\"permissions_id_seq\" OWNED BY \"public\".\"permissions\".\"id\"")
); );
assert_eq!( assert_eq!(
@@ -7834,6 +7943,20 @@ mod tests {
); );
} }
#[test]
fn postgres_table_generated_sequence_restores_the_table_bound_sequence() {
let mut sequence = postgres_sequence("renamed_identity_sequence");
sequence.created_by_table = true;
let ddl = generate_postgres_sequence_post_table_ddl(&sequence, "public").unwrap();
assert!(ddl.starts_with("DO "));
assert!(ddl.contains("ALTER SEQUENCE "));
assert!(!ddl.contains("OWNED BY"));
assert_eq!(
generate_postgres_sequence_setval_sql(&sequence, "public").as_deref(),
Some("SELECT setval(pg_get_serial_sequence('\"public\".\"permissions\"', 'id'), GREATEST(COALESCE(MAX(\"id\"), 42), 42), true) FROM \"public\".\"permissions\"")
);
}
#[test] #[test]
fn strict_exports_return_object_errors_instead_of_writing_error_comments() { fn strict_exports_return_object_errors_instead_of_writing_error_comments() {
let path = std::env::temp_dir().join(format!("dbx-strict-export-{}.sql", uuid::Uuid::new_v4())); let path = std::env::temp_dir().join(format!("dbx-strict-export-{}.sql", uuid::Uuid::new_v4()));
@@ -7908,7 +8031,12 @@ mod tests {
async fn test_app_state(scratch_dir: &std::path::Path) -> AppState { async fn test_app_state(scratch_dir: &std::path::Path) -> AppState {
let storage = crate::persistence::test_storage::open(&scratch_dir.join("storage.db")).await.unwrap(); let storage = crate::persistence::test_storage::open(&scratch_dir.join("storage.db")).await.unwrap();
AppState::new(storage) AppState::new_with_plugin_and_agent_dir_and_app_version(
storage,
scratch_dir.join("plugins"),
scratch_dir.join("agents"),
env!("CARGO_PKG_VERSION"),
)
} }
// Regression tests for #6327: create_dir_all on every export run is unsafe // Regression tests for #6327: create_dir_all on every export run is unsafe
+8 -2
View File
@@ -1634,7 +1634,7 @@ fn is_postgres_identity_extra(extra: Option<&str>) -> bool {
}) })
} }
fn is_postgres_generated_always_identity_extra(extra: Option<&str>) -> bool { pub(crate) fn is_postgres_generated_always_identity_extra(extra: Option<&str>) -> bool {
extra.is_some_and(|value| { extra.is_some_and(|value| {
let mut parts = value.split_whitespace(); let mut parts = value.split_whitespace();
parts.next().is_some_and(|part| part.eq_ignore_ascii_case("generated")) parts.next().is_some_and(|part| part.eq_ignore_ascii_case("generated"))
@@ -12415,7 +12415,13 @@ CREATE TABLE "Other"."prefix""Source"."NAME" ("ID" INT);"#;
let dir = std::env::temp_dir().join(format!("dbx-transfer-test-{}", uuid::Uuid::new_v4())); let dir = std::env::temp_dir().join(format!("dbx-transfer-test-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap(); std::fs::create_dir_all(&dir).unwrap();
let storage = crate::persistence::test_storage::open(&dir.join("storage.db")).await.unwrap(); let storage = crate::persistence::test_storage::open(&dir.join("storage.db")).await.unwrap();
(AppState::new(storage), dir) let state = AppState::new_with_plugin_and_agent_dir_and_app_version(
storage,
dir.join("plugins"),
dir.join("agents"),
env!("CARGO_PKG_VERSION"),
);
(state, dir)
} }
async fn spawn_influxdb3_transfer_server() -> (String, tokio::task::JoinHandle<String>) { async fn spawn_influxdb3_transfer_server() -> (String, tokio::task::JoinHandle<String>) {
@@ -2,8 +2,10 @@ use dbx_core::connection::AppState;
use dbx_core::database_export::{export_database_sql_core, DatabaseExportRequest, ExportStatus}; use dbx_core::database_export::{export_database_sql_core, DatabaseExportRequest, ExportStatus};
use dbx_core::models::connection::{ConnectionConfig, DatabaseType}; use dbx_core::models::connection::{ConnectionConfig, DatabaseType};
use dbx_core::query::execute_sql_statement; use dbx_core::query::execute_sql_statement;
use dbx_core::query_result_export::{export_query_result_core, QueryResultExportRequest};
use dbx_core::sql::SqlFileRequest; use dbx_core::sql::SqlFileRequest;
use dbx_core::sql_file_import::execute_sql_file_path; use dbx_core::sql_file_import::execute_sql_file_path;
use dbx_core::table_export::{export_table_data_core, TableExportRequest};
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
@@ -45,7 +47,12 @@ async fn live_postgres_all_schema_export_restores_one_sql_file() {
let dir = tempfile::tempdir().expect("create export temp directory"); let dir = tempfile::tempdir().expect("create export temp directory");
let storage = let storage =
dbx_core::persistence::test_storage::open(&dir.path().join("storage.db")).await.expect("open temp storage"); dbx_core::persistence::test_storage::open(&dir.path().join("storage.db")).await.expect("open temp storage");
let state = Arc::new(AppState::new(storage)); let state = Arc::new(AppState::new_with_plugin_and_agent_dir_and_app_version(
storage,
dir.path().join("plugins"),
dir.path().join("agents"),
env!("CARGO_PKG_VERSION"),
));
state state
.configs .configs
.write() .write()
@@ -78,12 +85,41 @@ async fn live_postgres_all_schema_export_restores_one_sql_file() {
"CREATE SCHEMA inventory", "CREATE SCHEMA inventory",
"CREATE SCHEMA reporting", "CREATE SCHEMA reporting",
"CREATE TABLE public.accounts (id integer PRIMARY KEY, name text NOT NULL)", "CREATE TABLE public.accounts (id integer PRIMARY KEY, name text NOT NULL)",
"CREATE TABLE public.identity_always (id integer GENERATED ALWAYS AS IDENTITY PRIMARY KEY, name text NOT NULL)",
"CREATE TABLE public.identity_default (id integer GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, name text NOT NULL)",
"CREATE TABLE public.serial_ids (id serial PRIMARY KEY, name text NOT NULL)",
"CREATE TABLE public.identity_desc (id integer GENERATED ALWAYS AS IDENTITY (START WITH 0 INCREMENT BY -1 MINVALUE -100 MAXVALUE 0) PRIMARY KEY, name text NOT NULL)",
"CREATE TABLE public.identity_unused (id integer GENERATED ALWAYS AS IDENTITY PRIMARY KEY, name text NOT NULL)",
"CREATE TABLE public.serial_unused (id serial PRIMARY KEY, name text NOT NULL)",
"CREATE TABLE public.identity_empty (id integer GENERATED ALWAYS AS IDENTITY PRIMARY KEY)",
"CREATE TABLE public.identity_cycle (id integer GENERATED ALWAYS AS IDENTITY (MINVALUE 1 MAXVALUE 5 CYCLE) PRIMARY KEY, name text NOT NULL)",
"CREATE TABLE inventory.products (id integer PRIMARY KEY, sku text NOT NULL)", "CREATE TABLE inventory.products (id integer PRIMARY KEY, sku text NOT NULL)",
"CREATE TABLE reporting.daily_totals (day date PRIMARY KEY, amount numeric(12,2) NOT NULL)", "CREATE TABLE reporting.daily_totals (day date PRIMARY KEY, amount numeric(12,2) NOT NULL)",
"CREATE VIEW reporting.positive_totals AS SELECT day, amount FROM reporting.daily_totals WHERE amount > 0", "CREATE VIEW reporting.positive_totals AS SELECT day, amount FROM reporting.daily_totals WHERE amount > 0",
"INSERT INTO public.accounts VALUES (1, 'Alice'), (2, 'Bob')", "INSERT INTO public.accounts VALUES (1, 'Alice'), (2, 'Bob')",
"INSERT INTO inventory.products VALUES (10, 'SKU-10')", "INSERT INTO inventory.products VALUES (10, 'SKU-10')",
"INSERT INTO reporting.daily_totals VALUES ('2026-09-07', 42.50)", "INSERT INTO reporting.daily_totals VALUES ('2026-09-07', 42.50)",
"INSERT INTO public.identity_always (name) VALUES ('one'), ('two')",
"INSERT INTO public.identity_default (name) VALUES ('one'), ('two')",
"INSERT INTO public.serial_ids (name) VALUES ('one'), ('two')",
"INSERT INTO public.identity_always OVERRIDING SYSTEM VALUE VALUES (42, 'explicit')",
"INSERT INTO public.identity_default VALUES (42, 'explicit')",
"INSERT INTO public.serial_ids VALUES (42, 'explicit')",
"ALTER SEQUENCE public.identity_always_id_seq RENAME TO renamed_identity_sequence",
"ALTER SEQUENCE public.renamed_identity_sequence MINVALUE -100 MAXVALUE 1000 CACHE 3 CYCLE",
"ALTER SEQUENCE public.serial_ids_id_seq START WITH 10 INCREMENT BY 5 MINVALUE 1 MAXVALUE 1000 CACHE 3 CYCLE",
"SELECT setval('public.renamed_identity_sequence', 80)",
"SELECT setval('public.identity_default_id_seq', 80)",
"SELECT setval('public.serial_ids_id_seq', 80)",
"INSERT INTO public.identity_desc (name) VALUES ('zero'), ('minus one')",
"INSERT INTO public.identity_desc OVERRIDING SYSTEM VALUE VALUES (-42, 'explicit')",
"SELECT setval('public.identity_desc_id_seq', -80)",
"INSERT INTO public.identity_unused OVERRIDING SYSTEM VALUE VALUES (1, 'explicit')",
"INSERT INTO public.serial_unused VALUES (1, 'explicit')",
"INSERT INTO public.identity_cycle (name) VALUES ('one')",
"INSERT INTO public.identity_cycle OVERRIDING SYSTEM VALUE VALUES (5, 'explicit')",
"SELECT setval('public.identity_cycle_id_seq', 5)",
"SELECT nextval('public.identity_cycle_id_seq')",
] { ] {
Box::pin(execute_sql_statement(&state, &source_connection_id, &source_database, statement, None, None)) Box::pin(execute_sql_statement(&state, &source_connection_id, &source_database, statement, None, None))
.await .await
@@ -97,7 +133,7 @@ async fn live_postgres_all_schema_export_restores_one_sql_file() {
&state, &state,
&DatabaseExportRequest { &DatabaseExportRequest {
export_id: format!("postgres-all-schema-export-{suffix}"), export_id: format!("postgres-all-schema-export-{suffix}"),
connection_id: source_connection_id, connection_id: source_connection_id.clone(),
database: source_database.clone(), database: source_database.clone(),
schema: String::new(), schema: String::new(),
file_path: export_path.display().to_string(), file_path: export_path.display().to_string(),
@@ -181,6 +217,219 @@ async fn live_postgres_all_schema_export_restores_one_sql_file() {
.expect("query restored schemas"); .expect("query restored schemas");
assert_eq!(restored.rows, vec![vec![serde_json::json!(2), serde_json::json!(1), serde_json::json!("42.50")]]); assert_eq!(restored.rows, vec![vec![serde_json::json!(2), serde_json::json!(1), serde_json::json!("42.50")]]);
for (table, next_id, increment, duplicate_id) in [
("identity_always", 81, 1, 42),
("identity_default", 81, 1, 42),
("serial_ids", 85, 5, 42),
("identity_desc", -81, -1, -42),
("identity_unused", 2, 1, 1),
("serial_unused", 2, 1, 1),
("identity_cycle", 2, 1, 5),
] {
let select = format!("SELECT id, name FROM public.{table} ORDER BY id");
let expected =
Box::pin(execute_sql_statement(&state, &source_connection_id, &source_database, &select, None, None))
.await
.unwrap()
.rows;
let restored =
Box::pin(execute_sql_statement(&state, &target_connection_id, &target_database, &select, None, None))
.await
.unwrap();
assert_eq!(restored.rows, expected, "database export must preserve {table} IDs");
let parameters_sql = format!(
"SELECT seqstart, seqmin, seqmax, seqincrement, seqcache, seqcycle FROM pg_sequence \
WHERE seqrelid = pg_get_serial_sequence('public.{table}', 'id')::regclass"
);
let source_parameters = Box::pin(execute_sql_statement(
&state,
&source_connection_id,
&source_database,
&parameters_sql,
None,
None,
))
.await
.unwrap();
let target_parameters = Box::pin(execute_sql_statement(
&state,
&target_connection_id,
&target_database,
&parameters_sql,
None,
None,
))
.await
.unwrap();
assert_eq!(target_parameters.rows, source_parameters.rows, "restore {table} sequence parameters");
let next = Box::pin(execute_sql_statement(
&state,
&target_connection_id,
&target_database,
&format!("INSERT INTO public.{table} (name) VALUES ('next') RETURNING id"),
None,
None,
))
.await
.unwrap();
assert_eq!(next.rows, vec![vec![serde_json::json!(next_id)]], "restore {table} sequence state");
let sequence_state_sql =
format!("SELECT pg_sequence_last_value(pg_get_serial_sequence('public.{table}', 'id')::regclass)::text");
let before_data_import = Box::pin(execute_sql_statement(
&state,
&target_connection_id,
&target_database,
&sequence_state_sql,
None,
None,
))
.await
.unwrap();
let columns =
dbx_core::schema::get_columns_core(&state, &source_connection_id, &source_database, "public", table)
.await
.unwrap();
for mode in ["single", "batch"] {
for entry in ["table", "query"] {
let path = dir.path().join(format!("{table}-{mode}-{entry}.sql"));
if entry == "table" {
// No supplied columns: exercise the table exporter's metadata query.
let request: TableExportRequest = serde_json::from_value(serde_json::json!({
"exportId": format!("{suffix}-{table}-{mode}-{entry}"),
"connectionId": source_connection_id, "database": source_database,
"schema": "public", "tableName": table, "filePath": path,
"format": "sql", "insertMode": mode, "batchSize": 2
}))
.unwrap();
export_table_data_core(&state, &request, |_| {}).await.unwrap();
} else {
// Query export receives extras from the result's table metadata.
let request: QueryResultExportRequest = serde_json::from_value(serde_json::json!({
"exportId": format!("{suffix}-{table}-{mode}-{entry}"),
"connectionId": source_connection_id, "database": source_database,
"schema": "public", "databaseType": "postgres", "sql": select,
"queryBaseSql": select, "exportTableName": table, "filePath": path,
"format": "sql", "insertMode": mode, "pageSize": 2,
"exportColumnExtras": columns.iter().map(|c| c.extra.clone()).collect::<Vec<_>>()
}))
.unwrap();
export_query_result_core(&state, &request, None, |_| {}).await.unwrap();
}
Box::pin(execute_sql_statement(
&state,
&target_connection_id,
&target_database,
&format!("TRUNCATE public.{table} CONTINUE IDENTITY"),
None,
None,
))
.await
.unwrap();
execute_sql_file_path(
&state,
&SqlFileRequest {
txn_session_id: None,
execution_id: uuid::Uuid::new_v4().to_string(),
connection_id: target_connection_id.clone(),
database: target_database.clone(),
schema: None,
file_path: path.display().to_string(),
continue_on_error: false,
selected_tables: None,
part_cooldown_ms: 0,
skip_relational_constraints: false,
},
&path,
CancellationToken::new(),
std::time::Instant::now(),
|_| {},
)
.await
.unwrap();
let restored = Box::pin(execute_sql_statement(
&state,
&target_connection_id,
&target_database,
&select,
None,
None,
))
.await
.unwrap();
assert_eq!(restored.rows, expected, "{entry}/{mode} must preserve {table} IDs");
}
}
// Data-only exports preserve values, but deliberately leave the target sequence alone.
let after_data_import = Box::pin(execute_sql_statement(
&state,
&target_connection_id,
&target_database,
&sequence_state_sql,
None,
None,
))
.await
.unwrap();
assert_eq!(after_data_import.rows, before_data_import.rows, "data import must not reset {table} sequence");
let next = Box::pin(execute_sql_statement(
&state,
&target_connection_id,
&target_database,
&format!("INSERT INTO public.{table} (name) VALUES ('after data import') RETURNING id"),
None,
None,
))
.await
.unwrap();
let after_id = next.rows[0][0].as_i64().unwrap();
assert!(
(after_id - next_id) * increment > 0 && (after_id - next_id) % increment == 0,
"{table} must continue in the configured direction/step: {after_id}"
);
assert!(
Box::pin(execute_sql_statement(
&state,
&target_connection_id,
&target_database,
&format!("INSERT INTO public.{table} OVERRIDING SYSTEM VALUE VALUES ({duplicate_id}, 'duplicate')"),
None,
None,
))
.await
.is_err(),
"primary key remains enforced for {table}"
);
}
let empty_next = Box::pin(execute_sql_statement(
&state,
&target_connection_id,
&target_database,
"INSERT INTO public.identity_empty DEFAULT VALUES RETURNING id",
None,
None,
))
.await
.unwrap();
assert_eq!(
empty_next.rows,
vec![vec![serde_json::json!(1)]],
"an empty uncalled sequence must retain its first value"
);
assert!(
Box::pin(execute_sql_statement(
&state,
&target_connection_id,
&target_database,
"INSERT INTO public.identity_always VALUES (99, 'forbidden')",
None,
None,
))
.await
.is_err(),
"ALWAYS identity remains enforced after restore"
);
for database in [&source_database, &target_database] { for database in [&source_database, &target_database] {
Box::pin(execute_sql_statement( Box::pin(execute_sql_statement(
&state, &state,
+4 -4
View File
@@ -4736,8 +4736,8 @@ fn postgres_columns_for_relations_sql() -> &'static str {
) AS is_pk, \ ) AS is_pk, \
col_description(a.attrelid, a.attnum) AS column_comment, \ col_description(a.attrelid, a.attnum) AS column_comment, \
CASE a.attidentity \ CASE a.attidentity \
WHEN 'd' THEN 'generated by default as identity' || CASE WHEN pseq.seqstart IS NOT NULL THEN format(' (start with %s increment by %s)', pseq.seqstart, pseq.seqincrement) ELSE '' END \ WHEN 'd' THEN 'generated by default as identity' || CASE WHEN pseq.seqstart IS NOT NULL THEN format(' (start with %s increment by %s minvalue %s maxvalue %s cache %s %s)', pseq.seqstart, pseq.seqincrement, pseq.seqmin, pseq.seqmax, pseq.seqcache, CASE WHEN pseq.seqcycle THEN 'cycle' ELSE 'no cycle' END) ELSE '' END \
WHEN 'a' THEN 'generated always as identity' || CASE WHEN pseq.seqstart IS NOT NULL THEN format(' (start with %s increment by %s)', pseq.seqstart, pseq.seqincrement) ELSE '' END \ WHEN 'a' THEN 'generated always as identity' || CASE WHEN pseq.seqstart IS NOT NULL THEN format(' (start with %s increment by %s minvalue %s maxvalue %s cache %s %s)', pseq.seqstart, pseq.seqincrement, pseq.seqmin, pseq.seqmax, pseq.seqcache, CASE WHEN pseq.seqcycle THEN 'cycle' ELSE 'no cycle' END) ELSE '' END \
ELSE CASE a.attgenerated \ ELSE CASE a.attgenerated \
WHEN 's' THEN 'generated always as (' || pg_get_expr(ad.adbin, ad.adrelid) || ') stored' \ WHEN 's' THEN 'generated always as (' || pg_get_expr(ad.adbin, ad.adrelid) || ') stored' \
WHEN 'v' THEN 'generated always as (' || pg_get_expr(ad.adbin, ad.adrelid) || ') virtual' \ WHEN 'v' THEN 'generated always as (' || pg_get_expr(ad.adbin, ad.adrelid) || ') virtual' \
@@ -7594,8 +7594,8 @@ const POSTGRES_COLUMNS_SQL: &str = "SELECT a.attname AS column_name, \
) AS is_pk, \ ) AS is_pk, \
col_description(a.attrelid, a.attnum) AS column_comment, \ col_description(a.attrelid, a.attnum) AS column_comment, \
CASE a.attidentity \ CASE a.attidentity \
WHEN 'd' THEN 'generated by default as identity' || CASE WHEN pseq.seqstart IS NOT NULL THEN format(' (start with %s increment by %s)', pseq.seqstart, pseq.seqincrement) ELSE '' END \ WHEN 'd' THEN 'generated by default as identity' || CASE WHEN pseq.seqstart IS NOT NULL THEN format(' (start with %s increment by %s minvalue %s maxvalue %s cache %s %s)', pseq.seqstart, pseq.seqincrement, pseq.seqmin, pseq.seqmax, pseq.seqcache, CASE WHEN pseq.seqcycle THEN 'cycle' ELSE 'no cycle' END) ELSE '' END \
WHEN 'a' THEN 'generated always as identity' || CASE WHEN pseq.seqstart IS NOT NULL THEN format(' (start with %s increment by %s)', pseq.seqstart, pseq.seqincrement) ELSE '' END \ WHEN 'a' THEN 'generated always as identity' || CASE WHEN pseq.seqstart IS NOT NULL THEN format(' (start with %s increment by %s minvalue %s maxvalue %s cache %s %s)', pseq.seqstart, pseq.seqincrement, pseq.seqmin, pseq.seqmax, pseq.seqcache, CASE WHEN pseq.seqcycle THEN 'cycle' ELSE 'no cycle' END) ELSE '' END \
ELSE CASE a.attgenerated \ ELSE CASE a.attgenerated \
WHEN 's' THEN 'generated always as (' || pg_get_expr(ad.adbin, ad.adrelid) || ') stored' \ WHEN 's' THEN 'generated always as (' || pg_get_expr(ad.adbin, ad.adrelid) || ') stored' \
WHEN 'v' THEN 'generated always as (' || pg_get_expr(ad.adbin, ad.adrelid) || ') virtual' \ WHEN 'v' THEN 'generated always as (' || pg_get_expr(ad.adbin, ad.adrelid) || ') virtual' \