fix(transfer): handle sqlserver identity columns and sqlite int width
This commit is contained in:
3 files changed
+549
-1
No files matched your search
@@ -3440,6 +3440,25 @@ pub fn map_column_type(source_type: &str, source_db: &DatabaseType, target_db: &
|
|||||||
// Extract basic type, `bigint unsigned` -> `bigint`
|
// Extract basic type, `bigint unsigned` -> `bigint`
|
||||||
base = base.split(' ').next().unwrap_or(base).trim();
|
base = base.split(' ').next().unwrap_or(base).trim();
|
||||||
|
|
||||||
|
// SQLite has no integer widths: every column whose declared type carries the
|
||||||
|
// `INT` substring — `INTEGER`, `INT`, `TINYINT`, `SMALLINT`, even the
|
||||||
|
// SQLite-only `UNSIGNED BIG INT` spelling — is stored as a full 64-bit signed
|
||||||
|
// integer (that is SQLite's documented INTEGER affinity rule). Routing those
|
||||||
|
// names through the 32-bit arms below builds a target column that cannot hold
|
||||||
|
// the source's own values, and the transfer then dies mid-batch instead of
|
||||||
|
// writing the row: SQL Server rejects the batch with code 248
|
||||||
|
// ("The conversion of the nvarchar value '...' overflowed an int column").
|
||||||
|
// rqlite, Turso and Cloudflare D1 are SQLite underneath and share the storage
|
||||||
|
// classes. Send integer-affinity columns down the 64-bit arm instead;
|
||||||
|
// Oracle-family targets keep `INTEGER`, which is already `NUMBER(38, 0)`.
|
||||||
|
if is_sqlite_transfer_dialect(source_db) && t.contains("int") {
|
||||||
|
base = if matches!(target_db, DatabaseType::Oracle | DatabaseType::OceanbaseOracle) {
|
||||||
|
"integer"
|
||||||
|
} else {
|
||||||
|
"bigint"
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
if matches!(
|
if matches!(
|
||||||
target_db,
|
target_db,
|
||||||
DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala | DatabaseType::Argo | DatabaseType::Transwarp
|
DatabaseType::Hive | DatabaseType::Kyuubi | DatabaseType::Impala | DatabaseType::Argo | DatabaseType::Transwarp
|
||||||
@@ -3612,6 +3631,13 @@ pub fn map_column_type(source_type: &str, source_db: &DatabaseType, target_db: &
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// SQLite and the databases that embed it (`rqlite`, Turso, Cloudflare D1) all
|
||||||
|
/// use SQLite's storage classes, where any column with INTEGER affinity holds a
|
||||||
|
/// 64-bit signed integer regardless of the width its declared type suggests.
|
||||||
|
fn is_sqlite_transfer_dialect(db_type: &DatabaseType) -> bool {
|
||||||
|
matches!(db_type, DatabaseType::Sqlite | DatabaseType::Rqlite | DatabaseType::Turso | DatabaseType::CloudflareD1)
|
||||||
|
}
|
||||||
|
|
||||||
fn mysql_type_needs_key_prefix(mapped_type: &str) -> bool {
|
fn mysql_type_needs_key_prefix(mapped_type: &str) -> bool {
|
||||||
let base = mapped_type.split('(').next().unwrap_or(mapped_type).trim().to_ascii_lowercase();
|
let base = mapped_type.split('(').next().unwrap_or(mapped_type).trim().to_ascii_lowercase();
|
||||||
matches!(
|
matches!(
|
||||||
@@ -9787,6 +9813,14 @@ where
|
|||||||
// A preexisting target also needs its columns read, even for a data-only
|
// A preexisting target also needs its columns read, even for a data-only
|
||||||
// transfer: the write SQL has to address the target's declared column
|
// transfer: the write SQL has to address the target's declared column
|
||||||
// names, which can differ from the source in case (#9320).
|
// names, which can differ from the source in case (#9320).
|
||||||
|
//
|
||||||
|
// SQL Server belongs to the always-read set for its identity flags, not just
|
||||||
|
// for a preexisting target: a freshly created target reuses the source DDL
|
||||||
|
// (`can_reuse`), which carries the source's `IDENTITY` clause, so the new
|
||||||
|
// target is an identity target too. `writes_identity_insert_columns` decides
|
||||||
|
// whether the batch needs the `SET IDENTITY_INSERT` wrapper, and without the
|
||||||
|
// target metadata it stays false — SQL Server then rejects the explicit
|
||||||
|
// identity values with 544.
|
||||||
let needs_target_columns = default_rows_only
|
let needs_target_columns = default_rows_only
|
||||||
|| target_table_preexisting
|
|| target_table_preexisting
|
||||||
|| (request.mode == TransferMode::Upsert
|
|| (request.mode == TransferMode::Upsert
|
||||||
@@ -9799,7 +9833,10 @@ where
|
|||||||
| DatabaseType::Argo
|
| DatabaseType::Argo
|
||||||
| DatabaseType::Transwarp
|
| DatabaseType::Transwarp
|
||||||
))
|
))
|
||||||
|| matches!(target_db_type, DatabaseType::Postgres | DatabaseType::Dameng | DatabaseType::H2);
|
|| matches!(
|
||||||
|
target_db_type,
|
||||||
|
DatabaseType::Postgres | DatabaseType::Dameng | DatabaseType::H2 | DatabaseType::SqlServer
|
||||||
|
);
|
||||||
let target_columns = if needs_target_columns {
|
let target_columns = if needs_target_columns {
|
||||||
get_columns_for_transfer(
|
get_columns_for_transfer(
|
||||||
state,
|
state,
|
||||||
@@ -18007,6 +18044,52 @@ SELECT 1 FROM dual"#
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn map_column_type_keeps_mysql_and_duckdb_integer_widths() {
|
||||||
|
// The widening below is specific to SQLite's storage classes: MySQL's
|
||||||
|
// `int` really is 32-bit and DuckDB's `INTEGER` is 32-bit too, so their
|
||||||
|
// mappings must not change.
|
||||||
|
assert_eq!(map_column_type("int", &DatabaseType::Mysql, &DatabaseType::SqlServer), "INT");
|
||||||
|
assert_eq!(map_column_type("INTEGER", &DatabaseType::DuckDb, &DatabaseType::SqlServer), "INT");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn map_column_type_widens_sqlite_integer_affinity_columns() {
|
||||||
|
// User report (SQL Server data transfer): a SQLite `INTEGER` column was
|
||||||
|
// mapped onto SQL Server `INT`, and the first value above 2^31 killed the
|
||||||
|
// batch with code 248 ("... overflowed an int column"). SQLite stores every
|
||||||
|
// integer-affinity column — the declared name is not a width — in a 64-bit
|
||||||
|
// signed integer, so the mapping has to stay 64-bit wide.
|
||||||
|
for source in [DatabaseType::Sqlite, DatabaseType::Rqlite, DatabaseType::Turso, DatabaseType::CloudflareD1] {
|
||||||
|
assert_eq!(map_column_type("INTEGER", &source, &DatabaseType::SqlServer), "BIGINT");
|
||||||
|
assert_eq!(map_column_type("int", &source, &DatabaseType::SqlServer), "BIGINT");
|
||||||
|
assert_eq!(map_column_type("INT(11)", &source, &DatabaseType::SqlServer), "BIGINT");
|
||||||
|
assert_eq!(map_column_type("smallint", &source, &DatabaseType::SqlServer), "BIGINT");
|
||||||
|
assert_eq!(map_column_type("tinyint", &source, &DatabaseType::SqlServer), "BIGINT");
|
||||||
|
assert_eq!(map_column_type("BIGINT", &source, &DatabaseType::SqlServer), "BIGINT");
|
||||||
|
// SQLite's own affinity rule matches the `INT` substring anywhere in
|
||||||
|
// the declared type, which is why `UNSIGNED BIG INT` is an integer.
|
||||||
|
assert_eq!(map_column_type("UNSIGNED BIG INT", &source, &DatabaseType::SqlServer), "BIGINT");
|
||||||
|
assert_eq!(map_column_type("INTEGER", &source, &DatabaseType::Mysql), "BIGINT");
|
||||||
|
assert_eq!(map_column_type("INTEGER", &source, &DatabaseType::Postgres), "BIGINT");
|
||||||
|
assert_eq!(map_column_type("INTEGER", &source, &DatabaseType::Kingbase), "BIGINT");
|
||||||
|
assert_eq!(map_column_type("INTEGER", &source, &DatabaseType::Hive), "BIGINT");
|
||||||
|
// Oracle-family targets keep `INTEGER`, which is already NUMBER(38, 0)
|
||||||
|
// and covers the whole 64-bit range without a `BIGINT` spelling Oracle
|
||||||
|
// does not define.
|
||||||
|
assert_eq!(map_column_type("INTEGER", &source, &DatabaseType::Oracle), "INTEGER");
|
||||||
|
assert_eq!(map_column_type("INTEGER", &source, &DatabaseType::OceanbaseOracle), "INTEGER");
|
||||||
|
// Non-integer declared types keep their existing mapping.
|
||||||
|
assert_eq!(map_column_type("TEXT", &source, &DatabaseType::SqlServer), "TEXT");
|
||||||
|
assert_eq!(map_column_type("REAL", &source, &DatabaseType::SqlServer), "FLOAT");
|
||||||
|
assert_eq!(map_column_type("NUMERIC", &source, &DatabaseType::SqlServer), "NUMERIC");
|
||||||
|
assert_eq!(map_column_type("BLOB", &source, &DatabaseType::SqlServer), "VARBINARY(MAX)");
|
||||||
|
}
|
||||||
|
// A SQLite target is unchanged: it accepts any 64-bit value in `INTEGER`.
|
||||||
|
assert_eq!(map_column_type("int", &DatabaseType::Mysql, &DatabaseType::Sqlite), "INTEGER");
|
||||||
|
assert_eq!(map_column_type("bigint", &DatabaseType::Mysql, &DatabaseType::Sqlite), "BIGINT");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn map_column_type_longtext_falls_back_to_text_for_non_mysql_target() {
|
fn map_column_type_longtext_falls_back_to_text_for_non_mysql_target() {
|
||||||
assert_eq!(map_column_type("longtext", &DatabaseType::Mysql, &DatabaseType::Postgres), "TEXT");
|
assert_eq!(map_column_type("longtext", &DatabaseType::Mysql, &DatabaseType::Postgres), "TEXT");
|
||||||
|
|||||||
@@ -0,0 +1,216 @@
|
|||||||
|
//! Cross-dialect transfer probes: SQLite source -> SQL Server target.
|
||||||
|
//!
|
||||||
|
//! SQLite's `INTEGER` affinity is a 64-bit signed integer, while the transfer
|
||||||
|
//! type mapper used to send any `int`-family source type to SQL Server `INT`
|
||||||
|
//! (32-bit). A user report shows the resulting T-SQL error verbatim:
|
||||||
|
//! `转换 nvarchar 值 '2098620070434758658' 时溢出了整数列。` (code 248).
|
||||||
|
|
||||||
|
use dbx_core::connection::AppState;
|
||||||
|
use dbx_core::models::connection::{ConnectionConfig, DatabaseType};
|
||||||
|
use dbx_core::transfer::{
|
||||||
|
transfer_table, TransferContent, TransferMode, TransferOwnershipPolicy, TransferRequest, TransferTableNameCase,
|
||||||
|
};
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
fn sqlite_config(id: &str, path: &str) -> ConnectionConfig {
|
||||||
|
ConnectionConfig {
|
||||||
|
docs_notes_path: None,
|
||||||
|
id: id.to_string(),
|
||||||
|
name: id.to_string(),
|
||||||
|
note: String::new(),
|
||||||
|
db_type: DatabaseType::Sqlite,
|
||||||
|
driver_profile: None,
|
||||||
|
driver_label: None,
|
||||||
|
url_params: None,
|
||||||
|
agent_java_options: Vec::new(),
|
||||||
|
host: path.to_string(),
|
||||||
|
port: 0,
|
||||||
|
username: String::new(),
|
||||||
|
password: String::new(),
|
||||||
|
database: None,
|
||||||
|
default_schema: None,
|
||||||
|
visible_databases: None,
|
||||||
|
visible_database_patterns: None,
|
||||||
|
visible_schemas: None,
|
||||||
|
attached_databases: Vec::new(),
|
||||||
|
init_script: None,
|
||||||
|
color: None,
|
||||||
|
transport_layers: Vec::new(),
|
||||||
|
connect_timeout_secs: 15,
|
||||||
|
query_timeout_secs: 30,
|
||||||
|
idle_timeout_secs: 60,
|
||||||
|
keepalive_interval_secs: 0,
|
||||||
|
ssl: false,
|
||||||
|
ca_cert_path: String::new(),
|
||||||
|
client_cert_path: String::new(),
|
||||||
|
client_key_path: String::new(),
|
||||||
|
sysdba: false,
|
||||||
|
oracle_connection_type: None,
|
||||||
|
connection_string: None,
|
||||||
|
redis_connection_mode: None,
|
||||||
|
redis_sentinel_master: String::new(),
|
||||||
|
redis_sentinel_nodes: String::new(),
|
||||||
|
redis_sentinel_username: String::new(),
|
||||||
|
redis_sentinel_password: String::new(),
|
||||||
|
redis_sentinel_tls: false,
|
||||||
|
redis_cluster_nodes: String::new(),
|
||||||
|
redis_key_separator: dbx_core::models::connection::default_redis_key_separator(),
|
||||||
|
redis_scan_page_size: None,
|
||||||
|
redis_database_aliases: Default::default(),
|
||||||
|
redis_key_templates: Vec::new(),
|
||||||
|
redis_key_grouping: None,
|
||||||
|
etcd_endpoints: String::new(),
|
||||||
|
gbase_server: String::new(),
|
||||||
|
informix_server: String::new(),
|
||||||
|
external_config: None,
|
||||||
|
plugin_id: None,
|
||||||
|
plugin_connection_provider: None,
|
||||||
|
plugin_connection_type: None,
|
||||||
|
connection_secrets: Default::default(),
|
||||||
|
jdbc_driver_class: None,
|
||||||
|
jdbc_driver_paths: Vec::new(),
|
||||||
|
one_time: false,
|
||||||
|
save_password: true,
|
||||||
|
read_only: false,
|
||||||
|
is_production: false,
|
||||||
|
production_databases: vec![],
|
||||||
|
show_system_schemas: false,
|
||||||
|
sidebar_auto_load_all_tables: false,
|
||||||
|
database_info: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sqlserver_config(id: &str, database: &str) -> ConnectionConfig {
|
||||||
|
let mut config = sqlite_config(id, "");
|
||||||
|
config.db_type = DatabaseType::SqlServer;
|
||||||
|
config.host = std::env::var("DBX_LIVE_SQLSERVER_HOST").unwrap_or_else(|_| "127.0.0.1".to_string());
|
||||||
|
config.port = std::env::var("DBX_LIVE_SQLSERVER_PORT").ok().and_then(|value| value.parse().ok()).unwrap_or(1433);
|
||||||
|
config.username = std::env::var("DBX_LIVE_SQLSERVER_USER").unwrap_or_else(|_| "sa".to_string());
|
||||||
|
config.password = std::env::var("DBX_LIVE_SQLSERVER_PASSWORD").expect("DBX_LIVE_SQLSERVER_PASSWORD");
|
||||||
|
config.database = Some(database.to_string());
|
||||||
|
config
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn sqlserver_connect(database: &str) -> dbx_core::db::sqlserver::SqlServerClient {
|
||||||
|
let host = std::env::var("DBX_LIVE_SQLSERVER_HOST").unwrap_or_else(|_| "127.0.0.1".to_string());
|
||||||
|
let port = std::env::var("DBX_LIVE_SQLSERVER_PORT").ok().and_then(|value| value.parse().ok()).unwrap_or(1433);
|
||||||
|
let user = std::env::var("DBX_LIVE_SQLSERVER_USER").unwrap_or_else(|_| "sa".to_string());
|
||||||
|
let password = std::env::var("DBX_LIVE_SQLSERVER_PASSWORD").expect("DBX_LIVE_SQLSERVER_PASSWORD");
|
||||||
|
dbx_core::db::sqlserver::connect(&host, port, &user, &password, Some(database), None, Duration::from_secs(20))
|
||||||
|
.await
|
||||||
|
.expect("connect SQL Server")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[ignore = "requires DBX_LIVE_SQLSERVER_HOST/PORT/USER/PASSWORD pointing at SQL Server"]
|
||||||
|
async fn live_sqlite_to_sqlserver_keeps_sixty_four_bit_integer_values() {
|
||||||
|
let suffix = uuid::Uuid::new_v4().simple().to_string();
|
||||||
|
let target_db = format!("dbx_sqlite_int_dst_{}", &suffix[..12]);
|
||||||
|
let sqlite_connection_id = format!("live-sqlite-source-{suffix}");
|
||||||
|
let sqlserver_connection_id = format!("live-sqlserver-target-{suffix}");
|
||||||
|
let table = "daq_standard_setting";
|
||||||
|
|
||||||
|
let mut master = sqlserver_connect("master").await;
|
||||||
|
dbx_core::db::sqlserver::execute_batch(&mut master, &format!("CREATE DATABASE [{target_db}];"))
|
||||||
|
.await
|
||||||
|
.expect("create target database");
|
||||||
|
|
||||||
|
let dir = std::env::temp_dir().join(format!("dbx-live-sqlite-int-{suffix}"));
|
||||||
|
std::fs::create_dir_all(&dir).expect("create temp directory");
|
||||||
|
let sqlite_path = dir.join("source.sqlite");
|
||||||
|
let sqlite_path = sqlite_path.to_string_lossy().into_owned();
|
||||||
|
|
||||||
|
let sqlite_pool =
|
||||||
|
dbx_core::db::sqlite::connect_path_create_if_missing(&sqlite_path).await.expect("open sqlite source");
|
||||||
|
dbx_core::db::sqlite::execute_query(
|
||||||
|
&sqlite_pool,
|
||||||
|
&format!(
|
||||||
|
"CREATE TABLE {table} (id INTEGER NOT NULL PRIMARY KEY, amount INTEGER NOT NULL, label TEXT); \
|
||||||
|
INSERT INTO {table} (id, amount, label) VALUES (1, 2098620070434758658, 'snowflake');"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("seed sqlite source table");
|
||||||
|
|
||||||
|
let storage = dbx_core::persistence::test_storage::open(&dir.join("storage.db")).await.expect("open storage");
|
||||||
|
let state = Arc::new(AppState::new(storage));
|
||||||
|
state
|
||||||
|
.configs
|
||||||
|
.write()
|
||||||
|
.await
|
||||||
|
.insert(sqlite_connection_id.clone(), sqlite_config(&sqlite_connection_id, &sqlite_path));
|
||||||
|
state
|
||||||
|
.configs
|
||||||
|
.write()
|
||||||
|
.await
|
||||||
|
.insert(sqlserver_connection_id.clone(), sqlserver_config(&sqlserver_connection_id, &target_db));
|
||||||
|
let source_pool_key =
|
||||||
|
state.get_or_create_pool(&sqlite_connection_id, Some("main")).await.expect("sqlite source pool");
|
||||||
|
let target_pool_key =
|
||||||
|
state.get_or_create_pool(&sqlserver_connection_id, Some(&target_db)).await.expect("target pool");
|
||||||
|
|
||||||
|
let request = TransferRequest {
|
||||||
|
transfer_id: format!("live-sqlite-int-{suffix}"),
|
||||||
|
source_connection_id: sqlite_connection_id.clone(),
|
||||||
|
source_database: "main".to_string(),
|
||||||
|
source_schema: "main".to_string(),
|
||||||
|
source_catalog: None,
|
||||||
|
target_connection_id: sqlserver_connection_id.clone(),
|
||||||
|
target_database: target_db.clone(),
|
||||||
|
target_schema: "dbo".to_string(),
|
||||||
|
target_catalog: None,
|
||||||
|
tables: vec![table.to_string()],
|
||||||
|
create_table: true,
|
||||||
|
drop_target_before_create: false,
|
||||||
|
drop_target_confirmed: false,
|
||||||
|
content: TransferContent::default(),
|
||||||
|
objects: Vec::new(),
|
||||||
|
mode: TransferMode::Append,
|
||||||
|
target_table_name_case: TransferTableNameCase::Preserve,
|
||||||
|
quote_target_column_names: true,
|
||||||
|
ownership_policy: TransferOwnershipPolicy::Preserve,
|
||||||
|
batch_size: 100,
|
||||||
|
};
|
||||||
|
|
||||||
|
let test_result = async {
|
||||||
|
let transferred = transfer_table(
|
||||||
|
&state,
|
||||||
|
&request,
|
||||||
|
table,
|
||||||
|
0,
|
||||||
|
&DatabaseType::Sqlite,
|
||||||
|
&DatabaseType::SqlServer,
|
||||||
|
&source_pool_key,
|
||||||
|
&target_pool_key,
|
||||||
|
&HashMap::new(),
|
||||||
|
&mut Vec::new(),
|
||||||
|
None,
|
||||||
|
|_| {},
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
assert_eq!(transferred, 1);
|
||||||
|
|
||||||
|
let mut target_client = sqlserver_connect(&target_db).await;
|
||||||
|
let rows = dbx_core::db::sqlserver::execute_query(
|
||||||
|
&mut target_client,
|
||||||
|
&format!("SELECT id, amount, label FROM dbo.[{table}]"),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|error| format!("read transferred rows: {error}"))?;
|
||||||
|
assert_eq!(rows.rows.len(), 1);
|
||||||
|
assert_eq!(rows.rows[0][1].as_str(), Some("2098620070434758658"));
|
||||||
|
Ok::<_, String>(())
|
||||||
|
}
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let cleanup = dbx_core::db::sqlserver::execute_batch(
|
||||||
|
&mut master,
|
||||||
|
&format!("ALTER DATABASE [{target_db}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE [{target_db}];"),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let _ = std::fs::remove_dir_all(dir);
|
||||||
|
cleanup.expect("drop target database");
|
||||||
|
test_result.expect("SQLite INTEGER values must survive a transfer into SQL Server");
|
||||||
|
}
|
||||||
@@ -748,3 +748,252 @@ async fn live_sqlserver_keyset_uniqueidentifier_datetime2_composite_key() {
|
|||||||
cleanup.expect("drop typed databases");
|
cleanup.expect("drop typed databases");
|
||||||
test_result.unwrap();
|
test_result.unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// User report (SQL Server 数据同步 / issue #9734 family): transferring a SQL Server
|
||||||
|
/// table into a **new** target database reuses the source `CREATE TABLE` DDL, so the
|
||||||
|
/// created target keeps its `IDENTITY` column. The batched INSERT then writes explicit
|
||||||
|
/// identity values and SQL Server rejects the batch with 544 unless the write is wrapped
|
||||||
|
/// in `SET IDENTITY_INSERT`.
|
||||||
|
#[tokio::test]
|
||||||
|
#[ignore = "requires DBX_LIVE_SQLSERVER_HOST/PORT/USER/PASSWORD pointing at SQL Server"]
|
||||||
|
async fn live_sqlserver_transfer_new_identity_target_keeps_explicit_identity_values() {
|
||||||
|
let suffix = uuid::Uuid::new_v4().simple().to_string();
|
||||||
|
let source_db = format!("dbx_identity_src_{}", &suffix[..12]);
|
||||||
|
let target_db = format!("dbx_identity_dst_{}", &suffix[..12]);
|
||||||
|
let connection_id = format!("live-sqlserver-identity-{suffix}");
|
||||||
|
let table = "daq_electest";
|
||||||
|
|
||||||
|
let mut master = sqlserver_connect("master").await;
|
||||||
|
dbx_core::db::sqlserver::execute_batch(
|
||||||
|
&mut master,
|
||||||
|
&format!("CREATE DATABASE [{source_db}]; CREATE DATABASE [{target_db}];"),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("create identity databases");
|
||||||
|
|
||||||
|
let mut source_client = sqlserver_connect(&source_db).await;
|
||||||
|
dbx_core::db::sqlserver::execute_batch(
|
||||||
|
&mut source_client,
|
||||||
|
&format!(
|
||||||
|
"CREATE TABLE dbo.[{table}] ( \
|
||||||
|
id INT IDENTITY(1,1) NOT NULL CONSTRAINT [PK_daq_electest] PRIMARY KEY, \
|
||||||
|
name NVARCHAR(64) NOT NULL); \
|
||||||
|
INSERT INTO dbo.[{table}] (name) VALUES (N'first'), (N'second');"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("create identity source table");
|
||||||
|
|
||||||
|
let dir = std::env::temp_dir().join(format!("dbx-live-sqlserver-identity-{suffix}"));
|
||||||
|
std::fs::create_dir_all(&dir).expect("create identity directory");
|
||||||
|
let storage =
|
||||||
|
dbx_core::persistence::test_storage::open(&dir.join("storage.db")).await.expect("open identity storage");
|
||||||
|
let state = Arc::new(AppState::new(storage));
|
||||||
|
state.configs.write().await.insert(connection_id.clone(), live_sqlserver_config(&connection_id, &source_db));
|
||||||
|
let source_pool_key = state.get_or_create_pool(&connection_id, Some(&source_db)).await.expect("source pool");
|
||||||
|
let target_pool_key = state.get_or_create_pool(&connection_id, Some(&target_db)).await.expect("target pool");
|
||||||
|
|
||||||
|
let request = TransferRequest {
|
||||||
|
transfer_id: format!("live-sqlserver-identity-{suffix}"),
|
||||||
|
source_connection_id: connection_id.clone(),
|
||||||
|
source_database: source_db.clone(),
|
||||||
|
source_schema: "dbo".to_string(),
|
||||||
|
source_catalog: None,
|
||||||
|
target_connection_id: connection_id.clone(),
|
||||||
|
target_database: target_db.clone(),
|
||||||
|
target_schema: "dbo".to_string(),
|
||||||
|
target_catalog: None,
|
||||||
|
tables: vec![table.to_string()],
|
||||||
|
create_table: true,
|
||||||
|
drop_target_before_create: false,
|
||||||
|
drop_target_confirmed: false,
|
||||||
|
content: TransferContent::default(),
|
||||||
|
objects: Vec::new(),
|
||||||
|
mode: TransferMode::Append,
|
||||||
|
target_table_name_case: TransferTableNameCase::Preserve,
|
||||||
|
quote_target_column_names: true,
|
||||||
|
ownership_policy: TransferOwnershipPolicy::Preserve,
|
||||||
|
batch_size: 1,
|
||||||
|
};
|
||||||
|
|
||||||
|
let test_result = async {
|
||||||
|
let transferred = transfer_table(
|
||||||
|
&state,
|
||||||
|
&request,
|
||||||
|
table,
|
||||||
|
0,
|
||||||
|
&DatabaseType::SqlServer,
|
||||||
|
&DatabaseType::SqlServer,
|
||||||
|
&source_pool_key,
|
||||||
|
&target_pool_key,
|
||||||
|
&HashMap::new(),
|
||||||
|
&mut Vec::new(),
|
||||||
|
None,
|
||||||
|
|_| {},
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
assert_eq!(transferred, 2);
|
||||||
|
|
||||||
|
let mut target_client = sqlserver_connect(&target_db).await;
|
||||||
|
let rows = dbx_core::db::sqlserver::execute_query(
|
||||||
|
&mut target_client,
|
||||||
|
&format!("SELECT id, name FROM dbo.[{table}] ORDER BY id"),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|error| format!("read transferred rows: {error}"))?;
|
||||||
|
assert_eq!(rows.rows.len(), 2);
|
||||||
|
assert_eq!(rows.rows[0][0].as_i64(), Some(1));
|
||||||
|
assert_eq!(rows.rows[1][0].as_i64(), Some(2));
|
||||||
|
assert_eq!(rows.rows[0][1].as_str(), Some("first"));
|
||||||
|
assert_eq!(rows.rows[1][1].as_str(), Some("second"));
|
||||||
|
Ok::<_, String>(())
|
||||||
|
}
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let cleanup = dbx_core::db::sqlserver::execute_batch(
|
||||||
|
&mut master,
|
||||||
|
&format!(
|
||||||
|
"ALTER DATABASE [{source_db}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE [{source_db}]; \
|
||||||
|
ALTER DATABASE [{target_db}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE [{target_db}];"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let _ = std::fs::remove_dir_all(dir);
|
||||||
|
cleanup.expect("drop identity databases");
|
||||||
|
test_result.expect("transfer into a freshly created identity target must keep the source identity values");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// User report (issue #9729): DBX's **database export** emitted one `CONSTRAINT` line per
|
||||||
|
/// column of a composite foreign key, all sharing the source constraint name. SQL Server
|
||||||
|
/// requires constraint names to be unique in the schema, so replaying the exported script
|
||||||
|
/// failed with 8168 (`名称不允许重复`) — the user hit it while importing `test1.sql`.
|
||||||
|
///
|
||||||
|
/// This drives the real export path against a real SQL Server and replays the produced
|
||||||
|
/// script into a fresh database, which is exactly what the user did by hand.
|
||||||
|
#[tokio::test]
|
||||||
|
#[ignore = "requires DBX_LIVE_SQLSERVER_HOST/PORT/USER/PASSWORD pointing at SQL Server"]
|
||||||
|
async fn live_sqlserver_database_export_replays_composite_foreign_keys() {
|
||||||
|
use dbx_core::data::database_export::{export_database_sql_core, DatabaseExportRequest};
|
||||||
|
|
||||||
|
let suffix = uuid::Uuid::new_v4().simple().to_string();
|
||||||
|
let source_db = format!("dbx_export_src_{}", &suffix[..12]);
|
||||||
|
let target_db = format!("dbx_export_dst_{}", &suffix[..12]);
|
||||||
|
let connection_id = format!("live-sqlserver-export-{suffix}");
|
||||||
|
|
||||||
|
let mut master = sqlserver_connect("master").await;
|
||||||
|
dbx_core::db::sqlserver::execute_batch(
|
||||||
|
&mut master,
|
||||||
|
&format!("CREATE DATABASE [{source_db}]; CREATE DATABASE [{target_db}];"),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("create export databases");
|
||||||
|
|
||||||
|
let mut source_client = sqlserver_connect(&source_db).await;
|
||||||
|
dbx_core::db::sqlserver::execute_batch(
|
||||||
|
&mut source_client,
|
||||||
|
"CREATE TABLE dbo.QRTZ_JOB_DETAILS ( \
|
||||||
|
sched_name NVARCHAR(100) NOT NULL, \
|
||||||
|
job_name NVARCHAR(170) NOT NULL, \
|
||||||
|
job_group NVARCHAR(170) NOT NULL, \
|
||||||
|
description NVARCHAR(250) NULL, \
|
||||||
|
CONSTRAINT PK_QRTZ_JOB_DETAILS PRIMARY KEY (sched_name, job_name, job_group)); \
|
||||||
|
CREATE TABLE dbo.QRTZ_TRIGGERS ( \
|
||||||
|
sched_name NVARCHAR(100) NOT NULL, \
|
||||||
|
trigger_name NVARCHAR(170) NOT NULL, \
|
||||||
|
trigger_group NVARCHAR(170) NOT NULL, \
|
||||||
|
job_name NVARCHAR(170) NOT NULL, \
|
||||||
|
job_group NVARCHAR(170) NOT NULL, \
|
||||||
|
description NVARCHAR(250) NULL, \
|
||||||
|
next_fire_time BIGINT NULL, \
|
||||||
|
PRIMARY KEY (sched_name, trigger_name, trigger_group), \
|
||||||
|
CONSTRAINT [FK_QRTRZ_TRIGGERS_JOB_DETAILS] FOREIGN KEY (sched_name, job_name, job_group) \
|
||||||
|
REFERENCES dbo.QRTZ_JOB_DETAILS (sched_name, job_name, job_group)); \
|
||||||
|
INSERT INTO dbo.QRTZ_JOB_DETAILS (sched_name, job_name, job_group, description) \
|
||||||
|
VALUES (N'sched', N'job', N'group', N'quartz job'); \
|
||||||
|
INSERT INTO dbo.QRTZ_TRIGGERS (sched_name, trigger_name, trigger_group, job_name, job_group, next_fire_time) \
|
||||||
|
VALUES (N'sched', N'trigger', N'group', N'job', N'group', 1700000000000);",
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("create QRTZ fixtures");
|
||||||
|
|
||||||
|
let dir = std::env::temp_dir().join(format!("dbx-live-sqlserver-export-{suffix}"));
|
||||||
|
std::fs::create_dir_all(&dir).expect("create export directory");
|
||||||
|
let storage =
|
||||||
|
dbx_core::persistence::test_storage::open(&dir.join("storage.db")).await.expect("open export storage");
|
||||||
|
let state = Arc::new(AppState::new(storage));
|
||||||
|
state.configs.write().await.insert(connection_id.clone(), live_sqlserver_config(&connection_id, &source_db));
|
||||||
|
let _ = state.get_or_create_pool(&connection_id, Some(&source_db)).await.expect("export source pool");
|
||||||
|
|
||||||
|
let export_path = dir.join("test1.sql");
|
||||||
|
let request = DatabaseExportRequest {
|
||||||
|
export_id: format!("live-sqlserver-export-{suffix}"),
|
||||||
|
connection_id: connection_id.clone(),
|
||||||
|
database: source_db.clone(),
|
||||||
|
schema: "dbo".to_string(),
|
||||||
|
file_path: export_path.to_string_lossy().into_owned(),
|
||||||
|
selected_tables: vec!["QRTZ_JOB_DETAILS".to_string(), "QRTZ_TRIGGERS".to_string()],
|
||||||
|
excluded_tables: Vec::new(),
|
||||||
|
include_structure: true,
|
||||||
|
include_data: true,
|
||||||
|
include_objects: false,
|
||||||
|
include_create_database: false,
|
||||||
|
drop_table_if_exists: false,
|
||||||
|
omit_auto_increment: false,
|
||||||
|
fail_on_error: false,
|
||||||
|
prevent_overwrite: false,
|
||||||
|
output_compression: Default::default(),
|
||||||
|
snapshot_session_id: None,
|
||||||
|
insert_dialect: Default::default(),
|
||||||
|
batch_size: 1000,
|
||||||
|
split_max_mb: None,
|
||||||
|
};
|
||||||
|
|
||||||
|
let test_result = async {
|
||||||
|
export_database_sql_core(&state, &request, |_| {}).await?;
|
||||||
|
let script = std::fs::read_to_string(&export_path).map_err(|error| format!("read export script: {error}"))?;
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
script.matches("CONSTRAINT [FK_QRTRZ_TRIGGERS_JOB_DETAILS]").count(),
|
||||||
|
1,
|
||||||
|
"a composite foreign key must be exported as one constraint, not one per column:\n{script}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
script.contains(
|
||||||
|
"CONSTRAINT [FK_QRTRZ_TRIGGERS_JOB_DETAILS] FOREIGN KEY ([sched_name], [job_name], [job_group]) \
|
||||||
|
REFERENCES [dbo].[QRTZ_JOB_DETAILS]([sched_name], [job_name], [job_group])"
|
||||||
|
),
|
||||||
|
"composite foreign key columns must stay grouped:\n{script}"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Replay: this is the step that used to fail with 8168.
|
||||||
|
let mut target_client = sqlserver_connect(&target_db).await;
|
||||||
|
for statement in script.split(";\n").map(str::trim).filter(|statement| !statement.is_empty()) {
|
||||||
|
dbx_core::db::sqlserver::execute_batch(&mut target_client, statement)
|
||||||
|
.await
|
||||||
|
.map_err(|error| format!("replay failed on `{statement}`: {error}"))?;
|
||||||
|
}
|
||||||
|
|
||||||
|
let rows = dbx_core::db::sqlserver::execute_query(
|
||||||
|
&mut target_client,
|
||||||
|
"SELECT COUNT(*) FROM dbo.QRTZ_TRIGGERS t JOIN dbo.QRTZ_JOB_DETAILS j \
|
||||||
|
ON t.sched_name = j.sched_name AND t.job_name = j.job_name AND t.job_group = j.job_group",
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(|error| format!("read replayed rows: {error}"))?;
|
||||||
|
assert_eq!(rows.rows[0][0].as_i64(), Some(1));
|
||||||
|
Ok::<_, String>(())
|
||||||
|
}
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let cleanup = dbx_core::db::sqlserver::execute_batch(
|
||||||
|
&mut master,
|
||||||
|
&format!(
|
||||||
|
"ALTER DATABASE [{source_db}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE [{source_db}]; \
|
||||||
|
ALTER DATABASE [{target_db}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE [{target_db}];"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let _ = std::fs::remove_dir_all(dir);
|
||||||
|
cleanup.expect("drop export databases");
|
||||||
|
test_result.expect("database export must replay a composite foreign key without 8168");
|
||||||
|
}
|
||||||
Reference in new issue
Block a user