perf(import): bound bulk import memory and cancellation
This commit is contained in:
1 parent
843f945581
commit
d0c3a3fd2e
7 files changed
+3761
-193
No files matched your search
@@ -4,10 +4,16 @@ use dbx_core::query_result_export::{export_query_result_core, ExportStatus, Quer
|
||||
use dbx_core::sql::{SqlFileRequest, SqlFileStatus};
|
||||
use dbx_core::sql_file_import::execute_sql_file_content;
|
||||
use dbx_core::storage::Storage;
|
||||
use dbx_core::table_import::{
|
||||
build_import_insert_batches, import_table_file_core, parse_delimited_file_with_options, TableImportColumnMapping,
|
||||
TableImportMode, TableImportParseOptions, TableImportRequest, TableImportSourceFormat, TableImportStatus,
|
||||
};
|
||||
use dbx_core::table_structure_sql::{
|
||||
build_table_structure_change_sql, ColumnInfo, EditableStructureColumn, TableStructureSqlOptions,
|
||||
};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use dbx_core::xlsx_export::{build_xlsx_workbook, XlsxWorksheetData};
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
@@ -68,6 +74,740 @@ fn live_sqlserver_config(id: &str, database: &str) -> dbx_core::models::connecti
|
||||
}
|
||||
}
|
||||
|
||||
async fn live_sqlserver_import_state(
|
||||
connection_id: &str,
|
||||
database: &str,
|
||||
suffix: &str,
|
||||
) -> (AppState, String, std::path::PathBuf) {
|
||||
let dir = std::env::temp_dir().join(format!("dbx-live-sqlserver-import-{suffix}"));
|
||||
std::fs::create_dir_all(&dir).expect("create live import directory");
|
||||
let storage = Storage::open(&dir.join("storage.db")).await.expect("open live import storage");
|
||||
let state = AppState::new(storage);
|
||||
let config = live_sqlserver_config(connection_id, database);
|
||||
state.configs.write().await.insert(connection_id.to_string(), config);
|
||||
let pool_key =
|
||||
state.get_or_create_pool(connection_id, Some(database)).await.expect("connect live SQL Server import pool");
|
||||
(state, pool_key, dir)
|
||||
}
|
||||
|
||||
fn live_sqlserver_import_mapping(source: &str, target: &str) -> TableImportColumnMapping {
|
||||
TableImportColumnMapping {
|
||||
source_column: source.to_string(),
|
||||
target_column: target.to_string(),
|
||||
target_data_type: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn live_sqlserver_import_request(
|
||||
connection_id: &str,
|
||||
database: &str,
|
||||
table: &str,
|
||||
file_path: &std::path::Path,
|
||||
mappings: Vec<TableImportColumnMapping>,
|
||||
mode: TableImportMode,
|
||||
) -> TableImportRequest {
|
||||
TableImportRequest {
|
||||
import_id: format!("live-sqlserver-import-{}", uuid::Uuid::new_v4().simple()),
|
||||
connection_id: connection_id.to_string(),
|
||||
database: database.to_string(),
|
||||
schema: "dbo".to_string(),
|
||||
table: table.to_string(),
|
||||
file_path: file_path.to_string_lossy().to_string(),
|
||||
source_ref: None,
|
||||
source_format: Some(TableImportSourceFormat::Csv),
|
||||
parse_options: TableImportParseOptions::default(),
|
||||
mappings,
|
||||
mode,
|
||||
create_table: false,
|
||||
batch_size: 500,
|
||||
date_time_format: None,
|
||||
prepared_source: None,
|
||||
retain_source: false,
|
||||
}
|
||||
}
|
||||
|
||||
async fn run_live_sqlserver_import(
|
||||
state: &AppState,
|
||||
pool_key: &str,
|
||||
request: &TableImportRequest,
|
||||
) -> Result<dbx_core::table_import::TableImportSummary, String> {
|
||||
import_table_file_core(state, request, &DatabaseType::SqlServer, pool_key, |_| Box::pin(async { false }), |_| {})
|
||||
.await
|
||||
}
|
||||
|
||||
fn live_sqlserver_matrix_column_types() -> Vec<(String, String)> {
|
||||
[
|
||||
("id", "int"),
|
||||
("code", "nvarchar(40)"),
|
||||
("nullable_text", "nvarchar(100)"),
|
||||
("amount", "decimal(38,10)"),
|
||||
("occurred_at", "datetime2(7)"),
|
||||
("offset_at", "datetimeoffset(7)"),
|
||||
("event_id", "uniqueidentifier"),
|
||||
("document", "xml"),
|
||||
("payload", "varbinary(max)"),
|
||||
]
|
||||
.into_iter()
|
||||
.map(|(name, data_type)| (name.to_string(), data_type.to_string()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn live_sqlserver_matrix_mappings(include_identity: bool) -> Vec<TableImportColumnMapping> {
|
||||
let columns =
|
||||
["id", "code", "nullable_text", "amount", "occurred_at", "offset_at", "event_id", "document", "payload"];
|
||||
columns
|
||||
.into_iter()
|
||||
.filter(|column| include_identity || *column != "id")
|
||||
.map(|column| live_sqlserver_import_mapping(column, column))
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn run_live_sqlserver_generated_insert(
|
||||
client: &mut dbx_core::db::sqlserver::SqlServerClient,
|
||||
table: &str,
|
||||
file_path: &std::path::Path,
|
||||
parse_options: &TableImportParseOptions,
|
||||
include_identity: bool,
|
||||
) -> Result<usize, String> {
|
||||
let parsed = parse_delimited_file_with_options(
|
||||
&file_path.to_string_lossy(),
|
||||
TableImportSourceFormat::Csv,
|
||||
parse_options,
|
||||
usize::MAX,
|
||||
)?;
|
||||
let batches = build_import_insert_batches(
|
||||
&parsed,
|
||||
&live_sqlserver_matrix_mappings(include_identity),
|
||||
&live_sqlserver_matrix_column_types(),
|
||||
table,
|
||||
"dbo",
|
||||
&DatabaseType::SqlServer,
|
||||
500,
|
||||
)?;
|
||||
if include_identity {
|
||||
let rows_imported = batches.iter().map(|batch| batch.row_count).sum();
|
||||
let statements = batches.into_iter().map(|batch| batch.sql).collect::<Vec<_>>().join(";\n");
|
||||
let sql =
|
||||
format!("SET IDENTITY_INSERT [dbo].[{table}] ON;\n{statements};\nSET IDENTITY_INSERT [dbo].[{table}] OFF");
|
||||
dbx_core::db::sqlserver::execute_batch(client, &sql).await?;
|
||||
return Ok(rows_imported);
|
||||
}
|
||||
let mut rows_imported = 0;
|
||||
for batch in batches {
|
||||
dbx_core::db::sqlserver::execute_batch(client, &batch.sql).await?;
|
||||
rows_imported += batch.row_count;
|
||||
}
|
||||
Ok(rows_imported)
|
||||
}
|
||||
|
||||
fn live_sqlserver_matrix_table_ddl(table: &str, audit_table: &str) -> String {
|
||||
format!(
|
||||
"CREATE TABLE [dbo].[{table}] (\
|
||||
[id] INT IDENTITY(1,1) NOT NULL PRIMARY KEY, \
|
||||
[code] NVARCHAR(40) NOT NULL UNIQUE, \
|
||||
[nullable_text] NVARCHAR(100) NULL, \
|
||||
[amount] DECIMAL(38,10) NOT NULL CHECK ([amount] > 0), \
|
||||
[occurred_at] DATETIME2(7) NOT NULL, \
|
||||
[offset_at] DATETIMEOFFSET(7) NOT NULL, \
|
||||
[event_id] UNIQUEIDENTIFIER NOT NULL, \
|
||||
[document] XML NULL, \
|
||||
[payload] VARBINARY(MAX) NULL, \
|
||||
[default_text] NVARCHAR(40) NOT NULL DEFAULT N'defaulted'); \
|
||||
CREATE TABLE [dbo].[{audit_table}] (\
|
||||
[target_id] INT NOT NULL, [code] NVARCHAR(40) NOT NULL, [default_text] NVARCHAR(40) NOT NULL);"
|
||||
)
|
||||
}
|
||||
|
||||
fn live_sqlserver_matrix_trigger_ddl(table: &str, audit_table: &str, trigger: &str) -> String {
|
||||
format!(
|
||||
"CREATE TRIGGER [dbo].[{trigger}] ON [dbo].[{table}] AFTER INSERT AS \
|
||||
INSERT INTO [dbo].[{audit_table}] ([target_id], [code], [default_text]) \
|
||||
SELECT [id], [code], [default_text] FROM inserted;"
|
||||
)
|
||||
}
|
||||
|
||||
async fn live_sqlserver_matrix_rows(
|
||||
client: &mut dbx_core::db::sqlserver::SqlServerClient,
|
||||
table: &str,
|
||||
) -> dbx_core::db::QueryResult {
|
||||
dbx_core::db::sqlserver::execute_query(
|
||||
client,
|
||||
&format!(
|
||||
"SELECT CONVERT(VARCHAR(12), [id]), [code], \
|
||||
CASE WHEN [nullable_text] IS NULL THEN N'<NULL>' ELSE N'<' + [nullable_text] + N'>' END, \
|
||||
CONVERT(VARCHAR(50), [amount]), CONVERT(VARCHAR(33), [occurred_at], 126), \
|
||||
CONVERT(VARCHAR(48), [offset_at], 127), CONVERT(VARCHAR(36), [event_id]), \
|
||||
CASE WHEN [document] IS NULL THEN N'<NULL>' ELSE CONVERT(NVARCHAR(MAX), [document]) END, \
|
||||
CASE WHEN [payload] IS NULL THEN N'<NULL>' ELSE sys.fn_varbintohexstr([payload]) END, \
|
||||
[default_text] FROM [dbo].[{table}] ORDER BY [id]"
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("query SQL Server import matrix rows")
|
||||
}
|
||||
|
||||
async fn live_sqlserver_table_count(client: &mut dbx_core::db::sqlserver::SqlServerClient, table: &str) -> i64 {
|
||||
let result = dbx_core::db::sqlserver::execute_query(client, &format!("SELECT COUNT_BIG(*) FROM [dbo].[{table}]"))
|
||||
.await
|
||||
.expect("count SQL Server matrix rows");
|
||||
result.rows[0][0].as_i64().expect("SQL Server COUNT_BIG result")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DBX_LIVE_SQLSERVER_HOST/PORT/USER/PASSWORD pointing at a writable SQL Server database"]
|
||||
async fn live_sqlserver_bulk_matches_generated_insert_type_and_constraint_matrix() {
|
||||
let database = std::env::var("DBX_LIVE_SQLSERVER_DATABASE").unwrap_or_else(|_| "tempdb".to_string());
|
||||
let suffix = uuid::Uuid::new_v4().simple().to_string();
|
||||
let connection_id = format!("live-sqlserver-matrix-{suffix}");
|
||||
let bulk_table = format!("dbx_bulk_matrix_{suffix}");
|
||||
let generated_table = format!("dbx_insert_matrix_{suffix}");
|
||||
let bulk_audit = format!("dbx_bulk_matrix_audit_{suffix}");
|
||||
let generated_audit = format!("dbx_insert_matrix_audit_{suffix}");
|
||||
let bulk_trigger = format!("dbx_bulk_matrix_trigger_{suffix}");
|
||||
let generated_trigger = format!("dbx_insert_matrix_trigger_{suffix}");
|
||||
let mut client = dbx_core::db::sqlserver::connect(
|
||||
&std::env::var("DBX_LIVE_SQLSERVER_HOST").unwrap_or_else(|_| "127.0.0.1".to_string()),
|
||||
std::env::var("DBX_LIVE_SQLSERVER_PORT").ok().and_then(|value| value.parse().ok()).unwrap_or(1433),
|
||||
&std::env::var("DBX_LIVE_SQLSERVER_USER").unwrap_or_else(|_| "sa".to_string()),
|
||||
&std::env::var("DBX_LIVE_SQLSERVER_PASSWORD").expect("DBX_LIVE_SQLSERVER_PASSWORD"),
|
||||
Some(&database),
|
||||
None,
|
||||
Duration::from_secs(10),
|
||||
)
|
||||
.await
|
||||
.expect("connect SQL Server");
|
||||
for ddl in [
|
||||
live_sqlserver_matrix_table_ddl(&bulk_table, &bulk_audit),
|
||||
live_sqlserver_matrix_table_ddl(&generated_table, &generated_audit),
|
||||
live_sqlserver_matrix_trigger_ddl(&bulk_table, &bulk_audit, &bulk_trigger),
|
||||
live_sqlserver_matrix_trigger_ddl(&generated_table, &generated_audit, &generated_trigger),
|
||||
] {
|
||||
dbx_core::db::sqlserver::execute_batch(&mut client, &ddl)
|
||||
.await
|
||||
.expect("create SQL Server import matrix objects");
|
||||
}
|
||||
|
||||
let (state, pool_key, dir) = live_sqlserver_import_state(&connection_id, &database, &suffix).await;
|
||||
let default_options = TableImportParseOptions::default();
|
||||
let empty_string_options =
|
||||
TableImportParseOptions { empty_string_as_null: Some(false), ..TableImportParseOptions::default() };
|
||||
|
||||
let null_csv = dir.join("matrix-null.csv");
|
||||
std::fs::write(
|
||||
&null_csv,
|
||||
"code,nullable_text,amount,occurred_at,offset_at,event_id,document,payload\n\
|
||||
auto-null,,12345678901234567890.1234567890,2026-07-27T12:34:56.1234567,2026-07-27T12:34:56.1234567+08:00,11111111-2222-3333-4444-555555555555,,\n",
|
||||
)
|
||||
.expect("write SQL Server NULL matrix CSV");
|
||||
let null_request = live_sqlserver_import_request(
|
||||
&connection_id,
|
||||
&database,
|
||||
&bulk_table,
|
||||
&null_csv,
|
||||
live_sqlserver_matrix_mappings(false),
|
||||
TableImportMode::Append,
|
||||
);
|
||||
let null_bulk = run_live_sqlserver_import(&state, &pool_key, &null_request)
|
||||
.await
|
||||
.expect("bulk import SQL Server NULL matrix row");
|
||||
let null_generated =
|
||||
run_live_sqlserver_generated_insert(&mut client, &generated_table, &null_csv, &default_options, false)
|
||||
.await
|
||||
.expect("generated INSERT SQL Server NULL matrix row");
|
||||
|
||||
let empty_csv = dir.join("matrix-empty-string.csv");
|
||||
std::fs::write(
|
||||
&empty_csv,
|
||||
"code,nullable_text,amount,occurred_at,offset_at,event_id,document,payload\n\
|
||||
empty,,0.0000000001,2026-07-28T01:02:03.0000001,2026-07-28T01:02:03.0000001-05:30,aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee,<root><value>empty-text</value></root>,0x00FF10\n",
|
||||
)
|
||||
.expect("write SQL Server empty-string matrix CSV");
|
||||
let mut empty_request = live_sqlserver_import_request(
|
||||
&connection_id,
|
||||
&database,
|
||||
&bulk_table,
|
||||
&empty_csv,
|
||||
live_sqlserver_matrix_mappings(false),
|
||||
TableImportMode::Append,
|
||||
);
|
||||
empty_request.parse_options = empty_string_options.clone();
|
||||
let empty_bulk = run_live_sqlserver_import(&state, &pool_key, &empty_request)
|
||||
.await
|
||||
.expect("bulk import SQL Server empty-string matrix row");
|
||||
let empty_generated =
|
||||
run_live_sqlserver_generated_insert(&mut client, &generated_table, &empty_csv, &empty_string_options, false)
|
||||
.await
|
||||
.expect("generated INSERT SQL Server empty-string matrix row");
|
||||
|
||||
let identity_csv = dir.join("matrix-explicit-identity.csv");
|
||||
std::fs::write(
|
||||
&identity_csv,
|
||||
"id,code,nullable_text,amount,occurred_at,offset_at,event_id,document,payload\n\
|
||||
42,explicit,identity,9999999999999999999999999999.9999999999,2026-07-29T23:59:59.9999999,2026-07-29T23:59:59.9999999+13:45,01234567-89ab-cdef-0123-456789abcdef,<root><value>identity</value></root>,0xABCDEF\n",
|
||||
)
|
||||
.expect("write SQL Server explicit-identity matrix CSV");
|
||||
let identity_request = live_sqlserver_import_request(
|
||||
&connection_id,
|
||||
&database,
|
||||
&bulk_table,
|
||||
&identity_csv,
|
||||
live_sqlserver_matrix_mappings(true),
|
||||
TableImportMode::Append,
|
||||
);
|
||||
let identity_bulk = run_live_sqlserver_import(&state, &pool_key, &identity_request)
|
||||
.await
|
||||
.expect("bulk import SQL Server explicit-identity matrix row");
|
||||
let identity_generated =
|
||||
run_live_sqlserver_generated_insert(&mut client, &generated_table, &identity_csv, &default_options, true)
|
||||
.await
|
||||
.expect("generated INSERT SQL Server explicit-identity matrix row");
|
||||
|
||||
let bulk_rows = live_sqlserver_matrix_rows(&mut client, &bulk_table).await;
|
||||
let generated_rows = live_sqlserver_matrix_rows(&mut client, &generated_table).await;
|
||||
let bulk_audit_rows = dbx_core::db::sqlserver::execute_query(
|
||||
&mut client,
|
||||
&format!("SELECT [target_id], [code], [default_text] FROM [dbo].[{bulk_audit}] ORDER BY [target_id]"),
|
||||
)
|
||||
.await
|
||||
.expect("query SQL Server bulk audit rows");
|
||||
let generated_audit_rows = dbx_core::db::sqlserver::execute_query(
|
||||
&mut client,
|
||||
&format!("SELECT [target_id], [code], [default_text] FROM [dbo].[{generated_audit}] ORDER BY [target_id]"),
|
||||
)
|
||||
.await
|
||||
.expect("query SQL Server generated INSERT audit rows");
|
||||
|
||||
let invalid_cases = [
|
||||
(
|
||||
"unique",
|
||||
"code,nullable_text,amount,occurred_at,offset_at,event_id,document,payload\n\
|
||||
unique-first,value,1.0000000000,2026-08-01T00:00:00,2026-08-01T00:00:00+00:00,10000000-0000-0000-0000-000000000001,<ok/>,0x01\n\
|
||||
auto-null,value,2.0000000000,2026-08-01T00:00:01,2026-08-01T00:00:01+00:00,10000000-0000-0000-0000-000000000002,<ok/>,0x02\n",
|
||||
),
|
||||
(
|
||||
"check",
|
||||
"code,nullable_text,amount,occurred_at,offset_at,event_id,document,payload\n\
|
||||
check-first,value,1.0000000000,2026-08-02T00:00:00,2026-08-02T00:00:00+00:00,20000000-0000-0000-0000-000000000001,<ok/>,0x01\n\
|
||||
check-invalid,value,-0.0000000001,2026-08-02T00:00:01,2026-08-02T00:00:01+00:00,20000000-0000-0000-0000-000000000002,<ok/>,0x02\n",
|
||||
),
|
||||
(
|
||||
"not-null",
|
||||
"code,nullable_text,amount,occurred_at,offset_at,event_id,document,payload\n\
|
||||
not-null-first,value,1.0000000000,2026-08-03T00:00:00,2026-08-03T00:00:00+00:00,30000000-0000-0000-0000-000000000001,<ok/>,0x01\n\
|
||||
,value,2.0000000000,2026-08-03T00:00:01,2026-08-03T00:00:01+00:00,30000000-0000-0000-0000-000000000002,<ok/>,0x02\n",
|
||||
),
|
||||
(
|
||||
"decimal-overflow",
|
||||
"code,nullable_text,amount,occurred_at,offset_at,event_id,document,payload\n\
|
||||
decimal-first,value,1.0000000000,2026-08-04T00:00:00,2026-08-04T00:00:00+00:00,40000000-0000-0000-0000-000000000001,<ok/>,0x01\n\
|
||||
decimal-invalid,value,10000000000000000000000000000.0000000000,2026-08-04T00:00:01,2026-08-04T00:00:01+00:00,40000000-0000-0000-0000-000000000002,<ok/>,0x02\n",
|
||||
),
|
||||
(
|
||||
"uuid",
|
||||
"code,nullable_text,amount,occurred_at,offset_at,event_id,document,payload\n\
|
||||
uuid-first,value,1.0000000000,2026-08-05T00:00:00,2026-08-05T00:00:00+00:00,50000000-0000-0000-0000-000000000001,<ok/>,0x01\n\
|
||||
uuid-invalid,value,2.0000000000,2026-08-05T00:00:01,2026-08-05T00:00:01+00:00,not-a-uuid,<ok/>,0x02\n",
|
||||
),
|
||||
(
|
||||
"xml",
|
||||
"code,nullable_text,amount,occurred_at,offset_at,event_id,document,payload\n\
|
||||
xml-first,value,1.0000000000,2026-08-06T00:00:00,2026-08-06T00:00:00+00:00,60000000-0000-0000-0000-000000000001,<ok/>,0x01\n\
|
||||
xml-invalid,value,2.0000000000,2026-08-06T00:00:01,2026-08-06T00:00:01+00:00,60000000-0000-0000-0000-000000000002,<unclosed>,0x02\n",
|
||||
),
|
||||
];
|
||||
let stable_bulk_count = live_sqlserver_table_count(&mut client, &bulk_table).await;
|
||||
let stable_generated_count = live_sqlserver_table_count(&mut client, &generated_table).await;
|
||||
let mut rejection_results = Vec::new();
|
||||
for (case, csv) in invalid_cases {
|
||||
let path = dir.join(format!("matrix-invalid-{case}.csv"));
|
||||
std::fs::write(&path, csv).expect("write SQL Server invalid matrix CSV");
|
||||
let request = live_sqlserver_import_request(
|
||||
&connection_id,
|
||||
&database,
|
||||
&bulk_table,
|
||||
&path,
|
||||
live_sqlserver_matrix_mappings(false),
|
||||
TableImportMode::Append,
|
||||
);
|
||||
let bulk_result = run_live_sqlserver_import(&state, &pool_key, &request).await;
|
||||
let generated_result =
|
||||
run_live_sqlserver_generated_insert(&mut client, &generated_table, &path, &default_options, false).await;
|
||||
let bulk_count = live_sqlserver_table_count(&mut client, &bulk_table).await;
|
||||
let generated_count = live_sqlserver_table_count(&mut client, &generated_table).await;
|
||||
rejection_results.push((case, bulk_result, generated_result, bulk_count, generated_count));
|
||||
}
|
||||
let final_bulk_audit_count = live_sqlserver_table_count(&mut client, &bulk_audit).await;
|
||||
let final_generated_audit_count = live_sqlserver_table_count(&mut client, &generated_audit).await;
|
||||
|
||||
let cleanup = format!(
|
||||
"DROP TRIGGER IF EXISTS [dbo].[{bulk_trigger}]; \
|
||||
DROP TRIGGER IF EXISTS [dbo].[{generated_trigger}]; \
|
||||
DROP TABLE IF EXISTS [dbo].[{bulk_audit}]; DROP TABLE IF EXISTS [dbo].[{generated_audit}]; \
|
||||
DROP TABLE IF EXISTS [dbo].[{bulk_table}]; DROP TABLE IF EXISTS [dbo].[{generated_table}];"
|
||||
);
|
||||
let _ = dbx_core::db::sqlserver::execute_batch(&mut client, &cleanup).await;
|
||||
state.remove_connection_pools_detached(&connection_id).await;
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
|
||||
assert_eq!(null_bulk.rows_imported, null_generated);
|
||||
assert_eq!(empty_bulk.rows_imported, empty_generated);
|
||||
assert_eq!(identity_bulk.rows_imported, identity_generated);
|
||||
assert_eq!(bulk_rows.rows, generated_rows.rows, "Bulk and generated INSERT values differ");
|
||||
assert_eq!(bulk_rows.rows.len(), 3);
|
||||
assert_eq!(bulk_rows.rows[0][2], serde_json::json!("<NULL>"));
|
||||
assert_eq!(bulk_rows.rows[0][3], serde_json::json!("12345678901234567890.1234567890"));
|
||||
assert_eq!(bulk_rows.rows[0][5], serde_json::json!("2026-07-27T04:34:56.1234567Z"));
|
||||
assert_eq!(bulk_rows.rows[0][7], serde_json::json!("<NULL>"));
|
||||
assert_eq!(bulk_rows.rows[0][8], serde_json::json!("<NULL>"));
|
||||
assert_eq!(bulk_rows.rows[1][2], serde_json::json!("<>"));
|
||||
assert_eq!(bulk_rows.rows[1][3], serde_json::json!("0.0000000001"));
|
||||
assert_eq!(bulk_rows.rows[1][5], serde_json::json!("2026-07-28T06:32:03.0000001Z"));
|
||||
assert_eq!(bulk_rows.rows[1][8], serde_json::json!("0x00ff10"));
|
||||
assert_eq!(bulk_rows.rows[2][0], serde_json::json!("42"));
|
||||
assert_eq!(bulk_rows.rows[2][3], serde_json::json!("9999999999999999999999999999.9999999999"));
|
||||
assert_eq!(bulk_rows.rows[2][5], serde_json::json!("2026-07-29T10:14:59.9999999Z"));
|
||||
assert!(bulk_rows.rows.iter().all(|row| row[9] == serde_json::json!("defaulted")));
|
||||
assert_eq!(bulk_audit_rows.rows, generated_audit_rows.rows, "trigger side effects differ");
|
||||
assert_eq!(bulk_audit_rows.rows.len(), 3);
|
||||
assert_eq!(final_bulk_audit_count, 3, "Bulk constraint failures left trigger side effects");
|
||||
assert_eq!(final_generated_audit_count, 3, "generated INSERT failures left trigger side effects");
|
||||
for (case, bulk_result, generated_result, bulk_count, generated_count) in rejection_results {
|
||||
assert!(bulk_result.is_err(), "Bulk path unexpectedly accepted {case} case");
|
||||
assert!(generated_result.is_err(), "generated INSERT unexpectedly accepted {case} case");
|
||||
assert_eq!(bulk_count, stable_bulk_count, "Bulk path partially wrote {case} case");
|
||||
assert_eq!(generated_count, stable_generated_count, "generated INSERT partially wrote {case} case");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DBX_LIVE_SQLSERVER_HOST/PORT/USER/PASSWORD pointing at a writable SQL Server database"]
|
||||
async fn live_sqlserver_bulk_imports_zero_fraction_xlsx_numbers_into_bigint() {
|
||||
let database = std::env::var("DBX_LIVE_SQLSERVER_DATABASE").unwrap_or_else(|_| "tempdb".to_string());
|
||||
let suffix = uuid::Uuid::new_v4().simple().to_string();
|
||||
let connection_id = format!("live-sqlserver-xlsx-integer-{suffix}");
|
||||
let table = format!("dbx_bulk_xlsx_integer_{suffix}");
|
||||
let mut client = dbx_core::db::sqlserver::connect(
|
||||
&std::env::var("DBX_LIVE_SQLSERVER_HOST").unwrap_or_else(|_| "127.0.0.1".to_string()),
|
||||
std::env::var("DBX_LIVE_SQLSERVER_PORT").ok().and_then(|value| value.parse().ok()).unwrap_or(1433),
|
||||
&std::env::var("DBX_LIVE_SQLSERVER_USER").unwrap_or_else(|_| "sa".to_string()),
|
||||
&std::env::var("DBX_LIVE_SQLSERVER_PASSWORD").expect("DBX_LIVE_SQLSERVER_PASSWORD"),
|
||||
Some(&database),
|
||||
None,
|
||||
Duration::from_secs(10),
|
||||
)
|
||||
.await
|
||||
.expect("connect SQL Server");
|
||||
dbx_core::db::sqlserver::execute_batch(
|
||||
&mut client,
|
||||
&format!("CREATE TABLE [dbo].[{table}] ([id] BIGINT NOT NULL, [label] NVARCHAR(40) NOT NULL)"),
|
||||
)
|
||||
.await
|
||||
.expect("create SQL Server XLSX integer table");
|
||||
|
||||
let (state, pool_key, dir) = live_sqlserver_import_state(&connection_id, &database, &suffix).await;
|
||||
let xlsx = build_xlsx_workbook(&XlsxWorksheetData {
|
||||
sheet_name: Some("Numbers".to_string()),
|
||||
columns: vec!["id".to_string(), "label".to_string()],
|
||||
column_types: Vec::new(),
|
||||
rows: vec![vec![serde_json::json!(1.0), serde_json::json!("xlsx")]],
|
||||
numeric_column_right_align: false,
|
||||
})
|
||||
.expect("build SQL Server XLSX integer fixture");
|
||||
let path = dir.join("zero-fraction-integer.xlsx");
|
||||
std::fs::write(&path, xlsx).expect("write SQL Server XLSX integer fixture");
|
||||
let mut request = live_sqlserver_import_request(
|
||||
&connection_id,
|
||||
&database,
|
||||
&table,
|
||||
&path,
|
||||
vec![live_sqlserver_import_mapping("id", "id"), live_sqlserver_import_mapping("label", "label")],
|
||||
TableImportMode::Append,
|
||||
);
|
||||
request.source_format = Some(TableImportSourceFormat::Excel);
|
||||
let result = run_live_sqlserver_import(&state, &pool_key, &request).await;
|
||||
let rows =
|
||||
dbx_core::db::sqlserver::execute_query(&mut client, &format!("SELECT [id], [label] FROM [dbo].[{table}]"))
|
||||
.await
|
||||
.expect("query SQL Server XLSX integer row");
|
||||
|
||||
let _ = dbx_core::db::sqlserver::execute_batch(&mut client, &format!("DROP TABLE IF EXISTS [dbo].[{table}]")).await;
|
||||
state.remove_connection_pools_detached(&connection_id).await;
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
|
||||
assert_eq!(result.expect("bulk import SQL Server XLSX integer row").rows_imported, 1);
|
||||
assert_eq!(rows.rows, vec![vec![serde_json::json!(1), serde_json::json!("xlsx")]]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DBX_LIVE_SQLSERVER_HOST/PORT/USER/PASSWORD pointing at a writable SQL Server database"]
|
||||
async fn live_sqlserver_table_import_bulk_preserves_target_semantics() {
|
||||
let database = std::env::var("DBX_LIVE_SQLSERVER_DATABASE").unwrap_or_else(|_| "tempdb".to_string());
|
||||
let suffix = uuid::Uuid::new_v4().simple().to_string();
|
||||
let connection_id = format!("live-sqlserver-import-{suffix}");
|
||||
let table = format!("dbx_bulk_target_{suffix}");
|
||||
let audit_table = format!("dbx_bulk_audit_{suffix}");
|
||||
let trigger = format!("dbx_bulk_trigger_{suffix}");
|
||||
let mut setup_client = dbx_core::db::sqlserver::connect(
|
||||
&std::env::var("DBX_LIVE_SQLSERVER_HOST").unwrap_or_else(|_| "127.0.0.1".to_string()),
|
||||
std::env::var("DBX_LIVE_SQLSERVER_PORT").ok().and_then(|value| value.parse().ok()).unwrap_or(1433),
|
||||
&std::env::var("DBX_LIVE_SQLSERVER_USER").unwrap_or_else(|_| "sa".to_string()),
|
||||
&std::env::var("DBX_LIVE_SQLSERVER_PASSWORD").expect("DBX_LIVE_SQLSERVER_PASSWORD"),
|
||||
Some(&database),
|
||||
None,
|
||||
Duration::from_secs(10),
|
||||
)
|
||||
.await
|
||||
.expect("connect SQL Server");
|
||||
dbx_core::db::sqlserver::execute_batch(
|
||||
&mut setup_client,
|
||||
&format!(
|
||||
"CREATE TABLE [dbo].[{table}] (\
|
||||
[id] INT IDENTITY(1,1) NOT NULL PRIMARY KEY, \
|
||||
[code] NVARCHAR(40) NOT NULL UNIQUE, \
|
||||
[occurred_at] DATETIME2(7) NOT NULL, \
|
||||
[amount] DECIMAL(38,10) NOT NULL, \
|
||||
[name] NVARCHAR(100) NOT NULL, \
|
||||
[payload] VARBINARY(MAX) NULL, \
|
||||
[created_at] DATETIME2(7) NOT NULL DEFAULT SYSUTCDATETIME()); \
|
||||
CREATE TABLE [dbo].[{audit_table}] ([target_id] INT NOT NULL, [name] NVARCHAR(100) NOT NULL);"
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("create bulk target tables");
|
||||
dbx_core::db::sqlserver::execute_batch(
|
||||
&mut setup_client,
|
||||
&format!(
|
||||
"CREATE TRIGGER [dbo].[{trigger}] ON [dbo].[{table}] AFTER INSERT AS \
|
||||
INSERT INTO [dbo].[{audit_table}] ([target_id], [name]) SELECT [id], [name] FROM inserted"
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("create bulk target trigger");
|
||||
|
||||
let (state, pool_key, dir) = live_sqlserver_import_state(&connection_id, &database, &suffix).await;
|
||||
let generated_identity_csv = dir.join("generated-identity.csv");
|
||||
std::fs::write(
|
||||
&generated_identity_csv,
|
||||
"code,occurred_at,amount,name,payload\nauto,2026-07-27T12:34:56.1234567,12345678901234567890.1234567890,\u{8d8a}\u{5357}\u{82b1},0x00FF10\n",
|
||||
)
|
||||
.expect("write generated identity CSV");
|
||||
let generated_identity_request = live_sqlserver_import_request(
|
||||
&connection_id,
|
||||
&database,
|
||||
&table,
|
||||
&generated_identity_csv,
|
||||
["code", "occurred_at", "amount", "name", "payload"]
|
||||
.into_iter()
|
||||
.map(|column| live_sqlserver_import_mapping(column, column))
|
||||
.collect(),
|
||||
TableImportMode::Append,
|
||||
);
|
||||
let generated_summary = run_live_sqlserver_import(&state, &pool_key, &generated_identity_request)
|
||||
.await
|
||||
.expect("bulk import with generated identity");
|
||||
|
||||
let explicit_identity_csv = dir.join("explicit-identity.csv");
|
||||
std::fs::write(
|
||||
&explicit_identity_csv,
|
||||
"id,code,occurred_at,amount,name,payload\n42,explicit,2026-07-28T01:02:03.0000000,1.0000000000,Tieng Viet,0xABCDEF\n",
|
||||
)
|
||||
.expect("write explicit identity CSV");
|
||||
let explicit_identity_request = live_sqlserver_import_request(
|
||||
&connection_id,
|
||||
&database,
|
||||
&table,
|
||||
&explicit_identity_csv,
|
||||
["id", "code", "occurred_at", "amount", "name", "payload"]
|
||||
.into_iter()
|
||||
.map(|column| live_sqlserver_import_mapping(column, column))
|
||||
.collect(),
|
||||
TableImportMode::Append,
|
||||
);
|
||||
let explicit_summary = run_live_sqlserver_import(&state, &pool_key, &explicit_identity_request)
|
||||
.await
|
||||
.expect("bulk import with explicit identity");
|
||||
|
||||
let plain_binary_csv = dir.join("plain-binary.csv");
|
||||
std::fs::write(
|
||||
&plain_binary_csv,
|
||||
"code,occurred_at,amount,name,payload\nplain,2026-07-28T02:03:04.0000000,2.0000000000,plain binary,plain\n",
|
||||
)
|
||||
.expect("write plain binary CSV");
|
||||
let plain_binary_request = live_sqlserver_import_request(
|
||||
&connection_id,
|
||||
&database,
|
||||
&table,
|
||||
&plain_binary_csv,
|
||||
["code", "occurred_at", "amount", "name", "payload"]
|
||||
.into_iter()
|
||||
.map(|column| live_sqlserver_import_mapping(column, column))
|
||||
.collect(),
|
||||
TableImportMode::Append,
|
||||
);
|
||||
let plain_binary_summary = run_live_sqlserver_import(&state, &pool_key, &plain_binary_request)
|
||||
.await
|
||||
.expect("SQL fallback import with plain-text varbinary input");
|
||||
|
||||
let duplicate_csv = dir.join("duplicate.csv");
|
||||
std::fs::write(
|
||||
&duplicate_csv,
|
||||
"code,occurred_at,amount,name,payload\nauto,2026-07-29T00:00:00,2.0000000000,duplicate,0x01\n",
|
||||
)
|
||||
.expect("write duplicate CSV");
|
||||
let duplicate_request = live_sqlserver_import_request(
|
||||
&connection_id,
|
||||
&database,
|
||||
&table,
|
||||
&duplicate_csv,
|
||||
["code", "occurred_at", "amount", "name", "payload"]
|
||||
.into_iter()
|
||||
.map(|column| live_sqlserver_import_mapping(column, column))
|
||||
.collect(),
|
||||
TableImportMode::Append,
|
||||
);
|
||||
let duplicate_error = run_live_sqlserver_import(&state, &pool_key, &duplicate_request).await;
|
||||
|
||||
let rows = dbx_core::db::sqlserver::execute_query(
|
||||
&mut setup_client,
|
||||
&format!(
|
||||
"SELECT CONVERT(VARCHAR(12), [id]), [code], CONVERT(VARCHAR(33), [occurred_at], 126), \
|
||||
CONVERT(VARCHAR(50), [amount]), [name], sys.fn_varbintohexstr([payload]), \
|
||||
CASE WHEN [created_at] IS NULL THEN N'missing' ELSE N'set' END, \
|
||||
CASE WHEN [code] = N'plain' THEN CONVERT(NVARCHAR(100), [payload]) END \
|
||||
FROM [dbo].[{table}] ORDER BY [id]"
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("verify bulk target rows");
|
||||
let audit_count = dbx_core::db::sqlserver::execute_query(
|
||||
&mut setup_client,
|
||||
&format!("SELECT COUNT(*) FROM [dbo].[{audit_table}]"),
|
||||
)
|
||||
.await
|
||||
.expect("verify trigger rows");
|
||||
let cleanup = format!(
|
||||
"DROP TRIGGER IF EXISTS [dbo].[{trigger}]; DROP TABLE IF EXISTS [dbo].[{audit_table}]; DROP TABLE IF EXISTS [dbo].[{table}];"
|
||||
);
|
||||
let _ = dbx_core::db::sqlserver::execute_batch(&mut setup_client, &cleanup).await;
|
||||
state.remove_connection_pools_detached(&connection_id).await;
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
|
||||
assert_eq!(generated_summary.rows_imported, 1);
|
||||
assert_eq!(explicit_summary.rows_imported, 1);
|
||||
assert_eq!(plain_binary_summary.rows_imported, 1);
|
||||
assert!(duplicate_error.is_err(), "unique constraint violation must fail the import");
|
||||
assert_eq!(rows.rows.len(), 3);
|
||||
assert_eq!(rows.rows[0][0], serde_json::json!("1"));
|
||||
assert_eq!(rows.rows[0][1], serde_json::json!("auto"));
|
||||
assert_eq!(rows.rows[0][2], serde_json::json!("2026-07-27T12:34:56.1234567"));
|
||||
assert_eq!(rows.rows[0][3], serde_json::json!("12345678901234567890.1234567890"));
|
||||
assert_eq!(rows.rows[0][4], serde_json::json!("\u{8d8a}\u{5357}\u{82b1}"));
|
||||
assert_eq!(rows.rows[0][5], serde_json::json!("0x00ff10"));
|
||||
assert_eq!(rows.rows[0][6], serde_json::json!("set"));
|
||||
assert_eq!(rows.rows[1][0], serde_json::json!("42"));
|
||||
assert_eq!(rows.rows[1][4], serde_json::json!("Tieng Viet"));
|
||||
assert_eq!(rows.rows[1][5], serde_json::json!("0xabcdef"));
|
||||
assert_eq!(rows.rows[2][1], serde_json::json!("plain"));
|
||||
assert_eq!(rows.rows[2][7], serde_json::json!("plain"));
|
||||
assert_eq!(audit_count.rows[0][0], serde_json::json!(3));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DBX_LIVE_SQLSERVER_HOST/PORT/USER/PASSWORD pointing at a writable SQL Server database"]
|
||||
async fn live_sqlserver_table_import_bulk_cancels_before_target_write_and_rolls_back_truncate() {
|
||||
let database = std::env::var("DBX_LIVE_SQLSERVER_DATABASE").unwrap_or_else(|_| "tempdb".to_string());
|
||||
let suffix = uuid::Uuid::new_v4().simple().to_string();
|
||||
let connection_id = format!("live-sqlserver-import-rollback-{suffix}");
|
||||
let table = format!("dbx_bulk_rollback_{suffix}");
|
||||
let mut setup_client = dbx_core::db::sqlserver::connect(
|
||||
&std::env::var("DBX_LIVE_SQLSERVER_HOST").unwrap_or_else(|_| "127.0.0.1".to_string()),
|
||||
std::env::var("DBX_LIVE_SQLSERVER_PORT").ok().and_then(|value| value.parse().ok()).unwrap_or(1433),
|
||||
&std::env::var("DBX_LIVE_SQLSERVER_USER").unwrap_or_else(|_| "sa".to_string()),
|
||||
&std::env::var("DBX_LIVE_SQLSERVER_PASSWORD").expect("DBX_LIVE_SQLSERVER_PASSWORD"),
|
||||
Some(&database),
|
||||
None,
|
||||
Duration::from_secs(10),
|
||||
)
|
||||
.await
|
||||
.expect("connect SQL Server");
|
||||
dbx_core::db::sqlserver::execute_batch(
|
||||
&mut setup_client,
|
||||
&format!(
|
||||
"CREATE TABLE [dbo].[{table}] ([id] INT NOT NULL PRIMARY KEY, [amount] DECIMAL(38,10) NOT NULL CHECK ([amount] > 0)); \
|
||||
INSERT INTO [dbo].[{table}] VALUES (999, 9.0000000000);"
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("create rollback target");
|
||||
let (state, pool_key, dir) = live_sqlserver_import_state(&connection_id, &database, &suffix).await;
|
||||
|
||||
let invalid_csv = dir.join("invalid.csv");
|
||||
std::fs::write(&invalid_csv, "id,amount\n1,-1.0000000000\n").expect("write invalid CSV");
|
||||
let invalid_request = live_sqlserver_import_request(
|
||||
&connection_id,
|
||||
&database,
|
||||
&table,
|
||||
&invalid_csv,
|
||||
vec![live_sqlserver_import_mapping("id", "id"), live_sqlserver_import_mapping("amount", "amount")],
|
||||
TableImportMode::Truncate,
|
||||
);
|
||||
let truncate_error = run_live_sqlserver_import(&state, &pool_key, &invalid_request).await;
|
||||
let rows_after_failed_truncate = dbx_core::db::sqlserver::execute_query(
|
||||
&mut setup_client,
|
||||
&format!("SELECT [id], CONVERT(VARCHAR(50), [amount]) FROM [dbo].[{table}]"),
|
||||
)
|
||||
.await
|
||||
.expect("verify truncate rollback");
|
||||
|
||||
let valid_csv = dir.join("cancelled.csv");
|
||||
std::fs::write(&valid_csv, "id,amount\n1,1.0000000000\n").expect("write cancellation CSV");
|
||||
let cancel_request = live_sqlserver_import_request(
|
||||
&connection_id,
|
||||
&database,
|
||||
&table,
|
||||
&valid_csv,
|
||||
vec![live_sqlserver_import_mapping("id", "id"), live_sqlserver_import_mapping("amount", "amount")],
|
||||
TableImportMode::Truncate,
|
||||
);
|
||||
let cancellation_checks = Arc::new(AtomicUsize::new(0));
|
||||
let cancellation_checks_for_import = cancellation_checks.clone();
|
||||
let cancelled_progress = Arc::new(AtomicBool::new(false));
|
||||
let cancelled_progress_for_import = cancelled_progress.clone();
|
||||
let cancel_error = import_table_file_core(
|
||||
&state,
|
||||
&cancel_request,
|
||||
&DatabaseType::SqlServer,
|
||||
&pool_key,
|
||||
move |_| {
|
||||
let checks = cancellation_checks_for_import.clone();
|
||||
Box::pin(async move { checks.fetch_add(1, Ordering::SeqCst) >= 1 })
|
||||
},
|
||||
move |progress| {
|
||||
if progress.status == TableImportStatus::Cancelled {
|
||||
cancelled_progress_for_import.store(true, Ordering::SeqCst);
|
||||
}
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let rows_after_cancel = dbx_core::db::sqlserver::execute_query(
|
||||
&mut setup_client,
|
||||
&format!("SELECT [id], CONVERT(VARCHAR(50), [amount]) FROM [dbo].[{table}]"),
|
||||
)
|
||||
.await
|
||||
.expect("verify cancellation leaves target unchanged");
|
||||
let _ = dbx_core::db::sqlserver::execute_batch(&mut setup_client, &format!("DROP TABLE IF EXISTS [dbo].[{table}]"))
|
||||
.await;
|
||||
state.remove_connection_pools_detached(&connection_id).await;
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
|
||||
assert!(truncate_error.is_err(), "check constraint must fail the truncate import");
|
||||
assert_eq!(rows_after_failed_truncate.rows, vec![vec![serde_json::json!(999), serde_json::json!("9.0000000000")]]);
|
||||
assert_eq!(cancel_error.unwrap_err(), "Import cancelled");
|
||||
assert!(cancellation_checks.load(Ordering::SeqCst) >= 2);
|
||||
assert!(cancelled_progress.load(Ordering::SeqCst));
|
||||
assert_eq!(rows_after_cancel.rows, vec![vec![serde_json::json!(999), serde_json::json!("9.0000000000")]]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DBX_LIVE_SQLSERVER_HOST/PORT/USER/PASSWORD pointing at a writable SQL Server database"]
|
||||
async fn live_sqlserver_column_metadata_marks_non_positional_insert_columns() {
|
||||
|
||||
Reference in new issue
Block a user