From 2384a2e2c2a7965d9552109762f90c260313ab38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=93=D0=91=20Lieu?= <60683892+hanhvs@users.noreply.github.com> Date: Mon, 28 Sep 2026 21:57:39 +0700 Subject: [PATCH] fix(sql-file): preserve statements across split ZIP parts --- crates/dbx-core/src/data/database_export.rs | 104 ++++++- crates/dbx-core/src/data/sql_file_import.rs | 57 ++-- crates/dbx-core/src/data/table_export.rs | 81 ++++-- .../tests/live_sqlserver_split_zip_e2e.rs | 267 ++++++++++++++++++ crates/dbx-formats/src/export_split_zip.rs | 140 ++++++++- 5 files changed, 596 insertions(+), 53 deletions(-) create mode 100644 crates/dbx-core/tests/live_sqlserver_split_zip_e2e.rs diff --git a/crates/dbx-core/src/data/database_export.rs b/crates/dbx-core/src/data/database_export.rs index 0facacdb0b..4cd3dd33d1 100644 --- a/crates/dbx-core/src/data/database_export.rs +++ b/crates/dbx-core/src/data/database_export.rs @@ -15,6 +15,7 @@ use crate::connection::task_client_session_id; use crate::models::connection::DatabaseType; use crate::mysql_ddl_normalize::DdlNormalizeOptions; use crate::object_source_sql::build_export_object_source_sql; +use crate::sql::{SqlParsingOptions, SqlStatementSplitter}; use crate::sql_dialect::{qualified_table_name, uses_single_row_insert_statements}; use crate::transfer::{ format_ch_array_sql_literal, format_pg_array_sql_literal, format_postgres_vector_sql_literal, @@ -166,6 +167,16 @@ enum DatabaseExportWriter { SplitZip(Box), } +impl DatabaseExportWriter { + fn write_sql_unit(&mut self, unit: &[u8]) -> std::io::Result<()> { + match self { + Self::Plain(writer) => writer.write_all(unit), + Self::Gzip(writer) => writer.write_all(unit), + Self::SplitZip(writer) => writer.write_sql_unit(unit), + } + } +} + impl Write for DatabaseExportWriter { fn write(&mut self, buffer: &[u8]) -> std::io::Result { match self { @@ -175,6 +186,12 @@ impl Write for DatabaseExportWriter { } } + fn write_fmt(&mut self, fmt: std::fmt::Arguments<'_>) -> std::io::Result<()> { + let mut unit = String::new(); + std::fmt::write(&mut unit, fmt).map_err(|_| std::io::Error::other("Failed to format SQL export unit"))?; + self.write_sql_unit(unit.as_bytes()) + } + fn flush(&mut self) -> std::io::Result<()> { match self { Self::Plain(writer) => writer.flush(), @@ -3221,17 +3238,21 @@ fn postgres_create_schema_sql(schema: &str) -> String { format!("CREATE SCHEMA IF NOT EXISTS {};", quote_identifier(schema, &DatabaseType::Postgres)) } -// Copy one line at a time so a `SplitZipExportWriter` can only rotate between -// complete SQL statements; `std::io::copy` would feed it arbitrary 8KB chunks. fn combine_schema_sql_export(source: &mut dyn BufRead, destination: &mut W) -> std::io::Result<()> { - let mut line = Vec::new(); + let mut splitter = SqlStatementSplitter::with_options(SqlParsingOptions::for_database_type(DatabaseType::Postgres)); + let mut chunk = String::new(); loop { - line.clear(); - let bytes_read = source.read_until(b'\n', &mut line)?; + chunk.clear(); + let bytes_read = source.read_line(&mut chunk)?; if bytes_read == 0 { break; } - destination.write_all(&line)?; + for statement in splitter.push_chunk(&chunk) { + destination.write_all(format!("{statement};\n").as_bytes())?; + } + } + for statement in splitter.finish() { + destination.write_all(format!("{statement};\n").as_bytes())?; } Ok(()) } @@ -6706,6 +6727,75 @@ mod tests { assert_eq!(combined.matches("INSERT INTO").count(), 20); } + #[test] + fn split_zip_export_writer_keeps_an_oversized_create_table_ddl_whole_before_the_next_unit() { + // Exercises the "SQL unit, not just INSERT" contract: a CREATE TABLE + // DDL statement, an INSERT, and a SQL Server-style multi-statement + // routine body must each land in exactly one part, never split at an + // arbitrary byte offset inside one, even when writeln! formats the + // unit from several interpolated arguments (real call sites do this, + // e.g. `writeln!(file, "{source}\n")` for exported routine bodies). + let directory = tempfile::tempdir().unwrap(); + let zip_path = directory.path().join("schema.zip"); + let mut writer = crate::export_split_zip::SplitZipExportWriter::create( + &zip_path, + crate::export_split_zip::MIN_SPLIT_PART_MAX_MB, + "schema", + "sql", + ) + .unwrap(); + + let table_name = "dbo.orders"; + // part_max_bytes for MIN_SPLIT_PART_MAX_MB is 1 MiB. Make the CREATE + // TABLE unit alone exceed that -- it stays whole as the oversized-unit + // exception, but the *next* unit must then rotate into a new part. + let columns = (0..100_000).map(|index| format!("col_{index} INT")).collect::>().join(", "); + let create_table_ddl = format!("CREATE TABLE {table_name} ({columns});"); + // A SQL Server routine body: multiple statements inside one BEGIN/END + // block plus a batch-separator comment, matching what + // build_export_object_source_sql emits for a procedure/trigger -- + // the writer must never cut between the inner statements. + let procedure_source = "CREATE PROCEDURE dbo.recalc_totals AS\nBEGIN\n UPDATE dbo.orders SET total = total + 1;\n INSERT INTO dbo.audit_log (message) VALUES ('recalculated');\nEND".to_string(); + let go_batch_separator = "GO"; + let insert = "INSERT INTO dbo.orders (id) VALUES (1);".to_string(); + + writeln!(writer, "{create_table_ddl}").unwrap(); + writeln!(writer, "{procedure_source}").unwrap(); + writeln!(writer, "{go_batch_separator}").unwrap(); + writeln!(writer, "{insert}").unwrap(); + writer.finish("schema.sql").unwrap(); + + let file = std::fs::File::open(&zip_path).unwrap(); + let mut archive = zip::ZipArchive::new(file).unwrap(); + let mut sql_parts = Vec::new(); + for index in 0..archive.len() { + let mut entry = archive.by_index(index).unwrap(); + if !entry.name().ends_with(".sql") { + continue; + } + let mut contents = String::new(); + entry.read_to_string(&mut contents).unwrap(); + sql_parts.push((entry.name().to_string(), contents)); + } + sql_parts.sort_by(|a, b| a.0.cmp(&b.0)); + + assert!(sql_parts.len() >= 2, "the oversized CREATE TABLE alone must rotate before later units"); + + let combined: String = sql_parts.iter().map(|(_, contents)| contents.as_str()).collect(); + assert_eq!( + combined, + format!("{create_table_ddl}\n{procedure_source}\n{go_batch_separator}\n{insert}\n"), + "reassembling every part must reproduce the original stream byte for byte" + ); + + // Every unit must be found intact inside exactly one part -- proof + // that none of them was fragmented across a rotation boundary. + for unit in [create_table_ddl.as_str(), procedure_source.as_str(), go_batch_separator, insert.as_str()] { + let containing_parts = sql_parts.iter().filter(|(_, contents)| contents.contains(unit)).count(); + assert_eq!(containing_parts, 1, "unit must appear whole in exactly one part: {unit:?}"); + } + } + #[test] fn all_schemas_combine_keeps_split_part_boundaries_statement_safe() { // Mirror of the per-schema temporary files that @@ -6758,7 +6848,7 @@ mod tests { for line in contents.lines().filter(|line| !line.trim().is_empty()) { assert!( line.trim_start().starts_with("INSERT INTO") && line.trim_end().ends_with(';'), - "{name} has a malformed line from a mid-statement cut" + "{name} has a malformed line from a mid-statement cut: {line:?}" ); } } diff --git a/crates/dbx-core/src/data/sql_file_import.rs b/crates/dbx-core/src/data/sql_file_import.rs index 6ce84870ec..1ebe70272d 100644 --- a/crates/dbx-core/src/data/sql_file_import.rs +++ b/crates/dbx-core/src/data/sql_file_import.rs @@ -856,6 +856,8 @@ async fn execute_sql_file_paths_inner( let mut prev_success_count = 0usize; let mut prev_failure_count = 0usize; let mut prev_affected_rows = 0u64; + let mut splitter = Some(StreamingSqlFileSplitter::new(database_type, options)); + let mut pending_statements = Vec::with_capacity(SQL_FILE_STATEMENT_BATCH_SIZE); let import_result = async { for (file_index, file_path) in file_paths.iter().enumerate() { let file_name = file_path.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(); @@ -881,8 +883,6 @@ async fn execute_sql_file_paths_inner( }); } - let mut splitter = StreamingSqlFileSplitter::new(database_type, options); - let mut pending_statements = Vec::with_capacity(SQL_FILE_STATEMENT_BATCH_SIZE); let normalize_mysql_binary_literals = import_target.as_ref().is_some_and(|target| { crate::sql::is_mysql_compatible_import_target(&target.db_type, target.driver_profile.as_deref()) }); @@ -941,7 +941,10 @@ async fn execute_sql_file_paths_inner( "", None, )); - let mut next_statements = splitter.push_chunk(&chunk); + let mut next_statements = splitter + .as_mut() + .expect("splitter is only taken by finish() on the last file, after this loop exits") + .push_chunk(&chunk); if let Some(filter) = restore_filter.as_mut() { if let Err(error) = filter_restore_statements( &mut next_statements, @@ -979,23 +982,27 @@ async fn execute_sql_file_paths_inner( .await?; } - let mut next_statements = splitter.finish(); - if let Some(filter) = restore_filter.as_mut() { - if let Err(error) = filter_restore_statements( - &mut next_statements, - filter, - request.selected_tables.as_deref().unwrap_or_default(), - ) { - emit(sql_file_execution_error_progress( - &request.execution_id, - started_at, - &progress, - error.clone(), - )); - return Err(error); + let is_last_file = file_index + 1 == file_count; + if is_last_file { + let mut next_statements = + splitter.take().expect("splitter is only taken once, on the last file").finish(); + if let Some(filter) = restore_filter.as_mut() { + if let Err(error) = filter_restore_statements( + &mut next_statements, + filter, + request.selected_tables.as_deref().unwrap_or_default(), + ) { + emit(sql_file_execution_error_progress( + &request.execution_id, + started_at, + &progress, + error.clone(), + )); + return Err(error); + } } + pending_statements.extend(next_statements); } - pending_statements.extend(next_statements); execute_sql_file_statement_batch( state, request, @@ -3183,6 +3190,20 @@ mod tests { assert_eq!(statements[1], "SELECT 3;"); } + #[test] + fn streaming_sqlserver_splitter_keeps_split_json_literal_across_parts() { + let part_one = "INSERT INTO [dbo].[VersionValue] ([value]) VALUES (N'{\"materialOrSymbolMate"; + let part_two = "rialId\":\"9625a891-3682-4151-acdb-6480db860033\",\"label\":\"B\\\\u1ebb ch\\\\u00e2n/Th\\\\u1eb3ng/\",\"quote\":\"it''s valid\"}');\n"; + let expected = format!("{part_one}{part_two}").trim().to_string(); + let mut splitter = StreamingSqlFileSplitter::new(Some(DatabaseType::SqlServer), SqlParsingOptions::default()); + + assert!(splitter.push_chunk(part_one).is_empty()); + assert!(splitter.push_chunk(part_two).is_empty()); + let statements = splitter.finish().into_iter().map(|statement| statement.sql).collect::>(); + + assert_eq!(statements, vec![expected]); + } + #[test] fn streaming_sqlserver_splitter_handles_go_across_chunks() { let mut splitter = SqlServerBatchSplitter::default(); diff --git a/crates/dbx-core/src/data/table_export.rs b/crates/dbx-core/src/data/table_export.rs index 67f1b85040..0cfbe8c4c2 100644 --- a/crates/dbx-core/src/data/table_export.rs +++ b/crates/dbx-core/src/data/table_export.rs @@ -1068,6 +1068,15 @@ enum TableExportSqlWriter { SplitZip(Box), } +impl TableExportSqlWriter { + fn write_sql_unit(&mut self, unit: &[u8]) -> std::io::Result<()> { + match self { + Self::Plain(writer) => writer.write_all(unit), + Self::SplitZip(writer) => writer.write_sql_unit(unit), + } + } +} + impl Write for TableExportSqlWriter { fn write(&mut self, buffer: &[u8]) -> std::io::Result { match self { @@ -1076,6 +1085,12 @@ impl Write for TableExportSqlWriter { } } + fn write_fmt(&mut self, fmt: std::fmt::Arguments<'_>) -> std::io::Result<()> { + let mut unit = String::new(); + std::fmt::write(&mut unit, fmt).map_err(|_| std::io::Error::other("Failed to format SQL export unit"))?; + self.write_sql_unit(unit.as_bytes()) + } + fn flush(&mut self) -> std::io::Result<()> { match self { Self::Plain(writer) => writer.flush(), @@ -1093,6 +1108,21 @@ impl TableExportSqlWriter { } } +fn write_sql_export_statements( + file: &mut impl Write, + statements: Vec, + wrote_statements: &mut bool, +) -> Result<(), String> { + for statement in statements { + if *wrote_statements { + file.write_all(b"\n").map_err(|e| format!("Failed to write SQL: {e}"))?; + } + file.write_all(statement.as_bytes()).map_err(|e| format!("Failed to write SQL: {e}"))?; + *wrote_statements = true; + } + Ok(()) +} + fn create_table_export_sql_writer(request: &TableExportRequest) -> Result { if let Some(max_mb) = request.split_max_mb { let zip_path = std::path::Path::new(&request.file_path); @@ -1423,14 +1453,7 @@ async fn try_export_native_table_stream( &sql_export_excluded_columns(request, primary_keys, col_names, column_extras), request.insert_dialect, )?; - if !statements.is_empty() { - if wrote_statements { - file.write_all(b"\n").map_err(|e| format!("Failed to write SQL: {e}"))?; - } - file.write_all(statements.join("\n").as_bytes()) - .map_err(|e| format!("Failed to write SQL: {e}"))?; - wrote_statements = true; - } + write_sql_export_statements(file, statements, &mut wrote_statements)?; Ok(()) }; let result = stream_native_table_rows( @@ -2289,14 +2312,7 @@ async fn export_table_data_core_inner( &sql_export_excluded_columns(request, &primary_keys, &col_names, &column_extras), request.insert_dialect, )?; - if !statements.is_empty() { - if wrote_statements { - file.write_all(b"\n").map_err(|e| format!("Failed to write SQL: {e}"))?; - } - file.write_all(statements.join("\n").as_bytes()) - .map_err(|e| format!("Failed to write SQL: {e}"))?; - wrote_statements = true; - } + write_sql_export_statements(&mut file, statements, &mut wrote_statements)?; rows_exported += row_count as u64; if use_keyset { @@ -2361,6 +2377,39 @@ mod tests { #[cfg(unix)] use std::os::unix::fs::PermissionsExt; + #[test] + fn split_zip_sql_export_writes_batch_statements_separately() { + let dir = tempfile::tempdir().unwrap(); + let zip_path = dir.path().join("export.zip"); + let statement = format!("INSERT INTO t VALUES ('{}');", "x".repeat(700 * 1024)); + let mut writer = crate::export_split_zip::SplitZipExportWriter::create( + &zip_path, + crate::export_split_zip::MIN_SPLIT_PART_MAX_MB, + "table", + "sql", + ) + .unwrap(); + let mut wrote_statements = false; + + write_sql_export_statements(&mut writer, vec![statement.clone(), statement.clone()], &mut wrote_statements) + .unwrap(); + writer.finish("table.sql").unwrap(); + + let mut archive = zip::ZipArchive::new(std::fs::File::open(zip_path).unwrap()).unwrap(); + let mut sql_parts = Vec::new(); + for index in 0..archive.len() { + let mut entry = archive.by_index(index).unwrap(); + if entry.name().ends_with(".sql") { + let mut contents = String::new(); + entry.read_to_string(&mut contents).unwrap(); + sql_parts.push(contents); + } + } + + assert_eq!(sql_parts, vec![format!("{statement}\n"), statement]); + assert!(sql_parts.iter().all(|part| part.len() <= 1024 * 1024)); + } + #[test] fn table_export_request_defaults_to_source_dialect_and_batch_insert_mode() { let mut payload = json!({ diff --git a/crates/dbx-core/tests/live_sqlserver_split_zip_e2e.rs b/crates/dbx-core/tests/live_sqlserver_split_zip_e2e.rs new file mode 100644 index 0000000000..8f889dc7e5 --- /dev/null +++ b/crates/dbx-core/tests/live_sqlserver_split_zip_e2e.rs @@ -0,0 +1,267 @@ +// E2E test executed manually against a real local SQL Server 2025 container +// (`database-mssql-1`, database `default`) already loaded with a production-shaped +// dataset (50 tables / 52 foreign keys), as part of task #01a0db6e-2276. +// +// Flow: export database `default` (with `split_max_mb` set, i.e. the split-ZIP +// contract under review) -> verify manifest/parts -> re-import the resulting ZIP +// via `execute_sql_file_paths` with `skip_relational_constraints = true` ("Skip +// relationship") into a fresh scratch database, exactly like the desktop app does. +// +// Run manually with: +// DBX_LIVE_SQLSERVER_PASSWORD='MSSQLStrongPassword123!' \ +// cargo test -p dbx-core --test live_sqlserver_split_zip_e2e -- --nocapture --test-threads=1 + +use dbx_core::connection::AppState; +use dbx_core::database_export::{export_database_sql_core, DatabaseExportOutputCompression, DatabaseExportRequest}; +use dbx_core::models::connection::DatabaseType; +use dbx_core::sql::SqlFileRequest; +use dbx_core::sql_file_import::execute_sql_file_paths; +use std::sync::Arc; +use std::time::Instant; +use tokio_util::sync::CancellationToken; + +fn live_sqlserver_config(id: &str, database: &str) -> dbx_core::models::connection::ConnectionConfig { + dbx_core::models::connection::ConnectionConfig { + docs_notes_path: None, + id: id.to_string(), + name: id.to_string(), + note: String::new(), + db_type: DatabaseType::SqlServer, + driver_profile: None, + driver_label: None, + url_params: None, + agent_java_options: Vec::new(), + host: std::env::var("DBX_LIVE_SQLSERVER_HOST").unwrap_or_else(|_| "127.0.0.1".to_string()), + port: std::env::var("DBX_LIVE_SQLSERVER_PORT").ok().and_then(|value| value.parse().ok()).unwrap_or(1433), + username: std::env::var("DBX_LIVE_SQLSERVER_USER").unwrap_or_else(|_| "sa".to_string()), + password: std::env::var("DBX_LIVE_SQLSERVER_PASSWORD").expect("DBX_LIVE_SQLSERVER_PASSWORD"), + database: Some(database.to_string()), + default_schema: None, + visible_databases: None, + visible_database_patterns: None, + visible_schemas: None, + attached_databases: Vec::new(), + init_script: None, + color: None, + transport_layers: Vec::new(), + connect_timeout_secs: 10, + query_timeout_secs: 30, + idle_timeout_secs: 60, + keepalive_interval_secs: 0, + ssl: false, + ca_cert_path: String::new(), + client_cert_path: String::new(), + client_key_path: String::new(), + sysdba: false, + oracle_connection_type: None, + connection_string: None, + redis_connection_mode: None, + redis_sentinel_master: String::new(), + redis_sentinel_nodes: String::new(), + redis_sentinel_username: String::new(), + redis_sentinel_password: String::new(), + redis_sentinel_tls: false, + redis_cluster_nodes: String::new(), + redis_key_separator: dbx_core::models::connection::default_redis_key_separator(), + redis_scan_page_size: None, + redis_database_aliases: Default::default(), + redis_key_templates: Vec::new(), + redis_key_grouping: None, + etcd_endpoints: String::new(), + gbase_server: String::new(), + informix_server: String::new(), + external_config: None, + plugin_id: None, + plugin_connection_provider: None, + plugin_connection_type: None, + connection_secrets: Default::default(), + jdbc_driver_class: None, + jdbc_driver_paths: Vec::new(), + one_time: false, + save_password: true, + read_only: false, + is_production: false, + production_databases: vec![], + show_system_schemas: false, + database_info: None, + sidebar_auto_load_all_tables: false, + } +} + +async fn live_sqlserver_state(connection_id: &str, database: &str, suffix: &str) -> (AppState, std::path::PathBuf) { + let dir = std::env::temp_dir().join(format!("dbx-live-sqlserver-splitzip-{suffix}")); + std::fs::create_dir_all(&dir).expect("create live test directory"); + let storage = + dbx_core::persistence::test_storage::open(&dir.join("storage.db")).await.expect("open live test storage"); + let state = AppState::new(storage); + let config = live_sqlserver_config(connection_id, database); + state.configs.write().await.insert(connection_id.to_string(), config); + (state, dir) +} + +#[tokio::test] +#[ignore = "manual E2E against a real local SQL Server instance -- see file header"] +async fn manual_e2e_sqlserver_default_split_zip_export_then_import_skip_relationship() { + let export_connection_id = "e2e-mssql-export"; + let import_connection_id = "e2e-mssql-import"; + let source_database = "default"; + // Import target is a separate scratch database, NOT the just-emptied + // `default`, per the task's own guidance to isolate the destructive + // re-import from the production-shaped database whenever possible. + let import_database = "dbx_e2e_split_zip_restore"; + + let (export_state, work_dir) = live_sqlserver_state(export_connection_id, source_database, "export").await; + let export_state = Arc::new(export_state); + + let zip_path = work_dir.join("default-export.zip"); + let export_request = DatabaseExportRequest { + export_id: format!("e2e-export-{}", uuid::Uuid::new_v4()), + connection_id: export_connection_id.to_string(), + database: source_database.to_string(), + schema: String::new(), + file_path: zip_path.to_string_lossy().to_string(), + selected_tables: Vec::new(), + excluded_tables: Vec::new(), + include_structure: true, + include_data: true, + include_objects: true, + include_create_database: false, + drop_table_if_exists: true, + omit_auto_increment: false, + fail_on_error: true, + prevent_overwrite: false, + output_compression: DatabaseExportOutputCompression::None, + snapshot_session_id: None, + batch_size: 1000, + split_max_mb: Some(1), + insert_dialect: Default::default(), + }; + + let progress_log = std::sync::Mutex::new(Vec::new()); + let export_result = export_database_sql_core(&export_state, &export_request, |progress| { + progress_log.lock().unwrap().push(format!("{progress:?}")); + }) + .await; + + eprintln!("=== EXPORT PROGRESS LOG ==="); + for line in progress_log.lock().unwrap().iter() { + eprintln!("{line}"); + } + eprintln!("=== EXPORT RESULT ===\n{export_result:?}"); + export_result.expect("split-ZIP database export should succeed"); + + assert!(zip_path.exists(), "expected export ZIP at {}", zip_path.display()); + let zip_metadata = std::fs::metadata(&zip_path).expect("stat export zip"); + eprintln!("Export ZIP size: {} bytes", zip_metadata.len()); + + // --- Verify manifest.json + parts against the dbx-sql-export-parts-v1 contract --- + let file = std::fs::File::open(&zip_path).expect("open export zip"); + let mut archive = zip::ZipArchive::new(file).expect("read export zip as a zip archive"); + let mut entry_names: Vec = + (0..archive.len()).map(|index| archive.by_index(index).expect("zip entry").name().to_string()).collect(); + entry_names.sort(); + eprintln!("=== ZIP ENTRIES ===\n{entry_names:#?}"); + assert!(entry_names.contains(&"manifest.json".to_string()), "manifest.json entry must be present"); + let sql_part_count = entry_names.iter().filter(|name| name.ends_with(".sql")).count(); + assert!(sql_part_count >= 1, "at least one .sql part expected"); + + let manifest_bytes = { + let mut entry = archive.by_name("manifest.json").expect("manifest.json entry"); + let mut bytes = Vec::new(); + std::io::Read::read_to_end(&mut entry, &mut bytes).expect("read manifest.json"); + bytes + }; + let manifest: serde_json::Value = serde_json::from_slice(&manifest_bytes).expect("parse manifest.json"); + eprintln!("=== MANIFEST ===\n{}", serde_json::to_string_pretty(&manifest).unwrap()); + assert_eq!(manifest["format"], "dbx-sql-export-parts-v1", "manifest format must match the documented contract"); + let manifest_parts = manifest["parts"].as_array().expect("manifest.parts array").len(); + assert_eq!(manifest_parts, sql_part_count, "manifest part count must match actual .sql entries in the zip"); + + // --- Prepare a clean scratch database on the same SQL Server instance for import --- + let (import_state, _import_work_dir) = + live_sqlserver_state(import_connection_id, "master", "import-bootstrap").await; + let bootstrap_pool = import_state + .get_or_create_pool(import_connection_id, Some("master")) + .await + .expect("connect to master to bootstrap scratch database"); + let drop_sql = format!( + "IF DB_ID(N'{import_database}') IS NOT NULL BEGIN ALTER DATABASE [{import_database}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE [{import_database}]; END" + ); + dbx_core::query::execute_sql_statement(&import_state, import_connection_id, "master", &drop_sql, None, None) + .await + .expect("drop pre-existing scratch database if present"); + let create_sql = format!("CREATE DATABASE [{import_database}]"); + dbx_core::query::execute_sql_statement(&import_state, import_connection_id, "master", &create_sql, None, None) + .await + .expect("create scratch database for import"); + let _ = bootstrap_pool; + + // --- Extract the ZIP the same way the app does before feeding SQL files to the importer --- + let extraction_dir = work_dir.join("extracted-parts"); + let package = dbx_core::sql_file_zip_package::extract_sql_file_zip_package(&zip_path, &extraction_dir) + .expect("extract split-ZIP package using the shared sql_file_zip_package contract"); + let extracted_paths = dbx_core::sql_file_zip_package::extracted_sql_zip_paths(&extraction_dir, &package); + eprintln!("=== EXTRACTED PART PATHS ===\n{extracted_paths:#?}"); + assert_eq!(extracted_paths.len(), sql_part_count, "extracted part count must match the zip's .sql entries"); + + let (reimport_state, _reimport_work_dir) = + live_sqlserver_state(import_connection_id, import_database, "import-exec").await; + let import_request = SqlFileRequest { + execution_id: format!("e2e-import-{}", uuid::Uuid::new_v4()), + connection_id: import_connection_id.to_string(), + database: import_database.to_string(), + file_path: extracted_paths[0].to_string_lossy().to_string(), + continue_on_error: false, + txn_session_id: None, + selected_tables: None, + part_cooldown_ms: 0, + // This is the "Skip relationship" option from the task: SQL Server + // constraint bypass runs `ALTER TABLE ... NOCHECK CONSTRAINT ALL` + // before the import and `WITH NOCHECK CHECK CONSTRAINT ALL` after. + skip_relational_constraints: true, + }; + + let file_path_refs: Vec<&std::path::Path> = extracted_paths.iter().map(|path| path.as_path()).collect(); + let import_progress_log = std::sync::Mutex::new(Vec::new()); + let started_at = Instant::now(); + let import_result = execute_sql_file_paths( + &reimport_state, + &import_request, + &file_path_refs, + CancellationToken::new(), + started_at, + |progress| import_progress_log.lock().unwrap().push(format!("{progress:?}")), + ) + .await; + + eprintln!("=== IMPORT PROGRESS LOG ==="); + for line in import_progress_log.lock().unwrap().iter() { + eprintln!("{line}"); + } + eprintln!("=== IMPORT RESULT ===\n{import_result:?}"); + + // Intentionally do not assert success here: the whole point of this manual + // E2E run is to surface whatever real error the split-ZIP + skip-relationship + // path produces (or confirm it succeeds), and report it verbatim, per the + // task's explicit instruction not to fabricate outcomes. + match &import_result { + Ok(()) => eprintln!("IMPORT SUCCEEDED"), + Err(error) => eprintln!("IMPORT FAILED WITH ERROR: {error}"), + } + + // --- Post-import verification: table/row counts on the scratch database --- + let verify_sql = "SELECT COUNT(*) AS tbl_count FROM sys.tables WHERE is_ms_shipped = 0"; + match dbx_core::query::execute_sql_statement( + &reimport_state, + import_connection_id, + import_database, + verify_sql, + None, + None, + ) + .await + { + Ok(result) => eprintln!("=== POST-IMPORT TABLE COUNT ===\n{result:?}"), + Err(error) => eprintln!("=== POST-IMPORT VERIFY QUERY FAILED ===\n{error}"), + } +} diff --git a/crates/dbx-formats/src/export_split_zip.rs b/crates/dbx-formats/src/export_split_zip.rs index 15626b083c..2c73fe1895 100644 --- a/crates/dbx-formats/src/export_split_zip.rs +++ b/crates/dbx-formats/src/export_split_zip.rs @@ -122,6 +122,21 @@ impl SplitZipExportWriter { Ok(()) } + /// Writes one complete SQL export unit into a single part. A unit may be + /// a DDL statement, routine body, INSERT, comment, or client-side batch + /// delimiter such as SQL Server `GO`; it must never be assembled from + /// multiple calls. A unit that fits starts a new part before it would + /// exceed the configured size; an oversized individual unit remains whole + /// as the only unavoidable exception. + pub fn write_sql_unit(&mut self, unit: &[u8]) -> std::io::Result<()> { + if self.current_part_bytes > 0 && self.current_part_bytes + unit.len() as u64 > self.part_max_bytes { + self.start_next_part().map_err(std::io::Error::other)?; + } + self.zip.write_all(unit)?; + self.current_part_bytes += unit.len() as u64; + Ok(()) + } + /// Writes the trailing `manifest.json` entry (listing every part in /// order) and finalizes the zip archive. Consumes the writer because the /// underlying `zip::ZipWriter::finish` does. @@ -148,12 +163,14 @@ impl SplitZipExportWriter { impl Write for SplitZipExportWriter { fn write(&mut self, buffer: &[u8]) -> std::io::Result { - if self.current_part_bytes >= self.part_max_bytes { - self.start_next_part().map_err(std::io::Error::other)?; - } - let written = self.zip.write(buffer)?; - self.current_part_bytes += written as u64; - Ok(written) + self.write_sql_unit(buffer)?; + Ok(buffer.len()) + } + + fn write_fmt(&mut self, fmt: std::fmt::Arguments<'_>) -> std::io::Result<()> { + let mut unit = String::new(); + std::fmt::write(&mut unit, fmt).map_err(|_| std::io::Error::other("Failed to format SQL export unit"))?; + self.write_sql_unit(unit.as_bytes()) } fn flush(&mut self) -> std::io::Result<()> { @@ -240,13 +257,9 @@ mod tests { { let mut writer = SplitZipExportWriter::create(&zip_path, MIN_SPLIT_PART_MAX_MB, "mydb", "sql").unwrap(); - // Each write() call is one full statement; the threshold check - // happens before the call, so a small per-write payload combined - // with a below-1MB threshold still needs enough total bytes to - // exceed 1MB and trigger a second part. Use the internal minimum - // by writing repeatedly. for index in 0..6000 { - writeln!(writer, "{long_statement} -- row {index}").unwrap(); + let line = format!("{long_statement} -- row {index}\n"); + writer.write_all(line.as_bytes()).unwrap(); } writer.finish("mydb.sql").unwrap(); } @@ -272,6 +285,109 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } + #[test] + fn rotates_before_a_complete_statement_that_would_exceed_the_part_limit() { + let dir = std::env::temp_dir().join(format!("dbx-split-zip-test-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&dir).unwrap(); + let zip_path = dir.join("export.zip"); + let statement = format!("INSERT INTO t VALUES ('{}');", "x".repeat(700 * 1024)); + + { + let mut writer = SplitZipExportWriter::create(&zip_path, MIN_SPLIT_PART_MAX_MB, "mydb", "sql").unwrap(); + writer.write_all(statement.as_bytes()).unwrap(); + writer.write_all(b"\n").unwrap(); + writer.write_all(statement.as_bytes()).unwrap(); + writer.finish("mydb.sql").unwrap(); + } + + let entries = read_zip_entries(&zip_path); + let sql_entries: Vec<_> = entries.iter().filter(|(name, _)| name.ends_with(".sql")).collect(); + assert_eq!(sql_entries.len(), 2); + assert!(sql_entries.iter().all(|(_, contents)| contents.len() <= 1024 * 1024)); + assert_eq!(String::from_utf8(sql_entries[0].1.clone()).unwrap(), format!("{statement}\n")); + assert_eq!(String::from_utf8(sql_entries[1].1.clone()).unwrap(), statement); + + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn writeln_with_multiple_format_arguments_stays_one_sql_unit() { + // std::fmt's default Write::write_fmt dispatches one write() call per + // format-string fragment, which would let a rotation slip in between + // "CREATE TABLE " and the column list even though the whole macro + // invocation is meant to emit a single DDL unit. Every real export + // call site formats a whole statement/DDL/routine body with `writeln!` + // (e.g. `writeln!(file, "{source}\n")`, `writeln!(file, "-- Date: + // {timestamp}")`), so the writer overrides `write_fmt` to buffer the + // formatted text and hand it to `write_sql_unit` in one call. + let dir = std::env::temp_dir().join(format!("dbx-split-zip-test-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&dir).unwrap(); + let zip_path = dir.join("export.zip"); + let table_name = "orders"; + let insert = "INSERT INTO orders VALUES (1);"; + // part_max_bytes for MIN_SPLIT_PART_MAX_MB is 1 MiB. Size the column + // list so the CREATE TABLE unit alone fits (leaving less headroom + // than the INSERT needs), forcing the rotation to land *between* the + // two units instead of never triggering or landing mid-DDL. + let overhead = "CREATE TABLE ".len() + table_name.len() + " (".len() + ");".len() + "\n".len(); + let columns = "x".repeat(1024 * 1024 - overhead - 10); + let create_table = format!("CREATE TABLE {table_name} ({columns});"); + + { + let mut writer = SplitZipExportWriter::create(&zip_path, MIN_SPLIT_PART_MAX_MB, "mydb", "sql").unwrap(); + writeln!(writer, "CREATE TABLE {table_name} ({columns});").unwrap(); + writeln!(writer, "{insert}").unwrap(); + writer.finish("mydb.sql").unwrap(); + } + + let entries = read_zip_entries(&zip_path); + let mut sql_entries: Vec<_> = entries.iter().filter(|(name, _)| name.ends_with(".sql")).collect(); + sql_entries.sort_by(|a, b| a.0.cmp(&b.0)); + + assert_eq!(sql_entries.len(), 2, "the oversized CREATE TABLE must rotate before the INSERT, not mid-DDL"); + let first = String::from_utf8(sql_entries[0].1.clone()).unwrap(); + let second = String::from_utf8(sql_entries[1].1.clone()).unwrap(); + assert_eq!(first, format!("{create_table}\n"), "the whole CREATE TABLE must land in one part intact"); + assert_eq!(second, format!("{insert}\n")); + + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn keeps_an_oversized_individual_statement_whole_as_the_only_limit_exception() { + let dir = std::env::temp_dir().join(format!("dbx-split-zip-test-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&dir).unwrap(); + let zip_path = dir.join("export.zip"); + + let mut seed = 0x1234_5678_u32; + let huge_payload = (0..3 * 1024 * 1024) + .map(|_| { + seed = seed.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + char::from(b'a' + (seed >> 24) as u8 % 26) + }) + .collect::(); + let statement = format!("INSERT INTO t VALUES ('{huge_payload}');\n"); + + { + let mut writer = SplitZipExportWriter::create(&zip_path, MIN_SPLIT_PART_MAX_MB, "mydb", "sql").unwrap(); + writer.write_all(statement.as_bytes()).unwrap(); + writer.finish("mydb.sql").unwrap(); + } + + let entries = read_zip_entries(&zip_path); + let mut sql_entries: Vec<_> = entries.iter().filter(|(name, _)| name.ends_with(".sql")).collect(); + sql_entries.sort_by(|a, b| a.0.cmp(&b.0)); + + assert_eq!(sql_entries.len(), 1, "a single write must stay in one SQL part"); + assert_eq!(String::from_utf8(sql_entries[0].1.clone()).unwrap(), statement); + + let manifest_entry = entries.iter().find(|(name, _)| name == "manifest.json").unwrap(); + let manifest: serde_json::Value = serde_json::from_slice(&manifest_entry.1).unwrap(); + assert_eq!(manifest["totalParts"].as_u64().unwrap() as usize, sql_entries.len()); + + let _ = std::fs::remove_dir_all(&dir); + } + #[test] fn sanitizes_zip_entry_stem_from_untrusted_names() { assert_eq!(sanitize_zip_entry_component("../../etc/passwd"), ".._.._etc_passwd");