fix(sql-file): preserve statements across split ZIP parts

This commit is contained in:
КГБ Lieu authored and GitHub committed 2026-09-28 22:57:39 +08:00
1 parent 30b351c079
commit 2384a2e2c2
5 files changed
+596 -53

No files matched your search

+97 -7
View File
@@ -15,6 +15,7 @@ use crate::connection::task_client_session_id;
use crate::models::connection::DatabaseType;
use crate::mysql_ddl_normalize::DdlNormalizeOptions;
use crate::object_source_sql::build_export_object_source_sql;
use crate::sql::{SqlParsingOptions, SqlStatementSplitter};
use crate::sql_dialect::{qualified_table_name, uses_single_row_insert_statements};
use crate::transfer::{
format_ch_array_sql_literal, format_pg_array_sql_literal, format_postgres_vector_sql_literal,
@@ -166,6 +167,16 @@ enum DatabaseExportWriter {
SplitZip(Box<crate::export_split_zip::SplitZipExportWriter>),
}
impl DatabaseExportWriter {
fn write_sql_unit(&mut self, unit: &[u8]) -> std::io::Result<()> {
match self {
Self::Plain(writer) => writer.write_all(unit),
Self::Gzip(writer) => writer.write_all(unit),
Self::SplitZip(writer) => writer.write_sql_unit(unit),
}
}
}
impl Write for DatabaseExportWriter {
fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
match self {
@@ -175,6 +186,12 @@ impl Write for DatabaseExportWriter {
}
}
fn write_fmt(&mut self, fmt: std::fmt::Arguments<'_>) -> std::io::Result<()> {
let mut unit = String::new();
std::fmt::write(&mut unit, fmt).map_err(|_| std::io::Error::other("Failed to format SQL export unit"))?;
self.write_sql_unit(unit.as_bytes())
}
fn flush(&mut self) -> std::io::Result<()> {
match self {
Self::Plain(writer) => writer.flush(),
@@ -3221,17 +3238,21 @@ fn postgres_create_schema_sql(schema: &str) -> String {
format!("CREATE SCHEMA IF NOT EXISTS {};", quote_identifier(schema, &DatabaseType::Postgres))
}
// Copy one line at a time so a `SplitZipExportWriter` can only rotate between
// complete SQL statements; `std::io::copy` would feed it arbitrary 8KB chunks.
fn combine_schema_sql_export<W: Write>(source: &mut dyn BufRead, destination: &mut W) -> std::io::Result<()> {
let mut line = Vec::new();
let mut splitter = SqlStatementSplitter::with_options(SqlParsingOptions::for_database_type(DatabaseType::Postgres));
let mut chunk = String::new();
loop {
line.clear();
let bytes_read = source.read_until(b'\n', &mut line)?;
chunk.clear();
let bytes_read = source.read_line(&mut chunk)?;
if bytes_read == 0 {
break;
}
destination.write_all(&line)?;
for statement in splitter.push_chunk(&chunk) {
destination.write_all(format!("{statement};\n").as_bytes())?;
}
}
for statement in splitter.finish() {
destination.write_all(format!("{statement};\n").as_bytes())?;
}
Ok(())
}
@@ -6706,6 +6727,75 @@ mod tests {
assert_eq!(combined.matches("INSERT INTO").count(), 20);
}
#[test]
fn split_zip_export_writer_keeps_an_oversized_create_table_ddl_whole_before_the_next_unit() {
// Exercises the "SQL unit, not just INSERT" contract: a CREATE TABLE
// DDL statement, an INSERT, and a SQL Server-style multi-statement
// routine body must each land in exactly one part, never split at an
// arbitrary byte offset inside one, even when writeln! formats the
// unit from several interpolated arguments (real call sites do this,
// e.g. `writeln!(file, "{source}\n")` for exported routine bodies).
let directory = tempfile::tempdir().unwrap();
let zip_path = directory.path().join("schema.zip");
let mut writer = crate::export_split_zip::SplitZipExportWriter::create(
&zip_path,
crate::export_split_zip::MIN_SPLIT_PART_MAX_MB,
"schema",
"sql",
)
.unwrap();
let table_name = "dbo.orders";
// part_max_bytes for MIN_SPLIT_PART_MAX_MB is 1 MiB. Make the CREATE
// TABLE unit alone exceed that -- it stays whole as the oversized-unit
// exception, but the *next* unit must then rotate into a new part.
let columns = (0..100_000).map(|index| format!("col_{index} INT")).collect::<Vec<_>>().join(", ");
let create_table_ddl = format!("CREATE TABLE {table_name} ({columns});");
// A SQL Server routine body: multiple statements inside one BEGIN/END
// block plus a batch-separator comment, matching what
// build_export_object_source_sql emits for a procedure/trigger --
// the writer must never cut between the inner statements.
let procedure_source = "CREATE PROCEDURE dbo.recalc_totals AS\nBEGIN\n UPDATE dbo.orders SET total = total + 1;\n INSERT INTO dbo.audit_log (message) VALUES ('recalculated');\nEND".to_string();
let go_batch_separator = "GO";
let insert = "INSERT INTO dbo.orders (id) VALUES (1);".to_string();
writeln!(writer, "{create_table_ddl}").unwrap();
writeln!(writer, "{procedure_source}").unwrap();
writeln!(writer, "{go_batch_separator}").unwrap();
writeln!(writer, "{insert}").unwrap();
writer.finish("schema.sql").unwrap();
let file = std::fs::File::open(&zip_path).unwrap();
let mut archive = zip::ZipArchive::new(file).unwrap();
let mut sql_parts = Vec::new();
for index in 0..archive.len() {
let mut entry = archive.by_index(index).unwrap();
if !entry.name().ends_with(".sql") {
continue;
}
let mut contents = String::new();
entry.read_to_string(&mut contents).unwrap();
sql_parts.push((entry.name().to_string(), contents));
}
sql_parts.sort_by(|a, b| a.0.cmp(&b.0));
assert!(sql_parts.len() >= 2, "the oversized CREATE TABLE alone must rotate before later units");
let combined: String = sql_parts.iter().map(|(_, contents)| contents.as_str()).collect();
assert_eq!(
combined,
format!("{create_table_ddl}\n{procedure_source}\n{go_batch_separator}\n{insert}\n"),
"reassembling every part must reproduce the original stream byte for byte"
);
// Every unit must be found intact inside exactly one part -- proof
// that none of them was fragmented across a rotation boundary.
for unit in [create_table_ddl.as_str(), procedure_source.as_str(), go_batch_separator, insert.as_str()] {
let containing_parts = sql_parts.iter().filter(|(_, contents)| contents.contains(unit)).count();
assert_eq!(containing_parts, 1, "unit must appear whole in exactly one part: {unit:?}");
}
}
#[test]
fn all_schemas_combine_keeps_split_part_boundaries_statement_safe() {
// Mirror of the per-schema temporary files that
@@ -6758,7 +6848,7 @@ mod tests {
for line in contents.lines().filter(|line| !line.trim().is_empty()) {
assert!(
line.trim_start().starts_with("INSERT INTO") && line.trim_end().ends_with(';'),
"{name} has a malformed line from a mid-statement cut"
"{name} has a malformed line from a mid-statement cut: {line:?}"
);
}
}
+39 -18
View File
@@ -856,6 +856,8 @@ async fn execute_sql_file_paths_inner(
let mut prev_success_count = 0usize;
let mut prev_failure_count = 0usize;
let mut prev_affected_rows = 0u64;
let mut splitter = Some(StreamingSqlFileSplitter::new(database_type, options));
let mut pending_statements = Vec::with_capacity(SQL_FILE_STATEMENT_BATCH_SIZE);
let import_result = async {
for (file_index, file_path) in file_paths.iter().enumerate() {
let file_name = file_path.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
@@ -881,8 +883,6 @@ async fn execute_sql_file_paths_inner(
});
}
let mut splitter = StreamingSqlFileSplitter::new(database_type, options);
let mut pending_statements = Vec::with_capacity(SQL_FILE_STATEMENT_BATCH_SIZE);
let normalize_mysql_binary_literals = import_target.as_ref().is_some_and(|target| {
crate::sql::is_mysql_compatible_import_target(&target.db_type, target.driver_profile.as_deref())
});
@@ -941,7 +941,10 @@ async fn execute_sql_file_paths_inner(
"",
None,
));
let mut next_statements = splitter.push_chunk(&chunk);
let mut next_statements = splitter
.as_mut()
.expect("splitter is only taken by finish() on the last file, after this loop exits")
.push_chunk(&chunk);
if let Some(filter) = restore_filter.as_mut() {
if let Err(error) = filter_restore_statements(
&mut next_statements,
@@ -979,23 +982,27 @@ async fn execute_sql_file_paths_inner(
.await?;
}
let mut next_statements = splitter.finish();
if let Some(filter) = restore_filter.as_mut() {
if let Err(error) = filter_restore_statements(
&mut next_statements,
filter,
request.selected_tables.as_deref().unwrap_or_default(),
) {
emit(sql_file_execution_error_progress(
&request.execution_id,
started_at,
&progress,
error.clone(),
));
return Err(error);
let is_last_file = file_index + 1 == file_count;
if is_last_file {
let mut next_statements =
splitter.take().expect("splitter is only taken once, on the last file").finish();
if let Some(filter) = restore_filter.as_mut() {
if let Err(error) = filter_restore_statements(
&mut next_statements,
filter,
request.selected_tables.as_deref().unwrap_or_default(),
) {
emit(sql_file_execution_error_progress(
&request.execution_id,
started_at,
&progress,
error.clone(),
));
return Err(error);
}
}
pending_statements.extend(next_statements);
}
pending_statements.extend(next_statements);
execute_sql_file_statement_batch(
state,
request,
@@ -3183,6 +3190,20 @@ mod tests {
assert_eq!(statements[1], "SELECT 3;");
}
#[test]
fn streaming_sqlserver_splitter_keeps_split_json_literal_across_parts() {
let part_one = "INSERT INTO [dbo].[VersionValue] ([value]) VALUES (N'{\"materialOrSymbolMate";
let part_two = "rialId\":\"9625a891-3682-4151-acdb-6480db860033\",\"label\":\"B\\\\u1ebb ch\\\\u00e2n/Th\\\\u1eb3ng/\",\"quote\":\"it''s valid\"}');\n";
let expected = format!("{part_one}{part_two}").trim().to_string();
let mut splitter = StreamingSqlFileSplitter::new(Some(DatabaseType::SqlServer), SqlParsingOptions::default());
assert!(splitter.push_chunk(part_one).is_empty());
assert!(splitter.push_chunk(part_two).is_empty());
let statements = splitter.finish().into_iter().map(|statement| statement.sql).collect::<Vec<_>>();
assert_eq!(statements, vec![expected]);
}
#[test]
fn streaming_sqlserver_splitter_handles_go_across_chunks() {
let mut splitter = SqlServerBatchSplitter::default();
+65 -16
View File
@@ -1068,6 +1068,15 @@ enum TableExportSqlWriter {
SplitZip(Box<crate::export_split_zip::SplitZipExportWriter>),
}
impl TableExportSqlWriter {
fn write_sql_unit(&mut self, unit: &[u8]) -> std::io::Result<()> {
match self {
Self::Plain(writer) => writer.write_all(unit),
Self::SplitZip(writer) => writer.write_sql_unit(unit),
}
}
}
impl Write for TableExportSqlWriter {
fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
match self {
@@ -1076,6 +1085,12 @@ impl Write for TableExportSqlWriter {
}
}
fn write_fmt(&mut self, fmt: std::fmt::Arguments<'_>) -> std::io::Result<()> {
let mut unit = String::new();
std::fmt::write(&mut unit, fmt).map_err(|_| std::io::Error::other("Failed to format SQL export unit"))?;
self.write_sql_unit(unit.as_bytes())
}
fn flush(&mut self) -> std::io::Result<()> {
match self {
Self::Plain(writer) => writer.flush(),
@@ -1093,6 +1108,21 @@ impl TableExportSqlWriter {
}
}
fn write_sql_export_statements(
file: &mut impl Write,
statements: Vec<String>,
wrote_statements: &mut bool,
) -> Result<(), String> {
for statement in statements {
if *wrote_statements {
file.write_all(b"\n").map_err(|e| format!("Failed to write SQL: {e}"))?;
}
file.write_all(statement.as_bytes()).map_err(|e| format!("Failed to write SQL: {e}"))?;
*wrote_statements = true;
}
Ok(())
}
fn create_table_export_sql_writer(request: &TableExportRequest) -> Result<TableExportSqlWriter, String> {
if let Some(max_mb) = request.split_max_mb {
let zip_path = std::path::Path::new(&request.file_path);
@@ -1423,14 +1453,7 @@ async fn try_export_native_table_stream(
&sql_export_excluded_columns(request, primary_keys, col_names, column_extras),
request.insert_dialect,
)?;
if !statements.is_empty() {
if wrote_statements {
file.write_all(b"\n").map_err(|e| format!("Failed to write SQL: {e}"))?;
}
file.write_all(statements.join("\n").as_bytes())
.map_err(|e| format!("Failed to write SQL: {e}"))?;
wrote_statements = true;
}
write_sql_export_statements(file, statements, &mut wrote_statements)?;
Ok(())
};
let result = stream_native_table_rows(
@@ -2289,14 +2312,7 @@ async fn export_table_data_core_inner(
&sql_export_excluded_columns(request, &primary_keys, &col_names, &column_extras),
request.insert_dialect,
)?;
if !statements.is_empty() {
if wrote_statements {
file.write_all(b"\n").map_err(|e| format!("Failed to write SQL: {e}"))?;
}
file.write_all(statements.join("\n").as_bytes())
.map_err(|e| format!("Failed to write SQL: {e}"))?;
wrote_statements = true;
}
write_sql_export_statements(&mut file, statements, &mut wrote_statements)?;
rows_exported += row_count as u64;
if use_keyset {
@@ -2361,6 +2377,39 @@ mod tests {
#[cfg(unix)]
use std::os::unix::fs::PermissionsExt;
#[test]
fn split_zip_sql_export_writes_batch_statements_separately() {
let dir = tempfile::tempdir().unwrap();
let zip_path = dir.path().join("export.zip");
let statement = format!("INSERT INTO t VALUES ('{}');", "x".repeat(700 * 1024));
let mut writer = crate::export_split_zip::SplitZipExportWriter::create(
&zip_path,
crate::export_split_zip::MIN_SPLIT_PART_MAX_MB,
"table",
"sql",
)
.unwrap();
let mut wrote_statements = false;
write_sql_export_statements(&mut writer, vec![statement.clone(), statement.clone()], &mut wrote_statements)
.unwrap();
writer.finish("table.sql").unwrap();
let mut archive = zip::ZipArchive::new(std::fs::File::open(zip_path).unwrap()).unwrap();
let mut sql_parts = Vec::new();
for index in 0..archive.len() {
let mut entry = archive.by_index(index).unwrap();
if entry.name().ends_with(".sql") {
let mut contents = String::new();
entry.read_to_string(&mut contents).unwrap();
sql_parts.push(contents);
}
}
assert_eq!(sql_parts, vec![format!("{statement}\n"), statement]);
assert!(sql_parts.iter().all(|part| part.len() <= 1024 * 1024));
}
#[test]
fn table_export_request_defaults_to_source_dialect_and_batch_insert_mode() {
let mut payload = json!({
@@ -0,0 +1,267 @@
// E2E test executed manually against a real local SQL Server 2025 container
// (`database-mssql-1`, database `default`) already loaded with a production-shaped
// dataset (50 tables / 52 foreign keys), as part of task #01a0db6e-2276.
//
// Flow: export database `default` (with `split_max_mb` set, i.e. the split-ZIP
// contract under review) -> verify manifest/parts -> re-import the resulting ZIP
// via `execute_sql_file_paths` with `skip_relational_constraints = true` ("Skip
// relationship") into a fresh scratch database, exactly like the desktop app does.
//
// Run manually with:
// DBX_LIVE_SQLSERVER_PASSWORD='MSSQLStrongPassword123!' \
// cargo test -p dbx-core --test live_sqlserver_split_zip_e2e -- --nocapture --test-threads=1
use dbx_core::connection::AppState;
use dbx_core::database_export::{export_database_sql_core, DatabaseExportOutputCompression, DatabaseExportRequest};
use dbx_core::models::connection::DatabaseType;
use dbx_core::sql::SqlFileRequest;
use dbx_core::sql_file_import::execute_sql_file_paths;
use std::sync::Arc;
use std::time::Instant;
use tokio_util::sync::CancellationToken;
fn live_sqlserver_config(id: &str, database: &str) -> dbx_core::models::connection::ConnectionConfig {
dbx_core::models::connection::ConnectionConfig {
docs_notes_path: None,
id: id.to_string(),
name: id.to_string(),
note: String::new(),
db_type: DatabaseType::SqlServer,
driver_profile: None,
driver_label: None,
url_params: None,
agent_java_options: Vec::new(),
host: std::env::var("DBX_LIVE_SQLSERVER_HOST").unwrap_or_else(|_| "127.0.0.1".to_string()),
port: std::env::var("DBX_LIVE_SQLSERVER_PORT").ok().and_then(|value| value.parse().ok()).unwrap_or(1433),
username: std::env::var("DBX_LIVE_SQLSERVER_USER").unwrap_or_else(|_| "sa".to_string()),
password: std::env::var("DBX_LIVE_SQLSERVER_PASSWORD").expect("DBX_LIVE_SQLSERVER_PASSWORD"),
database: Some(database.to_string()),
default_schema: None,
visible_databases: None,
visible_database_patterns: None,
visible_schemas: None,
attached_databases: Vec::new(),
init_script: None,
color: None,
transport_layers: Vec::new(),
connect_timeout_secs: 10,
query_timeout_secs: 30,
idle_timeout_secs: 60,
keepalive_interval_secs: 0,
ssl: false,
ca_cert_path: String::new(),
client_cert_path: String::new(),
client_key_path: String::new(),
sysdba: false,
oracle_connection_type: None,
connection_string: None,
redis_connection_mode: None,
redis_sentinel_master: String::new(),
redis_sentinel_nodes: String::new(),
redis_sentinel_username: String::new(),
redis_sentinel_password: String::new(),
redis_sentinel_tls: false,
redis_cluster_nodes: String::new(),
redis_key_separator: dbx_core::models::connection::default_redis_key_separator(),
redis_scan_page_size: None,
redis_database_aliases: Default::default(),
redis_key_templates: Vec::new(),
redis_key_grouping: None,
etcd_endpoints: String::new(),
gbase_server: String::new(),
informix_server: String::new(),
external_config: None,
plugin_id: None,
plugin_connection_provider: None,
plugin_connection_type: None,
connection_secrets: Default::default(),
jdbc_driver_class: None,
jdbc_driver_paths: Vec::new(),
one_time: false,
save_password: true,
read_only: false,
is_production: false,
production_databases: vec![],
show_system_schemas: false,
database_info: None,
sidebar_auto_load_all_tables: false,
}
}
async fn live_sqlserver_state(connection_id: &str, database: &str, suffix: &str) -> (AppState, std::path::PathBuf) {
let dir = std::env::temp_dir().join(format!("dbx-live-sqlserver-splitzip-{suffix}"));
std::fs::create_dir_all(&dir).expect("create live test directory");
let storage =
dbx_core::persistence::test_storage::open(&dir.join("storage.db")).await.expect("open live test storage");
let state = AppState::new(storage);
let config = live_sqlserver_config(connection_id, database);
state.configs.write().await.insert(connection_id.to_string(), config);
(state, dir)
}
#[tokio::test]
#[ignore = "manual E2E against a real local SQL Server instance -- see file header"]
async fn manual_e2e_sqlserver_default_split_zip_export_then_import_skip_relationship() {
let export_connection_id = "e2e-mssql-export";
let import_connection_id = "e2e-mssql-import";
let source_database = "default";
// Import target is a separate scratch database, NOT the just-emptied
// `default`, per the task's own guidance to isolate the destructive
// re-import from the production-shaped database whenever possible.
let import_database = "dbx_e2e_split_zip_restore";
let (export_state, work_dir) = live_sqlserver_state(export_connection_id, source_database, "export").await;
let export_state = Arc::new(export_state);
let zip_path = work_dir.join("default-export.zip");
let export_request = DatabaseExportRequest {
export_id: format!("e2e-export-{}", uuid::Uuid::new_v4()),
connection_id: export_connection_id.to_string(),
database: source_database.to_string(),
schema: String::new(),
file_path: zip_path.to_string_lossy().to_string(),
selected_tables: Vec::new(),
excluded_tables: Vec::new(),
include_structure: true,
include_data: true,
include_objects: true,
include_create_database: false,
drop_table_if_exists: true,
omit_auto_increment: false,
fail_on_error: true,
prevent_overwrite: false,
output_compression: DatabaseExportOutputCompression::None,
snapshot_session_id: None,
batch_size: 1000,
split_max_mb: Some(1),
insert_dialect: Default::default(),
};
let progress_log = std::sync::Mutex::new(Vec::new());
let export_result = export_database_sql_core(&export_state, &export_request, |progress| {
progress_log.lock().unwrap().push(format!("{progress:?}"));
})
.await;
eprintln!("=== EXPORT PROGRESS LOG ===");
for line in progress_log.lock().unwrap().iter() {
eprintln!("{line}");
}
eprintln!("=== EXPORT RESULT ===\n{export_result:?}");
export_result.expect("split-ZIP database export should succeed");
assert!(zip_path.exists(), "expected export ZIP at {}", zip_path.display());
let zip_metadata = std::fs::metadata(&zip_path).expect("stat export zip");
eprintln!("Export ZIP size: {} bytes", zip_metadata.len());
// --- Verify manifest.json + parts against the dbx-sql-export-parts-v1 contract ---
let file = std::fs::File::open(&zip_path).expect("open export zip");
let mut archive = zip::ZipArchive::new(file).expect("read export zip as a zip archive");
let mut entry_names: Vec<String> =
(0..archive.len()).map(|index| archive.by_index(index).expect("zip entry").name().to_string()).collect();
entry_names.sort();
eprintln!("=== ZIP ENTRIES ===\n{entry_names:#?}");
assert!(entry_names.contains(&"manifest.json".to_string()), "manifest.json entry must be present");
let sql_part_count = entry_names.iter().filter(|name| name.ends_with(".sql")).count();
assert!(sql_part_count >= 1, "at least one .sql part expected");
let manifest_bytes = {
let mut entry = archive.by_name("manifest.json").expect("manifest.json entry");
let mut bytes = Vec::new();
std::io::Read::read_to_end(&mut entry, &mut bytes).expect("read manifest.json");
bytes
};
let manifest: serde_json::Value = serde_json::from_slice(&manifest_bytes).expect("parse manifest.json");
eprintln!("=== MANIFEST ===\n{}", serde_json::to_string_pretty(&manifest).unwrap());
assert_eq!(manifest["format"], "dbx-sql-export-parts-v1", "manifest format must match the documented contract");
let manifest_parts = manifest["parts"].as_array().expect("manifest.parts array").len();
assert_eq!(manifest_parts, sql_part_count, "manifest part count must match actual .sql entries in the zip");
// --- Prepare a clean scratch database on the same SQL Server instance for import ---
let (import_state, _import_work_dir) =
live_sqlserver_state(import_connection_id, "master", "import-bootstrap").await;
let bootstrap_pool = import_state
.get_or_create_pool(import_connection_id, Some("master"))
.await
.expect("connect to master to bootstrap scratch database");
let drop_sql = format!(
"IF DB_ID(N'{import_database}') IS NOT NULL BEGIN ALTER DATABASE [{import_database}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE [{import_database}]; END"
);
dbx_core::query::execute_sql_statement(&import_state, import_connection_id, "master", &drop_sql, None, None)
.await
.expect("drop pre-existing scratch database if present");
let create_sql = format!("CREATE DATABASE [{import_database}]");
dbx_core::query::execute_sql_statement(&import_state, import_connection_id, "master", &create_sql, None, None)
.await
.expect("create scratch database for import");
let _ = bootstrap_pool;
// --- Extract the ZIP the same way the app does before feeding SQL files to the importer ---
let extraction_dir = work_dir.join("extracted-parts");
let package = dbx_core::sql_file_zip_package::extract_sql_file_zip_package(&zip_path, &extraction_dir)
.expect("extract split-ZIP package using the shared sql_file_zip_package contract");
let extracted_paths = dbx_core::sql_file_zip_package::extracted_sql_zip_paths(&extraction_dir, &package);
eprintln!("=== EXTRACTED PART PATHS ===\n{extracted_paths:#?}");
assert_eq!(extracted_paths.len(), sql_part_count, "extracted part count must match the zip's .sql entries");
let (reimport_state, _reimport_work_dir) =
live_sqlserver_state(import_connection_id, import_database, "import-exec").await;
let import_request = SqlFileRequest {
execution_id: format!("e2e-import-{}", uuid::Uuid::new_v4()),
connection_id: import_connection_id.to_string(),
database: import_database.to_string(),
file_path: extracted_paths[0].to_string_lossy().to_string(),
continue_on_error: false,
txn_session_id: None,
selected_tables: None,
part_cooldown_ms: 0,
// This is the "Skip relationship" option from the task: SQL Server
// constraint bypass runs `ALTER TABLE ... NOCHECK CONSTRAINT ALL`
// before the import and `WITH NOCHECK CHECK CONSTRAINT ALL` after.
skip_relational_constraints: true,
};
let file_path_refs: Vec<&std::path::Path> = extracted_paths.iter().map(|path| path.as_path()).collect();
let import_progress_log = std::sync::Mutex::new(Vec::new());
let started_at = Instant::now();
let import_result = execute_sql_file_paths(
&reimport_state,
&import_request,
&file_path_refs,
CancellationToken::new(),
started_at,
|progress| import_progress_log.lock().unwrap().push(format!("{progress:?}")),
)
.await;
eprintln!("=== IMPORT PROGRESS LOG ===");
for line in import_progress_log.lock().unwrap().iter() {
eprintln!("{line}");
}
eprintln!("=== IMPORT RESULT ===\n{import_result:?}");
// Intentionally do not assert success here: the whole point of this manual
// E2E run is to surface whatever real error the split-ZIP + skip-relationship
// path produces (or confirm it succeeds), and report it verbatim, per the
// task's explicit instruction not to fabricate outcomes.
match &import_result {
Ok(()) => eprintln!("IMPORT SUCCEEDED"),
Err(error) => eprintln!("IMPORT FAILED WITH ERROR: {error}"),
}
// --- Post-import verification: table/row counts on the scratch database ---
let verify_sql = "SELECT COUNT(*) AS tbl_count FROM sys.tables WHERE is_ms_shipped = 0";
match dbx_core::query::execute_sql_statement(
&reimport_state,
import_connection_id,
import_database,
verify_sql,
None,
None,
)
.await
{
Ok(result) => eprintln!("=== POST-IMPORT TABLE COUNT ===\n{result:?}"),
Err(error) => eprintln!("=== POST-IMPORT VERIFY QUERY FAILED ===\n{error}"),
}
}
+128 -12
View File
@@ -122,6 +122,21 @@ impl SplitZipExportWriter {
Ok(())
}
/// Writes one complete SQL export unit into a single part. A unit may be
/// a DDL statement, routine body, INSERT, comment, or client-side batch
/// delimiter such as SQL Server `GO`; it must never be assembled from
/// multiple calls. A unit that fits starts a new part before it would
/// exceed the configured size; an oversized individual unit remains whole
/// as the only unavoidable exception.
pub fn write_sql_unit(&mut self, unit: &[u8]) -> std::io::Result<()> {
if self.current_part_bytes > 0 && self.current_part_bytes + unit.len() as u64 > self.part_max_bytes {
self.start_next_part().map_err(std::io::Error::other)?;
}
self.zip.write_all(unit)?;
self.current_part_bytes += unit.len() as u64;
Ok(())
}
/// Writes the trailing `manifest.json` entry (listing every part in
/// order) and finalizes the zip archive. Consumes the writer because the
/// underlying `zip::ZipWriter::finish` does.
@@ -148,12 +163,14 @@ impl SplitZipExportWriter {
impl Write for SplitZipExportWriter {
fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
if self.current_part_bytes >= self.part_max_bytes {
self.start_next_part().map_err(std::io::Error::other)?;
}
let written = self.zip.write(buffer)?;
self.current_part_bytes += written as u64;
Ok(written)
self.write_sql_unit(buffer)?;
Ok(buffer.len())
}
fn write_fmt(&mut self, fmt: std::fmt::Arguments<'_>) -> std::io::Result<()> {
let mut unit = String::new();
std::fmt::write(&mut unit, fmt).map_err(|_| std::io::Error::other("Failed to format SQL export unit"))?;
self.write_sql_unit(unit.as_bytes())
}
fn flush(&mut self) -> std::io::Result<()> {
@@ -240,13 +257,9 @@ mod tests {
{
let mut writer = SplitZipExportWriter::create(&zip_path, MIN_SPLIT_PART_MAX_MB, "mydb", "sql").unwrap();
// Each write() call is one full statement; the threshold check
// happens before the call, so a small per-write payload combined
// with a below-1MB threshold still needs enough total bytes to
// exceed 1MB and trigger a second part. Use the internal minimum
// by writing repeatedly.
for index in 0..6000 {
writeln!(writer, "{long_statement} -- row {index}").unwrap();
let line = format!("{long_statement} -- row {index}\n");
writer.write_all(line.as_bytes()).unwrap();
}
writer.finish("mydb.sql").unwrap();
}
@@ -272,6 +285,109 @@ mod tests {
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn rotates_before_a_complete_statement_that_would_exceed_the_part_limit() {
let dir = std::env::temp_dir().join(format!("dbx-split-zip-test-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
let zip_path = dir.join("export.zip");
let statement = format!("INSERT INTO t VALUES ('{}');", "x".repeat(700 * 1024));
{
let mut writer = SplitZipExportWriter::create(&zip_path, MIN_SPLIT_PART_MAX_MB, "mydb", "sql").unwrap();
writer.write_all(statement.as_bytes()).unwrap();
writer.write_all(b"\n").unwrap();
writer.write_all(statement.as_bytes()).unwrap();
writer.finish("mydb.sql").unwrap();
}
let entries = read_zip_entries(&zip_path);
let sql_entries: Vec<_> = entries.iter().filter(|(name, _)| name.ends_with(".sql")).collect();
assert_eq!(sql_entries.len(), 2);
assert!(sql_entries.iter().all(|(_, contents)| contents.len() <= 1024 * 1024));
assert_eq!(String::from_utf8(sql_entries[0].1.clone()).unwrap(), format!("{statement}\n"));
assert_eq!(String::from_utf8(sql_entries[1].1.clone()).unwrap(), statement);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn writeln_with_multiple_format_arguments_stays_one_sql_unit() {
// std::fmt's default Write::write_fmt dispatches one write() call per
// format-string fragment, which would let a rotation slip in between
// "CREATE TABLE " and the column list even though the whole macro
// invocation is meant to emit a single DDL unit. Every real export
// call site formats a whole statement/DDL/routine body with `writeln!`
// (e.g. `writeln!(file, "{source}\n")`, `writeln!(file, "-- Date:
// {timestamp}")`), so the writer overrides `write_fmt` to buffer the
// formatted text and hand it to `write_sql_unit` in one call.
let dir = std::env::temp_dir().join(format!("dbx-split-zip-test-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
let zip_path = dir.join("export.zip");
let table_name = "orders";
let insert = "INSERT INTO orders VALUES (1);";
// part_max_bytes for MIN_SPLIT_PART_MAX_MB is 1 MiB. Size the column
// list so the CREATE TABLE unit alone fits (leaving less headroom
// than the INSERT needs), forcing the rotation to land *between* the
// two units instead of never triggering or landing mid-DDL.
let overhead = "CREATE TABLE ".len() + table_name.len() + " (".len() + ");".len() + "\n".len();
let columns = "x".repeat(1024 * 1024 - overhead - 10);
let create_table = format!("CREATE TABLE {table_name} ({columns});");
{
let mut writer = SplitZipExportWriter::create(&zip_path, MIN_SPLIT_PART_MAX_MB, "mydb", "sql").unwrap();
writeln!(writer, "CREATE TABLE {table_name} ({columns});").unwrap();
writeln!(writer, "{insert}").unwrap();
writer.finish("mydb.sql").unwrap();
}
let entries = read_zip_entries(&zip_path);
let mut sql_entries: Vec<_> = entries.iter().filter(|(name, _)| name.ends_with(".sql")).collect();
sql_entries.sort_by(|a, b| a.0.cmp(&b.0));
assert_eq!(sql_entries.len(), 2, "the oversized CREATE TABLE must rotate before the INSERT, not mid-DDL");
let first = String::from_utf8(sql_entries[0].1.clone()).unwrap();
let second = String::from_utf8(sql_entries[1].1.clone()).unwrap();
assert_eq!(first, format!("{create_table}\n"), "the whole CREATE TABLE must land in one part intact");
assert_eq!(second, format!("{insert}\n"));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn keeps_an_oversized_individual_statement_whole_as_the_only_limit_exception() {
let dir = std::env::temp_dir().join(format!("dbx-split-zip-test-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
let zip_path = dir.join("export.zip");
let mut seed = 0x1234_5678_u32;
let huge_payload = (0..3 * 1024 * 1024)
.map(|_| {
seed = seed.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
char::from(b'a' + (seed >> 24) as u8 % 26)
})
.collect::<String>();
let statement = format!("INSERT INTO t VALUES ('{huge_payload}');\n");
{
let mut writer = SplitZipExportWriter::create(&zip_path, MIN_SPLIT_PART_MAX_MB, "mydb", "sql").unwrap();
writer.write_all(statement.as_bytes()).unwrap();
writer.finish("mydb.sql").unwrap();
}
let entries = read_zip_entries(&zip_path);
let mut sql_entries: Vec<_> = entries.iter().filter(|(name, _)| name.ends_with(".sql")).collect();
sql_entries.sort_by(|a, b| a.0.cmp(&b.0));
assert_eq!(sql_entries.len(), 1, "a single write must stay in one SQL part");
assert_eq!(String::from_utf8(sql_entries[0].1.clone()).unwrap(), statement);
let manifest_entry = entries.iter().find(|(name, _)| name == "manifest.json").unwrap();
let manifest: serde_json::Value = serde_json::from_slice(&manifest_entry.1).unwrap();
assert_eq!(manifest["totalParts"].as_u64().unwrap() as usize, sql_entries.len());
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn sanitizes_zip_entry_stem_from_untrusted_names() {
assert_eq!(sanitize_zip_entry_component("../../etc/passwd"), ".._.._etc_passwd");