fix(sql-file): preserve statements across split ZIP parts
This commit is contained in:
1 parent
30b351c079
commit
2384a2e2c2
5 files changed
+596
-53
No files matched your search
@@ -15,6 +15,7 @@ use crate::connection::task_client_session_id;
|
||||
use crate::models::connection::DatabaseType;
|
||||
use crate::mysql_ddl_normalize::DdlNormalizeOptions;
|
||||
use crate::object_source_sql::build_export_object_source_sql;
|
||||
use crate::sql::{SqlParsingOptions, SqlStatementSplitter};
|
||||
use crate::sql_dialect::{qualified_table_name, uses_single_row_insert_statements};
|
||||
use crate::transfer::{
|
||||
format_ch_array_sql_literal, format_pg_array_sql_literal, format_postgres_vector_sql_literal,
|
||||
@@ -166,6 +167,16 @@ enum DatabaseExportWriter {
|
||||
SplitZip(Box<crate::export_split_zip::SplitZipExportWriter>),
|
||||
}
|
||||
|
||||
impl DatabaseExportWriter {
|
||||
fn write_sql_unit(&mut self, unit: &[u8]) -> std::io::Result<()> {
|
||||
match self {
|
||||
Self::Plain(writer) => writer.write_all(unit),
|
||||
Self::Gzip(writer) => writer.write_all(unit),
|
||||
Self::SplitZip(writer) => writer.write_sql_unit(unit),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for DatabaseExportWriter {
|
||||
fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
|
||||
match self {
|
||||
@@ -175,6 +186,12 @@ impl Write for DatabaseExportWriter {
|
||||
}
|
||||
}
|
||||
|
||||
fn write_fmt(&mut self, fmt: std::fmt::Arguments<'_>) -> std::io::Result<()> {
|
||||
let mut unit = String::new();
|
||||
std::fmt::write(&mut unit, fmt).map_err(|_| std::io::Error::other("Failed to format SQL export unit"))?;
|
||||
self.write_sql_unit(unit.as_bytes())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
match self {
|
||||
Self::Plain(writer) => writer.flush(),
|
||||
@@ -3221,17 +3238,21 @@ fn postgres_create_schema_sql(schema: &str) -> String {
|
||||
format!("CREATE SCHEMA IF NOT EXISTS {};", quote_identifier(schema, &DatabaseType::Postgres))
|
||||
}
|
||||
|
||||
// Copy one line at a time so a `SplitZipExportWriter` can only rotate between
|
||||
// complete SQL statements; `std::io::copy` would feed it arbitrary 8KB chunks.
|
||||
fn combine_schema_sql_export<W: Write>(source: &mut dyn BufRead, destination: &mut W) -> std::io::Result<()> {
|
||||
let mut line = Vec::new();
|
||||
let mut splitter = SqlStatementSplitter::with_options(SqlParsingOptions::for_database_type(DatabaseType::Postgres));
|
||||
let mut chunk = String::new();
|
||||
loop {
|
||||
line.clear();
|
||||
let bytes_read = source.read_until(b'\n', &mut line)?;
|
||||
chunk.clear();
|
||||
let bytes_read = source.read_line(&mut chunk)?;
|
||||
if bytes_read == 0 {
|
||||
break;
|
||||
}
|
||||
destination.write_all(&line)?;
|
||||
for statement in splitter.push_chunk(&chunk) {
|
||||
destination.write_all(format!("{statement};\n").as_bytes())?;
|
||||
}
|
||||
}
|
||||
for statement in splitter.finish() {
|
||||
destination.write_all(format!("{statement};\n").as_bytes())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -6706,6 +6727,75 @@ mod tests {
|
||||
assert_eq!(combined.matches("INSERT INTO").count(), 20);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_zip_export_writer_keeps_an_oversized_create_table_ddl_whole_before_the_next_unit() {
|
||||
// Exercises the "SQL unit, not just INSERT" contract: a CREATE TABLE
|
||||
// DDL statement, an INSERT, and a SQL Server-style multi-statement
|
||||
// routine body must each land in exactly one part, never split at an
|
||||
// arbitrary byte offset inside one, even when writeln! formats the
|
||||
// unit from several interpolated arguments (real call sites do this,
|
||||
// e.g. `writeln!(file, "{source}\n")` for exported routine bodies).
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let zip_path = directory.path().join("schema.zip");
|
||||
let mut writer = crate::export_split_zip::SplitZipExportWriter::create(
|
||||
&zip_path,
|
||||
crate::export_split_zip::MIN_SPLIT_PART_MAX_MB,
|
||||
"schema",
|
||||
"sql",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let table_name = "dbo.orders";
|
||||
// part_max_bytes for MIN_SPLIT_PART_MAX_MB is 1 MiB. Make the CREATE
|
||||
// TABLE unit alone exceed that -- it stays whole as the oversized-unit
|
||||
// exception, but the *next* unit must then rotate into a new part.
|
||||
let columns = (0..100_000).map(|index| format!("col_{index} INT")).collect::<Vec<_>>().join(", ");
|
||||
let create_table_ddl = format!("CREATE TABLE {table_name} ({columns});");
|
||||
// A SQL Server routine body: multiple statements inside one BEGIN/END
|
||||
// block plus a batch-separator comment, matching what
|
||||
// build_export_object_source_sql emits for a procedure/trigger --
|
||||
// the writer must never cut between the inner statements.
|
||||
let procedure_source = "CREATE PROCEDURE dbo.recalc_totals AS\nBEGIN\n UPDATE dbo.orders SET total = total + 1;\n INSERT INTO dbo.audit_log (message) VALUES ('recalculated');\nEND".to_string();
|
||||
let go_batch_separator = "GO";
|
||||
let insert = "INSERT INTO dbo.orders (id) VALUES (1);".to_string();
|
||||
|
||||
writeln!(writer, "{create_table_ddl}").unwrap();
|
||||
writeln!(writer, "{procedure_source}").unwrap();
|
||||
writeln!(writer, "{go_batch_separator}").unwrap();
|
||||
writeln!(writer, "{insert}").unwrap();
|
||||
writer.finish("schema.sql").unwrap();
|
||||
|
||||
let file = std::fs::File::open(&zip_path).unwrap();
|
||||
let mut archive = zip::ZipArchive::new(file).unwrap();
|
||||
let mut sql_parts = Vec::new();
|
||||
for index in 0..archive.len() {
|
||||
let mut entry = archive.by_index(index).unwrap();
|
||||
if !entry.name().ends_with(".sql") {
|
||||
continue;
|
||||
}
|
||||
let mut contents = String::new();
|
||||
entry.read_to_string(&mut contents).unwrap();
|
||||
sql_parts.push((entry.name().to_string(), contents));
|
||||
}
|
||||
sql_parts.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
|
||||
assert!(sql_parts.len() >= 2, "the oversized CREATE TABLE alone must rotate before later units");
|
||||
|
||||
let combined: String = sql_parts.iter().map(|(_, contents)| contents.as_str()).collect();
|
||||
assert_eq!(
|
||||
combined,
|
||||
format!("{create_table_ddl}\n{procedure_source}\n{go_batch_separator}\n{insert}\n"),
|
||||
"reassembling every part must reproduce the original stream byte for byte"
|
||||
);
|
||||
|
||||
// Every unit must be found intact inside exactly one part -- proof
|
||||
// that none of them was fragmented across a rotation boundary.
|
||||
for unit in [create_table_ddl.as_str(), procedure_source.as_str(), go_batch_separator, insert.as_str()] {
|
||||
let containing_parts = sql_parts.iter().filter(|(_, contents)| contents.contains(unit)).count();
|
||||
assert_eq!(containing_parts, 1, "unit must appear whole in exactly one part: {unit:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_schemas_combine_keeps_split_part_boundaries_statement_safe() {
|
||||
// Mirror of the per-schema temporary files that
|
||||
@@ -6758,7 +6848,7 @@ mod tests {
|
||||
for line in contents.lines().filter(|line| !line.trim().is_empty()) {
|
||||
assert!(
|
||||
line.trim_start().starts_with("INSERT INTO") && line.trim_end().ends_with(';'),
|
||||
"{name} has a malformed line from a mid-statement cut"
|
||||
"{name} has a malformed line from a mid-statement cut: {line:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -856,6 +856,8 @@ async fn execute_sql_file_paths_inner(
|
||||
let mut prev_success_count = 0usize;
|
||||
let mut prev_failure_count = 0usize;
|
||||
let mut prev_affected_rows = 0u64;
|
||||
let mut splitter = Some(StreamingSqlFileSplitter::new(database_type, options));
|
||||
let mut pending_statements = Vec::with_capacity(SQL_FILE_STATEMENT_BATCH_SIZE);
|
||||
let import_result = async {
|
||||
for (file_index, file_path) in file_paths.iter().enumerate() {
|
||||
let file_name = file_path.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
|
||||
@@ -881,8 +883,6 @@ async fn execute_sql_file_paths_inner(
|
||||
});
|
||||
}
|
||||
|
||||
let mut splitter = StreamingSqlFileSplitter::new(database_type, options);
|
||||
let mut pending_statements = Vec::with_capacity(SQL_FILE_STATEMENT_BATCH_SIZE);
|
||||
let normalize_mysql_binary_literals = import_target.as_ref().is_some_and(|target| {
|
||||
crate::sql::is_mysql_compatible_import_target(&target.db_type, target.driver_profile.as_deref())
|
||||
});
|
||||
@@ -941,7 +941,10 @@ async fn execute_sql_file_paths_inner(
|
||||
"",
|
||||
None,
|
||||
));
|
||||
let mut next_statements = splitter.push_chunk(&chunk);
|
||||
let mut next_statements = splitter
|
||||
.as_mut()
|
||||
.expect("splitter is only taken by finish() on the last file, after this loop exits")
|
||||
.push_chunk(&chunk);
|
||||
if let Some(filter) = restore_filter.as_mut() {
|
||||
if let Err(error) = filter_restore_statements(
|
||||
&mut next_statements,
|
||||
@@ -979,23 +982,27 @@ async fn execute_sql_file_paths_inner(
|
||||
.await?;
|
||||
}
|
||||
|
||||
let mut next_statements = splitter.finish();
|
||||
if let Some(filter) = restore_filter.as_mut() {
|
||||
if let Err(error) = filter_restore_statements(
|
||||
&mut next_statements,
|
||||
filter,
|
||||
request.selected_tables.as_deref().unwrap_or_default(),
|
||||
) {
|
||||
emit(sql_file_execution_error_progress(
|
||||
&request.execution_id,
|
||||
started_at,
|
||||
&progress,
|
||||
error.clone(),
|
||||
));
|
||||
return Err(error);
|
||||
let is_last_file = file_index + 1 == file_count;
|
||||
if is_last_file {
|
||||
let mut next_statements =
|
||||
splitter.take().expect("splitter is only taken once, on the last file").finish();
|
||||
if let Some(filter) = restore_filter.as_mut() {
|
||||
if let Err(error) = filter_restore_statements(
|
||||
&mut next_statements,
|
||||
filter,
|
||||
request.selected_tables.as_deref().unwrap_or_default(),
|
||||
) {
|
||||
emit(sql_file_execution_error_progress(
|
||||
&request.execution_id,
|
||||
started_at,
|
||||
&progress,
|
||||
error.clone(),
|
||||
));
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
pending_statements.extend(next_statements);
|
||||
}
|
||||
pending_statements.extend(next_statements);
|
||||
execute_sql_file_statement_batch(
|
||||
state,
|
||||
request,
|
||||
@@ -3183,6 +3190,20 @@ mod tests {
|
||||
assert_eq!(statements[1], "SELECT 3;");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn streaming_sqlserver_splitter_keeps_split_json_literal_across_parts() {
|
||||
let part_one = "INSERT INTO [dbo].[VersionValue] ([value]) VALUES (N'{\"materialOrSymbolMate";
|
||||
let part_two = "rialId\":\"9625a891-3682-4151-acdb-6480db860033\",\"label\":\"B\\\\u1ebb ch\\\\u00e2n/Th\\\\u1eb3ng/\",\"quote\":\"it''s valid\"}');\n";
|
||||
let expected = format!("{part_one}{part_two}").trim().to_string();
|
||||
let mut splitter = StreamingSqlFileSplitter::new(Some(DatabaseType::SqlServer), SqlParsingOptions::default());
|
||||
|
||||
assert!(splitter.push_chunk(part_one).is_empty());
|
||||
assert!(splitter.push_chunk(part_two).is_empty());
|
||||
let statements = splitter.finish().into_iter().map(|statement| statement.sql).collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(statements, vec![expected]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn streaming_sqlserver_splitter_handles_go_across_chunks() {
|
||||
let mut splitter = SqlServerBatchSplitter::default();
|
||||
|
||||
@@ -1068,6 +1068,15 @@ enum TableExportSqlWriter {
|
||||
SplitZip(Box<crate::export_split_zip::SplitZipExportWriter>),
|
||||
}
|
||||
|
||||
impl TableExportSqlWriter {
|
||||
fn write_sql_unit(&mut self, unit: &[u8]) -> std::io::Result<()> {
|
||||
match self {
|
||||
Self::Plain(writer) => writer.write_all(unit),
|
||||
Self::SplitZip(writer) => writer.write_sql_unit(unit),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for TableExportSqlWriter {
|
||||
fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
|
||||
match self {
|
||||
@@ -1076,6 +1085,12 @@ impl Write for TableExportSqlWriter {
|
||||
}
|
||||
}
|
||||
|
||||
fn write_fmt(&mut self, fmt: std::fmt::Arguments<'_>) -> std::io::Result<()> {
|
||||
let mut unit = String::new();
|
||||
std::fmt::write(&mut unit, fmt).map_err(|_| std::io::Error::other("Failed to format SQL export unit"))?;
|
||||
self.write_sql_unit(unit.as_bytes())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
match self {
|
||||
Self::Plain(writer) => writer.flush(),
|
||||
@@ -1093,6 +1108,21 @@ impl TableExportSqlWriter {
|
||||
}
|
||||
}
|
||||
|
||||
fn write_sql_export_statements(
|
||||
file: &mut impl Write,
|
||||
statements: Vec<String>,
|
||||
wrote_statements: &mut bool,
|
||||
) -> Result<(), String> {
|
||||
for statement in statements {
|
||||
if *wrote_statements {
|
||||
file.write_all(b"\n").map_err(|e| format!("Failed to write SQL: {e}"))?;
|
||||
}
|
||||
file.write_all(statement.as_bytes()).map_err(|e| format!("Failed to write SQL: {e}"))?;
|
||||
*wrote_statements = true;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn create_table_export_sql_writer(request: &TableExportRequest) -> Result<TableExportSqlWriter, String> {
|
||||
if let Some(max_mb) = request.split_max_mb {
|
||||
let zip_path = std::path::Path::new(&request.file_path);
|
||||
@@ -1423,14 +1453,7 @@ async fn try_export_native_table_stream(
|
||||
&sql_export_excluded_columns(request, primary_keys, col_names, column_extras),
|
||||
request.insert_dialect,
|
||||
)?;
|
||||
if !statements.is_empty() {
|
||||
if wrote_statements {
|
||||
file.write_all(b"\n").map_err(|e| format!("Failed to write SQL: {e}"))?;
|
||||
}
|
||||
file.write_all(statements.join("\n").as_bytes())
|
||||
.map_err(|e| format!("Failed to write SQL: {e}"))?;
|
||||
wrote_statements = true;
|
||||
}
|
||||
write_sql_export_statements(file, statements, &mut wrote_statements)?;
|
||||
Ok(())
|
||||
};
|
||||
let result = stream_native_table_rows(
|
||||
@@ -2289,14 +2312,7 @@ async fn export_table_data_core_inner(
|
||||
&sql_export_excluded_columns(request, &primary_keys, &col_names, &column_extras),
|
||||
request.insert_dialect,
|
||||
)?;
|
||||
if !statements.is_empty() {
|
||||
if wrote_statements {
|
||||
file.write_all(b"\n").map_err(|e| format!("Failed to write SQL: {e}"))?;
|
||||
}
|
||||
file.write_all(statements.join("\n").as_bytes())
|
||||
.map_err(|e| format!("Failed to write SQL: {e}"))?;
|
||||
wrote_statements = true;
|
||||
}
|
||||
write_sql_export_statements(&mut file, statements, &mut wrote_statements)?;
|
||||
|
||||
rows_exported += row_count as u64;
|
||||
if use_keyset {
|
||||
@@ -2361,6 +2377,39 @@ mod tests {
|
||||
#[cfg(unix)]
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
#[test]
|
||||
fn split_zip_sql_export_writes_batch_statements_separately() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let zip_path = dir.path().join("export.zip");
|
||||
let statement = format!("INSERT INTO t VALUES ('{}');", "x".repeat(700 * 1024));
|
||||
let mut writer = crate::export_split_zip::SplitZipExportWriter::create(
|
||||
&zip_path,
|
||||
crate::export_split_zip::MIN_SPLIT_PART_MAX_MB,
|
||||
"table",
|
||||
"sql",
|
||||
)
|
||||
.unwrap();
|
||||
let mut wrote_statements = false;
|
||||
|
||||
write_sql_export_statements(&mut writer, vec![statement.clone(), statement.clone()], &mut wrote_statements)
|
||||
.unwrap();
|
||||
writer.finish("table.sql").unwrap();
|
||||
|
||||
let mut archive = zip::ZipArchive::new(std::fs::File::open(zip_path).unwrap()).unwrap();
|
||||
let mut sql_parts = Vec::new();
|
||||
for index in 0..archive.len() {
|
||||
let mut entry = archive.by_index(index).unwrap();
|
||||
if entry.name().ends_with(".sql") {
|
||||
let mut contents = String::new();
|
||||
entry.read_to_string(&mut contents).unwrap();
|
||||
sql_parts.push(contents);
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(sql_parts, vec![format!("{statement}\n"), statement]);
|
||||
assert!(sql_parts.iter().all(|part| part.len() <= 1024 * 1024));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn table_export_request_defaults_to_source_dialect_and_batch_insert_mode() {
|
||||
let mut payload = json!({
|
||||
|
||||
@@ -0,0 +1,267 @@
|
||||
// E2E test executed manually against a real local SQL Server 2025 container
|
||||
// (`database-mssql-1`, database `default`) already loaded with a production-shaped
|
||||
// dataset (50 tables / 52 foreign keys), as part of task #01a0db6e-2276.
|
||||
//
|
||||
// Flow: export database `default` (with `split_max_mb` set, i.e. the split-ZIP
|
||||
// contract under review) -> verify manifest/parts -> re-import the resulting ZIP
|
||||
// via `execute_sql_file_paths` with `skip_relational_constraints = true` ("Skip
|
||||
// relationship") into a fresh scratch database, exactly like the desktop app does.
|
||||
//
|
||||
// Run manually with:
|
||||
// DBX_LIVE_SQLSERVER_PASSWORD='MSSQLStrongPassword123!' \
|
||||
// cargo test -p dbx-core --test live_sqlserver_split_zip_e2e -- --nocapture --test-threads=1
|
||||
|
||||
use dbx_core::connection::AppState;
|
||||
use dbx_core::database_export::{export_database_sql_core, DatabaseExportOutputCompression, DatabaseExportRequest};
|
||||
use dbx_core::models::connection::DatabaseType;
|
||||
use dbx_core::sql::SqlFileRequest;
|
||||
use dbx_core::sql_file_import::execute_sql_file_paths;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
fn live_sqlserver_config(id: &str, database: &str) -> dbx_core::models::connection::ConnectionConfig {
|
||||
dbx_core::models::connection::ConnectionConfig {
|
||||
docs_notes_path: None,
|
||||
id: id.to_string(),
|
||||
name: id.to_string(),
|
||||
note: String::new(),
|
||||
db_type: DatabaseType::SqlServer,
|
||||
driver_profile: None,
|
||||
driver_label: None,
|
||||
url_params: None,
|
||||
agent_java_options: Vec::new(),
|
||||
host: std::env::var("DBX_LIVE_SQLSERVER_HOST").unwrap_or_else(|_| "127.0.0.1".to_string()),
|
||||
port: std::env::var("DBX_LIVE_SQLSERVER_PORT").ok().and_then(|value| value.parse().ok()).unwrap_or(1433),
|
||||
username: std::env::var("DBX_LIVE_SQLSERVER_USER").unwrap_or_else(|_| "sa".to_string()),
|
||||
password: std::env::var("DBX_LIVE_SQLSERVER_PASSWORD").expect("DBX_LIVE_SQLSERVER_PASSWORD"),
|
||||
database: Some(database.to_string()),
|
||||
default_schema: None,
|
||||
visible_databases: None,
|
||||
visible_database_patterns: None,
|
||||
visible_schemas: None,
|
||||
attached_databases: Vec::new(),
|
||||
init_script: None,
|
||||
color: None,
|
||||
transport_layers: Vec::new(),
|
||||
connect_timeout_secs: 10,
|
||||
query_timeout_secs: 30,
|
||||
idle_timeout_secs: 60,
|
||||
keepalive_interval_secs: 0,
|
||||
ssl: false,
|
||||
ca_cert_path: String::new(),
|
||||
client_cert_path: String::new(),
|
||||
client_key_path: String::new(),
|
||||
sysdba: false,
|
||||
oracle_connection_type: None,
|
||||
connection_string: None,
|
||||
redis_connection_mode: None,
|
||||
redis_sentinel_master: String::new(),
|
||||
redis_sentinel_nodes: String::new(),
|
||||
redis_sentinel_username: String::new(),
|
||||
redis_sentinel_password: String::new(),
|
||||
redis_sentinel_tls: false,
|
||||
redis_cluster_nodes: String::new(),
|
||||
redis_key_separator: dbx_core::models::connection::default_redis_key_separator(),
|
||||
redis_scan_page_size: None,
|
||||
redis_database_aliases: Default::default(),
|
||||
redis_key_templates: Vec::new(),
|
||||
redis_key_grouping: None,
|
||||
etcd_endpoints: String::new(),
|
||||
gbase_server: String::new(),
|
||||
informix_server: String::new(),
|
||||
external_config: None,
|
||||
plugin_id: None,
|
||||
plugin_connection_provider: None,
|
||||
plugin_connection_type: None,
|
||||
connection_secrets: Default::default(),
|
||||
jdbc_driver_class: None,
|
||||
jdbc_driver_paths: Vec::new(),
|
||||
one_time: false,
|
||||
save_password: true,
|
||||
read_only: false,
|
||||
is_production: false,
|
||||
production_databases: vec![],
|
||||
show_system_schemas: false,
|
||||
database_info: None,
|
||||
sidebar_auto_load_all_tables: false,
|
||||
}
|
||||
}
|
||||
|
||||
async fn live_sqlserver_state(connection_id: &str, database: &str, suffix: &str) -> (AppState, std::path::PathBuf) {
|
||||
let dir = std::env::temp_dir().join(format!("dbx-live-sqlserver-splitzip-{suffix}"));
|
||||
std::fs::create_dir_all(&dir).expect("create live test directory");
|
||||
let storage =
|
||||
dbx_core::persistence::test_storage::open(&dir.join("storage.db")).await.expect("open live test storage");
|
||||
let state = AppState::new(storage);
|
||||
let config = live_sqlserver_config(connection_id, database);
|
||||
state.configs.write().await.insert(connection_id.to_string(), config);
|
||||
(state, dir)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "manual E2E against a real local SQL Server instance -- see file header"]
|
||||
async fn manual_e2e_sqlserver_default_split_zip_export_then_import_skip_relationship() {
|
||||
let export_connection_id = "e2e-mssql-export";
|
||||
let import_connection_id = "e2e-mssql-import";
|
||||
let source_database = "default";
|
||||
// Import target is a separate scratch database, NOT the just-emptied
|
||||
// `default`, per the task's own guidance to isolate the destructive
|
||||
// re-import from the production-shaped database whenever possible.
|
||||
let import_database = "dbx_e2e_split_zip_restore";
|
||||
|
||||
let (export_state, work_dir) = live_sqlserver_state(export_connection_id, source_database, "export").await;
|
||||
let export_state = Arc::new(export_state);
|
||||
|
||||
let zip_path = work_dir.join("default-export.zip");
|
||||
let export_request = DatabaseExportRequest {
|
||||
export_id: format!("e2e-export-{}", uuid::Uuid::new_v4()),
|
||||
connection_id: export_connection_id.to_string(),
|
||||
database: source_database.to_string(),
|
||||
schema: String::new(),
|
||||
file_path: zip_path.to_string_lossy().to_string(),
|
||||
selected_tables: Vec::new(),
|
||||
excluded_tables: Vec::new(),
|
||||
include_structure: true,
|
||||
include_data: true,
|
||||
include_objects: true,
|
||||
include_create_database: false,
|
||||
drop_table_if_exists: true,
|
||||
omit_auto_increment: false,
|
||||
fail_on_error: true,
|
||||
prevent_overwrite: false,
|
||||
output_compression: DatabaseExportOutputCompression::None,
|
||||
snapshot_session_id: None,
|
||||
batch_size: 1000,
|
||||
split_max_mb: Some(1),
|
||||
insert_dialect: Default::default(),
|
||||
};
|
||||
|
||||
let progress_log = std::sync::Mutex::new(Vec::new());
|
||||
let export_result = export_database_sql_core(&export_state, &export_request, |progress| {
|
||||
progress_log.lock().unwrap().push(format!("{progress:?}"));
|
||||
})
|
||||
.await;
|
||||
|
||||
eprintln!("=== EXPORT PROGRESS LOG ===");
|
||||
for line in progress_log.lock().unwrap().iter() {
|
||||
eprintln!("{line}");
|
||||
}
|
||||
eprintln!("=== EXPORT RESULT ===\n{export_result:?}");
|
||||
export_result.expect("split-ZIP database export should succeed");
|
||||
|
||||
assert!(zip_path.exists(), "expected export ZIP at {}", zip_path.display());
|
||||
let zip_metadata = std::fs::metadata(&zip_path).expect("stat export zip");
|
||||
eprintln!("Export ZIP size: {} bytes", zip_metadata.len());
|
||||
|
||||
// --- Verify manifest.json + parts against the dbx-sql-export-parts-v1 contract ---
|
||||
let file = std::fs::File::open(&zip_path).expect("open export zip");
|
||||
let mut archive = zip::ZipArchive::new(file).expect("read export zip as a zip archive");
|
||||
let mut entry_names: Vec<String> =
|
||||
(0..archive.len()).map(|index| archive.by_index(index).expect("zip entry").name().to_string()).collect();
|
||||
entry_names.sort();
|
||||
eprintln!("=== ZIP ENTRIES ===\n{entry_names:#?}");
|
||||
assert!(entry_names.contains(&"manifest.json".to_string()), "manifest.json entry must be present");
|
||||
let sql_part_count = entry_names.iter().filter(|name| name.ends_with(".sql")).count();
|
||||
assert!(sql_part_count >= 1, "at least one .sql part expected");
|
||||
|
||||
let manifest_bytes = {
|
||||
let mut entry = archive.by_name("manifest.json").expect("manifest.json entry");
|
||||
let mut bytes = Vec::new();
|
||||
std::io::Read::read_to_end(&mut entry, &mut bytes).expect("read manifest.json");
|
||||
bytes
|
||||
};
|
||||
let manifest: serde_json::Value = serde_json::from_slice(&manifest_bytes).expect("parse manifest.json");
|
||||
eprintln!("=== MANIFEST ===\n{}", serde_json::to_string_pretty(&manifest).unwrap());
|
||||
assert_eq!(manifest["format"], "dbx-sql-export-parts-v1", "manifest format must match the documented contract");
|
||||
let manifest_parts = manifest["parts"].as_array().expect("manifest.parts array").len();
|
||||
assert_eq!(manifest_parts, sql_part_count, "manifest part count must match actual .sql entries in the zip");
|
||||
|
||||
// --- Prepare a clean scratch database on the same SQL Server instance for import ---
|
||||
let (import_state, _import_work_dir) =
|
||||
live_sqlserver_state(import_connection_id, "master", "import-bootstrap").await;
|
||||
let bootstrap_pool = import_state
|
||||
.get_or_create_pool(import_connection_id, Some("master"))
|
||||
.await
|
||||
.expect("connect to master to bootstrap scratch database");
|
||||
let drop_sql = format!(
|
||||
"IF DB_ID(N'{import_database}') IS NOT NULL BEGIN ALTER DATABASE [{import_database}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE [{import_database}]; END"
|
||||
);
|
||||
dbx_core::query::execute_sql_statement(&import_state, import_connection_id, "master", &drop_sql, None, None)
|
||||
.await
|
||||
.expect("drop pre-existing scratch database if present");
|
||||
let create_sql = format!("CREATE DATABASE [{import_database}]");
|
||||
dbx_core::query::execute_sql_statement(&import_state, import_connection_id, "master", &create_sql, None, None)
|
||||
.await
|
||||
.expect("create scratch database for import");
|
||||
let _ = bootstrap_pool;
|
||||
|
||||
// --- Extract the ZIP the same way the app does before feeding SQL files to the importer ---
|
||||
let extraction_dir = work_dir.join("extracted-parts");
|
||||
let package = dbx_core::sql_file_zip_package::extract_sql_file_zip_package(&zip_path, &extraction_dir)
|
||||
.expect("extract split-ZIP package using the shared sql_file_zip_package contract");
|
||||
let extracted_paths = dbx_core::sql_file_zip_package::extracted_sql_zip_paths(&extraction_dir, &package);
|
||||
eprintln!("=== EXTRACTED PART PATHS ===\n{extracted_paths:#?}");
|
||||
assert_eq!(extracted_paths.len(), sql_part_count, "extracted part count must match the zip's .sql entries");
|
||||
|
||||
let (reimport_state, _reimport_work_dir) =
|
||||
live_sqlserver_state(import_connection_id, import_database, "import-exec").await;
|
||||
let import_request = SqlFileRequest {
|
||||
execution_id: format!("e2e-import-{}", uuid::Uuid::new_v4()),
|
||||
connection_id: import_connection_id.to_string(),
|
||||
database: import_database.to_string(),
|
||||
file_path: extracted_paths[0].to_string_lossy().to_string(),
|
||||
continue_on_error: false,
|
||||
txn_session_id: None,
|
||||
selected_tables: None,
|
||||
part_cooldown_ms: 0,
|
||||
// This is the "Skip relationship" option from the task: SQL Server
|
||||
// constraint bypass runs `ALTER TABLE ... NOCHECK CONSTRAINT ALL`
|
||||
// before the import and `WITH NOCHECK CHECK CONSTRAINT ALL` after.
|
||||
skip_relational_constraints: true,
|
||||
};
|
||||
|
||||
let file_path_refs: Vec<&std::path::Path> = extracted_paths.iter().map(|path| path.as_path()).collect();
|
||||
let import_progress_log = std::sync::Mutex::new(Vec::new());
|
||||
let started_at = Instant::now();
|
||||
let import_result = execute_sql_file_paths(
|
||||
&reimport_state,
|
||||
&import_request,
|
||||
&file_path_refs,
|
||||
CancellationToken::new(),
|
||||
started_at,
|
||||
|progress| import_progress_log.lock().unwrap().push(format!("{progress:?}")),
|
||||
)
|
||||
.await;
|
||||
|
||||
eprintln!("=== IMPORT PROGRESS LOG ===");
|
||||
for line in import_progress_log.lock().unwrap().iter() {
|
||||
eprintln!("{line}");
|
||||
}
|
||||
eprintln!("=== IMPORT RESULT ===\n{import_result:?}");
|
||||
|
||||
// Intentionally do not assert success here: the whole point of this manual
|
||||
// E2E run is to surface whatever real error the split-ZIP + skip-relationship
|
||||
// path produces (or confirm it succeeds), and report it verbatim, per the
|
||||
// task's explicit instruction not to fabricate outcomes.
|
||||
match &import_result {
|
||||
Ok(()) => eprintln!("IMPORT SUCCEEDED"),
|
||||
Err(error) => eprintln!("IMPORT FAILED WITH ERROR: {error}"),
|
||||
}
|
||||
|
||||
// --- Post-import verification: table/row counts on the scratch database ---
|
||||
let verify_sql = "SELECT COUNT(*) AS tbl_count FROM sys.tables WHERE is_ms_shipped = 0";
|
||||
match dbx_core::query::execute_sql_statement(
|
||||
&reimport_state,
|
||||
import_connection_id,
|
||||
import_database,
|
||||
verify_sql,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => eprintln!("=== POST-IMPORT TABLE COUNT ===\n{result:?}"),
|
||||
Err(error) => eprintln!("=== POST-IMPORT VERIFY QUERY FAILED ===\n{error}"),
|
||||
}
|
||||
}
|
||||
@@ -122,6 +122,21 @@ impl SplitZipExportWriter {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Writes one complete SQL export unit into a single part. A unit may be
|
||||
/// a DDL statement, routine body, INSERT, comment, or client-side batch
|
||||
/// delimiter such as SQL Server `GO`; it must never be assembled from
|
||||
/// multiple calls. A unit that fits starts a new part before it would
|
||||
/// exceed the configured size; an oversized individual unit remains whole
|
||||
/// as the only unavoidable exception.
|
||||
pub fn write_sql_unit(&mut self, unit: &[u8]) -> std::io::Result<()> {
|
||||
if self.current_part_bytes > 0 && self.current_part_bytes + unit.len() as u64 > self.part_max_bytes {
|
||||
self.start_next_part().map_err(std::io::Error::other)?;
|
||||
}
|
||||
self.zip.write_all(unit)?;
|
||||
self.current_part_bytes += unit.len() as u64;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Writes the trailing `manifest.json` entry (listing every part in
|
||||
/// order) and finalizes the zip archive. Consumes the writer because the
|
||||
/// underlying `zip::ZipWriter::finish` does.
|
||||
@@ -148,12 +163,14 @@ impl SplitZipExportWriter {
|
||||
|
||||
impl Write for SplitZipExportWriter {
|
||||
fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
|
||||
if self.current_part_bytes >= self.part_max_bytes {
|
||||
self.start_next_part().map_err(std::io::Error::other)?;
|
||||
}
|
||||
let written = self.zip.write(buffer)?;
|
||||
self.current_part_bytes += written as u64;
|
||||
Ok(written)
|
||||
self.write_sql_unit(buffer)?;
|
||||
Ok(buffer.len())
|
||||
}
|
||||
|
||||
fn write_fmt(&mut self, fmt: std::fmt::Arguments<'_>) -> std::io::Result<()> {
|
||||
let mut unit = String::new();
|
||||
std::fmt::write(&mut unit, fmt).map_err(|_| std::io::Error::other("Failed to format SQL export unit"))?;
|
||||
self.write_sql_unit(unit.as_bytes())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
@@ -240,13 +257,9 @@ mod tests {
|
||||
|
||||
{
|
||||
let mut writer = SplitZipExportWriter::create(&zip_path, MIN_SPLIT_PART_MAX_MB, "mydb", "sql").unwrap();
|
||||
// Each write() call is one full statement; the threshold check
|
||||
// happens before the call, so a small per-write payload combined
|
||||
// with a below-1MB threshold still needs enough total bytes to
|
||||
// exceed 1MB and trigger a second part. Use the internal minimum
|
||||
// by writing repeatedly.
|
||||
for index in 0..6000 {
|
||||
writeln!(writer, "{long_statement} -- row {index}").unwrap();
|
||||
let line = format!("{long_statement} -- row {index}\n");
|
||||
writer.write_all(line.as_bytes()).unwrap();
|
||||
}
|
||||
writer.finish("mydb.sql").unwrap();
|
||||
}
|
||||
@@ -272,6 +285,109 @@ mod tests {
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rotates_before_a_complete_statement_that_would_exceed_the_part_limit() {
|
||||
let dir = std::env::temp_dir().join(format!("dbx-split-zip-test-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let zip_path = dir.join("export.zip");
|
||||
let statement = format!("INSERT INTO t VALUES ('{}');", "x".repeat(700 * 1024));
|
||||
|
||||
{
|
||||
let mut writer = SplitZipExportWriter::create(&zip_path, MIN_SPLIT_PART_MAX_MB, "mydb", "sql").unwrap();
|
||||
writer.write_all(statement.as_bytes()).unwrap();
|
||||
writer.write_all(b"\n").unwrap();
|
||||
writer.write_all(statement.as_bytes()).unwrap();
|
||||
writer.finish("mydb.sql").unwrap();
|
||||
}
|
||||
|
||||
let entries = read_zip_entries(&zip_path);
|
||||
let sql_entries: Vec<_> = entries.iter().filter(|(name, _)| name.ends_with(".sql")).collect();
|
||||
assert_eq!(sql_entries.len(), 2);
|
||||
assert!(sql_entries.iter().all(|(_, contents)| contents.len() <= 1024 * 1024));
|
||||
assert_eq!(String::from_utf8(sql_entries[0].1.clone()).unwrap(), format!("{statement}\n"));
|
||||
assert_eq!(String::from_utf8(sql_entries[1].1.clone()).unwrap(), statement);
|
||||
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn writeln_with_multiple_format_arguments_stays_one_sql_unit() {
|
||||
// std::fmt's default Write::write_fmt dispatches one write() call per
|
||||
// format-string fragment, which would let a rotation slip in between
|
||||
// "CREATE TABLE " and the column list even though the whole macro
|
||||
// invocation is meant to emit a single DDL unit. Every real export
|
||||
// call site formats a whole statement/DDL/routine body with `writeln!`
|
||||
// (e.g. `writeln!(file, "{source}\n")`, `writeln!(file, "-- Date:
|
||||
// {timestamp}")`), so the writer overrides `write_fmt` to buffer the
|
||||
// formatted text and hand it to `write_sql_unit` in one call.
|
||||
let dir = std::env::temp_dir().join(format!("dbx-split-zip-test-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let zip_path = dir.join("export.zip");
|
||||
let table_name = "orders";
|
||||
let insert = "INSERT INTO orders VALUES (1);";
|
||||
// part_max_bytes for MIN_SPLIT_PART_MAX_MB is 1 MiB. Size the column
|
||||
// list so the CREATE TABLE unit alone fits (leaving less headroom
|
||||
// than the INSERT needs), forcing the rotation to land *between* the
|
||||
// two units instead of never triggering or landing mid-DDL.
|
||||
let overhead = "CREATE TABLE ".len() + table_name.len() + " (".len() + ");".len() + "\n".len();
|
||||
let columns = "x".repeat(1024 * 1024 - overhead - 10);
|
||||
let create_table = format!("CREATE TABLE {table_name} ({columns});");
|
||||
|
||||
{
|
||||
let mut writer = SplitZipExportWriter::create(&zip_path, MIN_SPLIT_PART_MAX_MB, "mydb", "sql").unwrap();
|
||||
writeln!(writer, "CREATE TABLE {table_name} ({columns});").unwrap();
|
||||
writeln!(writer, "{insert}").unwrap();
|
||||
writer.finish("mydb.sql").unwrap();
|
||||
}
|
||||
|
||||
let entries = read_zip_entries(&zip_path);
|
||||
let mut sql_entries: Vec<_> = entries.iter().filter(|(name, _)| name.ends_with(".sql")).collect();
|
||||
sql_entries.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
|
||||
assert_eq!(sql_entries.len(), 2, "the oversized CREATE TABLE must rotate before the INSERT, not mid-DDL");
|
||||
let first = String::from_utf8(sql_entries[0].1.clone()).unwrap();
|
||||
let second = String::from_utf8(sql_entries[1].1.clone()).unwrap();
|
||||
assert_eq!(first, format!("{create_table}\n"), "the whole CREATE TABLE must land in one part intact");
|
||||
assert_eq!(second, format!("{insert}\n"));
|
||||
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keeps_an_oversized_individual_statement_whole_as_the_only_limit_exception() {
|
||||
let dir = std::env::temp_dir().join(format!("dbx-split-zip-test-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let zip_path = dir.join("export.zip");
|
||||
|
||||
let mut seed = 0x1234_5678_u32;
|
||||
let huge_payload = (0..3 * 1024 * 1024)
|
||||
.map(|_| {
|
||||
seed = seed.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
|
||||
char::from(b'a' + (seed >> 24) as u8 % 26)
|
||||
})
|
||||
.collect::<String>();
|
||||
let statement = format!("INSERT INTO t VALUES ('{huge_payload}');\n");
|
||||
|
||||
{
|
||||
let mut writer = SplitZipExportWriter::create(&zip_path, MIN_SPLIT_PART_MAX_MB, "mydb", "sql").unwrap();
|
||||
writer.write_all(statement.as_bytes()).unwrap();
|
||||
writer.finish("mydb.sql").unwrap();
|
||||
}
|
||||
|
||||
let entries = read_zip_entries(&zip_path);
|
||||
let mut sql_entries: Vec<_> = entries.iter().filter(|(name, _)| name.ends_with(".sql")).collect();
|
||||
sql_entries.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
|
||||
assert_eq!(sql_entries.len(), 1, "a single write must stay in one SQL part");
|
||||
assert_eq!(String::from_utf8(sql_entries[0].1.clone()).unwrap(), statement);
|
||||
|
||||
let manifest_entry = entries.iter().find(|(name, _)| name == "manifest.json").unwrap();
|
||||
let manifest: serde_json::Value = serde_json::from_slice(&manifest_entry.1).unwrap();
|
||||
assert_eq!(manifest["totalParts"].as_u64().unwrap() as usize, sql_entries.len());
|
||||
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sanitizes_zip_entry_stem_from_untrusted_names() {
|
||||
assert_eq!(sanitize_zip_entry_component("../../etc/passwd"), ".._.._etc_passwd");
|
||||
|
||||
Reference in new issue
Block a user