fix(sql-file): preserve statements across split ZIP parts
This commit is contained in:
1 parent
30b351c079
commit
2384a2e2c2
5 files changed
+596
-53
No files matched your search
@@ -15,6 +15,7 @@ use crate::connection::task_client_session_id;
|
||||
use crate::models::connection::DatabaseType;
|
||||
use crate::mysql_ddl_normalize::DdlNormalizeOptions;
|
||||
use crate::object_source_sql::build_export_object_source_sql;
|
||||
use crate::sql::{SqlParsingOptions, SqlStatementSplitter};
|
||||
use crate::sql_dialect::{qualified_table_name, uses_single_row_insert_statements};
|
||||
use crate::transfer::{
|
||||
format_ch_array_sql_literal, format_pg_array_sql_literal, format_postgres_vector_sql_literal,
|
||||
@@ -166,6 +167,16 @@ enum DatabaseExportWriter {
|
||||
SplitZip(Box<crate::export_split_zip::SplitZipExportWriter>),
|
||||
}
|
||||
|
||||
impl DatabaseExportWriter {
|
||||
fn write_sql_unit(&mut self, unit: &[u8]) -> std::io::Result<()> {
|
||||
match self {
|
||||
Self::Plain(writer) => writer.write_all(unit),
|
||||
Self::Gzip(writer) => writer.write_all(unit),
|
||||
Self::SplitZip(writer) => writer.write_sql_unit(unit),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for DatabaseExportWriter {
|
||||
fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
|
||||
match self {
|
||||
@@ -175,6 +186,12 @@ impl Write for DatabaseExportWriter {
|
||||
}
|
||||
}
|
||||
|
||||
fn write_fmt(&mut self, fmt: std::fmt::Arguments<'_>) -> std::io::Result<()> {
|
||||
let mut unit = String::new();
|
||||
std::fmt::write(&mut unit, fmt).map_err(|_| std::io::Error::other("Failed to format SQL export unit"))?;
|
||||
self.write_sql_unit(unit.as_bytes())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
match self {
|
||||
Self::Plain(writer) => writer.flush(),
|
||||
@@ -3221,17 +3238,21 @@ fn postgres_create_schema_sql(schema: &str) -> String {
|
||||
format!("CREATE SCHEMA IF NOT EXISTS {};", quote_identifier(schema, &DatabaseType::Postgres))
|
||||
}
|
||||
|
||||
// Copy one line at a time so a `SplitZipExportWriter` can only rotate between
|
||||
// complete SQL statements; `std::io::copy` would feed it arbitrary 8KB chunks.
|
||||
fn combine_schema_sql_export<W: Write>(source: &mut dyn BufRead, destination: &mut W) -> std::io::Result<()> {
|
||||
let mut line = Vec::new();
|
||||
let mut splitter = SqlStatementSplitter::with_options(SqlParsingOptions::for_database_type(DatabaseType::Postgres));
|
||||
let mut chunk = String::new();
|
||||
loop {
|
||||
line.clear();
|
||||
let bytes_read = source.read_until(b'\n', &mut line)?;
|
||||
chunk.clear();
|
||||
let bytes_read = source.read_line(&mut chunk)?;
|
||||
if bytes_read == 0 {
|
||||
break;
|
||||
}
|
||||
destination.write_all(&line)?;
|
||||
for statement in splitter.push_chunk(&chunk) {
|
||||
destination.write_all(format!("{statement};\n").as_bytes())?;
|
||||
}
|
||||
}
|
||||
for statement in splitter.finish() {
|
||||
destination.write_all(format!("{statement};\n").as_bytes())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -6706,6 +6727,75 @@ mod tests {
|
||||
assert_eq!(combined.matches("INSERT INTO").count(), 20);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_zip_export_writer_keeps_an_oversized_create_table_ddl_whole_before_the_next_unit() {
|
||||
// Exercises the "SQL unit, not just INSERT" contract: a CREATE TABLE
|
||||
// DDL statement, an INSERT, and a SQL Server-style multi-statement
|
||||
// routine body must each land in exactly one part, never split at an
|
||||
// arbitrary byte offset inside one, even when writeln! formats the
|
||||
// unit from several interpolated arguments (real call sites do this,
|
||||
// e.g. `writeln!(file, "{source}\n")` for exported routine bodies).
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let zip_path = directory.path().join("schema.zip");
|
||||
let mut writer = crate::export_split_zip::SplitZipExportWriter::create(
|
||||
&zip_path,
|
||||
crate::export_split_zip::MIN_SPLIT_PART_MAX_MB,
|
||||
"schema",
|
||||
"sql",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let table_name = "dbo.orders";
|
||||
// part_max_bytes for MIN_SPLIT_PART_MAX_MB is 1 MiB. Make the CREATE
|
||||
// TABLE unit alone exceed that -- it stays whole as the oversized-unit
|
||||
// exception, but the *next* unit must then rotate into a new part.
|
||||
let columns = (0..100_000).map(|index| format!("col_{index} INT")).collect::<Vec<_>>().join(", ");
|
||||
let create_table_ddl = format!("CREATE TABLE {table_name} ({columns});");
|
||||
// A SQL Server routine body: multiple statements inside one BEGIN/END
|
||||
// block plus a batch-separator comment, matching what
|
||||
// build_export_object_source_sql emits for a procedure/trigger --
|
||||
// the writer must never cut between the inner statements.
|
||||
let procedure_source = "CREATE PROCEDURE dbo.recalc_totals AS\nBEGIN\n UPDATE dbo.orders SET total = total + 1;\n INSERT INTO dbo.audit_log (message) VALUES ('recalculated');\nEND".to_string();
|
||||
let go_batch_separator = "GO";
|
||||
let insert = "INSERT INTO dbo.orders (id) VALUES (1);".to_string();
|
||||
|
||||
writeln!(writer, "{create_table_ddl}").unwrap();
|
||||
writeln!(writer, "{procedure_source}").unwrap();
|
||||
writeln!(writer, "{go_batch_separator}").unwrap();
|
||||
writeln!(writer, "{insert}").unwrap();
|
||||
writer.finish("schema.sql").unwrap();
|
||||
|
||||
let file = std::fs::File::open(&zip_path).unwrap();
|
||||
let mut archive = zip::ZipArchive::new(file).unwrap();
|
||||
let mut sql_parts = Vec::new();
|
||||
for index in 0..archive.len() {
|
||||
let mut entry = archive.by_index(index).unwrap();
|
||||
if !entry.name().ends_with(".sql") {
|
||||
continue;
|
||||
}
|
||||
let mut contents = String::new();
|
||||
entry.read_to_string(&mut contents).unwrap();
|
||||
sql_parts.push((entry.name().to_string(), contents));
|
||||
}
|
||||
sql_parts.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
|
||||
assert!(sql_parts.len() >= 2, "the oversized CREATE TABLE alone must rotate before later units");
|
||||
|
||||
let combined: String = sql_parts.iter().map(|(_, contents)| contents.as_str()).collect();
|
||||
assert_eq!(
|
||||
combined,
|
||||
format!("{create_table_ddl}\n{procedure_source}\n{go_batch_separator}\n{insert}\n"),
|
||||
"reassembling every part must reproduce the original stream byte for byte"
|
||||
);
|
||||
|
||||
// Every unit must be found intact inside exactly one part -- proof
|
||||
// that none of them was fragmented across a rotation boundary.
|
||||
for unit in [create_table_ddl.as_str(), procedure_source.as_str(), go_batch_separator, insert.as_str()] {
|
||||
let containing_parts = sql_parts.iter().filter(|(_, contents)| contents.contains(unit)).count();
|
||||
assert_eq!(containing_parts, 1, "unit must appear whole in exactly one part: {unit:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_schemas_combine_keeps_split_part_boundaries_statement_safe() {
|
||||
// Mirror of the per-schema temporary files that
|
||||
@@ -6758,7 +6848,7 @@ mod tests {
|
||||
for line in contents.lines().filter(|line| !line.trim().is_empty()) {
|
||||
assert!(
|
||||
line.trim_start().starts_with("INSERT INTO") && line.trim_end().ends_with(';'),
|
||||
"{name} has a malformed line from a mid-statement cut"
|
||||
"{name} has a malformed line from a mid-statement cut: {line:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -856,6 +856,8 @@ async fn execute_sql_file_paths_inner(
|
||||
let mut prev_success_count = 0usize;
|
||||
let mut prev_failure_count = 0usize;
|
||||
let mut prev_affected_rows = 0u64;
|
||||
let mut splitter = Some(StreamingSqlFileSplitter::new(database_type, options));
|
||||
let mut pending_statements = Vec::with_capacity(SQL_FILE_STATEMENT_BATCH_SIZE);
|
||||
let import_result = async {
|
||||
for (file_index, file_path) in file_paths.iter().enumerate() {
|
||||
let file_name = file_path.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
|
||||
@@ -881,8 +883,6 @@ async fn execute_sql_file_paths_inner(
|
||||
});
|
||||
}
|
||||
|
||||
let mut splitter = StreamingSqlFileSplitter::new(database_type, options);
|
||||
let mut pending_statements = Vec::with_capacity(SQL_FILE_STATEMENT_BATCH_SIZE);
|
||||
let normalize_mysql_binary_literals = import_target.as_ref().is_some_and(|target| {
|
||||
crate::sql::is_mysql_compatible_import_target(&target.db_type, target.driver_profile.as_deref())
|
||||
});
|
||||
@@ -941,7 +941,10 @@ async fn execute_sql_file_paths_inner(
|
||||
"",
|
||||
None,
|
||||
));
|
||||
let mut next_statements = splitter.push_chunk(&chunk);
|
||||
let mut next_statements = splitter
|
||||
.as_mut()
|
||||
.expect("splitter is only taken by finish() on the last file, after this loop exits")
|
||||
.push_chunk(&chunk);
|
||||
if let Some(filter) = restore_filter.as_mut() {
|
||||
if let Err(error) = filter_restore_statements(
|
||||
&mut next_statements,
|
||||
@@ -979,23 +982,27 @@ async fn execute_sql_file_paths_inner(
|
||||
.await?;
|
||||
}
|
||||
|
||||
let mut next_statements = splitter.finish();
|
||||
if let Some(filter) = restore_filter.as_mut() {
|
||||
if let Err(error) = filter_restore_statements(
|
||||
&mut next_statements,
|
||||
filter,
|
||||
request.selected_tables.as_deref().unwrap_or_default(),
|
||||
) {
|
||||
emit(sql_file_execution_error_progress(
|
||||
&request.execution_id,
|
||||
started_at,
|
||||
&progress,
|
||||
error.clone(),
|
||||
));
|
||||
return Err(error);
|
||||
let is_last_file = file_index + 1 == file_count;
|
||||
if is_last_file {
|
||||
let mut next_statements =
|
||||
splitter.take().expect("splitter is only taken once, on the last file").finish();
|
||||
if let Some(filter) = restore_filter.as_mut() {
|
||||
if let Err(error) = filter_restore_statements(
|
||||
&mut next_statements,
|
||||
filter,
|
||||
request.selected_tables.as_deref().unwrap_or_default(),
|
||||
) {
|
||||
emit(sql_file_execution_error_progress(
|
||||
&request.execution_id,
|
||||
started_at,
|
||||
&progress,
|
||||
error.clone(),
|
||||
));
|
||||
return Err(error);
|
||||
}
|
||||
}
|
||||
pending_statements.extend(next_statements);
|
||||
}
|
||||
pending_statements.extend(next_statements);
|
||||
execute_sql_file_statement_batch(
|
||||
state,
|
||||
request,
|
||||
@@ -3183,6 +3190,20 @@ mod tests {
|
||||
assert_eq!(statements[1], "SELECT 3;");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn streaming_sqlserver_splitter_keeps_split_json_literal_across_parts() {
|
||||
let part_one = "INSERT INTO [dbo].[VersionValue] ([value]) VALUES (N'{\"materialOrSymbolMate";
|
||||
let part_two = "rialId\":\"9625a891-3682-4151-acdb-6480db860033\",\"label\":\"B\\\\u1ebb ch\\\\u00e2n/Th\\\\u1eb3ng/\",\"quote\":\"it''s valid\"}');\n";
|
||||
let expected = format!("{part_one}{part_two}").trim().to_string();
|
||||
let mut splitter = StreamingSqlFileSplitter::new(Some(DatabaseType::SqlServer), SqlParsingOptions::default());
|
||||
|
||||
assert!(splitter.push_chunk(part_one).is_empty());
|
||||
assert!(splitter.push_chunk(part_two).is_empty());
|
||||
let statements = splitter.finish().into_iter().map(|statement| statement.sql).collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(statements, vec![expected]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn streaming_sqlserver_splitter_handles_go_across_chunks() {
|
||||
let mut splitter = SqlServerBatchSplitter::default();
|
||||
|
||||
@@ -1068,6 +1068,15 @@ enum TableExportSqlWriter {
|
||||
SplitZip(Box<crate::export_split_zip::SplitZipExportWriter>),
|
||||
}
|
||||
|
||||
impl TableExportSqlWriter {
|
||||
fn write_sql_unit(&mut self, unit: &[u8]) -> std::io::Result<()> {
|
||||
match self {
|
||||
Self::Plain(writer) => writer.write_all(unit),
|
||||
Self::SplitZip(writer) => writer.write_sql_unit(unit),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for TableExportSqlWriter {
|
||||
fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
|
||||
match self {
|
||||
@@ -1076,6 +1085,12 @@ impl Write for TableExportSqlWriter {
|
||||
}
|
||||
}
|
||||
|
||||
fn write_fmt(&mut self, fmt: std::fmt::Arguments<'_>) -> std::io::Result<()> {
|
||||
let mut unit = String::new();
|
||||
std::fmt::write(&mut unit, fmt).map_err(|_| std::io::Error::other("Failed to format SQL export unit"))?;
|
||||
self.write_sql_unit(unit.as_bytes())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
match self {
|
||||
Self::Plain(writer) => writer.flush(),
|
||||
@@ -1093,6 +1108,21 @@ impl TableExportSqlWriter {
|
||||
}
|
||||
}
|
||||
|
||||
fn write_sql_export_statements(
|
||||
file: &mut impl Write,
|
||||
statements: Vec<String>,
|
||||
wrote_statements: &mut bool,
|
||||
) -> Result<(), String> {
|
||||
for statement in statements {
|
||||
if *wrote_statements {
|
||||
file.write_all(b"\n").map_err(|e| format!("Failed to write SQL: {e}"))?;
|
||||
}
|
||||
file.write_all(statement.as_bytes()).map_err(|e| format!("Failed to write SQL: {e}"))?;
|
||||
*wrote_statements = true;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn create_table_export_sql_writer(request: &TableExportRequest) -> Result<TableExportSqlWriter, String> {
|
||||
if let Some(max_mb) = request.split_max_mb {
|
||||
let zip_path = std::path::Path::new(&request.file_path);
|
||||
@@ -1423,14 +1453,7 @@ async fn try_export_native_table_stream(
|
||||
&sql_export_excluded_columns(request, primary_keys, col_names, column_extras),
|
||||
request.insert_dialect,
|
||||
)?;
|
||||
if !statements.is_empty() {
|
||||
if wrote_statements {
|
||||
file.write_all(b"\n").map_err(|e| format!("Failed to write SQL: {e}"))?;
|
||||
}
|
||||
file.write_all(statements.join("\n").as_bytes())
|
||||
.map_err(|e| format!("Failed to write SQL: {e}"))?;
|
||||
wrote_statements = true;
|
||||
}
|
||||
write_sql_export_statements(file, statements, &mut wrote_statements)?;
|
||||
Ok(())
|
||||
};
|
||||
let result = stream_native_table_rows(
|
||||
@@ -2289,14 +2312,7 @@ async fn export_table_data_core_inner(
|
||||
&sql_export_excluded_columns(request, &primary_keys, &col_names, &column_extras),
|
||||
request.insert_dialect,
|
||||
)?;
|
||||
if !statements.is_empty() {
|
||||
if wrote_statements {
|
||||
file.write_all(b"\n").map_err(|e| format!("Failed to write SQL: {e}"))?;
|
||||
}
|
||||
file.write_all(statements.join("\n").as_bytes())
|
||||
.map_err(|e| format!("Failed to write SQL: {e}"))?;
|
||||
wrote_statements = true;
|
||||
}
|
||||
write_sql_export_statements(&mut file, statements, &mut wrote_statements)?;
|
||||
|
||||
rows_exported += row_count as u64;
|
||||
if use_keyset {
|
||||
@@ -2361,6 +2377,39 @@ mod tests {
|
||||
#[cfg(unix)]
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
#[test]
|
||||
fn split_zip_sql_export_writes_batch_statements_separately() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let zip_path = dir.path().join("export.zip");
|
||||
let statement = format!("INSERT INTO t VALUES ('{}');", "x".repeat(700 * 1024));
|
||||
let mut writer = crate::export_split_zip::SplitZipExportWriter::create(
|
||||
&zip_path,
|
||||
crate::export_split_zip::MIN_SPLIT_PART_MAX_MB,
|
||||
"table",
|
||||
"sql",
|
||||
)
|
||||
.unwrap();
|
||||
let mut wrote_statements = false;
|
||||
|
||||
write_sql_export_statements(&mut writer, vec![statement.clone(), statement.clone()], &mut wrote_statements)
|
||||
.unwrap();
|
||||
writer.finish("table.sql").unwrap();
|
||||
|
||||
let mut archive = zip::ZipArchive::new(std::fs::File::open(zip_path).unwrap()).unwrap();
|
||||
let mut sql_parts = Vec::new();
|
||||
for index in 0..archive.len() {
|
||||
let mut entry = archive.by_index(index).unwrap();
|
||||
if entry.name().ends_with(".sql") {
|
||||
let mut contents = String::new();
|
||||
entry.read_to_string(&mut contents).unwrap();
|
||||
sql_parts.push(contents);
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(sql_parts, vec![format!("{statement}\n"), statement]);
|
||||
assert!(sql_parts.iter().all(|part| part.len() <= 1024 * 1024));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn table_export_request_defaults_to_source_dialect_and_batch_insert_mode() {
|
||||
let mut payload = json!({
|
||||
|
||||
Reference in new issue
Block a user