fix(sql-file): preserve statements across split ZIP parts

This commit is contained in:
КГБ Lieu authored and GitHub committed 2026-09-28 22:57:39 +08:00
1 parent 30b351c079
commit 2384a2e2c2
5 files changed
+596 -53

No files matched your search

+97 -7
View File
@@ -15,6 +15,7 @@ use crate::connection::task_client_session_id;
use crate::models::connection::DatabaseType;
use crate::mysql_ddl_normalize::DdlNormalizeOptions;
use crate::object_source_sql::build_export_object_source_sql;
use crate::sql::{SqlParsingOptions, SqlStatementSplitter};
use crate::sql_dialect::{qualified_table_name, uses_single_row_insert_statements};
use crate::transfer::{
format_ch_array_sql_literal, format_pg_array_sql_literal, format_postgres_vector_sql_literal,
@@ -166,6 +167,16 @@ enum DatabaseExportWriter {
SplitZip(Box<crate::export_split_zip::SplitZipExportWriter>),
}
impl DatabaseExportWriter {
fn write_sql_unit(&mut self, unit: &[u8]) -> std::io::Result<()> {
match self {
Self::Plain(writer) => writer.write_all(unit),
Self::Gzip(writer) => writer.write_all(unit),
Self::SplitZip(writer) => writer.write_sql_unit(unit),
}
}
}
impl Write for DatabaseExportWriter {
fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
match self {
@@ -175,6 +186,12 @@ impl Write for DatabaseExportWriter {
}
}
fn write_fmt(&mut self, fmt: std::fmt::Arguments<'_>) -> std::io::Result<()> {
let mut unit = String::new();
std::fmt::write(&mut unit, fmt).map_err(|_| std::io::Error::other("Failed to format SQL export unit"))?;
self.write_sql_unit(unit.as_bytes())
}
fn flush(&mut self) -> std::io::Result<()> {
match self {
Self::Plain(writer) => writer.flush(),
@@ -3221,17 +3238,21 @@ fn postgres_create_schema_sql(schema: &str) -> String {
format!("CREATE SCHEMA IF NOT EXISTS {};", quote_identifier(schema, &DatabaseType::Postgres))
}
// Copy one line at a time so a `SplitZipExportWriter` can only rotate between
// complete SQL statements; `std::io::copy` would feed it arbitrary 8KB chunks.
fn combine_schema_sql_export<W: Write>(source: &mut dyn BufRead, destination: &mut W) -> std::io::Result<()> {
let mut line = Vec::new();
let mut splitter = SqlStatementSplitter::with_options(SqlParsingOptions::for_database_type(DatabaseType::Postgres));
let mut chunk = String::new();
loop {
line.clear();
let bytes_read = source.read_until(b'\n', &mut line)?;
chunk.clear();
let bytes_read = source.read_line(&mut chunk)?;
if bytes_read == 0 {
break;
}
destination.write_all(&line)?;
for statement in splitter.push_chunk(&chunk) {
destination.write_all(format!("{statement};\n").as_bytes())?;
}
}
for statement in splitter.finish() {
destination.write_all(format!("{statement};\n").as_bytes())?;
}
Ok(())
}
@@ -6706,6 +6727,75 @@ mod tests {
assert_eq!(combined.matches("INSERT INTO").count(), 20);
}
#[test]
fn split_zip_export_writer_keeps_an_oversized_create_table_ddl_whole_before_the_next_unit() {
// Exercises the "SQL unit, not just INSERT" contract: a CREATE TABLE
// DDL statement, an INSERT, and a SQL Server-style multi-statement
// routine body must each land in exactly one part, never split at an
// arbitrary byte offset inside one, even when writeln! formats the
// unit from several interpolated arguments (real call sites do this,
// e.g. `writeln!(file, "{source}\n")` for exported routine bodies).
let directory = tempfile::tempdir().unwrap();
let zip_path = directory.path().join("schema.zip");
let mut writer = crate::export_split_zip::SplitZipExportWriter::create(
&zip_path,
crate::export_split_zip::MIN_SPLIT_PART_MAX_MB,
"schema",
"sql",
)
.unwrap();
let table_name = "dbo.orders";
// part_max_bytes for MIN_SPLIT_PART_MAX_MB is 1 MiB. Make the CREATE
// TABLE unit alone exceed that -- it stays whole as the oversized-unit
// exception, but the *next* unit must then rotate into a new part.
let columns = (0..100_000).map(|index| format!("col_{index} INT")).collect::<Vec<_>>().join(", ");
let create_table_ddl = format!("CREATE TABLE {table_name} ({columns});");
// A SQL Server routine body: multiple statements inside one BEGIN/END
// block plus a batch-separator comment, matching what
// build_export_object_source_sql emits for a procedure/trigger --
// the writer must never cut between the inner statements.
let procedure_source = "CREATE PROCEDURE dbo.recalc_totals AS\nBEGIN\n UPDATE dbo.orders SET total = total + 1;\n INSERT INTO dbo.audit_log (message) VALUES ('recalculated');\nEND".to_string();
let go_batch_separator = "GO";
let insert = "INSERT INTO dbo.orders (id) VALUES (1);".to_string();
writeln!(writer, "{create_table_ddl}").unwrap();
writeln!(writer, "{procedure_source}").unwrap();
writeln!(writer, "{go_batch_separator}").unwrap();
writeln!(writer, "{insert}").unwrap();
writer.finish("schema.sql").unwrap();
let file = std::fs::File::open(&zip_path).unwrap();
let mut archive = zip::ZipArchive::new(file).unwrap();
let mut sql_parts = Vec::new();
for index in 0..archive.len() {
let mut entry = archive.by_index(index).unwrap();
if !entry.name().ends_with(".sql") {
continue;
}
let mut contents = String::new();
entry.read_to_string(&mut contents).unwrap();
sql_parts.push((entry.name().to_string(), contents));
}
sql_parts.sort_by(|a, b| a.0.cmp(&b.0));
assert!(sql_parts.len() >= 2, "the oversized CREATE TABLE alone must rotate before later units");
let combined: String = sql_parts.iter().map(|(_, contents)| contents.as_str()).collect();
assert_eq!(
combined,
format!("{create_table_ddl}\n{procedure_source}\n{go_batch_separator}\n{insert}\n"),
"reassembling every part must reproduce the original stream byte for byte"
);
// Every unit must be found intact inside exactly one part -- proof
// that none of them was fragmented across a rotation boundary.
for unit in [create_table_ddl.as_str(), procedure_source.as_str(), go_batch_separator, insert.as_str()] {
let containing_parts = sql_parts.iter().filter(|(_, contents)| contents.contains(unit)).count();
assert_eq!(containing_parts, 1, "unit must appear whole in exactly one part: {unit:?}");
}
}
#[test]
fn all_schemas_combine_keeps_split_part_boundaries_statement_safe() {
// Mirror of the per-schema temporary files that
@@ -6758,7 +6848,7 @@ mod tests {
for line in contents.lines().filter(|line| !line.trim().is_empty()) {
assert!(
line.trim_start().starts_with("INSERT INTO") && line.trim_end().ends_with(';'),
"{name} has a malformed line from a mid-statement cut"
"{name} has a malformed line from a mid-statement cut: {line:?}"
);
}
}
+39 -18
View File
@@ -856,6 +856,8 @@ async fn execute_sql_file_paths_inner(
let mut prev_success_count = 0usize;
let mut prev_failure_count = 0usize;
let mut prev_affected_rows = 0u64;
let mut splitter = Some(StreamingSqlFileSplitter::new(database_type, options));
let mut pending_statements = Vec::with_capacity(SQL_FILE_STATEMENT_BATCH_SIZE);
let import_result = async {
for (file_index, file_path) in file_paths.iter().enumerate() {
let file_name = file_path.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
@@ -881,8 +883,6 @@ async fn execute_sql_file_paths_inner(
});
}
let mut splitter = StreamingSqlFileSplitter::new(database_type, options);
let mut pending_statements = Vec::with_capacity(SQL_FILE_STATEMENT_BATCH_SIZE);
let normalize_mysql_binary_literals = import_target.as_ref().is_some_and(|target| {
crate::sql::is_mysql_compatible_import_target(&target.db_type, target.driver_profile.as_deref())
});
@@ -941,7 +941,10 @@ async fn execute_sql_file_paths_inner(
"",
None,
));
let mut next_statements = splitter.push_chunk(&chunk);
let mut next_statements = splitter
.as_mut()
.expect("splitter is only taken by finish() on the last file, after this loop exits")
.push_chunk(&chunk);
if let Some(filter) = restore_filter.as_mut() {
if let Err(error) = filter_restore_statements(
&mut next_statements,
@@ -979,23 +982,27 @@ async fn execute_sql_file_paths_inner(
.await?;
}
let mut next_statements = splitter.finish();
if let Some(filter) = restore_filter.as_mut() {
if let Err(error) = filter_restore_statements(
&mut next_statements,
filter,
request.selected_tables.as_deref().unwrap_or_default(),
) {
emit(sql_file_execution_error_progress(
&request.execution_id,
started_at,
&progress,
error.clone(),
));
return Err(error);
let is_last_file = file_index + 1 == file_count;
if is_last_file {
let mut next_statements =
splitter.take().expect("splitter is only taken once, on the last file").finish();
if let Some(filter) = restore_filter.as_mut() {
if let Err(error) = filter_restore_statements(
&mut next_statements,
filter,
request.selected_tables.as_deref().unwrap_or_default(),
) {
emit(sql_file_execution_error_progress(
&request.execution_id,
started_at,
&progress,
error.clone(),
));
return Err(error);
}
}
pending_statements.extend(next_statements);
}
pending_statements.extend(next_statements);
execute_sql_file_statement_batch(
state,
request,
@@ -3183,6 +3190,20 @@ mod tests {
assert_eq!(statements[1], "SELECT 3;");
}
#[test]
fn streaming_sqlserver_splitter_keeps_split_json_literal_across_parts() {
let part_one = "INSERT INTO [dbo].[VersionValue] ([value]) VALUES (N'{\"materialOrSymbolMate";
let part_two = "rialId\":\"9625a891-3682-4151-acdb-6480db860033\",\"label\":\"B\\\\u1ebb ch\\\\u00e2n/Th\\\\u1eb3ng/\",\"quote\":\"it''s valid\"}');\n";
let expected = format!("{part_one}{part_two}").trim().to_string();
let mut splitter = StreamingSqlFileSplitter::new(Some(DatabaseType::SqlServer), SqlParsingOptions::default());
assert!(splitter.push_chunk(part_one).is_empty());
assert!(splitter.push_chunk(part_two).is_empty());
let statements = splitter.finish().into_iter().map(|statement| statement.sql).collect::<Vec<_>>();
assert_eq!(statements, vec![expected]);
}
#[test]
fn streaming_sqlserver_splitter_handles_go_across_chunks() {
let mut splitter = SqlServerBatchSplitter::default();
+65 -16
View File
@@ -1068,6 +1068,15 @@ enum TableExportSqlWriter {
SplitZip(Box<crate::export_split_zip::SplitZipExportWriter>),
}
impl TableExportSqlWriter {
fn write_sql_unit(&mut self, unit: &[u8]) -> std::io::Result<()> {
match self {
Self::Plain(writer) => writer.write_all(unit),
Self::SplitZip(writer) => writer.write_sql_unit(unit),
}
}
}
impl Write for TableExportSqlWriter {
fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
match self {
@@ -1076,6 +1085,12 @@ impl Write for TableExportSqlWriter {
}
}
fn write_fmt(&mut self, fmt: std::fmt::Arguments<'_>) -> std::io::Result<()> {
let mut unit = String::new();
std::fmt::write(&mut unit, fmt).map_err(|_| std::io::Error::other("Failed to format SQL export unit"))?;
self.write_sql_unit(unit.as_bytes())
}
fn flush(&mut self) -> std::io::Result<()> {
match self {
Self::Plain(writer) => writer.flush(),
@@ -1093,6 +1108,21 @@ impl TableExportSqlWriter {
}
}
fn write_sql_export_statements(
file: &mut impl Write,
statements: Vec<String>,
wrote_statements: &mut bool,
) -> Result<(), String> {
for statement in statements {
if *wrote_statements {
file.write_all(b"\n").map_err(|e| format!("Failed to write SQL: {e}"))?;
}
file.write_all(statement.as_bytes()).map_err(|e| format!("Failed to write SQL: {e}"))?;
*wrote_statements = true;
}
Ok(())
}
fn create_table_export_sql_writer(request: &TableExportRequest) -> Result<TableExportSqlWriter, String> {
if let Some(max_mb) = request.split_max_mb {
let zip_path = std::path::Path::new(&request.file_path);
@@ -1423,14 +1453,7 @@ async fn try_export_native_table_stream(
&sql_export_excluded_columns(request, primary_keys, col_names, column_extras),
request.insert_dialect,
)?;
if !statements.is_empty() {
if wrote_statements {
file.write_all(b"\n").map_err(|e| format!("Failed to write SQL: {e}"))?;
}
file.write_all(statements.join("\n").as_bytes())
.map_err(|e| format!("Failed to write SQL: {e}"))?;
wrote_statements = true;
}
write_sql_export_statements(file, statements, &mut wrote_statements)?;
Ok(())
};
let result = stream_native_table_rows(
@@ -2289,14 +2312,7 @@ async fn export_table_data_core_inner(
&sql_export_excluded_columns(request, &primary_keys, &col_names, &column_extras),
request.insert_dialect,
)?;
if !statements.is_empty() {
if wrote_statements {
file.write_all(b"\n").map_err(|e| format!("Failed to write SQL: {e}"))?;
}
file.write_all(statements.join("\n").as_bytes())
.map_err(|e| format!("Failed to write SQL: {e}"))?;
wrote_statements = true;
}
write_sql_export_statements(&mut file, statements, &mut wrote_statements)?;
rows_exported += row_count as u64;
if use_keyset {
@@ -2361,6 +2377,39 @@ mod tests {
#[cfg(unix)]
use std::os::unix::fs::PermissionsExt;
#[test]
fn split_zip_sql_export_writes_batch_statements_separately() {
let dir = tempfile::tempdir().unwrap();
let zip_path = dir.path().join("export.zip");
let statement = format!("INSERT INTO t VALUES ('{}');", "x".repeat(700 * 1024));
let mut writer = crate::export_split_zip::SplitZipExportWriter::create(
&zip_path,
crate::export_split_zip::MIN_SPLIT_PART_MAX_MB,
"table",
"sql",
)
.unwrap();
let mut wrote_statements = false;
write_sql_export_statements(&mut writer, vec![statement.clone(), statement.clone()], &mut wrote_statements)
.unwrap();
writer.finish("table.sql").unwrap();
let mut archive = zip::ZipArchive::new(std::fs::File::open(zip_path).unwrap()).unwrap();
let mut sql_parts = Vec::new();
for index in 0..archive.len() {
let mut entry = archive.by_index(index).unwrap();
if entry.name().ends_with(".sql") {
let mut contents = String::new();
entry.read_to_string(&mut contents).unwrap();
sql_parts.push(contents);
}
}
assert_eq!(sql_parts, vec![format!("{statement}\n"), statement]);
assert!(sql_parts.iter().all(|part| part.len() <= 1024 * 1024));
}
#[test]
fn table_export_request_defaults_to_source_dialect_and_batch_insert_mode() {
let mut payload = json!({