feat(export): split large sql exports into zip parts
This commit is contained in:
1 parent
339c96cc81
commit
16866293a3
30 files changed
+933
-57
No files matched your search
@@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::future::Future;
|
||||
use std::io::{BufWriter, Write};
|
||||
use std::io::{BufRead, BufWriter, Write};
|
||||
use std::pin::Pin;
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::time::Duration;
|
||||
@@ -126,11 +126,19 @@ pub struct DatabaseExportRequest {
|
||||
#[serde(default)]
|
||||
pub snapshot_session_id: Option<String>,
|
||||
pub batch_size: usize,
|
||||
/// When set, the export is packaged as a `.zip` archive containing
|
||||
/// multiple `part-N.sql` entries (plus a `manifest.json`), each capped
|
||||
/// at this many megabytes, instead of one unbounded `.sql`/`.sql.gz`
|
||||
/// file. Mutually exclusive with `output_compression` -- a zip archive
|
||||
/// is its own compressed container.
|
||||
#[serde(default)]
|
||||
pub split_max_mb: Option<u32>,
|
||||
}
|
||||
|
||||
enum DatabaseExportWriter {
|
||||
Plain(BufWriter<std::fs::File>),
|
||||
Gzip(Box<GzEncoder<BufWriter<std::fs::File>>>),
|
||||
SplitZip(Box<crate::export_split_zip::SplitZipExportWriter>),
|
||||
}
|
||||
|
||||
impl Write for DatabaseExportWriter {
|
||||
@@ -138,6 +146,7 @@ impl Write for DatabaseExportWriter {
|
||||
match self {
|
||||
Self::Plain(writer) => writer.write(buffer),
|
||||
Self::Gzip(writer) => writer.write(buffer),
|
||||
Self::SplitZip(writer) => writer.write(buffer),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -145,17 +154,23 @@ impl Write for DatabaseExportWriter {
|
||||
match self {
|
||||
Self::Plain(writer) => writer.flush(),
|
||||
Self::Gzip(writer) => writer.flush(),
|
||||
Self::SplitZip(writer) => writer.flush(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl DatabaseExportWriter {
|
||||
fn finish(self) -> Result<(), String> {
|
||||
fn finish(self, source_file_name: &str) -> Result<(), String> {
|
||||
match self {
|
||||
Self::Plain(mut writer) => writer.flush(),
|
||||
Self::Gzip(writer) => writer.finish().and_then(|mut output| output.flush()),
|
||||
Self::Plain(mut writer) => {
|
||||
writer.flush().map_err(|error| format!("Failed to finalize export file: {error}"))
|
||||
}
|
||||
Self::Gzip(writer) => writer
|
||||
.finish()
|
||||
.and_then(|mut output| output.flush())
|
||||
.map_err(|error| format!("Failed to finalize export file: {error}")),
|
||||
Self::SplitZip(writer) => writer.finish(source_file_name),
|
||||
}
|
||||
.map_err(|error| format!("Failed to finalize export file: {error}"))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2574,6 +2589,15 @@ fn export_destination_parent_dir(file_path: &str) -> Option<&std::path::Path> {
|
||||
(!parent.as_os_str().is_empty()).then_some(parent)
|
||||
}
|
||||
|
||||
/// Name recorded inside `manifest.json` as the logical "source file" the
|
||||
/// split parts represent, e.g. `mydb.zip` -> `mydb.sql`. When splitting is
|
||||
/// disabled this is unused (the writer variant never calls it).
|
||||
fn export_source_file_name(file_path: &str) -> String {
|
||||
let path = std::path::Path::new(file_path);
|
||||
let stem = path.file_stem().and_then(|stem| stem.to_str()).unwrap_or("export");
|
||||
format!("{stem}.sql")
|
||||
}
|
||||
|
||||
async fn create_database_export_writer(
|
||||
state: &Arc<crate::connection::AppState>,
|
||||
request: &DatabaseExportRequest,
|
||||
@@ -2582,6 +2606,32 @@ async fn create_database_export_writer(
|
||||
if let Some(parent) = export_destination_parent_dir(&request.file_path) {
|
||||
expected_destination_identity = ensure_export_destination_dir(state, parent).await?;
|
||||
}
|
||||
if let Some(max_mb) = request.split_max_mb {
|
||||
let zip_path = std::path::Path::new(&request.file_path);
|
||||
let stem = zip_path.file_stem().and_then(|stem| stem.to_str()).unwrap_or("export");
|
||||
let writer = crate::export_split_zip::SplitZipExportWriter::create_with_overwrite(
|
||||
zip_path,
|
||||
max_mb,
|
||||
stem,
|
||||
"sql",
|
||||
request.prevent_overwrite,
|
||||
)?;
|
||||
let opened_destination_identity =
|
||||
std::fs::File::open(&request.file_path).ok().as_ref().and_then(export_destination_identity_for_file);
|
||||
if export_destination_identity_mismatch(
|
||||
expected_destination_identity.as_ref(),
|
||||
opened_destination_identity.as_ref(),
|
||||
) {
|
||||
let _ = std::fs::remove_file(&request.file_path);
|
||||
return Err(format!(
|
||||
"Backup destination for {} changed while opening the output file -- the directory now \
|
||||
resolves to a different filesystem than the one just verified. If a removable or network \
|
||||
drive was disconnected and reconnected, retry the backup.",
|
||||
request.file_path
|
||||
));
|
||||
}
|
||||
return Ok(DatabaseExportWriter::SplitZip(Box::new(writer)));
|
||||
}
|
||||
let file = std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create(true)
|
||||
@@ -2636,6 +2686,21 @@ fn postgres_create_schema_sql(schema: &str) -> String {
|
||||
format!("CREATE SCHEMA IF NOT EXISTS {};", quote_identifier(schema, &DatabaseType::Postgres))
|
||||
}
|
||||
|
||||
// Copy one line at a time so a `SplitZipExportWriter` can only rotate between
|
||||
// complete SQL statements; `std::io::copy` would feed it arbitrary 8KB chunks.
|
||||
fn combine_schema_sql_export<W: Write>(source: &mut dyn BufRead, destination: &mut W) -> std::io::Result<()> {
|
||||
let mut line = Vec::new();
|
||||
loop {
|
||||
line.clear();
|
||||
let bytes_read = source.read_until(b'\n', &mut line)?;
|
||||
if bytes_read == 0 {
|
||||
break;
|
||||
}
|
||||
destination.write_all(&line)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn export_postgres_all_schemas_sql_core(
|
||||
state: &Arc<crate::connection::AppState>,
|
||||
request: &DatabaseExportRequest,
|
||||
@@ -2670,6 +2735,7 @@ async fn export_postgres_all_schemas_sql_core(
|
||||
schema_request.schema = schema_name.clone();
|
||||
schema_request.file_path = temp_dir.path().join(format!("schema-{schema_index}.sql")).display().to_string();
|
||||
schema_request.output_compression = DatabaseExportOutputCompression::None;
|
||||
schema_request.split_max_mb = None;
|
||||
|
||||
let terminal = Arc::new(std::sync::Mutex::new(None::<ExportProgress>));
|
||||
let terminal_for_callback = terminal.clone();
|
||||
@@ -2730,10 +2796,11 @@ async fn export_postgres_all_schemas_sql_core(
|
||||
let mut source = std::io::BufReader::new(
|
||||
std::fs::File::open(path).map_err(|e| format!("Failed to read temporary schema export: {e}"))?,
|
||||
);
|
||||
std::io::copy(&mut source, &mut file).map_err(|e| format!("Failed to combine schema export: {e}"))?;
|
||||
combine_schema_sql_export(&mut source, &mut file)
|
||||
.map_err(|e| format!("Failed to combine schema export: {e}"))?;
|
||||
writeln!(file).map_err(|e| format!("Failed to write file: {e}"))?;
|
||||
}
|
||||
file.finish()?;
|
||||
file.finish(&export_source_file_name(&request.file_path))?;
|
||||
on_progress(ExportProgress {
|
||||
export_id: request.export_id.clone(),
|
||||
current_object: request.database.clone(),
|
||||
@@ -3695,7 +3762,7 @@ async fn export_database_sql_core_inner(
|
||||
writeln!(file, "SET FOREIGN_KEY_CHECKS = 1;").map_err(|e| format!("Failed to write file: {e}"))?;
|
||||
}
|
||||
|
||||
file.finish()?;
|
||||
file.finish(&export_source_file_name(&request.file_path))?;
|
||||
|
||||
// Emit Done progress
|
||||
on_progress(ExportProgress {
|
||||
@@ -3758,7 +3825,7 @@ fn build_database_export_object_source_sql(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
await_export_operation, await_export_stream_operation, clear_export_cancelled,
|
||||
await_export_operation, await_export_stream_operation, clear_export_cancelled, combine_schema_sql_export,
|
||||
concurrent_metadata_prefetch_allowed, database_export_metadata_prefetch_concurrency,
|
||||
emit_database_export_cancelled, postgres_create_schema_sql, postgres_export_schema_names, set_export_cancelled,
|
||||
snapshot_batch_cancelled, ExportStatus, EXPORT_CANCELLED_ERROR,
|
||||
@@ -3978,6 +4045,7 @@ mod tests {
|
||||
output_compression: Default::default(),
|
||||
snapshot_session_id: None,
|
||||
batch_size: 1000,
|
||||
split_max_mb: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3991,7 +4059,7 @@ mod tests {
|
||||
flate2::Compression::default(),
|
||||
)));
|
||||
writer.write_all(b"SELECT 1;\n").unwrap();
|
||||
writer.finish().unwrap();
|
||||
writer.finish("backup.sql").unwrap();
|
||||
|
||||
let mut output = String::new();
|
||||
flate2::read::GzDecoder::new(std::fs::File::open(path).unwrap()).read_to_string(&mut output).unwrap();
|
||||
@@ -5471,6 +5539,136 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_zip_export_writer_splits_across_insert_batches_into_valid_sql() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let zip_path = directory.path().join("orders.zip");
|
||||
let mut writer = crate::export_split_zip::SplitZipExportWriter::create(
|
||||
&zip_path,
|
||||
crate::export_split_zip::MIN_SPLIT_PART_MAX_MB,
|
||||
"orders",
|
||||
"sql",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Each call is one full INSERT batch statement, exactly like the real
|
||||
// write_database_export_rows call sites -- the writer must only cut
|
||||
// between these calls, never inside one.
|
||||
let long_value = "x".repeat(200_000);
|
||||
for row_index in 0..20 {
|
||||
write_database_export_rows(
|
||||
&mut writer,
|
||||
&[vec![json!(row_index), json!(long_value.clone())]],
|
||||
&["id".to_string(), "payload".to_string()],
|
||||
&[Some("bigint".to_string()), Some("text".to_string())],
|
||||
&[None, None],
|
||||
"orders",
|
||||
"shop",
|
||||
&DatabaseType::Postgres,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
writer.finish("orders.sql").unwrap();
|
||||
|
||||
let file = std::fs::File::open(&zip_path).unwrap();
|
||||
let mut archive = zip::ZipArchive::new(file).unwrap();
|
||||
let mut sql_parts = Vec::new();
|
||||
for index in 0..archive.len() {
|
||||
let mut entry = archive.by_index(index).unwrap();
|
||||
if !entry.name().ends_with(".sql") {
|
||||
continue;
|
||||
}
|
||||
let mut contents = String::new();
|
||||
entry.read_to_string(&mut contents).unwrap();
|
||||
sql_parts.push((entry.name().to_string(), contents));
|
||||
}
|
||||
sql_parts.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
|
||||
assert!(
|
||||
sql_parts.len() > 1,
|
||||
"expected the large export to be split into multiple parts, got {}",
|
||||
sql_parts.len()
|
||||
);
|
||||
for (name, contents) in &sql_parts {
|
||||
assert!(!contents.is_empty(), "{name} must not be empty");
|
||||
// Every non-blank line must be a syntactically complete INSERT
|
||||
// statement -- proof that no cut landed inside one.
|
||||
for line in contents.lines().filter(|line| !line.trim().is_empty()) {
|
||||
assert!(
|
||||
line.trim_start().starts_with("INSERT INTO") && line.trim_end().ends_with(';'),
|
||||
"{name} has a malformed line from a mid-statement cut: {line}"
|
||||
);
|
||||
}
|
||||
}
|
||||
// Reassembling every part in order must reproduce all 20 rows.
|
||||
let combined: String = sql_parts.iter().map(|(_, contents)| contents.as_str()).collect();
|
||||
assert_eq!(combined.matches("INSERT INTO").count(), 20);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_schemas_combine_keeps_split_part_boundaries_statement_safe() {
|
||||
// Mirror of the per-schema temporary files that
|
||||
// `export_postgres_all_schemas_sql_core` combines: whole SQL
|
||||
// statements, one per line, each newline-terminated.
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let schema_path = directory.path().join("schema-0.sql");
|
||||
let long_value = "x".repeat(200_000);
|
||||
let mut schema_sql = String::new();
|
||||
for row_index in 0..20 {
|
||||
schema_sql.push_str(&format!("INSERT INTO orders VALUES ({row_index}, '{long_value}');\n"));
|
||||
}
|
||||
std::fs::write(&schema_path, &schema_sql).unwrap();
|
||||
|
||||
let zip_path = directory.path().join("combined.zip");
|
||||
let mut writer = crate::export_split_zip::SplitZipExportWriter::create(
|
||||
&zip_path,
|
||||
crate::export_split_zip::MIN_SPLIT_PART_MAX_MB,
|
||||
"combined",
|
||||
"sql",
|
||||
)
|
||||
.unwrap();
|
||||
let mut source = std::io::BufReader::new(std::fs::File::open(&schema_path).unwrap());
|
||||
combine_schema_sql_export(&mut source, &mut writer).unwrap();
|
||||
writer.finish("combined.sql").unwrap();
|
||||
|
||||
let file = std::fs::File::open(&zip_path).unwrap();
|
||||
let mut archive = zip::ZipArchive::new(file).unwrap();
|
||||
let mut sql_parts = Vec::new();
|
||||
for index in 0..archive.len() {
|
||||
let mut entry = archive.by_index(index).unwrap();
|
||||
if !entry.name().ends_with(".sql") {
|
||||
continue;
|
||||
}
|
||||
let mut contents = String::new();
|
||||
entry.read_to_string(&mut contents).unwrap();
|
||||
sql_parts.push((entry.name().to_string(), contents));
|
||||
}
|
||||
sql_parts.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
|
||||
assert!(
|
||||
sql_parts.len() > 1,
|
||||
"expected the combined export to be split into multiple parts, got {}",
|
||||
sql_parts.len()
|
||||
);
|
||||
for (name, contents) in &sql_parts {
|
||||
assert!(!contents.is_empty(), "{name} must not be empty");
|
||||
// Every non-blank line must be a complete statement -- proof that
|
||||
// the copy never cut inside one.
|
||||
for line in contents.lines().filter(|line| !line.trim().is_empty()) {
|
||||
assert!(
|
||||
line.trim_start().starts_with("INSERT INTO") && line.trim_end().ends_with(';'),
|
||||
"{name} has a malformed line from a mid-statement cut"
|
||||
);
|
||||
}
|
||||
}
|
||||
// Reassembling the parts in order must reproduce the temporary file
|
||||
// byte for byte: the line-by-line copy adds, drops, and alters
|
||||
// nothing, including the trailing newline.
|
||||
let combined: String = sql_parts.iter().map(|(_, contents)| contents.as_str()).collect();
|
||||
assert_eq!(combined, schema_sql);
|
||||
assert_eq!(combined.matches("INSERT INTO").count(), 20);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dameng_identity_export_inserts_enable_identity_insert() {
|
||||
let statements = build_export_insert_statements(BuildExportInsertStatementsOptions {
|
||||
|
||||
@@ -0,0 +1,281 @@
|
||||
//! Shared support for splitting a large generated SQL export into multiple
|
||||
//! parts packaged inside a single `.zip` archive, instead of one unbounded
|
||||
//! `.sql` file. Used by both the whole-database export (`database_export.rs`)
|
||||
//! and the single-table SQL export (`table_export.rs`).
|
||||
//!
|
||||
//! # Cut boundary
|
||||
//! The writer never splits mid-write: it only checks the size threshold
|
||||
//! *before* starting a new `write()` call and, if exceeded, rotates to the
|
||||
//! next zip entry first. Every caller in this codebase writes one complete
|
||||
//! SQL statement (or comment line) per `write`/`writeln!` call, so a part
|
||||
//! boundary can land between two INSERT batches of the same table but never
|
||||
//! inside a single statement -- every part is independently valid, replayable
|
||||
//! SQL.
|
||||
|
||||
use std::io::Write;
|
||||
|
||||
use serde::Serialize;
|
||||
use zip::write::SimpleFileOptions;
|
||||
use zip::{CompressionMethod, ZipWriter};
|
||||
|
||||
/// Default part size when a caller enables splitting without specifying a
|
||||
/// size (should not normally happen -- the frontend always sends an explicit
|
||||
/// value -- but keeps the backend safe against malformed requests).
|
||||
pub const DEFAULT_SPLIT_PART_MAX_MB: u32 = 100;
|
||||
pub const MIN_SPLIT_PART_MAX_MB: u32 = 1;
|
||||
pub const MAX_SPLIT_PART_MAX_MB: u32 = 4096;
|
||||
|
||||
/// Clamp a user-supplied "max MB per part" value to a sane range so a
|
||||
/// malformed or hostile request cannot create an unbounded number of zip
|
||||
/// entries (too-small) or a single-part export that defeats the point
|
||||
/// (too-large is still allowed up to a generous ceiling).
|
||||
pub fn clamp_split_part_max_mb(value: u32) -> u32 {
|
||||
value.clamp(MIN_SPLIT_PART_MAX_MB, MAX_SPLIT_PART_MAX_MB)
|
||||
}
|
||||
|
||||
fn split_part_max_bytes(max_mb: u32) -> u64 {
|
||||
u64::from(clamp_split_part_max_mb(max_mb)) * 1024 * 1024
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct SplitExportManifest<'a> {
|
||||
format: &'static str,
|
||||
generated_at: String,
|
||||
source_file_name: &'a str,
|
||||
part_max_bytes: u64,
|
||||
total_parts: u32,
|
||||
parts: &'a [String],
|
||||
}
|
||||
|
||||
/// Writes SQL export content across multiple `part-N.sql` entries inside a
|
||||
/// single `.zip` archive, rotating to a new entry once the current one
|
||||
/// reaches `part_max_bytes`. Call [`SplitZipExportWriter::finish`] to write
|
||||
/// the trailing `manifest.json` entry and flush the archive to disk.
|
||||
pub struct SplitZipExportWriter {
|
||||
zip: ZipWriter<std::fs::File>,
|
||||
options: SimpleFileOptions,
|
||||
part_max_bytes: u64,
|
||||
current_part_bytes: u64,
|
||||
part_index: u32,
|
||||
part_stem: String,
|
||||
part_extension: String,
|
||||
part_names: Vec<String>,
|
||||
}
|
||||
|
||||
impl SplitZipExportWriter {
|
||||
/// `part_stem`/`part_extension` name each entry as `{stem}-part-{n}.{extension}`
|
||||
/// (e.g. `mydb-part-1.sql`).
|
||||
pub fn create(
|
||||
zip_path: &std::path::Path,
|
||||
max_mb: u32,
|
||||
part_stem: &str,
|
||||
part_extension: &str,
|
||||
) -> Result<Self, String> {
|
||||
Self::create_with_overwrite(zip_path, max_mb, part_stem, part_extension, false)
|
||||
}
|
||||
|
||||
pub fn create_with_overwrite(
|
||||
zip_path: &std::path::Path,
|
||||
max_mb: u32,
|
||||
part_stem: &str,
|
||||
part_extension: &str,
|
||||
prevent_overwrite: bool,
|
||||
) -> Result<Self, String> {
|
||||
let file = std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(!prevent_overwrite)
|
||||
.create_new(prevent_overwrite)
|
||||
.open(zip_path)
|
||||
.map_err(|error| {
|
||||
if prevent_overwrite && error.kind() == std::io::ErrorKind::AlreadyExists {
|
||||
format!("Backup file already exists: {}", zip_path.display())
|
||||
} else {
|
||||
format!("Failed to create zip file: {error}")
|
||||
}
|
||||
})?;
|
||||
let zip = ZipWriter::new(file);
|
||||
let options = SimpleFileOptions::default().compression_method(CompressionMethod::Deflated);
|
||||
let mut writer = Self {
|
||||
zip,
|
||||
options,
|
||||
part_max_bytes: split_part_max_bytes(max_mb),
|
||||
current_part_bytes: 0,
|
||||
part_index: 0,
|
||||
part_stem: sanitize_zip_entry_component(part_stem),
|
||||
part_extension: sanitize_zip_entry_component(part_extension),
|
||||
part_names: Vec::new(),
|
||||
};
|
||||
writer.start_next_part()?;
|
||||
Ok(writer)
|
||||
}
|
||||
|
||||
fn start_next_part(&mut self) -> Result<(), String> {
|
||||
self.part_index += 1;
|
||||
let name = format!("{}-part-{}.{}", self.part_stem, self.part_index, self.part_extension);
|
||||
self.zip
|
||||
.start_file(&name, self.options)
|
||||
.map_err(|error| format!("Failed to start zip entry {name}: {error}"))?;
|
||||
self.part_names.push(name);
|
||||
self.current_part_bytes = 0;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Writes the trailing `manifest.json` entry (listing every part in
|
||||
/// order) and finalizes the zip archive. Consumes the writer because the
|
||||
/// underlying `zip::ZipWriter::finish` does.
|
||||
pub fn finish(mut self, source_file_name: &str) -> Result<(), String> {
|
||||
let manifest = SplitExportManifest {
|
||||
format: "dbx-sql-export-parts-v1",
|
||||
generated_at: chrono::Local::now().to_rfc3339(),
|
||||
source_file_name,
|
||||
part_max_bytes: self.part_max_bytes,
|
||||
total_parts: self.part_index,
|
||||
parts: &self.part_names,
|
||||
};
|
||||
let manifest_json = serde_json::to_vec_pretty(&manifest)
|
||||
.map_err(|error| format!("Failed to encode export manifest: {error}"))?;
|
||||
let stored_options = SimpleFileOptions::default().compression_method(CompressionMethod::Stored);
|
||||
self.zip
|
||||
.start_file("manifest.json", stored_options)
|
||||
.map_err(|error| format!("Failed to start manifest entry: {error}"))?;
|
||||
self.zip.write_all(&manifest_json).map_err(|error| format!("Failed to write export manifest: {error}"))?;
|
||||
let mut file = self.zip.finish().map_err(|error| format!("Failed to finalize zip archive: {error}"))?;
|
||||
file.flush().map_err(|error| format!("Failed to flush zip archive: {error}"))
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for SplitZipExportWriter {
|
||||
fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
|
||||
if self.current_part_bytes >= self.part_max_bytes {
|
||||
self.start_next_part().map_err(std::io::Error::other)?;
|
||||
}
|
||||
let written = self.zip.write(buffer)?;
|
||||
self.current_part_bytes += written as u64;
|
||||
Ok(written)
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
self.zip.flush()
|
||||
}
|
||||
}
|
||||
|
||||
/// Zip entry names must not contain path separators or other characters that
|
||||
/// could be misread as a directory traversal; export file stems come from
|
||||
/// user-controlled database/table names, so sanitize defensively.
|
||||
fn sanitize_zip_entry_component(value: &str) -> String {
|
||||
let sanitized: String = value
|
||||
.chars()
|
||||
.map(|c| if c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.' { c } else { '_' })
|
||||
.collect();
|
||||
let trimmed = sanitized.trim_matches('_').to_string();
|
||||
if trimmed.is_empty() {
|
||||
"export".to_string()
|
||||
} else {
|
||||
trimmed
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Read;
|
||||
|
||||
fn read_zip_entries(path: &std::path::Path) -> Vec<(String, Vec<u8>)> {
|
||||
let file = std::fs::File::open(path).unwrap();
|
||||
let mut archive = zip::ZipArchive::new(file).unwrap();
|
||||
let mut entries = Vec::new();
|
||||
for index in 0..archive.len() {
|
||||
let mut entry = archive.by_index(index).unwrap();
|
||||
let mut contents = Vec::new();
|
||||
entry.read_to_end(&mut contents).unwrap();
|
||||
entries.push((entry.name().to_string(), contents));
|
||||
}
|
||||
entries
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clamps_split_part_max_mb_to_a_sane_range() {
|
||||
assert_eq!(clamp_split_part_max_mb(0), MIN_SPLIT_PART_MAX_MB);
|
||||
assert_eq!(clamp_split_part_max_mb(50), 50);
|
||||
assert_eq!(clamp_split_part_max_mb(u32::MAX), MAX_SPLIT_PART_MAX_MB);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rotates_to_a_new_part_once_the_threshold_is_reached() {
|
||||
let dir = std::env::temp_dir().join(format!("dbx-split-zip-test-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let zip_path = dir.join("export.zip");
|
||||
|
||||
{
|
||||
// 1 MB minimum threshold; each write is far smaller so every
|
||||
// write should land in the very first part with no rotation.
|
||||
let mut writer = SplitZipExportWriter::create(&zip_path, 1, "mydb", "sql").unwrap();
|
||||
for index in 0..5 {
|
||||
writeln!(writer, "INSERT INTO t VALUES ({index});").unwrap();
|
||||
}
|
||||
writer.finish("mydb.sql").unwrap();
|
||||
}
|
||||
|
||||
let entries = read_zip_entries(&zip_path);
|
||||
let sql_entries: Vec<_> = entries.iter().filter(|(name, _)| name.ends_with(".sql")).collect();
|
||||
assert_eq!(sql_entries.len(), 1);
|
||||
assert_eq!(sql_entries[0].0, "mydb-part-1.sql");
|
||||
let manifest_entry = entries.iter().find(|(name, _)| name == "manifest.json").unwrap();
|
||||
let manifest: serde_json::Value = serde_json::from_slice(&manifest_entry.1).unwrap();
|
||||
assert_eq!(manifest["totalParts"], 1);
|
||||
assert_eq!(manifest["parts"].as_array().unwrap().len(), 1);
|
||||
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn splits_into_multiple_parts_and_every_part_is_valid_statements() {
|
||||
let dir = std::env::temp_dir().join(format!("dbx-split-zip-test-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let zip_path = dir.join("export.zip");
|
||||
// Force a rotation by using a threshold smaller than a single write.
|
||||
let long_statement = format!("INSERT INTO t VALUES ('{}');", "x".repeat(200));
|
||||
|
||||
{
|
||||
let mut writer = SplitZipExportWriter::create(&zip_path, MIN_SPLIT_PART_MAX_MB, "mydb", "sql").unwrap();
|
||||
// Each write() call is one full statement; the threshold check
|
||||
// happens before the call, so a small per-write payload combined
|
||||
// with a below-1MB threshold still needs enough total bytes to
|
||||
// exceed 1MB and trigger a second part. Use the internal minimum
|
||||
// by writing repeatedly.
|
||||
for index in 0..6000 {
|
||||
writeln!(writer, "{long_statement} -- row {index}").unwrap();
|
||||
}
|
||||
writer.finish("mydb.sql").unwrap();
|
||||
}
|
||||
|
||||
let entries = read_zip_entries(&zip_path);
|
||||
let mut sql_entries: Vec<_> = entries.iter().filter(|(name, _)| name.ends_with(".sql")).collect();
|
||||
sql_entries.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
assert!(sql_entries.len() >= 2, "expected at least 2 parts, got {}", sql_entries.len());
|
||||
|
||||
for (name, contents) in &sql_entries {
|
||||
let text = String::from_utf8(contents.clone()).unwrap();
|
||||
assert!(!text.is_empty(), "{name} must not be empty");
|
||||
for line in text.lines().filter(|line| !line.trim().is_empty()) {
|
||||
assert!(line.contains(';'), "{name} contains a line that is not a complete statement: {line}");
|
||||
}
|
||||
}
|
||||
|
||||
let manifest_entry = entries.iter().find(|(name, _)| name == "manifest.json").unwrap();
|
||||
let manifest: serde_json::Value = serde_json::from_slice(&manifest_entry.1).unwrap();
|
||||
assert_eq!(manifest["totalParts"].as_u64().unwrap() as usize, sql_entries.len());
|
||||
assert_eq!(manifest["parts"].as_array().unwrap().len(), sql_entries.len());
|
||||
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sanitizes_zip_entry_stem_from_untrusted_names() {
|
||||
assert_eq!(sanitize_zip_entry_component("../../etc/passwd"), ".._.._etc_passwd");
|
||||
assert_eq!(sanitize_zip_entry_component(""), "export");
|
||||
assert_eq!(sanitize_zip_entry_component("my db (prod)"), "my_db__prod");
|
||||
}
|
||||
}
|
||||
@@ -48,6 +48,7 @@ pub mod docs;
|
||||
pub mod document_ops;
|
||||
pub mod driver_runtime;
|
||||
pub mod export_runtime;
|
||||
pub mod export_split_zip;
|
||||
pub mod external;
|
||||
pub mod hbase_ops;
|
||||
pub mod history;
|
||||
|
||||
@@ -83,6 +83,12 @@ pub struct TableExportRequest {
|
||||
pub column_comments: Option<Vec<Option<String>>>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub auto_filter: Option<bool>,
|
||||
/// SQL format only: when set, the export is packaged as a `.zip` archive
|
||||
/// containing multiple `part-N.sql` entries (plus a `manifest.json`)
|
||||
/// capped at this many megabytes each, instead of one unbounded `.sql`
|
||||
/// file. Ignored for every other format.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub split_max_mb: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
@@ -975,6 +981,53 @@ async fn stream_native_table_rows(
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
/// Output writer for the single-table SQL export. `Plain` is the historical
|
||||
/// unbounded `.sql` file; `SplitZip` packages the SQL into multiple
|
||||
/// `part-N.sql` entries inside a `.zip` when `TableExportRequest::split_max_mb`
|
||||
/// is set.
|
||||
enum TableExportSqlWriter {
|
||||
Plain(BufWriter<std::fs::File>),
|
||||
SplitZip(Box<crate::export_split_zip::SplitZipExportWriter>),
|
||||
}
|
||||
|
||||
impl Write for TableExportSqlWriter {
|
||||
fn write(&mut self, buffer: &[u8]) -> std::io::Result<usize> {
|
||||
match self {
|
||||
Self::Plain(writer) => writer.write(buffer),
|
||||
Self::SplitZip(writer) => writer.write(buffer),
|
||||
}
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
match self {
|
||||
Self::Plain(writer) => writer.flush(),
|
||||
Self::SplitZip(writer) => writer.flush(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TableExportSqlWriter {
|
||||
fn finish(self, source_file_name: &str) -> Result<(), String> {
|
||||
match self {
|
||||
Self::Plain(mut writer) => writer.flush().map_err(|error| format!("Failed to flush export file: {error}")),
|
||||
Self::SplitZip(writer) => writer.finish(source_file_name),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn create_table_export_sql_writer(request: &TableExportRequest) -> Result<TableExportSqlWriter, String> {
|
||||
if let Some(max_mb) = request.split_max_mb {
|
||||
let zip_path = std::path::Path::new(&request.file_path);
|
||||
let stem = zip_path.file_stem().and_then(|stem| stem.to_str()).unwrap_or(&request.table_name);
|
||||
let writer = crate::export_split_zip::SplitZipExportWriter::create(zip_path, max_mb, stem, "sql")?;
|
||||
return Ok(TableExportSqlWriter::SplitZip(Box::new(writer)));
|
||||
}
|
||||
Ok(TableExportSqlWriter::Plain(BufWriter::new(
|
||||
std::fs::File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?,
|
||||
)))
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn try_export_native_table_stream(
|
||||
state: &AppState,
|
||||
@@ -1254,13 +1307,11 @@ async fn try_export_native_table_stream(
|
||||
result
|
||||
}
|
||||
"sql" => {
|
||||
let mut file = BufWriter::new(
|
||||
std::fs::File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?,
|
||||
);
|
||||
let mut file = create_table_export_sql_writer(request)?;
|
||||
let mut pending_rows: Vec<Vec<Value>> = Vec::new();
|
||||
let mut wrote_statements = false;
|
||||
let mut flush_pending =
|
||||
|file: &mut BufWriter<std::fs::File>, pending_rows: &mut Vec<Vec<Value>>| -> Result<(), String> {
|
||||
|file: &mut TableExportSqlWriter, pending_rows: &mut Vec<Vec<Value>>| -> Result<(), String> {
|
||||
if pending_rows.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
@@ -1321,7 +1372,7 @@ async fn try_export_native_table_stream(
|
||||
if wrote_statements {
|
||||
file.write_all(b"\n").map_err(|e| format!("Failed to write SQL: {e}"))?;
|
||||
}
|
||||
file.flush().map_err(|e| format!("Failed to flush export file: {e}"))?;
|
||||
file.finish(&format!("{}.sql", request.table_name))?;
|
||||
}
|
||||
result
|
||||
}
|
||||
@@ -1562,8 +1613,13 @@ async fn export_table_data_core_inner(
|
||||
}
|
||||
|
||||
// 8. Create output file
|
||||
let file = std::fs::File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?;
|
||||
let mut file = BufWriter::new(file);
|
||||
let mut file = if request.format.eq_ignore_ascii_case("sql") {
|
||||
create_table_export_sql_writer(request)?
|
||||
} else {
|
||||
TableExportSqlWriter::Plain(BufWriter::new(
|
||||
std::fs::File::create(&request.file_path).map_err(|e| format!("Failed to create file: {e}"))?,
|
||||
))
|
||||
};
|
||||
let mut text_buffer = String::new();
|
||||
|
||||
let mut rows_exported: u64 = 0;
|
||||
@@ -2154,7 +2210,7 @@ async fn export_table_data_core_inner(
|
||||
}
|
||||
|
||||
close_table_export_cursor_if_open(state, &pool_key, request, &mut cursor_session).await;
|
||||
file.flush().map_err(|e| format!("Failed to flush export file: {e}"))?;
|
||||
file.finish(&format!("{}.sql", request.table_name))?;
|
||||
|
||||
// 8. Emit Done progress
|
||||
on_progress(TableExportProgress {
|
||||
@@ -2304,6 +2360,7 @@ mod tests {
|
||||
numeric_column_right_align: false,
|
||||
column_comments: None,
|
||||
auto_filter: None,
|
||||
split_max_mb: None,
|
||||
};
|
||||
|
||||
ExternalDriverExportFixture { state, request, calls, output, dir }
|
||||
@@ -2466,6 +2523,7 @@ mod tests {
|
||||
numeric_column_right_align: false,
|
||||
column_comments: None,
|
||||
auto_filter: None,
|
||||
split_max_mb: None,
|
||||
};
|
||||
|
||||
export_table_data_core(&state, &request, |_| {}).await.unwrap();
|
||||
@@ -2631,6 +2689,7 @@ mod tests {
|
||||
numeric_column_right_align: false,
|
||||
column_comments: None,
|
||||
auto_filter: None,
|
||||
split_max_mb: None,
|
||||
};
|
||||
let context = table_export_sql_context(DatabaseType::Iotdb, None, request.schema.as_deref());
|
||||
let columns = vec!["Time".to_string(), "root.test.device2.temperature".to_string()];
|
||||
@@ -2689,6 +2748,7 @@ mod tests {
|
||||
numeric_column_right_align: false,
|
||||
column_comments: None,
|
||||
auto_filter: None,
|
||||
split_max_mb: None,
|
||||
};
|
||||
let context = table_export_sql_context(DatabaseType::Iotdb, None, request.schema.as_deref());
|
||||
let columns = vec!["tImE".to_string(), "temperature".to_string()];
|
||||
@@ -2725,6 +2785,7 @@ mod tests {
|
||||
numeric_column_right_align: false,
|
||||
column_comments: None,
|
||||
auto_filter: None,
|
||||
split_max_mb: None,
|
||||
};
|
||||
let context = table_export_sql_context(DatabaseType::Iotdb, None, request.schema.as_deref());
|
||||
let error = table_export_query_columns(&request, &context, &["TIME".to_string()]).unwrap_err();
|
||||
@@ -2756,6 +2817,7 @@ mod tests {
|
||||
numeric_column_right_align: false,
|
||||
column_comments: None,
|
||||
auto_filter: None,
|
||||
split_max_mb: None,
|
||||
};
|
||||
let context = table_export_sql_context(DatabaseType::Iotdb, None, request.schema.as_deref());
|
||||
let columns = vec![
|
||||
@@ -2795,6 +2857,7 @@ mod tests {
|
||||
numeric_column_right_align: false,
|
||||
column_comments: None,
|
||||
auto_filter: None,
|
||||
split_max_mb: None,
|
||||
};
|
||||
let columns = vec!["Time".to_string(), "value".to_string()];
|
||||
|
||||
@@ -2846,6 +2909,7 @@ mod tests {
|
||||
numeric_column_right_align: false,
|
||||
column_comments: None,
|
||||
auto_filter: None,
|
||||
split_max_mb: None,
|
||||
};
|
||||
let context = table_export_sql_context(DatabaseType::Oracle, None, request.schema.as_deref());
|
||||
|
||||
@@ -2891,6 +2955,7 @@ mod tests {
|
||||
numeric_column_right_align: false,
|
||||
column_comments: None,
|
||||
auto_filter: None,
|
||||
split_max_mb: None,
|
||||
};
|
||||
let columns = vec!["id".to_string(), "payload".to_string()];
|
||||
let primary_keys = vec!["id".to_string()];
|
||||
@@ -2964,6 +3029,7 @@ mod tests {
|
||||
numeric_column_right_align: false,
|
||||
column_comments: None,
|
||||
auto_filter: None,
|
||||
split_max_mb: None,
|
||||
};
|
||||
let columns = vec!["id".to_string(), "DisplayName".to_string()];
|
||||
let primary_keys = vec!["id".to_string()];
|
||||
@@ -3019,6 +3085,7 @@ mod tests {
|
||||
numeric_column_right_align: false,
|
||||
column_comments: None,
|
||||
auto_filter: None,
|
||||
split_max_mb: None,
|
||||
};
|
||||
let columns = vec!["id".to_string(), "geom".to_string(), "name".to_string()];
|
||||
let column_types = vec![Some("int".to_string()), Some("geometry".to_string()), Some("varchar".to_string())];
|
||||
@@ -3076,6 +3143,7 @@ mod tests {
|
||||
numeric_column_right_align: false,
|
||||
column_comments: None,
|
||||
auto_filter: None,
|
||||
split_max_mb: None,
|
||||
};
|
||||
let context = table_export_sql_context(DatabaseType::Oracle, None, request.schema.as_deref());
|
||||
let sql = table_cursor_sql(&request, &context, &columns, &[], &primary_keys);
|
||||
@@ -3200,6 +3268,187 @@ mod tests {
|
||||
cleanup_external_driver_export_fixture(fixture);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn external_driver_sql_export_without_split_writes_plain_sql() {
|
||||
// The fallback export future is too large for the default test-thread
|
||||
// stack in debug builds; mirror the dedicated-thread pattern used by
|
||||
// the other fallback-path tests.
|
||||
let handle = std::thread::Builder::new()
|
||||
.name("table-export-plain-sql".to_string())
|
||||
.stack_size(8 * 1024 * 1024)
|
||||
.spawn(|| {
|
||||
tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("build table export plain SQL test runtime")
|
||||
.block_on(run_external_driver_sql_export_without_split_writes_plain_sql());
|
||||
})
|
||||
.expect("spawn table export plain SQL test thread");
|
||||
if let Err(panic) = handle.join() {
|
||||
std::panic::resume_unwind(panic);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
async fn run_external_driver_sql_export_without_split_writes_plain_sql() {
|
||||
let fixture = external_driver_export_fixture(
|
||||
r#" case "$line" in
|
||||
*'"method":"executeQueryPage"'*)
|
||||
echo executeQueryPage >> "$CALLS"
|
||||
printf '{"id":%s,"result":{"columns":["id","name"],"rows":[[1,"Ada"],[2,"Grace"]],"affected_rows":0,"execution_time_ms":1,"session_id":null,"has_more":false}}\n' "$id"
|
||||
;;
|
||||
esac"#,
|
||||
2,
|
||||
None,
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
let output = fixture.dir.join("export.sql");
|
||||
let mut request = fixture.request.clone();
|
||||
request.format = "sql".to_string();
|
||||
request.file_path = output.to_string_lossy().into_owned();
|
||||
|
||||
let terminal = Arc::new(std::sync::Mutex::new(None::<ExportStatus>));
|
||||
let captured = terminal.clone();
|
||||
export_table_data_core(&fixture.state, &request, move |event| {
|
||||
*captured.lock().unwrap() = Some(event.status);
|
||||
})
|
||||
.await
|
||||
.expect("plain JDBC SQL export should succeed");
|
||||
|
||||
let sql = std::fs::read_to_string(&output).unwrap();
|
||||
// Default batch insert mode emits one multi-row INSERT statement.
|
||||
assert_eq!(sql.matches("INSERT INTO").count(), 1, "both rows should be exported: {sql}");
|
||||
assert!(sql.contains("Ada"));
|
||||
assert!(sql.contains("Grace"));
|
||||
assert!(sql.ends_with(";\n"));
|
||||
// Without `split_max_mb` the fallback arm must keep producing a plain
|
||||
// SQL file, not a zip archive.
|
||||
assert!(!sql.starts_with("PK"));
|
||||
assert_eq!(std::fs::read_to_string(&fixture.calls).unwrap(), "executeQueryPage\n");
|
||||
assert!(matches!(*terminal.lock().unwrap(), Some(ExportStatus::Done)));
|
||||
|
||||
cleanup_external_driver_export_fixture(fixture);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn external_driver_sql_export_honors_split_max_mb_into_zip_parts() {
|
||||
// See external_driver_sql_export_without_split_writes_plain_sql: the
|
||||
// fallback export future needs a larger-than-default test stack.
|
||||
let handle = std::thread::Builder::new()
|
||||
.name("table-export-split-sql".to_string())
|
||||
.stack_size(8 * 1024 * 1024)
|
||||
.spawn(|| {
|
||||
tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("build table export split SQL test runtime")
|
||||
.block_on(run_external_driver_sql_export_honors_split_max_mb_into_zip_parts());
|
||||
})
|
||||
.expect("spawn table export split SQL test thread");
|
||||
if let Err(panic) = handle.join() {
|
||||
std::panic::resume_unwind(panic);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
async fn run_external_driver_sql_export_honors_split_max_mb_into_zip_parts() {
|
||||
// Two cursor pages: 200 wide rows (~1.2MB of INSERT statements) then a
|
||||
// short 100-row page. The first page alone must exceed the 1MB part
|
||||
// threshold, forcing the second page into a new zip part, and every
|
||||
// part must still contain only complete statements.
|
||||
let fixture = external_driver_export_fixture(
|
||||
r#"LONG=$(head -c 6000 /dev/zero | tr '\0' 'x')
|
||||
ROWS=''
|
||||
i=1
|
||||
while [ $i -le 100 ]; do ROWS="$ROWS,[$i,\"$LONG\"]"; i=$((i+1)); done
|
||||
ROWS=${ROWS#,}
|
||||
case "$line" in
|
||||
*'"method":"executeQueryPage"'*)
|
||||
echo executeQueryPage >> "$CALLS"
|
||||
printf '{"id":%s,"result":{"columns":["id","name"],"rows":[%s,%s],"affected_rows":0,"execution_time_ms":1,"session_id":"cursor-1","has_more":true}}\n' "$id" "$ROWS" "$ROWS"
|
||||
;;
|
||||
*'"method":"fetchQueryPage"'*)
|
||||
echo fetchQueryPage >> "$CALLS"
|
||||
printf '{"id":%s,"result":{"columns":["id","name"],"rows":[%s],"affected_rows":0,"execution_time_ms":1,"session_id":null,"has_more":false}}\n' "$id" "$ROWS"
|
||||
;;
|
||||
*'"method":"closeQuerySession"'*)
|
||||
echo closeQuerySession >> "$CALLS"
|
||||
printf '{"id":%s,"result":{"ok":true}}\n' "$id"
|
||||
;;
|
||||
esac"#,
|
||||
200,
|
||||
None,
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
let zip_path = fixture.dir.join("export.zip");
|
||||
let mut request = fixture.request.clone();
|
||||
request.format = "sql".to_string();
|
||||
request.file_path = zip_path.to_string_lossy().into_owned();
|
||||
request.split_max_mb = Some(crate::export_split_zip::MIN_SPLIT_PART_MAX_MB);
|
||||
|
||||
let terminal = Arc::new(std::sync::Mutex::new(None::<ExportStatus>));
|
||||
let captured = terminal.clone();
|
||||
export_table_data_core(&fixture.state, &request, move |event| {
|
||||
*captured.lock().unwrap() = Some(event.status);
|
||||
})
|
||||
.await
|
||||
.expect("split JDBC SQL export should succeed");
|
||||
|
||||
// The final page reports `has_more: false` with no session id, so the
|
||||
// cursor is closed implicitly and no closeQuerySession RPC is issued.
|
||||
assert_eq!(std::fs::read_to_string(&fixture.calls).unwrap(), "executeQueryPage\nfetchQueryPage\n");
|
||||
let file = std::fs::File::open(&zip_path).unwrap();
|
||||
let mut archive = zip::ZipArchive::new(file).unwrap();
|
||||
let mut sql_parts = Vec::new();
|
||||
let mut manifest = None;
|
||||
for index in 0..archive.len() {
|
||||
let mut entry = archive.by_index(index).unwrap();
|
||||
if entry.name() == "manifest.json" {
|
||||
let mut contents = Vec::new();
|
||||
entry.read_to_end(&mut contents).unwrap();
|
||||
manifest = Some(serde_json::from_slice::<serde_json::Value>(&contents).unwrap());
|
||||
continue;
|
||||
}
|
||||
let mut contents = String::new();
|
||||
entry.read_to_string(&mut contents).unwrap();
|
||||
sql_parts.push((entry.name().to_string(), contents));
|
||||
}
|
||||
sql_parts.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
|
||||
assert!(
|
||||
sql_parts.len() >= 2,
|
||||
"expected the fallback SQL export to be split into multiple parts, got {}",
|
||||
sql_parts.len()
|
||||
);
|
||||
for (name, contents) in &sql_parts {
|
||||
assert!(!contents.is_empty(), "{name} must not be empty");
|
||||
// Every non-blank line must be a complete statement -- the part
|
||||
// boundary may only land between the fallback arm's per-batch
|
||||
// writes, never inside a statement.
|
||||
for line in contents.lines().filter(|line| !line.trim().is_empty()) {
|
||||
assert!(
|
||||
line.trim_start().starts_with("INSERT INTO") && line.trim_end().ends_with(';'),
|
||||
"{name} has a malformed line from a mid-statement cut"
|
||||
);
|
||||
}
|
||||
}
|
||||
// All 300 rows must be present across the parts: every row value is
|
||||
// `(n, 'xxxx…')`, so one `, '` occurrence per row.
|
||||
let combined: String = sql_parts.iter().map(|(_, contents)| contents.as_str()).collect();
|
||||
assert_eq!(combined.matches(", '").count(), 300);
|
||||
|
||||
let manifest = manifest.expect("split export must contain a manifest");
|
||||
assert_eq!(manifest["totalParts"].as_u64().unwrap() as usize, sql_parts.len());
|
||||
assert_eq!(manifest["sourceFileName"], "EXPORT_SAMPLE.sql");
|
||||
assert!(matches!(*terminal.lock().unwrap(), Some(ExportStatus::Done)));
|
||||
|
||||
cleanup_external_driver_export_fixture(fixture);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn external_driver_table_export_does_not_repeat_legacy_one_shot_results() {
|
||||
|
||||
Reference in new issue
Block a user