feat(export): support single row insert mode in database export
This commit is contained in:
13 files changed
+103
-9
No files matched your search
@@ -150,6 +150,8 @@ pub struct DatabaseExportRequest {
|
||||
#[serde(default)]
|
||||
pub insert_dialect: SqlInsertDialect,
|
||||
#[serde(default)]
|
||||
pub insert_mode: SqlInsertMode,
|
||||
#[serde(default)]
|
||||
pub snapshot_session_id: Option<String>,
|
||||
pub batch_size: usize,
|
||||
/// When set, the export is packaged as a `.zip` archive containing
|
||||
@@ -2619,6 +2621,7 @@ fn database_export_select_sql(
|
||||
format!("SELECT {columns} FROM {table}")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn write_database_export_rows<W: Write>(
|
||||
file: &mut W,
|
||||
rows: &[Vec<Value>],
|
||||
@@ -2629,6 +2632,32 @@ fn write_database_export_rows<W: Write>(
|
||||
schema: &str,
|
||||
db_type: &DatabaseType,
|
||||
insert_dialect: SqlInsertDialect,
|
||||
) -> Result<(), String> {
|
||||
write_database_export_rows_with_mode(
|
||||
file,
|
||||
rows,
|
||||
columns,
|
||||
column_types,
|
||||
column_extras,
|
||||
table,
|
||||
schema,
|
||||
db_type,
|
||||
insert_dialect,
|
||||
SqlInsertMode::default(),
|
||||
)
|
||||
}
|
||||
|
||||
fn write_database_export_rows_with_mode<W: Write>(
|
||||
file: &mut W,
|
||||
rows: &[Vec<Value>],
|
||||
columns: &[String],
|
||||
column_types: &[Option<String>],
|
||||
column_extras: &[Option<String>],
|
||||
table: &str,
|
||||
schema: &str,
|
||||
db_type: &DatabaseType,
|
||||
insert_dialect: SqlInsertDialect,
|
||||
insert_mode: SqlInsertMode,
|
||||
) -> Result<(), String> {
|
||||
let insert_indices = columns
|
||||
.iter()
|
||||
@@ -2691,7 +2720,7 @@ fn write_database_export_rows<W: Write>(
|
||||
spatial_columns: Vec::new(),
|
||||
spatial_values: Vec::new(),
|
||||
rows: insert_rows.to_vec(),
|
||||
batch_size: Some(DATABASE_EXPORT_INSERT_BATCH_SIZE),
|
||||
batch_size: Some(insert_mode.batch_size(DATABASE_EXPORT_INSERT_BATCH_SIZE)),
|
||||
},
|
||||
&[],
|
||||
insert_dialect,
|
||||
@@ -4126,7 +4155,7 @@ async fn export_database_sql_core_inner(
|
||||
if snapshot_batch_cancelled(&db_type, &request.export_id) {
|
||||
return Err(EXPORT_CANCELLED_ERROR.to_string());
|
||||
}
|
||||
write_database_export_rows(
|
||||
write_database_export_rows_with_mode(
|
||||
&mut file,
|
||||
&rows,
|
||||
&col_names,
|
||||
@@ -4136,6 +4165,7 @@ async fn export_database_sql_core_inner(
|
||||
&request.schema,
|
||||
&db_type,
|
||||
request.insert_dialect,
|
||||
request.insert_mode,
|
||||
)?;
|
||||
total_rows_exported += rows.len() as u64;
|
||||
on_progress(ExportProgress {
|
||||
@@ -4243,7 +4273,7 @@ async fn export_database_sql_core_inner(
|
||||
if row_count == 0 {
|
||||
break;
|
||||
}
|
||||
write_database_export_rows(
|
||||
write_database_export_rows_with_mode(
|
||||
&mut file,
|
||||
&result.rows,
|
||||
&col_names,
|
||||
@@ -4253,6 +4283,7 @@ async fn export_database_sql_core_inner(
|
||||
&request.schema,
|
||||
&db_type,
|
||||
request.insert_dialect,
|
||||
request.insert_mode,
|
||||
)?;
|
||||
total_rows_exported += row_count as u64;
|
||||
if use_keyset {
|
||||
@@ -4650,11 +4681,11 @@ mod tests {
|
||||
mysql_view_dependencies_from_rows, mysql_view_dependencies_sql, normalize_export_table_ddl,
|
||||
record_export_destination_identity, record_export_error, replace_database_export_select_list,
|
||||
sort_export_views_by_dependencies, split_postgres_export_table_triggers, write_database_export_rows,
|
||||
BuildDatabaseSqlExportOptions, BuildExportInsertStatementsOptions, BuildExportSqlInsertOptions,
|
||||
DatabaseExportObjectCounts, DatabaseExportRequest, DatabaseExportWriter, DdlNormalizeOptions, ExportedTableSql,
|
||||
PostgresExportEnum, PostgresExportExtension, PostgresExportSequence, PostgresExtensionMembers,
|
||||
SqlInsertDialect, DATABASE_EXPORT_INSERT_BATCH_SIZE, DATABASE_EXPORT_ROW_LIMIT,
|
||||
POSTGRES_EXPORT_SEQUENCES_COMPAT_SQL, POSTGRES_EXPORT_SEQUENCES_SQL,
|
||||
write_database_export_rows_with_mode, BuildDatabaseSqlExportOptions, BuildExportInsertStatementsOptions,
|
||||
BuildExportSqlInsertOptions, DatabaseExportObjectCounts, DatabaseExportRequest, DatabaseExportWriter,
|
||||
DdlNormalizeOptions, ExportedTableSql, PostgresExportEnum, PostgresExportExtension, PostgresExportSequence,
|
||||
PostgresExtensionMembers, SqlInsertDialect, SqlInsertMode, DATABASE_EXPORT_INSERT_BATCH_SIZE,
|
||||
DATABASE_EXPORT_ROW_LIMIT, POSTGRES_EXPORT_SEQUENCES_COMPAT_SQL, POSTGRES_EXPORT_SEQUENCES_SQL,
|
||||
};
|
||||
use super::{ExportProgress, LenientExportErrors};
|
||||
use crate::connection::AppState;
|
||||
@@ -4855,6 +4886,7 @@ mod tests {
|
||||
prevent_overwrite: false,
|
||||
output_compression: Default::default(),
|
||||
insert_dialect: Default::default(),
|
||||
insert_mode: Default::default(),
|
||||
snapshot_session_id: None,
|
||||
batch_size: 1000,
|
||||
split_max_mb: None,
|
||||
@@ -6908,6 +6940,33 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn database_row_writer_supports_one_insert_statement_per_row() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let path = directory.path().join("users.sql");
|
||||
let mut file = std::fs::File::create(&path).unwrap();
|
||||
|
||||
write_database_export_rows_with_mode(
|
||||
&mut file,
|
||||
&[vec![json!(1), json!("Ada")], vec![json!(2), json!("Linus")]],
|
||||
&["id".to_string(), "name".to_string()],
|
||||
&[Some("int".to_string()), Some("varchar(32)".to_string())],
|
||||
&[None, None],
|
||||
"users",
|
||||
"app",
|
||||
&DatabaseType::Mysql,
|
||||
SqlInsertDialect::Source,
|
||||
SqlInsertMode::Single,
|
||||
)
|
||||
.unwrap();
|
||||
drop(file);
|
||||
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(path).unwrap(),
|
||||
"INSERT INTO `users` (`id`, `name`) VALUES (1, 'Ada');\n\nINSERT INTO `users` (`id`, `name`) VALUES (2, 'Linus');\n\n"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_zip_export_writer_splits_across_insert_batches_into_valid_sql() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
|
||||
@@ -339,6 +339,7 @@ impl BackupService {
|
||||
DatabaseExportOutputCompression::None
|
||||
},
|
||||
insert_dialect: Default::default(),
|
||||
insert_mode: Default::default(),
|
||||
snapshot_session_id: Some(snapshot.session_id.clone()),
|
||||
batch_size: 1000,
|
||||
split_max_mb: None,
|
||||
|
||||
@@ -139,6 +139,7 @@ async fn run_database_export_of_partition_tree_has_no_duplicates_and_replays() {
|
||||
prevent_overwrite: false,
|
||||
output_compression: Default::default(),
|
||||
insert_dialect: Default::default(),
|
||||
insert_mode: Default::default(),
|
||||
snapshot_session_id: None,
|
||||
batch_size: 1000,
|
||||
split_max_mb: None,
|
||||
|
||||
@@ -78,6 +78,7 @@ async fn run_database_export_writes_structure_and_data_for_all_tables() {
|
||||
prevent_overwrite: false,
|
||||
output_compression: Default::default(),
|
||||
insert_dialect: Default::default(),
|
||||
insert_mode: Default::default(),
|
||||
snapshot_session_id: None,
|
||||
batch_size: 1000,
|
||||
split_max_mb: None,
|
||||
|
||||
@@ -331,6 +331,7 @@ async fn live_postgres_backup_snapshot_exports_wide_jsonb_then_next_table_inner(
|
||||
prevent_overwrite: false,
|
||||
output_compression: Default::default(),
|
||||
insert_dialect: Default::default(),
|
||||
insert_mode: Default::default(),
|
||||
snapshot_session_id: Some(snapshot.session_id.clone()),
|
||||
batch_size: 1000,
|
||||
split_max_mb: None,
|
||||
@@ -447,6 +448,7 @@ async fn live_postgres_backup_snapshot_cancel_interrupts_pending_row_inner() {
|
||||
prevent_overwrite: false,
|
||||
output_compression: Default::default(),
|
||||
insert_dialect: Default::default(),
|
||||
insert_mode: Default::default(),
|
||||
snapshot_session_id: Some(snapshot.session_id.clone()),
|
||||
batch_size: 1,
|
||||
split_max_mb: None,
|
||||
@@ -565,6 +567,7 @@ async fn live_mysql_database_backup_refreshes_an_idle_snapshot_before_export() {
|
||||
prevent_overwrite: false,
|
||||
output_compression: Default::default(),
|
||||
insert_dialect: Default::default(),
|
||||
insert_mode: Default::default(),
|
||||
snapshot_session_id: Some(snapshot.session_id.clone()),
|
||||
batch_size: 100,
|
||||
split_max_mb: None,
|
||||
|
||||
@@ -154,6 +154,7 @@ async fn live_mysql_database_export_restores_dependent_views() {
|
||||
prevent_overwrite: false,
|
||||
output_compression: Default::default(),
|
||||
insert_dialect: Default::default(),
|
||||
insert_mode: Default::default(),
|
||||
snapshot_session_id: None,
|
||||
batch_size: 1000,
|
||||
split_max_mb: None,
|
||||
@@ -296,6 +297,7 @@ async fn run_live_mysql_database_export_handles_many_tables_including_empty_tabl
|
||||
prevent_overwrite: false,
|
||||
output_compression: Default::default(),
|
||||
insert_dialect: Default::default(),
|
||||
insert_mode: Default::default(),
|
||||
snapshot_session_id: None,
|
||||
batch_size: 1000,
|
||||
split_max_mb: None,
|
||||
@@ -392,6 +394,7 @@ async fn live_mysql_database_export_creates_missing_destination_directory() {
|
||||
prevent_overwrite: false,
|
||||
output_compression: Default::default(),
|
||||
insert_dialect: Default::default(),
|
||||
insert_mode: Default::default(),
|
||||
snapshot_session_id: None,
|
||||
batch_size: 1000,
|
||||
split_max_mb: None,
|
||||
@@ -470,6 +473,7 @@ async fn live_mysql_database_export_refuses_to_recreate_a_destination_that_disap
|
||||
prevent_overwrite: false,
|
||||
output_compression: Default::default(),
|
||||
insert_dialect: Default::default(),
|
||||
insert_mode: Default::default(),
|
||||
snapshot_session_id: None,
|
||||
batch_size: 1000,
|
||||
split_max_mb: None,
|
||||
@@ -559,6 +563,7 @@ async fn live_mysql_database_export_refuses_a_destination_that_vanished_before_i
|
||||
prevent_overwrite: false,
|
||||
output_compression: Default::default(),
|
||||
insert_dialect: Default::default(),
|
||||
insert_mode: Default::default(),
|
||||
snapshot_session_id: None,
|
||||
batch_size: 1000,
|
||||
split_max_mb: None,
|
||||
|
||||
@@ -103,6 +103,7 @@ async fn live_mysql_database_export_table_order_is_not_alphabetical_when_fk_reor
|
||||
prevent_overwrite: false,
|
||||
output_compression: Default::default(),
|
||||
insert_dialect: Default::default(),
|
||||
insert_mode: Default::default(),
|
||||
snapshot_session_id: None,
|
||||
batch_size: 1000,
|
||||
split_max_mb: None,
|
||||
|
||||
@@ -113,6 +113,7 @@ async fn live_postgres_all_schema_export_restores_one_sql_file() {
|
||||
prevent_overwrite: false,
|
||||
output_compression: Default::default(),
|
||||
insert_dialect: Default::default(),
|
||||
insert_mode: Default::default(),
|
||||
snapshot_session_id: None,
|
||||
batch_size: 1000,
|
||||
split_max_mb: None,
|
||||
|
||||
@@ -139,6 +139,7 @@ async fn live_postgres_export_with_foreign_key_cycle_restores_into_empty_databas
|
||||
prevent_overwrite: false,
|
||||
output_compression: Default::default(),
|
||||
insert_dialect: Default::default(),
|
||||
insert_mode: Default::default(),
|
||||
snapshot_session_id: None,
|
||||
batch_size: 1000,
|
||||
split_max_mb: None,
|
||||
|
||||
@@ -137,6 +137,7 @@ async fn manual_e2e_sqlserver_default_split_zip_export_then_import_skip_relation
|
||||
batch_size: 1000,
|
||||
split_max_mb: Some(1),
|
||||
insert_dialect: Default::default(),
|
||||
insert_mode: Default::default(),
|
||||
};
|
||||
|
||||
let progress_log = std::sync::Mutex::new(Vec::new());
|
||||
|
||||
@@ -946,6 +946,7 @@ async fn live_sqlserver_database_export_replays_composite_foreign_keys() {
|
||||
output_compression: Default::default(),
|
||||
snapshot_session_id: None,
|
||||
insert_dialect: Default::default(),
|
||||
insert_mode: Default::default(),
|
||||
batch_size: 1000,
|
||||
split_max_mb: None,
|
||||
};
|
||||
|
||||
Reference in new issue
Block a user