feat(export): support single row insert mode in database export

This commit is contained in:
二丫讲梵 authored and GitHub committed 2026-10-01 16:02:59 +08:00
1 parent 660071870b
commit 5afe0b4248
13 files changed
+103 -9

No files matched your search

@@ -20,7 +20,7 @@ import { Input } from "@/components/ui/input";
import { Download, Square, CheckSquare, Search, X, Loader2, Wrench } from "@lucide/vue"; import { Download, Square, CheckSquare, Search, X, Loader2, Wrench } from "@lucide/vue";
import { formatDataTransferDuration, useExportTracker } from "@/composables/useExportTracker"; import { formatDataTransferDuration, useExportTracker } from "@/composables/useExportTracker";
import { isQueryTimeoutErrorMessage } from "@/lib/sql/queryError"; import { isQueryTimeoutErrorMessage } from "@/lib/sql/queryError";
import type { SqlInsertDialect } from "@/lib/export/sqlInsertMode"; import type { SqlInsertDialect, SqlInsertMode } from "@/lib/export/sqlInsertMode";
const { t } = useI18n(); const { t } = useI18n();
const { toast } = useToast(); const { toast } = useToast();
@@ -66,6 +66,7 @@ const POSTGRES_ALL_SCHEMAS = "__DBX_ALL_SCHEMAS__";
const includeStructure = ref(true); const includeStructure = ref(true);
const includeData = ref(true); const includeData = ref(true);
const insertDialect = ref<SqlInsertDialect>("source"); const insertDialect = ref<SqlInsertDialect>("source");
const insertMode = ref<SqlInsertMode>("batch");
const includeObjects = ref(true); const includeObjects = ref(true);
const includeCreateDatabase = ref(false); const includeCreateDatabase = ref(false);
const dropTableIfExists = ref(false); const dropTableIfExists = ref(false);
@@ -399,6 +400,7 @@ async function startExport() {
includeStructure: includeStructure.value, includeStructure: includeStructure.value,
includeData: includeData.value, includeData: includeData.value,
insertDialect: insertDialect.value, insertDialect: insertDialect.value,
insertMode: insertMode.value,
includeObjects: includeObjects.value, includeObjects: includeObjects.value,
includeCreateDatabase: includeCreateDatabase.value, includeCreateDatabase: includeCreateDatabase.value,
dropTableIfExists: dropTableIfExists.value, dropTableIfExists: dropTableIfExists.value,
@@ -542,6 +544,7 @@ async function startAllDatabasesExport() {
includeStructure: includeStructure.value, includeStructure: includeStructure.value,
includeData: includeData.value, includeData: includeData.value,
insertDialect: insertDialect.value, insertDialect: insertDialect.value,
insertMode: insertMode.value,
includeObjects: includeObjects.value, includeObjects: includeObjects.value,
includeCreateDatabase: includeCreateDatabase.value, includeCreateDatabase: includeCreateDatabase.value,
dropTableIfExists: dropTableIfExists.value, dropTableIfExists: dropTableIfExists.value,
@@ -664,6 +667,7 @@ function resetState() {
includeStructure.value = true; includeStructure.value = true;
includeData.value = true; includeData.value = true;
insertDialect.value = "source"; insertDialect.value = "source";
insertMode.value = "batch";
includeObjects.value = true; includeObjects.value = true;
includeCreateDatabase.value = false; includeCreateDatabase.value = false;
dropTableIfExists.value = false; dropTableIfExists.value = false;
@@ -962,6 +966,20 @@ watch(
{{ t("databaseExport.standardSqlInsertDescription") }} {{ t("databaseExport.standardSqlInsertDescription") }}
</p> </p>
</div> </div>
<div v-if="includeData" class="space-y-1">
<div class="text-xs font-medium">{{ t("grid.sqlInsertModeTitle") }}</div>
<label class="flex cursor-pointer items-center gap-2 text-xs">
<input v-model="insertMode" type="radio" value="batch" class="h-3.5 w-3.5" />
<span>{{ t("grid.sqlInsertModeBatch") }}</span>
</label>
<label class="flex cursor-pointer items-center gap-2 text-xs">
<input v-model="insertMode" type="radio" value="single" class="h-3.5 w-3.5" />
<span>{{ t("grid.sqlInsertModeSingle") }}</span>
</label>
<p class="pl-5 text-[11px] text-muted-foreground">
{{ insertMode === "single" ? t("grid.sqlInsertModeSingleDescription") : t("grid.sqlInsertModeBatchDescription") }}
</p>
</div>
<div class="flex items-center gap-2 cursor-pointer text-xs" @click="includeObjects = !includeObjects"> <div class="flex items-center gap-2 cursor-pointer text-xs" @click="includeObjects = !includeObjects">
<CheckSquare v-if="includeObjects" class="w-3.5 h-3.5 text-primary shrink-0" /> <CheckSquare v-if="includeObjects" class="w-3.5 h-3.5 text-primary shrink-0" />
<Square v-else class="w-3.5 h-3.5 text-muted-foreground/40 shrink-0" /> <Square v-else class="w-3.5 h-3.5 text-muted-foreground/40 shrink-0" />
+1
View File
@@ -5901,6 +5901,7 @@ export interface DatabaseExportRequest {
preventOverwrite?: boolean; preventOverwrite?: boolean;
outputCompression?: "none" | "gzip"; outputCompression?: "none" | "gzip";
insertDialect?: SqlInsertDialect; insertDialect?: SqlInsertDialect;
insertMode?: "batch" | "single";
snapshotSessionId?: string; snapshotSessionId?: string;
batchSize: number; batchSize: number;
splitMaxMb?: number; splitMaxMb?: number;
+67 -8
View File
@@ -150,6 +150,8 @@ pub struct DatabaseExportRequest {
#[serde(default)] #[serde(default)]
pub insert_dialect: SqlInsertDialect, pub insert_dialect: SqlInsertDialect,
#[serde(default)] #[serde(default)]
pub insert_mode: SqlInsertMode,
#[serde(default)]
pub snapshot_session_id: Option<String>, pub snapshot_session_id: Option<String>,
pub batch_size: usize, pub batch_size: usize,
/// When set, the export is packaged as a `.zip` archive containing /// When set, the export is packaged as a `.zip` archive containing
@@ -2619,6 +2621,7 @@ fn database_export_select_sql(
format!("SELECT {columns} FROM {table}") format!("SELECT {columns} FROM {table}")
} }
#[cfg(test)]
fn write_database_export_rows<W: Write>( fn write_database_export_rows<W: Write>(
file: &mut W, file: &mut W,
rows: &[Vec<Value>], rows: &[Vec<Value>],
@@ -2629,6 +2632,32 @@ fn write_database_export_rows<W: Write>(
schema: &str, schema: &str,
db_type: &DatabaseType, db_type: &DatabaseType,
insert_dialect: SqlInsertDialect, insert_dialect: SqlInsertDialect,
) -> Result<(), String> {
write_database_export_rows_with_mode(
file,
rows,
columns,
column_types,
column_extras,
table,
schema,
db_type,
insert_dialect,
SqlInsertMode::default(),
)
}
fn write_database_export_rows_with_mode<W: Write>(
file: &mut W,
rows: &[Vec<Value>],
columns: &[String],
column_types: &[Option<String>],
column_extras: &[Option<String>],
table: &str,
schema: &str,
db_type: &DatabaseType,
insert_dialect: SqlInsertDialect,
insert_mode: SqlInsertMode,
) -> Result<(), String> { ) -> Result<(), String> {
let insert_indices = columns let insert_indices = columns
.iter() .iter()
@@ -2691,7 +2720,7 @@ fn write_database_export_rows<W: Write>(
spatial_columns: Vec::new(), spatial_columns: Vec::new(),
spatial_values: Vec::new(), spatial_values: Vec::new(),
rows: insert_rows.to_vec(), rows: insert_rows.to_vec(),
batch_size: Some(DATABASE_EXPORT_INSERT_BATCH_SIZE), batch_size: Some(insert_mode.batch_size(DATABASE_EXPORT_INSERT_BATCH_SIZE)),
}, },
&[], &[],
insert_dialect, insert_dialect,
@@ -4126,7 +4155,7 @@ async fn export_database_sql_core_inner(
if snapshot_batch_cancelled(&db_type, &request.export_id) { if snapshot_batch_cancelled(&db_type, &request.export_id) {
return Err(EXPORT_CANCELLED_ERROR.to_string()); return Err(EXPORT_CANCELLED_ERROR.to_string());
} }
write_database_export_rows( write_database_export_rows_with_mode(
&mut file, &mut file,
&rows, &rows,
&col_names, &col_names,
@@ -4136,6 +4165,7 @@ async fn export_database_sql_core_inner(
&request.schema, &request.schema,
&db_type, &db_type,
request.insert_dialect, request.insert_dialect,
request.insert_mode,
)?; )?;
total_rows_exported += rows.len() as u64; total_rows_exported += rows.len() as u64;
on_progress(ExportProgress { on_progress(ExportProgress {
@@ -4243,7 +4273,7 @@ async fn export_database_sql_core_inner(
if row_count == 0 { if row_count == 0 {
break; break;
} }
write_database_export_rows( write_database_export_rows_with_mode(
&mut file, &mut file,
&result.rows, &result.rows,
&col_names, &col_names,
@@ -4253,6 +4283,7 @@ async fn export_database_sql_core_inner(
&request.schema, &request.schema,
&db_type, &db_type,
request.insert_dialect, request.insert_dialect,
request.insert_mode,
)?; )?;
total_rows_exported += row_count as u64; total_rows_exported += row_count as u64;
if use_keyset { if use_keyset {
@@ -4650,11 +4681,11 @@ mod tests {
mysql_view_dependencies_from_rows, mysql_view_dependencies_sql, normalize_export_table_ddl, mysql_view_dependencies_from_rows, mysql_view_dependencies_sql, normalize_export_table_ddl,
record_export_destination_identity, record_export_error, replace_database_export_select_list, record_export_destination_identity, record_export_error, replace_database_export_select_list,
sort_export_views_by_dependencies, split_postgres_export_table_triggers, write_database_export_rows, sort_export_views_by_dependencies, split_postgres_export_table_triggers, write_database_export_rows,
BuildDatabaseSqlExportOptions, BuildExportInsertStatementsOptions, BuildExportSqlInsertOptions, write_database_export_rows_with_mode, BuildDatabaseSqlExportOptions, BuildExportInsertStatementsOptions,
DatabaseExportObjectCounts, DatabaseExportRequest, DatabaseExportWriter, DdlNormalizeOptions, ExportedTableSql, BuildExportSqlInsertOptions, DatabaseExportObjectCounts, DatabaseExportRequest, DatabaseExportWriter,
PostgresExportEnum, PostgresExportExtension, PostgresExportSequence, PostgresExtensionMembers, DdlNormalizeOptions, ExportedTableSql, PostgresExportEnum, PostgresExportExtension, PostgresExportSequence,
SqlInsertDialect, DATABASE_EXPORT_INSERT_BATCH_SIZE, DATABASE_EXPORT_ROW_LIMIT, PostgresExtensionMembers, SqlInsertDialect, SqlInsertMode, DATABASE_EXPORT_INSERT_BATCH_SIZE,
POSTGRES_EXPORT_SEQUENCES_COMPAT_SQL, POSTGRES_EXPORT_SEQUENCES_SQL, DATABASE_EXPORT_ROW_LIMIT, POSTGRES_EXPORT_SEQUENCES_COMPAT_SQL, POSTGRES_EXPORT_SEQUENCES_SQL,
}; };
use super::{ExportProgress, LenientExportErrors}; use super::{ExportProgress, LenientExportErrors};
use crate::connection::AppState; use crate::connection::AppState;
@@ -4855,6 +4886,7 @@ mod tests {
prevent_overwrite: false, prevent_overwrite: false,
output_compression: Default::default(), output_compression: Default::default(),
insert_dialect: Default::default(), insert_dialect: Default::default(),
insert_mode: Default::default(),
snapshot_session_id: None, snapshot_session_id: None,
batch_size: 1000, batch_size: 1000,
split_max_mb: None, split_max_mb: None,
@@ -6908,6 +6940,33 @@ mod tests {
); );
} }
#[test]
fn database_row_writer_supports_one_insert_statement_per_row() {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("users.sql");
let mut file = std::fs::File::create(&path).unwrap();
write_database_export_rows_with_mode(
&mut file,
&[vec![json!(1), json!("Ada")], vec![json!(2), json!("Linus")]],
&["id".to_string(), "name".to_string()],
&[Some("int".to_string()), Some("varchar(32)".to_string())],
&[None, None],
"users",
"app",
&DatabaseType::Mysql,
SqlInsertDialect::Source,
SqlInsertMode::Single,
)
.unwrap();
drop(file);
assert_eq!(
std::fs::read_to_string(path).unwrap(),
"INSERT INTO `users` (`id`, `name`) VALUES (1, 'Ada');\n\nINSERT INTO `users` (`id`, `name`) VALUES (2, 'Linus');\n\n"
);
}
#[test] #[test]
fn split_zip_export_writer_splits_across_insert_batches_into_valid_sql() { fn split_zip_export_writer_splits_across_insert_batches_into_valid_sql() {
let directory = tempfile::tempdir().unwrap(); let directory = tempfile::tempdir().unwrap();
@@ -339,6 +339,7 @@ impl BackupService {
DatabaseExportOutputCompression::None DatabaseExportOutputCompression::None
}, },
insert_dialect: Default::default(), insert_dialect: Default::default(),
insert_mode: Default::default(),
snapshot_session_id: Some(snapshot.session_id.clone()), snapshot_session_id: Some(snapshot.session_id.clone()),
batch_size: 1000, batch_size: 1000,
split_max_mb: None, split_max_mb: None,
@@ -139,6 +139,7 @@ async fn run_database_export_of_partition_tree_has_no_duplicates_and_replays() {
prevent_overwrite: false, prevent_overwrite: false,
output_compression: Default::default(), output_compression: Default::default(),
insert_dialect: Default::default(), insert_dialect: Default::default(),
insert_mode: Default::default(),
snapshot_session_id: None, snapshot_session_id: None,
batch_size: 1000, batch_size: 1000,
split_max_mb: None, split_max_mb: None,
@@ -78,6 +78,7 @@ async fn run_database_export_writes_structure_and_data_for_all_tables() {
prevent_overwrite: false, prevent_overwrite: false,
output_compression: Default::default(), output_compression: Default::default(),
insert_dialect: Default::default(), insert_dialect: Default::default(),
insert_mode: Default::default(),
snapshot_session_id: None, snapshot_session_id: None,
batch_size: 1000, batch_size: 1000,
split_max_mb: None, split_max_mb: None,
@@ -331,6 +331,7 @@ async fn live_postgres_backup_snapshot_exports_wide_jsonb_then_next_table_inner(
prevent_overwrite: false, prevent_overwrite: false,
output_compression: Default::default(), output_compression: Default::default(),
insert_dialect: Default::default(), insert_dialect: Default::default(),
insert_mode: Default::default(),
snapshot_session_id: Some(snapshot.session_id.clone()), snapshot_session_id: Some(snapshot.session_id.clone()),
batch_size: 1000, batch_size: 1000,
split_max_mb: None, split_max_mb: None,
@@ -447,6 +448,7 @@ async fn live_postgres_backup_snapshot_cancel_interrupts_pending_row_inner() {
prevent_overwrite: false, prevent_overwrite: false,
output_compression: Default::default(), output_compression: Default::default(),
insert_dialect: Default::default(), insert_dialect: Default::default(),
insert_mode: Default::default(),
snapshot_session_id: Some(snapshot.session_id.clone()), snapshot_session_id: Some(snapshot.session_id.clone()),
batch_size: 1, batch_size: 1,
split_max_mb: None, split_max_mb: None,
@@ -565,6 +567,7 @@ async fn live_mysql_database_backup_refreshes_an_idle_snapshot_before_export() {
prevent_overwrite: false, prevent_overwrite: false,
output_compression: Default::default(), output_compression: Default::default(),
insert_dialect: Default::default(), insert_dialect: Default::default(),
insert_mode: Default::default(),
snapshot_session_id: Some(snapshot.session_id.clone()), snapshot_session_id: Some(snapshot.session_id.clone()),
batch_size: 100, batch_size: 100,
split_max_mb: None, split_max_mb: None,
@@ -154,6 +154,7 @@ async fn live_mysql_database_export_restores_dependent_views() {
prevent_overwrite: false, prevent_overwrite: false,
output_compression: Default::default(), output_compression: Default::default(),
insert_dialect: Default::default(), insert_dialect: Default::default(),
insert_mode: Default::default(),
snapshot_session_id: None, snapshot_session_id: None,
batch_size: 1000, batch_size: 1000,
split_max_mb: None, split_max_mb: None,
@@ -296,6 +297,7 @@ async fn run_live_mysql_database_export_handles_many_tables_including_empty_tabl
prevent_overwrite: false, prevent_overwrite: false,
output_compression: Default::default(), output_compression: Default::default(),
insert_dialect: Default::default(), insert_dialect: Default::default(),
insert_mode: Default::default(),
snapshot_session_id: None, snapshot_session_id: None,
batch_size: 1000, batch_size: 1000,
split_max_mb: None, split_max_mb: None,
@@ -392,6 +394,7 @@ async fn live_mysql_database_export_creates_missing_destination_directory() {
prevent_overwrite: false, prevent_overwrite: false,
output_compression: Default::default(), output_compression: Default::default(),
insert_dialect: Default::default(), insert_dialect: Default::default(),
insert_mode: Default::default(),
snapshot_session_id: None, snapshot_session_id: None,
batch_size: 1000, batch_size: 1000,
split_max_mb: None, split_max_mb: None,
@@ -470,6 +473,7 @@ async fn live_mysql_database_export_refuses_to_recreate_a_destination_that_disap
prevent_overwrite: false, prevent_overwrite: false,
output_compression: Default::default(), output_compression: Default::default(),
insert_dialect: Default::default(), insert_dialect: Default::default(),
insert_mode: Default::default(),
snapshot_session_id: None, snapshot_session_id: None,
batch_size: 1000, batch_size: 1000,
split_max_mb: None, split_max_mb: None,
@@ -559,6 +563,7 @@ async fn live_mysql_database_export_refuses_a_destination_that_vanished_before_i
prevent_overwrite: false, prevent_overwrite: false,
output_compression: Default::default(), output_compression: Default::default(),
insert_dialect: Default::default(), insert_dialect: Default::default(),
insert_mode: Default::default(),
snapshot_session_id: None, snapshot_session_id: None,
batch_size: 1000, batch_size: 1000,
split_max_mb: None, split_max_mb: None,
@@ -103,6 +103,7 @@ async fn live_mysql_database_export_table_order_is_not_alphabetical_when_fk_reor
prevent_overwrite: false, prevent_overwrite: false,
output_compression: Default::default(), output_compression: Default::default(),
insert_dialect: Default::default(), insert_dialect: Default::default(),
insert_mode: Default::default(),
snapshot_session_id: None, snapshot_session_id: None,
batch_size: 1000, batch_size: 1000,
split_max_mb: None, split_max_mb: None,
@@ -113,6 +113,7 @@ async fn live_postgres_all_schema_export_restores_one_sql_file() {
prevent_overwrite: false, prevent_overwrite: false,
output_compression: Default::default(), output_compression: Default::default(),
insert_dialect: Default::default(), insert_dialect: Default::default(),
insert_mode: Default::default(),
snapshot_session_id: None, snapshot_session_id: None,
batch_size: 1000, batch_size: 1000,
split_max_mb: None, split_max_mb: None,
@@ -139,6 +139,7 @@ async fn live_postgres_export_with_foreign_key_cycle_restores_into_empty_databas
prevent_overwrite: false, prevent_overwrite: false,
output_compression: Default::default(), output_compression: Default::default(),
insert_dialect: Default::default(), insert_dialect: Default::default(),
insert_mode: Default::default(),
snapshot_session_id: None, snapshot_session_id: None,
batch_size: 1000, batch_size: 1000,
split_max_mb: None, split_max_mb: None,
@@ -137,6 +137,7 @@ async fn manual_e2e_sqlserver_default_split_zip_export_then_import_skip_relation
batch_size: 1000, batch_size: 1000,
split_max_mb: Some(1), split_max_mb: Some(1),
insert_dialect: Default::default(), insert_dialect: Default::default(),
insert_mode: Default::default(),
}; };
let progress_log = std::sync::Mutex::new(Vec::new()); let progress_log = std::sync::Mutex::new(Vec::new());
@@ -946,6 +946,7 @@ async fn live_sqlserver_database_export_replays_composite_foreign_keys() {
output_compression: Default::default(), output_compression: Default::default(),
snapshot_session_id: None, snapshot_session_id: None,
insert_dialect: Default::default(), insert_dialect: Default::default(),
insert_mode: Default::default(),
batch_size: 1000, batch_size: 1000,
split_max_mb: None, split_max_mb: None,
}; };