diff --git a/apps/desktop/src/components/export/DatabaseExportDialog.vue b/apps/desktop/src/components/export/DatabaseExportDialog.vue index 5286d05658..dd4bcc6efb 100644 --- a/apps/desktop/src/components/export/DatabaseExportDialog.vue +++ b/apps/desktop/src/components/export/DatabaseExportDialog.vue @@ -74,6 +74,7 @@ const includeObjects = ref(true); const includeCreateDatabase = ref(false); const dropTableIfExists = ref(false); const omitAutoIncrement = ref(false); +const preserveOriginalLanguage = ref(false); const splitSqlOutput = ref(false); const splitSqlPartMaxMb = ref(100); const MIN_SPLIT_SQL_PART_MB = 1; @@ -424,6 +425,7 @@ async function startExport() { includeCreateDatabase: includeCreateDatabase.value, dropTableIfExists: dropTableIfExists.value, omitAutoIncrement: omitAutoIncrement.value, + preserveOriginalLanguage: preserveOriginalLanguage.value, snapshotSessionId, batchSize: 1000, splitMaxMb: splitSqlOutput.value ? normalizedSplitSqlPartMaxMb() : undefined, @@ -569,6 +571,7 @@ async function startAllDatabasesExport() { includeCreateDatabase: includeCreateDatabase.value, dropTableIfExists: dropTableIfExists.value, omitAutoIncrement: omitAutoIncrement.value, + preserveOriginalLanguage: preserveOriginalLanguage.value, snapshotSessionId, batchSize: 1000, splitMaxMb: splitSqlOutput.value ? normalizedSplitSqlPartMaxMb() : undefined, @@ -692,6 +695,7 @@ function resetState() { includeCreateDatabase.value = false; dropTableIfExists.value = false; omitAutoIncrement.value = false; + preserveOriginalLanguage.value = false; splitSqlOutput.value = false; splitSqlPartMaxMb.value = 100; isExporting.value = false; @@ -972,6 +976,11 @@ watch( {{ t("databaseExport.omitAutoIncrement") }} +
+ + + {{ t("databaseExport.preserveOriginalLanguage") }} +
diff --git a/apps/desktop/src/i18n/locales/az.ts b/apps/desktop/src/i18n/locales/az.ts index 0f5015d230..9df9f69356 100644 --- a/apps/desktop/src/i18n/locales/az.ts +++ b/apps/desktop/src/i18n/locales/az.ts @@ -8460,6 +8460,7 @@ export default withEnglishFallback({ includeCreateDatabase: "Verilənlər bazasının qurulmasını daxil et (CREATE DATABASE / USE)", dropTableIfExists: "DDL-dən əvvəl DROP TABLE IF EXISTS ifadəsini əlavə et", omitAutoIncrement: "AUTO_INCREMENT parametrini daxil etmə (yeni quraşdırma skriptləri üçün)", + preserveOriginalLanguage: "SQL-də orijinal dili saxla (\\uXXXX kaçış ardıcıllıqlarını deşifr et)", includeData: "Cədvəl məlumatları (INSERT)", sqlColumnSelection: "İxrac ediləcək sütunlar", selectedColumns: "{selected}/{total} sütun seçilib", diff --git a/apps/desktop/src/i18n/locales/en.ts b/apps/desktop/src/i18n/locales/en.ts index b949676b3d..f46ceb04d2 100644 --- a/apps/desktop/src/i18n/locales/en.ts +++ b/apps/desktop/src/i18n/locales/en.ts @@ -9435,6 +9435,7 @@ export default { includeCreateDatabase: "Include database setup (CREATE DATABASE / USE)", dropTableIfExists: "Add DROP TABLE IF EXISTS before DDL", omitAutoIncrement: "Omit AUTO_INCREMENT (for fresh-install scripts)", + preserveOriginalLanguage: "Preserve original language in SQL (decode \\uXXXX escapes)", includeData: "Table data (INSERT)", sqlColumnSelection: "Columns to export", selectedColumns: "{selected}/{total} columns selected", diff --git a/apps/desktop/src/i18n/locales/es.ts b/apps/desktop/src/i18n/locales/es.ts index c65ae02457..1d00a9d4a8 100644 --- a/apps/desktop/src/i18n/locales/es.ts +++ b/apps/desktop/src/i18n/locales/es.ts @@ -8776,6 +8776,7 @@ export default withEnglishFallback({ includeCreateDatabase: "Incluir creación de base de datos (CREATE DATABASE / USE)", dropTableIfExists: "Agregar DROP TABLE IF EXISTS antes del DDL", omitAutoIncrement: "Omitir AUTO_INCREMENT (para scripts de instalación nueva)", + preserveOriginalLanguage: "Preservar el idioma original en SQL (decodificar escapes \\uXXXX)", includeData: "Datos de tablas (INSERT)", sqlColumnSelection: "Columnas para exportar", selectedColumns: "{selected}/{total} columnas seleccionadas", diff --git a/apps/desktop/src/i18n/locales/id.ts b/apps/desktop/src/i18n/locales/id.ts index fd16cdb041..a64fc9eba0 100644 --- a/apps/desktop/src/i18n/locales/id.ts +++ b/apps/desktop/src/i18n/locales/id.ts @@ -9264,6 +9264,7 @@ export default withEnglishFallback({ includeCreateDatabase: "Sertakan pengaturan database (CREATE DATABASE / USE)", dropTableIfExists: "Tambahkan DROP TABLE IF EXISTS sebelum DDL", omitAutoIncrement: "Abaikan AUTO_INCREMENT (untuk skrip instalasi baru)", + preserveOriginalLanguage: "Pertahankan bahasa asli dalam SQL (dekode escape \\uXXXX)", includeData: "Data tabel (INSERT)", sqlColumnSelection: "Kolom yang diekspor", selectedColumns: "{selected}/{total} kolom dipilih", diff --git a/apps/desktop/src/i18n/locales/it.ts b/apps/desktop/src/i18n/locales/it.ts index 0d68d5c6ac..42002889b2 100644 --- a/apps/desktop/src/i18n/locales/it.ts +++ b/apps/desktop/src/i18n/locales/it.ts @@ -8523,6 +8523,7 @@ export default withEnglishFallback({ includeCreateDatabase: "Includi creazione database (CREATE DATABASE / USE)", dropTableIfExists: "Aggiungi DROP TABLE IF EXISTS prima del DDL", omitAutoIncrement: "Ometti AUTO_INCREMENT (per script di installazione pulita)", + preserveOriginalLanguage: "Mantieni la lingua originale nell'SQL (decodifica gli escape \\uXXXX)", includeData: "Dati tabella (INSERT)", sqlColumnSelection: "Colonne da esportare", selectedColumns: "{selected}/{total} colonne selezionate", diff --git a/apps/desktop/src/i18n/locales/ja.ts b/apps/desktop/src/i18n/locales/ja.ts index 01d62237fe..ab2457c18f 100644 --- a/apps/desktop/src/i18n/locales/ja.ts +++ b/apps/desktop/src/i18n/locales/ja.ts @@ -8564,6 +8564,7 @@ export default withEnglishFallback({ includeCreateDatabase: "データベース作成文を含める(CREATE DATABASE / USE)", dropTableIfExists: "DDLの前にDROP TABLE IF EXISTSを追加", omitAutoIncrement: "AUTO_INCREMENT を省略(新規インストール用スクリプト向け)", + preserveOriginalLanguage: "元の言語を SQL に保持する(\\uXXXX エスケープをデコード)", includeData: "テーブルデータ (INSERT)", sqlColumnSelection: "エクスポートする列", selectedColumns: "{selected}/{total} 列を選択", diff --git a/apps/desktop/src/i18n/locales/ko.ts b/apps/desktop/src/i18n/locales/ko.ts index 1fbf337e4b..aeca00e7fa 100644 --- a/apps/desktop/src/i18n/locales/ko.ts +++ b/apps/desktop/src/i18n/locales/ko.ts @@ -8292,6 +8292,7 @@ export default withEnglishFallback({ includeCreateDatabase: "데이터베이스 설정 포함 (CREATE DATABASE / USE)", dropTableIfExists: "DDL 앞에 DROP TABLE IF EXISTS 추가", omitAutoIncrement: "AUTO_INCREMENT 생략 (초기 설치 스크립트용)", + preserveOriginalLanguage: "SQL에 원본 언어 유지 (\\uXXXX 이스케이프 디코딩)", includeData: "테이블 데이터 (INSERT)", sqlColumnSelection: "내보낼 열", selectedColumns: "{selected}/{total}개 열 선택됨", diff --git a/apps/desktop/src/i18n/locales/pt-BR.ts b/apps/desktop/src/i18n/locales/pt-BR.ts index c436387f6b..69bc430b2e 100644 --- a/apps/desktop/src/i18n/locales/pt-BR.ts +++ b/apps/desktop/src/i18n/locales/pt-BR.ts @@ -8522,6 +8522,7 @@ export default withEnglishFallback({ includeCreateDatabase: "Incluir criação do banco (CREATE DATABASE / USE)", dropTableIfExists: "Adicionar DROP TABLE IF EXISTS antes do DDL", omitAutoIncrement: "Omitir AUTO_INCREMENT (para scripts de instalação do zero)", + preserveOriginalLanguage: "Preservar o idioma original no SQL (decodificar escapes \\uXXXX)", includeData: "Dados da tabela (INSERT)", sqlColumnSelection: "Colunas para exportar", selectedColumns: "{selected}/{total} colunas selecionadas", diff --git a/apps/desktop/src/i18n/locales/ru.ts b/apps/desktop/src/i18n/locales/ru.ts index 32f8b78545..ac69959daf 100644 --- a/apps/desktop/src/i18n/locales/ru.ts +++ b/apps/desktop/src/i18n/locales/ru.ts @@ -9741,6 +9741,7 @@ export default withEnglishFallback({ includeCreateDatabase: "Включить создание базы данных (CREATE DATABASE / USE)", dropTableIfExists: "Добавлять DROP TABLE IF EXISTS перед DDL", omitAutoIncrement: "Пропускать AUTO_INCREMENT (для скриптов чистой установки)", + preserveOriginalLanguage: "Сохранять исходный язык в SQL (декодировать escape-последовательности \\uXXXX)", includeData: "Данные таблиц (INSERT)", sqlColumnSelection: "Столбцы для экспорта", selectedColumns: "Выбрано столбцов: {selected}/{total}", diff --git a/apps/desktop/src/i18n/locales/tr.ts b/apps/desktop/src/i18n/locales/tr.ts index 02709c6f18..65282252ec 100644 --- a/apps/desktop/src/i18n/locales/tr.ts +++ b/apps/desktop/src/i18n/locales/tr.ts @@ -8332,6 +8332,7 @@ export default withEnglishFallback({ includeCreateDatabase: "Veritabanı kurulumunu dâhil et (CREATE DATABASE / USE)", dropTableIfExists: "DDL'den önce DROP TABLE IF EXISTS ekle", omitAutoIncrement: "AUTO_INCREMENT'i atla (sıfırdan kurulum betikleri için)", + preserveOriginalLanguage: "SQL'de orijinal dili koru (\\uXXXX kaçış dizilerini çöz)", includeData: "Tablo verisi (INSERT)", sqlColumnSelection: "Dışa aktarılacak sütunlar", selectedColumns: "{selected}/{total} sütun seçildi", diff --git a/apps/desktop/src/i18n/locales/zh-CN.ts b/apps/desktop/src/i18n/locales/zh-CN.ts index ac02076efd..ba44548fa9 100644 --- a/apps/desktop/src/i18n/locales/zh-CN.ts +++ b/apps/desktop/src/i18n/locales/zh-CN.ts @@ -9417,6 +9417,7 @@ export default withEnglishFallback({ includeCreateDatabase: "导出建库语句(CREATE DATABASE / USE)", dropTableIfExists: "导出前添加 DROP TABLE IF EXISTS", omitAutoIncrement: "省略 AUTO_INCREMENT(用于全新安装的初始化脚本)", + preserveOriginalLanguage: "在 SQL 中保留原始语言(解码 \\uXXXX 转义)", includeData: "表数据 (INSERT)", sqlColumnSelection: "要导出的列", selectedColumns: "已选择 {selected}/{total} 列", diff --git a/apps/desktop/src/i18n/locales/zh-TW.ts b/apps/desktop/src/i18n/locales/zh-TW.ts index 61c9cf299f..e064f89d56 100644 --- a/apps/desktop/src/i18n/locales/zh-TW.ts +++ b/apps/desktop/src/i18n/locales/zh-TW.ts @@ -7876,6 +7876,7 @@ export default withEnglishFallback({ includeCreateDatabase: "匯出建庫語句(CREATE DATABASE / USE)", dropTableIfExists: "匯出前新增 DROP TABLE IF EXISTS", omitAutoIncrement: "省略 AUTO_INCREMENT(用於全新安裝的初始化腳本)", + preserveOriginalLanguage: "在 SQL 中保留原始語言(解碼 \\uXXXX 轉義)", includeData: "資料表資料 (INSERT)", sqlColumnSelection: "要匯出的欄位", selectedColumns: "已選取 {selected}/{total} 個欄位", diff --git a/apps/desktop/src/lib/backend/tauri.ts b/apps/desktop/src/lib/backend/tauri.ts index 6609819408..0cca5b399f 100644 --- a/apps/desktop/src/lib/backend/tauri.ts +++ b/apps/desktop/src/lib/backend/tauri.ts @@ -5917,6 +5917,7 @@ export interface DatabaseExportRequest { includeCreateDatabase?: boolean; dropTableIfExists?: boolean; omitAutoIncrement?: boolean; + preserveOriginalLanguage?: boolean; failOnError?: boolean; preventOverwrite?: boolean; outputCompression?: "none" | "gzip"; diff --git a/crates/dbx-core/examples/data_transfer_bench.rs b/crates/dbx-core/examples/data_transfer_bench.rs index fb785465a7..9181cbd812 100644 --- a/crates/dbx-core/examples/data_transfer_bench.rs +++ b/crates/dbx-core/examples/data_transfer_bench.rs @@ -104,6 +104,7 @@ fn run() -> Result<(), String> { spatial_values: Vec::new(), rows: rows.clone(), batch_size: Some(options.batch_size), + preserve_original_language: false, })?; let sql_ms = elapsed_ms(sql_started); diff --git a/crates/dbx-core/src/data/database_export.rs b/crates/dbx-core/src/data/database_export.rs index f67f82bfaf..1c43cfa9c1 100644 --- a/crates/dbx-core/src/data/database_export.rs +++ b/crates/dbx-core/src/data/database_export.rs @@ -139,6 +139,14 @@ pub struct DatabaseExportRequest { /// position. No-op for non-MySQL databases. Defaults to `false` (preserve). #[serde(default)] pub omit_auto_increment: bool, + /// Decode JSON-style `\uXXXX` Unicode escapes back into literal characters + /// in exported string literals, so values an application deliberately + /// ASCII-escaped before storing (e.g. to keep Vietnamese text safe in a + /// non-Unicode TEXT column) read as the original language directly in the + /// `.sql` file. Opt-in; defaults to `false` to preserve the exact stored + /// bytes on export/import round-trips. + #[serde(default)] + pub preserve_original_language: bool, #[serde(default)] pub fail_on_error: bool, /// Refuse to truncate an existing destination. Scheduled backups enable @@ -537,6 +545,19 @@ pub struct BuildExportInsertStatementsOptions { pub rows: Vec>, #[serde(default, skip_serializing_if = "Option::is_none")] pub batch_size: Option, + /// Decode JSON-style `\uXXXX` Unicode escapes back into literal characters + /// in exported string literals (e.g. `\u1ed0` -> `ố`). Opt-in and defaults + /// to `false`: application code may deliberately store ASCII-escaped + /// Unicode in a TEXT column (see `decode_unicode_escapes_for_export`), and + /// the default export keeps that exact byte-for-byte round trip. Enable + /// this only to make the `.sql` file itself readable in the original + /// language. Re-importing such an export stores the decoded literal + /// characters rather than the original ASCII-escaped form, so it is not a + /// byte-faithful backup; any `\uXXXX`-shaped run forming a valid hex code + /// point (e.g. `\uface`) is decoded as well, including text that merely + /// documents such escapes. + #[serde(default)] + pub preserve_original_language: bool, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -578,6 +599,9 @@ pub struct BuildDatabaseSqlExportOptions { /// Defaults to `false` (preserve). See `DatabaseExportRequest::omit_auto_increment`. #[serde(default)] pub omit_auto_increment: bool, + /// See `DatabaseExportRequest::preserve_original_language`. + #[serde(default)] + pub preserve_original_language: bool, } pub fn format_export_sql_literal(value: &Value) -> String { @@ -716,6 +740,64 @@ fn quote_standard_export_sql_string(text: &str) -> String { format!("'{}'", text.replace('\'', "''")) } +/// Decodes JSON-style `\uXXXX` Unicode escapes (including surrogate pairs) +/// into their literal characters, leaving every other backslash sequence +/// untouched. Used only by the opt-in "preserve original language" export +/// option: application code (e.g. `encodeJsonValuesForDb`-style helpers) may +/// deliberately store ASCII-escaped Unicode in a TEXT column so the value +/// survives non-Unicode-aware storage; this renders that text human-readable +/// directly in the exported `.sql` file without touching the stored bytes in +/// the source database or breaking the round-trip for any other backslash +/// usage (e.g. a literal `\n`, a Windows path, or a doubled SQL quote). +fn decode_unicode_escapes_for_export(text: &str) -> String { + if !text.contains("\\u") { + return text.to_string(); + } + let chars: Vec = text.chars().collect(); + let mut result = String::with_capacity(text.len()); + let mut i = 0; + while i < chars.len() { + if chars[i] == '\\' && chars.get(i + 1) == Some(&'u') { + if let Some(high) = parse_hex4(&chars, i + 2) { + if (0xD800..=0xDBFF).contains(&high) { + // Possible surrogate pair: look for a trailing \uDC00-\uDFFF. + let low = if chars.get(i + 6) == Some(&'\\') && chars.get(i + 7) == Some(&'u') { + parse_hex4(&chars, i + 8).filter(|low| (0xDC00..=0xDFFF).contains(low)) + } else { + None + }; + if let Some(low) = low { + let combined = 0x10000u32 + ((high - 0xD800) << 10) + (low - 0xDC00); + if let Some(decoded) = char::from_u32(combined) { + result.push(decoded); + i += 12; + continue; + } + } + // Lone high surrogate: not representable as a char, keep as-is. + result.push(chars[i]); + i += 1; + continue; + } + if let Some(decoded) = char::from_u32(high) { + result.push(decoded); + i += 6; + continue; + } + } + } + result.push(chars[i]); + i += 1; + } + result +} + +fn parse_hex4(chars: &[char], start: usize) -> Option { + let slice = chars.get(start..start + 4)?; + let hex: String = slice.iter().collect(); + u32::from_str_radix(&hex, 16).ok() +} + /// SQL Server binary column types. The driver exposes their values as /// `0x`-prefixed hex text, and T-SQL only accepts that text as an unquoted /// binary literal: exporting it as a quoted string makes the import fail with @@ -1488,6 +1570,17 @@ pub(crate) fn build_export_insert_statements_excluding_with_dialect( .copied() .flatten() .or_else(|| spatial_columns.get(index).copied().flatten()); + let decoded_value; + let value = if options.preserve_original_language { + if let Some(text) = value.as_str() { + decoded_value = Value::String(decode_unicode_escapes_for_export(text)); + &decoded_value + } else { + value + } + } else { + value + }; let literal = if insert_dialect == SqlInsertDialect::Standard { format_standard_sql_literal(value) } else { @@ -1678,6 +1771,7 @@ mod sql_export_projection_tests { spatial_values: vec![vec![None, Some(4326), None, None]], rows: vec![vec![json!(1), json!("POINT(1 2)"), json!([0, 255]), Value::Null]], batch_size: Some(100), + preserve_original_language: false, }; let projected = projection.project_insert_options(options); assert_eq!(columns, vec!["id", "shape", "blob", "optional"]); @@ -1796,6 +1890,7 @@ pub fn build_database_sql_export(options: BuildDatabaseSqlExportOptions) -> Resu spatial_values: table.spatial_values, rows: table.rows, batch_size: Some(insert_batch_size), + preserve_original_language: options.preserve_original_language, }, &[], options.insert_dialect, @@ -2681,9 +2776,11 @@ fn write_database_export_rows( db_type, insert_dialect, SqlInsertMode::default(), + false, ) } +#[allow(clippy::too_many_arguments)] fn write_database_export_rows_with_mode( file: &mut W, rows: &[Vec], @@ -2695,6 +2792,7 @@ fn write_database_export_rows_with_mode( db_type: &DatabaseType, insert_dialect: SqlInsertDialect, insert_mode: SqlInsertMode, + preserve_original_language: bool, ) -> Result<(), String> { let insert_indices = columns .iter() @@ -2758,6 +2856,7 @@ fn write_database_export_rows_with_mode( spatial_values: Vec::new(), rows: insert_rows.to_vec(), batch_size: Some(insert_mode.batch_size(DATABASE_EXPORT_INSERT_BATCH_SIZE)), + preserve_original_language, }, &[], insert_dialect, @@ -4203,6 +4302,7 @@ async fn export_database_sql_core_inner( &db_type, request.insert_dialect, request.insert_mode, + request.preserve_original_language, )?; total_rows_exported += rows.len() as u64; on_progress(ExportProgress { @@ -4321,6 +4421,7 @@ async fn export_database_sql_core_inner( &db_type, request.insert_dialect, request.insert_mode, + request.preserve_original_language, )?; total_rows_exported += row_count as u64; if use_keyset { @@ -4709,11 +4810,11 @@ mod tests { build_database_export_object_source_sql, build_database_sql_export, build_export_insert_statements, build_export_insert_statements_excluding, build_export_object_source_sql, build_export_sql_insert, create_database_export_writer, database_export_query_options_for_timeout, database_export_select_sql, - database_export_total_objects, drop_table_if_exists_sql, ensure_export_destination_dir, - export_destination_identity_mismatch, extract_postgres_deferred_foreign_keys, filter_export_table_infos, - format_export_sql_literal, format_export_table_ddl, format_mysql_spatial_export_literal, - format_xugu_spatial_export_literal, generate_postgres_enum_ddl, generate_postgres_extension_ddl, - generate_postgres_sequence_create_ddl, generate_postgres_sequence_owner_ddl, + database_export_total_objects, decode_unicode_escapes_for_export, drop_table_if_exists_sql, + ensure_export_destination_dir, export_destination_identity_mismatch, extract_postgres_deferred_foreign_keys, + filter_export_table_infos, format_export_sql_literal, format_export_table_ddl, + format_mysql_spatial_export_literal, format_xugu_spatial_export_literal, generate_postgres_enum_ddl, + generate_postgres_extension_ddl, generate_postgres_sequence_create_ddl, generate_postgres_sequence_owner_ddl, generate_postgres_sequence_setval_sql, is_postgres_extension_member_routine, mysql_database_export_preamble, mysql_view_dependencies_from_rows, mysql_view_dependencies_sql, normalize_export_table_ddl, record_export_destination_identity, record_export_error, replace_database_export_select_list, @@ -4927,6 +5028,7 @@ mod tests { snapshot_session_id: None, batch_size: 1000, split_max_mb: None, + preserve_original_language: false, } } @@ -5535,6 +5637,7 @@ mod tests { json!("DBX_WKB:0:0101000000000000000000F03F0000000000000040"), ]], batch_size: Some(10), + preserve_original_language: false, }) .unwrap(); @@ -5665,6 +5768,7 @@ mod tests { spatial_values: vec![vec![Some(3857)]], rows: vec![vec![json!("POINT(1 2)")]], batch_size: None, + preserve_original_language: false, }) .unwrap(); @@ -5691,6 +5795,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!(true), json!(false), Value::Null]], batch_size: Some(10), + preserve_original_language: false, }) .unwrap(); let postgres_statements = build_export_insert_statements(BuildExportInsertStatementsOptions { @@ -5706,6 +5811,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!(true), json!(false), Value::Null]], batch_size: Some(10), + preserve_original_language: false, }) .unwrap(); @@ -5736,6 +5842,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!(r"C:\new\it's")]], batch_size: Some(10), + preserve_original_language: false, }) .unwrap(); @@ -5758,6 +5865,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!(r"C:\new\it's")]], batch_size: Some(10), + preserve_original_language: false, }) .unwrap(); @@ -5797,6 +5905,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!("a\x1ab")]], batch_size: Some(10), + preserve_original_language: false, }) .unwrap(); @@ -5826,6 +5935,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!("a\x1ab")]], batch_size: Some(10), + preserve_original_language: false, }) .unwrap(); @@ -5868,6 +5978,7 @@ mod tests { Value::Null, ]], batch_size: Some(10), + preserve_original_language: false, }) .unwrap(); @@ -5903,6 +6014,7 @@ mod tests { vec![json!(3), json!("0xzz"), json!("text"), json!("not-hex")], ], batch_size: Some(10), + preserve_original_language: false, }) .unwrap(); @@ -5942,6 +6054,7 @@ mod tests { database: None, schema: None, omit_auto_increment: false, + preserve_original_language: false, }) .expect("build SQL Server database export"); @@ -5964,6 +6077,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!("line1\nline2\tcol\rend\\slash\0\x1aO'Hara")]], batch_size: Some(10), + preserve_original_language: false, }) .unwrap(); @@ -5989,6 +6103,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!(long_value.clone())], vec![json!(long_value)]], batch_size: Some(100), + preserve_original_language: false, }) .unwrap(); @@ -6011,6 +6126,7 @@ mod tests { spatial_values: Vec::new(), rows: (0..1001).map(|id| vec![json!(id)]).collect(), batch_size: Some(2000), + preserve_original_language: false, }) .unwrap(); @@ -6034,6 +6150,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!("first\nsecond\tthird")]], batch_size: Some(10), + preserve_original_language: false, }) .unwrap(); @@ -6055,6 +6172,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!("line1\nline2\tend")]], batch_size: Some(10), + preserve_original_language: false, }) .unwrap(); @@ -6089,6 +6207,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!(format!("0x{ZIP_HEX}")), json!("0x"), json!("0XABcd"), Value::Null, json!("0xABcd")]], batch_size: Some(10), + preserve_original_language: false, }) .unwrap(); @@ -6115,6 +6234,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!("0xabc"), json!("0xgg"), json!(7)]], batch_size: Some(10), + preserve_original_language: false, }) .unwrap(); @@ -6146,6 +6266,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!("line1\rline2"), json!("O'Hara"), json!(r"C:\tmp"), json!("plain")]], batch_size: Some(10), + preserve_original_language: false, }) .unwrap(); @@ -6172,6 +6293,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!(r#"{"text":"say \"hi\"","path":"C:\\tmp","quote":"O'Hara"}"#)]], batch_size: Some(10), + preserve_original_language: false, }) .unwrap(); @@ -6199,6 +6321,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!("逻辑删除标志:'0'-未删除,'1'-已删除"), json!(style)]], batch_size: Some(10), + preserve_original_language: false, }) .unwrap(); @@ -6227,6 +6350,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!(style)]], batch_size: Some(10), + preserve_original_language: false, }) .unwrap(); @@ -6273,6 +6397,7 @@ mod tests { json!([[1.2, 3.4], [5, 6]]), ]], batch_size: Some(10), + preserve_original_language: false, }) .unwrap(); @@ -6299,6 +6424,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!(1), json!("Ada")], vec![json!(2), json!("O'Hara")], vec![json!(3), json!("Linus")]], batch_size: Some(2), + preserve_original_language: false, }) .unwrap(); @@ -6326,6 +6452,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!(1), json!("Ada")], vec![json!(2), json!("O'Hara")], vec![json!(3), json!("Linus")]], batch_size: Some(10), + preserve_original_language: false, }) .unwrap(); @@ -6350,6 +6477,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!(1), json!("Ada")], vec![json!(2), json!("Linus")]], batch_size: Some(1), + preserve_original_language: false, }) .unwrap(); @@ -6362,6 +6490,71 @@ mod tests { ); } + #[test] + fn decode_unicode_escapes_for_export_round_trips_vietnamese_text() { + assert_eq!(decode_unicode_escapes_for_export("\\u1ed0ng gi\\u00f3 th\\u1eb3ng"), "Ống gió thẳng"); + // Leaves non-escape backslashes untouched (e.g. a literal path or doubled quote escape). + assert_eq!(decode_unicode_escapes_for_export("600x200\\\" width"), "600x200\\\" width"); + // A text run with no `\u` escapes at all is returned unchanged (fast path). + assert_eq!(decode_unicode_escapes_for_export("plain text"), "plain text"); + // Surrogate pairs decode to a single non-BMP character. + assert_eq!(decode_unicode_escapes_for_export("\\ud83d\\ude00"), "😀"); + // A malformed/lone high surrogate is left as literal text instead of panicking or corrupting data. + assert_eq!(decode_unicode_escapes_for_export("\\ud83dxyz"), "\\ud83dxyz"); + } + + #[test] + fn preserve_original_language_option_decodes_unicode_escapes_in_exported_literals() { + let statements = build_export_insert_statements(BuildExportInsertStatementsOptions { + database_type: Some(DatabaseType::SqlServer), + identifier_quote: None, + schema: None, + table_name: Some("VersionValue".to_string()), + qualified_table_name: None, + columns: vec!["jsonValues".to_string()], + column_types: Vec::new(), + column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), + rows: vec![vec![json!("{\"clientBoqItemName\":\"\\u1ed0ng gi\\u00f3 th\\u1eb3ng\"}")]], + batch_size: Some(10), + preserve_original_language: true, + }) + .unwrap(); + + assert_eq!( + statements, + vec!["INSERT INTO [VersionValue] ([jsonValues]) VALUES ('{\"clientBoqItemName\":\"Ống gió thẳng\"}');"] + ); + } + + #[test] + fn preserve_original_language_defaults_to_false_and_keeps_ascii_escapes() { + let statements = build_export_insert_statements(BuildExportInsertStatementsOptions { + database_type: Some(DatabaseType::SqlServer), + identifier_quote: None, + schema: None, + table_name: Some("VersionValue".to_string()), + qualified_table_name: None, + columns: vec!["jsonValues".to_string()], + column_types: Vec::new(), + column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), + rows: vec![vec![json!("{\"clientBoqItemName\":\"\\u1ed0ng gi\\u00f3 th\\u1eb3ng\"}")]], + batch_size: Some(10), + preserve_original_language: false, + }) + .unwrap(); + + assert_eq!( + statements, + vec![ + "INSERT INTO [VersionValue] ([jsonValues]) VALUES ('{\"clientBoqItemName\":\"\\u1ed0ng gi\\u00f3 th\\u1eb3ng\"}');" + ] + ); + } + #[test] fn oracle_export_inserts_use_one_statement_per_row() { let statements = build_export_insert_statements(BuildExportInsertStatementsOptions { @@ -6377,6 +6570,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!(1), json!("Ada")], vec![json!(2), json!("Linus")]], batch_size: Some(100), + preserve_original_language: false, }) .unwrap(); @@ -6404,6 +6598,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!("AAAPr9AAEAAAAGfAAA"), json!(1), json!("Ada")]], batch_size: Some(100), + preserve_original_language: false, }) .unwrap(); @@ -6425,6 +6620,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!("*AAABk1AAEAAAAAgAAA"), json!(1), json!("Ada")]], batch_size: Some(100), + preserve_original_language: false, }) .unwrap(); @@ -6446,6 +6642,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!(7), json!("Ada")]], batch_size: Some(100), + preserve_original_language: false, }) .unwrap(); @@ -6470,6 +6667,7 @@ mod tests { vec![json!(2), json!("2022-08-25T00:00:00Z"), json!("2022-08-25T00:00:00Z")], ], batch_size: Some(100), + preserve_original_language: false, }) .unwrap(); @@ -6534,6 +6732,7 @@ mod tests { Value::Null, ]], batch_size: Some(100), + preserve_original_language: false, }) .unwrap(); @@ -6572,6 +6771,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!("2022-08-25T09:58:43.123456Z"), json!("2022-08-26T10:59:44+08:00")]], batch_size: Some(100), + preserve_original_language: false, }) .unwrap(); @@ -6599,6 +6799,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!("1"), json!("1010"), json!("1010")], vec![json!(false), json!(3), json!("off")]], batch_size: Some(10), + preserve_original_language: false, }) .unwrap(); @@ -6625,6 +6826,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!(true), json!(false), Value::Null]], batch_size: Some(10), + preserve_original_language: false, }) .unwrap(); @@ -6663,6 +6865,7 @@ mod tests { json!("\0"), ]], batch_size: Some(10), + preserve_original_language: false, }) .unwrap(); @@ -6696,6 +6899,7 @@ mod tests { vec![json!("2"), json!("0X"), json!("1")], ], batch_size: Some(10), + preserve_original_language: false, }) .unwrap(); @@ -6732,6 +6936,7 @@ mod tests { json!("2026-06-12T10:11:12Z"), ]], batch_size: Some(10), + preserve_original_language: false, }) .unwrap(); @@ -6761,6 +6966,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!("2026-06-12T10:11:12Z"), json!("2026-06-12T18:11:12+08:00")]], batch_size: Some(10), + preserve_original_language: false, }) .unwrap(); @@ -6787,6 +6993,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!("2026-06-12T10:11:12Z"), json!("2026-06-12T10:11:12.1234567Z")]], batch_size: Some(10), + preserve_original_language: false, }) .unwrap(); @@ -6813,6 +7020,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!(1), json!("Hello"), json!("'hello':1A")]], batch_size: Some(10), + preserve_original_language: false, }) .unwrap(); @@ -6835,6 +7043,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!(1), json!("Ada"), json!("ada@example.com")]], batch_size: Some(10), + preserve_original_language: false, }, &["id".to_string()], ) @@ -6862,6 +7071,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!(1), json!("0x00ff")]], batch_size: Some(10), + preserve_original_language: false, }, &["v2".to_string()], ) @@ -6886,6 +7096,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!(1)]], batch_size: Some(10), + preserve_original_language: false, }, &["user_id".to_string()], ); @@ -7016,6 +7227,7 @@ mod tests { database: None, schema: None, omit_auto_increment: false, + preserve_original_language: false, }) .unwrap(); @@ -7105,6 +7317,7 @@ mod tests { &DatabaseType::Mysql, SqlInsertDialect::Source, SqlInsertMode::Single, + false, ) .unwrap(); drop(file); @@ -7330,6 +7543,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!(1), json!("Ada")]], batch_size: Some(10), + preserve_original_language: false, }) .unwrap(); @@ -7359,6 +7573,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!(1), json!("Ada")]], batch_size: Some(10), + preserve_original_language: false, }) .unwrap(); @@ -7398,6 +7613,7 @@ mod tests { database: None, schema: None, omit_auto_increment: false, + preserve_original_language: false, }) .unwrap(); @@ -8027,6 +8243,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!(1), json!("login")]], batch_size: Some(100), + preserve_original_language: false, }) .unwrap(); assert_eq!(statements, vec!["INSERT INTO `audit-schema`.`events` (`id`, `event_type`) VALUES (1, 'login');"]); diff --git a/crates/dbx-core/src/data/query_result_export.rs b/crates/dbx-core/src/data/query_result_export.rs index c3681f1f3f..634c35438c 100644 --- a/crates/dbx-core/src/data/query_result_export.rs +++ b/crates/dbx-core/src/data/query_result_export.rs @@ -519,6 +519,7 @@ impl SqlInsertWriter { spatial_values: mem::take(&mut self.pending_spatial_values), rows: mem::take(&mut self.pending_rows), batch_size: Some(self.insert_mode.batch_size(SQL_INSERT_BATCH_SIZE)), + preserve_original_language: false, }, &self.exclude_columns, )?; diff --git a/crates/dbx-core/src/data/table_export.rs b/crates/dbx-core/src/data/table_export.rs index 796266479a..7ffbba6a58 100644 --- a/crates/dbx-core/src/data/table_export.rs +++ b/crates/dbx-core/src/data/table_export.rs @@ -1471,6 +1471,7 @@ async fn try_export_native_table_stream( spatial_values: Vec::new(), rows: std::mem::take(pending_rows), batch_size: Some(request.insert_mode.batch_size(SQL_INSERT_BATCH_SIZE)), + preserve_original_language: false, }), &sql_export_excluded_columns(request, primary_keys, col_names, column_extras), request.insert_dialect, @@ -2330,6 +2331,7 @@ async fn export_table_data_core_inner( spatial_values: result.spatial_values.clone(), rows: result.rows.clone(), batch_size: Some(request.insert_mode.batch_size(SQL_INSERT_BATCH_SIZE)), + preserve_original_language: false, }), &sql_export_excluded_columns(request, &primary_keys, &col_names, &column_extras), request.insert_dialect, @@ -3447,6 +3449,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!(1), json!("Ada")]], batch_size: Some(100), + preserve_original_language: false, }) .unwrap(); assert_eq!(statements, vec!["INSERT INTO \"APP\".\"USERS\" (\"ID\", \"NAME\") VALUES (1, 'Ada');"]); @@ -3545,6 +3548,7 @@ mod tests { spatial_values: Vec::new(), rows: vec![vec![json!(1)]], batch_size: Some(100), + preserve_original_language: false, }) .unwrap() }; diff --git a/crates/dbx-core/src/scheduled_backup/engine.rs b/crates/dbx-core/src/scheduled_backup/engine.rs index 05435f5f8f..ef908546be 100644 --- a/crates/dbx-core/src/scheduled_backup/engine.rs +++ b/crates/dbx-core/src/scheduled_backup/engine.rs @@ -361,6 +361,7 @@ impl BackupService { include_create_database: false, drop_table_if_exists: job.config.drop_table_if_exists, omit_auto_increment: false, + preserve_original_language: false, fail_on_error: true, prevent_overwrite: true, output_compression: if job.config.output_compression == "gzip" { diff --git a/crates/dbx-core/tests/database_export_partition_ddl.rs b/crates/dbx-core/tests/database_export_partition_ddl.rs index d4a8b82612..2c73be0a22 100644 --- a/crates/dbx-core/tests/database_export_partition_ddl.rs +++ b/crates/dbx-core/tests/database_export_partition_ddl.rs @@ -135,6 +135,7 @@ async fn run_database_export_of_partition_tree_has_no_duplicates_and_replays() { include_create_database: false, drop_table_if_exists: false, omit_auto_increment: false, + preserve_original_language: false, fail_on_error: true, prevent_overwrite: false, output_compression: Default::default(), diff --git a/crates/dbx-core/tests/database_export_prefetch.rs b/crates/dbx-core/tests/database_export_prefetch.rs index 3292f869ad..da7505f5ef 100644 --- a/crates/dbx-core/tests/database_export_prefetch.rs +++ b/crates/dbx-core/tests/database_export_prefetch.rs @@ -74,6 +74,7 @@ async fn run_database_export_writes_structure_and_data_for_all_tables() { include_create_database: false, drop_table_if_exists: true, omit_auto_increment: false, + preserve_original_language: false, fail_on_error: true, prevent_overwrite: false, output_compression: Default::default(), diff --git a/crates/dbx-core/tests/live_manual_transaction.rs b/crates/dbx-core/tests/live_manual_transaction.rs index 8814a83cef..40e2743161 100644 --- a/crates/dbx-core/tests/live_manual_transaction.rs +++ b/crates/dbx-core/tests/live_manual_transaction.rs @@ -327,6 +327,7 @@ async fn live_postgres_backup_snapshot_exports_wide_jsonb_then_next_table_inner( include_create_database: false, drop_table_if_exists: false, omit_auto_increment: false, + preserve_original_language: false, fail_on_error: true, prevent_overwrite: false, output_compression: Default::default(), @@ -444,6 +445,7 @@ async fn live_postgres_backup_snapshot_cancel_interrupts_pending_row_inner() { include_create_database: false, drop_table_if_exists: false, omit_auto_increment: false, + preserve_original_language: false, fail_on_error: true, prevent_overwrite: false, output_compression: Default::default(), @@ -563,6 +565,7 @@ async fn live_mysql_database_backup_refreshes_an_idle_snapshot_before_export() { include_create_database: false, drop_table_if_exists: false, omit_auto_increment: false, + preserve_original_language: false, fail_on_error: true, prevent_overwrite: false, output_compression: Default::default(), diff --git a/crates/dbx-core/tests/live_mysql_database_export.rs b/crates/dbx-core/tests/live_mysql_database_export.rs index 519cba3039..5e478fc528 100644 --- a/crates/dbx-core/tests/live_mysql_database_export.rs +++ b/crates/dbx-core/tests/live_mysql_database_export.rs @@ -150,6 +150,7 @@ async fn live_mysql_database_export_restores_dependent_views() { include_create_database: true, drop_table_if_exists: true, omit_auto_increment: false, + preserve_original_language: false, fail_on_error: true, prevent_overwrite: false, output_compression: Default::default(), @@ -293,6 +294,7 @@ async fn run_live_mysql_database_export_handles_many_tables_including_empty_tabl include_create_database: false, drop_table_if_exists: false, omit_auto_increment: false, + preserve_original_language: false, fail_on_error: false, prevent_overwrite: false, output_compression: Default::default(), @@ -390,6 +392,7 @@ async fn live_mysql_database_export_creates_missing_destination_directory() { include_create_database: true, drop_table_if_exists: true, omit_auto_increment: false, + preserve_original_language: false, fail_on_error: true, prevent_overwrite: false, output_compression: Default::default(), @@ -469,6 +472,7 @@ async fn live_mysql_database_export_refuses_to_recreate_a_destination_that_disap include_create_database: true, drop_table_if_exists: true, omit_auto_increment: false, + preserve_original_language: false, fail_on_error: true, prevent_overwrite: false, output_compression: Default::default(), @@ -559,6 +563,7 @@ async fn live_mysql_database_export_refuses_a_destination_that_vanished_before_i include_create_database: true, drop_table_if_exists: true, omit_auto_increment: false, + preserve_original_language: false, fail_on_error: true, prevent_overwrite: false, output_compression: Default::default(), diff --git a/crates/dbx-core/tests/live_mysql_export_table_order.rs b/crates/dbx-core/tests/live_mysql_export_table_order.rs index 7863d2532c..f05017df38 100644 --- a/crates/dbx-core/tests/live_mysql_export_table_order.rs +++ b/crates/dbx-core/tests/live_mysql_export_table_order.rs @@ -99,6 +99,7 @@ async fn live_mysql_database_export_table_order_is_not_alphabetical_when_fk_reor include_create_database: false, drop_table_if_exists: false, omit_auto_increment: false, + preserve_original_language: false, fail_on_error: true, prevent_overwrite: false, output_compression: Default::default(), diff --git a/crates/dbx-core/tests/live_postgres_all_schema_export.rs b/crates/dbx-core/tests/live_postgres_all_schema_export.rs index 6057ca41ec..dccd8bce48 100644 --- a/crates/dbx-core/tests/live_postgres_all_schema_export.rs +++ b/crates/dbx-core/tests/live_postgres_all_schema_export.rs @@ -109,6 +109,7 @@ async fn live_postgres_all_schema_export_restores_one_sql_file() { include_create_database: false, drop_table_if_exists: false, omit_auto_increment: false, + preserve_original_language: false, fail_on_error: true, prevent_overwrite: false, output_compression: Default::default(), diff --git a/crates/dbx-core/tests/live_postgres_export_deferred_foreign_keys.rs b/crates/dbx-core/tests/live_postgres_export_deferred_foreign_keys.rs index 53e3b47f78..e8ecca2c25 100644 --- a/crates/dbx-core/tests/live_postgres_export_deferred_foreign_keys.rs +++ b/crates/dbx-core/tests/live_postgres_export_deferred_foreign_keys.rs @@ -135,6 +135,7 @@ async fn live_postgres_export_with_foreign_key_cycle_restores_into_empty_databas include_create_database: false, drop_table_if_exists: false, omit_auto_increment: false, + preserve_original_language: false, fail_on_error: true, prevent_overwrite: false, output_compression: Default::default(), diff --git a/crates/dbx-core/tests/live_postgres_tsvector.rs b/crates/dbx-core/tests/live_postgres_tsvector.rs index 8ab317d6f8..3e5cd92638 100644 --- a/crates/dbx-core/tests/live_postgres_tsvector.rs +++ b/crates/dbx-core/tests/live_postgres_tsvector.rs @@ -98,6 +98,7 @@ async fn postgres_tsvector_generated_columns_are_readable_and_omitted_from_inser spatial_values: Vec::new(), rows: result.rows.clone(), batch_size: Some(10), + preserve_original_language: false, }) .expect("export insert") .join("\n"); diff --git a/crates/dbx-core/tests/live_sqlserver_split_zip_e2e.rs b/crates/dbx-core/tests/live_sqlserver_split_zip_e2e.rs index 6d6b1d0f1c..3ee10d9f76 100644 --- a/crates/dbx-core/tests/live_sqlserver_split_zip_e2e.rs +++ b/crates/dbx-core/tests/live_sqlserver_split_zip_e2e.rs @@ -130,6 +130,7 @@ async fn manual_e2e_sqlserver_default_split_zip_export_then_import_skip_relation include_create_database: false, drop_table_if_exists: true, omit_auto_increment: false, + preserve_original_language: false, fail_on_error: true, prevent_overwrite: false, output_compression: DatabaseExportOutputCompression::None, diff --git a/crates/dbx-core/tests/live_sqlserver_transfer.rs b/crates/dbx-core/tests/live_sqlserver_transfer.rs index ae4de04ca0..f2545c9001 100644 --- a/crates/dbx-core/tests/live_sqlserver_transfer.rs +++ b/crates/dbx-core/tests/live_sqlserver_transfer.rs @@ -941,6 +941,7 @@ async fn live_sqlserver_database_export_replays_composite_foreign_keys() { include_create_database: false, drop_table_if_exists: false, omit_auto_increment: false, + preserve_original_language: false, fail_on_error: false, prevent_overwrite: false, output_compression: Default::default(),