feat(sql-file): import split SQL ZIP packages with bounded extraction
This commit is contained in:
1 parent
f43bf80015
commit
b0d30e6f73
22 files changed
+550
-118
No files matched your search
@@ -85,6 +85,7 @@ const database = ref("");
|
|||||||
const databaseOptions = ref<string[]>([]);
|
const databaseOptions = ref<string[]>([]);
|
||||||
const loadingDatabases = ref(false);
|
const loadingDatabases = ref(false);
|
||||||
const continueOnError = ref(false);
|
const continueOnError = ref(false);
|
||||||
|
const skipRelationalConstraints = ref(false);
|
||||||
|
|
||||||
const running = ref(false);
|
const running = ref(false);
|
||||||
const cancelling = ref(false);
|
const cancelling = ref(false);
|
||||||
@@ -132,6 +133,15 @@ function resetPerFileState() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const sqlConnections = computed(() => store.connections.filter((c) => !["redis", "mongodb", "elasticsearch", "easysearch", "meilisearch", "qdrant", "milvus", "weaviate", "chromadb", "etcd", "zookeeper", "consul", "mq", "nacos"].includes(c.db_type)));
|
const sqlConnections = computed(() => store.connections.filter((c) => !["redis", "mongodb", "elasticsearch", "easysearch", "meilisearch", "qdrant", "milvus", "weaviate", "chromadb", "etcd", "zookeeper", "consul", "mq", "nacos"].includes(c.db_type)));
|
||||||
|
// Mirrors the core executor gate (`supports_connection_level_database_bootstrap_target`): the
|
||||||
|
// MySQL-family types it runs for, so the constraint toggle appears wherever the backend honors it.
|
||||||
|
const MYSQL_BOOTSTRAP_IMPORT_TYPES = new Set(["mysql", "doris", "starrocks", "goldendb"]);
|
||||||
|
const MYSQL_BOOTSTRAP_IMPORT_PROFILES = new Set(["mariadb", "tidb", "oceanbase", "custom_mysql", "doris", "starrocks", "selectdb", "goldendb"]);
|
||||||
|
const isMysqlCompatibleTarget = computed(() => {
|
||||||
|
const config = store.getConfig(connectionId.value);
|
||||||
|
if (!config) return false;
|
||||||
|
return MYSQL_BOOTSTRAP_IMPORT_TYPES.has(config.db_type) || (!!config.driver_profile && MYSQL_BOOTSTRAP_IMPORT_PROFILES.has(config.driver_profile.toLowerCase()));
|
||||||
|
});
|
||||||
|
|
||||||
const selectedConnection = computed(() => sqlConnections.value.find((c) => c.id === connectionId.value));
|
const selectedConnection = computed(() => sqlConnections.value.find((c) => c.id === connectionId.value));
|
||||||
|
|
||||||
@@ -309,6 +319,7 @@ function resetState() {
|
|||||||
databaseOptions.value = [];
|
databaseOptions.value = [];
|
||||||
loadingDatabases.value = false;
|
loadingDatabases.value = false;
|
||||||
continueOnError.value = false;
|
continueOnError.value = false;
|
||||||
|
skipRelationalConstraints.value = false;
|
||||||
restoreSelectedTables.value = false;
|
restoreSelectedTables.value = false;
|
||||||
resetExecution();
|
resetExecution();
|
||||||
}
|
}
|
||||||
@@ -386,7 +397,7 @@ async function selectFile() {
|
|||||||
const { open } = await import("@tauri-apps/plugin-dialog");
|
const { open } = await import("@tauri-apps/plugin-dialog");
|
||||||
const selected = await open({
|
const selected = await open({
|
||||||
multiple: true,
|
multiple: true,
|
||||||
filters: [{ name: "SQL", extensions: ["sql", "gz"] }],
|
filters: [{ name: "SQL package", extensions: ["sql", "gz", "zip"] }],
|
||||||
});
|
});
|
||||||
const paths = Array.isArray(selected) ? selected : selected ? [selected] : [];
|
const paths = Array.isArray(selected) ? selected : selected ? [selected] : [];
|
||||||
if (paths.length > 0) {
|
if (paths.length > 0) {
|
||||||
@@ -539,16 +550,19 @@ async function startExecution() {
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
executionStarted.value = true;
|
executionStarted.value = true;
|
||||||
|
const executionPaths = previews.value.flatMap((item) => item.packageFilePaths ?? [item.filePath]);
|
||||||
await executeSqlFiles(
|
await executeSqlFiles(
|
||||||
{
|
{
|
||||||
executionId: batchId,
|
executionId: batchId,
|
||||||
connectionId: connectionId.value,
|
connectionId: connectionId.value,
|
||||||
database: database.value.trim(),
|
database: database.value.trim(),
|
||||||
filePath: previews.value[0]!.filePath,
|
filePath: executionPaths[0]!,
|
||||||
continueOnError: continueOnError.value,
|
continueOnError: continueOnError.value,
|
||||||
...(restoreSelectedTables.value ? { selectedTables: selectedTables.value.map((table) => ({ ...table })) } : {}),
|
...(restoreSelectedTables.value ? { selectedTables: selectedTables.value.map((table) => ({ ...table })) } : {}),
|
||||||
|
partCooldownMs: previews.value.some((item) => item.packageFilePaths) ? 500 : 0,
|
||||||
|
skipRelationalConstraints: skipRelationalConstraints.value,
|
||||||
},
|
},
|
||||||
previews.value.map((item) => item.filePath),
|
executionPaths,
|
||||||
);
|
);
|
||||||
const terminal = await terminalProgress;
|
const terminal = await terminalProgress;
|
||||||
if (terminal.status === "error") {
|
if (terminal.status === "error") {
|
||||||
@@ -656,7 +670,7 @@ watch(
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="flex items-center gap-2">
|
<div class="flex items-center gap-2">
|
||||||
<input ref="fileInput" type="file" accept=".sql,.sql.gz,text/sql,application/gzip" multiple class="hidden" @change="handleFileInputChange" />
|
<input ref="fileInput" type="file" accept=".sql,.sql.gz,.zip,text/sql,application/gzip,application/zip" multiple class="hidden" @change="handleFileInputChange" />
|
||||||
<Input :model-value="filePathDisplay" readonly class="h-8 text-xs font-mono" :placeholder="t('sqlFile.selectSqlFile')" />
|
<Input :model-value="filePathDisplay" readonly class="h-8 text-xs font-mono" :placeholder="t('sqlFile.selectSqlFile')" />
|
||||||
<Button variant="outline" size="sm" class="h-8 shrink-0" :disabled="running || selectingFile" @click="selectFile">
|
<Button variant="outline" size="sm" class="h-8 shrink-0" :disabled="running || selectingFile" @click="selectFile">
|
||||||
<Loader2 v-if="selectingFile || loadingPreview" class="w-3.5 h-3.5 mr-1.5 animate-spin" />
|
<Loader2 v-if="selectingFile || loadingPreview" class="w-3.5 h-3.5 mr-1.5 animate-spin" />
|
||||||
@@ -696,6 +710,7 @@ watch(
|
|||||||
<span>{{ previewLineSummary(activePreview) }}</span>
|
<span>{{ previewLineSummary(activePreview) }}</span>
|
||||||
<span class="h-3 w-px bg-border" />
|
<span class="h-3 w-px bg-border" />
|
||||||
<span>{{ formatBytes(activePreview.sizeBytes) }}</span>
|
<span>{{ formatBytes(activePreview.sizeBytes) }}</span>
|
||||||
|
<span v-if="activePreview.packagePartCount">{{ t("sqlFile.packageParts", { count: activePreview.packagePartCount }) }}</span>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="sql-file-preview-viewer flex max-w-full overflow-auto bg-muted/15 text-xs rounded-b-md border border-t-0" :class="previews.length === 1 ? 'min-h-56 max-h-[min(46vh,420px)]' : 'min-h-0 max-h-[min(46vh,420px)]'">
|
<div class="sql-file-preview-viewer flex max-w-full overflow-auto bg-muted/15 text-xs rounded-b-md border border-t-0" :class="previews.length === 1 ? 'min-h-56 max-h-[min(46vh,420px)]' : 'min-h-0 max-h-[min(46vh,420px)]'">
|
||||||
@@ -791,6 +806,11 @@ watch(
|
|||||||
<Square v-else class="w-3.5 h-3.5 text-muted-foreground/40 shrink-0" />
|
<Square v-else class="w-3.5 h-3.5 text-muted-foreground/40 shrink-0" />
|
||||||
{{ t("sqlFile.continueOnError") }}
|
{{ t("sqlFile.continueOnError") }}
|
||||||
</button>
|
</button>
|
||||||
|
<button v-if="isMysqlCompatibleTarget" type="button" class="flex items-center gap-2 text-xs text-left" :disabled="running" @click="skipRelationalConstraints = !skipRelationalConstraints">
|
||||||
|
<CheckSquare v-if="skipRelationalConstraints" class="w-3.5 h-3.5 text-primary shrink-0" />
|
||||||
|
<Square v-else class="w-3.5 h-3.5 text-muted-foreground/40 shrink-0" />
|
||||||
|
{{ t("sqlFile.skipRelationalConstraints") }}
|
||||||
|
</button>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div v-if="running || terminalStatus !== 'idle' || progress" class="min-w-0 space-y-3">
|
<div v-if="running || terminalStatus !== 'idle' || progress" class="min-w-0 space-y-3">
|
||||||
|
|||||||
@@ -6270,6 +6270,8 @@ export default withEnglishFallback({
|
|||||||
selectDatabase: "Verilənlər bazası seç",
|
selectDatabase: "Verilənlər bazası seç",
|
||||||
databasePlaceholder: "Verilənlər bazasının adı",
|
databasePlaceholder: "Verilənlər bazasının adı",
|
||||||
options: "Seçimlər",
|
options: "Seçimlər",
|
||||||
|
packageParts: "{count} SQL hissəsi",
|
||||||
|
skipRelationalConstraints: "İdxal zamanı əlaqəli məhdudiyyətləri müvəqqəti söndür",
|
||||||
continueOnError: "Xəta olduqda davam et",
|
continueOnError: "Xəta olduqda davam et",
|
||||||
runInBackground: "Arxa planda işlət",
|
runInBackground: "Arxa planda işlət",
|
||||||
cancelling: "Ləğv edilir...",
|
cancelling: "Ləğv edilir...",
|
||||||
|
|||||||
@@ -6424,6 +6424,8 @@ export default {
|
|||||||
databasePlaceholder: "Database name",
|
databasePlaceholder: "Database name",
|
||||||
options: "Options",
|
options: "Options",
|
||||||
continueOnError: "Continue on error",
|
continueOnError: "Continue on error",
|
||||||
|
skipRelationalConstraints: "Temporarily disable relational constraints during import",
|
||||||
|
packageParts: "{count} SQL parts",
|
||||||
runInBackground: "Run in background",
|
runInBackground: "Run in background",
|
||||||
cancelling: "Cancelling...",
|
cancelling: "Cancelling...",
|
||||||
cancel: "Cancel",
|
cancel: "Cancel",
|
||||||
|
|||||||
@@ -6040,6 +6040,8 @@ export default withEnglishFallback({
|
|||||||
selectDatabase: "Seleccionar base de datos",
|
selectDatabase: "Seleccionar base de datos",
|
||||||
databasePlaceholder: "Nombre de la base de datos",
|
databasePlaceholder: "Nombre de la base de datos",
|
||||||
options: "Opciones",
|
options: "Opciones",
|
||||||
|
packageParts: "{count} partes SQL",
|
||||||
|
skipRelationalConstraints: "Desactivar temporalmente las restricciones relacionales durante la importación",
|
||||||
continueOnError: "Continuar en caso de error",
|
continueOnError: "Continuar en caso de error",
|
||||||
runInBackground: "Ejecutar en segundo plano",
|
runInBackground: "Ejecutar en segundo plano",
|
||||||
cancelling: "Cancelando...",
|
cancelling: "Cancelando...",
|
||||||
|
|||||||
@@ -6038,6 +6038,8 @@ export default withEnglishFallback({
|
|||||||
selectDatabase: "Seleziona database",
|
selectDatabase: "Seleziona database",
|
||||||
databasePlaceholder: "Nome database",
|
databasePlaceholder: "Nome database",
|
||||||
options: "Opzioni",
|
options: "Opzioni",
|
||||||
|
packageParts: "{count} parti SQL",
|
||||||
|
skipRelationalConstraints: "Disattiva temporaneamente i vincoli relazionali durante l'importazione",
|
||||||
continueOnError: "Continua in caso di errore",
|
continueOnError: "Continua in caso di errore",
|
||||||
runInBackground: "Esegui in background",
|
runInBackground: "Esegui in background",
|
||||||
cancelling: "Annullamento in corso...",
|
cancelling: "Annullamento in corso...",
|
||||||
|
|||||||
@@ -6077,6 +6077,8 @@ export default withEnglishFallback({
|
|||||||
selectDatabase: "データベースを選択",
|
selectDatabase: "データベースを選択",
|
||||||
databasePlaceholder: "データベース名",
|
databasePlaceholder: "データベース名",
|
||||||
options: "オプション",
|
options: "オプション",
|
||||||
|
packageParts: "{count} 個のSQLパート",
|
||||||
|
skipRelationalConstraints: "インポート中に関連性制約を一時的に無効化",
|
||||||
continueOnError: "エラー時に続行",
|
continueOnError: "エラー時に続行",
|
||||||
cancelling: "キャンセル中...",
|
cancelling: "キャンセル中...",
|
||||||
cancel: "キャンセル",
|
cancel: "キャンセル",
|
||||||
|
|||||||
@@ -5755,6 +5755,8 @@ export default withEnglishFallback({
|
|||||||
selectDatabase: "데이터베이스 선택",
|
selectDatabase: "데이터베이스 선택",
|
||||||
databasePlaceholder: "데이터베이스 이름",
|
databasePlaceholder: "데이터베이스 이름",
|
||||||
options: "옵션",
|
options: "옵션",
|
||||||
|
packageParts: "SQL 파트 {count}개",
|
||||||
|
skipRelationalConstraints: "가져오기 중 관계형 제약 조건을 일시적으로 해제",
|
||||||
continueOnError: "오류 시 계속",
|
continueOnError: "오류 시 계속",
|
||||||
runInBackground: "백그라운드에서 실행",
|
runInBackground: "백그라운드에서 실행",
|
||||||
cancelling: "취소하는 중...",
|
cancelling: "취소하는 중...",
|
||||||
|
|||||||
@@ -6040,6 +6040,8 @@ export default withEnglishFallback({
|
|||||||
selectDatabase: "Selecionar banco de dados",
|
selectDatabase: "Selecionar banco de dados",
|
||||||
databasePlaceholder: "Nome do banco de dados",
|
databasePlaceholder: "Nome do banco de dados",
|
||||||
options: "Opções",
|
options: "Opções",
|
||||||
|
packageParts: "{count} partes SQL",
|
||||||
|
skipRelationalConstraints: "Desabilitar temporariamente restrições relacionais durante a importação",
|
||||||
continueOnError: "Continuar em caso de erro",
|
continueOnError: "Continuar em caso de erro",
|
||||||
runInBackground: "Executar em segundo plano",
|
runInBackground: "Executar em segundo plano",
|
||||||
cancelling: "Cancelando...",
|
cancelling: "Cancelando...",
|
||||||
|
|||||||
@@ -6152,6 +6152,8 @@ export default withEnglishFallback({
|
|||||||
selectDatabase: "Veritabanı seç",
|
selectDatabase: "Veritabanı seç",
|
||||||
databasePlaceholder: "Veritabanı adı",
|
databasePlaceholder: "Veritabanı adı",
|
||||||
options: "Seçenekler",
|
options: "Seçenekler",
|
||||||
|
packageParts: "{count} SQL bölümü",
|
||||||
|
skipRelationalConstraints: "İçe aktarma sırasında ilişkisel kısıtlamaları geçici olarak devre dışı bırak",
|
||||||
continueOnError: "Hatada devam et",
|
continueOnError: "Hatada devam et",
|
||||||
runInBackground: "Arka planda çalıştır",
|
runInBackground: "Arka planda çalıştır",
|
||||||
cancelling: "İptal ediliyor...",
|
cancelling: "İptal ediliyor...",
|
||||||
|
|||||||
@@ -6398,6 +6398,8 @@ export default withEnglishFallback({
|
|||||||
selectDatabase: "选择数据库",
|
selectDatabase: "选择数据库",
|
||||||
databasePlaceholder: "数据库名称",
|
databasePlaceholder: "数据库名称",
|
||||||
options: "选项",
|
options: "选项",
|
||||||
|
packageParts: "{count} 个 SQL 分片",
|
||||||
|
skipRelationalConstraints: "导入期间临时禁用关系约束",
|
||||||
continueOnError: "出错后继续",
|
continueOnError: "出错后继续",
|
||||||
runInBackground: "后台运行",
|
runInBackground: "后台运行",
|
||||||
cancelling: "正在取消...",
|
cancelling: "正在取消...",
|
||||||
|
|||||||
@@ -5360,6 +5360,8 @@ export default withEnglishFallback({
|
|||||||
selectDatabase: "選擇資料庫",
|
selectDatabase: "選擇資料庫",
|
||||||
databasePlaceholder: "資料庫名稱",
|
databasePlaceholder: "資料庫名稱",
|
||||||
options: "選項",
|
options: "選項",
|
||||||
|
packageParts: "{count} 個 SQL 分片",
|
||||||
|
skipRelationalConstraints: "匯入期間暫時停用關聯式約束",
|
||||||
continueOnError: "錯誤後繼續",
|
continueOnError: "錯誤後繼續",
|
||||||
cancelling: "正在取消……",
|
cancelling: "正在取消……",
|
||||||
cancel: "取消",
|
cancel: "取消",
|
||||||
|
|||||||
@@ -4769,6 +4769,8 @@ export interface SqlFileRequest {
|
|||||||
filePath: string;
|
filePath: string;
|
||||||
continueOnError: boolean;
|
continueOnError: boolean;
|
||||||
selectedTables?: SqlFileTable[];
|
selectedTables?: SqlFileTable[];
|
||||||
|
partCooldownMs?: number;
|
||||||
|
skipRelationalConstraints?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface SqlFileTable {
|
export interface SqlFileTable {
|
||||||
@@ -4787,6 +4789,8 @@ export interface SqlFilePreview {
|
|||||||
preview: string;
|
preview: string;
|
||||||
canExecuteWithoutSelectedDatabase: boolean;
|
canExecuteWithoutSelectedDatabase: boolean;
|
||||||
establishesDatabaseContext?: boolean;
|
establishesDatabaseContext?: boolean;
|
||||||
|
packageFilePaths?: string[];
|
||||||
|
packagePartCount?: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface SqlFileProgress {
|
export interface SqlFileProgress {
|
||||||
|
|||||||
@@ -95,6 +95,7 @@ pub mod sql_dialect;
|
|||||||
pub mod sql_editability;
|
pub mod sql_editability;
|
||||||
pub mod sql_error_position;
|
pub mod sql_error_position;
|
||||||
pub mod sql_file_import;
|
pub mod sql_file_import;
|
||||||
|
pub mod sql_file_zip_package;
|
||||||
pub mod sql_parser;
|
pub mod sql_parser;
|
||||||
pub mod sql_risk;
|
pub mod sql_risk;
|
||||||
pub mod sqlite_backup;
|
pub mod sqlite_backup;
|
||||||
|
|||||||
@@ -61,6 +61,13 @@ pub struct SqlFileRequest {
|
|||||||
pub continue_on_error: bool,
|
pub continue_on_error: bool,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub selected_tables: Option<Vec<crate::sql_file_import::SqlFileTable>>,
|
pub selected_tables: Option<Vec<crate::sql_file_import::SqlFileTable>>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub part_cooldown_ms: u64,
|
||||||
|
/// Temporarily disable MySQL `FOREIGN_KEY_CHECKS` for this import and
|
||||||
|
/// restore them on completion, error, or cancellation. Only applies to
|
||||||
|
/// MySQL-compatible connections that reuse one pinned session.
|
||||||
|
#[serde(default)]
|
||||||
|
pub skip_relational_constraints: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
@@ -73,6 +80,10 @@ pub struct SqlFilePreview {
|
|||||||
pub can_execute_without_selected_database: bool,
|
pub can_execute_without_selected_database: bool,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub establishes_database_context: bool,
|
pub establishes_database_context: bool,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub package_file_paths: Option<Vec<String>>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub package_part_count: Option<usize>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ use std::path::Path;
|
|||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use mysql_async::prelude::Queryable;
|
||||||
use tokio::io::{AsyncReadExt, BufReader};
|
use tokio::io::{AsyncReadExt, BufReader};
|
||||||
use tokio_util::sync::CancellationToken;
|
use tokio_util::sync::CancellationToken;
|
||||||
|
|
||||||
@@ -183,6 +184,7 @@ struct MySqlSqlFileExecutor {
|
|||||||
dialect: db::mysql::MySqlQueryDialect,
|
dialect: db::mysql::MySqlQueryDialect,
|
||||||
budget: DbOperationBudget,
|
budget: DbOperationBudget,
|
||||||
conn: Option<mysql_async::Conn>,
|
conn: Option<mysql_async::Conn>,
|
||||||
|
constraints_disabled: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl MySqlSqlFileExecutor {
|
impl MySqlSqlFileExecutor {
|
||||||
@@ -229,6 +231,7 @@ impl MySqlSqlFileExecutor {
|
|||||||
),
|
),
|
||||||
budget,
|
budget,
|
||||||
conn: None,
|
conn: None,
|
||||||
|
constraints_disabled: false,
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -320,6 +323,11 @@ impl MySqlSqlFileExecutor {
|
|||||||
let database = self.database.trim();
|
let database = self.database.trim();
|
||||||
let database = (!database.is_empty()).then_some(database);
|
let database = (!database.is_empty()).then_some(database);
|
||||||
self.pool_key = state.reconnect_pool_for_session(&self.connection_id, database, None).await?;
|
self.pool_key = state.reconnect_pool_for_session(&self.connection_id, database, None).await?;
|
||||||
|
// A fresh session restores the server default FOREIGN_KEY_CHECKS = 1;
|
||||||
|
// re-issue the bypass so the remaining parts match the caller's toggle.
|
||||||
|
if self.constraints_disabled {
|
||||||
|
self.set_foreign_key_checks(state, child_token, false).await?;
|
||||||
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
// Cancelled, or the retry itself failed with another
|
// Cancelled, or the retry itself failed with another
|
||||||
@@ -338,6 +346,19 @@ impl MySqlSqlFileExecutor {
|
|||||||
unreachable!("MySQL SQL file executor retry loop runs at most twice")
|
unreachable!("MySQL SQL file executor retry loop runs at most twice")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn set_foreign_key_checks(
|
||||||
|
&mut self,
|
||||||
|
state: &AppState,
|
||||||
|
token: &CancellationToken,
|
||||||
|
enabled: bool,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
self.ensure_conn(state, token).await?;
|
||||||
|
let conn = self.conn.as_mut().ok_or("MySQL SQL file executor is missing a connection".to_string())?;
|
||||||
|
conn.query_drop(if enabled { "SET FOREIGN_KEY_CHECKS = 1" } else { "SET FOREIGN_KEY_CHECKS = 0" })
|
||||||
|
.await
|
||||||
|
.map_err(|error| error.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
async fn ensure_conn(&mut self, state: &AppState, token: &CancellationToken) -> Result<(), String> {
|
async fn ensure_conn(&mut self, state: &AppState, token: &CancellationToken) -> Result<(), String> {
|
||||||
if self.conn.is_some() {
|
if self.conn.is_some() {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
@@ -489,72 +510,121 @@ pub async fn execute_sql_file_paths(
|
|||||||
return Err(error);
|
return Err(error);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
let constraints_disabled = request.skip_relational_constraints && mysql_executor.is_some();
|
||||||
|
if constraints_disabled {
|
||||||
|
let executor = mysql_executor.as_mut().expect("missing MySQL executor while disabling constraints");
|
||||||
|
if let Err(error) = executor.set_foreign_key_checks(state, &token, false).await {
|
||||||
|
emit(sql_file_execution_error_progress(&request.execution_id, started_at, &progress, error.clone()));
|
||||||
|
return Err(error);
|
||||||
|
}
|
||||||
|
executor.constraints_disabled = true;
|
||||||
|
}
|
||||||
let file_count = file_paths.len();
|
let file_count = file_paths.len();
|
||||||
let mut prev_statement_index = 0usize;
|
let mut prev_statement_index = 0usize;
|
||||||
let mut prev_success_count = 0usize;
|
let mut prev_success_count = 0usize;
|
||||||
let mut prev_failure_count = 0usize;
|
let mut prev_failure_count = 0usize;
|
||||||
let mut prev_affected_rows = 0u64;
|
let mut prev_affected_rows = 0u64;
|
||||||
for (file_index, file_path) in file_paths.iter().enumerate() {
|
let import_result = async {
|
||||||
let file_name = file_path.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
|
for (file_index, file_path) in file_paths.iter().enumerate() {
|
||||||
|
let file_name = file_path.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
|
||||||
|
|
||||||
// Emit a file-boundary progress event so the frontend knows which file is
|
// Emit a file-boundary progress event so the frontend knows which file is
|
||||||
// currently executing and can display a "File N/M" indicator.
|
// currently executing and can display a "File N/M" indicator.
|
||||||
if file_count > 1 {
|
if file_count > 1 {
|
||||||
emit(SqlFileProgress {
|
emit(SqlFileProgress {
|
||||||
execution_id: request.execution_id.clone(),
|
execution_id: request.execution_id.clone(),
|
||||||
status: SqlFileStatus::Running,
|
status: SqlFileStatus::Running,
|
||||||
statement_index: progress.statement_index,
|
statement_index: progress.statement_index,
|
||||||
success_count: progress.success_count,
|
success_count: progress.success_count,
|
||||||
failure_count: progress.failure_count,
|
failure_count: progress.failure_count,
|
||||||
affected_rows: progress.affected_rows,
|
affected_rows: progress.affected_rows,
|
||||||
elapsed_ms: started_at.elapsed().as_millis(),
|
elapsed_ms: started_at.elapsed().as_millis(),
|
||||||
statement_summary: String::new(),
|
statement_summary: String::new(),
|
||||||
error: None,
|
error: None,
|
||||||
file_index: Some(file_index),
|
file_index: Some(file_index),
|
||||||
file_name: Some(file_name.clone()),
|
file_name: Some(file_name.clone()),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut splitter = StreamingSqlFileSplitter::new(database_type, options);
|
||||||
|
let mut pending_statements = Vec::with_capacity(SQL_FILE_STATEMENT_BATCH_SIZE);
|
||||||
|
let normalize_mysql_binary_literals = import_target.as_ref().is_some_and(|target| {
|
||||||
|
crate::sql::is_mysql_compatible_import_target(&target.db_type, target.driver_profile.as_deref())
|
||||||
});
|
});
|
||||||
}
|
let mut decoder =
|
||||||
|
match SqlFileStreamDecoder::open_for_target(file_path, normalize_mysql_binary_literals).await {
|
||||||
|
Ok(decoder) => decoder,
|
||||||
|
Err(error) => {
|
||||||
|
emit(sql_file_execution_error_progress(
|
||||||
|
&request.execution_id,
|
||||||
|
started_at,
|
||||||
|
&progress,
|
||||||
|
error.clone(),
|
||||||
|
));
|
||||||
|
return Err(error);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
let mut splitter = StreamingSqlFileSplitter::new(database_type, options);
|
loop {
|
||||||
let mut pending_statements = Vec::with_capacity(SQL_FILE_STATEMENT_BATCH_SIZE);
|
let chunk = match decoder.next_chunk().await {
|
||||||
let normalize_mysql_binary_literals = import_target.as_ref().is_some_and(|target| {
|
Ok(chunk) => chunk,
|
||||||
crate::sql::is_mysql_compatible_import_target(&target.db_type, target.driver_profile.as_deref())
|
Err(error) => {
|
||||||
});
|
emit(sql_file_progress(
|
||||||
let mut decoder = match SqlFileStreamDecoder::open_for_target(file_path, normalize_mysql_binary_literals).await
|
&request.execution_id,
|
||||||
{
|
SqlFileStatus::Error,
|
||||||
Ok(decoder) => decoder,
|
progress.statement_index,
|
||||||
Err(error) => {
|
progress.success_count,
|
||||||
emit(sql_file_execution_error_progress(&request.execution_id, started_at, &progress, error.clone()));
|
progress.failure_count,
|
||||||
return Err(error);
|
progress.affected_rows,
|
||||||
}
|
started_at,
|
||||||
};
|
"",
|
||||||
|
Some(error.clone()),
|
||||||
loop {
|
));
|
||||||
let chunk = match decoder.next_chunk().await {
|
return Err(error);
|
||||||
Ok(chunk) => chunk,
|
}
|
||||||
Err(error) => {
|
};
|
||||||
emit(sql_file_progress(
|
let Some(chunk) = chunk else {
|
||||||
&request.execution_id,
|
break;
|
||||||
SqlFileStatus::Error,
|
};
|
||||||
progress.statement_index,
|
if token.is_cancelled() {
|
||||||
progress.success_count,
|
emit_sql_file_terminal_progress(request, &token, started_at, &progress, &mut emit);
|
||||||
progress.failure_count,
|
return Ok(());
|
||||||
progress.affected_rows,
|
|
||||||
started_at,
|
|
||||||
"",
|
|
||||||
Some(error.clone()),
|
|
||||||
));
|
|
||||||
return Err(error);
|
|
||||||
}
|
}
|
||||||
};
|
let mut next_statements = splitter.push_chunk(&chunk);
|
||||||
let Some(chunk) = chunk else {
|
if let Some(filter) = restore_filter.as_mut() {
|
||||||
break;
|
if let Err(error) = filter_restore_statements(
|
||||||
};
|
&mut next_statements,
|
||||||
if token.is_cancelled() {
|
filter,
|
||||||
emit_sql_file_terminal_progress(request, &token, started_at, &progress, &mut emit);
|
request.selected_tables.as_deref().unwrap_or_default(),
|
||||||
return Ok(());
|
) {
|
||||||
|
emit(sql_file_execution_error_progress(
|
||||||
|
&request.execution_id,
|
||||||
|
started_at,
|
||||||
|
&progress,
|
||||||
|
error.clone(),
|
||||||
|
));
|
||||||
|
return Err(error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pending_statements.extend(next_statements);
|
||||||
|
if pending_statements.len() < SQL_FILE_STATEMENT_BATCH_SIZE {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
execute_sql_file_statement_batch(
|
||||||
|
state,
|
||||||
|
request,
|
||||||
|
&token,
|
||||||
|
started_at,
|
||||||
|
&mut pending_statements,
|
||||||
|
import_target.as_ref(),
|
||||||
|
mysql_executor.as_mut(),
|
||||||
|
&mut progress,
|
||||||
|
&mut emit,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
}
|
}
|
||||||
let mut next_statements = splitter.push_chunk(&chunk);
|
|
||||||
|
let mut next_statements = splitter.finish();
|
||||||
if let Some(filter) = restore_filter.as_mut() {
|
if let Some(filter) = restore_filter.as_mut() {
|
||||||
if let Err(error) = filter_restore_statements(
|
if let Err(error) = filter_restore_statements(
|
||||||
&mut next_statements,
|
&mut next_statements,
|
||||||
@@ -571,9 +641,6 @@ pub async fn execute_sql_file_paths(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
pending_statements.extend(next_statements);
|
pending_statements.extend(next_statements);
|
||||||
if pending_statements.len() < SQL_FILE_STATEMENT_BATCH_SIZE {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
execute_sql_file_statement_batch(
|
execute_sql_file_statement_batch(
|
||||||
state,
|
state,
|
||||||
request,
|
request,
|
||||||
@@ -586,57 +653,48 @@ pub async fn execute_sql_file_paths(
|
|||||||
&mut emit,
|
&mut emit,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
}
|
|
||||||
|
|
||||||
let mut next_statements = splitter.finish();
|
// After each file, emit a per-file summary with diff-based counters so
|
||||||
if let Some(filter) = restore_filter.as_mut() {
|
// the frontend can build a per-file breakdown table.
|
||||||
if let Err(error) = filter_restore_statements(
|
if file_count > 1 {
|
||||||
&mut next_statements,
|
emit(SqlFileProgress {
|
||||||
filter,
|
execution_id: request.execution_id.clone(),
|
||||||
request.selected_tables.as_deref().unwrap_or_default(),
|
status: SqlFileStatus::StatementDone,
|
||||||
) {
|
statement_index: progress.statement_index - prev_statement_index,
|
||||||
emit(sql_file_execution_error_progress(&request.execution_id, started_at, &progress, error.clone()));
|
success_count: progress.success_count - prev_success_count,
|
||||||
return Err(error);
|
failure_count: progress.failure_count - prev_failure_count,
|
||||||
|
affected_rows: progress.affected_rows - prev_affected_rows,
|
||||||
|
elapsed_ms: started_at.elapsed().as_millis(),
|
||||||
|
statement_summary: String::new(),
|
||||||
|
error: None,
|
||||||
|
file_index: Some(file_index),
|
||||||
|
file_name: Some(file_name),
|
||||||
|
});
|
||||||
|
prev_statement_index = progress.statement_index;
|
||||||
|
prev_success_count = progress.success_count;
|
||||||
|
prev_failure_count = progress.failure_count;
|
||||||
|
prev_affected_rows = progress.affected_rows;
|
||||||
|
}
|
||||||
|
if file_index + 1 < file_count && request.part_cooldown_ms > 0 {
|
||||||
|
tokio::select! {
|
||||||
|
_ = token.cancelled() => {
|
||||||
|
emit_sql_file_terminal_progress(request, &token, started_at, &progress, &mut emit);
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
_ = tokio::time::sleep(Duration::from_millis(request.part_cooldown_ms)) => {}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
pending_statements.extend(next_statements);
|
emit_sql_file_terminal_progress(request, &token, started_at, &progress, &mut emit);
|
||||||
execute_sql_file_statement_batch(
|
Ok(())
|
||||||
state,
|
}
|
||||||
request,
|
.await;
|
||||||
&token,
|
if constraints_disabled {
|
||||||
started_at,
|
if let Some(executor) = mysql_executor.as_mut() {
|
||||||
&mut pending_statements,
|
let _ = executor.set_foreign_key_checks(state, &token, true).await;
|
||||||
import_target.as_ref(),
|
|
||||||
mysql_executor.as_mut(),
|
|
||||||
&mut progress,
|
|
||||||
&mut emit,
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
// After each file, emit a per-file summary with diff-based counters so
|
|
||||||
// the frontend can build a per-file breakdown table.
|
|
||||||
if file_count > 1 {
|
|
||||||
emit(SqlFileProgress {
|
|
||||||
execution_id: request.execution_id.clone(),
|
|
||||||
status: SqlFileStatus::StatementDone,
|
|
||||||
statement_index: progress.statement_index - prev_statement_index,
|
|
||||||
success_count: progress.success_count - prev_success_count,
|
|
||||||
failure_count: progress.failure_count - prev_failure_count,
|
|
||||||
affected_rows: progress.affected_rows - prev_affected_rows,
|
|
||||||
elapsed_ms: started_at.elapsed().as_millis(),
|
|
||||||
statement_summary: String::new(),
|
|
||||||
error: None,
|
|
||||||
file_index: Some(file_index),
|
|
||||||
file_name: Some(file_name),
|
|
||||||
});
|
|
||||||
prev_statement_index = progress.statement_index;
|
|
||||||
prev_success_count = progress.success_count;
|
|
||||||
prev_failure_count = progress.failure_count;
|
|
||||||
prev_affected_rows = progress.affected_rows;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
emit_sql_file_terminal_progress(request, &token, started_at, &progress, &mut emit);
|
import_result
|
||||||
Ok(())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn inspect_sql_file_tables(file_path: &Path) -> Result<Vec<SqlFileTable>, String> {
|
pub async fn inspect_sql_file_tables(file_path: &Path) -> Result<Vec<SqlFileTable>, String> {
|
||||||
@@ -2589,6 +2647,8 @@ mod tests {
|
|||||||
file_path: path.to_string_lossy().to_string(),
|
file_path: path.to_string_lossy().to_string(),
|
||||||
continue_on_error: true,
|
continue_on_error: true,
|
||||||
selected_tables: None,
|
selected_tables: None,
|
||||||
|
part_cooldown_ms: 0,
|
||||||
|
skip_relational_constraints: false,
|
||||||
};
|
};
|
||||||
let mut progress = Vec::new();
|
let mut progress = Vec::new();
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,178 @@
|
|||||||
|
use std::io::{Read, Write};
|
||||||
|
use std::path::{Component, Path, PathBuf};
|
||||||
|
|
||||||
|
use serde::Deserialize;
|
||||||
|
|
||||||
|
pub const SQL_FILE_ZIP_MAX_PARTS: usize = 10_000;
|
||||||
|
pub const SQL_FILE_ZIP_MAX_MANIFEST_BYTES: u64 = 1024 * 1024;
|
||||||
|
pub const SQL_FILE_ZIP_MAX_TOTAL_BYTES: u64 = 4 * 1024 * 1024 * 1024;
|
||||||
|
pub const SQL_FILE_ZIP_MAX_PART_BYTES: u64 = 1024 * 1024 * 1024;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Deserialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
struct SqlZipManifest {
|
||||||
|
format: String,
|
||||||
|
source_file_name: String,
|
||||||
|
parts: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct SqlFileZipPackage {
|
||||||
|
pub source_file_name: String,
|
||||||
|
pub part_names: Vec<String>,
|
||||||
|
pub total_bytes: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn inspect_sql_file_zip_package(path: &Path) -> Result<SqlFileZipPackage, String> {
|
||||||
|
let file = std::fs::File::open(path).map_err(|error| format!("Failed to open SQL ZIP package: {error}"))?;
|
||||||
|
let mut archive = zip::ZipArchive::new(file).map_err(|error| format!("Invalid SQL ZIP package: {error}"))?;
|
||||||
|
let manifest_bytes = read_zip_entry(&mut archive, "manifest.json", SQL_FILE_ZIP_MAX_MANIFEST_BYTES)?;
|
||||||
|
let manifest: SqlZipManifest =
|
||||||
|
serde_json::from_slice(&manifest_bytes).map_err(|error| format!("Invalid SQL ZIP manifest: {error}"))?;
|
||||||
|
if manifest.format != "dbx-sql-export-parts-v1" {
|
||||||
|
return Err("Unsupported SQL ZIP package format".to_string());
|
||||||
|
}
|
||||||
|
if manifest.parts.is_empty() || manifest.parts.len() > SQL_FILE_ZIP_MAX_PARTS {
|
||||||
|
return Err("SQL ZIP manifest has an invalid number of parts".to_string());
|
||||||
|
}
|
||||||
|
let mut seen = std::collections::HashSet::new();
|
||||||
|
let mut total_bytes = 0_u64;
|
||||||
|
for part in &manifest.parts {
|
||||||
|
validate_part_name(part)?;
|
||||||
|
if !seen.insert(part) {
|
||||||
|
return Err(format!("SQL ZIP manifest contains duplicate part: {part}"));
|
||||||
|
}
|
||||||
|
let entry = archive.by_name(part).map_err(|_| format!("SQL ZIP package is missing manifest part: {part}"))?;
|
||||||
|
if entry.is_dir() || entry.size() > SQL_FILE_ZIP_MAX_PART_BYTES {
|
||||||
|
return Err(format!("SQL ZIP part exceeds limits: {part}"));
|
||||||
|
}
|
||||||
|
total_bytes = total_bytes.checked_add(entry.size()).ok_or("SQL ZIP package size overflow")?;
|
||||||
|
if total_bytes > SQL_FILE_ZIP_MAX_TOTAL_BYTES {
|
||||||
|
return Err("SQL ZIP package exceeds total extracted size limit".to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(SqlFileZipPackage { source_file_name: manifest.source_file_name, part_names: manifest.parts, total_bytes })
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn extract_sql_file_zip_package(path: &Path, destination: &Path) -> Result<SqlFileZipPackage, String> {
|
||||||
|
let package = inspect_sql_file_zip_package(path)?;
|
||||||
|
std::fs::create_dir_all(destination)
|
||||||
|
.map_err(|error| format!("Failed to create SQL ZIP extraction directory: {error}"))?;
|
||||||
|
let file = std::fs::File::open(path).map_err(|error| format!("Failed to open SQL ZIP package: {error}"))?;
|
||||||
|
let mut archive = zip::ZipArchive::new(file).map_err(|error| format!("Invalid SQL ZIP package: {error}"))?;
|
||||||
|
for (index, part) in package.part_names.iter().enumerate() {
|
||||||
|
let entry = archive.by_name(part).map_err(|_| format!("SQL ZIP package is missing manifest part: {part}"))?;
|
||||||
|
let output = destination.join(format!("{:05}-{}", index + 1, part));
|
||||||
|
let mut target =
|
||||||
|
std::fs::File::create(&output).map_err(|error| format!("Failed to create extracted SQL part: {error}"))?;
|
||||||
|
// `entry.size()` is declared metadata; the zip reader does not clamp decompressed output
|
||||||
|
// to it, so the copy itself must be bounded to keep a lying header from writing past the
|
||||||
|
// inspected limit before the mismatch check runs.
|
||||||
|
let declared = entry.size();
|
||||||
|
let mut bounded = entry.take(declared + 1);
|
||||||
|
let copied =
|
||||||
|
std::io::copy(&mut bounded, &mut target).map_err(|error| format!("Failed to extract SQL part: {error}"))?;
|
||||||
|
if copied != declared {
|
||||||
|
return Err(format!("Failed to fully extract SQL part: {part}"));
|
||||||
|
}
|
||||||
|
target.flush().map_err(|error| format!("Failed to flush extracted SQL part: {error}"))?;
|
||||||
|
}
|
||||||
|
Ok(package)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_zip_entry(archive: &mut zip::ZipArchive<std::fs::File>, name: &str, max_bytes: u64) -> Result<Vec<u8>, String> {
|
||||||
|
let entry = archive.by_name(name).map_err(|_| format!("SQL ZIP package is missing {name}"))?;
|
||||||
|
if entry.is_dir() || entry.size() > max_bytes {
|
||||||
|
return Err(format!("SQL ZIP package {name} exceeds limits"));
|
||||||
|
}
|
||||||
|
let mut bytes = Vec::with_capacity(entry.size() as usize);
|
||||||
|
let declared = entry.size();
|
||||||
|
let mut bounded = entry.take(max_bytes + 1);
|
||||||
|
bounded.read_to_end(&mut bytes).map_err(|error| format!("Failed to read SQL ZIP {name}: {error}"))?;
|
||||||
|
if bytes.len() as u64 != declared {
|
||||||
|
return Err(format!("SQL ZIP {name} does not match its declared size"));
|
||||||
|
}
|
||||||
|
Ok(bytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_part_name(name: &str) -> Result<(), String> {
|
||||||
|
let path = Path::new(name);
|
||||||
|
if name.trim().is_empty()
|
||||||
|
|| !name.ends_with(".sql")
|
||||||
|
|| path.is_absolute()
|
||||||
|
|| path.components().any(|component| !matches!(component, Component::Normal(_)))
|
||||||
|
{
|
||||||
|
return Err(format!("Unsafe SQL ZIP manifest part: {name}"));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn extracted_sql_zip_paths(destination: &Path, package: &SqlFileZipPackage) -> Vec<PathBuf> {
|
||||||
|
package
|
||||||
|
.part_names
|
||||||
|
.iter()
|
||||||
|
.enumerate()
|
||||||
|
.map(|(index, part)| destination.join(format!("{:05}-{}", index + 1, part)))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use zip::write::SimpleFileOptions;
|
||||||
|
|
||||||
|
fn write_package(path: &Path, parts: &[(&str, &str)], manifest_parts: &[&str]) {
|
||||||
|
let file = std::fs::File::create(path).unwrap();
|
||||||
|
let mut zip = zip::ZipWriter::new(file);
|
||||||
|
let options = SimpleFileOptions::default();
|
||||||
|
for (name, contents) in parts {
|
||||||
|
zip.start_file(*name, options).unwrap();
|
||||||
|
zip.write_all(contents.as_bytes()).unwrap();
|
||||||
|
}
|
||||||
|
let manifest = serde_json::json!({ "format": "dbx-sql-export-parts-v1", "sourceFileName": "source.sql", "parts": manifest_parts });
|
||||||
|
zip.start_file("manifest.json", options).unwrap();
|
||||||
|
zip.write_all(serde_json::to_string(&manifest).unwrap().as_bytes()).unwrap();
|
||||||
|
zip.finish().unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn preserves_manifest_order_when_extracting_parts() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let package_path = dir.path().join("package.zip");
|
||||||
|
write_package(&package_path, &[("z.sql", "SELECT 2;\n"), ("a.sql", "SELECT 1;\n")], &["a.sql", "z.sql"]);
|
||||||
|
let destination = dir.path().join("parts");
|
||||||
|
let package = extract_sql_file_zip_package(&package_path, &destination).unwrap();
|
||||||
|
let paths = extracted_sql_zip_paths(&destination, &package);
|
||||||
|
assert_eq!(std::fs::read_to_string(&paths[0]).unwrap(), "SELECT 1;\n");
|
||||||
|
assert_eq!(std::fs::read_to_string(&paths[1]).unwrap(), "SELECT 2;\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rejects_unsafe_manifest_paths() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let package_path = dir.path().join("package.zip");
|
||||||
|
write_package(&package_path, &[("safe.sql", "SELECT 1;\n")], &["../safe.sql"]);
|
||||||
|
assert!(inspect_sql_file_zip_package(&package_path).unwrap_err().contains("Unsafe"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rejects_parts_whose_declared_size_understates_content() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let package_path = dir.path().join("package.zip");
|
||||||
|
write_package(&package_path, &[("a.sql", &"SELECT 1;\n".repeat(64))], &["a.sql"]);
|
||||||
|
// Shrink the central-directory uncompressed size of the first entry so the declared
|
||||||
|
// metadata understates the real decompressed content: the bounded copy must stop at the
|
||||||
|
// declared size instead of inflating whatever the stream contains onto disk.
|
||||||
|
let mut bytes = std::fs::read(&package_path).unwrap();
|
||||||
|
let eocd = bytes.windows(4).rposition(|window| window == b"PK\x05\x06").unwrap();
|
||||||
|
let directory_offset = u32::from_le_bytes(bytes[eocd + 16..eocd + 20].try_into().unwrap()) as usize;
|
||||||
|
assert_eq!(&bytes[directory_offset..directory_offset + 4], b"PK\x01\x02");
|
||||||
|
let declared_size_at = directory_offset + 24;
|
||||||
|
bytes[declared_size_at..declared_size_at + 4].copy_from_slice(&1u32.to_le_bytes());
|
||||||
|
std::fs::write(&package_path, bytes).unwrap();
|
||||||
|
|
||||||
|
let destination = dir.path().join("parts");
|
||||||
|
let error = extract_sql_file_zip_package(&package_path, &destination).unwrap_err();
|
||||||
|
assert!(error.contains("Failed to fully extract SQL part"), "{error}");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1288,6 +1288,8 @@ INSERT INTO install_check (id) VALUES (1), (2);
|
|||||||
.into_owned(),
|
.into_owned(),
|
||||||
continue_on_error: false,
|
continue_on_error: false,
|
||||||
selected_tables: None,
|
selected_tables: None,
|
||||||
|
part_cooldown_ms: 0,
|
||||||
|
skip_relational_constraints: false,
|
||||||
};
|
};
|
||||||
|
|
||||||
let _ = execute_sql_statement(
|
let _ = execute_sql_statement(
|
||||||
@@ -1374,6 +1376,8 @@ INSERT INTO children (parent_id) VALUES (LAST_INSERT_ID());
|
|||||||
.into_owned(),
|
.into_owned(),
|
||||||
continue_on_error: false,
|
continue_on_error: false,
|
||||||
selected_tables: None,
|
selected_tables: None,
|
||||||
|
part_cooldown_ms: 0,
|
||||||
|
skip_relational_constraints: false,
|
||||||
};
|
};
|
||||||
|
|
||||||
let _ = execute_sql_statement(
|
let _ = execute_sql_statement(
|
||||||
@@ -1444,6 +1448,8 @@ async fn live_sql_file_import_preserves_raw_mysql_binary_literal_bytes() {
|
|||||||
file_path: std::env::temp_dir().join(format!("mysql-binary-dump-{suffix}.sql")).to_string_lossy().into_owned(),
|
file_path: std::env::temp_dir().join(format!("mysql-binary-dump-{suffix}.sql")).to_string_lossy().into_owned(),
|
||||||
continue_on_error: false,
|
continue_on_error: false,
|
||||||
selected_tables: None,
|
selected_tables: None,
|
||||||
|
part_cooldown_ms: 0,
|
||||||
|
skip_relational_constraints: false,
|
||||||
};
|
};
|
||||||
|
|
||||||
tokio::fs::write(&request.file_path, script).await.unwrap();
|
tokio::fs::write(&request.file_path, script).await.unwrap();
|
||||||
|
|||||||
@@ -76,6 +76,8 @@ async fn live_mysql_selected_table_restore_preserves_unselected_tables() {
|
|||||||
let request = SqlFileRequest {
|
let request = SqlFileRequest {
|
||||||
execution_id: format!("restore-{suffix}-{compressed}"), connection_id: connection_id.clone(), database: database.clone(), file_path: path.display().to_string(), continue_on_error: false,
|
execution_id: format!("restore-{suffix}-{compressed}"), connection_id: connection_id.clone(), database: database.clone(), file_path: path.display().to_string(), continue_on_error: false,
|
||||||
selected_tables: Some(vec![SqlFileTable { database: Some(database.clone()), name: "chosen".into() }]),
|
selected_tables: Some(vec![SqlFileTable { database: Some(database.clone()), name: "chosen".into() }]),
|
||||||
|
part_cooldown_ms: 0,
|
||||||
|
skip_relational_constraints: false,
|
||||||
};
|
};
|
||||||
execute_sql_file_path(&state, &request, &path, CancellationToken::new(), std::time::Instant::now(), |_| {}).await?;
|
execute_sql_file_path(&state, &request, &path, CancellationToken::new(), std::time::Instant::now(), |_| {}).await?;
|
||||||
let rows = execute_sql_statement(&state, &connection_id, &database, "SELECT id, LENGTH(body) FROM chosen", None, None).await?;
|
let rows = execute_sql_statement(&state, &connection_id, &database, "SELECT id, LENGTH(body) FROM chosen", None, None).await?;
|
||||||
@@ -90,6 +92,8 @@ async fn live_mysql_selected_table_restore_preserves_unselected_tables() {
|
|||||||
let request = SqlFileRequest {
|
let request = SqlFileRequest {
|
||||||
execution_id: format!("invalid-{suffix}"), connection_id: connection_id.clone(), database: database.clone(), file_path: path.display().to_string(), continue_on_error: true,
|
execution_id: format!("invalid-{suffix}"), connection_id: connection_id.clone(), database: database.clone(), file_path: path.display().to_string(), continue_on_error: true,
|
||||||
selected_tables: Some(vec![SqlFileTable { database: None, name: "chosen".into() }]),
|
selected_tables: Some(vec![SqlFileTable { database: None, name: "chosen".into() }]),
|
||||||
|
part_cooldown_ms: 0,
|
||||||
|
skip_relational_constraints: false,
|
||||||
};
|
};
|
||||||
let mut events = Vec::new();
|
let mut events = Vec::new();
|
||||||
assert!(execute_sql_file_path(&state, &request, &path, CancellationToken::new(), std::time::Instant::now(), |event| events.push(event)).await.is_err());
|
assert!(execute_sql_file_path(&state, &request, &path, CancellationToken::new(), std::time::Instant::now(), |event| events.push(event)).await.is_err());
|
||||||
@@ -171,6 +175,8 @@ async fn live_mysql_database_export_restores_dependent_views() {
|
|||||||
file_path: file_path.to_string_lossy().to_string(),
|
file_path: file_path.to_string_lossy().to_string(),
|
||||||
continue_on_error: false,
|
continue_on_error: false,
|
||||||
selected_tables: None,
|
selected_tables: None,
|
||||||
|
part_cooldown_ms: 0,
|
||||||
|
skip_relational_constraints: false,
|
||||||
};
|
};
|
||||||
execute_sql_file_path(
|
execute_sql_file_path(
|
||||||
&state,
|
&state,
|
||||||
|
|||||||
@@ -150,6 +150,8 @@ async fn live_postgres_all_schema_export_restores_one_sql_file() {
|
|||||||
file_path: export_path.display().to_string(),
|
file_path: export_path.display().to_string(),
|
||||||
continue_on_error: false,
|
continue_on_error: false,
|
||||||
selected_tables: None,
|
selected_tables: None,
|
||||||
|
part_cooldown_ms: 0,
|
||||||
|
skip_relational_constraints: false,
|
||||||
},
|
},
|
||||||
&export_path,
|
&export_path,
|
||||||
CancellationToken::new(),
|
CancellationToken::new(),
|
||||||
|
|||||||
@@ -1295,6 +1295,8 @@ async fn live_sqlserver_sql_file_import_executes_go_batches() {
|
|||||||
file_path: "fixture.sql".to_string(),
|
file_path: "fixture.sql".to_string(),
|
||||||
continue_on_error: false,
|
continue_on_error: false,
|
||||||
selected_tables: None,
|
selected_tables: None,
|
||||||
|
part_cooldown_ms: 0,
|
||||||
|
skip_relational_constraints: false,
|
||||||
};
|
};
|
||||||
let done_seen = AtomicBool::new(false);
|
let done_seen = AtomicBool::new(false);
|
||||||
|
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ use std::time::Duration;
|
|||||||
use axum::extract::{Multipart, Path as AxumPath, State};
|
use axum::extract::{Multipart, Path as AxumPath, State};
|
||||||
use axum::response::sse::{Event, Sse};
|
use axum::response::sse::{Event, Sse};
|
||||||
use axum::Json;
|
use axum::Json;
|
||||||
use dbx_core::sql::{SqlFileProgress, SqlFileRequest, SqlFileStatus};
|
use dbx_core::sql::{self, SqlFileProgress, SqlFileRequest, SqlFileStatus};
|
||||||
use dbx_core::sql_file_import::{
|
use dbx_core::sql_file_import::{
|
||||||
execute_sql_file_paths, sql_file_error_progress, sql_file_progress as build_sql_file_progress,
|
execute_sql_file_paths, sql_file_error_progress, sql_file_progress as build_sql_file_progress,
|
||||||
SqlFileProgressEmitter,
|
SqlFileProgressEmitter,
|
||||||
@@ -95,6 +95,31 @@ pub async fn preview_sql_file(
|
|||||||
let file_path = safe_uploaded_sql_path(&tmp_dir, &file_name)?;
|
let file_path = safe_uploaded_sql_path(&tmp_dir, &file_name)?;
|
||||||
std::fs::write(&file_path, &data).map_err(|e| AppError::from(e.to_string()))?;
|
std::fs::write(&file_path, &data).map_err(|e| AppError::from(e.to_string()))?;
|
||||||
|
|
||||||
|
if file_name.to_ascii_lowercase().ends_with(".zip") {
|
||||||
|
let extraction_dir = tmp_dir.join(format!("package-{}", Uuid::new_v4()));
|
||||||
|
let package = dbx_core::sql_file_zip_package::extract_sql_file_zip_package(&file_path, &extraction_dir)
|
||||||
|
.map_err(AppError::from)?;
|
||||||
|
let paths = dbx_core::sql_file_zip_package::extracted_sql_zip_paths(&extraction_dir, &package)
|
||||||
|
.into_iter()
|
||||||
|
.map(|path| path.to_string_lossy().to_string())
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
let content =
|
||||||
|
sql::decode_sql_file_bytes(&std::fs::read(&paths[0]).map_err(|e| AppError::from(e.to_string()))?)
|
||||||
|
.map_err(AppError::from)?;
|
||||||
|
let preview: String = content.chars().take(20_000).collect();
|
||||||
|
let bootstrap_analysis = dbx_core::sql_file_import::mysql_like_sql_file_bootstrap_analysis(&content);
|
||||||
|
return Ok(Json(serde_json::json!({
|
||||||
|
"fileName": file_name,
|
||||||
|
"filePath": file_path.to_string_lossy(),
|
||||||
|
"sizeBytes": data.len(),
|
||||||
|
"preview": preview,
|
||||||
|
"canExecuteWithoutSelectedDatabase": bootstrap_analysis.can_execute_without_selected_database,
|
||||||
|
"establishesDatabaseContext": bootstrap_analysis.establishes_database_context,
|
||||||
|
"packageFilePaths": paths,
|
||||||
|
"packagePartCount": package.part_names.len(),
|
||||||
|
})));
|
||||||
|
}
|
||||||
|
|
||||||
let size_bytes = data.len() as u64;
|
let size_bytes = data.len() as u64;
|
||||||
let content =
|
let content =
|
||||||
dbx_core::sql_file_import::read_sql_file_preview(&file_path, 1_000_000).await.map_err(AppError::from)?;
|
dbx_core::sql_file_import::read_sql_file_preview(&file_path, 1_000_000).await.map_err(AppError::from)?;
|
||||||
@@ -199,6 +224,7 @@ pub async fn execute_sql_file(
|
|||||||
})
|
})
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
|
cleanup_sql_file_package_paths(&file_paths);
|
||||||
cleanup_sql_file_execution(&state_clone, &req.execution_id).await;
|
cleanup_sql_file_execution(&state_clone, &req.execution_id).await;
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -211,6 +237,21 @@ fn send_sql_file_progress(tx: &broadcast::Sender<String>, progress: SqlFileProgr
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn cleanup_sql_file_package_paths(file_paths: &[PathBuf]) {
|
||||||
|
let mut directories = std::collections::HashSet::new();
|
||||||
|
for path in file_paths {
|
||||||
|
let Some(parent) = path.parent() else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if parent.file_name().and_then(|name| name.to_str()).is_some_and(|name| name.starts_with("package-")) {
|
||||||
|
directories.insert(parent.to_path_buf());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for directory in directories {
|
||||||
|
let _ = std::fs::remove_dir_all(directory);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn cleanup_sql_file_execution(state: &WebState, execution_id: &str) {
|
async fn cleanup_sql_file_execution(state: &WebState, execution_id: &str) {
|
||||||
state.remove_sse_channel(execution_id).await;
|
state.remove_sse_channel(execution_id).await;
|
||||||
state.sql_file_executions.write().await.remove(execution_id);
|
state.sql_file_executions.write().await.remove(execution_id);
|
||||||
@@ -245,14 +286,19 @@ fn cleanup_sql_file_uploads_older_than(tmp_dir: &Path, max_age: Duration) {
|
|||||||
};
|
};
|
||||||
for entry in entries.flatten() {
|
for entry in entries.flatten() {
|
||||||
let path = entry.path();
|
let path = entry.path();
|
||||||
let expired = entry
|
let Ok(metadata) = entry.metadata() else {
|
||||||
.metadata()
|
continue;
|
||||||
.ok()
|
};
|
||||||
.filter(|metadata| metadata.is_file())
|
// Extracted `package-*` directories count too: a preview that never reaches execution
|
||||||
.and_then(|metadata| metadata.modified().ok())
|
// leaves them behind, and they must expire with the uploaded files.
|
||||||
.and_then(|modified| modified.elapsed().ok())
|
let expired =
|
||||||
.is_some_and(|age| age >= max_age);
|
metadata.modified().ok().and_then(|modified| modified.elapsed().ok()).is_some_and(|age| age >= max_age);
|
||||||
if expired {
|
if !expired {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if metadata.is_dir() {
|
||||||
|
let _ = std::fs::remove_dir_all(path);
|
||||||
|
} else {
|
||||||
let _ = std::fs::remove_file(path);
|
let _ = std::fs::remove_file(path);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -41,6 +41,40 @@ pub async fn inspect_sql_file_tables(
|
|||||||
pub async fn preview_sql_file(file_path: String) -> Result<SqlFilePreview, String> {
|
pub async fn preview_sql_file(file_path: String) -> Result<SqlFilePreview, String> {
|
||||||
let path = PathBuf::from(&file_path);
|
let path = PathBuf::from(&file_path);
|
||||||
let metadata = tokio::fs::metadata(&path).await.map_err(|e| e.to_string())?;
|
let metadata = tokio::fs::metadata(&path).await.map_err(|e| e.to_string())?;
|
||||||
|
if path
|
||||||
|
.extension()
|
||||||
|
.and_then(|extension| extension.to_str())
|
||||||
|
.is_some_and(|extension| extension.eq_ignore_ascii_case("zip"))
|
||||||
|
{
|
||||||
|
sweep_stale_sql_zip_packages();
|
||||||
|
let extraction_dir = std::env::temp_dir().join(format!("dbx-sql-package-{}", uuid::Uuid::new_v4()));
|
||||||
|
let (package, extracted_paths) = tokio::task::spawn_blocking({
|
||||||
|
let path = path.clone();
|
||||||
|
let extraction_dir = extraction_dir.clone();
|
||||||
|
move || {
|
||||||
|
let package = dbx_core::sql_file_zip_package::extract_sql_file_zip_package(&path, &extraction_dir)?;
|
||||||
|
let paths = dbx_core::sql_file_zip_package::extracted_sql_zip_paths(&extraction_dir, &package)
|
||||||
|
.into_iter()
|
||||||
|
.map(|part| part.to_string_lossy().to_string())
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
Ok::<_, String>((package, paths))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|error| format!("Failed to extract SQL ZIP package: {error}"))??;
|
||||||
|
let prefix = read_sql_file_preview(PathBuf::from(&extracted_paths[0]).as_path(), 1_000_000).await?;
|
||||||
|
let bootstrap_analysis = mysql_like_sql_file_bootstrap_analysis(&prefix);
|
||||||
|
return Ok(SqlFilePreview {
|
||||||
|
file_name: path.file_name().and_then(|name| name.to_str()).unwrap_or("package.zip").to_string(),
|
||||||
|
file_path,
|
||||||
|
size_bytes: metadata.len(),
|
||||||
|
preview: prefix.chars().take(20_000).collect(),
|
||||||
|
can_execute_without_selected_database: bootstrap_analysis.can_execute_without_selected_database,
|
||||||
|
establishes_database_context: bootstrap_analysis.establishes_database_context,
|
||||||
|
package_file_paths: Some(extracted_paths),
|
||||||
|
package_part_count: Some(package.part_names.len()),
|
||||||
|
});
|
||||||
|
}
|
||||||
let prefix = read_sql_file_preview(&path, 1_000_000).await?;
|
let prefix = read_sql_file_preview(&path, 1_000_000).await?;
|
||||||
let bootstrap_analysis = mysql_like_sql_file_bootstrap_analysis(&prefix);
|
let bootstrap_analysis = mysql_like_sql_file_bootstrap_analysis(&prefix);
|
||||||
let preview = prefix.chars().take(20_000).collect();
|
let preview = prefix.chars().take(20_000).collect();
|
||||||
@@ -52,6 +86,8 @@ pub async fn preview_sql_file(file_path: String) -> Result<SqlFilePreview, Strin
|
|||||||
preview,
|
preview,
|
||||||
can_execute_without_selected_database: bootstrap_analysis.can_execute_without_selected_database,
|
can_execute_without_selected_database: bootstrap_analysis.can_execute_without_selected_database,
|
||||||
establishes_database_context: bootstrap_analysis.establishes_database_context,
|
establishes_database_context: bootstrap_analysis.establishes_database_context,
|
||||||
|
package_file_paths: None,
|
||||||
|
package_part_count: None,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -84,6 +120,7 @@ pub async fn execute_sql_files(
|
|||||||
|
|
||||||
let started_at = Instant::now();
|
let started_at = Instant::now();
|
||||||
let result = execute_sql_files_inner(&app, &state, &request, &file_paths, token, started_at).await;
|
let result = execute_sql_files_inner(&app, &state, &request, &file_paths, token, started_at).await;
|
||||||
|
cleanup_sql_zip_package_paths(&file_paths);
|
||||||
{
|
{
|
||||||
let mut executions = sql_file_executions().write().await;
|
let mut executions = sql_file_executions().write().await;
|
||||||
remove_sql_file_execution(&mut executions, &request.execution_id);
|
remove_sql_file_execution(&mut executions, &request.execution_id);
|
||||||
@@ -91,6 +128,45 @@ pub async fn execute_sql_files(
|
|||||||
result
|
result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn cleanup_sql_zip_package_paths(file_paths: &[String]) {
|
||||||
|
let mut directories = std::collections::HashSet::new();
|
||||||
|
for path in file_paths {
|
||||||
|
let path = PathBuf::from(path);
|
||||||
|
let Some(parent) = path.parent() else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if parent.file_name().and_then(|name| name.to_str()).is_some_and(|name| name.starts_with("dbx-sql-package-")) {
|
||||||
|
directories.insert(parent.to_path_buf());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for directory in directories {
|
||||||
|
let _ = std::fs::remove_dir_all(directory);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A preview that never reaches execution leaves its extraction directory behind, so each new
|
||||||
|
/// preview also sweeps `dbx-sql-package-*` directories that have outlived a day.
|
||||||
|
fn sweep_stale_sql_zip_packages() {
|
||||||
|
const MAX_AGE: std::time::Duration = std::time::Duration::from_secs(24 * 60 * 60);
|
||||||
|
let Ok(entries) = std::fs::read_dir(std::env::temp_dir()) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
for entry in entries.flatten() {
|
||||||
|
if !entry.file_name().to_str().is_some_and(|name| name.starts_with("dbx-sql-package-")) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let expired = entry
|
||||||
|
.metadata()
|
||||||
|
.ok()
|
||||||
|
.and_then(|metadata| metadata.modified().ok())
|
||||||
|
.and_then(|modified| modified.elapsed().ok())
|
||||||
|
.is_some_and(|age| age >= MAX_AGE);
|
||||||
|
if expired {
|
||||||
|
let _ = std::fs::remove_dir_all(entry.path());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tauri::command]
|
#[tauri::command]
|
||||||
pub async fn cancel_sql_file_execution(execution_id: String) -> Result<bool, String> {
|
pub async fn cancel_sql_file_execution(execution_id: String) -> Result<bool, String> {
|
||||||
let executions = sql_file_executions().read().await;
|
let executions = sql_file_executions().read().await;
|
||||||
|
|||||||
Reference in new issue
Block a user