fix(sql-file): stream sqlserver zip imports incrementally
This commit is contained in:
1 parent
e0f802c031
commit
64e5847f74
7 files changed
+659
-66
No files matched your search
@@ -14,6 +14,7 @@ use crate::models::connection::DatabaseType;
|
|||||||
|
|
||||||
pub const DEFAULT_SQL_FILE_UPLOAD_MAX_MB: u32 = 200;
|
pub const DEFAULT_SQL_FILE_UPLOAD_MAX_MB: u32 = 200;
|
||||||
pub const MAX_SQL_FILE_UPLOAD_MAX_MB: u32 = 4096;
|
pub const MAX_SQL_FILE_UPLOAD_MAX_MB: u32 = 4096;
|
||||||
|
const SQLSERVER_ZIP_INSERT_BATCH_MAX_STATEMENTS: usize = 100;
|
||||||
|
|
||||||
pub fn clamp_sql_file_upload_max_mb(value: u32) -> u32 {
|
pub fn clamp_sql_file_upload_max_mb(value: u32) -> u32 {
|
||||||
value.clamp(1, MAX_SQL_FILE_UPLOAD_MAX_MB)
|
value.clamp(1, MAX_SQL_FILE_UPLOAD_MAX_MB)
|
||||||
@@ -24,7 +25,8 @@ use crate::query::{
|
|||||||
QueryExecutionOptions,
|
QueryExecutionOptions,
|
||||||
};
|
};
|
||||||
use crate::sql::{
|
use crate::sql::{
|
||||||
optimize_sql_file_import_statements, prepare_sql_file_statement, split_sql_batches, statement_summary,
|
optimize_sql_file_import_statements, optimize_sql_file_import_statements_with_max_insert_batch_statements,
|
||||||
|
prepare_sql_file_statement, split_sql_batches, starts_with_sqlserver_module_ddl, statement_summary,
|
||||||
SqlFileImportStatement, SqlFileImportStatementKind, SqlFilePhase, SqlFileProgress, SqlFileRequest,
|
SqlFileImportStatement, SqlFileImportStatementKind, SqlFilePhase, SqlFileProgress, SqlFileRequest,
|
||||||
SqlFileStatementAction, SqlFileStatus, SqlParsingOptions, SqlStatementSplitter, SqlStatementWithControl,
|
SqlFileStatementAction, SqlFileStatus, SqlParsingOptions, SqlStatementSplitter, SqlStatementWithControl,
|
||||||
};
|
};
|
||||||
@@ -691,6 +693,7 @@ async fn execute_sql_file_content_inner(
|
|||||||
import_target.as_ref().map(|target| target.db_type),
|
import_target.as_ref().map(|target| target.db_type),
|
||||||
import_target.as_ref().and_then(|target| target.driver_profile.as_deref()),
|
import_target.as_ref().and_then(|target| target.driver_profile.as_deref()),
|
||||||
request.txn_session_id.is_some(),
|
request.txn_session_id.is_some(),
|
||||||
|
None,
|
||||||
);
|
);
|
||||||
// MySQL-family imports need one pinned connection so `USE` and session
|
// MySQL-family imports need one pinned connection so `USE` and session
|
||||||
// state survive across the whole file.
|
// state survive across the whole file.
|
||||||
@@ -761,12 +764,43 @@ pub async fn execute_sql_file_paths(
|
|||||||
token: CancellationToken,
|
token: CancellationToken,
|
||||||
started_at: Instant,
|
started_at: Instant,
|
||||||
emit: impl FnMut(SqlFileProgress),
|
emit: impl FnMut(SqlFileProgress),
|
||||||
|
) -> Result<(), String> {
|
||||||
|
execute_sql_file_paths_with_options(state, request, file_paths, token, started_at, false, emit).await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn execute_sql_file_zip_package_paths(
|
||||||
|
state: &AppState,
|
||||||
|
request: &SqlFileRequest,
|
||||||
|
file_paths: &[&Path],
|
||||||
|
token: CancellationToken,
|
||||||
|
started_at: Instant,
|
||||||
|
emit: impl FnMut(SqlFileProgress),
|
||||||
|
) -> Result<(), String> {
|
||||||
|
execute_sql_file_paths_with_options(state, request, file_paths, token, started_at, true, emit).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn execute_sql_file_paths_with_options(
|
||||||
|
state: &AppState,
|
||||||
|
request: &SqlFileRequest,
|
||||||
|
file_paths: &[&Path],
|
||||||
|
token: CancellationToken,
|
||||||
|
started_at: Instant,
|
||||||
|
zip_package: bool,
|
||||||
|
emit: impl FnMut(SqlFileProgress),
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
with_sql_file_transaction(
|
with_sql_file_transaction(
|
||||||
state,
|
state,
|
||||||
request,
|
request,
|
||||||
&token,
|
&token,
|
||||||
Box::pin(execute_sql_file_paths_inner(state, request, file_paths, token.clone(), started_at, emit)),
|
Box::pin(execute_sql_file_paths_inner(
|
||||||
|
state,
|
||||||
|
request,
|
||||||
|
file_paths,
|
||||||
|
token.clone(),
|
||||||
|
started_at,
|
||||||
|
zip_package,
|
||||||
|
emit,
|
||||||
|
)),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
@@ -777,6 +811,7 @@ async fn execute_sql_file_paths_inner(
|
|||||||
file_paths: &[&Path],
|
file_paths: &[&Path],
|
||||||
token: CancellationToken,
|
token: CancellationToken,
|
||||||
started_at: Instant,
|
started_at: Instant,
|
||||||
|
zip_package: bool,
|
||||||
mut emit: impl FnMut(SqlFileProgress),
|
mut emit: impl FnMut(SqlFileProgress),
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
if file_paths.is_empty() {
|
if file_paths.is_empty() {
|
||||||
@@ -800,6 +835,7 @@ async fn execute_sql_file_paths_inner(
|
|||||||
})
|
})
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
let database_type = import_target.as_ref().map(|target| target.db_type);
|
let database_type = import_target.as_ref().map(|target| target.db_type);
|
||||||
|
let max_insert_batch_statements = sql_file_import_max_insert_batch_statements(zip_package, database_type);
|
||||||
let mut progress = SqlFileExecutionProgress::new();
|
let mut progress = SqlFileExecutionProgress::new();
|
||||||
// Validate the entire dump before any SQL reaches the database. This also
|
// Validate the entire dump before any SQL reaches the database. This also
|
||||||
// identifies view placeholders whose final CREATE VIEW appears later.
|
// identifies view placeholders whose final CREATE VIEW appears later.
|
||||||
@@ -859,10 +895,13 @@ async fn execute_sql_file_paths_inner(
|
|||||||
let mut prev_success_count = 0usize;
|
let mut prev_success_count = 0usize;
|
||||||
let mut prev_failure_count = 0usize;
|
let mut prev_failure_count = 0usize;
|
||||||
let mut prev_affected_rows = 0u64;
|
let mut prev_affected_rows = 0u64;
|
||||||
let mut splitter = Some(StreamingSqlFileSplitter::new(database_type, options));
|
let mut splitter = zip_package.then(|| StreamingSqlFileSplitter::new(database_type, options, true));
|
||||||
let mut pending_statements = Vec::with_capacity(SQL_FILE_STATEMENT_BATCH_SIZE);
|
let mut pending_statements = Vec::with_capacity(SQL_FILE_STATEMENT_BATCH_SIZE);
|
||||||
let import_result = async {
|
let import_result = async {
|
||||||
for (file_index, file_path) in file_paths.iter().enumerate() {
|
for (file_index, file_path) in file_paths.iter().enumerate() {
|
||||||
|
if !zip_package {
|
||||||
|
splitter = Some(StreamingSqlFileSplitter::new(database_type, options, false));
|
||||||
|
}
|
||||||
let file_name = file_path.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
|
let file_name = file_path.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
|
||||||
|
|
||||||
// Emit a file-boundary progress event so the frontend knows which file is
|
// Emit a file-boundary progress event so the frontend knows which file is
|
||||||
@@ -977,6 +1016,7 @@ async fn execute_sql_file_paths_inner(
|
|||||||
&token,
|
&token,
|
||||||
started_at,
|
started_at,
|
||||||
&mut pending_statements,
|
&mut pending_statements,
|
||||||
|
max_insert_batch_statements,
|
||||||
import_target.as_ref(),
|
import_target.as_ref(),
|
||||||
mysql_executor.as_mut(),
|
mysql_executor.as_mut(),
|
||||||
&mut progress,
|
&mut progress,
|
||||||
@@ -985,10 +1025,10 @@ async fn execute_sql_file_paths_inner(
|
|||||||
.await?;
|
.await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
let is_last_file = file_index + 1 == file_count;
|
let should_finish_splitter = !zip_package || file_index + 1 == file_count;
|
||||||
if is_last_file {
|
if should_finish_splitter {
|
||||||
let mut next_statements =
|
let mut next_statements =
|
||||||
splitter.take().expect("splitter is only taken once, on the last file").finish();
|
splitter.take().expect("splitter is finished at each ordinary file or the last ZIP part").finish();
|
||||||
if let Some(filter) = restore_filter.as_mut() {
|
if let Some(filter) = restore_filter.as_mut() {
|
||||||
if let Err(error) = filter_restore_statements(
|
if let Err(error) = filter_restore_statements(
|
||||||
&mut next_statements,
|
&mut next_statements,
|
||||||
@@ -1012,6 +1052,7 @@ async fn execute_sql_file_paths_inner(
|
|||||||
&token,
|
&token,
|
||||||
started_at,
|
started_at,
|
||||||
&mut pending_statements,
|
&mut pending_statements,
|
||||||
|
max_insert_batch_statements,
|
||||||
import_target.as_ref(),
|
import_target.as_ref(),
|
||||||
mysql_executor.as_mut(),
|
mysql_executor.as_mut(),
|
||||||
&mut progress,
|
&mut progress,
|
||||||
@@ -1088,7 +1129,8 @@ async fn scan_sql_file_tables(
|
|||||||
token: &CancellationToken,
|
token: &CancellationToken,
|
||||||
) -> Result<(Vec<SqlFileTable>, std::collections::BTreeSet<SqlFileTable>), String> {
|
) -> Result<(Vec<SqlFileTable>, std::collections::BTreeSet<SqlFileTable>), String> {
|
||||||
let mut decoder = SqlFileStreamDecoder::open_for_target(file_path, true).await?;
|
let mut decoder = SqlFileStreamDecoder::open_for_target(file_path, true).await?;
|
||||||
let mut splitter = StreamingSqlFileSplitter::new(Some(DatabaseType::Mysql), SqlParsingOptions::mysql_compatible());
|
let mut splitter =
|
||||||
|
StreamingSqlFileSplitter::new(Some(DatabaseType::Mysql), SqlParsingOptions::mysql_compatible(), false);
|
||||||
let mut scan = TableRestoreFilter::default();
|
let mut scan = TableRestoreFilter::default();
|
||||||
while let Some(chunk) = decoder.next_chunk().await? {
|
while let Some(chunk) = decoder.next_chunk().await? {
|
||||||
if token.is_cancelled() {
|
if token.is_cancelled() {
|
||||||
@@ -1732,8 +1774,12 @@ enum StreamingSqlFileSplitterKind {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl StreamingSqlFileSplitter {
|
impl StreamingSqlFileSplitter {
|
||||||
pub(crate) fn new(db_type: Option<DatabaseType>, options: SqlParsingOptions) -> Self {
|
pub(crate) fn new(
|
||||||
Self(StreamingSqlFileSplitterKind::new(db_type, options))
|
db_type: Option<DatabaseType>,
|
||||||
|
options: SqlParsingOptions,
|
||||||
|
flush_sqlserver_semicolons: bool,
|
||||||
|
) -> Self {
|
||||||
|
Self(StreamingSqlFileSplitterKind::new(db_type, options, flush_sqlserver_semicolons))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn push_chunk(&mut self, chunk: &str) -> Vec<SqlStatementWithControl> {
|
pub(crate) fn push_chunk(&mut self, chunk: &str) -> Vec<SqlStatementWithControl> {
|
||||||
@@ -1746,9 +1792,9 @@ impl StreamingSqlFileSplitter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl StreamingSqlFileSplitterKind {
|
impl StreamingSqlFileSplitterKind {
|
||||||
fn new(db_type: Option<DatabaseType>, options: SqlParsingOptions) -> Self {
|
fn new(db_type: Option<DatabaseType>, options: SqlParsingOptions, flush_sqlserver_semicolons: bool) -> Self {
|
||||||
if db_type == Some(DatabaseType::SqlServer) {
|
if db_type == Some(DatabaseType::SqlServer) {
|
||||||
Self::SqlServerBatches(SqlServerBatchSplitter::default())
|
Self::SqlServerBatches(SqlServerBatchSplitter::new(flush_sqlserver_semicolons))
|
||||||
} else {
|
} else {
|
||||||
Self::Statements(SqlStatementSplitter::with_options(options))
|
Self::Statements(SqlStatementSplitter::with_options(options))
|
||||||
}
|
}
|
||||||
@@ -1775,13 +1821,40 @@ impl StreamingSqlFileSplitterKind {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Default)]
|
|
||||||
struct SqlServerBatchSplitter {
|
struct SqlServerBatchSplitter {
|
||||||
batch: String,
|
batch: String,
|
||||||
partial_line: String,
|
partial_line: String,
|
||||||
|
lexical_state: SqlServerLexicalState,
|
||||||
|
flush_semicolons: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for SqlServerBatchSplitter {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::new(false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default, Clone, Copy)]
|
||||||
|
enum SqlServerLexicalState {
|
||||||
|
#[default]
|
||||||
|
Normal,
|
||||||
|
SingleQuote,
|
||||||
|
DoubleQuote,
|
||||||
|
BracketIdentifier,
|
||||||
|
LineComment,
|
||||||
|
BlockComment,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl SqlServerBatchSplitter {
|
impl SqlServerBatchSplitter {
|
||||||
|
fn new(flush_semicolons: bool) -> Self {
|
||||||
|
Self {
|
||||||
|
batch: String::new(),
|
||||||
|
partial_line: String::new(),
|
||||||
|
lexical_state: SqlServerLexicalState::default(),
|
||||||
|
flush_semicolons,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn push_chunk(&mut self, chunk: &str) -> Vec<String> {
|
fn push_chunk(&mut self, chunk: &str) -> Vec<String> {
|
||||||
self.partial_line.push_str(chunk);
|
self.partial_line.push_str(chunk);
|
||||||
let mut batches = Vec::new();
|
let mut batches = Vec::new();
|
||||||
@@ -1804,11 +1877,62 @@ impl SqlServerBatchSplitter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn push_line(&mut self, line: &str, batches: &mut Vec<String>) {
|
fn push_line(&mut self, line: &str, batches: &mut Vec<String>) {
|
||||||
if line.trim().eq_ignore_ascii_case("go") {
|
if matches!(self.lexical_state, SqlServerLexicalState::Normal) && line.trim().eq_ignore_ascii_case("go") {
|
||||||
self.push_batch(batches);
|
self.push_batch(batches);
|
||||||
} else {
|
return;
|
||||||
self.batch.push_str(line);
|
}
|
||||||
self.batch.push('\n');
|
let mut chars = line.chars().peekable();
|
||||||
|
while let Some(ch) = chars.next() {
|
||||||
|
self.batch.push(ch);
|
||||||
|
match self.lexical_state {
|
||||||
|
SqlServerLexicalState::Normal => match ch {
|
||||||
|
'\'' => self.lexical_state = SqlServerLexicalState::SingleQuote,
|
||||||
|
'"' => self.lexical_state = SqlServerLexicalState::DoubleQuote,
|
||||||
|
'[' => self.lexical_state = SqlServerLexicalState::BracketIdentifier,
|
||||||
|
'-' if chars.peek() == Some(&'-') => {
|
||||||
|
self.batch.push(chars.next().unwrap());
|
||||||
|
self.lexical_state = SqlServerLexicalState::LineComment;
|
||||||
|
}
|
||||||
|
'/' if chars.peek() == Some(&'*') => {
|
||||||
|
self.batch.push(chars.next().unwrap());
|
||||||
|
self.lexical_state = SqlServerLexicalState::BlockComment;
|
||||||
|
}
|
||||||
|
';' if self.flush_semicolons && !starts_with_sqlserver_module_ddl(&self.batch) => {
|
||||||
|
self.push_batch(batches)
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
},
|
||||||
|
SqlServerLexicalState::SingleQuote if ch == '\'' => {
|
||||||
|
if chars.peek() == Some(&'\'') {
|
||||||
|
self.batch.push(chars.next().unwrap());
|
||||||
|
} else {
|
||||||
|
self.lexical_state = SqlServerLexicalState::Normal;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
SqlServerLexicalState::DoubleQuote if ch == '"' => {
|
||||||
|
if chars.peek() == Some(&'"') {
|
||||||
|
self.batch.push(chars.next().unwrap());
|
||||||
|
} else {
|
||||||
|
self.lexical_state = SqlServerLexicalState::Normal;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
SqlServerLexicalState::BracketIdentifier if ch == ']' => {
|
||||||
|
if chars.peek() == Some(&']') {
|
||||||
|
self.batch.push(chars.next().unwrap());
|
||||||
|
} else {
|
||||||
|
self.lexical_state = SqlServerLexicalState::Normal;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
SqlServerLexicalState::BlockComment if ch == '*' && chars.peek() == Some(&'/') => {
|
||||||
|
self.batch.push(chars.next().unwrap());
|
||||||
|
self.lexical_state = SqlServerLexicalState::Normal;
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.batch.push('\n');
|
||||||
|
if matches!(self.lexical_state, SqlServerLexicalState::LineComment) {
|
||||||
|
self.lexical_state = SqlServerLexicalState::Normal;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1818,6 +1942,7 @@ impl SqlServerBatchSplitter {
|
|||||||
batches.push(batch.to_string());
|
batches.push(batch.to_string());
|
||||||
}
|
}
|
||||||
self.batch.clear();
|
self.batch.clear();
|
||||||
|
self.lexical_state = SqlServerLexicalState::Normal;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1828,6 +1953,7 @@ async fn execute_sql_file_statement_batch(
|
|||||||
token: &CancellationToken,
|
token: &CancellationToken,
|
||||||
started_at: Instant,
|
started_at: Instant,
|
||||||
statements: &mut Vec<SqlStatementWithControl>,
|
statements: &mut Vec<SqlStatementWithControl>,
|
||||||
|
max_insert_batch_statements: Option<usize>,
|
||||||
import_target: Option<&SqlFileImportTarget>,
|
import_target: Option<&SqlFileImportTarget>,
|
||||||
mysql_executor: Option<&mut MySqlSqlFileExecutor>,
|
mysql_executor: Option<&mut MySqlSqlFileExecutor>,
|
||||||
progress: &mut SqlFileExecutionProgress,
|
progress: &mut SqlFileExecutionProgress,
|
||||||
@@ -1842,6 +1968,7 @@ async fn execute_sql_file_statement_batch(
|
|||||||
import_target.map(|target| target.db_type),
|
import_target.map(|target| target.db_type),
|
||||||
import_target.and_then(|target| target.driver_profile.as_deref()),
|
import_target.and_then(|target| target.driver_profile.as_deref()),
|
||||||
request.txn_session_id.is_some(),
|
request.txn_session_id.is_some(),
|
||||||
|
max_insert_batch_statements,
|
||||||
);
|
);
|
||||||
execute_planned_statements_with_progress(
|
execute_planned_statements_with_progress(
|
||||||
state,
|
state,
|
||||||
@@ -1923,9 +2050,15 @@ fn plan_sql_file_statements(
|
|||||||
db_type: Option<DatabaseType>,
|
db_type: Option<DatabaseType>,
|
||||||
driver_profile: Option<&str>,
|
driver_profile: Option<&str>,
|
||||||
manual: bool,
|
manual: bool,
|
||||||
|
max_insert_batch_statements: Option<usize>,
|
||||||
) -> Vec<ControlledSqlFileImportStatement> {
|
) -> Vec<ControlledSqlFileImportStatement> {
|
||||||
if !manual {
|
if !manual {
|
||||||
return optimize_controlled_sql_file_import_statements(statements, db_type, driver_profile);
|
return optimize_controlled_sql_file_import_statements(
|
||||||
|
statements,
|
||||||
|
db_type,
|
||||||
|
driver_profile,
|
||||||
|
max_insert_batch_statements,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
// Preserve each statement and its error boundary. The normal import path
|
// Preserve each statement and its error boundary. The normal import path
|
||||||
// may merge INSERTs and retry them individually, which is not valid after
|
// may merge INSERTs and retry them individually, which is not valid after
|
||||||
@@ -1933,17 +2066,44 @@ fn plan_sql_file_statements(
|
|||||||
statements
|
statements
|
||||||
.iter()
|
.iter()
|
||||||
.flat_map(|statement| {
|
.flat_map(|statement| {
|
||||||
optimize_sql_file_import_statements(std::slice::from_ref(&statement.sql), db_type, driver_profile)
|
optimize_sql_file_import_statements_for_batch(
|
||||||
.into_iter()
|
std::slice::from_ref(&statement.sql),
|
||||||
.map(|statement| ControlledSqlFileImportStatement { statement, stop_on_error: true })
|
db_type,
|
||||||
|
driver_profile,
|
||||||
|
max_insert_batch_statements,
|
||||||
|
)
|
||||||
|
.into_iter()
|
||||||
|
.map(|statement| ControlledSqlFileImportStatement { statement, stop_on_error: true })
|
||||||
})
|
})
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn sql_file_import_max_insert_batch_statements(zip_package: bool, db_type: Option<DatabaseType>) -> Option<usize> {
|
||||||
|
(zip_package && db_type == Some(DatabaseType::SqlServer)).then_some(SQLSERVER_ZIP_INSERT_BATCH_MAX_STATEMENTS)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn optimize_sql_file_import_statements_for_batch(
|
||||||
|
sql: &[String],
|
||||||
|
db_type: Option<DatabaseType>,
|
||||||
|
driver_profile: Option<&str>,
|
||||||
|
max_insert_batch_statements: Option<usize>,
|
||||||
|
) -> Vec<SqlFileImportStatement> {
|
||||||
|
match max_insert_batch_statements {
|
||||||
|
Some(max_insert_batch_statements) => optimize_sql_file_import_statements_with_max_insert_batch_statements(
|
||||||
|
sql,
|
||||||
|
db_type,
|
||||||
|
driver_profile,
|
||||||
|
max_insert_batch_statements,
|
||||||
|
),
|
||||||
|
None => optimize_sql_file_import_statements(sql, db_type, driver_profile),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn optimize_controlled_sql_file_import_statements(
|
fn optimize_controlled_sql_file_import_statements(
|
||||||
statements: &[SqlStatementWithControl],
|
statements: &[SqlStatementWithControl],
|
||||||
db_type: Option<DatabaseType>,
|
db_type: Option<DatabaseType>,
|
||||||
driver_profile: Option<&str>,
|
driver_profile: Option<&str>,
|
||||||
|
max_insert_batch_statements: Option<usize>,
|
||||||
) -> Vec<ControlledSqlFileImportStatement> {
|
) -> Vec<ControlledSqlFileImportStatement> {
|
||||||
let mut controlled = Vec::new();
|
let mut controlled = Vec::new();
|
||||||
let mut start = 0;
|
let mut start = 0;
|
||||||
@@ -1955,7 +2115,7 @@ fn optimize_controlled_sql_file_import_statements(
|
|||||||
}
|
}
|
||||||
let sql = statements[start..end].iter().map(|statement| statement.sql.clone()).collect::<Vec<_>>();
|
let sql = statements[start..end].iter().map(|statement| statement.sql.clone()).collect::<Vec<_>>();
|
||||||
controlled.extend(
|
controlled.extend(
|
||||||
optimize_sql_file_import_statements(&sql, db_type, driver_profile)
|
optimize_sql_file_import_statements_for_batch(&sql, db_type, driver_profile, max_insert_batch_statements)
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|statement| ControlledSqlFileImportStatement { statement, stop_on_error }),
|
.map(|statement| ControlledSqlFileImportStatement { statement, stop_on_error }),
|
||||||
);
|
);
|
||||||
@@ -2688,9 +2848,9 @@ mod tests {
|
|||||||
Some(DatabaseType::Postgres),
|
Some(DatabaseType::Postgres),
|
||||||
None,
|
None,
|
||||||
);
|
);
|
||||||
let optimized = plan_sql_file_statements(&statements, Some(DatabaseType::Postgres), None, false);
|
let optimized = plan_sql_file_statements(&statements, Some(DatabaseType::Postgres), None, false, None);
|
||||||
assert_eq!(optimized.len(), 1);
|
assert_eq!(optimized.len(), 1);
|
||||||
let manual = plan_sql_file_statements(&statements, Some(DatabaseType::Postgres), None, true);
|
let manual = plan_sql_file_statements(&statements, Some(DatabaseType::Postgres), None, true, None);
|
||||||
assert_eq!(manual.len(), 2);
|
assert_eq!(manual.len(), 2);
|
||||||
assert!(manual.iter().all(|item| item.stop_on_error && item.statement.source_statement_count == 1));
|
assert!(manual.iter().all(|item| item.stop_on_error && item.statement.source_statement_count == 1));
|
||||||
assert_eq!(manual[0].statement.sql, statements[0].sql);
|
assert_eq!(manual[0].statement.sql, statements[0].sql);
|
||||||
@@ -2804,7 +2964,7 @@ mod tests {
|
|||||||
assert!(dump.len() > SQL_FILE_STATEMENT_BATCH_MAX_BYTES * 2, "the dump must exceed the byte bound");
|
assert!(dump.len() > SQL_FILE_STATEMENT_BATCH_MAX_BYTES * 2, "the dump must exceed the byte bound");
|
||||||
|
|
||||||
let mut splitter =
|
let mut splitter =
|
||||||
StreamingSqlFileSplitter::new(Some(DatabaseType::Mysql), SqlParsingOptions::mysql_compatible());
|
StreamingSqlFileSplitter::new(Some(DatabaseType::Mysql), SqlParsingOptions::mysql_compatible(), false);
|
||||||
let mut pending: Vec<SqlStatementWithControl> = Vec::new();
|
let mut pending: Vec<SqlStatementWithControl> = Vec::new();
|
||||||
let mut batches: Vec<(usize, usize)> = Vec::new();
|
let mut batches: Vec<(usize, usize)> = Vec::new();
|
||||||
let mut largest_statement = 0usize;
|
let mut largest_statement = 0usize;
|
||||||
@@ -2851,7 +3011,7 @@ mod tests {
|
|||||||
SqlFileTable { database: Some("second".to_string()), name: "b".to_string() },
|
SqlFileTable { database: Some("second".to_string()), name: "b".to_string() },
|
||||||
];
|
];
|
||||||
let mut splitter =
|
let mut splitter =
|
||||||
StreamingSqlFileSplitter::new(Some(DatabaseType::Mysql), SqlParsingOptions::mysql_compatible());
|
StreamingSqlFileSplitter::new(Some(DatabaseType::Mysql), SqlParsingOptions::mysql_compatible(), false);
|
||||||
let mut filter = TableRestoreFilter::default();
|
let mut filter = TableRestoreFilter::default();
|
||||||
let mut statements = Vec::new();
|
let mut statements = Vec::new();
|
||||||
for chunk in ["INSERT INTO a VALUES (1); USE second;", "INSERT INTO b VALUES (2); INSERT INTO a VALUES (3);"] {
|
for chunk in ["INSERT INTO a VALUES (1); USE second;", "INSERT INTO b VALUES (2); INSERT INTO a VALUES (3);"] {
|
||||||
@@ -3195,18 +3355,135 @@ mod tests {
|
|||||||
assert_eq!(statements[1], "SELECT 3;");
|
assert_eq!(statements[1], "SELECT 3;");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ordinary_multifile_import_does_not_share_splitter_state() {
|
||||||
|
let mut first_file =
|
||||||
|
StreamingSqlFileSplitter::new(Some(DatabaseType::SqlServer), SqlParsingOptions::default(), false);
|
||||||
|
assert!(first_file.push_chunk("INSERT INTO dbo.items VALUES (N'partial").is_empty());
|
||||||
|
assert_eq!(
|
||||||
|
first_file.finish().into_iter().map(|statement| statement.sql).collect::<Vec<_>>(),
|
||||||
|
vec!["INSERT INTO dbo.items VALUES (N'partial"]
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut second_file =
|
||||||
|
StreamingSqlFileSplitter::new(Some(DatabaseType::SqlServer), SqlParsingOptions::default(), false);
|
||||||
|
assert!(second_file.push_chunk("SELECT 2;\n").is_empty());
|
||||||
|
assert_eq!(
|
||||||
|
second_file.finish().into_iter().map(|statement| statement.sql).collect::<Vec<_>>(),
|
||||||
|
vec!["SELECT 2;"]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn sqlserver_zip_insert_cap_does_not_change_postgres_default_merge() {
|
||||||
|
let statements =
|
||||||
|
vec!["INSERT INTO items (id) VALUES (1);".to_string(), "INSERT INTO items (id) VALUES (2);".to_string()];
|
||||||
|
|
||||||
|
assert_eq!(sql_file_import_max_insert_batch_statements(true, Some(DatabaseType::Postgres)), None);
|
||||||
|
assert_eq!(sql_file_import_max_insert_batch_statements(false, Some(DatabaseType::SqlServer)), None);
|
||||||
|
assert_eq!(
|
||||||
|
sql_file_import_max_insert_batch_statements(true, Some(DatabaseType::SqlServer)),
|
||||||
|
Some(SQLSERVER_ZIP_INSERT_BATCH_MAX_STATEMENTS)
|
||||||
|
);
|
||||||
|
|
||||||
|
let postgres =
|
||||||
|
optimize_sql_file_import_statements_for_batch(&statements, Some(DatabaseType::Postgres), None, None);
|
||||||
|
let sqlserver_zip = optimize_sql_file_import_statements_for_batch(
|
||||||
|
&statements,
|
||||||
|
Some(DatabaseType::SqlServer),
|
||||||
|
None,
|
||||||
|
sql_file_import_max_insert_batch_statements(true, Some(DatabaseType::SqlServer)),
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(postgres.len(), 1);
|
||||||
|
assert_eq!(postgres[0].source_statement_count, 2);
|
||||||
|
assert_eq!(sqlserver_zip.len(), 1);
|
||||||
|
assert_eq!(sqlserver_zip[0].source_statement_count, 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn sqlserver_zip_insert_cap_still_bounds_very_large_merges() {
|
||||||
|
let statements: Vec<String> = (0..(SQLSERVER_ZIP_INSERT_BATCH_MAX_STATEMENTS * 2))
|
||||||
|
.map(|id| format!("INSERT INTO items (id) VALUES ({id});"))
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
let sqlserver_zip = optimize_sql_file_import_statements_for_batch(
|
||||||
|
&statements,
|
||||||
|
Some(DatabaseType::SqlServer),
|
||||||
|
None,
|
||||||
|
sql_file_import_max_insert_batch_statements(true, Some(DatabaseType::SqlServer)),
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(sqlserver_zip.len(), 2);
|
||||||
|
assert!(sqlserver_zip
|
||||||
|
.iter()
|
||||||
|
.all(|statement| statement.source_statement_count == SQLSERVER_ZIP_INSERT_BATCH_MAX_STATEMENTS));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn streaming_sqlserver_splitter_emits_semicolon_terminated_inserts_without_go() {
|
||||||
|
let mut splitter =
|
||||||
|
StreamingSqlFileSplitter::new(Some(DatabaseType::SqlServer), SqlParsingOptions::default(), true);
|
||||||
|
let statements = splitter
|
||||||
|
.push_chunk("INSERT INTO dbo.items VALUES (1);\nINSERT INTO dbo.items VALUES (2);\n")
|
||||||
|
.into_iter()
|
||||||
|
.map(|statement| statement.sql)
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
assert_eq!(statements, vec!["INSERT INTO dbo.items VALUES (1);", "INSERT INTO dbo.items VALUES (2);"]);
|
||||||
|
assert!(splitter.finish().is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn streaming_sqlserver_splitter_ignores_semicolons_inside_literals_and_comments() {
|
||||||
|
let mut splitter =
|
||||||
|
StreamingSqlFileSplitter::new(Some(DatabaseType::SqlServer), SqlParsingOptions::default(), true);
|
||||||
|
let statements = splitter
|
||||||
|
.push_chunk("INSERT INTO dbo.items VALUES (N'one; two', [a;]); -- trailing ;\n/* block ; comment */ INSERT INTO dbo.items VALUES (2);\n")
|
||||||
|
.into_iter()
|
||||||
|
.map(|statement| statement.sql)
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
statements,
|
||||||
|
vec![
|
||||||
|
"INSERT INTO dbo.items VALUES (N'one; two', [a;]);",
|
||||||
|
"-- trailing ;\n/* block ; comment */ INSERT INTO dbo.items VALUES (2);"
|
||||||
|
]
|
||||||
|
);
|
||||||
|
assert!(splitter.finish().is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn streaming_sqlserver_splitter_keeps_split_json_literal_across_parts() {
|
fn streaming_sqlserver_splitter_keeps_split_json_literal_across_parts() {
|
||||||
let part_one = "INSERT INTO [dbo].[VersionValue] ([value]) VALUES (N'{\"materialOrSymbolMate";
|
let part_one = "INSERT INTO [dbo].[VersionValue] ([value]) VALUES (N'{\"materialOrSymbolMate";
|
||||||
let part_two = "rialId\":\"9625a891-3682-4151-acdb-6480db860033\",\"label\":\"B\\\\u1ebb ch\\\\u00e2n/Th\\\\u1eb3ng/\",\"quote\":\"it''s valid\"}');\n";
|
let part_two = "rialId\":\"9625a891-3682-4151-acdb-6480db860033\",\"label\":\"B\\\\u1ebb ch\\\\u00e2n/Th\\\\u1eb3ng/\",\"quote\":\"it''s valid\"}');\n";
|
||||||
let expected = format!("{part_one}{part_two}").trim().to_string();
|
let expected = format!("{part_one}{part_two}").trim().to_string();
|
||||||
let mut splitter = StreamingSqlFileSplitter::new(Some(DatabaseType::SqlServer), SqlParsingOptions::default());
|
let mut splitter =
|
||||||
|
StreamingSqlFileSplitter::new(Some(DatabaseType::SqlServer), SqlParsingOptions::default(), true);
|
||||||
|
|
||||||
assert!(splitter.push_chunk(part_one).is_empty());
|
assert!(splitter.push_chunk(part_one).is_empty());
|
||||||
assert!(splitter.push_chunk(part_two).is_empty());
|
let statements = splitter.push_chunk(part_two).into_iter().map(|statement| statement.sql).collect::<Vec<_>>();
|
||||||
let statements = splitter.finish().into_iter().map(|statement| statement.sql).collect::<Vec<_>>();
|
|
||||||
|
|
||||||
assert_eq!(statements, vec![expected]);
|
assert_eq!(statements, vec![expected]);
|
||||||
|
assert!(splitter.finish().is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn streaming_sqlserver_splitter_keeps_leading_comment_module_semicolons_until_go() {
|
||||||
|
let mut splitter =
|
||||||
|
StreamingSqlFileSplitter::new(Some(DatabaseType::SqlServer), SqlParsingOptions::default(), true);
|
||||||
|
|
||||||
|
assert!(splitter
|
||||||
|
.push_chunk("/* setup */\nCREATE OR ALTER PROCEDURE dbo.demo AS\nBEGIN\n SELECT 1;\n SELECT 2;\nEND\n")
|
||||||
|
.is_empty());
|
||||||
|
let statements = splitter.push_chunk("GO\n").into_iter().map(|statement| statement.sql).collect::<Vec<_>>();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
statements,
|
||||||
|
vec!["/* setup */\nCREATE OR ALTER PROCEDURE dbo.demo AS\nBEGIN\n SELECT 1;\n SELECT 2;\nEND"]
|
||||||
|
);
|
||||||
|
assert!(splitter.finish().is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -3219,6 +3496,59 @@ mod tests {
|
|||||||
assert_eq!(batches, vec!["CREATE PROCEDURE dbo.demo AS\nBEGIN\nSELECT 1;\nEND", "SELECT 2;"]);
|
assert_eq!(batches, vec!["CREATE PROCEDURE dbo.demo AS\nBEGIN\nSELECT 1;\nEND", "SELECT 2;"]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ordinary_sqlserver_import_keeps_declared_variable_batch_scope() {
|
||||||
|
let mut splitter =
|
||||||
|
StreamingSqlFileSplitter::new(Some(DatabaseType::SqlServer), SqlParsingOptions::default(), false);
|
||||||
|
|
||||||
|
let statements = splitter
|
||||||
|
.push_chunk("DECLARE @name nvarchar(50);\nSET @name = 'x';\nSELECT * FROM users WHERE name = @name;\nGO\n")
|
||||||
|
.into_iter()
|
||||||
|
.map(|statement| statement.sql)
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
statements,
|
||||||
|
vec!["DECLARE @name nvarchar(50);\nSET @name = 'x';\nSELECT * FROM users WHERE name = @name;"]
|
||||||
|
);
|
||||||
|
assert!(splitter.finish().is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ordinary_sqlserver_import_keeps_control_flow_batch_together() {
|
||||||
|
let mut splitter =
|
||||||
|
StreamingSqlFileSplitter::new(Some(DatabaseType::SqlServer), SqlParsingOptions::default(), false);
|
||||||
|
|
||||||
|
let statements = splitter
|
||||||
|
.push_chunk(
|
||||||
|
"IF EXISTS (SELECT 1 FROM sys.tables WHERE name = 'demo')\nBEGIN\n SELECT 1;\n SELECT 2;\nEND\nELSE\nBEGIN\n SELECT 3;\nEND\nGO\n",
|
||||||
|
)
|
||||||
|
.into_iter()
|
||||||
|
.map(|statement| statement.sql)
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
statements,
|
||||||
|
vec!["IF EXISTS (SELECT 1 FROM sys.tables WHERE name = 'demo')\nBEGIN\n SELECT 1;\n SELECT 2;\nEND\nELSE\nBEGIN\n SELECT 3;\nEND"]
|
||||||
|
);
|
||||||
|
assert!(splitter.finish().is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn zip_sqlserver_import_still_keeps_module_body_whole_with_comment_before_keyword() {
|
||||||
|
let mut splitter =
|
||||||
|
StreamingSqlFileSplitter::new(Some(DatabaseType::SqlServer), SqlParsingOptions::default(), true);
|
||||||
|
|
||||||
|
let statements = splitter
|
||||||
|
.push_chunk("CREATE /*c*/ PROCEDURE dbo.demo AS\nBEGIN\n SELECT 1;\n SELECT 2;\nEND\nGO\n")
|
||||||
|
.into_iter()
|
||||||
|
.map(|statement| statement.sql)
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
assert_eq!(statements, vec!["CREATE /*c*/ PROCEDURE dbo.demo AS\nBEGIN\n SELECT 1;\n SELECT 2;\nEND"]);
|
||||||
|
assert!(splitter.finish().is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn streaming_decoder_detects_gbk_after_ascii_prefix() {
|
async fn streaming_decoder_detects_gbk_after_ascii_prefix() {
|
||||||
let (encoded, _, _) = encoding_rs::GBK.encode("-- Navicat dump\nINSERT INTO t VALUES ('中文');");
|
let (encoded, _, _) = encoding_rs::GBK.encode("-- Navicat dump\nINSERT INTO t VALUES ('中文');");
|
||||||
|
|||||||
@@ -2209,7 +2209,7 @@ impl SqlImportRowStream {
|
|||||||
.await?;
|
.await?;
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
decoder,
|
decoder,
|
||||||
splitter: Some(StreamingSqlFileSplitter::new(options.sql_dialect, parsing_options)),
|
splitter: Some(StreamingSqlFileSplitter::new(options.sql_dialect, parsing_options, false)),
|
||||||
family,
|
family,
|
||||||
target: None,
|
target: None,
|
||||||
rows: Vec::new(),
|
rows: Vec::new(),
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ use dbx_core::connection::AppState;
|
|||||||
use dbx_core::database_export::{export_database_sql_core, DatabaseExportOutputCompression, DatabaseExportRequest};
|
use dbx_core::database_export::{export_database_sql_core, DatabaseExportOutputCompression, DatabaseExportRequest};
|
||||||
use dbx_core::models::connection::DatabaseType;
|
use dbx_core::models::connection::DatabaseType;
|
||||||
use dbx_core::sql::SqlFileRequest;
|
use dbx_core::sql::SqlFileRequest;
|
||||||
use dbx_core::sql_file_import::execute_sql_file_paths;
|
use dbx_core::sql_file_import::execute_sql_file_zip_package_paths;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::Instant;
|
use std::time::Instant;
|
||||||
use tokio_util::sync::CancellationToken;
|
use tokio_util::sync::CancellationToken;
|
||||||
@@ -225,7 +225,12 @@ async fn manual_e2e_sqlserver_default_split_zip_export_then_import_skip_relation
|
|||||||
let file_path_refs: Vec<&std::path::Path> = extracted_paths.iter().map(|path| path.as_path()).collect();
|
let file_path_refs: Vec<&std::path::Path> = extracted_paths.iter().map(|path| path.as_path()).collect();
|
||||||
let import_progress_log = std::sync::Mutex::new(Vec::new());
|
let import_progress_log = std::sync::Mutex::new(Vec::new());
|
||||||
let started_at = Instant::now();
|
let started_at = Instant::now();
|
||||||
let import_result = execute_sql_file_paths(
|
// Use the dedicated ZIP-package entry point, exactly as the desktop/web
|
||||||
|
// dispatch sites now do once `is_zip_package` is correctly derived from
|
||||||
|
// these extracted part paths (see PR #10632 review: routing previously
|
||||||
|
// fell through to `execute_sql_file_paths`, silently losing the
|
||||||
|
// persistent splitter across parts this whole flow is meant to exercise).
|
||||||
|
let import_result = execute_sql_file_zip_package_paths(
|
||||||
&reimport_state,
|
&reimport_state,
|
||||||
&import_request,
|
&import_request,
|
||||||
&file_path_refs,
|
&file_path_refs,
|
||||||
|
|||||||
@@ -8,6 +8,35 @@ pub const SQL_FILE_ZIP_MAX_MANIFEST_BYTES: u64 = 1024 * 1024;
|
|||||||
pub const SQL_FILE_ZIP_MAX_TOTAL_BYTES: u64 = 4 * 1024 * 1024 * 1024;
|
pub const SQL_FILE_ZIP_MAX_TOTAL_BYTES: u64 = 4 * 1024 * 1024 * 1024;
|
||||||
pub const SQL_FILE_ZIP_MAX_PART_BYTES: u64 = 1024 * 1024 * 1024;
|
pub const SQL_FILE_ZIP_MAX_PART_BYTES: u64 = 1024 * 1024 * 1024;
|
||||||
|
|
||||||
|
/// Extraction-directory prefix used by the desktop (Tauri) dispatch site,
|
||||||
|
/// under the OS temp dir: `dbx-sql-package-<uuid>`.
|
||||||
|
pub const SQL_FILE_ZIP_EXTRACTION_DIR_PREFIX_DESKTOP: &str = "dbx-sql-package-";
|
||||||
|
|
||||||
|
/// Extraction-directory prefix used by the web dispatch site, under the
|
||||||
|
/// managed upload tmp dir: `package-<uuid>`.
|
||||||
|
pub const SQL_FILE_ZIP_EXTRACTION_DIR_PREFIX_WEB: &str = "package-";
|
||||||
|
|
||||||
|
/// True if `path`'s parent directory name starts with `prefix`, i.e. `path` is
|
||||||
|
/// one of the parts `extract_sql_file_zip_package` wrote out for a `.zip`
|
||||||
|
/// upload extracted under that prefix.
|
||||||
|
///
|
||||||
|
/// Dispatch sites must derive "is this execution a ZIP package" from the
|
||||||
|
/// actual paths they are about to execute (via this helper), not from the
|
||||||
|
/// original upload's file name/extension: the frontend always sends the
|
||||||
|
/// extracted `.sql` part paths as the execution paths, never the original
|
||||||
|
/// `.zip` path, so a suffix check on the original request field never
|
||||||
|
/// matches. Each site must pass only the prefix it extracts under itself:
|
||||||
|
/// the prefixes are per transport, and `package-` is generic enough that
|
||||||
|
/// user-chosen directories can share it, so such a path must never be
|
||||||
|
/// mistaken for an extraction dir and routed through the ZIP importer or its
|
||||||
|
/// `remove_dir_all` cleanup.
|
||||||
|
pub fn is_extracted_sql_zip_package_path(path: &Path, prefix: &str) -> bool {
|
||||||
|
path.parent()
|
||||||
|
.and_then(|parent| parent.file_name())
|
||||||
|
.and_then(|name| name.to_str())
|
||||||
|
.is_some_and(|name| name.starts_with(prefix))
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Deserialize)]
|
#[derive(Debug, Clone, Deserialize)]
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
struct SqlZipManifest {
|
struct SqlZipManifest {
|
||||||
@@ -135,6 +164,32 @@ mod tests {
|
|||||||
zip.finish().unwrap();
|
zip.finish().unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn is_extracted_sql_zip_package_path_matches_only_the_requested_prefix() {
|
||||||
|
let desktop_part = Path::new("/tmp/dbx-sql-package-11111111-1111-1111-1111-111111111111/00001-dump.sql");
|
||||||
|
let web_part = Path::new("/var/data/tmp/sql_file/package-22222222-2222-2222-2222-222222222222/00001-dump.sql");
|
||||||
|
assert!(is_extracted_sql_zip_package_path(desktop_part, SQL_FILE_ZIP_EXTRACTION_DIR_PREFIX_DESKTOP));
|
||||||
|
assert!(!is_extracted_sql_zip_package_path(desktop_part, SQL_FILE_ZIP_EXTRACTION_DIR_PREFIX_WEB));
|
||||||
|
assert!(is_extracted_sql_zip_package_path(web_part, SQL_FILE_ZIP_EXTRACTION_DIR_PREFIX_WEB));
|
||||||
|
assert!(!is_extracted_sql_zip_package_path(web_part, SQL_FILE_ZIP_EXTRACTION_DIR_PREFIX_DESKTOP));
|
||||||
|
assert!(!is_extracted_sql_zip_package_path(
|
||||||
|
Path::new("/tmp/upload-33333333.sql"),
|
||||||
|
SQL_FILE_ZIP_EXTRACTION_DIR_PREFIX_DESKTOP
|
||||||
|
));
|
||||||
|
assert!(!is_extracted_sql_zip_package_path(
|
||||||
|
Path::new("/tmp/upload-33333333.sql"),
|
||||||
|
SQL_FILE_ZIP_EXTRACTION_DIR_PREFIX_WEB
|
||||||
|
));
|
||||||
|
assert!(!is_extracted_sql_zip_package_path(
|
||||||
|
Path::new("/var/data/tmp/sql_file/restore-44444444/backup.sql"),
|
||||||
|
SQL_FILE_ZIP_EXTRACTION_DIR_PREFIX_WEB
|
||||||
|
));
|
||||||
|
assert!(!is_extracted_sql_zip_package_path(
|
||||||
|
Path::new("standalone.sql"),
|
||||||
|
SQL_FILE_ZIP_EXTRACTION_DIR_PREFIX_DESKTOP
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn preserves_manifest_order_when_extracting_parts() {
|
fn preserves_manifest_order_when_extracting_parts() {
|
||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
|||||||
@@ -1364,7 +1364,7 @@ fn starts_with_sqlserver_control_flow_batch(sql: &str) -> bool {
|
|||||||
&& tokens.iter().any(|token| token.eq_ignore_ascii_case("END"))
|
&& tokens.iter().any(|token| token.eq_ignore_ascii_case("END"))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn starts_with_sqlserver_module_ddl(sql: &str) -> bool {
|
pub fn starts_with_sqlserver_module_ddl(sql: &str) -> bool {
|
||||||
let tokens = first_sql_tokens(sql, 4);
|
let tokens = first_sql_tokens(sql, 4);
|
||||||
if tokens.len() >= 4
|
if tokens.len() >= 4
|
||||||
&& tokens[0].eq_ignore_ascii_case("CREATE")
|
&& tokens[0].eq_ignore_ascii_case("CREATE")
|
||||||
@@ -1617,15 +1617,46 @@ fn mysql_routine_tokens(sql: &str) -> Vec<String> {
|
|||||||
|
|
||||||
fn first_sql_tokens(sql: &str, limit: usize) -> Vec<String> {
|
fn first_sql_tokens(sql: &str, limit: usize) -> Vec<String> {
|
||||||
let mut tokens = Vec::new();
|
let mut tokens = Vec::new();
|
||||||
for token in sql.split(|ch: char| !ch.is_ascii_alphanumeric() && ch != '_') {
|
let mut token = String::new();
|
||||||
if token.is_empty() {
|
let mut chars = sql.chars().peekable();
|
||||||
continue;
|
|
||||||
|
while let Some(ch) = chars.next() {
|
||||||
|
if ch == '-' && chars.peek() == Some(&'-') {
|
||||||
|
chars.next();
|
||||||
|
if !token.is_empty() {
|
||||||
|
tokens.push(std::mem::take(&mut token));
|
||||||
|
}
|
||||||
|
for comment_char in chars.by_ref() {
|
||||||
|
if comment_char == '\n' {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if ch == '/' && chars.peek() == Some(&'*') {
|
||||||
|
chars.next();
|
||||||
|
if !token.is_empty() {
|
||||||
|
tokens.push(std::mem::take(&mut token));
|
||||||
|
}
|
||||||
|
let mut previous = None;
|
||||||
|
for comment_char in chars.by_ref() {
|
||||||
|
if previous == Some('*') && comment_char == '/' {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
previous = Some(comment_char);
|
||||||
|
}
|
||||||
|
} else if ch.is_ascii_alphanumeric() || ch == '_' {
|
||||||
|
token.push(ch);
|
||||||
|
} else if !token.is_empty() {
|
||||||
|
tokens.push(std::mem::take(&mut token));
|
||||||
}
|
}
|
||||||
tokens.push(token.to_string());
|
|
||||||
if tokens.len() >= limit {
|
if tokens.len() >= limit {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if tokens.len() < limit && !token.is_empty() {
|
||||||
|
tokens.push(token);
|
||||||
|
}
|
||||||
tokens
|
tokens
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1799,6 +1830,20 @@ pub fn optimize_sql_file_import_statements(
|
|||||||
statements: &[String],
|
statements: &[String],
|
||||||
db_type: Option<DatabaseType>,
|
db_type: Option<DatabaseType>,
|
||||||
driver_profile: Option<&str>,
|
driver_profile: Option<&str>,
|
||||||
|
) -> Vec<SqlFileImportStatement> {
|
||||||
|
optimize_sql_file_import_statements_with_max_insert_batch_statements(
|
||||||
|
statements,
|
||||||
|
db_type,
|
||||||
|
driver_profile,
|
||||||
|
SQL_FILE_INSERT_BATCH_MAX_STATEMENTS,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn optimize_sql_file_import_statements_with_max_insert_batch_statements(
|
||||||
|
statements: &[String],
|
||||||
|
db_type: Option<DatabaseType>,
|
||||||
|
driver_profile: Option<&str>,
|
||||||
|
max_insert_batch_statements: usize,
|
||||||
) -> Vec<SqlFileImportStatement> {
|
) -> Vec<SqlFileImportStatement> {
|
||||||
let mut optimized = Vec::new();
|
let mut optimized = Vec::new();
|
||||||
let mut pending_insert: Option<PendingInsertBatch> = None;
|
let mut pending_insert: Option<PendingInsertBatch> = None;
|
||||||
@@ -1833,7 +1878,7 @@ pub fn optimize_sql_file_import_statements(
|
|||||||
.flatten();
|
.flatten();
|
||||||
if let Some(insert) = mergeable_insert {
|
if let Some(insert) = mergeable_insert {
|
||||||
match pending_insert.as_mut() {
|
match pending_insert.as_mut() {
|
||||||
Some(batch) if batch.can_accept(&insert) => batch.push(insert),
|
Some(batch) if batch.can_accept(&insert, max_insert_batch_statements) => batch.push(insert),
|
||||||
Some(_) => {
|
Some(_) => {
|
||||||
flush_pending_insert(&mut optimized, &mut pending_insert);
|
flush_pending_insert(&mut optimized, &mut pending_insert);
|
||||||
pending_insert = Some(PendingInsertBatch::new(insert));
|
pending_insert = Some(PendingInsertBatch::new(insert));
|
||||||
@@ -1904,9 +1949,9 @@ impl PendingInsertBatch {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn can_accept(&self, insert: &MergeableInsert) -> bool {
|
fn can_accept(&self, insert: &MergeableInsert, max_insert_batch_statements: usize) -> bool {
|
||||||
self.prefix_key == insert.prefix_key
|
self.prefix_key == insert.prefix_key
|
||||||
&& self.source_statement_count < SQL_FILE_INSERT_BATCH_MAX_STATEMENTS
|
&& self.source_statement_count < max_insert_batch_statements
|
||||||
&& self.byte_len + insert.values.len() + 3 <= SQL_FILE_INSERT_BATCH_MAX_BYTES
|
&& self.byte_len + insert.values.len() + 3 <= SQL_FILE_INSERT_BATCH_MAX_BYTES
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -8,8 +8,8 @@ use axum::response::sse::{Event, Sse};
|
|||||||
use axum::Json;
|
use axum::Json;
|
||||||
use dbx_core::sql::{self, SqlFileProgress, SqlFileRequest, SqlFileStatus};
|
use dbx_core::sql::{self, SqlFileProgress, SqlFileRequest, SqlFileStatus};
|
||||||
use dbx_core::sql_file_import::{
|
use dbx_core::sql_file_import::{
|
||||||
execute_sql_file_paths, sql_file_error_progress, sql_file_progress as build_sql_file_progress,
|
execute_sql_file_paths, execute_sql_file_zip_package_paths, sql_file_error_progress,
|
||||||
SqlFileProgressEmitter,
|
sql_file_progress as build_sql_file_progress, SqlFileProgressEmitter,
|
||||||
};
|
};
|
||||||
use futures::stream::Stream;
|
use futures::stream::Stream;
|
||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
@@ -223,7 +223,11 @@ async fn preview_uploaded_sql_file(
|
|||||||
let tmp_dir = file_path.parent().ok_or_else(|| AppError::from("Invalid SQL file path".to_string()))?;
|
let tmp_dir = file_path.parent().ok_or_else(|| AppError::from("Invalid SQL file path".to_string()))?;
|
||||||
|
|
||||||
if file_name.to_ascii_lowercase().ends_with(".zip") {
|
if file_name.to_ascii_lowercase().ends_with(".zip") {
|
||||||
let extraction_dir = tmp_dir.join(format!("package-{}", Uuid::new_v4()));
|
let extraction_dir = tmp_dir.join(format!(
|
||||||
|
"{}{}",
|
||||||
|
dbx_core::sql_file_zip_package::SQL_FILE_ZIP_EXTRACTION_DIR_PREFIX_WEB,
|
||||||
|
Uuid::new_v4()
|
||||||
|
));
|
||||||
let package = dbx_core::sql_file_zip_package::extract_sql_file_zip_package(file_path, &extraction_dir)
|
let package = dbx_core::sql_file_zip_package::extract_sql_file_zip_package(file_path, &extraction_dir)
|
||||||
.map_err(AppError::from)?;
|
.map_err(AppError::from)?;
|
||||||
let paths = dbx_core::sql_file_zip_package::extracted_sql_zip_paths(&extraction_dir, &package)
|
let paths = dbx_core::sql_file_zip_package::extracted_sql_zip_paths(&extraction_dir, &package)
|
||||||
@@ -272,6 +276,7 @@ pub async fn execute_sql_file(
|
|||||||
.iter()
|
.iter()
|
||||||
.map(|file_path| validated_uploaded_sql_path(&state.data_dir, file_path))
|
.map(|file_path| validated_uploaded_sql_path(&state.data_dir, file_path))
|
||||||
.collect::<Result<Vec<_>, _>>()?;
|
.collect::<Result<Vec<_>, _>>()?;
|
||||||
|
let is_zip_package = execution_targets_zip_package(&file_paths);
|
||||||
let managed_previews = state.managed_sql_previews.claim(&file_paths)?;
|
let managed_previews = state.managed_sql_previews.claim(&file_paths)?;
|
||||||
|
|
||||||
// Fast-fail: reject early if the connection is read-only (individual statements are also checked in do_execute)
|
// Fast-fail: reject early if the connection is read-only (individual statements are also checked in do_execute)
|
||||||
@@ -344,10 +349,17 @@ pub async fn execute_sql_file(
|
|||||||
let file_path_refs: Vec<&Path> = file_paths.iter().map(PathBuf::as_path).collect();
|
let file_path_refs: Vec<&Path> = file_paths.iter().map(PathBuf::as_path).collect();
|
||||||
// The core executor emits exactly one terminal Error with the latest
|
// The core executor emits exactly one terminal Error with the latest
|
||||||
// cumulative counters before returning Err.
|
// cumulative counters before returning Err.
|
||||||
let _ = execute_sql_file_paths(&app, &req, &file_path_refs, token, started_at, |progress| {
|
let _ = if is_zip_package {
|
||||||
progress_emitter.emit(progress);
|
execute_sql_file_zip_package_paths(&app, &req, &file_path_refs, token, started_at, |progress| {
|
||||||
})
|
progress_emitter.emit(progress);
|
||||||
.await;
|
})
|
||||||
|
.await
|
||||||
|
} else {
|
||||||
|
execute_sql_file_paths(&app, &req, &file_path_refs, token, started_at, |progress| {
|
||||||
|
progress_emitter.emit(progress);
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
};
|
||||||
|
|
||||||
cleanup_sql_file_package_paths(&file_paths);
|
cleanup_sql_file_package_paths(&file_paths);
|
||||||
cleanup_sql_file_execution(&state_clone, &req.execution_id).await;
|
cleanup_sql_file_execution(&state_clone, &req.execution_id).await;
|
||||||
@@ -365,11 +377,11 @@ fn send_sql_file_progress(tx: &broadcast::Sender<String>, progress: SqlFileProgr
|
|||||||
fn cleanup_sql_file_package_paths(file_paths: &[PathBuf]) {
|
fn cleanup_sql_file_package_paths(file_paths: &[PathBuf]) {
|
||||||
let mut directories = std::collections::HashSet::new();
|
let mut directories = std::collections::HashSet::new();
|
||||||
for path in file_paths {
|
for path in file_paths {
|
||||||
let Some(parent) = path.parent() else {
|
if dbx_core::sql_file_zip_package::is_extracted_sql_zip_package_path(
|
||||||
continue;
|
path,
|
||||||
};
|
dbx_core::sql_file_zip_package::SQL_FILE_ZIP_EXTRACTION_DIR_PREFIX_WEB,
|
||||||
if parent.file_name().and_then(|name| name.to_str()).is_some_and(|name| name.starts_with("package-")) {
|
) {
|
||||||
directories.insert(parent.to_path_buf());
|
directories.insert(path.parent().expect("checked by is_extracted_sql_zip_package_path").to_path_buf());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for directory in directories {
|
for directory in directories {
|
||||||
@@ -436,6 +448,23 @@ fn cleanup_sql_file_uploads_except(tmp_dir: &Path, max_age: Duration, active: &H
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The frontend always sends the extracted `.sql` part paths as the execution
|
||||||
|
/// paths for a ZIP upload, never the original `.zip` path (that stays only in
|
||||||
|
/// `SqlFileRequest::file_path`), so routing to the ZIP-package incremental
|
||||||
|
/// -flush importer must be derived from those actual execution paths, not
|
||||||
|
/// from `req.file_path`'s extension. Web only ever extracts under the
|
||||||
|
/// `package-` prefix, and every path reaching here has already passed
|
||||||
|
/// `validated_uploaded_sql_path`, which confines it to the managed upload
|
||||||
|
/// tmp dir.
|
||||||
|
fn execution_targets_zip_package(file_paths: &[PathBuf]) -> bool {
|
||||||
|
file_paths.iter().any(|path| {
|
||||||
|
dbx_core::sql_file_zip_package::is_extracted_sql_zip_package_path(
|
||||||
|
path,
|
||||||
|
dbx_core::sql_file_zip_package::SQL_FILE_ZIP_EXTRACTION_DIR_PREFIX_WEB,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn validated_uploaded_sql_path(data_dir: &Path, file_path: &str) -> Result<PathBuf, AppError> {
|
fn validated_uploaded_sql_path(data_dir: &Path, file_path: &str) -> Result<PathBuf, AppError> {
|
||||||
let path = PathBuf::from(file_path);
|
let path = PathBuf::from(file_path);
|
||||||
if !path.is_absolute() {
|
if !path.is_absolute() {
|
||||||
@@ -633,6 +662,35 @@ mod tests {
|
|||||||
assert!(state.sql_file_executions.read().await.contains_key("existing-job"));
|
assert!(state.sql_file_executions.read().await.contains_key("existing-job"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn zip_package_routing_is_selected_from_extracted_part_paths_not_the_original_zip_name() {
|
||||||
|
// Regression for PR #10632 review: the frontend sends the extracted
|
||||||
|
// `.sql` part paths as execution paths, never the original `.zip`
|
||||||
|
// upload path, so routing must key off the actual paths, not
|
||||||
|
// `req.file_path`'s extension.
|
||||||
|
let zip_upload_part_paths = vec![
|
||||||
|
PathBuf::from("/data/tmp/sql_file/package-11111111-1111-1111-1111-111111111111/00001-dump.sql"),
|
||||||
|
PathBuf::from("/data/tmp/sql_file/package-11111111-1111-1111-1111-111111111111/00002-dump.sql"),
|
||||||
|
];
|
||||||
|
assert!(execution_targets_zip_package(&zip_upload_part_paths));
|
||||||
|
|
||||||
|
let ordinary_upload_paths = vec![PathBuf::from("/data/tmp/sql_file/upload-22222222.sql")];
|
||||||
|
assert!(!execution_targets_zip_package(&ordinary_upload_paths));
|
||||||
|
|
||||||
|
// A single-file (non-multi-file) request always executes exactly the
|
||||||
|
// `req.file_path` value as its only path, so this also exercises the
|
||||||
|
// no-`file_paths`-supplied call shape from `execute_sql_file`.
|
||||||
|
let single_zip_part_as_sole_path = vec![PathBuf::from("/data/tmp/sql_file/package-33333333/00001-dump.sql")];
|
||||||
|
assert!(execution_targets_zip_package(&single_zip_part_as_sole_path));
|
||||||
|
|
||||||
|
// Desktop extraction dirs (`dbx-sql-package-`, under the OS temp dir)
|
||||||
|
// are not this site's to match: web only ever extracts `package-*`
|
||||||
|
// dirs, and the generic name must not pull user directories into the
|
||||||
|
// ZIP importer or its cleanup.
|
||||||
|
let desktop_style_paths = vec![PathBuf::from("/tmp/dbx-sql-package-44444444/00001-dump.sql")];
|
||||||
|
assert!(!execution_targets_zip_package(&desktop_style_paths));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn sql_file_upload_limit_defaults_and_clamps() {
|
fn sql_file_upload_limit_defaults_and_clamps() {
|
||||||
assert_eq!(sql_file_upload_max_bytes_from_value(None), SQL_FILE_UPLOAD_MAX_BYTES);
|
assert_eq!(sql_file_upload_max_bytes_from_value(None), SQL_FILE_UPLOAD_MAX_BYTES);
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::path::PathBuf;
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::{Arc, OnceLock};
|
use std::sync::{Arc, OnceLock};
|
||||||
use std::time::Instant;
|
use std::time::Instant;
|
||||||
|
|
||||||
@@ -9,8 +9,8 @@ use tokio_util::sync::CancellationToken;
|
|||||||
|
|
||||||
use crate::commands::connection::{ensure_connection_writable, AppState};
|
use crate::commands::connection::{ensure_connection_writable, AppState};
|
||||||
use dbx_core::sql_file_import::{
|
use dbx_core::sql_file_import::{
|
||||||
execute_sql_file_paths, mysql_like_sql_file_bootstrap_analysis, read_sql_file_preview, sql_file_progress,
|
execute_sql_file_paths, execute_sql_file_zip_package_paths, mysql_like_sql_file_bootstrap_analysis,
|
||||||
SqlFileProgressEmitter,
|
read_sql_file_preview, sql_file_progress, SqlFileProgressEmitter,
|
||||||
};
|
};
|
||||||
|
|
||||||
pub use dbx_core::sql::{SqlFilePreview, SqlFileRequest, SqlFileStatus};
|
pub use dbx_core::sql::{SqlFilePreview, SqlFileRequest, SqlFileStatus};
|
||||||
@@ -47,7 +47,11 @@ pub async fn preview_sql_file(file_path: String) -> Result<SqlFilePreview, Strin
|
|||||||
.is_some_and(|extension| extension.eq_ignore_ascii_case("zip"))
|
.is_some_and(|extension| extension.eq_ignore_ascii_case("zip"))
|
||||||
{
|
{
|
||||||
sweep_stale_sql_zip_packages();
|
sweep_stale_sql_zip_packages();
|
||||||
let extraction_dir = std::env::temp_dir().join(format!("dbx-sql-package-{}", uuid::Uuid::new_v4()));
|
let extraction_dir = std::env::temp_dir().join(format!(
|
||||||
|
"{}{}",
|
||||||
|
dbx_core::sql_file_zip_package::SQL_FILE_ZIP_EXTRACTION_DIR_PREFIX_DESKTOP,
|
||||||
|
uuid::Uuid::new_v4()
|
||||||
|
));
|
||||||
let (package, extracted_paths) = tokio::task::spawn_blocking({
|
let (package, extracted_paths) = tokio::task::spawn_blocking({
|
||||||
let path = path.clone();
|
let path = path.clone();
|
||||||
let extraction_dir = extraction_dir.clone();
|
let extraction_dir = extraction_dir.clone();
|
||||||
@@ -100,6 +104,32 @@ pub async fn execute_sql_file(
|
|||||||
execute_sql_files(app, state, request.clone(), vec![request.file_path.clone()]).await
|
execute_sql_files(app, state, request.clone(), vec![request.file_path.clone()]).await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The frontend always sends the extracted `.sql` part paths as `file_paths`
|
||||||
|
/// for a ZIP upload, never the original `.zip` path (that only ever lives in
|
||||||
|
/// `SqlFileRequest::file_path`), so routing to the ZIP-package incremental
|
||||||
|
/// -flush importer must be derived from those actual execution paths, not
|
||||||
|
/// from `request.file_path`'s extension.
|
||||||
|
fn execution_targets_zip_package(file_paths: &[String]) -> bool {
|
||||||
|
file_paths.iter().any(|path| is_desktop_sql_zip_package_part(Path::new(path)))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A desktop ZIP-package part lives in a `dbx-sql-package-<uuid>` directory
|
||||||
|
/// under the OS temp dir, exactly where `preview_sql_file` extracts uploads.
|
||||||
|
/// Requiring both the prefix and the temp-dir parent confines matching to
|
||||||
|
/// directories DBX created itself: desktop `file_paths` are arbitrary
|
||||||
|
/// user-chosen paths, so a user directory named `package-1.0/` (or even
|
||||||
|
/// `dbx-sql-package-...`) must never be mistaken for an extraction dir and
|
||||||
|
/// routed through the ZIP importer or its `remove_dir_all` cleanup.
|
||||||
|
fn is_desktop_sql_zip_package_part(path: &Path) -> bool {
|
||||||
|
path.parent().is_some_and(|parent| {
|
||||||
|
parent.starts_with(std::env::temp_dir())
|
||||||
|
&& dbx_core::sql_file_zip_package::is_extracted_sql_zip_package_path(
|
||||||
|
path,
|
||||||
|
dbx_core::sql_file_zip_package::SQL_FILE_ZIP_EXTRACTION_DIR_PREFIX_DESKTOP,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
#[tauri::command]
|
#[tauri::command]
|
||||||
pub async fn execute_sql_files(
|
pub async fn execute_sql_files(
|
||||||
app: AppHandle,
|
app: AppHandle,
|
||||||
@@ -119,7 +149,8 @@ pub async fn execute_sql_files(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let started_at = Instant::now();
|
let started_at = Instant::now();
|
||||||
let result = execute_sql_files_inner(&app, &state, &request, &file_paths, token, started_at).await;
|
let is_zip_package = execution_targets_zip_package(&file_paths);
|
||||||
|
let result = execute_sql_files_inner(&app, &state, &request, &file_paths, token, started_at, is_zip_package).await;
|
||||||
cleanup_sql_zip_package_paths(&file_paths);
|
cleanup_sql_zip_package_paths(&file_paths);
|
||||||
{
|
{
|
||||||
let mut executions = sql_file_executions().write().await;
|
let mut executions = sql_file_executions().write().await;
|
||||||
@@ -132,11 +163,8 @@ fn cleanup_sql_zip_package_paths(file_paths: &[String]) {
|
|||||||
let mut directories = std::collections::HashSet::new();
|
let mut directories = std::collections::HashSet::new();
|
||||||
for path in file_paths {
|
for path in file_paths {
|
||||||
let path = PathBuf::from(path);
|
let path = PathBuf::from(path);
|
||||||
let Some(parent) = path.parent() else {
|
if is_desktop_sql_zip_package_part(&path) {
|
||||||
continue;
|
directories.insert(path.parent().expect("checked by is_desktop_sql_zip_package_part").to_path_buf());
|
||||||
};
|
|
||||||
if parent.file_name().and_then(|name| name.to_str()).is_some_and(|name| name.starts_with("dbx-sql-package-")) {
|
|
||||||
directories.insert(parent.to_path_buf());
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for directory in directories {
|
for directory in directories {
|
||||||
@@ -152,7 +180,9 @@ fn sweep_stale_sql_zip_packages() {
|
|||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
for entry in entries.flatten() {
|
for entry in entries.flatten() {
|
||||||
if !entry.file_name().to_str().is_some_and(|name| name.starts_with("dbx-sql-package-")) {
|
if !entry.file_name().to_str().is_some_and(|name| {
|
||||||
|
name.starts_with(dbx_core::sql_file_zip_package::SQL_FILE_ZIP_EXTRACTION_DIR_PREFIX_DESKTOP)
|
||||||
|
}) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
let expired = entry
|
let expired = entry
|
||||||
@@ -185,6 +215,7 @@ async fn execute_sql_files_inner(
|
|||||||
file_paths: &[String],
|
file_paths: &[String],
|
||||||
token: CancellationToken,
|
token: CancellationToken,
|
||||||
started_at: Instant,
|
started_at: Instant,
|
||||||
|
is_zip_package: bool,
|
||||||
) -> Result<(), String> {
|
) -> Result<(), String> {
|
||||||
let mut progress_emitter = SqlFileProgressEmitter::new(|progress| {
|
let mut progress_emitter = SqlFileProgressEmitter::new(|progress| {
|
||||||
let _ = app.emit("sql-file-progress", progress);
|
let _ = app.emit("sql-file-progress", progress);
|
||||||
@@ -202,10 +233,17 @@ async fn execute_sql_files_inner(
|
|||||||
));
|
));
|
||||||
let paths: Vec<PathBuf> = file_paths.iter().map(PathBuf::from).collect();
|
let paths: Vec<PathBuf> = file_paths.iter().map(PathBuf::from).collect();
|
||||||
let path_refs: Vec<&std::path::Path> = paths.iter().map(PathBuf::as_path).collect();
|
let path_refs: Vec<&std::path::Path> = paths.iter().map(PathBuf::as_path).collect();
|
||||||
execute_sql_file_paths(state.inner().as_ref(), request, &path_refs, token, started_at, |progress| {
|
if is_zip_package {
|
||||||
progress_emitter.emit(progress);
|
execute_sql_file_zip_package_paths(state.inner().as_ref(), request, &path_refs, token, started_at, |progress| {
|
||||||
})
|
progress_emitter.emit(progress);
|
||||||
.await
|
})
|
||||||
|
.await
|
||||||
|
} else {
|
||||||
|
execute_sql_file_paths(state.inner().as_ref(), request, &path_refs, token, started_at, |progress| {
|
||||||
|
progress_emitter.emit(progress);
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn register_sql_file_execution(
|
fn register_sql_file_execution(
|
||||||
@@ -315,6 +353,68 @@ mod execution_tests {
|
|||||||
assert_eq!(summary.status, SqlFileStatus::Cancelled);
|
assert_eq!(summary.status, SqlFileStatus::Cancelled);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn zip_package_routing_is_selected_from_extracted_part_paths_not_the_original_zip_name() {
|
||||||
|
// Regression for PR #10632 review: `SqlFileExecutionDialog.vue` sends
|
||||||
|
// `executionPaths = packageFilePaths ?? [filePath]`, i.e. the extracted
|
||||||
|
// `.sql` part paths, as `file_paths` -- `request.file_path` on its own
|
||||||
|
// (still `.zip` in that case) must never gate this decision.
|
||||||
|
// Extraction dirs live under the OS temp dir, so the fixtures must too.
|
||||||
|
let temp = std::env::temp_dir();
|
||||||
|
let zip_upload_part_paths = vec![
|
||||||
|
temp.join("dbx-sql-package-11111111-1111-1111-1111-111111111111/00001-dump.sql")
|
||||||
|
.to_string_lossy()
|
||||||
|
.into_owned(),
|
||||||
|
temp.join("dbx-sql-package-11111111-1111-1111-1111-111111111111/00002-dump.sql")
|
||||||
|
.to_string_lossy()
|
||||||
|
.into_owned(),
|
||||||
|
];
|
||||||
|
assert!(execution_targets_zip_package(&zip_upload_part_paths));
|
||||||
|
|
||||||
|
let ordinary_upload_paths = vec![temp.join("upload-22222222.sql").to_string_lossy().into_owned()];
|
||||||
|
assert!(!execution_targets_zip_package(&ordinary_upload_paths));
|
||||||
|
|
||||||
|
// `execute_sql_file` forwards `vec![request.file_path.clone()]` as the
|
||||||
|
// sole `file_paths` entry, so a single extracted part must also match.
|
||||||
|
let single_zip_part_as_sole_path =
|
||||||
|
vec![temp.join("dbx-sql-package-33333333/00001-dump.sql").to_string_lossy().into_owned()];
|
||||||
|
assert!(execution_targets_zip_package(&single_zip_part_as_sole_path));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn zip_package_routing_ignores_user_directories_sharing_an_extraction_prefix() {
|
||||||
|
// Scoping regression for PR #10632 review: desktop `file_paths` are
|
||||||
|
// arbitrary user-chosen paths, so only `dbx-sql-package-*` directories
|
||||||
|
// under the OS temp dir count as extraction dirs.
|
||||||
|
let temp = std::env::temp_dir();
|
||||||
|
let web_style_user_dir = vec![temp.join("package-1.0/00001-dump.sql").to_string_lossy().into_owned()];
|
||||||
|
assert!(!execution_targets_zip_package(&web_style_user_dir));
|
||||||
|
|
||||||
|
let desktop_named_dir_outside_temp =
|
||||||
|
vec!["/definitely/not/the/temp/dbx-sql-package-44444444/00001-dump.sql".to_string()];
|
||||||
|
assert!(!execution_targets_zip_package(&desktop_named_dir_outside_temp));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn zip_package_cleanup_never_removes_user_directories_sharing_a_prefix() {
|
||||||
|
let temp = std::env::temp_dir();
|
||||||
|
let extraction = temp.join(format!("dbx-sql-package-{}", uuid::Uuid::new_v4()));
|
||||||
|
std::fs::create_dir_all(&extraction).unwrap();
|
||||||
|
std::fs::write(extraction.join("00001-dump.sql"), "SELECT 1;\n").unwrap();
|
||||||
|
let user_dir = temp.join(format!("package-{}", uuid::Uuid::new_v4()));
|
||||||
|
std::fs::create_dir_all(&user_dir).unwrap();
|
||||||
|
std::fs::write(user_dir.join("user.sql"), "SELECT 2;\n").unwrap();
|
||||||
|
|
||||||
|
cleanup_sql_zip_package_paths(&[
|
||||||
|
extraction.join("00001-dump.sql").to_string_lossy().into_owned(),
|
||||||
|
user_dir.join("user.sql").to_string_lossy().into_owned(),
|
||||||
|
]);
|
||||||
|
|
||||||
|
assert!(!extraction.exists());
|
||||||
|
assert!(user_dir.exists());
|
||||||
|
let _ = std::fs::remove_dir_all(&user_dir);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn duplicate_execution_id_is_rejected_without_replacing_token() {
|
fn duplicate_execution_id_is_rejected_without_replacing_token() {
|
||||||
let mut executions = HashMap::new();
|
let mut executions = HashMap::new();
|
||||||
|
|||||||
Reference in new issue
Block a user