fix(sql-file): stream sqlserver zip imports incrementally

This commit is contained in:
КГБ Lieu authored and GitHub committed 2026-10-01 02:40:02 +08:00
1 parent e0f802c031
commit 64e5847f74
7 files changed
+659 -66

No files matched your search

+53 -8
View File
@@ -1364,7 +1364,7 @@ fn starts_with_sqlserver_control_flow_batch(sql: &str) -> bool {
&& tokens.iter().any(|token| token.eq_ignore_ascii_case("END"))
}
fn starts_with_sqlserver_module_ddl(sql: &str) -> bool {
pub fn starts_with_sqlserver_module_ddl(sql: &str) -> bool {
let tokens = first_sql_tokens(sql, 4);
if tokens.len() >= 4
&& tokens[0].eq_ignore_ascii_case("CREATE")
@@ -1617,15 +1617,46 @@ fn mysql_routine_tokens(sql: &str) -> Vec<String> {
fn first_sql_tokens(sql: &str, limit: usize) -> Vec<String> {
let mut tokens = Vec::new();
for token in sql.split(|ch: char| !ch.is_ascii_alphanumeric() && ch != '_') {
if token.is_empty() {
continue;
let mut token = String::new();
let mut chars = sql.chars().peekable();
while let Some(ch) = chars.next() {
if ch == '-' && chars.peek() == Some(&'-') {
chars.next();
if !token.is_empty() {
tokens.push(std::mem::take(&mut token));
}
for comment_char in chars.by_ref() {
if comment_char == '\n' {
break;
}
}
} else if ch == '/' && chars.peek() == Some(&'*') {
chars.next();
if !token.is_empty() {
tokens.push(std::mem::take(&mut token));
}
let mut previous = None;
for comment_char in chars.by_ref() {
if previous == Some('*') && comment_char == '/' {
break;
}
previous = Some(comment_char);
}
} else if ch.is_ascii_alphanumeric() || ch == '_' {
token.push(ch);
} else if !token.is_empty() {
tokens.push(std::mem::take(&mut token));
}
tokens.push(token.to_string());
if tokens.len() >= limit {
break;
}
}
if tokens.len() < limit && !token.is_empty() {
tokens.push(token);
}
tokens
}
@@ -1799,6 +1830,20 @@ pub fn optimize_sql_file_import_statements(
statements: &[String],
db_type: Option<DatabaseType>,
driver_profile: Option<&str>,
) -> Vec<SqlFileImportStatement> {
optimize_sql_file_import_statements_with_max_insert_batch_statements(
statements,
db_type,
driver_profile,
SQL_FILE_INSERT_BATCH_MAX_STATEMENTS,
)
}
pub fn optimize_sql_file_import_statements_with_max_insert_batch_statements(
statements: &[String],
db_type: Option<DatabaseType>,
driver_profile: Option<&str>,
max_insert_batch_statements: usize,
) -> Vec<SqlFileImportStatement> {
let mut optimized = Vec::new();
let mut pending_insert: Option<PendingInsertBatch> = None;
@@ -1833,7 +1878,7 @@ pub fn optimize_sql_file_import_statements(
.flatten();
if let Some(insert) = mergeable_insert {
match pending_insert.as_mut() {
Some(batch) if batch.can_accept(&insert) => batch.push(insert),
Some(batch) if batch.can_accept(&insert, max_insert_batch_statements) => batch.push(insert),
Some(_) => {
flush_pending_insert(&mut optimized, &mut pending_insert);
pending_insert = Some(PendingInsertBatch::new(insert));
@@ -1904,9 +1949,9 @@ impl PendingInsertBatch {
}
}
fn can_accept(&self, insert: &MergeableInsert) -> bool {
fn can_accept(&self, insert: &MergeableInsert, max_insert_batch_statements: usize) -> bool {
self.prefix_key == insert.prefix_key
&& self.source_statement_count < SQL_FILE_INSERT_BATCH_MAX_STATEMENTS
&& self.source_statement_count < max_insert_batch_statements
&& self.byte_len + insert.values.len() + 3 <= SQL_FILE_INSERT_BATCH_MAX_BYTES
}