perf(sql-file): optimize import batching

This commit is contained in:
t8y2 committed 2026-05-31 11:14:14 +08:00
1 parent bc4e912504
commit f5cc81f761
3 files changed
+702 -91

No files matched your search

+402 -3
View File
@@ -39,6 +39,20 @@ pub enum SqlFileStatementAction {
Skip,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum SqlFileImportStatementKind {
Execute,
Skip,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SqlFileImportStatement {
pub kind: SqlFileImportStatementKind,
pub sql: String,
pub source_sqls: Vec<String>,
pub source_statement_count: usize,
}
#[derive(Debug, Clone, Copy, Default)]
pub struct SqlParsingOptions {
pub supports_hash_line_comments: bool,
@@ -685,6 +699,343 @@ pub fn prepare_sql_file_statement(
SqlFileStatementAction::Execute(body.to_string())
}
pub fn optimize_sql_file_import_statements(
statements: &[String],
db_type: Option<DatabaseType>,
driver_profile: Option<&str>,
) -> Vec<SqlFileImportStatement> {
let mut optimized = Vec::new();
let mut pending_insert: Option<PendingInsertBatch> = None;
for statement in statements {
let action = db_type
.as_ref()
.map(|db_type| prepare_sql_file_statement(statement, db_type, driver_profile))
.unwrap_or_else(|| SqlFileStatementAction::Execute(statement.trim().to_string()));
match action {
SqlFileStatementAction::Skip => {
flush_pending_insert(&mut optimized, &mut pending_insert);
optimized.push(SqlFileImportStatement {
kind: SqlFileImportStatementKind::Skip,
sql: statement.trim().to_string(),
source_sqls: vec![statement.trim().to_string()],
source_statement_count: 1,
});
}
SqlFileStatementAction::Execute(sql) => {
let options = db_type.map(SqlParsingOptions::for_database_type).unwrap_or_default();
if let Some(insert) = parse_mergeable_insert(&sql, options) {
match pending_insert.as_mut() {
Some(batch) if batch.can_accept(&insert) => batch.push(insert),
Some(_) => {
flush_pending_insert(&mut optimized, &mut pending_insert);
pending_insert = Some(PendingInsertBatch::new(insert));
}
None => {
pending_insert = Some(PendingInsertBatch::new(insert));
}
}
} else {
flush_pending_insert(&mut optimized, &mut pending_insert);
optimized.push(SqlFileImportStatement {
kind: SqlFileImportStatementKind::Execute,
sql: sql.clone(),
source_sqls: vec![sql],
source_statement_count: 1,
});
}
}
}
}
flush_pending_insert(&mut optimized, &mut pending_insert);
optimized
}
fn flush_pending_insert(optimized: &mut Vec<SqlFileImportStatement>, pending_insert: &mut Option<PendingInsertBatch>) {
if let Some(batch) = pending_insert.take() {
optimized.push(SqlFileImportStatement {
kind: SqlFileImportStatementKind::Execute,
sql: batch.to_sql(),
source_sqls: batch.source_sqls,
source_statement_count: batch.source_statement_count,
});
}
}
const SQL_FILE_INSERT_BATCH_MAX_STATEMENTS: usize = 500;
const SQL_FILE_INSERT_BATCH_MAX_BYTES: usize = 4 * 1024 * 1024;
#[derive(Debug, Clone)]
struct MergeableInsert {
prefix: String,
prefix_key: String,
values: String,
sql: String,
}
#[derive(Debug, Clone)]
struct PendingInsertBatch {
prefix: String,
prefix_key: String,
values: Vec<String>,
source_sqls: Vec<String>,
source_statement_count: usize,
byte_len: usize,
}
impl PendingInsertBatch {
fn new(insert: MergeableInsert) -> Self {
let byte_len = insert.prefix.len() + insert.values.len() + 16;
Self {
prefix: insert.prefix,
prefix_key: insert.prefix_key,
values: vec![insert.values],
source_sqls: vec![insert.sql],
source_statement_count: 1,
byte_len,
}
}
fn can_accept(&self, insert: &MergeableInsert) -> bool {
self.prefix_key == insert.prefix_key
&& self.source_statement_count < SQL_FILE_INSERT_BATCH_MAX_STATEMENTS
&& self.byte_len + insert.values.len() + 3 <= SQL_FILE_INSERT_BATCH_MAX_BYTES
}
fn push(&mut self, insert: MergeableInsert) {
self.byte_len += insert.values.len() + 3;
self.values.push(insert.values);
self.source_sqls.push(insert.sql);
self.source_statement_count += 1;
}
fn to_sql(&self) -> String {
if self.source_statement_count == 1 {
return self.source_sqls.first().cloned().unwrap_or_default();
}
format!("{} VALUES\n{}", self.prefix, self.values.join(",\n"))
}
}
fn parse_mergeable_insert(sql: &str, options: SqlParsingOptions) -> Option<MergeableInsert> {
let executable = leading_executable_sql_with_options(sql, options).trim().trim_end_matches(';').trim();
if !starts_with_keyword(executable, "insert") {
return None;
}
let (values_start, values_end) = find_top_level_values_keyword(executable)?;
let prefix_without_values = executable[..values_start].trim_end();
let values = executable[values_end..].trim();
let values = parse_insert_values_tail(values)?;
let prefix = prefix_without_values.to_string();
Some(MergeableInsert {
prefix_key: normalize_insert_prefix_key(&prefix),
prefix,
values,
sql: executable.to_string(),
})
}
fn find_top_level_values_keyword(sql: &str) -> Option<(usize, usize)> {
let mut scanner = SqlScanner::default();
let mut depth = 0usize;
for (idx, ch) in sql.char_indices() {
scanner.step(sql, idx, ch);
if scanner.is_masked() {
continue;
}
match ch {
'(' => depth += 1,
')' => depth = depth.saturating_sub(1),
_ if depth == 0 => {
for keyword in ["values", "value"] {
if keyword_at(sql, idx, keyword) {
return Some((idx, idx + keyword.len()));
}
}
}
_ => {}
}
}
None
}
fn parse_insert_values_tail(tail: &str) -> Option<String> {
let tail = tail.trim().trim_end_matches(';').trim();
if tail.is_empty() {
return None;
}
let mut scanner = SqlScanner::default();
let mut depth = 0usize;
let mut saw_tuple = false;
let mut expecting_tuple = true;
for (idx, ch) in tail.char_indices() {
scanner.step(tail, idx, ch);
if scanner.is_masked() {
continue;
}
if expecting_tuple {
if ch.is_whitespace() || (saw_tuple && ch == ',') {
continue;
}
if ch != '(' {
return None;
}
expecting_tuple = false;
saw_tuple = true;
depth = 1;
continue;
}
match ch {
'(' => depth += 1,
')' => {
depth = depth.checked_sub(1)?;
if depth == 0 {
expecting_tuple = true;
}
}
',' if depth == 0 => {}
_ if depth == 0 && !ch.is_whitespace() => return None,
_ => {}
}
}
if saw_tuple && depth == 0 {
Some(tail.to_string())
} else {
None
}
}
#[derive(Default)]
struct SqlScanner {
in_single_quote: bool,
in_double_quote: bool,
in_backtick: bool,
in_line_comment: bool,
in_block_comment: bool,
dollar_quote_tag: Option<String>,
previous: Option<char>,
}
impl SqlScanner {
fn step(&mut self, sql: &str, idx: usize, ch: char) {
if let Some(tag) = self.dollar_quote_tag.clone() {
if sql[idx..].starts_with(&tag) {
self.dollar_quote_tag = None;
}
self.previous = Some(ch);
return;
}
let next = next_char_at(sql, idx + ch.len_utf8());
if self.in_line_comment {
if ch == '\n' {
self.in_line_comment = false;
}
self.previous = Some(ch);
return;
}
if self.in_block_comment {
if self.previous == Some('*') && ch == '/' {
self.in_block_comment = false;
}
self.previous = Some(ch);
return;
}
if !self.in_single_quote && !self.in_double_quote && !self.in_backtick {
if ch == '-' && next == Some('-') {
self.in_line_comment = true;
} else if ch == '/' && next == Some('*') {
self.in_block_comment = true;
} else if let Some(tag) = dollar_quote_tag_at_str(sql, idx) {
self.dollar_quote_tag = Some(tag);
}
}
match ch {
'\'' if !self.in_double_quote && !self.in_backtick && self.previous != Some('\\') => {
self.in_single_quote = !self.in_single_quote;
}
'"' if !self.in_single_quote && !self.in_backtick && self.previous != Some('\\') => {
self.in_double_quote = !self.in_double_quote;
}
'`' if !self.in_single_quote && !self.in_double_quote => {
self.in_backtick = !self.in_backtick;
}
_ => {}
}
self.previous = Some(ch);
}
fn is_masked(&self) -> bool {
self.in_single_quote
|| self.in_double_quote
|| self.in_backtick
|| self.in_line_comment
|| self.in_block_comment
|| self.dollar_quote_tag.is_some()
}
}
fn keyword_at(sql: &str, idx: usize, keyword: &str) -> bool {
let end = idx + keyword.len();
sql.get(idx..end).is_some_and(|candidate| candidate.eq_ignore_ascii_case(keyword))
&& sql[..idx].chars().next_back().is_none_or(|ch| !is_sql_ident_char(ch))
&& sql.get(end..).and_then(|tail| tail.chars().next()).is_none_or(|ch| !is_sql_ident_char(ch))
}
fn starts_with_keyword(sql: &str, keyword: &str) -> bool {
sql.get(..keyword.len()).is_some_and(|candidate| candidate.eq_ignore_ascii_case(keyword))
&& sql.get(keyword.len()..).and_then(|tail| tail.chars().next()).is_none_or(|ch| !is_sql_ident_char(ch))
}
fn is_sql_ident_char(ch: char) -> bool {
ch == '_' || ch == '$' || ch.is_ascii_alphanumeric()
}
fn normalize_insert_prefix_key(prefix: &str) -> String {
let mut scanner = SqlScanner::default();
let mut key = String::with_capacity(prefix.len());
let mut previous_space = false;
for (idx, ch) in prefix.char_indices() {
scanner.step(prefix, idx, ch);
if scanner.in_single_quote
|| scanner.in_double_quote
|| scanner.in_backtick
|| scanner.dollar_quote_tag.is_some()
{
key.push(ch);
previous_space = false;
continue;
}
if ch.is_whitespace() {
if !previous_space {
key.push(' ');
}
previous_space = true;
} else {
key.push(ch.to_ascii_lowercase());
previous_space = false;
}
}
key.trim().to_string()
}
pub fn starts_with_executable_sql_keyword(sql: &str, keywords: &[&str]) -> bool {
starts_with_executable_sql_keyword_with_options(sql, keywords, SqlParsingOptions::default())
}
@@ -1041,9 +1392,10 @@ mod tests {
use crate::models::connection::DatabaseType;
use super::{
decode_sql_file_bytes, find_statement_at_cursor_for_database, prepare_sql_file_statement, split_sql_script,
split_sql_statements_for_database, starts_with_executable_sql_keyword,
starts_with_executable_sql_keyword_for_database, SqlFileStatementAction, SqlStatementSplitter,
decode_sql_file_bytes, find_statement_at_cursor_for_database, optimize_sql_file_import_statements,
prepare_sql_file_statement, split_sql_script, split_sql_statements_for_database,
starts_with_executable_sql_keyword, starts_with_executable_sql_keyword_for_database, SqlFileStatementAction,
SqlStatementSplitter,
};
#[test]
@@ -1271,6 +1623,53 @@ mod tests {
);
}
#[test]
fn optimizes_adjacent_single_row_inserts_into_multi_row_insert() {
let statements = vec![
"INSERT INTO users (id, name) VALUES (1, 'Ada')".to_string(),
"insert into users (id, name) values (2, 'Linus')".to_string(),
"SELECT 1".to_string(),
];
let optimized = optimize_sql_file_import_statements(&statements, Some(DatabaseType::Mysql), None);
assert_eq!(optimized.len(), 2);
assert_eq!(optimized[0].source_statement_count, 2);
assert_eq!(optimized[0].sql, "INSERT INTO users (id, name) VALUES\n(1, 'Ada'),\n(2, 'Linus')");
assert_eq!(optimized[1].sql, "SELECT 1");
}
#[test]
fn keeps_insert_batches_separate_for_different_targets_or_suffixes() {
let statements = vec![
"INSERT INTO users (id) VALUES (1)".to_string(),
"INSERT INTO teams (id) VALUES (1)".to_string(),
"INSERT INTO users (id) VALUES (2) RETURNING id".to_string(),
];
let optimized = optimize_sql_file_import_statements(&statements, Some(DatabaseType::Postgres), None);
assert_eq!(optimized.len(), 3);
assert!(optimized.iter().all(|statement| statement.source_statement_count == 1));
}
#[test]
fn optimized_sql_file_import_keeps_skipped_mysql_dump_statements() {
let statements = vec![
"LOCK TABLES `users` WRITE".to_string(),
"INSERT INTO `users` VALUES (1)".to_string(),
"INSERT INTO `users` VALUES (2)".to_string(),
"UNLOCK TABLES".to_string(),
];
let optimized = optimize_sql_file_import_statements(&statements, Some(DatabaseType::Mysql), None);
assert_eq!(optimized.len(), 3);
assert_eq!(optimized[0].kind, super::SqlFileImportStatementKind::Skip);
assert_eq!(optimized[1].source_statement_count, 2);
assert_eq!(optimized[2].kind, super::SqlFileImportStatementKind::Skip);
}
#[test]
fn split_batches_by_go() {
assert_eq!(super::split_sql_batches("SELECT 1\nGO\nSELECT 2"), vec!["SELECT 1", "SELECT 2"]);