perf(sql-file): optimize import batching
This commit is contained in:
1 parent
bc4e912504
commit
f5cc81f761
3 files changed
+702
-91
No files matched your search
+402
-3
@@ -39,6 +39,20 @@ pub enum SqlFileStatementAction {
|
||||
Skip,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum SqlFileImportStatementKind {
|
||||
Execute,
|
||||
Skip,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SqlFileImportStatement {
|
||||
pub kind: SqlFileImportStatementKind,
|
||||
pub sql: String,
|
||||
pub source_sqls: Vec<String>,
|
||||
pub source_statement_count: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
pub struct SqlParsingOptions {
|
||||
pub supports_hash_line_comments: bool,
|
||||
@@ -685,6 +699,343 @@ pub fn prepare_sql_file_statement(
|
||||
SqlFileStatementAction::Execute(body.to_string())
|
||||
}
|
||||
|
||||
pub fn optimize_sql_file_import_statements(
|
||||
statements: &[String],
|
||||
db_type: Option<DatabaseType>,
|
||||
driver_profile: Option<&str>,
|
||||
) -> Vec<SqlFileImportStatement> {
|
||||
let mut optimized = Vec::new();
|
||||
let mut pending_insert: Option<PendingInsertBatch> = None;
|
||||
|
||||
for statement in statements {
|
||||
let action = db_type
|
||||
.as_ref()
|
||||
.map(|db_type| prepare_sql_file_statement(statement, db_type, driver_profile))
|
||||
.unwrap_or_else(|| SqlFileStatementAction::Execute(statement.trim().to_string()));
|
||||
|
||||
match action {
|
||||
SqlFileStatementAction::Skip => {
|
||||
flush_pending_insert(&mut optimized, &mut pending_insert);
|
||||
optimized.push(SqlFileImportStatement {
|
||||
kind: SqlFileImportStatementKind::Skip,
|
||||
sql: statement.trim().to_string(),
|
||||
source_sqls: vec![statement.trim().to_string()],
|
||||
source_statement_count: 1,
|
||||
});
|
||||
}
|
||||
SqlFileStatementAction::Execute(sql) => {
|
||||
let options = db_type.map(SqlParsingOptions::for_database_type).unwrap_or_default();
|
||||
if let Some(insert) = parse_mergeable_insert(&sql, options) {
|
||||
match pending_insert.as_mut() {
|
||||
Some(batch) if batch.can_accept(&insert) => batch.push(insert),
|
||||
Some(_) => {
|
||||
flush_pending_insert(&mut optimized, &mut pending_insert);
|
||||
pending_insert = Some(PendingInsertBatch::new(insert));
|
||||
}
|
||||
None => {
|
||||
pending_insert = Some(PendingInsertBatch::new(insert));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
flush_pending_insert(&mut optimized, &mut pending_insert);
|
||||
optimized.push(SqlFileImportStatement {
|
||||
kind: SqlFileImportStatementKind::Execute,
|
||||
sql: sql.clone(),
|
||||
source_sqls: vec![sql],
|
||||
source_statement_count: 1,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
flush_pending_insert(&mut optimized, &mut pending_insert);
|
||||
optimized
|
||||
}
|
||||
|
||||
fn flush_pending_insert(optimized: &mut Vec<SqlFileImportStatement>, pending_insert: &mut Option<PendingInsertBatch>) {
|
||||
if let Some(batch) = pending_insert.take() {
|
||||
optimized.push(SqlFileImportStatement {
|
||||
kind: SqlFileImportStatementKind::Execute,
|
||||
sql: batch.to_sql(),
|
||||
source_sqls: batch.source_sqls,
|
||||
source_statement_count: batch.source_statement_count,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const SQL_FILE_INSERT_BATCH_MAX_STATEMENTS: usize = 500;
|
||||
const SQL_FILE_INSERT_BATCH_MAX_BYTES: usize = 4 * 1024 * 1024;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct MergeableInsert {
|
||||
prefix: String,
|
||||
prefix_key: String,
|
||||
values: String,
|
||||
sql: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct PendingInsertBatch {
|
||||
prefix: String,
|
||||
prefix_key: String,
|
||||
values: Vec<String>,
|
||||
source_sqls: Vec<String>,
|
||||
source_statement_count: usize,
|
||||
byte_len: usize,
|
||||
}
|
||||
|
||||
impl PendingInsertBatch {
|
||||
fn new(insert: MergeableInsert) -> Self {
|
||||
let byte_len = insert.prefix.len() + insert.values.len() + 16;
|
||||
Self {
|
||||
prefix: insert.prefix,
|
||||
prefix_key: insert.prefix_key,
|
||||
values: vec![insert.values],
|
||||
source_sqls: vec![insert.sql],
|
||||
source_statement_count: 1,
|
||||
byte_len,
|
||||
}
|
||||
}
|
||||
|
||||
fn can_accept(&self, insert: &MergeableInsert) -> bool {
|
||||
self.prefix_key == insert.prefix_key
|
||||
&& self.source_statement_count < SQL_FILE_INSERT_BATCH_MAX_STATEMENTS
|
||||
&& self.byte_len + insert.values.len() + 3 <= SQL_FILE_INSERT_BATCH_MAX_BYTES
|
||||
}
|
||||
|
||||
fn push(&mut self, insert: MergeableInsert) {
|
||||
self.byte_len += insert.values.len() + 3;
|
||||
self.values.push(insert.values);
|
||||
self.source_sqls.push(insert.sql);
|
||||
self.source_statement_count += 1;
|
||||
}
|
||||
|
||||
fn to_sql(&self) -> String {
|
||||
if self.source_statement_count == 1 {
|
||||
return self.source_sqls.first().cloned().unwrap_or_default();
|
||||
}
|
||||
format!("{} VALUES\n{}", self.prefix, self.values.join(",\n"))
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_mergeable_insert(sql: &str, options: SqlParsingOptions) -> Option<MergeableInsert> {
|
||||
let executable = leading_executable_sql_with_options(sql, options).trim().trim_end_matches(';').trim();
|
||||
if !starts_with_keyword(executable, "insert") {
|
||||
return None;
|
||||
}
|
||||
|
||||
let (values_start, values_end) = find_top_level_values_keyword(executable)?;
|
||||
let prefix_without_values = executable[..values_start].trim_end();
|
||||
let values = executable[values_end..].trim();
|
||||
let values = parse_insert_values_tail(values)?;
|
||||
|
||||
let prefix = prefix_without_values.to_string();
|
||||
Some(MergeableInsert {
|
||||
prefix_key: normalize_insert_prefix_key(&prefix),
|
||||
prefix,
|
||||
values,
|
||||
sql: executable.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
fn find_top_level_values_keyword(sql: &str) -> Option<(usize, usize)> {
|
||||
let mut scanner = SqlScanner::default();
|
||||
let mut depth = 0usize;
|
||||
|
||||
for (idx, ch) in sql.char_indices() {
|
||||
scanner.step(sql, idx, ch);
|
||||
if scanner.is_masked() {
|
||||
continue;
|
||||
}
|
||||
|
||||
match ch {
|
||||
'(' => depth += 1,
|
||||
')' => depth = depth.saturating_sub(1),
|
||||
_ if depth == 0 => {
|
||||
for keyword in ["values", "value"] {
|
||||
if keyword_at(sql, idx, keyword) {
|
||||
return Some((idx, idx + keyword.len()));
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
fn parse_insert_values_tail(tail: &str) -> Option<String> {
|
||||
let tail = tail.trim().trim_end_matches(';').trim();
|
||||
if tail.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut scanner = SqlScanner::default();
|
||||
let mut depth = 0usize;
|
||||
let mut saw_tuple = false;
|
||||
let mut expecting_tuple = true;
|
||||
|
||||
for (idx, ch) in tail.char_indices() {
|
||||
scanner.step(tail, idx, ch);
|
||||
if scanner.is_masked() {
|
||||
continue;
|
||||
}
|
||||
|
||||
if expecting_tuple {
|
||||
if ch.is_whitespace() || (saw_tuple && ch == ',') {
|
||||
continue;
|
||||
}
|
||||
if ch != '(' {
|
||||
return None;
|
||||
}
|
||||
expecting_tuple = false;
|
||||
saw_tuple = true;
|
||||
depth = 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
match ch {
|
||||
'(' => depth += 1,
|
||||
')' => {
|
||||
depth = depth.checked_sub(1)?;
|
||||
if depth == 0 {
|
||||
expecting_tuple = true;
|
||||
}
|
||||
}
|
||||
',' if depth == 0 => {}
|
||||
_ if depth == 0 && !ch.is_whitespace() => return None,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
if saw_tuple && depth == 0 {
|
||||
Some(tail.to_string())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct SqlScanner {
|
||||
in_single_quote: bool,
|
||||
in_double_quote: bool,
|
||||
in_backtick: bool,
|
||||
in_line_comment: bool,
|
||||
in_block_comment: bool,
|
||||
dollar_quote_tag: Option<String>,
|
||||
previous: Option<char>,
|
||||
}
|
||||
|
||||
impl SqlScanner {
|
||||
fn step(&mut self, sql: &str, idx: usize, ch: char) {
|
||||
if let Some(tag) = self.dollar_quote_tag.clone() {
|
||||
if sql[idx..].starts_with(&tag) {
|
||||
self.dollar_quote_tag = None;
|
||||
}
|
||||
self.previous = Some(ch);
|
||||
return;
|
||||
}
|
||||
|
||||
let next = next_char_at(sql, idx + ch.len_utf8());
|
||||
if self.in_line_comment {
|
||||
if ch == '\n' {
|
||||
self.in_line_comment = false;
|
||||
}
|
||||
self.previous = Some(ch);
|
||||
return;
|
||||
}
|
||||
if self.in_block_comment {
|
||||
if self.previous == Some('*') && ch == '/' {
|
||||
self.in_block_comment = false;
|
||||
}
|
||||
self.previous = Some(ch);
|
||||
return;
|
||||
}
|
||||
|
||||
if !self.in_single_quote && !self.in_double_quote && !self.in_backtick {
|
||||
if ch == '-' && next == Some('-') {
|
||||
self.in_line_comment = true;
|
||||
} else if ch == '/' && next == Some('*') {
|
||||
self.in_block_comment = true;
|
||||
} else if let Some(tag) = dollar_quote_tag_at_str(sql, idx) {
|
||||
self.dollar_quote_tag = Some(tag);
|
||||
}
|
||||
}
|
||||
|
||||
match ch {
|
||||
'\'' if !self.in_double_quote && !self.in_backtick && self.previous != Some('\\') => {
|
||||
self.in_single_quote = !self.in_single_quote;
|
||||
}
|
||||
'"' if !self.in_single_quote && !self.in_backtick && self.previous != Some('\\') => {
|
||||
self.in_double_quote = !self.in_double_quote;
|
||||
}
|
||||
'`' if !self.in_single_quote && !self.in_double_quote => {
|
||||
self.in_backtick = !self.in_backtick;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
self.previous = Some(ch);
|
||||
}
|
||||
|
||||
fn is_masked(&self) -> bool {
|
||||
self.in_single_quote
|
||||
|| self.in_double_quote
|
||||
|| self.in_backtick
|
||||
|| self.in_line_comment
|
||||
|| self.in_block_comment
|
||||
|| self.dollar_quote_tag.is_some()
|
||||
}
|
||||
}
|
||||
|
||||
fn keyword_at(sql: &str, idx: usize, keyword: &str) -> bool {
|
||||
let end = idx + keyword.len();
|
||||
sql.get(idx..end).is_some_and(|candidate| candidate.eq_ignore_ascii_case(keyword))
|
||||
&& sql[..idx].chars().next_back().is_none_or(|ch| !is_sql_ident_char(ch))
|
||||
&& sql.get(end..).and_then(|tail| tail.chars().next()).is_none_or(|ch| !is_sql_ident_char(ch))
|
||||
}
|
||||
|
||||
fn starts_with_keyword(sql: &str, keyword: &str) -> bool {
|
||||
sql.get(..keyword.len()).is_some_and(|candidate| candidate.eq_ignore_ascii_case(keyword))
|
||||
&& sql.get(keyword.len()..).and_then(|tail| tail.chars().next()).is_none_or(|ch| !is_sql_ident_char(ch))
|
||||
}
|
||||
|
||||
fn is_sql_ident_char(ch: char) -> bool {
|
||||
ch == '_' || ch == '$' || ch.is_ascii_alphanumeric()
|
||||
}
|
||||
|
||||
fn normalize_insert_prefix_key(prefix: &str) -> String {
|
||||
let mut scanner = SqlScanner::default();
|
||||
let mut key = String::with_capacity(prefix.len());
|
||||
let mut previous_space = false;
|
||||
|
||||
for (idx, ch) in prefix.char_indices() {
|
||||
scanner.step(prefix, idx, ch);
|
||||
if scanner.in_single_quote
|
||||
|| scanner.in_double_quote
|
||||
|| scanner.in_backtick
|
||||
|| scanner.dollar_quote_tag.is_some()
|
||||
{
|
||||
key.push(ch);
|
||||
previous_space = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
if ch.is_whitespace() {
|
||||
if !previous_space {
|
||||
key.push(' ');
|
||||
}
|
||||
previous_space = true;
|
||||
} else {
|
||||
key.push(ch.to_ascii_lowercase());
|
||||
previous_space = false;
|
||||
}
|
||||
}
|
||||
|
||||
key.trim().to_string()
|
||||
}
|
||||
|
||||
pub fn starts_with_executable_sql_keyword(sql: &str, keywords: &[&str]) -> bool {
|
||||
starts_with_executable_sql_keyword_with_options(sql, keywords, SqlParsingOptions::default())
|
||||
}
|
||||
@@ -1041,9 +1392,10 @@ mod tests {
|
||||
use crate::models::connection::DatabaseType;
|
||||
|
||||
use super::{
|
||||
decode_sql_file_bytes, find_statement_at_cursor_for_database, prepare_sql_file_statement, split_sql_script,
|
||||
split_sql_statements_for_database, starts_with_executable_sql_keyword,
|
||||
starts_with_executable_sql_keyword_for_database, SqlFileStatementAction, SqlStatementSplitter,
|
||||
decode_sql_file_bytes, find_statement_at_cursor_for_database, optimize_sql_file_import_statements,
|
||||
prepare_sql_file_statement, split_sql_script, split_sql_statements_for_database,
|
||||
starts_with_executable_sql_keyword, starts_with_executable_sql_keyword_for_database, SqlFileStatementAction,
|
||||
SqlStatementSplitter,
|
||||
};
|
||||
|
||||
#[test]
|
||||
@@ -1271,6 +1623,53 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn optimizes_adjacent_single_row_inserts_into_multi_row_insert() {
|
||||
let statements = vec![
|
||||
"INSERT INTO users (id, name) VALUES (1, 'Ada')".to_string(),
|
||||
"insert into users (id, name) values (2, 'Linus')".to_string(),
|
||||
"SELECT 1".to_string(),
|
||||
];
|
||||
|
||||
let optimized = optimize_sql_file_import_statements(&statements, Some(DatabaseType::Mysql), None);
|
||||
|
||||
assert_eq!(optimized.len(), 2);
|
||||
assert_eq!(optimized[0].source_statement_count, 2);
|
||||
assert_eq!(optimized[0].sql, "INSERT INTO users (id, name) VALUES\n(1, 'Ada'),\n(2, 'Linus')");
|
||||
assert_eq!(optimized[1].sql, "SELECT 1");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keeps_insert_batches_separate_for_different_targets_or_suffixes() {
|
||||
let statements = vec![
|
||||
"INSERT INTO users (id) VALUES (1)".to_string(),
|
||||
"INSERT INTO teams (id) VALUES (1)".to_string(),
|
||||
"INSERT INTO users (id) VALUES (2) RETURNING id".to_string(),
|
||||
];
|
||||
|
||||
let optimized = optimize_sql_file_import_statements(&statements, Some(DatabaseType::Postgres), None);
|
||||
|
||||
assert_eq!(optimized.len(), 3);
|
||||
assert!(optimized.iter().all(|statement| statement.source_statement_count == 1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn optimized_sql_file_import_keeps_skipped_mysql_dump_statements() {
|
||||
let statements = vec![
|
||||
"LOCK TABLES `users` WRITE".to_string(),
|
||||
"INSERT INTO `users` VALUES (1)".to_string(),
|
||||
"INSERT INTO `users` VALUES (2)".to_string(),
|
||||
"UNLOCK TABLES".to_string(),
|
||||
];
|
||||
|
||||
let optimized = optimize_sql_file_import_statements(&statements, Some(DatabaseType::Mysql), None);
|
||||
|
||||
assert_eq!(optimized.len(), 3);
|
||||
assert_eq!(optimized[0].kind, super::SqlFileImportStatementKind::Skip);
|
||||
assert_eq!(optimized[1].source_statement_count, 2);
|
||||
assert_eq!(optimized[2].kind, super::SqlFileImportStatementKind::Skip);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn split_batches_by_go() {
|
||||
assert_eq!(super::split_sql_batches("SELECT 1\nGO\nSELECT 2"), vec!["SELECT 1", "SELECT 2"]);
|
||||
|
||||
Reference in new issue
Block a user