use serde::{Deserialize, Serialize}; use std::collections::HashSet; use std::sync::LazyLock; use regex::Regex; use sqlparser::ast::{ Expr, FunctionArg, FunctionArgExpr, FunctionArguments, GroupByExpr, Ident, JoinConstraint, JoinOperator, ObjectName, ObjectNamePart, OrderByKind, Query, Select, SelectItem, SetExpr, Statement, TableFactor, TableWithJoins, }; use sqlparser::dialect::{ ClickHouseDialect, DuckDbDialect, GenericDialect, MsSqlDialect, MySqlDialect, PostgreSqlDialect, SQLiteDialect, SparkSqlDialect, }; use sqlparser::keywords::Keyword; use sqlparser::parser::{Parser, ParserError}; use sqlparser::tokenizer::{Span, Token, TokenWithSpan, Tokenizer}; use crate::models::connection::DatabaseType; use crate::sql::{ starts_with_duckdb_result_sql_keyword, starts_with_executable_sql_keyword, statement_ranges_for_database, }; static CLICKHOUSE_STRICTNESS_FIRST_JOIN_RE: LazyLock = LazyLock::new(|| { Regex::new(r"(?i)\b(ANY|ALL|SEMI|ANTI|ASOF)\s+(LEFT|RIGHT|FULL|INNER|CROSS)(\s+OUTER)?\s+JOIN\b") .expect("valid ClickHouse join strictness regex") }); static POSTGRES_DEFAULT_PRIVILEGES_RE: LazyLock = LazyLock::new(|| { Regex::new(r"(?i)^\s*ALTER\s+DEFAULT\s+PRIVILEGES\b").expect("valid PostgreSQL default privileges regex") }); /// Oracle administrative DDL that sqlparser has no grammar for: Oracle and the /// Oracle-compatible engines (OceanBase Oracle mode, Dameng) accept these statements, but /// the analyzer routes them through sqlparser's generic dialect, which reports a bogus /// `sql parser error` on statements the server happily runs (issue #9104). static ORACLE_ADMIN_DDL_RE: LazyLock = LazyLock::new(|| { Regex::new( r"(?is)^\s*(?:(?:create|alter|drop)\s+(?:or\s+replace\s+)?(?:public\s+)?(?:user|tablespace|profile|directory|synonym|role|session|system)\b|(?:grant|revoke)\b)", ) .expect("valid Oracle administrative DDL regex") }); #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct SqlReferenceAnalysis { pub tables: Vec, pub columns: Vec, pub scopes: Vec, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct SqlTableReference { pub name: String, pub schema: Option, pub alias: Option, pub span: SqlTextSpan, pub scope_id: usize, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct SqlColumnReference { pub name: String, pub qualifier: Option, pub span: SqlTextSpan, pub scope_id: usize, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct SqlReferenceScope { pub id: usize, pub parent_id: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct SqlTextSpan { pub start_line: usize, pub start_column: usize, pub end_line: usize, pub end_column: usize, } impl From for SqlTextSpan { fn from(span: Span) -> Self { Self { start_line: span.start.line as usize, start_column: span.start.column as usize, end_line: span.end.line as usize, end_column: span.end.column as usize, } } } #[derive(Default)] struct Analyzer { tables: Vec, columns: Vec, scopes: Vec, scope_stack: Vec, cte_scope_stack: Vec>, next_scope_id: usize, is_sqlserver: bool, is_spark: bool, } pub fn analyze_sql_references(sql: &str, dialect: Option<&str>) -> Result { let normalized_dialect = normalize_dialect(dialect); if normalized_dialect == "duckdb" && starts_with_duckdb_parser_gap_sql(sql) { return Ok(SqlReferenceAnalysis { tables: vec![], columns: vec![], scopes: vec![] }); } if normalized_dialect == "postgres" && starts_with_postgres_parser_gap_sql(sql) { return Ok(SqlReferenceAnalysis { tables: vec![], columns: vec![], scopes: vec![] }); } let parser_sql = if normalized_dialect == "clickhouse" { normalize_clickhouse_join_order_for_parser(sql) } else { sql.to_string() }; let parsed_statements = match normalized_dialect.as_str() { "postgres" => Parser::parse_sql(&PostgreSqlDialect {}, &parser_sql), "mysql" => Parser::parse_sql(&MySqlDialect {}, &parser_sql), "sqlite" => Parser::parse_sql(&SQLiteDialect {}, &parser_sql), "sqlserver" => parse_sqlserver(&parser_sql), "clickhouse" => Parser::parse_sql(&ClickHouseDialect {}, &parser_sql), "duckdb" => Parser::parse_sql(&DuckDbDialect {}, &parser_sql), "spark" => parse_spark_sql(&parser_sql), _ => Parser::parse_sql(&GenericDialect {}, &parser_sql), }; let statements = match parsed_statements { Ok(statements) => { if normalized_dialect == "postgres" && postgres_create_procedure_is_missing_body(&parser_sql) { return Err("Expected: procedure body after AS, found: EOF".to_string()); } statements } Err(error) if normalized_dialect == "postgres" && is_postgres_create_procedure_parser_gap(&parser_sql, &error) => { return Ok(SqlReferenceAnalysis { tables: vec![], columns: vec![], scopes: vec![] }); } Err(error) => match oracle_admin_ddl_gap_statements(dialect, &parser_sql) { Some(statements) => statements, None => return Err(error.to_string()), }, }; let mut analyzer = Analyzer { is_sqlserver: normalized_dialect == "sqlserver", is_spark: normalized_dialect == "spark", ..Analyzer::default() }; for statement in statements { analyzer.visit_statement(&statement); } Ok(SqlReferenceAnalysis { tables: analyzer.tables, columns: analyzer.columns, scopes: analyzer.scopes }) } fn parse_sqlserver(sql: &str) -> Result, ParserError> { let dialect = MsSqlDialect {}; let mut tokens = Tokenizer::new(&dialect, sql).tokenize_with_location()?; normalize_sqlserver_create_proc_tokens(&mut tokens); match Parser::new(&dialect).with_tokens_with_locations(tokens).parse_statements() { Ok(statements) => Ok(statements), Err(error) => { let mut fallback_tokens = Tokenizer::new(&dialect, sql).tokenize_with_location()?; normalize_sqlserver_create_proc_tokens(&mut fallback_tokens); let mut changed = normalize_sqlserver_cursor_tokens(&mut fallback_tokens); changed |= normalize_sqlserver_alter_table_add_tokens(&mut fallback_tokens); changed |= remove_sqlserver_query_hint_tokens(&mut fallback_tokens); if changed { if let Ok(statements) = Parser::new(&dialect).with_tokens_with_locations(fallback_tokens).parse_statements() { return Ok(statements); } } Err(error) } } } fn parse_spark_sql(sql: &str) -> Result, ParserError> { let dialect = SparkSqlDialect {}; let tokens = Tokenizer::new(&dialect, sql).tokenize_with_location()?; let mut fallback_tokens = tokens.clone(); let fallback_changed = normalize_spark_datasource_create_table_tokens(&mut fallback_tokens)?; match Parser::new(&dialect).with_tokens_with_locations(tokens).parse_statements() { Ok(statements) => Ok(statements), Err(error) => { // sqlparser 0.62 accepts Spark's USING clause but expects later datasource clauses in Hive order. // The fallback removes only clauses parsed independently from the same located token stream. if fallback_changed { if let Ok(statements) = Parser::new(&dialect).with_tokens_with_locations(fallback_tokens).parse_statements() { return Ok(statements); } } Err(error) } } } fn normalize_spark_datasource_create_table_tokens(tokens: &mut Vec) -> Result { let dialect = SparkSqlDialect {}; let mut removals = Vec::new(); let mut index = 0usize; while let Some(statement_start) = next_significant_token_index(tokens, index) { if matches!(tokens[statement_start].token, Token::EOF) { break; } if matches!(tokens[statement_start].token, Token::SemiColon) { index = statement_start + 1; continue; } let statement_end = sql_statement_end_index(tokens, statement_start); let significant_indexes = tokens .iter() .enumerate() .take(statement_end) .skip(statement_start) .filter_map(|(index, token)| (!matches!(token.token, Token::Whitespace(_))).then_some(index)) .collect::>(); removals.extend(spark_datasource_clause_removals(tokens, &significant_indexes, &dialect)?); index = statement_end.saturating_add(1); } removals.sort_unstable(); removals.dedup(); let changed = !removals.is_empty(); for index in removals.into_iter().rev() { tokens.remove(index); } Ok(changed) } fn spark_datasource_clause_removals( tokens: &[TokenWithSpan], indexes: &[usize], dialect: &SparkSqlDialect, ) -> Result, ParserError> { let Some(using_position) = spark_create_table_using_position(tokens, indexes) else { return Ok(vec![]); }; let Some(provider_index) = indexes.get(using_position + 1).copied() else { return Ok(vec![]); }; if !matches!(tokens[provider_index].token, Token::Word(_)) { return Parser::new(dialect).expected("data source identifier", tokens[provider_index].clone()); } let mut removals = Vec::new(); let mut seen_options = false; let mut seen_partitioning = false; let mut seen_comment = false; let mut seen_table_properties = false; let mut position = using_position + 2; while position < indexes.len() { let token = &tokens[indexes[position]]; match unquoted_token_keyword(token) { Some(Keyword::OPTIONS) | Some(Keyword::TBLPROPERTIES) => { let keyword = unquoted_token_keyword(token).expect("matched option keyword"); let (seen, clause_name) = match keyword { Keyword::OPTIONS => (&mut seen_options, "OPTIONS"), Keyword::TBLPROPERTIES => (&mut seen_table_properties, "TBLPROPERTIES"), _ => unreachable!("matched option keyword"), }; if *seen { return Parser::new(dialect).expected(&format!("at most one {clause_name} clause"), token.clone()); } *seen = true; let Some(open_index) = indexes.get(position + 1).copied() else { return Ok(vec![]); }; if !matches!(tokens[open_index].token, Token::LParen) { return Ok(vec![]); } let Some(close_index) = matching_parenthesis_index(tokens, open_index) else { return Ok(vec![]); }; let Some(close_position) = indexes.iter().position(|index| *index == close_index) else { return Ok(vec![]); }; if !spark_tokens_parse_options(tokens, &indexes[position..=close_position], keyword, dialect) { return Ok(vec![]); } removals.extend_from_slice(&indexes[position..=close_position]); position = close_position + 1; } Some(Keyword::PARTITIONED) => { if seen_partitioning { return Parser::new(dialect).expected("at most one PARTITIONED BY clause", token.clone()); } seen_partitioning = true; if indexes.get(position + 1).and_then(|index| unquoted_token_keyword(&tokens[*index])) != Some(Keyword::BY) { return Ok(vec![]); } let Some(open_index) = indexes.get(position + 2).copied() else { return Ok(vec![]); }; if !matches!(tokens[open_index].token, Token::LParen) { return Ok(vec![]); } let Some(close_index) = matching_parenthesis_index(tokens, open_index) else { return Ok(vec![]); }; let Some(close_position) = indexes.iter().position(|index| *index == close_index) else { return Ok(vec![]); }; if !spark_tokens_parse_partitioning(tokens, &indexes[position..=close_position], dialect) { return Ok(vec![]); } removals.extend_from_slice(&indexes[position..=close_position]); position = close_position + 1; } Some(Keyword::COMMENT) => { if seen_comment { return Parser::new(dialect).expected("at most one COMMENT clause", token.clone()); } seen_comment = true; let Some(comment_index) = indexes.get(position + 1).copied() else { return Ok(vec![]); }; if !matches!(tokens[comment_index].token, Token::SingleQuotedString(_)) { return Ok(vec![]); } removals.extend_from_slice(&indexes[position..=position + 1]); position += 2; } Some(Keyword::LOCATION) => { let Some(location_index) = indexes.get(position + 1).copied() else { return Ok(vec![]); }; if !matches!(tokens[location_index].token, Token::SingleQuotedString(_)) { return Ok(vec![]); } position += 2; } Some(Keyword::AS) => break, _ => return Ok(vec![]), } } Ok(removals) } fn spark_create_table_using_position(tokens: &[TokenWithSpan], indexes: &[usize]) -> Option { if indexes.first().and_then(|index| unquoted_token_keyword(&tokens[*index])) != Some(Keyword::CREATE) { return None; } let mut position = 1usize; if indexes.get(position).and_then(|index| unquoted_token_keyword(&tokens[*index])) == Some(Keyword::OR) && indexes.get(position + 1).and_then(|index| unquoted_token_keyword(&tokens[*index])) == Some(Keyword::REPLACE) { position += 2; } while indexes.get(position).is_some_and(|index| { ["GLOBAL", "LOCAL", "TEMP", "TEMPORARY", "EXTERNAL"] .iter() .any(|modifier| unquoted_token_word_eq(&tokens[*index], modifier)) }) { position += 1; } if indexes.get(position).and_then(|index| unquoted_token_keyword(&tokens[*index])) != Some(Keyword::TABLE) { return None; } let mut depth = 0usize; for (position, index) in indexes.iter().copied().enumerate().skip(position + 1) { match tokens[index].token { Token::LParen => depth += 1, Token::RParen => depth = depth.saturating_sub(1), _ if depth == 0 && unquoted_token_keyword(&tokens[index]) == Some(Keyword::AS) => return None, _ if depth == 0 && unquoted_token_keyword(&tokens[index]) == Some(Keyword::USING) => { return Some(position); } _ => {} } } None } fn spark_tokens_parse_options( tokens: &[TokenWithSpan], indexes: &[usize], keyword: Keyword, dialect: &SparkSqlDialect, ) -> bool { let clause_tokens = indexes.iter().map(|index| tokens[*index].clone()).collect(); let mut parser = Parser::new(dialect).with_tokens_with_locations(clause_tokens); parser.parse_options(keyword).is_ok() && matches!(parser.peek_token_ref().token, Token::EOF) } fn spark_tokens_parse_partitioning(tokens: &[TokenWithSpan], indexes: &[usize], dialect: &SparkSqlDialect) -> bool { let clause_tokens = indexes.iter().map(|index| tokens[*index].clone()).collect(); let mut parser = Parser::new(dialect).with_tokens_with_locations(clause_tokens); if parser.expect_keywords(&[Keyword::PARTITIONED, Keyword::BY]).is_err() || parser.expect_token(&Token::LParen).is_err() { return false; } let Ok(expressions) = parser.parse_comma_separated(Parser::parse_expr) else { return false; }; !expressions.is_empty() && parser.expect_token(&Token::RParen).is_ok() && matches!(parser.peek_token_ref().token, Token::EOF) } fn normalize_sqlserver_cursor_tokens(tokens: &mut Vec) -> bool { let mut removals = Vec::new(); let mut index = 0usize; while let Some(statement_start) = next_significant_token_index(tokens, index) { if matches!(tokens[statement_start].token, Token::EOF) { break; } if matches!(tokens[statement_start].token, Token::SemiColon) { index = statement_start + 1; continue; } let statement_end = sql_statement_end_index(tokens, statement_start); let significant_indexes = tokens .iter() .enumerate() .take(statement_end) .skip(statement_start) .filter_map(|(index, token)| (!matches!(token.token, Token::Whitespace(_))).then_some(index)) .collect::>(); if let Some(removal) = sqlserver_cursor_option_removal(tokens, &significant_indexes) { removals.push(removal); } index = statement_end.saturating_add(1); } let changed = !removals.is_empty(); for removal in removals.into_iter().rev() { tokens.drain(removal); } changed } fn sqlserver_cursor_option_removal(tokens: &[TokenWithSpan], indexes: &[usize]) -> Option> { if indexes.len() < 4 || !unquoted_token_word_eq(&tokens[indexes[0]], "DECLARE") || !is_sqlserver_named_cursor(&tokens[indexes[1]]) || !unquoted_token_word_eq(&tokens[indexes[2]], "CURSOR") { return None; } let mut position = 3usize; if indexes.get(position).is_some_and(|index| { unquoted_token_word_eq(&tokens[*index], "LOCAL") || unquoted_token_word_eq(&tokens[*index], "GLOBAL") }) { position += 1; } if indexes.get(position).is_some_and(|index| unquoted_token_word_eq(&tokens[*index], "FAST_FORWARD")) { position += 1; } if position == 3 || !indexes.get(position).is_some_and(|index| unquoted_token_word_eq(&tokens[*index], "FOR")) { return None; } let query_position = position + 1; let query_start = *indexes.get(query_position)?; if (!unquoted_token_word_eq(&tokens[query_start], "SELECT") && !unquoted_token_word_eq(&tokens[query_start], "WITH")) || sqlserver_cursor_query_has_forbidden_clause(tokens, &indexes[query_position..]) { return None; } Some(indexes[3]..indexes[position]) } fn is_sqlserver_named_cursor(token: &TokenWithSpan) -> bool { let Token::Word(word) = &token.token else { return false; }; !word.value.starts_with('@') && (word.quote_style.is_some() || word.keyword == Keyword::NoKeyword) } fn sqlserver_cursor_query_has_forbidden_clause(tokens: &[TokenWithSpan], indexes: &[usize]) -> bool { let mut depth = 0usize; for (position, index) in indexes.iter().copied().enumerate() { match tokens[index].token { Token::LParen => depth += 1, Token::RParen => depth = depth.saturating_sub(1), _ if depth == 0 && (unquoted_token_word_eq(&tokens[index], "INTO") || unquoted_token_word_eq(&tokens[index], "COMPUTE") || unquoted_token_word_eq(&tokens[index], "FOR") && indexes .get(position + 1) .is_some_and(|index| unquoted_token_word_eq(&tokens[*index], "BROWSE"))) => { return true; } _ => {} } } false } fn unquoted_token_word_eq(token: &TokenWithSpan, expected: &str) -> bool { matches!(&token.token, Token::Word(word) if word.quote_style.is_none() && word.value.eq_ignore_ascii_case(expected)) } fn normalize_sqlserver_alter_table_add_tokens(tokens: &mut Vec) -> bool { let mut insertions = Vec::new(); let mut index = 0usize; while index < tokens.len() { if unquoted_token_keyword(&tokens[index]) != Some(Keyword::ALTER) { index += 1; continue; } let Some(table_index) = next_significant_token_index(tokens, index + 1) else { break; }; if unquoted_token_keyword(&tokens[table_index]) != Some(Keyword::TABLE) { index += 1; continue; } let statement_end = sql_statement_end_index(tokens, table_index + 1); let Some(add_index) = sqlserver_alter_table_add_index(tokens, table_index + 1, statement_end) else { index = statement_end.saturating_add(1); continue; }; let add_token = tokens[add_index].clone(); let mut depth = 0usize; for token_index in add_index + 1..statement_end { match tokens[token_index].token { Token::LParen => depth += 1, Token::RParen => depth = depth.saturating_sub(1), Token::Comma if depth == 0 => { let Some(next_index) = next_significant_token_index(tokens, token_index + 1) else { continue; }; if next_index >= statement_end { continue; } if is_sqlserver_alter_table_operation_starter(&tokens[next_index]) { break; } insertions.push((next_index, add_token.clone())); } _ => {} } } index = statement_end.saturating_add(1); } let changed = !insertions.is_empty(); for (index, token) in insertions.into_iter().rev() { tokens.insert(index, token); } changed } fn sql_statement_end_index(tokens: &[TokenWithSpan], start: usize) -> usize { let mut depth = 0usize; for (index, token) in tokens.iter().enumerate().skip(start) { match token.token { Token::LParen => depth += 1, Token::RParen => depth = depth.saturating_sub(1), Token::SemiColon | Token::EOF if depth == 0 => return index, _ => {} } } tokens.len() } fn sqlserver_alter_table_add_index(tokens: &[TokenWithSpan], start: usize, end: usize) -> Option { let mut depth = 0usize; for (index, token) in tokens.iter().enumerate().take(end).skip(start) { match token.token { Token::LParen => depth += 1, Token::RParen => depth = depth.saturating_sub(1), _ if depth == 0 && unquoted_token_keyword(token) == Some(Keyword::ADD) => return Some(index), _ => {} } } None } fn is_sqlserver_alter_table_operation_starter(token: &TokenWithSpan) -> bool { let Token::Word(word) = &token.token else { return false; }; if word.quote_style.is_some() { return false; } matches!( word.value.to_ascii_uppercase().as_str(), "ADD" | "ALTER" | "DISABLE" | "DROP" | "ENABLE" | "NOCHECK" | "PARTITION" | "REBUILD" | "RENAME" | "REPLICA" | "SET" | "SWAP" | "SWITCH" ) } fn remove_sqlserver_query_hint_tokens(tokens: &mut Vec) -> bool { let mut depth = 0usize; let mut ranges = Vec::new(); for index in 0..tokens.len() { match &tokens[index].token { Token::LParen => depth += 1, Token::RParen => depth = depth.saturating_sub(1), Token::Word(word) if depth == 0 && word.quote_style.is_none() && word.value.eq_ignore_ascii_case("OPTION") => { if let Some(range) = sqlserver_query_hint_range(tokens, index) { if sqlserver_query_hint_removal_parses_statement(tokens, range) { ranges.push(range); } } } _ => {} } } let removed = !ranges.is_empty(); for (start, end) in ranges.into_iter().rev() { tokens.drain(start..=end); } removed } fn sqlserver_query_hint_removal_parses_statement(tokens: &[TokenWithSpan], range: (usize, usize)) -> bool { let statement_start = tokens[..range.0] .iter() .rposition(|token| matches!(token.token, Token::SemiColon)) .map_or(0, |index| index + 1); let statement_end = next_significant_token_index(tokens, range.1 + 1).unwrap_or(tokens.len() - 1); let mut statement_tokens = tokens[statement_start..=statement_end].to_vec(); if parse_sqlserver_tokens(statement_tokens.clone()).is_ok() { return false; } statement_tokens.drain(range.0 - statement_start..=range.1 - statement_start); parse_sqlserver_tokens(statement_tokens).is_ok() } fn parse_sqlserver_tokens(tokens: Vec) -> Result, ParserError> { Parser::new(&MsSqlDialect {}).with_tokens_with_locations(tokens).parse_statements() } fn sqlserver_query_hint_range(tokens: &[TokenWithSpan], option_index: usize) -> Option<(usize, usize)> { let open_index = next_significant_token_index(tokens, option_index + 1)?; if !matches!(tokens[open_index].token, Token::LParen) { return None; } let close_index = matching_parenthesis_index(tokens, open_index)?; let hint_index = next_significant_token_index(tokens, open_index + 1)?; if hint_index >= close_index || !is_sqlserver_query_hint_starter(&tokens[hint_index]) { return None; } if let Some(next_index) = next_significant_token_index(tokens, close_index + 1) { if !matches!(tokens[next_index].token, Token::SemiColon | Token::EOF) { return None; } } Some((option_index, close_index)) } fn next_significant_token_index(tokens: &[TokenWithSpan], mut index: usize) -> Option { while index < tokens.len() { if !matches!(tokens[index].token, Token::Whitespace(_)) { return Some(index); } index += 1; } None } fn matching_parenthesis_index(tokens: &[TokenWithSpan], open_index: usize) -> Option { let mut depth = 0usize; for (index, token) in tokens.iter().enumerate().skip(open_index) { match token.token { Token::LParen => depth += 1, Token::RParen => { depth = depth.checked_sub(1)?; if depth == 0 { return Some(index); } } _ => {} } } None } fn is_sqlserver_query_hint_starter(token: &TokenWithSpan) -> bool { let Token::Word(word) = &token.token else { return false; }; if word.quote_style.is_some() { return false; } matches!( word.value.to_ascii_uppercase().as_str(), "CONCAT" | "DISABLE" | "DISABLE_OPTIMIZED_PLAN_FORCING" | "EXPAND" | "FAST" | "FOR" | "FORCE" | "HASH" | "IGNORE_NONCLUSTERED_COLUMNSTORE_INDEX" | "KEEP" | "KEEPFIXED" | "LABEL" | "LOOP" | "MAX_GRANT_PERCENT" | "MAXDOP" | "MAXRECURSION" | "MERGE" | "MIN_GRANT_PERCENT" | "NO_PERFORMANCE_SPOOL" | "OPTIMIZE" | "ORDER" | "PARAMETERIZATION" | "QUERYTRACEON" | "RECOMPILE" | "ROBUST" | "TABLE" | "USE" ) } fn normalize_sqlserver_create_proc_tokens(tokens: &mut [TokenWithSpan]) { let significant_indexes: Vec = tokens .iter() .enumerate() .filter_map(|(index, token)| (!matches!(token.token, Token::Whitespace(_))).then_some(index)) .collect(); for (position, index) in significant_indexes.iter().copied().enumerate() { if token_keyword(&tokens[index]) != Some(Keyword::CREATE) { continue; } let mut proc_position = position + 1; if significant_indexes.get(proc_position).and_then(|index| token_keyword(&tokens[*index])) == Some(Keyword::OR) { proc_position += 1; if significant_indexes.get(proc_position).and_then(|index| token_keyword(&tokens[*index])) != Some(Keyword::ALTER) { continue; } proc_position += 1; } let Some(proc_index) = significant_indexes.get(proc_position).copied() else { continue; }; let Token::Word(word) = &mut tokens[proc_index].token else { continue; }; // SQL Server documents PROC as a contextual synonym for PROCEDURE after CREATE. if word.quote_style.is_none() && word.value.eq_ignore_ascii_case("proc") { word.keyword = Keyword::PROCEDURE; } } } fn token_keyword(token: &TokenWithSpan) -> Option { match &token.token { Token::Word(word) => Some(word.keyword), _ => None, } } fn unquoted_token_keyword(token: &TokenWithSpan) -> Option { match &token.token { Token::Word(word) if word.quote_style.is_none() => Some(word.keyword), _ => None, } } fn starts_with_duckdb_parser_gap_sql(sql: &str) -> bool { starts_with_duckdb_result_sql_keyword(sql) && starts_with_executable_sql_keyword(sql, &["FROM", "SUMMARIZE", "SUMMARISE", "PIVOT", "UNPIVOT"]) } /// Oracle-compatible dialects whose statements the analyzer parses with sqlparser's /// generic dialect. Oracle connections report the PL/SQL formatter id (`oracle`) as /// their analysis dialect, and Dameng follows the same Oracle-compatible grammar. fn is_oracle_compatible_analysis_dialect(dialect: Option<&str>) -> bool { let Some(name) = dialect else { return false; }; let name = name.trim().to_ascii_lowercase(); matches!(name.as_str(), "oracle" | "dameng") } /// Re-parses `sql` with every Oracle administrative DDL statement blanked out, so the /// statements around it keep their diagnostics instead of the whole script failing on a /// false syntax error. Returns `None` when nothing matched or when the rest of the script /// still does not parse, in which case the caller keeps the original parser error. fn oracle_admin_ddl_gap_statements(dialect: Option<&str>, sql: &str) -> Option> { if !is_oracle_compatible_analysis_dialect(dialect) { return None; } let mut masked: Option = None; for range in statement_ranges_for_database(sql, DatabaseType::Oracle) { if !ORACLE_ADMIN_DDL_RE.is_match(&range.text) { continue; } let target = masked.get_or_insert_with(|| sql.to_string()); let blanked = blank_out_statement_text(target.get(range.start..range.end)?); target.replace_range(range.start..range.end, &blanked); } Parser::parse_sql(&GenericDialect {}, masked.as_deref()?).ok() } /// Replaces every character of a statement with a space while keeping line breaks and the /// original byte length, so token positions of the statements that follow stay exact. fn blank_out_statement_text(statement: &str) -> String { let mut blanked = String::with_capacity(statement.len()); for character in statement.chars() { if character == '\n' || character == '\r' { blanked.push(character); } else { for _ in 0..character.len_utf8() { blanked.push(' '); } } } blanked } fn starts_with_postgres_parser_gap_sql(sql: &str) -> bool { POSTGRES_DEFAULT_PRIVILEGES_RE.is_match(sql) } struct PostgresCreateProcedureLayout { tokens: Vec, significant_indexes: Vec, or_replace_indexes: Option<[usize; 2]>, parameters_open_position: usize, parameters_close_position: usize, } fn postgres_create_procedure_layout(sql: &str) -> Option { let dialect = PostgreSqlDialect {}; let tokens = Tokenizer::new(&dialect, sql).tokenize_with_location().ok()?; let significant_indexes: Vec = tokens .iter() .enumerate() .filter_map(|(index, token)| (!matches!(token.token, Token::Whitespace(_))).then_some(index)) .collect(); let mut position = 0; if significant_indexes.get(position).and_then(|index| token_keyword(&tokens[*index])) != Some(Keyword::CREATE) { return None; } position += 1; let or_replace_indexes = if significant_indexes.get(position).and_then(|index| token_keyword(&tokens[*index])) == Some(Keyword::OR) && significant_indexes.get(position + 1).and_then(|index| token_keyword(&tokens[*index])) == Some(Keyword::REPLACE) { let indexes = [significant_indexes[position], significant_indexes[position + 1]]; position += 2; Some(indexes) } else { None }; if significant_indexes.get(position).and_then(|index| token_keyword(&tokens[*index])) != Some(Keyword::PROCEDURE) { return None; } position += 1; if !matches!(tokens[significant_indexes.get(position).copied()?].token, Token::Word(_)) { return None; } position += 1; while matches!(significant_indexes.get(position).map(|index| &tokens[*index].token), Some(Token::Period)) { position += 1; if !matches!(tokens[significant_indexes.get(position).copied()?].token, Token::Word(_)) { return None; } position += 1; } let parameters_open_index = significant_indexes.get(position).copied()?; if !matches!(tokens[parameters_open_index].token, Token::LParen) { return None; } let parameters_close_index = matching_parenthesis_index(&tokens, parameters_open_index)?; let parameters_close_position = significant_indexes.iter().position(|index| *index == parameters_close_index)?; if !postgres_procedure_parameters_are_nonempty(&tokens, &significant_indexes, position, parameters_close_position) { return None; } Some(PostgresCreateProcedureLayout { tokens, significant_indexes, or_replace_indexes, parameters_open_position: position, parameters_close_position, }) } fn postgres_procedure_parameters_are_nonempty( tokens: &[TokenWithSpan], significant_indexes: &[usize], open_position: usize, close_position: usize, ) -> bool { if close_position == open_position + 1 { return true; } let mut depth = 1; let mut parameter_has_tokens = false; for index in significant_indexes.iter().take(close_position).skip(open_position + 1).copied() { match tokens[index].token { Token::LParen => { depth += 1; parameter_has_tokens = true; } Token::RParen => depth -= 1, Token::Comma if depth == 1 => { if !parameter_has_tokens { return false; } parameter_has_tokens = false; } _ if depth == 1 => parameter_has_tokens = true, _ => {} } } parameter_has_tokens } fn postgres_create_procedure_is_missing_body(sql: &str) -> bool { let Some(layout) = postgres_create_procedure_layout(sql) else { return false; }; let mut tail = &layout.significant_indexes[layout.parameters_close_position + 1..]; while let Some(index) = tail.last() { if matches!(layout.tokens[*index].token, Token::SemiColon) { tail = &tail[..tail.len() - 1]; } else { break; } } tail.last().and_then(|index| token_keyword(&layout.tokens[*index])) == Some(Keyword::AS) } fn is_postgres_create_procedure_parser_gap(sql: &str, error: &ParserError) -> bool { let Some(mut layout) = postgres_create_procedure_layout(sql) else { return false; }; let Some(body_position) = (layout.parameters_close_position + 1..layout.significant_indexes.len()).find(|position| { matches!(layout.tokens[layout.significant_indexes[*position]].token, Token::DollarQuotedString(_)) }) else { return false; }; let as_index = layout.significant_indexes[body_position - 1]; if token_keyword(&layout.tokens[as_index]) != Some(Keyword::AS) { return false; } let trailing = &layout.significant_indexes[body_position + 1..]; if !(trailing.is_empty() || trailing.len() == 1 && matches!(layout.tokens[trailing[0]].token, Token::SemiColon)) { return false; } let ParserError::ParserError(message) = error else { return false; }; let error_matches_gap = if layout.or_replace_indexes.is_some() { message.starts_with( "Expected: [EXTERNAL] TABLE or [MATERIALIZED] VIEW or FUNCTION after CREATE OR REPLACE, found: PROCEDURE at ", ) } else { message.starts_with(&format!( "Expected: an SQL statement, found: {} at ", layout.tokens[layout.significant_indexes[body_position]] )) }; if !error_matches_gap { return false; } let mut depth = 1; for position in layout.parameters_open_position + 1..layout.parameters_close_position { let index = layout.significant_indexes[position]; match layout.tokens[index].token { Token::LParen => depth += 1, Token::RParen => depth -= 1, _ if depth == 1 && token_keyword(&layout.tokens[index]) == Some(Keyword::DEFAULT) => { layout.tokens[index].token = Token::Eq; } _ => {} } } let dialect = PostgreSqlDialect {}; let body_index = layout.significant_indexes[body_position]; let replacement = match Tokenizer::new(&dialect, "BEGIN SELECT 1; END").tokenize_with_location() { Ok(tokens) => tokens, Err(_) => return false, }; layout.tokens.splice(body_index..=body_index, replacement); if let Some([or_index, replace_index]) = layout.or_replace_indexes { layout.tokens.remove(replace_index); layout.tokens.remove(or_index); } Parser::new(&dialect).with_tokens_with_locations(layout.tokens).parse_statements().is_ok() } fn normalize_clickhouse_join_order_for_parser(sql: &str) -> String { CLICKHOUSE_STRICTNESS_FIRST_JOIN_RE .replace_all(sql, |captures: ®ex::Captures<'_>| { let matched_len = captures.get(0).map(|value| value.as_str().len()).unwrap_or(0); let strictness = captures.get(1).map(|value| value.as_str()).unwrap_or(""); let join_type = captures.get(2).map(|value| value.as_str()).unwrap_or(""); let outer = captures.get(3).map(|value| value.as_str()).unwrap_or(""); let mut normalized = match strictness.to_ascii_uppercase().as_str() { "SEMI" | "ANTI" => format!("{join_type}{outer} {strictness} JOIN"), _ => format!("{join_type}{outer} JOIN"), }; if normalized.len() < matched_len { normalized.push_str(&" ".repeat(matched_len - normalized.len())); } normalized }) .into_owned() } fn normalize_dialect(dialect: Option<&str>) -> String { match dialect.unwrap_or("generic").to_ascii_lowercase().as_str() { "postgres" | "postgresql" | "redshift" | "opengauss" | "gaussdb" | "highgo" | "uxdb" | "questdb" => { "postgres".to_string() } "mysql" | "mariadb" | "doris" | "starrocks" | "manticoresearch" | "oceanbase" => "mysql".to_string(), "sqlite" => "sqlite".to_string(), "sqlserver" | "mssql" => "sqlserver".to_string(), "clickhouse" => "clickhouse".to_string(), "duckdb" => "duckdb".to_string(), "spark" | "sparksql" => "spark".to_string(), _ => "generic".to_string(), } } impl Analyzer { fn visit_statement(&mut self, statement: &Statement) { match statement { Statement::Query(query) => self.visit_query_in_new_scope(query, None), Statement::CreateTable(create_table) if self.is_spark => { if let Some(query) = &create_table.query { self.visit_query_in_new_scope(query, None); } } Statement::Declare { stmts } if self.is_sqlserver => { for declaration in stmts { if let Some(query) = &declaration.for_query { self.visit_query_in_new_scope(query, None); } } } _ => {} } } fn visit_query_in_new_scope(&mut self, query: &Query, parent_id: Option) { let scope_id = self.next_scope_id; self.next_scope_id += 1; self.scopes.push(SqlReferenceScope { id: scope_id, parent_id }); self.scope_stack.push(scope_id); self.cte_scope_stack.push(HashSet::new()); self.visit_query(query); self.cte_scope_stack.pop(); self.scope_stack.pop(); } fn visit_child_query(&mut self, query: &Query) { self.visit_query_in_new_scope(query, self.current_scope_id()); } fn current_scope_id(&self) -> Option { self.scope_stack.last().copied() } fn add_visible_cte(&mut self, ident: &Ident) { let key = self.cte_name_key(ident); if let Some(visible_ctes) = self.cte_scope_stack.last_mut() { visible_ctes.insert(key); } } fn is_visible_cte(&self, name: &ObjectName) -> bool { if name.0.len() != 1 { return false; } let Some(ident) = name.0.first().and_then(ObjectNamePart::as_ident) else { return false; }; let key = self.cte_name_key(ident); self.cte_scope_stack.iter().rev().any(|visible_ctes| visible_ctes.contains(&key)) } fn cte_name_key(&self, ident: &Ident) -> String { if self.is_sqlserver || ident.quote_style.is_none() { ident.value.to_ascii_lowercase() } else { ident.value.clone() } } fn visit_query(&mut self, query: &Query) { if let Some(with) = &query.with { for cte in &with.cte_tables { // Add each name before its body: recursive/self and earlier CTEs are visible, later CTEs are not. self.add_visible_cte(&cte.alias.name); self.visit_child_query(&cte.query); } } self.visit_set_expr(&query.body); if let Some(order_by) = &query.order_by { if let OrderByKind::Expressions(exprs) = &order_by.kind { for expr in exprs { self.visit_expr(&expr.expr); } } } } fn visit_set_expr(&mut self, set_expr: &SetExpr) { match set_expr { SetExpr::Select(select) => self.visit_select(select), SetExpr::Query(query) => self.visit_child_query(query), SetExpr::SetOperation { left, right, .. } => { self.visit_set_expr_in_child_scope(left); self.visit_set_expr_in_child_scope(right); } _ => {} } } fn visit_set_expr_in_child_scope(&mut self, set_expr: &SetExpr) { let scope_id = self.next_scope_id; self.next_scope_id += 1; self.scopes.push(SqlReferenceScope { id: scope_id, parent_id: self.current_scope_id() }); self.scope_stack.push(scope_id); self.visit_set_expr(set_expr); self.scope_stack.pop(); } fn visit_select(&mut self, select: &Select) { for table in &select.from { self.visit_table_with_joins(table); } for item in &select.projection { match item { SelectItem::UnnamedExpr(expr) | SelectItem::ExprWithAlias { expr, .. } | SelectItem::ExprWithAliases { expr, .. } => self.visit_expr(expr), _ => {} } } if let Some(expr) = &select.prewhere { self.visit_expr(expr); } if let Some(expr) = &select.selection { self.visit_expr(expr); } if let GroupByExpr::Expressions(exprs, _) = &select.group_by { for expr in exprs { self.visit_expr(expr); } } for expr in &select.cluster_by { self.visit_expr(expr); } for expr in &select.distribute_by { self.visit_expr(expr); } for expr in &select.sort_by { self.visit_expr(&expr.expr); } if let Some(expr) = &select.having { self.visit_expr(expr); } if let Some(expr) = &select.qualify { self.visit_expr(expr); } } fn visit_table_with_joins(&mut self, table: &TableWithJoins) { self.visit_table_factor(&table.relation); for join in &table.joins { self.visit_table_factor(&join.relation); self.visit_join_operator(&join.join_operator); } } fn visit_join_operator(&mut self, operator: &JoinOperator) { match operator { JoinOperator::Join(constraint) | JoinOperator::Inner(constraint) | JoinOperator::Left(constraint) | JoinOperator::LeftOuter(constraint) | JoinOperator::Right(constraint) | JoinOperator::RightOuter(constraint) | JoinOperator::FullOuter(constraint) | JoinOperator::CrossJoin(constraint) | JoinOperator::Semi(constraint) | JoinOperator::LeftSemi(constraint) | JoinOperator::RightSemi(constraint) | JoinOperator::Anti(constraint) | JoinOperator::LeftAnti(constraint) | JoinOperator::RightAnti(constraint) | JoinOperator::StraightJoin(constraint) => self.visit_join_constraint(constraint), JoinOperator::AsOf { match_condition, constraint } => { self.visit_expr(match_condition); self.visit_join_constraint(constraint); } _ => {} } } fn visit_join_constraint(&mut self, constraint: &JoinConstraint) { match constraint { JoinConstraint::On(expr) => self.visit_expr(expr), JoinConstraint::Using(names) => { for name in names { if let Some(ident) = object_name_last_ident(name) { self.push_column(None, ident); } } } _ => {} } } fn visit_table_factor(&mut self, factor: &TableFactor) { match factor { TableFactor::Table { name, alias, args, .. } => { // Qualified names remain physical objects even when their final component matches a visible CTE. if args.is_none() && !self.is_visible_cte(name) { if let Some(table) = table_reference_from_name( name, alias.as_ref().map(|a| a.name.value.clone()), self.current_scope_id(), ) { self.tables.push(table); } } } TableFactor::Derived { subquery, .. } => self.visit_child_query(subquery), TableFactor::NestedJoin { table_with_joins, .. } => self.visit_table_with_joins(table_with_joins), TableFactor::TableFunction { expr, .. } => self.visit_expr(expr), TableFactor::Function { args, .. } => { for arg in args { self.visit_function_arg(arg); } } TableFactor::UNNEST { array_exprs, .. } => { for expr in array_exprs { self.visit_expr(expr); } } _ => {} } } fn visit_expr(&mut self, expr: &Expr) { match expr { Expr::Identifier(ident) => self.push_column(None, ident), Expr::CompoundIdentifier(idents) => { if idents.len() >= 2 { let column = idents.last().expect("checked length"); let qualifier = idents.get(idents.len() - 2).map(|ident| ident.value.clone()); self.push_column(qualifier, column); } } Expr::CompoundFieldAccess { root, .. } | Expr::JsonAccess { value: root, .. } => self.visit_expr(root), Expr::IsFalse(expr) | Expr::IsNotFalse(expr) | Expr::IsTrue(expr) | Expr::IsNotTrue(expr) | Expr::IsNull(expr) | Expr::IsNotNull(expr) | Expr::IsUnknown(expr) | Expr::IsNotUnknown(expr) | Expr::UnaryOp { expr, .. } | Expr::Nested(expr) => self.visit_expr(expr), Expr::IsDistinctFrom(left, right) | Expr::IsNotDistinctFrom(left, right) | Expr::BinaryOp { left, right, .. } | Expr::AnyOp { left, right, .. } | Expr::AllOp { left, right, .. } => { self.visit_expr(left); self.visit_expr(right); } Expr::InList { expr, list, .. } => { self.visit_expr(expr); for item in list { self.visit_expr(item); } } Expr::InSubquery { expr, subquery, .. } => { self.visit_expr(expr); self.visit_child_query(subquery); } Expr::InUnnest { expr, array_expr, .. } => { self.visit_expr(expr); self.visit_expr(array_expr); } Expr::Between { expr, low, high, .. } => { self.visit_expr(expr); self.visit_expr(low); self.visit_expr(high); } Expr::Like { expr, pattern, .. } | Expr::ILike { expr, pattern, .. } | Expr::SimilarTo { expr, pattern, .. } | Expr::RLike { expr, pattern, .. } => { self.visit_expr(expr); self.visit_expr(pattern); } Expr::Cast { expr, .. } | Expr::Extract { expr, .. } | Expr::Ceil { expr, .. } | Expr::Floor { expr, .. } => self.visit_expr(expr), Expr::AtTimeZone { timestamp, time_zone } => { self.visit_expr(timestamp); self.visit_expr(time_zone); } Expr::Position { expr, r#in } => { self.visit_expr(expr); self.visit_expr(r#in); } Expr::Function(function) => { self.visit_function_args(&function.parameters); self.visit_function_call_args(&function.name, &function.args); if let Some(filter) = &function.filter { self.visit_expr(filter); } for order in &function.within_group { self.visit_expr(&order.expr); } } Expr::Case { operand, conditions, else_result, .. } => { if let Some(operand) = operand { self.visit_expr(operand); } for condition in conditions { self.visit_expr(&condition.condition); self.visit_expr(&condition.result); } if let Some(else_result) = else_result { self.visit_expr(else_result); } } Expr::Subquery(query) | Expr::Exists { subquery: query, .. } => self.visit_child_query(query), _ => {} } } fn visit_function_args(&mut self, args: &FunctionArguments) { self.visit_function_args_skipping(args, |_, _| false); } fn visit_function_call_args(&mut self, name: &ObjectName, args: &FunctionArguments) { let sqlserver_datepart_function = self.is_sqlserver.then(|| sqlserver_datepart_function_name(name)).flatten(); self.visit_function_args_skipping(args, |index, arg| { index == 0 && sqlserver_datepart_function .is_some_and(|function_name| is_sqlserver_datepart_argument(function_name, arg)) }); } fn visit_function_args_skipping( &mut self, args: &FunctionArguments, mut should_skip: impl FnMut(usize, &FunctionArg) -> bool, ) { match args { FunctionArguments::Subquery(query) => self.visit_child_query(query), FunctionArguments::List(list) => { for (index, arg) in list.args.iter().enumerate() { if !should_skip(index, arg) { self.visit_function_arg(arg); } } for clause in &list.clauses { if let sqlparser::ast::FunctionArgumentClause::OrderBy(items) = clause { for item in items { self.visit_expr(&item.expr); } } } } FunctionArguments::None => {} } } fn visit_function_arg(&mut self, arg: &FunctionArg) { match arg { FunctionArg::Named { arg, .. } | FunctionArg::ExprNamed { arg, .. } | FunctionArg::Unnamed(arg) => { if let FunctionArgExpr::Expr(expr) = arg { self.visit_expr(expr); } } } } fn push_column(&mut self, qualifier: Option, ident: &Ident) { if let Some(scope_id) = self.current_scope_id() { self.columns.push(SqlColumnReference { name: ident.value.clone(), qualifier, span: ident.span.into(), scope_id, }); } } } fn table_reference_from_name( name: &ObjectName, alias: Option, scope_id: Option, ) -> Option { let parts: Vec<&Ident> = name.0.iter().filter_map(ObjectNamePart::as_ident).collect(); let table = parts.last()?; let schema = if parts.len() >= 2 { parts.get(parts.len() - 2).map(|ident| ident.value.clone()) } else { None }; Some(SqlTableReference { name: table.value.clone(), schema, alias, span: table.span.into(), scope_id: scope_id? }) } fn object_name_last_ident(name: &ObjectName) -> Option<&Ident> { name.0.iter().rev().find_map(ObjectNamePart::as_ident) } fn sqlserver_datepart_function_name(name: &ObjectName) -> Option<&str> { if name.0.len() != 1 { return None; } let ident = name.0.first()?.as_ident()?; ["DATEADD", "DATEDIFF", "DATEDIFF_BIG", "DATEPART", "DATENAME"] .iter() .copied() .find(|function_name| ident.value.eq_ignore_ascii_case(function_name)) } fn is_sqlserver_datepart_argument(function_name: &str, arg: &FunctionArg) -> bool { let FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Identifier(ident))) = arg else { return false; }; if ident.quote_style.is_some() { return false; } // SQL Server parses datepart tokens as identifiers, but these built-ins treat the first token as grammar, not a column. let datepart = ident.value.as_str(); let common_datepart = is_datepart( datepart, &[ "year", "yy", "yyyy", "quarter", "qq", "q", "month", "mm", "m", "dayofyear", "dy", "y", "day", "dd", "d", "week", "wk", "ww", "hour", "hh", "minute", "mi", "n", "second", "ss", "s", "millisecond", "ms", "microsecond", "mcs", "nanosecond", "ns", ], ); let weekday_datepart = is_datepart(datepart, &["weekday", "dw", "w"]); let extended_datepart = is_datepart(datepart, &["tzoffset", "tz", "iso_week", "isowk", "isoww"]); match function_name { "DATEADD" | "DATEDIFF" | "DATEDIFF_BIG" => common_datepart || weekday_datepart, "DATEPART" => common_datepart || weekday_datepart || extended_datepart, "DATENAME" => common_datepart || weekday_datepart || extended_datepart, _ => false, } } fn is_datepart(value: &str, valid_dateparts: &[&str]) -> bool { valid_dateparts.iter().any(|datepart| value.eq_ignore_ascii_case(datepart)) }