fix(schema): converge mysql structure sync and refresh metadata
This commit is contained in:
3 files changed
+224
-4
No files matched your search
@@ -6,11 +6,13 @@ use regex::Regex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::models::connection::DatabaseType;
|
||||
use crate::sql_dialect::ddl_profile::{profile_for, ColumnDdlSyntax, CommentDdlSyntax, DdlDialectProfile};
|
||||
use crate::sql_dialect::ddl_profile::{
|
||||
profile_for, AutoIncSyntax, ColumnDdlSyntax, CommentDdlSyntax, DdlDialectProfile,
|
||||
};
|
||||
use crate::sql_dialect::descriptor::DialectKind;
|
||||
use crate::sql_dialect::inference::{ColumnType, DefaultTypeInferenceEngine, TypeInferenceEngine};
|
||||
use crate::sql_dialect::type_rewrite::{
|
||||
apply_auto_inc_to_column_def, rewrite_column_type, type_looks_integer, AutoIncColumnBuild,
|
||||
apply_auto_inc_to_column_def, column_is_auto_increment, rewrite_column_type, type_looks_integer, AutoIncColumnBuild,
|
||||
};
|
||||
use crate::sql_parser::ast_filter::AstTransmitFilter;
|
||||
use crate::types::{
|
||||
@@ -2162,13 +2164,15 @@ fn diff_schema(options: &SchemaDiffPreparationOptions) -> Vec<TableDiff> {
|
||||
for name in common {
|
||||
let Some(source) = source_details.get(name.as_str()) else { continue };
|
||||
let Some(target) = target_details.get(name.as_str()) else { continue };
|
||||
let column_diffs = diff_columns_with_options(
|
||||
let column_diffs = diff_columns_with_dialect_options(
|
||||
&source.columns,
|
||||
&target.columns,
|
||||
options.ignore_comments,
|
||||
options.compare_column_order,
|
||||
options.detect_renames,
|
||||
options.rename_threshold,
|
||||
options.source_dialect,
|
||||
options.target_dialect,
|
||||
);
|
||||
let index_diffs = diff_indexes(&source.indexes, &target.indexes);
|
||||
let foreign_key_diffs = diff_foreign_keys(&source.foreign_keys, &target.foreign_keys);
|
||||
@@ -2474,6 +2478,43 @@ pub fn diff_columns(source: &[ColumnInfo], target: &[ColumnInfo]) -> Vec<ColumnD
|
||||
diff_columns_with_options(source, target, false, false, false, 0.5)
|
||||
}
|
||||
|
||||
fn normalize_mysql_integer_type_for_comparison(data_type: &str) -> String {
|
||||
let normalized = data_type.split_whitespace().collect::<Vec<_>>().join(" ").to_ascii_lowercase();
|
||||
let Some(open) = normalized.find('(') else {
|
||||
return normalized;
|
||||
};
|
||||
let Some(close) = normalized[open + 1..].find(')').map(|index| open + 1 + index) else {
|
||||
return normalized;
|
||||
};
|
||||
let base = normalized[..open].trim();
|
||||
let width = normalized[open + 1..close].trim();
|
||||
let integer_type = matches!(base, "tinyint" | "smallint" | "mediumint" | "int" | "integer" | "bigint" | "year");
|
||||
if !integer_type || width.is_empty() || !width.bytes().all(|byte| byte.is_ascii_digit()) {
|
||||
return normalized;
|
||||
}
|
||||
let suffix = normalized[close + 1..].trim();
|
||||
if suffix.is_empty() {
|
||||
base.to_string()
|
||||
} else {
|
||||
format!("{base} {suffix}")
|
||||
}
|
||||
}
|
||||
|
||||
fn column_types_equal_for_dialects(
|
||||
source_type: &str,
|
||||
target_type: &str,
|
||||
source_dialect: Option<DialectKind>,
|
||||
target_dialect: Option<DialectKind>,
|
||||
) -> bool {
|
||||
if source_type.eq_ignore_ascii_case(target_type) {
|
||||
return true;
|
||||
}
|
||||
source_dialect == Some(DialectKind::Mysql)
|
||||
&& target_dialect == Some(DialectKind::Mysql)
|
||||
&& normalize_mysql_integer_type_for_comparison(source_type)
|
||||
== normalize_mysql_integer_type_for_comparison(target_type)
|
||||
}
|
||||
|
||||
fn column_type_similarity_score(source_type: &str, target_type: &str) -> f64 {
|
||||
let s = ColumnType::parse(source_type).base_type.to_ascii_lowercase();
|
||||
let t = ColumnType::parse(target_type).base_type.to_ascii_lowercase();
|
||||
@@ -2513,6 +2554,29 @@ fn diff_columns_with_options(
|
||||
compare_column_order: bool,
|
||||
detect_renames: bool,
|
||||
rename_threshold: f64,
|
||||
) -> Vec<ColumnDiff> {
|
||||
diff_columns_with_dialect_options(
|
||||
source,
|
||||
target,
|
||||
ignore_comments,
|
||||
compare_column_order,
|
||||
detect_renames,
|
||||
rename_threshold,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn diff_columns_with_dialect_options(
|
||||
source: &[ColumnInfo],
|
||||
target: &[ColumnInfo],
|
||||
ignore_comments: bool,
|
||||
compare_column_order: bool,
|
||||
detect_renames: bool,
|
||||
rename_threshold: f64,
|
||||
source_dialect: Option<DialectKind>,
|
||||
target_dialect: Option<DialectKind>,
|
||||
) -> Vec<ColumnDiff> {
|
||||
let mut diffs = Vec::new();
|
||||
let target_map: HashMap<&str, &ColumnInfo> = target.iter().map(|column| (column.name.as_str(), column)).collect();
|
||||
@@ -2526,7 +2590,12 @@ fn diff_columns_with_options(
|
||||
for (source_index, source_column) in source.iter().enumerate() {
|
||||
if let Some(target_column) = target_map.get(source_column.name.as_str()) {
|
||||
let mut changes = Vec::new();
|
||||
if source_column.data_type.to_lowercase() != target_column.data_type.to_lowercase() {
|
||||
if !column_types_equal_for_dialects(
|
||||
&source_column.data_type,
|
||||
&target_column.data_type,
|
||||
source_dialect,
|
||||
target_dialect,
|
||||
) {
|
||||
changes.push(format!("type: {} → {}", target_column.data_type, source_column.data_type));
|
||||
}
|
||||
if source_column.is_nullable != target_column.is_nullable {
|
||||
@@ -3134,6 +3203,14 @@ fn column_def(col: &ColumnInfo, db_type: DatabaseType, source_dialect: Option<Di
|
||||
)
|
||||
));
|
||||
}
|
||||
// Suffix-style auto-increment is only valid in MySQL-family ALTER clauses
|
||||
// (ADD/MODIFY/CHANGE). Other dialects' identity clauses are order-sensitive
|
||||
// inside ADD COLUMN, so keep omitting them outside the MySQL family.
|
||||
if column_is_auto_increment(col) && profile.alter_uses_modify_column {
|
||||
if let AutoIncSyntax::Suffix(suffix) = profile.auto_inc {
|
||||
definition.push_str(suffix);
|
||||
}
|
||||
}
|
||||
if profile.inline_column_comment {
|
||||
if let Some(comment) = &col.comment {
|
||||
definition.push_str(&format!(" COMMENT {}", comment_literal(comment)));
|
||||
@@ -7571,6 +7648,99 @@ mod tests {
|
||||
assert_eq!(column_type_similarity_score("varchar(255)", "varchar(64)"), 1.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mysql_same_dialect_ignores_only_integer_display_widths() {
|
||||
let source = vec![
|
||||
column("id", "int(11) unsigned", None),
|
||||
column("status", "tinyint(4)", None),
|
||||
column("amount", "decimal(10,2)", None),
|
||||
column("name", "varchar(128)", None),
|
||||
];
|
||||
let target = vec![
|
||||
column("id", "int unsigned", None),
|
||||
column("status", "tinyint", None),
|
||||
column("amount", "decimal(12,2)", None),
|
||||
column("name", "varchar(64)", None),
|
||||
];
|
||||
|
||||
let diffs = diff_columns_with_dialect_options(
|
||||
&source,
|
||||
&target,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
0.5,
|
||||
Some(DialectKind::Mysql),
|
||||
Some(DialectKind::Mysql),
|
||||
);
|
||||
|
||||
assert_eq!(diffs.iter().map(|diff| diff.name.as_str()).collect::<Vec<_>>(), vec!["amount", "name"]);
|
||||
assert!(diffs.iter().all(|diff| diff.changes.iter().any(|change| change.starts_with("type:"))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mysql_modify_column_preserves_explicit_auto_increment() {
|
||||
let mut source = column("id", "int", Some("new comment"));
|
||||
source.is_primary_key = true;
|
||||
source.extra = Some("auto_increment".to_string());
|
||||
let mut target = source.clone();
|
||||
target.comment = Some("old comment".to_string());
|
||||
let diff = ColumnDiff {
|
||||
diff_type: "modified".to_string(),
|
||||
name: "id".to_string(),
|
||||
source: Some(source),
|
||||
target: Some(target),
|
||||
changes: vec!["comment: old comment → new comment".to_string()],
|
||||
add_position: None,
|
||||
};
|
||||
|
||||
let sql = gen_sql(wrap_table_diff("users", vec![diff]), DatabaseType::Mysql, Some(DialectKind::Mysql));
|
||||
|
||||
assert!(
|
||||
sql.contains("MODIFY COLUMN `id` int NOT NULL AUTO_INCREMENT COMMENT 'new comment'"),
|
||||
"MySQL MODIFY must preserve AUTO_INCREMENT: {sql}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mysql_add_column_keeps_auto_increment_suffix() {
|
||||
let mut source = column("seq", "int", None);
|
||||
source.extra = Some("auto_increment".to_string());
|
||||
let diff = ColumnDiff {
|
||||
diff_type: "added".to_string(),
|
||||
name: "seq".to_string(),
|
||||
source: Some(source),
|
||||
target: None,
|
||||
changes: vec![],
|
||||
add_position: None,
|
||||
};
|
||||
|
||||
let sql = gen_sql(wrap_table_diff("users", vec![diff]), DatabaseType::Mysql, Some(DialectKind::Mysql));
|
||||
|
||||
assert!(sql.contains("AUTO_INCREMENT"), "MySQL ADD COLUMN must keep AUTO_INCREMENT: {sql}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sqlserver_add_column_omits_identity_suffix() {
|
||||
let mut source = column("seq", "int", None);
|
||||
source.extra = Some("auto_increment".to_string());
|
||||
let diff = ColumnDiff {
|
||||
diff_type: "added".to_string(),
|
||||
name: "seq".to_string(),
|
||||
source: Some(source),
|
||||
target: None,
|
||||
changes: vec![],
|
||||
add_position: None,
|
||||
};
|
||||
|
||||
let sql = gen_sql(wrap_table_diff("users", vec![diff]), DatabaseType::SqlServer, Some(DialectKind::Mysql));
|
||||
|
||||
assert!(
|
||||
!sql.to_uppercase().contains("IDENTITY("),
|
||||
"SQL Server ADD COLUMN must not append an order-sensitive IDENTITY clause: {sql}"
|
||||
);
|
||||
}
|
||||
|
||||
// -- 32. Multiple renames in one table --
|
||||
#[test]
|
||||
fn multiple_renames_in_one_table() {
|
||||
|
||||
@@ -725,9 +725,19 @@ pub async fn execute_script_with_2pc(
|
||||
req.destructive_confirmed.unwrap_or(false),
|
||||
)
|
||||
.await;
|
||||
if schema_diff_deploy_may_have_changed_schema(&result) {
|
||||
let prefix = super::schema::object_metadata_cache_prefix(&req.connection_id, &req.database);
|
||||
if let Err(error) = state.app.storage.delete_schema_cache_prefix(&prefix).await {
|
||||
tracing::warn!(connection_id = %req.connection_id, database = %req.database, %error, "failed to invalidate schema metadata cache after deploy");
|
||||
}
|
||||
}
|
||||
Ok(Json(result))
|
||||
}
|
||||
|
||||
fn schema_diff_deploy_may_have_changed_schema(result: &dbx_core::query::SchemaDiffDeployResult) -> bool {
|
||||
result.statement_count > 0 && matches!(result.status.as_str(), "committed" | "mixed")
|
||||
}
|
||||
|
||||
pub async fn analyze_sql_references(
|
||||
Json(req): Json<AnalyzeSqlReferencesRequest>,
|
||||
) -> Result<Json<dbx_core::sql_analysis::SqlReferenceAnalysis>, AppError> {
|
||||
@@ -1158,6 +1168,28 @@ mod tests {
|
||||
(state, dir)
|
||||
}
|
||||
|
||||
fn deploy_result(status: &str, statement_count: usize) -> dbx_core::query::SchemaDiffDeployResult {
|
||||
dbx_core::query::SchemaDiffDeployResult {
|
||||
transaction_id: "deploy-test".to_string(),
|
||||
status: status.to_string(),
|
||||
participants: Vec::new(),
|
||||
created_at: String::new(),
|
||||
updated_at: String::new(),
|
||||
executed_count: 0,
|
||||
statement_count,
|
||||
error: None,
|
||||
metadata: serde_json::Value::Null,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn schema_diff_cache_invalidation_covers_committed_and_partial_deploys() {
|
||||
assert!(schema_diff_deploy_may_have_changed_schema(&deploy_result("committed", 1)));
|
||||
assert!(schema_diff_deploy_may_have_changed_schema(&deploy_result("mixed", 1)));
|
||||
assert!(!schema_diff_deploy_may_have_changed_schema(&deploy_result("rolled_back", 1)));
|
||||
assert!(!schema_diff_deploy_may_have_changed_schema(&deploy_result("committed", 0)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_script_with_2pc_returns_structured_result() {
|
||||
let (state, _dir) = test_web_state().await;
|
||||
|
||||
@@ -421,6 +421,15 @@ fn metadata_cache_key(
|
||||
.join(":")
|
||||
}
|
||||
|
||||
pub(crate) fn object_metadata_cache_prefix(connection_id: &str, database: &str) -> String {
|
||||
format!(
|
||||
"{}:{}:{}:",
|
||||
OBJECT_METADATA_CACHE_PREFIX,
|
||||
metadata_cache_segment(connection_id),
|
||||
metadata_cache_segment(database)
|
||||
)
|
||||
}
|
||||
|
||||
fn decode_metadata_cache<T: DeserializeOwned>(value: serde_json::Value) -> Option<T> {
|
||||
serde_json::from_value(value).ok()
|
||||
}
|
||||
@@ -856,6 +865,15 @@ mod tests {
|
||||
assert!(key.starts_with("object-meta:v1:conn%3A1:db%25%20name:sch%2Fema:%E8%A1%A8%3A%E5%90%8D:"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn database_metadata_cache_prefix_covers_backend_object_facets() {
|
||||
let prefix = object_metadata_cache_prefix("conn:1", "db% name");
|
||||
let key = metadata_cache_key("conn:1", "db% name", "public", "users", None, "backend-columns");
|
||||
|
||||
assert_eq!(prefix, "object-meta:v1:conn%3A1:db%25%20name:");
|
||||
assert!(key.starts_with(&prefix));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_cache_keys_isolate_catalogs_and_facets() {
|
||||
let iceberg_columns = metadata_cache_key("c1", "sales", "", "orders", Some("iceberg"), "backend-columns");
|
||||
|
||||
Reference in new issue
Block a user