fix(schema): converge mysql structure sync and refresh metadata

This commit is contained in:
zipg authored and GitHub committed 2026-08-26 12:30:10 +08:00
1 parent 43289e2ce1
commit c8c9870dba
3 files changed
+224 -4

No files matched your search

+174 -4
View File
@@ -6,11 +6,13 @@ use regex::Regex;
use serde::{Deserialize, Serialize};
use crate::models::connection::DatabaseType;
use crate::sql_dialect::ddl_profile::{profile_for, ColumnDdlSyntax, CommentDdlSyntax, DdlDialectProfile};
use crate::sql_dialect::ddl_profile::{
profile_for, AutoIncSyntax, ColumnDdlSyntax, CommentDdlSyntax, DdlDialectProfile,
};
use crate::sql_dialect::descriptor::DialectKind;
use crate::sql_dialect::inference::{ColumnType, DefaultTypeInferenceEngine, TypeInferenceEngine};
use crate::sql_dialect::type_rewrite::{
apply_auto_inc_to_column_def, rewrite_column_type, type_looks_integer, AutoIncColumnBuild,
apply_auto_inc_to_column_def, column_is_auto_increment, rewrite_column_type, type_looks_integer, AutoIncColumnBuild,
};
use crate::sql_parser::ast_filter::AstTransmitFilter;
use crate::types::{
@@ -2162,13 +2164,15 @@ fn diff_schema(options: &SchemaDiffPreparationOptions) -> Vec<TableDiff> {
for name in common {
let Some(source) = source_details.get(name.as_str()) else { continue };
let Some(target) = target_details.get(name.as_str()) else { continue };
let column_diffs = diff_columns_with_options(
let column_diffs = diff_columns_with_dialect_options(
&source.columns,
&target.columns,
options.ignore_comments,
options.compare_column_order,
options.detect_renames,
options.rename_threshold,
options.source_dialect,
options.target_dialect,
);
let index_diffs = diff_indexes(&source.indexes, &target.indexes);
let foreign_key_diffs = diff_foreign_keys(&source.foreign_keys, &target.foreign_keys);
@@ -2474,6 +2478,43 @@ pub fn diff_columns(source: &[ColumnInfo], target: &[ColumnInfo]) -> Vec<ColumnD
diff_columns_with_options(source, target, false, false, false, 0.5)
}
fn normalize_mysql_integer_type_for_comparison(data_type: &str) -> String {
let normalized = data_type.split_whitespace().collect::<Vec<_>>().join(" ").to_ascii_lowercase();
let Some(open) = normalized.find('(') else {
return normalized;
};
let Some(close) = normalized[open + 1..].find(')').map(|index| open + 1 + index) else {
return normalized;
};
let base = normalized[..open].trim();
let width = normalized[open + 1..close].trim();
let integer_type = matches!(base, "tinyint" | "smallint" | "mediumint" | "int" | "integer" | "bigint" | "year");
if !integer_type || width.is_empty() || !width.bytes().all(|byte| byte.is_ascii_digit()) {
return normalized;
}
let suffix = normalized[close + 1..].trim();
if suffix.is_empty() {
base.to_string()
} else {
format!("{base} {suffix}")
}
}
fn column_types_equal_for_dialects(
source_type: &str,
target_type: &str,
source_dialect: Option<DialectKind>,
target_dialect: Option<DialectKind>,
) -> bool {
if source_type.eq_ignore_ascii_case(target_type) {
return true;
}
source_dialect == Some(DialectKind::Mysql)
&& target_dialect == Some(DialectKind::Mysql)
&& normalize_mysql_integer_type_for_comparison(source_type)
== normalize_mysql_integer_type_for_comparison(target_type)
}
fn column_type_similarity_score(source_type: &str, target_type: &str) -> f64 {
let s = ColumnType::parse(source_type).base_type.to_ascii_lowercase();
let t = ColumnType::parse(target_type).base_type.to_ascii_lowercase();
@@ -2513,6 +2554,29 @@ fn diff_columns_with_options(
compare_column_order: bool,
detect_renames: bool,
rename_threshold: f64,
) -> Vec<ColumnDiff> {
diff_columns_with_dialect_options(
source,
target,
ignore_comments,
compare_column_order,
detect_renames,
rename_threshold,
None,
None,
)
}
#[allow(clippy::too_many_arguments)]
fn diff_columns_with_dialect_options(
source: &[ColumnInfo],
target: &[ColumnInfo],
ignore_comments: bool,
compare_column_order: bool,
detect_renames: bool,
rename_threshold: f64,
source_dialect: Option<DialectKind>,
target_dialect: Option<DialectKind>,
) -> Vec<ColumnDiff> {
let mut diffs = Vec::new();
let target_map: HashMap<&str, &ColumnInfo> = target.iter().map(|column| (column.name.as_str(), column)).collect();
@@ -2526,7 +2590,12 @@ fn diff_columns_with_options(
for (source_index, source_column) in source.iter().enumerate() {
if let Some(target_column) = target_map.get(source_column.name.as_str()) {
let mut changes = Vec::new();
if source_column.data_type.to_lowercase() != target_column.data_type.to_lowercase() {
if !column_types_equal_for_dialects(
&source_column.data_type,
&target_column.data_type,
source_dialect,
target_dialect,
) {
changes.push(format!("type: {} → {}", target_column.data_type, source_column.data_type));
}
if source_column.is_nullable != target_column.is_nullable {
@@ -3134,6 +3203,14 @@ fn column_def(col: &ColumnInfo, db_type: DatabaseType, source_dialect: Option<Di
)
));
}
// Suffix-style auto-increment is only valid in MySQL-family ALTER clauses
// (ADD/MODIFY/CHANGE). Other dialects' identity clauses are order-sensitive
// inside ADD COLUMN, so keep omitting them outside the MySQL family.
if column_is_auto_increment(col) && profile.alter_uses_modify_column {
if let AutoIncSyntax::Suffix(suffix) = profile.auto_inc {
definition.push_str(suffix);
}
}
if profile.inline_column_comment {
if let Some(comment) = &col.comment {
definition.push_str(&format!(" COMMENT {}", comment_literal(comment)));
@@ -7571,6 +7648,99 @@ mod tests {
assert_eq!(column_type_similarity_score("varchar(255)", "varchar(64)"), 1.0);
}
#[test]
fn mysql_same_dialect_ignores_only_integer_display_widths() {
let source = vec![
column("id", "int(11) unsigned", None),
column("status", "tinyint(4)", None),
column("amount", "decimal(10,2)", None),
column("name", "varchar(128)", None),
];
let target = vec![
column("id", "int unsigned", None),
column("status", "tinyint", None),
column("amount", "decimal(12,2)", None),
column("name", "varchar(64)", None),
];
let diffs = diff_columns_with_dialect_options(
&source,
&target,
false,
false,
false,
0.5,
Some(DialectKind::Mysql),
Some(DialectKind::Mysql),
);
assert_eq!(diffs.iter().map(|diff| diff.name.as_str()).collect::<Vec<_>>(), vec!["amount", "name"]);
assert!(diffs.iter().all(|diff| diff.changes.iter().any(|change| change.starts_with("type:"))));
}
#[test]
fn mysql_modify_column_preserves_explicit_auto_increment() {
let mut source = column("id", "int", Some("new comment"));
source.is_primary_key = true;
source.extra = Some("auto_increment".to_string());
let mut target = source.clone();
target.comment = Some("old comment".to_string());
let diff = ColumnDiff {
diff_type: "modified".to_string(),
name: "id".to_string(),
source: Some(source),
target: Some(target),
changes: vec!["comment: old comment → new comment".to_string()],
add_position: None,
};
let sql = gen_sql(wrap_table_diff("users", vec![diff]), DatabaseType::Mysql, Some(DialectKind::Mysql));
assert!(
sql.contains("MODIFY COLUMN `id` int NOT NULL AUTO_INCREMENT COMMENT 'new comment'"),
"MySQL MODIFY must preserve AUTO_INCREMENT: {sql}"
);
}
#[test]
fn mysql_add_column_keeps_auto_increment_suffix() {
let mut source = column("seq", "int", None);
source.extra = Some("auto_increment".to_string());
let diff = ColumnDiff {
diff_type: "added".to_string(),
name: "seq".to_string(),
source: Some(source),
target: None,
changes: vec![],
add_position: None,
};
let sql = gen_sql(wrap_table_diff("users", vec![diff]), DatabaseType::Mysql, Some(DialectKind::Mysql));
assert!(sql.contains("AUTO_INCREMENT"), "MySQL ADD COLUMN must keep AUTO_INCREMENT: {sql}");
}
#[test]
fn sqlserver_add_column_omits_identity_suffix() {
let mut source = column("seq", "int", None);
source.extra = Some("auto_increment".to_string());
let diff = ColumnDiff {
diff_type: "added".to_string(),
name: "seq".to_string(),
source: Some(source),
target: None,
changes: vec![],
add_position: None,
};
let sql = gen_sql(wrap_table_diff("users", vec![diff]), DatabaseType::SqlServer, Some(DialectKind::Mysql));
assert!(
!sql.to_uppercase().contains("IDENTITY("),
"SQL Server ADD COLUMN must not append an order-sensitive IDENTITY clause: {sql}"
);
}
// -- 32. Multiple renames in one table --
#[test]
fn multiple_renames_in_one_table() {
+32
View File
@@ -725,9 +725,19 @@ pub async fn execute_script_with_2pc(
req.destructive_confirmed.unwrap_or(false),
)
.await;
if schema_diff_deploy_may_have_changed_schema(&result) {
let prefix = super::schema::object_metadata_cache_prefix(&req.connection_id, &req.database);
if let Err(error) = state.app.storage.delete_schema_cache_prefix(&prefix).await {
tracing::warn!(connection_id = %req.connection_id, database = %req.database, %error, "failed to invalidate schema metadata cache after deploy");
}
}
Ok(Json(result))
}
fn schema_diff_deploy_may_have_changed_schema(result: &dbx_core::query::SchemaDiffDeployResult) -> bool {
result.statement_count > 0 && matches!(result.status.as_str(), "committed" | "mixed")
}
pub async fn analyze_sql_references(
Json(req): Json<AnalyzeSqlReferencesRequest>,
) -> Result<Json<dbx_core::sql_analysis::SqlReferenceAnalysis>, AppError> {
@@ -1158,6 +1168,28 @@ mod tests {
(state, dir)
}
fn deploy_result(status: &str, statement_count: usize) -> dbx_core::query::SchemaDiffDeployResult {
dbx_core::query::SchemaDiffDeployResult {
transaction_id: "deploy-test".to_string(),
status: status.to_string(),
participants: Vec::new(),
created_at: String::new(),
updated_at: String::new(),
executed_count: 0,
statement_count,
error: None,
metadata: serde_json::Value::Null,
}
}
#[test]
fn schema_diff_cache_invalidation_covers_committed_and_partial_deploys() {
assert!(schema_diff_deploy_may_have_changed_schema(&deploy_result("committed", 1)));
assert!(schema_diff_deploy_may_have_changed_schema(&deploy_result("mixed", 1)));
assert!(!schema_diff_deploy_may_have_changed_schema(&deploy_result("rolled_back", 1)));
assert!(!schema_diff_deploy_may_have_changed_schema(&deploy_result("committed", 0)));
}
#[tokio::test]
async fn execute_script_with_2pc_returns_structured_result() {
let (state, _dir) = test_web_state().await;
+18
View File
@@ -421,6 +421,15 @@ fn metadata_cache_key(
.join(":")
}
pub(crate) fn object_metadata_cache_prefix(connection_id: &str, database: &str) -> String {
format!(
"{}:{}:{}:",
OBJECT_METADATA_CACHE_PREFIX,
metadata_cache_segment(connection_id),
metadata_cache_segment(database)
)
}
fn decode_metadata_cache<T: DeserializeOwned>(value: serde_json::Value) -> Option<T> {
serde_json::from_value(value).ok()
}
@@ -856,6 +865,15 @@ mod tests {
assert!(key.starts_with("object-meta:v1:conn%3A1:db%25%20name:sch%2Fema:%E8%A1%A8%3A%E5%90%8D:"));
}
#[test]
fn database_metadata_cache_prefix_covers_backend_object_facets() {
let prefix = object_metadata_cache_prefix("conn:1", "db% name");
let key = metadata_cache_key("conn:1", "db% name", "public", "users", None, "backend-columns");
assert_eq!(prefix, "object-meta:v1:conn%3A1:db%25%20name:");
assert!(key.starts_with(&prefix));
}
#[test]
fn metadata_cache_keys_isolate_catalogs_and_facets() {
let iceberg_columns = metadata_cache_key("c1", "sales", "", "orders", Some("iceberg"), "backend-columns");