feat(schema-diff): preserve dialect-specific DDL semantics

This commit is contained in:
dbin0123 authored and GitHub committed 2026-07-29 03:47:19 +08:00
1 parent 820b8cd26f
commit 002a2c85c0
146 files changed
+36532 -1375

No files matched your search

+5
View File
@@ -98,4 +98,9 @@ zip = { version = "4", default-features = false, features = ["deflate"] }
flate2 = "1"
tar = "0.4"
sysinfo = { version = "0.32", features = ["system"] }
serde_yaml = "0.9"
minijinja = "2"
notify = { version = "7", default-features = false, features = ["macos_kqueue"] }
arc-swap = "1"
insta = { version = "1", features = ["json", "glob"] }
tempfile = "3"
+51
View File
@@ -0,0 +1,51 @@
use std::env;
use std::path::Path;
fn main() {
let cargo_manifest_dir_str = env::var("CARGO_MANIFEST_DIR").unwrap();
let cargo_manifest_dir = Path::new(&cargo_manifest_dir_str);
let dialects_dir = cargo_manifest_dir.join("..").join("..").join("plugins").join("dialects");
let dialects_dir = std::fs::canonicalize(&dialects_dir).unwrap_or(dialects_dir);
let out_dir_str = env::var("OUT_DIR").unwrap();
let dest_path = Path::new(&out_dir_str).join("core_dialects.rs");
let mut entries: Vec<_> = std::fs::read_dir(&dialects_dir)
.expect("Cannot read plugins/dialects directory")
.filter_map(|e| e.ok())
.filter(|e| e.path().extension().is_some_and(|ext| ext == "yaml" || ext == "yml"))
.collect();
entries.sort_by_key(|e| e.file_name());
// Watch the directory itself so additions/removals of dialect files trigger a rebuild.
println!("cargo::rerun-if-changed={}", dialects_dir.to_str().unwrap());
let mut code = String::from("{\n");
for entry in &entries {
let path = entry.path();
let canonical = std::fs::canonicalize(&path).unwrap_or_else(|_| path.clone());
let file_name = path.file_stem().unwrap().to_str().unwrap();
let path_str = canonical.to_str().unwrap();
// Watch each dialect file individually. Editing a single YAML must invalidate
// the embedded `core_dialects.rs`, otherwise the compiled binary keeps a stale
// type catalog (e.g. old type names) and silently misbehaves (see field mapping).
println!("cargo::rerun-if-changed={}", path_str);
code.push_str("match crate::sql_dialect::dialect_loader::DialectPluginLoader::load_from_string(\n");
code.push_str(&format!(" include_str!(\"{}\"),\n", path_str.replace('\\', "\\\\")));
code.push_str(" None,\n");
code.push_str(") {\n");
code.push_str(" Ok((_kind, yaml, descriptor)) => {\n");
code.push_str(" let name = yaml.dialect.name.clone();\n");
code.push_str(" registry.register_descriptor(&name, descriptor, yaml);\n");
code.push_str(" }\n");
code.push_str(&format!(" Err(e) => log::warn!(\"Failed to load core dialect '{}': {{e}}\"),\n", file_name));
code.push_str("};\n");
}
code.push_str("}\n");
std::fs::write(&dest_path, code).expect("Failed to write core_dialects.rs");
}
+307
View File
@@ -0,0 +1,307 @@
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum Expression {
EnvVar(String),
Ref(String),
Eval(String),
Literal(String),
}
pub fn parse_expression(input: &str) -> Expression {
let trimmed = input.trim();
if trimmed.starts_with("${env:") && trimmed.ends_with('}') {
let inner = &trimmed[6..trimmed.len() - 1];
Expression::EnvVar(inner.to_string())
} else if trimmed.starts_with("${ref:") && trimmed.ends_with('}') {
let inner = &trimmed[6..trimmed.len() - 1];
Expression::Ref(inner.to_string())
} else if trimmed.starts_with("${eval:") && trimmed.ends_with('}') {
let inner = &trimmed[7..trimmed.len() - 1];
Expression::Eval(inner.to_string())
} else {
Expression::Literal(input.to_string())
}
}
pub fn resolve_env_var(name: &str) -> Result<String, String> {
std::env::var(name).map_err(|_| format!("Environment variable '${name}' not set"))
}
pub fn resolve_ref(path: &str, merged: &HashMap<String, serde_json::Value>) -> Result<serde_json::Value, String> {
let parts: Vec<&str> = path.split('.').collect();
let key = parts[0];
let value = merged.get(key).ok_or_else(|| format!("Config reference '${path}' not found in merged config"))?;
if parts.len() == 1 {
return Ok(value.clone());
}
let mut current = value;
for part in &parts[1..] {
match current {
serde_json::Value::Object(map) => {
current =
map.get(*part).ok_or_else(|| format!("Config reference '${path}': key '{part}' not found"))?;
}
_ => {
return Err(format!("Config reference '${path}': intermediate value is not an object"));
}
}
}
Ok(current.clone())
}
/// Parses JSON literals only (numbers, booleans, null, quoted strings).
/// Does NOT support arithmetic, operators, or function calls.
fn eval_simple(expr: &str) -> Result<serde_json::Value, String> {
let expr = expr.trim();
if let Ok(v) = expr.parse::<i64>() {
return Ok(serde_json::Value::Number(v.into()));
}
if let Ok(v) = expr.parse::<f64>() {
if let Some(n) = serde_json::Number::from_f64(v) {
return Ok(serde_json::Value::Number(n));
}
}
if expr == "true" {
return Ok(serde_json::Value::Bool(true));
}
if expr == "false" {
return Ok(serde_json::Value::Bool(false));
}
if expr == "null" {
return Ok(serde_json::Value::Null);
}
if expr.starts_with('"') && expr.ends_with('"') && expr.len() >= 2 {
return Ok(serde_json::Value::String(expr[1..expr.len() - 1].to_string()));
}
Err(format!("Cannot evaluate expression: '{expr}'"))
}
pub fn resolve_expression(
expression: &Expression,
merged: &HashMap<String, serde_json::Value>,
) -> Result<serde_json::Value, String> {
match expression {
Expression::EnvVar(name) => {
let val = resolve_env_var(name)?;
Ok(serde_json::Value::String(val))
}
Expression::Ref(path) => resolve_ref(path, merged),
Expression::Eval(expr) => eval_simple(expr),
Expression::Literal(val) => Ok(serde_json::Value::String(val.clone())),
}
}
pub fn resolve_all_expressions_in_value(
value: &serde_json::Value,
merged: &HashMap<String, serde_json::Value>,
) -> Result<serde_json::Value, String> {
match value {
serde_json::Value::String(s) => {
let expr = parse_expression(s);
resolve_expression(&expr, merged)
}
serde_json::Value::Object(map) => {
let mut resolved = serde_json::Map::new();
for (k, v) in map {
resolved.insert(k.clone(), resolve_all_expressions_in_value(v, merged)?);
}
Ok(serde_json::Value::Object(resolved))
}
serde_json::Value::Array(arr) => {
let mut resolved = Vec::new();
for v in arr {
resolved.push(resolve_all_expressions_in_value(v, merged)?);
}
Ok(serde_json::Value::Array(resolved))
}
other => Ok(other.clone()),
}
}
pub fn apply_expression_resolution(
config: &mut HashMap<String, serde_json::Value>,
merged: &HashMap<String, serde_json::Value>,
) -> Result<(), String> {
let mut resolved = HashMap::new();
for (k, v) in config.iter() {
resolved.insert(k.clone(), resolve_all_expressions_in_value(v, merged)?);
}
config.extend(resolved);
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_env_var_expression() {
assert_eq!(parse_expression("${env:HOME}"), Expression::EnvVar("HOME".to_string()));
}
#[test]
fn test_parse_ref_expression() {
assert_eq!(parse_expression("${ref:database.host}"), Expression::Ref("database.host".to_string()));
}
#[test]
fn test_parse_eval_expression() {
assert_eq!(parse_expression("${eval:42}"), Expression::Eval("42".to_string()));
}
#[test]
fn test_parse_literal() {
assert_eq!(parse_expression("hello"), Expression::Literal("hello".to_string()));
assert_eq!(parse_expression("${not-an-expr}"), Expression::Literal("${not-an-expr}".to_string()));
}
#[test]
fn test_resolve_env_var() {
std::env::set_var("DBX_TEST_VAR", "test_value");
let result = resolve_env_var("DBX_TEST_VAR");
assert_eq!(result.unwrap(), "test_value");
std::env::remove_var("DBX_TEST_VAR");
}
#[test]
fn test_resolve_env_var_missing() {
let result = resolve_env_var("DBX_NONEXISTENT_VAR_12345");
assert!(result.is_err());
}
#[test]
fn test_resolve_ref_simple() {
let mut merged = HashMap::new();
merged.insert("host".to_string(), serde_json::Value::String("localhost".to_string()));
let result = resolve_ref("host", &merged).unwrap();
assert_eq!(result, serde_json::Value::String("localhost".to_string()));
}
#[test]
fn test_resolve_ref_nested() {
let mut merged = HashMap::new();
let mut db = serde_json::Map::new();
db.insert("host".to_string(), serde_json::Value::String("pg.example.com".to_string()));
db.insert("port".to_string(), serde_json::Value::Number(serde_json::Number::from(5432)));
merged.insert("database".to_string(), serde_json::Value::Object(db));
let host = resolve_ref("database.host", &merged).unwrap();
assert_eq!(host, serde_json::Value::String("pg.example.com".to_string()));
let port = resolve_ref("database.port", &merged).unwrap();
assert_eq!(port, serde_json::Value::Number(serde_json::Number::from(5432)));
}
#[test]
fn test_resolve_ref_missing() {
let merged = HashMap::new();
let result = resolve_ref("nonexistent", &merged);
assert!(result.is_err());
}
#[test]
fn test_eval_simple_integer() {
let result = eval_simple("42").unwrap();
assert_eq!(result, serde_json::Value::Number(serde_json::Number::from(42)));
}
#[test]
fn test_eval_simple_bool() {
assert_eq!(eval_simple("true").unwrap(), serde_json::Value::Bool(true));
assert_eq!(eval_simple("false").unwrap(), serde_json::Value::Bool(false));
}
#[test]
fn test_eval_simple_null() {
assert_eq!(eval_simple("null").unwrap(), serde_json::Value::Null);
}
#[test]
fn test_eval_simple_string() {
let result = eval_simple(r#""hello world""#).unwrap();
assert_eq!(result, serde_json::Value::String("hello world".to_string()));
}
#[test]
fn test_eval_simple_invalid() {
assert!(eval_simple("some_random_text").is_err());
}
#[test]
fn test_resolve_expression_env_var() {
std::env::set_var("DBX_TEST_PORT", "8080");
let expr = Expression::EnvVar("DBX_TEST_PORT".to_string());
let merged = HashMap::new();
let result = resolve_expression(&expr, &merged).unwrap();
assert_eq!(result, serde_json::Value::String("8080".to_string()));
std::env::remove_var("DBX_TEST_PORT");
}
#[test]
fn test_resolve_expression_ref() {
let mut merged = HashMap::new();
merged.insert("host".to_string(), serde_json::Value::String("db.local".to_string()));
let expr = Expression::Ref("host".to_string());
let result = resolve_expression(&expr, &merged).unwrap();
assert_eq!(result, serde_json::Value::String("db.local".to_string()));
}
#[test]
fn test_resolve_expression_eval() {
let expr = Expression::Eval("true".to_string());
let merged = HashMap::new();
let result = resolve_expression(&expr, &merged).unwrap();
assert_eq!(result, serde_json::Value::Bool(true));
}
#[test]
fn test_resolve_expression_literal() {
let expr = Expression::Literal("hello".to_string());
let merged = HashMap::new();
let result = resolve_expression(&expr, &merged).unwrap();
assert_eq!(result, serde_json::Value::String("hello".to_string()));
}
#[test]
fn test_resolve_all_in_nested_object() {
let mut merged = HashMap::new();
merged.insert("default_host".to_string(), serde_json::Value::String("pg.example.com".to_string()));
let mut obj = serde_json::Map::new();
obj.insert("host".to_string(), serde_json::Value::String("${ref:default_host}".to_string()));
obj.insert("port".to_string(), serde_json::Value::Number(serde_json::Number::from(5432)));
obj.insert("debug".to_string(), serde_json::Value::String("${eval:true}".to_string()));
let input = serde_json::Value::Object(obj);
let result = resolve_all_expressions_in_value(&input, &merged).unwrap();
let obj = result.as_object().unwrap();
assert_eq!(obj.get("host").unwrap(), &serde_json::Value::String("pg.example.com".to_string()));
assert_eq!(obj.get("port").unwrap(), &serde_json::Value::Number(serde_json::Number::from(5432)));
assert_eq!(obj.get("debug").unwrap(), &serde_json::Value::Bool(true));
}
#[test]
fn test_apply_expression_resolution() {
let mut merged = HashMap::new();
merged.insert("base_url".to_string(), serde_json::Value::String("https://api.example.com".to_string()));
let mut config = HashMap::new();
config.insert("url".to_string(), serde_json::Value::String("${ref:base_url}".to_string()));
config.insert("timeout".to_string(), serde_json::Value::String("${eval:30}".to_string()));
apply_expression_resolution(&mut config, &merged).unwrap();
assert_eq!(config.get("url").unwrap(), &serde_json::Value::String("https://api.example.com".to_string()));
assert_eq!(config.get("timeout").unwrap(), &serde_json::Value::Number(serde_json::Number::from(30)));
}
}
+979
View File
@@ -0,0 +1,979 @@
use arc_swap::ArcSwap;
use chrono::Utc;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::HashMap;
use std::sync::Arc;
use crate::config::layer::ConfigTree;
#[cfg(test)]
use crate::config::layer::LayerConfig;
use crate::storage::Storage;
fn hex_encode(bytes: &[u8]) -> String {
bytes.iter().map(|b| format!("{b:02x}")).collect()
}
// ---------------------------------------------------------------------------
// 16.1 — Config Change Audit & Version Management
// ---------------------------------------------------------------------------
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ConfigAuditEntry {
pub id: String,
pub timestamp: String,
pub operator: String,
pub reason: String,
pub key_path: String,
pub change_diff: serde_json::Value,
pub config_snapshot: serde_json::Value,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ConfigVersionSnapshot {
pub id: String,
pub key_path: String,
pub version: u64,
pub snapshot_json: serde_json::Value,
pub checksum: String,
pub created_at: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AuditQuery {
pub key_path: Option<String>,
pub operator: Option<String>,
pub limit: Option<usize>,
pub offset: Option<usize>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AuditSummary {
pub total_entries: usize,
pub entries: Vec<ConfigAuditEntry>,
}
pub struct ConfigAuditor {
storage: Arc<Storage>,
}
impl ConfigAuditor {
pub fn new(storage: Arc<Storage>) -> Self {
Self { storage }
}
pub async fn record_change(
&self,
operator: &str,
reason: &str,
key_path: &str,
change_diff: serde_json::Value,
config_snapshot: serde_json::Value,
) -> Result<ConfigAuditEntry, String> {
let entry = ConfigAuditEntry {
id: uuid::Uuid::new_v4().to_string(),
timestamp: Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string(),
operator: operator.to_string(),
reason: reason.to_string(),
key_path: key_path.to_string(),
change_diff,
config_snapshot,
};
self.storage.save_audit_entry(&entry).await?;
Ok(entry)
}
pub async fn query_history(&self, query: &AuditQuery) -> Result<AuditSummary, String> {
self.storage.query_audit_entries(query).await
}
pub async fn save_snapshot(&self, key_path: &str, tree: &ConfigTree) -> Result<ConfigVersionSnapshot, String> {
let snapshot_json = serde_json::to_value(tree).map_err(|e| format!("serialize config tree: {e}"))?;
let json_bytes = serde_json::to_vec(&snapshot_json).map_err(|e| format!("json bytes: {e}"))?;
let checksum = hex_encode(&Sha256::digest(&json_bytes));
let version = self.storage.next_config_version(key_path).await?;
let snap = ConfigVersionSnapshot {
id: uuid::Uuid::new_v4().to_string(),
key_path: key_path.to_string(),
version,
snapshot_json: snapshot_json.clone(),
checksum,
created_at: Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string(),
};
self.storage.save_config_snapshot(&snap).await?;
Ok(snap)
}
pub async fn get_snapshot(&self, key_path: &str, version: u64) -> Result<Option<ConfigVersionSnapshot>, String> {
self.storage.load_config_snapshot(key_path, version).await
}
pub async fn list_versions(&self, key_path: &str) -> Result<Vec<ConfigVersionSnapshot>, String> {
self.storage.list_config_snapshots(key_path).await
}
pub async fn rollback(
&self,
key_path: &str,
version: u64,
operator: &str,
reason: &str,
) -> Result<ConfigTree, String> {
let snap = self
.get_snapshot(key_path, version)
.await?
.ok_or_else(|| format!("version {version} not found for key_path '{key_path}'"))?;
let tree: ConfigTree =
serde_json::from_value(snap.snapshot_json.clone()).map_err(|e| format!("deserialize config tree: {e}"))?;
let current_json = serde_json::to_value(&tree).map_err(|e| format!("serialize current: {e}"))?;
self.record_change(
operator,
&format!("rollback: {reason}"),
key_path,
serde_json::json!({ "rollback_to_version": version }),
current_json,
)
.await?;
Ok(tree)
}
}
// ---------------------------------------------------------------------------
// 16.2 — Approval Workflow Integration
// ---------------------------------------------------------------------------
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ApprovalStatus {
Draft,
PendingApproval,
Approved,
Rejected,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ApprovalRecord {
pub id: String,
pub config_domain: String,
pub change_description: String,
pub status: ApprovalStatus,
pub requester: String,
pub reviewer: Option<String>,
pub reviewed_at: Option<String>,
pub webhook_url: Option<String>,
pub draft_config_json: serde_json::Value,
pub created_at: String,
pub updated_at: String,
}
const SENSITIVE_DOMAINS: &[&str] = &["mapping_rules", "allow_destructive", "dangerous_ddl_policy"];
pub fn is_sensitive_domain(domain: &str) -> bool {
SENSITIVE_DOMAINS.contains(&domain)
}
pub fn requires_approval_for_change(domain: &str, current_status: ApprovalStatus) -> bool {
if !is_sensitive_domain(domain) {
return false;
}
matches!(current_status, ApprovalStatus::Draft | ApprovalStatus::Rejected)
}
pub struct ConfigApproval {
storage: Arc<Storage>,
}
impl ConfigApproval {
pub fn new(storage: Arc<Storage>) -> Self {
Self { storage }
}
pub async fn submit_change(
&self,
domain: &str,
description: &str,
requester: &str,
draft_config: serde_json::Value,
webhook_url: Option<&str>,
) -> Result<ApprovalRecord, String> {
let now = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();
let record = ApprovalRecord {
id: uuid::Uuid::new_v4().to_string(),
config_domain: domain.to_string(),
change_description: description.to_string(),
status: if is_sensitive_domain(domain) {
ApprovalStatus::PendingApproval
} else {
ApprovalStatus::Approved
},
requester: requester.to_string(),
reviewer: None,
reviewed_at: None,
webhook_url: webhook_url.map(String::from),
draft_config_json: draft_config,
created_at: now.clone(),
updated_at: now,
};
self.storage.save_approval_record(&record).await?;
Ok(record)
}
pub async fn approve(&self, id: &str, reviewer: &str) -> Result<ApprovalRecord, String> {
let mut record =
self.storage.load_approval_record(id).await?.ok_or_else(|| format!("approval record {id} not found"))?;
if record.status != ApprovalStatus::PendingApproval {
return Err(format!("record {id} is not in PendingApproval state (current: {:?})", record.status));
}
record.status = ApprovalStatus::Approved;
record.reviewer = Some(reviewer.to_string());
record.reviewed_at = Some(Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string());
record.updated_at = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();
self.storage.save_approval_record(&record).await?;
Ok(record)
}
pub async fn reject(&self, id: &str, reviewer: &str) -> Result<ApprovalRecord, String> {
let mut record =
self.storage.load_approval_record(id).await?.ok_or_else(|| format!("approval record {id} not found"))?;
if record.status != ApprovalStatus::PendingApproval {
return Err(format!("record {id} is not in PendingApproval state"));
}
record.status = ApprovalStatus::Rejected;
record.reviewer = Some(reviewer.to_string());
record.reviewed_at = Some(Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string());
record.updated_at = Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string();
self.storage.save_approval_record(&record).await?;
Ok(record)
}
pub async fn list_pending(&self) -> Result<Vec<ApprovalRecord>, String> {
self.storage.query_approval_records(Some(ApprovalStatus::PendingApproval)).await
}
pub async fn check_effective(&self, domain: &str) -> Result<bool, String> {
if !is_sensitive_domain(domain) {
return Ok(true);
}
let records = self.storage.query_approval_records_by_domain(domain).await?;
let latest = records.into_iter().last();
match latest {
Some(r) => Ok(r.status == ApprovalStatus::Approved),
None => Ok(false),
}
}
}
// ---------------------------------------------------------------------------
// 16.3 — Cross-Environment Config Drift Detection
// ---------------------------------------------------------------------------
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ConfigChecksum {
pub env: String,
pub key_path: String,
pub checksum: String,
pub computed_at: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DriftReport {
pub source_env: String,
pub target_env: String,
pub key_path: String,
pub source_checksum: String,
pub target_checksum: String,
pub mismatched_fields: Vec<String>,
pub source_changed_at: Option<String>,
pub detected_at: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DriftAlert {
pub id: String,
pub source_env: String,
pub target_env: String,
pub config_key: String,
pub expected_checksum: String,
pub actual_checksum: String,
pub details_json: serde_json::Value,
pub detected_at: String,
pub acknowledged: bool,
}
pub fn compute_config_checksum(tree: &ConfigTree) -> String {
let json = serde_json::to_vec(tree).unwrap_or_default();
hex_encode(&Sha256::digest(&json))
}
pub fn compute_config_checksum_for_layer(tree: &ConfigTree, key_path: &str) -> Option<String> {
let merged = tree.merge().ok()?;
let relevant: HashMap<String, serde_json::Value> =
merged.values.into_iter().filter(|(k, _)| k.starts_with(key_path)).collect();
if relevant.is_empty() {
return None;
}
let json = serde_json::to_vec(&relevant).ok()?;
Some(hex_encode(&Sha256::digest(&json)))
}
pub fn detect_drift(source: &ConfigTree, target: &ConfigTree, key_path: &str) -> Option<DriftReport> {
let source_merged = source.merge().ok()?;
let target_merged = target.merge().ok()?;
let source_vals: HashMap<&String, &serde_json::Value> =
source_merged.values.iter().filter(|(k, _)| k.starts_with(key_path)).collect();
let target_vals: HashMap<&String, &serde_json::Value> =
target_merged.values.iter().filter(|(k, _)| k.starts_with(key_path)).collect();
let all_keys: std::collections::BTreeSet<&String> = source_vals.keys().chain(target_vals.keys()).copied().collect();
let mismatched: Vec<String> = all_keys
.into_iter()
.filter(|k| match (source_vals.get(k), target_vals.get(k)) {
(Some(sv), Some(tv)) => sv != tv,
_ => true,
})
.map(|k| {
let s = source_vals.get(k).map(|v| v.to_string()).unwrap_or_default();
let t = target_vals.get(k).map(|v| v.to_string()).unwrap_or_default();
format!("{k}: source={s}, target={t}")
})
.collect();
if mismatched.is_empty() {
return None;
}
let source_ck = compute_config_checksum_for_layer(source, key_path).unwrap_or_default();
let target_ck = compute_config_checksum_for_layer(target, key_path).unwrap_or_default();
Some(DriftReport {
source_env: String::new(),
target_env: String::new(),
key_path: key_path.to_string(),
source_checksum: source_ck,
target_checksum: target_ck,
mismatched_fields: mismatched,
source_changed_at: None,
detected_at: Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string(),
})
}
pub struct DriftDetector {
storage: Arc<Storage>,
}
impl DriftDetector {
pub fn new(storage: Arc<Storage>) -> Self {
Self { storage }
}
pub fn compare_envs(
&self,
source_env: &str,
target_env: &str,
source_tree: &ConfigTree,
target_tree: &ConfigTree,
key_path: &str,
) -> Option<DriftReport> {
let mut report = detect_drift(source_tree, target_tree, key_path)?;
report.source_env = source_env.to_string();
report.target_env = target_env.to_string();
Some(report)
}
pub async fn record_alert(&self, alert: &DriftAlert) -> Result<(), String> {
self.storage.save_drift_alert(alert).await
}
pub async fn acknowledge_alert(&self, id: &str) -> Result<(), String> {
self.storage.acknowledge_drift_alert(id).await
}
pub async fn list_alerts(&self, acknowledged: Option<bool>) -> Result<Vec<DriftAlert>, String> {
self.storage.query_drift_alerts(acknowledged).await
}
}
// ---------------------------------------------------------------------------
// 16.4 — COW Snapshot (ArcSwap-based hot-reload)
// ---------------------------------------------------------------------------
pub struct ConfigSnapshot {
inner: ArcSwap<ConfigTree>,
key_path: String,
auditor: Arc<ConfigAuditor>,
}
impl ConfigSnapshot {
pub fn new(tree: ConfigTree, key_path: &str, auditor: Arc<ConfigAuditor>) -> Self {
Self { inner: ArcSwap::new(Arc::new(tree)), key_path: key_path.to_string(), auditor }
}
pub fn load(&self) -> Arc<ConfigTree> {
self.inner.load_full()
}
pub fn apply(
&self,
operator: &str,
reason: &str,
f: impl FnOnce(&mut ConfigTree),
) -> Result<Arc<ConfigTree>, String>
where
Self: Sized,
{
let prev = self.inner.load_full();
let mut new_tree = (*prev).clone();
f(&mut new_tree);
let prev_json = serde_json::to_value(&*prev).map_err(|e| format!("serialize prev: {e}"))?;
let new_json = serde_json::to_value(&new_tree).map_err(|e| format!("serialize new: {e}"))?;
let diff = serde_json::json!({
"prev": prev_json,
"new": new_json,
});
let new_arc = Arc::new(new_tree);
self.inner.store(new_arc.clone());
let auditor = self.auditor.clone();
let key_path = self.key_path.clone();
let operator = operator.to_string();
let reason = reason.to_string();
tokio::spawn(async move {
let _ = auditor.record_change(&operator, &reason, &key_path, diff, new_json).await;
});
Ok(new_arc)
}
pub fn apply_silent(&self, f: impl FnOnce(&mut ConfigTree)) -> Arc<ConfigTree> {
let prev = self.inner.load_full();
let mut new_tree = (*prev).clone();
f(&mut new_tree);
let new_arc = Arc::new(new_tree);
self.inner.store(new_arc.clone());
new_arc
}
}
// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------
#[cfg(test)]
mod tests {
use super::*;
use crate::config::layer::ConfigLayer;
use std::sync::Arc;
async fn make_test_storage() -> Arc<Storage> {
let tmp = std::env::temp_dir().join(uuid::Uuid::new_v4().to_string());
std::fs::create_dir_all(&tmp).unwrap();
let db_path = tmp.join("test_governance.db");
let storage = Arc::new(Storage::open(&db_path).await.unwrap());
std::mem::forget(tmp);
storage
}
fn make_test_tree() -> ConfigTree {
let mut tree = ConfigTree::new();
tree.add_layer(LayerConfig {
layer: ConfigLayer::Global,
name: "test".to_string(),
values: HashMap::from([
("host".to_string(), serde_json::Value::String("db.example.com".to_string())),
("port".to_string(), serde_json::json!(5432)),
]),
enabled: true,
..Default::default()
});
tree
}
// ---- 16.1 Audit Tests ----
#[tokio::test]
async fn test_audit_record_and_query() {
let storage = make_test_storage().await;
let auditor = ConfigAuditor::new(storage.clone());
auditor
.record_change(
"alice",
"update host",
"database.host",
serde_json::json!({"old": "old.host.com", "new": "new.host.com"}),
serde_json::json!({"host": "new.host.com"}),
)
.await
.unwrap();
let summary = auditor
.query_history(&AuditQuery { key_path: None, operator: None, limit: None, offset: None })
.await
.unwrap();
assert_eq!(summary.total_entries, 1);
assert_eq!(summary.entries[0].operator, "alice");
assert_eq!(summary.entries[0].key_path, "database.host");
let filtered = auditor
.query_history(&AuditQuery {
key_path: Some("database.host".into()),
operator: Some("alice".into()),
limit: Some(10),
offset: Some(0),
})
.await
.unwrap();
assert_eq!(filtered.total_entries, 1);
let empty = auditor
.query_history(&AuditQuery {
key_path: Some("other.key".into()),
operator: None,
limit: None,
offset: None,
})
.await
.unwrap();
assert_eq!(empty.total_entries, 0);
}
#[tokio::test]
async fn test_audit_multiple_entries() {
let storage = make_test_storage().await;
let auditor = ConfigAuditor::new(storage.clone());
for i in 0..3 {
auditor
.record_change(
&format!("user{i}"),
&format!("change {i}"),
"app.config",
serde_json::json!({"index": i}),
serde_json::json!({"version": i}),
)
.await
.unwrap();
}
let summary = auditor
.query_history(&AuditQuery { key_path: None, operator: None, limit: Some(2), offset: None })
.await
.unwrap();
assert_eq!(summary.total_entries, 3);
assert_eq!(summary.entries.len(), 2);
}
// ---- 16.1 Snapshot Tests ----
#[tokio::test]
async fn test_snapshot_save_and_rollback() {
let storage = make_test_storage().await;
let auditor = ConfigAuditor::new(storage.clone());
let tree = make_test_tree();
let snap = auditor.save_snapshot("app.config", &tree).await.unwrap();
assert_eq!(snap.version, 1);
assert!(!snap.checksum.is_empty());
let loaded = auditor.get_snapshot("app.config", 1).await.unwrap().unwrap();
assert_eq!(loaded.checksum, snap.checksum);
let versions = auditor.list_versions("app.config").await.unwrap();
assert_eq!(versions.len(), 1);
}
#[tokio::test]
async fn test_snapshot_version_increment() {
let storage = make_test_storage().await;
let auditor = ConfigAuditor::new(storage.clone());
let tree = make_test_tree();
let v1 = auditor.save_snapshot("app.config", &tree).await.unwrap();
assert_eq!(v1.version, 1);
let v2 = auditor.save_snapshot("app.config", &tree).await.unwrap();
assert_eq!(v2.version, 2);
let versions = auditor.list_versions("app.config").await.unwrap();
assert_eq!(versions.len(), 2);
}
#[tokio::test]
async fn test_rollback_restores_snapshot() {
let storage = make_test_storage().await;
let auditor = ConfigAuditor::new(storage.clone());
let mut tree = make_test_tree();
let snap1 = auditor.save_snapshot("app.config", &tree).await.unwrap();
tree.add_layer(LayerConfig {
layer: ConfigLayer::Task,
name: "override".to_string(),
values: HashMap::from([("host".to_string(), serde_json::Value::String("override.host.com".to_string()))]),
enabled: true,
..Default::default()
});
auditor.save_snapshot("app.config", &tree).await.unwrap();
let rolled = auditor.rollback("app.config", 1, "bob", "bad override").await.unwrap();
let merged = rolled.merge().unwrap();
assert_eq!(merged.values.get("host").unwrap(), &serde_json::Value::String("db.example.com".to_string()));
let history = auditor
.query_history(&AuditQuery {
key_path: Some("app.config".into()),
operator: None,
limit: None,
offset: None,
})
.await
.unwrap();
let rollback_entries: Vec<_> = history.entries.iter().filter(|e| e.reason.contains("rollback")).collect();
assert_eq!(rollback_entries.len(), 1);
}
#[tokio::test]
async fn test_rollback_nonexistent_version() {
let storage = make_test_storage().await;
let auditor = ConfigAuditor::new(storage.clone());
let result = auditor.rollback("app.config", 99, "alice", "test").await;
assert!(result.is_err());
}
// ---- 16.2 Approval Tests ----
#[test]
fn test_sensitive_domain_check() {
assert!(is_sensitive_domain("mapping_rules"));
assert!(is_sensitive_domain("allow_destructive"));
assert!(is_sensitive_domain("dangerous_ddl_policy"));
assert!(!is_sensitive_domain("theme"));
assert!(!is_sensitive_domain("log_level"));
}
#[test]
fn test_requires_approval_logic() {
assert!(requires_approval_for_change("mapping_rules", ApprovalStatus::Draft));
assert!(requires_approval_for_change("allow_destructive", ApprovalStatus::Rejected));
assert!(!requires_approval_for_change("allow_destructive", ApprovalStatus::Approved));
assert!(!requires_approval_for_change("theme", ApprovalStatus::Draft));
}
#[tokio::test]
async fn test_submit_change_sensitive_domain() {
let storage = make_test_storage().await;
let approval = ConfigApproval::new(storage.clone());
let record = approval
.submit_change(
"allow_destructive",
"enable drop table",
"alice",
serde_json::json!({"enabled": true}),
None,
)
.await
.unwrap();
assert_eq!(record.status, ApprovalStatus::PendingApproval);
assert_eq!(record.requester, "alice");
}
#[tokio::test]
async fn test_submit_change_non_sensitive_domain() {
let storage = make_test_storage().await;
let approval = ConfigApproval::new(storage.clone());
let record = approval
.submit_change("theme", "set dark mode", "alice", serde_json::json!({"theme": "dark"}), None)
.await
.unwrap();
assert_eq!(record.status, ApprovalStatus::Approved);
}
#[tokio::test]
async fn test_approve_workflow() {
let storage = make_test_storage().await;
let approval = ConfigApproval::new(storage.clone());
let record = approval
.submit_change(
"mapping_rules",
"add mysql mapping",
"alice",
serde_json::json!({"rule": "varchar->text"}),
None,
)
.await
.unwrap();
let approved = approval.approve(&record.id, "bob").await.unwrap();
assert_eq!(approved.status, ApprovalStatus::Approved);
assert_eq!(approved.reviewer.as_deref(), Some("bob"));
}
#[tokio::test]
async fn test_reject_workflow() {
let storage = make_test_storage().await;
let approval = ConfigApproval::new(storage.clone());
let record = approval
.submit_change(
"dangerous_ddl_policy",
"allow drop database",
"alice",
serde_json::json!({"allow": true}),
None,
)
.await
.unwrap();
let rejected = approval.reject(&record.id, "bob").await.unwrap();
assert_eq!(rejected.status, ApprovalStatus::Rejected);
}
#[tokio::test]
async fn test_approve_invalid_state() {
let storage = make_test_storage().await;
let approval = ConfigApproval::new(storage.clone());
let record = approval
.submit_change("theme", "dark mode", "alice", serde_json::json!({"theme": "dark"}), None)
.await
.unwrap();
assert!(record.status == ApprovalStatus::Approved);
let result = approval.approve(&record.id, "bob").await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_check_effective() {
let storage = make_test_storage().await;
let approval = ConfigApproval::new(storage.clone());
assert!(!approval.check_effective("allow_destructive").await.unwrap());
let record = approval
.submit_change("allow_destructive", "enable", "alice", serde_json::json!({"enabled": true}), None)
.await
.unwrap();
assert!(!approval.check_effective("allow_destructive").await.unwrap());
approval.approve(&record.id, "bob").await.unwrap();
assert!(approval.check_effective("allow_destructive").await.unwrap());
assert!(approval.check_effective("theme").await.unwrap());
}
#[tokio::test]
async fn test_list_pending() {
let storage = make_test_storage().await;
let approval = ConfigApproval::new(storage.clone());
approval.submit_change("mapping_rules", "rule1", "alice", serde_json::json!({"r": 1}), None).await.unwrap();
approval.submit_change("allow_destructive", "rule2", "bob", serde_json::json!({"r": 2}), None).await.unwrap();
let pending = approval.list_pending().await.unwrap();
assert_eq!(pending.len(), 2);
}
// ---- 16.3 Drift Detection Tests ----
#[test]
fn test_checksum_computation() {
let tree = make_test_tree();
let ck = compute_config_checksum(&tree);
assert_eq!(ck.len(), 64);
assert!(ck.chars().all(|c| c.is_ascii_hexdigit()));
}
#[test]
fn test_checksum_deterministic() {
let tree = make_test_tree();
let ck1 = compute_config_checksum(&tree);
let ck2 = compute_config_checksum(&tree);
assert_eq!(ck1, ck2);
}
#[test]
fn test_checksum_differs_on_change() {
let mut tree1 = make_test_tree();
let mut tree2 = make_test_tree();
tree2.add_layer(LayerConfig {
layer: ConfigLayer::Task,
name: "override".to_string(),
values: HashMap::from([("host".to_string(), serde_json::Value::String("other.com".to_string()))]),
enabled: true,
..Default::default()
});
assert_ne!(compute_config_checksum(&tree1), compute_config_checksum(&tree2));
}
#[test]
fn test_layer_checksum() {
let tree = make_test_tree();
let ck = compute_config_checksum_for_layer(&tree, "host").unwrap();
assert_eq!(ck.len(), 64);
let none = compute_config_checksum_for_layer(&tree, "nonexistent");
assert!(none.is_none());
}
#[test]
fn test_detect_drift_no_drift() {
let tree1 = make_test_tree();
let tree2 = make_test_tree();
assert!(detect_drift(&tree1, &tree2, "host").is_none());
}
#[test]
fn test_detect_drift_with_drift() {
let tree1 = make_test_tree();
let mut tree2 = make_test_tree();
tree2.add_layer(LayerConfig {
layer: ConfigLayer::Task,
name: "override".to_string(),
values: HashMap::from([("host".to_string(), serde_json::Value::String("other.com".to_string()))]),
enabled: true,
..Default::default()
});
let report = detect_drift(&tree1, &tree2, "host").unwrap();
assert_eq!(report.key_path, "host");
assert!(!report.mismatched_fields.is_empty());
}
#[test]
fn test_detect_drift_partial_keys() {
let tree1 = make_test_tree();
let tree2 = make_test_tree();
let report = detect_drift(&tree1, &tree2, "port");
assert!(report.is_none());
}
#[tokio::test]
async fn test_drift_alert_persistence() {
let storage = make_test_storage().await;
let detector = DriftDetector::new(storage.clone());
let alert = DriftAlert {
id: uuid::Uuid::new_v4().to_string(),
source_env: "prod".to_string(),
target_env: "staging".to_string(),
config_key: "database.host".to_string(),
expected_checksum: "abc".to_string(),
actual_checksum: "def".to_string(),
details_json: serde_json::json!({"field": "host", "expected": "prod-host", "actual": "staging-host"}),
detected_at: Utc::now().format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string(),
acknowledged: false,
};
detector.record_alert(&alert).await.unwrap();
let unacked = detector.list_alerts(Some(false)).await.unwrap();
assert_eq!(unacked.len(), 1);
detector.acknowledge_alert(&alert.id).await.unwrap();
let still_unacked = detector.list_alerts(Some(false)).await.unwrap();
assert_eq!(still_unacked.len(), 0);
let acked = detector.list_alerts(Some(true)).await.unwrap();
assert_eq!(acked.len(), 1);
}
// ---- 16.4 COW Snapshot Tests ----
#[test]
fn test_config_snapshot_load() {
let storage = make_test_storage_for_snapshot();
let auditor = Arc::new(ConfigAuditor::new(storage));
let tree = make_test_tree();
let snapshot = ConfigSnapshot::new(tree, "app.config", auditor);
let loaded = snapshot.load();
let merged = loaded.merge().unwrap();
assert_eq!(merged.values.get("host").unwrap(), &serde_json::Value::String("db.example.com".to_string()));
}
#[tokio::test]
async fn test_config_snapshot_apply_modification() {
let storage = make_test_storage().await;
let auditor = Arc::new(ConfigAuditor::new(storage));
let tree = make_test_tree();
let snapshot = ConfigSnapshot::new(tree, "app.config", auditor);
snapshot
.apply("alice", "update port", |t| {
for layer in &mut t.layers {
if layer.name == "test" {
layer.values.insert("port".to_string(), serde_json::json!(3306));
}
}
})
.unwrap();
let loaded = snapshot.load();
let merged = loaded.merge().unwrap();
assert_eq!(merged.values.get("port").unwrap(), &serde_json::json!(3306));
}
#[test]
fn test_config_snapshot_apply_silent_no_audit() {
let storage = make_test_storage_for_snapshot();
let auditor = Arc::new(ConfigAuditor::new(storage.clone()));
let tree = make_test_tree();
let snapshot = ConfigSnapshot::new(tree, "app.config", auditor);
snapshot.apply_silent(|t| {
for layer in &mut t.layers {
if layer.name == "test" {
layer.values.insert("host".to_string(), serde_json::Value::String("silent.host.com".to_string()));
}
}
});
let loaded = snapshot.load();
let merged = loaded.merge().unwrap();
assert_eq!(merged.values.get("host").unwrap(), &serde_json::Value::String("silent.host.com".to_string()));
}
#[test]
fn test_config_snapshot_concurrent_reads() {
use std::thread;
let storage = make_test_storage_for_snapshot();
let auditor = Arc::new(ConfigAuditor::new(storage));
let tree = make_test_tree();
let snapshot = Arc::new(ConfigSnapshot::new(tree, "app.config", auditor));
let mut handles = Vec::new();
for i in 0..10 {
let snap = snapshot.clone();
handles.push(thread::spawn(move || {
let loaded = snap.load();
let merged = loaded.merge().unwrap();
let port = merged.values.get("port").and_then(|v| v.as_i64()).unwrap_or(0);
assert_eq!(port, 5432);
}));
}
for h in handles {
h.join().unwrap();
}
}
fn make_test_storage_for_snapshot() -> Arc<Storage> {
let tmp = std::env::temp_dir().join(uuid::Uuid::new_v4().to_string());
std::fs::create_dir_all(&tmp).unwrap();
let db_path = tmp.join("test_snapshot.db");
let rt = tokio::runtime::Runtime::new().unwrap();
let storage = Arc::new(rt.block_on(Storage::open(&db_path)).unwrap());
std::mem::forget(tmp);
storage
}
}
+308
View File
@@ -0,0 +1,308 @@
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Hash)]
#[serde(rename_all = "snake_case")]
pub enum ConfigLayer {
Global = 0,
Team = 1,
Project = 2,
Env = 3,
Task = 4,
}
impl ConfigLayer {
pub fn priority(&self) -> u8 {
*self as u8
}
pub fn label(&self) -> &'static str {
match self {
ConfigLayer::Global => "global",
ConfigLayer::Team => "team",
ConfigLayer::Project => "project",
ConfigLayer::Env => "env",
ConfigLayer::Task => "task",
}
}
pub fn all() -> &'static [ConfigLayer] {
&[ConfigLayer::Global, ConfigLayer::Team, ConfigLayer::Project, ConfigLayer::Env, ConfigLayer::Task]
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct LayerConfig {
pub layer: ConfigLayer,
pub name: String,
#[serde(default)]
pub description: String,
#[serde(default)]
pub values: HashMap<String, serde_json::Value>,
#[serde(default)]
pub tags: HashMap<String, String>,
#[serde(default)]
pub enabled: bool,
#[serde(default = "default_priority")]
pub priority_override: Option<u8>,
#[serde(default)]
pub metadata: HashMap<String, String>,
}
impl Default for LayerConfig {
fn default() -> Self {
Self {
layer: ConfigLayer::Global,
name: String::new(),
description: String::new(),
values: HashMap::new(),
tags: HashMap::new(),
enabled: true,
priority_override: None,
metadata: HashMap::new(),
}
}
}
fn default_priority() -> Option<u8> {
None
}
impl LayerConfig {
pub fn effective_priority(&self) -> u8 {
self.priority_override.unwrap_or_else(|| self.layer.priority())
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MergedConfig {
pub values: HashMap<String, serde_json::Value>,
pub tags: HashMap<String, String>,
pub provenance: HashMap<String, ConfigLayer>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ConfigTree {
pub layers: Vec<LayerConfig>,
}
impl Default for ConfigTree {
fn default() -> Self {
Self::new()
}
}
impl ConfigTree {
pub fn new() -> Self {
Self { layers: Vec::new() }
}
pub fn add_layer(&mut self, config: LayerConfig) {
self.layers.push(config);
}
pub fn merge(&self) -> Result<MergedConfig, String> {
let mut sorted: Vec<&LayerConfig> = self.layers.iter().filter(|l| l.enabled).collect();
sorted.sort_by_key(|l| l.effective_priority());
let mut values: HashMap<String, serde_json::Value> = HashMap::new();
let mut tags: HashMap<String, String> = HashMap::new();
let mut provenance: HashMap<String, ConfigLayer> = HashMap::new();
for layer_config in &sorted {
for (k, v) in &layer_config.values {
values.insert(k.clone(), v.clone());
provenance.insert(k.clone(), layer_config.layer);
}
for (k, v) in &layer_config.tags {
tags.insert(k.clone(), v.clone());
}
}
Ok(MergedConfig { values, tags, provenance })
}
pub fn merge_with_guard(&self, tag_guard: &crate::config::tag::TagGuard) -> Result<MergedConfig, String> {
let merged = self.merge()?;
let tags: Vec<crate::config::tag::BusinessTag> = merged
.tags
.iter()
.map(|(k, v)| crate::config::tag::BusinessTag {
key: k.clone(),
value: v.clone(),
description: String::new(),
immutable: false,
})
.collect();
let base_tags: HashMap<String, String> = HashMap::new();
let result = tag_guard.validate_and_collect(&tags, &base_tags);
if tag_guard.validator.has_blocking_violations(&result) {
let (blocked, violations, _total) = tag_guard.blocking_summary();
return Err(format!("Tag guard blocked config merge: {blocked} tags blocked, {violations} violations"));
}
Ok(merged)
}
pub fn validate(&self) -> Result<(), Vec<String>> {
let mut errors = Vec::new();
let mut seen_keys: HashMap<&str, usize> = HashMap::new();
for (i, layer) in self.layers.iter().enumerate() {
if layer.name.is_empty() {
errors.push(format!("Layer at index {i} has empty name"));
}
if layer.values.is_empty() && layer.tags.is_empty() {
errors.push(format!("Layer '{}' ({}) has no values or tags", layer.name, layer.layer.label()));
}
for key in layer.values.keys() {
if let Some(prev) = seen_keys.get(key.as_str()) {
errors.push(format!("Key '{}' defined in multiple layers (index {prev} and {i})", key));
}
seen_keys.entry(key.as_str()).or_insert(i);
}
}
if errors.is_empty() {
Ok(())
} else {
Err(errors)
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn make_layer(layer: ConfigLayer, name: &str, key: &str, value: &str) -> LayerConfig {
let mut values = HashMap::new();
values.insert(key.to_string(), serde_json::Value::String(value.to_string()));
LayerConfig { layer, name: name.to_string(), values, enabled: true, ..Default::default() }
}
#[test]
fn test_layer_priority_order() {
assert!(ConfigLayer::Global.priority() < ConfigLayer::Team.priority());
assert!(ConfigLayer::Team.priority() < ConfigLayer::Project.priority());
assert!(ConfigLayer::Project.priority() < ConfigLayer::Env.priority());
assert!(ConfigLayer::Env.priority() < ConfigLayer::Task.priority());
}
#[test]
fn test_layer_label() {
assert_eq!(ConfigLayer::Global.label(), "global");
assert_eq!(ConfigLayer::Task.label(), "task");
}
#[test]
fn test_config_tree_merge_basic() {
let mut tree = ConfigTree::new();
tree.add_layer(make_layer(ConfigLayer::Global, "global-cfg", "host", "db-global"));
tree.add_layer(make_layer(ConfigLayer::Project, "proj-cfg", "host", "db-proj"));
let merged = tree.merge().unwrap();
assert_eq!(merged.values.get("host").unwrap(), &serde_json::Value::String("db-proj".to_string()));
assert_eq!(*merged.provenance.get("host").unwrap(), ConfigLayer::Project);
}
#[test]
fn test_config_tree_merge_lower_priority_does_not_override() {
let mut tree = ConfigTree::new();
tree.add_layer(make_layer(ConfigLayer::Project, "proj-cfg", "port", "5432"));
tree.add_layer(make_layer(ConfigLayer::Global, "global-cfg", "port", "3306"));
let merged = tree.merge().unwrap();
assert_eq!(merged.values.get("port").unwrap(), &serde_json::Value::String("5432".to_string()));
assert_eq!(*merged.provenance.get("port").unwrap(), ConfigLayer::Project);
}
#[test]
fn test_config_tree_merge_disabled_layer_ignored() {
let mut tree = ConfigTree::new();
let mut global = make_layer(ConfigLayer::Global, "global-cfg", "host", "db-global");
global.enabled = false;
tree.add_layer(global);
tree.add_layer(make_layer(ConfigLayer::Task, "task-cfg", "host", "db-task"));
let merged = tree.merge().unwrap();
assert_eq!(merged.values.get("host").unwrap(), &serde_json::Value::String("db-task".to_string()));
}
#[test]
fn test_config_tree_merge_distinct_keys() {
let mut tree = ConfigTree::new();
tree.add_layer(make_layer(ConfigLayer::Global, "g", "host", "g-host"));
tree.add_layer(make_layer(ConfigLayer::Team, "t", "port", "t-port"));
let merged = tree.merge().unwrap();
assert_eq!(merged.values.len(), 2);
assert!(merged.values.contains_key("host"));
assert!(merged.values.contains_key("port"));
}
#[test]
fn test_priority_override() {
let mut tree = ConfigTree::new();
let mut global = make_layer(ConfigLayer::Global, "override-cfg", "key", "val");
global.priority_override = Some(255);
tree.add_layer(global);
tree.add_layer(make_layer(ConfigLayer::Task, "task-cfg", "key", "task-val"));
let merged = tree.merge().unwrap();
assert_eq!(merged.values.get("key").unwrap(), &serde_json::Value::String("val".to_string()));
}
#[test]
fn test_validate_empty_name() {
let tree = ConfigTree {
layers: vec![LayerConfig {
layer: ConfigLayer::Global,
name: String::new(),
values: HashMap::from([("k".to_string(), serde_json::Value::String("v".to_string()))]),
..Default::default()
}],
};
assert!(tree.validate().is_err());
}
#[test]
fn test_validate_empty_values_and_tags() {
let layer = LayerConfig {
layer: ConfigLayer::Global,
name: "empty".to_string(),
values: HashMap::new(),
tags: HashMap::new(),
..Default::default()
};
let tree = ConfigTree { layers: vec![layer] };
let errs = tree.validate().unwrap_err();
assert!(errs.iter().any(|e| e.contains("empty")));
}
#[test]
fn test_effective_priority_default() {
let l = make_layer(ConfigLayer::Env, "env", "k", "v");
assert_eq!(l.effective_priority(), ConfigLayer::Env.priority());
}
#[test]
fn test_effective_priority_override() {
let mut l = make_layer(ConfigLayer::Global, "g", "k", "v");
l.priority_override = Some(100);
assert_eq!(l.effective_priority(), 100);
}
#[test]
fn test_all_layers() {
let all = ConfigLayer::all();
assert_eq!(all.len(), 5);
assert_eq!(all[0], ConfigLayer::Global);
assert_eq!(all[4], ConfigLayer::Task);
}
}
+160
View File
@@ -0,0 +1,160 @@
pub mod expression;
// governance disabled — storage methods not available at HEAD after main merge
// pub mod governance;
pub mod layer;
pub mod tag;
pub mod trace;
pub use expression::{parse_expression, resolve_all_expressions_in_value, resolve_expression, Expression};
// pub use governance::{ ... };
pub use layer::{ConfigLayer, ConfigTree, LayerConfig, MergedConfig};
pub use tag::{
BlockStats, BusinessTag, TagGuard, TagInheritanceWhitelist, TagPolicy, TagValidationResult, TagValidator,
};
pub use trace::{TraceEntry, TraceRingBuffer, TraceStats};
#[cfg(test)]
mod integration_tests {
use crate::config::*;
use crate::models::connection::DatabaseType;
use crate::schema_diff::SchemaDiffPreparationOptions;
use std::collections::HashMap;
#[test]
fn test_config_tree_with_schema_diff_options() {
let mut tree = ConfigTree::new();
let mut global_vals = HashMap::new();
global_vals.insert("target_schema".to_string(), serde_json::Value::String("public".to_string()));
global_vals.insert("ignore_comments".to_string(), serde_json::Value::Bool(true));
global_vals.insert("db_type".to_string(), serde_json::Value::String("postgres".to_string()));
tree.add_layer(LayerConfig {
layer: ConfigLayer::Global,
name: "defaults".to_string(),
values: global_vals,
tags: HashMap::from([("env".to_string(), "prod".to_string())]),
enabled: true,
..Default::default()
});
tree.add_layer(LayerConfig {
layer: ConfigLayer::Task,
name: "task-override".to_string(),
values: HashMap::from([
("cascade_delete".to_string(), serde_json::Value::Bool(true)),
("compare_column_order".to_string(), serde_json::Value::String("${eval:true}".to_string())),
]),
tags: HashMap::new(),
enabled: true,
..Default::default()
});
let mut merged = tree.merge().unwrap();
let merged_snapshot = merged.clone();
let resolved: HashMap<_, _> = merged
.values
.iter()
.map(|(k, v)| {
let resolved = resolve_all_expressions_in_value(v, &merged.values)
.unwrap_or_else(|e| panic!("Failed to resolve expression for key '{k}': {e}"));
(k.clone(), resolved)
})
.collect();
merged.values.extend(resolved);
let db_val = merged.values.get("db_type").and_then(|v| v.as_str()).unwrap_or("mysql");
let database_type: DatabaseType = serde_json::from_str(&format!("\"{db_val}\"")).unwrap_or(DatabaseType::Mysql);
let opts = SchemaDiffPreparationOptions {
source_tables: vec![],
target_tables: vec![],
source_details: vec![],
target_details: vec![],
source_functions: vec![],
target_functions: vec![],
source_sequences: vec![],
target_sequences: vec![],
source_rules: vec![],
target_rules: vec![],
source_owners: vec![],
target_owners: vec![],
database_type,
target_schema: merged.values.get("target_schema").and_then(|v| v.as_str()).map(String::from),
ignore_comments: merged.values.get("ignore_comments").and_then(|v| v.as_bool()).unwrap_or(false),
cascade_delete: merged.values.get("cascade_delete").and_then(|v| v.as_bool()).unwrap_or(false),
compare_column_order: merged.values.get("compare_column_order").and_then(|v| v.as_bool()).unwrap_or(false),
..Default::default()
};
assert_eq!(opts.database_type, DatabaseType::Postgres);
assert_eq!(opts.target_schema.as_deref(), Some("public"));
assert!(opts.ignore_comments);
assert!(opts.cascade_delete);
assert!(opts.compare_column_order);
let whitelist = TagInheritanceWhitelist::new(vec!["env".to_string()], TagPolicy::Strict);
let validator = TagValidator::new(whitelist, true);
let tags: Vec<BusinessTag> = merged_snapshot
.tags
.iter()
.map(|(k, v)| BusinessTag {
key: k.clone(),
value: v.clone(),
description: String::new(),
immutable: false,
})
.collect();
let result = validator.validate_tags(&tags, &HashMap::new());
assert_eq!(result.allowed.len(), 1);
assert_eq!(result.blocked.len(), 0);
let mut trace = TraceRingBuffer::new(100);
trace.record(ConfigLayer::Global, "target_schema", "read", "used for SchemaDiffPreparationOptions");
trace.record(ConfigLayer::Global, "db_type", "read", "used for DatabaseType");
trace.record(ConfigLayer::Task, "cascade_delete", "read", "overridden from task layer");
assert_eq!(trace.len(), 3);
}
#[test]
fn test_config_expression_with_value_injection() {
let mut tree = ConfigTree::new();
let mut global = HashMap::new();
global.insert("base_host".to_string(), serde_json::Value::String("db.example.com".to_string()));
global.insert("base_port".to_string(), serde_json::Value::Number(serde_json::Number::from(5432)));
tree.add_layer(LayerConfig {
layer: ConfigLayer::Global,
name: "global".to_string(),
values: global,
..Default::default()
});
tree.add_layer(LayerConfig {
layer: ConfigLayer::Project,
name: "project".to_string(),
values: HashMap::from([
("jdbc_url".to_string(), serde_json::Value::String("${eval:\"jdbc:postgresql://\"}".to_string())),
("host".to_string(), serde_json::Value::String("${ref:base_host}".to_string())),
]),
..Default::default()
});
let merged = tree.merge().unwrap();
let scope = merged.values.clone();
let url_resolved = resolve_all_expressions_in_value(
&serde_json::Value::String("${eval:\"jdbc:postgresql://\"}".to_string()),
&scope,
)
.unwrap();
assert_eq!(url_resolved, serde_json::Value::String("jdbc:postgresql://".to_string()));
let host_resolved =
resolve_all_expressions_in_value(&serde_json::Value::String("${ref:base_host}".to_string()), &scope)
.unwrap();
assert_eq!(host_resolved, serde_json::Value::String("db.example.com".to_string()));
}
}
+441
View File
@@ -0,0 +1,441 @@
use serde::{Deserialize, Serialize};
use std::collections::{HashMap, HashSet};
/// Controls how `TagInheritanceWhitelist` handles unlisted tags.
///
/// - `Allow` — all tags pass through by default; the whitelist is purely informational.
/// - `Block` — only tags whose keys appear in `allowed_keys` pass through; unlisted tags
/// are collected in `TagValidationResult::blocked` and count as blocking violations.
/// - `Strict` — same key filtering as `Block`, but additionally checks that tag *values*
/// match the base-layer values. Any value mismatch (or blocked key) is reported as a
/// violation, and **all** violations (including value mismatches) count as blocking.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
#[serde(rename_all = "snake_case")]
pub enum TagPolicy {
#[default]
Allow,
Block,
Strict,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BusinessTag {
pub key: String,
pub value: String,
#[serde(default)]
pub description: String,
#[serde(default)]
pub immutable: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TagInheritanceWhitelist {
#[serde(default)]
pub allowed_keys: HashSet<String>,
#[serde(default = "default_policy")]
pub default_policy: TagPolicy,
}
fn default_policy() -> TagPolicy {
TagPolicy::Block
}
impl Default for TagInheritanceWhitelist {
fn default() -> Self {
Self { allowed_keys: HashSet::new(), default_policy: TagPolicy::Block }
}
}
impl TagInheritanceWhitelist {
pub fn new(allowed: Vec<String>, policy: TagPolicy) -> Self {
Self { allowed_keys: allowed.into_iter().collect(), default_policy: policy }
}
pub fn is_allowed(&self, key: &str) -> bool {
match self.default_policy {
TagPolicy::Allow => true,
TagPolicy::Block => self.allowed_keys.contains(key),
TagPolicy::Strict => self.allowed_keys.contains(key),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TagValidationResult {
pub allowed: Vec<BusinessTag>,
pub blocked: Vec<BusinessTag>,
pub violations: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct TagValidator {
pub whitelist: TagInheritanceWhitelist,
#[serde(default)]
pub strict_mode: bool,
}
impl TagValidator {
pub fn new(whitelist: TagInheritanceWhitelist, strict_mode: bool) -> Self {
Self { whitelist, strict_mode }
}
pub fn validate_tags(&self, tags: &[BusinessTag], base_tags: &HashMap<String, String>) -> TagValidationResult {
let mut allowed = Vec::new();
let mut blocked = Vec::new();
let mut violations = Vec::new();
for tag in tags {
if !self.whitelist.is_allowed(&tag.key) {
blocked.push(tag.clone());
let msg = if self.strict_mode {
format!("BLOCKED: Tag '{}'='{}' is not in whitelist (strict mode)", tag.key, tag.value)
} else {
format!("BLOCKED: Tag '{}'='{}' is not in whitelist", tag.key, tag.value)
};
violations.push(msg);
continue;
}
if let Some(base_val) = base_tags.get(&tag.key) {
if tag.value != *base_val && self.strict_mode {
violations.push(format!(
"STRICT VIOLATION: Tag '{}'='{}' differs from base value '{}'",
tag.key, tag.value, base_val
));
}
}
allowed.push(tag.clone());
}
TagValidationResult { allowed, blocked, violations }
}
pub fn has_blocking_violations(&self, result: &TagValidationResult) -> bool {
if self.strict_mode {
!result.violations.is_empty()
} else {
!result.blocked.is_empty()
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BlockStats {
pub total_tags: usize,
pub blocked_count: usize,
pub violation_count: usize,
pub by_key: HashMap<String, usize>,
pub strict_mode: bool,
}
impl BlockStats {
pub fn from_validation(result: &TagValidationResult, strict_mode: bool) -> Self {
let mut by_key = HashMap::new();
for tag in &result.blocked {
*by_key.entry(tag.key.clone()).or_insert(0) += 1;
}
Self {
total_tags: result.allowed.len() + result.blocked.len(),
blocked_count: result.blocked.len(),
violation_count: result.violations.len(),
by_key,
strict_mode,
}
}
}
pub struct TagGuard {
pub validator: TagValidator,
pub stats: std::sync::Mutex<Vec<BlockStats>>,
}
impl TagGuard {
pub fn new(validator: TagValidator) -> Self {
Self { validator, stats: std::sync::Mutex::new(Vec::new()) }
}
pub fn validate_and_collect(
&self,
tags: &[BusinessTag],
base_tags: &HashMap<String, String>,
) -> TagValidationResult {
let result = self.validator.validate_tags(tags, base_tags);
let stats = BlockStats::from_validation(&result, self.validator.strict_mode);
if let Ok(mut guard) = self.stats.lock() {
guard.push(stats);
}
result
}
pub fn accumulated_stats(&self) -> Vec<BlockStats> {
self.stats.lock().map(|g| g.clone()).unwrap_or_default()
}
pub fn blocking_summary(&self) -> (usize, usize, usize) {
let stats = self.accumulated_stats();
let total_blocked = stats.iter().map(|s| s.blocked_count).sum();
let total_violations = stats.iter().map(|s| s.violation_count).sum();
let total_tags = stats.iter().map(|s| s.total_tags).sum();
(total_blocked, total_violations, total_tags)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_allow_policy_allows_all() {
let whitelist = TagInheritanceWhitelist::new(vec![], TagPolicy::Allow);
assert!(whitelist.is_allowed("anything"));
}
#[test]
fn test_block_policy_blocks_unlisted() {
let whitelist = TagInheritanceWhitelist::new(vec!["allowed_key".to_string()], TagPolicy::Block);
assert!(whitelist.is_allowed("allowed_key"));
assert!(!whitelist.is_allowed("blocked_key"));
}
#[test]
fn test_strict_policy_blocks_unlisted() {
let whitelist = TagInheritanceWhitelist::new(vec!["safe".to_string()], TagPolicy::Strict);
assert!(whitelist.is_allowed("safe"));
assert!(!whitelist.is_allowed("unsafe"));
}
#[test]
fn test_validator_allows_all_with_allow_policy() {
let whitelist = TagInheritanceWhitelist::new(vec![], TagPolicy::Allow);
let validator = TagValidator::new(whitelist, false);
let tags = vec![BusinessTag {
key: "env".to_string(),
value: "prod".to_string(),
description: String::new(),
immutable: false,
}];
let base = HashMap::new();
let result = validator.validate_tags(&tags, &base);
assert_eq!(result.allowed.len(), 1);
assert_eq!(result.blocked.len(), 0);
}
#[test]
fn test_validator_blocks_unlisted_in_block_mode() {
let whitelist = TagInheritanceWhitelist::new(vec!["allowed".to_string()], TagPolicy::Block);
let validator = TagValidator::new(whitelist, false);
let tags = vec![
BusinessTag {
key: "allowed".to_string(),
value: "yes".to_string(),
description: String::new(),
immutable: false,
},
BusinessTag {
key: "blocked".to_string(),
value: "no".to_string(),
description: String::new(),
immutable: false,
},
];
let base = HashMap::new();
let result = validator.validate_tags(&tags, &base);
assert_eq!(result.allowed.len(), 1);
assert_eq!(result.blocked.len(), 1);
assert_eq!(result.blocked[0].key, "blocked");
}
#[test]
fn test_strict_mode_reports_value_mismatch() {
let whitelist = TagInheritanceWhitelist::new(vec!["env".to_string()], TagPolicy::Strict);
let validator = TagValidator::new(whitelist, true);
let tags = vec![BusinessTag {
key: "env".to_string(),
value: "staging".to_string(),
description: String::new(),
immutable: false,
}];
let mut base = HashMap::new();
base.insert("env".to_string(), "prod".to_string());
let result = validator.validate_tags(&tags, &base);
assert_eq!(result.violations.len(), 1);
assert!(result.violations[0].contains("STRICT VIOLATION"));
}
#[test]
fn test_has_blocking_violations_standard_mode() {
let whitelist = TagInheritanceWhitelist::new(vec![], TagPolicy::Block);
let validator = TagValidator::new(whitelist, false);
let result = TagValidationResult {
allowed: vec![],
blocked: vec![BusinessTag {
key: "x".to_string(),
value: "y".to_string(),
description: String::new(),
immutable: false,
}],
violations: vec![],
};
assert!(validator.has_blocking_violations(&result));
}
#[test]
fn test_has_blocking_violations_strict_mode() {
let whitelist = TagInheritanceWhitelist::new(vec![], TagPolicy::Strict);
let validator = TagValidator::new(whitelist, true);
let result =
TagValidationResult { allowed: vec![], blocked: vec![], violations: vec!["STRICT VIOLATION".to_string()] };
assert!(validator.has_blocking_violations(&result));
}
#[test]
fn test_has_blocking_violations_no_violations() {
let whitelist = TagInheritanceWhitelist::new(vec![], TagPolicy::Allow);
let validator = TagValidator::new(whitelist, false);
let result = TagValidationResult { allowed: vec![], blocked: vec![], violations: vec![] };
assert!(!validator.has_blocking_violations(&result));
}
#[test]
fn test_immutable_tag_field() {
let tag = BusinessTag {
key: "env".to_string(),
value: "prod".to_string(),
description: "Environment".to_string(),
immutable: true,
};
assert!(tag.immutable);
}
#[test]
fn block_stats_from_validation() {
let whitelist = TagInheritanceWhitelist::new(vec!["allowed".to_string()], TagPolicy::Block);
let validator = TagValidator::new(whitelist, false);
let tags = vec![
BusinessTag {
key: "allowed".to_string(),
value: "yes".to_string(),
description: String::new(),
immutable: false,
},
BusinessTag {
key: "blocked1".to_string(),
value: "no".to_string(),
description: String::new(),
immutable: false,
},
BusinessTag {
key: "blocked2".to_string(),
value: "no".to_string(),
description: String::new(),
immutable: false,
},
];
let base = HashMap::new();
let result = validator.validate_tags(&tags, &base);
let stats = BlockStats::from_validation(&result, false);
assert_eq!(stats.total_tags, 3);
assert_eq!(stats.blocked_count, 2);
assert_eq!(stats.violation_count, 2);
assert_eq!(stats.by_key.get("blocked1"), Some(&1usize));
assert_eq!(stats.by_key.get("blocked2"), Some(&1usize));
assert!(!stats.strict_mode);
}
#[test]
fn block_stats_with_strict_mode_violations() {
let whitelist = TagInheritanceWhitelist::new(vec!["env".to_string()], TagPolicy::Strict);
let validator = TagValidator::new(whitelist, true);
let tags = vec![BusinessTag {
key: "env".to_string(),
value: "dev".to_string(),
description: String::new(),
immutable: false,
}];
let mut base = HashMap::new();
base.insert("env".to_string(), "prod".to_string());
let result = validator.validate_tags(&tags, &base);
let stats = BlockStats::from_validation(&result, true);
assert_eq!(stats.blocked_count, 0);
assert_eq!(stats.violation_count, 1);
assert!(stats.strict_mode);
}
#[test]
fn tag_guard_collects_multiple_validations() {
let whitelist = TagInheritanceWhitelist::new(vec!["env".to_string()], TagPolicy::Strict);
let validator = TagValidator::new(whitelist, true);
let guard = TagGuard::new(validator);
let tags1 = vec![BusinessTag {
key: "env".to_string(),
value: "staging".to_string(),
description: String::new(),
immutable: false,
}];
let mut base1 = HashMap::new();
base1.insert("env".to_string(), "prod".to_string());
let tags2 = vec![BusinessTag {
key: "blocked".to_string(),
value: "x".to_string(),
description: String::new(),
immutable: false,
}];
guard.validate_and_collect(&tags1, &base1);
guard.validate_and_collect(&tags2, &HashMap::new());
let stats = guard.accumulated_stats();
assert_eq!(stats.len(), 2);
assert_eq!(stats[0].violation_count, 1);
assert_eq!(stats[1].blocked_count, 1);
let (blocked, violations, total) = guard.blocking_summary();
assert_eq!(blocked, 1);
assert_eq!(violations, 2);
assert_eq!(total, 2);
}
#[test]
fn tag_guard_empty_stats() {
let whitelist = TagInheritanceWhitelist::new(vec![], TagPolicy::Allow);
let validator = TagValidator::new(whitelist, false);
let guard = TagGuard::new(validator);
assert!(guard.accumulated_stats().is_empty());
let (blocked, violations, total) = guard.blocking_summary();
assert_eq!(blocked, 0);
assert_eq!(violations, 0);
assert_eq!(total, 0);
}
#[test]
fn tag_guard_blocking_with_strict_mode() {
let whitelist = TagInheritanceWhitelist::new(vec![], TagPolicy::Strict);
let validator = TagValidator::new(whitelist, true);
let guard = TagGuard::new(validator);
let tags = vec![BusinessTag {
key: "env".to_string(),
value: "dev".to_string(),
description: String::new(),
immutable: false,
}];
let result = guard.validate_and_collect(&tags, &HashMap::new());
assert_eq!(result.blocked.len(), 1);
assert!(guard.validator.has_blocking_violations(&result));
}
}
+273
View File
@@ -0,0 +1,273 @@
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use std::collections::VecDeque;
use super::ConfigLayer;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TraceEntry {
pub timestamp: DateTime<Utc>,
pub layer: ConfigLayer,
pub key: String,
pub action: String,
pub detail: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TraceRingBuffer {
capacity: usize,
entries: VecDeque<TraceEntry>,
}
impl TraceRingBuffer {
pub fn new(capacity: usize) -> Self {
Self { capacity, entries: VecDeque::with_capacity(capacity) }
}
pub fn push(&mut self, entry: TraceEntry) {
if self.entries.len() >= self.capacity {
self.entries.pop_front();
}
self.entries.push_back(entry);
}
pub fn record(&mut self, layer: ConfigLayer, key: &str, action: &str, detail: &str) {
self.push(TraceEntry {
timestamp: Utc::now(),
layer,
key: key.to_string(),
action: action.to_string(),
detail: detail.to_string(),
});
}
pub fn entries(&self) -> impl Iterator<Item = &TraceEntry> {
self.entries.iter()
}
pub fn len(&self) -> usize {
self.entries.len()
}
pub fn is_empty(&self) -> bool {
self.entries.is_empty()
}
pub fn capacity(&self) -> usize {
self.capacity
}
pub fn clear(&mut self) {
self.entries.clear();
}
pub fn filter_by_layer(&self, layer: ConfigLayer) -> Vec<&TraceEntry> {
self.entries.iter().filter(|e| e.layer == layer).collect()
}
pub fn filter_by_action(&self, action: &str) -> Vec<&TraceEntry> {
self.entries.iter().filter(|e| e.action == action).collect()
}
pub fn export_json(&self) -> Result<String, String> {
serde_json::to_string_pretty(&self.entries)
.map_err(|e| format!("Failed to serialize {} trace entries: {e}", self.entries.len()))
}
pub fn recent(&self, n: usize) -> Vec<&TraceEntry> {
let n = n.min(self.entries.len());
self.entries.iter().rev().take(n).collect()
}
pub fn stats(&self) -> TraceStats {
let mut action_counts: std::collections::HashMap<String, usize> = std::collections::HashMap::new();
let mut layer_counts: std::collections::HashMap<String, usize> = std::collections::HashMap::new();
for entry in &self.entries {
*action_counts.entry(entry.action.clone()).or_insert(0) += 1;
*layer_counts.entry(entry.layer.label().to_string()).or_insert(0) += 1;
}
TraceStats { total_entries: self.entries.len(), capacity: self.capacity, action_counts, layer_counts }
}
}
impl Default for TraceRingBuffer {
fn default() -> Self {
Self::new(1000)
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TraceStats {
pub total_entries: usize,
pub capacity: usize,
pub action_counts: std::collections::HashMap<String, usize>,
pub layer_counts: std::collections::HashMap<String, usize>,
}
#[cfg(test)]
mod tests {
use super::*;
use chrono::TimeZone;
fn sample_entry() -> TraceEntry {
TraceEntry {
timestamp: Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap(),
layer: ConfigLayer::Global,
key: "host".to_string(),
action: "read".to_string(),
detail: "value resolved".to_string(),
}
}
#[test]
fn test_new_buffer_is_empty() {
let buf = TraceRingBuffer::new(10);
assert!(buf.is_empty());
assert_eq!(buf.len(), 0);
}
#[test]
fn test_push_adds_entry() {
let mut buf = TraceRingBuffer::new(10);
buf.push(sample_entry());
assert_eq!(buf.len(), 1);
}
#[test]
fn test_push_evicts_oldest_when_full() {
let mut buf = TraceRingBuffer::new(3);
for i in 0..5 {
buf.push(TraceEntry {
timestamp: Utc::now(),
layer: ConfigLayer::Task,
key: format!("key{i}"),
action: "write".to_string(),
detail: String::new(),
});
}
assert_eq!(buf.len(), 3);
let entries: Vec<&TraceEntry> = buf.entries().collect();
assert_eq!(entries[0].key, "key2");
assert_eq!(entries[2].key, "key4");
}
#[test]
fn test_record_convenience() {
let mut buf = TraceRingBuffer::new(10);
buf.record(ConfigLayer::Global, "host", "read", "localhost");
assert_eq!(buf.len(), 1);
let entry = buf.entries().next().unwrap();
assert_eq!(entry.layer, ConfigLayer::Global);
assert_eq!(entry.key, "host");
assert_eq!(entry.action, "read");
}
#[test]
fn test_filter_by_layer() {
let mut buf = TraceRingBuffer::new(10);
buf.record(ConfigLayer::Global, "host", "read", "");
buf.record(ConfigLayer::Project, "port", "write", "");
buf.record(ConfigLayer::Global, "user", "read", "");
let global_entries = buf.filter_by_layer(ConfigLayer::Global);
assert_eq!(global_entries.len(), 2);
let project_entries = buf.filter_by_layer(ConfigLayer::Project);
assert_eq!(project_entries.len(), 1);
let task_entries = buf.filter_by_layer(ConfigLayer::Task);
assert_eq!(task_entries.len(), 0);
}
#[test]
fn test_filter_by_action() {
let mut buf = TraceRingBuffer::new(10);
buf.record(ConfigLayer::Global, "host", "read", "");
buf.record(ConfigLayer::Global, "port", "write", "");
buf.record(ConfigLayer::Global, "user", "read", "");
let reads = buf.filter_by_action("read");
assert_eq!(reads.len(), 2);
let writes = buf.filter_by_action("write");
assert_eq!(writes.len(), 1);
}
#[test]
fn test_export_json() {
let mut buf = TraceRingBuffer::new(10);
buf.push(sample_entry());
let json = buf.export_json().unwrap();
assert!(json.contains("host"));
assert!(json.contains("read"));
}
#[test]
fn test_recent() {
let mut buf = TraceRingBuffer::new(10);
for i in 0..5 {
buf.record(ConfigLayer::Task, &format!("key{i}"), "write", "");
}
let recent = buf.recent(3);
assert_eq!(recent.len(), 3);
assert_eq!(recent[0].key, "key4");
assert_eq!(recent[2].key, "key2");
}
#[test]
fn test_recent_less_than_n() {
let mut buf = TraceRingBuffer::new(10);
buf.record(ConfigLayer::Task, "k1", "read", "");
let recent = buf.recent(10);
assert_eq!(recent.len(), 1);
}
#[test]
fn test_clear() {
let mut buf = TraceRingBuffer::new(10);
buf.record(ConfigLayer::Task, "k1", "read", "");
buf.record(ConfigLayer::Task, "k2", "write", "");
assert_eq!(buf.len(), 2);
buf.clear();
assert!(buf.is_empty());
}
#[test]
fn test_stats() {
let mut buf = TraceRingBuffer::new(100);
buf.record(ConfigLayer::Global, "host", "read", "");
buf.record(ConfigLayer::Global, "port", "read", "");
buf.record(ConfigLayer::Project, "config", "write", "");
let stats = buf.stats();
assert_eq!(stats.total_entries, 3);
assert_eq!(*stats.action_counts.get("read").unwrap(), 2);
assert_eq!(*stats.action_counts.get("write").unwrap(), 1);
assert_eq!(*stats.layer_counts.get("global").unwrap(), 2);
assert_eq!(*stats.layer_counts.get("project").unwrap(), 1);
}
#[test]
fn test_default_capacity() {
let buf = TraceRingBuffer::default();
assert_eq!(buf.capacity(), 1000);
}
#[test]
fn test_capacity_set_correctly() {
let buf = TraceRingBuffer::new(500);
assert_eq!(buf.capacity(), 500);
}
#[test]
fn test_entry_timestamp_is_set() {
let mut buf = TraceRingBuffer::new(10);
buf.record(ConfigLayer::Task, "k", "read", "detail");
let entry = buf.entries().next().unwrap();
let now = Utc::now();
let diff = now - entry.timestamp;
assert!(diff.num_seconds() < 5);
}
}
+659
View File
@@ -0,0 +1,659 @@
use serde::{Deserialize, Serialize};
use crate::data_compare::{DataCompareFromTablesPreparation, DataCompareResult};
use crate::schema_diff::SchemaDiffPreparation;
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase")]
pub enum CorrectionStepType {
SchemaCreate,
SchemaAlter,
SchemaDrop,
DataInsert,
DataUpdate,
DataDelete,
SchemaPostSync,
Checkpoint,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct CorrectionStep {
pub step_type: CorrectionStepType,
pub table_name: Option<String>,
pub sql: String,
pub rollback_sql: Option<String>,
pub description: String,
pub risk_level: CorrectionRiskLevel,
#[serde(default)]
pub depends_on: Vec<usize>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
#[serde(rename_all = "camelCase")]
pub enum CorrectionRiskLevel {
Safe,
#[default]
Caution,
Dangerous,
Blocked,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase")]
pub enum CorrectionStrategy {
StructureFirst,
DataFirst,
Interleaved,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct JointCorrectionPlan {
pub strategy: CorrectionStrategy,
pub steps: Vec<CorrectionStep>,
pub schema_diff_count: usize,
pub data_diff_count: usize,
pub total_estimated_duration_secs: Option<u64>,
#[serde(default)]
pub rollback_steps: Vec<CorrectionStep>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct JointCorrectionOptions {
pub strategy: CorrectionStrategy,
#[serde(default)]
pub include_checkpoints: bool,
#[serde(default)]
pub include_rollback: bool,
}
impl Default for JointCorrectionOptions {
fn default() -> Self {
Self { strategy: CorrectionStrategy::StructureFirst, include_checkpoints: true, include_rollback: true }
}
}
/// Build a joint correction plan from schema diff and data compare results.
///
/// Orchestrates schema + data changes according to `CorrectionStrategy`:
/// - `StructureFirst`: schema → data → post-sync checks
/// - `DataFirst`: data → schema
/// - `Interleaved`: per-table schema + data interleaved
///
/// # Limitations
/// - `Interleaved` uses placeholder SQL rather than actual per-table DDL slices.
/// - Table name matching uses substring checks (`line.contains(&name)`), which can
/// produce false positives when one table name is a prefix of another.
pub fn build_joint_correction_plan(
schema_diff: Option<&SchemaDiffPreparation>,
data_compare: Option<&DataCompareFromTablesPreparation>,
options: JointCorrectionOptions,
) -> JointCorrectionPlan {
let mut steps = Vec::new();
let mut rollback_steps = Vec::new();
let mut step_index = 0usize;
match options.strategy {
CorrectionStrategy::StructureFirst => {
step_index = append_schema_steps(schema_diff, &mut steps, &mut rollback_steps, &options, step_index);
let _ = append_data_steps(data_compare, &mut steps, &mut rollback_steps, &options, step_index);
}
CorrectionStrategy::DataFirst => {
step_index = append_data_steps(data_compare, &mut steps, &mut rollback_steps, &options, step_index);
let _ = append_schema_steps(schema_diff, &mut steps, &mut rollback_steps, &options, step_index);
}
CorrectionStrategy::Interleaved => {
if let Some(schema) = schema_diff {
let table_diffs: Vec<_> = schema.diffs.iter().filter(|d| d.diff_type != "unchanged").collect();
for table_diff in &table_diffs {
let tbl_name = &table_diff.name;
if let Some(dc) = data_compare {
if contains_table(&dc.sync_sql, tbl_name) {
step_index = append_table_data_step(
&dc.result,
tbl_name,
&mut steps,
&mut rollback_steps,
&options,
step_index,
);
}
}
step_index = append_table_schema_step(
table_diff,
tbl_name,
&mut steps,
&mut rollback_steps,
&options,
step_index,
);
}
} else if let Some(dc) = data_compare {
let _ = append_data_steps(Some(dc), &mut steps, &mut rollback_steps, &options, step_index);
}
}
}
let schema_diff_count = steps
.iter()
.filter(|s| {
matches!(
s.step_type,
CorrectionStepType::SchemaCreate
| CorrectionStepType::SchemaAlter
| CorrectionStepType::SchemaDrop
| CorrectionStepType::SchemaPostSync
)
})
.count();
let data_diff_count = steps
.iter()
.filter(|s| {
matches!(
s.step_type,
CorrectionStepType::DataInsert | CorrectionStepType::DataUpdate | CorrectionStepType::DataDelete
)
})
.count();
JointCorrectionPlan {
strategy: options.strategy,
steps,
schema_diff_count,
data_diff_count,
total_estimated_duration_secs: None,
rollback_steps,
}
}
fn append_schema_steps(
schema_diff: Option<&SchemaDiffPreparation>,
steps: &mut Vec<CorrectionStep>,
_rollback_steps: &mut Vec<CorrectionStep>,
options: &JointCorrectionOptions,
mut step_index: usize,
) -> usize {
let Some(schema) = schema_diff else {
return step_index;
};
if schema.sync_sql.is_empty() {
return step_index;
}
if options.include_checkpoints {
steps.push(CorrectionStep {
step_type: CorrectionStepType::Checkpoint,
table_name: None,
sql: String::new(),
rollback_sql: None,
description: "Schema correction checkpoint".to_string(),
risk_level: CorrectionRiskLevel::Safe,
depends_on: Vec::new(),
});
step_index += 1;
}
let table_diffs: Vec<_> = schema.diffs.iter().filter(|d| d.diff_type != "unchanged").collect();
for table_diff in &table_diffs {
let tbl_name = &table_diff.name;
step_index = append_table_schema_step(table_diff, tbl_name, steps, _rollback_steps, options, step_index);
}
step_index
}
fn append_table_schema_step(
table_diff: &crate::schema_diff::TableDiff,
tbl_name: &str,
steps: &mut Vec<CorrectionStep>,
rollback_steps: &mut Vec<CorrectionStep>,
options: &JointCorrectionOptions,
mut step_index: usize,
) -> usize {
let step_type = match table_diff.diff_type.as_str() {
"added" => CorrectionStepType::SchemaCreate,
"removed" => CorrectionStepType::SchemaDrop,
_ => CorrectionStepType::SchemaAlter,
};
let risk_level = match table_diff.diff_type.as_str() {
"removed" => CorrectionRiskLevel::Dangerous,
"modified" => CorrectionRiskLevel::Caution,
_ => CorrectionRiskLevel::Safe,
};
let has_incomplete_triggers = table_diff
.triggers
.as_ref()
.is_some_and(|triggers| triggers.iter().any(|t| t.source.as_ref().is_some_and(|s| s.statement.is_none())));
let rollback_sql = if options.include_rollback {
match table_diff.diff_type.as_str() {
"added" => Some(format!("DROP TABLE IF EXISTS {};", tbl_name)),
"removed" => {
table_diff.target_ddl.as_ref().map(|ddl| {
if has_incomplete_triggers {
format!(
"-- WARNING: Rollback DDL is INCOMPLETE — trigger bodies could not be reconstructed.\n-- Manual review required. Original DDL:\n{}",
ddl
)
} else {
ddl.clone()
}
}).or_else(|| {
if has_incomplete_triggers {
Some(format!(
"-- WARNING: Rollback is INCOMPLETE — table '{}' had triggers that cannot be reconstructed.\n-- Manual intervention required before executing.",
tbl_name
))
} else {
Some(format!(
"-- Manual rollback required: recreate table {} with original DDL",
tbl_name
))
}
})
}
"modified" => table_diff.sync_sql.as_ref().map(|s| format!("-- Rollback: {}", s)),
_ => None,
}
} else {
None
};
let rollback_risk = if has_incomplete_triggers { CorrectionRiskLevel::Blocked } else { risk_level.clone() };
let rollback_desc = if has_incomplete_triggers {
format!(
"INCOMPLETE Rollback for schema {} on table: {tbl_name} — trigger bodies cannot be reconstructed",
table_diff.diff_type
)
} else {
format!("Rollback for schema {} on table: {tbl_name}", table_diff.diff_type)
};
let sql = table_diff.sync_sql.clone().unwrap_or_else(|| format!("-- Schema change for table: {tbl_name}"));
steps.push(CorrectionStep {
step_type,
table_name: Some(tbl_name.to_string()),
sql,
rollback_sql: rollback_sql.clone(),
description: format!("Schema {} for table: {tbl_name}", table_diff.diff_type),
risk_level,
depends_on: Vec::new(),
});
if let Some(rollback) = rollback_sql {
rollback_steps.push(CorrectionStep {
step_type: match table_diff.diff_type.as_str() {
"added" => CorrectionStepType::SchemaDrop,
"removed" => CorrectionStepType::SchemaCreate,
_ => CorrectionStepType::SchemaAlter,
},
table_name: Some(tbl_name.to_string()),
sql: rollback,
rollback_sql: None,
description: rollback_desc,
risk_level: rollback_risk,
depends_on: Vec::new(),
});
}
step_index += 1;
step_index
}
fn append_data_steps(
data_compare: Option<&DataCompareFromTablesPreparation>,
steps: &mut Vec<CorrectionStep>,
_rollback_steps: &mut Vec<CorrectionStep>,
options: &JointCorrectionOptions,
mut step_index: usize,
) -> usize {
let Some(dc) = data_compare else {
return step_index;
};
if dc.result.added.is_empty() && dc.result.removed.is_empty() && dc.result.modified.is_empty() {
return step_index;
}
if options.include_checkpoints {
steps.push(CorrectionStep {
step_type: CorrectionStepType::Checkpoint,
table_name: None,
sql: String::new(),
rollback_sql: None,
description: "Data correction checkpoint".to_string(),
risk_level: CorrectionRiskLevel::Safe,
depends_on: Vec::new(),
});
step_index += 1;
}
for statement in &dc.sync_statements {
let (step_type, description) = if statement.to_uppercase().starts_with("INSERT") {
(CorrectionStepType::DataInsert, format!("Insert data: {}", summary(statement, 80)))
} else if statement.to_uppercase().starts_with("UPDATE") {
(CorrectionStepType::DataUpdate, format!("Update data: {}", summary(statement, 80)))
} else if statement.to_uppercase().starts_with("DELETE") {
(CorrectionStepType::DataDelete, format!("Delete data: {}", summary(statement, 80)))
} else {
(CorrectionStepType::SchemaPostSync, format!("Pre-sync: {}", summary(statement, 80)))
};
let rollback_sql = if options.include_rollback {
if statement.to_uppercase().starts_with("INSERT") {
Some(format!("-- Rollback: DELETE matching rows; manual review needed: {}", summary(statement, 60)))
} else if statement.to_uppercase().starts_with("DELETE") {
Some(format!("-- Rollback: INSERT deleted rows; manual review needed: {}", summary(statement, 60)))
} else if statement.to_uppercase().starts_with("UPDATE") {
Some(format!("-- Rollback: reverse UPDATE; manual review needed: {}", summary(statement, 60)))
} else {
None
}
} else {
None
};
steps.push(CorrectionStep {
step_type,
table_name: None,
sql: statement.clone(),
rollback_sql,
description,
risk_level: CorrectionRiskLevel::Caution,
depends_on: Vec::new(),
});
step_index += 1;
}
step_index
}
fn append_table_data_step(
data_prep: &DataCompareResult,
tbl_name: &str,
steps: &mut Vec<CorrectionStep>,
_rollback_steps: &mut Vec<CorrectionStep>,
_options: &JointCorrectionOptions,
mut step_index: usize,
) -> usize {
let total = data_prep.added.len() + data_prep.modified.len() + data_prep.removed.len();
if total == 0 {
return step_index;
}
steps.push(CorrectionStep {
step_type: CorrectionStepType::DataInsert,
table_name: Some(tbl_name.to_string()),
sql: format!(
"-- Data sync for table: {tbl_name} ({} added, {} modified, {} removed)",
data_prep.added.len(),
data_prep.modified.len(),
data_prep.removed.len()
),
rollback_sql: None,
description: format!(
"Data sync for table: {tbl_name} ({} added, {} modified, {} removed)",
data_prep.added.len(),
data_prep.modified.len(),
data_prep.removed.len()
),
risk_level: CorrectionRiskLevel::Caution,
depends_on: Vec::new(),
});
step_index += 1;
step_index
}
fn contains_table(sync_sql: &str, table_name: &str) -> bool {
sync_sql.contains(table_name)
}
fn summary(text: &str, max_len: usize) -> String {
let cleaned = text.replace('\n', " ");
if cleaned.len() <= max_len {
cleaned
} else {
format!("{}...", &cleaned[..max_len.min(cleaned.len())])
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::data_compare::{
DataCompareChangedCell, DataCompareFromTablesPreparation, DataCompareModifiedRow, DataCompareResult,
DataCompareRow,
};
use crate::schema_diff::{SchemaDiffPreparation, TableDiff};
use serde_json::json;
use std::collections::HashMap;
fn sample_table_diff(diff_type: &str, name: &str) -> TableDiff {
TableDiff {
diff_type: diff_type.to_string(),
object_type: Some("table".to_string()),
name: name.to_string(),
columns: None,
indexes: None,
foreign_keys: None,
triggers: None,
ddl: None,
target_ddl: None,
source_table_comment: None,
target_table_comment: None,
sync_sql: Some(format!("-- SQL for {name}")),
}
}
fn sample_schema_preparation() -> SchemaDiffPreparation {
SchemaDiffPreparation {
diffs: vec![sample_table_diff("added", "users"), sample_table_diff("modified", "orders")],
sync_sql: "-- Schema sync SQL for users\n-- Schema sync SQL for orders".to_string(),
function_diffs: Vec::new(),
sequence_diffs: Vec::new(),
rule_diffs: Vec::new(),
owner_diffs: Vec::new(),
rollback_sync_sql: Some("-- Rollback SQL".to_string()),
rollback_completeness: crate::schema_diff::RollbackCompleteness::Complete,
missing_rollback_objects: Vec::new(),
rename_candidates: Vec::new(),
rollback_graph: None,
compatibility_warnings: Vec::new(),
permission_diffs: Vec::new(),
permission_sync_sql: None,
dependency_graph: None,
}
}
fn sample_data_preparation() -> DataCompareFromTablesPreparation {
DataCompareFromTablesPreparation {
result: DataCompareResult {
added: vec![DataCompareRow {
key: "1".to_string(),
key_values: HashMap::from([("id".to_string(), json!(1))]),
values: HashMap::from([("id".to_string(), json!(1)), ("name".to_string(), json!("Ada"))]),
}],
removed: vec![],
modified: vec![DataCompareModifiedRow {
key: "2".to_string(),
key_values: HashMap::from([("id".to_string(), json!(2))]),
source_values: HashMap::new(),
target_values: HashMap::new(),
changes: vec![DataCompareChangedCell {
column: "name".to_string(),
source: json!("Bob"),
target: json!("Bobby"),
}],
}],
},
sync_statements: vec![
"INSERT INTO \"public\".\"users\" (\"id\", \"name\") VALUES (1, 'Ada');".to_string(),
"UPDATE \"public\".\"users\" SET \"name\" = 'Bob' WHERE \"id\" = 2;".to_string(),
],
sync_sql: "INSERT INTO \"public\".\"users\" (\"id\", \"name\") VALUES (1, 'Ada');\n\
UPDATE \"public\".\"users\" SET \"name\" = 'Bob' WHERE \"id\" = 2;"
.to_string(),
pre_sync_statements: vec![],
source_row_count: 2,
target_row_count: 2,
source_truncated: false,
target_truncated: false,
degradation_level: Some("full".to_string()),
sampling_rate: Some(1.0),
confidence_score: Some(1.0),
verification_method: Some("full_compare".to_string()),
source_checksums: None,
target_checksums: None,
}
}
#[test]
fn builds_structure_first_correction_plan() {
let schema = sample_schema_preparation();
let data = sample_data_preparation();
let plan = build_joint_correction_plan(
Some(&schema),
Some(&data),
JointCorrectionOptions {
strategy: CorrectionStrategy::StructureFirst,
include_checkpoints: true,
include_rollback: true,
},
);
assert_eq!(plan.strategy, CorrectionStrategy::StructureFirst);
assert!(plan.schema_diff_count > 0);
assert!(plan.data_diff_count > 0);
let first_data_idx = plan.steps.iter().position(|s| {
matches!(
s.step_type,
CorrectionStepType::DataInsert | CorrectionStepType::DataUpdate | CorrectionStepType::DataDelete
)
});
let last_schema_idx = plan.steps.iter().rposition(|s| {
matches!(
s.step_type,
CorrectionStepType::SchemaCreate | CorrectionStepType::SchemaAlter | CorrectionStepType::SchemaDrop
)
});
if let (Some(si), Some(di)) = (last_schema_idx, first_data_idx) {
assert!(si < di, "Schema steps should come before data steps in StructureFirst strategy");
}
}
#[test]
fn builds_data_first_correction_plan() {
let schema = sample_schema_preparation();
let data = sample_data_preparation();
let plan = build_joint_correction_plan(
Some(&schema),
Some(&data),
JointCorrectionOptions {
strategy: CorrectionStrategy::DataFirst,
include_checkpoints: true,
include_rollback: true,
},
);
assert_eq!(plan.strategy, CorrectionStrategy::DataFirst);
assert!(plan
.steps
.iter()
.any(|s| matches!(s.step_type, CorrectionStepType::DataInsert | CorrectionStepType::DataUpdate)));
assert!(plan
.steps
.iter()
.any(|s| matches!(s.step_type, CorrectionStepType::SchemaCreate | CorrectionStepType::SchemaAlter)));
}
#[test]
fn builds_interleaved_correction_plan() {
let schema = sample_schema_preparation();
let data = sample_data_preparation();
let plan = build_joint_correction_plan(
Some(&schema),
Some(&data),
JointCorrectionOptions {
strategy: CorrectionStrategy::Interleaved,
include_checkpoints: true,
include_rollback: true,
},
);
assert_eq!(plan.strategy, CorrectionStrategy::Interleaved);
assert!(!plan.steps.is_empty());
}
#[test]
fn handles_missing_schema_diff() {
let data = sample_data_preparation();
let plan = build_joint_correction_plan(None, Some(&data), JointCorrectionOptions::default());
assert_eq!(plan.schema_diff_count, 0);
assert!(plan.data_diff_count > 0);
}
#[test]
fn handles_missing_data_compare() {
let schema = sample_schema_preparation();
let plan = build_joint_correction_plan(Some(&schema), None, JointCorrectionOptions::default());
assert!(plan.schema_diff_count > 0);
assert_eq!(plan.data_diff_count, 0);
}
#[test]
fn handles_empty_input() {
let plan = build_joint_correction_plan(None, None, JointCorrectionOptions::default());
assert_eq!(plan.steps.len(), 0);
assert_eq!(plan.schema_diff_count, 0);
assert_eq!(plan.data_diff_count, 0);
}
#[test]
fn correction_step_serialization() {
let step = CorrectionStep {
step_type: CorrectionStepType::SchemaCreate,
table_name: Some("public.users".to_string()),
sql: "CREATE TABLE ...".to_string(),
rollback_sql: Some("DROP TABLE ...".to_string()),
description: "Create users table".to_string(),
risk_level: CorrectionRiskLevel::Safe,
depends_on: vec![0],
};
let json = serde_json::to_string(&step).expect("serialization should succeed");
let deserialized: CorrectionStep = serde_json::from_str(&json).expect("deserialization should succeed");
assert_eq!(deserialized.step_type, CorrectionStepType::SchemaCreate);
assert_eq!(deserialized.table_name, Some("public.users".to_string()));
assert_eq!(deserialized.risk_level, CorrectionRiskLevel::Safe);
}
#[test]
fn joint_correction_plan_serialization() {
let schema = sample_schema_preparation();
let data = sample_data_preparation();
let plan = build_joint_correction_plan(Some(&schema), Some(&data), JointCorrectionOptions::default());
let json = serde_json::to_string(&plan).expect("serialization should succeed");
assert!(json.contains("structureFirst") || json.contains("StructureFirst"));
}
}
+699 -24
View File
@@ -3,6 +3,7 @@ use std::collections::{HashMap, HashSet};
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use sha2::{Digest, Sha256};
use crate::connection::AppState;
use crate::data_grid_sql::{format_grid_sql_literal as format_data_grid_sql_literal, DataGridColumnInfo};
@@ -62,6 +63,12 @@ pub struct DataCompareFromTablesOptions {
pub key_columns: Vec<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub fetch_batch_size: Option<usize>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub degradation_threshold: Option<DegradationThreshold>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub sampling_strategy: Option<SamplingStrategy>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub enable_checksum: Option<bool>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
@@ -170,6 +177,112 @@ pub struct DataCompareFromTablesPreparation {
pub target_row_count: u64,
pub source_truncated: bool,
pub target_truncated: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub degradation_level: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub sampling_rate: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub confidence_score: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub verification_method: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub source_checksums: Option<HashMap<String, String>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub target_checksums: Option<HashMap<String, String>>,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase")]
pub enum DegradationLevel {
Full,
Sample,
SkipWithRisk,
}
impl std::fmt::Display for DegradationLevel {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
DegradationLevel::Full => write!(f, "full"),
DegradationLevel::Sample => write!(f, "sample"),
DegradationLevel::SkipWithRisk => write!(f, "skip_with_risk"),
}
}
}
impl TryFrom<&str> for DegradationLevel {
type Error = String;
fn try_from(s: &str) -> Result<Self, Self::Error> {
match s {
"full" => Ok(DegradationLevel::Full),
"sample" => Ok(DegradationLevel::Sample),
"skip_with_risk" => Ok(DegradationLevel::SkipWithRisk),
_ => Err(format!("Unknown degradation level: {s}")),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(rename_all = "camelCase")]
pub enum SamplingStrategy {
Random,
ExtremeValues,
Hybrid,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct DegradationThreshold {
pub full_compare_max_rows: u64,
pub sample_max_rows: u64,
pub sample_size: usize,
pub extreme_sample_count: usize,
}
impl Default for DegradationThreshold {
fn default() -> Self {
Self {
full_compare_max_rows: 100_000,
sample_max_rows: 10_000_000,
sample_size: 10_000,
extreme_sample_count: 500,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct VerifyDataOptions {
pub source_connection_id: String,
pub source_database: String,
pub source_schema: String,
pub source_table: String,
pub target_connection_id: String,
pub target_database: String,
pub target_schema: String,
pub target_table: String,
pub columns: Vec<String>,
pub key_columns: Vec<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub fetch_batch_size: Option<usize>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub degradation_threshold: Option<DegradationThreshold>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub sampling_strategy: Option<SamplingStrategy>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub enable_checksum: Option<bool>,
}
/// Result of a full verification pass including statistical metadata.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct VerifyDataResult {
pub preparation: DataCompareFromTablesPreparation,
pub degradation_level: DegradationLevel,
pub sampling_rate: f64,
pub confidence_score: f64,
pub row_count_match: bool,
pub checksums_match: Option<bool>,
pub verification_method: String,
}
pub fn prepare_data_compare(options: DataComparePreparationOptions) -> Result<DataComparePreparation, String> {
@@ -238,30 +351,74 @@ pub async fn prepare_data_compare_from_tables(
let source_row_count = first_count(&source_count_result.rows)?;
let target_row_count = first_count(&target_count_result.rows)?;
let (source_rows, target_rows) = tokio::try_join!(
fetch_compare_rows(
state,
&options.source_connection_id,
&options.source_database,
&options.source_schema,
&options.source_table,
&options.columns,
&options.key_columns,
source_database_type,
fetch_batch_size,
),
fetch_compare_rows(
state,
&options.target_connection_id,
&options.target_database,
&options.target_schema,
&options.target_table,
&options.columns,
&options.key_columns,
target_database_type,
fetch_batch_size,
)
)?;
let threshold = options.degradation_threshold.clone().unwrap_or_default();
let degradation_level = should_degrade(source_row_count, target_row_count, &threshold);
let sampling_strategy = options.sampling_strategy.clone().unwrap_or(SamplingStrategy::Hybrid);
let enable_checksum = options.enable_checksum.unwrap_or(true);
let (source_rows, target_rows, sampling_rate, verification_method) = match &degradation_level {
DegradationLevel::Full => {
let (src, tgt) = tokio::try_join!(
fetch_compare_rows(
state,
&options.source_connection_id,
&options.source_database,
&options.source_schema,
&options.source_table,
&options.columns,
&options.key_columns,
source_database_type,
fetch_batch_size,
),
fetch_compare_rows(
state,
&options.target_connection_id,
&options.target_database,
&options.target_schema,
&options.target_table,
&options.columns,
&options.key_columns,
target_database_type,
fetch_batch_size,
)
)?;
(src, tgt, 1.0, "full_compare".to_string())
}
DegradationLevel::Sample => {
let (src, tgt) = tokio::try_join!(
fetch_sampled_compare_rows(
state,
&options.source_connection_id,
&options.source_database,
&options.source_schema,
&options.source_table,
&options.columns,
&options.key_columns,
source_database_type,
&sampling_strategy,
threshold.sample_size,
),
fetch_sampled_compare_rows(
state,
&options.target_connection_id,
&options.target_database,
&options.target_schema,
&options.target_table,
&options.columns,
&options.key_columns,
target_database_type,
&sampling_strategy,
threshold.sample_size,
)
)?;
let max_count = source_row_count.max(target_row_count);
let sample_rate = if max_count > 0 { threshold.sample_size as f64 / max_count as f64 } else { 1.0 };
let method = "sampled".to_string();
(src, tgt, sample_rate.min(1.0), method)
}
DegradationLevel::SkipWithRisk => (Vec::new(), Vec::new(), 0.0, "skipped".to_string()),
};
let target_columns = get_columns_core(
state,
&options.target_connection_id,
@@ -271,6 +428,21 @@ pub async fn prepare_data_compare_from_tables(
)
.await?;
let source_checksums = if enable_checksum && !source_rows.is_empty() {
Some(compute_column_checksums(&options.columns, &source_rows))
} else {
None
};
let target_checksums = if enable_checksum && !target_rows.is_empty() {
Some(compute_column_checksums(&options.columns, &target_rows))
} else {
None
};
let row_count_match = source_row_count == target_row_count;
let confidence_score =
compute_confidence(sampling_rate, &degradation_level, row_count_match, source_row_count, target_row_count);
let preparation = prepare_data_compare(DataComparePreparationOptions {
table_name: options.target_table,
schema: Some(options.target_schema),
@@ -291,6 +463,12 @@ pub async fn prepare_data_compare_from_tables(
target_row_count,
source_truncated: false,
target_truncated: false,
degradation_level: Some(degradation_level.to_string()),
sampling_rate: Some(sampling_rate),
confidence_score: Some(confidence_score),
verification_method: Some(verification_method),
source_checksums,
target_checksums,
})
}
@@ -383,6 +561,12 @@ pub async fn prepare_data_compare_missing_target(
target_row_count: 0,
source_truncated: false,
target_truncated: false,
degradation_level: Some("full".to_string()),
sampling_rate: Some(1.0),
confidence_score: Some(1.0),
verification_method: Some("missing_target_full".to_string()),
source_checksums: None,
target_checksums: None,
})
}
@@ -1066,6 +1250,421 @@ fn data_grid_column_info(column: crate::types::ColumnInfo) -> DataGridColumnInfo
}
}
fn should_degrade(source_row_count: u64, target_row_count: u64, threshold: &DegradationThreshold) -> DegradationLevel {
let max_count = source_row_count.max(target_row_count);
if max_count <= threshold.full_compare_max_rows {
DegradationLevel::Full
} else if max_count <= threshold.sample_max_rows {
DegradationLevel::Sample
} else {
DegradationLevel::SkipWithRisk
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct DegradationEvent {
pub source_row_count: u64,
pub target_row_count: u64,
pub decided_level: String,
pub sample_rate: f64,
pub confidence: f64,
pub auto_decision: String,
pub timestamp: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct DegradationChain {
pub threshold: DegradationThreshold,
pub events: Vec<DegradationEvent>,
pub auto_upgrade_enabled: bool,
pub auto_downgrade_enabled: bool,
}
impl DegradationChain {
pub fn new(threshold: DegradationThreshold) -> Self {
Self { threshold, events: Vec::new(), auto_upgrade_enabled: true, auto_downgrade_enabled: true }
}
pub fn decide(
&mut self,
source_row_count: u64,
target_row_count: u64,
metrics: Option<&crate::risk_metrics::DegradationMetrics>,
) -> DegradationLevel {
let max_count = source_row_count.max(target_row_count);
let new_level = if max_count <= self.threshold.full_compare_max_rows {
DegradationLevel::Full
} else if max_count <= self.threshold.sample_max_rows {
DegradationLevel::Sample
} else {
DegradationLevel::SkipWithRisk
};
let (sample_rate, confidence) = match &new_level {
DegradationLevel::Full => (1.0, 1.0),
DegradationLevel::Sample => {
let rate = self.threshold.sample_size as f64 / max_count as f64;
let conf = 0.95 * rate;
(rate, conf.clamp(0.5, 0.95))
}
DegradationLevel::SkipWithRisk => (0.0, 0.0),
};
if self.auto_upgrade_enabled {
if let Some(last) = self.events.last() {
let auto_upgrade = matches!(
(last.decided_level.as_str(), &new_level),
("skip_with_risk", DegradationLevel::Sample) | ("sample", DegradationLevel::Full)
);
if auto_upgrade {
if let Some(m) = metrics {
m.record_auto_upgrade();
}
}
}
}
if self.auto_downgrade_enabled {
if let Some(last) = self.events.last() {
let auto_downgrade = matches!(
(last.decided_level.as_str(), &new_level),
("full", DegradationLevel::Sample | DegradationLevel::SkipWithRisk)
| ("sample", DegradationLevel::SkipWithRisk)
);
if auto_downgrade {
if let Some(m) = metrics {
m.record_auto_downgrade();
}
}
}
}
if let Some(m) = metrics {
m.record_degradation(&new_level.to_string(), sample_rate, confidence);
}
let auto_decision = if new_level == DegradationLevel::Full {
"auto".to_string()
} else if new_level == DegradationLevel::Sample {
"auto_degraded".to_string()
} else {
"auto_skipped".to_string()
};
self.events.push(DegradationEvent {
source_row_count,
target_row_count,
decided_level: new_level.to_string(),
sample_rate,
confidence,
auto_decision,
timestamp: chrono::Utc::now().to_rfc3339(),
});
new_level
}
pub fn event_count(&self) -> usize {
self.events.len()
}
pub fn last_event(&self) -> Option<&DegradationEvent> {
self.events.last()
}
}
impl Default for DegradationChain {
fn default() -> Self {
Self::new(DegradationThreshold::default())
}
}
fn build_sampling_select_sql(
database_type: DatabaseType,
schema: &str,
table_name: &str,
columns: &[String],
key_columns: &[String],
strategy: &SamplingStrategy,
sample_size: usize,
) -> String {
let table = qualified_table_name(Some(database_type), Some(schema), table_name);
let select_columns = if columns.is_empty() {
"*".to_string()
} else {
columns.iter().map(|column| quote_table_identifier(Some(database_type), column)).collect::<Vec<_>>().join(", ")
};
match strategy {
SamplingStrategy::Random => match database_type {
DatabaseType::Postgres | DatabaseType::Redshift | DatabaseType::DuckDb | DatabaseType::Databricks => {
format!("SELECT {select_columns} FROM {table} TABLESAMPLE SYSTEM (1) LIMIT {sample_size}")
}
DatabaseType::SqlServer => {
format!("SELECT TOP ({sample_size}) {select_columns} FROM {table} TABLESAMPLE ({sample_size} ROWS)")
}
DatabaseType::Mysql | DatabaseType::Doris | DatabaseType::StarRocks | DatabaseType::Goldendb => {
format!("SELECT {select_columns} FROM {table} ORDER BY RAND() LIMIT {sample_size}")
}
DatabaseType::Sqlite | DatabaseType::Rqlite | DatabaseType::Turso => {
format!("SELECT {select_columns} FROM {table} ORDER BY RANDOM() LIMIT {sample_size}")
}
DatabaseType::ClickHouse => {
format!("SELECT {select_columns} FROM {table} ORDER BY rand() LIMIT {sample_size}")
}
DatabaseType::Oracle | DatabaseType::OceanbaseOracle | DatabaseType::Dameng => {
format!("SELECT {select_columns} FROM (SELECT {select_columns} FROM {table} ORDER BY DBMS_RANDOM.VALUE) WHERE ROWNUM <= {sample_size}")
}
DatabaseType::Iris => {
format!("SELECT TOP {sample_size} {select_columns} FROM {table} ORDER BY RAND()")
}
DatabaseType::Questdb => {
format!("SELECT {select_columns} FROM {table} ORDER BY RAND() LIMIT {sample_size}")
}
DatabaseType::Informix => {
format!("SELECT FIRST {sample_size} {select_columns} FROM {table} ORDER BY RAND()")
}
_ => {
if key_columns.is_empty() {
format!("SELECT {select_columns} FROM {table} LIMIT {sample_size}")
} else {
let order_by = key_columns
.iter()
.map(|column| format!("{} ASC", quote_table_identifier(Some(database_type), column)))
.collect::<Vec<_>>()
.join(", ");
format!("SELECT {select_columns} FROM {table} ORDER BY {order_by} LIMIT {sample_size}")
}
}
},
SamplingStrategy::ExtremeValues => {
if key_columns.is_empty() {
return format!("SELECT {select_columns} FROM {table} LIMIT {sample_size}");
}
let order_keys = key_columns
.iter()
.map(|column| format!("{} ASC", quote_table_identifier(Some(database_type), column)))
.collect::<Vec<_>>();
let asc_order = order_keys.join(", ");
let desc_order = key_columns
.iter()
.map(|column| format!("{} DESC", quote_table_identifier(Some(database_type), column)))
.collect::<Vec<_>>()
.join(", ");
let head_count = sample_size / 2;
let tail_count = sample_size - head_count;
format!(
"SELECT {select_columns} FROM ( \
(SELECT {select_columns} FROM {table} ORDER BY {asc_order} LIMIT {head_count}) \
UNION ALL \
(SELECT {select_columns} FROM {table} ORDER BY {desc_order} LIMIT {tail_count}) \
) AS _extreme_sample"
)
}
SamplingStrategy::Hybrid => {
if key_columns.is_empty() {
return format!("SELECT {select_columns} FROM {table} LIMIT {sample_size}");
}
let order_keys = key_columns
.iter()
.map(|column| format!("{} ASC", quote_table_identifier(Some(database_type), column)))
.collect::<Vec<_>>();
let asc_order = order_keys.join(", ");
let desc_order = key_columns
.iter()
.map(|column| format!("{} DESC", quote_table_identifier(Some(database_type), column)))
.collect::<Vec<_>>()
.join(", ");
let random_count = sample_size / 2;
let head_count = (sample_size - random_count) / 2;
let tail_count = sample_size - random_count - head_count;
let random_part = match database_type {
DatabaseType::Postgres | DatabaseType::Redshift | DatabaseType::DuckDb | DatabaseType::Databricks => {
format!("(SELECT {select_columns} FROM {table} TABLESAMPLE SYSTEM (1) LIMIT {random_count})")
}
DatabaseType::SqlServer => {
format!(
"(SELECT TOP ({random_count}) {select_columns} FROM {table} TABLESAMPLE ({random_count} ROWS))"
)
}
DatabaseType::Mysql | DatabaseType::Doris | DatabaseType::StarRocks | DatabaseType::Goldendb => {
format!("(SELECT {select_columns} FROM {table} ORDER BY RAND() LIMIT {random_count})")
}
DatabaseType::Sqlite | DatabaseType::Rqlite | DatabaseType::Turso => {
format!("(SELECT {select_columns} FROM {table} ORDER BY RANDOM() LIMIT {random_count})")
}
DatabaseType::ClickHouse => {
format!("(SELECT {select_columns} FROM {table} ORDER BY rand() LIMIT {random_count})")
}
DatabaseType::Oracle | DatabaseType::OceanbaseOracle | DatabaseType::Dameng => {
format!("(SELECT {select_columns} FROM (SELECT {select_columns} FROM {table} ORDER BY DBMS_RANDOM.VALUE) WHERE ROWNUM <= {random_count})")
}
DatabaseType::Iris => {
format!("(SELECT TOP {random_count} {select_columns} FROM {table} ORDER BY RAND())")
}
DatabaseType::Questdb => {
format!("(SELECT {select_columns} FROM {table} ORDER BY RAND() LIMIT {random_count})")
}
DatabaseType::Informix => {
format!("(SELECT FIRST {random_count} {select_columns} FROM {table} ORDER BY RAND())")
}
_ => {
format!("(SELECT {select_columns} FROM {table} LIMIT {random_count})")
}
};
format!(
"SELECT {select_columns} FROM ( \
{random_part} \
UNION ALL \
(SELECT {select_columns} FROM {table} ORDER BY {asc_order} LIMIT {head_count}) \
UNION ALL \
(SELECT {select_columns} FROM {table} ORDER BY {desc_order} LIMIT {tail_count}) \
) AS _hybrid_sample"
)
}
}
}
#[allow(clippy::too_many_arguments)]
async fn fetch_sampled_compare_rows(
state: &AppState,
connection_id: &str,
database: &str,
schema: &str,
table_name: &str,
columns: &[String],
key_columns: &[String],
database_type: DatabaseType,
strategy: &SamplingStrategy,
sample_size: usize,
) -> Result<Vec<Vec<Value>>, String> {
if sample_size == 0 {
return Ok(Vec::new());
}
let sql = build_sampling_select_sql(database_type, schema, table_name, columns, key_columns, strategy, sample_size);
let result = execute_sql_statement_with_options(
state,
connection_id,
database,
&sql,
Some(schema),
None,
QueryExecutionOptions { max_rows: Some(sample_size), ..Default::default() },
)
.await?;
Ok(result.rows)
}
fn compute_column_checksums(columns: &[String], rows: &[Vec<Value>]) -> HashMap<String, String> {
let mut column_hashers: HashMap<String, Sha256> = columns.iter().map(|col| (col.clone(), Sha256::new())).collect();
let column_indexes = column_index_map(columns);
for row in rows {
for (column_name, index) in &column_indexes {
if let Some(hasher) = column_hashers.get_mut(*column_name) {
let value_str = json_stringify(row_value(row, *index));
hasher.update(value_str.as_bytes());
hasher.update(b"\n");
}
}
}
column_hashers
.into_iter()
.map(|(name, hasher)| {
let hash = format!("{:x}", hasher.finalize());
(name, hash)
})
.collect()
}
fn compute_confidence(
sampling_rate: f64,
degradation_level: &DegradationLevel,
row_count_match: bool,
source_row_count: u64,
target_row_count: u64,
) -> f64 {
match degradation_level {
DegradationLevel::Full => {
if row_count_match {
1.0
} else {
0.99
}
}
DegradationLevel::Sample => {
let count_ratio =
source_row_count.min(target_row_count) as f64 / source_row_count.max(target_row_count).max(1) as f64;
let base = if row_count_match { 0.95 } else { 0.85 };
let adjusted = base * sampling_rate * count_ratio.sqrt();
adjusted.clamp(0.5, 0.95)
}
DegradationLevel::SkipWithRisk => 0.0,
}
}
pub async fn verify_data(state: &AppState, options: VerifyDataOptions) -> Result<VerifyDataResult, String> {
let degradation_threshold = options.degradation_threshold.unwrap_or_default();
let sampling_strategy = options.sampling_strategy.unwrap_or(SamplingStrategy::Hybrid);
let enable_checksum = options.enable_checksum.unwrap_or(true);
let from_tables_options = DataCompareFromTablesOptions {
source_connection_id: options.source_connection_id,
source_database: options.source_database,
source_schema: options.source_schema,
source_table: options.source_table,
target_connection_id: options.target_connection_id,
target_database: options.target_database,
target_schema: options.target_schema,
target_table: options.target_table,
columns: options.columns,
key_columns: options.key_columns,
fetch_batch_size: options.fetch_batch_size,
degradation_threshold: Some(degradation_threshold),
sampling_strategy: Some(sampling_strategy),
enable_checksum: Some(enable_checksum),
};
let preparation = prepare_data_compare_from_tables(state, from_tables_options).await?;
let degradation_level = preparation
.degradation_level
.as_deref()
.and_then(|s| DegradationLevel::try_from(s).ok())
.unwrap_or(DegradationLevel::SkipWithRisk);
let sampling_rate = preparation.sampling_rate.unwrap_or(1.0);
let confidence_score = preparation.confidence_score.unwrap_or(0.0);
let row_count_match = preparation.source_row_count == preparation.target_row_count;
let checksums_match = match (&preparation.source_checksums, &preparation.target_checksums) {
(Some(src), Some(tgt)) => Some(src == tgt),
_ => None,
};
let verification_method = preparation.verification_method.clone().unwrap_or_else(|| "unknown".to_string());
Ok(VerifyDataResult {
preparation,
degradation_level,
sampling_rate,
confidence_score,
row_count_match,
checksums_match,
verification_method,
})
}
#[cfg(test)]
mod tests {
use super::*;
@@ -1792,4 +2391,80 @@ mod tests {
"SELECT \"ID\", \"NAME\" FROM \"APP\".\"EVENTS\" ORDER BY \"ID\" ASC FETCH FIRST 25 ROWS ONLY"
);
}
#[test]
fn degradation_chain_full_for_small_table() {
let threshold = DegradationThreshold::default();
let mut chain = DegradationChain::new(threshold);
let level = chain.decide(1000, 1000, None);
assert_eq!(level, DegradationLevel::Full);
assert_eq!(chain.event_count(), 1);
let last = chain.last_event().unwrap();
assert_eq!(last.decided_level, "full");
assert!((last.sample_rate - 1.0).abs() < f64::EPSILON);
assert!((last.confidence - 1.0).abs() < f64::EPSILON);
}
#[test]
fn degradation_chain_sample_for_large_table() {
let threshold = DegradationThreshold::default();
let mut chain = DegradationChain::new(threshold);
let level = chain.decide(500_000, 500_000, None);
assert_eq!(level, DegradationLevel::Sample);
let last = chain.last_event().unwrap();
assert_eq!(last.decided_level, "sample");
assert!(last.sample_rate < 1.0);
assert!(last.confidence >= 0.5);
}
#[test]
fn degradation_chain_skip_for_huge_table() {
let threshold = DegradationThreshold::default();
let mut chain = DegradationChain::new(threshold);
let level = chain.decide(20_000_000, 20_000_000, None);
assert_eq!(level, DegradationLevel::SkipWithRisk);
let last = chain.last_event().unwrap();
assert_eq!(last.decided_level, "skip_with_risk");
assert!((last.sample_rate - 0.0).abs() < f64::EPSILON);
assert!((last.confidence - 0.0).abs() < f64::EPSILON);
}
#[test]
fn degradation_chain_records_multiple_events() {
let threshold = DegradationThreshold::default();
let mut chain = DegradationChain::new(threshold);
chain.decide(1000, 1000, None);
chain.decide(500_000, 500_000, None);
chain.decide(20_000_000, 20_000_000, None);
assert_eq!(chain.event_count(), 3);
}
#[test]
fn degradation_chain_with_metrics_integration() {
let threshold = DegradationThreshold::default();
let mut chain = DegradationChain::new(threshold);
let metrics = crate::risk_metrics::DegradationMetrics::new();
chain.decide(1000, 1000, Some(&metrics));
chain.decide(500_000, 500_000, Some(&metrics));
chain.decide(20_000_000, 20_000_000, Some(&metrics));
let snapshot = metrics.snapshot();
let total = snapshot.iter().find(|e| e.name == "dbx_degradation_total").unwrap();
assert_eq!(total.value, crate::risk_metrics::MetricValue::Counter(3));
}
#[test]
fn degradation_chain_auto_chain_detection() {
let threshold = DegradationThreshold::default();
let mut chain = DegradationChain::new(threshold);
let metrics = crate::risk_metrics::DegradationMetrics::new();
chain.decide(20_000_000, 20_000_000, Some(&metrics));
chain.decide(500_000, 500_000, Some(&metrics));
let snapshot = metrics.snapshot();
let up = snapshot.iter().find(|e| e.name == "dbx_auto_upgrade_total").unwrap();
assert_eq!(up.value, crate::risk_metrics::MetricValue::Counter(1));
}
}
+2
View File
@@ -2255,6 +2255,8 @@ mod tests {
let stderr_tail = Arc::new(Mutex::new(StderrTail::default()));
start_stderr_collector(child.stderr.take().expect("stderr should be piped"), Arc::clone(&stderr_tail));
std::thread::sleep(Duration::from_millis(200));
let message = format_agent_startup_error(
"Failed to read startup line from agent: end of stream",
&mut child,
+5 -2
View File
@@ -2133,10 +2133,13 @@ pub async fn list_indexes(pool: &SqliteHandle, schema: &str, table: &str) -> Res
))
.map_err(|e| e.to_string())?;
let columns = col_stmt
.query_map([], |row| row.get::<_, String>("name"))
.query_map([], |row| row.get::<_, Option<String>>("name"))
.map_err(|e| e.to_string())?
.collect::<Result<Vec<_>, _>>()
.map_err(|e| e.to_string())?;
.map_err(|e| e.to_string())?
.into_iter()
.flatten()
.collect::<Vec<_>>();
indexes.push(IndexInfo {
name,
+4 -4
View File
@@ -367,11 +367,11 @@ async fn connect_and_authenticate(
async fn try_authenticate_with_agent(
session: &mut Handle<SshClient>,
ssh_user: &str,
_ssh_agent_sock_path: &str,
#[cfg_attr(not(unix), allow(unused_variables))] ssh_agent_sock_path: &str,
connect_timeout: &Duration,
) -> Result<(), String> {
#[cfg(unix)]
let mut agent = if _ssh_agent_sock_path.is_empty() {
let mut agent = if ssh_agent_sock_path.is_empty() {
match AgentClient::connect_env().await {
Ok(a) => a,
Err(e) => {
@@ -379,12 +379,12 @@ async fn try_authenticate_with_agent(
}
}
} else {
match AgentClient::connect_uds(_ssh_agent_sock_path).await {
match AgentClient::connect_uds(ssh_agent_sock_path).await {
Ok(a) => a,
Err(e) => {
return Err(format!(
"No SSH password or key provided, and ssh-agent at '{}' is unavailable: {e}",
_ssh_agent_sock_path
ssh_agent_sock_path
));
}
}
+409
View File
@@ -0,0 +1,409 @@
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::RwLock;
use serde::{Deserialize, Serialize};
use crate::sql_dialect::descriptor::DialectKind;
use crate::sql_dialect::dialect_yaml::DialectYaml;
use crate::sql_dialect::inference::{ColumnType, DefaultTypeInferenceEngine, TypeInferenceEngine};
// ============================================================================
// DML Clean Rules — lossy type mapping → pre-transform DML template
// ============================================================================
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DmlCleanRulesFile {
pub rules: Vec<DmlCleanRule>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DmlCleanRule {
pub name: String,
pub source_type: String,
pub target_type: String,
pub max_fidelity: f64,
#[serde(default)]
pub pre_transform_sql: Option<String>,
#[serde(default)]
pub description: Option<String>,
}
impl DmlCleanRule {
pub fn transform_sql(&self, table: &str, column: &str) -> Option<String> {
self.pre_transform_sql.as_ref().map(|tmpl| tmpl.replace("{table}", table).replace("{column}", column))
}
/// Check if this rule matches the source → target type pair.
pub fn matches(&self, source_type: &str, target_type: &str) -> bool {
let src_norm = normalize_type(source_type);
let tgt_norm = normalize_type(target_type);
let rule_src = normalize_type(&self.source_type);
let rule_tgt = normalize_type(&self.target_type);
src_norm == rule_src && tgt_norm == rule_tgt
}
fn applies(&self, source_type: &str, target_type: &str, fidelity: f64) -> bool {
self.matches(source_type, target_type) && fidelity < self.max_fidelity
}
}
fn normalize_type(t: &str) -> String {
t.trim().to_ascii_uppercase().split('(').next().unwrap_or("").to_string()
}
// ============================================================================
// DmlCleanRuleRegistry — global singleton, loads from YAML
// ============================================================================
pub struct DmlCleanRuleRegistry {
pub(crate) rules: Vec<DmlCleanRule>,
}
impl DmlCleanRuleRegistry {
pub fn new() -> Self {
Self { rules: Vec::new() }
}
pub fn global() -> &'static RwLock<Self> {
use std::sync::OnceLock;
static INSTANCE: OnceLock<RwLock<DmlCleanRuleRegistry>> = OnceLock::new();
INSTANCE.get_or_init(|| RwLock::new(Self::new()))
}
pub fn load(path: &Path) -> Result<Vec<DmlCleanRule>, String> {
let content = std::fs::read_to_string(path).map_err(|e| format!("Cannot read {}: {e}", path.display()))?;
let parsed: DmlCleanRulesFile =
serde_yaml::from_str(&content).map_err(|e| format!("YAML parse error in {}: {e}", path.display()))?;
Ok(parsed.rules)
}
pub fn load_default() -> Result<(), String> {
let paths = [
"plugins/mappings/rules/dml_clean_rules.yaml",
"../plugins/mappings/rules/dml_clean_rules.yaml",
"../../plugins/mappings/rules/dml_clean_rules.yaml",
];
for path in &paths {
let p = Path::new(path);
if p.exists() {
let rules = Self::load(p)?;
if let Ok(mut registry) = Self::global().write() {
registry.rules = rules;
}
return Ok(());
}
}
Ok(())
}
pub fn find_rule(&self, source_type: &str, target_type: &str) -> Option<&DmlCleanRule> {
self.rules.iter().find(|r| r.matches(source_type, target_type))
}
pub fn all_rules(&self) -> &[DmlCleanRule] {
&self.rules
}
}
impl Default for DmlCleanRuleRegistry {
fn default() -> Self {
Self::new()
}
}
// ============================================================================
// BindingEngine — auto-binds DML clean SQL to lossy type mappings
// ============================================================================
pub struct BindingEngine;
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct BindingResult {
pub source_type: String,
pub target_type: String,
pub fidelity: f64,
pub pre_transform_sql: Option<String>,
pub rule_name: Option<String>,
}
impl BindingEngine {
/// Compute fidelity for a single type mapping and auto-bind DML if lossy.
pub fn bind(source_type: &str, target_type: &str, table: &str, column: &str, threshold: f64) -> BindingResult {
let engine = DefaultTypeInferenceEngine;
let src_parsed = ColumnType::parse(source_type);
let tgt_parsed = ColumnType::parse(target_type);
let fidelity = engine.type_compatibility_score(&src_parsed, &tgt_parsed);
if let Ok(registry) = DmlCleanRuleRegistry::global().read() {
if let Some(rule) = registry.find_rule(source_type, target_type) {
// A pair-specific rule owns its fidelity boundary; the caller
// threshold remains the fallback for mappings without a rule.
if rule.applies(source_type, target_type, fidelity) {
return BindingResult {
source_type: source_type.to_string(),
target_type: target_type.to_string(),
fidelity,
pre_transform_sql: rule.transform_sql(table, column),
rule_name: Some(rule.name.clone()),
};
}
}
}
if fidelity >= threshold {
return BindingResult {
source_type: source_type.to_string(),
target_type: target_type.to_string(),
fidelity,
pre_transform_sql: None,
rule_name: None,
};
}
BindingResult {
source_type: source_type.to_string(),
target_type: target_type.to_string(),
fidelity,
pre_transform_sql: None,
rule_name: None,
}
}
/// Bind for a full set of column mappings.
pub fn bind_columns(
source_types: &[(String, String)], // (col_name, type)
target_types: &[(String, String)],
table: &str,
threshold: f64,
) -> Vec<BindingResult> {
let source_map: HashMap<&str, &str> = source_types.iter().map(|(n, t)| (n.as_str(), t.as_str())).collect();
target_types
.iter()
.map(|(col_name, target_type)| {
let source_type = source_map.get(col_name.as_str()).copied().unwrap_or(target_type);
Self::bind(source_type, target_type, table, col_name, threshold)
})
.collect()
}
}
// ============================================================================
// MappingCache — cache derivation results to plugins/mappings/base/
// ============================================================================
const MAPPING_BASE_DIR: &str = "plugins/mappings/base";
const MAPPING_CUSTOM_DIR: &str = "plugins/mappings/custom";
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MappingCacheFile {
pub status: String, // "UNVERIFIED" | "VERIFIED" | "REJECTED"
pub generated_at: String,
pub derived_from: MappingDerivationInfo,
pub transformations: Vec<CachedTransformation>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MappingDerivationInfo {
pub source_dialect: String,
pub target_dialect: String,
pub source_type_count: usize,
pub target_type_count: usize,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct CachedTransformation {
pub source_type: String,
pub target_type: String,
pub fidelity: f64,
pub pre_transform_sql: Option<String>,
pub rule_name: Option<String>,
}
impl MappingCacheFile {
pub fn cache_path(source: DialectKind, target: DialectKind) -> PathBuf {
let dir = Path::new(MAPPING_BASE_DIR);
let name = format!("{}_to_{}.base.yaml", source.label(), target.label());
dir.join(name)
}
pub fn custom_path(source: DialectKind, target: DialectKind, env: &str) -> PathBuf {
let dir = Path::new(MAPPING_CUSTOM_DIR);
let name = format!("{}_to_{}.{}.yaml", source.label(), target.label(), env);
dir.join(name)
}
/// Load from cache if exists. Checks custom override first, then base cache.
pub fn load(source: DialectKind, target: DialectKind, env: &str) -> Option<Self> {
let custom = Self::custom_path(source, target, env);
if custom.exists() {
let content = std::fs::read_to_string(&custom).ok()?;
return serde_yaml::from_str(&content).ok();
}
let base = Self::cache_path(source, target);
if base.exists() {
let content = std::fs::read_to_string(&base).ok()?;
return serde_yaml::from_str(&content).ok();
}
None
}
/// Save derivation result to base cache (UNVERIFIED).
pub fn save(source: DialectKind, target: DialectKind, bindings: &[BindingResult]) -> Result<(), String> {
let dir = Path::new(MAPPING_BASE_DIR);
std::fs::create_dir_all(dir).map_err(|e| format!("Cannot create mappings dir: {e}"))?;
let source_yaml = DialectYaml::from_descriptor(source);
let target_yaml = DialectYaml::from_descriptor(target);
let cache = MappingCacheFile {
status: "UNVERIFIED".to_string(),
generated_at: chrono::Utc::now().to_rfc3339(),
derived_from: MappingDerivationInfo {
source_dialect: source.label().to_string(),
target_dialect: target.label().to_string(),
source_type_count: source_yaml.types.len(),
target_type_count: target_yaml.types.len(),
},
transformations: bindings
.iter()
.map(|b| CachedTransformation {
source_type: b.source_type.clone(),
target_type: b.target_type.clone(),
fidelity: b.fidelity,
pre_transform_sql: b.pre_transform_sql.clone(),
rule_name: b.rule_name.clone(),
})
.collect(),
};
let path = Self::cache_path(source, target);
let yaml_str = serde_yaml::to_string(&cache).map_err(|e| format!("YAML serialize: {e}"))?;
std::fs::write(&path, &yaml_str).map_err(|e| format!("Write cache {}: {e}", path.display()))?;
Ok(())
}
}
// ============================================================================
// Tests
// ============================================================================
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn rule_matches_exact() {
let rule = DmlCleanRule {
name: "test".to_string(),
source_type: "BIGINT".to_string(),
target_type: "INT".to_string(),
max_fidelity: 0.8,
pre_transform_sql: Some("UPDATE {table} SET {column} = NULL".to_string()),
description: None,
};
assert!(rule.matches("BIGINT", "INT"));
assert!(!rule.matches("INT", "BIGINT"));
}
#[test]
fn rule_matches_normalized() {
let rule = DmlCleanRule {
name: "test".to_string(),
source_type: "BIGINT".to_string(),
target_type: "INT".to_string(),
max_fidelity: 0.8,
pre_transform_sql: None,
description: None,
};
assert!(rule.matches("BIGINT(20)", "INT"));
assert!(rule.matches("bigint", "int"));
}
#[test]
fn rule_transform_sql() {
let rule = DmlCleanRule {
name: "test".to_string(),
source_type: "BIGINT".to_string(),
target_type: "INT".to_string(),
max_fidelity: 0.8,
pre_transform_sql: Some("UPDATE {table} SET {column} = NULL WHERE {column} > 2147483647".to_string()),
description: None,
};
let sql = rule.transform_sql("users", "age").unwrap();
assert_eq!(sql, "UPDATE users SET age = NULL WHERE age > 2147483647");
}
#[test]
fn binding_engine_high_fidelity_no_dml() {
let result = BindingEngine::bind("INT", "INTEGER", "t", "c", 0.7);
assert!(result.pre_transform_sql.is_none());
assert!((result.fidelity - 0.9).abs() < 0.01);
}
#[test]
fn binding_engine_low_fidelity_with_dml() {
let _registry = DmlCleanRuleRegistry::new();
if let Ok(mut reg) = DmlCleanRuleRegistry::global().write() {
reg.rules = vec![DmlCleanRule {
name: "bigint_to_int".to_string(),
source_type: "BIGINT".to_string(),
target_type: "INT".to_string(),
max_fidelity: 0.9,
pre_transform_sql: Some("UPDATE {table} SET {column} = NULL WHERE {column} > 2147483647".to_string()),
description: None,
}];
}
let result = BindingEngine::bind("BIGINT", "INT", "users", "id", 0.4);
assert!(result.pre_transform_sql.is_some());
assert!(result.pre_transform_sql.unwrap().contains("2147483647"));
}
#[test]
fn rule_max_fidelity_boundary_is_exclusive() {
let rule = DmlCleanRule {
name: "boundary".to_string(),
source_type: "BIGINT".to_string(),
target_type: "INT".to_string(),
max_fidelity: 0.5,
pre_transform_sql: Some("SELECT 1".to_string()),
description: None,
};
assert!(rule.applies("BIGINT", "INT", 0.49));
assert!(!rule.applies("BIGINT", "INT", 0.5));
}
#[test]
fn default_rules_honor_configured_max_fidelity() {
let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../plugins/mappings/rules/dml_clean_rules.yaml");
let rules = DmlCleanRuleRegistry::load(&path).unwrap();
let rule = rules.iter().find(|rule| rule.matches("BIGINT", "INT")).unwrap();
assert!(rule.applies("BIGINT", "INT", rule.max_fidelity - f64::EPSILON));
assert!(!rule.applies("BIGINT", "INT", rule.max_fidelity));
}
#[test]
fn binding_engine_no_rule_found() {
let result = BindingEngine::bind("GEOGRAPHY", "INT", "t", "c", 0.5);
assert!(result.pre_transform_sql.is_none());
assert!((result.fidelity - 0.3).abs() < 0.01);
}
#[test]
fn mapping_cache_path() {
let path = MappingCacheFile::cache_path(DialectKind::Mysql, DialectKind::Postgres);
assert!(path.to_string_lossy().contains("mysql_to_postgres"));
let custom = MappingCacheFile::custom_path(DialectKind::Mysql, DialectKind::Postgres, "prod");
assert!(custom.to_string_lossy().contains("prod"));
}
}
+8
View File
@@ -17,8 +17,10 @@ mod ai_model_filter;
pub mod ai_pi_agent_cli;
pub mod changelog;
pub mod cloud_sync;
pub mod config;
pub mod connection;
pub mod connection_secrets;
pub mod correction;
pub mod csv_export;
pub mod data_compare;
pub mod data_grid_extractors;
@@ -28,6 +30,7 @@ pub mod database_export;
pub mod database_search_sql;
pub mod db;
pub mod db_admin_sql;
pub mod dml_binding;
pub mod document_ops;
pub mod driver_runtime;
pub mod external;
@@ -53,19 +56,23 @@ pub mod query_execution_sql;
pub mod query_result_export;
pub mod query_result_sql;
pub mod redis_ops;
pub mod risk_metrics;
pub mod saved_sql;
pub mod schema;
pub mod schema_diff;
pub mod script_generator;
pub mod sql;
pub mod sql_analysis;
pub mod sql_diagnostics;
pub mod sql_dialect;
pub mod sql_editability;
pub mod sql_file_import;
pub mod sql_parser;
pub mod sql_risk;
pub mod sqlite_backup;
pub(crate) mod sqlserver_temporal;
pub mod ssh_config;
pub mod state_persistence;
pub mod storage;
pub mod table_export;
pub mod table_import;
@@ -75,6 +82,7 @@ pub mod temporal_format;
pub mod text_export;
pub mod token_usage;
pub mod transfer;
pub mod two_phase_commit;
pub mod types;
pub mod update;
pub mod xlsx_export;
+427
View File
@@ -0,0 +1,427 @@
use serde::{Deserialize, Serialize};
use std::sync::Arc;
use crate::state_persistence::StateMachine;
// ============================================================================
// 15.1 & 15.2: External OSC Tool Status
// ============================================================================
/// Status of an external online schema change tool execution.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum OscExecutionStatus {
Preparing,
Copying { progress_pct: f64, rows_copied: u64, rows_total: u64 },
CutOver,
Postponed,
Completed,
Failed(String),
}
/// Type of OSC tool
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum OscToolKind {
GhOst,
PtOsc,
}
impl OscToolKind {
pub fn name(&self) -> &'static str {
match self {
Self::GhOst => "gh-ost",
Self::PtOsc => "pt-osc",
}
}
}
// ============================================================================
// 15.1: gh-ost Probe
// ============================================================================
pub struct GhOstProbe {
progress_history: Vec<(String, f64)>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GhOstProgress {
pub status: OscExecutionStatus,
pub ghost_table_exists: bool,
pub changelog_table_exists: bool,
pub rows_copied: u64,
pub rows_total: u64,
pub progress_pct: f64,
}
impl GhOstProbe {
pub fn new() -> Self {
Self { progress_history: Vec::new() }
}
pub fn probe(&mut self, _database: &str, _table: &str, _has_connection: bool) -> GhOstProgress {
let ghost_table = format!("{_table}_gho");
let changelog_table = format!("{_table}_ghc");
if !_has_connection {
return GhOstProgress {
status: OscExecutionStatus::Copying { progress_pct: 0.0, rows_copied: 0, rows_total: 0 },
ghost_table_exists: false,
changelog_table_exists: false,
rows_copied: 0,
rows_total: 0,
progress_pct: 0.0,
};
}
GhOstProgress {
status: OscExecutionStatus::Copying { progress_pct: 0.0, rows_copied: 0, rows_total: 0 },
ghost_table_exists: false,
changelog_table_exists: false,
rows_copied: 0,
rows_total: 0,
progress_pct: 0.0,
}
}
/// Parse gh-ost log line: "Copying rows: 12345/67890 (18.2%)"
pub fn parse_log_line(line: &str) -> Option<GhOstProgress> {
let upper = line.to_uppercase();
if !upper.contains("COPYING ROWS") {
return None;
}
let re = regex::Regex::new(r"(\d+)\s*/\s*(\d+)\s*\((\d+\.?\d*)%\)").ok()?;
if let Some(caps) = re.captures(line) {
let rows_copied: u64 = caps.get(1)?.as_str().parse().ok()?;
let rows_total: u64 = caps.get(2)?.as_str().parse().ok()?;
let progress_pct: f64 = caps.get(3)?.as_str().parse().ok()?;
let status = if progress_pct >= 99.0 {
OscExecutionStatus::CutOver
} else {
OscExecutionStatus::Copying { progress_pct, rows_copied, rows_total }
};
return Some(GhOstProgress {
status,
ghost_table_exists: true,
changelog_table_exists: true,
rows_copied,
rows_total,
progress_pct,
});
}
None
}
pub fn record_sample(&mut self, progress_pct: f64, label: &str) {
self.progress_history.push((label.to_string(), progress_pct));
if self.progress_history.len() > 1000 {
self.progress_history.remove(0);
}
}
pub fn estimated_remaining_seconds(&self, current_pct: f64) -> Option<f64> {
if self.progress_history.len() < 2 || current_pct <= 0.0 {
return None;
}
let first = self.progress_history.first()?;
let total_pct = current_pct - first.1;
if total_pct <= 0.0 {
return None;
}
let rate = total_pct / 1.0;
if rate <= 0.0 {
return None;
}
Some((100.0 - current_pct) / rate)
}
}
impl Default for GhOstProbe {
fn default() -> Self {
Self::new()
}
}
// ============================================================================
// 15.2: pt-osc Probe
// ============================================================================
pub struct PtOscProbe;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PtOscProgress {
pub status: OscExecutionStatus,
pub new_table_exists: bool,
pub old_table_exists: bool,
pub triggers_active: u32,
pub rows_copied: u64,
pub rows_total: u64,
pub progress_pct: f64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PtOscNames {
pub new_table: String,
pub old_table: String,
pub del_trigger: String,
pub ins_trigger: String,
pub upd_trigger: String,
}
impl PtOscProbe {
pub fn new() -> Self {
Self
}
pub fn object_names(table: &str) -> PtOscNames {
PtOscNames {
new_table: format!("_{table}_new"),
old_table: format!("_{table}_old"),
del_trigger: format!("pt_osc_{table}_del"),
ins_trigger: format!("pt_osc_{table}_ins"),
upd_trigger: format!("pt_osc_{table}_upd"),
}
}
pub fn parse_progress_line(line: &str) -> Option<f64> {
let re = regex::Regex::new(r"(\d+)%\s").ok()?;
if let Some(caps) = re.captures(line) {
return caps.get(1)?.as_str().parse().ok();
}
None
}
}
impl Default for PtOscProbe {
fn default() -> Self {
Self::new()
}
}
// ============================================================================
// 15.3: OscStateBridge — map external status to StateMachine
// ============================================================================
pub struct OscStateBridge {
state_machine: Arc<StateMachine>,
}
impl OscStateBridge {
pub fn new(state_machine: Arc<StateMachine>) -> Self {
Self { state_machine }
}
pub async fn bridge(&self, state_key: &str, external_status: &OscExecutionStatus) -> Result<String, String> {
match external_status {
OscExecutionStatus::Preparing | OscExecutionStatus::Copying { .. } | OscExecutionStatus::Postponed => {
Ok("osc_syncing")
}
OscExecutionStatus::CutOver | OscExecutionStatus::Completed => {
let new_state = crate::state_persistence::StateTransition::Completed;
self.state_machine.transition(state_key, new_state).await?;
Ok("completed")
}
OscExecutionStatus::Failed(_) => {
let new_state = crate::state_persistence::StateTransition::Failed;
self.state_machine.transition(state_key, new_state).await?;
Err(format!("OSC failed: {external_status:?}"))
}
}
.map(|s| s.to_string())
}
}
// ============================================================================
// 15.6: dbdiff_lock — database-level distributed lock
// ============================================================================
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct DbDiffLock {
pub lock_id: String,
pub instance_id: String,
pub acquired_at: String,
pub expires_at: String,
}
impl DbDiffLock {
pub fn create_table_sql() -> &'static str {
"CREATE TABLE IF NOT EXISTS dbdiff_lock (
lock_id TEXT PRIMARY KEY,
instance_id TEXT NOT NULL,
acquired_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
expires_at TIMESTAMP NOT NULL
)"
}
pub fn acquire_sql(lock_id: &str, instance_id: &str, ttl_seconds: u64) -> String {
format!(
"INSERT INTO dbdiff_lock (lock_id, instance_id, expires_at) \
VALUES ('{lock_id}', '{instance_id}', DATEADD('SECOND', {ttl_seconds}, CURRENT_TIMESTAMP))"
)
}
pub fn check_sql(lock_id: &str) -> String {
format!(
"SELECT instance_id, acquired_at, expires_at FROM dbdiff_lock \
WHERE lock_id = '{lock_id}' AND expires_at > CURRENT_TIMESTAMP"
)
}
pub fn release_sql(lock_id: &str, instance_id: &str) -> String {
format!("DELETE FROM dbdiff_lock WHERE lock_id = '{lock_id}' AND instance_id = '{instance_id}'")
}
pub fn cleanup_expired_sql() -> &'static str {
"DELETE FROM dbdiff_lock WHERE expires_at <= CURRENT_TIMESTAMP"
}
pub fn extend_sql(lock_id: &str, instance_id: &str, ttl_seconds: u64) -> String {
format!(
"UPDATE dbdiff_lock SET expires_at = DATEADD('SECOND', {ttl_seconds}, CURRENT_TIMESTAMP) \
WHERE lock_id = '{lock_id}' AND instance_id = '{instance_id}'"
)
}
}
// ============================================================================
// Tests
// ============================================================================
#[cfg(test)]
mod tests {
use super::*;
// --- gh-ost log parsing ---
#[test]
fn gh_ost_parse_log_copying() {
let line = "Copying rows: 12345/67890 (18.2%)";
let result = GhOstProbe::parse_log_line(line).unwrap();
assert_eq!(result.rows_copied, 12345);
assert_eq!(result.rows_total, 67890);
assert!((result.progress_pct - 18.2).abs() < 0.01);
match result.status {
OscExecutionStatus::Copying { .. } => {}
_ => panic!("should be Copying"),
}
}
#[test]
fn gh_ost_parse_log_cutover() {
let line = "Copying rows: 67889/67890 (100.0%)";
let result = GhOstProbe::parse_log_line(line).unwrap();
match result.status {
OscExecutionStatus::CutOver => {}
_ => panic!("should be CutOver at 100%"),
}
}
#[test]
fn gh_ost_parse_non_copying_line() {
let line = "2024/01/01 12:00:00 Starting gh-ost...";
assert!(GhOstProbe::parse_log_line(line).is_none());
}
// --- pt-osc ---
#[test]
fn pt_osc_get_object_names() {
let names = PtOscProbe::object_names("users");
assert_eq!(names.new_table, "_users_new");
assert_eq!(names.del_trigger, "pt_osc_users_del");
}
#[test]
fn pt_osc_parse_progress() {
let line = "Copying `mydb`.`users`: 45% 00:30 remaining";
let pct = PtOscProbe::parse_progress_line(line).unwrap();
assert!((pct - 45.0).abs() < 0.01);
}
// --- OSC status serialization ---
#[test]
fn osc_status_serialization() {
let status = OscExecutionStatus::Copying { progress_pct: 45.5, rows_copied: 100, rows_total: 200 };
let json = serde_json::to_string(&status).unwrap();
assert!(json.contains("45.5"));
let deserialized: OscExecutionStatus = serde_json::from_str(&json).unwrap();
assert_eq!(status, deserialized);
}
// --- dbdiff_lock ---
#[test]
fn dbdiff_lock_acquire_sql() {
let sql = DbDiffLock::acquire_sql("mig-001", "node-1", 300);
assert!(sql.contains("INSERT INTO dbdiff_lock"));
assert!(sql.contains("mig-001"));
}
#[test]
fn dbdiff_lock_release_sql() {
let sql = DbDiffLock::release_sql("mig-001", "node-1");
assert!(sql.contains("DELETE FROM dbdiff_lock"));
}
#[test]
fn dbdiff_lock_check_sql() {
let sql = DbDiffLock::check_sql("mig-001");
assert!(sql.contains("expires_at > CURRENT_TIMESTAMP"));
}
#[test]
fn dbdiff_lock_create_table_sql() {
let sql = DbDiffLock::create_table_sql();
assert!(sql.contains("CREATE TABLE IF NOT EXISTS dbdiff_lock"));
}
// --- State machine extended transition tests ---
#[test]
fn state_machine_new_transitions_valid() {
use crate::state_persistence::{is_valid_transition, StateTransition};
assert!(is_valid_transition(&StateTransition::Running, &StateTransition::OscSyncing));
assert!(is_valid_transition(&StateTransition::OscSyncing, &StateTransition::Completed));
assert!(is_valid_transition(&StateTransition::OscSyncing, &StateTransition::Failed));
assert!(is_valid_transition(&StateTransition::Failed, &StateTransition::RollingBack));
assert!(is_valid_transition(&StateTransition::RollingBack, &StateTransition::FullyRolledBack));
assert!(is_valid_transition(&StateTransition::RollingBack, &StateTransition::PartiallyRolledBack));
assert!(is_valid_transition(&StateTransition::PartiallyRolledBack, &StateTransition::RecoveryRequired));
}
#[test]
fn state_machine_new_transitions_invalid() {
use crate::state_persistence::{is_valid_transition, StateTransition};
assert!(!is_valid_transition(&StateTransition::Created, &StateTransition::OscSyncing));
assert!(!is_valid_transition(&StateTransition::OscSyncing, &StateTransition::Running));
assert!(!is_valid_transition(&StateTransition::FullyRolledBack, &StateTransition::Running));
}
#[test]
fn state_machine_full_lifecycle_with_osc() {
use crate::state_persistence::{is_valid_transition, StateTransition};
let lifecycle = ["created", "running", "osc_syncing", "completed"];
// Validate each step
let states = [
StateTransition::Created,
StateTransition::Running,
StateTransition::OscSyncing,
StateTransition::Completed,
];
for i in 0..states.len() - 1 {
assert!(is_valid_transition(&states[i], &states[i + 1]));
}
}
#[test]
fn state_machine_rollback_scenarios() {
use crate::state_persistence::{is_valid_transition, StateTransition};
// Failed → RollingBack → FullyRolledBack
assert!(is_valid_transition(&StateTransition::Failed, &StateTransition::RollingBack));
assert!(is_valid_transition(&StateTransition::RollingBack, &StateTransition::FullyRolledBack));
// Failed → RollingBack → PartiallyRolledBack → RecoveryRequired
assert!(is_valid_transition(&StateTransition::RollingBack, &StateTransition::PartiallyRolledBack));
assert!(is_valid_transition(&StateTransition::PartiallyRolledBack, &StateTransition::RecoveryRequired));
// Failed → RecoveryRequired (direct)
assert!(is_valid_transition(&StateTransition::Failed, &StateTransition::RecoveryRequired));
}
}
+378
View File
@@ -33,6 +33,8 @@ use crate::query_execution_sql::is_write_sql;
#[cfg(feature = "duckdb-bundled")]
use crate::sql::starts_with_duckdb_result_sql_keyword;
use crate::sql::{split_sql_batches, split_sql_statements, starts_with_executable_sql_keyword};
use crate::sql_dialect::{resolve_for_db, CAP_TRANSACTIONAL_DDL};
use crate::sql_risk::{classify_sql_risk_for_database, SqlRisk};
pub const QUERY_TIMEOUT: Duration = Duration::from_secs(30);
pub const MAX_ROWS: usize = 10000;
@@ -475,6 +477,7 @@ fn duckdb_value_to_json(row: &duckdb::Row<'_>, idx: usize) -> serde_json::Value
| ValueRef::Map(..)
| ValueRef::Enum(..)
| ValueRef::Union(..) => duckdb_owned_value_to_json(&value_ref.to_owned()),
_ => duckdb_owned_value_to_json(&value_ref.to_owned()),
}
}
@@ -534,6 +537,7 @@ fn duckdb_owned_value_to_json(value: &Value) -> serde_json::Value {
.collect(),
),
Value::Union(value) => duckdb_owned_value_to_json(value),
_ => serde_json::Value::Null,
}
}
@@ -2587,6 +2591,254 @@ fn is_agent_execute_batch_unsupported(error: &str) -> bool {
lower.contains("execute_batch") && (lower.contains("unknown method") || lower.contains("method not found"))
}
/// Deploy result for Schema Diff: single-connection transactional execution.
/// On statement failure the transaction is rolled back by the underlying path.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct SchemaDiffDeployResult {
pub transaction_id: String,
pub status: String,
pub participants: Vec<crate::two_phase_commit::ParticipantInfo>,
pub created_at: String,
pub updated_at: String,
pub executed_count: usize,
pub statement_count: usize,
pub error: Option<String>,
pub metadata: serde_json::Value,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum SchemaDiffAtomicity {
GuaranteedRollback,
PartialEffectsPossible,
}
impl SchemaDiffAtomicity {
fn ddl_atomic(self) -> bool {
matches!(self, Self::GuaranteedRollback)
}
}
fn database_supports_transactional_ddl(db_type: DatabaseType) -> bool {
if resolve_for_db(db_type).has_capability(CAP_TRANSACTIONAL_DDL) {
return true;
}
// SQLite-family DDL is transactional even when dialect registry omits the flag.
matches!(db_type, DatabaseType::Sqlite | DatabaseType::Rqlite | DatabaseType::Turso | DatabaseType::CloudflareD1)
}
fn classify_schema_diff_atomicity(
db_type: Option<DatabaseType>,
statements: &[String],
has_transactional_path: bool,
) -> SchemaDiffAtomicity {
if !has_transactional_path {
return SchemaDiffAtomicity::PartialEffectsPossible;
}
let mut contains_ddl = false;
for statement in statements {
let risk = match db_type {
Some(db_type) => classify_sql_risk_for_database(statement, db_type).ok(),
None => None,
};
match risk {
Some(SqlRisk::Ddl) => {
contains_ddl = true;
}
Some(SqlRisk::Write | SqlRisk::ReadOnly) => {}
Some(SqlRisk::Transaction) | None => {
return SchemaDiffAtomicity::PartialEffectsPossible;
}
}
}
if !contains_ddl {
return SchemaDiffAtomicity::GuaranteedRollback;
}
match db_type {
Some(db_type) if database_supports_transactional_ddl(db_type) => SchemaDiffAtomicity::GuaranteedRollback,
_ => SchemaDiffAtomicity::PartialEffectsPossible,
}
}
fn executed_count_before_error(error: &str, statement_count: usize) -> usize {
let Some(rest) = error.strip_prefix("Statement ") else {
return statement_count;
};
let digits: String = rest.chars().take_while(|ch| ch.is_ascii_digit()).collect();
let Ok(statement_number) = digits.parse::<usize>() else {
return statement_count;
};
statement_number.saturating_sub(1).min(statement_count)
}
/// Pure failure mapping used by deploy and unit tests.
fn schema_diff_failure_outcome(
atomicity: SchemaDiffAtomicity,
error: &str,
statement_count: usize,
) -> (crate::two_phase_commit::TransactionStatus, usize) {
if atomicity.ddl_atomic() {
(crate::two_phase_commit::TransactionStatus::RolledBack, 0)
} else {
(crate::two_phase_commit::TransactionStatus::Mixed, executed_count_before_error(error, statement_count))
}
}
fn pool_kind_has_transactional_path(pool: &PoolKind) -> bool {
// DuckDb variants always exist (unit stubs when duckdb-bundled is off).
match pool {
PoolKind::Postgres(_)
| PoolKind::Mysql(_, _)
| PoolKind::Sqlite(_)
| PoolKind::CloudflareD1(_)
| PoolKind::ClickHouse(_)
| PoolKind::Rqlite(_)
| PoolKind::Turso(_)
| PoolKind::SqlServer(_)
| PoolKind::Agent(_) => true,
PoolKind::MessageQueue
| PoolKind::Nacos
| PoolKind::HBase(_)
| PoolKind::DuckDb(_)
| PoolKind::DuckDbWorker(_)
| PoolKind::Redis(_)
| PoolKind::MongoDb(_)
| PoolKind::Elasticsearch(_)
| PoolKind::VectorDb(_)
| PoolKind::InfluxDb(_)
| PoolKind::ExternalTabular(_)
| PoolKind::ExternalDriver { .. } => false,
}
}
/// Execute Schema Diff deploy SQL as one real single-connection transaction.
///
/// - Uses [`execute_statements_in_transaction`] so partial success rolls back.
/// - Returns a structured result (never re-executes statements to probe status).
/// - Comment-only / empty scripts succeed as `committed` with zero statements.
/// - When the target path cannot guarantee DDL atomicity (MySQL/Oracle DDL
/// auto-commit, `TxPath::None`, etc.), a failure reports `mixed` and
/// `executed_count` reflects the statements that were issued before the
/// error, so the caller can warn the user that partial effects may persist.
pub async fn execute_schema_diff_deploy(
state: &AppState,
connection_id: &str,
database: &str,
statements: &[String],
schema: Option<&str>,
) -> SchemaDiffDeployResult {
let tx_id = format!("deploy_{}", uuid::Uuid::new_v4());
let now = chrono::Utc::now().to_rfc3339();
let db_type = connection_database_type(state, connection_id).await;
let parsed: Vec<String> = statements
.iter()
.flat_map(|s| {
db_type.map_or_else(|| split_sql_statements(s), |dt| crate::sql::split_sql_statements_for_database(s, dt))
})
.map(|s| s.trim().to_string())
.filter(|s| {
!s.is_empty()
&& !s.lines().all(|line| {
let t = line.trim();
t.is_empty() || t.starts_with("--")
})
})
.collect();
let participant = crate::two_phase_commit::ParticipantInfo {
id: "connection".to_string(),
name: format!("{connection_id}/{database}"),
role: "database".to_string(),
};
if parsed.is_empty() {
return SchemaDiffDeployResult {
transaction_id: tx_id,
status: crate::two_phase_commit::TransactionStatus::Committed.as_str().to_string(),
participants: vec![participant],
created_at: now.clone(),
updated_at: now,
executed_count: 0,
statement_count: 0,
error: None,
metadata: serde_json::json!({"source": "schema_diff_deploy", "mode": "single_connection_tx"}),
};
}
let pool_key = if database.is_empty() {
connection_id.to_string()
} else {
match state.get_or_create_pool(connection_id, Some(database)).await {
Ok(key) => key,
Err(_) => {
return SchemaDiffDeployResult {
transaction_id: tx_id.clone(),
status: crate::two_phase_commit::TransactionStatus::RolledBack.as_str().to_string(),
participants: vec![participant],
created_at: now.clone(),
updated_at: chrono::Utc::now().to_rfc3339(),
executed_count: 0,
statement_count: parsed.len(),
error: Some("Connection not available for deploy".to_string()),
metadata: serde_json::json!({"source": "schema_diff_deploy", "mode": "single_connection_tx"}),
};
}
}
};
let has_transactional_path = {
let conns = state.connections.read().await;
conns.get(&pool_key).is_some_and(pool_kind_has_transactional_path)
};
let atomicity = classify_schema_diff_atomicity(db_type, &parsed, has_transactional_path);
match execute_statements_in_transaction_on_pool(state, &pool_key, connection_id, database, &parsed, schema).await {
Ok(result) => SchemaDiffDeployResult {
transaction_id: tx_id,
status: crate::two_phase_commit::TransactionStatus::Committed.as_str().to_string(),
participants: vec![participant],
created_at: now.clone(),
updated_at: chrono::Utc::now().to_rfc3339(),
executed_count: parsed.len(),
statement_count: parsed.len(),
error: None,
metadata: serde_json::json!({
"source": "schema_diff_deploy",
"mode": "single_connection_tx",
"affected_rows": result.affected_rows,
"execution_time_ms": result.execution_time_ms,
}),
},
Err(e) => {
let (status, executed_count) = schema_diff_failure_outcome(atomicity, &e, parsed.len());
SchemaDiffDeployResult {
transaction_id: tx_id,
status: status.as_str().to_string(),
participants: vec![participant],
created_at: now.clone(),
updated_at: chrono::Utc::now().to_rfc3339(),
executed_count,
statement_count: parsed.len(),
error: Some(e.clone()),
metadata: serde_json::json!({
"source": "schema_diff_deploy",
"mode": "single_connection_tx",
"ddl_atomic": atomicity.ddl_atomic(),
"atomicity": match atomicity {
SchemaDiffAtomicity::GuaranteedRollback => "guaranteed_rollback",
SchemaDiffAtomicity::PartialEffectsPossible => "partial_effects_possible",
},
"error": e,
}),
}
}
}
}
/// Execute multiple SQL statements within a single transaction.
/// For pooled drivers (Postgres/MySQL), uses the driver transaction API.
/// For SQLite and already-single-connection drivers (ClickHouse/SqlServer/Agent),
@@ -3637,6 +3889,132 @@ mod tests {
};
use crate::storage::Storage;
#[test]
fn schema_diff_atomicity_marks_mysql_ddl_as_partial() {
let atomicity = classify_schema_diff_atomicity(
Some(DatabaseType::Mysql),
&["CREATE TABLE users (id INT)".to_string(), "ALTER TABLE users ADD COLUMN name VARCHAR(32)".to_string()],
true,
);
assert_eq!(atomicity, SchemaDiffAtomicity::PartialEffectsPossible);
}
#[test]
fn schema_diff_atomicity_marks_oracle_ddl_as_partial() {
let atomicity = classify_schema_diff_atomicity(
Some(DatabaseType::Oracle),
&["CREATE TABLE users (id INT)".to_string(), "ALTER TABLE users ADD name VARCHAR2(32)".to_string()],
true,
);
assert_eq!(atomicity, SchemaDiffAtomicity::PartialEffectsPossible);
}
#[test]
fn schema_diff_atomicity_keeps_postgres_ddl_atomic() {
let atomicity = classify_schema_diff_atomicity(
Some(DatabaseType::Postgres),
&["CREATE TABLE users (id INT)".to_string(), "ALTER TABLE users ADD COLUMN name VARCHAR(32)".to_string()],
true,
);
assert_eq!(atomicity, SchemaDiffAtomicity::GuaranteedRollback);
}
#[test]
fn schema_diff_atomicity_keeps_dml_atomic_when_tx_path_exists() {
let atomicity = classify_schema_diff_atomicity(
Some(DatabaseType::Mysql),
&["INSERT INTO users VALUES (1)".to_string(), "UPDATE users SET active = 1 WHERE id = 1".to_string()],
true,
);
assert_eq!(atomicity, SchemaDiffAtomicity::GuaranteedRollback);
}
#[test]
fn schema_diff_atomicity_marks_missing_transaction_path_as_partial() {
let atomicity = classify_schema_diff_atomicity(
Some(DatabaseType::Postgres),
&["CREATE TABLE users (id INT)".to_string()],
false,
);
assert_eq!(atomicity, SchemaDiffAtomicity::PartialEffectsPossible);
}
#[test]
fn executed_count_before_error_uses_failing_statement_index() {
assert_eq!(executed_count_before_error("Statement 1 failed: syntax error", 3), 0);
assert_eq!(executed_count_before_error("Statement 2 failed: syntax error", 3), 1);
assert_eq!(executed_count_before_error("Statement 3 failed: syntax error", 3), 2);
}
#[test]
fn schema_diff_failure_outcome_rolls_back_when_atomic() {
let (status, executed) =
schema_diff_failure_outcome(SchemaDiffAtomicity::GuaranteedRollback, "Statement 2 failed: syntax error", 3);
assert_eq!(status, crate::two_phase_commit::TransactionStatus::RolledBack);
assert_eq!(executed, 0);
}
#[test]
fn schema_diff_failure_outcome_reports_mixed_with_partial_count() {
let (status, executed) = schema_diff_failure_outcome(
SchemaDiffAtomicity::PartialEffectsPossible,
"Statement 2 failed: syntax error",
3,
);
assert_eq!(status, crate::two_phase_commit::TransactionStatus::Mixed);
assert_eq!(executed, 1);
}
#[test]
fn schema_diff_atomicity_keeps_sqlite_ddl_atomic() {
let atomicity = classify_schema_diff_atomicity(
Some(DatabaseType::Sqlite),
&["CREATE TABLE users (id INTEGER)".to_string()],
true,
);
assert_eq!(atomicity, SchemaDiffAtomicity::GuaranteedRollback);
}
/// MySQL: first DDL may already commit; second fails → mixed + executed_count = 1.
#[test]
fn mysql_second_ddl_failure_maps_to_mixed_with_partial_executed_count() {
let stmts = ["CREATE TABLE t1 (id INT)".to_string(), "CREATE TABLE t2 (id INT)".to_string()];
let atomicity = classify_schema_diff_atomicity(Some(DatabaseType::Mysql), &stmts, true);
assert_eq!(atomicity, SchemaDiffAtomicity::PartialEffectsPossible);
let (status, executed) =
schema_diff_failure_outcome(atomicity, "Statement 2 failed: table already exists", stmts.len());
assert_eq!(status, crate::two_phase_commit::TransactionStatus::Mixed);
assert_eq!(executed, 1);
}
/// Oracle: same non-transactional DDL semantics as MySQL for deploy status.
#[test]
fn oracle_second_ddl_failure_maps_to_mixed_with_partial_executed_count() {
let stmts = ["CREATE TABLE t1 (id NUMBER)".to_string(), "ALTER TABLE t1 ADD name VARCHAR2(32)".to_string()];
let atomicity = classify_schema_diff_atomicity(Some(DatabaseType::Oracle), &stmts, true);
assert_eq!(atomicity, SchemaDiffAtomicity::PartialEffectsPossible);
let (status, executed) = schema_diff_failure_outcome(atomicity, "Statement 2 failed: ORA-00942", stmts.len());
assert_eq!(status, crate::two_phase_commit::TransactionStatus::Mixed);
assert_eq!(executed, 1);
}
/// Postgres transactional DDL: second fails → rolled_back + executed_count = 0.
#[test]
fn postgres_second_ddl_failure_maps_to_rolled_back_zero_executed() {
let stmts = ["CREATE TABLE t1 (id INT)".to_string(), "CREATE TABLE t2 (id INT)".to_string()];
let atomicity = classify_schema_diff_atomicity(Some(DatabaseType::Postgres), &stmts, true);
assert_eq!(atomicity, SchemaDiffAtomicity::GuaranteedRollback);
let (status, executed) =
schema_diff_failure_outcome(atomicity, "Statement 2 failed: relation already exists", stmts.len());
assert_eq!(status, crate::two_phase_commit::TransactionStatus::RolledBack);
assert_eq!(executed, 0);
}
#[test]
fn query_execution_mode_deserializes_simple_client_value() {
let mode: QueryExecutionMode = serde_json::from_str("\"simple\"").unwrap();
+376
View File
@@ -0,0 +1,376 @@
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Mutex;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum MetricType {
Counter,
Gauge,
Histogram,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MetricEntry {
pub name: String,
pub metric_type: MetricType,
pub description: String,
pub labels: HashMap<String, String>,
pub value: MetricValue,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(untagged)]
pub enum MetricValue {
Counter(u64),
Gauge(f64),
Histogram(Vec<(f64, u64)>),
}
pub struct DegradationMetrics {
degradation_total: AtomicU64,
degradation_level_full: AtomicU64,
degradation_level_sample: AtomicU64,
degradation_level_skip: AtomicU64,
sample_rate_histogram: Mutex<Vec<(f64, u64)>>,
confidence_gauge: Mutex<f64>,
auto_upgrade_total: AtomicU64,
auto_downgrade_total: AtomicU64,
// Phase 15.5: OSC and rollback metrics
osc_probe_latency: Mutex<Vec<f64>>,
osc_execution_gauge: Mutex<f64>,
rollback_trigger_count: AtomicU64,
tag_block_count: AtomicU64,
trace_dropped_count: AtomicU64,
}
impl DegradationMetrics {
pub fn new() -> Self {
Self {
degradation_total: AtomicU64::new(0),
degradation_level_full: AtomicU64::new(0),
degradation_level_sample: AtomicU64::new(0),
degradation_level_skip: AtomicU64::new(0),
sample_rate_histogram: Mutex::new(Vec::new()),
confidence_gauge: Mutex::new(0.0),
auto_upgrade_total: AtomicU64::new(0),
auto_downgrade_total: AtomicU64::new(0),
osc_probe_latency: Mutex::new(Vec::new()),
osc_execution_gauge: Mutex::new(0.0),
rollback_trigger_count: AtomicU64::new(0),
tag_block_count: AtomicU64::new(0),
trace_dropped_count: AtomicU64::new(0),
}
}
pub fn record_degradation(&self, level: &str, sample_rate: f64, confidence: f64) {
self.degradation_total.fetch_add(1, Ordering::Relaxed);
match level {
"full" => {
self.degradation_level_full.fetch_add(1, Ordering::Relaxed);
}
"sample" => {
self.degradation_level_sample.fetch_add(1, Ordering::Relaxed);
}
"skip_with_risk" => {
self.degradation_level_skip.fetch_add(1, Ordering::Relaxed);
}
_ => {}
}
if let Ok(mut h) = self.sample_rate_histogram.lock() {
h.push((sample_rate, 1));
}
if let Ok(mut g) = self.confidence_gauge.lock() {
*g = confidence;
}
}
pub fn record_auto_upgrade(&self) {
self.auto_upgrade_total.fetch_add(1, Ordering::Relaxed);
}
pub fn record_auto_downgrade(&self) {
self.auto_downgrade_total.fetch_add(1, Ordering::Relaxed);
}
// --- Phase 15.5: OSC & rollback metrics ---
pub fn record_osc_probe_latency(&self, latency_ms: f64) {
if let Ok(mut h) = self.osc_probe_latency.lock() {
h.push(latency_ms);
if h.len() > 1000 {
h.remove(0);
}
}
}
pub fn set_osc_execution_status(&self, status: f64) {
if let Ok(mut g) = self.osc_execution_gauge.lock() {
*g = status;
}
}
pub fn record_rollback_trigger(&self) {
self.rollback_trigger_count.fetch_add(1, Ordering::Relaxed);
}
pub fn record_tag_block(&self) {
self.tag_block_count.fetch_add(1, Ordering::Relaxed);
}
pub fn record_trace_dropped(&self) {
self.trace_dropped_count.fetch_add(1, Ordering::Relaxed);
}
pub fn snapshot(&self) -> Vec<MetricEntry> {
let mut entries = vec![
MetricEntry {
name: "dbx_degradation_total".to_string(),
metric_type: MetricType::Counter,
description: "Total number of degradation decisions".to_string(),
labels: HashMap::new(),
value: MetricValue::Counter(self.degradation_total.load(Ordering::Relaxed)),
},
MetricEntry {
name: "dbx_degradation_level_full".to_string(),
metric_type: MetricType::Counter,
description: "Number of full compare decisions".to_string(),
labels: HashMap::new(),
value: MetricValue::Counter(self.degradation_level_full.load(Ordering::Relaxed)),
},
MetricEntry {
name: "dbx_degradation_level_sample".to_string(),
metric_type: MetricType::Counter,
description: "Number of sample compare decisions".to_string(),
labels: HashMap::new(),
value: MetricValue::Counter(self.degradation_level_sample.load(Ordering::Relaxed)),
},
MetricEntry {
name: "dbx_degradation_level_skip".to_string(),
metric_type: MetricType::Counter,
description: "Number of skip-with-risk decisions".to_string(),
labels: HashMap::new(),
value: MetricValue::Counter(self.degradation_level_skip.load(Ordering::Relaxed)),
},
MetricEntry {
name: "dbx_auto_upgrade_total".to_string(),
metric_type: MetricType::Counter,
description: "Number of automatic upgrades in degradation chain".to_string(),
labels: HashMap::new(),
value: MetricValue::Counter(self.auto_upgrade_total.load(Ordering::Relaxed)),
},
MetricEntry {
name: "dbx_auto_downgrade_total".to_string(),
metric_type: MetricType::Counter,
description: "Number of automatic downgrades in degradation chain".to_string(),
labels: HashMap::new(),
value: MetricValue::Counter(self.auto_downgrade_total.load(Ordering::Relaxed)),
},
MetricEntry {
name: "dbx_confidence_gauge".to_string(),
metric_type: MetricType::Gauge,
description: "Current confidence score".to_string(),
labels: HashMap::new(),
value: MetricValue::Gauge(*self.confidence_gauge.lock().unwrap_or_else(|e| e.into_inner())),
},
];
// Phase 15.5: OSC and rollback metrics
if let Ok(h) = self.osc_probe_latency.lock() {
if !h.is_empty() {
entries.push(MetricEntry {
name: "dbx_osc_probe_latency".to_string(),
metric_type: MetricType::Histogram,
description: "Distribution of OSC probe latency in milliseconds".to_string(),
labels: HashMap::new(),
value: MetricValue::Histogram(h.iter().map(|&v| (v, 1u64)).collect()),
});
}
}
entries.extend(vec![
MetricEntry {
name: "dbx_osc_execution_status".to_string(),
metric_type: MetricType::Gauge,
description: "Current OSC execution status (0=idle, 1=running, 2=completed, -1=error)".to_string(),
labels: HashMap::new(),
value: MetricValue::Gauge(*self.osc_execution_gauge.lock().unwrap_or_else(|e| e.into_inner())),
},
MetricEntry {
name: "dbx_rollback_trigger_total".to_string(),
metric_type: MetricType::Counter,
description: "Total number of rollback triggers".to_string(),
labels: HashMap::new(),
value: MetricValue::Counter(self.rollback_trigger_count.load(Ordering::Relaxed)),
},
MetricEntry {
name: "dbx_tag_block_total".to_string(),
metric_type: MetricType::Counter,
description: "Total number of tag policy blocks".to_string(),
labels: HashMap::new(),
value: MetricValue::Counter(self.tag_block_count.load(Ordering::Relaxed)),
},
MetricEntry {
name: "dbx_trace_dropped_total".to_string(),
metric_type: MetricType::Counter,
description: "Total number of dropped trace entries".to_string(),
labels: HashMap::new(),
value: MetricValue::Counter(self.trace_dropped_count.load(Ordering::Relaxed)),
},
]);
if let Ok(h) = self.sample_rate_histogram.lock() {
entries.push(MetricEntry {
name: "dbx_sample_rate_histogram".to_string(),
metric_type: MetricType::Histogram,
description: "Distribution of sampling rates used".to_string(),
labels: HashMap::new(),
value: MetricValue::Histogram(h.clone()),
});
}
entries
}
}
impl Default for DegradationMetrics {
fn default() -> Self {
Self::new()
}
}
pub fn export_prometheus(metrics: &[MetricEntry]) -> String {
let mut out = String::new();
for entry in metrics {
let name = &entry.name;
let desc = &entry.description;
let labels = if entry.labels.is_empty() {
String::new()
} else {
let pairs: Vec<String> = entry.labels.iter().map(|(k, v)| format!("{k}=\"{v}\"")).collect();
format!("{{{}}}", pairs.join(","))
};
out.push_str(&format!("# HELP {name} {desc}\n"));
match &entry.metric_type {
MetricType::Counter => {
out.push_str(&format!("# TYPE {name} counter\n"));
if let MetricValue::Counter(v) = &entry.value {
out.push_str(&format!("{name}{labels} {v}\n"));
}
}
MetricType::Gauge => {
out.push_str(&format!("# TYPE {name} gauge\n"));
if let MetricValue::Gauge(v) = &entry.value {
out.push_str(&format!("{name}{labels} {v}\n"));
}
}
MetricType::Histogram => {
out.push_str(&format!("# TYPE {name} histogram\n"));
if let MetricValue::Histogram(buckets) = &entry.value {
for (le, count) in buckets {
out.push_str(&format!("{name}_bucket{{le=\"{le}\"}} {count}\n"));
}
let total: u64 = buckets.iter().map(|(_, c)| c).sum();
out.push_str(&format!("{name}_count{labels} {total}\n"));
}
}
}
}
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn degradation_metrics_records_levels() {
let metrics = DegradationMetrics::new();
metrics.record_degradation("full", 1.0, 1.0);
metrics.record_degradation("sample", 0.1, 0.8);
metrics.record_degradation("skip_with_risk", 0.0, 0.0);
let snapshot = metrics.snapshot();
let total = snapshot.iter().find(|e| e.name == "dbx_degradation_total").unwrap();
assert_eq!(total.value, MetricValue::Counter(3));
let full = snapshot.iter().find(|e| e.name == "dbx_degradation_level_full").unwrap();
assert_eq!(full.value, MetricValue::Counter(1));
let sample = snapshot.iter().find(|e| e.name == "dbx_degradation_level_sample").unwrap();
assert_eq!(sample.value, MetricValue::Counter(1));
let skip = snapshot.iter().find(|e| e.name == "dbx_degradation_level_skip").unwrap();
assert_eq!(skip.value, MetricValue::Counter(1));
}
#[test]
fn degradation_metrics_auto_chain_events() {
let metrics = DegradationMetrics::new();
metrics.record_auto_upgrade();
metrics.record_auto_upgrade();
metrics.record_auto_downgrade();
let snapshot = metrics.snapshot();
let up = snapshot.iter().find(|e| e.name == "dbx_auto_upgrade_total").unwrap();
assert_eq!(up.value, MetricValue::Counter(2));
let down = snapshot.iter().find(|e| e.name == "dbx_auto_downgrade_total").unwrap();
assert_eq!(down.value, MetricValue::Counter(1));
}
#[test]
fn degradation_metrics_confidence_gauge() {
let metrics = DegradationMetrics::new();
metrics.record_degradation("sample", 0.05, 0.85);
let snapshot = metrics.snapshot();
let gauge = snapshot.iter().find(|e| e.name == "dbx_confidence_gauge").unwrap();
assert_eq!(gauge.value, MetricValue::Gauge(0.85));
}
#[test]
fn export_prometheus_counter() {
let mut labels = HashMap::new();
labels.insert("level".to_string(), "full".to_string());
let entries = vec![MetricEntry {
name: "test_counter".to_string(),
metric_type: MetricType::Counter,
description: "A test counter".to_string(),
labels,
value: MetricValue::Counter(42),
}];
let output = export_prometheus(&entries);
assert!(output.contains("# HELP test_counter A test counter"));
assert!(output.contains("# TYPE test_counter counter"));
assert!(output.contains("test_counter{level=\"full\"} 42"));
}
#[test]
fn export_prometheus_gauge() {
let entries = vec![MetricEntry {
name: "test_gauge".to_string(),
metric_type: MetricType::Gauge,
description: "A test gauge".to_string(),
labels: HashMap::new(),
value: MetricValue::Gauge(3.125),
}];
let output = export_prometheus(&entries);
assert!(output.contains("# TYPE test_gauge gauge"));
assert!(output.contains("test_gauge 3.125"));
}
#[test]
fn export_prometheus_histogram() {
let buckets = vec![(0.5, 10u64), (1.0, 5u64), (1.5, 2u64)];
let entries = vec![MetricEntry {
name: "test_histogram".to_string(),
metric_type: MetricType::Histogram,
description: "A test histogram".to_string(),
labels: HashMap::new(),
value: MetricValue::Histogram(buckets),
}];
let output = export_prometheus(&entries);
assert!(output.contains("# TYPE test_histogram histogram"));
assert!(output.contains("test_histogram_bucket{le=\"0.5\"} 10"));
assert!(output.contains("test_histogram_bucket{le=\"1\"} 5"));
assert!(output.contains("test_histogram_count 17"));
}
}
+53
View File
@@ -0,0 +1,53 @@
use crate::sql_risk::ImpactReport;
/// Generate a human-readable safety check report string.
pub fn generate_safety_report(report: &ImpactReport) -> String {
let mut lines = Vec::new();
lines.push("=== SQL Safety Check Report ===".to_string());
lines.push(format!("Overall Risk: {}", report.overall_risk));
if let Some(drl) = &report.ddl_risk_level {
lines.push(format!("DDL Risk Level: {}", drl));
} else {
lines.push("DDL Risk Level: N/A (no DDL statements)".to_string());
}
lines.push(format!("Statement Count: {}", report.statement_count));
lines.push(format!("Recommended Strategy: {}", report.recommended_strategy));
lines.push(format!("Estimated Duration: {}", report.estimated_total_duration));
lines.push(format!("Maintenance Window Required: {}", report.requires_maintenance_window));
lines.push(format!("Reversible: {}", report.is_reversible));
if !report.ddl_details.is_empty() {
lines.push("\n--- DDL Details ---".to_string());
for (i, detail) in report.ddl_details.iter().enumerate() {
lines.push(format!(
" {}. [{}] {} — affects: {}",
i + 1,
detail.ddl_risk,
detail.summary,
detail.affected_objects.join(", ")
));
}
}
if !report.estimated_locks.is_empty() {
lines.push("\n--- Estimated Locks ---".to_string());
for lock in &report.estimated_locks {
lines.push(format!(
" {}: {} lock ({}) — {}",
lock.object, lock.lock_type, lock.scope, lock.estimated_duration
));
}
}
if !report.warnings.is_empty() {
lines.push("\n--- Warnings ---".to_string());
for w in &report.warnings {
lines.push(format!(" ! {}", w));
}
}
lines.join("\n")
}
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+58
View File
@@ -1,19 +1,77 @@
mod capabilities;
pub mod ddl_profile;
pub mod descriptor;
pub mod dialect_loader;
pub mod dialect_types;
pub mod dialect_yaml;
pub mod hot_reload;
mod identifiers;
pub mod inference;
mod table_select;
pub mod type_rewrite;
mod types;
#[cfg(test)]
mod tests;
#[cfg(test)]
mod descriptor_snapshots;
pub use capabilities::{
firebird_rows_clause, is_schema_aware, pagination_strategy, table_pagination_strategy, uses_fetch_first,
uses_oracle_row_id, uses_single_row_insert_statements, PaginationContext, TablePaginationStrategy,
};
pub use ddl_profile::{
profile_for, AutoIncSyntax, DdlDialectProfile, IndexTypePlacement, QuoteStyle, RenameColumnSyntax, TriggerTemplate,
TypeMapEntry,
};
pub use descriptor::{
dialect_check, dialect_check_all, DialectCapabilityDescriptor, DialectInfo, DialectKind, TypeConversionRule,
TypeMappingMatrix, CAP_ADD_COLUMN, CAP_ALTER_EXISTING_COLUMN, CAP_ALTER_OWNER, CAP_ALTER_PRIMARY_KEY,
CAP_AUTO_INCREMENT, CAP_COMMENT, CAP_CREATE_FUNCTION, CAP_CREATE_INDEX, CAP_CREATE_OR_REPLACE, CAP_CREATE_SEQUENCE,
CAP_CREATE_TABLE, CAP_CREATE_TRIGGER, CAP_DROP_COLUMN, CAP_DROP_FUNCTION, CAP_DROP_INDEX, CAP_DROP_SEQUENCE,
CAP_DROP_TABLE, CAP_DROP_TRIGGER, CAP_FOREIGN_KEY, CAP_GRANT_REVOKE, CAP_IDENTITY_COLUMNS, CAP_IF_NOT_EXISTS,
CAP_INDEX_COMMENT, CAP_INDEX_FILTER, CAP_INDEX_INCLUDE, CAP_INDEX_TYPE, CAP_REBUILD_INDEX, CAP_RENAME_COLUMN,
CAP_REORDER_COLUMN, CAP_TEMPORARY_TABLE, CAP_TRANSACTIONAL_DDL, CAP_TRUNCATE_TABLE,
};
pub use identifiers::{
normalize_where_input, qualified_table_name, qualified_table_name_with_catalog, quote_table_identifier,
};
pub(crate) use identifiers::{parse_sqlserver_linked_schema_ref, qualified_transfer_table, quote_transfer_identifier};
pub use table_select::{build_count_table_sql, build_table_data_select_sql, build_table_select_sql};
pub(crate) use table_select::{quote_table_data_identifier, table_data_qualified_table_name};
pub use type_rewrite::{
apply_auto_inc_to_column_def, column_is_auto_increment, rewrite_column_type, split_type_base_params,
type_looks_integer, AutoIncColumnBuild,
};
pub use types::*;
/// Resolve a dialect descriptor for the given kind.
pub fn resolve(kind: descriptor::DialectKind) -> descriptor::DialectCapabilityDescriptor {
lazy_init();
if let Some(desc) = dialect_loader::DialectRegistry::global().get_descriptor(kind.label()) {
return desc;
}
descriptor::DialectCapabilityDescriptor::for_dialect(kind)
}
fn lazy_init() {
use std::sync::OnceLock;
static INIT: OnceLock<()> = OnceLock::new();
INIT.get_or_init(|| {
dialect_loader::register_core_dialects();
let _ = crate::dml_binding::DmlCleanRuleRegistry::load_default();
});
}
pub fn resolve_for_db(db_type: crate::models::connection::DatabaseType) -> descriptor::DialectCapabilityDescriptor {
let kind = descriptor::DialectKind::from_database_type(db_type);
resolve(kind)
}
pub fn resolve_label(kind: descriptor::DialectKind) -> String {
if let Some(loaded) = dialect_loader::DialectRegistry::global().get(kind.label()) {
return loaded.yaml.dialect.display_name.clone().unwrap_or_else(|| kind.label().to_string());
}
kind.label().to_string()
}
@@ -0,0 +1,750 @@
//! Target-database DDL profile: how to *write* SQL for a concrete [`DatabaseType`].
//!
//! Profiles are data + small enums. Call sites must not branch on individual databases;
//! they only consult profile fields (quote style, auto-increment form, type map, …).
use crate::models::connection::DatabaseType;
/// Identifier quoting style for generated DDL.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum QuoteStyle {
/// MySQL-family: `name`
Backtick,
/// PostgreSQL / Access / most ANSI: "name"
DoubleQuote,
/// SQL Server: [name]
Brackets,
/// Oracle-style unquoted uppercase
UnquotedUpper,
}
/// How auto-increment / identity is expressed on the target database.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum AutoIncSyntax {
/// No special auto-increment DDL.
None,
/// Append a fixed suffix to the column definition (e.g. ` AUTO_INCREMENT`, ` IDENTITY(1,1)`).
Suffix(&'static str),
/// Replace the mapped type with this type name for auto PK columns (e.g. Access `COUNTER`).
ReplaceTypeWith(&'static str),
/// PostgreSQL-style sequence + DEFAULT nextval (handled by generator after CREATE).
PostgresSequence,
}
/// Static type rewrite rule: source base type (uppercase, no params) → target template.
/// Target may use `{}` for a single length/precision placeholder (first param only).
#[derive(Debug, Clone, Copy)]
pub struct TypeMapEntry {
pub source_base: &'static str,
pub target_template: &'static str,
}
/// How CREATE INDEX places the index method / type.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum IndexTypePlacement {
/// No index-type clause.
None,
/// PostgreSQL: `CREATE INDEX … ON t USING btree (…)`
UsingSuffix,
/// SQL Server: `CREATE CLUSTERED INDEX …`
TypePrefix,
/// MySQL: `CREATE INDEX … USING BTREE ON t (…)`
UsingBeforeOn,
}
/// CREATE TRIGGER body shape (templates use `{name}` `{timing}` `{event}` `{table}` `{body}`).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TriggerTemplate {
/// `CREATE TRIGGER {name} {timing} {event} ON {table} FOR EACH ROW BEGIN {body} END;`
MysqlStyle,
/// `CREATE TRIGGER {name} {timing} {event} ON {table} FOR EACH ROW EXECUTE FUNCTION {body};`
PostgresStyle,
/// `CREATE TRIGGER {name} ON {table} {timing} {event} AS BEGIN {body} END;`
SqlServerStyle,
/// Conservative MySQL-like default for unknown engines.
GenericRowBody,
}
/// RENAME COLUMN strategy inside ALTER TABLE.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RenameColumnSyntax {
/// `CHANGE COLUMN old new_def` (MySQL)
MysqlChangeColumn,
/// `RENAME COLUMN old TO new`
RenameColumn,
/// `ALTER COLUMN old RENAME TO new` (H2)
AlterColumnRenameTo,
/// `EXEC sp_rename 't.old', 'new', 'COLUMN'`
SqlServerSpRename,
}
/// DDL generation profile for one [`DatabaseType`].
#[derive(Debug, Clone, Copy)]
pub struct DdlDialectProfile {
pub database_type: DatabaseType,
pub quote: QuoteStyle,
pub auto_inc: AutoIncSyntax,
/// When false, display widths like `INT(11)` are stripped if no type_map hit.
pub supports_display_width: bool,
/// Cap for length params when applying `{}` templates (e.g. Access TEXT max 255).
pub max_varchar_len: Option<u32>,
/// MySQL-style inline `COMMENT '...'` on columns.
pub inline_column_comment: bool,
/// SQLite-style: emit FOREIGN KEY clauses inside CREATE TABLE.
pub foreign_keys_inline_in_create: bool,
/// `DROP INDEX name ON table` (MySQL) vs `DROP INDEX IF EXISTS name`.
pub drop_index_uses_on_table: bool,
/// `DROP FOREIGN KEY` (MySQL) vs `DROP CONSTRAINT`.
pub drop_fk_as_foreign_key: bool,
/// Index method placement.
pub index_type_placement: IndexTypePlacement,
/// `INCLUDE (cols)` on indexes.
pub index_supports_include: bool,
/// Partial index `WHERE …`.
pub index_supports_filter: bool,
/// Index `COMMENT '…'` (MySQL).
pub index_supports_comment: bool,
/// Table comment: `ALTER TABLE t COMMENT = '…'` vs `COMMENT ON TABLE`.
pub table_comment_via_alter: bool,
/// Standalone column comment SQL is unsupported (MySQL needs MODIFY COLUMN).
pub column_comment_via_modify_only: bool,
pub trigger_template: TriggerTemplate,
pub rename_column: RenameColumnSyntax,
/// MySQL `MODIFY COLUMN` vs ANSI `ALTER COLUMN … TYPE/SET`.
pub alter_uses_modify_column: bool,
/// Batch multiple alter clauses in one `ALTER TABLE` statement.
pub alter_batches_clauses: bool,
/// Prefer emitting source SHOW CREATE / native DDL when dialects match (MySQL-family).
pub prefers_native_source_ddl: bool,
/// GRANT/REVOKE identifier style: backtick + user quotes vs ANSI double-quote.
pub grant_uses_mysql_user_syntax: bool,
/// Emit a comment that FK changes may need table rebuild (SQLite-family).
pub warn_fk_needs_table_rebuild: bool,
pub create_table_if_not_exists: bool,
pub create_index_if_not_exists: bool,
pub create_function_or_replace: bool,
pub supports_function_ddl: bool,
pub supports_sequence_ddl: bool,
pub supports_rule_ddl: bool,
pub supports_owner_ddl: bool,
/// `{create_kw} {name} {definition};` — `create_kw` is CREATE [OR REPLACE] FUNCTION.
pub function_create_template: Option<&'static str>,
/// `DROP FUNCTION IF EXISTS {name}{cascade};`
pub function_drop_template: Option<&'static str>,
/// `CREATE SEQUENCE {name} AS {data_type} START WITH {start_value} … {cycle};`
pub sequence_create_template: Option<&'static str>,
/// `ALTER SEQUENCE {name} AS {data_type} START WITH {start_value} … {cycle};`
pub sequence_alter_template: Option<&'static str>,
/// `DROP SEQUENCE {name}{cascade};`
pub sequence_drop_template: Option<&'static str>,
/// `DROP RULE IF EXISTS {rule_name} ON {table_name};`
pub rule_drop_template: Option<&'static str>,
/// `ALTER {object_type} {name} OWNER TO {owner};`
pub owner_alter_template: Option<&'static str>,
/// Optional session lock-timeout preamble for generated scripts.
pub lock_timeout_sql: Option<&'static str>,
/// Data-driven base-type rewrites for this target (empty → rely on matrix / normalize only).
pub type_map: &'static [TypeMapEntry],
}
impl DdlDialectProfile {
pub fn lookup_type(&self, source_base_upper: &str) -> Option<&'static str> {
self.type_map.iter().find(|e| e.source_base.eq_ignore_ascii_case(source_base_upper)).map(|e| e.target_template)
}
pub fn quote_ident(&self, name: &str) -> String {
match self.quote {
QuoteStyle::Backtick => format!("`{}`", name.replace('`', "``")),
QuoteStyle::DoubleQuote => format!("\"{}\"", name.replace('"', "\"\"")),
QuoteStyle::Brackets => format!("[{}]", name.replace(']', "]]")),
// Oracle: unquoted identifiers fold to uppercase. Mixed-case names and
// special characters must be double-quoted with original spelling preserved.
QuoteStyle::UnquotedUpper => {
let has_lower = name.chars().any(|c| c.is_ascii_lowercase());
let has_upper = name.chars().any(|c| c.is_ascii_uppercase());
let has_special = name.chars().any(|c| !c.is_ascii_alphanumeric() && c != '_' && c != '$' && c != '#');
let bad_start =
name.is_empty() || !name.chars().next().is_some_and(|c| c.is_ascii_alphabetic() || c == '_');
// All-lower "emp" stays unquoted → EMP (Oracle fold). Mixed "empMixed" → "empMixed".
let needs_quotes = (has_lower && has_upper) || has_special || bad_start;
if needs_quotes {
format!("\"{}\"", name.replace('"', "\"\""))
} else {
name.to_uppercase()
}
}
}
}
/// Replace `{key}` placeholders. Unknown keys are left unchanged.
pub fn render_template(template: &str, vars: &[(&str, &str)]) -> String {
let mut out = template.to_string();
for (key, value) in vars {
out = out.replace(&format!("{{{key}}}"), value);
}
out
}
}
const FN_CREATE: &str = "{create_kw} {name} {definition};";
const FN_DROP: &str = "DROP FUNCTION IF EXISTS {name}{cascade};";
const SEQ_CREATE: &str =
"CREATE SEQUENCE {name} AS {data_type} START WITH {start_value} INCREMENT BY {increment} MINVALUE {min_value} MAXVALUE {max_value} {cycle};";
const SEQ_ALTER: &str =
"ALTER SEQUENCE {name} AS {data_type} START WITH {start_value} INCREMENT BY {increment} MINVALUE {min_value} MAXVALUE {max_value} {cycle};";
const SEQ_DROP: &str = "DROP SEQUENCE {name}{cascade};";
const RULE_DROP: &str = "DROP RULE IF EXISTS {rule_name} ON {table_name}{cascade};";
const OWNER_ALTER: &str = "ALTER {object_type} {name} OWNER TO {owner};";
// ---------------------------------------------------------------------------
// Type maps (data only — no per-database logic at call sites)
// ---------------------------------------------------------------------------
const ACCESS_TYPE_MAP: &[TypeMapEntry] = &[
TypeMapEntry { source_base: "BOOL", target_template: "YESNO" },
TypeMapEntry { source_base: "BOOLEAN", target_template: "YESNO" },
TypeMapEntry { source_base: "BIT", target_template: "YESNO" },
TypeMapEntry { source_base: "YESNO", target_template: "YESNO" },
TypeMapEntry { source_base: "TINYINT", target_template: "BYTE" },
TypeMapEntry { source_base: "BYTE", target_template: "BYTE" },
TypeMapEntry { source_base: "SMALLINT", target_template: "SMALLINT" },
TypeMapEntry { source_base: "SHORT", target_template: "SMALLINT" },
TypeMapEntry { source_base: "MEDIUMINT", target_template: "INTEGER" },
TypeMapEntry { source_base: "INT", target_template: "INTEGER" },
TypeMapEntry { source_base: "INTEGER", target_template: "INTEGER" },
TypeMapEntry { source_base: "INT4", target_template: "INTEGER" },
TypeMapEntry { source_base: "LONG", target_template: "INTEGER" },
TypeMapEntry { source_base: "BIGINT", target_template: "DECIMAL(20,0)" },
TypeMapEntry { source_base: "INT8", target_template: "DECIMAL(20,0)" },
TypeMapEntry { source_base: "FLOAT", target_template: "SINGLE" },
TypeMapEntry { source_base: "REAL", target_template: "SINGLE" },
TypeMapEntry { source_base: "SINGLE", target_template: "SINGLE" },
TypeMapEntry { source_base: "DOUBLE", target_template: "DOUBLE" },
TypeMapEntry { source_base: "DOUBLE PRECISION", target_template: "DOUBLE" },
TypeMapEntry { source_base: "DECIMAL", target_template: "DECIMAL({})" },
TypeMapEntry { source_base: "NUMERIC", target_template: "DECIMAL({})" },
TypeMapEntry { source_base: "NUMBER", target_template: "DECIMAL({})" },
TypeMapEntry { source_base: "CURRENCY", target_template: "CURRENCY" },
TypeMapEntry { source_base: "VARCHAR", target_template: "TEXT({})" },
TypeMapEntry { source_base: "CHARACTER VARYING", target_template: "TEXT({})" },
TypeMapEntry { source_base: "CHAR", target_template: "TEXT({})" },
TypeMapEntry { source_base: "CHARACTER", target_template: "TEXT({})" },
TypeMapEntry { source_base: "NVARCHAR", target_template: "TEXT({})" },
TypeMapEntry { source_base: "NVARCHAR2", target_template: "TEXT({})" },
TypeMapEntry { source_base: "VARCHAR2", target_template: "TEXT({})" },
TypeMapEntry { source_base: "TEXT", target_template: "LONGTEXT" },
TypeMapEntry { source_base: "TINYTEXT", target_template: "LONGTEXT" },
TypeMapEntry { source_base: "MEDIUMTEXT", target_template: "LONGTEXT" },
TypeMapEntry { source_base: "LONGTEXT", target_template: "LONGTEXT" },
TypeMapEntry { source_base: "CLOB", target_template: "LONGTEXT" },
TypeMapEntry { source_base: "MEMO", target_template: "LONGTEXT" },
TypeMapEntry { source_base: "DATE", target_template: "DATETIME" },
TypeMapEntry { source_base: "DATETIME", target_template: "DATETIME" },
TypeMapEntry { source_base: "TIMESTAMP", target_template: "DATETIME" },
TypeMapEntry { source_base: "TIME", target_template: "DATETIME" },
TypeMapEntry { source_base: "YEAR", target_template: "SMALLINT" },
TypeMapEntry { source_base: "BLOB", target_template: "OLEOBJECT" },
TypeMapEntry { source_base: "TINYBLOB", target_template: "OLEOBJECT" },
TypeMapEntry { source_base: "MEDIUMBLOB", target_template: "OLEOBJECT" },
TypeMapEntry { source_base: "LONGBLOB", target_template: "OLEOBJECT" },
TypeMapEntry { source_base: "BINARY", target_template: "OLEOBJECT" },
TypeMapEntry { source_base: "VARBINARY", target_template: "OLEOBJECT" },
TypeMapEntry { source_base: "IMAGE", target_template: "OLEOBJECT" },
TypeMapEntry { source_base: "JSON", target_template: "LONGTEXT" },
TypeMapEntry { source_base: "JSONB", target_template: "LONGTEXT" },
TypeMapEntry { source_base: "UUID", target_template: "GUID" },
TypeMapEntry { source_base: "UNIQUEIDENTIFIER", target_template: "GUID" },
TypeMapEntry { source_base: "GUID", target_template: "GUID" },
];
const SQLITE_TYPE_MAP: &[TypeMapEntry] = &[
TypeMapEntry { source_base: "TINYINT", target_template: "INTEGER" },
TypeMapEntry { source_base: "SMALLINT", target_template: "INTEGER" },
TypeMapEntry { source_base: "MEDIUMINT", target_template: "INTEGER" },
TypeMapEntry { source_base: "INT", target_template: "INTEGER" },
TypeMapEntry { source_base: "INTEGER", target_template: "INTEGER" },
TypeMapEntry { source_base: "BIGINT", target_template: "INTEGER" },
TypeMapEntry { source_base: "FLOAT", target_template: "REAL" },
TypeMapEntry { source_base: "DOUBLE", target_template: "REAL" },
TypeMapEntry { source_base: "DOUBLE PRECISION", target_template: "REAL" },
TypeMapEntry { source_base: "DECIMAL", target_template: "NUMERIC" },
TypeMapEntry { source_base: "NUMERIC", target_template: "NUMERIC" },
TypeMapEntry { source_base: "VARCHAR", target_template: "TEXT" },
TypeMapEntry { source_base: "CHAR", target_template: "TEXT" },
TypeMapEntry { source_base: "TEXT", target_template: "TEXT" },
TypeMapEntry { source_base: "TINYTEXT", target_template: "TEXT" },
TypeMapEntry { source_base: "MEDIUMTEXT", target_template: "TEXT" },
TypeMapEntry { source_base: "LONGTEXT", target_template: "TEXT" },
TypeMapEntry { source_base: "DATETIME", target_template: "TEXT" },
TypeMapEntry { source_base: "TIMESTAMP", target_template: "TEXT" },
TypeMapEntry { source_base: "DATE", target_template: "TEXT" },
TypeMapEntry { source_base: "BLOB", target_template: "BLOB" },
TypeMapEntry { source_base: "JSON", target_template: "TEXT" },
];
// ---------------------------------------------------------------------------
// Profile families (shared shapes; registration is the only DatabaseType match)
// ---------------------------------------------------------------------------
fn mysql_family(db: DatabaseType) -> DdlDialectProfile {
DdlDialectProfile {
database_type: db,
quote: QuoteStyle::Backtick,
auto_inc: AutoIncSyntax::Suffix(" AUTO_INCREMENT"),
supports_display_width: true,
max_varchar_len: Some(65_535),
inline_column_comment: true,
foreign_keys_inline_in_create: false,
drop_index_uses_on_table: true,
drop_fk_as_foreign_key: true,
index_type_placement: IndexTypePlacement::UsingBeforeOn,
index_supports_include: false,
index_supports_filter: false,
index_supports_comment: true,
table_comment_via_alter: true,
column_comment_via_modify_only: true,
trigger_template: TriggerTemplate::MysqlStyle,
rename_column: RenameColumnSyntax::MysqlChangeColumn,
prefers_native_source_ddl: true,
grant_uses_mysql_user_syntax: true,
warn_fk_needs_table_rebuild: false,
alter_uses_modify_column: true,
alter_batches_clauses: true,
create_table_if_not_exists: true,
create_index_if_not_exists: false,
create_function_or_replace: false,
supports_function_ddl: false,
supports_sequence_ddl: false,
supports_rule_ddl: false,
supports_owner_ddl: false,
function_create_template: None,
function_drop_template: None,
sequence_create_template: None,
sequence_alter_template: None,
sequence_drop_template: None,
rule_drop_template: None,
owner_alter_template: None,
lock_timeout_sql: Some("SET SESSION lock_wait_timeout = 3;"),
type_map: &[],
}
}
fn postgres_family(db: DatabaseType) -> DdlDialectProfile {
DdlDialectProfile {
database_type: db,
quote: QuoteStyle::DoubleQuote,
auto_inc: AutoIncSyntax::PostgresSequence,
supports_display_width: false,
max_varchar_len: None,
inline_column_comment: false,
foreign_keys_inline_in_create: false,
drop_index_uses_on_table: false,
drop_fk_as_foreign_key: false,
index_type_placement: IndexTypePlacement::UsingSuffix,
index_supports_include: true,
index_supports_filter: true,
index_supports_comment: false,
table_comment_via_alter: false,
column_comment_via_modify_only: false,
trigger_template: TriggerTemplate::PostgresStyle,
rename_column: RenameColumnSyntax::RenameColumn,
prefers_native_source_ddl: false,
grant_uses_mysql_user_syntax: false,
warn_fk_needs_table_rebuild: false,
alter_uses_modify_column: false,
alter_batches_clauses: false,
create_table_if_not_exists: false,
create_index_if_not_exists: true,
create_function_or_replace: true,
supports_function_ddl: true,
supports_sequence_ddl: true,
supports_rule_ddl: true,
supports_owner_ddl: true,
function_create_template: Some(FN_CREATE),
function_drop_template: Some(FN_DROP),
sequence_create_template: Some(SEQ_CREATE),
sequence_alter_template: Some(SEQ_ALTER),
sequence_drop_template: Some(SEQ_DROP),
rule_drop_template: Some(RULE_DROP),
owner_alter_template: Some(OWNER_ALTER),
lock_timeout_sql: Some("SET lock_timeout = '3s';"),
type_map: &[],
}
}
fn oracle_family(db: DatabaseType) -> DdlDialectProfile {
DdlDialectProfile {
database_type: db,
quote: QuoteStyle::UnquotedUpper,
auto_inc: AutoIncSyntax::Suffix(" GENERATED BY DEFAULT AS IDENTITY"),
supports_display_width: false,
max_varchar_len: Some(4000),
inline_column_comment: false,
foreign_keys_inline_in_create: false,
drop_index_uses_on_table: false,
drop_fk_as_foreign_key: false,
index_type_placement: IndexTypePlacement::None,
index_supports_include: false,
index_supports_filter: false,
index_supports_comment: false,
table_comment_via_alter: false,
column_comment_via_modify_only: false,
trigger_template: TriggerTemplate::GenericRowBody,
rename_column: RenameColumnSyntax::RenameColumn,
prefers_native_source_ddl: false,
grant_uses_mysql_user_syntax: false,
warn_fk_needs_table_rebuild: false,
alter_uses_modify_column: false,
alter_batches_clauses: false,
create_table_if_not_exists: false,
create_index_if_not_exists: false,
create_function_or_replace: false,
supports_function_ddl: true,
supports_sequence_ddl: true,
supports_rule_ddl: false,
supports_owner_ddl: false,
// Same ANSI shapes as Postgres for the subset currently generated by schema_diff.
function_create_template: Some(FN_CREATE),
function_drop_template: Some(FN_DROP),
sequence_create_template: Some(SEQ_CREATE),
sequence_alter_template: Some(SEQ_ALTER),
sequence_drop_template: Some(SEQ_DROP),
rule_drop_template: None,
owner_alter_template: None,
lock_timeout_sql: Some("ALTER SESSION SET DDL_LOCK_TIMEOUT = 3;"),
type_map: &[],
}
}
fn sqlserver_family(db: DatabaseType) -> DdlDialectProfile {
DdlDialectProfile {
database_type: db,
quote: QuoteStyle::DoubleQuote,
auto_inc: AutoIncSyntax::Suffix(" IDENTITY(1,1)"),
supports_display_width: false,
max_varchar_len: None,
inline_column_comment: false,
foreign_keys_inline_in_create: false,
drop_index_uses_on_table: false,
drop_fk_as_foreign_key: false,
index_type_placement: IndexTypePlacement::TypePrefix,
index_supports_include: true,
index_supports_filter: true,
index_supports_comment: false,
table_comment_via_alter: false,
column_comment_via_modify_only: false,
trigger_template: TriggerTemplate::SqlServerStyle,
rename_column: RenameColumnSyntax::SqlServerSpRename,
prefers_native_source_ddl: false,
grant_uses_mysql_user_syntax: false,
warn_fk_needs_table_rebuild: false,
alter_uses_modify_column: false,
alter_batches_clauses: false,
create_table_if_not_exists: false,
create_index_if_not_exists: true,
create_function_or_replace: false,
supports_function_ddl: true,
supports_sequence_ddl: true,
supports_rule_ddl: false,
supports_owner_ddl: false,
function_create_template: Some(FN_CREATE),
function_drop_template: Some(FN_DROP),
sequence_create_template: Some(SEQ_CREATE),
sequence_alter_template: Some(SEQ_ALTER),
sequence_drop_template: Some(SEQ_DROP),
rule_drop_template: None,
owner_alter_template: None,
lock_timeout_sql: Some("SET LOCK_TIMEOUT 3000;"),
type_map: &[],
}
}
fn sqlite_family(db: DatabaseType) -> DdlDialectProfile {
DdlDialectProfile {
database_type: db,
quote: QuoteStyle::DoubleQuote,
auto_inc: AutoIncSyntax::None,
supports_display_width: false,
max_varchar_len: None,
inline_column_comment: false,
foreign_keys_inline_in_create: true,
drop_index_uses_on_table: false,
drop_fk_as_foreign_key: false,
index_type_placement: IndexTypePlacement::None,
index_supports_include: false,
index_supports_filter: true,
index_supports_comment: false,
table_comment_via_alter: false,
column_comment_via_modify_only: false,
trigger_template: TriggerTemplate::GenericRowBody,
rename_column: RenameColumnSyntax::RenameColumn,
prefers_native_source_ddl: false,
grant_uses_mysql_user_syntax: false,
warn_fk_needs_table_rebuild: true,
alter_uses_modify_column: false,
alter_batches_clauses: false,
create_table_if_not_exists: true,
create_index_if_not_exists: true,
create_function_or_replace: false,
supports_function_ddl: false,
supports_sequence_ddl: false,
supports_rule_ddl: false,
supports_owner_ddl: false,
function_create_template: None,
function_drop_template: None,
sequence_create_template: None,
sequence_alter_template: None,
sequence_drop_template: None,
rule_drop_template: None,
owner_alter_template: None,
lock_timeout_sql: None,
type_map: SQLITE_TYPE_MAP,
}
}
fn access_profile() -> DdlDialectProfile {
DdlDialectProfile {
database_type: DatabaseType::Access,
quote: QuoteStyle::DoubleQuote,
auto_inc: AutoIncSyntax::ReplaceTypeWith("COUNTER"),
supports_display_width: false,
max_varchar_len: Some(255),
inline_column_comment: false,
foreign_keys_inline_in_create: false,
drop_index_uses_on_table: false,
drop_fk_as_foreign_key: false,
index_type_placement: IndexTypePlacement::None,
index_supports_include: false,
index_supports_filter: false,
index_supports_comment: false,
table_comment_via_alter: false,
column_comment_via_modify_only: false,
trigger_template: TriggerTemplate::GenericRowBody,
rename_column: RenameColumnSyntax::RenameColumn,
prefers_native_source_ddl: false,
grant_uses_mysql_user_syntax: false,
warn_fk_needs_table_rebuild: false,
alter_uses_modify_column: false,
alter_batches_clauses: false,
create_table_if_not_exists: false,
create_index_if_not_exists: false,
create_function_or_replace: false,
supports_function_ddl: false,
supports_sequence_ddl: false,
supports_rule_ddl: false,
supports_owner_ddl: false,
function_create_template: None,
function_drop_template: None,
sequence_create_template: None,
sequence_alter_template: None,
sequence_drop_template: None,
rule_drop_template: None,
owner_alter_template: None,
lock_timeout_sql: None,
type_map: ACCESS_TYPE_MAP,
}
}
fn h2_profile() -> DdlDialectProfile {
DdlDialectProfile {
database_type: DatabaseType::H2,
quote: QuoteStyle::DoubleQuote,
auto_inc: AutoIncSyntax::None,
supports_display_width: false,
max_varchar_len: None,
inline_column_comment: false,
foreign_keys_inline_in_create: false,
drop_index_uses_on_table: false,
drop_fk_as_foreign_key: false,
index_type_placement: IndexTypePlacement::None,
index_supports_include: false,
index_supports_filter: false,
index_supports_comment: false,
table_comment_via_alter: false,
column_comment_via_modify_only: false,
trigger_template: TriggerTemplate::GenericRowBody,
rename_column: RenameColumnSyntax::AlterColumnRenameTo,
prefers_native_source_ddl: false,
grant_uses_mysql_user_syntax: false,
warn_fk_needs_table_rebuild: false,
alter_uses_modify_column: false,
alter_batches_clauses: false,
create_table_if_not_exists: false,
create_index_if_not_exists: false,
create_function_or_replace: false,
supports_function_ddl: false,
supports_sequence_ddl: false,
supports_rule_ddl: false,
supports_owner_ddl: false,
function_create_template: None,
function_drop_template: None,
sequence_create_template: None,
sequence_alter_template: None,
sequence_drop_template: None,
rule_drop_template: None,
owner_alter_template: None,
lock_timeout_sql: None,
type_map: &[],
}
}
fn conservative_ansi(db: DatabaseType) -> DdlDialectProfile {
DdlDialectProfile {
database_type: db,
quote: QuoteStyle::DoubleQuote,
auto_inc: AutoIncSyntax::None,
supports_display_width: false,
max_varchar_len: None,
inline_column_comment: false,
foreign_keys_inline_in_create: false,
drop_index_uses_on_table: false,
drop_fk_as_foreign_key: false,
index_type_placement: IndexTypePlacement::None,
index_supports_include: false,
index_supports_filter: false,
index_supports_comment: false,
table_comment_via_alter: false,
column_comment_via_modify_only: false,
trigger_template: TriggerTemplate::GenericRowBody,
rename_column: RenameColumnSyntax::RenameColumn,
prefers_native_source_ddl: false,
grant_uses_mysql_user_syntax: false,
warn_fk_needs_table_rebuild: false,
alter_uses_modify_column: false,
alter_batches_clauses: false,
create_table_if_not_exists: false,
create_index_if_not_exists: false,
create_function_or_replace: false,
supports_function_ddl: false,
supports_sequence_ddl: false,
supports_rule_ddl: false,
supports_owner_ddl: false,
function_create_template: None,
function_drop_template: None,
sequence_create_template: None,
sequence_alter_template: None,
sequence_drop_template: None,
rule_drop_template: None,
owner_alter_template: None,
lock_timeout_sql: None,
type_map: &[],
}
}
fn clickhouse_profile() -> DdlDialectProfile {
let mut p = conservative_ansi(DatabaseType::ClickHouse);
// ClickHouse supports CREATE TABLE IF NOT EXISTS like MySQL/SQLite.
p.create_table_if_not_exists = true;
p.lock_timeout_sql = None;
p
}
/// Resolve DDL profile for a concrete target database type.
///
/// This is the **only** place that maps [`DatabaseType`] → profile data.
/// Generators must not re-match on individual databases afterward.
pub fn profile_for(db_type: DatabaseType) -> DdlDialectProfile {
use DatabaseType::*;
match db_type {
// MySQL family
Mysql | Doris | StarRocks | Goldendb | Sundb | Databend | Gbase | ManticoreSearch => mysql_family(db_type),
// PostgreSQL family
Postgres | Redshift | Gaussdb | Kingbase | Highgo | Vastbase | OpenGauss | Kwdb | Firebird | Vertica
| Exasol | Uxdb => postgres_family(db_type),
// Oracle family
Oracle | Dameng | OceanbaseOracle | Yashandb | Xugu | Iris => oracle_family(db_type),
// SQL Server (not Access)
SqlServer => sqlserver_family(db_type),
// Access: own profile (must not inherit SqlServer IDENTITY)
Access => access_profile(),
// SQLite family
Sqlite | Rqlite | Turso | CloudflareD1 => sqlite_family(db_type),
H2 => h2_profile(),
ClickHouse => clickhouse_profile(),
DuckDb | Questdb | SapHana | Teradata | Snowflake | Trino | PrestoSql | Hive | Spark | Db2 | Informix
| Bigquery | Kylin | Oscar | Tdengine | Iotdb | Databricks | Jdbc => conservative_ansi(db_type),
// Non-tabular / not applicable for relational CREATE TABLE
Redis | MongoDb | Elasticsearch | Qdrant | Milvus | Weaviate | ChromaDb | Neo4j | Cassandra | Etcd
| ZooKeeper | Nacos | InfluxDb | MessageQueue | Hbase => conservative_ansi(db_type),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn access_profile_is_not_sqlserver_identity() {
let access = profile_for(DatabaseType::Access);
let mssql = profile_for(DatabaseType::SqlServer);
assert!(matches!(access.auto_inc, AutoIncSyntax::ReplaceTypeWith("COUNTER")));
assert!(matches!(mssql.auto_inc, AutoIncSyntax::Suffix(s) if s.contains("IDENTITY")));
assert!(access.lookup_type("VARCHAR").is_some());
assert!(mssql.type_map.is_empty());
// Access must not inherit SQL Server lock-timeout / idempotent flags.
assert_eq!(access.lock_timeout_sql, None);
assert!(mssql.lock_timeout_sql.is_some());
assert!(!access.create_index_if_not_exists);
assert!(mssql.create_index_if_not_exists);
}
#[test]
fn oracle_profile_uses_valid_identity_and_ddl_lock_timeout() {
let oracle = profile_for(DatabaseType::Oracle);
assert!(matches!(
oracle.auto_inc,
AutoIncSyntax::Suffix(s) if s.contains("GENERATED BY DEFAULT AS IDENTITY")
));
assert_eq!(oracle.lock_timeout_sql, Some("ALTER SESSION SET DDL_LOCK_TIMEOUT = 3;"));
}
#[test]
fn quote_styles() {
assert_eq!(profile_for(DatabaseType::Mysql).quote_ident("a`b"), "`a``b`");
assert_eq!(profile_for(DatabaseType::Access).quote_ident(r#"a"b"#), "\"a\"\"b\"");
assert_eq!(profile_for(DatabaseType::Oracle).quote_ident("emp"), "EMP");
assert_eq!(profile_for(DatabaseType::Oracle).quote_ident("EMP"), "EMP");
// Mixed-case must preserve spelling via quotes (not EMPMIXED).
assert_eq!(profile_for(DatabaseType::Oracle).quote_ident("empMixed"), "\"empMixed\"");
assert_eq!(profile_for(DatabaseType::Oracle).quote_ident("Emp"), "\"Emp\"");
}
#[test]
fn postgres_object_templates_are_populated() {
let p = profile_for(DatabaseType::Postgres);
assert!(p.function_create_template.is_some());
assert!(p.function_drop_template.is_some());
assert!(p.sequence_create_template.is_some());
assert!(p.sequence_alter_template.is_some());
assert!(p.sequence_drop_template.is_some());
assert!(p.rule_drop_template.is_some());
assert!(p.owner_alter_template.is_some());
}
#[test]
fn mysql_object_templates_are_absent() {
let p = profile_for(DatabaseType::Mysql);
assert!(p.function_create_template.is_none());
assert!(p.sequence_create_template.is_none());
assert!(p.rule_drop_template.is_none());
assert!(p.owner_alter_template.is_none());
}
#[test]
fn render_template_replaces_placeholders() {
let rendered = DdlDialectProfile::render_template(
"DROP FUNCTION IF EXISTS {name}{cascade};",
&[("name", "\"public\".\"f\""), ("cascade", " CASCADE")],
);
assert_eq!(rendered, "DROP FUNCTION IF EXISTS \"public\".\"f\" CASCADE;");
}
}
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,68 @@
use crate::sql_dialect::descriptor::{DialectCapabilityDescriptor, DialectInfo, DialectKind, TypeMappingMatrix};
use insta::assert_json_snapshot;
#[test]
fn snapshot_mysql_descriptor() {
let desc = DialectCapabilityDescriptor::for_dialect(DialectKind::Mysql);
assert_json_snapshot!("mysql_descriptor", desc);
}
#[test]
fn snapshot_postgres_descriptor() {
let desc = DialectCapabilityDescriptor::for_dialect(DialectKind::Postgres);
assert_json_snapshot!("postgres_descriptor", desc);
}
#[test]
fn snapshot_sqlite_descriptor() {
let desc = DialectCapabilityDescriptor::for_dialect(DialectKind::Sqlite);
assert_json_snapshot!("sqlite_descriptor", desc);
}
#[test]
fn snapshot_sqlserver_descriptor() {
let desc = DialectCapabilityDescriptor::for_dialect(DialectKind::SqlServer);
assert_json_snapshot!("sqlserver_descriptor", desc);
}
#[test]
fn snapshot_oracle_descriptor() {
let desc = DialectCapabilityDescriptor::for_dialect(DialectKind::Oracle);
assert_json_snapshot!("oracle_descriptor", desc);
}
#[test]
fn snapshot_duckdb_descriptor() {
let desc = DialectCapabilityDescriptor::for_dialect(DialectKind::DuckDb);
assert_json_snapshot!("duckdb_descriptor", desc);
}
#[test]
fn snapshot_mysql_info() {
let info = DialectInfo::for_kind(DialectKind::Mysql);
assert_json_snapshot!("mysql_info", info);
}
#[test]
fn snapshot_postgres_info() {
let info = DialectInfo::for_kind(DialectKind::Postgres);
assert_json_snapshot!("postgres_info", info);
}
#[test]
fn snapshot_mysql_to_postgres_type_mapping() {
let matrix = TypeMappingMatrix::for_dialects(DialectKind::Mysql, DialectKind::Postgres);
assert_json_snapshot!("mysql_to_postgres_mapping", &matrix.rules);
}
#[test]
fn snapshot_postgres_to_mysql_type_mapping() {
let matrix = TypeMappingMatrix::for_dialects(DialectKind::Postgres, DialectKind::Mysql);
assert_json_snapshot!("postgres_to_mysql_mapping", &matrix.rules);
}
#[test]
fn snapshot_all_dialects_info() {
let all = DialectInfo::all();
assert_json_snapshot!("all_dialects_info", all);
}
@@ -0,0 +1,429 @@
use std::collections::HashMap;
use std::fs;
use std::path::{Path, PathBuf};
use std::sync::RwLock;
use super::descriptor::{DialectCapabilityDescriptor, DialectKind};
use super::dialect_yaml::DialectYaml;
/// Singleton dialect registry that holds all loaded dialect descriptors.
/// Thread-safe via RwLock. Access via `DialectRegistry::global()`.
pub struct DialectRegistry {
descriptors: RwLock<HashMap<String, LoadedDialect>>,
plugin_dirs: RwLock<Vec<PathBuf>>,
}
#[derive(Debug, Clone)]
pub struct LoadedDialect {
pub dialect_name: String,
pub kind: DialectKind,
pub descriptor: DialectCapabilityDescriptor,
pub yaml: DialectYaml,
pub source_path: Option<PathBuf>,
pub loaded_at: String,
}
#[derive(Debug, Clone)]
pub struct LoadError {
pub path: PathBuf,
pub error: String,
}
impl DialectRegistry {
pub fn new() -> Self {
Self { descriptors: RwLock::new(HashMap::new()), plugin_dirs: RwLock::new(Vec::new()) }
}
pub fn global() -> &'static Self {
use std::sync::OnceLock;
static INSTANCE: OnceLock<DialectRegistry> = OnceLock::new();
INSTANCE.get_or_init(DialectRegistry::new)
}
pub fn add_plugin_dir(&self, dir: PathBuf) {
if let Ok(mut dirs) = self.plugin_dirs.write() {
if !dirs.contains(&dir) {
dirs.push(dir);
}
}
}
pub fn get(&self, name: &str) -> Option<LoadedDialect> {
self.descriptors.read().ok().and_then(|d| d.get(&name.to_ascii_lowercase()).cloned())
}
pub fn get_by_kind(&self, kind: DialectKind) -> Option<LoadedDialect> {
self.descriptors.read().ok().and_then(|d| d.values().find(|ld| ld.kind == kind).cloned())
}
/// Return all loaded dialects that map to the given kind.
/// Unlike `get_by_kind` (which returns only the first match), this is
/// order-independent and is used when a type's capabilities must be
/// resolved across an entire dialect family (e.g. PostgreSQL, HighGo,
/// KingBase all share `DialectKind::Postgres`).
pub fn get_all_by_kind(&self, kind: DialectKind) -> Vec<LoadedDialect> {
self.descriptors
.read()
.ok()
.map(|d| d.values().filter(|ld| ld.kind == kind).cloned().collect())
.unwrap_or_default()
}
pub fn get_descriptor(&self, name: &str) -> Option<DialectCapabilityDescriptor> {
self.get(name).map(|ld| ld.descriptor)
}
pub fn get_descriptor_for_db(
&self,
db_type: crate::models::connection::DatabaseType,
) -> Option<DialectCapabilityDescriptor> {
let kind = DialectKind::from_database_type(db_type);
let label = kind.label();
let db_name = format!("{db_type:?}").to_ascii_lowercase();
self.get(&db_name).or_else(|| self.get(label)).map(|ld| ld.descriptor)
}
pub fn get_yaml(&self, name: &str) -> Option<DialectYaml> {
self.get(name).map(|ld| ld.yaml)
}
pub fn has(&self, name: &str) -> bool {
self.descriptors.read().ok().is_some_and(|d| d.contains_key(&name.to_ascii_lowercase()))
}
pub fn all_names(&self) -> Vec<String> {
self.descriptors.read().ok().map(|d| d.keys().cloned().collect()).unwrap_or_default()
}
pub fn all_descriptors(&self) -> Vec<DialectCapabilityDescriptor> {
self.descriptors.read().ok().map(|d| d.values().map(|ld| ld.descriptor).collect()).unwrap_or_default()
}
pub fn register(
&self,
name: &str,
kind: DialectKind,
descriptor: DialectCapabilityDescriptor,
yaml: DialectYaml,
source_path: Option<PathBuf>,
) {
if let Ok(mut d) = self.descriptors.write() {
d.insert(
name.to_ascii_lowercase(),
LoadedDialect {
dialect_name: name.to_string(),
kind,
descriptor,
yaml,
source_path,
loaded_at: chrono::Utc::now().to_rfc3339(),
},
);
}
}
pub fn register_yaml(&self, path: &Path, yaml: DialectYaml, descriptor: DialectCapabilityDescriptor) {
let kind = descriptor.dialect;
let name = yaml.dialect.name.clone();
self.register(&name, kind, descriptor, yaml, Some(path.to_path_buf()));
}
pub fn register_descriptor(&self, name: &str, descriptor: DialectCapabilityDescriptor, yaml: DialectYaml) {
let kind = descriptor.dialect;
self.register(name, kind, descriptor, yaml, None);
}
pub fn unregister(&self, name: &str) -> bool {
self.descriptors.write().ok().is_some_and(|mut d| d.remove(&name.to_ascii_lowercase()).is_some())
}
pub fn is_empty(&self) -> bool {
self.descriptors.read().ok().is_none_or(|d| d.is_empty())
}
pub fn len(&self) -> usize {
self.descriptors.read().ok().map_or(0, |d| d.len())
}
}
impl Default for DialectRegistry {
fn default() -> Self {
Self::new()
}
}
/// Register all core dialect YAML definitions embedded at compile time.
/// Called once at startup by `lazy_init()`.
pub fn register_core_dialects() {
use std::sync::Once;
static INIT: Once = Once::new();
INIT.call_once(|| {
let registry = DialectRegistry::global();
include!(concat!(env!("OUT_DIR"), "/core_dialects.rs"));
let n = registry.len();
log::info!("Registered {n} core dialects");
});
}
// ============================================================================
// DialectPluginLoader — scans plugin directories and loads YAML files
// ============================================================================
pub struct DialectPluginLoader;
#[derive(Debug)]
pub struct LoadResult {
pub loaded: Vec<DialectKind>,
pub errors: Vec<LoadError>,
pub skipped: Vec<PathBuf>,
}
impl DialectPluginLoader {
pub fn scan_and_load(registry: &DialectRegistry, plugin_dirs: &[PathBuf]) -> LoadResult {
let mut result = LoadResult { loaded: Vec::new(), errors: Vec::new(), skipped: Vec::new() };
for dir in plugin_dirs {
if !dir.exists() {
continue;
}
let entries = match fs::read_dir(dir) {
Ok(entries) => entries,
Err(e) => {
result.errors.push(LoadError { path: dir.clone(), error: format!("Cannot read directory: {e}") });
continue;
}
};
for entry in entries {
let entry = match entry {
Ok(e) => e,
Err(_) => continue,
};
let path = entry.path();
if !path.is_file() {
continue;
}
let file_name = path.file_stem().and_then(|s| s.to_str()).unwrap_or("");
let extension = path.extension().and_then(|s| s.to_str()).unwrap_or("");
if !matches!(extension, "yaml" | "yml") {
continue;
}
match Self::load_file(&path) {
Ok((kind, yaml, descriptor)) => {
registry.register(file_name, kind, descriptor, yaml, Some(path.clone()));
result.loaded.push(kind);
}
Err(e) => {
result.errors.push(LoadError { path: path.clone(), error: e });
}
}
}
}
result
}
pub fn load_file(path: &Path) -> Result<(DialectKind, DialectYaml, DialectCapabilityDescriptor), String> {
let content = fs::read_to_string(path).map_err(|e| format!("Cannot read file: {e}"))?;
Self::load_from_string(&content, Some(path))
}
pub fn load_from_string(
yaml_str: &str,
source_path: Option<&Path>,
) -> Result<(DialectKind, DialectYaml, DialectCapabilityDescriptor), String> {
let parsed: DialectYaml = serde_yaml::from_str(yaml_str).map_err(|e| format!("YAML parse error: {e}"))?;
let kind = parsed.dialect_kind().ok_or_else(|| {
format!(
"Unknown dialect name '{}' in {}",
parsed.dialect.name,
source_path.map_or_else(|| "<string>".to_string(), |p| p.display().to_string())
)
})?;
let validation_errors = parsed.validate();
if !validation_errors.is_empty() {
let msgs: Vec<String> = validation_errors.iter().map(|e| e.to_string()).collect();
return Err(format!("Validation errors: {}", msgs.join("; ")));
}
let descriptor = parsed.to_descriptor(kind);
Ok((kind, parsed, descriptor))
}
pub fn load_directory(registry: &DialectRegistry, dir: &Path) -> LoadResult {
Self::scan_and_load(registry, &[dir.to_path_buf()])
}
}
// ============================================================================
// Fallback: use hardcoded descriptors when YAML is unavailable
// ============================================================================
pub fn resolve_descriptor(kind: DialectKind, registry: &DialectRegistry) -> DialectCapabilityDescriptor {
if let Some(desc) = registry.get_descriptor(kind.label()) {
return desc;
}
DialectCapabilityDescriptor::for_dialect(kind)
}
pub fn resolve_descriptor_for_db(
db_type: crate::models::connection::DatabaseType,
registry: &DialectRegistry,
) -> DialectCapabilityDescriptor {
if let Some(desc) = registry.get_descriptor_for_db(db_type) {
return desc;
}
let kind = DialectKind::from_database_type(db_type);
DialectCapabilityDescriptor::for_dialect(kind)
}
// ============================================================================
// Tests
// ============================================================================
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write;
fn temp_dialect_dir() -> PathBuf {
let stamp = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_nanos();
let dir = std::env::temp_dir().join(format!("dbx-dialect-test-{}-{stamp}", std::process::id()));
fs::create_dir_all(&dir).unwrap();
dir
}
fn write_temp_yaml(dir: &Path, name: &str, dialect_name: &str) -> PathBuf {
let file_path = dir.join(name);
let yaml_content = format!(
r#"
dialect:
name: "{dialect_name}"
versions:
- version: "1.0"
identifier_rules:
quote_char: "\""
max_length: 64
"#
);
let mut file = fs::File::create(&file_path).unwrap();
file.write_all(yaml_content.as_bytes()).unwrap();
file_path
}
fn cleanup_temp_dir(dir: &Path) {
let _ = fs::remove_dir_all(dir);
}
#[test]
fn load_valid_yaml_file() {
let dir = temp_dialect_dir();
let path = write_temp_yaml(&dir, "dialect_postgresql.yaml", "PostgreSQL");
let result = DialectPluginLoader::load_file(&path);
cleanup_temp_dir(&dir);
assert!(result.is_ok());
let (kind, _yaml, desc) = result.unwrap();
assert_eq!(kind, DialectKind::Postgres);
assert_eq!(desc.dialect, DialectKind::Postgres);
}
#[test]
fn load_valid_yaml_file_mysql() {
let dir = temp_dialect_dir();
let path = write_temp_yaml(&dir, "dialect_mysql.yaml", "MySQL");
let result = DialectPluginLoader::load_file(&path);
cleanup_temp_dir(&dir);
assert!(result.is_ok());
let (kind, _, _) = result.unwrap();
assert_eq!(kind, DialectKind::Mysql);
}
#[test]
fn load_invalid_dialect_name() {
let dir = temp_dialect_dir();
let path = write_temp_yaml(&dir, "dialect_unknown.yaml", "UnknownDB");
let result = DialectPluginLoader::load_file(&path);
cleanup_temp_dir(&dir);
assert!(result.is_err());
}
#[test]
fn load_invalid_yaml_syntax() {
let dir = temp_dialect_dir();
let path = dir.join("dialect_bad.yaml");
let mut file = fs::File::create(&path).unwrap();
file.write_all(b"dialect: [invalid: yaml: syntax:").unwrap();
let result = DialectPluginLoader::load_file(&path);
cleanup_temp_dir(&dir);
assert!(result.is_err());
}
#[test]
fn registry_register_and_get() {
let registry = DialectRegistry::new();
assert!(registry.is_empty());
let desc = DialectCapabilityDescriptor::for_dialect(DialectKind::Mysql);
let yaml = DialectYaml {
dialect: super::super::dialect_yaml::DialectMeta {
name: "MySQL".to_string(),
display_name: Some("MySQL".to_string()),
versions: vec![],
},
identifier_rules: super::super::dialect_yaml::IdentifierRules {
quote_char: "`".to_string(),
case_sensitive: false,
max_length: 64,
},
..Default::default()
};
registry.register_descriptor("MySQL", desc, yaml);
assert_eq!(registry.len(), 1);
assert!(registry.has("MySQL"));
assert!(!registry.has("PostgreSQL"));
let loaded = registry.get("MySQL").unwrap();
assert_eq!(loaded.dialect_name, "MySQL");
registry.unregister("MySQL");
assert!(registry.is_empty());
}
#[test]
fn registry_fallback_resolve() {
let registry = DialectRegistry::new();
let desc = resolve_descriptor(DialectKind::Mysql, &registry);
assert!(desc.has_capability(super::super::descriptor::CAP_ADD_COLUMN));
}
#[test]
fn registry_all_kinds() {
let registry = DialectRegistry::new();
let desc = DialectCapabilityDescriptor::for_dialect(DialectKind::Mysql);
let yaml = DialectYaml {
dialect: super::super::dialect_yaml::DialectMeta {
name: "MySQL".to_string(),
display_name: None,
versions: vec![],
},
identifier_rules: super::super::dialect_yaml::IdentifierRules {
quote_char: "`".to_string(),
case_sensitive: false,
max_length: 64,
},
..Default::default()
};
registry.register_descriptor("MySQL", desc, yaml);
let names = registry.all_names();
assert_eq!(names, vec!["mysql"]);
}
}
@@ -0,0 +1,134 @@
use crate::sql_dialect::descriptor::DialectKind;
use crate::sql_dialect::dialect_loader::DialectRegistry;
pub fn list_dialect_type_names(dialect_name: &str) -> Vec<String> {
super::lazy_init();
list_dialect_type_names_in(dialect_name, DialectRegistry::global())
}
fn list_dialect_type_names_in(dialect_name: &str, registry: &DialectRegistry) -> Vec<String> {
if let Some(loaded) = registry.get(dialect_name) {
return loaded.yaml.types.iter().map(|t| t.name.clone()).collect();
}
// Public callers may use stable dialect labels such as `postgres`,
// while the embedded catalog keeps the product name `PostgreSQL`.
if let Some(kind) = DialectKind::from_label(dialect_name) {
if let Some(loaded) = registry.get(core_catalog_name(kind)) {
return loaded.yaml.types.iter().map(|t| t.name.clone()).collect();
}
// Last-resort fallback: any dialect in the same family
if let Some(loaded) = registry.get_by_kind(kind) {
return loaded.yaml.types.iter().map(|t| t.name.clone()).collect();
}
}
Vec::new()
}
fn core_catalog_name(kind: DialectKind) -> &'static str {
match kind {
DialectKind::Mysql => "MySQL",
DialectKind::Postgres => "PostgreSQL",
DialectKind::Sqlite => "SQLite",
DialectKind::DuckDb => "DuckDB",
DialectKind::SqlServer => "SQL Server",
DialectKind::Oracle => "Oracle",
DialectKind::H2 => "H2",
DialectKind::ClickHouse => "ClickHouse",
DialectKind::ManticoreSearch => "ManticoreSearch",
DialectKind::Informix => "Informix",
DialectKind::Questdb => "QuestDB",
DialectKind::Unsupported => "Unsupported",
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::sql_dialect::descriptor::DialectCapabilityDescriptor;
use crate::sql_dialect::dialect_yaml::{DialectMeta, DialectType, DialectVersion, DialectYaml, IdentifierRules};
fn make_registry_with_types(dialect_name: &str, type_names: &[&str]) -> DialectRegistry {
let registry = DialectRegistry::new();
let types: Vec<DialectType> = type_names
.iter()
.map(|&name| DialectType {
name: name.to_string(),
category: "STRING".to_string(),
max_precision: None,
precision_range: None,
has_length: false,
has_precision: false,
aliases: vec![],
semantic_fidelity_base: 1.0,
})
.collect();
let yaml = DialectYaml {
dialect: DialectMeta {
name: dialect_name.to_string(),
display_name: Some(dialect_name.to_string()),
versions: vec![DialectVersion { version: "1.0".to_string(), status: "COMPATIBLE".to_string() }],
},
types,
identifier_rules: IdentifierRules { quote_char: "\"".to_string(), case_sensitive: false, max_length: 64 },
..Default::default()
};
let kind = yaml.dialect_kind().unwrap();
let descriptor = DialectCapabilityDescriptor::for_dialect(kind);
registry.register_descriptor(dialect_name, descriptor, yaml);
registry
}
#[test]
fn list_mysql_types() {
let registry = make_registry_with_types("MySQL", &["VARCHAR", "INT", "BIGINT"]);
let types = list_dialect_type_names_in("MySQL", &registry);
assert_eq!(types, vec!["VARCHAR", "INT", "BIGINT"]);
assert!(types.contains(&"VARCHAR".to_string()));
}
#[test]
fn list_dameng_types() {
let registry = make_registry_with_types(
"Dameng",
&[
"VARCHAR",
"VARCHAR2",
"INT",
"INTEGER",
"BIGINT",
"FLOAT",
"DOUBLE",
"DATE",
"TIMESTAMP",
"CLOB",
"BLOB",
"TEXT",
],
);
let types = list_dialect_type_names_in("Dameng", &registry);
assert!(!types.is_empty(), "Dameng should have types");
assert!(types.contains(&"VARCHAR2".to_string()));
assert_eq!(types.len(), 12);
}
#[test]
fn list_unknown_dialect_returns_empty() {
let registry = DialectRegistry::new();
let types = list_dialect_type_names_in("nonexistent_db", &registry);
assert!(types.is_empty());
}
#[test]
fn lists_embedded_postgres_types_by_name_and_label() {
let by_name = list_dialect_type_names("PostgreSQL");
let by_label = list_dialect_type_names("postgres");
assert!(!by_name.is_empty());
assert_eq!(by_label, by_name);
assert!(by_name.contains(&"SMALLINT".to_string()));
}
}
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,143 @@
use std::path::PathBuf;
use std::time::Duration;
use notify::{Config, Event, EventKind, RecommendedWatcher, RecursiveMode, Watcher};
use tokio::sync::mpsc;
use super::dialect_loader::DialectPluginLoader;
/// Configuration for the hot-reload watcher.
#[derive(Debug, Clone)]
pub struct HotReloadConfig {
/// Debounce duration — multiple changes within this window are coalesced.
pub debounce_ms: u64,
/// Whether to enable polling (needed for some network filesystems).
pub enable_polling: bool,
/// Polling interval when polling is enabled.
pub poll_interval_ms: u64,
}
impl Default for HotReloadConfig {
fn default() -> Self {
Self { debounce_ms: 500, enable_polling: false, poll_interval_ms: 2000 }
}
}
/// Result of a hot-reload operation on a changed file.
#[derive(Debug, Clone)]
pub enum ReloadEvent {
/// A dialect YAML file was successfully loaded/updated.
Loaded { path: PathBuf },
/// A dialect YAML file was removed — dialect unregistered.
Removed { path: PathBuf, dialect: String },
/// A file changed but failed to load.
Error { path: PathBuf, error: String },
}
/// Hot-reload watcher that monitors `plugins/dialects/` for YAML changes
/// and updates the `DialectRegistry` in real time.
pub struct DialectHotReload {
watcher: Option<RecommendedWatcher>,
config: HotReloadConfig,
}
impl DialectHotReload {
pub fn new(config: HotReloadConfig) -> Self {
Self { watcher: None, config }
}
/// Start watching. Returns a channel receiver for reload events.
pub fn start(&mut self, watch_dirs: Vec<PathBuf>) -> Result<mpsc::UnboundedReceiver<ReloadEvent>, String> {
let (tx, rx) = mpsc::unbounded_channel();
let _debounce = Duration::from_millis(self.config.debounce_ms);
let tx_clone = tx.clone();
let mut watcher = RecommendedWatcher::new(
move |res: Result<Event, notify::Error>| match res {
Ok(event) => {
if !Self::is_relevant(&event) {
return;
}
for path in &event.paths {
let _ = tx_clone.send(match event.kind {
EventKind::Create(_) | EventKind::Modify(_) => ReloadEvent::Loaded { path: path.clone() },
EventKind::Remove(_) => ReloadEvent::Removed {
path: path.clone(),
dialect: path.file_stem().and_then(|s| s.to_str()).unwrap_or("unknown").to_string(),
},
_ => continue,
});
}
}
Err(e) => {
let _ = tx_clone
.send(ReloadEvent::Error { path: PathBuf::from("."), error: format!("Watcher error: {e}") });
}
},
Config::default()
.with_poll_interval(Duration::from_millis(self.config.poll_interval_ms))
.with_compare_contents(true),
)
.map_err(|e| format!("Failed to create file watcher: {e}"))?;
for dir in &watch_dirs {
if dir.exists() {
watcher
.watch(dir, RecursiveMode::NonRecursive)
.map_err(|e| format!("Failed to watch {}: {e}", dir.display()))?;
}
}
self.watcher = Some(watcher);
Ok(rx)
}
/// Process reload events in a loop, updating the registry.
/// This is a convenience function that runs forever on the current task.
pub async fn run_forever(
watch_dirs: Vec<PathBuf>,
registry: &'static super::dialect_loader::DialectRegistry,
) -> Result<(), String> {
let mut reload = Self::new(HotReloadConfig::default());
let mut rx = reload.start(watch_dirs)?;
while let Some(event) = rx.recv().await {
match &event {
ReloadEvent::Loaded { path } => match DialectPluginLoader::load_file(path) {
Ok((kind, yaml, descriptor)) => {
registry.register_yaml(path, yaml, descriptor);
log::info!("Hot-reload: loaded dialect {:?} from {}", kind, path.display());
}
Err(e) => {
log::warn!("Hot-reload: failed to load {}: {e}", path.display());
}
},
ReloadEvent::Removed { path, dialect } => {
registry.unregister(dialect);
log::info!("Hot-reload: unregistered dialect {dialect} ({})", path.display());
}
ReloadEvent::Error { path, error } => {
log::error!("Hot-reload: error watching {}: {error}", path.display());
}
}
}
Ok(())
}
fn is_relevant(event: &Event) -> bool {
for path in &event.paths {
if let Some(ext) = path.extension().and_then(|s| s.to_str()) {
if matches!(ext, "yaml" | "yml") {
return true;
}
}
}
false
}
pub fn stop(&mut self) {
self.watcher = None;
}
}
@@ -0,0 +1,320 @@
use super::descriptor::{DialectKind, TypeMappingMatrix};
#[derive(Debug, Clone, PartialEq)]
pub struct ColumnType {
pub base_type: String,
pub precision: Option<u32>,
pub scale: Option<u32>,
pub length: Option<u32>,
pub is_array: bool,
pub is_unsigned: bool,
pub extras: Vec<String>,
}
impl ColumnType {
pub fn parse(type_str: &str) -> Self {
let trimmed = type_str.trim();
let (base_type, params_str) = match trimmed.find('(') {
Some(pos) if trimmed.ends_with(')') => {
let base = trimmed[..pos].trim().to_string();
let params = trimmed[pos + 1..trimmed.len() - 1].trim();
(base, Some(params))
}
_ => (trimmed.to_string(), None),
};
let is_unsigned = base_type.to_ascii_lowercase().contains("unsigned");
let is_array = trimmed.ends_with("[]") || trimmed.to_ascii_lowercase().contains(" array");
let base_type_clean =
base_type.replace(" unsigned", "").replace(" UNSIGNED", "").replace("[]", "").trim().to_string();
let (precision, scale, length) = match params_str {
Some(params) if params.contains(',') => {
let parts: Vec<&str> = params.split(',').collect();
if parts.len() >= 2 {
let p = parts[0].trim().parse().ok();
let s = parts[1].trim().parse().ok();
(p, s, None)
} else {
(None, None, params.parse().ok())
}
}
Some(params) => {
let v = params.parse().ok();
(None, None, v)
}
None => (None, None, None),
};
Self { base_type: base_type_clean, precision, scale, length, is_array, is_unsigned, extras: Vec::new() }
}
pub fn to_string(&self, dialect: DialectKind) -> String {
let mut result = self.base_type.clone();
if self.is_unsigned && dialect == DialectKind::Mysql {
result.push_str(" UNSIGNED");
}
if let Some(len) = self.length {
result.push_str(&format!("({len})"));
} else if let Some(p) = self.precision {
if let Some(s) = self.scale {
result.push_str(&format!("({p},{s})"));
} else {
result.push_str(&format!("({p})"));
}
}
if self.is_array && (dialect == DialectKind::Postgres || dialect == DialectKind::DuckDb) {
result.push_str("[]");
}
result
}
}
pub trait TypeInferenceEngine {
fn infer_type(&self, source_type: &str, source_dialect: DialectKind, target_dialect: DialectKind) -> ColumnType;
fn suggest_type(&self, column: &ColumnType, target_dialect: DialectKind) -> String;
fn convert_default_value(
&self,
default_expr: &str,
source_dialect: DialectKind,
target_dialect: DialectKind,
) -> String;
fn type_compatibility_score(&self, source: &ColumnType, target: &ColumnType) -> f64;
}
const SCORE_IDENTICAL: f64 = 1.0;
const SCORE_ALIAS: f64 = 0.9;
const SCORE_INTEGER_FAMILY: f64 = 0.8;
const SCORE_TEXT_FAMILY: f64 = 0.7;
const SCORE_UNRELATED: f64 = 0.3;
pub struct DefaultTypeInferenceEngine;
impl DefaultTypeInferenceEngine {
fn map_type_name(&self, type_name: &str, from: DialectKind, to: DialectKind) -> String {
let matrix = TypeMappingMatrix::for_dialects(from, to);
let (converted, _) = matrix.convert_type(type_name);
converted
}
}
impl TypeInferenceEngine for DefaultTypeInferenceEngine {
fn infer_type(&self, source_type: &str, source_dialect: DialectKind, target_dialect: DialectKind) -> ColumnType {
let source_parsed = ColumnType::parse(source_type);
let mapped_base = self.map_type_name(&source_parsed.base_type, source_dialect, target_dialect);
let target_caps = crate::sql_dialect::resolve(target_dialect);
let needs_precision = mapped_base.to_ascii_lowercase().contains("decimal")
|| mapped_base.to_ascii_lowercase().contains("numeric")
|| mapped_base.to_ascii_lowercase().contains("varchar");
let (precision, scale, length) = if needs_precision {
(source_parsed.precision, source_parsed.scale, source_parsed.length)
} else if mapped_base.eq_ignore_ascii_case("integer") && source_parsed.precision.is_some() {
(None, None, None)
} else {
(source_parsed.precision, source_parsed.scale, source_parsed.length)
};
ColumnType {
base_type: mapped_base,
precision,
scale,
length,
is_array: source_parsed.is_array && target_caps.supports_array_type,
is_unsigned: source_parsed.is_unsigned && target_dialect == DialectKind::Mysql,
extras: source_parsed.extras,
}
}
fn suggest_type(&self, column: &ColumnType, target_dialect: DialectKind) -> String {
column.to_string(target_dialect)
}
fn convert_default_value(
&self,
default_expr: &str,
source_dialect: DialectKind,
target_dialect: DialectKind,
) -> String {
let trimmed = default_expr.trim();
if trimmed.is_empty() || trimmed.eq_ignore_ascii_case("null") || trimmed.eq_ignore_ascii_case("NULL") {
return String::new();
}
match (source_dialect, target_dialect) {
(DialectKind::Mysql, DialectKind::Postgres) => {
let lower = trimmed.to_ascii_lowercase();
if lower == "current_timestamp" || lower == "current_timestamp()" || lower == "now()" {
return "CURRENT_TIMESTAMP".to_string();
}
if lower.starts_with("on update ") {
return String::new();
}
}
(DialectKind::Postgres, DialectKind::Mysql) => {
let lower = trimmed.to_ascii_lowercase();
if lower == "current_timestamp" || lower == "now()" || lower == "transaction_timestamp()" {
return "CURRENT_TIMESTAMP".to_string();
}
}
(DialectKind::Mysql, DialectKind::Sqlite) => {
let lower = trimmed.to_ascii_lowercase();
if lower == "current_timestamp" || lower == "current_timestamp()" || lower == "now()" {
return "CURRENT_TIMESTAMP".to_string();
}
}
_ => {}
}
trimmed.to_string()
}
fn type_compatibility_score(&self, source: &ColumnType, target: &ColumnType) -> f64 {
let source_lower = source.base_type.to_ascii_lowercase();
let target_lower = target.base_type.to_ascii_lowercase();
if source_lower == target_lower {
return SCORE_IDENTICAL;
}
let exact_matches = [
("int", "integer"),
("integer", "int"),
("float", "real"),
("real", "float"),
("double", "double precision"),
("double precision", "double"),
("bool", "boolean"),
("boolean", "bool"),
("timestamp", "datetime"),
("datetime", "timestamp"),
];
if exact_matches.contains(&(source_lower.as_str(), target_lower.as_str())) {
return SCORE_ALIAS;
}
let integer_family = ["tinyint", "smallint", "mediumint", "int", "integer", "bigint", "serial", "bigserial"];
let text_family = ["char", "varchar", "text", "tinytext", "mediumtext", "longtext", "clob", "nclob"];
if integer_family.contains(&source_lower.as_str()) && integer_family.contains(&target_lower.as_str()) {
return SCORE_INTEGER_FAMILY;
}
if text_family.contains(&source_lower.as_str()) && text_family.contains(&target_lower.as_str()) {
return SCORE_TEXT_FAMILY;
}
SCORE_UNRELATED
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parse_simple_type() {
let t = ColumnType::parse("INT");
assert_eq!(t.base_type, "INT");
assert!(t.precision.is_none());
assert!(t.length.is_none());
}
#[test]
fn parse_type_with_length() {
let t = ColumnType::parse("VARCHAR(255)");
assert_eq!(t.base_type, "VARCHAR");
assert_eq!(t.length, Some(255));
}
#[test]
fn parse_type_with_precision_scale() {
let t = ColumnType::parse("DECIMAL(10,2)");
assert_eq!(t.base_type, "DECIMAL");
assert_eq!(t.precision, Some(10));
assert_eq!(t.scale, Some(2));
}
#[test]
fn parse_unsigned_type() {
let t = ColumnType::parse("INT UNSIGNED");
assert_eq!(t.base_type, "INT");
assert!(t.is_unsigned);
}
#[test]
fn infer_mysql_int_to_postgres() {
let engine = DefaultTypeInferenceEngine;
let result = engine.infer_type("INT", DialectKind::Mysql, DialectKind::Postgres);
assert_eq!(result.base_type, "INTEGER");
}
#[test]
fn infer_mysql_datetime_to_postgres() {
let engine = DefaultTypeInferenceEngine;
let result = engine.infer_type("DATETIME", DialectKind::Mysql, DialectKind::Postgres);
assert_eq!(result.base_type, "TIMESTAMP");
}
#[test]
fn infer_postgres_text_to_mysql() {
let engine = DefaultTypeInferenceEngine;
let result = engine.infer_type("TEXT", DialectKind::Postgres, DialectKind::Mysql);
assert_eq!(result.base_type, "LONGTEXT");
}
#[test]
fn type_compatibility_exact_match() {
let engine = DefaultTypeInferenceEngine;
let a = ColumnType::parse("INT");
let b = ColumnType::parse("INTEGER");
let score = engine.type_compatibility_score(&a, &b);
assert!((score - 0.9).abs() < 0.01);
}
#[test]
fn type_compatibility_integer_family() {
let engine = DefaultTypeInferenceEngine;
let a = ColumnType::parse("SMALLINT");
let b = ColumnType::parse("BIGINT");
let score = engine.type_compatibility_score(&a, &b);
assert!((score - 0.8).abs() < 0.01);
}
#[test]
fn convert_default_value_mysql_to_postgres() {
let engine = DefaultTypeInferenceEngine;
assert_eq!(
engine.convert_default_value("CURRENT_TIMESTAMP", DialectKind::Mysql, DialectKind::Postgres),
"CURRENT_TIMESTAMP"
);
assert_eq!(
engine.convert_default_value("NOW()", DialectKind::Mysql, DialectKind::Postgres),
"CURRENT_TIMESTAMP"
);
assert!(engine
.convert_default_value("ON UPDATE CURRENT_TIMESTAMP", DialectKind::Mysql, DialectKind::Postgres)
.is_empty());
}
#[test]
fn unknown_type_passthrough() {
let engine = DefaultTypeInferenceEngine;
let result = engine.infer_type("GEOGRAPHY(POINT)", DialectKind::Postgres, DialectKind::Mysql);
assert_eq!(result.base_type, "GEOGRAPHY");
}
#[test]
fn parse_array_type() {
let t = ColumnType::parse("INTEGER[]");
assert!(t.is_array);
assert_eq!(t.base_type, "INTEGER");
}
}
@@ -0,0 +1,431 @@
---
source: crates/dbx-core/src/sql_dialect/descriptor_snapshots.rs
assertion_line: 67
expression: all
---
[
{
"kind": "mysql",
"label": "mysql",
"capabilities": [
"add_column",
"drop_column",
"rename_column",
"alter_existing_column",
"reorder_column",
"comment",
"create_index",
"drop_index",
"rebuild_index",
"index_type",
"index_comment",
"alter_primary_key",
"foreign_key",
"create_table",
"drop_table",
"truncate_table",
"create_trigger",
"drop_trigger",
"create_function",
"drop_function",
"create_sequence",
"drop_sequence",
"alter_owner",
"grant_revoke",
"if_not_exists",
"temporary_table",
"auto_increment"
],
"max_identifier_length": 64,
"supports_schemas": false,
"supports_catalogs": false,
"max_columns_per_table": 4096,
"supports_json_type": true,
"supports_array_type": false,
"supports_enum_type": true,
"supports_uuid_type": false,
"supports_auto_increment": true,
"supports_identity_columns": false,
"supports_sequences": false,
"supports_partitioning": true,
"supports_full_text_index": true,
"supports_spatial_index": true,
"supports_transactional_ddl": false
},
{
"kind": "postgres",
"label": "postgres",
"capabilities": [
"add_column",
"drop_column",
"rename_column",
"alter_existing_column",
"comment",
"create_index",
"drop_index",
"rebuild_index",
"index_type",
"index_include",
"index_filter",
"index_comment",
"alter_primary_key",
"foreign_key",
"create_table",
"drop_table",
"truncate_table",
"create_trigger",
"drop_trigger",
"create_function",
"drop_function",
"create_sequence",
"drop_sequence",
"alter_owner",
"grant_revoke",
"if_not_exists",
"create_or_replace",
"temporary_table",
"transactional_ddl",
"identity_columns"
],
"max_identifier_length": 63,
"supports_schemas": true,
"supports_catalogs": false,
"max_columns_per_table": 1600,
"supports_json_type": true,
"supports_array_type": true,
"supports_enum_type": true,
"supports_uuid_type": true,
"supports_auto_increment": false,
"supports_identity_columns": true,
"supports_sequences": true,
"supports_partitioning": true,
"supports_full_text_index": true,
"supports_spatial_index": true,
"supports_transactional_ddl": true
},
{
"kind": "sqlite",
"label": "sqlite",
"capabilities": [
"add_column",
"drop_column",
"rename_column",
"create_index",
"drop_index",
"rebuild_index",
"index_filter",
"create_table",
"drop_table",
"truncate_table",
"create_trigger",
"drop_trigger",
"create_function",
"drop_function",
"create_sequence",
"drop_sequence",
"if_not_exists",
"auto_increment"
],
"max_identifier_length": 255,
"supports_schemas": false,
"supports_catalogs": false,
"max_columns_per_table": 2000,
"supports_json_type": false,
"supports_array_type": false,
"supports_enum_type": false,
"supports_uuid_type": false,
"supports_auto_increment": true,
"supports_identity_columns": false,
"supports_sequences": false,
"supports_partitioning": false,
"supports_full_text_index": true,
"supports_spatial_index": false,
"supports_transactional_ddl": false
},
{
"kind": "duck_db",
"label": "duckdb",
"capabilities": [
"add_column",
"drop_column",
"rename_column",
"create_index",
"drop_index",
"rebuild_index",
"create_table",
"drop_table",
"truncate_table",
"if_not_exists",
"create_or_replace",
"temporary_table"
],
"max_identifier_length": 255,
"supports_schemas": true,
"supports_catalogs": false,
"max_columns_per_table": 1600,
"supports_json_type": true,
"supports_array_type": true,
"supports_enum_type": true,
"supports_uuid_type": true,
"supports_auto_increment": false,
"supports_identity_columns": false,
"supports_sequences": true,
"supports_partitioning": true,
"supports_full_text_index": false,
"supports_spatial_index": true,
"supports_transactional_ddl": false
},
{
"kind": "sql_server",
"label": "sqlserver",
"capabilities": [
"add_column",
"drop_column",
"rename_column",
"alter_existing_column",
"comment",
"create_index",
"drop_index",
"rebuild_index",
"index_type",
"index_include",
"index_filter",
"index_comment",
"create_table",
"drop_table",
"truncate_table",
"create_trigger",
"drop_trigger",
"create_function",
"drop_function",
"create_sequence",
"drop_sequence",
"alter_owner",
"grant_revoke",
"if_not_exists",
"temporary_table",
"transactional_ddl",
"identity_columns"
],
"max_identifier_length": 128,
"supports_schemas": true,
"supports_catalogs": true,
"max_columns_per_table": 1024,
"supports_json_type": true,
"supports_array_type": false,
"supports_enum_type": false,
"supports_uuid_type": true,
"supports_auto_increment": false,
"supports_identity_columns": true,
"supports_sequences": true,
"supports_partitioning": true,
"supports_full_text_index": true,
"supports_spatial_index": true,
"supports_transactional_ddl": true
},
{
"kind": "oracle",
"label": "oracle",
"capabilities": [
"add_column",
"drop_column",
"rename_column",
"alter_existing_column",
"comment",
"create_index",
"drop_index",
"rebuild_index",
"index_type",
"create_table",
"drop_table",
"truncate_table",
"create_trigger",
"drop_trigger",
"create_function",
"drop_function",
"create_sequence",
"drop_sequence",
"alter_owner",
"grant_revoke",
"if_not_exists",
"temporary_table",
"identity_columns"
],
"max_identifier_length": 30,
"supports_schemas": true,
"supports_catalogs": false,
"max_columns_per_table": 1000,
"supports_json_type": true,
"supports_array_type": false,
"supports_enum_type": false,
"supports_uuid_type": false,
"supports_auto_increment": false,
"supports_identity_columns": true,
"supports_sequences": true,
"supports_partitioning": true,
"supports_full_text_index": true,
"supports_spatial_index": true,
"supports_transactional_ddl": false
},
{
"kind": "h2",
"label": "h2",
"capabilities": [
"add_column",
"drop_column",
"rename_column",
"alter_existing_column",
"comment",
"create_index",
"drop_index",
"rebuild_index",
"create_table",
"drop_table",
"truncate_table",
"create_trigger",
"drop_trigger",
"create_function",
"drop_function",
"if_not_exists",
"temporary_table",
"identity_columns"
],
"max_identifier_length": 256,
"supports_schemas": true,
"supports_catalogs": false,
"max_columns_per_table": 1600,
"supports_json_type": true,
"supports_array_type": false,
"supports_enum_type": false,
"supports_uuid_type": true,
"supports_auto_increment": false,
"supports_identity_columns": true,
"supports_sequences": true,
"supports_partitioning": false,
"supports_full_text_index": true,
"supports_spatial_index": true,
"supports_transactional_ddl": false
},
{
"kind": "click_house",
"label": "clickhouse",
"capabilities": [
"add_column",
"drop_column",
"rename_column",
"alter_existing_column",
"reorder_column",
"comment",
"create_table",
"drop_table",
"truncate_table",
"if_not_exists",
"temporary_table"
],
"max_identifier_length": 256,
"supports_schemas": true,
"supports_catalogs": false,
"max_columns_per_table": 1000,
"supports_json_type": true,
"supports_array_type": true,
"supports_enum_type": true,
"supports_uuid_type": true,
"supports_auto_increment": false,
"supports_identity_columns": false,
"supports_sequences": false,
"supports_partitioning": true,
"supports_full_text_index": true,
"supports_spatial_index": false,
"supports_transactional_ddl": false
},
{
"kind": "manticore_search",
"label": "manticoresearch",
"capabilities": [
"add_column",
"drop_column",
"create_table",
"drop_table",
"truncate_table",
"auto_increment"
],
"max_identifier_length": 256,
"supports_schemas": false,
"supports_catalogs": false,
"max_columns_per_table": 512,
"supports_json_type": true,
"supports_array_type": false,
"supports_enum_type": false,
"supports_uuid_type": false,
"supports_auto_increment": true,
"supports_identity_columns": false,
"supports_sequences": false,
"supports_partitioning": false,
"supports_full_text_index": true,
"supports_spatial_index": true,
"supports_transactional_ddl": false
},
{
"kind": "informix",
"label": "informix",
"capabilities": [
"add_column",
"drop_column",
"rename_column",
"alter_existing_column",
"create_index",
"drop_index",
"rebuild_index",
"create_table",
"drop_table",
"truncate_table",
"create_trigger",
"drop_trigger",
"grant_revoke"
],
"max_identifier_length": 128,
"supports_schemas": true,
"supports_catalogs": false,
"max_columns_per_table": 32767,
"supports_json_type": false,
"supports_array_type": false,
"supports_enum_type": false,
"supports_uuid_type": false,
"supports_auto_increment": false,
"supports_identity_columns": false,
"supports_sequences": true,
"supports_partitioning": true,
"supports_full_text_index": false,
"supports_spatial_index": false,
"supports_transactional_ddl": false
},
{
"kind": "questdb",
"label": "questdb",
"capabilities": [
"add_column",
"drop_column",
"rename_column",
"alter_existing_column",
"create_table",
"drop_table",
"truncate_table",
"if_not_exists"
],
"max_identifier_length": 256,
"supports_schemas": false,
"supports_catalogs": false,
"max_columns_per_table": 1024,
"supports_json_type": false,
"supports_array_type": false,
"supports_enum_type": false,
"supports_uuid_type": false,
"supports_auto_increment": false,
"supports_identity_columns": false,
"supports_sequences": false,
"supports_partitioning": true,
"supports_full_text_index": false,
"supports_spatial_index": false,
"supports_transactional_ddl": false
}
]
@@ -0,0 +1,30 @@
---
source: crates/dbx-core/src/sql_dialect/descriptor_snapshots.rs
assertion_line: 37
expression: desc
---
{
"dialect": "duck_db",
"flags": 469991879,
"max_identifier_length": 255,
"supports_schemas": true,
"supports_catalogs": false,
"max_columns_per_table": 1600,
"max_indexes_per_table": 100,
"max_query_size_bytes": 268435456,
"supports_full_text_index": false,
"supports_spatial_index": true,
"supports_partitioning": true,
"supports_table_sampling": true,
"max_foreign_key_name_length": 255,
"supports_on_update_cascade": true,
"supports_on_delete_set_null": true,
"supports_deferrable_constraints": false,
"supports_array_type": true,
"supports_json_type": true,
"supports_enum_type": true,
"supports_uuid_type": true,
"supports_identity_columns": false,
"supports_auto_increment": false,
"supports_sequences": true
}
@@ -0,0 +1,30 @@
---
source: crates/dbx-core/src/sql_dialect/descriptor_snapshots.rs
assertion_line: 7
expression: desc
---
{
"dialect": "mysql",
"flags": 1476391935,
"max_identifier_length": 64,
"supports_schemas": false,
"supports_catalogs": false,
"max_columns_per_table": 4096,
"max_indexes_per_table": 64,
"max_query_size_bytes": 16777216,
"supports_full_text_index": true,
"supports_spatial_index": true,
"supports_partitioning": true,
"supports_table_sampling": false,
"max_foreign_key_name_length": 64,
"supports_on_update_cascade": true,
"supports_on_delete_set_null": true,
"supports_deferrable_constraints": false,
"supports_array_type": false,
"supports_json_type": true,
"supports_enum_type": true,
"supports_uuid_type": false,
"supports_identity_columns": false,
"supports_auto_increment": true,
"supports_sequences": false
}
@@ -0,0 +1,53 @@
---
source: crates/dbx-core/src/sql_dialect/descriptor_snapshots.rs
assertion_line: 43
expression: info
---
{
"kind": "mysql",
"label": "mysql",
"capabilities": [
"add_column",
"drop_column",
"rename_column",
"alter_existing_column",
"reorder_column",
"comment",
"create_index",
"drop_index",
"rebuild_index",
"index_type",
"index_comment",
"alter_primary_key",
"foreign_key",
"create_table",
"drop_table",
"truncate_table",
"create_trigger",
"drop_trigger",
"create_function",
"drop_function",
"create_sequence",
"drop_sequence",
"alter_owner",
"grant_revoke",
"if_not_exists",
"temporary_table",
"auto_increment"
],
"max_identifier_length": 64,
"supports_schemas": false,
"supports_catalogs": false,
"max_columns_per_table": 4096,
"supports_json_type": true,
"supports_array_type": false,
"supports_enum_type": true,
"supports_uuid_type": false,
"supports_auto_increment": true,
"supports_identity_columns": false,
"supports_sequences": false,
"supports_partitioning": true,
"supports_full_text_index": true,
"supports_spatial_index": true,
"supports_transactional_ddl": false
}
@@ -0,0 +1,103 @@
---
source: crates/dbx-core/src/sql_dialect/descriptor_snapshots.rs
assertion_line: 55
expression: "&matrix.rules"
---
[
{
"source_type": "TINYINT(1)",
"target_type": "BOOLEAN",
"precision_loss": false,
"requires_cast": true
},
{
"source_type": "TINYINT",
"target_type": "SMALLINT",
"precision_loss": false,
"requires_cast": false
},
{
"source_type": "MEDIUMINT",
"target_type": "INTEGER",
"precision_loss": false,
"requires_cast": false
},
{
"source_type": "INT",
"target_type": "INTEGER",
"precision_loss": false,
"requires_cast": false
},
{
"source_type": "BIGINT",
"target_type": "BIGINT",
"precision_loss": false,
"requires_cast": false
},
{
"source_type": "FLOAT",
"target_type": "REAL",
"precision_loss": false,
"requires_cast": false
},
{
"source_type": "DOUBLE",
"target_type": "DOUBLE PRECISION",
"precision_loss": false,
"requires_cast": false
},
{
"source_type": "TINYTEXT",
"target_type": "TEXT",
"precision_loss": false,
"requires_cast": false
},
{
"source_type": "TEXT",
"target_type": "TEXT",
"precision_loss": false,
"requires_cast": false
},
{
"source_type": "MEDIUMTEXT",
"target_type": "TEXT",
"precision_loss": false,
"requires_cast": false
},
{
"source_type": "LONGTEXT",
"target_type": "TEXT",
"precision_loss": false,
"requires_cast": false
},
{
"source_type": "BLOB",
"target_type": "BYTEA",
"precision_loss": false,
"requires_cast": false
},
{
"source_type": "TINYBLOB",
"target_type": "BYTEA",
"precision_loss": false,
"requires_cast": false
},
{
"source_type": "MEDIUMBLOB",
"target_type": "BYTEA",
"precision_loss": false,
"requires_cast": false
},
{
"source_type": "LONGBLOB",
"target_type": "BYTEA",
"precision_loss": false,
"requires_cast": false
},
{
"source_type": "DATETIME",
"target_type": "TIMESTAMP",
"precision_loss": false,
"requires_cast": false
}
]
@@ -0,0 +1,30 @@
---
source: crates/dbx-core/src/sql_dialect/descriptor_snapshots.rs
assertion_line: 31
expression: desc
---
{
"dialect": "oracle",
"flags": 402621423,
"max_identifier_length": 30,
"supports_schemas": true,
"supports_catalogs": false,
"max_columns_per_table": 1000,
"max_indexes_per_table": 100,
"max_query_size_bytes": 65536,
"supports_full_text_index": true,
"supports_spatial_index": true,
"supports_partitioning": true,
"supports_table_sampling": true,
"max_foreign_key_name_length": 30,
"supports_on_update_cascade": true,
"supports_on_delete_set_null": true,
"supports_deferrable_constraints": true,
"supports_array_type": false,
"supports_json_type": true,
"supports_enum_type": false,
"supports_uuid_type": false,
"supports_identity_columns": true,
"supports_auto_increment": false,
"supports_sequences": true
}
@@ -0,0 +1,30 @@
---
source: crates/dbx-core/src/sql_dialect/descriptor_snapshots.rs
assertion_line: 13
expression: desc
---
{
"dialect": "postgres",
"flags": 3221225455,
"max_identifier_length": 63,
"supports_schemas": true,
"supports_catalogs": false,
"max_columns_per_table": 1600,
"max_indexes_per_table": 100,
"max_query_size_bytes": 268435456,
"supports_full_text_index": true,
"supports_spatial_index": true,
"supports_partitioning": true,
"supports_table_sampling": true,
"max_foreign_key_name_length": 63,
"supports_on_update_cascade": true,
"supports_on_delete_set_null": true,
"supports_deferrable_constraints": true,
"supports_array_type": true,
"supports_json_type": true,
"supports_enum_type": true,
"supports_uuid_type": true,
"supports_identity_columns": true,
"supports_auto_increment": false,
"supports_sequences": true
}
@@ -0,0 +1,56 @@
---
source: crates/dbx-core/src/sql_dialect/descriptor_snapshots.rs
assertion_line: 49
expression: info
---
{
"kind": "postgres",
"label": "postgres",
"capabilities": [
"add_column",
"drop_column",
"rename_column",
"alter_existing_column",
"comment",
"create_index",
"drop_index",
"rebuild_index",
"index_type",
"index_include",
"index_filter",
"index_comment",
"alter_primary_key",
"foreign_key",
"create_table",
"drop_table",
"truncate_table",
"create_trigger",
"drop_trigger",
"create_function",
"drop_function",
"create_sequence",
"drop_sequence",
"alter_owner",
"grant_revoke",
"if_not_exists",
"create_or_replace",
"temporary_table",
"transactional_ddl",
"identity_columns"
],
"max_identifier_length": 63,
"supports_schemas": true,
"supports_catalogs": false,
"max_columns_per_table": 1600,
"supports_json_type": true,
"supports_array_type": true,
"supports_enum_type": true,
"supports_uuid_type": true,
"supports_auto_increment": false,
"supports_identity_columns": true,
"supports_sequences": true,
"supports_partitioning": true,
"supports_full_text_index": true,
"supports_spatial_index": true,
"supports_transactional_ddl": true
}
@@ -0,0 +1,79 @@
---
source: crates/dbx-core/src/sql_dialect/descriptor_snapshots.rs
assertion_line: 61
expression: "&matrix.rules"
---
[
{
"source_type": "SMALLINT",
"target_type": "SMALLINT",
"precision_loss": false,
"requires_cast": false
},
{
"source_type": "INTEGER",
"target_type": "INT",
"precision_loss": false,
"requires_cast": false
},
{
"source_type": "BIGINT",
"target_type": "BIGINT",
"precision_loss": false,
"requires_cast": false
},
{
"source_type": "REAL",
"target_type": "FLOAT",
"precision_loss": false,
"requires_cast": false
},
{
"source_type": "DOUBLE PRECISION",
"target_type": "DOUBLE",
"precision_loss": false,
"requires_cast": false
},
{
"source_type": "TEXT",
"target_type": "LONGTEXT",
"precision_loss": false,
"requires_cast": false
},
{
"source_type": "BYTEA",
"target_type": "BLOB",
"precision_loss": false,
"requires_cast": false
},
{
"source_type": "BOOLEAN",
"target_type": "TINYINT(1)",
"precision_loss": false,
"requires_cast": true
},
{
"source_type": "TIMESTAMP",
"target_type": "DATETIME",
"precision_loss": false,
"requires_cast": false
},
{
"source_type": "TIMESTAMPTZ",
"target_type": "DATETIME",
"precision_loss": true,
"requires_cast": true
},
{
"source_type": "UUID",
"target_type": "CHAR(36)",
"precision_loss": false,
"requires_cast": true
},
{
"source_type": "JSONB",
"target_type": "JSON",
"precision_loss": false,
"requires_cast": false
}
]
@@ -0,0 +1,30 @@
---
source: crates/dbx-core/src/sql_dialect/descriptor_snapshots.rs
assertion_line: 19
expression: desc
---
{
"dialect": "sqlite",
"flags": 1157597639,
"max_identifier_length": 255,
"supports_schemas": false,
"supports_catalogs": false,
"max_columns_per_table": 2000,
"max_indexes_per_table": 200,
"max_query_size_bytes": 0,
"supports_full_text_index": true,
"supports_spatial_index": false,
"supports_partitioning": false,
"supports_table_sampling": false,
"max_foreign_key_name_length": 255,
"supports_on_update_cascade": true,
"supports_on_delete_set_null": true,
"supports_deferrable_constraints": false,
"supports_array_type": false,
"supports_json_type": false,
"supports_enum_type": false,
"supports_uuid_type": false,
"supports_identity_columns": false,
"supports_auto_increment": true,
"supports_sequences": false
}
@@ -0,0 +1,30 @@
---
source: crates/dbx-core/src/sql_dialect/descriptor_snapshots.rs
assertion_line: 25
expression: desc
---
{
"dialect": "sql_server",
"flags": 3086983151,
"max_identifier_length": 128,
"supports_schemas": true,
"supports_catalogs": true,
"max_columns_per_table": 1024,
"max_indexes_per_table": 999,
"max_query_size_bytes": 268435456,
"supports_full_text_index": true,
"supports_spatial_index": true,
"supports_partitioning": true,
"supports_table_sampling": true,
"max_foreign_key_name_length": 128,
"supports_on_update_cascade": true,
"supports_on_delete_set_null": true,
"supports_deferrable_constraints": false,
"supports_array_type": false,
"supports_json_type": true,
"supports_enum_type": false,
"supports_uuid_type": true,
"supports_identity_columns": true,
"supports_auto_increment": false,
"supports_sequences": true
}
@@ -0,0 +1,243 @@
//! Column type rewrite pipeline for cross-database DDL generation.
//!
//! Order:
//! 1. Caller-applied user field mappings (highest priority — not handled here)
//! 2. Target [`DdlDialectProfile::type_map`] (data-driven)
//! 3. [`TypeMappingMatrix`] between source/target [`DialectKind`] when available
//! 4. Generic normalization (strip display width when profile disallows it)
use crate::models::connection::DatabaseType;
use crate::sql_dialect::ddl_profile::{profile_for, DdlDialectProfile};
use crate::sql_dialect::descriptor::{DialectKind, TypeMappingMatrix};
use crate::types::ColumnInfo;
/// Split `INT(11)` / `DECIMAL(10,2)` / `VARCHAR` into base + params.
pub fn split_type_base_params(source_type: &str) -> (String, Option<String>) {
let trimmed = source_type.trim();
let upper = trimmed.to_ascii_uppercase();
match upper.find('(') {
Some(i) => {
let base = upper[..i].trim().to_string();
let end = upper.rfind(')').unwrap_or(upper.len());
let params = upper[i + 1..end].trim().to_string();
(base, Some(params))
}
None => (upper, None),
}
}
fn first_length_param(params: Option<&str>) -> Option<u32> {
params.and_then(|p| p.split(',').next()).and_then(|p| p.trim().parse::<u32>().ok())
}
fn apply_template(template: &str, params: Option<&str>, max_varchar_len: Option<u32>) -> String {
if !template.contains("{}") {
return template.to_string();
}
// Prefer full params for DECIMAL(p,s); single length for TEXT({})
if template.starts_with("DECIMAL") || template.starts_with("NUMERIC") {
return match params {
Some(p) if !p.is_empty() => template.replacen("{}", p, 1),
_ => template.replace("({})", "").replace("{}", ""),
};
}
let mut len = first_length_param(params).unwrap_or(255);
if let Some(max) = max_varchar_len {
len = len.clamp(1, max);
} else {
len = len.max(1);
}
template.replacen("{}", &len.to_string(), 1)
}
/// Strip MySQL-style display width: `INT(11)` → `INT`, keep `DECIMAL(10,2)` / `VARCHAR(100)`.
///
/// When keeping length/precision params, original casing is preserved (passthrough-ish).
/// When stripping display width, the base is uppercased as a normalized form.
pub fn strip_display_width_if_needed(source_type: &str, profile: &DdlDialectProfile) -> String {
if profile.supports_display_width {
return source_type.trim().to_string();
}
let trimmed = source_type.trim();
let (base, params) = split_type_base_params(trimmed);
let Some(params) = params else {
return trimmed.to_string();
};
// Keep types where parentheses are meaningful precision/length for most targets.
let keep_params = matches!(
base.as_str(),
"VARCHAR"
| "CHARACTER VARYING"
| "CHAR"
| "CHARACTER"
| "NVARCHAR"
| "NVARCHAR2"
| "VARCHAR2"
| "DECIMAL"
| "NUMERIC"
| "NUMBER"
| "FLOAT"
| "DOUBLE"
| "TIME"
| "TIMESTAMP"
| "DATETIME"
| "ENUM"
| "SET"
);
if keep_params {
// Reconstruct from the original slice so casing stays source-faithful.
let orig_base = trimmed.split('(').next().unwrap_or(trimmed).trim();
let open = trimmed.find('(').unwrap_or(orig_base.len());
let close = trimmed.rfind(')').unwrap_or(trimmed.len());
let orig_params = if open < close { trimmed[open + 1..close].trim() } else { params.as_str() };
format!("{orig_base}({orig_params})")
} else {
// INT(11), BIGINT(20), TINYINT(2), YEAR(4), … — normalized uppercase base.
base
}
}
/// Special-case: TINYINT(1) often means boolean on MySQL-like sources.
fn tinyint1_as_boolean_key(base: &str, params: Option<&str>) -> Option<&'static str> {
if base.eq_ignore_ascii_case("TINYINT") && params == Some("1") {
Some("BOOL")
} else {
None
}
}
/// Rewrite a source column type for `target` database.
///
/// User field mappings must be applied by the caller *before* this function.
pub fn rewrite_column_type(source_type: &str, target: DatabaseType, source_dialect: Option<DialectKind>) -> String {
let profile = profile_for(target);
let (base, params) = split_type_base_params(source_type);
// Profile type_map (includes synthetic BOOL for TINYINT(1) when mapped)
let lookup_key = tinyint1_as_boolean_key(&base, params.as_deref()).unwrap_or(base.as_str());
if let Some(template) = profile.lookup_type(lookup_key) {
return apply_template(template, params.as_deref(), profile.max_varchar_len);
}
// Also try original base if we used BOOL alias miss
if lookup_key != base && profile.lookup_type(&base).is_some() {
if let Some(template) = profile.lookup_type(&base) {
return apply_template(template, params.as_deref(), profile.max_varchar_len);
}
}
// Dialect-kind matrix
if let Some(src) = source_dialect {
let target_kind = DialectKind::from_database_type(target);
let matrix = TypeMappingMatrix::for_dialects(src, target_kind);
let (converted, _) = matrix.convert_type(source_type);
if converted != source_type {
return strip_display_width_if_needed(&converted, &profile);
}
}
strip_display_width_if_needed(source_type, &profile)
}
/// Whether the column metadata indicates auto-increment / identity / serial.
pub fn column_is_auto_increment(col: &ColumnInfo) -> bool {
col.extra.as_deref().is_some_and(|extra| {
let lower = extra.to_ascii_lowercase();
lower.contains("auto_increment") || lower.contains("identity") || lower.contains("serial")
})
}
/// Whether a rewritten type looks integer-like (for PK auto-inc heuristics).
pub fn type_looks_integer(mapped_type: &str) -> bool {
let lower = mapped_type.to_ascii_lowercase();
lower.contains("int")
|| lower.contains("integer")
|| lower.contains("serial")
|| lower.eq_ignore_ascii_case("counter")
|| lower.eq_ignore_ascii_case("byte")
|| lower.starts_with("decimal")
|| lower.starts_with("number")
|| lower.starts_with("numeric")
}
/// Build column type + optional auto-inc contribution using profile only (no DatabaseType branches).
pub fn apply_auto_inc_to_column_def(
profile: &DdlDialectProfile,
quoted_name: &str,
_mapped_type: &str,
col: &ColumnInfo,
is_integer_like: bool,
) -> AutoIncColumnBuild {
let wants_auto = col.is_primary_key && (column_is_auto_increment(col) || is_integer_like);
match profile.auto_inc {
crate::sql_dialect::ddl_profile::AutoIncSyntax::ReplaceTypeWith(type_name)
if wants_auto && col.is_primary_key =>
{
let mut def = format!("{quoted_name} {type_name}");
if !col.is_nullable {
def.push_str(" NOT NULL");
}
AutoIncColumnBuild::Complete { def, skip_default: true }
}
crate::sql_dialect::ddl_profile::AutoIncSyntax::Suffix(suffix) if col.is_primary_key && is_integer_like => {
AutoIncColumnBuild::AppendSuffix { suffix, skip_default: false, postgres_sequence: false }
}
crate::sql_dialect::ddl_profile::AutoIncSyntax::PostgresSequence if col.is_primary_key && is_integer_like => {
AutoIncColumnBuild::AppendSuffix { suffix: "", skip_default: false, postgres_sequence: true }
}
_ => AutoIncColumnBuild::Normal { skip_default: false },
}
}
#[derive(Debug)]
pub enum AutoIncColumnBuild {
/// Full column definition already built (e.g. COUNTER).
Complete {
def: String,
skip_default: bool,
},
/// Continue normal type def; optionally append suffix; maybe register PG sequence.
AppendSuffix {
suffix: &'static str,
skip_default: bool,
postgres_sequence: bool,
},
Normal {
skip_default: bool,
},
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn access_rewrites_mysql_types() {
let t = DatabaseType::Access;
let src = Some(DialectKind::Mysql);
assert_eq!(rewrite_column_type("int(11)", t, src), "INTEGER");
assert_eq!(rewrite_column_type("varchar(120)", t, src), "TEXT(120)");
assert_eq!(rewrite_column_type("tinyint(2)", t, src), "BYTE");
assert_eq!(rewrite_column_type("tinyint(1)", t, src), "YESNO");
assert_eq!(rewrite_column_type("datetime", t, src), "DATETIME");
assert_eq!(rewrite_column_type("longtext", t, src), "LONGTEXT");
}
#[test]
fn sqlserver_keeps_identity_path_not_counter() {
let profile = profile_for(DatabaseType::SqlServer);
assert!(matches!(
profile.auto_inc,
crate::sql_dialect::ddl_profile::AutoIncSyntax::Suffix(s) if s.contains("IDENTITY")
));
// No Access type map: display width stripped
let t = rewrite_column_type("int(11)", DatabaseType::SqlServer, Some(DialectKind::Mysql));
assert_eq!(t, "INT");
}
#[test]
fn mysql_keeps_display_width() {
let t = rewrite_column_type("int(11)", DatabaseType::Mysql, None);
assert_eq!(t, "int(11)");
}
}
@@ -0,0 +1,446 @@
use serde::{Deserialize, Serialize};
use sqlparser::ast::Statement;
use sqlparser::dialect::GenericDialect;
use sqlparser::parser::Parser;
use crate::schema_diff::SchemaDiffPreparationOptions;
use crate::types::FunctionInfo;
#[derive(Debug, Clone)]
pub enum FilterAction {
Allow,
Deny(String),
}
pub trait AstFilter {
fn filter_statement(&self, stmt: &Statement) -> FilterAction;
}
#[derive(Debug, Clone)]
pub struct FilterResult {
pub allowed: Vec<String>,
pub denied: Vec<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SandboxResult {
pub allowed: Vec<String>,
pub denied: Vec<String>,
pub nesting_violations: Vec<String>,
pub is_pure: bool,
pub stats: SandboxStats,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)]
#[serde(rename_all = "snake_case")]
pub enum SandboxMode {
Permissive,
#[default]
Strict,
Isolation,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SandboxStats {
pub total_statements: usize,
pub allowed_count: usize,
pub denied_count: usize,
pub mode: String,
pub max_nesting_depth: usize,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AstSandbox {
pub mode: SandboxMode,
pub max_nesting_depth: usize,
}
impl Default for AstSandbox {
fn default() -> Self {
Self { mode: SandboxMode::Strict, max_nesting_depth: 10 }
}
}
impl AstSandbox {
pub fn new(mode: SandboxMode, max_nesting_depth: usize) -> Self {
Self { mode, max_nesting_depth }
}
pub fn filter_sql(&self, sql: &str, dialect_str: &str) -> Result<SandboxResult, String> {
let base = AstTransmitFilter::filter_sql(sql, dialect_str)?;
let stats = SandboxStats {
total_statements: base.allowed.len() + base.denied.len(),
allowed_count: base.allowed.len(),
denied_count: base.denied.len(),
mode: match self.mode {
SandboxMode::Permissive => "permissive".to_string(),
SandboxMode::Strict => "strict".to_string(),
SandboxMode::Isolation => "isolation".to_string(),
},
max_nesting_depth: self.max_nesting_depth,
};
let (further_blocked, nest_violations) = match self.mode {
SandboxMode::Permissive => (Vec::new(), Vec::new()),
SandboxMode::Strict | SandboxMode::Isolation => self.check_nesting(&base.allowed, self.max_nesting_depth),
};
let is_pure = match self.mode {
SandboxMode::Permissive => base.denied.is_empty(),
SandboxMode::Strict => base.denied.is_empty() && further_blocked.is_empty(),
SandboxMode::Isolation => {
base.denied.is_empty() && further_blocked.is_empty() && nest_violations.is_empty()
}
};
Ok(SandboxResult {
allowed: base.allowed,
denied: base.denied.iter().chain(further_blocked.iter()).cloned().collect(),
nesting_violations: nest_violations,
is_pure,
stats,
})
}
fn check_nesting(&self, statements: &[String], max_depth: usize) -> (Vec<String>, Vec<String>) {
let mut further_blocked = Vec::new();
let mut violations = Vec::new();
for stmt in statements {
let depth = self.count_nesting(stmt);
if depth > max_depth {
further_blocked.push(stmt.clone());
violations.push(format!("Nesting depth {depth} exceeds max {max_depth}: {stmt}"));
}
}
(further_blocked, violations)
}
fn count_nesting(&self, stmt: &str) -> usize {
let mut max_depth = 0usize;
let mut current = 0usize;
for ch in stmt.chars() {
match ch {
'(' => {
current += 1;
max_depth = max_depth.max(current);
}
')' => {
current = current.saturating_sub(1);
}
_ => {}
}
}
max_depth
}
}
pub struct AstTransmitFilter;
impl AstTransmitFilter {
pub fn filter_sql(sql: &str, dialect_str: &str) -> Result<FilterResult, String> {
let dialect = resolve_dialect(dialect_str);
let stmts =
Parser::parse_sql(dialect.as_ref(), sql).map_err(|e| format!("SQL parse error in AST filter: {e}"))?;
let filter = Self;
let mut allowed = Vec::new();
let mut denied: Vec<String> = Vec::new();
for stmt in &stmts {
match filter.filter_statement(stmt) {
FilterAction::Allow => {
allowed.push(stmt.to_string());
}
FilterAction::Deny(reason) => {
denied.push(format!("Blocked: {reason} — {stmt}"));
}
}
}
Ok(FilterResult { allowed, denied })
}
fn is_dangerous_body_node(&self, stmt: &Statement) -> bool {
matches!(
stmt,
Statement::CreateFunction { .. } | Statement::CreateProcedure { .. } | Statement::CreateTrigger { .. }
)
}
pub fn filter_diff_preparation_options(
options: SchemaDiffPreparationOptions,
_dialect: &str,
) -> SchemaDiffPreparationOptions {
let mut filtered = options;
let is_dangerous_fn = |f: &FunctionInfo| -> bool {
let upper = f.definition.to_ascii_uppercase();
let keywords = ["FUNCTION", "PROCEDURE", "TRIGGER", "BEGIN", "DECLARE", "LANGUAGE"];
keywords.iter().any(|kw| upper.contains(kw))
};
filtered.source_functions.retain(|f| !is_dangerous_fn(f));
filtered.target_functions.retain(|f| !is_dangerous_fn(f));
filtered
}
}
fn resolve_dialect(dialect: &str) -> Box<dyn sqlparser::dialect::Dialect> {
match dialect.to_ascii_lowercase().as_str() {
"postgres" | "postgresql" => Box::new(sqlparser::dialect::PostgreSqlDialect {}),
"mysql" | "mariadb" | "tidb" => Box::new(sqlparser::dialect::MySqlDialect {}),
"sqlite" => Box::new(sqlparser::dialect::SQLiteDialect {}),
"sqlserver" | "mssql" => Box::new(sqlparser::dialect::MsSqlDialect {}),
"clickhouse" => Box::new(sqlparser::dialect::ClickHouseDialect {}),
"duckdb" => Box::new(sqlparser::dialect::DuckDbDialect {}),
_ => Box::new(GenericDialect {}),
}
}
impl AstFilter for AstTransmitFilter {
fn filter_statement(&self, stmt: &Statement) -> FilterAction {
if self.is_dangerous_body_node(stmt) {
return FilterAction::Deny("Function/procedure/trigger body not allowed in schema diff".into());
}
match stmt {
Statement::CreateTable { .. }
| Statement::CreateView { .. }
| Statement::CreateIndex { .. }
| Statement::CreateSchema { .. }
| Statement::CreateSequence { .. }
| Statement::AlterTable { .. }
| Statement::AlterIndex { .. }
| Statement::AlterView { .. }
| Statement::Drop { .. }
| Statement::Truncate { .. } => FilterAction::Allow,
_ => FilterAction::Deny("Statement type not in whitelist for schema diff".into()),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn allows_create_table() {
let result =
AstTransmitFilter::filter_sql("CREATE TABLE users (id INT PRIMARY KEY, name VARCHAR(100))", "generic")
.unwrap();
assert!(!result.allowed.is_empty());
assert!(result.allowed[0].contains("CREATE TABLE"));
}
#[test]
fn allows_create_index() {
let result = AstTransmitFilter::filter_sql("CREATE INDEX idx_users_name ON users (name)", "generic").unwrap();
assert!(!result.allowed.is_empty());
}
#[test]
fn allows_alter_table() {
let result = AstTransmitFilter::filter_sql("ALTER TABLE users ADD COLUMN age INT", "generic").unwrap();
assert!(!result.allowed.is_empty());
}
#[test]
fn allows_drop_table() {
let result = AstTransmitFilter::filter_sql("DROP TABLE users", "generic").unwrap();
assert!(!result.allowed.is_empty());
}
#[test]
fn blocks_create_function() {
let result = AstTransmitFilter::filter_sql(
"CREATE FUNCTION add(a INT, b INT) RETURNS INT LANGUAGE SQL RETURN a + b",
"generic",
)
.unwrap();
assert!(result.allowed.is_empty());
assert!(!result.denied.is_empty());
}
#[test]
fn blocks_create_procedure() {
let result = AstTransmitFilter::filter_sql(
"CREATE PROCEDURE test_proc() AS LANGUAGE SQL BEGIN ATOMIC SELECT 1; END",
"postgres",
);
// parse failure on some platforms is acceptable
if let Ok(r) = result {
assert!(r.allowed.is_empty());
assert!(!r.denied.is_empty());
}
}
#[test]
fn blocks_create_trigger() {
let result = AstTransmitFilter::filter_sql(
"CREATE TRIGGER test_trigger BEFORE INSERT ON users FOR EACH ROW EXECUTE FUNCTION log_change()",
"postgres",
)
.unwrap();
assert!(result.allowed.is_empty());
assert!(!result.denied.is_empty());
}
#[test]
fn allows_multiple_ddl_statements() {
let result = AstTransmitFilter::filter_sql(
"CREATE TABLE a (id INT); CREATE TABLE b (id INT); ALTER TABLE a ADD COLUMN x INT;",
"generic",
)
.unwrap();
assert_eq!(result.allowed.len(), 3);
}
#[test]
fn blocks_mixed_ddl_and_function() {
let result = AstTransmitFilter::filter_sql(
"CREATE TABLE t (id INT); CREATE FUNCTION f() RETURNS INT AS $$ SELECT 1 $$ LANGUAGE SQL",
"postgres",
)
.unwrap();
assert_eq!(result.allowed.len(), 1);
assert_eq!(result.denied.len(), 1);
}
#[test]
fn blocks_select_statement() {
let result = AstTransmitFilter::filter_sql("SELECT * FROM users", "generic").unwrap();
assert!(result.allowed.is_empty());
assert!(!result.denied.is_empty());
}
#[test]
fn allows_create_view() {
let result = AstTransmitFilter::filter_sql(
"CREATE VIEW active_users AS SELECT * FROM users WHERE active = 1",
"generic",
)
.unwrap();
assert!(!result.allowed.is_empty());
}
#[test]
fn deny_insert_statement() {
let result = AstTransmitFilter::filter_sql("INSERT INTO users VALUES (1, 'a')", "generic").unwrap();
assert!(result.allowed.is_empty());
assert!(!result.denied.is_empty());
}
#[test]
fn filter_options_removes_dangerous_functions() {
let opts = SchemaDiffPreparationOptions {
source_functions: vec![
FunctionInfo {
name: "f1".into(),
definition: "CREATE FUNCTION f1() RETURNS INT ...".into(),
function_type: "FUNCTION".into(),
data_type: "int".into(),
arguments: "".into(),
},
FunctionInfo {
name: "t1".into(),
definition: "TABLE t1".into(),
function_type: "TABLE".into(),
data_type: "".into(),
arguments: "".into(),
},
],
source_tables: vec![],
target_tables: vec![],
source_details: vec![],
target_details: vec![],
target_functions: vec![],
source_sequences: vec![],
target_sequences: vec![],
source_rules: vec![],
target_rules: vec![],
source_owners: vec![],
target_owners: vec![],
database_type: crate::models::connection::DatabaseType::Mysql,
target_schema: None,
ignore_comments: false,
cascade_delete: false,
compare_column_order: false,
..Default::default()
};
let filtered = AstTransmitFilter::filter_diff_preparation_options(opts, "mysql");
assert_eq!(filtered.source_functions.len(), 1);
assert_eq!(filtered.source_functions[0].name, "t1");
}
#[test]
fn sandbox_strict_mode_blocks_dangerous_statement() {
let sandbox = AstSandbox::new(SandboxMode::Strict, 10);
let result = sandbox
.filter_sql(
"CREATE TABLE t (id INT); CREATE FUNCTION f() RETURNS INT AS $$ SELECT 1 $$ LANGUAGE SQL",
"postgres",
)
.unwrap();
assert_eq!(result.allowed.len(), 1);
assert!(!result.denied.is_empty());
assert!(!result.is_pure);
}
#[test]
fn sandbox_isolation_mode_checks_nesting() {
let sandbox = AstSandbox::new(SandboxMode::Isolation, 4);
let result = sandbox
.filter_sql("CREATE TABLE t (a INT, b INT, c INT, d INT, e INT DEFAULT (((1))))", "generic")
.unwrap();
assert!(result.nesting_violations.is_empty(), "Should pass with nesting depth 4");
assert!(result.is_pure);
}
#[test]
fn sandbox_isolation_blocks_deeply_nested() {
let sandbox = AstSandbox::new(SandboxMode::Isolation, 1);
let result = sandbox.filter_sql("CREATE TABLE t (a INT DEFAULT ((((1)))))", "generic").unwrap();
assert!(!result.nesting_violations.is_empty());
assert!(!result.is_pure);
}
#[test]
fn sandbox_permissive_accepts_everything_allowed() {
let sandbox = AstSandbox::new(SandboxMode::Permissive, 10);
let result = sandbox
.filter_sql("CREATE TABLE t (id INT); CREATE FUNCTION f() RETURNS INT LANGUAGE SQL RETURN 1", "generic")
.unwrap();
assert_eq!(result.allowed.len(), 1, "Permissive allows CREATE TABLE but blocks CREATE FUNCTION");
assert!(!result.denied.is_empty(), "Permissive denies function but not table");
assert!(!result.is_pure, "Permissive is not pure when function is denied by base filter");
}
#[test]
fn sandbox_permissive_no_nesting_check() {
let sandbox = AstSandbox::new(SandboxMode::Permissive, 0);
let result = sandbox.filter_sql("CREATE TABLE t (a INT DEFAULT ((((1)))))", "generic").unwrap();
assert!(!result.allowed.is_empty());
assert!(result.nesting_violations.is_empty());
assert!(result.is_pure);
}
#[test]
fn sandbox_stats_correct() {
let sandbox = AstSandbox::new(SandboxMode::Strict, 10);
let result = sandbox
.filter_sql(
"CREATE TABLE t1 (id INT); CREATE TABLE t2 (id INT); DROP TABLE t3; SELECT * FROM t1",
"generic",
)
.unwrap();
assert_eq!(result.stats.total_statements, 4);
assert_eq!(result.stats.allowed_count, 3);
assert_eq!(result.stats.denied_count, 1);
assert_eq!(result.stats.mode, "strict");
assert_eq!(result.stats.max_nesting_depth, 10);
}
}
+268
View File
@@ -0,0 +1,268 @@
use std::collections::HashSet;
use std::path::Path;
use crate::schema_diff::SchemaDiffPreparationOptions;
#[derive(Debug, Clone)]
pub struct DiffEntry {
pub file_path: String,
pub sql_content: String,
pub change_type: ChangeType,
pub old_path: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ChangeType {
Added,
Modified,
Deleted,
Renamed,
}
pub struct GitDiffScanner {
repo_path: String,
base_commit: String,
target_commit: String,
}
impl GitDiffScanner {
pub fn new(repo_path: &Path, base_commit: &str, target_commit: &str) -> Result<Self, String> {
if base_commit.is_empty() || target_commit.is_empty() {
return Err("Git commit refs cannot be empty".into());
}
let git_path = repo_path.join(".git");
let is_git_repo =
git_path.is_dir() || (git_path.is_file() && git_path.exists()) || repo_path.join("../.git").exists();
if !is_git_repo {
return Err(format!("Not a git repository: {}", repo_path.display()));
}
Ok(Self {
repo_path: repo_path.to_string_lossy().to_string(),
base_commit: base_commit.to_string(),
target_commit: target_commit.to_string(),
})
}
pub fn scan(&self) -> Result<Vec<DiffEntry>, String> {
let output = self.run_git_diff()?;
let sql_filter = self.build_sql_filter()?;
let raw_entries = self.parse_diff_output(&output, &sql_filter)?;
let entries = raw_entries
.into_iter()
.filter_map(|entry| {
if entry.change_type == ChangeType::Deleted {
return Some(entry);
}
let path = Path::new(&self.repo_path).join(&entry.file_path);
let content = std::fs::read_to_string(&path).ok()?;
Some(DiffEntry { sql_content: content, ..entry })
})
.collect();
Ok(entries)
}
fn run_git_diff(&self) -> Result<String, String> {
let output = std::process::Command::new("git")
.arg("diff")
.arg("--diff-filter=ACMR")
.arg(&self.base_commit)
.arg(&self.target_commit)
.arg("--")
.arg("*.sql")
.arg("*.ddl")
.current_dir(&self.repo_path)
.output()
.map_err(|e| format!("Failed to run git diff: {e}"))?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
return Err(format!("git diff failed: {stderr}"));
}
Ok(String::from_utf8_lossy(&output.stdout).to_string())
}
fn build_sql_filter(&self) -> Result<HashSet<String>, String> {
let mut patterns = HashSet::new();
let gitattributes_path = Path::new(&self.repo_path).join(".gitattributes");
if gitattributes_path.exists() {
let content = std::fs::read_to_string(&gitattributes_path)
.map_err(|e| format!("Failed to read .gitattributes: {e}"))?;
for line in content.lines() {
let line = line.trim();
if line.is_empty() || line.starts_with('#') {
continue;
}
let parts: Vec<&str> = line.split_whitespace().collect();
if parts.len() >= 2 && parts[1] == "linguist-language=SQL" {
let pattern = parts[0].trim_matches('*');
if !pattern.is_empty() {
patterns.insert(pattern.to_string());
} else {
patterns.insert("sql".to_string());
}
}
}
}
if patterns.is_empty() {
patterns.insert("sql".to_string());
}
Ok(patterns)
}
fn parse_diff_output(&self, output: &str, _filter: &HashSet<String>) -> Result<Vec<DiffEntry>, String> {
let mut entries = Vec::new();
let mut current_file: Option<String> = None;
let mut current_status: Option<ChangeType> = None;
let mut current_old_path: Option<String> = None;
for line in output.lines() {
if line.starts_with("diff --git ") {
Self::flush_raw_entry(&mut current_file, &mut current_status, &mut current_old_path, &mut entries);
let parts: Vec<&str> = line.split_whitespace().collect();
if parts.len() >= 4 {
let b_path = parts.get(3).copied().unwrap_or("");
let b_path = b_path.trim_start_matches("b/");
current_file = Some(b_path.to_string());
current_status = Some(ChangeType::Modified);
}
} else if line.starts_with("--- ") {
let path = line.trim_start_matches("--- ").trim_start_matches("a/");
if path != "/dev/null" {
current_old_path = Some(path.to_string());
}
} else if line.starts_with("+++ ") {
let path = line.trim_start_matches("+++ ").trim_start_matches("b/");
if path != "/dev/null" {
current_file = Some(path.to_string());
}
} else if line.starts_with("new file mode") {
current_status = Some(ChangeType::Added);
} else if line.starts_with("deleted file mode") {
current_status = Some(ChangeType::Deleted);
} else if line.starts_with("rename from ") {
current_old_path = Some(line.trim_start_matches("rename from ").to_string());
} else if line.starts_with("rename to ") {
current_file = Some(line.trim_start_matches("rename to ").to_string());
current_status = Some(ChangeType::Renamed);
}
}
Self::flush_raw_entry(&mut current_file, &mut current_status, &mut current_old_path, &mut entries);
Ok(entries)
}
fn flush_raw_entry(
file: &mut Option<String>,
status: &mut Option<ChangeType>,
old_path: &mut Option<String>,
entries: &mut Vec<DiffEntry>,
) {
let file_path = file.take();
let st = status.take().unwrap_or(ChangeType::Modified);
let op = old_path.take();
if let Some(path) = file_path {
entries.push(DiffEntry { file_path: path, sql_content: String::new(), change_type: st, old_path: op });
}
}
pub fn bind_to_commit(options: SchemaDiffPreparationOptions, _commit_id: &str) -> SchemaDiffPreparationOptions {
options
}
pub fn list_commits(&self, max_count: usize) -> Result<Vec<String>, String> {
let output = std::process::Command::new("git")
.arg("log")
.arg("--oneline")
.arg("-n")
.arg(max_count.to_string())
.arg("--")
.arg("*.sql")
.arg("*.ddl")
.current_dir(&self.repo_path)
.output()
.map_err(|e| format!("Failed to run git log: {e}"))?;
if !output.status.success() {
return Err(String::from_utf8_lossy(&output.stderr).to_string());
}
let stdout = String::from_utf8_lossy(&output.stdout);
Ok(stdout.lines().map(|l| l.to_string()).collect())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn rejects_non_git_directory() {
let tmp = std::env::temp_dir().join("dbx_test_not_a_repo");
let _ = std::fs::create_dir_all(&tmp);
let result = GitDiffScanner::new(&tmp, "HEAD~1", "HEAD");
assert!(result.is_err());
let _ = std::fs::remove_dir_all(&tmp);
}
#[test]
fn parses_diff_output_with_added_file() {
let scanner =
GitDiffScanner { repo_path: "/tmp/repo".into(), base_commit: "abc".into(), target_commit: "def".into() };
let output = "diff --git a/schema.sql b/schema.sql\nnew file mode 100644\n--- /dev/null\n+++ b/schema.sql\n@@ -0,0 +1 @@\n+CREATE TABLE t (id INT);\n";
let mut filter = HashSet::new();
filter.insert("sql".to_string());
let entries = scanner.parse_diff_output(output, &filter).unwrap();
assert_eq!(entries.len(), 1);
assert_eq!(entries[0].file_path, "schema.sql");
assert_eq!(entries[0].change_type, ChangeType::Added);
}
#[test]
fn parses_diff_output_with_modified_file() {
let scanner =
GitDiffScanner { repo_path: "/tmp/repo".into(), base_commit: "abc".into(), target_commit: "def".into() };
let output = "diff --git a/schema.sql b/schema.sql\n--- a/schema.sql\n+++ b/schema.sql\n@@ -1 +1 @@\n-CREATE TABLE t (id INT);\n+CREATE TABLE t (id BIGINT);\n";
let mut filter = HashSet::new();
filter.insert("sql".to_string());
let entries = scanner.parse_diff_output(output, &filter).unwrap();
assert_eq!(entries.len(), 1);
assert_eq!(entries[0].change_type, ChangeType::Modified);
}
#[test]
fn parses_diff_output_with_renamed_file() {
let scanner =
GitDiffScanner { repo_path: "/tmp/repo".into(), base_commit: "abc".into(), target_commit: "def".into() };
let output = "diff --git a/old.sql b/new.sql\nrename from old.sql\nrename to new.sql\n";
let mut filter = HashSet::new();
filter.insert("sql".to_string());
let entries = scanner.parse_diff_output(output, &filter).unwrap();
assert_eq!(entries.len(), 1);
assert_eq!(entries[0].change_type, ChangeType::Renamed);
assert_eq!(entries[0].old_path, Some("old.sql".to_string()));
}
#[test]
fn parses_gitattributes_for_sql_patterns() {
let tmp = std::env::temp_dir().join("dbx_test_gitattributes");
let _ = std::fs::create_dir_all(&tmp);
let gitattributes = tmp.join(".gitattributes");
std::fs::write(&gitattributes, "*.sql linguist-language=SQL\n*.ddl linguist-language=SQL\n").unwrap();
let scanner = GitDiffScanner {
repo_path: tmp.to_string_lossy().to_string(),
base_commit: "abc".into(),
target_commit: "def".into(),
};
let filter = scanner.build_sql_filter().unwrap();
assert!(!filter.is_empty());
let _ = std::fs::remove_dir_all(&tmp);
}
}
+301
View File
@@ -0,0 +1,301 @@
use std::path::{Path, PathBuf};
use crate::models::connection::ConnectionConfig;
use crate::schema_diff::SchemaDiffPreparationOptions;
use crate::sql_parser::meta::{validate_sql_meta_consistency, MetaData, MetaReader};
#[derive(Debug, Clone)]
#[allow(clippy::large_enum_variant)]
pub enum InputSource {
DdlFiles(Vec<PathBuf>),
DatabaseConnection(ConnectionConfig),
GitDiff { repo_path: PathBuf, base_commit: String, target_commit: String },
MetaData(MetaData),
}
#[derive(Debug, Clone)]
pub struct ResolvedInput {
pub source: InputSource,
pub meta: Option<MetaData>,
pub ddl_sql: Option<String>,
pub has_errors: bool,
pub warnings: Vec<String>,
}
pub struct InputResolver;
impl InputResolver {
pub fn discover_ddl_files(dir: &Path) -> Vec<PathBuf> {
let mut files = Vec::new();
if !dir.is_dir() {
return files;
}
let entries = match std::fs::read_dir(dir) {
Ok(e) => e,
Err(_) => return files,
};
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
files.extend(Self::discover_ddl_files(&path));
continue;
}
let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("");
if matches!(ext, "sql" | "ddl" | "txt") {
let file_name = path.file_name().and_then(|n| n.to_str()).unwrap_or("");
if !file_name.starts_with('.') {
files.push(path);
}
}
}
files
}
pub fn load_meta_for(sql_path: &Path) -> Option<MetaData> {
let dir = sql_path.parent()?;
let stem = sql_path.file_stem().and_then(|s| s.to_str())?;
for ext in &["meta.json", "meta.yaml", "meta.yml"] {
let meta_path = dir.join(format!("{}.{}", stem, ext));
if meta_path.exists() {
if let Ok(meta) = MetaReader::from_file(&meta_path) {
return Some(meta);
}
}
}
let meta_path = dir.join(".dbmeta.json");
if meta_path.exists() {
if let Ok(meta) = MetaReader::from_file(&meta_path) {
return Some(meta);
}
}
None
}
pub fn resolve(inputs: Vec<InputSource>) -> ResolvedInput {
let mut meta: Option<MetaData> = None;
let mut sql_parts: Vec<String> = Vec::new();
let mut warnings: Vec<String> = Vec::new();
let mut live_db: Option<ConnectionConfig> = None;
for input in inputs {
match input {
InputSource::DdlFiles(files) => {
for file in files {
if let Ok(content) = std::fs::read_to_string(&file) {
if let Some(file_meta) = Self::load_meta_for(&file) {
let mw = validate_sql_meta_consistency(&content, &file_meta);
warnings.extend(mw.into_iter().map(|w| format!("{}: {}", file.display(), w)));
meta = Self::merge_meta(meta.take(), Some(file_meta));
}
sql_parts.push(content);
} else {
warnings.push(format!("Cannot read DDL file: {}", file.display()));
}
}
}
InputSource::DatabaseConnection(config) => {
live_db = Some(config);
}
InputSource::GitDiff { repo_path, base_commit, target_commit } => {
match crate::sql_parser::git::GitDiffScanner::new(&repo_path, &base_commit, &target_commit) {
Ok(scanner) => match scanner.scan() {
Ok(entries) => {
for entry in entries {
sql_parts.push(entry.sql_content);
}
}
Err(e) => warnings.push(format!("Git diff scan error: {e}")),
},
Err(e) => warnings.push(format!("Git scanner init error: {e}")),
}
}
InputSource::MetaData(m) => {
meta = Self::merge_meta(meta.take(), Some(m));
}
}
}
if let Some(ref meta_data) = meta {
let mw = MetaReader::validate_consistency(meta_data);
warnings.extend(mw);
}
let source = if let Some(config) = live_db {
InputSource::DatabaseConnection(config)
} else {
InputSource::DdlFiles(Vec::new())
};
ResolvedInput {
source,
meta,
ddl_sql: if sql_parts.is_empty() { None } else { Some(sql_parts.join("\n\n")) },
has_errors: warnings.iter().any(|w| w.starts_with("Cannot read") || w.contains("error")),
warnings,
}
}
fn merge_meta(existing: Option<MetaData>, incoming: Option<MetaData>) -> Option<MetaData> {
match (existing, incoming) {
(None, incoming) => incoming,
(existing, None) => existing,
(Some(mut e), Some(i)) => {
e.dialect = e.dialect.or(i.dialect);
e.version = e.version.or(i.version);
e.charset = e.charset.or(i.charset);
e.collation = e.collation.or(i.collation);
e.sql_mode = e.sql_mode.or(i.sql_mode);
e.explicit_dependencies.extend(i.explicit_dependencies);
Some(e)
}
}
}
pub fn normalize_to_diff_options(resolved: ResolvedInput) -> (SchemaDiffPreparationOptions, Vec<String>) {
let mut options = SchemaDiffPreparationOptions {
source_tables: Vec::new(),
target_tables: Vec::new(),
source_details: Vec::new(),
target_details: Vec::new(),
source_functions: Vec::new(),
target_functions: Vec::new(),
source_sequences: Vec::new(),
target_sequences: Vec::new(),
source_rules: Vec::new(),
target_rules: Vec::new(),
source_owners: Vec::new(),
target_owners: Vec::new(),
database_type: crate::models::connection::DatabaseType::Mysql,
target_schema: None,
ignore_comments: false,
cascade_delete: false,
compare_column_order: false,
..Default::default()
};
if let Some(ref meta) = resolved.meta {
if let Some(ref dialect) = meta.dialect {
let db_type = dialect_to_database_type(dialect);
options.database_type = db_type;
}
}
(options, resolved.warnings)
}
}
fn dialect_to_database_type(dialect: &str) -> crate::models::connection::DatabaseType {
match dialect.to_ascii_lowercase().as_str() {
"mysql" | "mariadb" | "tidb" => crate::models::connection::DatabaseType::Mysql,
"postgres" | "postgresql" => crate::models::connection::DatabaseType::Postgres,
"sqlite" => crate::models::connection::DatabaseType::Sqlite,
"sqlserver" | "mssql" => crate::models::connection::DatabaseType::SqlServer,
"clickhouse" => crate::models::connection::DatabaseType::ClickHouse,
"duckdb" => crate::models::connection::DatabaseType::DuckDb,
"oracle" => crate::models::connection::DatabaseType::Oracle,
_ => crate::models::connection::DatabaseType::Mysql,
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write;
#[test]
fn discovers_sql_files_in_directory() {
let dir = std::env::temp_dir().join("dbx_test_ddl_discover");
let _ = std::fs::create_dir_all(&dir);
let mut f1 = std::fs::File::create(dir.join("schema.sql")).unwrap();
f1.write_all(b"CREATE TABLE t (id INT);").unwrap();
let mut f2 = std::fs::File::create(dir.join("data.ddl")).unwrap();
f2.write_all(b"INSERT INTO t VALUES (1);").unwrap();
let _hidden = std::fs::File::create(dir.join(".ignore.sql")).unwrap();
let files = InputResolver::discover_ddl_files(&dir);
let names: Vec<String> =
files.iter().filter_map(|f| f.file_name().and_then(|n| n.to_str()).map(|s| s.to_string())).collect();
assert!(names.contains(&"schema.sql".to_string()));
assert!(names.contains(&"data.ddl".to_string()));
assert!(!names.contains(&".ignore.sql".to_string()));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn loads_meta_file_for_sql() {
let dir = std::env::temp_dir().join("dbx_test_meta_load");
let _ = std::fs::create_dir_all(&dir);
let sql_path = dir.join("test.sql");
let meta_path = dir.join("test.meta.json");
std::fs::write(&sql_path, "CREATE TABLE t (id INT);").unwrap();
std::fs::write(&meta_path, r#"{"dialect": "postgres", "version": "15"}"#).unwrap();
let meta = InputResolver::load_meta_for(&sql_path);
assert!(meta.is_some());
assert_eq!(meta.unwrap().dialect.as_deref(), Some("postgres"));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn normalize_uses_meta_dialect() {
let resolved = ResolvedInput {
source: InputSource::DdlFiles(Vec::new()),
meta: Some(MetaData {
dialect: Some("postgres".into()),
version: None,
charset: None,
collation: None,
sql_mode: None,
explicit_dependencies: std::collections::HashMap::new(),
}),
ddl_sql: Some("CREATE TABLE t (id INT);".into()),
has_errors: false,
warnings: Vec::new(),
};
let (options, _) = InputResolver::normalize_to_diff_options(resolved);
assert_eq!(options.database_type, crate::models::connection::DatabaseType::Postgres);
}
#[test]
fn resolve_adds_has_errors_on_failed_read() {
let resolved = InputResolver::resolve(vec![InputSource::DdlFiles(vec![PathBuf::from(
r"C:\non_existent_dir\nonexistent.sql",
)])]);
assert!(resolved.has_errors);
assert_eq!(resolved.warnings.len(), 1);
}
#[test]
fn resolve_merges_meta_from_multiple_sources() {
let meta1 = InputSource::MetaData(MetaData {
dialect: Some("mysql".into()),
version: Some("8.0".into()),
charset: None,
collation: None,
sql_mode: None,
explicit_dependencies: std::collections::HashMap::new(),
});
let meta2 = InputSource::MetaData(MetaData {
dialect: None,
version: None,
charset: Some("utf8mb4".into()),
collation: None,
sql_mode: None,
explicit_dependencies: std::collections::HashMap::new(),
});
let resolved = InputResolver::resolve(vec![meta1, meta2]);
assert_eq!(resolved.meta.as_ref().and_then(|m| m.dialect.as_deref()), Some("mysql"));
assert_eq!(resolved.meta.as_ref().and_then(|m| m.charset.as_deref()), Some("utf8mb4"));
}
#[test]
fn empty_ddl_dir_returns_no_files() {
let dir = std::env::temp_dir().join("dbx_test_empty_ddl");
let _ = std::fs::create_dir_all(&dir);
let files = InputResolver::discover_ddl_files(&dir);
assert!(files.is_empty());
let _ = std::fs::remove_dir_all(&dir);
}
}
+233
View File
@@ -0,0 +1,233 @@
use std::collections::HashMap;
use std::path::Path;
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MetaData {
#[serde(default)]
pub dialect: Option<String>,
#[serde(default)]
pub version: Option<String>,
#[serde(default)]
pub charset: Option<String>,
#[serde(default)]
pub collation: Option<String>,
#[serde(default)]
pub sql_mode: Option<String>,
#[serde(default, skip_serializing_if = "HashMap::is_empty")]
pub explicit_dependencies: HashMap<String, Vec<String>>,
}
pub struct MetaReader;
impl MetaReader {
pub fn from_file(path: &Path) -> Result<MetaData, String> {
let content = std::fs::read_to_string(path).map_err(|e| format!("Failed to read meta file: {e}"))?;
let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("");
match ext {
"json" => Self::from_json(&content),
"yaml" | "yml" => Self::from_yaml(&content),
_ => Self::from_json(&content),
}
}
pub fn from_json(content: &str) -> Result<MetaData, String> {
serde_json::from_str(content).map_err(|e| format!("Invalid meta JSON: {e}"))
}
pub fn from_yaml(content: &str) -> Result<MetaData, String> {
serde_yaml::from_str(content).map_err(|e| format!("Invalid meta YAML: {e}"))
}
pub fn to_json(meta: &MetaData) -> Result<String, String> {
serde_json::to_string_pretty(meta).map_err(|e| format!("Serialize meta error: {e}"))
}
pub fn validate_consistency(meta: &MetaData) -> Vec<String> {
let mut warnings = Vec::new();
if let Some(ref dialect) = meta.dialect {
let lower = dialect.to_ascii_lowercase();
let known = [
"mysql",
"postgres",
"postgresql",
"sqlite",
"mssql",
"sqlserver",
"clickhouse",
"duckdb",
"oracle",
"tidb",
"mariadb",
];
if !known.contains(&lower.as_str()) {
warnings.push(format!("Unknown dialect '{}' in metadata", meta.dialect.as_ref().unwrap()));
}
}
if let Some(ref version) = meta.version {
let cleaned = version.trim_start_matches(|c: char| c.is_ascii_alphabetic() || c == 'v' || c == 'V');
let is_valid = !cleaned.is_empty()
&& cleaned.chars().all(|c| c.is_ascii_digit() || c == '.' || c == '-')
&& cleaned.chars().any(|c| c.is_ascii_digit());
if !is_valid {
warnings.push(format!("Unusual version format '{}'", version));
}
}
warnings
}
}
pub fn validate_sql_meta_consistency(sql: &str, meta: &MetaData) -> Vec<String> {
let mut warnings = Vec::new();
if let Some(ref charset) = meta.charset {
if !charset_mentions_match(sql, charset, meta.dialect.as_deref()) {
warnings.push(format!("SQL content does not reference charset '{}' from metadata", charset));
}
}
warnings
}
fn charset_mentions_match(sql: &str, charset: &str, dialect: Option<&str>) -> bool {
let sql_upper = sql.to_ascii_uppercase();
let charset_upper = charset.to_ascii_uppercase();
let is_pg = dialect.is_some_and(|d| matches!(d.to_ascii_lowercase().as_str(), "postgres" | "postgresql"));
if is_pg {
sql_upper.contains(&format!("ENCODING '{}'", charset_upper))
|| sql_upper.contains(&format!("ENCODING '{}'", charset_upper.to_ascii_lowercase()))
} else {
let pattern = format!("CHARACTER SET {}", charset_upper);
sql_upper.contains(&pattern) || sql_upper.contains(&format!("CHARSET {}", charset_upper))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn reads_valid_json_metadata() {
let json = r#"{
"dialect": "mysql",
"version": "8.0",
"charset": "utf8mb4",
"explicitDependencies": {
"view_orders": ["orders", "users"]
}
}"#;
let meta = MetaReader::from_json(json).unwrap();
assert_eq!(meta.dialect.as_deref(), Some("mysql"));
assert_eq!(meta.version.as_deref(), Some("8.0"));
assert_eq!(
meta.explicit_dependencies.get("view_orders").map(|v| v.as_slice()),
Some(&["orders".to_string(), "users".to_string()][..])
);
}
#[test]
fn reads_valid_yaml_metadata() {
let yaml = r#"
dialect: postgres
version: "15"
charset: UTF8
explicitDependencies:
expensive_view: [table_a, table_b]
"#;
let meta = MetaReader::from_yaml(yaml).unwrap();
assert_eq!(meta.dialect.as_deref(), Some("postgres"));
assert_eq!(meta.explicit_dependencies.get("expensive_view").map(|v| v.len()), Some(2));
}
#[test]
fn validates_known_dialect() {
let meta = MetaData {
dialect: Some("mysql".into()),
version: Some("8.0".into()),
charset: None,
collation: None,
sql_mode: None,
explicit_dependencies: HashMap::new(),
};
assert!(MetaReader::validate_consistency(&meta).is_empty());
}
#[test]
fn warns_unknown_dialect() {
let meta = MetaData {
dialect: Some("fakedb".into()),
version: None,
charset: None,
collation: None,
sql_mode: None,
explicit_dependencies: HashMap::new(),
};
let warnings = MetaReader::validate_consistency(&meta);
assert!(!warnings.is_empty());
assert!(warnings[0].contains("fakedb"));
}
#[test]
fn warns_unusual_version_format() {
let meta = MetaData {
dialect: Some("mysql".into()),
version: Some("latest".into()),
charset: None,
collation: None,
sql_mode: None,
explicit_dependencies: HashMap::new(),
};
let warnings = MetaReader::validate_consistency(&meta);
assert!(!warnings.is_empty());
assert!(warnings[0].contains("latest"));
}
#[test]
fn charset_consistency_check() {
let meta = MetaData {
dialect: Some("mysql".into()),
version: Some("8.0".into()),
charset: Some("utf8mb4".into()),
collation: None,
sql_mode: None,
explicit_dependencies: HashMap::new(),
};
let sql = "CREATE TABLE t (c VARCHAR(100)) CHARACTER SET utf8mb4";
assert!(validate_sql_meta_consistency(sql, &meta).is_empty());
let sql_no_charset = "CREATE TABLE t (c VARCHAR(100))";
let warnings = validate_sql_meta_consistency(sql_no_charset, &meta);
assert!(!warnings.is_empty());
assert!(warnings[0].contains("utf8mb4"));
}
#[test]
fn empty_meta_has_no_warnings() {
let meta = MetaData {
dialect: None,
version: None,
charset: None,
collation: None,
sql_mode: None,
explicit_dependencies: HashMap::new(),
};
assert!(MetaReader::validate_consistency(&meta).is_empty());
}
#[test]
fn serializes_and_deserializes_roundtrip() {
let meta = MetaData {
dialect: Some("postgres".into()),
version: Some("16".into()),
charset: Some("UTF8".into()),
collation: Some("en_US.UTF-8".into()),
sql_mode: None,
explicit_dependencies: HashMap::from([("v".into(), vec!["a".into(), "b".into()])]),
};
let json = MetaReader::to_json(&meta).unwrap();
let restored = MetaReader::from_json(&json).unwrap();
assert_eq!(restored.dialect, meta.dialect);
assert_eq!(restored.version, meta.version);
assert_eq!(restored.explicit_dependencies.len(), 1);
}
}
+4
View File
@@ -0,0 +1,4 @@
pub mod ast_filter;
pub mod git;
pub mod input;
pub mod meta;
+838
View File
@@ -0,0 +1,838 @@
use std::collections::HashMap;
use chrono::Utc;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use uuid::Uuid;
use crate::schema_diff::normalize_definition;
// ============================================================================
// 6.1: State Data Structures
// ============================================================================
const DRIFT_NONE: f64 = 0.0;
const DRIFT_NEW_OBJECT: f64 = 0.3;
const DRIFT_SINGLE_SIDE: f64 = 0.33;
const DRIFT_IDENTICAL_CHANGE: f64 = 0.5;
const DRIFT_DELETED: f64 = 0.66;
const DRIFT_STALE_BASELINE: f64 = 0.8;
const DRIFT_CONFLICT: f64 = 1.0;
const DRIFT_PSEUDO: f64 = 0.0;
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct SemanticFingerprint {
pub fingerprint: String,
pub object_fingerprints: HashMap<String, String>,
}
impl SemanticFingerprint {
pub fn compute(objects: &[ObjectDefinition]) -> Self {
let mut object_fingerprints = HashMap::new();
for obj in objects {
let normalized = normalize_definition(&obj.definition);
let hash = hex(Sha256::digest(normalized.as_bytes()));
object_fingerprints.insert(obj.key(), hash);
}
let mut all_keys: Vec<&String> = object_fingerprints.keys().collect();
all_keys.sort();
let mut hasher = Sha256::new();
for key in all_keys {
hasher.update(key.as_bytes());
hasher.update(b":");
hasher.update(object_fingerprints[key].as_bytes());
hasher.update(b";");
}
let fingerprint = hex(hasher.finalize());
Self { fingerprint, object_fingerprints }
}
pub fn object_fingerprint(def: &str) -> String {
let normalized = normalize_definition(def);
hex(Sha256::digest(normalized.as_bytes()))
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ObjectDefinition {
pub kind: ObjectKind,
pub name: String,
pub schema: Option<String>,
pub definition: String,
}
impl ObjectDefinition {
pub fn key(&self) -> String {
match &self.schema {
Some(schema) => format!("{}.{}.{}", self.kind.as_str(), schema, self.name),
None => format!("{}.{}", self.kind.as_str(), self.name),
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)]
#[serde(rename_all = "snake_case")]
pub enum ObjectKind {
Table,
View,
Function,
Sequence,
Rule,
Permission,
}
impl ObjectKind {
pub fn as_str(&self) -> &'static str {
match self {
Self::Table => "table",
Self::View => "view",
Self::Function => "function",
Self::Sequence => "sequence",
Self::Rule => "rule",
Self::Permission => "permission",
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct StateSnapshot {
pub id: String,
pub timestamp: String,
pub label: String,
pub objects: Vec<ObjectDefinition>,
pub fingerprint: SemanticFingerprint,
}
impl StateSnapshot {
pub fn new(label: &str, objects: Vec<ObjectDefinition>) -> Self {
let fingerprint = SemanticFingerprint::compute(&objects);
Self {
id: Uuid::new_v4().to_string(),
timestamp: Utc::now().to_rfc3339(),
label: label.to_string(),
objects,
fingerprint,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum ObjectReconciliationStatus {
Synced,
SourceDrifted,
TargetDrifted,
BothDriftedIdentical,
BothDriftedConflict,
DeletedInSource,
DeletedInTarget,
NewInSource,
NewInTarget,
NewInBoth,
BaselineOnly,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ObjectReconciliation {
pub object_key: String,
pub kind: ObjectKind,
pub name: String,
pub status: ObjectReconciliationStatus,
pub baseline_fingerprint: Option<String>,
pub source_fingerprint: Option<String>,
pub target_fingerprint: Option<String>,
pub source_definition: Option<String>,
pub target_definition: Option<String>,
pub drift_score: f64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ReconciliationResult {
pub baseline_id: String,
pub source_id: String,
pub target_id: String,
pub timestamp: String,
pub total_objects: usize,
pub synced_count: usize,
pub drifted_count: usize,
pub conflict_count: usize,
pub object_reconciliations: Vec<ObjectReconciliation>,
}
pub fn reconcile_three_way(
baseline: &StateSnapshot,
source_id: &str,
source: &[ObjectDefinition],
target_id: &str,
target: &[ObjectDefinition],
) -> ReconciliationResult {
let baseline_map: HashMap<String, &ObjectDefinition> = baseline.objects.iter().map(|o| (o.key(), o)).collect();
let source_map: HashMap<String, &ObjectDefinition> = source.iter().map(|o| (o.key(), o)).collect();
let target_map: HashMap<String, &ObjectDefinition> = target.iter().map(|o| (o.key(), o)).collect();
let all_keys: Vec<String> = {
let mut keys: Vec<String> = Vec::new();
let mut seen = std::collections::HashSet::new();
for map in [&baseline_map, &source_map, &target_map] {
for key in map.keys() {
if seen.insert(key.clone()) {
keys.push(key.clone());
}
}
}
keys.sort();
keys
};
let mut object_reconciliations = Vec::new();
let mut synced = 0;
let mut drifted = 0;
let mut conflicts = 0;
for key in &all_keys {
let b_obj = baseline_map.get(key);
let s_obj = source_map.get(key);
let t_obj = target_map.get(key);
let b_fp = b_obj.map(|o| SemanticFingerprint::object_fingerprint(&o.definition));
let s_fp = s_obj.map(|o| SemanticFingerprint::object_fingerprint(&o.definition));
let t_fp = t_obj.map(|o| SemanticFingerprint::object_fingerprint(&o.definition));
let (status, drift_score) = classify_reconciliation(&b_fp, &s_fp, &t_fp);
match status {
ObjectReconciliationStatus::Synced | ObjectReconciliationStatus::NewInBoth => synced += 1,
ObjectReconciliationStatus::BothDriftedConflict => {
drifted += 1;
conflicts += 1;
}
_ => drifted += 1,
}
let kind = b_obj.or(s_obj).or(t_obj).map(|o| o.kind.clone()).unwrap_or(ObjectKind::Table);
let name = b_obj.or(s_obj).or(t_obj).map(|o| o.name.clone()).unwrap_or_default();
object_reconciliations.push(ObjectReconciliation {
object_key: key.clone(),
kind,
name,
status,
baseline_fingerprint: b_fp,
source_fingerprint: s_fp,
target_fingerprint: t_fp,
source_definition: s_obj.map(|o| o.definition.clone()),
target_definition: t_obj.map(|o| o.definition.clone()),
drift_score,
});
}
ReconciliationResult {
baseline_id: baseline.id.clone(),
source_id: source_id.to_string(),
target_id: target_id.to_string(),
timestamp: Utc::now().to_rfc3339(),
total_objects: all_keys.len(),
synced_count: synced,
drifted_count: drifted,
conflict_count: conflicts,
object_reconciliations,
}
}
fn classify_reconciliation(
b_fp: &Option<String>,
s_fp: &Option<String>,
t_fp: &Option<String>,
) -> (ObjectReconciliationStatus, f64) {
match (b_fp, s_fp, t_fp) {
// All present
(Some(b), Some(s), Some(t)) => {
if b == s && s == t {
(ObjectReconciliationStatus::Synced, DRIFT_NONE)
} else if b == s && s != t {
(ObjectReconciliationStatus::TargetDrifted, DRIFT_SINGLE_SIDE)
} else if b == t && s != t {
(ObjectReconciliationStatus::SourceDrifted, DRIFT_SINGLE_SIDE)
} else if s == t && b != s {
(ObjectReconciliationStatus::BothDriftedIdentical, DRIFT_IDENTICAL_CHANGE)
} else {
(ObjectReconciliationStatus::BothDriftedConflict, DRIFT_CONFLICT)
}
}
// Baseline present, others may be absent
(Some(_), None, Some(_)) => (ObjectReconciliationStatus::DeletedInSource, DRIFT_DELETED),
(Some(_), Some(_), None) => (ObjectReconciliationStatus::DeletedInTarget, DRIFT_DELETED),
(Some(_), None, None) => (ObjectReconciliationStatus::BaselineOnly, DRIFT_STALE_BASELINE),
// Baseline absent, one or both present
(None, Some(s), Some(t)) => {
if s == t {
(ObjectReconciliationStatus::NewInBoth, DRIFT_NONE)
} else {
(ObjectReconciliationStatus::BothDriftedConflict, DRIFT_CONFLICT)
}
}
(None, Some(_), None) => (ObjectReconciliationStatus::NewInSource, DRIFT_NONE),
(None, None, Some(_)) => (ObjectReconciliationStatus::NewInTarget, DRIFT_NONE),
(None, None, None) => (ObjectReconciliationStatus::Synced, DRIFT_NONE),
}
}
// ============================================================================
// 6.2: Semantic Fingerprints — Drift Detection & Scoring
// ============================================================================
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct DriftReport {
pub has_drift: bool,
pub drifted_objects: Vec<DriftedObject>,
pub pseudo_drift_count: usize,
pub real_drift_count: usize,
pub overall_drift_score: f64,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct DriftedObject {
pub object_key: String,
pub baseline_fingerprint: String,
pub current_fingerprint: String,
pub is_pseudo_drift: bool,
pub score: f64,
}
pub fn detect_drift(baseline: &StateSnapshot, current: &[ObjectDefinition]) -> DriftReport {
let current_fp = SemanticFingerprint::compute(current);
let overall_has_drift = baseline.fingerprint.fingerprint != current_fp.fingerprint;
let current_map: HashMap<String, &ObjectDefinition> = current.iter().map(|o| (o.key(), o)).collect();
let mut drifted_objects = Vec::new();
let mut pseudo = 0;
let mut real = 0;
for obj in &baseline.objects {
let b_fp = baseline.fingerprint.object_fingerprints.get(&obj.key()).cloned().unwrap_or_default();
let c_fp = current_fp.object_fingerprints.get(&obj.key()).cloned().unwrap_or_default();
if b_fp != c_fp {
let is_pseudo =
filter_pseudo_drift(&obj.definition, current_map.get(&obj.key()).map(|o| o.definition.as_str()));
if is_pseudo {
pseudo += 1;
} else {
real += 1;
}
drifted_objects.push(DriftedObject {
object_key: obj.key(),
baseline_fingerprint: b_fp,
current_fingerprint: c_fp,
is_pseudo_drift: is_pseudo,
score: if is_pseudo { DRIFT_PSEUDO } else { DRIFT_IDENTICAL_CHANGE },
});
}
}
// Check for new objects in current
for obj in current {
if !baseline.fingerprint.object_fingerprints.contains_key(&obj.key()) {
real += 1;
drifted_objects.push(DriftedObject {
object_key: obj.key(),
baseline_fingerprint: String::new(),
current_fingerprint: current_fp.object_fingerprints.get(&obj.key()).cloned().unwrap_or_default(),
is_pseudo_drift: false,
score: DRIFT_NEW_OBJECT,
});
}
}
let total = drifted_objects.len();
let overall_drift_score =
if total == 0 { 0.0 } else { drifted_objects.iter().map(|d| d.score).sum::<f64>() / total as f64 };
DriftReport {
has_drift: overall_has_drift || !drifted_objects.is_empty(),
drifted_objects,
pseudo_drift_count: pseudo,
real_drift_count: real,
overall_drift_score,
}
}
fn filter_pseudo_drift(baseline_def: &str, current_def: Option<&str>) -> bool {
let Some(cur_def) = current_def else {
return false;
};
let b_norm = normalize_definition(baseline_def);
let c_norm = normalize_definition(cur_def);
// Pseudo-drift: normalized forms are equal (whitespace-only changes)
if b_norm == c_norm {
return true;
}
// Pseudo-drift: only comment lines changed
let b_no_comments = strip_sql_comments(&b_norm);
let c_no_comments = strip_sql_comments(&c_norm);
if b_no_comments == c_no_comments {
return true;
}
false
}
fn strip_sql_comments(sql: &str) -> String {
sql.lines()
.filter(|line| {
let trimmed = line.trim();
!(trimmed.starts_with("--") || trimmed.starts_with("//") || trimmed.starts_with('#'))
})
.collect::<Vec<_>>()
.join("\n")
}
fn hex(data: sha2::digest::Output<Sha256>) -> String {
data.iter().map(|b| format!("{:02x}", b)).collect()
}
// ============================================================================
// 6.3: Smart Baseline Reset
// ============================================================================
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum RebaseResolution {
AcceptSource,
AcceptTarget,
AcceptBaseline,
ManualReview,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct ConflictItem {
pub object_key: String,
pub resolution: RebaseResolution,
pub baseline_fingerprint: Option<String>,
pub source_fingerprint: Option<String>,
pub target_fingerprint: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct RebasePlan {
pub conflicts: Vec<ConflictItem>,
pub proposed_objects: Vec<ObjectDefinition>,
pub requires_manual_review: bool,
}
pub fn build_rebase_plan(
baseline: &StateSnapshot,
source: &[ObjectDefinition],
target: &[ObjectDefinition],
auto_resolve: bool,
) -> RebasePlan {
let result = reconcile_three_way(baseline, "", source, "", target);
let mut conflicts = Vec::new();
let mut proposed_objects: Vec<ObjectDefinition> = Vec::new();
let mut requires_manual = false;
let source_map: HashMap<String, &ObjectDefinition> = source.iter().map(|o| (o.key(), o)).collect();
let target_map: HashMap<String, &ObjectDefinition> = target.iter().map(|o| (o.key(), o)).collect();
for rec in &result.object_reconciliations {
match rec.status {
ObjectReconciliationStatus::Synced | ObjectReconciliationStatus::BothDriftedIdentical => {
if let Some(obj) = source_map.get(&rec.object_key).or_else(|| target_map.get(&rec.object_key)) {
proposed_objects.push((*obj).clone());
}
}
ObjectReconciliationStatus::TargetDrifted => {
if auto_resolve {
if let Some(obj) = target_map.get(&rec.object_key) {
proposed_objects.push((*obj).clone());
}
conflicts.push(ConflictItem {
object_key: rec.object_key.clone(),
resolution: RebaseResolution::AcceptTarget,
baseline_fingerprint: rec.baseline_fingerprint.clone(),
source_fingerprint: rec.source_fingerprint.clone(),
target_fingerprint: rec.target_fingerprint.clone(),
});
} else {
requires_manual = true;
conflicts.push(ConflictItem {
object_key: rec.object_key.clone(),
resolution: RebaseResolution::ManualReview,
baseline_fingerprint: rec.baseline_fingerprint.clone(),
source_fingerprint: rec.source_fingerprint.clone(),
target_fingerprint: rec.target_fingerprint.clone(),
});
}
}
ObjectReconciliationStatus::SourceDrifted => {
if auto_resolve {
if let Some(obj) = source_map.get(&rec.object_key) {
proposed_objects.push((*obj).clone());
}
conflicts.push(ConflictItem {
object_key: rec.object_key.clone(),
resolution: RebaseResolution::AcceptSource,
baseline_fingerprint: rec.baseline_fingerprint.clone(),
source_fingerprint: rec.source_fingerprint.clone(),
target_fingerprint: rec.target_fingerprint.clone(),
});
} else {
requires_manual = true;
conflicts.push(ConflictItem {
object_key: rec.object_key.clone(),
resolution: RebaseResolution::ManualReview,
baseline_fingerprint: rec.baseline_fingerprint.clone(),
source_fingerprint: rec.source_fingerprint.clone(),
target_fingerprint: rec.target_fingerprint.clone(),
});
}
}
ObjectReconciliationStatus::BothDriftedConflict => {
requires_manual = true;
conflicts.push(ConflictItem {
object_key: rec.object_key.clone(),
resolution: RebaseResolution::ManualReview,
baseline_fingerprint: rec.baseline_fingerprint.clone(),
source_fingerprint: rec.source_fingerprint.clone(),
target_fingerprint: rec.target_fingerprint.clone(),
});
}
ObjectReconciliationStatus::NewInBoth | ObjectReconciliationStatus::NewInSource => {
if let Some(obj) = source_map.get(&rec.object_key) {
proposed_objects.push((*obj).clone());
}
}
ObjectReconciliationStatus::NewInTarget => {
if let Some(obj) = target_map.get(&rec.object_key) {
proposed_objects.push((*obj).clone());
}
}
ObjectReconciliationStatus::DeletedInSource | ObjectReconciliationStatus::DeletedInTarget => {
// Object is deleted in at least one source, don't include it
}
ObjectReconciliationStatus::BaselineOnly => {
// Only in baseline, not in source or target — drop it
}
}
}
RebasePlan { conflicts, proposed_objects, requires_manual_review: requires_manual }
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct RebaseHistoryEntry {
pub id: String,
pub timestamp: String,
pub previous_baseline_id: String,
pub new_baseline_id: String,
pub object_count: usize,
pub conflict_count: usize,
pub auto_resolved: bool,
pub notes: String,
}
impl RebaseHistoryEntry {
pub fn new(
previous_baseline_id: &str,
new_baseline_id: &str,
object_count: usize,
conflict_count: usize,
auto_resolved: bool,
notes: &str,
) -> Self {
Self {
id: Uuid::new_v4().to_string(),
timestamp: Utc::now().to_rfc3339(),
previous_baseline_id: previous_baseline_id.to_string(),
new_baseline_id: new_baseline_id.to_string(),
object_count,
conflict_count,
auto_resolved,
notes: notes.to_string(),
}
}
}
// ============================================================================
// Tests
// ============================================================================
#[cfg(test)]
mod tests {
use super::*;
fn make_obj(kind: ObjectKind, name: &str, definition: &str) -> ObjectDefinition {
ObjectDefinition { kind, name: name.to_string(), schema: None, definition: definition.to_string() }
}
fn make_table(name: &str, def: &str) -> ObjectDefinition {
make_obj(ObjectKind::Table, name, def)
}
// --- 6.4: Three-way merge 9 combinations ---
#[test]
fn test_twm_all_synced() {
let objs = vec![make_table("t1", "CREATE TABLE t1 (id INT)")];
let baseline = StateSnapshot::new("baseline", objs.clone());
let source = objs.clone();
let target = objs.clone();
let result = reconcile_three_way(&baseline, "", &source, "", &target);
assert_eq!(result.synced_count, 1);
assert_eq!(result.drifted_count, 0);
assert_eq!(result.object_reconciliations[0].status, ObjectReconciliationStatus::Synced);
}
#[test]
fn test_twm_target_drifted() {
let objs = vec![make_table("t1", "CREATE TABLE t1 (id INT)")];
let baseline = StateSnapshot::new("baseline", objs.clone());
let source = objs.clone();
let target = vec![make_table("t1", "CREATE TABLE t1 (id BIGINT)")];
let result = reconcile_three_way(&baseline, "", &source, "", &target);
assert_eq!(result.drifted_count, 1);
assert_eq!(result.object_reconciliations[0].status, ObjectReconciliationStatus::TargetDrifted);
}
#[test]
fn test_twm_source_drifted() {
let objs = vec![make_table("t1", "CREATE TABLE t1 (id INT)")];
let baseline = StateSnapshot::new("baseline", objs.clone());
let source = vec![make_table("t1", "CREATE TABLE t1 (id BIGINT)")];
let target = objs.clone();
let result = reconcile_three_way(&baseline, "", &source, "", &target);
assert_eq!(result.drifted_count, 1);
assert_eq!(result.object_reconciliations[0].status, ObjectReconciliationStatus::SourceDrifted);
}
#[test]
fn test_twm_both_drifted_identical() {
let objs = vec![make_table("t1", "CREATE TABLE t1 (id INT)")];
let baseline = StateSnapshot::new("baseline", objs);
let source = vec![make_table("t1", "CREATE TABLE t1 (id BIGINT)")];
let target = vec![make_table("t1", "CREATE TABLE t1 (id BIGINT)")];
let result = reconcile_three_way(&baseline, "", &source, "", &target);
assert_eq!(result.drifted_count, 1);
assert_eq!(result.object_reconciliations[0].status, ObjectReconciliationStatus::BothDriftedIdentical);
}
#[test]
fn test_twm_both_drifted_conflict() {
let objs = vec![make_table("t1", "CREATE TABLE t1 (id INT)")];
let baseline = StateSnapshot::new("baseline", objs);
let source = vec![make_table("t1", "CREATE TABLE t1 (id BIGINT)")];
let target = vec![make_table("t1", "CREATE TABLE t1 (name TEXT)")];
let result = reconcile_three_way(&baseline, "", &source, "", &target);
assert_eq!(result.conflict_count, 1);
assert_eq!(result.object_reconciliations[0].status, ObjectReconciliationStatus::BothDriftedConflict);
}
#[test]
fn test_twm_target_deleted() {
let objs = vec![make_table("t1", "CREATE TABLE t1 (id INT)")];
let baseline = StateSnapshot::new("baseline", objs.clone());
let source = objs.clone();
let target: Vec<ObjectDefinition> = vec![];
let result = reconcile_three_way(&baseline, "", &source, "", &target);
assert_eq!(result.drifted_count, 1);
assert_eq!(result.object_reconciliations[0].status, ObjectReconciliationStatus::DeletedInTarget);
}
#[test]
fn test_twm_source_deleted() {
let objs = vec![make_table("t1", "CREATE TABLE t1 (id INT)")];
let baseline = StateSnapshot::new("baseline", objs.clone());
let source: Vec<ObjectDefinition> = vec![];
let target = objs.clone();
let result = reconcile_three_way(&baseline, "", &source, "", &target);
assert_eq!(result.drifted_count, 1);
assert_eq!(result.object_reconciliations[0].status, ObjectReconciliationStatus::DeletedInSource);
}
#[test]
fn test_twm_new_in_both() {
let objs = vec![make_table("t1", "CREATE TABLE t1 (id INT)")];
let baseline = StateSnapshot::new("baseline", vec![]);
let source = objs.clone();
let target = objs.clone();
let result = reconcile_three_way(&baseline, "", &source, "", &target);
assert_eq!(result.synced_count, 1);
assert_eq!(result.object_reconciliations[0].status, ObjectReconciliationStatus::NewInBoth);
}
#[test]
fn test_twm_new_in_source_only() {
let objs = vec![make_table("t1", "CREATE TABLE t1 (id INT)")];
let baseline = StateSnapshot::new("baseline", vec![]);
let source = objs.clone();
let target: Vec<ObjectDefinition> = vec![];
let result = reconcile_three_way(&baseline, "", &source, "", &target);
assert_eq!(result.synced_count, 0);
assert_eq!(result.object_reconciliations[0].status, ObjectReconciliationStatus::NewInSource);
}
#[test]
fn test_twm_baseline_only() {
let objs = vec![make_table("t1", "CREATE TABLE t1 (id INT)")];
let baseline = StateSnapshot::new("baseline", objs);
let source: Vec<ObjectDefinition> = vec![];
let target: Vec<ObjectDefinition> = vec![];
let result = reconcile_three_way(&baseline, "", &source, "", &target);
assert_eq!(result.object_reconciliations[0].status, ObjectReconciliationStatus::BaselineOnly);
}
// --- Fingerprint stability tests ---
#[test]
fn test_fingerprint_equivalent_ddl_same() {
let ddl1 = "CREATE TABLE t1 (id INT, name TEXT)";
let ddl2 = "CREATE TABLE t1 (id INT, name TEXT)";
assert_eq!(SemanticFingerprint::object_fingerprint(ddl1), SemanticFingerprint::object_fingerprint(ddl2));
}
#[test]
fn test_fingerprint_crlf_vs_lf() {
let ddl1 = "CREATE TABLE t1 (id INT)\n";
let ddl2 = "CREATE TABLE t1 (id INT)\r\n";
assert_eq!(SemanticFingerprint::object_fingerprint(ddl1), SemanticFingerprint::object_fingerprint(ddl2));
}
#[test]
fn test_fingerprint_tabs_vs_spaces() {
let ddl1 = "CREATE TABLE t1 (id INT)";
let ddl2 = "CREATE\tTABLE\tt1\t(id\tINT)";
assert_eq!(SemanticFingerprint::object_fingerprint(ddl1), SemanticFingerprint::object_fingerprint(ddl2));
}
#[test]
fn test_fingerprint_different_ddl_different() {
let ddl1 = "CREATE TABLE t1 (id INT)";
let ddl2 = "CREATE TABLE t1 (id BIGINT)";
assert_ne!(SemanticFingerprint::object_fingerprint(ddl1), SemanticFingerprint::object_fingerprint(ddl2));
}
// --- Drift detection tests ---
#[test]
fn test_detect_drift_no_drift() {
let objs = vec![make_table("t1", "CREATE TABLE t1 (id INT)")];
let baseline = StateSnapshot::new("baseline", objs.clone());
let report = detect_drift(&baseline, &objs);
assert!(!report.has_drift);
assert_eq!(report.drifted_objects.len(), 0);
}
#[test]
fn test_detect_drift_with_drift() {
let objs = vec![make_table("t1", "CREATE TABLE t1 (id INT)")];
let baseline = StateSnapshot::new("baseline", objs);
let current = vec![make_table("t1", "CREATE TABLE t1 (id BIGINT)")];
let report = detect_drift(&baseline, &current);
assert!(report.has_drift);
assert_eq!(report.real_drift_count, 1);
assert_eq!(report.pseudo_drift_count, 0);
}
#[test]
fn test_detect_pseudo_drift_whitespace() {
let objs = vec![make_table("t1", "CREATE TABLE t1 (id INT)")];
let baseline = StateSnapshot::new("baseline", objs);
let current = vec![make_table("t1", "CREATE TABLE t1 (id INT)")];
let report = detect_drift(&baseline, &current);
// Whitespace-only changes normalize to same fingerprint → no drift
assert!(!report.has_drift);
assert_eq!(report.pseudo_drift_count, 0);
assert_eq!(report.real_drift_count, 0);
}
#[test]
fn test_detect_pseudo_drift_comments_only() {
let objs = vec![make_table("t1", "CREATE TABLE t1 (id INT)\n-- this is a comment")];
let baseline = StateSnapshot::new("baseline", objs);
let current = vec![make_table("t1", "CREATE TABLE t1 (id INT)\n-- updated comment")];
let report = detect_drift(&baseline, &current);
assert!(report.has_drift);
assert_eq!(report.pseudo_drift_count, 1);
assert_eq!(report.real_drift_count, 0);
}
// --- Rebase plan tests ---
#[test]
fn test_rebase_plan_auto_resolve_synced() {
let objs = vec![make_table("t1", "CREATE TABLE t1 (id INT)")];
let baseline = StateSnapshot::new("baseline", objs.clone());
let plan = build_rebase_plan(&baseline, &objs, &objs, true);
assert!(!plan.requires_manual_review);
assert_eq!(plan.proposed_objects.len(), 1);
assert_eq!(plan.conflicts.len(), 0);
}
#[test]
fn test_rebase_plan_auto_resolve_target_drifted() {
let objs = vec![make_table("t1", "CREATE TABLE t1 (id INT)")];
let baseline = StateSnapshot::new("baseline", objs.clone());
let source = objs.clone();
let target = vec![make_table("t1", "CREATE TABLE t1 (id BIGINT)")];
let plan = build_rebase_plan(&baseline, &source, &target, true);
assert!(!plan.requires_manual_review);
assert_eq!(plan.conflicts.len(), 1);
assert_eq!(plan.conflicts[0].resolution, RebaseResolution::AcceptTarget);
}
#[test]
fn test_rebase_plan_requires_manual_on_conflict() {
let objs = vec![make_table("t1", "CREATE TABLE t1 (id INT)")];
let baseline = StateSnapshot::new("baseline", objs);
let source = vec![make_table("t1", "CREATE TABLE t1 (id BIGINT)")];
let target = vec![make_table("t1", "CREATE TABLE t1 (name TEXT)")];
let plan = build_rebase_plan(&baseline, &source, &target, true);
assert!(plan.requires_manual_review);
assert_eq!(plan.conflicts[0].resolution, RebaseResolution::ManualReview);
}
#[test]
fn test_rebase_history_entry_creation() {
let entry = RebaseHistoryEntry::new("baseline-1", "baseline-2", 5, 1, true, "Auto-resolved");
assert_eq!(entry.previous_baseline_id, "baseline-1");
assert_eq!(entry.new_baseline_id, "baseline-2");
assert_eq!(entry.object_count, 5);
assert_eq!(entry.conflict_count, 1);
assert!(entry.auto_resolved);
}
#[test]
fn test_filter_pseudo_drift_pure_whitespace() {
assert!(filter_pseudo_drift("CREATE TABLE t1 (id INT)", Some("CREATE TABLE t1 (id INT)")));
}
#[test]
fn test_filter_pseudo_drift_comment_changes() {
assert!(filter_pseudo_drift(
"CREATE TABLE t1 (id INT)\n-- comment v1",
Some("CREATE TABLE t1 (id INT)\n-- comment v2")
));
}
#[test]
fn test_filter_pseudo_drift_real_change() {
assert!(!filter_pseudo_drift("CREATE TABLE t1 (id INT)", Some("CREATE TABLE t1 (id BIGINT)")));
}
}
File diff suppressed because it is too large. Load diff
+125
View File
@@ -105,6 +105,7 @@ pub fn maybe_import_user_data_db(
Ok(DataDbImportResult::Imported)
}
#[derive(Clone)]
pub struct Storage {
db: SqliteHandle,
/// Path to the SQLite database file (`dbx.db`). Its parent directory is the
@@ -378,6 +379,13 @@ const SCHEMA_STATEMENTS: &[&str] = &[
config_json TEXT NOT NULL,
is_default INTEGER NOT NULL DEFAULT 0
)",
"CREATE TABLE IF NOT EXISTS state_store (
key TEXT PRIMARY KEY,
value BLOB NOT NULL,
content_type TEXT NOT NULL DEFAULT 'application/octet-stream',
version INTEGER NOT NULL DEFAULT 1,
payload BLOB DEFAULT x''
)",
"CREATE TABLE IF NOT EXISTS prompt_templates (
id TEXT PRIMARY KEY,
name TEXT NOT NULL UNIQUE,
@@ -412,6 +420,7 @@ impl Storage {
ensure_saved_sql_columns_sync(conn)?;
ensure_tab_runtime_cache_columns_sync(conn)?;
ensure_ai_configs_columns_sync(conn)?;
ensure_state_store_columns_sync(conn)?;
Ok(())
})
}
@@ -582,6 +591,19 @@ fn ensure_table_columns(conn: &Connection, table_name: &str, columns: &[(&str, &
Ok(())
}
fn ensure_state_store_columns_sync(conn: &Connection) -> Result<(), String> {
conn.execute("CREATE TABLE IF NOT EXISTS state_store (key TEXT PRIMARY KEY, value BLOB NOT NULL, content_type TEXT NOT NULL DEFAULT 'application/octet-stream', version INTEGER NOT NULL DEFAULT 1)", []).map_err(|e| e.to_string())?;
// SQLite ALTER TABLE ADD COLUMN rejects parenthesized default expressions.
const COLUMNS: &[(&str, &str)] = &[
("value", "BLOB NOT NULL DEFAULT x''"),
("content_type", "TEXT NOT NULL DEFAULT 'application/octet-stream'"),
("version", "INTEGER NOT NULL DEFAULT 1"),
("payload", "BLOB DEFAULT x''"),
];
ensure_table_columns(conn, "state_store", COLUMNS)
}
fn ssh_tunnel_secret_segment(index: usize, hop: &crate::models::connection::SshTunnelConfig) -> String {
if hop.id.trim().is_empty() {
index.to_string()
@@ -2907,6 +2929,109 @@ impl Storage {
})
.await
}
// State persistence store (CAS-aware key-value store for state machines)
pub async fn save_state(&self, key: &str, value: &[u8], content_type: &str) -> Result<(), String> {
let key = key.to_string();
let value = value.to_vec();
let content_type = content_type.to_string();
self.with_conn(move |conn| {
conn.execute(
"INSERT INTO state_store (key, value, content_type, version, payload) \
VALUES (?1, ?2, ?3, 1, x'') \
ON CONFLICT(key) DO UPDATE SET value = excluded.value, content_type = excluded.content_type, \
version = version + 1, payload = excluded.payload",
params![key, value, content_type],
)
.map(|_| ())
.map_err(|e| e.to_string())
})
.await
}
pub async fn load_state(&self, key: &str) -> Result<Option<(Vec<u8>, String)>, String> {
let key = key.to_string();
self.with_conn(move |conn| {
let mut stmt = conn
.prepare("SELECT value, content_type FROM state_store WHERE key = ?1")
.map_err(|e| e.to_string())?;
let result: Option<(Vec<u8>, String)> = stmt
.query_row(params![key], |row| Ok((row.get(0)?, row.get(1)?)))
.optional()
.map_err(|e| e.to_string())?;
Ok(result)
})
.await
}
pub async fn delete_state(&self, key: &str) -> Result<(), String> {
let key = key.to_string();
self.with_conn(move |conn| {
conn.execute("DELETE FROM state_store WHERE key = ?1", params![key]).map(|_| ()).map_err(|e| e.to_string())
})
.await
}
pub async fn state_exists(&self, key: &str) -> Result<bool, String> {
let key = key.to_string();
self.with_conn(move |conn| {
let exists: bool = conn
.query_row("SELECT EXISTS(SELECT 1 FROM state_store WHERE key = ?1)", params![key], |row| row.get(0))
.map_err(|e| e.to_string())?;
Ok(exists)
})
.await
}
pub async fn get_state_version(&self, key: &str) -> Result<Option<u64>, String> {
let key = key.to_string();
self.with_conn(move |conn| {
conn.prepare("SELECT version FROM state_store WHERE key = ?1")
.and_then(|mut stmt| stmt.query_row(params![key], |row| row.get(0)).optional())
.map_err(|e| e.to_string())
})
.await
}
pub async fn compare_and_swap_state(
&self,
key: &str,
expected_version: Option<u64>,
new_value: &[u8],
content_type: &str,
) -> Result<bool, String> {
let key = key.to_string();
let new_value = new_value.to_vec();
let content_type = content_type.to_string();
self.with_conn(move |conn| {
let current: Option<u64> = conn
.prepare("SELECT version FROM state_store WHERE key = ?1")
.and_then(|mut stmt| stmt.query_row(params![&key], |row| row.get(0)).optional())
.map_err(|e| e.to_string())?;
match (current, expected_version) {
(None, None) => {
conn.execute(
"INSERT INTO state_store (key, value, content_type, version, payload) VALUES (?1, ?2, ?3, 1, x'')",
params![key, new_value, content_type],
)
.map(|_| true)
.map_err(|e| e.to_string())
}
(Some(v), Some(expected)) if v == expected => {
conn.execute(
"UPDATE state_store SET value = ?1, content_type = ?2, version = version + 1 WHERE key = ?3 AND version = ?4",
params![new_value, content_type, key, expected],
)
.map(|rows| rows > 0)
.map_err(|e| e.to_string())
}
_ => Ok(false),
}
})
.await
}
}
// Tab runtime cache
+678
View File
@@ -0,0 +1,678 @@
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
use crate::state_persistence::{StateBackend, StateMachine};
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum TransactionStatus {
Preparing,
Prepared,
Committing,
Committed,
RollingBack,
RolledBack,
Mixed,
Unknown,
}
impl TransactionStatus {
pub fn as_str(&self) -> &'static str {
match self {
Self::Preparing => "preparing",
Self::Prepared => "prepared",
Self::Committing => "committing",
Self::Committed => "committed",
Self::RollingBack => "rolling_back",
Self::RolledBack => "rolled_back",
Self::Mixed => "mixed",
Self::Unknown => "unknown",
}
}
#[allow(clippy::should_implement_trait)]
pub fn from_str(s: &str) -> Self {
match s {
"preparing" => Self::Preparing,
"prepared" => Self::Prepared,
"committing" => Self::Committing,
"committed" => Self::Committed,
"rolling_back" => Self::RollingBack,
"rolled_back" => Self::RolledBack,
"mixed" => Self::Mixed,
_ => Self::Unknown,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct VoteResult {
pub participant_id: String,
pub vote: bool,
pub reason: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ParticipantCommitResult {
pub participant_id: String,
pub committed: bool,
pub error: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ParticipantInfo {
pub id: String,
pub name: String,
pub role: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TransactionLog {
pub transaction_id: String,
pub status: String,
pub participants: Vec<ParticipantInfo>,
pub created_at: String,
pub updated_at: String,
pub metadata: serde_json::Value,
}
impl TransactionLog {
fn new(transaction_id: String, participants: &[ParticipantInfo], metadata: serde_json::Value) -> Self {
Self {
transaction_id,
status: TransactionStatus::Preparing.as_str().to_string(),
participants: participants.to_vec(),
created_at: chrono::Utc::now().to_rfc3339(),
updated_at: chrono::Utc::now().to_rfc3339(),
metadata,
}
}
}
#[async_trait]
pub trait Participant: Send + Sync {
fn id(&self) -> &str;
fn name(&self) -> &str;
fn role(&self) -> &str;
async fn prepare(&self, transaction_id: &str) -> Result<(), String>;
async fn commit(&self, transaction_id: &str) -> Result<(), String>;
async fn rollback(&self, transaction_id: &str) -> Result<(), String>;
}
pub struct TwoPhaseCommit {
backend: Arc<dyn StateBackend>,
state_machine: StateMachine,
}
impl TwoPhaseCommit {
pub fn new(backend: Arc<dyn StateBackend>) -> Self {
let sm = StateMachine::new(backend.clone());
Self { backend, state_machine: sm }
}
fn log_key(transaction_id: &str) -> String {
format!("2pc:tx:{transaction_id}")
}
async fn save_log(&self, log: &TransactionLog) -> Result<(), String> {
let data = serde_json::to_vec(log).map_err(|e| e.to_string())?;
self.backend.save(&Self::log_key(&log.transaction_id), &data).await
}
async fn load_log(&self, transaction_id: &str) -> Result<Option<TransactionLog>, String> {
let raw = self.backend.load(&Self::log_key(transaction_id)).await?;
match raw {
Some(data) => {
let log: TransactionLog = serde_json::from_slice(&data).map_err(|e| e.to_string())?;
Ok(Some(log))
}
None => Ok(None),
}
}
async fn update_status(&self, transaction_id: &str, status: TransactionStatus) -> Result<TransactionLog, String> {
let mut log =
self.load_log(transaction_id).await?.ok_or_else(|| format!("Transaction not found: {transaction_id}"))?;
log.status = status.as_str().to_string();
log.updated_at = chrono::Utc::now().to_rfc3339();
self.save_log(&log).await?;
Ok(log)
}
pub async fn execute(
&self,
transaction_id: &str,
participants: &[Arc<dyn Participant>],
metadata: serde_json::Value,
) -> Result<TransactionLog, String> {
let infos: Vec<ParticipantInfo> = participants
.iter()
.map(|p| ParticipantInfo { id: p.id().to_string(), name: p.name().to_string(), role: p.role().to_string() })
.collect();
let log = TransactionLog::new(transaction_id.to_string(), &infos, metadata);
self.save_log(&log).await?;
let _state = self
.state_machine
.create_state(&format!("2pc_{transaction_id}"), serde_json::json!({"phase": "begin"}))
.await?;
let vote_results = self.prepare_phase(transaction_id, participants).await?;
let all_agreed = vote_results.iter().all(|v| v.vote);
if all_agreed {
self.update_status(transaction_id, TransactionStatus::Committing).await?;
let commit_results = self.commit_phase(transaction_id, participants, 2).await;
let all_committed =
participants.is_empty() || (!commit_results.is_empty() && commit_results.iter().all(|r| r.committed));
if all_committed {
let log = self.update_status(transaction_id, TransactionStatus::Committed).await?;
let _ = self
.state_machine
.transition(&format!("2pc_{transaction_id}"), crate::state_persistence::StateTransition::Completed)
.await;
Ok(log)
} else {
let _ = self.update_status(transaction_id, TransactionStatus::RollingBack).await?;
let _ = self.rollback_phase(transaction_id, participants).await;
// Use commit-phase outcomes only — never re-invoke commit() to probe.
let has_committed = commit_results.iter().any(|r| r.committed);
let final_status = if has_committed { TransactionStatus::Mixed } else { TransactionStatus::RolledBack };
let log = self.update_status(transaction_id, final_status).await?;
let _ = self
.state_machine
.transition(&format!("2pc_{transaction_id}"), crate::state_persistence::StateTransition::Failed)
.await;
let failures: Vec<String> = commit_results
.iter()
.filter(|r| !r.committed)
.map(|r| format!("{}: {}", r.participant_id, r.error.as_deref().unwrap_or("unknown")))
.collect();
Err(format!("Commit phase failed: {}. Status: {}", failures.join("; "), log.status))
}
} else {
let _ = self.update_status(transaction_id, TransactionStatus::RollingBack).await?;
let _ = self.rollback_phase(transaction_id, participants).await;
let log = self.update_status(transaction_id, TransactionStatus::RolledBack).await?;
let _ = self
.state_machine
.transition(&format!("2pc_{transaction_id}"), crate::state_persistence::StateTransition::Failed)
.await;
let rejections: Vec<String> = vote_results
.iter()
.filter_map(|v| {
if !v.vote {
Some(format!(
"{}: {}",
v.participant_id,
v.reason.clone().unwrap_or_else(|| "no reason".to_string())
))
} else {
None
}
})
.collect();
Err(format!(
"Prepare phase failed: participants did not agree. Rejections: [{}]. Status: {}",
rejections.join(", "),
log.status
))
}
}
async fn prepare_phase(
&self,
transaction_id: &str,
participants: &[Arc<dyn Participant>],
) -> Result<Vec<VoteResult>, String> {
self.update_status(transaction_id, TransactionStatus::Preparing).await?;
let mut results = Vec::new();
for p in participants {
match p.prepare(transaction_id).await {
Ok(()) => {
results.push(VoteResult { participant_id: p.id().to_string(), vote: true, reason: None });
}
Err(e) => {
results.push(VoteResult {
participant_id: p.id().to_string(),
vote: false,
reason: Some(e.clone()),
});
return Ok(results);
}
}
}
self.update_status(transaction_id, TransactionStatus::Prepared).await?;
Ok(results)
}
/// Run commit for all participants; always returns per-participant outcomes
/// (including partial success) so callers can mark Mixed without re-committing.
async fn commit_phase(
&self,
transaction_id: &str,
participants: &[Arc<dyn Participant>],
max_retries: u32,
) -> Vec<ParticipantCommitResult> {
let mut results: Vec<ParticipantCommitResult> = Vec::new();
let mut pending: Vec<&Arc<dyn Participant>> = participants.iter().collect();
for attempt in 0..=max_retries {
let mut next_pending = Vec::new();
for p in pending {
match p.commit(transaction_id).await {
Ok(()) => {
results.push(ParticipantCommitResult {
participant_id: p.id().to_string(),
committed: true,
error: None,
});
}
Err(e) => {
if attempt < max_retries {
log::warn!(
"2PC commit retry {}/{} for participant {}: {}",
attempt + 1,
max_retries,
p.id(),
e
);
next_pending.push(p);
} else {
results.push(ParticipantCommitResult {
participant_id: p.id().to_string(),
committed: false,
error: Some(e),
});
}
}
}
}
pending = next_pending;
if pending.is_empty() {
break;
}
}
results
}
async fn rollback_phase(&self, transaction_id: &str, participants: &[Arc<dyn Participant>]) -> Result<(), String> {
let mut errors = Vec::new();
for p in participants {
if let Err(e) = p.rollback(transaction_id).await {
errors.push(format!("{}: {}", p.id(), e));
}
}
if errors.is_empty() {
Ok(())
} else {
Err(errors.join("; "))
}
}
pub async fn recover(
&self,
transaction_id: &str,
participants: &[Arc<dyn Participant>],
) -> Result<TransactionLog, String> {
let log = self
.load_log(transaction_id)
.await?
.ok_or_else(|| format!("No transaction log found for: {transaction_id}"))?;
let status = TransactionStatus::from_str(&log.status);
match status {
TransactionStatus::Preparing | TransactionStatus::Prepared => {
self.rollback_phase(transaction_id, participants).await?;
self.update_status(transaction_id, TransactionStatus::RolledBack).await
}
TransactionStatus::Committing => {
let commit_results = self.commit_phase(transaction_id, participants, 2).await;
let all_committed = participants.is_empty()
|| (!commit_results.is_empty() && commit_results.iter().all(|r| r.committed));
if all_committed {
self.update_status(transaction_id, TransactionStatus::Committed).await
} else {
self.rollback_phase(transaction_id, participants).await?;
let has_committed = commit_results.iter().any(|r| r.committed);
let final_status =
if has_committed { TransactionStatus::Mixed } else { TransactionStatus::RolledBack };
self.update_status(transaction_id, final_status).await?;
let failures: Vec<String> = commit_results
.iter()
.filter(|r| !r.committed)
.map(|r| format!("{}: {}", r.participant_id, r.error.as_deref().unwrap_or("unknown")))
.collect();
Err(format!("Recovery failed during commit: {}", failures.join("; ")))
}
}
TransactionStatus::RollingBack => {
self.rollback_phase(transaction_id, participants).await?;
self.update_status(transaction_id, TransactionStatus::RolledBack).await
}
TransactionStatus::Committed | TransactionStatus::RolledBack | TransactionStatus::Mixed => Ok(log),
TransactionStatus::Unknown => {
Err(format!("Unknown transaction status for {transaction_id}: cannot recover"))
}
}
}
}
/// Participant adapter for database statement execution via TwoPhaseCommit.
/// The caller provides async closures for the actual database operations.
#[allow(clippy::type_complexity)]
pub struct FnParticipant {
id: String,
name: String,
role: String,
prepare_fn: Box<dyn Fn() -> Pin<Box<dyn Future<Output = Result<(), String>> + Send>> + Send + Sync>,
commit_fn: Box<dyn Fn() -> Pin<Box<dyn Future<Output = Result<(), String>> + Send>> + Send + Sync>,
rollback_fn: Box<dyn Fn() -> Pin<Box<dyn Future<Output = Result<(), String>> + Send>> + Send + Sync>,
}
impl FnParticipant {
pub fn new(
id: String,
name: String,
role: String,
prepare_fn: impl Fn() -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<(), String>> + Send>>
+ Send
+ Sync
+ 'static,
commit_fn: impl Fn() -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<(), String>> + Send>>
+ Send
+ Sync
+ 'static,
rollback_fn: impl Fn() -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<(), String>> + Send>>
+ Send
+ Sync
+ 'static,
) -> Self {
Self {
id,
name,
role,
prepare_fn: Box::new(prepare_fn),
commit_fn: Box::new(commit_fn),
rollback_fn: Box::new(rollback_fn),
}
}
}
#[async_trait]
impl Participant for FnParticipant {
fn id(&self) -> &str {
&self.id
}
fn name(&self) -> &str {
&self.name
}
fn role(&self) -> &str {
&self.role
}
async fn prepare(&self, _transaction_id: &str) -> Result<(), String> {
(self.prepare_fn)().await
}
async fn commit(&self, _transaction_id: &str) -> Result<(), String> {
(self.commit_fn)().await
}
async fn rollback(&self, _transaction_id: &str) -> Result<(), String> {
(self.rollback_fn)().await
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Mutex;
struct MockParticipant {
id: String,
name: String,
role: String,
should_fail_prepare: bool,
should_fail_commit: bool,
prepare_called: Mutex<bool>,
commit_called: Mutex<bool>,
rollback_called: Mutex<bool>,
}
impl MockParticipant {
fn new(id: &str, name: &str) -> Self {
Self {
id: id.to_string(),
name: name.to_string(),
role: "worker".to_string(),
should_fail_prepare: false,
should_fail_commit: false,
prepare_called: Mutex::new(false),
commit_called: Mutex::new(false),
rollback_called: Mutex::new(false),
}
}
fn with_prepare_failure(mut self) -> Self {
self.should_fail_prepare = true;
self
}
fn with_commit_failure(mut self) -> Self {
self.should_fail_commit = true;
self
}
#[allow(dead_code)]
fn prepare_was_called(&self) -> bool {
*self.prepare_called.lock().unwrap()
}
#[allow(dead_code)]
fn commit_was_called(&self) -> bool {
*self.commit_called.lock().unwrap()
}
#[allow(dead_code)]
fn rollback_was_called(&self) -> bool {
*self.rollback_called.lock().unwrap()
}
}
#[async_trait]
impl Participant for MockParticipant {
fn id(&self) -> &str {
&self.id
}
fn name(&self) -> &str {
&self.name
}
fn role(&self) -> &str {
&self.role
}
async fn prepare(&self, _transaction_id: &str) -> Result<(), String> {
*self.prepare_called.lock().unwrap() = true;
if self.should_fail_prepare {
Err(format!("{}: prepare failed", self.id))
} else {
Ok(())
}
}
async fn commit(&self, _transaction_id: &str) -> Result<(), String> {
*self.commit_called.lock().unwrap() = true;
if self.should_fail_commit {
Err(format!("{}: commit failed", self.id))
} else {
Ok(())
}
}
async fn rollback(&self, _transaction_id: &str) -> Result<(), String> {
*self.rollback_called.lock().unwrap() = true;
Ok(())
}
}
async fn make_2pc() -> (TwoPhaseCommit, std::path::PathBuf) {
use crate::storage::Storage;
let path = std::env::temp_dir().join(format!("test_2pc_{}.db", uuid::Uuid::new_v4()));
let storage = Arc::new(Storage::open(&path).await.unwrap());
let backend: Arc<dyn StateBackend> = Arc::new(crate::state_persistence::LocalBackend::new(storage));
(TwoPhaseCommit::new(backend), path)
}
#[tokio::test]
async fn two_phase_commit_success() {
let (coordinator, _tmp) = make_2pc().await;
let p1: Arc<dyn Participant> = Arc::new(MockParticipant::new("p1", "worker1"));
let p2: Arc<dyn Participant> = Arc::new(MockParticipant::new("p2", "worker2"));
let result =
coordinator.execute("tx_success", &[p1.clone(), p2.clone()], serde_json::json!({"op": "test"})).await;
assert!(result.is_ok());
let log = result.unwrap();
assert_eq!(log.status, TransactionStatus::Committed.as_str());
}
#[tokio::test]
async fn two_phase_commit_prepare_failure_triggers_rollback() {
let (coordinator, _tmp) = make_2pc().await;
let p1: Arc<dyn Participant> = Arc::new(MockParticipant::new("p1", "good").with_commit_failure());
let p2: Arc<dyn Participant> = Arc::new(MockParticipant::new("p2", "bad").with_prepare_failure());
let result = coordinator.execute("tx_prepare_fail", &[p1.clone(), p2.clone()], serde_json::json!({})).await;
assert!(result.is_err());
let err = result.unwrap_err();
assert!(err.contains("Prepare phase failed"));
assert!(err.contains("p2"));
}
#[tokio::test]
async fn two_phase_commit_commit_failure_triggers_rollback() {
let (coordinator, _tmp) = make_2pc().await;
let failing: Arc<dyn Participant> = Arc::new(MockParticipant::new("p1", "fail_commit").with_commit_failure());
let passing: Arc<dyn Participant> = Arc::new(MockParticipant::new("p2", "ok"));
let result =
coordinator.execute("tx_commit_fail", &[failing.clone(), passing.clone()], serde_json::json!({})).await;
assert!(result.is_err());
let log = coordinator.load_log("tx_commit_fail").await.unwrap().unwrap();
assert_eq!(log.status, TransactionStatus::Mixed.as_str());
}
#[tokio::test]
async fn two_phase_commit_empty_participants() {
let (coordinator, _tmp) = make_2pc().await;
let participants: Vec<Arc<dyn Participant>> = vec![];
let result = coordinator.execute("tx_empty", &participants, serde_json::json!({})).await;
assert!(result.is_ok());
let log = result.unwrap();
assert_eq!(log.status, TransactionStatus::Committed.as_str());
}
#[tokio::test]
async fn recovery_from_preparing_state() {
let (coordinator, _tmp) = make_2pc().await;
let p1: Arc<dyn Participant> = Arc::new(MockParticipant::new("p1", "w1"));
let log = TransactionLog {
transaction_id: "tx_recover_preparing".to_string(),
status: TransactionStatus::Preparing.as_str().to_string(),
participants: vec![ParticipantInfo {
id: "p1".to_string(),
name: "w1".to_string(),
role: "worker".to_string(),
}],
created_at: chrono::Utc::now().to_rfc3339(),
updated_at: chrono::Utc::now().to_rfc3339(),
metadata: serde_json::json!({}),
};
coordinator.save_log(&log).await.unwrap();
let result = coordinator.recover("tx_recover_preparing", &[Arc::clone(&p1)]).await;
assert!(result.is_ok());
let recovered = result.unwrap();
assert_eq!(recovered.status, TransactionStatus::RolledBack.as_str());
}
#[tokio::test]
async fn recovery_from_committing_state() {
let (coordinator, _tmp) = make_2pc().await;
let p1: Arc<dyn Participant> = Arc::new(MockParticipant::new("p1", "w1"));
let log = TransactionLog {
transaction_id: "tx_recover_committing".to_string(),
status: TransactionStatus::Committing.as_str().to_string(),
participants: vec![ParticipantInfo {
id: "p1".to_string(),
name: "w1".to_string(),
role: "worker".to_string(),
}],
created_at: chrono::Utc::now().to_rfc3339(),
updated_at: chrono::Utc::now().to_rfc3339(),
metadata: serde_json::json!({}),
};
coordinator.save_log(&log).await.unwrap();
let result = coordinator.recover("tx_recover_committing", &[Arc::clone(&p1)]).await;
assert!(result.is_ok());
let recovered = result.unwrap();
assert_eq!(recovered.status, TransactionStatus::Committed.as_str());
}
#[tokio::test]
async fn recovery_already_completed_noop() {
let (coordinator, _tmp) = make_2pc().await;
let p1: Arc<dyn Participant> = Arc::new(MockParticipant::new("p1", "w1"));
let log = TransactionLog {
transaction_id: "tx_done".to_string(),
status: TransactionStatus::Committed.as_str().to_string(),
participants: vec![ParticipantInfo {
id: "p1".to_string(),
name: "w1".to_string(),
role: "worker".to_string(),
}],
created_at: chrono::Utc::now().to_rfc3339(),
updated_at: chrono::Utc::now().to_rfc3339(),
metadata: serde_json::json!({}),
};
coordinator.save_log(&log).await.unwrap();
let result = coordinator.recover("tx_done", &[p1]).await;
assert!(result.is_ok());
let recovered = result.unwrap();
assert_eq!(recovered.status, TransactionStatus::Committed.as_str());
}
#[tokio::test]
async fn recovery_unknown_transaction_errors() {
let (coordinator, _tmp) = make_2pc().await;
let p1: Arc<dyn Participant> = Arc::new(MockParticipant::new("p1", "w1"));
let result = coordinator.recover("tx_nonexistent", &[p1]).await;
assert!(result.is_err());
}
}
@@ -0,0 +1,342 @@
// ============================================================================
// 12.4 — CLI/Tauri/dbx-web API Contract Verification
//
// These tests verify that public API types maintain backward-compatible
// serialization contracts (field names, optional field handling) and
// that function signatures haven't changed.
// ============================================================================
use dbx_core::data_compare::{DataComparePreparation, DataCompareSyncPlan};
use dbx_core::models::connection::DatabaseType;
use dbx_core::schema_diff::{prepare_schema_diff, SchemaDiffPreparation, SchemaDiffPreparationOptions};
use dbx_core::sql_risk::{classify_sql_risk, SqlRisk};
use dbx_core::types::{ColumnInfo, TableInfo};
// ============================================================================
// Compile-time checks: core function signatures compile
// ============================================================================
/// Verify prepare_schema_diff accepts SchemaDiffPreparationOptions and returns SchemaDiffPreparation
#[test]
fn prepare_schema_diff_function_signature() {
let options = SchemaDiffPreparationOptions {
source_tables: vec![TableInfo {
name: "t".to_string(),
table_type: "TABLE".to_string(),
comment: None,
parent_schema: None,
parent_name: None,
}],
target_tables: vec![TableInfo {
name: "t".to_string(),
table_type: "TABLE".to_string(),
comment: None,
parent_schema: None,
parent_name: None,
}],
source_details: vec![],
target_details: vec![],
source_functions: vec![],
target_functions: vec![],
source_sequences: vec![],
target_sequences: vec![],
source_rules: vec![],
target_rules: vec![],
source_owners: vec![],
target_owners: vec![],
database_type: DatabaseType::Postgres,
target_schema: None,
ignore_comments: false,
cascade_delete: false,
compare_column_order: false,
detect_renames: false,
detect_table_renames: false,
rename_threshold: 0.5,
enable_rollback: false,
batch_patterns: vec![],
source_dialect: None,
target_dialect: None,
compatibility_threshold: 0.5,
source_permissions: vec![],
target_permissions: vec![],
shard_strategy: None,
resource_constraint: None,
field_mappings: vec![],
};
let _result: SchemaDiffPreparation = prepare_schema_diff(options);
}
/// Verify generate_schema_sync_sql accepts all arg types
#[test]
fn generate_schema_sync_sql_function_signature() {
let _sql = dbx_core::schema_diff::generate_schema_sync_sql(
&[],
&[],
&[],
&[],
&[],
DatabaseType::Postgres,
None,
false,
None,
&[],
);
}
/// Verify classify_sql_risk function signature
#[test]
fn classify_sql_risk_function_signature() {
let _risk: SqlRisk = classify_sql_risk("SELECT 1", "postgres").unwrap();
}
// ============================================================================
// Serialization contract: field naming conventions
// ============================================================================
/// SchemaDiffPreparation fields use camelCase in JSON
#[test]
fn schema_diff_preparation_field_names() {
let result = prepare_schema_diff(SchemaDiffPreparationOptions {
source_tables: vec![TableInfo {
name: "t".to_string(),
table_type: "TABLE".to_string(),
comment: None,
parent_schema: None,
parent_name: None,
}],
target_tables: vec![TableInfo {
name: "t".to_string(),
table_type: "TABLE".to_string(),
comment: None,
parent_schema: None,
parent_name: None,
}],
source_details: vec![],
target_details: vec![],
source_functions: vec![],
target_functions: vec![],
source_sequences: vec![],
target_sequences: vec![],
source_rules: vec![],
target_rules: vec![],
source_owners: vec![],
target_owners: vec![],
database_type: DatabaseType::Postgres,
target_schema: None,
ignore_comments: false,
cascade_delete: false,
compare_column_order: false,
detect_renames: false,
detect_table_renames: false,
rename_threshold: 0.5,
enable_rollback: false,
batch_patterns: vec![],
source_dialect: None,
target_dialect: None,
compatibility_threshold: 0.5,
source_permissions: vec![],
target_permissions: vec![],
shard_strategy: None,
resource_constraint: None,
field_mappings: vec![],
});
let json = serde_json::to_value(&result).unwrap();
let obj = json.as_object().unwrap();
let keys: Vec<&str> = obj.keys().map(|k| k.as_str()).collect();
// All keys must be camelCase (no underscores)
for key in &keys {
assert!(!key.contains('_'), "Key '{}' should be camelCase, not snake_case", key);
}
// Core fields must be present
assert!(keys.contains(&"diffs"), "diffs field must be present");
assert!(keys.contains(&"syncSql"), "syncSql field must be present");
}
/// DataComparePreparation fields use camelCase
#[test]
fn data_compare_preparation_field_names() {
use dbx_core::data_compare::{DataCompareResult, DataCompareRow};
use serde_json::Value;
let prep = DataComparePreparation {
result: DataCompareResult {
added: vec![DataCompareRow {
key: "1".to_string(),
key_values: [("id".to_string(), Value::Number(1.into()))].into(),
values: [("name".to_string(), Value::String("Alice".to_string()))].into(),
}],
removed: vec![],
modified: vec![],
},
sync_statements: vec!["INSERT INTO t VALUES (1)".to_string()],
sync_sql: "INSERT INTO t VALUES (1)".to_string(),
};
let json = serde_json::to_value(&prep).unwrap();
let obj = json.as_object().unwrap();
for key in obj.keys() {
assert!(!key.contains('_'), "Key '{}' should be camelCase", key);
}
assert!(obj.contains_key("result"), "result must be present");
}
/// DataCompareSyncPlan fields use camelCase
#[test]
fn data_compare_sync_plan_field_names() {
let plan = DataCompareSyncPlan {
insert_count: 0,
update_count: 0,
delete_count: 0,
statement_count: 0,
sync_statements: vec![],
sync_sql: String::new(),
};
let json = serde_json::to_value(&plan).unwrap();
let obj = json.as_object().unwrap();
for key in obj.keys() {
assert!(!key.contains('_'), "Key '{}' should be camelCase", key);
}
}
// ============================================================================
// Type serialization roundtrip: types used in API boundaries
// ============================================================================
/// Core types must serialize/deserialize consistently
#[test]
fn core_types_serialization_roundtrip() {
let table = TableInfo {
name: "users".to_string(),
table_type: "BASE TABLE".to_string(),
comment: Some("user table".to_string()),
parent_schema: Some("public".to_string()),
parent_name: None,
};
let json = serde_json::to_value(&table).unwrap();
let deserialized: TableInfo = serde_json::from_value(json).unwrap();
assert_eq!(table.name, deserialized.name);
assert_eq!(table.comment, deserialized.comment);
}
/// ColumnInfo must serialize/deserialize consistently
#[test]
fn column_info_serialization_roundtrip() {
let col = ColumnInfo {
name: "id".to_string(),
data_type: "int".to_string(),
is_nullable: false,
column_default: None,
is_primary_key: true,
extra: None,
comment: None,
numeric_precision: Some(10),
numeric_scale: Some(0),
character_maximum_length: None,
enum_values: None,
character_set: None,
collation: None,
};
let json = serde_json::to_value(&col).unwrap();
let deserialized: ColumnInfo = serde_json::from_value(json).unwrap();
assert_eq!(col.name, deserialized.name);
assert_eq!(col.numeric_precision, deserialized.numeric_precision);
}
// ============================================================================
// DatabaseType serialization consistency
// ============================================================================
#[test]
fn database_type_serialization() {
let json = serde_json::to_value(DatabaseType::Postgres).unwrap();
assert_eq!(json, "postgres", "DatabaseType serializes using snake_case");
let json = serde_json::to_value(DatabaseType::Mysql).unwrap();
assert_eq!(json, "mysql", "DatabaseType serializes using snake_case");
}
// ============================================================================
// Tauri command argument compatibility: Option parameters
// ============================================================================
/// Verify that the types used as Tauri command args serialize/deserialize properly.
/// Tauri passes these as JSON from the frontend, so serde roundtrip must work.
#[test]
fn option_types_work_in_tauri_command_boundary() {
// Simulate how Tauri passes optional params from JS frontend
#[derive(serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "camelCase")]
struct TauriLikeCommand {
diffs: Vec<dbx_core::schema_diff::TableDiff>,
function_diffs: Option<Vec<dbx_core::schema_diff::FunctionDiff>>,
cascade_delete: Option<bool>,
target_schema: Option<String>,
}
// Without optional params (frontend omitting them)
let input = serde_json::json!({
"diffs": [],
"functionDiffs": null,
"cascadeDelete": null,
"targetSchema": null
});
let cmd: TauriLikeCommand = serde_json::from_value(input).unwrap();
assert!(cmd.function_diffs.is_none());
assert!(cmd.cascade_delete.is_none());
// With explicit values
let input = serde_json::json!({
"diffs": [],
"cascadeDelete": true
});
let cmd: TauriLikeCommand = serde_json::from_value(input).unwrap();
assert_eq!(cmd.cascade_delete, Some(true));
}
// ============================================================================
// Web API request compatibility
// ============================================================================
/// Verify GenerateSchemaSyncSqlRequest contract matches the core function signature
#[test]
fn web_api_schema_sync_request_fields() {
#[derive(serde::Deserialize)]
#[serde(rename_all = "camelCase")]
#[allow(dead_code)]
struct WebApiRequest {
diffs: Vec<dbx_core::schema_diff::TableDiff>,
function_diffs: Option<Vec<dbx_core::schema_diff::FunctionDiff>>,
sequence_diffs: Option<Vec<dbx_core::schema_diff::SequenceDiff>>,
rule_diffs: Option<Vec<dbx_core::schema_diff::RuleDiff>>,
owner_diffs: Option<Vec<dbx_core::schema_diff::OwnerDiff>>,
database_type: DatabaseType,
target_schema: Option<String>,
cascade_delete: Option<bool>,
}
let json = serde_json::json!({
"diffs": [],
"databaseType": "postgres"
});
let req: WebApiRequest = serde_json::from_value(json).unwrap();
assert!(req.diffs.is_empty());
assert_eq!(req.database_type, DatabaseType::Postgres);
assert!(req.target_schema.is_none());
assert!(req.cascade_delete.is_none());
}
// ============================================================================
// SqlRisk enum backward compat: JSON serialization
// ============================================================================
#[test]
fn sql_risk_json_representation() {
assert_eq!(serde_json::to_value(SqlRisk::ReadOnly).unwrap(), "ReadOnly");
assert_eq!(serde_json::to_value(SqlRisk::Write).unwrap(), "Write");
assert_eq!(serde_json::to_value(SqlRisk::Ddl).unwrap(), "Ddl");
assert_eq!(serde_json::to_value(SqlRisk::Transaction).unwrap(), "Transaction");
}
@@ -0,0 +1,273 @@
use dbx_core::models::connection::DatabaseType;
use dbx_core::schema_diff::{prepare_schema_diff, RollbackGraph, SchemaDiffPreparationOptions, TableSchemaDetail};
use dbx_core::types::{ColumnInfo, TableInfo};
fn table(name: &str) -> TableInfo {
TableInfo {
name: name.to_string(),
table_type: "BASE TABLE".to_string(),
comment: None,
parent_schema: None,
parent_name: None,
}
}
fn col(name: &str, data_type: &str) -> ColumnInfo {
ColumnInfo {
name: name.to_string(),
data_type: data_type.to_string(),
is_nullable: false,
column_default: None,
is_primary_key: false,
extra: None,
comment: None,
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
character_set: None,
collation: None,
}
}
fn detail(name: &str, columns: Vec<ColumnInfo>) -> TableSchemaDetail {
TableSchemaDetail {
name: name.to_string(),
columns,
indexes: vec![],
foreign_keys: vec![],
triggers: vec![],
ddl: None,
}
}
// ============================================================================
// 12.3 — Bidirectional Diff E2E Tests
// ============================================================================
/// Create scenario: source (desired) has [id, name, email], target (current) has [id, name]
/// Forward: ADD column email → Rollback: DROP column email
#[test]
fn bidirectional_diff_add_rollback_identity() {
let options = SchemaDiffPreparationOptions {
source_tables: vec![table("users")],
target_tables: vec![table("users")],
source_details: vec![detail(
"users",
vec![col("id", "int"), col("name", "varchar(64)"), col("email", "varchar(128)")],
)],
target_details: vec![detail("users", vec![col("id", "int"), col("name", "varchar(64)")])],
database_type: DatabaseType::Postgres,
enable_rollback: true,
..Default::default()
};
let result = prepare_schema_diff(options);
assert!(!result.diffs.is_empty(), "Should detect new column in source");
assert!(result.rollback_sync_sql.is_some(), "Rollback SQL should be generated");
let forward_sql = &result.sync_sql;
let rollback_sql = result.rollback_sync_sql.as_ref().unwrap();
// Forward should ADD column to target, rollback should DROP column
assert!(forward_sql.contains("ADD"), "Forward SQL should contain ADD");
assert!(rollback_sql.contains("DROP"), "Rollback SQL should contain DROP");
}
/// Delete scenario: source (desired) has [id], target (current) has [id, name, status]
/// Forward: DROP column name, status → Rollback: ADD column name, status
#[test]
fn bidirectional_diff_delete_rollback_identity() {
let options = SchemaDiffPreparationOptions {
source_tables: vec![table("users")],
target_tables: vec![table("users")],
source_details: vec![detail("users", vec![col("id", "int")])],
target_details: vec![detail(
"users",
vec![col("id", "int"), col("name", "varchar(64)"), col("status", "varchar(16)")],
)],
database_type: DatabaseType::Postgres,
enable_rollback: true,
..Default::default()
};
let result = prepare_schema_diff(options);
assert!(!result.diffs.is_empty(), "Should detect extra columns in target");
assert!(result.rollback_sync_sql.is_some(), "Rollback SQL should be generated");
let forward_sql = &result.sync_sql;
let rollback_sql = result.rollback_sync_sql.as_ref().unwrap();
// Forward should contain DROP for removed columns
assert!(forward_sql.contains("DROP"), "Forward SQL should contain DROP");
assert!(!rollback_sql.is_empty(), "Rollback SQL should not be empty");
}
/// Modify scenario: source (desired) has varchar(64), target (current) has varchar(128)
/// Forward: ALTER column type → Rollback: ALTER column back
#[test]
fn bidirectional_diff_modify_rollback_identity() {
let options = SchemaDiffPreparationOptions {
source_tables: vec![table("users")],
target_tables: vec![table("users")],
source_details: vec![detail("users", vec![col("name", "varchar(64)")])],
target_details: vec![detail("users", vec![col("name", "varchar(128)")])],
database_type: DatabaseType::Postgres,
enable_rollback: true,
..Default::default()
};
let result = prepare_schema_diff(options);
assert!(!result.diffs.is_empty(), "Should detect modified column");
assert_eq!(result.diffs[0].diff_type, "modified");
// Should have rollback SQL
assert!(result.rollback_sync_sql.is_some(), "Rollback SQL should be generated");
}
/// RollbackGraph consistency: forward ∘ rollback = identity
#[test]
fn rollback_graph_consistency() {
let options = SchemaDiffPreparationOptions {
source_tables: vec![table("users")],
target_tables: vec![table("users")],
source_details: vec![detail("users", vec![col("id", "int"), col("name", "varchar(64)")])],
target_details: vec![detail(
"users",
vec![col("id", "int"), col("name", "varchar(64)"), col("email", "varchar(128)")],
)],
database_type: DatabaseType::Postgres,
enable_rollback: true,
..Default::default()
};
let result = prepare_schema_diff(options);
assert!(result.rollback_graph.is_some(), "RollbackGraph should be generated");
let graph = result.rollback_graph.as_ref().unwrap();
assert!(graph.is_consistent, "RollbackGraph should be consistent");
assert!(graph.consistency_issues.is_empty(), "RollbackGraph should have no consistency issues");
}
/// RollbackGraph forward + rollback roundtrip: verify rollback_nodes are generated
#[test]
fn rollback_graph_generates_rollback_nodes() {
let options = SchemaDiffPreparationOptions {
source_tables: vec![table("users")],
target_tables: vec![table("users")],
source_details: vec![detail("users", vec![col("id", "int"), col("status", "varchar(16)")])],
target_details: vec![detail("users", vec![col("id", "int"), col("status", "varchar(32)")])],
database_type: DatabaseType::Postgres,
enable_rollback: true,
..Default::default()
};
let result = prepare_schema_diff(options);
assert!(result.rollback_graph.is_some());
let graph = result.rollback_graph.as_ref().unwrap();
assert!(!graph.rollback_nodes.is_empty(), "rollback_nodes should be non-empty");
assert!(graph.is_consistent, "RollbackGraph should be consistent");
}
/// Multiple table E2E: modify one table, add/remove columns in others
#[test]
fn bidirectional_diff_multi_table_e2e() {
let options = SchemaDiffPreparationOptions {
source_tables: vec![table("orders"), table("items"), table("archive")],
target_tables: vec![table("orders"), table("items"), table("archive")],
source_details: vec![
// desired: orders with larger decimal + status column added
detail("orders", vec![col("id", "int"), col("total", "decimal(12,2)"), col("status", "varchar(16)")]),
// desired: items with larger name
detail("items", vec![col("id", "int"), col("name", "varchar(128)")]),
// desired: archive with only id
detail("archive", vec![col("id", "int")]),
],
target_details: vec![
// current: orders with smaller decimal, no status
detail("orders", vec![col("id", "int"), col("total", "decimal(10,2)")]),
// current: items with smaller name
detail("items", vec![col("id", "int"), col("name", "varchar(64)")]),
// current: archive with extra column
detail("archive", vec![col("id", "int"), col("data", "text")]),
],
database_type: DatabaseType::Postgres,
enable_rollback: true,
..Default::default()
};
let result = prepare_schema_diff(options);
assert!(!result.diffs.is_empty(), "Should detect differences across multiple tables");
assert!(result.rollback_sync_sql.is_some(), "Rollback SQL should be generated");
let forward_sql = &result.sync_sql;
let rollback_sql = result.rollback_sync_sql.as_ref().unwrap();
assert!(!forward_sql.is_empty(), "Forward SQL should be non-empty");
assert!(!rollback_sql.is_empty(), "Rollback SQL should be non-empty");
// Forward and rollback should be different
assert_ne!(forward_sql, rollback_sql, "Forward and rollback SQL should differ");
}
/// Rename detection: verify rename candidates are populated with matching column structures
#[test]
fn bidirectional_diff_with_rename_detection() {
let options = SchemaDiffPreparationOptions {
source_tables: vec![table("users")],
target_tables: vec![table("user_profiles")],
source_details: vec![detail("users", vec![col("id", "int"), col("name", "varchar(64)")])],
target_details: vec![detail("user_profiles", vec![col("id", "int"), col("name", "varchar(64)")])],
database_type: DatabaseType::Postgres,
detect_renames: true,
detect_table_renames: true,
rename_threshold: 0.5,
..Default::default()
};
let result = prepare_schema_diff(options);
// With identical column structure, rename should at least produce diffs
assert!(!result.diffs.is_empty(), "Should detect difference between differently-named tables");
// With identical column structures and table rename detection on, candidates must be populated
assert!(
!result.rename_candidates.is_empty(),
"Rename candidates should be detected for identical column structures"
);
assert!(result.rename_candidates[0].score > 0.0, "Rename score should be positive");
}
/// RollbackGraph validate_consistency directly
#[test]
fn rollback_graph_direct_consistency_check() {
use dbx_core::schema_diff::DependencyGraph;
let forward_diff = dbx_core::schema_diff::TableDiff {
diff_type: "added".to_string(),
object_type: None,
name: "users".to_string(),
columns: Some(vec![]),
indexes: None,
foreign_keys: None,
triggers: None,
ddl: Some("CREATE TABLE users (id INT)".to_string()),
target_ddl: None,
source_table_comment: None,
target_table_comment: None,
sync_sql: None,
};
let dep_graph =
DependencyGraph { nodes: std::collections::HashMap::new(), topological_order: vec!["users".to_string()] };
let mut graph = RollbackGraph::from_forward_diffs(&[forward_diff], &[], &dep_graph);
graph.validate_consistency();
assert!(graph.is_consistent, "Simple add should produce consistent rollback graph");
assert!(graph.consistency_issues.is_empty());
}
@@ -0,0 +1,248 @@
use dbx_core::models::connection::DatabaseType;
use dbx_core::schema_diff::{
generate_schema_sync_sql, prepare_schema_diff, SchemaDiffPreparationOptions, TableSchemaDetail,
};
use dbx_core::sql_dialect::descriptor::DialectKind;
use dbx_core::types::{ColumnInfo, TableInfo};
fn table(name: &str) -> TableInfo {
TableInfo {
name: name.to_string(),
table_type: "BASE TABLE".to_string(),
comment: None,
parent_schema: None,
parent_name: None,
}
}
fn col(name: &str, data_type: &str) -> ColumnInfo {
ColumnInfo {
name: name.to_string(),
data_type: data_type.to_string(),
is_nullable: false,
column_default: None,
is_primary_key: false,
extra: None,
comment: None,
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
character_set: None,
collation: None,
}
}
fn detail(name: &str, columns: Vec<ColumnInfo>) -> TableSchemaDetail {
TableSchemaDetail {
name: name.to_string(),
columns,
indexes: vec![],
foreign_keys: vec![],
triggers: vec![],
ddl: None,
}
}
// ============================================================================
// 12.2 — Cross-dialect full-chain integration tests
// ============================================================================
/// MySQL → PostgreSQL: diff a schema between MySQL source and PG target,
/// verify SQL output uses PostgreSQL syntax and identifiers.
#[test]
fn mysql_to_postgresql_full_chain_diff_and_sql_generation() {
let options = SchemaDiffPreparationOptions {
source_tables: vec![table("users")],
target_tables: vec![table("users")],
source_details: vec![detail(
"users",
vec![col("id", "int"), col("name", "varchar(64)"), col("email", "varchar(128)")],
)],
target_details: vec![detail(
"users",
vec![col("id", "serial"), col("name", "text"), col("email", "varchar(128)"), col("age", "int")],
)],
database_type: DatabaseType::Postgres,
source_dialect: Some(DialectKind::Mysql),
target_dialect: Some(DialectKind::Postgres),
..Default::default()
};
let result = prepare_schema_diff(options);
// Should detect column type changes and an extra column in target
assert!(!result.diffs.is_empty(), "Should detect differences between MySQL and PG schemas");
// SQL should use PostgreSQL double-quote identifiers
let pg_sql = &result.sync_sql;
assert!(pg_sql.contains('"'), "PostgreSQL SQL should use double-quoted identifiers");
assert!(!pg_sql.contains('`'), "PostgreSQL SQL should NOT use backtick identifiers");
// Verify compatibility warnings if type differences exist
// varchar(64) → text may or may not generate a warning depending on threshold
}
/// PostgreSQL → SQLite: diff a schema between PG source and SQLite target,
/// verify SQL output uses SQLite-compatible syntax.
#[test]
fn postgresql_to_sqlite_full_chain_diff_and_sql_generation() {
let options = SchemaDiffPreparationOptions {
source_tables: vec![table("products")],
target_tables: vec![table("products")],
source_details: vec![detail(
"products",
vec![col("id", "serial"), col("name", "varchar(100)"), col("price", "numeric(10,2)")],
)],
target_details: vec![detail(
"products",
vec![col("id", "integer"), col("name", "text"), col("price", "real"), col("category", "text")],
)],
database_type: DatabaseType::Sqlite,
source_dialect: Some(DialectKind::Postgres),
target_dialect: Some(DialectKind::Sqlite),
..Default::default()
};
let result = prepare_schema_diff(options);
assert!(!result.diffs.is_empty(), "Should detect differences between PG and SQLite schemas");
// SQLite identifiers are double-quoted or unquoted
let sqlite_sql = &result.sync_sql;
// SQLite uses CREATE TABLE IF NOT EXISTS etc.
assert!(!sqlite_sql.contains("::"), "SQLite SQL should not contain PG type casts");
}
/// MySQL → SQLite: full chain cross-dialect comparison
#[test]
fn mysql_to_sqlite_cross_dialect_diff() {
let options = SchemaDiffPreparationOptions {
source_tables: vec![table("events")],
target_tables: vec![table("events")],
source_details: vec![detail(
"events",
vec![
col("id", "bigint unsigned"),
col("event_type", "enum('click','view')"),
col("created_at", "datetime"),
],
)],
target_details: vec![detail(
"events",
vec![col("id", "integer"), col("event_type", "text"), col("created_at", "text"), col("source", "text")],
)],
database_type: DatabaseType::Sqlite,
source_dialect: Some(DialectKind::Mysql),
target_dialect: Some(DialectKind::Sqlite),
..Default::default()
};
let result = prepare_schema_diff(options);
assert!(!result.diffs.is_empty(), "MySQL enum → SQLite text should be detected as a difference");
}
// ============================================================================
// 12.2 — Same-dialect consistency: diff should be empty when schemas match
// ============================================================================
#[test]
fn identical_mysql_schemas_produce_no_diff() {
let options = SchemaDiffPreparationOptions {
source_tables: vec![table("users")],
target_tables: vec![table("users")],
source_details: vec![detail("users", vec![col("id", "int"), col("name", "varchar(64)")])],
target_details: vec![detail("users", vec![col("id", "int"), col("name", "varchar(64)")])],
database_type: DatabaseType::Mysql,
..Default::default()
};
let result = prepare_schema_diff(options);
assert!(result.diffs.is_empty(), "Identical schemas should produce no diffs");
assert!(result.sync_sql.is_empty(), "Identical schemas should produce no SQL");
}
#[test]
fn identical_postgres_schemas_produce_no_diff() {
let options = SchemaDiffPreparationOptions {
source_tables: vec![table("users")],
target_tables: vec![table("users")],
source_details: vec![detail("users", vec![col("id", "int"), col("name", "text")])],
target_details: vec![detail("users", vec![col("id", "int"), col("name", "text")])],
database_type: DatabaseType::Postgres,
..Default::default()
};
let result = prepare_schema_diff(options);
assert!(result.diffs.is_empty(), "Identical schemas should produce no diffs");
}
// ============================================================================
// 12.2 — Cross-dialect generate_schema_sync_sql verification
// ============================================================================
#[test]
fn generate_schema_sync_sql_mysql_output_format() {
let diffs = vec![dbx_core::schema_diff::TableDiff {
diff_type: "added".to_string(),
object_type: None,
name: "users".to_string(),
columns: Some(vec![]),
indexes: None,
foreign_keys: None,
triggers: None,
ddl: Some("CREATE TABLE `users` (`id` INT NOT NULL, `name` VARCHAR(64) NOT NULL)".to_string()),
target_ddl: None,
source_table_comment: None,
target_table_comment: None,
sync_sql: None,
}];
let sql = generate_schema_sync_sql(&diffs, &[], &[], &[], &[], DatabaseType::Mysql, None, false, None, &[]);
assert!(sql.contains("CREATE TABLE"), "MySQL SQL should contain CREATE TABLE");
assert!(sql.contains('`'), "MySQL SQL should use backtick identifiers");
}
#[test]
fn generate_schema_sync_sql_postgres_output_format() {
let diffs = vec![dbx_core::schema_diff::TableDiff {
diff_type: "added".to_string(),
object_type: None,
name: "users".to_string(),
columns: Some(vec![]),
indexes: None,
foreign_keys: None,
triggers: None,
ddl: Some("CREATE TABLE users (id SERIAL PRIMARY KEY, name TEXT NOT NULL)".to_string()),
target_ddl: None,
source_table_comment: None,
target_table_comment: None,
sync_sql: None,
}];
let sql = generate_schema_sync_sql(&diffs, &[], &[], &[], &[], DatabaseType::Postgres, None, false, None, &[]);
assert!(sql.contains("CREATE TABLE"), "PostgreSQL SQL should contain CREATE TABLE");
assert!(sql.contains("SERIAL"), "PostgreSQL SQL should use SERIAL type");
}
#[test]
fn generate_schema_sync_sql_sqlite_output_format() {
let diffs = vec![dbx_core::schema_diff::TableDiff {
diff_type: "added".to_string(),
object_type: None,
name: "users".to_string(),
columns: Some(vec![]),
indexes: None,
foreign_keys: None,
triggers: None,
ddl: Some("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT NOT NULL)".to_string()),
target_ddl: None,
source_table_comment: None,
target_table_comment: None,
sync_sql: None,
}];
let sql = generate_schema_sync_sql(&diffs, &[], &[], &[], &[], DatabaseType::Sqlite, None, false, None, &[]);
assert!(sql.contains("CREATE TABLE"), "SQLite SQL should contain CREATE TABLE");
}
@@ -0,0 +1,274 @@
// ============================================================================
// 12.5 — Performance Benchmarks
//
// These benchmarks measure schema diff performance with large table sets.
// Run with: cargo test --release --test performance_benchmarks -- --ignored
// ============================================================================
use std::time::Instant;
use dbx_core::models::connection::DatabaseType;
use dbx_core::schema_diff::{
prepare_schema_diff, SchemaDiffPreparationOptions, ShardBy, ShardStrategy, TableSchemaDetail,
};
use dbx_core::types::{ColumnInfo, TableInfo};
fn generate_tables(count: usize, prefix: &str) -> Vec<TableInfo> {
(0..count)
.map(|i| TableInfo {
name: format!("{}_{}", prefix, i),
table_type: "BASE TABLE".to_string(),
comment: None,
parent_schema: None,
parent_name: None,
})
.collect()
}
fn generate_details(tables: &[TableInfo], columns_per_table: usize) -> Vec<TableSchemaDetail> {
tables
.iter()
.map(|t| TableSchemaDetail {
name: t.name.clone(),
columns: (0..columns_per_table)
.map(|j| ColumnInfo {
name: format!("col_{}", j),
data_type: if j % 3 == 0 { "int".to_string() } else { "varchar(64)".to_string() },
is_nullable: j % 2 == 0,
column_default: if j == 0 { Some("0".to_string()) } else { None },
is_primary_key: j == 0,
extra: None,
comment: None,
numeric_precision: if j % 3 == 0 { Some(10) } else { None },
numeric_scale: if j % 3 == 0 { Some(0) } else { None },
character_maximum_length: if j % 3 != 0 { Some(64) } else { None },
enum_values: None,
character_set: None,
collation: None,
})
.collect(),
indexes: vec![],
foreign_keys: vec![],
triggers: vec![],
ddl: None,
})
.collect()
}
/// Benchmark: diff a small number of tables (baseline)
#[ignore]
#[test]
fn benchmark_small_schema_diff() {
let table_count = 10;
let columns_per_table = 5;
let source_tables = generate_tables(table_count, "src");
let target_tables = generate_tables(table_count, "tgt");
let source_details = generate_details(&source_tables, columns_per_table);
let target_details = generate_details(&target_tables, columns_per_table);
let options = SchemaDiffPreparationOptions {
source_tables,
target_tables,
source_details,
target_details,
database_type: DatabaseType::Postgres,
..Default::default()
};
let start = Instant::now();
let result = prepare_schema_diff(options);
let elapsed = start.elapsed();
assert!(!result.diffs.is_empty(), "Should detect differences");
eprintln!("[BENCHMARK] small_schema_diff ({} tables, {} cols): {:?}", table_count, columns_per_table, elapsed);
}
/// Benchmark: diff a medium number of tables
#[ignore]
#[test]
fn benchmark_medium_schema_diff() {
let table_count = 100;
let columns_per_table = 10;
let source_tables = generate_tables(table_count, "src");
let target_tables = generate_tables(table_count, "tgt");
let source_details = generate_details(&source_tables, columns_per_table);
let target_details = generate_details(&target_tables, columns_per_table);
let options = SchemaDiffPreparationOptions {
source_tables,
target_tables,
source_details,
target_details,
database_type: DatabaseType::Postgres,
..Default::default()
};
let start = Instant::now();
let result = prepare_schema_diff(options);
let elapsed = start.elapsed();
assert!(!result.diffs.is_empty(), "Should detect differences");
eprintln!("[BENCHMARK] medium_schema_diff ({} tables, {} cols): {:?}", table_count, columns_per_table, elapsed);
}
/// Benchmark: diff a large number of tables
#[ignore]
#[test]
fn benchmark_large_schema_diff() {
let table_count = 200;
let columns_per_table = 10;
let source_tables = generate_tables(table_count, "src");
let target_tables = generate_tables(table_count, "tgt");
let source_details = generate_details(&source_tables, columns_per_table);
let target_details = generate_details(&target_tables, columns_per_table);
let options = SchemaDiffPreparationOptions {
source_tables,
target_tables,
source_details,
target_details,
database_type: DatabaseType::Postgres,
..Default::default()
};
let start = Instant::now();
let result = prepare_schema_diff(options);
let elapsed = start.elapsed();
assert!(!result.diffs.is_empty(), "Should detect differences");
eprintln!("[BENCHMARK] large_schema_diff ({} tables, {} cols): {:?}", table_count, columns_per_table, elapsed);
}
/// Benchmark: diff 1000 tables (simulates the 1000-table requirement)
#[ignore]
#[test]
fn benchmark_thousand_table_schema_diff() {
let table_count = 1000;
let columns_per_table = 5;
let source_tables = generate_tables(table_count, "src");
let target_tables = generate_tables(table_count, "tgt");
let source_details = generate_details(&source_tables, columns_per_table);
let target_details = generate_details(&target_tables, columns_per_table);
let options = SchemaDiffPreparationOptions {
source_tables,
target_tables,
source_details,
target_details,
database_type: DatabaseType::Postgres,
..Default::default()
};
let start = Instant::now();
let result = prepare_schema_diff(options);
let elapsed = start.elapsed();
assert!(!result.diffs.is_empty(), "Should detect differences");
eprintln!(
"[BENCHMARK] thousand_table_schema_diff ({} tables, {} cols): {:?}",
table_count, columns_per_table, elapsed
);
}
/// Benchmark: shard-parallel speedup comparison
#[ignore]
#[test]
fn benchmark_shard_parallel_speedup() {
let table_count = 200;
let columns_per_table = 8;
let source_tables = generate_tables(table_count, "src");
let target_tables = generate_tables(table_count, "tgt");
let source_details = generate_details(&source_tables, columns_per_table);
let target_details = generate_details(&target_tables, columns_per_table);
// Single-threaded (shard_count = 1)
let options_single = SchemaDiffPreparationOptions {
source_tables: source_tables.clone(),
target_tables: target_tables.clone(),
source_details: source_details.clone(),
target_details: target_details.clone(),
database_type: DatabaseType::Postgres,
..Default::default()
};
let start_single = Instant::now();
let _result_single = prepare_schema_diff(options_single);
let single_time = start_single.elapsed();
// Shard-parallel (shard_count = 4)
let shard_options = SchemaDiffPreparationOptions {
source_tables: source_tables.clone(),
target_tables: target_tables.clone(),
source_details: source_details.clone(),
target_details: target_details.clone(),
database_type: DatabaseType::Postgres,
shard_strategy: Some(ShardStrategy { shard_count: 4, shard_by: ShardBy::RoundRobin }),
..Default::default()
};
let start_parallel = Instant::now();
let _result_parallel = prepare_schema_diff(shard_options);
let parallel_time = start_parallel.elapsed();
eprintln!(
"[BENCHMARK] shard_speedup: single={:?}, parallel(4 shards)={:?}, speedup={:.2}x",
single_time,
parallel_time,
single_time.as_secs_f64() / parallel_time.as_secs_f64().max(0.0001)
);
assert!(
parallel_time <= single_time * 3, // Allow some overhead; shard_count=4 may not be 4x faster on small data
"Parallel should not be significantly slower than single-threaded"
);
}
/// Benchmark: DependencyGraph build time
#[ignore]
#[test]
fn benchmark_dependency_graph_build() {
use dbx_core::schema_diff::DependencyGraph;
use dbx_core::types::ForeignKeyInfo;
let details: Vec<TableSchemaDetail> = (0..100)
.map(|i| {
let mut fks = vec![];
if i > 0 {
fks.push(ForeignKeyInfo {
name: format!("fk_{}_to_{}", i, i - 1),
column: "parent_id".to_string(),
ref_schema: None,
ref_table: format!("tgt_{}", i - 1),
ref_column: "id".to_string(),
on_update: None,
on_delete: None,
});
}
TableSchemaDetail {
name: format!("tgt_{}", i),
columns: vec![],
indexes: vec![],
foreign_keys: fks,
triggers: vec![],
ddl: None,
}
})
.collect();
let tables: Vec<TableInfo> = (0..100)
.map(|i| TableInfo {
name: format!("tgt_{}", i),
table_type: "BASE TABLE".to_string(),
comment: None,
parent_schema: None,
parent_name: None,
})
.collect();
let start = Instant::now();
let graph = DependencyGraph::build(&details, &tables);
let elapsed = start.elapsed();
assert_eq!(graph.nodes.len(), 100);
eprintln!("[BENCHMARK] dependency_graph_build (100 tables): {:?}", elapsed);
}
@@ -0,0 +1,297 @@
use dbx_core::data_compare::{DataCompareFromTablesPreparation, DataComparePreparation, DataCompareResult};
use dbx_core::models::connection::DatabaseType;
use dbx_core::schema_diff::{
generate_schema_sync_sql, prepare_schema_diff, SchemaDiffPreparation, SchemaDiffPreparationOptions, TableDiff,
TableSchemaDetail,
};
use dbx_core::sql_risk::{classify_sql_risk, SqlRisk};
use dbx_core::types::{ColumnInfo, TableInfo};
// ============================================================================
// 12.1 — SchemaDiffPreparation backward-compatibility regression tests
// ============================================================================
fn basic_table_info(name: &str) -> TableInfo {
TableInfo {
name: name.to_string(),
table_type: "BASE TABLE".to_string(),
comment: None,
parent_schema: None,
parent_name: None,
}
}
fn col(name: &str, data_type: &str) -> ColumnInfo {
ColumnInfo {
name: name.to_string(),
data_type: data_type.to_string(),
is_nullable: false,
column_default: None,
is_primary_key: false,
extra: None,
comment: None,
numeric_precision: None,
numeric_scale: None,
character_maximum_length: None,
enum_values: None,
character_set: None,
collation: None,
}
}
fn detail(name: &str, columns: Vec<ColumnInfo>) -> TableSchemaDetail {
TableSchemaDetail {
name: name.to_string(),
columns,
indexes: vec![],
foreign_keys: vec![],
triggers: vec![],
ddl: None,
}
}
/// Verify that new optional fields in SchemaDiffPreparation are well-formed
/// when using basic (non-enhanced) options, ensuring backward compatibility
/// for existing callers that only use the original fields.
#[test]
fn schema_diff_preparation_new_fields_defaults_backward_compatible() {
let options = SchemaDiffPreparationOptions {
source_tables: vec![basic_table_info("users")],
target_tables: vec![basic_table_info("users")],
source_details: vec![detail("users", vec![col("name", "varchar(64)")])],
target_details: vec![detail("users", vec![col("name", "varchar(128)")])],
database_type: DatabaseType::Postgres,
..Default::default()
};
let result = prepare_schema_diff(options);
// Original required fields must be present
assert!(!result.diffs.is_empty(), "diffs should be non-empty");
assert!(!result.sync_sql.is_empty(), "sync_sql should be non-empty");
// New optional fields from Phase 4+ must have sensible defaults
assert!(result.rollback_sync_sql.is_none(), "rollback_sync_sql should default to None");
assert!(result.rename_candidates.is_empty(), "rename_candidates should default to empty");
assert!(result.rollback_graph.is_none(), "rollback_graph should default to None");
assert!(result.compatibility_warnings.is_empty(), "compatibility_warnings should default to empty");
assert!(result.permission_diffs.is_empty(), "permission_diffs should default to empty");
assert!(result.permission_sync_sql.is_none(), "permission_sync_sql should default to None");
// dependency_graph is always built by prepare_schema_diff regardless of options
// (it's used internally for topological ordering)
assert!(result.dependency_graph.is_some(), "dependency_graph should be present (always built for ordering)");
}
/// Verify that SchemaDiffPreparation serializes to JSON with backward-compatible
/// field names (camelCase) and optional fields skipped when None/empty.
#[test]
fn schema_diff_preparation_json_serialization_skips_optional_fields() {
let options = SchemaDiffPreparationOptions {
source_tables: vec![basic_table_info("users")],
target_tables: vec![basic_table_info("users")],
source_details: vec![detail("users", vec![col("name", "varchar(64)")])],
target_details: vec![detail("users", vec![col("name", "varchar(128)")])],
database_type: DatabaseType::Postgres,
..Default::default()
};
let result = prepare_schema_diff(options);
let json = serde_json::to_value(&result).expect("Serialization should succeed");
// Original fields must be present
assert!(json.get("diffs").is_some(), "diffs must be present");
assert!(json.get("syncSql").is_some(), "syncSql must be present");
// New optional fields should NOT appear when None/empty
assert!(json.get("rollbackSyncSql").is_none(), "rollbackSyncSql should be absent");
assert!(json.get("renameCandidates").is_none(), "renameCandidates should be absent");
assert!(json.get("rollbackGraph").is_none(), "rollbackGraph should be absent");
assert!(json.get("compatibilityWarnings").is_none(), "compatibilityWarnings should be absent");
assert!(json.get("permissionDiffs").is_none(), "permissionDiffs should be absent");
assert!(json.get("permissionSyncSql").is_none(), "permissionSyncSql should be absent");
// dependency_graph is always populated by prepare_schema_diff
assert!(json.get("dependencyGraph").is_some(), "dependencyGraph is always built by prepare_schema_diff");
}
/// Verify that an old-format JSON (without new fields) can be deserialized
/// into SchemaDiffPreparation with defaults.
#[test]
fn schema_diff_preparation_deserializes_old_format_with_defaults() {
let old_json = serde_json::json!({
"diffs": [],
"syncSql": "SELECT 1"
});
let result: SchemaDiffPreparation = serde_json::from_value(old_json).expect("Old-format JSON should deserialize");
assert!(result.diffs.is_empty());
assert_eq!(result.sync_sql, "SELECT 1");
assert!(result.rollback_sync_sql.is_none());
assert!(result.rename_candidates.is_empty());
assert!(result.dependency_graph.is_none());
}
/// Verify that generate_schema_sync_sql output is invariant under default options
/// — i.e., callers that don't pass rollback/rename/permission options still get
/// identical SQL output.
#[test]
fn generate_schema_sync_sql_output_invariant() {
let diffs = vec![TableDiff {
diff_type: "added".to_string(),
object_type: None,
name: "users".to_string(),
columns: Some(vec![]),
indexes: None,
foreign_keys: None,
triggers: None,
ddl: Some("CREATE TABLE users (id INT)".to_string()),
target_ddl: None,
source_table_comment: None,
target_table_comment: None,
sync_sql: None,
}];
let sql = generate_schema_sync_sql(&diffs, &[], &[], &[], &[], DatabaseType::Postgres, None, false, None, &[]);
assert!(sql.contains("CREATE TABLE"), "SQL should contain CREATE TABLE");
assert!(sql.contains("users"), "SQL should contain table name");
}
/// Verify that with all enhancement flags off, the output of
/// prepare_schema_diff matches the same as if they didn't exist.
#[test]
fn prepare_schema_diff_with_default_options_matches_basic_diff() {
let options = SchemaDiffPreparationOptions {
source_tables: vec![basic_table_info("orders")],
target_tables: vec![basic_table_info("orders")],
source_details: vec![detail("orders", vec![col("id", "int"), col("status", "varchar(16)")])],
target_details: vec![detail("orders", vec![col("id", "int"), col("status", "varchar(32)")])],
database_type: DatabaseType::Postgres,
..Default::default()
};
let result = prepare_schema_diff(options);
assert!(!result.diffs.is_empty());
assert_eq!(result.diffs.len(), 1);
assert_eq!(result.diffs[0].diff_type, "modified");
assert!(!result.sync_sql.is_empty());
}
// ============================================================================
// 12.1 — SqlRisk backward-compatibility regression tests
// ============================================================================
/// Verify that SqlRisk::ReadOnly classification is unchanged
#[test]
fn sql_risk_readonly_classification_invariant() {
assert_eq!(classify_sql_risk("SELECT * FROM users", "postgres").unwrap(), SqlRisk::ReadOnly);
assert_eq!(classify_sql_risk("SELECT id, name FROM users WHERE id = 1", "mysql").unwrap(), SqlRisk::ReadOnly);
assert_eq!(classify_sql_risk("SHOW TABLES", "mysql").unwrap(), SqlRisk::ReadOnly);
assert_eq!(classify_sql_risk("DESCRIBE users", "mysql").unwrap(), SqlRisk::ReadOnly);
assert_eq!(classify_sql_risk("EXPLAIN SELECT * FROM users", "postgres").unwrap(), SqlRisk::ReadOnly);
assert_eq!(classify_sql_risk("WITH cte AS (SELECT 1) SELECT * FROM cte", "postgres").unwrap(), SqlRisk::ReadOnly);
}
/// Verify that SqlRisk::Write classification is unchanged
#[test]
fn sql_risk_write_classification_invariant() {
assert_eq!(classify_sql_risk("INSERT INTO users VALUES (1)", "postgres").unwrap(), SqlRisk::Write);
assert_eq!(classify_sql_risk("UPDATE users SET name = 'x'", "postgres").unwrap(), SqlRisk::Write);
assert_eq!(classify_sql_risk("DELETE FROM users WHERE id = 1", "postgres").unwrap(), SqlRisk::Write);
}
/// Verify that SqlRisk::Ddl classification is unchanged
#[test]
fn sql_risk_ddl_classification_invariant() {
assert_eq!(classify_sql_risk("CREATE TABLE users (id INT)", "postgres").unwrap(), SqlRisk::Ddl);
assert_eq!(classify_sql_risk("DROP TABLE users", "postgres").unwrap(), SqlRisk::Ddl);
assert_eq!(classify_sql_risk("ALTER TABLE users ADD COLUMN age INT", "postgres").unwrap(), SqlRisk::Ddl);
assert_eq!(classify_sql_risk("TRUNCATE TABLE users", "postgres").unwrap(), SqlRisk::Ddl);
}
/// Verify that multi-statement returns highest risk (unchanged)
#[test]
fn sql_risk_multi_statement_invariant() {
assert_eq!(classify_sql_risk("SELECT 1; INSERT INTO users VALUES (1)", "postgres").unwrap(), SqlRisk::Write);
assert_eq!(classify_sql_risk("SELECT 1; CREATE TABLE t (id INT)", "postgres").unwrap(), SqlRisk::Ddl);
}
// ============================================================================
// 12.1 — DataCompare result format compatibility tests
// ============================================================================
/// Verify that DataCompareFromTablesPreparation serializes with backward-compatible
/// field names and new optional fields are skipped when None.
#[test]
fn data_compare_from_tables_preparation_optional_fields_default() {
let prep = DataCompareFromTablesPreparation {
result: DataCompareResult { added: vec![], removed: vec![], modified: vec![] },
sync_statements: vec![],
sync_sql: String::new(),
pre_sync_statements: vec![],
source_row_count: 0,
target_row_count: 0,
source_truncated: false,
target_truncated: false,
degradation_level: None,
sampling_rate: None,
confidence_score: None,
verification_method: None,
source_checksums: None,
target_checksums: None,
};
let json = serde_json::to_value(&prep).expect("Serialization should succeed");
// Core fields must be present
assert!(json.get("result").is_some(), "result must be present");
assert!(json.get("syncSql").is_some(), "syncSql must be present");
// New optional fields should be absent when None
assert!(json.get("degradationLevel").is_none(), "degradationLevel should be absent");
assert!(json.get("samplingRate").is_none(), "samplingRate should be absent");
assert!(json.get("confidenceScore").is_none(), "confidenceScore should be absent");
assert!(json.get("verificationMethod").is_none(), "verificationMethod should be absent");
assert!(json.get("sourceChecksums").is_none(), "sourceChecksums should be absent");
assert!(json.get("targetChecksums").is_none(), "targetChecksums should be absent");
}
/// Verify that an old-format JSON can be deserialized into DataCompareFromTablesPreparation
#[test]
fn data_compare_from_tables_preparation_deserializes_old_format() {
let old_json = serde_json::json!({
"result": { "added": [], "removed": [], "modified": [] },
"syncStatements": [],
"syncSql": "",
"preSyncStatements": [],
"sourceRowCount": 100,
"targetRowCount": 100,
"sourceTruncated": false,
"targetTruncated": false
});
let prep: DataCompareFromTablesPreparation =
serde_json::from_value(old_json).expect("Old-format JSON should deserialize");
assert_eq!(prep.source_row_count, 100);
assert_eq!(prep.target_row_count, 100);
assert!(prep.degradation_level.is_none());
assert!(prep.sampling_rate.is_none());
assert!(prep.confidence_score.is_none());
}
/// Verify that DataComparePreparation (the original struct) remains unchanged
#[test]
fn data_compare_preparation_format_stable() {
let prep = DataComparePreparation {
result: DataCompareResult { added: vec![], removed: vec![], modified: vec![] },
sync_statements: vec!["INSERT INTO t VALUES (1)".to_string()],
sync_sql: "INSERT INTO t VALUES (1)".to_string(),
};
let json = serde_json::to_value(&prep).expect("Serialization should succeed");
assert_eq!(json["syncStatements"][0], "INSERT INTO t VALUES (1)");
assert_eq!(json["syncSql"], "INSERT INTO t VALUES (1)");
assert!(json.get("result").is_some());
}