fix(query): invalidate persisted object metadata after DDL

This commit is contained in:
soddygo authored and GitHub committed 2026-09-14 22:26:56 +08:00
1 parent c6ecd9808f
commit 5918414127
9 files changed
+689 -31

No files matched your search

+1
View File
@@ -68,6 +68,7 @@ pub mod mysql_event_sql;
pub mod nacos;
#[cfg(all(target_os = "windows", target_env = "gnu"))]
mod nanosleep_stub;
pub mod object_cache;
pub mod object_source_sql;
pub mod path_utils;
pub mod plugins;
+136
View File
@@ -0,0 +1,136 @@
//! Persisted object metadata shared by Web and SQL execution entry points.
//!
//! These keys are also produced by the frontend's encodeURIComponent-based
//! caches. Keep the encoding and namespace stable when invalidating snapshots.
use crate::models::connection::DatabaseType;
use crate::storage::Storage;
pub const OBJECT_METADATA_CACHE_PREFIX: &str = "object-meta:v1";
const OBJECT_DDL_CACHE_PREFIX: &str = "object-ddl:v1";
pub fn metadata_cache_segment(value: &str) -> String {
const HEX: &[u8; 16] = b"0123456789ABCDEF";
let mut encoded = String::with_capacity(value.len());
for byte in value.bytes() {
match byte {
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'!' | b'~' | b'*' | b'\'' | b'(' | b')' => {
encoded.push(byte as char)
}
_ => {
encoded.push('%');
encoded.push(HEX[(byte >> 4) as usize] as char);
encoded.push(HEX[(byte & 0x0f) as usize] as char);
}
}
}
encoded
}
pub fn object_metadata_cache_prefix(connection_id: &str, database: &str) -> String {
format!(
"{}:{}:{}:",
OBJECT_METADATA_CACHE_PREFIX,
metadata_cache_segment(connection_id),
metadata_cache_segment(database)
)
}
/// Conservative cache policy, separate from SQL authorization. A failed DDL
/// attempt may already have committed, so callers apply this even on errors.
pub(crate) fn sql_may_change_object_metadata(sql: &str, db_type: Option<DatabaseType>) -> bool {
let statements = match db_type {
Some(db_type) => crate::sql::split_sql_statements_for_database(sql, db_type),
None => crate::sql::split_sql_statements(sql),
};
statements.iter().any(|statement| {
let risk = match db_type {
Some(db_type) => crate::sql_risk::classify_sql_risk_for_database(statement, db_type),
None => crate::sql_risk::classify_sql_risk(statement, "generic"),
};
if matches!(risk, Ok(crate::sql_risk::SqlRisk::Ddl)) {
return true;
}
// Include transaction completion: a previous DDL call may have run
// before commit, while another connection cached the old structure.
let keywords = [
"CREATE", "ALTER", "DROP", "TRUNCATE", "COMMENT", "GRANT", "REVOKE", "RENAME", "COMMIT", "END", "ROLLBACK",
"ABORT",
];
match db_type {
Some(db_type) => crate::sql::starts_with_executable_sql_keyword_for_database(statement, &keywords, db_type),
None => crate::sql::starts_with_executable_sql_keyword(statement, &keywords),
}
})
}
/// Connection scope also covers qualified cross-database DDL and session USE.
/// Await before returning the SQL result; cache failures must not turn an
/// already committed database operation into an apparent execution failure.
pub(crate) async fn invalidate_connection_object_cache(storage: &Storage, connection_id: &str) {
let connection = metadata_cache_segment(connection_id);
for namespace in [OBJECT_METADATA_CACHE_PREFIX, OBJECT_DDL_CACHE_PREFIX] {
let prefix = format!("{namespace}:{connection}:");
if let Err(error) = storage.delete_schema_cache_prefix(&prefix).await {
log::warn!(
"failed to invalidate object cache: connection_id={connection_id} namespace={namespace} error={error}"
);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn ddl_schema_cache_classifies_scripts_without_changing_permissions() {
for sql in [
"ALTER TABLE users ADD COLUMN test4 INTEGER",
"BEGIN; ALTER TABLE users ADD COLUMN test4 INTEGER; COMMIT;",
"-- leading comment\nCOMMENT ON COLUMN users.test4 IS 'new'",
"CREATE SOMETHING dialect_specific",
"COMMIT",
"END",
"ROLLBACK",
"ABORT",
] {
assert!(sql_may_change_object_metadata(sql, Some(DatabaseType::Postgres)), "{sql}");
}
for sql in [
"SELECT 'ALTER TABLE users'",
"-- DROP TABLE users\nSELECT 1",
"INSERT INTO users VALUES (1)",
"UPDATE users SET id = 1",
"DELETE FROM users",
"BEGIN",
"",
"/* ALTER TABLE */",
] {
assert!(!sql_may_change_object_metadata(sql, Some(DatabaseType::Postgres)), "{sql}");
}
}
#[tokio::test]
async fn ddl_schema_cache_encoding_and_connection_isolation() {
let dir = tempfile::tempdir().unwrap();
let storage = Storage::open(&dir.path().join("storage.db")).await.unwrap();
let id = "conn:% 中文";
assert_eq!(metadata_cache_segment(id), "conn%3A%25%20%E4%B8%AD%E6%96%87");
let prefix = object_metadata_cache_prefix(id, "db% name");
assert_eq!(prefix, "object-meta:v1:conn%3A%25%20%E4%B8%AD%E6%96%87:db%25%20name:");
let removed = [
format!("{prefix}public:users::backend-columns:"),
format!("object-ddl:v1:{}:other-db:public:users::TABLE:", metadata_cache_segment(id)),
];
let kept = format!("{}columns:", object_metadata_cache_prefix(&format!("{id}2"), "db% name"));
for key in removed.iter().chain(std::iter::once(&kept)) {
storage.save_schema_cache(key, &serde_json::json!([])).await.unwrap();
}
invalidate_connection_object_cache(&storage, id).await;
for key in removed {
assert!(storage.load_schema_cache(&key).await.unwrap().is_none());
}
assert!(storage.load_schema_cache(&kept).await.unwrap().is_some());
}
}
+203
View File
@@ -2774,6 +2774,33 @@ pub async fn execute_sql_statement_with_options_typed(
schema: Option<&str>,
cancel_token: Option<CancellationToken>,
options: QueryExecutionOptions,
) -> Result<db::QueryResult, QueryExecutionError> {
let db_type = connection_database_type(state, connection_id).await;
let invalidate = crate::object_cache::sql_may_change_object_metadata(sql, db_type);
let result = execute_sql_statement_with_options_typed_inner(
state,
connection_id,
database,
sql,
schema,
cancel_token,
options,
)
.await;
if invalidate {
crate::object_cache::invalidate_connection_object_cache(&state.storage, connection_id).await;
}
result
}
async fn execute_sql_statement_with_options_typed_inner(
state: &AppState,
connection_id: &str,
database: &str,
sql: &str,
schema: Option<&str>,
cancel_token: Option<CancellationToken>,
options: QueryExecutionOptions,
) -> Result<db::QueryResult, QueryExecutionError> {
// MongoDB connections use shell-style commands dispatched through the
// frontend parser. Queries that fall through to the generic SQL executor
@@ -3093,6 +3120,35 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed(
cancel_token: Option<CancellationToken>,
options: QueryExecutionOptions,
progress: Option<ExecuteMultiProgressCallback>,
) -> Result<Vec<ExecuteMultiResult>, QueryExecutionError> {
let db_type = connection_database_type(state, connection_id).await;
let invalidate = crate::object_cache::sql_may_change_object_metadata(sql, db_type);
let result = execute_multi_core_with_options_for_client_and_progress_typed_inner(
state,
connection_id,
database,
sql,
schema,
cancel_token,
options,
progress,
)
.await;
if invalidate {
crate::object_cache::invalidate_connection_object_cache(&state.storage, connection_id).await;
}
result
}
async fn execute_multi_core_with_options_for_client_and_progress_typed_inner(
state: &AppState,
connection_id: &str,
database: &str,
sql: &str,
schema: Option<&str>,
cancel_token: Option<CancellationToken>,
options: QueryExecutionOptions,
progress: Option<ExecuteMultiProgressCallback>,
) -> Result<Vec<ExecuteMultiResult>, QueryExecutionError> {
let pool_database = query_pool_database(database, options.catalog.as_deref());
// Reject MongoDB queries that fall through to the generic executor.
@@ -3931,6 +3987,23 @@ pub async fn execute_statements(
statements: &[String],
schema: Option<&str>,
timeout_secs: Option<u64>,
) -> Result<db::QueryResult, String> {
let db_type = connection_database_type(state, connection_id).await;
let invalidate = statements.iter().any(|sql| crate::object_cache::sql_may_change_object_metadata(sql, db_type));
let result = execute_statements_inner(state, connection_id, database, statements, schema, timeout_secs).await;
if invalidate {
crate::object_cache::invalidate_connection_object_cache(&state.storage, connection_id).await;
}
result
}
async fn execute_statements_inner(
state: &AppState,
connection_id: &str,
database: &str,
statements: &[String],
schema: Option<&str>,
timeout_secs: Option<u64>,
) -> Result<db::QueryResult, String> {
let sql_ctx = statements.first().map(|s| s.as_str()).unwrap_or("");
let pool_key = if database.is_empty() {
@@ -7486,6 +7559,136 @@ for line in sys.stdin:
}
}
#[tokio::test]
async fn ddl_schema_cache_invalidates_persisted_object_snapshots() {
let dir = tempfile::tempdir().unwrap();
let storage = Storage::open(&dir.path().join("storage.db")).await.unwrap();
let state = AppState::new(storage);
let connection_id = "schema-cache";
let sqlite =
db::sqlite::connect_path_create_if_missing(dir.path().join("query.db").to_str().unwrap()).await.unwrap();
state
.update_connection_pools(|pools| {
pools.insert(connection_id.to_string(), PoolKind::Sqlite(sqlite));
})
.await;
state.configs.write().await.insert(connection_id.to_string(), test_connection_config(DatabaseType::Sqlite));
let keys = [
"object-meta:v1:schema-cache:db:public:users::TABLE:columns:",
"object-meta:v1:schema-cache:db:public:users::backend-columns:",
"object-ddl:v1:schema-cache:db:public:users::TABLE:",
];
for key in keys {
state.storage.save_schema_cache(key, &serde_json::json!({"old": true})).await.unwrap();
}
let unrelated = "object-meta:v1:schema-cache-other:db:public:users::backend-columns:";
state.storage.save_schema_cache(unrelated, &serde_json::json!([])).await.unwrap();
execute_sql_statement(&state, connection_id, "", "SELECT 1", None, None).await.unwrap();
for key in keys {
assert!(state.storage.load_schema_cache(key).await.unwrap().is_some());
}
execute_sql_statement(&state, connection_id, "", "CREATE TABLE users (id INTEGER)", None, None).await.unwrap();
for key in keys {
assert!(state.storage.load_schema_cache(key).await.unwrap().is_none(), "stale cache: {key}");
}
assert!(state.storage.load_schema_cache(unrelated).await.unwrap().is_some());
}
#[tokio::test]
async fn ddl_schema_cache_batch_outcomes_and_transaction_completion() {
for (sql, use_transaction, expected_error, expected_column) in [
("ALTER TABLE users ADD COLUMN added INTEGER; SELECT 1", false, false, true),
("ALTER TABLE users ADD COLUMN added INTEGER; SELECT * FROM missing_table", false, true, true),
("ALTER TABLE users ADD COLUMN added INTEGER; SELECT 1", true, false, true),
("ALTER TABLE users ADD COLUMN added INTEGER; SELECT * FROM missing_table", true, true, false),
("BEGIN; ALTER TABLE users ADD COLUMN added INTEGER; COMMIT", false, false, true),
] {
let dir = tempfile::tempdir().unwrap();
let storage = Storage::open(&dir.path().join("storage.db")).await.unwrap();
let state = AppState::new(storage);
let sqlite = db::sqlite::connect_path_create_if_missing(dir.path().join("query.db").to_str().unwrap())
.await
.unwrap();
state
.update_connection_pools(|pools| {
pools.insert("cache".into(), PoolKind::Sqlite(sqlite));
})
.await;
state.configs.write().await.insert("cache".into(), test_connection_config(DatabaseType::Sqlite));
execute_sql_statement(&state, "cache", "", "CREATE TABLE users (id INTEGER)", None, None).await.unwrap();
let key = "object-meta:v1:cache:db:public:users::backend-columns:";
state.storage.save_schema_cache(key, &serde_json::json!([])).await.unwrap();
let result = execute_multi_core_with_options(
&state,
"cache",
"",
sql,
None,
None,
QueryExecutionOptions { use_transaction: Some(use_transaction), ..Default::default() },
)
.await;
let failed = match &result {
Err(_) => true,
Ok(results) => results.iter().any(|r| r.columns == vec!["Error"]),
};
assert_eq!(failed, expected_error, "{sql}");
assert!(state.storage.load_schema_cache(key).await.unwrap().is_none(), "{sql}");
let columns =
execute_sql_statement(&state, "cache", "", "PRAGMA table_info(users)", None, None).await.unwrap();
assert_eq!(columns.rows.iter().any(|row| row[1] == "added"), expected_column, "{sql}");
// A failed DDL still keeps its execution error while clearing cache.
state.storage.save_schema_cache(key, &serde_json::json!([])).await.unwrap();
let error = execute_sql_statement(
&state,
"cache",
"",
"ALTER TABLE missing_table ADD COLUMN x INTEGER",
None,
None,
)
.await
.unwrap_err();
assert!(error.contains("missing_table"));
assert!(state.storage.load_schema_cache(key).await.unwrap().is_none());
for sql in ["BEGIN", "COMMIT", "BEGIN", "ROLLBACK"] {
state.storage.save_schema_cache(key, &serde_json::json!([])).await.unwrap();
execute_sql_statement(&state, "cache", "", sql, None, None).await.unwrap();
assert_eq!(state.storage.load_schema_cache(key).await.unwrap().is_some(), sql == "BEGIN");
}
}
}
#[tokio::test]
async fn ddl_schema_cache_storage_failure_preserves_sql_outcome() {
let dir = tempfile::tempdir().unwrap();
let storage_path = dir.path().join("storage.db");
let state = AppState::new(Storage::open(&storage_path).await.unwrap());
let sqlite =
db::sqlite::connect_path_create_if_missing(dir.path().join("query.db").to_str().unwrap()).await.unwrap();
state
.update_connection_pools(|pools| {
pools.insert("cache".into(), PoolKind::Sqlite(sqlite));
})
.await;
state.configs.write().await.insert("cache".into(), test_connection_config(DatabaseType::Sqlite));
let key = "object-meta:v1:cache:db:public:users::backend-columns:";
state.storage.save_schema_cache(key, &serde_json::json!([])).await.unwrap();
// Fail only deletion; SQL execution and cache reads remain available.
let fault = rusqlite::Connection::open(&storage_path).unwrap();
fault.execute_batch("CREATE TRIGGER reject_cache_delete BEFORE DELETE ON schema_cache BEGIN SELECT RAISE(FAIL, 'injected cache delete failure'); END;").unwrap();
execute_sql_statement(&state, "cache", "", "CREATE TABLE users (id INTEGER)", None, None).await.unwrap();
assert!(state.storage.load_schema_cache(key).await.unwrap().is_some());
let result = execute_sql_statement(&state, "cache", "", "SELECT id FROM users", None, None).await.unwrap();
assert_eq!(result.columns, vec!["id"]);
let error =
execute_sql_statement(&state, "cache", "", "ALTER TABLE missing_table ADD COLUMN x INTEGER", None, None)
.await
.unwrap_err();
assert!(error.contains("missing_table"));
assert!(!error.contains("injected cache delete failure"));
}
async fn assert_sqlite_batch_error_behavior(failure_first: bool, continue_on_error: bool) {
let dir = std::env::temp_dir().join(format!("dbx-query-batch-error-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();