fix(query): invalidate persisted object metadata after DDL
This commit is contained in:
1 parent
c6ecd9808f
commit
5918414127
9 files changed
+689
-31
No files matched your search
@@ -68,6 +68,7 @@ pub mod mysql_event_sql;
|
||||
pub mod nacos;
|
||||
#[cfg(all(target_os = "windows", target_env = "gnu"))]
|
||||
mod nanosleep_stub;
|
||||
pub mod object_cache;
|
||||
pub mod object_source_sql;
|
||||
pub mod path_utils;
|
||||
pub mod plugins;
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
//! Persisted object metadata shared by Web and SQL execution entry points.
|
||||
//!
|
||||
//! These keys are also produced by the frontend's encodeURIComponent-based
|
||||
//! caches. Keep the encoding and namespace stable when invalidating snapshots.
|
||||
|
||||
use crate::models::connection::DatabaseType;
|
||||
use crate::storage::Storage;
|
||||
|
||||
pub const OBJECT_METADATA_CACHE_PREFIX: &str = "object-meta:v1";
|
||||
const OBJECT_DDL_CACHE_PREFIX: &str = "object-ddl:v1";
|
||||
|
||||
pub fn metadata_cache_segment(value: &str) -> String {
|
||||
const HEX: &[u8; 16] = b"0123456789ABCDEF";
|
||||
let mut encoded = String::with_capacity(value.len());
|
||||
for byte in value.bytes() {
|
||||
match byte {
|
||||
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'!' | b'~' | b'*' | b'\'' | b'(' | b')' => {
|
||||
encoded.push(byte as char)
|
||||
}
|
||||
_ => {
|
||||
encoded.push('%');
|
||||
encoded.push(HEX[(byte >> 4) as usize] as char);
|
||||
encoded.push(HEX[(byte & 0x0f) as usize] as char);
|
||||
}
|
||||
}
|
||||
}
|
||||
encoded
|
||||
}
|
||||
|
||||
pub fn object_metadata_cache_prefix(connection_id: &str, database: &str) -> String {
|
||||
format!(
|
||||
"{}:{}:{}:",
|
||||
OBJECT_METADATA_CACHE_PREFIX,
|
||||
metadata_cache_segment(connection_id),
|
||||
metadata_cache_segment(database)
|
||||
)
|
||||
}
|
||||
|
||||
/// Conservative cache policy, separate from SQL authorization. A failed DDL
|
||||
/// attempt may already have committed, so callers apply this even on errors.
|
||||
pub(crate) fn sql_may_change_object_metadata(sql: &str, db_type: Option<DatabaseType>) -> bool {
|
||||
let statements = match db_type {
|
||||
Some(db_type) => crate::sql::split_sql_statements_for_database(sql, db_type),
|
||||
None => crate::sql::split_sql_statements(sql),
|
||||
};
|
||||
statements.iter().any(|statement| {
|
||||
let risk = match db_type {
|
||||
Some(db_type) => crate::sql_risk::classify_sql_risk_for_database(statement, db_type),
|
||||
None => crate::sql_risk::classify_sql_risk(statement, "generic"),
|
||||
};
|
||||
if matches!(risk, Ok(crate::sql_risk::SqlRisk::Ddl)) {
|
||||
return true;
|
||||
}
|
||||
// Include transaction completion: a previous DDL call may have run
|
||||
// before commit, while another connection cached the old structure.
|
||||
let keywords = [
|
||||
"CREATE", "ALTER", "DROP", "TRUNCATE", "COMMENT", "GRANT", "REVOKE", "RENAME", "COMMIT", "END", "ROLLBACK",
|
||||
"ABORT",
|
||||
];
|
||||
match db_type {
|
||||
Some(db_type) => crate::sql::starts_with_executable_sql_keyword_for_database(statement, &keywords, db_type),
|
||||
None => crate::sql::starts_with_executable_sql_keyword(statement, &keywords),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Connection scope also covers qualified cross-database DDL and session USE.
|
||||
/// Await before returning the SQL result; cache failures must not turn an
|
||||
/// already committed database operation into an apparent execution failure.
|
||||
pub(crate) async fn invalidate_connection_object_cache(storage: &Storage, connection_id: &str) {
|
||||
let connection = metadata_cache_segment(connection_id);
|
||||
for namespace in [OBJECT_METADATA_CACHE_PREFIX, OBJECT_DDL_CACHE_PREFIX] {
|
||||
let prefix = format!("{namespace}:{connection}:");
|
||||
if let Err(error) = storage.delete_schema_cache_prefix(&prefix).await {
|
||||
log::warn!(
|
||||
"failed to invalidate object cache: connection_id={connection_id} namespace={namespace} error={error}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn ddl_schema_cache_classifies_scripts_without_changing_permissions() {
|
||||
for sql in [
|
||||
"ALTER TABLE users ADD COLUMN test4 INTEGER",
|
||||
"BEGIN; ALTER TABLE users ADD COLUMN test4 INTEGER; COMMIT;",
|
||||
"-- leading comment\nCOMMENT ON COLUMN users.test4 IS 'new'",
|
||||
"CREATE SOMETHING dialect_specific",
|
||||
"COMMIT",
|
||||
"END",
|
||||
"ROLLBACK",
|
||||
"ABORT",
|
||||
] {
|
||||
assert!(sql_may_change_object_metadata(sql, Some(DatabaseType::Postgres)), "{sql}");
|
||||
}
|
||||
for sql in [
|
||||
"SELECT 'ALTER TABLE users'",
|
||||
"-- DROP TABLE users\nSELECT 1",
|
||||
"INSERT INTO users VALUES (1)",
|
||||
"UPDATE users SET id = 1",
|
||||
"DELETE FROM users",
|
||||
"BEGIN",
|
||||
"",
|
||||
"/* ALTER TABLE */",
|
||||
] {
|
||||
assert!(!sql_may_change_object_metadata(sql, Some(DatabaseType::Postgres)), "{sql}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ddl_schema_cache_encoding_and_connection_isolation() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let storage = Storage::open(&dir.path().join("storage.db")).await.unwrap();
|
||||
let id = "conn:% 中文";
|
||||
assert_eq!(metadata_cache_segment(id), "conn%3A%25%20%E4%B8%AD%E6%96%87");
|
||||
let prefix = object_metadata_cache_prefix(id, "db% name");
|
||||
assert_eq!(prefix, "object-meta:v1:conn%3A%25%20%E4%B8%AD%E6%96%87:db%25%20name:");
|
||||
let removed = [
|
||||
format!("{prefix}public:users::backend-columns:"),
|
||||
format!("object-ddl:v1:{}:other-db:public:users::TABLE:", metadata_cache_segment(id)),
|
||||
];
|
||||
let kept = format!("{}columns:", object_metadata_cache_prefix(&format!("{id}2"), "db% name"));
|
||||
for key in removed.iter().chain(std::iter::once(&kept)) {
|
||||
storage.save_schema_cache(key, &serde_json::json!([])).await.unwrap();
|
||||
}
|
||||
invalidate_connection_object_cache(&storage, id).await;
|
||||
for key in removed {
|
||||
assert!(storage.load_schema_cache(&key).await.unwrap().is_none());
|
||||
}
|
||||
assert!(storage.load_schema_cache(&kept).await.unwrap().is_some());
|
||||
}
|
||||
}
|
||||
@@ -2774,6 +2774,33 @@ pub async fn execute_sql_statement_with_options_typed(
|
||||
schema: Option<&str>,
|
||||
cancel_token: Option<CancellationToken>,
|
||||
options: QueryExecutionOptions,
|
||||
) -> Result<db::QueryResult, QueryExecutionError> {
|
||||
let db_type = connection_database_type(state, connection_id).await;
|
||||
let invalidate = crate::object_cache::sql_may_change_object_metadata(sql, db_type);
|
||||
let result = execute_sql_statement_with_options_typed_inner(
|
||||
state,
|
||||
connection_id,
|
||||
database,
|
||||
sql,
|
||||
schema,
|
||||
cancel_token,
|
||||
options,
|
||||
)
|
||||
.await;
|
||||
if invalidate {
|
||||
crate::object_cache::invalidate_connection_object_cache(&state.storage, connection_id).await;
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
async fn execute_sql_statement_with_options_typed_inner(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
database: &str,
|
||||
sql: &str,
|
||||
schema: Option<&str>,
|
||||
cancel_token: Option<CancellationToken>,
|
||||
options: QueryExecutionOptions,
|
||||
) -> Result<db::QueryResult, QueryExecutionError> {
|
||||
// MongoDB connections use shell-style commands dispatched through the
|
||||
// frontend parser. Queries that fall through to the generic SQL executor
|
||||
@@ -3093,6 +3120,35 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed(
|
||||
cancel_token: Option<CancellationToken>,
|
||||
options: QueryExecutionOptions,
|
||||
progress: Option<ExecuteMultiProgressCallback>,
|
||||
) -> Result<Vec<ExecuteMultiResult>, QueryExecutionError> {
|
||||
let db_type = connection_database_type(state, connection_id).await;
|
||||
let invalidate = crate::object_cache::sql_may_change_object_metadata(sql, db_type);
|
||||
let result = execute_multi_core_with_options_for_client_and_progress_typed_inner(
|
||||
state,
|
||||
connection_id,
|
||||
database,
|
||||
sql,
|
||||
schema,
|
||||
cancel_token,
|
||||
options,
|
||||
progress,
|
||||
)
|
||||
.await;
|
||||
if invalidate {
|
||||
crate::object_cache::invalidate_connection_object_cache(&state.storage, connection_id).await;
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
async fn execute_multi_core_with_options_for_client_and_progress_typed_inner(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
database: &str,
|
||||
sql: &str,
|
||||
schema: Option<&str>,
|
||||
cancel_token: Option<CancellationToken>,
|
||||
options: QueryExecutionOptions,
|
||||
progress: Option<ExecuteMultiProgressCallback>,
|
||||
) -> Result<Vec<ExecuteMultiResult>, QueryExecutionError> {
|
||||
let pool_database = query_pool_database(database, options.catalog.as_deref());
|
||||
// Reject MongoDB queries that fall through to the generic executor.
|
||||
@@ -3931,6 +3987,23 @@ pub async fn execute_statements(
|
||||
statements: &[String],
|
||||
schema: Option<&str>,
|
||||
timeout_secs: Option<u64>,
|
||||
) -> Result<db::QueryResult, String> {
|
||||
let db_type = connection_database_type(state, connection_id).await;
|
||||
let invalidate = statements.iter().any(|sql| crate::object_cache::sql_may_change_object_metadata(sql, db_type));
|
||||
let result = execute_statements_inner(state, connection_id, database, statements, schema, timeout_secs).await;
|
||||
if invalidate {
|
||||
crate::object_cache::invalidate_connection_object_cache(&state.storage, connection_id).await;
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
async fn execute_statements_inner(
|
||||
state: &AppState,
|
||||
connection_id: &str,
|
||||
database: &str,
|
||||
statements: &[String],
|
||||
schema: Option<&str>,
|
||||
timeout_secs: Option<u64>,
|
||||
) -> Result<db::QueryResult, String> {
|
||||
let sql_ctx = statements.first().map(|s| s.as_str()).unwrap_or("");
|
||||
let pool_key = if database.is_empty() {
|
||||
@@ -7486,6 +7559,136 @@ for line in sys.stdin:
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ddl_schema_cache_invalidates_persisted_object_snapshots() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let storage = Storage::open(&dir.path().join("storage.db")).await.unwrap();
|
||||
let state = AppState::new(storage);
|
||||
let connection_id = "schema-cache";
|
||||
let sqlite =
|
||||
db::sqlite::connect_path_create_if_missing(dir.path().join("query.db").to_str().unwrap()).await.unwrap();
|
||||
state
|
||||
.update_connection_pools(|pools| {
|
||||
pools.insert(connection_id.to_string(), PoolKind::Sqlite(sqlite));
|
||||
})
|
||||
.await;
|
||||
state.configs.write().await.insert(connection_id.to_string(), test_connection_config(DatabaseType::Sqlite));
|
||||
let keys = [
|
||||
"object-meta:v1:schema-cache:db:public:users::TABLE:columns:",
|
||||
"object-meta:v1:schema-cache:db:public:users::backend-columns:",
|
||||
"object-ddl:v1:schema-cache:db:public:users::TABLE:",
|
||||
];
|
||||
for key in keys {
|
||||
state.storage.save_schema_cache(key, &serde_json::json!({"old": true})).await.unwrap();
|
||||
}
|
||||
let unrelated = "object-meta:v1:schema-cache-other:db:public:users::backend-columns:";
|
||||
state.storage.save_schema_cache(unrelated, &serde_json::json!([])).await.unwrap();
|
||||
execute_sql_statement(&state, connection_id, "", "SELECT 1", None, None).await.unwrap();
|
||||
for key in keys {
|
||||
assert!(state.storage.load_schema_cache(key).await.unwrap().is_some());
|
||||
}
|
||||
execute_sql_statement(&state, connection_id, "", "CREATE TABLE users (id INTEGER)", None, None).await.unwrap();
|
||||
for key in keys {
|
||||
assert!(state.storage.load_schema_cache(key).await.unwrap().is_none(), "stale cache: {key}");
|
||||
}
|
||||
assert!(state.storage.load_schema_cache(unrelated).await.unwrap().is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ddl_schema_cache_batch_outcomes_and_transaction_completion() {
|
||||
for (sql, use_transaction, expected_error, expected_column) in [
|
||||
("ALTER TABLE users ADD COLUMN added INTEGER; SELECT 1", false, false, true),
|
||||
("ALTER TABLE users ADD COLUMN added INTEGER; SELECT * FROM missing_table", false, true, true),
|
||||
("ALTER TABLE users ADD COLUMN added INTEGER; SELECT 1", true, false, true),
|
||||
("ALTER TABLE users ADD COLUMN added INTEGER; SELECT * FROM missing_table", true, true, false),
|
||||
("BEGIN; ALTER TABLE users ADD COLUMN added INTEGER; COMMIT", false, false, true),
|
||||
] {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let storage = Storage::open(&dir.path().join("storage.db")).await.unwrap();
|
||||
let state = AppState::new(storage);
|
||||
let sqlite = db::sqlite::connect_path_create_if_missing(dir.path().join("query.db").to_str().unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
state
|
||||
.update_connection_pools(|pools| {
|
||||
pools.insert("cache".into(), PoolKind::Sqlite(sqlite));
|
||||
})
|
||||
.await;
|
||||
state.configs.write().await.insert("cache".into(), test_connection_config(DatabaseType::Sqlite));
|
||||
execute_sql_statement(&state, "cache", "", "CREATE TABLE users (id INTEGER)", None, None).await.unwrap();
|
||||
let key = "object-meta:v1:cache:db:public:users::backend-columns:";
|
||||
state.storage.save_schema_cache(key, &serde_json::json!([])).await.unwrap();
|
||||
let result = execute_multi_core_with_options(
|
||||
&state,
|
||||
"cache",
|
||||
"",
|
||||
sql,
|
||||
None,
|
||||
None,
|
||||
QueryExecutionOptions { use_transaction: Some(use_transaction), ..Default::default() },
|
||||
)
|
||||
.await;
|
||||
let failed = match &result {
|
||||
Err(_) => true,
|
||||
Ok(results) => results.iter().any(|r| r.columns == vec!["Error"]),
|
||||
};
|
||||
assert_eq!(failed, expected_error, "{sql}");
|
||||
assert!(state.storage.load_schema_cache(key).await.unwrap().is_none(), "{sql}");
|
||||
let columns =
|
||||
execute_sql_statement(&state, "cache", "", "PRAGMA table_info(users)", None, None).await.unwrap();
|
||||
assert_eq!(columns.rows.iter().any(|row| row[1] == "added"), expected_column, "{sql}");
|
||||
// A failed DDL still keeps its execution error while clearing cache.
|
||||
state.storage.save_schema_cache(key, &serde_json::json!([])).await.unwrap();
|
||||
let error = execute_sql_statement(
|
||||
&state,
|
||||
"cache",
|
||||
"",
|
||||
"ALTER TABLE missing_table ADD COLUMN x INTEGER",
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(error.contains("missing_table"));
|
||||
assert!(state.storage.load_schema_cache(key).await.unwrap().is_none());
|
||||
for sql in ["BEGIN", "COMMIT", "BEGIN", "ROLLBACK"] {
|
||||
state.storage.save_schema_cache(key, &serde_json::json!([])).await.unwrap();
|
||||
execute_sql_statement(&state, "cache", "", sql, None, None).await.unwrap();
|
||||
assert_eq!(state.storage.load_schema_cache(key).await.unwrap().is_some(), sql == "BEGIN");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ddl_schema_cache_storage_failure_preserves_sql_outcome() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let storage_path = dir.path().join("storage.db");
|
||||
let state = AppState::new(Storage::open(&storage_path).await.unwrap());
|
||||
let sqlite =
|
||||
db::sqlite::connect_path_create_if_missing(dir.path().join("query.db").to_str().unwrap()).await.unwrap();
|
||||
state
|
||||
.update_connection_pools(|pools| {
|
||||
pools.insert("cache".into(), PoolKind::Sqlite(sqlite));
|
||||
})
|
||||
.await;
|
||||
state.configs.write().await.insert("cache".into(), test_connection_config(DatabaseType::Sqlite));
|
||||
let key = "object-meta:v1:cache:db:public:users::backend-columns:";
|
||||
state.storage.save_schema_cache(key, &serde_json::json!([])).await.unwrap();
|
||||
// Fail only deletion; SQL execution and cache reads remain available.
|
||||
let fault = rusqlite::Connection::open(&storage_path).unwrap();
|
||||
fault.execute_batch("CREATE TRIGGER reject_cache_delete BEFORE DELETE ON schema_cache BEGIN SELECT RAISE(FAIL, 'injected cache delete failure'); END;").unwrap();
|
||||
execute_sql_statement(&state, "cache", "", "CREATE TABLE users (id INTEGER)", None, None).await.unwrap();
|
||||
assert!(state.storage.load_schema_cache(key).await.unwrap().is_some());
|
||||
let result = execute_sql_statement(&state, "cache", "", "SELECT id FROM users", None, None).await.unwrap();
|
||||
assert_eq!(result.columns, vec!["id"]);
|
||||
let error =
|
||||
execute_sql_statement(&state, "cache", "", "ALTER TABLE missing_table ADD COLUMN x INTEGER", None, None)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(error.contains("missing_table"));
|
||||
assert!(!error.contains("injected cache delete failure"));
|
||||
}
|
||||
|
||||
async fn assert_sqlite_batch_error_behavior(failure_first: bool, continue_on_error: bool) {
|
||||
let dir = std::env::temp_dir().join(format!("dbx-query-batch-error-{}", uuid::Uuid::new_v4()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
|
||||
Reference in new issue
Block a user