fix(query): invalidate persisted object metadata after DDL

This commit is contained in:
soddygo authored and GitHub committed 2026-09-14 22:26:56 +08:00
1 parent c6ecd9808f
commit 5918414127
9 files changed
+689 -31

No files matched your search

+3
View File
@@ -43,3 +43,6 @@ pub mod tunnel_profiles;
pub mod update;
pub mod vector;
pub mod zookeeper;
#[cfg(test)]
mod object_cache_tests;
@@ -0,0 +1,214 @@
//! Regression coverage across real Web handlers and MCP backends.
use super::{connection, query, schema, schema_cache};
use crate::state::WebState;
use axum::{
routing::{get, post},
Router,
};
use dbx_core::{
agent_tools::AgentSqlPermissions,
connection::AppState,
models::connection::ConnectionConfig,
query::QueryExecutionOptions,
storage::{McpGlobalPolicy, Storage},
};
use dbx_mcp::{DbxBackend, LocalBackend, WebBackend};
use serde_json::{json, Value};
use std::sync::Arc;
async fn run_object_cache_regression(postgres: bool) {
let dir = std::env::temp_dir().join(format!("dbx-object-cache-{}", uuid::Uuid::new_v4()));
std::fs::create_dir_all(&dir).unwrap();
let storage = Storage::open(&dir.join("storage.db")).await.unwrap();
let database =
if postgres { std::env::var("DBX_LIVE_POSTGRES_DATABASE").expect("test database") } else { "main".into() };
let config: ConnectionConfig = serde_json::from_value(json!({
"id": "cache:中文%", "name": "cache regression", "db_type": if postgres { "postgres" } else { "sqlite" },
"host": if postgres { std::env::var("DBX_LIVE_POSTGRES_HOST").expect("test host") } else { dir.join("query.db").to_str().unwrap().to_string() },
"port": if postgres { std::env::var("DBX_LIVE_POSTGRES_PORT").expect("test port").parse::<u16>().unwrap() } else { 0 },
"username": if postgres { std::env::var("DBX_LIVE_POSTGRES_USER").expect("test user") } else { String::new() },
"password": if postgres { std::env::var("DBX_LIVE_POSTGRES_PASSWORD").expect("test password") } else { String::new() },
"database": database, "save_password": true
})).unwrap();
if !postgres {
dbx_core::db::sqlite::connect_path_create_if_missing(&config.host).await.unwrap();
}
storage.save_connections(std::slice::from_ref(&config)).await.unwrap();
storage
.save_mcp_global_policy(&McpGlobalPolicy { read_only: false, allow_dangerous_sql: true, ..Default::default() })
.await
.unwrap();
let app = Arc::new(AppState::new_with_plugin_dir(storage, dir.join("plugins")));
app.configs.write().await.insert(config.id.clone(), config.clone());
let mut state = WebState::for_tests(app.clone(), dir.clone());
state.password_disabled = true;
let state = Arc::new(state);
let router = Router::new()
.route("/api/auth/check", get(crate::auth::check))
.route("/api/connection/connect", post(connection::connect_db))
.route("/api/query/execute", post(query::execute_query))
.route("/api/query/execute-multi", post(query::execute_multi))
.route("/api/query/execute-batch", post(query::execute_batch))
.route("/api/schema/columns", get(schema::list_columns))
.route("/api/schema/ddl", get(schema::get_ddl))
.route("/api/schema/cache", get(schema_cache::load_schema_cache).post(schema_cache::save_schema_cache))
.with_state(state);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let base = format!("http://{}", listener.local_addr().unwrap());
let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() });
let local = LocalBackend::from_app_state(app.clone(), dir.clone());
let web = WebBackend::new(base.clone(), String::new()).unwrap();
let client = reqwest::Client::new();
let schema_name = if postgres { format!("dbx_cache_{}", uuid::Uuid::new_v4().simple()) } else { "main".into() };
if postgres {
local.execute_query(&config, &database, &format!("CREATE SCHEMA {schema_name}"), None, None).await.unwrap();
}
let table = format!("{schema_name}.users");
local.execute_query(&config, &database, &format!("CREATE TABLE {table} (id INTEGER)"), None, None).await.unwrap();
let prefix = dbx_core::object_cache::object_metadata_cache_prefix(&config.id, &database);
let schema_segment = dbx_core::object_cache::metadata_cache_segment(&schema_name);
let front_key = format!("{prefix}{schema_segment}:users::TABLE:columns:");
let ddl_key = format!(
"object-ddl:v1:{}:{}:{schema_segment}:users::TABLE:",
dbx_core::object_cache::metadata_cache_segment(&config.id),
dbx_core::object_cache::metadata_cache_segment(&database)
);
let backend_key = format!("{prefix}{schema_segment}:users::backend-columns:");
for mode in 0..5 {
let parameters = [
("connection_id", config.id.as_str()),
("database", database.as_str()),
("schema", schema_name.as_str()),
("table", "users"),
];
let old_columns: Value = client
.get(format!("{base}/api/schema/columns"))
.query(&parameters)
.send()
.await
.unwrap()
.error_for_status()
.unwrap()
.json()
.await
.unwrap();
let old_ddl: Value = client
.get(format!("{base}/api/schema/ddl"))
.query(&parameters)
.send()
.await
.unwrap()
.error_for_status()
.unwrap()
.json()
.await
.unwrap();
for (key, payload) in [
(&front_key, json!({"version": 1, "cachedAt": chrono::Utc::now().to_rfc3339(), "value": old_columns})),
(&ddl_key, json!({"version": 1, "cachedAt": chrono::Utc::now().to_rfc3339(), "ddl": old_ddl})),
] {
client
.post(format!("{base}/api/schema/cache"))
.json(&json!({"cacheKey": key, "payload": payload}))
.send()
.await
.unwrap()
.error_for_status()
.unwrap();
}
assert!(app.storage.load_schema_cache(&backend_key).await.unwrap().is_some());
let column = format!("added_{mode}");
let sql = format!("ALTER TABLE {table} ADD COLUMN {column} INTEGER");
if mode == 0 || mode == 2 {
let backend: &dyn DbxBackend = if mode == 0 { &local } else { &web };
let result = backend
.execute_agent_tool(
&config,
&database,
"execute_query",
json!({"sql": sql}),
AgentSqlPermissions { allow_writes: true, allow_dangerous: true, confirmed_write_sql: None },
)
.await;
assert!(!result.is_error, "{}", result.content);
} else if mode == 1 || mode == 3 {
let backend: &dyn DbxBackend = if mode == 1 { &local } else { &web };
let results = backend
.execute_batch(
&config,
&database,
Some(&schema_name),
&format!("{sql}; SELECT 1"),
QueryExecutionOptions::default(),
)
.await
.unwrap();
assert_eq!(results.len(), 2);
assert!(results.iter().all(|result| !result.execution_error), "{results:?}");
} else {
client.post(format!("{base}/api/query/execute-batch")).json(&json!({"connectionId": config.id, "database": database, "statements": [sql], "schema": schema_name})).send().await.unwrap().error_for_status().unwrap();
}
for key in [&front_key, &ddl_key, &backend_key] {
assert!(app.storage.load_schema_cache(key).await.unwrap().is_none(), "mode {mode}: {key}");
}
// A fresh frontend read must miss persisted snapshots, then fetch the
// actual updated columns and DDL through the real Web handlers.
let cached: Value = client
.get(format!("{base}/api/schema/cache"))
.query(&[("cache_key", &front_key)])
.send()
.await
.unwrap()
.json()
.await
.unwrap();
assert!(cached.is_null());
let columns: Value = client
.get(format!("{base}/api/schema/columns"))
.query(&parameters)
.send()
.await
.unwrap()
.error_for_status()
.unwrap()
.json()
.await
.unwrap();
assert!(columns.as_array().unwrap().iter().any(|c| c["name"] == column), "{columns}");
let ddl: Value = client
.get(format!("{base}/api/schema/ddl"))
.query(&parameters)
.send()
.await
.unwrap()
.error_for_status()
.unwrap()
.json()
.await
.unwrap();
assert!(ddl.to_string().contains(&column), "{ddl}");
}
if postgres {
local
.execute_query(&config, &database, &format!("DROP SCHEMA {schema_name} CASCADE"), None, None)
.await
.unwrap();
}
server.abort();
let _ = server.await;
drop(web);
drop(local);
drop(app);
std::fs::remove_dir_all(dir).unwrap();
}
#[tokio::test]
async fn ddl_schema_cache_real_web_handlers_and_mcp_backends() {
run_object_cache_regression(false).await;
}
#[tokio::test]
#[ignore = "requires isolated writable DBX_LIVE_POSTGRES_HOST/PORT/USER/PASSWORD/DATABASE"]
async fn ddl_schema_cache_live_postgres_web_handlers_and_mcp_backends() {
run_object_cache_regression(true).await;
}
+2 -28
View File
@@ -402,25 +402,8 @@ pub async fn get_custom_type_details(
Ok(Json(result))
}
const OBJECT_METADATA_CACHE_PREFIX: &str = "object-meta:v1";
fn metadata_cache_segment(value: &str) -> String {
const HEX: &[u8; 16] = b"0123456789ABCDEF";
let mut encoded = String::with_capacity(value.len());
for byte in value.bytes() {
match byte {
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'!' | b'~' | b'*' | b'\'' | b'(' | b')' => {
encoded.push(byte as char)
}
_ => {
encoded.push('%');
encoded.push(HEX[(byte >> 4) as usize] as char);
encoded.push(HEX[(byte & 0x0f) as usize] as char);
}
}
}
encoded
}
pub(crate) use dbx_core::object_cache::object_metadata_cache_prefix;
use dbx_core::object_cache::{metadata_cache_segment, OBJECT_METADATA_CACHE_PREFIX};
fn metadata_cache_key(
connection_id: &str,
@@ -443,15 +426,6 @@ fn metadata_cache_key(
.join(":")
}
pub(crate) fn object_metadata_cache_prefix(connection_id: &str, database: &str) -> String {
format!(
"{}:{}:{}:",
OBJECT_METADATA_CACHE_PREFIX,
metadata_cache_segment(connection_id),
metadata_cache_segment(database)
)
}
fn decode_metadata_cache<T: DeserializeOwned>(value: serde_json::Value) -> Option<T> {
serde_json::from_value(value).ok()
}