feat(connection): notify sidebar when keepalive loses connection

This commit is contained in:
Abeautifulsnow authored and GitHub committed 2026-09-30 19:49:52 +08:00
1 parent 3ffff92c8f
commit dc503877f9
13 files changed
+864 -27

No files matched your search

+402 -23
View File
@@ -33,7 +33,8 @@ use crate::db::proxy_tunnel::ProxyTunnelManager;
use crate::db::ssh_tunnel::TunnelManager;
use crate::models::connection::{
database_info_from_protocol_value, parse_jdbc_host_port, parse_mongo_first_host, rewrite_jdbc_url_host,
ConnectionConfig, ConnectionTestResult, DatabaseConnectionInfo, DatabaseType, TransportLayerConfig,
ConnectionConfig, ConnectionLivenessFailureKind, ConnectionLivenessMessage, ConnectionTestResult,
DatabaseConnectionInfo, DatabaseType, TransportLayerConfig,
};
use crate::mongo_oidc::MongoOidcBrowserOpener;
use crate::nacos::config::{NACOS_CONSOLE_SESSION_PASSWORD, NACOS_PRIMARY_SESSION_PASSWORD};
@@ -428,6 +429,10 @@ pub struct AppState {
/// In-memory, never-persisted 1/5 minute write overrides for read-only connections.
pub write_unlock_windows: crate::write_unlock::WriteUnlockWindows,
metadata_gates: Arc<Mutex<HashMap<String, Arc<Semaphore>>>>,
/// Backend-confirmed liveness losses plus resync requests (#4339). The keepalive task
/// publishes here once a connection has no pools left; each shell relays it to its own
/// frontend, so DBX core never needs a UI handle of its own.
connection_liveness: tokio::sync::broadcast::Sender<ConnectionLivenessMessage>,
mongo_oidc_browser_opener: std::sync::RwLock<Option<MongoOidcBrowserOpener>>,
salesforce_browser_opener: std::sync::RwLock<Option<SfBrowserOpener>>,
#[cfg(feature = "mq-admin")]
@@ -1429,6 +1434,15 @@ impl AppState {
self.connections.read().await.keys().any(|key| pool_key_belongs_to_connection(key, connection_id))
}
/// Subscribe to backend-confirmed connection liveness messages (#4339).
///
/// Each shell relays these to its own frontend. A fresh receiver only sees messages
/// published after it subscribed, so shells subscribe once at startup and keep the
/// receiver alive rather than re-subscribing per request.
pub fn subscribe_connection_liveness(&self) -> tokio::sync::broadcast::Receiver<ConnectionLivenessMessage> {
self.connection_liveness.subscribe()
}
/// Find an already-registered metadata/workload pool for a metadata read.
/// Unlike `get_or_create_metadata_pool_for_session`, this is a pure lookup:
/// it never validates credentials, starts an agent, or opens a transport.
@@ -1623,6 +1637,10 @@ impl AppState {
session_credentials: SessionCredentialStore::new(),
write_unlock_windows: crate::write_unlock::WriteUnlockWindows::default(),
metadata_gates: Arc::new(Mutex::new(HashMap::new())),
// Bounded so a shell that stops draining cannot grow memory without bound. A
// lagged receiver re-syncs from the read-only status check on its next use, so
// dropping events here is safe.
connection_liveness: tokio::sync::broadcast::Sender::new(64),
mongo_oidc_browser_opener: std::sync::RwLock::new(None),
salesforce_browser_opener: std::sync::RwLock::new(None),
#[cfg(feature = "mq-admin")]
@@ -2167,6 +2185,10 @@ impl AppState {
let routing = self.pool_routing_control();
let connections = self.connections.clone();
let running_queries = self.running_queries.clone();
let liveness = self.connection_liveness.clone();
// Liveness is keyed by connection, not by pool: pool keys are per session/role/
// database derived, so only the connection id identifies "this connection is gone".
let connection_id = config.id.clone();
// MQ pool markers are empty; close_pool_kind is a no-op, so keepalive must
// drop the registry adapter or reconnect would reuse a dead agent.
#[cfg(feature = "mq-admin")]
@@ -2209,22 +2231,25 @@ impl AppState {
}
#[cfg(feature = "mq-admin")]
let drop_mq = matches!(target, KeepaliveTarget::MessageQueue(_));
if !detach_keepalive_target_if_current(
// Only a pool we actually detached may report liveness loss; a
// probe failure on an already-replaced pool must not.
let detached = detach_and_report_keepalive_loss(
&routing,
&connections,
&liveness,
&key,
target,
&connection_id,
ConnectionLivenessFailureKind::ProbeFailed,
replace_runtime,
)
.await
{
log::debug!("Skipping stale keepalive result for replaced pool '{key}'");
} else {
#[cfg(feature = "mq-admin")]
if drop_mq {
mq_registry.drop_connection(&mq_connection_id).await;
}
.await;
#[cfg(feature = "mq-admin")]
if detached && drop_mq {
mq_registry.drop_connection(&mq_connection_id).await;
}
#[cfg(not(feature = "mq-admin"))]
let _ = detached;
break;
}
Err(_) => {
@@ -2241,14 +2266,26 @@ impl AppState {
}
#[cfg(feature = "mq-admin")]
let drop_mq = matches!(target, KeepaliveTarget::MessageQueue(_));
if !detach_keepalive_target_if_current(&routing, &connections, &key, target, false).await {
log::debug!("Skipping stale keepalive timeout for replaced pool '{key}'");
} else {
#[cfg(feature = "mq-admin")]
if drop_mq {
mq_registry.drop_connection(&mq_connection_id).await;
}
// The pool is torn down either way, so a timed-out probe is reported
// too: staying silent would keep the sidebar claiming the connection
// is usable.
let detached = detach_and_report_keepalive_loss(
&routing,
&connections,
&liveness,
&key,
target,
&connection_id,
ConnectionLivenessFailureKind::TimedOut,
false,
)
.await;
#[cfg(feature = "mq-admin")]
if detached && drop_mq {
mq_registry.drop_connection(&mq_connection_id).await;
}
#[cfg(not(feature = "mq-admin"))]
let _ = detached;
break;
}
}
@@ -5965,6 +6002,50 @@ fn keepalive_failure_proves_pool_dead(error: &str) -> bool {
!crate::query::is_pool_saturation_error(error)
}
/// Tells the shells that `connection_id` has no pools left, so the frontend can stop
/// claiming the connection is connected (#4339).
///
/// Pool keys are per session/role/database derived, so one dying derived pool says nothing
/// about the connection root: only the last pool to go makes the connection really unusable.
/// Gating on "no pools left" is also what collapses N per-pool keepalive failures into one
/// logical disconnect. Two pools of the same connection can both finish detaching before
/// either evaluates this, publishing twice; the frontend path is idempotent.
async fn report_connection_liveness_lost(
connections: &Arc<RwLock<ConnectionPoolRegistry>>,
liveness: &tokio::sync::broadcast::Sender<ConnectionLivenessMessage>,
connection_id: &str,
failure_kind: ConnectionLivenessFailureKind,
) {
if connections.read().await.keys().any(|key| pool_key_belongs_to_connection(key, connection_id)) {
return;
}
// A missing subscriber (headless shell) is not an error worth reporting.
let _ = liveness.send(ConnectionLivenessMessage::Lost { connection_id: connection_id.to_string(), failure_kind });
}
/// Detaches a probe's pool when it is still the current one and reports the loss if the
/// connection has nothing left (#4339).
///
/// Returns whether this probe's pool was still current and was detached. A stale probe (its
/// pool was already replaced) reports nothing: its result says nothing about the connection.
async fn detach_and_report_keepalive_loss(
routing: &PoolRoutingControl,
connections: &Arc<RwLock<ConnectionPoolRegistry>>,
liveness: &tokio::sync::broadcast::Sender<ConnectionLivenessMessage>,
pool_key: &str,
target: &KeepaliveTarget,
connection_id: &str,
failure_kind: ConnectionLivenessFailureKind,
replace_agent_runtime: bool,
) -> bool {
if !detach_keepalive_target_if_current(routing, connections, pool_key, target, replace_agent_runtime).await {
log::debug!("Skipping stale keepalive {failure_kind:?} result for replaced pool '{pool_key}'");
return false;
}
report_connection_liveness_lost(connections, liveness, connection_id, failure_kind).await;
true
}
impl KeepaliveTarget {
fn matches_pool(&self, pool: &PoolKind) -> bool {
match (self, pool) {
@@ -6945,11 +7026,11 @@ mod tests {
kafka_single_loopback_bootstrap_endpoint, keepalive_failure_proves_pool_dead, metadata_connection_config,
metadata_pool_database, mysql_metadata_fallback_url, mysql_pool_setup_queries, oceanbase_mysql_setup_queries,
prestosql_jdbc_config_for_endpoint, redacted_connection_url_for_endpoint, redis_sentinel_transport_id,
redis_sentinel_transport_prefix, sqlserver_legacy_agent_config, sqlserver_legacy_driver_error,
sqlserver_uses_legacy_driver, task_client_session_id, transport_layers_through_last_ssh,
upsert_connection_url_param, uses_bare_mysql_pool, uses_tcp_probe, validate_connection_url_params,
validate_h2_database_path, AppState, MysqlMode, PoolKind, TxnConnection, GAUSSDB_M_JDBC_DRIVER_CLASS,
GAUSSDB_M_JDBC_DRIVER_PROFILE, PRESTOSQL_JDBC_DRIVER_CLASS,
redis_sentinel_transport_prefix, report_connection_liveness_lost, sqlserver_legacy_agent_config,
sqlserver_legacy_driver_error, sqlserver_uses_legacy_driver, task_client_session_id,
transport_layers_through_last_ssh, upsert_connection_url_param, uses_bare_mysql_pool, uses_tcp_probe,
validate_connection_url_params, validate_h2_database_path, AppState, MysqlMode, PoolKind, TxnConnection,
GAUSSDB_M_JDBC_DRIVER_CLASS, GAUSSDB_M_JDBC_DRIVER_PROFILE, PRESTOSQL_JDBC_DRIVER_CLASS,
};
use crate::agent_connection::{
agent_connect_params, mongo_legacy_error_with_auth_hint, mongo_uses_legacy_driver,
@@ -6960,7 +7041,8 @@ mod tests {
use crate::db;
use crate::models::connection::{
default_connect_timeout_secs, default_redis_key_separator, AttachedDatabaseConfig, ConnectionConfig,
DatabaseType, HttpTunnelConfig, ProxyTunnelConfig, ProxyType, SshTunnelConfig, TransportLayerConfig,
ConnectionLivenessFailureKind, ConnectionLivenessMessage, DatabaseType, HttpTunnelConfig, ProxyTunnelConfig,
ProxyType, SshTunnelConfig, TransportLayerConfig,
};
use crate::query;
use crate::schema;
@@ -9432,6 +9514,303 @@ sleep 30
let _ = std::fs::remove_dir_all(dir);
}
/// Minimal `ConnectionConfig` for driving `start_keepalive_task` directly.
///
/// The keepalive path only reads `id`, the effective timeouts and
/// `keepalive_interval_secs`; the rest is filler so the struct can be built without a
/// live database. `connect_timeout_secs` is 1 so a failing probe resolves fast.
fn keepalive_test_config(id: &str, interval_secs: u64) -> ConnectionConfig {
ConnectionConfig {
docs_notes_path: None,
id: id.to_string(),
name: id.to_string(),
note: String::new(),
db_type: DatabaseType::Oracle,
driver_profile: None,
driver_label: None,
url_params: None,
agent_java_options: Vec::new(),
host: "127.0.0.1".to_string(),
port: 1521,
username: "dbx".to_string(),
password: "dbx".to_string(),
database: None,
default_schema: None,
visible_databases: None,
visible_database_patterns: None,
visible_schemas: None,
attached_databases: Vec::new(),
init_script: None,
color: None,
transport_layers: Vec::new(),
connect_timeout_secs: 1,
query_timeout_secs: 30,
idle_timeout_secs: 60,
keepalive_interval_secs: interval_secs,
ssl: false,
ca_cert_path: String::new(),
client_cert_path: String::new(),
client_key_path: String::new(),
sysdba: false,
oracle_connection_type: None,
connection_string: None,
redis_connection_mode: None,
redis_sentinel_master: String::new(),
redis_sentinel_nodes: String::new(),
redis_sentinel_username: String::new(),
redis_sentinel_password: String::new(),
redis_sentinel_tls: false,
redis_cluster_nodes: String::new(),
redis_key_separator: crate::models::connection::default_redis_key_separator(),
redis_scan_page_size: None,
redis_database_aliases: Default::default(),
redis_key_templates: Vec::new(),
redis_key_grouping: None,
etcd_endpoints: String::new(),
gbase_server: String::new(),
informix_server: String::new(),
external_config: None,
plugin_id: None,
plugin_connection_provider: None,
plugin_connection_type: None,
connection_secrets: Default::default(),
jdbc_driver_class: None,
jdbc_driver_paths: Vec::new(),
one_time: false,
save_password: true,
read_only: false,
is_production: false,
production_databases: vec![],
show_system_schemas: false,
sidebar_auto_load_all_tables: false,
database_info: None,
}
}
/// Fake agent whose `validate_connection` reports a real failure.
///
/// It must NOT answer with an unsupported-method error: `ping_keepalive_target` treats that
/// as a healthy connection (`is_agent_validate_connection_unsupported`), which would make
/// this fixture silently green.
const KEEPALIVE_FAILING_AGENT: &str = r#"import json, sys
print(json.dumps({'ready': True}), flush=True)
for line in sys.stdin:
req = json.loads(line)
if req['method'] == 'handshake':
result = {'protocolVersion': 2, 'agentProtocolVersion': 2, 'capabilities': ['multi_session']}
print(json.dumps({'jsonrpc': '2.0', 'id': req['id'], 'result': result}), flush=True)
elif 'validate_connection' in req['method']:
print(json.dumps({'jsonrpc': '2.0', 'id': req['id'], 'error': {'code': -32000, 'message': 'connection refused by keepalive fixture'}}), flush=True)
else:
print(json.dumps({'jsonrpc': '2.0', 'id': req['id'], 'result': {}}), flush=True)
"#;
async fn spawn_keepalive_agent(
dir: &std::path::Path,
name: &str,
) -> std::sync::Arc<crate::db::agent_driver::PooledAgentClient> {
let script_path = dir.join(name);
std::fs::write(&script_path, KEEPALIVE_FAILING_AGENT).unwrap();
let python = if cfg!(windows) { "python" } else { "python3" };
let runtime = crate::db::agent_driver::AgentRuntimeClient::spawn(
crate::db::agent_driver::AgentLaunchSpec::new(python)
.with_args([script_path.to_string_lossy().to_string()]),
"test",
)
.await
.unwrap();
runtime.increment_session_count();
std::sync::Arc::new(crate::db::agent_driver::PooledAgentClient::new(
crate::db::agent_driver::AgentDriverClient::shared_session(runtime, format!("{name}-session")),
))
}
/// A failing probe on the connection's current pool must reach the UI (#4339). This drives
/// the real keepalive task rather than the publish helper, so it pins the wiring between
/// the probe's failure branch and the report.
#[tokio::test]
async fn keepalive_probe_failure_reports_a_lost_connection() {
let (state, dir) = test_app_state().await;
let config = keepalive_test_config("conn", 1);
state.configs.write().await.insert(config.id.clone(), config.clone());
let client = spawn_keepalive_agent(&dir, "failing-keepalive-agent.py").await;
state.connections.write().await.insert("conn".to_string(), PoolKind::Agent(std::sync::Arc::clone(&client)));
let pool = PoolKind::Agent(client);
let mut events = state.subscribe_connection_liveness();
state.start_keepalive_task(
"conn",
&pool,
&config,
#[cfg(feature = "mq-admin")]
None,
);
let message = tokio::time::timeout(Duration::from_secs(10), events.recv())
.await
.expect("a failing probe must report a liveness loss")
.expect("the channel must stay open");
assert_eq!(
message,
ConnectionLivenessMessage::Lost {
connection_id: "conn".to_string(),
failure_kind: ConnectionLivenessFailureKind::ProbeFailed,
}
);
state.stop_keepalive_task("conn").await;
let _ = std::fs::remove_dir_all(dir);
}
/// A pool that is busy with a running query is not evidence of a dead connection: the probe
/// is skipped entirely, so nothing may be reported (#4339).
#[tokio::test]
async fn keepalive_busy_pool_reports_nothing() {
let (state, dir) = test_app_state().await;
let config = keepalive_test_config("conn", 1);
state.configs.write().await.insert(config.id.clone(), config.clone());
let client = spawn_keepalive_agent(&dir, "busy-keepalive-agent.py").await;
state.connections.write().await.insert("conn".to_string(), PoolKind::Agent(std::sync::Arc::clone(&client)));
let pool = PoolKind::Agent(client);
// `is_pool_active` is true whenever a registered execution owns this pool key.
let registered = state.running_queries.register("keepalive-busy-exec".to_string());
state.running_queries.set_pool_key("keepalive-busy-exec", "conn");
assert!(state.running_queries.is_pool_active("conn"));
let mut events = state.subscribe_connection_liveness();
state.start_keepalive_task(
"conn",
&pool,
&config,
#[cfg(feature = "mq-admin")]
None,
);
// Long enough for more than one interval to elapse.
let outcome = tokio::time::timeout(Duration::from_millis(1500), events.recv()).await;
assert!(outcome.is_err(), "a busy pool must not report a liveness loss: {outcome:?}");
state.stop_keepalive_task("conn").await;
drop(registered);
let _ = std::fs::remove_dir_all(dir);
}
/// A probe whose pool was already replaced reports nothing: its result says nothing about
/// the connection that is live now (#4339).
#[tokio::test]
async fn keepalive_stale_target_reports_nothing() {
let (state, dir) = test_app_state().await;
let config = keepalive_test_config("conn", 1);
state.configs.write().await.insert(config.id.clone(), config.clone());
// The registry holds `current`, but the keepalive task is started for a different client,
// so the detach cannot match the live pool and must stay silent.
let current = spawn_keepalive_agent(&dir, "stale-current-agent.py").await;
let stale = spawn_keepalive_agent(&dir, "stale-replaced-agent.py").await;
state.connections.write().await.insert("conn".to_string(), PoolKind::Agent(current));
let mut events = state.subscribe_connection_liveness();
state.start_keepalive_task(
"conn",
&PoolKind::Agent(stale),
&config,
#[cfg(feature = "mq-admin")]
None,
);
let outcome = tokio::time::timeout(Duration::from_millis(1500), events.recv()).await;
assert!(outcome.is_err(), "a stale probe must not report a liveness loss: {outcome:?}");
state.stop_keepalive_task("conn").await;
let _ = std::fs::remove_dir_all(dir);
}
/// Stopping the keepalive task (explicit disconnect / reconnect) must not report a loss for
/// a pool that was never detached (#4339).
#[tokio::test]
async fn keepalive_stop_reports_nothing() {
let (state, dir) = test_app_state().await;
let config = keepalive_test_config("conn", 1);
state.configs.write().await.insert(config.id.clone(), config.clone());
let client = spawn_keepalive_agent(&dir, "stopped-keepalive-agent.py").await;
state.connections.write().await.insert("conn".to_string(), PoolKind::Agent(std::sync::Arc::clone(&client)));
let pool = PoolKind::Agent(client);
let mut events = state.subscribe_connection_liveness();
state.start_keepalive_task(
"conn",
&pool,
&config,
#[cfg(feature = "mq-admin")]
None,
);
state.stop_keepalive_task("conn").await;
let outcome = tokio::time::timeout(Duration::from_millis(1500), events.recv()).await;
assert!(outcome.is_err(), "a stopped keepalive must not report a liveness loss: {outcome:?}");
let _ = std::fs::remove_dir_all(dir);
}
/// The sidebar must not grey out a connection because one derived pool died: pool keys
/// are per session/role/database derived, so only the last pool going makes the
/// connection really unusable. This gate is also what collapses N per-pool keepalive
/// failures into a single logical disconnect (#4339).
#[tokio::test]
async fn liveness_is_published_only_after_the_connections_last_pool_is_gone() {
let (state, dir) = test_app_state().await;
let pool = crate::db::sqlite::connect_path(":memory:").await.unwrap();
let mut events = state.subscribe_connection_liveness();
state
.update_connection_pools(|connections| {
connections.insert("conn".to_string(), PoolKind::Sqlite(pool.clone()));
connections.insert("conn:analytics:session:tab-1".to_string(), PoolKind::Sqlite(pool.clone()));
// A sibling id sharing the prefix is a different connection and must not
// keep "conn" alive.
connections.insert("conn-2".to_string(), PoolKind::Sqlite(pool));
})
.await;
// A derived pool dying while another pool of the same connection survives says
// nothing about the connection root.
state.update_connection_pools(|connections| connections.remove("conn:analytics:session:tab-1")).await;
report_connection_liveness_lost(
&state.connections,
&state.connection_liveness,
"conn",
ConnectionLivenessFailureKind::ProbeFailed,
)
.await;
assert!(
matches!(events.try_recv(), Err(tokio::sync::broadcast::error::TryRecvError::Empty)),
"a surviving pool must suppress the liveness message"
);
// The last pool going is a real disconnect, and only then is it reported.
state.update_connection_pools(|connections| connections.remove("conn")).await;
report_connection_liveness_lost(
&state.connections,
&state.connection_liveness,
"conn",
ConnectionLivenessFailureKind::TimedOut,
)
.await;
let message = events.try_recv().expect("the last pool going must report a liveness loss");
assert_eq!(
message,
ConnectionLivenessMessage::Lost {
connection_id: "conn".to_string(),
failure_kind: ConnectionLivenessFailureKind::TimedOut,
}
);
assert!(
matches!(events.try_recv(), Err(tokio::sync::broadcast::error::TryRecvError::Empty)),
"one outage publishes exactly one message"
);
let _ = std::fs::remove_dir_all(dir);
}
#[tokio::test]
async fn connection_slot_recovery_waits_for_agent_metadata_and_confirms_close() {
let (state, dir) = test_app_state().await;