feat(core): optimize query resource lifecycle

This commit is contained in:
t8y2 committed 2026-07-16 00:19:24 +08:00
1 parent b4bd6c42a7
commit 295b67dd36
29 files changed
+1857 -335

No files matched your search

+269 -10
View File
@@ -1,5 +1,6 @@
use std::collections::{HashMap, HashSet};
use std::path::Path;
use std::time::{SystemTime, UNIX_EPOCH};
use log::warn;
use rusqlite::{params, params_from_iter, Connection, DatabaseName, OpenFlags, OptionalExtension, ToSql};
@@ -102,6 +103,7 @@ pub struct Storage {
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct TabRuntimeCacheEntry {
pub key: String,
pub payload: Vec<u8>,
@@ -109,6 +111,32 @@ pub struct TabRuntimeCacheEntry {
pub column_count: i64,
pub byte_size: i64,
pub updated_at: String,
pub created_at: i64,
pub last_accessed_at: i64,
pub owner_id: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct TabRuntimeCacheMetadata {
pub key: String,
pub row_count: i64,
pub column_count: i64,
pub byte_size: i64,
pub updated_at: String,
pub created_at: i64,
pub last_accessed_at: i64,
pub owner_id: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct TabRuntimeCachePruneResult {
pub deleted_entries: usize,
pub deleted_bytes: i64,
pub orphan_deletions: usize,
pub remaining_entries: usize,
pub remaining_bytes: i64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
@@ -260,7 +288,10 @@ const SCHEMA_STATEMENTS: &[&str] = &[
row_count INTEGER NOT NULL DEFAULT 0,
column_count INTEGER NOT NULL DEFAULT 0,
byte_size INTEGER NOT NULL DEFAULT 0,
updated_at TEXT NOT NULL
updated_at TEXT NOT NULL,
created_at INTEGER NOT NULL DEFAULT 0,
last_accessed_at INTEGER NOT NULL DEFAULT 0,
owner_id TEXT
)",
"CREATE TABLE IF NOT EXISTS mq_token_records (
id TEXT PRIMARY KEY,
@@ -319,6 +350,7 @@ impl Storage {
}
ensure_history_columns_sync(conn)?;
ensure_saved_sql_columns_sync(conn)?;
ensure_tab_runtime_cache_columns_sync(conn)?;
Ok(())
})
}
@@ -432,6 +464,26 @@ fn ensure_saved_sql_columns_sync(conn: &Connection) -> Result<(), String> {
Ok(())
}
fn ensure_tab_runtime_cache_columns_sync(conn: &Connection) -> Result<(), String> {
const COLUMNS: &[(&str, &str)] = &[
("created_at", "INTEGER NOT NULL DEFAULT 0"),
("last_accessed_at", "INTEGER NOT NULL DEFAULT 0"),
("owner_id", "TEXT"),
];
ensure_table_columns(conn, "tab_runtime_cache", COLUMNS)?;
let now = unix_timestamp_millis();
// Legacy rows must receive a grace period instead of being treated as ancient crash leftovers.
conn.execute("UPDATE tab_runtime_cache SET created_at = ?1 WHERE created_at = 0", [now])
.map_err(|e| e.to_string())?;
conn.execute("UPDATE tab_runtime_cache SET last_accessed_at = created_at WHERE last_accessed_at = 0", [])
.map_err(|e| e.to_string())?;
Ok(())
}
fn unix_timestamp_millis() -> i64 {
SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis().min(i64::MAX as u128) as i64
}
fn ensure_table_columns(conn: &Connection, table_name: &str, columns: &[(&str, &str)]) -> Result<(), String> {
let mut stmt =
conn.prepare(&format!("SELECT name FROM pragma_table_info('{table_name}')")).map_err(|e| e.to_string())?;
@@ -2082,18 +2134,21 @@ impl Storage {
payload: Vec<u8>,
row_count: i64,
column_count: i64,
owner_id: Option<String>,
) -> Result<(), String> {
let key = key.to_string();
let byte_size = payload.len() as i64;
let now = unix_timestamp_millis();
self.with_conn(move |conn| {
conn.execute(
"INSERT INTO tab_runtime_cache \
(cache_key, payload, row_count, column_count, byte_size, updated_at) \
VALUES (?1, ?2, ?3, ?4, ?5, datetime('now')) \
(cache_key, payload, row_count, column_count, byte_size, updated_at, created_at, last_accessed_at, owner_id) \
VALUES (?1, ?2, ?3, ?4, ?5, datetime('now'), ?6, ?6, ?7) \
ON CONFLICT(cache_key) DO UPDATE SET \
payload = excluded.payload, row_count = excluded.row_count, column_count = excluded.column_count, \
byte_size = excluded.byte_size, updated_at = excluded.updated_at",
params![key, payload, row_count, column_count, byte_size],
byte_size = excluded.byte_size, updated_at = excluded.updated_at, \
last_accessed_at = excluded.last_accessed_at, owner_id = excluded.owner_id",
params![key, payload, row_count, column_count, byte_size, now, owner_id],
)
.map(|_| ())
.map_err(|e| e.to_string())
@@ -2103,11 +2158,13 @@ impl Storage {
pub async fn load_tab_runtime_cache(&self, key: &str) -> Result<Option<TabRuntimeCacheEntry>, String> {
let key = key.to_string();
let now = unix_timestamp_millis();
self.with_conn(move |conn| {
conn.query_row(
"SELECT cache_key, payload, row_count, column_count, byte_size, updated_at \
let entry = conn
.query_row(
"SELECT cache_key, payload, row_count, column_count, byte_size, updated_at, created_at, last_accessed_at, owner_id \
FROM tab_runtime_cache WHERE cache_key = ?1",
[key],
[&key],
|row| {
Ok(TabRuntimeCacheEntry {
key: row.get(0)?,
@@ -2116,11 +2173,134 @@ impl Storage {
column_count: row.get(3)?,
byte_size: row.get(4)?,
updated_at: row.get(5)?,
created_at: row.get(6)?,
last_accessed_at: now,
owner_id: row.get(8)?,
})
},
)
.optional()
.map_err(|e| e.to_string())
.map_err(|e| e.to_string())?;
if entry.is_some() {
conn.execute(
"UPDATE tab_runtime_cache SET last_accessed_at = ?2 WHERE cache_key = ?1",
params![key, now],
)
.map_err(|e| e.to_string())?;
}
Ok(entry)
})
.await
}
pub async fn list_tab_runtime_cache_metadata(&self) -> Result<Vec<TabRuntimeCacheMetadata>, String> {
self.with_conn(move |conn| {
let mut statement = conn
.prepare(
"SELECT cache_key, row_count, column_count, byte_size, updated_at, created_at, last_accessed_at, owner_id \
FROM tab_runtime_cache ORDER BY last_accessed_at ASC, cache_key ASC",
)
.map_err(|e| e.to_string())?;
let metadata = statement
.query_map([], |row| {
Ok(TabRuntimeCacheMetadata {
key: row.get(0)?,
row_count: row.get(1)?,
column_count: row.get(2)?,
byte_size: row.get(3)?,
updated_at: row.get(4)?,
created_at: row.get(5)?,
last_accessed_at: row.get(6)?,
owner_id: row.get(7)?,
})
})
.map_err(|e| e.to_string())?
.collect::<Result<Vec<_>, _>>()
.map_err(|e| e.to_string())?;
Ok(metadata)
})
.await
}
pub async fn prune_tab_runtime_cache(
&self,
live_keys: Vec<String>,
max_bytes: i64,
orphan_grace_ms: i64,
max_age_ms: Option<i64>,
) -> Result<TabRuntimeCachePruneResult, String> {
let now = unix_timestamp_millis();
self.with_conn(move |conn| {
let live_keys: HashSet<String> = live_keys.into_iter().collect();
let mut statement = conn
.prepare(
"SELECT cache_key, byte_size, created_at, last_accessed_at \
FROM tab_runtime_cache ORDER BY last_accessed_at ASC, cache_key ASC",
)
.map_err(|e| e.to_string())?;
let entries = statement
.query_map([], |row| {
Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?, row.get::<_, i64>(2)?, row.get::<_, i64>(3)?))
})
.map_err(|e| e.to_string())?
.collect::<Result<Vec<_>, _>>()
.map_err(|e| e.to_string())?;
drop(statement);
let mut total_bytes = entries.iter().map(|(_, bytes, _, _)| *bytes).sum::<i64>();
let mut deleted = HashSet::new();
let mut orphan_deletions = 0usize;
for (key, bytes, created_at, last_accessed_at) in &entries {
if live_keys.contains(key) {
continue;
}
let orphan_expired = now.saturating_sub(*created_at) >= orphan_grace_ms.max(0);
let age_expired =
max_age_ms.is_some_and(|max_age| now.saturating_sub(*last_accessed_at) >= max_age.max(0));
if orphan_expired || age_expired {
deleted.insert(key.clone());
total_bytes = total_bytes.saturating_sub(*bytes);
if orphan_expired {
orphan_deletions += 1;
}
}
}
for (key, bytes, _, _) in &entries {
if total_bytes <= max_bytes.max(0) {
break;
}
if live_keys.contains(key) || deleted.contains(key) {
continue;
}
deleted.insert(key.clone());
total_bytes = total_bytes.saturating_sub(*bytes);
}
let deleted_bytes =
entries.iter().filter(|(key, _, _, _)| deleted.contains(key)).map(|(_, bytes, _, _)| *bytes).sum();
let transaction = conn.transaction().map_err(|e| e.to_string())?;
for key in &deleted {
transaction
.execute("DELETE FROM tab_runtime_cache WHERE cache_key = ?1", [key])
.map_err(|e| e.to_string())?;
}
transaction.commit().map_err(|e| e.to_string())?;
Ok(TabRuntimeCachePruneResult {
deleted_entries: deleted.len(),
deleted_bytes,
orphan_deletions,
remaining_entries: entries.len().saturating_sub(deleted.len()),
remaining_bytes: total_bytes,
})
})
.await
}
pub async fn delete_tab_runtime_cache_owner(&self, owner_id: &str) -> Result<usize, String> {
let owner_id = owner_id.to_string();
self.with_conn(move |conn| {
conn.execute("DELETE FROM tab_runtime_cache WHERE owner_id = ?1", [owner_id]).map_err(|e| e.to_string())
})
.await
}
@@ -2535,6 +2715,7 @@ mod tests {
ConnectionConfig, DatabaseConnectionInfo, DatabaseType, SshTunnelConfig, TransportLayerConfig,
};
use crate::saved_sql::SavedSqlFile;
use rusqlite::Connection;
use std::time::{SystemTime, UNIX_EPOCH};
fn temp_db_path(name: &str) -> std::path::PathBuf {
@@ -3256,7 +3437,10 @@ mod tests {
let path = temp_db_path("tab-runtime-cache");
let storage = Storage::open(&path).await.unwrap();
storage.save_tab_runtime_cache("tab:1:result", vec![1, 2, 3, 4], 10, 3).await.unwrap();
storage
.save_tab_runtime_cache("tab:1:result", vec![1, 2, 3, 4], 10, 3, Some("connection-1".to_string()))
.await
.unwrap();
let entry = storage.load_tab_runtime_cache("tab:1:result").await.unwrap().unwrap();
assert_eq!(entry.key, "tab:1:result");
@@ -3264,11 +3448,86 @@ mod tests {
assert_eq!(entry.row_count, 10);
assert_eq!(entry.column_count, 3);
assert_eq!(entry.byte_size, 4);
assert_eq!(entry.owner_id.as_deref(), Some("connection-1"));
assert!(entry.created_at > 0);
assert!(entry.last_accessed_at >= entry.created_at);
storage.delete_tab_runtime_cache("tab:1:result").await.unwrap();
assert_eq!(storage.load_tab_runtime_cache("tab:1:result").await.unwrap(), None);
}
#[tokio::test]
async fn tab_runtime_cache_pruning_retains_live_entries_and_enforces_byte_budget() {
let path = temp_db_path("tab-runtime-cache-prune");
let storage = Storage::open(&path).await.unwrap();
for (key, size) in [("live", 6usize), ("old", 5), ("new", 4)] {
storage.save_tab_runtime_cache(key, vec![1; size], 1, 1, Some("connection-1".to_string())).await.unwrap();
}
storage
.with_conn(|conn| {
conn.execute("UPDATE tab_runtime_cache SET last_accessed_at = 1 WHERE cache_key = 'old'", [])
.map_err(|e| e.to_string())?;
conn.execute("UPDATE tab_runtime_cache SET last_accessed_at = 2 WHERE cache_key = 'new'", [])
.map_err(|e| e.to_string())?;
Ok(())
})
.await
.unwrap();
let result = storage.prune_tab_runtime_cache(vec!["live".to_string()], 10, i64::MAX, None).await.unwrap();
assert_eq!(result.deleted_entries, 1);
assert!(storage.load_tab_runtime_cache("live").await.unwrap().is_some());
assert!(storage.load_tab_runtime_cache("old").await.unwrap().is_none());
assert!(storage.load_tab_runtime_cache("new").await.unwrap().is_some());
}
#[tokio::test]
async fn tab_runtime_cache_pruning_respects_orphan_grace_period() {
let path = temp_db_path("tab-runtime-cache-orphan-grace");
let storage = Storage::open(&path).await.unwrap();
storage.save_tab_runtime_cache("fresh", vec![1], 1, 1, None).await.unwrap();
storage.save_tab_runtime_cache("crash-leftover", vec![2], 1, 1, None).await.unwrap();
storage
.with_conn(|conn| {
conn.execute("UPDATE tab_runtime_cache SET created_at = 1 WHERE cache_key = 'crash-leftover'", [])
.map(|_| ())
.map_err(|e| e.to_string())
})
.await
.unwrap();
let result = storage.prune_tab_runtime_cache(Vec::new(), 1024, 60_000, None).await.unwrap();
assert_eq!(result.orphan_deletions, 1);
assert!(storage.load_tab_runtime_cache("fresh").await.unwrap().is_some());
assert!(storage.load_tab_runtime_cache("crash-leftover").await.unwrap().is_none());
}
#[tokio::test]
async fn tab_runtime_cache_migrates_legacy_schema_without_dropping_entries() {
let path = temp_db_path("tab-runtime-cache-legacy-schema");
{
let connection = Connection::open(&path).unwrap();
connection
.execute(
"CREATE TABLE tab_runtime_cache (cache_key TEXT PRIMARY KEY, payload BLOB NOT NULL, row_count INTEGER NOT NULL DEFAULT 0, column_count INTEGER NOT NULL DEFAULT 0, byte_size INTEGER NOT NULL DEFAULT 0, updated_at TEXT NOT NULL)",
[],
)
.unwrap();
connection
.execute("INSERT INTO tab_runtime_cache VALUES ('legacy', X'0102', 1, 1, 2, '2026-01-01')", [])
.unwrap();
}
let storage = Storage::open(&path).await.unwrap();
let entry = storage.load_tab_runtime_cache("legacy").await.unwrap().unwrap();
assert_eq!(entry.payload, vec![1, 2]);
assert!(entry.created_at > 0);
assert!(entry.last_accessed_at >= entry.created_at);
}
#[tokio::test]
async fn saved_sql_summary_omits_sql_text_and_loads_file_on_demand() {
let path = temp_db_path("saved-sql-summary");