feat(core): optimize query resource lifecycle
This commit is contained in:
1 parent
b4bd6c42a7
commit
295b67dd36
29 files changed
+1857
-335
No files matched your search
+269
-10
@@ -1,5 +1,6 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::path::Path;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use log::warn;
|
||||
use rusqlite::{params, params_from_iter, Connection, DatabaseName, OpenFlags, OptionalExtension, ToSql};
|
||||
@@ -102,6 +103,7 @@ pub struct Storage {
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct TabRuntimeCacheEntry {
|
||||
pub key: String,
|
||||
pub payload: Vec<u8>,
|
||||
@@ -109,6 +111,32 @@ pub struct TabRuntimeCacheEntry {
|
||||
pub column_count: i64,
|
||||
pub byte_size: i64,
|
||||
pub updated_at: String,
|
||||
pub created_at: i64,
|
||||
pub last_accessed_at: i64,
|
||||
pub owner_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct TabRuntimeCacheMetadata {
|
||||
pub key: String,
|
||||
pub row_count: i64,
|
||||
pub column_count: i64,
|
||||
pub byte_size: i64,
|
||||
pub updated_at: String,
|
||||
pub created_at: i64,
|
||||
pub last_accessed_at: i64,
|
||||
pub owner_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct TabRuntimeCachePruneResult {
|
||||
pub deleted_entries: usize,
|
||||
pub deleted_bytes: i64,
|
||||
pub orphan_deletions: usize,
|
||||
pub remaining_entries: usize,
|
||||
pub remaining_bytes: i64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
@@ -260,7 +288,10 @@ const SCHEMA_STATEMENTS: &[&str] = &[
|
||||
row_count INTEGER NOT NULL DEFAULT 0,
|
||||
column_count INTEGER NOT NULL DEFAULT 0,
|
||||
byte_size INTEGER NOT NULL DEFAULT 0,
|
||||
updated_at TEXT NOT NULL
|
||||
updated_at TEXT NOT NULL,
|
||||
created_at INTEGER NOT NULL DEFAULT 0,
|
||||
last_accessed_at INTEGER NOT NULL DEFAULT 0,
|
||||
owner_id TEXT
|
||||
)",
|
||||
"CREATE TABLE IF NOT EXISTS mq_token_records (
|
||||
id TEXT PRIMARY KEY,
|
||||
@@ -319,6 +350,7 @@ impl Storage {
|
||||
}
|
||||
ensure_history_columns_sync(conn)?;
|
||||
ensure_saved_sql_columns_sync(conn)?;
|
||||
ensure_tab_runtime_cache_columns_sync(conn)?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
@@ -432,6 +464,26 @@ fn ensure_saved_sql_columns_sync(conn: &Connection) -> Result<(), String> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ensure_tab_runtime_cache_columns_sync(conn: &Connection) -> Result<(), String> {
|
||||
const COLUMNS: &[(&str, &str)] = &[
|
||||
("created_at", "INTEGER NOT NULL DEFAULT 0"),
|
||||
("last_accessed_at", "INTEGER NOT NULL DEFAULT 0"),
|
||||
("owner_id", "TEXT"),
|
||||
];
|
||||
ensure_table_columns(conn, "tab_runtime_cache", COLUMNS)?;
|
||||
let now = unix_timestamp_millis();
|
||||
// Legacy rows must receive a grace period instead of being treated as ancient crash leftovers.
|
||||
conn.execute("UPDATE tab_runtime_cache SET created_at = ?1 WHERE created_at = 0", [now])
|
||||
.map_err(|e| e.to_string())?;
|
||||
conn.execute("UPDATE tab_runtime_cache SET last_accessed_at = created_at WHERE last_accessed_at = 0", [])
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn unix_timestamp_millis() -> i64 {
|
||||
SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_millis().min(i64::MAX as u128) as i64
|
||||
}
|
||||
|
||||
fn ensure_table_columns(conn: &Connection, table_name: &str, columns: &[(&str, &str)]) -> Result<(), String> {
|
||||
let mut stmt =
|
||||
conn.prepare(&format!("SELECT name FROM pragma_table_info('{table_name}')")).map_err(|e| e.to_string())?;
|
||||
@@ -2082,18 +2134,21 @@ impl Storage {
|
||||
payload: Vec<u8>,
|
||||
row_count: i64,
|
||||
column_count: i64,
|
||||
owner_id: Option<String>,
|
||||
) -> Result<(), String> {
|
||||
let key = key.to_string();
|
||||
let byte_size = payload.len() as i64;
|
||||
let now = unix_timestamp_millis();
|
||||
self.with_conn(move |conn| {
|
||||
conn.execute(
|
||||
"INSERT INTO tab_runtime_cache \
|
||||
(cache_key, payload, row_count, column_count, byte_size, updated_at) \
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, datetime('now')) \
|
||||
(cache_key, payload, row_count, column_count, byte_size, updated_at, created_at, last_accessed_at, owner_id) \
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, datetime('now'), ?6, ?6, ?7) \
|
||||
ON CONFLICT(cache_key) DO UPDATE SET \
|
||||
payload = excluded.payload, row_count = excluded.row_count, column_count = excluded.column_count, \
|
||||
byte_size = excluded.byte_size, updated_at = excluded.updated_at",
|
||||
params![key, payload, row_count, column_count, byte_size],
|
||||
byte_size = excluded.byte_size, updated_at = excluded.updated_at, \
|
||||
last_accessed_at = excluded.last_accessed_at, owner_id = excluded.owner_id",
|
||||
params![key, payload, row_count, column_count, byte_size, now, owner_id],
|
||||
)
|
||||
.map(|_| ())
|
||||
.map_err(|e| e.to_string())
|
||||
@@ -2103,11 +2158,13 @@ impl Storage {
|
||||
|
||||
pub async fn load_tab_runtime_cache(&self, key: &str) -> Result<Option<TabRuntimeCacheEntry>, String> {
|
||||
let key = key.to_string();
|
||||
let now = unix_timestamp_millis();
|
||||
self.with_conn(move |conn| {
|
||||
conn.query_row(
|
||||
"SELECT cache_key, payload, row_count, column_count, byte_size, updated_at \
|
||||
let entry = conn
|
||||
.query_row(
|
||||
"SELECT cache_key, payload, row_count, column_count, byte_size, updated_at, created_at, last_accessed_at, owner_id \
|
||||
FROM tab_runtime_cache WHERE cache_key = ?1",
|
||||
[key],
|
||||
[&key],
|
||||
|row| {
|
||||
Ok(TabRuntimeCacheEntry {
|
||||
key: row.get(0)?,
|
||||
@@ -2116,11 +2173,134 @@ impl Storage {
|
||||
column_count: row.get(3)?,
|
||||
byte_size: row.get(4)?,
|
||||
updated_at: row.get(5)?,
|
||||
created_at: row.get(6)?,
|
||||
last_accessed_at: now,
|
||||
owner_id: row.get(8)?,
|
||||
})
|
||||
},
|
||||
)
|
||||
.optional()
|
||||
.map_err(|e| e.to_string())
|
||||
.map_err(|e| e.to_string())?;
|
||||
if entry.is_some() {
|
||||
conn.execute(
|
||||
"UPDATE tab_runtime_cache SET last_accessed_at = ?2 WHERE cache_key = ?1",
|
||||
params![key, now],
|
||||
)
|
||||
.map_err(|e| e.to_string())?;
|
||||
}
|
||||
Ok(entry)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn list_tab_runtime_cache_metadata(&self) -> Result<Vec<TabRuntimeCacheMetadata>, String> {
|
||||
self.with_conn(move |conn| {
|
||||
let mut statement = conn
|
||||
.prepare(
|
||||
"SELECT cache_key, row_count, column_count, byte_size, updated_at, created_at, last_accessed_at, owner_id \
|
||||
FROM tab_runtime_cache ORDER BY last_accessed_at ASC, cache_key ASC",
|
||||
)
|
||||
.map_err(|e| e.to_string())?;
|
||||
let metadata = statement
|
||||
.query_map([], |row| {
|
||||
Ok(TabRuntimeCacheMetadata {
|
||||
key: row.get(0)?,
|
||||
row_count: row.get(1)?,
|
||||
column_count: row.get(2)?,
|
||||
byte_size: row.get(3)?,
|
||||
updated_at: row.get(4)?,
|
||||
created_at: row.get(5)?,
|
||||
last_accessed_at: row.get(6)?,
|
||||
owner_id: row.get(7)?,
|
||||
})
|
||||
})
|
||||
.map_err(|e| e.to_string())?
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(metadata)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn prune_tab_runtime_cache(
|
||||
&self,
|
||||
live_keys: Vec<String>,
|
||||
max_bytes: i64,
|
||||
orphan_grace_ms: i64,
|
||||
max_age_ms: Option<i64>,
|
||||
) -> Result<TabRuntimeCachePruneResult, String> {
|
||||
let now = unix_timestamp_millis();
|
||||
self.with_conn(move |conn| {
|
||||
let live_keys: HashSet<String> = live_keys.into_iter().collect();
|
||||
let mut statement = conn
|
||||
.prepare(
|
||||
"SELECT cache_key, byte_size, created_at, last_accessed_at \
|
||||
FROM tab_runtime_cache ORDER BY last_accessed_at ASC, cache_key ASC",
|
||||
)
|
||||
.map_err(|e| e.to_string())?;
|
||||
let entries = statement
|
||||
.query_map([], |row| {
|
||||
Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?, row.get::<_, i64>(2)?, row.get::<_, i64>(3)?))
|
||||
})
|
||||
.map_err(|e| e.to_string())?
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.map_err(|e| e.to_string())?;
|
||||
drop(statement);
|
||||
|
||||
let mut total_bytes = entries.iter().map(|(_, bytes, _, _)| *bytes).sum::<i64>();
|
||||
let mut deleted = HashSet::new();
|
||||
let mut orphan_deletions = 0usize;
|
||||
for (key, bytes, created_at, last_accessed_at) in &entries {
|
||||
if live_keys.contains(key) {
|
||||
continue;
|
||||
}
|
||||
let orphan_expired = now.saturating_sub(*created_at) >= orphan_grace_ms.max(0);
|
||||
let age_expired =
|
||||
max_age_ms.is_some_and(|max_age| now.saturating_sub(*last_accessed_at) >= max_age.max(0));
|
||||
if orphan_expired || age_expired {
|
||||
deleted.insert(key.clone());
|
||||
total_bytes = total_bytes.saturating_sub(*bytes);
|
||||
if orphan_expired {
|
||||
orphan_deletions += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (key, bytes, _, _) in &entries {
|
||||
if total_bytes <= max_bytes.max(0) {
|
||||
break;
|
||||
}
|
||||
if live_keys.contains(key) || deleted.contains(key) {
|
||||
continue;
|
||||
}
|
||||
deleted.insert(key.clone());
|
||||
total_bytes = total_bytes.saturating_sub(*bytes);
|
||||
}
|
||||
|
||||
let deleted_bytes =
|
||||
entries.iter().filter(|(key, _, _, _)| deleted.contains(key)).map(|(_, bytes, _, _)| *bytes).sum();
|
||||
let transaction = conn.transaction().map_err(|e| e.to_string())?;
|
||||
for key in &deleted {
|
||||
transaction
|
||||
.execute("DELETE FROM tab_runtime_cache WHERE cache_key = ?1", [key])
|
||||
.map_err(|e| e.to_string())?;
|
||||
}
|
||||
transaction.commit().map_err(|e| e.to_string())?;
|
||||
Ok(TabRuntimeCachePruneResult {
|
||||
deleted_entries: deleted.len(),
|
||||
deleted_bytes,
|
||||
orphan_deletions,
|
||||
remaining_entries: entries.len().saturating_sub(deleted.len()),
|
||||
remaining_bytes: total_bytes,
|
||||
})
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn delete_tab_runtime_cache_owner(&self, owner_id: &str) -> Result<usize, String> {
|
||||
let owner_id = owner_id.to_string();
|
||||
self.with_conn(move |conn| {
|
||||
conn.execute("DELETE FROM tab_runtime_cache WHERE owner_id = ?1", [owner_id]).map_err(|e| e.to_string())
|
||||
})
|
||||
.await
|
||||
}
|
||||
@@ -2535,6 +2715,7 @@ mod tests {
|
||||
ConnectionConfig, DatabaseConnectionInfo, DatabaseType, SshTunnelConfig, TransportLayerConfig,
|
||||
};
|
||||
use crate::saved_sql::SavedSqlFile;
|
||||
use rusqlite::Connection;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
fn temp_db_path(name: &str) -> std::path::PathBuf {
|
||||
@@ -3256,7 +3437,10 @@ mod tests {
|
||||
let path = temp_db_path("tab-runtime-cache");
|
||||
let storage = Storage::open(&path).await.unwrap();
|
||||
|
||||
storage.save_tab_runtime_cache("tab:1:result", vec![1, 2, 3, 4], 10, 3).await.unwrap();
|
||||
storage
|
||||
.save_tab_runtime_cache("tab:1:result", vec![1, 2, 3, 4], 10, 3, Some("connection-1".to_string()))
|
||||
.await
|
||||
.unwrap();
|
||||
let entry = storage.load_tab_runtime_cache("tab:1:result").await.unwrap().unwrap();
|
||||
|
||||
assert_eq!(entry.key, "tab:1:result");
|
||||
@@ -3264,11 +3448,86 @@ mod tests {
|
||||
assert_eq!(entry.row_count, 10);
|
||||
assert_eq!(entry.column_count, 3);
|
||||
assert_eq!(entry.byte_size, 4);
|
||||
assert_eq!(entry.owner_id.as_deref(), Some("connection-1"));
|
||||
assert!(entry.created_at > 0);
|
||||
assert!(entry.last_accessed_at >= entry.created_at);
|
||||
|
||||
storage.delete_tab_runtime_cache("tab:1:result").await.unwrap();
|
||||
assert_eq!(storage.load_tab_runtime_cache("tab:1:result").await.unwrap(), None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tab_runtime_cache_pruning_retains_live_entries_and_enforces_byte_budget() {
|
||||
let path = temp_db_path("tab-runtime-cache-prune");
|
||||
let storage = Storage::open(&path).await.unwrap();
|
||||
for (key, size) in [("live", 6usize), ("old", 5), ("new", 4)] {
|
||||
storage.save_tab_runtime_cache(key, vec![1; size], 1, 1, Some("connection-1".to_string())).await.unwrap();
|
||||
}
|
||||
storage
|
||||
.with_conn(|conn| {
|
||||
conn.execute("UPDATE tab_runtime_cache SET last_accessed_at = 1 WHERE cache_key = 'old'", [])
|
||||
.map_err(|e| e.to_string())?;
|
||||
conn.execute("UPDATE tab_runtime_cache SET last_accessed_at = 2 WHERE cache_key = 'new'", [])
|
||||
.map_err(|e| e.to_string())?;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let result = storage.prune_tab_runtime_cache(vec!["live".to_string()], 10, i64::MAX, None).await.unwrap();
|
||||
|
||||
assert_eq!(result.deleted_entries, 1);
|
||||
assert!(storage.load_tab_runtime_cache("live").await.unwrap().is_some());
|
||||
assert!(storage.load_tab_runtime_cache("old").await.unwrap().is_none());
|
||||
assert!(storage.load_tab_runtime_cache("new").await.unwrap().is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tab_runtime_cache_pruning_respects_orphan_grace_period() {
|
||||
let path = temp_db_path("tab-runtime-cache-orphan-grace");
|
||||
let storage = Storage::open(&path).await.unwrap();
|
||||
storage.save_tab_runtime_cache("fresh", vec![1], 1, 1, None).await.unwrap();
|
||||
storage.save_tab_runtime_cache("crash-leftover", vec![2], 1, 1, None).await.unwrap();
|
||||
storage
|
||||
.with_conn(|conn| {
|
||||
conn.execute("UPDATE tab_runtime_cache SET created_at = 1 WHERE cache_key = 'crash-leftover'", [])
|
||||
.map(|_| ())
|
||||
.map_err(|e| e.to_string())
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let result = storage.prune_tab_runtime_cache(Vec::new(), 1024, 60_000, None).await.unwrap();
|
||||
|
||||
assert_eq!(result.orphan_deletions, 1);
|
||||
assert!(storage.load_tab_runtime_cache("fresh").await.unwrap().is_some());
|
||||
assert!(storage.load_tab_runtime_cache("crash-leftover").await.unwrap().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tab_runtime_cache_migrates_legacy_schema_without_dropping_entries() {
|
||||
let path = temp_db_path("tab-runtime-cache-legacy-schema");
|
||||
{
|
||||
let connection = Connection::open(&path).unwrap();
|
||||
connection
|
||||
.execute(
|
||||
"CREATE TABLE tab_runtime_cache (cache_key TEXT PRIMARY KEY, payload BLOB NOT NULL, row_count INTEGER NOT NULL DEFAULT 0, column_count INTEGER NOT NULL DEFAULT 0, byte_size INTEGER NOT NULL DEFAULT 0, updated_at TEXT NOT NULL)",
|
||||
[],
|
||||
)
|
||||
.unwrap();
|
||||
connection
|
||||
.execute("INSERT INTO tab_runtime_cache VALUES ('legacy', X'0102', 1, 1, 2, '2026-01-01')", [])
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let storage = Storage::open(&path).await.unwrap();
|
||||
let entry = storage.load_tab_runtime_cache("legacy").await.unwrap().unwrap();
|
||||
|
||||
assert_eq!(entry.payload, vec![1, 2]);
|
||||
assert!(entry.created_at > 0);
|
||||
assert!(entry.last_accessed_at >= entry.created_at);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn saved_sql_summary_omits_sql_text_and_loads_file_on_demand() {
|
||||
let path = temp_db_path("saved-sql-summary");
|
||||
|
||||
Reference in new issue
Block a user