fix(tdengine): keep table paging stable via sorted snapshot
This commit is contained in:
2 files changed
+84
-66
No files matched your search
@@ -158,9 +158,9 @@ impl TdengineDriver {
|
|||||||
return Ok(Vec::new());
|
return Ok(Vec::new());
|
||||||
}
|
}
|
||||||
let database = effective_database(database, &self.current_database)?.to_string();
|
let database = effective_database(database, &self.current_database)?.to_string();
|
||||||
if constraints.limit > 0 && constraints.filter.trim().is_empty() {
|
// `SHOW <db>.TABLES` / `SHOW <db>.STABLES` are unordered, so a server-side page
|
||||||
return self.list_tables_page(&database, constraints.offset, constraints.limit, token).await;
|
// offset can skip or repeat rows between two requests. Always read the full
|
||||||
}
|
// (sorted, cached) list and slice it locally instead.
|
||||||
let tables = self.all_tables(&database, token).await?;
|
let tables = self.all_tables(&database, token).await?;
|
||||||
Ok(filter_tables(tables, &constraints))
|
Ok(filter_tables(tables, &constraints))
|
||||||
}
|
}
|
||||||
@@ -522,42 +522,6 @@ impl TdengineDriver {
|
|||||||
Ok(tables)
|
Ok(tables)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn list_tables_page(
|
|
||||||
&self,
|
|
||||||
database: &str,
|
|
||||||
offset: usize,
|
|
||||||
limit: usize,
|
|
||||||
token: &CancellationToken,
|
|
||||||
) -> Result<Vec<TableInfo>> {
|
|
||||||
let (mut stables, scanned) = self
|
|
||||||
.query_tables_page(
|
|
||||||
&format!("SHOW {}STABLES", quote_qualified_prefix(database)),
|
|
||||||
"STABLE",
|
|
||||||
false,
|
|
||||||
offset,
|
|
||||||
limit,
|
|
||||||
token,
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
if stables.len() >= limit {
|
|
||||||
return Ok(stables);
|
|
||||||
}
|
|
||||||
let table_offset = offset.saturating_sub(scanned);
|
|
||||||
let (tables, _) = self
|
|
||||||
.query_tables_page(
|
|
||||||
&format!("SHOW {}TABLES", quote_qualified_prefix(database)),
|
|
||||||
"TABLE",
|
|
||||||
true,
|
|
||||||
table_offset,
|
|
||||||
limit - stables.len(),
|
|
||||||
token,
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
stables.extend(tables);
|
|
||||||
self.enrich_table_parents(database, &mut stables, token).await?;
|
|
||||||
Ok(stables)
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn enrich_table_parents(
|
async fn enrich_table_parents(
|
||||||
&self,
|
&self,
|
||||||
database: &str,
|
database: &str,
|
||||||
@@ -603,33 +567,6 @@ impl TdengineDriver {
|
|||||||
Ok(rows.into_iter().filter_map(|row| table_from_show_row(row, table_type, includes_stable_name)).collect())
|
Ok(rows.into_iter().filter_map(|row| table_from_show_row(row, table_type, includes_stable_name)).collect())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn query_tables_page(
|
|
||||||
&self,
|
|
||||||
sql: &str,
|
|
||||||
table_type: &str,
|
|
||||||
includes_stable_name: bool,
|
|
||||||
offset: usize,
|
|
||||||
limit: usize,
|
|
||||||
token: &CancellationToken,
|
|
||||||
) -> Result<(Vec<TableInfo>, usize)> {
|
|
||||||
let mut cursor = self.start_cursor(sql, offset.saturating_add(limit), token, 0).await?;
|
|
||||||
let mut scanned = 0usize;
|
|
||||||
let mut result = Vec::with_capacity(limit);
|
|
||||||
while result.len() < limit {
|
|
||||||
let Some(row) = cursor.next_row(token, 0).await? else {
|
|
||||||
break;
|
|
||||||
};
|
|
||||||
scanned += 1;
|
|
||||||
if scanned <= offset {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if let Some(table) = table_from_show_row(row, table_type, includes_stable_name) {
|
|
||||||
result.push(table);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok((result, scanned))
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn get_create_sql(
|
async fn get_create_sql(
|
||||||
&self,
|
&self,
|
||||||
database: &str,
|
database: &str,
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
use std::collections::HashSet;
|
||||||
use std::env;
|
use std::env;
|
||||||
use std::io::{BufRead, BufReader, Write};
|
use std::io::{BufRead, BufReader, Write};
|
||||||
use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio};
|
use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio};
|
||||||
@@ -222,3 +223,83 @@ fn tdengine_websocket_live_compatibility() {
|
|||||||
);
|
);
|
||||||
agent.shutdown();
|
agent.shutdown();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// `SHOW <db>.TABLES` has no `ORDER BY`, so TDengine may answer two identical
|
||||||
|
/// statements with different row orders. A table list must still be identical
|
||||||
|
/// whichever page size the caller asks for: paging used to skip and repeat rows,
|
||||||
|
/// which hid tables (including freshly created normal tables) from the sidebar.
|
||||||
|
#[test]
|
||||||
|
fn tdengine_websocket_live_paged_table_list_matches_the_full_list() {
|
||||||
|
let Some(connect_params) = integration_params() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let database = format!("dbx_rust_page_{}", std::process::id());
|
||||||
|
let session_id = "tdengine-live-page";
|
||||||
|
let page_size = 100usize;
|
||||||
|
let mut agent = AgentProcess::spawn();
|
||||||
|
|
||||||
|
let mut open_params = connect_params.clone();
|
||||||
|
open_params["agentSessionId"] = json!(session_id);
|
||||||
|
assert_eq!(agent.call("open_session", open_params), json!({"ok": true}));
|
||||||
|
|
||||||
|
let mut statements = vec![
|
||||||
|
format!("CREATE DATABASE IF NOT EXISTS {database}"),
|
||||||
|
format!("CREATE STABLE IF NOT EXISTS {database}.st_paged (ts TIMESTAMP, v INT) TAGS (gid INT)"),
|
||||||
|
];
|
||||||
|
// Enough child tables that the sidebar's paging walks several pages.
|
||||||
|
for index in 0..300 {
|
||||||
|
statements.push(format!(
|
||||||
|
"CREATE TABLE IF NOT EXISTS {database}.c_{index:04} USING {database}.st_paged TAGS ({index})"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
statements.push(format!("CREATE TABLE IF NOT EXISTS {database}.normal_tbl (ts TIMESTAMP, v INT)"));
|
||||||
|
agent.call("execute_batch", json!({"agentSessionId": session_id, "statements": statements}));
|
||||||
|
|
||||||
|
fn page(agent: &mut AgentProcess, session_id: &str, database: &str, limit: usize, offset: usize) -> Value {
|
||||||
|
agent.call(
|
||||||
|
"list_tables",
|
||||||
|
json!({
|
||||||
|
"agentSessionId": session_id,
|
||||||
|
"schema": database,
|
||||||
|
"filter": "",
|
||||||
|
"limit": limit,
|
||||||
|
"offset": offset,
|
||||||
|
"object_types": []
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
fn names(tables: &Value) -> HashSet<(String, String)> {
|
||||||
|
tables
|
||||||
|
.as_array()
|
||||||
|
.expect("table list")
|
||||||
|
.iter()
|
||||||
|
.map(|table| {
|
||||||
|
(table["name"].as_str().unwrap_or_default().to_string(), table["table_type"].as_str().unwrap_or_default().to_string())
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
let full = names(&page(&mut agent, session_id, &database, 0, 0));
|
||||||
|
assert!(full.contains(&("normal_tbl".to_string(), "TABLE".to_string())));
|
||||||
|
|
||||||
|
let mut seen = HashSet::new();
|
||||||
|
let mut offset = 0usize;
|
||||||
|
loop {
|
||||||
|
let tables = page(&mut agent, session_id, &database, page_size, offset);
|
||||||
|
let count = tables.as_array().expect("table page").len();
|
||||||
|
if count == 0 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
for table in names(&tables) {
|
||||||
|
assert!(seen.insert(table.clone()), "table {table:?} was returned twice while paging at offset {offset}");
|
||||||
|
}
|
||||||
|
if count < page_size {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
offset += page_size;
|
||||||
|
}
|
||||||
|
assert_eq!(seen, full, "paging dropped or invented tables");
|
||||||
|
|
||||||
|
agent.call("execute_batch", json!({"agentSessionId": session_id, "statements": [format!("DROP DATABASE IF EXISTS {database}")]}));
|
||||||
|
agent.shutdown();
|
||||||
|
}
|
||||||
Reference in new issue
Block a user