fix(grid): bound large table previews by byte budget

This commit is contained in:
zipg authored and GitHub committed 2026-08-13 13:25:04 +08:00
1 parent 020654c33c
commit e85d3dcc59
19 files changed
+823 -23

No files matched your search

+442 -5
View File
@@ -266,15 +266,43 @@ impl ExecuteMultiResult {
}
fn success_with_index_and_large_values(
mut result: db::QueryResult,
statement_index: usize,
mut large_value_cells: Vec<db::LargeValueCell>,
table_data_preview: bool,
) -> Self {
let server_cells = if table_data_preview {
remap_large_value_cells_around_server_markers(&result, &mut large_value_cells);
extract_server_large_value_markers(&mut result)
} else {
Vec::new()
};
Self {
result,
large_value_cells: merge_large_value_cells(large_value_cells, server_cells),
execution_error: false,
statement_index: Some(statement_index),
error: None,
server_message: false,
}
}
fn success_with_index_and_optional_server_large_values(
result: db::QueryResult,
statement_index: usize,
large_value_cells: Vec<db::LargeValueCell>,
table_data_preview: bool,
) -> Self {
Self::success_with_index_and_large_values(result, statement_index, Vec::new(), table_data_preview)
}
fn success_with_optional_server_large_values(mut result: db::QueryResult, table_data_preview: bool) -> Self {
let large_value_cells =
if table_data_preview { extract_server_large_value_markers(&mut result) } else { Vec::new() };
Self {
result,
large_value_cells,
execution_error: false,
statement_index: Some(statement_index),
statement_index: None,
error: None,
server_message: false,
}
@@ -290,6 +318,228 @@ impl ExecuteMultiResult {
}
}
const SERVER_LARGE_VALUE_UNKNOWN_BYTES: usize = 64 * 1024 * 1024;
#[derive(Clone, Copy, PartialEq, Eq)]
enum ServerLargeValuePreviewKind {
Text,
Binary,
Vector,
}
#[derive(Clone, Copy)]
struct ServerLargeValueMarker {
result_index: usize,
source_index: usize,
preview_kind: Option<ServerLargeValuePreviewKind>,
source_type: Option<&'static str>,
}
fn server_large_value_alias(
suffix: &str,
) -> Option<(usize, Option<ServerLargeValuePreviewKind>, Option<&'static str>)> {
if let Ok(source_index) = suffix.parse::<usize>() {
return Some((source_index, None, None));
}
let (kind, source_index) = suffix.split_once('_')?;
let source_index = source_index.parse::<usize>().ok()?;
let (preview_kind, source_type) = match kind {
"T" => (ServerLargeValuePreviewKind::Text, None),
"B" => (ServerLargeValuePreviewKind::Binary, None),
"V" => (ServerLargeValuePreviewKind::Vector, Some("vector")),
"J" => (ServerLargeValuePreviewKind::Text, Some("json")),
"K" => (ServerLargeValuePreviewKind::Text, Some("jsonb")),
"S" => (ServerLargeValuePreviewKind::Text, Some("tsvector")),
_ => return None,
};
Some((source_index, Some(preview_kind), source_type))
}
fn server_large_value_marker(value: &serde_json::Value) -> Option<(ServerLargeValuePreviewKind, usize)> {
let (kind, preview_size) = value.as_str()?.split_once(':')?;
let kind = match kind {
"T" => ServerLargeValuePreviewKind::Text,
"B" => ServerLargeValuePreviewKind::Binary,
"V" => ServerLargeValuePreviewKind::Vector,
_ => return None,
};
Some((kind, preview_size.parse::<usize>().ok()?.max(1)))
}
fn truncate_server_large_value_preview(
value: &mut serde_json::Value,
kind: ServerLargeValuePreviewKind,
preview_size: usize,
) -> bool {
let serde_json::Value::String(text) = value else {
return false;
};
if matches!(kind, ServerLargeValuePreviewKind::Vector) {
let truncated = text.chars().count() > preview_size;
let vector_text = if truncated {
let prefix: String = text.chars().take(preview_size).collect();
let Some(last_separator) = prefix.rfind(',') else {
return false;
};
format!("{}]", &prefix[..last_separator])
} else {
text.clone()
};
let Ok(vector) = serde_json::from_str::<Vec<serde_json::Value>>(&vector_text) else {
return false;
};
*value = serde_json::Value::Array(vector);
return truncated;
}
let truncate_at = match kind {
ServerLargeValuePreviewKind::Text => text.char_indices().nth(preview_size).map(|(index, _)| index),
ServerLargeValuePreviewKind::Binary => text
.strip_prefix("0x")
.filter(|hex| hex.len() > preview_size.saturating_mul(2))
.map(|_| 2usize.saturating_add(preview_size.saturating_mul(2))),
ServerLargeValuePreviewKind::Vector => unreachable!(),
};
let Some(truncate_at) = truncate_at else {
return false;
};
text.truncate(truncate_at);
text.push_str("...");
true
}
fn server_large_value_markers(result: &db::QueryResult) -> Vec<ServerLargeValueMarker> {
let mut markers = Vec::new();
for (result_index, column) in result.columns.iter().enumerate() {
let Some((source_index, preview_kind, source_type)) = column
.strip_prefix(crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX)
.and_then(server_large_value_alias)
else {
continue;
};
let expected_source_index = result_index.checked_sub(markers.len() + 1);
if expected_source_index == Some(source_index) {
markers.push(ServerLargeValueMarker { result_index, source_index, preview_kind, source_type });
}
}
markers
}
fn extract_server_large_value_markers(result: &mut db::QueryResult) -> Vec<db::LargeValueCell> {
let markers = server_large_value_markers(result);
if markers.is_empty() {
return Vec::new();
}
let mut large_value_cells = Vec::new();
for marker in &markers {
let source_result_index = marker.result_index.saturating_sub(1);
let row_kind = result
.rows
.iter()
.find_map(|row| row.get(marker.result_index).and_then(server_large_value_marker).map(|value| value.0));
let source_type = marker.source_type.or_else(|| {
(marker.preview_kind.or(row_kind) == Some(ServerLargeValuePreviewKind::Vector)).then_some("vector")
});
if let (Some(source_type), Some(column_type)) = (source_type, result.column_types.get_mut(source_result_index))
{
*column_type = source_type.to_string();
}
}
for (row_index, row) in result.rows.iter_mut().enumerate() {
for marker in &markers {
let marker_value = row.get(marker.result_index).and_then(server_large_value_marker);
let source_result_index = marker.result_index.saturating_sub(1);
if marker_value.is_some_and(|(kind, preview_size)| {
row.get_mut(source_result_index)
.is_some_and(|value| truncate_server_large_value_preview(value, kind, preview_size))
}) {
large_value_cells.push(db::LargeValueCell {
row_index,
column_index: marker.source_index,
original_bytes: SERVER_LARGE_VALUE_UNKNOWN_BYTES,
});
}
}
}
let removed: std::collections::HashSet<usize> = markers.iter().map(|marker| marker.result_index).collect();
let retained_index = |index: usize| index - removed.iter().filter(|removed_index| **removed_index < index).count();
result.spatial_columns.retain_mut(|column| {
if removed.contains(&column.column_index) {
return false;
}
column.column_index = retained_index(column.column_index);
true
});
for row in &mut result.rows {
let mut index = 0;
row.retain(|_| {
let keep = !removed.contains(&index);
index += 1;
keep
});
}
for row in &mut result.spatial_values {
let mut index = 0;
row.retain(|_| {
let keep = !removed.contains(&index);
index += 1;
keep
});
}
let mut index = 0;
result.columns.retain(|_| {
let keep = !removed.contains(&index);
index += 1;
keep
});
let mut index = 0;
result.column_types.retain(|_| {
let keep = !removed.contains(&index);
index += 1;
keep
});
let mut index = 0;
result.column_sortables.retain(|_| {
let keep = !removed.contains(&index);
index += 1;
keep
});
large_value_cells
}
fn remap_large_value_cells_around_server_markers(result: &db::QueryResult, cells: &mut Vec<db::LargeValueCell>) {
let removed: std::collections::HashSet<usize> =
server_large_value_markers(result).into_iter().map(|marker| marker.result_index).collect();
if removed.is_empty() {
return;
}
cells.retain_mut(|cell| {
if removed.contains(&cell.column_index) {
return false;
}
cell.column_index -= removed.iter().filter(|index| **index < cell.column_index).count();
true
});
}
fn merge_large_value_cells(
mut driver_cells: Vec<db::LargeValueCell>,
server_cells: Vec<db::LargeValueCell>,
) -> Vec<db::LargeValueCell> {
for server_cell in server_cells {
if let Some(driver_cell) = driver_cells
.iter_mut()
.find(|cell| cell.row_index == server_cell.row_index && cell.column_index == server_cell.column_index)
{
*driver_cell = server_cell;
} else {
driver_cells.push(server_cell);
}
}
driver_cells
}
impl From<db::QueryResult> for ExecuteMultiResult {
fn from(result: db::QueryResult) -> Self {
Self {
@@ -639,6 +889,9 @@ pub struct QueryExecutionOptions {
/// Result columns that must stay exact because clients use them as stable
/// row identifiers when fetching full large-cell values on demand.
pub result_key_columns: Vec<String>,
/// Enables extraction of hidden server-side preview metadata. This must be
/// set only for generated table-data preview SQL, never arbitrary queries.
pub table_data_preview: bool,
/// Doris / StarRocks catalog selected for this query tab.
pub catalog: Option<String>,
pub result_session_id: Option<String>,
@@ -2252,6 +2505,7 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed(
// HTTP SQLite providers send all statements in one request so the provider
// can preserve batch ordering and atomicity.
if is_http_sqlite {
let table_data_preview = options.table_data_preview;
return single_statement_multi_result(
execute_sql_statement_with_options_typed(
state,
@@ -2263,6 +2517,7 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed(
options,
)
.await,
table_data_preview,
);
}
@@ -2306,6 +2561,7 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed(
)
{
let single_sql = statements.into_iter().next().unwrap_or_default();
let table_data_preview = options.table_data_preview;
return single_statement_multi_result(
execute_sql_statement_with_options_typed(
state,
@@ -2317,6 +2573,7 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed(
options,
)
.await,
table_data_preview,
);
}
@@ -2372,7 +2629,11 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed(
{
Ok(r) => {
report_execute_multi_progress(progress.as_ref(), statement_index, statements.len(), &r, true, None);
results.push(ExecuteMultiResult::success_with_index(r, statement_index));
results.push(ExecuteMultiResult::success_with_index_and_optional_server_large_values(
r,
statement_index,
options.table_data_preview,
));
}
Err(error) => {
let action = query_execution_error_action(db_type, stmt, &error);
@@ -2403,8 +2664,9 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed(
fn single_statement_multi_result(
result: Result<db::QueryResult, QueryExecutionError>,
table_data_preview: bool,
) -> Result<Vec<ExecuteMultiResult>, QueryExecutionError> {
result.map(|result| vec![result.into()])
result.map(|result| vec![ExecuteMultiResult::success_with_optional_server_large_values(result, table_data_preview)])
}
fn mysql_single_statement_uses_batch_route(
@@ -2420,6 +2682,10 @@ fn mysql_single_statement_uses_batch_route(
}
trait MysqlBatchStatementExecutor {
fn table_data_preview(&self) -> bool {
false
}
async fn execute_statement(&mut self, statement: &str) -> Result<Vec<db::mysql::MySqlQueryResult>, String>;
async fn execute_non_result_batch(&mut self, statements: &[String]) -> db::mysql::MySqlNonResultBatchOutcome {
@@ -2450,10 +2716,15 @@ struct MysqlBatchConnection<'a> {
max_rows: Option<usize>,
max_result_bytes: Option<usize>,
result_key_columns: &'a [String],
table_data_preview: bool,
dialect: db::mysql::MySqlQueryDialect,
}
impl MysqlBatchStatementExecutor for MysqlBatchConnection<'_> {
fn table_data_preview(&self) -> bool {
self.table_data_preview
}
async fn execute_statement(&mut self, statement: &str) -> Result<Vec<db::mysql::MySqlQueryResult>, String> {
wait_for_result_opt(
self.cancel_token.clone(),
@@ -2553,6 +2824,7 @@ where
{
let mut results = Vec::with_capacity(statements.len());
let mut statement_index = 0usize;
let table_data_preview = executor.table_data_preview();
while statement_index < statements.len() {
if is_canceled(&cancel_token) {
results.push(ExecuteMultiResult::execution_error(error_query_result(canceled_error())));
@@ -2618,6 +2890,7 @@ where
result.result,
statement_index,
result.large_value_cells,
table_data_preview,
)
}));
}
@@ -2711,6 +2984,7 @@ async fn execute_multi_mysql(
max_rows,
max_result_bytes,
result_key_columns: &options.result_key_columns,
table_data_preview: options.table_data_preview,
dialect,
};
let (results, error_action) = execute_mysql_batch_statements(
@@ -5766,6 +6040,7 @@ for line in sys.stdin:
empty_query_result(1),
0,
vec![db::LargeValueCell { row_index: 2, column_index: 3, original_bytes: 65_536 }],
false,
);
let serialized = serde_json::to_value(result).unwrap();
@@ -5901,7 +6176,7 @@ for line in sys.stdin:
)
.with_omitted_sql_context("SELECT * FROM dbx_table_that_does_not_exist");
let error = single_statement_multi_result(Err(error)).unwrap_err();
let error = single_statement_multi_result(Err(error), false).unwrap_err();
let backend_error = error.into_backend_error();
assert_eq!(backend_error.code(), "DBX-JDBC-4001");
@@ -6900,6 +7175,168 @@ for line in sys.stdin:
assert_eq!(normalized.rows[0][1], serde_json::json!([1, "2041797190226354178"]));
}
#[test]
fn extracts_server_large_value_markers_and_restores_source_columns() {
let mut result = db::QueryResult {
columns: vec![
"id".to_string(),
"payload".to_string(),
format!("{}T_1", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX),
"note".to_string(),
],
column_types: vec!["integer".to_string(), "text".to_string(), "bigint".to_string(), "text".to_string()],
column_sortables: vec![true, true, true, true],
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![
vec![
serde_json::json!(1),
serde_json::json!("预览文本多"),
serde_json::json!("T:4"),
serde_json::json!("a"),
],
vec![serde_json::json!(2), serde_json::json!("短值"), serde_json::json!("T:4"), serde_json::json!("b")],
],
affected_rows: 0,
execution_time_ms: 0,
truncated: false,
session_id: None,
has_more: false,
elasticsearch_raw_body: None,
messages: Vec::new(),
};
let cells = extract_server_large_value_markers(&mut result);
assert_eq!(result.columns, vec!["id", "payload", "note"]);
assert_eq!(result.column_types, vec!["integer", "text", "text"]);
assert_eq!(
result.rows[0],
vec![serde_json::json!(1), serde_json::json!("预览文本..."), serde_json::json!("a")]
);
assert_eq!(result.rows[1], vec![serde_json::json!(2), serde_json::json!("短值"), serde_json::json!("b")]);
assert_eq!(
cells,
vec![db::LargeValueCell {
row_index: 0,
column_index: 1,
original_bytes: SERVER_LARGE_VALUE_UNKNOWN_BYTES,
}]
);
}
#[test]
fn truncates_server_binary_preview_after_the_configured_byte_count() {
let mut result = db::QueryResult {
columns: vec![
"raw_value".to_string(),
format!("{}B_0", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX),
],
column_types: vec!["bytea".to_string(), "text".to_string()],
column_sortables: vec![true, true],
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![serde_json::json!("0x0102030405"), serde_json::json!("B:4")]],
affected_rows: 0,
execution_time_ms: 0,
truncated: false,
session_id: None,
has_more: false,
elasticsearch_raw_body: None,
messages: Vec::new(),
};
let cells = extract_server_large_value_markers(&mut result);
assert_eq!(result.rows, vec![vec![serde_json::json!("0x01020304...")]]);
assert_eq!(cells.len(), 1);
}
#[test]
fn restores_pgvector_previews_as_arrays_and_marks_only_truncated_values() {
let marker = format!("{}V_0", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX);
let mut result = db::QueryResult {
columns: vec!["embedding".to_string(), marker],
column_types: vec!["text".to_string(), "text".to_string()],
column_sortables: vec![true, true],
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![
vec![serde_json::json!("[0.1,0.2,0.3]"), serde_json::json!("V:9")],
vec![serde_json::json!("[0.1,0.2]"), serde_json::json!("V:20")],
],
affected_rows: 0,
execution_time_ms: 0,
truncated: false,
session_id: None,
has_more: false,
elasticsearch_raw_body: None,
messages: Vec::new(),
};
let cells = extract_server_large_value_markers(&mut result);
assert_eq!(result.column_types, vec!["vector"]);
assert_eq!(result.rows[0][0], serde_json::json!([0.1, 0.2]));
assert_eq!(result.rows[1][0], serde_json::json!([0.1, 0.2]));
assert_eq!(cells.len(), 1);
assert_eq!(cells[0].row_index, 0);
}
#[test]
fn restores_server_preview_types_without_rows() {
let mut result = db::QueryResult {
columns: vec![
"embedding".to_string(),
format!("{}V_0", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX),
"metadata".to_string(),
format!("{}K_1", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX),
],
column_types: vec!["text".to_string(), "text".to_string(), "text".to_string(), "text".to_string()],
column_sortables: vec![true; 4],
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: Vec::new(),
affected_rows: 0,
execution_time_ms: 0,
truncated: false,
session_id: None,
has_more: false,
elasticsearch_raw_body: None,
messages: Vec::new(),
};
let cells = extract_server_large_value_markers(&mut result);
assert_eq!(result.columns, vec!["embedding", "metadata"]);
assert_eq!(result.column_types, vec!["vector", "jsonb"]);
assert!(cells.is_empty());
}
#[test]
fn ordinary_queries_preserve_columns_that_resemble_preview_markers() {
let marker = format!("{}0", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX);
let mut result = empty_query_result(1);
result.columns = vec!["payload".to_string(), marker.clone()];
result.column_types = vec!["text".to_string(), "bigint".to_string()];
result.column_sortables = vec![true, true];
result.rows = vec![vec![serde_json::json!("value"), serde_json::json!(123)]];
let ordinary = ExecuteMultiResult::success_with_index_and_optional_server_large_values(result, 0, false);
assert_eq!(ordinary.result.columns, vec!["payload".to_string(), marker]);
assert_eq!(ordinary.result.rows[0], vec![serde_json::json!("value"), serde_json::json!(123)]);
assert!(ordinary.large_value_cells.is_empty());
}
#[test]
fn single_statement_preview_preserves_absent_statement_index() {
let result = single_statement_multi_result(Ok(empty_query_result(1)), true).unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].statement_index, None);
}
#[test]
fn query_execution_options_default_use_transaction_is_none() {
let opts = QueryExecutionOptions::default();
+3 -1
View File
@@ -38,7 +38,9 @@ pub use identifiers::{
normalize_where_input, qualified_table_name, qualified_table_name_with_catalog, quote_table_identifier,
};
pub(crate) use identifiers::{parse_sqlserver_linked_schema_ref, qualified_transfer_table, quote_transfer_identifier};
pub use table_select::{build_count_table_sql, build_table_data_select_sql, build_table_select_sql};
pub use table_select::{
build_count_table_sql, build_table_data_select_sql, build_table_select_sql, DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX,
};
pub(crate) use table_select::{
quote_table_data_identifier, table_data_qualified_table_name, table_data_schema, uses_connection_identifier_quote,
};
@@ -12,6 +12,131 @@ use super::types::{
DBX_TDENGINE_TBNAME_COLUMN,
};
pub const DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX: &str = "__DBX_LARGE_VALUE_BYTES_";
#[derive(Clone, Copy, PartialEq, Eq)]
enum LargeValuePreviewKind {
Text,
Binary,
TextCast,
Vector,
}
fn large_value_marker_alias_kind(kind: LargeValuePreviewKind, data_type: &str) -> &'static str {
match kind {
LargeValuePreviewKind::Binary => "B",
LargeValuePreviewKind::Vector => "V",
LargeValuePreviewKind::TextCast => match normalized_data_type_base(data_type).as_str() {
"json" => "J",
"jsonb" => "K",
"tsvector" => "S",
_ => "T",
},
LargeValuePreviewKind::Text => "T",
}
}
fn normalized_data_type_base(data_type: &str) -> String {
data_type.trim().split(['(', '[']).next().unwrap_or_default().trim().to_ascii_lowercase()
}
fn large_value_preview_kind(database_type: Option<DatabaseType>, data_type: &str) -> Option<LargeValuePreviewKind> {
let normalized = data_type.trim().to_ascii_lowercase();
let base = normalized_data_type_base(data_type);
match database_type {
Some(DatabaseType::Mysql) => {
if matches!(base.as_str(), "binary" | "varbinary" | "tinyblob" | "blob" | "mediumblob" | "longblob") {
Some(LargeValuePreviewKind::Binary)
} else if base == "json" {
Some(LargeValuePreviewKind::TextCast)
} else if matches!(base.as_str(), "char" | "varchar" | "tinytext" | "text" | "mediumtext" | "longtext") {
Some(LargeValuePreviewKind::Text)
} else {
None
}
}
Some(DatabaseType::Postgres) => {
if base == "bytea" {
Some(LargeValuePreviewKind::Binary)
} else if matches!(base.as_str(), "char" | "character" | "varchar" | "text" | "citext" | "name" | "xml")
|| normalized.starts_with("character varying")
{
Some(LargeValuePreviewKind::Text)
} else if base == "vector" {
Some(LargeValuePreviewKind::Vector)
} else if matches!(base.as_str(), "json" | "jsonb" | "tsvector") {
Some(LargeValuePreviewKind::TextCast)
} else {
None
}
}
_ => None,
}
}
fn build_large_value_preview_columns(options: &TableDataSelectSqlOptions) -> Option<String> {
let database_type = options.database_type;
let preview_size = options.large_value_preview_size?.max(1);
if options.columns.is_empty()
|| options.columns.len() != options.column_types.len()
|| options.primary_keys.is_empty()
|| options
.columns
.iter()
.any(|column| column.to_ascii_uppercase().starts_with(DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX))
{
return None;
}
let protected: std::collections::HashSet<String> =
options.primary_keys.iter().map(|column| column.to_ascii_lowercase()).collect();
let mut projections = Vec::with_capacity(options.columns.len() * 2);
let mut marker_count = 0;
for (column_index, (column, data_type)) in options.columns.iter().zip(&options.column_types).enumerate() {
let quoted = if uses_connection_identifier_quote(database_type, options.identifier_quote.as_deref()) {
quote_table_data_identifier(database_type, column, options.identifier_quote.as_deref())
} else {
quote_table_identifier(database_type, column)
};
let kind = (!protected.contains(&column.to_ascii_lowercase()))
.then(|| large_value_preview_kind(database_type, data_type))
.flatten();
let Some(kind) = kind else {
projections.push(quoted);
continue;
};
let alias_kind = large_value_marker_alias_kind(kind, data_type);
let marker_alias = quote_table_identifier(
database_type,
&format!("{DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX}{alias_kind}_{column_index}"),
);
let prefix_size = preview_size.saturating_add(1);
let (preview, marker_kind) = match database_type {
Some(DatabaseType::Mysql) if kind == LargeValuePreviewKind::Binary => {
(format!("LEFT({quoted}, {prefix_size}) AS {quoted}"), "B")
}
Some(DatabaseType::Mysql) => (format!("LEFT({quoted}, {prefix_size}) AS {quoted}"), "T"),
Some(DatabaseType::Postgres) if kind == LargeValuePreviewKind::Binary => {
(format!("substring({quoted} from 1 for {prefix_size}) AS {quoted}"), "B")
}
Some(DatabaseType::Postgres) if kind == LargeValuePreviewKind::TextCast => {
(format!("left({quoted}::text, {prefix_size}) AS {quoted}"), "T")
}
Some(DatabaseType::Postgres) if kind == LargeValuePreviewKind::Vector => {
(format!("left({quoted}::text, {prefix_size}) AS {quoted}"), "V")
}
Some(DatabaseType::Postgres) => (format!("left({quoted}, {prefix_size}) AS {quoted}"), "T"),
_ => return None,
};
let marker = format!("'{marker_kind}:{preview_size}' AS {marker_alias}");
projections.push(preview);
projections.push(marker);
marker_count += 1;
}
(marker_count > 0).then(|| projections.join(", "))
}
pub fn build_count_table_sql(database_type: Option<DatabaseType>, schema: Option<&str>, table_name: &str) -> String {
if database_type == Some(DatabaseType::VictoriaMetrics) {
return format!("count({})", victoriametrics_metric_selector(table_name));
@@ -77,6 +202,8 @@ pub fn build_table_data_select_sql(options: TableDataSelectSqlOptions) -> String
let select_columns = if include_oracle_row_id {
format!("ROWIDTOCHAR(t.ROWID) AS \"{DBX_ROWID_COLUMN}\", t.*")
} else if let Some(preview_columns) = build_large_value_preview_columns(&options) {
preview_columns
} else {
build_select_columns(
database_type,
@@ -521,6 +648,8 @@ mod tests {
table_type: None,
primary_keys: Vec::new(),
columns: Vec::new(),
column_types: Vec::new(),
large_value_preview_size: None,
fallback_order_columns: Vec::new(),
order_by: None,
limit: Some(10),
+91
View File
@@ -641,6 +641,97 @@ fn builds_table_data_where_and_schema_queries() {
);
}
#[test]
fn builds_mysql_table_data_large_value_previews_without_truncating_keys() {
let sql = build_table_data_select_sql(TableDataSelectSqlOptions {
database_type: Some(DatabaseType::Mysql),
table_name: "large_rows".to_string(),
primary_keys: vec!["id".to_string()],
columns: vec!["id".to_string(), "payload".to_string(), "raw_value".to_string(), "metadata".to_string()],
column_types: vec!["bigint".to_string(), "longtext".to_string(), "longblob".to_string(), "json".to_string()],
large_value_preview_size: Some(4096),
limit: Some(100),
..Default::default()
});
assert!(sql.starts_with("SELECT `id`, LEFT(`payload`, 4097) AS `payload`"));
assert!(sql.contains("'T:4096' AS `__DBX_LARGE_VALUE_BYTES_T_1`"));
assert!(sql.contains("LEFT(`raw_value`, 4097) AS `raw_value`"));
assert!(sql.contains("'B:4096' AS `__DBX_LARGE_VALUE_BYTES_B_2`"));
assert!(sql.contains("LEFT(`metadata`, 4097) AS `metadata`"));
assert!(sql.contains("'T:4096' AS `__DBX_LARGE_VALUE_BYTES_J_3`"));
assert!(!sql.contains("__DBX_LARGE_VALUE_BYTES_0"));
}
#[test]
fn builds_postgres_table_data_large_value_previews() {
let sql = build_table_data_select_sql(TableDataSelectSqlOptions {
database_type: Some(DatabaseType::Postgres),
schema: Some("public".to_string()),
table_name: "large_rows".to_string(),
primary_keys: vec!["id".to_string()],
columns: vec!["id".to_string(), "payload".to_string(), "metadata".to_string()],
column_types: vec!["integer".to_string(), "text".to_string(), "jsonb".to_string()],
large_value_preview_size: Some(8192),
limit: Some(100),
..Default::default()
});
assert!(sql.starts_with("SELECT \"id\", left(\"payload\", 8193) AS \"payload\""));
assert!(sql.contains("'T:8192' AS \"__DBX_LARGE_VALUE_BYTES_T_1\""));
assert!(sql.contains("left(\"metadata\"::text, 8193) AS \"metadata\""));
assert!(sql.contains("'T:8192' AS \"__DBX_LARGE_VALUE_BYTES_K_2\""));
}
#[test]
fn builds_pgvector_table_data_preview_with_compatible_text_cast() {
let sql = build_table_data_select_sql(TableDataSelectSqlOptions {
database_type: Some(DatabaseType::Postgres),
schema: Some("public".to_string()),
table_name: "embeddings".to_string(),
primary_keys: vec!["id".to_string()],
columns: vec!["id".to_string(), "embedding".to_string()],
column_types: vec!["integer".to_string(), "vector(16000)".to_string()],
large_value_preview_size: Some(8192),
limit: Some(100),
..Default::default()
});
assert!(sql.contains("left(\"embedding\"::text, 8193) AS \"embedding\""));
assert!(sql.contains("'V:8192' AS \"__DBX_LARGE_VALUE_BYTES_V_1\""));
}
#[test]
fn table_data_preview_requires_stable_keys_and_parallel_types() {
let base = TableDataSelectSqlOptions {
database_type: Some(DatabaseType::Postgres),
table_name: "large_rows".to_string(),
columns: vec!["payload".to_string()],
column_types: vec!["text".to_string()],
large_value_preview_size: Some(8192),
limit: Some(100),
..Default::default()
};
assert_eq!(build_table_data_select_sql(base.clone()), "SELECT * FROM \"large_rows\" LIMIT 100;");
assert_eq!(
build_table_data_select_sql(TableDataSelectSqlOptions {
primary_keys: vec!["id".to_string()],
column_types: Vec::new(),
..base.clone()
}),
"SELECT * FROM \"large_rows\" LIMIT 100;"
);
assert_eq!(
build_table_data_select_sql(TableDataSelectSqlOptions {
primary_keys: vec!["id".to_string()],
columns: vec!["id".to_string(), "__dbx_large_value_bytes_t_0".to_string()],
column_types: vec!["integer".to_string(), "text".to_string()],
..base
}),
"SELECT * FROM \"large_rows\" LIMIT 100;"
);
}
#[test]
fn builds_informix_table_data_with_skip_first_pagination() {
assert_eq!(
+9
View File
@@ -44,6 +44,15 @@ pub struct TableDataSelectSqlOptions {
pub primary_keys: Vec<String>,
#[serde(default)]
pub columns: Vec<String>,
/// Database type names parallel to `columns`. Table-data callers can use
/// these to request bounded server-side projections for variable-length
/// values without changing arbitrary query execution.
#[serde(default)]
pub column_types: Vec<String>,
/// Maximum characters (text-like values) or bytes (binary values) kept in
/// a table-data preview. Omitted for exports and ordinary SQL queries.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub large_value_preview_size: Option<usize>,
#[serde(default)]
pub fallback_order_columns: Vec<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]