fix(grid): bound large table previews by byte budget
This commit is contained in:
19 files changed
+823
-23
No files matched your search
@@ -266,15 +266,43 @@ impl ExecuteMultiResult {
|
||||
}
|
||||
|
||||
fn success_with_index_and_large_values(
|
||||
mut result: db::QueryResult,
|
||||
statement_index: usize,
|
||||
mut large_value_cells: Vec<db::LargeValueCell>,
|
||||
table_data_preview: bool,
|
||||
) -> Self {
|
||||
let server_cells = if table_data_preview {
|
||||
remap_large_value_cells_around_server_markers(&result, &mut large_value_cells);
|
||||
extract_server_large_value_markers(&mut result)
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
Self {
|
||||
result,
|
||||
large_value_cells: merge_large_value_cells(large_value_cells, server_cells),
|
||||
execution_error: false,
|
||||
statement_index: Some(statement_index),
|
||||
error: None,
|
||||
server_message: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn success_with_index_and_optional_server_large_values(
|
||||
result: db::QueryResult,
|
||||
statement_index: usize,
|
||||
large_value_cells: Vec<db::LargeValueCell>,
|
||||
table_data_preview: bool,
|
||||
) -> Self {
|
||||
Self::success_with_index_and_large_values(result, statement_index, Vec::new(), table_data_preview)
|
||||
}
|
||||
|
||||
fn success_with_optional_server_large_values(mut result: db::QueryResult, table_data_preview: bool) -> Self {
|
||||
let large_value_cells =
|
||||
if table_data_preview { extract_server_large_value_markers(&mut result) } else { Vec::new() };
|
||||
Self {
|
||||
result,
|
||||
large_value_cells,
|
||||
execution_error: false,
|
||||
statement_index: Some(statement_index),
|
||||
statement_index: None,
|
||||
error: None,
|
||||
server_message: false,
|
||||
}
|
||||
@@ -290,6 +318,228 @@ impl ExecuteMultiResult {
|
||||
}
|
||||
}
|
||||
|
||||
const SERVER_LARGE_VALUE_UNKNOWN_BYTES: usize = 64 * 1024 * 1024;
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
enum ServerLargeValuePreviewKind {
|
||||
Text,
|
||||
Binary,
|
||||
Vector,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct ServerLargeValueMarker {
|
||||
result_index: usize,
|
||||
source_index: usize,
|
||||
preview_kind: Option<ServerLargeValuePreviewKind>,
|
||||
source_type: Option<&'static str>,
|
||||
}
|
||||
|
||||
fn server_large_value_alias(
|
||||
suffix: &str,
|
||||
) -> Option<(usize, Option<ServerLargeValuePreviewKind>, Option<&'static str>)> {
|
||||
if let Ok(source_index) = suffix.parse::<usize>() {
|
||||
return Some((source_index, None, None));
|
||||
}
|
||||
let (kind, source_index) = suffix.split_once('_')?;
|
||||
let source_index = source_index.parse::<usize>().ok()?;
|
||||
let (preview_kind, source_type) = match kind {
|
||||
"T" => (ServerLargeValuePreviewKind::Text, None),
|
||||
"B" => (ServerLargeValuePreviewKind::Binary, None),
|
||||
"V" => (ServerLargeValuePreviewKind::Vector, Some("vector")),
|
||||
"J" => (ServerLargeValuePreviewKind::Text, Some("json")),
|
||||
"K" => (ServerLargeValuePreviewKind::Text, Some("jsonb")),
|
||||
"S" => (ServerLargeValuePreviewKind::Text, Some("tsvector")),
|
||||
_ => return None,
|
||||
};
|
||||
Some((source_index, Some(preview_kind), source_type))
|
||||
}
|
||||
|
||||
fn server_large_value_marker(value: &serde_json::Value) -> Option<(ServerLargeValuePreviewKind, usize)> {
|
||||
let (kind, preview_size) = value.as_str()?.split_once(':')?;
|
||||
let kind = match kind {
|
||||
"T" => ServerLargeValuePreviewKind::Text,
|
||||
"B" => ServerLargeValuePreviewKind::Binary,
|
||||
"V" => ServerLargeValuePreviewKind::Vector,
|
||||
_ => return None,
|
||||
};
|
||||
Some((kind, preview_size.parse::<usize>().ok()?.max(1)))
|
||||
}
|
||||
|
||||
fn truncate_server_large_value_preview(
|
||||
value: &mut serde_json::Value,
|
||||
kind: ServerLargeValuePreviewKind,
|
||||
preview_size: usize,
|
||||
) -> bool {
|
||||
let serde_json::Value::String(text) = value else {
|
||||
return false;
|
||||
};
|
||||
if matches!(kind, ServerLargeValuePreviewKind::Vector) {
|
||||
let truncated = text.chars().count() > preview_size;
|
||||
let vector_text = if truncated {
|
||||
let prefix: String = text.chars().take(preview_size).collect();
|
||||
let Some(last_separator) = prefix.rfind(',') else {
|
||||
return false;
|
||||
};
|
||||
format!("{}]", &prefix[..last_separator])
|
||||
} else {
|
||||
text.clone()
|
||||
};
|
||||
let Ok(vector) = serde_json::from_str::<Vec<serde_json::Value>>(&vector_text) else {
|
||||
return false;
|
||||
};
|
||||
*value = serde_json::Value::Array(vector);
|
||||
return truncated;
|
||||
}
|
||||
let truncate_at = match kind {
|
||||
ServerLargeValuePreviewKind::Text => text.char_indices().nth(preview_size).map(|(index, _)| index),
|
||||
ServerLargeValuePreviewKind::Binary => text
|
||||
.strip_prefix("0x")
|
||||
.filter(|hex| hex.len() > preview_size.saturating_mul(2))
|
||||
.map(|_| 2usize.saturating_add(preview_size.saturating_mul(2))),
|
||||
ServerLargeValuePreviewKind::Vector => unreachable!(),
|
||||
};
|
||||
let Some(truncate_at) = truncate_at else {
|
||||
return false;
|
||||
};
|
||||
text.truncate(truncate_at);
|
||||
text.push_str("...");
|
||||
true
|
||||
}
|
||||
|
||||
fn server_large_value_markers(result: &db::QueryResult) -> Vec<ServerLargeValueMarker> {
|
||||
let mut markers = Vec::new();
|
||||
for (result_index, column) in result.columns.iter().enumerate() {
|
||||
let Some((source_index, preview_kind, source_type)) = column
|
||||
.strip_prefix(crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX)
|
||||
.and_then(server_large_value_alias)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
let expected_source_index = result_index.checked_sub(markers.len() + 1);
|
||||
if expected_source_index == Some(source_index) {
|
||||
markers.push(ServerLargeValueMarker { result_index, source_index, preview_kind, source_type });
|
||||
}
|
||||
}
|
||||
markers
|
||||
}
|
||||
|
||||
fn extract_server_large_value_markers(result: &mut db::QueryResult) -> Vec<db::LargeValueCell> {
|
||||
let markers = server_large_value_markers(result);
|
||||
if markers.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let mut large_value_cells = Vec::new();
|
||||
for marker in &markers {
|
||||
let source_result_index = marker.result_index.saturating_sub(1);
|
||||
let row_kind = result
|
||||
.rows
|
||||
.iter()
|
||||
.find_map(|row| row.get(marker.result_index).and_then(server_large_value_marker).map(|value| value.0));
|
||||
let source_type = marker.source_type.or_else(|| {
|
||||
(marker.preview_kind.or(row_kind) == Some(ServerLargeValuePreviewKind::Vector)).then_some("vector")
|
||||
});
|
||||
if let (Some(source_type), Some(column_type)) = (source_type, result.column_types.get_mut(source_result_index))
|
||||
{
|
||||
*column_type = source_type.to_string();
|
||||
}
|
||||
}
|
||||
for (row_index, row) in result.rows.iter_mut().enumerate() {
|
||||
for marker in &markers {
|
||||
let marker_value = row.get(marker.result_index).and_then(server_large_value_marker);
|
||||
let source_result_index = marker.result_index.saturating_sub(1);
|
||||
if marker_value.is_some_and(|(kind, preview_size)| {
|
||||
row.get_mut(source_result_index)
|
||||
.is_some_and(|value| truncate_server_large_value_preview(value, kind, preview_size))
|
||||
}) {
|
||||
large_value_cells.push(db::LargeValueCell {
|
||||
row_index,
|
||||
column_index: marker.source_index,
|
||||
original_bytes: SERVER_LARGE_VALUE_UNKNOWN_BYTES,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let removed: std::collections::HashSet<usize> = markers.iter().map(|marker| marker.result_index).collect();
|
||||
let retained_index = |index: usize| index - removed.iter().filter(|removed_index| **removed_index < index).count();
|
||||
result.spatial_columns.retain_mut(|column| {
|
||||
if removed.contains(&column.column_index) {
|
||||
return false;
|
||||
}
|
||||
column.column_index = retained_index(column.column_index);
|
||||
true
|
||||
});
|
||||
for row in &mut result.rows {
|
||||
let mut index = 0;
|
||||
row.retain(|_| {
|
||||
let keep = !removed.contains(&index);
|
||||
index += 1;
|
||||
keep
|
||||
});
|
||||
}
|
||||
for row in &mut result.spatial_values {
|
||||
let mut index = 0;
|
||||
row.retain(|_| {
|
||||
let keep = !removed.contains(&index);
|
||||
index += 1;
|
||||
keep
|
||||
});
|
||||
}
|
||||
let mut index = 0;
|
||||
result.columns.retain(|_| {
|
||||
let keep = !removed.contains(&index);
|
||||
index += 1;
|
||||
keep
|
||||
});
|
||||
let mut index = 0;
|
||||
result.column_types.retain(|_| {
|
||||
let keep = !removed.contains(&index);
|
||||
index += 1;
|
||||
keep
|
||||
});
|
||||
let mut index = 0;
|
||||
result.column_sortables.retain(|_| {
|
||||
let keep = !removed.contains(&index);
|
||||
index += 1;
|
||||
keep
|
||||
});
|
||||
large_value_cells
|
||||
}
|
||||
|
||||
fn remap_large_value_cells_around_server_markers(result: &db::QueryResult, cells: &mut Vec<db::LargeValueCell>) {
|
||||
let removed: std::collections::HashSet<usize> =
|
||||
server_large_value_markers(result).into_iter().map(|marker| marker.result_index).collect();
|
||||
if removed.is_empty() {
|
||||
return;
|
||||
}
|
||||
cells.retain_mut(|cell| {
|
||||
if removed.contains(&cell.column_index) {
|
||||
return false;
|
||||
}
|
||||
cell.column_index -= removed.iter().filter(|index| **index < cell.column_index).count();
|
||||
true
|
||||
});
|
||||
}
|
||||
|
||||
fn merge_large_value_cells(
|
||||
mut driver_cells: Vec<db::LargeValueCell>,
|
||||
server_cells: Vec<db::LargeValueCell>,
|
||||
) -> Vec<db::LargeValueCell> {
|
||||
for server_cell in server_cells {
|
||||
if let Some(driver_cell) = driver_cells
|
||||
.iter_mut()
|
||||
.find(|cell| cell.row_index == server_cell.row_index && cell.column_index == server_cell.column_index)
|
||||
{
|
||||
*driver_cell = server_cell;
|
||||
} else {
|
||||
driver_cells.push(server_cell);
|
||||
}
|
||||
}
|
||||
driver_cells
|
||||
}
|
||||
|
||||
impl From<db::QueryResult> for ExecuteMultiResult {
|
||||
fn from(result: db::QueryResult) -> Self {
|
||||
Self {
|
||||
@@ -639,6 +889,9 @@ pub struct QueryExecutionOptions {
|
||||
/// Result columns that must stay exact because clients use them as stable
|
||||
/// row identifiers when fetching full large-cell values on demand.
|
||||
pub result_key_columns: Vec<String>,
|
||||
/// Enables extraction of hidden server-side preview metadata. This must be
|
||||
/// set only for generated table-data preview SQL, never arbitrary queries.
|
||||
pub table_data_preview: bool,
|
||||
/// Doris / StarRocks catalog selected for this query tab.
|
||||
pub catalog: Option<String>,
|
||||
pub result_session_id: Option<String>,
|
||||
@@ -2252,6 +2505,7 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed(
|
||||
// HTTP SQLite providers send all statements in one request so the provider
|
||||
// can preserve batch ordering and atomicity.
|
||||
if is_http_sqlite {
|
||||
let table_data_preview = options.table_data_preview;
|
||||
return single_statement_multi_result(
|
||||
execute_sql_statement_with_options_typed(
|
||||
state,
|
||||
@@ -2263,6 +2517,7 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed(
|
||||
options,
|
||||
)
|
||||
.await,
|
||||
table_data_preview,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2306,6 +2561,7 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed(
|
||||
)
|
||||
{
|
||||
let single_sql = statements.into_iter().next().unwrap_or_default();
|
||||
let table_data_preview = options.table_data_preview;
|
||||
return single_statement_multi_result(
|
||||
execute_sql_statement_with_options_typed(
|
||||
state,
|
||||
@@ -2317,6 +2573,7 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed(
|
||||
options,
|
||||
)
|
||||
.await,
|
||||
table_data_preview,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2372,7 +2629,11 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed(
|
||||
{
|
||||
Ok(r) => {
|
||||
report_execute_multi_progress(progress.as_ref(), statement_index, statements.len(), &r, true, None);
|
||||
results.push(ExecuteMultiResult::success_with_index(r, statement_index));
|
||||
results.push(ExecuteMultiResult::success_with_index_and_optional_server_large_values(
|
||||
r,
|
||||
statement_index,
|
||||
options.table_data_preview,
|
||||
));
|
||||
}
|
||||
Err(error) => {
|
||||
let action = query_execution_error_action(db_type, stmt, &error);
|
||||
@@ -2403,8 +2664,9 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed(
|
||||
|
||||
fn single_statement_multi_result(
|
||||
result: Result<db::QueryResult, QueryExecutionError>,
|
||||
table_data_preview: bool,
|
||||
) -> Result<Vec<ExecuteMultiResult>, QueryExecutionError> {
|
||||
result.map(|result| vec![result.into()])
|
||||
result.map(|result| vec![ExecuteMultiResult::success_with_optional_server_large_values(result, table_data_preview)])
|
||||
}
|
||||
|
||||
fn mysql_single_statement_uses_batch_route(
|
||||
@@ -2420,6 +2682,10 @@ fn mysql_single_statement_uses_batch_route(
|
||||
}
|
||||
|
||||
trait MysqlBatchStatementExecutor {
|
||||
fn table_data_preview(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
async fn execute_statement(&mut self, statement: &str) -> Result<Vec<db::mysql::MySqlQueryResult>, String>;
|
||||
|
||||
async fn execute_non_result_batch(&mut self, statements: &[String]) -> db::mysql::MySqlNonResultBatchOutcome {
|
||||
@@ -2450,10 +2716,15 @@ struct MysqlBatchConnection<'a> {
|
||||
max_rows: Option<usize>,
|
||||
max_result_bytes: Option<usize>,
|
||||
result_key_columns: &'a [String],
|
||||
table_data_preview: bool,
|
||||
dialect: db::mysql::MySqlQueryDialect,
|
||||
}
|
||||
|
||||
impl MysqlBatchStatementExecutor for MysqlBatchConnection<'_> {
|
||||
fn table_data_preview(&self) -> bool {
|
||||
self.table_data_preview
|
||||
}
|
||||
|
||||
async fn execute_statement(&mut self, statement: &str) -> Result<Vec<db::mysql::MySqlQueryResult>, String> {
|
||||
wait_for_result_opt(
|
||||
self.cancel_token.clone(),
|
||||
@@ -2553,6 +2824,7 @@ where
|
||||
{
|
||||
let mut results = Vec::with_capacity(statements.len());
|
||||
let mut statement_index = 0usize;
|
||||
let table_data_preview = executor.table_data_preview();
|
||||
while statement_index < statements.len() {
|
||||
if is_canceled(&cancel_token) {
|
||||
results.push(ExecuteMultiResult::execution_error(error_query_result(canceled_error())));
|
||||
@@ -2618,6 +2890,7 @@ where
|
||||
result.result,
|
||||
statement_index,
|
||||
result.large_value_cells,
|
||||
table_data_preview,
|
||||
)
|
||||
}));
|
||||
}
|
||||
@@ -2711,6 +2984,7 @@ async fn execute_multi_mysql(
|
||||
max_rows,
|
||||
max_result_bytes,
|
||||
result_key_columns: &options.result_key_columns,
|
||||
table_data_preview: options.table_data_preview,
|
||||
dialect,
|
||||
};
|
||||
let (results, error_action) = execute_mysql_batch_statements(
|
||||
@@ -5766,6 +6040,7 @@ for line in sys.stdin:
|
||||
empty_query_result(1),
|
||||
0,
|
||||
vec![db::LargeValueCell { row_index: 2, column_index: 3, original_bytes: 65_536 }],
|
||||
false,
|
||||
);
|
||||
|
||||
let serialized = serde_json::to_value(result).unwrap();
|
||||
@@ -5901,7 +6176,7 @@ for line in sys.stdin:
|
||||
)
|
||||
.with_omitted_sql_context("SELECT * FROM dbx_table_that_does_not_exist");
|
||||
|
||||
let error = single_statement_multi_result(Err(error)).unwrap_err();
|
||||
let error = single_statement_multi_result(Err(error), false).unwrap_err();
|
||||
let backend_error = error.into_backend_error();
|
||||
|
||||
assert_eq!(backend_error.code(), "DBX-JDBC-4001");
|
||||
@@ -6900,6 +7175,168 @@ for line in sys.stdin:
|
||||
assert_eq!(normalized.rows[0][1], serde_json::json!([1, "2041797190226354178"]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_server_large_value_markers_and_restores_source_columns() {
|
||||
let mut result = db::QueryResult {
|
||||
columns: vec![
|
||||
"id".to_string(),
|
||||
"payload".to_string(),
|
||||
format!("{}T_1", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX),
|
||||
"note".to_string(),
|
||||
],
|
||||
column_types: vec!["integer".to_string(), "text".to_string(), "bigint".to_string(), "text".to_string()],
|
||||
column_sortables: vec![true, true, true, true],
|
||||
spatial_columns: Vec::new(),
|
||||
spatial_values: Vec::new(),
|
||||
rows: vec![
|
||||
vec![
|
||||
serde_json::json!(1),
|
||||
serde_json::json!("预览文本多"),
|
||||
serde_json::json!("T:4"),
|
||||
serde_json::json!("a"),
|
||||
],
|
||||
vec![serde_json::json!(2), serde_json::json!("短值"), serde_json::json!("T:4"), serde_json::json!("b")],
|
||||
],
|
||||
affected_rows: 0,
|
||||
execution_time_ms: 0,
|
||||
truncated: false,
|
||||
session_id: None,
|
||||
has_more: false,
|
||||
elasticsearch_raw_body: None,
|
||||
messages: Vec::new(),
|
||||
};
|
||||
|
||||
let cells = extract_server_large_value_markers(&mut result);
|
||||
|
||||
assert_eq!(result.columns, vec!["id", "payload", "note"]);
|
||||
assert_eq!(result.column_types, vec!["integer", "text", "text"]);
|
||||
assert_eq!(
|
||||
result.rows[0],
|
||||
vec![serde_json::json!(1), serde_json::json!("预览文本..."), serde_json::json!("a")]
|
||||
);
|
||||
assert_eq!(result.rows[1], vec![serde_json::json!(2), serde_json::json!("短值"), serde_json::json!("b")]);
|
||||
assert_eq!(
|
||||
cells,
|
||||
vec![db::LargeValueCell {
|
||||
row_index: 0,
|
||||
column_index: 1,
|
||||
original_bytes: SERVER_LARGE_VALUE_UNKNOWN_BYTES,
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncates_server_binary_preview_after_the_configured_byte_count() {
|
||||
let mut result = db::QueryResult {
|
||||
columns: vec![
|
||||
"raw_value".to_string(),
|
||||
format!("{}B_0", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX),
|
||||
],
|
||||
column_types: vec!["bytea".to_string(), "text".to_string()],
|
||||
column_sortables: vec![true, true],
|
||||
spatial_columns: Vec::new(),
|
||||
spatial_values: Vec::new(),
|
||||
rows: vec![vec![serde_json::json!("0x0102030405"), serde_json::json!("B:4")]],
|
||||
affected_rows: 0,
|
||||
execution_time_ms: 0,
|
||||
truncated: false,
|
||||
session_id: None,
|
||||
has_more: false,
|
||||
elasticsearch_raw_body: None,
|
||||
messages: Vec::new(),
|
||||
};
|
||||
|
||||
let cells = extract_server_large_value_markers(&mut result);
|
||||
|
||||
assert_eq!(result.rows, vec![vec![serde_json::json!("0x01020304...")]]);
|
||||
assert_eq!(cells.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restores_pgvector_previews_as_arrays_and_marks_only_truncated_values() {
|
||||
let marker = format!("{}V_0", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX);
|
||||
let mut result = db::QueryResult {
|
||||
columns: vec!["embedding".to_string(), marker],
|
||||
column_types: vec!["text".to_string(), "text".to_string()],
|
||||
column_sortables: vec![true, true],
|
||||
spatial_columns: Vec::new(),
|
||||
spatial_values: Vec::new(),
|
||||
rows: vec![
|
||||
vec![serde_json::json!("[0.1,0.2,0.3]"), serde_json::json!("V:9")],
|
||||
vec![serde_json::json!("[0.1,0.2]"), serde_json::json!("V:20")],
|
||||
],
|
||||
affected_rows: 0,
|
||||
execution_time_ms: 0,
|
||||
truncated: false,
|
||||
session_id: None,
|
||||
has_more: false,
|
||||
elasticsearch_raw_body: None,
|
||||
messages: Vec::new(),
|
||||
};
|
||||
|
||||
let cells = extract_server_large_value_markers(&mut result);
|
||||
|
||||
assert_eq!(result.column_types, vec!["vector"]);
|
||||
assert_eq!(result.rows[0][0], serde_json::json!([0.1, 0.2]));
|
||||
assert_eq!(result.rows[1][0], serde_json::json!([0.1, 0.2]));
|
||||
assert_eq!(cells.len(), 1);
|
||||
assert_eq!(cells[0].row_index, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn restores_server_preview_types_without_rows() {
|
||||
let mut result = db::QueryResult {
|
||||
columns: vec![
|
||||
"embedding".to_string(),
|
||||
format!("{}V_0", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX),
|
||||
"metadata".to_string(),
|
||||
format!("{}K_1", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX),
|
||||
],
|
||||
column_types: vec!["text".to_string(), "text".to_string(), "text".to_string(), "text".to_string()],
|
||||
column_sortables: vec![true; 4],
|
||||
spatial_columns: Vec::new(),
|
||||
spatial_values: Vec::new(),
|
||||
rows: Vec::new(),
|
||||
affected_rows: 0,
|
||||
execution_time_ms: 0,
|
||||
truncated: false,
|
||||
session_id: None,
|
||||
has_more: false,
|
||||
elasticsearch_raw_body: None,
|
||||
messages: Vec::new(),
|
||||
};
|
||||
|
||||
let cells = extract_server_large_value_markers(&mut result);
|
||||
|
||||
assert_eq!(result.columns, vec!["embedding", "metadata"]);
|
||||
assert_eq!(result.column_types, vec!["vector", "jsonb"]);
|
||||
assert!(cells.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ordinary_queries_preserve_columns_that_resemble_preview_markers() {
|
||||
let marker = format!("{}0", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX);
|
||||
let mut result = empty_query_result(1);
|
||||
result.columns = vec!["payload".to_string(), marker.clone()];
|
||||
result.column_types = vec!["text".to_string(), "bigint".to_string()];
|
||||
result.column_sortables = vec![true, true];
|
||||
result.rows = vec![vec![serde_json::json!("value"), serde_json::json!(123)]];
|
||||
|
||||
let ordinary = ExecuteMultiResult::success_with_index_and_optional_server_large_values(result, 0, false);
|
||||
|
||||
assert_eq!(ordinary.result.columns, vec!["payload".to_string(), marker]);
|
||||
assert_eq!(ordinary.result.rows[0], vec![serde_json::json!("value"), serde_json::json!(123)]);
|
||||
assert!(ordinary.large_value_cells.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_statement_preview_preserves_absent_statement_index() {
|
||||
let result = single_statement_multi_result(Ok(empty_query_result(1)), true).unwrap();
|
||||
|
||||
assert_eq!(result.len(), 1);
|
||||
assert_eq!(result[0].statement_index, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn query_execution_options_default_use_transaction_is_none() {
|
||||
let opts = QueryExecutionOptions::default();
|
||||
|
||||
@@ -38,7 +38,9 @@ pub use identifiers::{
|
||||
normalize_where_input, qualified_table_name, qualified_table_name_with_catalog, quote_table_identifier,
|
||||
};
|
||||
pub(crate) use identifiers::{parse_sqlserver_linked_schema_ref, qualified_transfer_table, quote_transfer_identifier};
|
||||
pub use table_select::{build_count_table_sql, build_table_data_select_sql, build_table_select_sql};
|
||||
pub use table_select::{
|
||||
build_count_table_sql, build_table_data_select_sql, build_table_select_sql, DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX,
|
||||
};
|
||||
pub(crate) use table_select::{
|
||||
quote_table_data_identifier, table_data_qualified_table_name, table_data_schema, uses_connection_identifier_quote,
|
||||
};
|
||||
|
||||
@@ -12,6 +12,131 @@ use super::types::{
|
||||
DBX_TDENGINE_TBNAME_COLUMN,
|
||||
};
|
||||
|
||||
pub const DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX: &str = "__DBX_LARGE_VALUE_BYTES_";
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
enum LargeValuePreviewKind {
|
||||
Text,
|
||||
Binary,
|
||||
TextCast,
|
||||
Vector,
|
||||
}
|
||||
|
||||
fn large_value_marker_alias_kind(kind: LargeValuePreviewKind, data_type: &str) -> &'static str {
|
||||
match kind {
|
||||
LargeValuePreviewKind::Binary => "B",
|
||||
LargeValuePreviewKind::Vector => "V",
|
||||
LargeValuePreviewKind::TextCast => match normalized_data_type_base(data_type).as_str() {
|
||||
"json" => "J",
|
||||
"jsonb" => "K",
|
||||
"tsvector" => "S",
|
||||
_ => "T",
|
||||
},
|
||||
LargeValuePreviewKind::Text => "T",
|
||||
}
|
||||
}
|
||||
|
||||
fn normalized_data_type_base(data_type: &str) -> String {
|
||||
data_type.trim().split(['(', '[']).next().unwrap_or_default().trim().to_ascii_lowercase()
|
||||
}
|
||||
|
||||
fn large_value_preview_kind(database_type: Option<DatabaseType>, data_type: &str) -> Option<LargeValuePreviewKind> {
|
||||
let normalized = data_type.trim().to_ascii_lowercase();
|
||||
let base = normalized_data_type_base(data_type);
|
||||
match database_type {
|
||||
Some(DatabaseType::Mysql) => {
|
||||
if matches!(base.as_str(), "binary" | "varbinary" | "tinyblob" | "blob" | "mediumblob" | "longblob") {
|
||||
Some(LargeValuePreviewKind::Binary)
|
||||
} else if base == "json" {
|
||||
Some(LargeValuePreviewKind::TextCast)
|
||||
} else if matches!(base.as_str(), "char" | "varchar" | "tinytext" | "text" | "mediumtext" | "longtext") {
|
||||
Some(LargeValuePreviewKind::Text)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
Some(DatabaseType::Postgres) => {
|
||||
if base == "bytea" {
|
||||
Some(LargeValuePreviewKind::Binary)
|
||||
} else if matches!(base.as_str(), "char" | "character" | "varchar" | "text" | "citext" | "name" | "xml")
|
||||
|| normalized.starts_with("character varying")
|
||||
{
|
||||
Some(LargeValuePreviewKind::Text)
|
||||
} else if base == "vector" {
|
||||
Some(LargeValuePreviewKind::Vector)
|
||||
} else if matches!(base.as_str(), "json" | "jsonb" | "tsvector") {
|
||||
Some(LargeValuePreviewKind::TextCast)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn build_large_value_preview_columns(options: &TableDataSelectSqlOptions) -> Option<String> {
|
||||
let database_type = options.database_type;
|
||||
let preview_size = options.large_value_preview_size?.max(1);
|
||||
if options.columns.is_empty()
|
||||
|| options.columns.len() != options.column_types.len()
|
||||
|| options.primary_keys.is_empty()
|
||||
|| options
|
||||
.columns
|
||||
.iter()
|
||||
.any(|column| column.to_ascii_uppercase().starts_with(DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
|
||||
let protected: std::collections::HashSet<String> =
|
||||
options.primary_keys.iter().map(|column| column.to_ascii_lowercase()).collect();
|
||||
let mut projections = Vec::with_capacity(options.columns.len() * 2);
|
||||
let mut marker_count = 0;
|
||||
for (column_index, (column, data_type)) in options.columns.iter().zip(&options.column_types).enumerate() {
|
||||
let quoted = if uses_connection_identifier_quote(database_type, options.identifier_quote.as_deref()) {
|
||||
quote_table_data_identifier(database_type, column, options.identifier_quote.as_deref())
|
||||
} else {
|
||||
quote_table_identifier(database_type, column)
|
||||
};
|
||||
let kind = (!protected.contains(&column.to_ascii_lowercase()))
|
||||
.then(|| large_value_preview_kind(database_type, data_type))
|
||||
.flatten();
|
||||
let Some(kind) = kind else {
|
||||
projections.push(quoted);
|
||||
continue;
|
||||
};
|
||||
|
||||
let alias_kind = large_value_marker_alias_kind(kind, data_type);
|
||||
let marker_alias = quote_table_identifier(
|
||||
database_type,
|
||||
&format!("{DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX}{alias_kind}_{column_index}"),
|
||||
);
|
||||
let prefix_size = preview_size.saturating_add(1);
|
||||
let (preview, marker_kind) = match database_type {
|
||||
Some(DatabaseType::Mysql) if kind == LargeValuePreviewKind::Binary => {
|
||||
(format!("LEFT({quoted}, {prefix_size}) AS {quoted}"), "B")
|
||||
}
|
||||
Some(DatabaseType::Mysql) => (format!("LEFT({quoted}, {prefix_size}) AS {quoted}"), "T"),
|
||||
Some(DatabaseType::Postgres) if kind == LargeValuePreviewKind::Binary => {
|
||||
(format!("substring({quoted} from 1 for {prefix_size}) AS {quoted}"), "B")
|
||||
}
|
||||
Some(DatabaseType::Postgres) if kind == LargeValuePreviewKind::TextCast => {
|
||||
(format!("left({quoted}::text, {prefix_size}) AS {quoted}"), "T")
|
||||
}
|
||||
Some(DatabaseType::Postgres) if kind == LargeValuePreviewKind::Vector => {
|
||||
(format!("left({quoted}::text, {prefix_size}) AS {quoted}"), "V")
|
||||
}
|
||||
Some(DatabaseType::Postgres) => (format!("left({quoted}, {prefix_size}) AS {quoted}"), "T"),
|
||||
_ => return None,
|
||||
};
|
||||
let marker = format!("'{marker_kind}:{preview_size}' AS {marker_alias}");
|
||||
projections.push(preview);
|
||||
projections.push(marker);
|
||||
marker_count += 1;
|
||||
}
|
||||
(marker_count > 0).then(|| projections.join(", "))
|
||||
}
|
||||
|
||||
pub fn build_count_table_sql(database_type: Option<DatabaseType>, schema: Option<&str>, table_name: &str) -> String {
|
||||
if database_type == Some(DatabaseType::VictoriaMetrics) {
|
||||
return format!("count({})", victoriametrics_metric_selector(table_name));
|
||||
@@ -77,6 +202,8 @@ pub fn build_table_data_select_sql(options: TableDataSelectSqlOptions) -> String
|
||||
|
||||
let select_columns = if include_oracle_row_id {
|
||||
format!("ROWIDTOCHAR(t.ROWID) AS \"{DBX_ROWID_COLUMN}\", t.*")
|
||||
} else if let Some(preview_columns) = build_large_value_preview_columns(&options) {
|
||||
preview_columns
|
||||
} else {
|
||||
build_select_columns(
|
||||
database_type,
|
||||
@@ -521,6 +648,8 @@ mod tests {
|
||||
table_type: None,
|
||||
primary_keys: Vec::new(),
|
||||
columns: Vec::new(),
|
||||
column_types: Vec::new(),
|
||||
large_value_preview_size: None,
|
||||
fallback_order_columns: Vec::new(),
|
||||
order_by: None,
|
||||
limit: Some(10),
|
||||
|
||||
@@ -641,6 +641,97 @@ fn builds_table_data_where_and_schema_queries() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_mysql_table_data_large_value_previews_without_truncating_keys() {
|
||||
let sql = build_table_data_select_sql(TableDataSelectSqlOptions {
|
||||
database_type: Some(DatabaseType::Mysql),
|
||||
table_name: "large_rows".to_string(),
|
||||
primary_keys: vec!["id".to_string()],
|
||||
columns: vec!["id".to_string(), "payload".to_string(), "raw_value".to_string(), "metadata".to_string()],
|
||||
column_types: vec!["bigint".to_string(), "longtext".to_string(), "longblob".to_string(), "json".to_string()],
|
||||
large_value_preview_size: Some(4096),
|
||||
limit: Some(100),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
assert!(sql.starts_with("SELECT `id`, LEFT(`payload`, 4097) AS `payload`"));
|
||||
assert!(sql.contains("'T:4096' AS `__DBX_LARGE_VALUE_BYTES_T_1`"));
|
||||
assert!(sql.contains("LEFT(`raw_value`, 4097) AS `raw_value`"));
|
||||
assert!(sql.contains("'B:4096' AS `__DBX_LARGE_VALUE_BYTES_B_2`"));
|
||||
assert!(sql.contains("LEFT(`metadata`, 4097) AS `metadata`"));
|
||||
assert!(sql.contains("'T:4096' AS `__DBX_LARGE_VALUE_BYTES_J_3`"));
|
||||
assert!(!sql.contains("__DBX_LARGE_VALUE_BYTES_0"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_postgres_table_data_large_value_previews() {
|
||||
let sql = build_table_data_select_sql(TableDataSelectSqlOptions {
|
||||
database_type: Some(DatabaseType::Postgres),
|
||||
schema: Some("public".to_string()),
|
||||
table_name: "large_rows".to_string(),
|
||||
primary_keys: vec!["id".to_string()],
|
||||
columns: vec!["id".to_string(), "payload".to_string(), "metadata".to_string()],
|
||||
column_types: vec!["integer".to_string(), "text".to_string(), "jsonb".to_string()],
|
||||
large_value_preview_size: Some(8192),
|
||||
limit: Some(100),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
assert!(sql.starts_with("SELECT \"id\", left(\"payload\", 8193) AS \"payload\""));
|
||||
assert!(sql.contains("'T:8192' AS \"__DBX_LARGE_VALUE_BYTES_T_1\""));
|
||||
assert!(sql.contains("left(\"metadata\"::text, 8193) AS \"metadata\""));
|
||||
assert!(sql.contains("'T:8192' AS \"__DBX_LARGE_VALUE_BYTES_K_2\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_pgvector_table_data_preview_with_compatible_text_cast() {
|
||||
let sql = build_table_data_select_sql(TableDataSelectSqlOptions {
|
||||
database_type: Some(DatabaseType::Postgres),
|
||||
schema: Some("public".to_string()),
|
||||
table_name: "embeddings".to_string(),
|
||||
primary_keys: vec!["id".to_string()],
|
||||
columns: vec!["id".to_string(), "embedding".to_string()],
|
||||
column_types: vec!["integer".to_string(), "vector(16000)".to_string()],
|
||||
large_value_preview_size: Some(8192),
|
||||
limit: Some(100),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
assert!(sql.contains("left(\"embedding\"::text, 8193) AS \"embedding\""));
|
||||
assert!(sql.contains("'V:8192' AS \"__DBX_LARGE_VALUE_BYTES_V_1\""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn table_data_preview_requires_stable_keys_and_parallel_types() {
|
||||
let base = TableDataSelectSqlOptions {
|
||||
database_type: Some(DatabaseType::Postgres),
|
||||
table_name: "large_rows".to_string(),
|
||||
columns: vec!["payload".to_string()],
|
||||
column_types: vec!["text".to_string()],
|
||||
large_value_preview_size: Some(8192),
|
||||
limit: Some(100),
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(build_table_data_select_sql(base.clone()), "SELECT * FROM \"large_rows\" LIMIT 100;");
|
||||
assert_eq!(
|
||||
build_table_data_select_sql(TableDataSelectSqlOptions {
|
||||
primary_keys: vec!["id".to_string()],
|
||||
column_types: Vec::new(),
|
||||
..base.clone()
|
||||
}),
|
||||
"SELECT * FROM \"large_rows\" LIMIT 100;"
|
||||
);
|
||||
assert_eq!(
|
||||
build_table_data_select_sql(TableDataSelectSqlOptions {
|
||||
primary_keys: vec!["id".to_string()],
|
||||
columns: vec!["id".to_string(), "__dbx_large_value_bytes_t_0".to_string()],
|
||||
column_types: vec!["integer".to_string(), "text".to_string()],
|
||||
..base
|
||||
}),
|
||||
"SELECT * FROM \"large_rows\" LIMIT 100;"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_informix_table_data_with_skip_first_pagination() {
|
||||
assert_eq!(
|
||||
|
||||
@@ -44,6 +44,15 @@ pub struct TableDataSelectSqlOptions {
|
||||
pub primary_keys: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub columns: Vec<String>,
|
||||
/// Database type names parallel to `columns`. Table-data callers can use
|
||||
/// these to request bounded server-side projections for variable-length
|
||||
/// values without changing arbitrary query execution.
|
||||
#[serde(default)]
|
||||
pub column_types: Vec<String>,
|
||||
/// Maximum characters (text-like values) or bytes (binary values) kept in
|
||||
/// a table-data preview. Omitted for exports and ordinary SQL queries.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub large_value_preview_size: Option<usize>,
|
||||
#[serde(default)]
|
||||
pub fallback_order_columns: Vec<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
|
||||
Reference in new issue
Block a user