mirror of
https://github.com/t8y2/dbx.git
synced 2026-10-19 16:16:45 +00:00
fix(mysql): improve large-value table previews
This commit is contained in:
zipg
@@ -4097,6 +4097,15 @@ function isLargeValuePreview(item: RowItem | undefined, columnIndex: number): bo
|
||||
return largeValueCellsByKey.value.has(largeValueCellKey(item.sourceIndex, columnIndex));
|
||||
}
|
||||
|
||||
function largeValueOriginalBytes(item: Pick<RowItem, "sourceIndex" | "isNew" | "isDraft" | "isDirtyCol"> | undefined, columnIndex: number): number | undefined {
|
||||
if (resolvedDatabaseType.value !== "mysql" || !item || item.isNew || item.isDraft || item.sourceIndex === undefined || item.isDirtyCol[columnIndex]) return undefined;
|
||||
return largeValueCellsByKey.value.get(largeValueCellKey(item.sourceIndex, columnIndex))?.original_bytes;
|
||||
}
|
||||
|
||||
function formatGridItemCell(item: RowItem, columnIndex: number): string {
|
||||
return formatCellCached(item.data[columnIndex], columnIndex, largeValueOriginalBytes(item, columnIndex));
|
||||
}
|
||||
|
||||
function normalizedLargeValueIdentityPart(value: CellValue): string {
|
||||
if (value === null) return "null";
|
||||
return `${typeof value}:${JSON.stringify(value)}`;
|
||||
@@ -5385,7 +5394,7 @@ function primitiveCellFormatKey(value: CellValue, columnIndex?: number): string
|
||||
return `${columnIndex ?? -1}\u0000${typeof value}\u0000${String(value)}`;
|
||||
}
|
||||
|
||||
function formatCell(value: CellValue, columnIndex?: number): string {
|
||||
function formatCell(value: CellValue, columnIndex?: number, originalBytes?: number): string {
|
||||
const formatter = columnIndex === undefined ? undefined : resolvedColumnFormatters.value[columnIndex];
|
||||
const columnInfo = columnIndex === undefined ? undefined : tableColumnForGridColumn(columnIndex);
|
||||
const displayColumnInfo = columnInfo ?? (columnIndex === undefined ? undefined : resultColumnInfoForGridColumn(columnIndex));
|
||||
@@ -5397,14 +5406,14 @@ function formatCell(value: CellValue, columnIndex?: number): string {
|
||||
columnInfo: displayColumnInfo,
|
||||
});
|
||||
if (arrayDisplay !== undefined) return arrayDisplay;
|
||||
const binaryDisplay = formatter ? null : binaryCellDisplayText(value, columnIndex === undefined ? undefined : allColumnTypes.value[columnIndex]);
|
||||
const binaryDisplay = formatter ? null : binaryCellDisplayText(value, columnIndex === undefined ? undefined : allColumnTypes.value[columnIndex], originalBytes);
|
||||
if (binaryDisplay !== null) return binaryDisplay;
|
||||
const s = applyColumnFormatter(value, formatter);
|
||||
return limitDataGridCellDisplay(s, resolvedDatabaseType.value === "sqlserver" ? SQLSERVER_DATA_GRID_CELL_DISPLAY_MAX_LENGTH : undefined);
|
||||
}
|
||||
|
||||
function formatCellCached(value: CellValue, columnIndex?: number): string {
|
||||
if (value !== null && typeof (value as unknown) === "object") {
|
||||
function formatCellCached(value: CellValue, columnIndex?: number, originalBytes?: number): string {
|
||||
if (originalBytes === undefined && value !== null && typeof (value as unknown) === "object") {
|
||||
const objectValue = value as unknown as object;
|
||||
const cacheColumn = columnIndex ?? -1;
|
||||
const columnCache = objectCellFormatCache.get(objectValue);
|
||||
@@ -5424,13 +5433,13 @@ function formatCellCached(value: CellValue, columnIndex?: number): string {
|
||||
// values are truncated for display anyway, so formatting them on demand is
|
||||
// cheaper than keeping the full source string in the primitive cache.
|
||||
if (typeof value === "string" && value.length > CELL_FORMAT_CACHE_STRING_KEY_MAX_LENGTH) {
|
||||
return formatCell(value, columnIndex);
|
||||
return formatCell(value, columnIndex, originalBytes);
|
||||
}
|
||||
|
||||
const key = primitiveCellFormatKey(value, columnIndex);
|
||||
const key = `${primitiveCellFormatKey(value, columnIndex)}\u0000${originalBytes ?? ""}`;
|
||||
const cached = primitiveCellFormatCache.get(key);
|
||||
if (cached !== undefined) return cached;
|
||||
return rememberPrimitiveCellFormat(key, formatCell(value, columnIndex));
|
||||
return rememberPrimitiveCellFormat(key, formatCell(value, columnIndex, originalBytes));
|
||||
}
|
||||
|
||||
function rowNumberPageOffset(): number {
|
||||
@@ -6207,7 +6216,7 @@ function drawCanvasGrid() {
|
||||
editingCell: editingCell.value,
|
||||
searchMatchKeys: searchMatchSet.value,
|
||||
currentSearchMatch: currentSearchMatch.value,
|
||||
formatCell: formatCellCached,
|
||||
formatCell: (value, columnIndex, row) => formatCellCached(value, columnIndex, largeValueOriginalBytes(row, columnIndex)),
|
||||
draftCellPlaceholder: t("grid.quickEntryDraftPlaceholder"),
|
||||
isRowActive,
|
||||
rowCellsUseSelectionVisual,
|
||||
@@ -6394,6 +6403,45 @@ async function cancelActiveExport() {
|
||||
await exportCancelHandler.value?.();
|
||||
}
|
||||
|
||||
const userFacingSql = ref("");
|
||||
let userFacingSqlGeneration = 0;
|
||||
|
||||
async function syncUserFacingSql() {
|
||||
const generation = ++userFacingSqlGeneration;
|
||||
const executionSql = props.sql?.trim() ?? "";
|
||||
if (props.context !== "table-data" || !executionSql.includes("__DBX_LARGE_VALUE_BYTES_") || !props.tableMeta?.tableName) {
|
||||
userFacingSql.value = executionSql;
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const config = props.connectionId ? connectionStore.getConfig(props.connectionId) : undefined;
|
||||
const sql = await buildTableSelectSql({
|
||||
databaseType: resolvedDatabaseType.value,
|
||||
driverProfile: config?.driver_profile,
|
||||
identifierQuote: props.connectionId ? connectionStore.connectionIdentifierQuote?.(props.connectionId) : undefined,
|
||||
catalog: props.tableMeta.catalog,
|
||||
database: props.tableMeta.database,
|
||||
schema: props.tableMeta.schema,
|
||||
tableName: props.tableMeta.tableName,
|
||||
tableType: props.tableMeta.tableType,
|
||||
whereInput: currentWhereInput(),
|
||||
orderBy: currentOrderBy(),
|
||||
limit: props.pageLimit ?? pageSize.value,
|
||||
offset: props.pageOffset ?? Math.max(0, currentPage.value - 1) * pageSize.value,
|
||||
});
|
||||
if (generation === userFacingSqlGeneration) userFacingSql.value = sql;
|
||||
} catch {
|
||||
if (generation === userFacingSqlGeneration) userFacingSql.value = executionSql;
|
||||
}
|
||||
}
|
||||
|
||||
watch(
|
||||
() => [props.sql, props.context, props.tableMeta, props.pageLimit, props.pageOffset, currentWhereInput(), currentOrderBy()],
|
||||
() => void syncUserFacingSql(),
|
||||
{ immediate: true },
|
||||
);
|
||||
|
||||
// --- Export composable ---
|
||||
const {
|
||||
copyText,
|
||||
@@ -6422,7 +6470,6 @@ const {
|
||||
exportCurrentPageSql,
|
||||
exportTxt,
|
||||
exportCurrentPageTxt,
|
||||
copySql,
|
||||
} = useDataGridExport({
|
||||
columns: visibleColumns,
|
||||
displayItems: visibleDisplayItems,
|
||||
@@ -8665,7 +8712,11 @@ function onRowContext(rowId: number, rowIndex: number) {
|
||||
}
|
||||
}
|
||||
|
||||
const sqlOneLiner = computed(() => props.sql?.replace(/\s+/g, " ").trim() || "");
|
||||
const sqlOneLiner = computed(() => userFacingSql.value.replace(/\s+/g, " ").trim());
|
||||
|
||||
async function copyUserFacingSql() {
|
||||
if (userFacingSql.value) await copyText(userFacingSql.value);
|
||||
}
|
||||
|
||||
type TableInfoTabItem = {
|
||||
id: TableInfoTab;
|
||||
@@ -11195,7 +11246,7 @@ const gridContextMenuItems = computed<ContextMenuItem[]>(() => {
|
||||
<template v-if="draftCellPlaceholder(item, col.actualColIdx)">
|
||||
<span class="text-muted-foreground/70 italic">{{ draftCellPlaceholder(item, col.actualColIdx) }}</span>
|
||||
</template>
|
||||
<template v-else>{{ firstLineCellDisplayValue(formatCellCached(item.data[col.actualColIdx], col.actualColIdx), flatteningMultiLineEnabled) }}</template>
|
||||
<template v-else>{{ firstLineCellDisplayValue(formatGridItemCell(item, col.actualColIdx), flatteningMultiLineEnabled) }}</template>
|
||||
<div v-if="cellDetailButtonVisible(item.displayIndex, col.actualColIdx)" class="absolute right-2 top-1/2 flex -translate-y-1/2 items-center gap-1">
|
||||
<LightDropdownMenu
|
||||
v-if="canQuickDownloadCellValue(item.displayIndex, col.actualColIdx)"
|
||||
@@ -11699,12 +11750,12 @@ const gridContextMenuItems = computed<ContextMenuItem[]>(() => {
|
||||
|
||||
<Tooltip v-if="sqlOneLiner">
|
||||
<TooltipTrigger as-child>
|
||||
<span class="min-w-0 max-w-full justify-self-center truncate opacity-60 cursor-pointer hover:opacity-100" @click="copySql">
|
||||
<span class="min-w-0 max-w-full justify-self-center truncate opacity-60 cursor-pointer hover:opacity-100" @click="copyUserFacingSql">
|
||||
{{ sqlOneLiner }}
|
||||
</span>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent side="top" class="max-w-md">
|
||||
<pre class="text-xs font-mono whitespace-pre-wrap">{{ props.sql }}</pre>
|
||||
<pre class="text-xs font-mono whitespace-pre-wrap">{{ userFacingSql }}</pre>
|
||||
</TooltipContent>
|
||||
</Tooltip>
|
||||
<span v-else class="min-w-0" />
|
||||
|
||||
@@ -66,6 +66,31 @@ describe("data grid large-value metadata", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("uses the active MySQL page budget to select bounded preview columns", () => {
|
||||
const columns = [column("id", "int", true), column("image_mime", "varchar(64)"), column("image_data", "longblob"), column("image_url", "varchar(512)"), column("image_note", "varchar(255)")];
|
||||
|
||||
expect(tableDataLargeValuePreviewOptions("mysql", columns, ["id"], 10_000)).toEqual({
|
||||
columnTypes: columns.map((item) => item.data_type),
|
||||
largeValuePreviewSize: 139,
|
||||
});
|
||||
expect(tableDataLargeValuePreviewOptions("mysql", columns, ["id"], 100)).toEqual({
|
||||
columnTypes: columns.map((item) => item.data_type),
|
||||
largeValuePreviewSize: TABLE_DATA_CELL_PREVIEW_SIZE,
|
||||
});
|
||||
expect(
|
||||
tableDataLargeValuePreviewOptions(
|
||||
"mysql",
|
||||
columns.filter((item) => item.name !== "image_data"),
|
||||
["id"],
|
||||
100,
|
||||
),
|
||||
).toEqual({});
|
||||
expect(tableDataLargeValuePreviewOptions("mysql", [column("id", "int", true), column("payload", "varchar(8000)")], ["id"], 10_000)).toEqual({
|
||||
columnTypes: ["int", "varchar(8000)"],
|
||||
largeValuePreviewSize: 419,
|
||||
});
|
||||
});
|
||||
|
||||
it("offsets appended segment rows and discards metadata beyond the append cap", () => {
|
||||
expect(
|
||||
appendLargeValueCells(
|
||||
|
||||
@@ -108,6 +108,7 @@ export function retainBinaryCellDownloadMenuForHover(openCell: BinaryCellPositio
|
||||
}
|
||||
|
||||
const HEX_VALUE_RE = /^(?:0[xX]|\\x)([0-9a-fA-F\s]*)$/;
|
||||
const TRUNCATED_HEX_VALUE_RE = /^(?:0[xX]|\\x)[0-9a-fA-F\s]+\.\.\.$/;
|
||||
const BARE_HEX_RE = /^[0-9a-fA-F\s]+$/;
|
||||
const HEX_ESCAPE_RE = /^(?:\\x[0-9a-fA-F]{2}|\s)+$/;
|
||||
const BINARY_TYPE_RE = /^(?:blob|tinyblob|mediumblob|longblob|bytea|bytes|binary|varbinary|image|raw|long\s+raw)(?:\b|\()/i;
|
||||
@@ -192,7 +193,11 @@ export function canDownloadBinaryCellValue(value: unknown, columnType?: string):
|
||||
return !!parseBinaryCellBytes(value, columnType);
|
||||
}
|
||||
|
||||
export function binaryCellDisplayText(value: unknown, columnType?: string): string | null {
|
||||
export function binaryCellDisplayText(value: unknown, columnType?: string, originalBytes?: number): string | null {
|
||||
if (typeof value === "string" && isBinaryCellColumnType(columnType) && TRUNCATED_HEX_VALUE_RE.test(value.trim())) {
|
||||
const size = originalBytes === undefined ? "..." : formatBinaryCellByteSize(originalBytes);
|
||||
return `${binaryCellDisplayLabel(columnType)} [${size}]`;
|
||||
}
|
||||
const bytes = parseBinaryCellBytes(value, columnType);
|
||||
if (!bytes || !isBinaryCellColumnType(columnType)) return null;
|
||||
if (BINARY_STRING_TYPE_RE.test((columnType ?? "").trim())) {
|
||||
|
||||
@@ -26,6 +26,7 @@ export interface CanvasDataGridRow {
|
||||
isDeleted: boolean;
|
||||
isDirtyCol: boolean[];
|
||||
status: RowStatus;
|
||||
sourceIndex?: number;
|
||||
}
|
||||
|
||||
export interface CanvasHoverCell {
|
||||
@@ -77,7 +78,7 @@ export interface DrawCanvasDataGridOptions {
|
||||
editingCell: CanvasEditingCell | null;
|
||||
searchMatchKeys: ReadonlySet<number>;
|
||||
currentSearchMatch: CanvasSearchMatch | null;
|
||||
formatCell: (value: CellValue, columnIndex: number) => string;
|
||||
formatCell: (value: CellValue, columnIndex: number, row: CanvasDataGridRow) => string;
|
||||
columnIsBoolean?: (columnIndex: number) => boolean;
|
||||
draftCellPlaceholder?: string;
|
||||
isRowActive: (rowIndex: number) => boolean;
|
||||
@@ -576,7 +577,7 @@ export function drawCanvasDataGrid(options: DrawCanvasDataGridOptions) {
|
||||
ctx.stroke();
|
||||
}
|
||||
} else {
|
||||
const rawDisplayText = item.isDraft && value === null ? (draftCellPlaceholder ?? "") : formatCell(value, actualColIdx);
|
||||
const rawDisplayText = item.isDraft && value === null ? (draftCellPlaceholder ?? "") : formatCell(value, actualColIdx, item);
|
||||
const displayText = isEditingThisCell ? "" : firstLineCellDisplayValue(rawDisplayText, flatteningMultiLineEnabled);
|
||||
const text = isEditingThisCell ? displayText : fitCanvasText(ctx, displayText, cellMaxWidth, isBooleanNullCell ? "left" : isRightAlign ? "right" : "left");
|
||||
const anchorX = isBooleanNullCell ? alignCanvasPixel(drawX + colWidth / 2, scaleX) : textAnchorX;
|
||||
|
||||
@@ -7,6 +7,46 @@ export const TABLE_DATA_PREVIEW_CONTENT_MAX_BYTES = 24 * 1024 * 1024;
|
||||
export const TABLE_DATA_TEXT_SERIALIZED_BYTES_PER_CHARACTER = 6;
|
||||
const TABLE_DATA_LARGE_VALUE_MARKER_PREFIX = "__DBX_LARGE_VALUE_BYTES_";
|
||||
|
||||
function normalizedDataTypeBase(dataType: string): string {
|
||||
return dataType.trim().split(/[([]/, 1)[0]?.trim().toLocaleLowerCase() ?? "";
|
||||
}
|
||||
|
||||
function declaredDataTypeLength(dataType: string): number | undefined {
|
||||
const match = dataType.match(/\(\s*(\d+)/);
|
||||
if (!match?.[1]) return undefined;
|
||||
const length = Number.parseInt(match[1], 10);
|
||||
return Number.isSafeInteger(length) ? length : undefined;
|
||||
}
|
||||
|
||||
function mysqlColumnNeedsLargeValuePreview(column: ColumnInfo, previewSize: number): boolean {
|
||||
const base = normalizedDataTypeBase(column.data_type);
|
||||
if (matchesMysqlUnboundedLargeValueType(base)) return true;
|
||||
if (base !== "varchar" && base !== "varbinary") return false;
|
||||
return (declaredDataTypeLength(column.data_type) ?? 0) > previewSize;
|
||||
}
|
||||
|
||||
function previewSizeForColumnCount(pageSize: number | undefined, previewColumnCount: number): number {
|
||||
const budgetedSize = Math.floor(TABLE_DATA_PREVIEW_CONTENT_MAX_BYTES / Math.max(1, pageSize ?? 1) / previewColumnCount / TABLE_DATA_TEXT_SERIALIZED_BYTES_PER_CHARACTER);
|
||||
return Math.max(TABLE_DATA_CELL_PREVIEW_MIN_SIZE, Math.min(TABLE_DATA_CELL_PREVIEW_SIZE, budgetedSize));
|
||||
}
|
||||
|
||||
function mysqlPreviewBudget(columns: readonly ColumnInfo[], keyColumns: ReadonlySet<string>, pageSize: number | undefined): { previewColumnCount: number; previewSize: number } | null {
|
||||
let previewColumnCount = 1;
|
||||
let previewSize = previewSizeForColumnCount(pageSize, previewColumnCount);
|
||||
for (;;) {
|
||||
const nextCount = columns.filter((column) => !keyColumns.has(column.name.toLocaleLowerCase()) && mysqlColumnNeedsLargeValuePreview(column, previewSize)).length;
|
||||
if (nextCount === 0) return null;
|
||||
const nextSize = previewSizeForColumnCount(pageSize, nextCount);
|
||||
if (nextCount === previewColumnCount && nextSize === previewSize) return { previewColumnCount: nextCount, previewSize: nextSize };
|
||||
previewColumnCount = nextCount;
|
||||
previewSize = nextSize;
|
||||
}
|
||||
}
|
||||
|
||||
function matchesMysqlUnboundedLargeValueType(base: string): boolean {
|
||||
return base === "blob" || base === "mediumblob" || base === "longblob" || base === "text" || base === "mediumtext" || base === "longtext" || base === "json";
|
||||
}
|
||||
|
||||
export function canUseTableDataLargeValuePreview(databaseType: DatabaseType | undefined, columns: readonly ColumnInfo[], primaryKeys: readonly string[]): boolean {
|
||||
return (databaseType === "mysql" || databaseType === "postgres") && columns.length > 0 && primaryKeys.length > 0 && !columns.some((column) => column.name.toLocaleUpperCase().startsWith(TABLE_DATA_LARGE_VALUE_MARKER_PREFIX));
|
||||
}
|
||||
@@ -14,12 +54,12 @@ export function canUseTableDataLargeValuePreview(databaseType: DatabaseType | un
|
||||
export function tableDataLargeValuePreviewOptions(databaseType: DatabaseType | undefined, columns: readonly ColumnInfo[], primaryKeys: readonly string[], pageSize?: number): { columnTypes: string[]; largeValuePreviewSize: number } | Record<string, never> {
|
||||
if (!canUseTableDataLargeValuePreview(databaseType, columns, primaryKeys)) return {};
|
||||
const keyColumns = new Set(primaryKeys.map((column) => column.toLocaleLowerCase()));
|
||||
const previewColumnCount = columns.filter((column) => !keyColumns.has(column.name.toLocaleLowerCase())).length;
|
||||
const mysqlBudget = databaseType === "mysql" ? mysqlPreviewBudget(columns, keyColumns, pageSize) : null;
|
||||
const previewColumnCount = databaseType === "mysql" ? (mysqlBudget?.previewColumnCount ?? 0) : columns.filter((column) => !keyColumns.has(column.name.toLocaleLowerCase())).length;
|
||||
if (previewColumnCount === 0) return {};
|
||||
const budgetedSize = Math.floor(TABLE_DATA_PREVIEW_CONTENT_MAX_BYTES / Math.max(1, pageSize ?? 1) / previewColumnCount / TABLE_DATA_TEXT_SERIALIZED_BYTES_PER_CHARACTER);
|
||||
return {
|
||||
columnTypes: columns.map((column) => column.data_type),
|
||||
largeValuePreviewSize: Math.max(TABLE_DATA_CELL_PREVIEW_MIN_SIZE, Math.min(TABLE_DATA_CELL_PREVIEW_SIZE, budgetedSize)),
|
||||
largeValuePreviewSize: mysqlBudget?.previewSize ?? previewSizeForColumnCount(pageSize, previewColumnCount),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -606,17 +606,17 @@ fn mysql_result_cell_preview_bytes(
|
||||
}
|
||||
|
||||
fn mysql_column_can_use_bounded_preview(column: &mysql_async::Column) -> bool {
|
||||
matches!(
|
||||
column.column_type(),
|
||||
match column.column_type() {
|
||||
ColumnType::MYSQL_TYPE_JSON
|
||||
| ColumnType::MYSQL_TYPE_BLOB
|
||||
| ColumnType::MYSQL_TYPE_LONG_BLOB
|
||||
| ColumnType::MYSQL_TYPE_MEDIUM_BLOB
|
||||
| ColumnType::MYSQL_TYPE_TINY_BLOB
|
||||
| ColumnType::MYSQL_TYPE_STRING
|
||||
| ColumnType::MYSQL_TYPE_VAR_STRING
|
||||
| ColumnType::MYSQL_TYPE_VARCHAR
|
||||
)
|
||||
| ColumnType::MYSQL_TYPE_BLOB
|
||||
| ColumnType::MYSQL_TYPE_LONG_BLOB
|
||||
| ColumnType::MYSQL_TYPE_MEDIUM_BLOB => true,
|
||||
ColumnType::MYSQL_TYPE_TINY_BLOB => false,
|
||||
ColumnType::MYSQL_TYPE_STRING | ColumnType::MYSQL_TYPE_VAR_STRING | ColumnType::MYSQL_TYPE_VARCHAR => {
|
||||
column.column_length() > 8 * 1024
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn mysql_bounded_value_preview(
|
||||
@@ -665,7 +665,7 @@ fn mysql_result_protected_column_indexes(
|
||||
protected_columns: &[String],
|
||||
) -> HashSet<usize> {
|
||||
let protected: HashSet<String> = protected_columns.iter().map(|column| column.to_ascii_lowercase()).collect();
|
||||
columns
|
||||
let mut indexes = columns
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(index, column)| {
|
||||
@@ -673,7 +673,16 @@ fn mysql_result_protected_column_indexes(
|
||||
|| protected.contains(&column.name_str().to_ascii_lowercase()))
|
||||
.then_some(index)
|
||||
})
|
||||
.collect()
|
||||
.collect::<HashSet<_>>();
|
||||
for (index, column) in columns.iter().enumerate() {
|
||||
if column.name_str().to_ascii_uppercase().starts_with(crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX) {
|
||||
indexes.insert(index);
|
||||
if index > 0 {
|
||||
indexes.insert(index - 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
indexes
|
||||
}
|
||||
|
||||
fn mysql_row_to_json_with_srids(
|
||||
@@ -6410,6 +6419,46 @@ mod tests {
|
||||
assert_eq!(cells, vec![LargeValueCell { row_index: 0, column_index: 1, original_bytes: 4096 }]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mysql_large_value_preview_skips_bounded_strings_and_server_preview_columns() {
|
||||
let ordinary_column = Column::new(ColumnType::MYSQL_TYPE_VAR_STRING)
|
||||
.with_name(b"image_url")
|
||||
.with_character_set(45)
|
||||
.with_column_length(2048);
|
||||
let large_column = Column::new(ColumnType::MYSQL_TYPE_VAR_STRING)
|
||||
.with_name(b"large_note")
|
||||
.with_character_set(45)
|
||||
.with_column_length(40_000);
|
||||
let server_preview_column = Column::new(ColumnType::MYSQL_TYPE_VAR_STRING)
|
||||
.with_name(b"image_data")
|
||||
.with_character_set(63)
|
||||
.with_column_length(1680);
|
||||
let marker_column = Column::new(ColumnType::MYSQL_TYPE_VAR_STRING)
|
||||
.with_name(format!("{}B_2", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX).as_bytes())
|
||||
.with_character_set(45)
|
||||
.with_column_length(64);
|
||||
let row = mysql_test_row_with_columns(
|
||||
vec![
|
||||
Value::Bytes(vec![b'u'; 500]),
|
||||
Value::Bytes(vec![b'n'; 10_000]),
|
||||
Value::Bytes(vec![0xab; 420]),
|
||||
Value::Bytes(b"B:419:25143".to_vec()),
|
||||
],
|
||||
vec![ordinary_column, large_column, server_preview_column, marker_column],
|
||||
);
|
||||
let protected = mysql_result_protected_column_indexes(row.columns_ref(), &[]);
|
||||
let mut spatial_columns = mysql_spatial_column_builder(row.columns_ref());
|
||||
|
||||
let (values, _srids, cells) =
|
||||
mysql_row_to_json_with_srids_and_previews(&row, &mut spatial_columns, 0, Some(256), &protected);
|
||||
|
||||
assert_eq!(values[0].as_str().map(str::len), Some(500));
|
||||
assert!(values[1].as_str().is_some_and(|value| value.ends_with("...")));
|
||||
assert_eq!(values[2].as_str().map(str::len), Some(2 + 420 * 2));
|
||||
assert_eq!(values[3], serde_json::json!("B:419:25143"));
|
||||
assert_eq!(cells, vec![LargeValueCell { row_index: 0, column_index: 1, original_bytes: 10_000 }]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mysql_large_binary_preview_is_bounded_hex() {
|
||||
let payload_column = Column::new(ColumnType::MYSQL_TYPE_LONG_BLOB)
|
||||
|
||||
@@ -336,6 +336,13 @@ struct ServerLargeValueMarker {
|
||||
source_type: Option<&'static str>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct ServerLargeValueMarkerValue {
|
||||
kind: ServerLargeValuePreviewKind,
|
||||
preview_size: usize,
|
||||
original_bytes: Option<usize>,
|
||||
}
|
||||
|
||||
fn server_large_value_alias(
|
||||
suffix: &str,
|
||||
) -> Option<(usize, Option<ServerLargeValuePreviewKind>, Option<&'static str>)> {
|
||||
@@ -360,8 +367,10 @@ fn server_large_value_alias(
|
||||
Some((source_index, Some(preview_kind), source_type))
|
||||
}
|
||||
|
||||
fn server_large_value_marker(value: &serde_json::Value) -> Option<(ServerLargeValuePreviewKind, usize)> {
|
||||
let (kind, preview_size) = value.as_str()?.split_once(':')?;
|
||||
fn server_large_value_marker(value: &serde_json::Value) -> Option<ServerLargeValueMarkerValue> {
|
||||
let mut parts = value.as_str()?.split(':');
|
||||
let kind = parts.next()?;
|
||||
let preview_size = parts.next()?;
|
||||
let kind = match kind {
|
||||
"T" => ServerLargeValuePreviewKind::Text,
|
||||
"B" => ServerLargeValuePreviewKind::Binary,
|
||||
@@ -369,7 +378,8 @@ fn server_large_value_marker(value: &serde_json::Value) -> Option<(ServerLargeVa
|
||||
"D" => ServerLargeValuePreviewKind::Deferred,
|
||||
_ => return None,
|
||||
};
|
||||
Some((kind, preview_size.parse::<usize>().ok()?.max(1)))
|
||||
let original_bytes = parts.next().and_then(|value| value.parse::<usize>().ok());
|
||||
Some(ServerLargeValueMarkerValue { kind, preview_size: preview_size.parse::<usize>().ok()?.max(1), original_bytes })
|
||||
}
|
||||
|
||||
fn truncate_server_large_value_preview(
|
||||
@@ -445,7 +455,7 @@ fn extract_server_large_value_markers(result: &mut db::QueryResult) -> Vec<db::L
|
||||
let row_kind = result
|
||||
.rows
|
||||
.iter()
|
||||
.find_map(|row| row.get(marker.result_index).and_then(server_large_value_marker).map(|value| value.0));
|
||||
.find_map(|row| row.get(marker.result_index).and_then(server_large_value_marker).map(|value| value.kind));
|
||||
let source_type = marker.source_type.or_else(|| {
|
||||
(marker.preview_kind.or(row_kind) == Some(ServerLargeValuePreviewKind::Vector)).then_some("vector")
|
||||
});
|
||||
@@ -458,14 +468,16 @@ fn extract_server_large_value_markers(result: &mut db::QueryResult) -> Vec<db::L
|
||||
for marker in &markers {
|
||||
let marker_value = row.get(marker.result_index).and_then(server_large_value_marker);
|
||||
let source_result_index = marker.result_index.saturating_sub(1);
|
||||
if marker_value.is_some_and(|(kind, preview_size)| {
|
||||
if marker_value.is_some_and(|value| {
|
||||
row.get_mut(source_result_index)
|
||||
.is_some_and(|value| truncate_server_large_value_preview(value, kind, preview_size))
|
||||
.is_some_and(|source| truncate_server_large_value_preview(source, value.kind, value.preview_size))
|
||||
}) {
|
||||
large_value_cells.push(db::LargeValueCell {
|
||||
row_index,
|
||||
column_index: marker.source_index,
|
||||
original_bytes: SERVER_LARGE_VALUE_UNKNOWN_BYTES,
|
||||
original_bytes: marker_value
|
||||
.and_then(|value| value.original_bytes)
|
||||
.unwrap_or(SERVER_LARGE_VALUE_UNKNOWN_BYTES),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -7354,7 +7366,7 @@ for line in sys.stdin:
|
||||
column_sortables: vec![true, true],
|
||||
spatial_columns: Vec::new(),
|
||||
spatial_values: Vec::new(),
|
||||
rows: vec![vec![serde_json::json!("0x0102030405"), serde_json::json!("B:4")]],
|
||||
rows: vec![vec![serde_json::json!("0x0102030405"), serde_json::json!("B:4:5")]],
|
||||
affected_rows: 0,
|
||||
execution_time_ms: 0,
|
||||
truncated: false,
|
||||
@@ -7367,7 +7379,7 @@ for line in sys.stdin:
|
||||
let cells = extract_server_large_value_markers(&mut result);
|
||||
|
||||
assert_eq!(result.rows, vec![vec![serde_json::json!("0x01020304...")]]);
|
||||
assert_eq!(cells.len(), 1);
|
||||
assert_eq!(cells, vec![db::LargeValueCell { row_index: 0, column_index: 0, original_bytes: 5 }]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -40,16 +40,32 @@ fn normalized_data_type_base(data_type: &str) -> String {
|
||||
data_type.trim().split(['(', '[']).next().unwrap_or_default().trim().to_ascii_lowercase()
|
||||
}
|
||||
|
||||
fn large_value_preview_kind(database_type: Option<DatabaseType>, data_type: &str) -> Option<LargeValuePreviewKind> {
|
||||
fn declared_data_type_length(data_type: &str) -> Option<usize> {
|
||||
let parameters = data_type.split_once('(')?.1;
|
||||
let digits = parameters.trim_start().chars().take_while(char::is_ascii_digit).collect::<String>();
|
||||
(!digits.is_empty()).then(|| digits.parse::<usize>().ok()).flatten()
|
||||
}
|
||||
|
||||
fn large_value_preview_kind(
|
||||
database_type: Option<DatabaseType>,
|
||||
data_type: &str,
|
||||
preview_size: usize,
|
||||
) -> Option<LargeValuePreviewKind> {
|
||||
let normalized = data_type.trim().to_ascii_lowercase();
|
||||
let base = normalized_data_type_base(data_type);
|
||||
match database_type {
|
||||
Some(DatabaseType::Mysql) => {
|
||||
if matches!(base.as_str(), "binary" | "varbinary" | "tinyblob" | "blob" | "mediumblob" | "longblob") {
|
||||
if matches!(base.as_str(), "blob" | "mediumblob" | "longblob")
|
||||
|| (base == "varbinary"
|
||||
&& declared_data_type_length(data_type).is_some_and(|length| length > preview_size))
|
||||
{
|
||||
Some(LargeValuePreviewKind::Binary)
|
||||
} else if base == "json" {
|
||||
Some(LargeValuePreviewKind::TextCast)
|
||||
} else if matches!(base.as_str(), "char" | "varchar" | "tinytext" | "text" | "mediumtext" | "longtext") {
|
||||
} else if matches!(base.as_str(), "text" | "mediumtext" | "longtext")
|
||||
|| (base == "varchar"
|
||||
&& declared_data_type_length(data_type).is_some_and(|length| length > preview_size))
|
||||
{
|
||||
Some(LargeValuePreviewKind::Text)
|
||||
} else {
|
||||
None
|
||||
@@ -101,7 +117,7 @@ fn build_large_value_preview_columns(options: &TableDataSelectSqlOptions) -> Opt
|
||||
quote_table_identifier(database_type, column)
|
||||
};
|
||||
let kind = (!protected.contains(&column.to_ascii_lowercase()))
|
||||
.then(|| large_value_preview_kind(database_type, data_type))
|
||||
.then(|| large_value_preview_kind(database_type, data_type, preview_size))
|
||||
.flatten();
|
||||
let Some(kind) = kind else {
|
||||
projections.push(quoted);
|
||||
@@ -131,7 +147,11 @@ fn build_large_value_preview_columns(options: &TableDataSelectSqlOptions) -> Opt
|
||||
Some(DatabaseType::Postgres) => (format!("left({quoted}, {prefix_size}) AS {quoted}"), "T"),
|
||||
_ => return None,
|
||||
};
|
||||
let marker = format!("'{marker_kind}:{preview_size}' AS {marker_alias}");
|
||||
let marker = if database_type == Some(DatabaseType::Mysql) {
|
||||
format!("CONCAT('{marker_kind}:{preview_size}:', OCTET_LENGTH({quoted})) AS {marker_alias}")
|
||||
} else {
|
||||
format!("'{marker_kind}:{preview_size}' AS {marker_alias}")
|
||||
};
|
||||
projections.push(preview);
|
||||
projections.push(marker);
|
||||
marker_count += 1;
|
||||
|
||||
@@ -705,14 +705,50 @@ fn builds_mysql_table_data_large_value_previews_without_truncating_keys() {
|
||||
});
|
||||
|
||||
assert!(sql.starts_with("SELECT `id`, LEFT(`payload`, 4097) AS `payload`"));
|
||||
assert!(sql.contains("'T:4096' AS `__DBX_LARGE_VALUE_BYTES_T_1`"));
|
||||
assert!(sql.contains("CONCAT('T:4096:', OCTET_LENGTH(`payload`)) AS `__DBX_LARGE_VALUE_BYTES_T_1`"));
|
||||
assert!(sql.contains("LEFT(`raw_value`, 4097) AS `raw_value`"));
|
||||
assert!(sql.contains("'B:4096' AS `__DBX_LARGE_VALUE_BYTES_B_2`"));
|
||||
assert!(sql.contains("CONCAT('B:4096:', OCTET_LENGTH(`raw_value`)) AS `__DBX_LARGE_VALUE_BYTES_B_2`"));
|
||||
assert!(sql.contains("LEFT(`metadata`, 4097) AS `metadata`"));
|
||||
assert!(sql.contains("'T:4096' AS `__DBX_LARGE_VALUE_BYTES_J_3`"));
|
||||
assert!(sql.contains("CONCAT('T:4096:', OCTET_LENGTH(`metadata`)) AS `__DBX_LARGE_VALUE_BYTES_J_3`"));
|
||||
assert!(!sql.contains("__DBX_LARGE_VALUE_BYTES_0"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn previews_mysql_bounded_string_columns_only_above_the_active_budget() {
|
||||
let sql = build_table_data_select_sql(TableDataSelectSqlOptions {
|
||||
database_type: Some(DatabaseType::Mysql),
|
||||
table_name: "t_0001".to_string(),
|
||||
primary_keys: vec!["id".to_string()],
|
||||
columns: vec![
|
||||
"id".to_string(),
|
||||
"image_mime".to_string(),
|
||||
"image_data".to_string(),
|
||||
"image_url".to_string(),
|
||||
"large_note".to_string(),
|
||||
"large_binary".to_string(),
|
||||
],
|
||||
column_types: vec![
|
||||
"int".to_string(),
|
||||
"varchar(64)".to_string(),
|
||||
"longblob".to_string(),
|
||||
"varchar(512)".to_string(),
|
||||
"varchar(10000)".to_string(),
|
||||
"varbinary(10000)".to_string(),
|
||||
],
|
||||
large_value_preview_size: Some(419),
|
||||
limit: Some(10_000),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
assert!(sql.starts_with("SELECT `id`, `image_mime`, LEFT(`image_data`, 420) AS `image_data`"));
|
||||
assert!(sql.contains("CONCAT('B:419:', OCTET_LENGTH(`image_data`)) AS `__DBX_LARGE_VALUE_BYTES_B_2`, LEFT(`image_url`, 420) AS `image_url`"));
|
||||
assert!(sql.contains("CONCAT('T:419:', OCTET_LENGTH(`image_url`)) AS `__DBX_LARGE_VALUE_BYTES_T_3`"));
|
||||
assert!(sql.contains("LEFT(`large_note`, 420) AS `large_note`"));
|
||||
assert!(sql.contains("CONCAT('T:419:', OCTET_LENGTH(`large_note`)) AS `__DBX_LARGE_VALUE_BYTES_T_4`"));
|
||||
assert!(sql.contains("LEFT(`large_binary`, 420) AS `large_binary`"));
|
||||
assert!(!sql.contains("LEFT(`image_mime`"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_postgres_table_data_large_value_previews() {
|
||||
let sql = build_table_data_select_sql(TableDataSelectSqlOptions {
|
||||
|
||||
+4
-1
@@ -75,6 +75,9 @@ test("binaryCellDisplayText previews printable binary strings without changing t
|
||||
assert.equal(binaryCellDisplayText("0xdeadbeef", "VARBINARY(4)"), "VARBINARY [4 bytes]");
|
||||
assert.equal(binaryCellDisplayText("0x89504e47", "BLOB"), "BLOB [4 bytes]");
|
||||
assert.equal(binaryCellDisplayText(`0x${"00".repeat(2048)}`, "VARBINARY(2048)"), "VARBINARY [2.0 KB]");
|
||||
assert.equal(binaryCellDisplayText("0xffd8ffe000104a46...", "VARBINARY"), "VARBINARY [...]");
|
||||
assert.equal(binaryCellDisplayText("0x89504e47...", "LONGBLOB"), "BLOB [...]");
|
||||
assert.equal(binaryCellDisplayText("0xffd8ffe000104a46...", "VARBINARY", 25_143), "VARBINARY [25 KB]");
|
||||
assert.equal(binaryCellDisplayText("0x89504e47"), null);
|
||||
assert.equal(binaryCellDisplayText("0x", "VARBINARY(0)"), "");
|
||||
});
|
||||
@@ -101,7 +104,7 @@ test("DataGrid binary preview prefers ResultSet column types when table metadata
|
||||
const source = readFileSync("apps/desktop/src/components/grid/DataGrid.vue", "utf8");
|
||||
const formatter = source.match(/function formatCell\([^]*?\n\}/)?.[0] ?? "";
|
||||
|
||||
assert.match(formatter, /binaryCellDisplayText\(value, columnIndex === undefined \? undefined : allColumnTypes\.value\[columnIndex\]\)/);
|
||||
assert.match(formatter, /binaryCellDisplayText\(value, columnIndex === undefined \? undefined : allColumnTypes\.value\[columnIndex\], originalBytes\)/);
|
||||
});
|
||||
|
||||
test("binaryCellDownloadPayload decodes GBK text bytes", () => {
|
||||
|
||||
Reference in New Issue
Block a user