diff --git a/apps/desktop/src/components/grid/DataGrid.vue b/apps/desktop/src/components/grid/DataGrid.vue index 39680f799c..b2d7203c6c 100644 --- a/apps/desktop/src/components/grid/DataGrid.vue +++ b/apps/desktop/src/components/grid/DataGrid.vue @@ -4097,6 +4097,15 @@ function isLargeValuePreview(item: RowItem | undefined, columnIndex: number): bo return largeValueCellsByKey.value.has(largeValueCellKey(item.sourceIndex, columnIndex)); } +function largeValueOriginalBytes(item: Pick | undefined, columnIndex: number): number | undefined { + if (resolvedDatabaseType.value !== "mysql" || !item || item.isNew || item.isDraft || item.sourceIndex === undefined || item.isDirtyCol[columnIndex]) return undefined; + return largeValueCellsByKey.value.get(largeValueCellKey(item.sourceIndex, columnIndex))?.original_bytes; +} + +function formatGridItemCell(item: RowItem, columnIndex: number): string { + return formatCellCached(item.data[columnIndex], columnIndex, largeValueOriginalBytes(item, columnIndex)); +} + function normalizedLargeValueIdentityPart(value: CellValue): string { if (value === null) return "null"; return `${typeof value}:${JSON.stringify(value)}`; @@ -5385,7 +5394,7 @@ function primitiveCellFormatKey(value: CellValue, columnIndex?: number): string return `${columnIndex ?? -1}\u0000${typeof value}\u0000${String(value)}`; } -function formatCell(value: CellValue, columnIndex?: number): string { +function formatCell(value: CellValue, columnIndex?: number, originalBytes?: number): string { const formatter = columnIndex === undefined ? undefined : resolvedColumnFormatters.value[columnIndex]; const columnInfo = columnIndex === undefined ? undefined : tableColumnForGridColumn(columnIndex); const displayColumnInfo = columnInfo ?? (columnIndex === undefined ? undefined : resultColumnInfoForGridColumn(columnIndex)); @@ -5397,14 +5406,14 @@ function formatCell(value: CellValue, columnIndex?: number): string { columnInfo: displayColumnInfo, }); if (arrayDisplay !== undefined) return arrayDisplay; - const binaryDisplay = formatter ? null : binaryCellDisplayText(value, columnIndex === undefined ? undefined : allColumnTypes.value[columnIndex]); + const binaryDisplay = formatter ? null : binaryCellDisplayText(value, columnIndex === undefined ? undefined : allColumnTypes.value[columnIndex], originalBytes); if (binaryDisplay !== null) return binaryDisplay; const s = applyColumnFormatter(value, formatter); return limitDataGridCellDisplay(s, resolvedDatabaseType.value === "sqlserver" ? SQLSERVER_DATA_GRID_CELL_DISPLAY_MAX_LENGTH : undefined); } -function formatCellCached(value: CellValue, columnIndex?: number): string { - if (value !== null && typeof (value as unknown) === "object") { +function formatCellCached(value: CellValue, columnIndex?: number, originalBytes?: number): string { + if (originalBytes === undefined && value !== null && typeof (value as unknown) === "object") { const objectValue = value as unknown as object; const cacheColumn = columnIndex ?? -1; const columnCache = objectCellFormatCache.get(objectValue); @@ -5424,13 +5433,13 @@ function formatCellCached(value: CellValue, columnIndex?: number): string { // values are truncated for display anyway, so formatting them on demand is // cheaper than keeping the full source string in the primitive cache. if (typeof value === "string" && value.length > CELL_FORMAT_CACHE_STRING_KEY_MAX_LENGTH) { - return formatCell(value, columnIndex); + return formatCell(value, columnIndex, originalBytes); } - const key = primitiveCellFormatKey(value, columnIndex); + const key = `${primitiveCellFormatKey(value, columnIndex)}\u0000${originalBytes ?? ""}`; const cached = primitiveCellFormatCache.get(key); if (cached !== undefined) return cached; - return rememberPrimitiveCellFormat(key, formatCell(value, columnIndex)); + return rememberPrimitiveCellFormat(key, formatCell(value, columnIndex, originalBytes)); } function rowNumberPageOffset(): number { @@ -6207,7 +6216,7 @@ function drawCanvasGrid() { editingCell: editingCell.value, searchMatchKeys: searchMatchSet.value, currentSearchMatch: currentSearchMatch.value, - formatCell: formatCellCached, + formatCell: (value, columnIndex, row) => formatCellCached(value, columnIndex, largeValueOriginalBytes(row, columnIndex)), draftCellPlaceholder: t("grid.quickEntryDraftPlaceholder"), isRowActive, rowCellsUseSelectionVisual, @@ -6394,6 +6403,45 @@ async function cancelActiveExport() { await exportCancelHandler.value?.(); } +const userFacingSql = ref(""); +let userFacingSqlGeneration = 0; + +async function syncUserFacingSql() { + const generation = ++userFacingSqlGeneration; + const executionSql = props.sql?.trim() ?? ""; + if (props.context !== "table-data" || !executionSql.includes("__DBX_LARGE_VALUE_BYTES_") || !props.tableMeta?.tableName) { + userFacingSql.value = executionSql; + return; + } + + try { + const config = props.connectionId ? connectionStore.getConfig(props.connectionId) : undefined; + const sql = await buildTableSelectSql({ + databaseType: resolvedDatabaseType.value, + driverProfile: config?.driver_profile, + identifierQuote: props.connectionId ? connectionStore.connectionIdentifierQuote?.(props.connectionId) : undefined, + catalog: props.tableMeta.catalog, + database: props.tableMeta.database, + schema: props.tableMeta.schema, + tableName: props.tableMeta.tableName, + tableType: props.tableMeta.tableType, + whereInput: currentWhereInput(), + orderBy: currentOrderBy(), + limit: props.pageLimit ?? pageSize.value, + offset: props.pageOffset ?? Math.max(0, currentPage.value - 1) * pageSize.value, + }); + if (generation === userFacingSqlGeneration) userFacingSql.value = sql; + } catch { + if (generation === userFacingSqlGeneration) userFacingSql.value = executionSql; + } +} + +watch( + () => [props.sql, props.context, props.tableMeta, props.pageLimit, props.pageOffset, currentWhereInput(), currentOrderBy()], + () => void syncUserFacingSql(), + { immediate: true }, +); + // --- Export composable --- const { copyText, @@ -6422,7 +6470,6 @@ const { exportCurrentPageSql, exportTxt, exportCurrentPageTxt, - copySql, } = useDataGridExport({ columns: visibleColumns, displayItems: visibleDisplayItems, @@ -8665,7 +8712,11 @@ function onRowContext(rowId: number, rowIndex: number) { } } -const sqlOneLiner = computed(() => props.sql?.replace(/\s+/g, " ").trim() || ""); +const sqlOneLiner = computed(() => userFacingSql.value.replace(/\s+/g, " ").trim()); + +async function copyUserFacingSql() { + if (userFacingSql.value) await copyText(userFacingSql.value); +} type TableInfoTabItem = { id: TableInfoTab; @@ -11195,7 +11246,7 @@ const gridContextMenuItems = computed(() => { - +
(() => { - + {{ sqlOneLiner }} - 
{{ props.sql }}
+ 
{{ userFacingSql }}
 diff --git a/apps/desktop/src/lib/__tests__/dataGrid/dataGridLargeValues.spec.ts b/apps/desktop/src/lib/__tests__/dataGrid/dataGridLargeValues.spec.ts index 299014a79c..b45508d115 100644 --- a/apps/desktop/src/lib/__tests__/dataGrid/dataGridLargeValues.spec.ts +++ b/apps/desktop/src/lib/__tests__/dataGrid/dataGridLargeValues.spec.ts @@ -66,6 +66,31 @@ describe("data grid large-value metadata", () => { } }); + it("uses the active MySQL page budget to select bounded preview columns", () => { + const columns = [column("id", "int", true), column("image_mime", "varchar(64)"), column("image_data", "longblob"), column("image_url", "varchar(512)"), column("image_note", "varchar(255)")]; + + expect(tableDataLargeValuePreviewOptions("mysql", columns, ["id"], 10_000)).toEqual({ + columnTypes: columns.map((item) => item.data_type), + largeValuePreviewSize: 139, + }); + expect(tableDataLargeValuePreviewOptions("mysql", columns, ["id"], 100)).toEqual({ + columnTypes: columns.map((item) => item.data_type), + largeValuePreviewSize: TABLE_DATA_CELL_PREVIEW_SIZE, + }); + expect( + tableDataLargeValuePreviewOptions( + "mysql", + columns.filter((item) => item.name !== "image_data"), + ["id"], + 100, + ), + ).toEqual({}); + expect(tableDataLargeValuePreviewOptions("mysql", [column("id", "int", true), column("payload", "varchar(8000)")], ["id"], 10_000)).toEqual({ + columnTypes: ["int", "varchar(8000)"], + largeValuePreviewSize: 419, + }); + }); + it("offsets appended segment rows and discards metadata beyond the append cap", () => { expect( appendLargeValueCells( diff --git a/apps/desktop/src/lib/dataGrid/binaryCellDownload.ts b/apps/desktop/src/lib/dataGrid/binaryCellDownload.ts index a83c421863..60683f84c8 100644 --- a/apps/desktop/src/lib/dataGrid/binaryCellDownload.ts +++ b/apps/desktop/src/lib/dataGrid/binaryCellDownload.ts @@ -108,6 +108,7 @@ export function retainBinaryCellDownloadMenuForHover(openCell: BinaryCellPositio } const HEX_VALUE_RE = /^(?:0[xX]|\\x)([0-9a-fA-F\s]*)$/; +const TRUNCATED_HEX_VALUE_RE = /^(?:0[xX]|\\x)[0-9a-fA-F\s]+\.\.\.$/; const BARE_HEX_RE = /^[0-9a-fA-F\s]+$/; const HEX_ESCAPE_RE = /^(?:\\x[0-9a-fA-F]{2}|\s)+$/; const BINARY_TYPE_RE = /^(?:blob|tinyblob|mediumblob|longblob|bytea|bytes|binary|varbinary|image|raw|long\s+raw)(?:\b|\()/i; @@ -192,7 +193,11 @@ export function canDownloadBinaryCellValue(value: unknown, columnType?: string): return !!parseBinaryCellBytes(value, columnType); } -export function binaryCellDisplayText(value: unknown, columnType?: string): string | null { +export function binaryCellDisplayText(value: unknown, columnType?: string, originalBytes?: number): string | null { + if (typeof value === "string" && isBinaryCellColumnType(columnType) && TRUNCATED_HEX_VALUE_RE.test(value.trim())) { + const size = originalBytes === undefined ? "..." : formatBinaryCellByteSize(originalBytes); + return `${binaryCellDisplayLabel(columnType)} [${size}]`; + } const bytes = parseBinaryCellBytes(value, columnType); if (!bytes || !isBinaryCellColumnType(columnType)) return null; if (BINARY_STRING_TYPE_RE.test((columnType ?? "").trim())) { diff --git a/apps/desktop/src/lib/dataGrid/canvasDataGridRenderer.ts b/apps/desktop/src/lib/dataGrid/canvasDataGridRenderer.ts index 7f454673cb..109947fdd1 100644 --- a/apps/desktop/src/lib/dataGrid/canvasDataGridRenderer.ts +++ b/apps/desktop/src/lib/dataGrid/canvasDataGridRenderer.ts @@ -26,6 +26,7 @@ export interface CanvasDataGridRow { isDeleted: boolean; isDirtyCol: boolean[]; status: RowStatus; + sourceIndex?: number; } export interface CanvasHoverCell { @@ -77,7 +78,7 @@ export interface DrawCanvasDataGridOptions { editingCell: CanvasEditingCell | null; searchMatchKeys: ReadonlySet; currentSearchMatch: CanvasSearchMatch | null; - formatCell: (value: CellValue, columnIndex: number) => string; + formatCell: (value: CellValue, columnIndex: number, row: CanvasDataGridRow) => string; columnIsBoolean?: (columnIndex: number) => boolean; draftCellPlaceholder?: string; isRowActive: (rowIndex: number) => boolean; @@ -576,7 +577,7 @@ export function drawCanvasDataGrid(options: DrawCanvasDataGridOptions) { ctx.stroke(); } } else { - const rawDisplayText = item.isDraft && value === null ? (draftCellPlaceholder ?? "") : formatCell(value, actualColIdx); + const rawDisplayText = item.isDraft && value === null ? (draftCellPlaceholder ?? "") : formatCell(value, actualColIdx, item); const displayText = isEditingThisCell ? "" : firstLineCellDisplayValue(rawDisplayText, flatteningMultiLineEnabled); const text = isEditingThisCell ? displayText : fitCanvasText(ctx, displayText, cellMaxWidth, isBooleanNullCell ? "left" : isRightAlign ? "right" : "left"); const anchorX = isBooleanNullCell ? alignCanvasPixel(drawX + colWidth / 2, scaleX) : textAnchorX; diff --git a/apps/desktop/src/lib/dataGrid/dataGridLargeValues.ts b/apps/desktop/src/lib/dataGrid/dataGridLargeValues.ts index d48af57e58..a40f5dd131 100644 --- a/apps/desktop/src/lib/dataGrid/dataGridLargeValues.ts +++ b/apps/desktop/src/lib/dataGrid/dataGridLargeValues.ts @@ -7,6 +7,46 @@ export const TABLE_DATA_PREVIEW_CONTENT_MAX_BYTES = 24 * 1024 * 1024; export const TABLE_DATA_TEXT_SERIALIZED_BYTES_PER_CHARACTER = 6; const TABLE_DATA_LARGE_VALUE_MARKER_PREFIX = "__DBX_LARGE_VALUE_BYTES_"; +function normalizedDataTypeBase(dataType: string): string { + return dataType.trim().split(/[([]/, 1)[0]?.trim().toLocaleLowerCase() ?? ""; +} + +function declaredDataTypeLength(dataType: string): number | undefined { + const match = dataType.match(/\(\s*(\d+)/); + if (!match?.[1]) return undefined; + const length = Number.parseInt(match[1], 10); + return Number.isSafeInteger(length) ? length : undefined; +} + +function mysqlColumnNeedsLargeValuePreview(column: ColumnInfo, previewSize: number): boolean { + const base = normalizedDataTypeBase(column.data_type); + if (matchesMysqlUnboundedLargeValueType(base)) return true; + if (base !== "varchar" && base !== "varbinary") return false; + return (declaredDataTypeLength(column.data_type) ?? 0) > previewSize; +} + +function previewSizeForColumnCount(pageSize: number | undefined, previewColumnCount: number): number { + const budgetedSize = Math.floor(TABLE_DATA_PREVIEW_CONTENT_MAX_BYTES / Math.max(1, pageSize ?? 1) / previewColumnCount / TABLE_DATA_TEXT_SERIALIZED_BYTES_PER_CHARACTER); + return Math.max(TABLE_DATA_CELL_PREVIEW_MIN_SIZE, Math.min(TABLE_DATA_CELL_PREVIEW_SIZE, budgetedSize)); +} + +function mysqlPreviewBudget(columns: readonly ColumnInfo[], keyColumns: ReadonlySet, pageSize: number | undefined): { previewColumnCount: number; previewSize: number } | null { + let previewColumnCount = 1; + let previewSize = previewSizeForColumnCount(pageSize, previewColumnCount); + for (;;) { + const nextCount = columns.filter((column) => !keyColumns.has(column.name.toLocaleLowerCase()) && mysqlColumnNeedsLargeValuePreview(column, previewSize)).length; + if (nextCount === 0) return null; + const nextSize = previewSizeForColumnCount(pageSize, nextCount); + if (nextCount === previewColumnCount && nextSize === previewSize) return { previewColumnCount: nextCount, previewSize: nextSize }; + previewColumnCount = nextCount; + previewSize = nextSize; + } +} + +function matchesMysqlUnboundedLargeValueType(base: string): boolean { + return base === "blob" || base === "mediumblob" || base === "longblob" || base === "text" || base === "mediumtext" || base === "longtext" || base === "json"; +} + export function canUseTableDataLargeValuePreview(databaseType: DatabaseType | undefined, columns: readonly ColumnInfo[], primaryKeys: readonly string[]): boolean { return (databaseType === "mysql" || databaseType === "postgres") && columns.length > 0 && primaryKeys.length > 0 && !columns.some((column) => column.name.toLocaleUpperCase().startsWith(TABLE_DATA_LARGE_VALUE_MARKER_PREFIX)); } @@ -14,12 +54,12 @@ export function canUseTableDataLargeValuePreview(databaseType: DatabaseType | un export function tableDataLargeValuePreviewOptions(databaseType: DatabaseType | undefined, columns: readonly ColumnInfo[], primaryKeys: readonly string[], pageSize?: number): { columnTypes: string[]; largeValuePreviewSize: number } | Record { if (!canUseTableDataLargeValuePreview(databaseType, columns, primaryKeys)) return {}; const keyColumns = new Set(primaryKeys.map((column) => column.toLocaleLowerCase())); - const previewColumnCount = columns.filter((column) => !keyColumns.has(column.name.toLocaleLowerCase())).length; + const mysqlBudget = databaseType === "mysql" ? mysqlPreviewBudget(columns, keyColumns, pageSize) : null; + const previewColumnCount = databaseType === "mysql" ? (mysqlBudget?.previewColumnCount ?? 0) : columns.filter((column) => !keyColumns.has(column.name.toLocaleLowerCase())).length; if (previewColumnCount === 0) return {}; - const budgetedSize = Math.floor(TABLE_DATA_PREVIEW_CONTENT_MAX_BYTES / Math.max(1, pageSize ?? 1) / previewColumnCount / TABLE_DATA_TEXT_SERIALIZED_BYTES_PER_CHARACTER); return { columnTypes: columns.map((column) => column.data_type), - largeValuePreviewSize: Math.max(TABLE_DATA_CELL_PREVIEW_MIN_SIZE, Math.min(TABLE_DATA_CELL_PREVIEW_SIZE, budgetedSize)), + largeValuePreviewSize: mysqlBudget?.previewSize ?? previewSizeForColumnCount(pageSize, previewColumnCount), }; } diff --git a/crates/dbx-core/src/db/mysql.rs b/crates/dbx-core/src/db/mysql.rs index 1c1d563916..9cbb3014d0 100644 --- a/crates/dbx-core/src/db/mysql.rs +++ b/crates/dbx-core/src/db/mysql.rs @@ -606,17 +606,17 @@ fn mysql_result_cell_preview_bytes( } fn mysql_column_can_use_bounded_preview(column: &mysql_async::Column) -> bool { - matches!( - column.column_type(), + match column.column_type() { ColumnType::MYSQL_TYPE_JSON - | ColumnType::MYSQL_TYPE_BLOB - | ColumnType::MYSQL_TYPE_LONG_BLOB - | ColumnType::MYSQL_TYPE_MEDIUM_BLOB - | ColumnType::MYSQL_TYPE_TINY_BLOB - | ColumnType::MYSQL_TYPE_STRING - | ColumnType::MYSQL_TYPE_VAR_STRING - | ColumnType::MYSQL_TYPE_VARCHAR - ) + | ColumnType::MYSQL_TYPE_BLOB + | ColumnType::MYSQL_TYPE_LONG_BLOB + | ColumnType::MYSQL_TYPE_MEDIUM_BLOB => true, + ColumnType::MYSQL_TYPE_TINY_BLOB => false, + ColumnType::MYSQL_TYPE_STRING | ColumnType::MYSQL_TYPE_VAR_STRING | ColumnType::MYSQL_TYPE_VARCHAR => { + column.column_length() > 8 * 1024 + } + _ => false, + } } fn mysql_bounded_value_preview( @@ -665,7 +665,7 @@ fn mysql_result_protected_column_indexes( protected_columns: &[String], ) -> HashSet { let protected: HashSet = protected_columns.iter().map(|column| column.to_ascii_lowercase()).collect(); - columns + let mut indexes = columns .iter() .enumerate() .filter_map(|(index, column)| { @@ -673,7 +673,16 @@ fn mysql_result_protected_column_indexes( || protected.contains(&column.name_str().to_ascii_lowercase())) .then_some(index) }) - .collect() + .collect::>(); + for (index, column) in columns.iter().enumerate() { + if column.name_str().to_ascii_uppercase().starts_with(crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX) { + indexes.insert(index); + if index > 0 { + indexes.insert(index - 1); + } + } + } + indexes } fn mysql_row_to_json_with_srids( @@ -6410,6 +6419,46 @@ mod tests { assert_eq!(cells, vec![LargeValueCell { row_index: 0, column_index: 1, original_bytes: 4096 }]); } + #[test] + fn mysql_large_value_preview_skips_bounded_strings_and_server_preview_columns() { + let ordinary_column = Column::new(ColumnType::MYSQL_TYPE_VAR_STRING) + .with_name(b"image_url") + .with_character_set(45) + .with_column_length(2048); + let large_column = Column::new(ColumnType::MYSQL_TYPE_VAR_STRING) + .with_name(b"large_note") + .with_character_set(45) + .with_column_length(40_000); + let server_preview_column = Column::new(ColumnType::MYSQL_TYPE_VAR_STRING) + .with_name(b"image_data") + .with_character_set(63) + .with_column_length(1680); + let marker_column = Column::new(ColumnType::MYSQL_TYPE_VAR_STRING) + .with_name(format!("{}B_2", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX).as_bytes()) + .with_character_set(45) + .with_column_length(64); + let row = mysql_test_row_with_columns( + vec![ + Value::Bytes(vec![b'u'; 500]), + Value::Bytes(vec![b'n'; 10_000]), + Value::Bytes(vec![0xab; 420]), + Value::Bytes(b"B:419:25143".to_vec()), + ], + vec![ordinary_column, large_column, server_preview_column, marker_column], + ); + let protected = mysql_result_protected_column_indexes(row.columns_ref(), &[]); + let mut spatial_columns = mysql_spatial_column_builder(row.columns_ref()); + + let (values, _srids, cells) = + mysql_row_to_json_with_srids_and_previews(&row, &mut spatial_columns, 0, Some(256), &protected); + + assert_eq!(values[0].as_str().map(str::len), Some(500)); + assert!(values[1].as_str().is_some_and(|value| value.ends_with("..."))); + assert_eq!(values[2].as_str().map(str::len), Some(2 + 420 * 2)); + assert_eq!(values[3], serde_json::json!("B:419:25143")); + assert_eq!(cells, vec![LargeValueCell { row_index: 0, column_index: 1, original_bytes: 10_000 }]); + } + #[test] fn mysql_large_binary_preview_is_bounded_hex() { let payload_column = Column::new(ColumnType::MYSQL_TYPE_LONG_BLOB) diff --git a/crates/dbx-core/src/query.rs b/crates/dbx-core/src/query.rs index 09f4ffec83..4f5d20940a 100644 --- a/crates/dbx-core/src/query.rs +++ b/crates/dbx-core/src/query.rs @@ -336,6 +336,13 @@ struct ServerLargeValueMarker { source_type: Option<&'static str>, } +#[derive(Clone, Copy)] +struct ServerLargeValueMarkerValue { + kind: ServerLargeValuePreviewKind, + preview_size: usize, + original_bytes: Option, +} + fn server_large_value_alias( suffix: &str, ) -> Option<(usize, Option, Option<&'static str>)> { @@ -360,8 +367,10 @@ fn server_large_value_alias( Some((source_index, Some(preview_kind), source_type)) } -fn server_large_value_marker(value: &serde_json::Value) -> Option<(ServerLargeValuePreviewKind, usize)> { - let (kind, preview_size) = value.as_str()?.split_once(':')?; +fn server_large_value_marker(value: &serde_json::Value) -> Option { + let mut parts = value.as_str()?.split(':'); + let kind = parts.next()?; + let preview_size = parts.next()?; let kind = match kind { "T" => ServerLargeValuePreviewKind::Text, "B" => ServerLargeValuePreviewKind::Binary, @@ -369,7 +378,8 @@ fn server_large_value_marker(value: &serde_json::Value) -> Option<(ServerLargeVa "D" => ServerLargeValuePreviewKind::Deferred, _ => return None, }; - Some((kind, preview_size.parse::().ok()?.max(1))) + let original_bytes = parts.next().and_then(|value| value.parse::().ok()); + Some(ServerLargeValueMarkerValue { kind, preview_size: preview_size.parse::().ok()?.max(1), original_bytes }) } fn truncate_server_large_value_preview( @@ -445,7 +455,7 @@ fn extract_server_large_value_markers(result: &mut db::QueryResult) -> Vec Vec String { data_type.trim().split(['(', '[']).next().unwrap_or_default().trim().to_ascii_lowercase() } -fn large_value_preview_kind(database_type: Option, data_type: &str) -> Option { +fn declared_data_type_length(data_type: &str) -> Option { + let parameters = data_type.split_once('(')?.1; + let digits = parameters.trim_start().chars().take_while(char::is_ascii_digit).collect::(); + (!digits.is_empty()).then(|| digits.parse::().ok()).flatten() +} + +fn large_value_preview_kind( + database_type: Option, + data_type: &str, + preview_size: usize, +) -> Option { let normalized = data_type.trim().to_ascii_lowercase(); let base = normalized_data_type_base(data_type); match database_type { Some(DatabaseType::Mysql) => { - if matches!(base.as_str(), "binary" | "varbinary" | "tinyblob" | "blob" | "mediumblob" | "longblob") { + if matches!(base.as_str(), "blob" | "mediumblob" | "longblob") + || (base == "varbinary" + && declared_data_type_length(data_type).is_some_and(|length| length > preview_size)) + { Some(LargeValuePreviewKind::Binary) } else if base == "json" { Some(LargeValuePreviewKind::TextCast) - } else if matches!(base.as_str(), "char" | "varchar" | "tinytext" | "text" | "mediumtext" | "longtext") { + } else if matches!(base.as_str(), "text" | "mediumtext" | "longtext") + || (base == "varchar" + && declared_data_type_length(data_type).is_some_and(|length| length > preview_size)) + { Some(LargeValuePreviewKind::Text) } else { None @@ -101,7 +117,7 @@ fn build_large_value_preview_columns(options: &TableDataSelectSqlOptions) -> Opt quote_table_identifier(database_type, column) }; let kind = (!protected.contains(&column.to_ascii_lowercase())) - .then(|| large_value_preview_kind(database_type, data_type)) + .then(|| large_value_preview_kind(database_type, data_type, preview_size)) .flatten(); let Some(kind) = kind else { projections.push(quoted); @@ -131,7 +147,11 @@ fn build_large_value_preview_columns(options: &TableDataSelectSqlOptions) -> Opt Some(DatabaseType::Postgres) => (format!("left({quoted}, {prefix_size}) AS {quoted}"), "T"), _ => return None, }; - let marker = format!("'{marker_kind}:{preview_size}' AS {marker_alias}"); + let marker = if database_type == Some(DatabaseType::Mysql) { + format!("CONCAT('{marker_kind}:{preview_size}:', OCTET_LENGTH({quoted})) AS {marker_alias}") + } else { + format!("'{marker_kind}:{preview_size}' AS {marker_alias}") + }; projections.push(preview); projections.push(marker); marker_count += 1; diff --git a/crates/dbx-core/src/sql_dialect/tests.rs b/crates/dbx-core/src/sql_dialect/tests.rs index 6d5fe0b947..738987d2b7 100644 --- a/crates/dbx-core/src/sql_dialect/tests.rs +++ b/crates/dbx-core/src/sql_dialect/tests.rs @@ -705,14 +705,50 @@ fn builds_mysql_table_data_large_value_previews_without_truncating_keys() { }); assert!(sql.starts_with("SELECT `id`, LEFT(`payload`, 4097) AS `payload`")); - assert!(sql.contains("'T:4096' AS `__DBX_LARGE_VALUE_BYTES_T_1`")); + assert!(sql.contains("CONCAT('T:4096:', OCTET_LENGTH(`payload`)) AS `__DBX_LARGE_VALUE_BYTES_T_1`")); assert!(sql.contains("LEFT(`raw_value`, 4097) AS `raw_value`")); - assert!(sql.contains("'B:4096' AS `__DBX_LARGE_VALUE_BYTES_B_2`")); + assert!(sql.contains("CONCAT('B:4096:', OCTET_LENGTH(`raw_value`)) AS `__DBX_LARGE_VALUE_BYTES_B_2`")); assert!(sql.contains("LEFT(`metadata`, 4097) AS `metadata`")); - assert!(sql.contains("'T:4096' AS `__DBX_LARGE_VALUE_BYTES_J_3`")); + assert!(sql.contains("CONCAT('T:4096:', OCTET_LENGTH(`metadata`)) AS `__DBX_LARGE_VALUE_BYTES_J_3`")); assert!(!sql.contains("__DBX_LARGE_VALUE_BYTES_0")); } +#[test] +fn previews_mysql_bounded_string_columns_only_above_the_active_budget() { + let sql = build_table_data_select_sql(TableDataSelectSqlOptions { + database_type: Some(DatabaseType::Mysql), + table_name: "t_0001".to_string(), + primary_keys: vec!["id".to_string()], + columns: vec![ + "id".to_string(), + "image_mime".to_string(), + "image_data".to_string(), + "image_url".to_string(), + "large_note".to_string(), + "large_binary".to_string(), + ], + column_types: vec![ + "int".to_string(), + "varchar(64)".to_string(), + "longblob".to_string(), + "varchar(512)".to_string(), + "varchar(10000)".to_string(), + "varbinary(10000)".to_string(), + ], + large_value_preview_size: Some(419), + limit: Some(10_000), + ..Default::default() + }); + + assert!(sql.starts_with("SELECT `id`, `image_mime`, LEFT(`image_data`, 420) AS `image_data`")); + assert!(sql.contains("CONCAT('B:419:', OCTET_LENGTH(`image_data`)) AS `__DBX_LARGE_VALUE_BYTES_B_2`, LEFT(`image_url`, 420) AS `image_url`")); + assert!(sql.contains("CONCAT('T:419:', OCTET_LENGTH(`image_url`)) AS `__DBX_LARGE_VALUE_BYTES_T_3`")); + assert!(sql.contains("LEFT(`large_note`, 420) AS `large_note`")); + assert!(sql.contains("CONCAT('T:419:', OCTET_LENGTH(`large_note`)) AS `__DBX_LARGE_VALUE_BYTES_T_4`")); + assert!(sql.contains("LEFT(`large_binary`, 420) AS `large_binary`")); + assert!(!sql.contains("LEFT(`image_mime`")); +} + #[test] fn builds_postgres_table_data_large_value_previews() { let sql = build_table_data_select_sql(TableDataSelectSqlOptions { diff --git a/packages/app-tests/binaryCellDownload.test.ts b/packages/app-tests/binaryCellDownload.test.ts index 7183676564..9a2f243ba2 100644 --- a/packages/app-tests/binaryCellDownload.test.ts +++ b/packages/app-tests/binaryCellDownload.test.ts @@ -75,6 +75,9 @@ test("binaryCellDisplayText previews printable binary strings without changing t assert.equal(binaryCellDisplayText("0xdeadbeef", "VARBINARY(4)"), "VARBINARY [4 bytes]"); assert.equal(binaryCellDisplayText("0x89504e47", "BLOB"), "BLOB [4 bytes]"); assert.equal(binaryCellDisplayText(`0x${"00".repeat(2048)}`, "VARBINARY(2048)"), "VARBINARY [2.0 KB]"); + assert.equal(binaryCellDisplayText("0xffd8ffe000104a46...", "VARBINARY"), "VARBINARY [...]"); + assert.equal(binaryCellDisplayText("0x89504e47...", "LONGBLOB"), "BLOB [...]"); + assert.equal(binaryCellDisplayText("0xffd8ffe000104a46...", "VARBINARY", 25_143), "VARBINARY [25 KB]"); assert.equal(binaryCellDisplayText("0x89504e47"), null); assert.equal(binaryCellDisplayText("0x", "VARBINARY(0)"), ""); }); @@ -101,7 +104,7 @@ test("DataGrid binary preview prefers ResultSet column types when table metadata const source = readFileSync("apps/desktop/src/components/grid/DataGrid.vue", "utf8"); const formatter = source.match(/function formatCell\([^]*?\n\}/)?.[0] ?? ""; - assert.match(formatter, /binaryCellDisplayText\(value, columnIndex === undefined \? undefined : allColumnTypes\.value\[columnIndex\]\)/); + assert.match(formatter, /binaryCellDisplayText\(value, columnIndex === undefined \? undefined : allColumnTypes\.value\[columnIndex\], originalBytes\)/); }); test("binaryCellDownloadPayload decodes GBK text bytes", () => {