diff --git a/apps/desktop/src/components/grid/DataGrid.vue b/apps/desktop/src/components/grid/DataGrid.vue index 83412d4d2e..651992b74b 100644 --- a/apps/desktop/src/components/grid/DataGrid.vue +++ b/apps/desktop/src/components/grid/DataGrid.vue @@ -151,7 +151,7 @@ import { isCancelSearchShortcut, isCopyCurrentRowShortcut, isDeleteCurrentRowSho import { dataGridHeaderContentWidth, scrollbarGutterWidth } from "@/lib/dataGrid/dataGridScrollGutter"; import { canFetchNextDataGridSegment, canGoNextDataGridPage, dataGridTotalRowCountLabelKey, dataGridTruncationHintKey, hasCompleteLocalDataGridResult, resolveDataGridPaginationTotal, type DataGridInexactTotalRowCountMode } from "@/lib/dataGrid/dataGridPagination"; import { dataGridCountQueryOptions } from "@/lib/dataGrid/dataGridQueryOptions"; -import { largeValueCellKey, largeValueCellMap } from "@/lib/dataGrid/dataGridLargeValues"; +import { largeValueCellKey, largeValueCellMap, tableDataLargeValuePreviewOptions } from "@/lib/dataGrid/dataGridLargeValues"; import { dataGridBottomScrollTop, dataGridScrollPosition, isDataGridAtScrollBottom, isDataGridNearScrollBottom, isDataGridPrefixAppend, shouldCheckInfiniteScrollAfterScroll, type DataGridScrollPosition } from "@/lib/dataGrid/dataGridInfiniteScroll"; import { CANVAS_DATA_GRID_ROW_HEIGHT, MAX_CANVAS_DATA_GRID_PIXEL_RATIO, canvasDataGridActionOverlayWidth, canvasDataGridActionReservedWidth, dataGridSearchMatchKey, drawCanvasDataGrid, type CanvasDevicePixelSize } from "@/lib/dataGrid/canvasDataGridRenderer"; import { DATA_GRID_DARK_STRIPED_ROW_BG, DATA_GRID_LIGHT_STRIPED_ROW_BG, dataGridActiveRowBackground } from "@/lib/dataGrid/dataGridPaintTheme"; @@ -4141,7 +4141,7 @@ async function resolveLargeValueCells(rowIds: number[], columnIndexes: number[]) } } if (requestsByColumn.size === 0) return resolved; - if (resolvedDatabaseType.value !== "mysql" || !props.connectionId || !props.tableMeta?.tableName || props.tableMeta.primaryKeys.length === 0) { + if ((resolvedDatabaseType.value !== "mysql" && resolvedDatabaseType.value !== "postgres") || !props.connectionId || !props.tableMeta?.tableName || props.tableMeta.primaryKeys.length === 0) { throw new Error(t("grid.largeValueNeedsStableKey")); } const primaryKeyIndexes = props.tableMeta.primaryKeys.map(largeValueSourceColumnIndex); @@ -5224,6 +5224,7 @@ async function applyOrderBySearch() { tableType: tableMeta.tableType, columns: tableMeta.columns.map((column) => column.name), primaryKeys: tableMeta.primaryKeys, + ...tableDataLargeValuePreviewOptions(resolvedDatabaseType.value, tableMeta.columns, tableMeta.primaryKeys, pageSize.value), orderBy: orderByClause, limit: pageSize.value, whereInput: currentWhereInput(), @@ -5259,6 +5260,7 @@ async function applyWhereFilter() { tableType: tableMeta.tableType, columns: tableMeta.columns.map((column) => column.name), primaryKeys: tableMeta.primaryKeys, + ...tableDataLargeValuePreviewOptions(resolvedDatabaseType.value, tableMeta.columns, tableMeta.primaryKeys, pageSize.value), orderBy: orderByInput.value.trim() || (sortCol.value ? `${queryColumnRef(sortCol.value)} ${sortDir.value.toUpperCase()}` : undefined), limit: pageSize.value, whereInput, diff --git a/apps/desktop/src/composables/__tests__/useNavigationTargets.spec.ts b/apps/desktop/src/composables/__tests__/useNavigationTargets.spec.ts index 134e214825..58e5ed2c90 100644 --- a/apps/desktop/src/composables/__tests__/useNavigationTargets.spec.ts +++ b/apps/desktop/src/composables/__tests__/useNavigationTargets.spec.ts @@ -6,6 +6,7 @@ import type { QueryTab } from "@/types/database"; const mocks = vi.hoisted(() => ({ tabs: [] as QueryTab[], activeTabId: "", + buildTableSelectSql: vi.fn(), databaseType: "postgres" as string, ensureConnected: vi.fn(), executeTabSql: vi.fn(), @@ -83,7 +84,7 @@ vi.mock("@/stores/settingsStore", () => ({ })); vi.mock("@/lib/table/tableSelectSql", () => ({ - buildTableSelectSql: async ({ tableName }: { tableName: string }) => `SELECT * FROM ${tableName}`, + buildTableSelectSql: mocks.buildTableSelectSql, })); const dialogs = { @@ -103,12 +104,13 @@ describe("useNavigationTargets openTableTarget", () => { mocks.tabs.length = 0; mocks.databaseType = "postgres"; mocks.ensureConnected.mockResolvedValue(undefined); + mocks.buildTableSelectSql.mockImplementation(async ({ tableName }: { tableName: string }) => `SELECT * FROM ${tableName}`); mocks.executeTabSql.mockResolvedValue(undefined); mocks.getColumns.mockResolvedValue([column("id")]); mocks.listIndexes.mockResolvedValue([]); }); - it("marks row identity pending until real metadata lands", async () => { + it("loads stable row identity before the first PostgreSQL table query", async () => { let releaseColumns: (columns: unknown[]) => void = () => {}; mocks.getColumns.mockReturnValueOnce( new Promise((resolve) => { @@ -124,10 +126,15 @@ describe("useNavigationTargets openTableTarget", () => { releaseColumns([column("id")]); await open; - // 数据查询执行期间元数据未落地：行标识等待必须已挂起 - expect(pendingDuringQuery).toBe(true); + expect(pendingDuringQuery).toBe(false); expect(mocks.tabs[0]?.tableMeta?.primaryKeys).toEqual(["id"]); expect(mocks.tabs[0]?.tableMetaPending).toBe(false); + expect(mocks.buildTableSelectSql).toHaveBeenCalledWith( + expect.objectContaining({ + columns: ["id"], + primaryKeys: ["id"], + }), + ); }); it("reuses cached metadata and force-refreshes it once per catalog after a structure save", async () => { diff --git a/apps/desktop/src/composables/useDataGridActions.ts b/apps/desktop/src/composables/useDataGridActions.ts index ef5560c019..7fdf887aa5 100644 --- a/apps/desktop/src/composables/useDataGridActions.ts +++ b/apps/desktop/src/composables/useDataGridActions.ts @@ -5,6 +5,7 @@ import { useQueryStore } from "@/stores/queryStore"; import { useSettingsStore } from "@/stores/settingsStore"; import { buildTableSelectSql, quoteTableDataIdentifier } from "@/lib/table/tableSelectSql"; import { tableOpenPageLimit } from "@/lib/table/tableOpenPageLimit"; +import { tableDataLargeValuePreviewOptions } from "@/lib/dataGrid/dataGridLargeValues"; import { usesSyntheticRowIdKey } from "@/lib/table/tableEditing"; import { tableMetaForDataTab } from "@/lib/table/tableDataTabMeta"; import * as api from "@/lib/backend/api"; @@ -65,6 +66,7 @@ export function useDataGridActions(activeTab: ComputedRef) // 结果（可能是失败结果的 ["Error"]），进入 SQL 会生成非法投影； // 真实列缺失时省略 columns 让 builder 生成 SELECT * const realColumns = tab.tableMeta?.columns.length ? tab.tableMeta.columns : undefined; + const limit = options.limit ?? tab.resultPageLimit ?? tableOpenPageLimit(settingsStore.editorSettings.tableOpenPageSize); return buildTableSelectSql({ databaseType: effectiveDbType, driverProfile: config?.driver_profile, @@ -76,8 +78,9 @@ export function useDataGridActions(activeTab: ComputedRef) catalog: tableMeta?.catalog, columns: realColumns?.map((column) => column.name), primaryKeys, + ...tableDataLargeValuePreviewOptions(effectiveDbType, realColumns ?? [], primaryKeys, limit), includeRowId: useRowId, - limit: options.limit ?? tab.resultPageLimit ?? tableOpenPageLimit(settingsStore.editorSettings.tableOpenPageSize), + limit, ...options, }); } diff --git a/apps/desktop/src/composables/useNavigationTargets.ts b/apps/desktop/src/composables/useNavigationTargets.ts index 209e7cf466..386975b780 100644 --- a/apps/desktop/src/composables/useNavigationTargets.ts +++ b/apps/desktop/src/composables/useNavigationTargets.ts @@ -4,6 +4,7 @@ import { invalidateTableMetadataCache, loadTableMetadata } from "@/lib/metadata/ import { canApplyDataTabMetadata } from "@/lib/sidebar/dataTabOpenPolicy"; import { isNoSnapshotErrorResult, isQueryExecutionErrorResult } from "@/lib/query/queryResultError"; import { buildTableSelectSql } from "@/lib/table/tableSelectSql"; +import { tableDataLargeValuePreviewOptions } from "@/lib/dataGrid/dataGridLargeValues"; import { editableRowIdentifierColumns, usesSyntheticRowIdKey } from "@/lib/table/tableEditing"; import { tableOpenPageLimit } from "@/lib/table/tableOpenPageLimit"; import { uuid } from "@/lib/common/utils"; @@ -136,6 +137,21 @@ async function openTableTarget(target: NavigationTarget, options: { tableInfoTab await queryStore.executeTabSql(tabId, sql, { pagination: { limit: pageLimit, offset: 0 } }); return; } + const eagerMetadata = + effectiveDbType === "mysql" || effectiveDbType === "postgres" + ? await loadTableMetadata({ + connectionId: target.connectionId, + database: target.database, + schema: querySchema, + tableName: target.tableName, + tableType: targetTableType, + databaseType: effectiveDbType, + driverProfile: config.driver_profile || config.db_type, + catalog: target.catalog, + }) + : undefined; + const eagerColumns = eagerMetadata?.metadata.columns ?? []; + const eagerPrimaryKeys = eagerMetadata?.metadata.primaryKeys ?? []; const sql = await buildTableSelectSql({ databaseType: effectiveDbType, driverProfile: config.driver_profile, @@ -145,6 +161,9 @@ async function openTableTarget(target: NavigationTarget, options: { tableInfoTab database: target.database, tableName: target.tableName, tableType: targetTableType, + columns: eagerColumns.map((column) => column.name), + primaryKeys: eagerPrimaryKeys, + ...tableDataLargeValuePreviewOptions(effectiveDbType, eagerColumns, eagerPrimaryKeys, pageLimit), whereInput: target.whereInput, limit: pageLimit, }); @@ -156,8 +175,8 @@ async function openTableTarget(target: NavigationTarget, options: { tableInfoTab database: target.database, tableName: target.tableName, tableType: targetTableType, - columns: [], - primaryKeys: [], + columns: eagerColumns, + primaryKeys: eagerPrimaryKeys, }); firstExecuteStarted = true; // 取消计数快照：isCancelling 是瞬态的（取消失败/查询先完成会被清掉）， diff --git a/apps/desktop/src/composables/useSidebarDataOpenRuntime.ts b/apps/desktop/src/composables/useSidebarDataOpenRuntime.ts index 6df69b4587..bceea3f0bf 100644 --- a/apps/desktop/src/composables/useSidebarDataOpenRuntime.ts +++ b/apps/desktop/src/composables/useSidebarDataOpenRuntime.ts @@ -13,6 +13,7 @@ import { hasTreeNodeDatabaseContext } from "@/lib/sidebar/treeNodeContext"; import { buildTableSelectSql } from "@/lib/table/tableSelectSql"; import { usesSyntheticRowIdKey } from "@/lib/table/tableEditing"; import { tableOpenPageLimit } from "@/lib/table/tableOpenPageLimit"; +import { tableDataLargeValuePreviewOptions } from "@/lib/dataGrid/dataGridLargeValues"; import { canActivateExistingDataTableTab } from "@/lib/tabs/dataTabActivation"; import { beginDataTabNavigation, endDataTabNavigation, isCurrentDataTabNavigation } from "@/lib/tabs/dataTabNavigationGeneration"; @@ -293,18 +294,23 @@ export function useSidebarDataOpenRuntime() { } else if (deferTableMetaRefresh) { logPhase("metadata-deferred", { tabId }); } else { - void refreshTableMetaInBackground(tabId); logPhase("metadata-started", { tabId }); } + const metadataRefresh = shouldRefreshTableMeta && !deferTableMetaRefresh ? refreshTableMetaInBackground(tabId) : undefined; + if (!cachedTableMeta && (effectiveDbType === "mysql" || effectiveDbType === "postgres")) { + await metadataRefresh; + } + // Check if superseded by a newer openData call if (!isActive()) { logPhase("superseded-before-build-sql", { tabId }); return; } - const columns = cachedTableMeta?.columns ?? []; - const primaryKeys = cachedTableMeta?.primaryKeys ?? []; + const loadedTableMeta = cachedTableMeta ?? queryStore.tabs.find((item) => item.id === tabId)?.tableMeta; + const columns = loadedTableMeta?.columns ?? []; + const primaryKeys = loadedTableMeta?.primaryKeys ?? []; const includeRowId = usesSyntheticRowIdKey(effectiveDbType, primaryKeys, tableType); const sql = await buildTableSelectSql({ databaseType: effectiveDbType, @@ -317,6 +323,7 @@ export function useSidebarDataOpenRuntime() { catalog: node.catalog, columns: columns.map((column) => column.name), primaryKeys, + ...tableDataLargeValuePreviewOptions(effectiveDbType, columns, primaryKeys, limit), limit, includeRowId, }); diff --git a/apps/desktop/src/lib/__tests__/dataGrid/dataGridLargeValues.spec.ts b/apps/desktop/src/lib/__tests__/dataGrid/dataGridLargeValues.spec.ts index d7391131bd..cee5320b00 100644 --- a/apps/desktop/src/lib/__tests__/dataGrid/dataGridLargeValues.spec.ts +++ b/apps/desktop/src/lib/__tests__/dataGrid/dataGridLargeValues.spec.ts @@ -1,8 +1,63 @@ import { describe, expect, it } from "vitest"; -import { appendLargeValueCells, largeValueCellMap, remapLargeValueCells } from "@/lib/dataGrid/dataGridLargeValues"; +import { appendLargeValueCells, canUseTableDataLargeValuePreview, largeValueCellMap, remapLargeValueCells, TABLE_DATA_CELL_PREVIEW_SIZE, TABLE_DATA_PREVIEW_CONTENT_MAX_BYTES, tableDataLargeValuePreviewOptions } from "@/lib/dataGrid/dataGridLargeValues"; import { buildDataGridCellDetail } from "@/lib/dataGrid/dataGridDetail"; +import type { ColumnInfo } from "@/types/database"; + +function column(name: string, dataType: string, isPrimaryKey = false): ColumnInfo { + return { + name, + data_type: dataType, + is_nullable: false, + column_default: null, + is_primary_key: isPrimaryKey, + extra: null, + }; +} describe("data grid large-value metadata", () => { + it("enables bounded table previews only for supported databases with stable keys", () => { + const columns = [column("id", "bigint", true), column("payload", "longtext")]; + + expect(tableDataLargeValuePreviewOptions("mysql", columns, ["id"], 100)).toEqual({ + columnTypes: ["bigint", "longtext"], + largeValuePreviewSize: TABLE_DATA_CELL_PREVIEW_SIZE, + }); + expect(tableDataLargeValuePreviewOptions("postgres", columns, ["id"], 100)).toEqual({ + columnTypes: ["bigint", "longtext"], + largeValuePreviewSize: TABLE_DATA_CELL_PREVIEW_SIZE, + }); + expect(tableDataLargeValuePreviewOptions("sqlite", columns, ["id"], 100)).toEqual({}); + expect(tableDataLargeValuePreviewOptions("mysql", columns, [], 100)).toEqual({}); + expect(tableDataLargeValuePreviewOptions("mysql", [columns[0]!], ["id"], 100)).toEqual({}); + }); + + it("disables marker parsing when a real column uses the reserved preview prefix", () => { + const columns = [column("id", "bigint", true), column("__dbx_large_value_bytes_t_0", "text")]; + + expect(canUseTableDataLargeValuePreview("postgres", columns, ["id"])).toBe(false); + expect(tableDataLargeValuePreviewOptions("postgres", columns, ["id"], 100)).toEqual({}); + }); + + it("shrinks each cell preview by serialized byte budget without changing page size", () => { + const columns = [column("id", "bigint", true), ...Array.from({ length: 256 }, (_, index) => column(`payload_${index}`, "text"))]; + const options = tableDataLargeValuePreviewOptions("postgres", columns, ["ID"], 1_000); + + expect(options).toEqual({ + columnTypes: columns.map((item) => item.data_type), + largeValuePreviewSize: 16, + }); + + if (!("largeValuePreviewSize" in options)) throw new Error("expected preview options"); + const previewSize = options.largeValuePreviewSize; + const previewCellCount = 1_000 * 256; + const serializedSamples = ["中".repeat(previewSize), "😀".repeat(previewSize), "\u0000".repeat(previewSize)]; + for (const sample of serializedSamples) { + const serializedContent = JSON.stringify(sample).slice(1, -1); + const serializedBytes = new TextEncoder().encode(serializedContent).byteLength * previewCellCount; + expect(serializedBytes).toBeLessThanOrEqual(TABLE_DATA_PREVIEW_CONTENT_MAX_BYTES); + } + }); + it("offsets appended segment rows and discards metadata beyond the append cap", () => { expect( appendLargeValueCells( diff --git a/apps/desktop/src/lib/backend/http.ts b/apps/desktop/src/lib/backend/http.ts index e71bb58d44..dd0db43e92 100644 --- a/apps/desktop/src/lib/backend/http.ts +++ b/apps/desktop/src/lib/backend/http.ts @@ -929,6 +929,7 @@ export async function executeMulti( pageSize?: number; maxResultBytes?: number; resultKeyColumns?: string[]; + tableDataPreview?: boolean; resultSessionId?: string; clientSessionId?: string; timeoutSecs?: number; @@ -971,6 +972,7 @@ export async function executeMultiWithProgress( pageSize?: number; maxResultBytes?: number; resultKeyColumns?: string[]; + tableDataPreview?: boolean; resultSessionId?: string; clientSessionId?: string; timeoutSecs?: number; diff --git a/apps/desktop/src/lib/backend/tauri.ts b/apps/desktop/src/lib/backend/tauri.ts index 76737d6a5a..949620db76 100644 --- a/apps/desktop/src/lib/backend/tauri.ts +++ b/apps/desktop/src/lib/backend/tauri.ts @@ -1148,6 +1148,7 @@ export async function executeMulti( pageSize?: number; maxResultBytes?: number; resultKeyColumns?: string[]; + tableDataPreview?: boolean; resultSessionId?: string; clientSessionId?: string; timeoutSecs?: number; @@ -1194,6 +1195,7 @@ export async function executeMultiWithProgress( pageSize?: number; maxResultBytes?: number; resultKeyColumns?: string[]; + tableDataPreview?: boolean; resultSessionId?: string; clientSessionId?: string; timeoutSecs?: number; diff --git a/apps/desktop/src/lib/dataGrid/dataGridLargeValues.ts b/apps/desktop/src/lib/dataGrid/dataGridLargeValues.ts index f2e3739709..3a9a39b6c9 100644 --- a/apps/desktop/src/lib/dataGrid/dataGridLargeValues.ts +++ b/apps/desktop/src/lib/dataGrid/dataGridLargeValues.ts @@ -1,6 +1,27 @@ -import type { QueryResult } from "@/types/database"; +import type { ColumnInfo, DatabaseType, QueryResult } from "@/types/database"; export const TABLE_DATA_RESULT_MAX_BYTES = 32 * 1024 * 1024; +export const TABLE_DATA_CELL_PREVIEW_MIN_SIZE = 1; +export const TABLE_DATA_CELL_PREVIEW_SIZE = 8 * 1024; +export const TABLE_DATA_PREVIEW_CONTENT_MAX_BYTES = 24 * 1024 * 1024; +export const TABLE_DATA_TEXT_SERIALIZED_BYTES_PER_CHARACTER = 6; +const TABLE_DATA_LARGE_VALUE_MARKER_PREFIX = "__DBX_LARGE_VALUE_BYTES_"; + +export function canUseTableDataLargeValuePreview(databaseType: DatabaseType | undefined, columns: readonly ColumnInfo[], primaryKeys: readonly string[]): boolean { + return (databaseType === "mysql" || databaseType === "postgres") && columns.length > 0 && primaryKeys.length > 0 && !columns.some((column) => column.name.toLocaleUpperCase().startsWith(TABLE_DATA_LARGE_VALUE_MARKER_PREFIX)); +} + +export function tableDataLargeValuePreviewOptions(databaseType: DatabaseType | undefined, columns: readonly ColumnInfo[], primaryKeys: readonly string[], pageSize?: number): { columnTypes: string[]; largeValuePreviewSize: number } | Record { + if (!canUseTableDataLargeValuePreview(databaseType, columns, primaryKeys)) return {}; + const keyColumns = new Set(primaryKeys.map((column) => column.toLocaleLowerCase())); + const previewColumnCount = columns.filter((column) => !keyColumns.has(column.name.toLocaleLowerCase())).length; + if (previewColumnCount === 0) return {}; + const budgetedSize = Math.floor(TABLE_DATA_PREVIEW_CONTENT_MAX_BYTES / Math.max(1, pageSize ?? 1) / previewColumnCount / TABLE_DATA_TEXT_SERIALIZED_BYTES_PER_CHARACTER); + return { + columnTypes: columns.map((column) => column.data_type), + largeValuePreviewSize: Math.max(TABLE_DATA_CELL_PREVIEW_MIN_SIZE, Math.min(TABLE_DATA_CELL_PREVIEW_SIZE, budgetedSize)), + }; +} export function largeValueCellKey(rowIndex: number, columnIndex: number): string { return `${rowIndex}:${columnIndex}`; diff --git a/apps/desktop/src/lib/table/tableSelectSql.ts b/apps/desktop/src/lib/table/tableSelectSql.ts index 3af4e5bc61..75182527f9 100644 --- a/apps/desktop/src/lib/table/tableSelectSql.ts +++ b/apps/desktop/src/lib/table/tableSelectSql.ts @@ -13,6 +13,8 @@ export interface BuildTableSelectSqlOptions { tableType?: string; primaryKeys?: string[]; columns?: string[]; + columnTypes?: string[]; + largeValuePreviewSize?: number; fallbackOrderColumns?: string[]; orderBy?: string; limit?: number; diff --git a/apps/desktop/src/stores/__tests__/queryStore.tableDataRefresh.spec.ts b/apps/desktop/src/stores/__tests__/queryStore.tableDataRefresh.spec.ts index 3a2df44263..e5bd6f0dda 100644 --- a/apps/desktop/src/stores/__tests__/queryStore.tableDataRefresh.spec.ts +++ b/apps/desktop/src/stores/__tests__/queryStore.tableDataRefresh.spec.ts @@ -114,7 +114,9 @@ describe("queryStore table data refresh", () => { tableType: "TABLE", catalog: undefined, columns: ["id", "status", "created_at"], + columnTypes: ["integer", "text", "timestamp"], primaryKeys: ["id"], + largeValuePreviewSize: 8192, includeRowId: false, whereInput: "status = 'ACTIVE'", orderBy: "created_at DESC", diff --git a/apps/desktop/src/stores/queryStore.ts b/apps/desktop/src/stores/queryStore.ts index 65a6af4c34..1c7aff8b41 100644 --- a/apps/desktop/src/stores/queryStore.ts +++ b/apps/desktop/src/stores/queryStore.ts @@ -46,7 +46,7 @@ import { connectionObjectTreeNodeSchema, connectionQueryExecutionSchema, connect import { frontendQueryTimeoutDelayMs, frontendQueryTimeoutSecsForSql, queryTimeoutSecsForConnection } from "@/lib/sql/queryTimeout"; import { queryResultNameFromPreamble, queryResultSourceLabel } from "@/lib/sql/queryResultSource"; import { beginDataGridNativeSelectionBlock, finishDataGridNativeSelectionBlock } from "@/lib/dataGrid/dataGridNativeSelection"; -import { appendLargeValueCells, remapLargeValueCells, TABLE_DATA_RESULT_MAX_BYTES } from "@/lib/dataGrid/dataGridLargeValues"; +import { appendLargeValueCells, canUseTableDataLargeValuePreview, remapLargeValueCells, tableDataLargeValuePreviewOptions, TABLE_DATA_RESULT_MAX_BYTES } from "@/lib/dataGrid/dataGridLargeValues"; import { simpleDataGridOrderByReferencesMissingColumn, sortDataGridRowIndexes, type DataGridSortDirection } from "@/lib/dataGrid/dataGridSort"; import { normalizeResultPageSize } from "@/lib/dataGrid/paginationPageSize"; import { agentProtocolQueryResultMaxRows, capQueryResultTotal, effectiveQueryResultMaxRows, limitQueryPagination, queryResultLimitReached } from "@/lib/dataGrid/queryResultRowLimit"; @@ -2782,6 +2782,7 @@ export const useQueryStore = defineStore("query", () => { catalog: tableMeta.catalog, columns: tableMeta.columns.map((column) => column.name), primaryKeys, + ...tableDataLargeValuePreviewOptions(effectiveDbType, tableMeta.columns, primaryKeys, limit), includeRowId: usesSyntheticRowIdKey(effectiveDbType, primaryKeys, tableMeta.tableType), whereInput: tab.whereInput, orderBy, @@ -4573,6 +4574,8 @@ export const useQueryStore = defineStore("query", () => { queryExecutionLog("info", "execute-multi:start", { traceId, elapsed: elapsed() }); // Query and data tabs use a tab-scoped pool so repeated executions keep // connection-local state and avoid MySQL pool resets on every refresh. + const dataTabMeta = tab.mode === "data" ? tableMetaForDataTab(tab) : undefined; + const useTableDataPreview = canUseTableDataLargeValuePreview(effectiveDbType, dataTabMeta?.columns ?? [], dataTabMeta?.primaryKeys ?? []); const executionOptions = { ...(typeof pageLimit === "number" ? useAgentResultSession @@ -4586,10 +4589,11 @@ export const useQueryStore = defineStore("query", () => { : { maxRows: pageLimit, fetchSize: pageLimit } : { maxRows: agentProtocolQueryResultMaxRows(queryResultMaxRows) }), ...(executionClientSessionId ? { clientSessionId: executionClientSessionId } : {}), - ...(tab.mode === "data" && effectiveDbType === "mysql" + ...(tab.mode === "data" && (effectiveDbType === "mysql" || effectiveDbType === "postgres") ? { maxResultBytes: TABLE_DATA_RESULT_MAX_BYTES, - resultKeyColumns: tableMetaForDataTab(tab)?.primaryKeys ?? [], + resultKeyColumns: dataTabMeta?.primaryKeys ?? [], + tableDataPreview: useTableDataPreview, } : {}), timeoutSecs: queryTimeoutSecs, diff --git a/crates/dbx-core/src/query.rs b/crates/dbx-core/src/query.rs index 5912c0742b..569076ff8f 100644 --- a/crates/dbx-core/src/query.rs +++ b/crates/dbx-core/src/query.rs @@ -266,15 +266,43 @@ impl ExecuteMultiResult { } fn success_with_index_and_large_values( + mut result: db::QueryResult, + statement_index: usize, + mut large_value_cells: Vec, + table_data_preview: bool, + ) -> Self { + let server_cells = if table_data_preview { + remap_large_value_cells_around_server_markers(&result, &mut large_value_cells); + extract_server_large_value_markers(&mut result) + } else { + Vec::new() + }; + Self { + result, + large_value_cells: merge_large_value_cells(large_value_cells, server_cells), + execution_error: false, + statement_index: Some(statement_index), + error: None, + server_message: false, + } + } + + fn success_with_index_and_optional_server_large_values( result: db::QueryResult, statement_index: usize, - large_value_cells: Vec, + table_data_preview: bool, ) -> Self { + Self::success_with_index_and_large_values(result, statement_index, Vec::new(), table_data_preview) + } + + fn success_with_optional_server_large_values(mut result: db::QueryResult, table_data_preview: bool) -> Self { + let large_value_cells = + if table_data_preview { extract_server_large_value_markers(&mut result) } else { Vec::new() }; Self { result, large_value_cells, execution_error: false, - statement_index: Some(statement_index), + statement_index: None, error: None, server_message: false, } @@ -290,6 +318,228 @@ impl ExecuteMultiResult { } } +const SERVER_LARGE_VALUE_UNKNOWN_BYTES: usize = 64 * 1024 * 1024; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum ServerLargeValuePreviewKind { + Text, + Binary, + Vector, +} + +#[derive(Clone, Copy)] +struct ServerLargeValueMarker { + result_index: usize, + source_index: usize, + preview_kind: Option, + source_type: Option<&'static str>, +} + +fn server_large_value_alias( + suffix: &str, +) -> Option<(usize, Option, Option<&'static str>)> { + if let Ok(source_index) = suffix.parse::() { + return Some((source_index, None, None)); + } + let (kind, source_index) = suffix.split_once('_')?; + let source_index = source_index.parse::().ok()?; + let (preview_kind, source_type) = match kind { + "T" => (ServerLargeValuePreviewKind::Text, None), + "B" => (ServerLargeValuePreviewKind::Binary, None), + "V" => (ServerLargeValuePreviewKind::Vector, Some("vector")), + "J" => (ServerLargeValuePreviewKind::Text, Some("json")), + "K" => (ServerLargeValuePreviewKind::Text, Some("jsonb")), + "S" => (ServerLargeValuePreviewKind::Text, Some("tsvector")), + _ => return None, + }; + Some((source_index, Some(preview_kind), source_type)) +} + +fn server_large_value_marker(value: &serde_json::Value) -> Option<(ServerLargeValuePreviewKind, usize)> { + let (kind, preview_size) = value.as_str()?.split_once(':')?; + let kind = match kind { + "T" => ServerLargeValuePreviewKind::Text, + "B" => ServerLargeValuePreviewKind::Binary, + "V" => ServerLargeValuePreviewKind::Vector, + _ => return None, + }; + Some((kind, preview_size.parse::().ok()?.max(1))) +} + +fn truncate_server_large_value_preview( + value: &mut serde_json::Value, + kind: ServerLargeValuePreviewKind, + preview_size: usize, +) -> bool { + let serde_json::Value::String(text) = value else { + return false; + }; + if matches!(kind, ServerLargeValuePreviewKind::Vector) { + let truncated = text.chars().count() > preview_size; + let vector_text = if truncated { + let prefix: String = text.chars().take(preview_size).collect(); + let Some(last_separator) = prefix.rfind(',') else { + return false; + }; + format!("{}]", &prefix[..last_separator]) + } else { + text.clone() + }; + let Ok(vector) = serde_json::from_str::>(&vector_text) else { + return false; + }; + *value = serde_json::Value::Array(vector); + return truncated; + } + let truncate_at = match kind { + ServerLargeValuePreviewKind::Text => text.char_indices().nth(preview_size).map(|(index, _)| index), + ServerLargeValuePreviewKind::Binary => text + .strip_prefix("0x") + .filter(|hex| hex.len() > preview_size.saturating_mul(2)) + .map(|_| 2usize.saturating_add(preview_size.saturating_mul(2))), + ServerLargeValuePreviewKind::Vector => unreachable!(), + }; + let Some(truncate_at) = truncate_at else { + return false; + }; + text.truncate(truncate_at); + text.push_str("..."); + true +} + +fn server_large_value_markers(result: &db::QueryResult) -> Vec { + let mut markers = Vec::new(); + for (result_index, column) in result.columns.iter().enumerate() { + let Some((source_index, preview_kind, source_type)) = column + .strip_prefix(crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX) + .and_then(server_large_value_alias) + else { + continue; + }; + let expected_source_index = result_index.checked_sub(markers.len() + 1); + if expected_source_index == Some(source_index) { + markers.push(ServerLargeValueMarker { result_index, source_index, preview_kind, source_type }); + } + } + markers +} + +fn extract_server_large_value_markers(result: &mut db::QueryResult) -> Vec { + let markers = server_large_value_markers(result); + if markers.is_empty() { + return Vec::new(); + } + + let mut large_value_cells = Vec::new(); + for marker in &markers { + let source_result_index = marker.result_index.saturating_sub(1); + let row_kind = result + .rows + .iter() + .find_map(|row| row.get(marker.result_index).and_then(server_large_value_marker).map(|value| value.0)); + let source_type = marker.source_type.or_else(|| { + (marker.preview_kind.or(row_kind) == Some(ServerLargeValuePreviewKind::Vector)).then_some("vector") + }); + if let (Some(source_type), Some(column_type)) = (source_type, result.column_types.get_mut(source_result_index)) + { + *column_type = source_type.to_string(); + } + } + for (row_index, row) in result.rows.iter_mut().enumerate() { + for marker in &markers { + let marker_value = row.get(marker.result_index).and_then(server_large_value_marker); + let source_result_index = marker.result_index.saturating_sub(1); + if marker_value.is_some_and(|(kind, preview_size)| { + row.get_mut(source_result_index) + .is_some_and(|value| truncate_server_large_value_preview(value, kind, preview_size)) + }) { + large_value_cells.push(db::LargeValueCell { + row_index, + column_index: marker.source_index, + original_bytes: SERVER_LARGE_VALUE_UNKNOWN_BYTES, + }); + } + } + } + + let removed: std::collections::HashSet = markers.iter().map(|marker| marker.result_index).collect(); + let retained_index = |index: usize| index - removed.iter().filter(|removed_index| **removed_index < index).count(); + result.spatial_columns.retain_mut(|column| { + if removed.contains(&column.column_index) { + return false; + } + column.column_index = retained_index(column.column_index); + true + }); + for row in &mut result.rows { + let mut index = 0; + row.retain(|_| { + let keep = !removed.contains(&index); + index += 1; + keep + }); + } + for row in &mut result.spatial_values { + let mut index = 0; + row.retain(|_| { + let keep = !removed.contains(&index); + index += 1; + keep + }); + } + let mut index = 0; + result.columns.retain(|_| { + let keep = !removed.contains(&index); + index += 1; + keep + }); + let mut index = 0; + result.column_types.retain(|_| { + let keep = !removed.contains(&index); + index += 1; + keep + }); + let mut index = 0; + result.column_sortables.retain(|_| { + let keep = !removed.contains(&index); + index += 1; + keep + }); + large_value_cells +} + +fn remap_large_value_cells_around_server_markers(result: &db::QueryResult, cells: &mut Vec) { + let removed: std::collections::HashSet = + server_large_value_markers(result).into_iter().map(|marker| marker.result_index).collect(); + if removed.is_empty() { + return; + } + cells.retain_mut(|cell| { + if removed.contains(&cell.column_index) { + return false; + } + cell.column_index -= removed.iter().filter(|index| **index < cell.column_index).count(); + true + }); +} + +fn merge_large_value_cells( + mut driver_cells: Vec, + server_cells: Vec, +) -> Vec { + for server_cell in server_cells { + if let Some(driver_cell) = driver_cells + .iter_mut() + .find(|cell| cell.row_index == server_cell.row_index && cell.column_index == server_cell.column_index) + { + *driver_cell = server_cell; + } else { + driver_cells.push(server_cell); + } + } + driver_cells +} + impl From for ExecuteMultiResult { fn from(result: db::QueryResult) -> Self { Self { @@ -639,6 +889,9 @@ pub struct QueryExecutionOptions { /// Result columns that must stay exact because clients use them as stable /// row identifiers when fetching full large-cell values on demand. pub result_key_columns: Vec, + /// Enables extraction of hidden server-side preview metadata. This must be + /// set only for generated table-data preview SQL, never arbitrary queries. + pub table_data_preview: bool, /// Doris / StarRocks catalog selected for this query tab. pub catalog: Option, pub result_session_id: Option, @@ -2252,6 +2505,7 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed( // HTTP SQLite providers send all statements in one request so the provider // can preserve batch ordering and atomicity. if is_http_sqlite { + let table_data_preview = options.table_data_preview; return single_statement_multi_result( execute_sql_statement_with_options_typed( state, @@ -2263,6 +2517,7 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed( options, ) .await, + table_data_preview, ); } @@ -2306,6 +2561,7 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed( ) { let single_sql = statements.into_iter().next().unwrap_or_default(); + let table_data_preview = options.table_data_preview; return single_statement_multi_result( execute_sql_statement_with_options_typed( state, @@ -2317,6 +2573,7 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed( options, ) .await, + table_data_preview, ); } @@ -2372,7 +2629,11 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed( { Ok(r) => { report_execute_multi_progress(progress.as_ref(), statement_index, statements.len(), &r, true, None); - results.push(ExecuteMultiResult::success_with_index(r, statement_index)); + results.push(ExecuteMultiResult::success_with_index_and_optional_server_large_values( + r, + statement_index, + options.table_data_preview, + )); } Err(error) => { let action = query_execution_error_action(db_type, stmt, &error); @@ -2403,8 +2664,9 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed( fn single_statement_multi_result( result: Result, + table_data_preview: bool, ) -> Result, QueryExecutionError> { - result.map(|result| vec![result.into()]) + result.map(|result| vec![ExecuteMultiResult::success_with_optional_server_large_values(result, table_data_preview)]) } fn mysql_single_statement_uses_batch_route( @@ -2420,6 +2682,10 @@ fn mysql_single_statement_uses_batch_route( } trait MysqlBatchStatementExecutor { + fn table_data_preview(&self) -> bool { + false + } + async fn execute_statement(&mut self, statement: &str) -> Result, String>; async fn execute_non_result_batch(&mut self, statements: &[String]) -> db::mysql::MySqlNonResultBatchOutcome { @@ -2450,10 +2716,15 @@ struct MysqlBatchConnection<'a> { max_rows: Option, max_result_bytes: Option, result_key_columns: &'a [String], + table_data_preview: bool, dialect: db::mysql::MySqlQueryDialect, } impl MysqlBatchStatementExecutor for MysqlBatchConnection<'_> { + fn table_data_preview(&self) -> bool { + self.table_data_preview + } + async fn execute_statement(&mut self, statement: &str) -> Result, String> { wait_for_result_opt( self.cancel_token.clone(), @@ -2553,6 +2824,7 @@ where { let mut results = Vec::with_capacity(statements.len()); let mut statement_index = 0usize; + let table_data_preview = executor.table_data_preview(); while statement_index < statements.len() { if is_canceled(&cancel_token) { results.push(ExecuteMultiResult::execution_error(error_query_result(canceled_error()))); @@ -2618,6 +2890,7 @@ where result.result, statement_index, result.large_value_cells, + table_data_preview, ) })); } @@ -2711,6 +2984,7 @@ async fn execute_multi_mysql( max_rows, max_result_bytes, result_key_columns: &options.result_key_columns, + table_data_preview: options.table_data_preview, dialect, }; let (results, error_action) = execute_mysql_batch_statements( @@ -5766,6 +6040,7 @@ for line in sys.stdin: empty_query_result(1), 0, vec![db::LargeValueCell { row_index: 2, column_index: 3, original_bytes: 65_536 }], + false, ); let serialized = serde_json::to_value(result).unwrap(); @@ -5901,7 +6176,7 @@ for line in sys.stdin: ) .with_omitted_sql_context("SELECT * FROM dbx_table_that_does_not_exist"); - let error = single_statement_multi_result(Err(error)).unwrap_err(); + let error = single_statement_multi_result(Err(error), false).unwrap_err(); let backend_error = error.into_backend_error(); assert_eq!(backend_error.code(), "DBX-JDBC-4001"); @@ -6900,6 +7175,168 @@ for line in sys.stdin: assert_eq!(normalized.rows[0][1], serde_json::json!([1, "2041797190226354178"])); } + #[test] + fn extracts_server_large_value_markers_and_restores_source_columns() { + let mut result = db::QueryResult { + columns: vec![ + "id".to_string(), + "payload".to_string(), + format!("{}T_1", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX), + "note".to_string(), + ], + column_types: vec!["integer".to_string(), "text".to_string(), "bigint".to_string(), "text".to_string()], + column_sortables: vec![true, true, true, true], + spatial_columns: Vec::new(), + spatial_values: Vec::new(), + rows: vec![ + vec![ + serde_json::json!(1), + serde_json::json!("预览文本多"), + serde_json::json!("T:4"), + serde_json::json!("a"), + ], + vec![serde_json::json!(2), serde_json::json!("短值"), serde_json::json!("T:4"), serde_json::json!("b")], + ], + affected_rows: 0, + execution_time_ms: 0, + truncated: false, + session_id: None, + has_more: false, + elasticsearch_raw_body: None, + messages: Vec::new(), + }; + + let cells = extract_server_large_value_markers(&mut result); + + assert_eq!(result.columns, vec!["id", "payload", "note"]); + assert_eq!(result.column_types, vec!["integer", "text", "text"]); + assert_eq!( + result.rows[0], + vec![serde_json::json!(1), serde_json::json!("预览文本..."), serde_json::json!("a")] + ); + assert_eq!(result.rows[1], vec![serde_json::json!(2), serde_json::json!("短值"), serde_json::json!("b")]); + assert_eq!( + cells, + vec![db::LargeValueCell { + row_index: 0, + column_index: 1, + original_bytes: SERVER_LARGE_VALUE_UNKNOWN_BYTES, + }] + ); + } + + #[test] + fn truncates_server_binary_preview_after_the_configured_byte_count() { + let mut result = db::QueryResult { + columns: vec![ + "raw_value".to_string(), + format!("{}B_0", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX), + ], + column_types: vec!["bytea".to_string(), "text".to_string()], + column_sortables: vec![true, true], + spatial_columns: Vec::new(), + spatial_values: Vec::new(), + rows: vec![vec![serde_json::json!("0x0102030405"), serde_json::json!("B:4")]], + affected_rows: 0, + execution_time_ms: 0, + truncated: false, + session_id: None, + has_more: false, + elasticsearch_raw_body: None, + messages: Vec::new(), + }; + + let cells = extract_server_large_value_markers(&mut result); + + assert_eq!(result.rows, vec![vec![serde_json::json!("0x01020304...")]]); + assert_eq!(cells.len(), 1); + } + + #[test] + fn restores_pgvector_previews_as_arrays_and_marks_only_truncated_values() { + let marker = format!("{}V_0", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX); + let mut result = db::QueryResult { + columns: vec!["embedding".to_string(), marker], + column_types: vec!["text".to_string(), "text".to_string()], + column_sortables: vec![true, true], + spatial_columns: Vec::new(), + spatial_values: Vec::new(), + rows: vec![ + vec![serde_json::json!("[0.1,0.2,0.3]"), serde_json::json!("V:9")], + vec![serde_json::json!("[0.1,0.2]"), serde_json::json!("V:20")], + ], + affected_rows: 0, + execution_time_ms: 0, + truncated: false, + session_id: None, + has_more: false, + elasticsearch_raw_body: None, + messages: Vec::new(), + }; + + let cells = extract_server_large_value_markers(&mut result); + + assert_eq!(result.column_types, vec!["vector"]); + assert_eq!(result.rows[0][0], serde_json::json!([0.1, 0.2])); + assert_eq!(result.rows[1][0], serde_json::json!([0.1, 0.2])); + assert_eq!(cells.len(), 1); + assert_eq!(cells[0].row_index, 0); + } + + #[test] + fn restores_server_preview_types_without_rows() { + let mut result = db::QueryResult { + columns: vec![ + "embedding".to_string(), + format!("{}V_0", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX), + "metadata".to_string(), + format!("{}K_1", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX), + ], + column_types: vec!["text".to_string(), "text".to_string(), "text".to_string(), "text".to_string()], + column_sortables: vec![true; 4], + spatial_columns: Vec::new(), + spatial_values: Vec::new(), + rows: Vec::new(), + affected_rows: 0, + execution_time_ms: 0, + truncated: false, + session_id: None, + has_more: false, + elasticsearch_raw_body: None, + messages: Vec::new(), + }; + + let cells = extract_server_large_value_markers(&mut result); + + assert_eq!(result.columns, vec!["embedding", "metadata"]); + assert_eq!(result.column_types, vec!["vector", "jsonb"]); + assert!(cells.is_empty()); + } + + #[test] + fn ordinary_queries_preserve_columns_that_resemble_preview_markers() { + let marker = format!("{}0", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX); + let mut result = empty_query_result(1); + result.columns = vec!["payload".to_string(), marker.clone()]; + result.column_types = vec!["text".to_string(), "bigint".to_string()]; + result.column_sortables = vec![true, true]; + result.rows = vec![vec![serde_json::json!("value"), serde_json::json!(123)]]; + + let ordinary = ExecuteMultiResult::success_with_index_and_optional_server_large_values(result, 0, false); + + assert_eq!(ordinary.result.columns, vec!["payload".to_string(), marker]); + assert_eq!(ordinary.result.rows[0], vec![serde_json::json!("value"), serde_json::json!(123)]); + assert!(ordinary.large_value_cells.is_empty()); + } + + #[test] + fn single_statement_preview_preserves_absent_statement_index() { + let result = single_statement_multi_result(Ok(empty_query_result(1)), true).unwrap(); + + assert_eq!(result.len(), 1); + assert_eq!(result[0].statement_index, None); + } + #[test] fn query_execution_options_default_use_transaction_is_none() { let opts = QueryExecutionOptions::default(); diff --git a/crates/dbx-core/src/sql_dialect.rs b/crates/dbx-core/src/sql_dialect.rs index 7d2e5eee5a..062c7e380c 100644 --- a/crates/dbx-core/src/sql_dialect.rs +++ b/crates/dbx-core/src/sql_dialect.rs @@ -38,7 +38,9 @@ pub use identifiers::{ normalize_where_input, qualified_table_name, qualified_table_name_with_catalog, quote_table_identifier, }; pub(crate) use identifiers::{parse_sqlserver_linked_schema_ref, qualified_transfer_table, quote_transfer_identifier}; -pub use table_select::{build_count_table_sql, build_table_data_select_sql, build_table_select_sql}; +pub use table_select::{ + build_count_table_sql, build_table_data_select_sql, build_table_select_sql, DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX, +}; pub(crate) use table_select::{ quote_table_data_identifier, table_data_qualified_table_name, table_data_schema, uses_connection_identifier_quote, }; diff --git a/crates/dbx-core/src/sql_dialect/table_select.rs b/crates/dbx-core/src/sql_dialect/table_select.rs index 5330491dc4..09684fb83d 100644 --- a/crates/dbx-core/src/sql_dialect/table_select.rs +++ b/crates/dbx-core/src/sql_dialect/table_select.rs @@ -12,6 +12,131 @@ use super::types::{ DBX_TDENGINE_TBNAME_COLUMN, }; +pub const DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX: &str = "__DBX_LARGE_VALUE_BYTES_"; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum LargeValuePreviewKind { + Text, + Binary, + TextCast, + Vector, +} + +fn large_value_marker_alias_kind(kind: LargeValuePreviewKind, data_type: &str) -> &'static str { + match kind { + LargeValuePreviewKind::Binary => "B", + LargeValuePreviewKind::Vector => "V", + LargeValuePreviewKind::TextCast => match normalized_data_type_base(data_type).as_str() { + "json" => "J", + "jsonb" => "K", + "tsvector" => "S", + _ => "T", + }, + LargeValuePreviewKind::Text => "T", + } +} + +fn normalized_data_type_base(data_type: &str) -> String { + data_type.trim().split(['(', '[']).next().unwrap_or_default().trim().to_ascii_lowercase() +} + +fn large_value_preview_kind(database_type: Option, data_type: &str) -> Option { + let normalized = data_type.trim().to_ascii_lowercase(); + let base = normalized_data_type_base(data_type); + match database_type { + Some(DatabaseType::Mysql) => { + if matches!(base.as_str(), "binary" | "varbinary" | "tinyblob" | "blob" | "mediumblob" | "longblob") { + Some(LargeValuePreviewKind::Binary) + } else if base == "json" { + Some(LargeValuePreviewKind::TextCast) + } else if matches!(base.as_str(), "char" | "varchar" | "tinytext" | "text" | "mediumtext" | "longtext") { + Some(LargeValuePreviewKind::Text) + } else { + None + } + } + Some(DatabaseType::Postgres) => { + if base == "bytea" { + Some(LargeValuePreviewKind::Binary) + } else if matches!(base.as_str(), "char" | "character" | "varchar" | "text" | "citext" | "name" | "xml") + || normalized.starts_with("character varying") + { + Some(LargeValuePreviewKind::Text) + } else if base == "vector" { + Some(LargeValuePreviewKind::Vector) + } else if matches!(base.as_str(), "json" | "jsonb" | "tsvector") { + Some(LargeValuePreviewKind::TextCast) + } else { + None + } + } + _ => None, + } +} + +fn build_large_value_preview_columns(options: &TableDataSelectSqlOptions) -> Option { + let database_type = options.database_type; + let preview_size = options.large_value_preview_size?.max(1); + if options.columns.is_empty() + || options.columns.len() != options.column_types.len() + || options.primary_keys.is_empty() + || options + .columns + .iter() + .any(|column| column.to_ascii_uppercase().starts_with(DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX)) + { + return None; + } + + let protected: std::collections::HashSet = + options.primary_keys.iter().map(|column| column.to_ascii_lowercase()).collect(); + let mut projections = Vec::with_capacity(options.columns.len() * 2); + let mut marker_count = 0; + for (column_index, (column, data_type)) in options.columns.iter().zip(&options.column_types).enumerate() { + let quoted = if uses_connection_identifier_quote(database_type, options.identifier_quote.as_deref()) { + quote_table_data_identifier(database_type, column, options.identifier_quote.as_deref()) + } else { + quote_table_identifier(database_type, column) + }; + let kind = (!protected.contains(&column.to_ascii_lowercase())) + .then(|| large_value_preview_kind(database_type, data_type)) + .flatten(); + let Some(kind) = kind else { + projections.push(quoted); + continue; + }; + + let alias_kind = large_value_marker_alias_kind(kind, data_type); + let marker_alias = quote_table_identifier( + database_type, + &format!("{DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX}{alias_kind}_{column_index}"), + ); + let prefix_size = preview_size.saturating_add(1); + let (preview, marker_kind) = match database_type { + Some(DatabaseType::Mysql) if kind == LargeValuePreviewKind::Binary => { + (format!("LEFT({quoted}, {prefix_size}) AS {quoted}"), "B") + } + Some(DatabaseType::Mysql) => (format!("LEFT({quoted}, {prefix_size}) AS {quoted}"), "T"), + Some(DatabaseType::Postgres) if kind == LargeValuePreviewKind::Binary => { + (format!("substring({quoted} from 1 for {prefix_size}) AS {quoted}"), "B") + } + Some(DatabaseType::Postgres) if kind == LargeValuePreviewKind::TextCast => { + (format!("left({quoted}::text, {prefix_size}) AS {quoted}"), "T") + } + Some(DatabaseType::Postgres) if kind == LargeValuePreviewKind::Vector => { + (format!("left({quoted}::text, {prefix_size}) AS {quoted}"), "V") + } + Some(DatabaseType::Postgres) => (format!("left({quoted}, {prefix_size}) AS {quoted}"), "T"), + _ => return None, + }; + let marker = format!("'{marker_kind}:{preview_size}' AS {marker_alias}"); + projections.push(preview); + projections.push(marker); + marker_count += 1; + } + (marker_count > 0).then(|| projections.join(", ")) +} + pub fn build_count_table_sql(database_type: Option, schema: Option<&str>, table_name: &str) -> String { if database_type == Some(DatabaseType::VictoriaMetrics) { return format!("count({})", victoriametrics_metric_selector(table_name)); @@ -77,6 +202,8 @@ pub fn build_table_data_select_sql(options: TableDataSelectSqlOptions) -> String let select_columns = if include_oracle_row_id { format!("ROWIDTOCHAR(t.ROWID) AS \"{DBX_ROWID_COLUMN}\", t.*") + } else if let Some(preview_columns) = build_large_value_preview_columns(&options) { + preview_columns } else { build_select_columns( database_type, @@ -521,6 +648,8 @@ mod tests { table_type: None, primary_keys: Vec::new(), columns: Vec::new(), + column_types: Vec::new(), + large_value_preview_size: None, fallback_order_columns: Vec::new(), order_by: None, limit: Some(10), diff --git a/crates/dbx-core/src/sql_dialect/tests.rs b/crates/dbx-core/src/sql_dialect/tests.rs index aac43a8d69..3683b5be11 100644 --- a/crates/dbx-core/src/sql_dialect/tests.rs +++ b/crates/dbx-core/src/sql_dialect/tests.rs @@ -641,6 +641,97 @@ fn builds_table_data_where_and_schema_queries() { ); } +#[test] +fn builds_mysql_table_data_large_value_previews_without_truncating_keys() { + let sql = build_table_data_select_sql(TableDataSelectSqlOptions { + database_type: Some(DatabaseType::Mysql), + table_name: "large_rows".to_string(), + primary_keys: vec!["id".to_string()], + columns: vec!["id".to_string(), "payload".to_string(), "raw_value".to_string(), "metadata".to_string()], + column_types: vec!["bigint".to_string(), "longtext".to_string(), "longblob".to_string(), "json".to_string()], + large_value_preview_size: Some(4096), + limit: Some(100), + ..Default::default() + }); + + assert!(sql.starts_with("SELECT `id`, LEFT(`payload`, 4097) AS `payload`")); + assert!(sql.contains("'T:4096' AS `__DBX_LARGE_VALUE_BYTES_T_1`")); + assert!(sql.contains("LEFT(`raw_value`, 4097) AS `raw_value`")); + assert!(sql.contains("'B:4096' AS `__DBX_LARGE_VALUE_BYTES_B_2`")); + assert!(sql.contains("LEFT(`metadata`, 4097) AS `metadata`")); + assert!(sql.contains("'T:4096' AS `__DBX_LARGE_VALUE_BYTES_J_3`")); + assert!(!sql.contains("__DBX_LARGE_VALUE_BYTES_0")); +} + +#[test] +fn builds_postgres_table_data_large_value_previews() { + let sql = build_table_data_select_sql(TableDataSelectSqlOptions { + database_type: Some(DatabaseType::Postgres), + schema: Some("public".to_string()), + table_name: "large_rows".to_string(), + primary_keys: vec!["id".to_string()], + columns: vec!["id".to_string(), "payload".to_string(), "metadata".to_string()], + column_types: vec!["integer".to_string(), "text".to_string(), "jsonb".to_string()], + large_value_preview_size: Some(8192), + limit: Some(100), + ..Default::default() + }); + + assert!(sql.starts_with("SELECT \"id\", left(\"payload\", 8193) AS \"payload\"")); + assert!(sql.contains("'T:8192' AS \"__DBX_LARGE_VALUE_BYTES_T_1\"")); + assert!(sql.contains("left(\"metadata\"::text, 8193) AS \"metadata\"")); + assert!(sql.contains("'T:8192' AS \"__DBX_LARGE_VALUE_BYTES_K_2\"")); +} + +#[test] +fn builds_pgvector_table_data_preview_with_compatible_text_cast() { + let sql = build_table_data_select_sql(TableDataSelectSqlOptions { + database_type: Some(DatabaseType::Postgres), + schema: Some("public".to_string()), + table_name: "embeddings".to_string(), + primary_keys: vec!["id".to_string()], + columns: vec!["id".to_string(), "embedding".to_string()], + column_types: vec!["integer".to_string(), "vector(16000)".to_string()], + large_value_preview_size: Some(8192), + limit: Some(100), + ..Default::default() + }); + + assert!(sql.contains("left(\"embedding\"::text, 8193) AS \"embedding\"")); + assert!(sql.contains("'V:8192' AS \"__DBX_LARGE_VALUE_BYTES_V_1\"")); +} + +#[test] +fn table_data_preview_requires_stable_keys_and_parallel_types() { + let base = TableDataSelectSqlOptions { + database_type: Some(DatabaseType::Postgres), + table_name: "large_rows".to_string(), + columns: vec!["payload".to_string()], + column_types: vec!["text".to_string()], + large_value_preview_size: Some(8192), + limit: Some(100), + ..Default::default() + }; + assert_eq!(build_table_data_select_sql(base.clone()), "SELECT * FROM \"large_rows\" LIMIT 100;"); + assert_eq!( + build_table_data_select_sql(TableDataSelectSqlOptions { + primary_keys: vec!["id".to_string()], + column_types: Vec::new(), + ..base.clone() + }), + "SELECT * FROM \"large_rows\" LIMIT 100;" + ); + assert_eq!( + build_table_data_select_sql(TableDataSelectSqlOptions { + primary_keys: vec!["id".to_string()], + columns: vec!["id".to_string(), "__dbx_large_value_bytes_t_0".to_string()], + column_types: vec!["integer".to_string(), "text".to_string()], + ..base + }), + "SELECT * FROM \"large_rows\" LIMIT 100;" + ); +} + #[test] fn builds_informix_table_data_with_skip_first_pagination() { assert_eq!( diff --git a/crates/dbx-core/src/sql_dialect/types.rs b/crates/dbx-core/src/sql_dialect/types.rs index 8b7141f428..5bb652b9c1 100644 --- a/crates/dbx-core/src/sql_dialect/types.rs +++ b/crates/dbx-core/src/sql_dialect/types.rs @@ -44,6 +44,15 @@ pub struct TableDataSelectSqlOptions { pub primary_keys: Vec, #[serde(default)] pub columns: Vec, + /// Database type names parallel to `columns`. Table-data callers can use + /// these to request bounded server-side projections for variable-length + /// values without changing arbitrary query execution. + #[serde(default)] + pub column_types: Vec, + /// Maximum characters (text-like values) or bytes (binary values) kept in + /// a table-data preview. Omitted for exports and ordinary SQL queries. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub large_value_preview_size: Option, #[serde(default)] pub fallback_order_columns: Vec, #[serde(default, skip_serializing_if = "Option::is_none")] diff --git a/crates/dbx-web/src/routes/query.rs b/crates/dbx-web/src/routes/query.rs index 6d1a0772be..a8e285f3eb 100644 --- a/crates/dbx-web/src/routes/query.rs +++ b/crates/dbx-web/src/routes/query.rs @@ -24,6 +24,8 @@ pub struct ExecuteQueryRequest { pub max_result_bytes: Option, #[serde(default)] pub result_key_columns: Vec, + #[serde(default)] + pub table_data_preview: bool, pub result_session_id: Option, pub client_session_id: Option, pub timeout_secs: Option, @@ -363,6 +365,7 @@ pub async fn execute_query( page_size: req.page_size, max_result_bytes: req.max_result_bytes, result_key_columns: req.result_key_columns, + table_data_preview: req.table_data_preview, catalog: req.catalog, result_session_id: req.result_session_id, client_session_id: req.client_session_id, @@ -411,6 +414,7 @@ pub async fn execute_multi( page_size: req.page_size, max_result_bytes: req.max_result_bytes, result_key_columns: req.result_key_columns, + table_data_preview: req.table_data_preview, catalog: req.catalog, result_session_id: req.result_session_id, client_session_id: req.client_session_id, diff --git a/src-tauri/src/commands/query.rs b/src-tauri/src/commands/query.rs index c8a77602c9..5964ebb01c 100644 --- a/src-tauri/src/commands/query.rs +++ b/src-tauri/src/commands/query.rs @@ -90,6 +90,7 @@ pub async fn execute_multi( page_size: Option, max_result_bytes: Option, result_key_columns: Option>, + table_data_preview: Option, result_session_id: Option, client_session_id: Option, timeout_secs: Option, @@ -148,6 +149,7 @@ pub async fn execute_multi( page_size, max_result_bytes, result_key_columns: result_key_columns.unwrap_or_default(), + table_data_preview: table_data_preview.unwrap_or(false), catalog, result_session_id, client_session_id,