fix(grid): bound large table previews by byte budget

This commit is contained in:
zipg authored and GitHub committed 2026-08-13 13:25:04 +08:00
1 parent 020654c33c
commit e85d3dcc59
19 files changed
+823 -23

No files matched your search

@@ -151,7 +151,7 @@ import { isCancelSearchShortcut, isCopyCurrentRowShortcut, isDeleteCurrentRowSho
import { dataGridHeaderContentWidth, scrollbarGutterWidth } from "@/lib/dataGrid/dataGridScrollGutter"; import { dataGridHeaderContentWidth, scrollbarGutterWidth } from "@/lib/dataGrid/dataGridScrollGutter";
import { canFetchNextDataGridSegment, canGoNextDataGridPage, dataGridTotalRowCountLabelKey, dataGridTruncationHintKey, hasCompleteLocalDataGridResult, resolveDataGridPaginationTotal, type DataGridInexactTotalRowCountMode } from "@/lib/dataGrid/dataGridPagination"; import { canFetchNextDataGridSegment, canGoNextDataGridPage, dataGridTotalRowCountLabelKey, dataGridTruncationHintKey, hasCompleteLocalDataGridResult, resolveDataGridPaginationTotal, type DataGridInexactTotalRowCountMode } from "@/lib/dataGrid/dataGridPagination";
import { dataGridCountQueryOptions } from "@/lib/dataGrid/dataGridQueryOptions"; import { dataGridCountQueryOptions } from "@/lib/dataGrid/dataGridQueryOptions";
import { largeValueCellKey, largeValueCellMap } from "@/lib/dataGrid/dataGridLargeValues"; import { largeValueCellKey, largeValueCellMap, tableDataLargeValuePreviewOptions } from "@/lib/dataGrid/dataGridLargeValues";
import { dataGridBottomScrollTop, dataGridScrollPosition, isDataGridAtScrollBottom, isDataGridNearScrollBottom, isDataGridPrefixAppend, shouldCheckInfiniteScrollAfterScroll, type DataGridScrollPosition } from "@/lib/dataGrid/dataGridInfiniteScroll"; import { dataGridBottomScrollTop, dataGridScrollPosition, isDataGridAtScrollBottom, isDataGridNearScrollBottom, isDataGridPrefixAppend, shouldCheckInfiniteScrollAfterScroll, type DataGridScrollPosition } from "@/lib/dataGrid/dataGridInfiniteScroll";
import { CANVAS_DATA_GRID_ROW_HEIGHT, MAX_CANVAS_DATA_GRID_PIXEL_RATIO, canvasDataGridActionOverlayWidth, canvasDataGridActionReservedWidth, dataGridSearchMatchKey, drawCanvasDataGrid, type CanvasDevicePixelSize } from "@/lib/dataGrid/canvasDataGridRenderer"; import { CANVAS_DATA_GRID_ROW_HEIGHT, MAX_CANVAS_DATA_GRID_PIXEL_RATIO, canvasDataGridActionOverlayWidth, canvasDataGridActionReservedWidth, dataGridSearchMatchKey, drawCanvasDataGrid, type CanvasDevicePixelSize } from "@/lib/dataGrid/canvasDataGridRenderer";
import { DATA_GRID_DARK_STRIPED_ROW_BG, DATA_GRID_LIGHT_STRIPED_ROW_BG, dataGridActiveRowBackground } from "@/lib/dataGrid/dataGridPaintTheme"; import { DATA_GRID_DARK_STRIPED_ROW_BG, DATA_GRID_LIGHT_STRIPED_ROW_BG, dataGridActiveRowBackground } from "@/lib/dataGrid/dataGridPaintTheme";
@@ -4141,7 +4141,7 @@ async function resolveLargeValueCells(rowIds: number[], columnIndexes: number[])
} }
} }
if (requestsByColumn.size === 0) return resolved; if (requestsByColumn.size === 0) return resolved;
if (resolvedDatabaseType.value !== "mysql" || !props.connectionId || !props.tableMeta?.tableName || props.tableMeta.primaryKeys.length === 0) { if ((resolvedDatabaseType.value !== "mysql" && resolvedDatabaseType.value !== "postgres") || !props.connectionId || !props.tableMeta?.tableName || props.tableMeta.primaryKeys.length === 0) {
throw new Error(t("grid.largeValueNeedsStableKey")); throw new Error(t("grid.largeValueNeedsStableKey"));
} }
const primaryKeyIndexes = props.tableMeta.primaryKeys.map(largeValueSourceColumnIndex); const primaryKeyIndexes = props.tableMeta.primaryKeys.map(largeValueSourceColumnIndex);
@@ -5224,6 +5224,7 @@ async function applyOrderBySearch() {
tableType: tableMeta.tableType, tableType: tableMeta.tableType,
columns: tableMeta.columns.map((column) => column.name), columns: tableMeta.columns.map((column) => column.name),
primaryKeys: tableMeta.primaryKeys, primaryKeys: tableMeta.primaryKeys,
...tableDataLargeValuePreviewOptions(resolvedDatabaseType.value, tableMeta.columns, tableMeta.primaryKeys, pageSize.value),
orderBy: orderByClause, orderBy: orderByClause,
limit: pageSize.value, limit: pageSize.value,
whereInput: currentWhereInput(), whereInput: currentWhereInput(),
@@ -5259,6 +5260,7 @@ async function applyWhereFilter() {
tableType: tableMeta.tableType, tableType: tableMeta.tableType,
columns: tableMeta.columns.map((column) => column.name), columns: tableMeta.columns.map((column) => column.name),
primaryKeys: tableMeta.primaryKeys, primaryKeys: tableMeta.primaryKeys,
...tableDataLargeValuePreviewOptions(resolvedDatabaseType.value, tableMeta.columns, tableMeta.primaryKeys, pageSize.value),
orderBy: orderByInput.value.trim() || (sortCol.value ? `${queryColumnRef(sortCol.value)} ${sortDir.value.toUpperCase()}` : undefined), orderBy: orderByInput.value.trim() || (sortCol.value ? `${queryColumnRef(sortCol.value)} ${sortDir.value.toUpperCase()}` : undefined),
limit: pageSize.value, limit: pageSize.value,
whereInput, whereInput,
@@ -6,6 +6,7 @@ import type { QueryTab } from "@/types/database";
const mocks = vi.hoisted(() => ({ const mocks = vi.hoisted(() => ({
tabs: [] as QueryTab[], tabs: [] as QueryTab[],
activeTabId: "", activeTabId: "",
buildTableSelectSql: vi.fn(),
databaseType: "postgres" as string, databaseType: "postgres" as string,
ensureConnected: vi.fn(), ensureConnected: vi.fn(),
executeTabSql: vi.fn(), executeTabSql: vi.fn(),
@@ -83,7 +84,7 @@ vi.mock("@/stores/settingsStore", () => ({
})); }));
vi.mock("@/lib/table/tableSelectSql", () => ({ vi.mock("@/lib/table/tableSelectSql", () => ({
buildTableSelectSql: async ({ tableName }: { tableName: string }) => `SELECT * FROM ${tableName}`, buildTableSelectSql: mocks.buildTableSelectSql,
})); }));
const dialogs = { const dialogs = {
@@ -103,12 +104,13 @@ describe("useNavigationTargets openTableTarget", () => {
mocks.tabs.length = 0; mocks.tabs.length = 0;
mocks.databaseType = "postgres"; mocks.databaseType = "postgres";
mocks.ensureConnected.mockResolvedValue(undefined); mocks.ensureConnected.mockResolvedValue(undefined);
mocks.buildTableSelectSql.mockImplementation(async ({ tableName }: { tableName: string }) => `SELECT * FROM ${tableName}`);
mocks.executeTabSql.mockResolvedValue(undefined); mocks.executeTabSql.mockResolvedValue(undefined);
mocks.getColumns.mockResolvedValue([column("id")]); mocks.getColumns.mockResolvedValue([column("id")]);
mocks.listIndexes.mockResolvedValue([]); mocks.listIndexes.mockResolvedValue([]);
}); });
it("marks row identity pending until real metadata lands", async () => { it("loads stable row identity before the first PostgreSQL table query", async () => {
let releaseColumns: (columns: unknown[]) => void = () => {}; let releaseColumns: (columns: unknown[]) => void = () => {};
mocks.getColumns.mockReturnValueOnce( mocks.getColumns.mockReturnValueOnce(
new Promise((resolve) => { new Promise((resolve) => {
@@ -124,10 +126,15 @@ describe("useNavigationTargets openTableTarget", () => {
releaseColumns([column("id")]); releaseColumns([column("id")]);
await open; await open;
// 数据查询执行期间元数据未落地:行标识等待必须已挂起 expect(pendingDuringQuery).toBe(false);
expect(pendingDuringQuery).toBe(true);
expect(mocks.tabs[0]?.tableMeta?.primaryKeys).toEqual(["id"]); expect(mocks.tabs[0]?.tableMeta?.primaryKeys).toEqual(["id"]);
expect(mocks.tabs[0]?.tableMetaPending).toBe(false); expect(mocks.tabs[0]?.tableMetaPending).toBe(false);
expect(mocks.buildTableSelectSql).toHaveBeenCalledWith(
expect.objectContaining({
columns: ["id"],
primaryKeys: ["id"],
}),
);
}); });
it("reuses cached metadata and force-refreshes it once per catalog after a structure save", async () => { it("reuses cached metadata and force-refreshes it once per catalog after a structure save", async () => {
@@ -5,6 +5,7 @@ import { useQueryStore } from "@/stores/queryStore";
import { useSettingsStore } from "@/stores/settingsStore"; import { useSettingsStore } from "@/stores/settingsStore";
import { buildTableSelectSql, quoteTableDataIdentifier } from "@/lib/table/tableSelectSql"; import { buildTableSelectSql, quoteTableDataIdentifier } from "@/lib/table/tableSelectSql";
import { tableOpenPageLimit } from "@/lib/table/tableOpenPageLimit"; import { tableOpenPageLimit } from "@/lib/table/tableOpenPageLimit";
import { tableDataLargeValuePreviewOptions } from "@/lib/dataGrid/dataGridLargeValues";
import { usesSyntheticRowIdKey } from "@/lib/table/tableEditing"; import { usesSyntheticRowIdKey } from "@/lib/table/tableEditing";
import { tableMetaForDataTab } from "@/lib/table/tableDataTabMeta"; import { tableMetaForDataTab } from "@/lib/table/tableDataTabMeta";
import * as api from "@/lib/backend/api"; import * as api from "@/lib/backend/api";
@@ -65,6 +66,7 @@ export function useDataGridActions(activeTab: ComputedRef<QueryTab | undefined>)
// 结果(可能是失败结果的 ["Error"]),进入 SQL 会生成非法投影; // 结果(可能是失败结果的 ["Error"]),进入 SQL 会生成非法投影;
// 真实列缺失时省略 columns 让 builder 生成 SELECT * // 真实列缺失时省略 columns 让 builder 生成 SELECT *
const realColumns = tab.tableMeta?.columns.length ? tab.tableMeta.columns : undefined; const realColumns = tab.tableMeta?.columns.length ? tab.tableMeta.columns : undefined;
const limit = options.limit ?? tab.resultPageLimit ?? tableOpenPageLimit(settingsStore.editorSettings.tableOpenPageSize);
return buildTableSelectSql({ return buildTableSelectSql({
databaseType: effectiveDbType, databaseType: effectiveDbType,
driverProfile: config?.driver_profile, driverProfile: config?.driver_profile,
@@ -76,8 +78,9 @@ export function useDataGridActions(activeTab: ComputedRef<QueryTab | undefined>)
catalog: tableMeta?.catalog, catalog: tableMeta?.catalog,
columns: realColumns?.map((column) => column.name), columns: realColumns?.map((column) => column.name),
primaryKeys, primaryKeys,
...tableDataLargeValuePreviewOptions(effectiveDbType, realColumns ?? [], primaryKeys, limit),
includeRowId: useRowId, includeRowId: useRowId,
limit: options.limit ?? tab.resultPageLimit ?? tableOpenPageLimit(settingsStore.editorSettings.tableOpenPageSize), limit,
...options, ...options,
}); });
} }
@@ -4,6 +4,7 @@ import { invalidateTableMetadataCache, loadTableMetadata } from "@/lib/metadata/
import { canApplyDataTabMetadata } from "@/lib/sidebar/dataTabOpenPolicy"; import { canApplyDataTabMetadata } from "@/lib/sidebar/dataTabOpenPolicy";
import { isNoSnapshotErrorResult, isQueryExecutionErrorResult } from "@/lib/query/queryResultError"; import { isNoSnapshotErrorResult, isQueryExecutionErrorResult } from "@/lib/query/queryResultError";
import { buildTableSelectSql } from "@/lib/table/tableSelectSql"; import { buildTableSelectSql } from "@/lib/table/tableSelectSql";
import { tableDataLargeValuePreviewOptions } from "@/lib/dataGrid/dataGridLargeValues";
import { editableRowIdentifierColumns, usesSyntheticRowIdKey } from "@/lib/table/tableEditing"; import { editableRowIdentifierColumns, usesSyntheticRowIdKey } from "@/lib/table/tableEditing";
import { tableOpenPageLimit } from "@/lib/table/tableOpenPageLimit"; import { tableOpenPageLimit } from "@/lib/table/tableOpenPageLimit";
import { uuid } from "@/lib/common/utils"; import { uuid } from "@/lib/common/utils";
@@ -136,6 +137,21 @@ async function openTableTarget(target: NavigationTarget, options: { tableInfoTab
await queryStore.executeTabSql(tabId, sql, { pagination: { limit: pageLimit, offset: 0 } }); await queryStore.executeTabSql(tabId, sql, { pagination: { limit: pageLimit, offset: 0 } });
return; return;
} }
const eagerMetadata =
effectiveDbType === "mysql" || effectiveDbType === "postgres"
? await loadTableMetadata({
connectionId: target.connectionId,
database: target.database,
schema: querySchema,
tableName: target.tableName,
tableType: targetTableType,
databaseType: effectiveDbType,
driverProfile: config.driver_profile || config.db_type,
catalog: target.catalog,
})
: undefined;
const eagerColumns = eagerMetadata?.metadata.columns ?? [];
const eagerPrimaryKeys = eagerMetadata?.metadata.primaryKeys ?? [];
const sql = await buildTableSelectSql({ const sql = await buildTableSelectSql({
databaseType: effectiveDbType, databaseType: effectiveDbType,
driverProfile: config.driver_profile, driverProfile: config.driver_profile,
@@ -145,6 +161,9 @@ async function openTableTarget(target: NavigationTarget, options: { tableInfoTab
database: target.database, database: target.database,
tableName: target.tableName, tableName: target.tableName,
tableType: targetTableType, tableType: targetTableType,
columns: eagerColumns.map((column) => column.name),
primaryKeys: eagerPrimaryKeys,
...tableDataLargeValuePreviewOptions(effectiveDbType, eagerColumns, eagerPrimaryKeys, pageLimit),
whereInput: target.whereInput, whereInput: target.whereInput,
limit: pageLimit, limit: pageLimit,
}); });
@@ -156,8 +175,8 @@ async function openTableTarget(target: NavigationTarget, options: { tableInfoTab
database: target.database, database: target.database,
tableName: target.tableName, tableName: target.tableName,
tableType: targetTableType, tableType: targetTableType,
columns: [], columns: eagerColumns,
primaryKeys: [], primaryKeys: eagerPrimaryKeys,
}); });
firstExecuteStarted = true; firstExecuteStarted = true;
// 取消计数快照:isCancelling 是瞬态的(取消失败/查询先完成会被清掉), // 取消计数快照:isCancelling 是瞬态的(取消失败/查询先完成会被清掉),
@@ -13,6 +13,7 @@ import { hasTreeNodeDatabaseContext } from "@/lib/sidebar/treeNodeContext";
import { buildTableSelectSql } from "@/lib/table/tableSelectSql"; import { buildTableSelectSql } from "@/lib/table/tableSelectSql";
import { usesSyntheticRowIdKey } from "@/lib/table/tableEditing"; import { usesSyntheticRowIdKey } from "@/lib/table/tableEditing";
import { tableOpenPageLimit } from "@/lib/table/tableOpenPageLimit"; import { tableOpenPageLimit } from "@/lib/table/tableOpenPageLimit";
import { tableDataLargeValuePreviewOptions } from "@/lib/dataGrid/dataGridLargeValues";
import { canActivateExistingDataTableTab } from "@/lib/tabs/dataTabActivation"; import { canActivateExistingDataTableTab } from "@/lib/tabs/dataTabActivation";
import { beginDataTabNavigation, endDataTabNavigation, isCurrentDataTabNavigation } from "@/lib/tabs/dataTabNavigationGeneration"; import { beginDataTabNavigation, endDataTabNavigation, isCurrentDataTabNavigation } from "@/lib/tabs/dataTabNavigationGeneration";
@@ -293,18 +294,23 @@ export function useSidebarDataOpenRuntime() {
} else if (deferTableMetaRefresh) { } else if (deferTableMetaRefresh) {
logPhase("metadata-deferred", { tabId }); logPhase("metadata-deferred", { tabId });
} else { } else {
void refreshTableMetaInBackground(tabId);
logPhase("metadata-started", { tabId }); logPhase("metadata-started", { tabId });
} }
const metadataRefresh = shouldRefreshTableMeta && !deferTableMetaRefresh ? refreshTableMetaInBackground(tabId) : undefined;
if (!cachedTableMeta && (effectiveDbType === "mysql" || effectiveDbType === "postgres")) {
await metadataRefresh;
}
// Check if superseded by a newer openData call // Check if superseded by a newer openData call
if (!isActive()) { if (!isActive()) {
logPhase("superseded-before-build-sql", { tabId }); logPhase("superseded-before-build-sql", { tabId });
return; return;
} }
const columns = cachedTableMeta?.columns ?? []; const loadedTableMeta = cachedTableMeta ?? queryStore.tabs.find((item) => item.id === tabId)?.tableMeta;
const primaryKeys = cachedTableMeta?.primaryKeys ?? []; const columns = loadedTableMeta?.columns ?? [];
const primaryKeys = loadedTableMeta?.primaryKeys ?? [];
const includeRowId = usesSyntheticRowIdKey(effectiveDbType, primaryKeys, tableType); const includeRowId = usesSyntheticRowIdKey(effectiveDbType, primaryKeys, tableType);
const sql = await buildTableSelectSql({ const sql = await buildTableSelectSql({
databaseType: effectiveDbType, databaseType: effectiveDbType,
@@ -317,6 +323,7 @@ export function useSidebarDataOpenRuntime() {
catalog: node.catalog, catalog: node.catalog,
columns: columns.map((column) => column.name), columns: columns.map((column) => column.name),
primaryKeys, primaryKeys,
...tableDataLargeValuePreviewOptions(effectiveDbType, columns, primaryKeys, limit),
limit, limit,
includeRowId, includeRowId,
}); });
@@ -1,8 +1,63 @@
import { describe, expect, it } from "vitest"; import { describe, expect, it } from "vitest";
import { appendLargeValueCells, largeValueCellMap, remapLargeValueCells } from "@/lib/dataGrid/dataGridLargeValues"; import { appendLargeValueCells, canUseTableDataLargeValuePreview, largeValueCellMap, remapLargeValueCells, TABLE_DATA_CELL_PREVIEW_SIZE, TABLE_DATA_PREVIEW_CONTENT_MAX_BYTES, tableDataLargeValuePreviewOptions } from "@/lib/dataGrid/dataGridLargeValues";
import { buildDataGridCellDetail } from "@/lib/dataGrid/dataGridDetail"; import { buildDataGridCellDetail } from "@/lib/dataGrid/dataGridDetail";
import type { ColumnInfo } from "@/types/database";
function column(name: string, dataType: string, isPrimaryKey = false): ColumnInfo {
return {
name,
data_type: dataType,
is_nullable: false,
column_default: null,
is_primary_key: isPrimaryKey,
extra: null,
};
}
describe("data grid large-value metadata", () => { describe("data grid large-value metadata", () => {
it("enables bounded table previews only for supported databases with stable keys", () => {
const columns = [column("id", "bigint", true), column("payload", "longtext")];
expect(tableDataLargeValuePreviewOptions("mysql", columns, ["id"], 100)).toEqual({
columnTypes: ["bigint", "longtext"],
largeValuePreviewSize: TABLE_DATA_CELL_PREVIEW_SIZE,
});
expect(tableDataLargeValuePreviewOptions("postgres", columns, ["id"], 100)).toEqual({
columnTypes: ["bigint", "longtext"],
largeValuePreviewSize: TABLE_DATA_CELL_PREVIEW_SIZE,
});
expect(tableDataLargeValuePreviewOptions("sqlite", columns, ["id"], 100)).toEqual({});
expect(tableDataLargeValuePreviewOptions("mysql", columns, [], 100)).toEqual({});
expect(tableDataLargeValuePreviewOptions("mysql", [columns[0]!], ["id"], 100)).toEqual({});
});
it("disables marker parsing when a real column uses the reserved preview prefix", () => {
const columns = [column("id", "bigint", true), column("__dbx_large_value_bytes_t_0", "text")];
expect(canUseTableDataLargeValuePreview("postgres", columns, ["id"])).toBe(false);
expect(tableDataLargeValuePreviewOptions("postgres", columns, ["id"], 100)).toEqual({});
});
it("shrinks each cell preview by serialized byte budget without changing page size", () => {
const columns = [column("id", "bigint", true), ...Array.from({ length: 256 }, (_, index) => column(`payload_${index}`, "text"))];
const options = tableDataLargeValuePreviewOptions("postgres", columns, ["ID"], 1_000);
expect(options).toEqual({
columnTypes: columns.map((item) => item.data_type),
largeValuePreviewSize: 16,
});
if (!("largeValuePreviewSize" in options)) throw new Error("expected preview options");
const previewSize = options.largeValuePreviewSize;
const previewCellCount = 1_000 * 256;
const serializedSamples = ["中".repeat(previewSize), "😀".repeat(previewSize), "\u0000".repeat(previewSize)];
for (const sample of serializedSamples) {
const serializedContent = JSON.stringify(sample).slice(1, -1);
const serializedBytes = new TextEncoder().encode(serializedContent).byteLength * previewCellCount;
expect(serializedBytes).toBeLessThanOrEqual(TABLE_DATA_PREVIEW_CONTENT_MAX_BYTES);
}
});
it("offsets appended segment rows and discards metadata beyond the append cap", () => { it("offsets appended segment rows and discards metadata beyond the append cap", () => {
expect( expect(
appendLargeValueCells( appendLargeValueCells(
+2
View File
@@ -929,6 +929,7 @@ export async function executeMulti(
pageSize?: number; pageSize?: number;
maxResultBytes?: number; maxResultBytes?: number;
resultKeyColumns?: string[]; resultKeyColumns?: string[];
tableDataPreview?: boolean;
resultSessionId?: string; resultSessionId?: string;
clientSessionId?: string; clientSessionId?: string;
timeoutSecs?: number; timeoutSecs?: number;
@@ -971,6 +972,7 @@ export async function executeMultiWithProgress(
pageSize?: number; pageSize?: number;
maxResultBytes?: number; maxResultBytes?: number;
resultKeyColumns?: string[]; resultKeyColumns?: string[];
tableDataPreview?: boolean;
resultSessionId?: string; resultSessionId?: string;
clientSessionId?: string; clientSessionId?: string;
timeoutSecs?: number; timeoutSecs?: number;
+2
View File
@@ -1148,6 +1148,7 @@ export async function executeMulti(
pageSize?: number; pageSize?: number;
maxResultBytes?: number; maxResultBytes?: number;
resultKeyColumns?: string[]; resultKeyColumns?: string[];
tableDataPreview?: boolean;
resultSessionId?: string; resultSessionId?: string;
clientSessionId?: string; clientSessionId?: string;
timeoutSecs?: number; timeoutSecs?: number;
@@ -1194,6 +1195,7 @@ export async function executeMultiWithProgress(
pageSize?: number; pageSize?: number;
maxResultBytes?: number; maxResultBytes?: number;
resultKeyColumns?: string[]; resultKeyColumns?: string[];
tableDataPreview?: boolean;
resultSessionId?: string; resultSessionId?: string;
clientSessionId?: string; clientSessionId?: string;
timeoutSecs?: number; timeoutSecs?: number;
@@ -1,6 +1,27 @@
import type { QueryResult } from "@/types/database"; import type { ColumnInfo, DatabaseType, QueryResult } from "@/types/database";
export const TABLE_DATA_RESULT_MAX_BYTES = 32 * 1024 * 1024; export const TABLE_DATA_RESULT_MAX_BYTES = 32 * 1024 * 1024;
export const TABLE_DATA_CELL_PREVIEW_MIN_SIZE = 1;
export const TABLE_DATA_CELL_PREVIEW_SIZE = 8 * 1024;
export const TABLE_DATA_PREVIEW_CONTENT_MAX_BYTES = 24 * 1024 * 1024;
export const TABLE_DATA_TEXT_SERIALIZED_BYTES_PER_CHARACTER = 6;
const TABLE_DATA_LARGE_VALUE_MARKER_PREFIX = "__DBX_LARGE_VALUE_BYTES_";
export function canUseTableDataLargeValuePreview(databaseType: DatabaseType | undefined, columns: readonly ColumnInfo[], primaryKeys: readonly string[]): boolean {
return (databaseType === "mysql" || databaseType === "postgres") && columns.length > 0 && primaryKeys.length > 0 && !columns.some((column) => column.name.toLocaleUpperCase().startsWith(TABLE_DATA_LARGE_VALUE_MARKER_PREFIX));
}
export function tableDataLargeValuePreviewOptions(databaseType: DatabaseType | undefined, columns: readonly ColumnInfo[], primaryKeys: readonly string[], pageSize?: number): { columnTypes: string[]; largeValuePreviewSize: number } | Record<string, never> {
if (!canUseTableDataLargeValuePreview(databaseType, columns, primaryKeys)) return {};
const keyColumns = new Set(primaryKeys.map((column) => column.toLocaleLowerCase()));
const previewColumnCount = columns.filter((column) => !keyColumns.has(column.name.toLocaleLowerCase())).length;
if (previewColumnCount === 0) return {};
const budgetedSize = Math.floor(TABLE_DATA_PREVIEW_CONTENT_MAX_BYTES / Math.max(1, pageSize ?? 1) / previewColumnCount / TABLE_DATA_TEXT_SERIALIZED_BYTES_PER_CHARACTER);
return {
columnTypes: columns.map((column) => column.data_type),
largeValuePreviewSize: Math.max(TABLE_DATA_CELL_PREVIEW_MIN_SIZE, Math.min(TABLE_DATA_CELL_PREVIEW_SIZE, budgetedSize)),
};
}
export function largeValueCellKey(rowIndex: number, columnIndex: number): string { export function largeValueCellKey(rowIndex: number, columnIndex: number): string {
return `${rowIndex}:${columnIndex}`; return `${rowIndex}:${columnIndex}`;
@@ -13,6 +13,8 @@ export interface BuildTableSelectSqlOptions {
tableType?: string; tableType?: string;
primaryKeys?: string[]; primaryKeys?: string[];
columns?: string[]; columns?: string[];
columnTypes?: string[];
largeValuePreviewSize?: number;
fallbackOrderColumns?: string[]; fallbackOrderColumns?: string[];
orderBy?: string; orderBy?: string;
limit?: number; limit?: number;
@@ -114,7 +114,9 @@ describe("queryStore table data refresh", () => {
tableType: "TABLE", tableType: "TABLE",
catalog: undefined, catalog: undefined,
columns: ["id", "status", "created_at"], columns: ["id", "status", "created_at"],
columnTypes: ["integer", "text", "timestamp"],
primaryKeys: ["id"], primaryKeys: ["id"],
largeValuePreviewSize: 8192,
includeRowId: false, includeRowId: false,
whereInput: "status = 'ACTIVE'", whereInput: "status = 'ACTIVE'",
orderBy: "created_at DESC", orderBy: "created_at DESC",
+7 -3
View File
@@ -46,7 +46,7 @@ import { connectionObjectTreeNodeSchema, connectionQueryExecutionSchema, connect
import { frontendQueryTimeoutDelayMs, frontendQueryTimeoutSecsForSql, queryTimeoutSecsForConnection } from "@/lib/sql/queryTimeout"; import { frontendQueryTimeoutDelayMs, frontendQueryTimeoutSecsForSql, queryTimeoutSecsForConnection } from "@/lib/sql/queryTimeout";
import { queryResultNameFromPreamble, queryResultSourceLabel } from "@/lib/sql/queryResultSource"; import { queryResultNameFromPreamble, queryResultSourceLabel } from "@/lib/sql/queryResultSource";
import { beginDataGridNativeSelectionBlock, finishDataGridNativeSelectionBlock } from "@/lib/dataGrid/dataGridNativeSelection"; import { beginDataGridNativeSelectionBlock, finishDataGridNativeSelectionBlock } from "@/lib/dataGrid/dataGridNativeSelection";
import { appendLargeValueCells, remapLargeValueCells, TABLE_DATA_RESULT_MAX_BYTES } from "@/lib/dataGrid/dataGridLargeValues"; import { appendLargeValueCells, canUseTableDataLargeValuePreview, remapLargeValueCells, tableDataLargeValuePreviewOptions, TABLE_DATA_RESULT_MAX_BYTES } from "@/lib/dataGrid/dataGridLargeValues";
import { simpleDataGridOrderByReferencesMissingColumn, sortDataGridRowIndexes, type DataGridSortDirection } from "@/lib/dataGrid/dataGridSort"; import { simpleDataGridOrderByReferencesMissingColumn, sortDataGridRowIndexes, type DataGridSortDirection } from "@/lib/dataGrid/dataGridSort";
import { normalizeResultPageSize } from "@/lib/dataGrid/paginationPageSize"; import { normalizeResultPageSize } from "@/lib/dataGrid/paginationPageSize";
import { agentProtocolQueryResultMaxRows, capQueryResultTotal, effectiveQueryResultMaxRows, limitQueryPagination, queryResultLimitReached } from "@/lib/dataGrid/queryResultRowLimit"; import { agentProtocolQueryResultMaxRows, capQueryResultTotal, effectiveQueryResultMaxRows, limitQueryPagination, queryResultLimitReached } from "@/lib/dataGrid/queryResultRowLimit";
@@ -2782,6 +2782,7 @@ export const useQueryStore = defineStore("query", () => {
catalog: tableMeta.catalog, catalog: tableMeta.catalog,
columns: tableMeta.columns.map((column) => column.name), columns: tableMeta.columns.map((column) => column.name),
primaryKeys, primaryKeys,
...tableDataLargeValuePreviewOptions(effectiveDbType, tableMeta.columns, primaryKeys, limit),
includeRowId: usesSyntheticRowIdKey(effectiveDbType, primaryKeys, tableMeta.tableType), includeRowId: usesSyntheticRowIdKey(effectiveDbType, primaryKeys, tableMeta.tableType),
whereInput: tab.whereInput, whereInput: tab.whereInput,
orderBy, orderBy,
@@ -4573,6 +4574,8 @@ export const useQueryStore = defineStore("query", () => {
queryExecutionLog("info", "execute-multi:start", { traceId, elapsed: elapsed() }); queryExecutionLog("info", "execute-multi:start", { traceId, elapsed: elapsed() });
// Query and data tabs use a tab-scoped pool so repeated executions keep // Query and data tabs use a tab-scoped pool so repeated executions keep
// connection-local state and avoid MySQL pool resets on every refresh. // connection-local state and avoid MySQL pool resets on every refresh.
const dataTabMeta = tab.mode === "data" ? tableMetaForDataTab(tab) : undefined;
const useTableDataPreview = canUseTableDataLargeValuePreview(effectiveDbType, dataTabMeta?.columns ?? [], dataTabMeta?.primaryKeys ?? []);
const executionOptions = { const executionOptions = {
...(typeof pageLimit === "number" ...(typeof pageLimit === "number"
? useAgentResultSession ? useAgentResultSession
@@ -4586,10 +4589,11 @@ export const useQueryStore = defineStore("query", () => {
: { maxRows: pageLimit, fetchSize: pageLimit } : { maxRows: pageLimit, fetchSize: pageLimit }
: { maxRows: agentProtocolQueryResultMaxRows(queryResultMaxRows) }), : { maxRows: agentProtocolQueryResultMaxRows(queryResultMaxRows) }),
...(executionClientSessionId ? { clientSessionId: executionClientSessionId } : {}), ...(executionClientSessionId ? { clientSessionId: executionClientSessionId } : {}),
...(tab.mode === "data" && effectiveDbType === "mysql" ...(tab.mode === "data" && (effectiveDbType === "mysql" || effectiveDbType === "postgres")
? { ? {
maxResultBytes: TABLE_DATA_RESULT_MAX_BYTES, maxResultBytes: TABLE_DATA_RESULT_MAX_BYTES,
resultKeyColumns: tableMetaForDataTab(tab)?.primaryKeys ?? [], resultKeyColumns: dataTabMeta?.primaryKeys ?? [],
tableDataPreview: useTableDataPreview,
} }
: {}), : {}),
timeoutSecs: queryTimeoutSecs, timeoutSecs: queryTimeoutSecs,
+442 -5
View File
@@ -266,15 +266,43 @@ impl ExecuteMultiResult {
} }
fn success_with_index_and_large_values( fn success_with_index_and_large_values(
mut result: db::QueryResult,
statement_index: usize,
mut large_value_cells: Vec<db::LargeValueCell>,
table_data_preview: bool,
) -> Self {
let server_cells = if table_data_preview {
remap_large_value_cells_around_server_markers(&result, &mut large_value_cells);
extract_server_large_value_markers(&mut result)
} else {
Vec::new()
};
Self {
result,
large_value_cells: merge_large_value_cells(large_value_cells, server_cells),
execution_error: false,
statement_index: Some(statement_index),
error: None,
server_message: false,
}
}
fn success_with_index_and_optional_server_large_values(
result: db::QueryResult, result: db::QueryResult,
statement_index: usize, statement_index: usize,
large_value_cells: Vec<db::LargeValueCell>, table_data_preview: bool,
) -> Self { ) -> Self {
Self::success_with_index_and_large_values(result, statement_index, Vec::new(), table_data_preview)
}
fn success_with_optional_server_large_values(mut result: db::QueryResult, table_data_preview: bool) -> Self {
let large_value_cells =
if table_data_preview { extract_server_large_value_markers(&mut result) } else { Vec::new() };
Self { Self {
result, result,
large_value_cells, large_value_cells,
execution_error: false, execution_error: false,
statement_index: Some(statement_index), statement_index: None,
error: None, error: None,
server_message: false, server_message: false,
} }
@@ -290,6 +318,228 @@ impl ExecuteMultiResult {
} }
} }
const SERVER_LARGE_VALUE_UNKNOWN_BYTES: usize = 64 * 1024 * 1024;
#[derive(Clone, Copy, PartialEq, Eq)]
enum ServerLargeValuePreviewKind {
Text,
Binary,
Vector,
}
#[derive(Clone, Copy)]
struct ServerLargeValueMarker {
result_index: usize,
source_index: usize,
preview_kind: Option<ServerLargeValuePreviewKind>,
source_type: Option<&'static str>,
}
fn server_large_value_alias(
suffix: &str,
) -> Option<(usize, Option<ServerLargeValuePreviewKind>, Option<&'static str>)> {
if let Ok(source_index) = suffix.parse::<usize>() {
return Some((source_index, None, None));
}
let (kind, source_index) = suffix.split_once('_')?;
let source_index = source_index.parse::<usize>().ok()?;
let (preview_kind, source_type) = match kind {
"T" => (ServerLargeValuePreviewKind::Text, None),
"B" => (ServerLargeValuePreviewKind::Binary, None),
"V" => (ServerLargeValuePreviewKind::Vector, Some("vector")),
"J" => (ServerLargeValuePreviewKind::Text, Some("json")),
"K" => (ServerLargeValuePreviewKind::Text, Some("jsonb")),
"S" => (ServerLargeValuePreviewKind::Text, Some("tsvector")),
_ => return None,
};
Some((source_index, Some(preview_kind), source_type))
}
fn server_large_value_marker(value: &serde_json::Value) -> Option<(ServerLargeValuePreviewKind, usize)> {
let (kind, preview_size) = value.as_str()?.split_once(':')?;
let kind = match kind {
"T" => ServerLargeValuePreviewKind::Text,
"B" => ServerLargeValuePreviewKind::Binary,
"V" => ServerLargeValuePreviewKind::Vector,
_ => return None,
};
Some((kind, preview_size.parse::<usize>().ok()?.max(1)))
}
fn truncate_server_large_value_preview(
value: &mut serde_json::Value,
kind: ServerLargeValuePreviewKind,
preview_size: usize,
) -> bool {
let serde_json::Value::String(text) = value else {
return false;
};
if matches!(kind, ServerLargeValuePreviewKind::Vector) {
let truncated = text.chars().count() > preview_size;
let vector_text = if truncated {
let prefix: String = text.chars().take(preview_size).collect();
let Some(last_separator) = prefix.rfind(',') else {
return false;
};
format!("{}]", &prefix[..last_separator])
} else {
text.clone()
};
let Ok(vector) = serde_json::from_str::<Vec<serde_json::Value>>(&vector_text) else {
return false;
};
*value = serde_json::Value::Array(vector);
return truncated;
}
let truncate_at = match kind {
ServerLargeValuePreviewKind::Text => text.char_indices().nth(preview_size).map(|(index, _)| index),
ServerLargeValuePreviewKind::Binary => text
.strip_prefix("0x")
.filter(|hex| hex.len() > preview_size.saturating_mul(2))
.map(|_| 2usize.saturating_add(preview_size.saturating_mul(2))),
ServerLargeValuePreviewKind::Vector => unreachable!(),
};
let Some(truncate_at) = truncate_at else {
return false;
};
text.truncate(truncate_at);
text.push_str("...");
true
}
fn server_large_value_markers(result: &db::QueryResult) -> Vec<ServerLargeValueMarker> {
let mut markers = Vec::new();
for (result_index, column) in result.columns.iter().enumerate() {
let Some((source_index, preview_kind, source_type)) = column
.strip_prefix(crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX)
.and_then(server_large_value_alias)
else {
continue;
};
let expected_source_index = result_index.checked_sub(markers.len() + 1);
if expected_source_index == Some(source_index) {
markers.push(ServerLargeValueMarker { result_index, source_index, preview_kind, source_type });
}
}
markers
}
fn extract_server_large_value_markers(result: &mut db::QueryResult) -> Vec<db::LargeValueCell> {
let markers = server_large_value_markers(result);
if markers.is_empty() {
return Vec::new();
}
let mut large_value_cells = Vec::new();
for marker in &markers {
let source_result_index = marker.result_index.saturating_sub(1);
let row_kind = result
.rows
.iter()
.find_map(|row| row.get(marker.result_index).and_then(server_large_value_marker).map(|value| value.0));
let source_type = marker.source_type.or_else(|| {
(marker.preview_kind.or(row_kind) == Some(ServerLargeValuePreviewKind::Vector)).then_some("vector")
});
if let (Some(source_type), Some(column_type)) = (source_type, result.column_types.get_mut(source_result_index))
{
*column_type = source_type.to_string();
}
}
for (row_index, row) in result.rows.iter_mut().enumerate() {
for marker in &markers {
let marker_value = row.get(marker.result_index).and_then(server_large_value_marker);
let source_result_index = marker.result_index.saturating_sub(1);
if marker_value.is_some_and(|(kind, preview_size)| {
row.get_mut(source_result_index)
.is_some_and(|value| truncate_server_large_value_preview(value, kind, preview_size))
}) {
large_value_cells.push(db::LargeValueCell {
row_index,
column_index: marker.source_index,
original_bytes: SERVER_LARGE_VALUE_UNKNOWN_BYTES,
});
}
}
}
let removed: std::collections::HashSet<usize> = markers.iter().map(|marker| marker.result_index).collect();
let retained_index = |index: usize| index - removed.iter().filter(|removed_index| **removed_index < index).count();
result.spatial_columns.retain_mut(|column| {
if removed.contains(&column.column_index) {
return false;
}
column.column_index = retained_index(column.column_index);
true
});
for row in &mut result.rows {
let mut index = 0;
row.retain(|_| {
let keep = !removed.contains(&index);
index += 1;
keep
});
}
for row in &mut result.spatial_values {
let mut index = 0;
row.retain(|_| {
let keep = !removed.contains(&index);
index += 1;
keep
});
}
let mut index = 0;
result.columns.retain(|_| {
let keep = !removed.contains(&index);
index += 1;
keep
});
let mut index = 0;
result.column_types.retain(|_| {
let keep = !removed.contains(&index);
index += 1;
keep
});
let mut index = 0;
result.column_sortables.retain(|_| {
let keep = !removed.contains(&index);
index += 1;
keep
});
large_value_cells
}
fn remap_large_value_cells_around_server_markers(result: &db::QueryResult, cells: &mut Vec<db::LargeValueCell>) {
let removed: std::collections::HashSet<usize> =
server_large_value_markers(result).into_iter().map(|marker| marker.result_index).collect();
if removed.is_empty() {
return;
}
cells.retain_mut(|cell| {
if removed.contains(&cell.column_index) {
return false;
}
cell.column_index -= removed.iter().filter(|index| **index < cell.column_index).count();
true
});
}
fn merge_large_value_cells(
mut driver_cells: Vec<db::LargeValueCell>,
server_cells: Vec<db::LargeValueCell>,
) -> Vec<db::LargeValueCell> {
for server_cell in server_cells {
if let Some(driver_cell) = driver_cells
.iter_mut()
.find(|cell| cell.row_index == server_cell.row_index && cell.column_index == server_cell.column_index)
{
*driver_cell = server_cell;
} else {
driver_cells.push(server_cell);
}
}
driver_cells
}
impl From<db::QueryResult> for ExecuteMultiResult { impl From<db::QueryResult> for ExecuteMultiResult {
fn from(result: db::QueryResult) -> Self { fn from(result: db::QueryResult) -> Self {
Self { Self {
@@ -639,6 +889,9 @@ pub struct QueryExecutionOptions {
/// Result columns that must stay exact because clients use them as stable /// Result columns that must stay exact because clients use them as stable
/// row identifiers when fetching full large-cell values on demand. /// row identifiers when fetching full large-cell values on demand.
pub result_key_columns: Vec<String>, pub result_key_columns: Vec<String>,
/// Enables extraction of hidden server-side preview metadata. This must be
/// set only for generated table-data preview SQL, never arbitrary queries.
pub table_data_preview: bool,
/// Doris / StarRocks catalog selected for this query tab. /// Doris / StarRocks catalog selected for this query tab.
pub catalog: Option<String>, pub catalog: Option<String>,
pub result_session_id: Option<String>, pub result_session_id: Option<String>,
@@ -2252,6 +2505,7 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed(
// HTTP SQLite providers send all statements in one request so the provider // HTTP SQLite providers send all statements in one request so the provider
// can preserve batch ordering and atomicity. // can preserve batch ordering and atomicity.
if is_http_sqlite { if is_http_sqlite {
let table_data_preview = options.table_data_preview;
return single_statement_multi_result( return single_statement_multi_result(
execute_sql_statement_with_options_typed( execute_sql_statement_with_options_typed(
state, state,
@@ -2263,6 +2517,7 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed(
options, options,
) )
.await, .await,
table_data_preview,
); );
} }
@@ -2306,6 +2561,7 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed(
) )
{ {
let single_sql = statements.into_iter().next().unwrap_or_default(); let single_sql = statements.into_iter().next().unwrap_or_default();
let table_data_preview = options.table_data_preview;
return single_statement_multi_result( return single_statement_multi_result(
execute_sql_statement_with_options_typed( execute_sql_statement_with_options_typed(
state, state,
@@ -2317,6 +2573,7 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed(
options, options,
) )
.await, .await,
table_data_preview,
); );
} }
@@ -2372,7 +2629,11 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed(
{ {
Ok(r) => { Ok(r) => {
report_execute_multi_progress(progress.as_ref(), statement_index, statements.len(), &r, true, None); report_execute_multi_progress(progress.as_ref(), statement_index, statements.len(), &r, true, None);
results.push(ExecuteMultiResult::success_with_index(r, statement_index)); results.push(ExecuteMultiResult::success_with_index_and_optional_server_large_values(
r,
statement_index,
options.table_data_preview,
));
} }
Err(error) => { Err(error) => {
let action = query_execution_error_action(db_type, stmt, &error); let action = query_execution_error_action(db_type, stmt, &error);
@@ -2403,8 +2664,9 @@ pub async fn execute_multi_core_with_options_for_client_and_progress_typed(
fn single_statement_multi_result( fn single_statement_multi_result(
result: Result<db::QueryResult, QueryExecutionError>, result: Result<db::QueryResult, QueryExecutionError>,
table_data_preview: bool,
) -> Result<Vec<ExecuteMultiResult>, QueryExecutionError> { ) -> Result<Vec<ExecuteMultiResult>, QueryExecutionError> {
result.map(|result| vec![result.into()]) result.map(|result| vec![ExecuteMultiResult::success_with_optional_server_large_values(result, table_data_preview)])
} }
fn mysql_single_statement_uses_batch_route( fn mysql_single_statement_uses_batch_route(
@@ -2420,6 +2682,10 @@ fn mysql_single_statement_uses_batch_route(
} }
trait MysqlBatchStatementExecutor { trait MysqlBatchStatementExecutor {
fn table_data_preview(&self) -> bool {
false
}
async fn execute_statement(&mut self, statement: &str) -> Result<Vec<db::mysql::MySqlQueryResult>, String>; async fn execute_statement(&mut self, statement: &str) -> Result<Vec<db::mysql::MySqlQueryResult>, String>;
async fn execute_non_result_batch(&mut self, statements: &[String]) -> db::mysql::MySqlNonResultBatchOutcome { async fn execute_non_result_batch(&mut self, statements: &[String]) -> db::mysql::MySqlNonResultBatchOutcome {
@@ -2450,10 +2716,15 @@ struct MysqlBatchConnection<'a> {
max_rows: Option<usize>, max_rows: Option<usize>,
max_result_bytes: Option<usize>, max_result_bytes: Option<usize>,
result_key_columns: &'a [String], result_key_columns: &'a [String],
table_data_preview: bool,
dialect: db::mysql::MySqlQueryDialect, dialect: db::mysql::MySqlQueryDialect,
} }
impl MysqlBatchStatementExecutor for MysqlBatchConnection<'_> { impl MysqlBatchStatementExecutor for MysqlBatchConnection<'_> {
fn table_data_preview(&self) -> bool {
self.table_data_preview
}
async fn execute_statement(&mut self, statement: &str) -> Result<Vec<db::mysql::MySqlQueryResult>, String> { async fn execute_statement(&mut self, statement: &str) -> Result<Vec<db::mysql::MySqlQueryResult>, String> {
wait_for_result_opt( wait_for_result_opt(
self.cancel_token.clone(), self.cancel_token.clone(),
@@ -2553,6 +2824,7 @@ where
{ {
let mut results = Vec::with_capacity(statements.len()); let mut results = Vec::with_capacity(statements.len());
let mut statement_index = 0usize; let mut statement_index = 0usize;
let table_data_preview = executor.table_data_preview();
while statement_index < statements.len() { while statement_index < statements.len() {
if is_canceled(&cancel_token) { if is_canceled(&cancel_token) {
results.push(ExecuteMultiResult::execution_error(error_query_result(canceled_error()))); results.push(ExecuteMultiResult::execution_error(error_query_result(canceled_error())));
@@ -2618,6 +2890,7 @@ where
result.result, result.result,
statement_index, statement_index,
result.large_value_cells, result.large_value_cells,
table_data_preview,
) )
})); }));
} }
@@ -2711,6 +2984,7 @@ async fn execute_multi_mysql(
max_rows, max_rows,
max_result_bytes, max_result_bytes,
result_key_columns: &options.result_key_columns, result_key_columns: &options.result_key_columns,
table_data_preview: options.table_data_preview,
dialect, dialect,
}; };
let (results, error_action) = execute_mysql_batch_statements( let (results, error_action) = execute_mysql_batch_statements(
@@ -5766,6 +6040,7 @@ for line in sys.stdin:
empty_query_result(1), empty_query_result(1),
0, 0,
vec![db::LargeValueCell { row_index: 2, column_index: 3, original_bytes: 65_536 }], vec![db::LargeValueCell { row_index: 2, column_index: 3, original_bytes: 65_536 }],
false,
); );
let serialized = serde_json::to_value(result).unwrap(); let serialized = serde_json::to_value(result).unwrap();
@@ -5901,7 +6176,7 @@ for line in sys.stdin:
) )
.with_omitted_sql_context("SELECT * FROM dbx_table_that_does_not_exist"); .with_omitted_sql_context("SELECT * FROM dbx_table_that_does_not_exist");
let error = single_statement_multi_result(Err(error)).unwrap_err(); let error = single_statement_multi_result(Err(error), false).unwrap_err();
let backend_error = error.into_backend_error(); let backend_error = error.into_backend_error();
assert_eq!(backend_error.code(), "DBX-JDBC-4001"); assert_eq!(backend_error.code(), "DBX-JDBC-4001");
@@ -6900,6 +7175,168 @@ for line in sys.stdin:
assert_eq!(normalized.rows[0][1], serde_json::json!([1, "2041797190226354178"])); assert_eq!(normalized.rows[0][1], serde_json::json!([1, "2041797190226354178"]));
} }
#[test]
fn extracts_server_large_value_markers_and_restores_source_columns() {
let mut result = db::QueryResult {
columns: vec![
"id".to_string(),
"payload".to_string(),
format!("{}T_1", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX),
"note".to_string(),
],
column_types: vec!["integer".to_string(), "text".to_string(), "bigint".to_string(), "text".to_string()],
column_sortables: vec![true, true, true, true],
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![
vec![
serde_json::json!(1),
serde_json::json!("预览文本多"),
serde_json::json!("T:4"),
serde_json::json!("a"),
],
vec![serde_json::json!(2), serde_json::json!("短值"), serde_json::json!("T:4"), serde_json::json!("b")],
],
affected_rows: 0,
execution_time_ms: 0,
truncated: false,
session_id: None,
has_more: false,
elasticsearch_raw_body: None,
messages: Vec::new(),
};
let cells = extract_server_large_value_markers(&mut result);
assert_eq!(result.columns, vec!["id", "payload", "note"]);
assert_eq!(result.column_types, vec!["integer", "text", "text"]);
assert_eq!(
result.rows[0],
vec![serde_json::json!(1), serde_json::json!("预览文本..."), serde_json::json!("a")]
);
assert_eq!(result.rows[1], vec![serde_json::json!(2), serde_json::json!("短值"), serde_json::json!("b")]);
assert_eq!(
cells,
vec![db::LargeValueCell {
row_index: 0,
column_index: 1,
original_bytes: SERVER_LARGE_VALUE_UNKNOWN_BYTES,
}]
);
}
#[test]
fn truncates_server_binary_preview_after_the_configured_byte_count() {
let mut result = db::QueryResult {
columns: vec![
"raw_value".to_string(),
format!("{}B_0", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX),
],
column_types: vec!["bytea".to_string(), "text".to_string()],
column_sortables: vec![true, true],
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![vec![serde_json::json!("0x0102030405"), serde_json::json!("B:4")]],
affected_rows: 0,
execution_time_ms: 0,
truncated: false,
session_id: None,
has_more: false,
elasticsearch_raw_body: None,
messages: Vec::new(),
};
let cells = extract_server_large_value_markers(&mut result);
assert_eq!(result.rows, vec![vec![serde_json::json!("0x01020304...")]]);
assert_eq!(cells.len(), 1);
}
#[test]
fn restores_pgvector_previews_as_arrays_and_marks_only_truncated_values() {
let marker = format!("{}V_0", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX);
let mut result = db::QueryResult {
columns: vec!["embedding".to_string(), marker],
column_types: vec!["text".to_string(), "text".to_string()],
column_sortables: vec![true, true],
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![
vec![serde_json::json!("[0.1,0.2,0.3]"), serde_json::json!("V:9")],
vec![serde_json::json!("[0.1,0.2]"), serde_json::json!("V:20")],
],
affected_rows: 0,
execution_time_ms: 0,
truncated: false,
session_id: None,
has_more: false,
elasticsearch_raw_body: None,
messages: Vec::new(),
};
let cells = extract_server_large_value_markers(&mut result);
assert_eq!(result.column_types, vec!["vector"]);
assert_eq!(result.rows[0][0], serde_json::json!([0.1, 0.2]));
assert_eq!(result.rows[1][0], serde_json::json!([0.1, 0.2]));
assert_eq!(cells.len(), 1);
assert_eq!(cells[0].row_index, 0);
}
#[test]
fn restores_server_preview_types_without_rows() {
let mut result = db::QueryResult {
columns: vec![
"embedding".to_string(),
format!("{}V_0", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX),
"metadata".to_string(),
format!("{}K_1", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX),
],
column_types: vec!["text".to_string(), "text".to_string(), "text".to_string(), "text".to_string()],
column_sortables: vec![true; 4],
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: Vec::new(),
affected_rows: 0,
execution_time_ms: 0,
truncated: false,
session_id: None,
has_more: false,
elasticsearch_raw_body: None,
messages: Vec::new(),
};
let cells = extract_server_large_value_markers(&mut result);
assert_eq!(result.columns, vec!["embedding", "metadata"]);
assert_eq!(result.column_types, vec!["vector", "jsonb"]);
assert!(cells.is_empty());
}
#[test]
fn ordinary_queries_preserve_columns_that_resemble_preview_markers() {
let marker = format!("{}0", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX);
let mut result = empty_query_result(1);
result.columns = vec!["payload".to_string(), marker.clone()];
result.column_types = vec!["text".to_string(), "bigint".to_string()];
result.column_sortables = vec![true, true];
result.rows = vec![vec![serde_json::json!("value"), serde_json::json!(123)]];
let ordinary = ExecuteMultiResult::success_with_index_and_optional_server_large_values(result, 0, false);
assert_eq!(ordinary.result.columns, vec!["payload".to_string(), marker]);
assert_eq!(ordinary.result.rows[0], vec![serde_json::json!("value"), serde_json::json!(123)]);
assert!(ordinary.large_value_cells.is_empty());
}
#[test]
fn single_statement_preview_preserves_absent_statement_index() {
let result = single_statement_multi_result(Ok(empty_query_result(1)), true).unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].statement_index, None);
}
#[test] #[test]
fn query_execution_options_default_use_transaction_is_none() { fn query_execution_options_default_use_transaction_is_none() {
let opts = QueryExecutionOptions::default(); let opts = QueryExecutionOptions::default();
+3 -1
View File
@@ -38,7 +38,9 @@ pub use identifiers::{
normalize_where_input, qualified_table_name, qualified_table_name_with_catalog, quote_table_identifier, normalize_where_input, qualified_table_name, qualified_table_name_with_catalog, quote_table_identifier,
}; };
pub(crate) use identifiers::{parse_sqlserver_linked_schema_ref, qualified_transfer_table, quote_transfer_identifier}; pub(crate) use identifiers::{parse_sqlserver_linked_schema_ref, qualified_transfer_table, quote_transfer_identifier};
pub use table_select::{build_count_table_sql, build_table_data_select_sql, build_table_select_sql}; pub use table_select::{
build_count_table_sql, build_table_data_select_sql, build_table_select_sql, DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX,
};
pub(crate) use table_select::{ pub(crate) use table_select::{
quote_table_data_identifier, table_data_qualified_table_name, table_data_schema, uses_connection_identifier_quote, quote_table_data_identifier, table_data_qualified_table_name, table_data_schema, uses_connection_identifier_quote,
}; };
@@ -12,6 +12,131 @@ use super::types::{
DBX_TDENGINE_TBNAME_COLUMN, DBX_TDENGINE_TBNAME_COLUMN,
}; };
pub const DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX: &str = "__DBX_LARGE_VALUE_BYTES_";
#[derive(Clone, Copy, PartialEq, Eq)]
enum LargeValuePreviewKind {
Text,
Binary,
TextCast,
Vector,
}
fn large_value_marker_alias_kind(kind: LargeValuePreviewKind, data_type: &str) -> &'static str {
match kind {
LargeValuePreviewKind::Binary => "B",
LargeValuePreviewKind::Vector => "V",
LargeValuePreviewKind::TextCast => match normalized_data_type_base(data_type).as_str() {
"json" => "J",
"jsonb" => "K",
"tsvector" => "S",
_ => "T",
},
LargeValuePreviewKind::Text => "T",
}
}
fn normalized_data_type_base(data_type: &str) -> String {
data_type.trim().split(['(', '[']).next().unwrap_or_default().trim().to_ascii_lowercase()
}
fn large_value_preview_kind(database_type: Option<DatabaseType>, data_type: &str) -> Option<LargeValuePreviewKind> {
let normalized = data_type.trim().to_ascii_lowercase();
let base = normalized_data_type_base(data_type);
match database_type {
Some(DatabaseType::Mysql) => {
if matches!(base.as_str(), "binary" | "varbinary" | "tinyblob" | "blob" | "mediumblob" | "longblob") {
Some(LargeValuePreviewKind::Binary)
} else if base == "json" {
Some(LargeValuePreviewKind::TextCast)
} else if matches!(base.as_str(), "char" | "varchar" | "tinytext" | "text" | "mediumtext" | "longtext") {
Some(LargeValuePreviewKind::Text)
} else {
None
}
}
Some(DatabaseType::Postgres) => {
if base == "bytea" {
Some(LargeValuePreviewKind::Binary)
} else if matches!(base.as_str(), "char" | "character" | "varchar" | "text" | "citext" | "name" | "xml")
|| normalized.starts_with("character varying")
{
Some(LargeValuePreviewKind::Text)
} else if base == "vector" {
Some(LargeValuePreviewKind::Vector)
} else if matches!(base.as_str(), "json" | "jsonb" | "tsvector") {
Some(LargeValuePreviewKind::TextCast)
} else {
None
}
}
_ => None,
}
}
fn build_large_value_preview_columns(options: &TableDataSelectSqlOptions) -> Option<String> {
let database_type = options.database_type;
let preview_size = options.large_value_preview_size?.max(1);
if options.columns.is_empty()
|| options.columns.len() != options.column_types.len()
|| options.primary_keys.is_empty()
|| options
.columns
.iter()
.any(|column| column.to_ascii_uppercase().starts_with(DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX))
{
return None;
}
let protected: std::collections::HashSet<String> =
options.primary_keys.iter().map(|column| column.to_ascii_lowercase()).collect();
let mut projections = Vec::with_capacity(options.columns.len() * 2);
let mut marker_count = 0;
for (column_index, (column, data_type)) in options.columns.iter().zip(&options.column_types).enumerate() {
let quoted = if uses_connection_identifier_quote(database_type, options.identifier_quote.as_deref()) {
quote_table_data_identifier(database_type, column, options.identifier_quote.as_deref())
} else {
quote_table_identifier(database_type, column)
};
let kind = (!protected.contains(&column.to_ascii_lowercase()))
.then(|| large_value_preview_kind(database_type, data_type))
.flatten();
let Some(kind) = kind else {
projections.push(quoted);
continue;
};
let alias_kind = large_value_marker_alias_kind(kind, data_type);
let marker_alias = quote_table_identifier(
database_type,
&format!("{DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX}{alias_kind}_{column_index}"),
);
let prefix_size = preview_size.saturating_add(1);
let (preview, marker_kind) = match database_type {
Some(DatabaseType::Mysql) if kind == LargeValuePreviewKind::Binary => {
(format!("LEFT({quoted}, {prefix_size}) AS {quoted}"), "B")
}
Some(DatabaseType::Mysql) => (format!("LEFT({quoted}, {prefix_size}) AS {quoted}"), "T"),
Some(DatabaseType::Postgres) if kind == LargeValuePreviewKind::Binary => {
(format!("substring({quoted} from 1 for {prefix_size}) AS {quoted}"), "B")
}
Some(DatabaseType::Postgres) if kind == LargeValuePreviewKind::TextCast => {
(format!("left({quoted}::text, {prefix_size}) AS {quoted}"), "T")
}
Some(DatabaseType::Postgres) if kind == LargeValuePreviewKind::Vector => {
(format!("left({quoted}::text, {prefix_size}) AS {quoted}"), "V")
}
Some(DatabaseType::Postgres) => (format!("left({quoted}, {prefix_size}) AS {quoted}"), "T"),
_ => return None,
};
let marker = format!("'{marker_kind}:{preview_size}' AS {marker_alias}");
projections.push(preview);
projections.push(marker);
marker_count += 1;
}
(marker_count > 0).then(|| projections.join(", "))
}
pub fn build_count_table_sql(database_type: Option<DatabaseType>, schema: Option<&str>, table_name: &str) -> String { pub fn build_count_table_sql(database_type: Option<DatabaseType>, schema: Option<&str>, table_name: &str) -> String {
if database_type == Some(DatabaseType::VictoriaMetrics) { if database_type == Some(DatabaseType::VictoriaMetrics) {
return format!("count({})", victoriametrics_metric_selector(table_name)); return format!("count({})", victoriametrics_metric_selector(table_name));
@@ -77,6 +202,8 @@ pub fn build_table_data_select_sql(options: TableDataSelectSqlOptions) -> String
let select_columns = if include_oracle_row_id { let select_columns = if include_oracle_row_id {
format!("ROWIDTOCHAR(t.ROWID) AS \"{DBX_ROWID_COLUMN}\", t.*") format!("ROWIDTOCHAR(t.ROWID) AS \"{DBX_ROWID_COLUMN}\", t.*")
} else if let Some(preview_columns) = build_large_value_preview_columns(&options) {
preview_columns
} else { } else {
build_select_columns( build_select_columns(
database_type, database_type,
@@ -521,6 +648,8 @@ mod tests {
table_type: None, table_type: None,
primary_keys: Vec::new(), primary_keys: Vec::new(),
columns: Vec::new(), columns: Vec::new(),
column_types: Vec::new(),
large_value_preview_size: None,
fallback_order_columns: Vec::new(), fallback_order_columns: Vec::new(),
order_by: None, order_by: None,
limit: Some(10), limit: Some(10),
+91
View File
@@ -641,6 +641,97 @@ fn builds_table_data_where_and_schema_queries() {
); );
} }
#[test]
fn builds_mysql_table_data_large_value_previews_without_truncating_keys() {
let sql = build_table_data_select_sql(TableDataSelectSqlOptions {
database_type: Some(DatabaseType::Mysql),
table_name: "large_rows".to_string(),
primary_keys: vec!["id".to_string()],
columns: vec!["id".to_string(), "payload".to_string(), "raw_value".to_string(), "metadata".to_string()],
column_types: vec!["bigint".to_string(), "longtext".to_string(), "longblob".to_string(), "json".to_string()],
large_value_preview_size: Some(4096),
limit: Some(100),
..Default::default()
});
assert!(sql.starts_with("SELECT `id`, LEFT(`payload`, 4097) AS `payload`"));
assert!(sql.contains("'T:4096' AS `__DBX_LARGE_VALUE_BYTES_T_1`"));
assert!(sql.contains("LEFT(`raw_value`, 4097) AS `raw_value`"));
assert!(sql.contains("'B:4096' AS `__DBX_LARGE_VALUE_BYTES_B_2`"));
assert!(sql.contains("LEFT(`metadata`, 4097) AS `metadata`"));
assert!(sql.contains("'T:4096' AS `__DBX_LARGE_VALUE_BYTES_J_3`"));
assert!(!sql.contains("__DBX_LARGE_VALUE_BYTES_0"));
}
#[test]
fn builds_postgres_table_data_large_value_previews() {
let sql = build_table_data_select_sql(TableDataSelectSqlOptions {
database_type: Some(DatabaseType::Postgres),
schema: Some("public".to_string()),
table_name: "large_rows".to_string(),
primary_keys: vec!["id".to_string()],
columns: vec!["id".to_string(), "payload".to_string(), "metadata".to_string()],
column_types: vec!["integer".to_string(), "text".to_string(), "jsonb".to_string()],
large_value_preview_size: Some(8192),
limit: Some(100),
..Default::default()
});
assert!(sql.starts_with("SELECT \"id\", left(\"payload\", 8193) AS \"payload\""));
assert!(sql.contains("'T:8192' AS \"__DBX_LARGE_VALUE_BYTES_T_1\""));
assert!(sql.contains("left(\"metadata\"::text, 8193) AS \"metadata\""));
assert!(sql.contains("'T:8192' AS \"__DBX_LARGE_VALUE_BYTES_K_2\""));
}
#[test]
fn builds_pgvector_table_data_preview_with_compatible_text_cast() {
let sql = build_table_data_select_sql(TableDataSelectSqlOptions {
database_type: Some(DatabaseType::Postgres),
schema: Some("public".to_string()),
table_name: "embeddings".to_string(),
primary_keys: vec!["id".to_string()],
columns: vec!["id".to_string(), "embedding".to_string()],
column_types: vec!["integer".to_string(), "vector(16000)".to_string()],
large_value_preview_size: Some(8192),
limit: Some(100),
..Default::default()
});
assert!(sql.contains("left(\"embedding\"::text, 8193) AS \"embedding\""));
assert!(sql.contains("'V:8192' AS \"__DBX_LARGE_VALUE_BYTES_V_1\""));
}
#[test]
fn table_data_preview_requires_stable_keys_and_parallel_types() {
let base = TableDataSelectSqlOptions {
database_type: Some(DatabaseType::Postgres),
table_name: "large_rows".to_string(),
columns: vec!["payload".to_string()],
column_types: vec!["text".to_string()],
large_value_preview_size: Some(8192),
limit: Some(100),
..Default::default()
};
assert_eq!(build_table_data_select_sql(base.clone()), "SELECT * FROM \"large_rows\" LIMIT 100;");
assert_eq!(
build_table_data_select_sql(TableDataSelectSqlOptions {
primary_keys: vec!["id".to_string()],
column_types: Vec::new(),
..base.clone()
}),
"SELECT * FROM \"large_rows\" LIMIT 100;"
);
assert_eq!(
build_table_data_select_sql(TableDataSelectSqlOptions {
primary_keys: vec!["id".to_string()],
columns: vec!["id".to_string(), "__dbx_large_value_bytes_t_0".to_string()],
column_types: vec!["integer".to_string(), "text".to_string()],
..base
}),
"SELECT * FROM \"large_rows\" LIMIT 100;"
);
}
#[test] #[test]
fn builds_informix_table_data_with_skip_first_pagination() { fn builds_informix_table_data_with_skip_first_pagination() {
assert_eq!( assert_eq!(
+9
View File
@@ -44,6 +44,15 @@ pub struct TableDataSelectSqlOptions {
pub primary_keys: Vec<String>, pub primary_keys: Vec<String>,
#[serde(default)] #[serde(default)]
pub columns: Vec<String>, pub columns: Vec<String>,
/// Database type names parallel to `columns`. Table-data callers can use
/// these to request bounded server-side projections for variable-length
/// values without changing arbitrary query execution.
#[serde(default)]
pub column_types: Vec<String>,
/// Maximum characters (text-like values) or bytes (binary values) kept in
/// a table-data preview. Omitted for exports and ordinary SQL queries.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub large_value_preview_size: Option<usize>,
#[serde(default)] #[serde(default)]
pub fallback_order_columns: Vec<String>, pub fallback_order_columns: Vec<String>,
#[serde(default, skip_serializing_if = "Option::is_none")] #[serde(default, skip_serializing_if = "Option::is_none")]
+4
View File
@@ -24,6 +24,8 @@ pub struct ExecuteQueryRequest {
pub max_result_bytes: Option<usize>, pub max_result_bytes: Option<usize>,
#[serde(default)] #[serde(default)]
pub result_key_columns: Vec<String>, pub result_key_columns: Vec<String>,
#[serde(default)]
pub table_data_preview: bool,
pub result_session_id: Option<String>, pub result_session_id: Option<String>,
pub client_session_id: Option<String>, pub client_session_id: Option<String>,
pub timeout_secs: Option<u64>, pub timeout_secs: Option<u64>,
@@ -363,6 +365,7 @@ pub async fn execute_query(
page_size: req.page_size, page_size: req.page_size,
max_result_bytes: req.max_result_bytes, max_result_bytes: req.max_result_bytes,
result_key_columns: req.result_key_columns, result_key_columns: req.result_key_columns,
table_data_preview: req.table_data_preview,
catalog: req.catalog, catalog: req.catalog,
result_session_id: req.result_session_id, result_session_id: req.result_session_id,
client_session_id: req.client_session_id, client_session_id: req.client_session_id,
@@ -411,6 +414,7 @@ pub async fn execute_multi(
page_size: req.page_size, page_size: req.page_size,
max_result_bytes: req.max_result_bytes, max_result_bytes: req.max_result_bytes,
result_key_columns: req.result_key_columns, result_key_columns: req.result_key_columns,
table_data_preview: req.table_data_preview,
catalog: req.catalog, catalog: req.catalog,
result_session_id: req.result_session_id, result_session_id: req.result_session_id,
client_session_id: req.client_session_id, client_session_id: req.client_session_id,
+2
View File
@@ -90,6 +90,7 @@ pub async fn execute_multi(
page_size: Option<usize>, page_size: Option<usize>,
max_result_bytes: Option<usize>, max_result_bytes: Option<usize>,
result_key_columns: Option<Vec<String>>, result_key_columns: Option<Vec<String>>,
table_data_preview: Option<bool>,
result_session_id: Option<String>, result_session_id: Option<String>,
client_session_id: Option<String>, client_session_id: Option<String>,
timeout_secs: Option<u64>, timeout_secs: Option<u64>,
@@ -148,6 +149,7 @@ pub async fn execute_multi(
page_size, page_size,
max_result_bytes, max_result_bytes,
result_key_columns: result_key_columns.unwrap_or_default(), result_key_columns: result_key_columns.unwrap_or_default(),
table_data_preview: table_data_preview.unwrap_or(false),
catalog, catalog,
result_session_id, result_session_id,
client_session_id, client_session_id,