fix(grid): bound large table previews by byte budget

This commit is contained in:
zipg authored and GitHub committed 2026-08-13 13:25:04 +08:00
1 parent 020654c33c
commit e85d3dcc59
19 files changed
+823 -23

No files matched your search

@@ -151,7 +151,7 @@ import { isCancelSearchShortcut, isCopyCurrentRowShortcut, isDeleteCurrentRowSho
import { dataGridHeaderContentWidth, scrollbarGutterWidth } from "@/lib/dataGrid/dataGridScrollGutter";
import { canFetchNextDataGridSegment, canGoNextDataGridPage, dataGridTotalRowCountLabelKey, dataGridTruncationHintKey, hasCompleteLocalDataGridResult, resolveDataGridPaginationTotal, type DataGridInexactTotalRowCountMode } from "@/lib/dataGrid/dataGridPagination";
import { dataGridCountQueryOptions } from "@/lib/dataGrid/dataGridQueryOptions";
import { largeValueCellKey, largeValueCellMap } from "@/lib/dataGrid/dataGridLargeValues";
import { largeValueCellKey, largeValueCellMap, tableDataLargeValuePreviewOptions } from "@/lib/dataGrid/dataGridLargeValues";
import { dataGridBottomScrollTop, dataGridScrollPosition, isDataGridAtScrollBottom, isDataGridNearScrollBottom, isDataGridPrefixAppend, shouldCheckInfiniteScrollAfterScroll, type DataGridScrollPosition } from "@/lib/dataGrid/dataGridInfiniteScroll";
import { CANVAS_DATA_GRID_ROW_HEIGHT, MAX_CANVAS_DATA_GRID_PIXEL_RATIO, canvasDataGridActionOverlayWidth, canvasDataGridActionReservedWidth, dataGridSearchMatchKey, drawCanvasDataGrid, type CanvasDevicePixelSize } from "@/lib/dataGrid/canvasDataGridRenderer";
import { DATA_GRID_DARK_STRIPED_ROW_BG, DATA_GRID_LIGHT_STRIPED_ROW_BG, dataGridActiveRowBackground } from "@/lib/dataGrid/dataGridPaintTheme";
@@ -4141,7 +4141,7 @@ async function resolveLargeValueCells(rowIds: number[], columnIndexes: number[])
}
}
if (requestsByColumn.size === 0) return resolved;
if (resolvedDatabaseType.value !== "mysql" || !props.connectionId || !props.tableMeta?.tableName || props.tableMeta.primaryKeys.length === 0) {
if ((resolvedDatabaseType.value !== "mysql" && resolvedDatabaseType.value !== "postgres") || !props.connectionId || !props.tableMeta?.tableName || props.tableMeta.primaryKeys.length === 0) {
throw new Error(t("grid.largeValueNeedsStableKey"));
}
const primaryKeyIndexes = props.tableMeta.primaryKeys.map(largeValueSourceColumnIndex);
@@ -5224,6 +5224,7 @@ async function applyOrderBySearch() {
tableType: tableMeta.tableType,
columns: tableMeta.columns.map((column) => column.name),
primaryKeys: tableMeta.primaryKeys,
...tableDataLargeValuePreviewOptions(resolvedDatabaseType.value, tableMeta.columns, tableMeta.primaryKeys, pageSize.value),
orderBy: orderByClause,
limit: pageSize.value,
whereInput: currentWhereInput(),
@@ -5259,6 +5260,7 @@ async function applyWhereFilter() {
tableType: tableMeta.tableType,
columns: tableMeta.columns.map((column) => column.name),
primaryKeys: tableMeta.primaryKeys,
...tableDataLargeValuePreviewOptions(resolvedDatabaseType.value, tableMeta.columns, tableMeta.primaryKeys, pageSize.value),
orderBy: orderByInput.value.trim() || (sortCol.value ? `${queryColumnRef(sortCol.value)} ${sortDir.value.toUpperCase()}` : undefined),
limit: pageSize.value,
whereInput,
@@ -6,6 +6,7 @@ import type { QueryTab } from "@/types/database";
const mocks = vi.hoisted(() => ({
tabs: [] as QueryTab[],
activeTabId: "",
buildTableSelectSql: vi.fn(),
databaseType: "postgres" as string,
ensureConnected: vi.fn(),
executeTabSql: vi.fn(),
@@ -83,7 +84,7 @@ vi.mock("@/stores/settingsStore", () => ({
}));
vi.mock("@/lib/table/tableSelectSql", () => ({
buildTableSelectSql: async ({ tableName }: { tableName: string }) => `SELECT * FROM ${tableName}`,
buildTableSelectSql: mocks.buildTableSelectSql,
}));
const dialogs = {
@@ -103,12 +104,13 @@ describe("useNavigationTargets openTableTarget", () => {
mocks.tabs.length = 0;
mocks.databaseType = "postgres";
mocks.ensureConnected.mockResolvedValue(undefined);
mocks.buildTableSelectSql.mockImplementation(async ({ tableName }: { tableName: string }) => `SELECT * FROM ${tableName}`);
mocks.executeTabSql.mockResolvedValue(undefined);
mocks.getColumns.mockResolvedValue([column("id")]);
mocks.listIndexes.mockResolvedValue([]);
});
it("marks row identity pending until real metadata lands", async () => {
it("loads stable row identity before the first PostgreSQL table query", async () => {
let releaseColumns: (columns: unknown[]) => void = () => {};
mocks.getColumns.mockReturnValueOnce(
new Promise((resolve) => {
@@ -124,10 +126,15 @@ describe("useNavigationTargets openTableTarget", () => {
releaseColumns([column("id")]);
await open;
// 数据查询执行期间元数据未落地:行标识等待必须已挂起
expect(pendingDuringQuery).toBe(true);
expect(pendingDuringQuery).toBe(false);
expect(mocks.tabs[0]?.tableMeta?.primaryKeys).toEqual(["id"]);
expect(mocks.tabs[0]?.tableMetaPending).toBe(false);
expect(mocks.buildTableSelectSql).toHaveBeenCalledWith(
expect.objectContaining({
columns: ["id"],
primaryKeys: ["id"],
}),
);
});
it("reuses cached metadata and force-refreshes it once per catalog after a structure save", async () => {
@@ -5,6 +5,7 @@ import { useQueryStore } from "@/stores/queryStore";
import { useSettingsStore } from "@/stores/settingsStore";
import { buildTableSelectSql, quoteTableDataIdentifier } from "@/lib/table/tableSelectSql";
import { tableOpenPageLimit } from "@/lib/table/tableOpenPageLimit";
import { tableDataLargeValuePreviewOptions } from "@/lib/dataGrid/dataGridLargeValues";
import { usesSyntheticRowIdKey } from "@/lib/table/tableEditing";
import { tableMetaForDataTab } from "@/lib/table/tableDataTabMeta";
import * as api from "@/lib/backend/api";
@@ -65,6 +66,7 @@ export function useDataGridActions(activeTab: ComputedRef<QueryTab | undefined>)
// 结果(可能是失败结果的 ["Error"]),进入 SQL 会生成非法投影;
// 真实列缺失时省略 columns 让 builder 生成 SELECT *
const realColumns = tab.tableMeta?.columns.length ? tab.tableMeta.columns : undefined;
const limit = options.limit ?? tab.resultPageLimit ?? tableOpenPageLimit(settingsStore.editorSettings.tableOpenPageSize);
return buildTableSelectSql({
databaseType: effectiveDbType,
driverProfile: config?.driver_profile,
@@ -76,8 +78,9 @@ export function useDataGridActions(activeTab: ComputedRef<QueryTab | undefined>)
catalog: tableMeta?.catalog,
columns: realColumns?.map((column) => column.name),
primaryKeys,
...tableDataLargeValuePreviewOptions(effectiveDbType, realColumns ?? [], primaryKeys, limit),
includeRowId: useRowId,
limit: options.limit ?? tab.resultPageLimit ?? tableOpenPageLimit(settingsStore.editorSettings.tableOpenPageSize),
limit,
...options,
});
}
@@ -4,6 +4,7 @@ import { invalidateTableMetadataCache, loadTableMetadata } from "@/lib/metadata/
import { canApplyDataTabMetadata } from "@/lib/sidebar/dataTabOpenPolicy";
import { isNoSnapshotErrorResult, isQueryExecutionErrorResult } from "@/lib/query/queryResultError";
import { buildTableSelectSql } from "@/lib/table/tableSelectSql";
import { tableDataLargeValuePreviewOptions } from "@/lib/dataGrid/dataGridLargeValues";
import { editableRowIdentifierColumns, usesSyntheticRowIdKey } from "@/lib/table/tableEditing";
import { tableOpenPageLimit } from "@/lib/table/tableOpenPageLimit";
import { uuid } from "@/lib/common/utils";
@@ -136,6 +137,21 @@ async function openTableTarget(target: NavigationTarget, options: { tableInfoTab
await queryStore.executeTabSql(tabId, sql, { pagination: { limit: pageLimit, offset: 0 } });
return;
}
const eagerMetadata =
effectiveDbType === "mysql" || effectiveDbType === "postgres"
? await loadTableMetadata({
connectionId: target.connectionId,
database: target.database,
schema: querySchema,
tableName: target.tableName,
tableType: targetTableType,
databaseType: effectiveDbType,
driverProfile: config.driver_profile || config.db_type,
catalog: target.catalog,
})
: undefined;
const eagerColumns = eagerMetadata?.metadata.columns ?? [];
const eagerPrimaryKeys = eagerMetadata?.metadata.primaryKeys ?? [];
const sql = await buildTableSelectSql({
databaseType: effectiveDbType,
driverProfile: config.driver_profile,
@@ -145,6 +161,9 @@ async function openTableTarget(target: NavigationTarget, options: { tableInfoTab
database: target.database,
tableName: target.tableName,
tableType: targetTableType,
columns: eagerColumns.map((column) => column.name),
primaryKeys: eagerPrimaryKeys,
...tableDataLargeValuePreviewOptions(effectiveDbType, eagerColumns, eagerPrimaryKeys, pageLimit),
whereInput: target.whereInput,
limit: pageLimit,
});
@@ -156,8 +175,8 @@ async function openTableTarget(target: NavigationTarget, options: { tableInfoTab
database: target.database,
tableName: target.tableName,
tableType: targetTableType,
columns: [],
primaryKeys: [],
columns: eagerColumns,
primaryKeys: eagerPrimaryKeys,
});
firstExecuteStarted = true;
// 取消计数快照:isCancelling 是瞬态的(取消失败/查询先完成会被清掉),
@@ -13,6 +13,7 @@ import { hasTreeNodeDatabaseContext } from "@/lib/sidebar/treeNodeContext";
import { buildTableSelectSql } from "@/lib/table/tableSelectSql";
import { usesSyntheticRowIdKey } from "@/lib/table/tableEditing";
import { tableOpenPageLimit } from "@/lib/table/tableOpenPageLimit";
import { tableDataLargeValuePreviewOptions } from "@/lib/dataGrid/dataGridLargeValues";
import { canActivateExistingDataTableTab } from "@/lib/tabs/dataTabActivation";
import { beginDataTabNavigation, endDataTabNavigation, isCurrentDataTabNavigation } from "@/lib/tabs/dataTabNavigationGeneration";
@@ -293,18 +294,23 @@ export function useSidebarDataOpenRuntime() {
} else if (deferTableMetaRefresh) {
logPhase("metadata-deferred", { tabId });
} else {
void refreshTableMetaInBackground(tabId);
logPhase("metadata-started", { tabId });
}
const metadataRefresh = shouldRefreshTableMeta && !deferTableMetaRefresh ? refreshTableMetaInBackground(tabId) : undefined;
if (!cachedTableMeta && (effectiveDbType === "mysql" || effectiveDbType === "postgres")) {
await metadataRefresh;
}
// Check if superseded by a newer openData call
if (!isActive()) {
logPhase("superseded-before-build-sql", { tabId });
return;
}
const columns = cachedTableMeta?.columns ?? [];
const primaryKeys = cachedTableMeta?.primaryKeys ?? [];
const loadedTableMeta = cachedTableMeta ?? queryStore.tabs.find((item) => item.id === tabId)?.tableMeta;
const columns = loadedTableMeta?.columns ?? [];
const primaryKeys = loadedTableMeta?.primaryKeys ?? [];
const includeRowId = usesSyntheticRowIdKey(effectiveDbType, primaryKeys, tableType);
const sql = await buildTableSelectSql({
databaseType: effectiveDbType,
@@ -317,6 +323,7 @@ export function useSidebarDataOpenRuntime() {
catalog: node.catalog,
columns: columns.map((column) => column.name),
primaryKeys,
...tableDataLargeValuePreviewOptions(effectiveDbType, columns, primaryKeys, limit),
limit,
includeRowId,
});
@@ -1,8 +1,63 @@
import { describe, expect, it } from "vitest";
import { appendLargeValueCells, largeValueCellMap, remapLargeValueCells } from "@/lib/dataGrid/dataGridLargeValues";
import { appendLargeValueCells, canUseTableDataLargeValuePreview, largeValueCellMap, remapLargeValueCells, TABLE_DATA_CELL_PREVIEW_SIZE, TABLE_DATA_PREVIEW_CONTENT_MAX_BYTES, tableDataLargeValuePreviewOptions } from "@/lib/dataGrid/dataGridLargeValues";
import { buildDataGridCellDetail } from "@/lib/dataGrid/dataGridDetail";
import type { ColumnInfo } from "@/types/database";
function column(name: string, dataType: string, isPrimaryKey = false): ColumnInfo {
return {
name,
data_type: dataType,
is_nullable: false,
column_default: null,
is_primary_key: isPrimaryKey,
extra: null,
};
}
describe("data grid large-value metadata", () => {
it("enables bounded table previews only for supported databases with stable keys", () => {
const columns = [column("id", "bigint", true), column("payload", "longtext")];
expect(tableDataLargeValuePreviewOptions("mysql", columns, ["id"], 100)).toEqual({
columnTypes: ["bigint", "longtext"],
largeValuePreviewSize: TABLE_DATA_CELL_PREVIEW_SIZE,
});
expect(tableDataLargeValuePreviewOptions("postgres", columns, ["id"], 100)).toEqual({
columnTypes: ["bigint", "longtext"],
largeValuePreviewSize: TABLE_DATA_CELL_PREVIEW_SIZE,
});
expect(tableDataLargeValuePreviewOptions("sqlite", columns, ["id"], 100)).toEqual({});
expect(tableDataLargeValuePreviewOptions("mysql", columns, [], 100)).toEqual({});
expect(tableDataLargeValuePreviewOptions("mysql", [columns[0]!], ["id"], 100)).toEqual({});
});
it("disables marker parsing when a real column uses the reserved preview prefix", () => {
const columns = [column("id", "bigint", true), column("__dbx_large_value_bytes_t_0", "text")];
expect(canUseTableDataLargeValuePreview("postgres", columns, ["id"])).toBe(false);
expect(tableDataLargeValuePreviewOptions("postgres", columns, ["id"], 100)).toEqual({});
});
it("shrinks each cell preview by serialized byte budget without changing page size", () => {
const columns = [column("id", "bigint", true), ...Array.from({ length: 256 }, (_, index) => column(`payload_${index}`, "text"))];
const options = tableDataLargeValuePreviewOptions("postgres", columns, ["ID"], 1_000);
expect(options).toEqual({
columnTypes: columns.map((item) => item.data_type),
largeValuePreviewSize: 16,
});
if (!("largeValuePreviewSize" in options)) throw new Error("expected preview options");
const previewSize = options.largeValuePreviewSize;
const previewCellCount = 1_000 * 256;
const serializedSamples = ["中".repeat(previewSize), "😀".repeat(previewSize), "\u0000".repeat(previewSize)];
for (const sample of serializedSamples) {
const serializedContent = JSON.stringify(sample).slice(1, -1);
const serializedBytes = new TextEncoder().encode(serializedContent).byteLength * previewCellCount;
expect(serializedBytes).toBeLessThanOrEqual(TABLE_DATA_PREVIEW_CONTENT_MAX_BYTES);
}
});
it("offsets appended segment rows and discards metadata beyond the append cap", () => {
expect(
appendLargeValueCells(
+2
View File
@@ -929,6 +929,7 @@ export async function executeMulti(
pageSize?: number;
maxResultBytes?: number;
resultKeyColumns?: string[];
tableDataPreview?: boolean;
resultSessionId?: string;
clientSessionId?: string;
timeoutSecs?: number;
@@ -971,6 +972,7 @@ export async function executeMultiWithProgress(
pageSize?: number;
maxResultBytes?: number;
resultKeyColumns?: string[];
tableDataPreview?: boolean;
resultSessionId?: string;
clientSessionId?: string;
timeoutSecs?: number;
+2
View File
@@ -1148,6 +1148,7 @@ export async function executeMulti(
pageSize?: number;
maxResultBytes?: number;
resultKeyColumns?: string[];
tableDataPreview?: boolean;
resultSessionId?: string;
clientSessionId?: string;
timeoutSecs?: number;
@@ -1194,6 +1195,7 @@ export async function executeMultiWithProgress(
pageSize?: number;
maxResultBytes?: number;
resultKeyColumns?: string[];
tableDataPreview?: boolean;
resultSessionId?: string;
clientSessionId?: string;
timeoutSecs?: number;
@@ -1,6 +1,27 @@
import type { QueryResult } from "@/types/database";
import type { ColumnInfo, DatabaseType, QueryResult } from "@/types/database";
export const TABLE_DATA_RESULT_MAX_BYTES = 32 * 1024 * 1024;
export const TABLE_DATA_CELL_PREVIEW_MIN_SIZE = 1;
export const TABLE_DATA_CELL_PREVIEW_SIZE = 8 * 1024;
export const TABLE_DATA_PREVIEW_CONTENT_MAX_BYTES = 24 * 1024 * 1024;
export const TABLE_DATA_TEXT_SERIALIZED_BYTES_PER_CHARACTER = 6;
const TABLE_DATA_LARGE_VALUE_MARKER_PREFIX = "__DBX_LARGE_VALUE_BYTES_";
export function canUseTableDataLargeValuePreview(databaseType: DatabaseType | undefined, columns: readonly ColumnInfo[], primaryKeys: readonly string[]): boolean {
return (databaseType === "mysql" || databaseType === "postgres") && columns.length > 0 && primaryKeys.length > 0 && !columns.some((column) => column.name.toLocaleUpperCase().startsWith(TABLE_DATA_LARGE_VALUE_MARKER_PREFIX));
}
export function tableDataLargeValuePreviewOptions(databaseType: DatabaseType | undefined, columns: readonly ColumnInfo[], primaryKeys: readonly string[], pageSize?: number): { columnTypes: string[]; largeValuePreviewSize: number } | Record<string, never> {
if (!canUseTableDataLargeValuePreview(databaseType, columns, primaryKeys)) return {};
const keyColumns = new Set(primaryKeys.map((column) => column.toLocaleLowerCase()));
const previewColumnCount = columns.filter((column) => !keyColumns.has(column.name.toLocaleLowerCase())).length;
if (previewColumnCount === 0) return {};
const budgetedSize = Math.floor(TABLE_DATA_PREVIEW_CONTENT_MAX_BYTES / Math.max(1, pageSize ?? 1) / previewColumnCount / TABLE_DATA_TEXT_SERIALIZED_BYTES_PER_CHARACTER);
return {
columnTypes: columns.map((column) => column.data_type),
largeValuePreviewSize: Math.max(TABLE_DATA_CELL_PREVIEW_MIN_SIZE, Math.min(TABLE_DATA_CELL_PREVIEW_SIZE, budgetedSize)),
};
}
export function largeValueCellKey(rowIndex: number, columnIndex: number): string {
return `${rowIndex}:${columnIndex}`;
@@ -13,6 +13,8 @@ export interface BuildTableSelectSqlOptions {
tableType?: string;
primaryKeys?: string[];
columns?: string[];
columnTypes?: string[];
largeValuePreviewSize?: number;
fallbackOrderColumns?: string[];
orderBy?: string;
limit?: number;
@@ -114,7 +114,9 @@ describe("queryStore table data refresh", () => {
tableType: "TABLE",
catalog: undefined,
columns: ["id", "status", "created_at"],
columnTypes: ["integer", "text", "timestamp"],
primaryKeys: ["id"],
largeValuePreviewSize: 8192,
includeRowId: false,
whereInput: "status = 'ACTIVE'",
orderBy: "created_at DESC",
+7 -3
View File
@@ -46,7 +46,7 @@ import { connectionObjectTreeNodeSchema, connectionQueryExecutionSchema, connect
import { frontendQueryTimeoutDelayMs, frontendQueryTimeoutSecsForSql, queryTimeoutSecsForConnection } from "@/lib/sql/queryTimeout";
import { queryResultNameFromPreamble, queryResultSourceLabel } from "@/lib/sql/queryResultSource";
import { beginDataGridNativeSelectionBlock, finishDataGridNativeSelectionBlock } from "@/lib/dataGrid/dataGridNativeSelection";
import { appendLargeValueCells, remapLargeValueCells, TABLE_DATA_RESULT_MAX_BYTES } from "@/lib/dataGrid/dataGridLargeValues";
import { appendLargeValueCells, canUseTableDataLargeValuePreview, remapLargeValueCells, tableDataLargeValuePreviewOptions, TABLE_DATA_RESULT_MAX_BYTES } from "@/lib/dataGrid/dataGridLargeValues";
import { simpleDataGridOrderByReferencesMissingColumn, sortDataGridRowIndexes, type DataGridSortDirection } from "@/lib/dataGrid/dataGridSort";
import { normalizeResultPageSize } from "@/lib/dataGrid/paginationPageSize";
import { agentProtocolQueryResultMaxRows, capQueryResultTotal, effectiveQueryResultMaxRows, limitQueryPagination, queryResultLimitReached } from "@/lib/dataGrid/queryResultRowLimit";
@@ -2782,6 +2782,7 @@ export const useQueryStore = defineStore("query", () => {
catalog: tableMeta.catalog,
columns: tableMeta.columns.map((column) => column.name),
primaryKeys,
...tableDataLargeValuePreviewOptions(effectiveDbType, tableMeta.columns, primaryKeys, limit),
includeRowId: usesSyntheticRowIdKey(effectiveDbType, primaryKeys, tableMeta.tableType),
whereInput: tab.whereInput,
orderBy,
@@ -4573,6 +4574,8 @@ export const useQueryStore = defineStore("query", () => {
queryExecutionLog("info", "execute-multi:start", { traceId, elapsed: elapsed() });
// Query and data tabs use a tab-scoped pool so repeated executions keep
// connection-local state and avoid MySQL pool resets on every refresh.
const dataTabMeta = tab.mode === "data" ? tableMetaForDataTab(tab) : undefined;
const useTableDataPreview = canUseTableDataLargeValuePreview(effectiveDbType, dataTabMeta?.columns ?? [], dataTabMeta?.primaryKeys ?? []);
const executionOptions = {
...(typeof pageLimit === "number"
? useAgentResultSession
@@ -4586,10 +4589,11 @@ export const useQueryStore = defineStore("query", () => {
: { maxRows: pageLimit, fetchSize: pageLimit }
: { maxRows: agentProtocolQueryResultMaxRows(queryResultMaxRows) }),
...(executionClientSessionId ? { clientSessionId: executionClientSessionId } : {}),
...(tab.mode === "data" && effectiveDbType === "mysql"
...(tab.mode === "data" && (effectiveDbType === "mysql" || effectiveDbType === "postgres")
? {
maxResultBytes: TABLE_DATA_RESULT_MAX_BYTES,
resultKeyColumns: tableMetaForDataTab(tab)?.primaryKeys ?? [],
resultKeyColumns: dataTabMeta?.primaryKeys ?? [],
tableDataPreview: useTableDataPreview,
}
: {}),
timeoutSecs: queryTimeoutSecs,