fix(grid): bound large table previews by byte budget

This commit is contained in:
zipg authored and GitHub committed 2026-08-13 13:25:04 +08:00
1 parent 020654c33c
commit e85d3dcc59
19 files changed
+823 -23

No files matched your search

@@ -1,8 +1,63 @@
import { describe, expect, it } from "vitest";
import { appendLargeValueCells, largeValueCellMap, remapLargeValueCells } from "@/lib/dataGrid/dataGridLargeValues";
import { appendLargeValueCells, canUseTableDataLargeValuePreview, largeValueCellMap, remapLargeValueCells, TABLE_DATA_CELL_PREVIEW_SIZE, TABLE_DATA_PREVIEW_CONTENT_MAX_BYTES, tableDataLargeValuePreviewOptions } from "@/lib/dataGrid/dataGridLargeValues";
import { buildDataGridCellDetail } from "@/lib/dataGrid/dataGridDetail";
import type { ColumnInfo } from "@/types/database";
function column(name: string, dataType: string, isPrimaryKey = false): ColumnInfo {
return {
name,
data_type: dataType,
is_nullable: false,
column_default: null,
is_primary_key: isPrimaryKey,
extra: null,
};
}
describe("data grid large-value metadata", () => {
it("enables bounded table previews only for supported databases with stable keys", () => {
const columns = [column("id", "bigint", true), column("payload", "longtext")];
expect(tableDataLargeValuePreviewOptions("mysql", columns, ["id"], 100)).toEqual({
columnTypes: ["bigint", "longtext"],
largeValuePreviewSize: TABLE_DATA_CELL_PREVIEW_SIZE,
});
expect(tableDataLargeValuePreviewOptions("postgres", columns, ["id"], 100)).toEqual({
columnTypes: ["bigint", "longtext"],
largeValuePreviewSize: TABLE_DATA_CELL_PREVIEW_SIZE,
});
expect(tableDataLargeValuePreviewOptions("sqlite", columns, ["id"], 100)).toEqual({});
expect(tableDataLargeValuePreviewOptions("mysql", columns, [], 100)).toEqual({});
expect(tableDataLargeValuePreviewOptions("mysql", [columns[0]!], ["id"], 100)).toEqual({});
});
it("disables marker parsing when a real column uses the reserved preview prefix", () => {
const columns = [column("id", "bigint", true), column("__dbx_large_value_bytes_t_0", "text")];
expect(canUseTableDataLargeValuePreview("postgres", columns, ["id"])).toBe(false);
expect(tableDataLargeValuePreviewOptions("postgres", columns, ["id"], 100)).toEqual({});
});
it("shrinks each cell preview by serialized byte budget without changing page size", () => {
const columns = [column("id", "bigint", true), ...Array.from({ length: 256 }, (_, index) => column(`payload_${index}`, "text"))];
const options = tableDataLargeValuePreviewOptions("postgres", columns, ["ID"], 1_000);
expect(options).toEqual({
columnTypes: columns.map((item) => item.data_type),
largeValuePreviewSize: 16,
});
if (!("largeValuePreviewSize" in options)) throw new Error("expected preview options");
const previewSize = options.largeValuePreviewSize;
const previewCellCount = 1_000 * 256;
const serializedSamples = ["中".repeat(previewSize), "😀".repeat(previewSize), "\u0000".repeat(previewSize)];
for (const sample of serializedSamples) {
const serializedContent = JSON.stringify(sample).slice(1, -1);
const serializedBytes = new TextEncoder().encode(serializedContent).byteLength * previewCellCount;
expect(serializedBytes).toBeLessThanOrEqual(TABLE_DATA_PREVIEW_CONTENT_MAX_BYTES);
}
});
it("offsets appended segment rows and discards metadata beyond the append cap", () => {
expect(
appendLargeValueCells(
+2
View File
@@ -929,6 +929,7 @@ export async function executeMulti(
pageSize?: number;
maxResultBytes?: number;
resultKeyColumns?: string[];
tableDataPreview?: boolean;
resultSessionId?: string;
clientSessionId?: string;
timeoutSecs?: number;
@@ -971,6 +972,7 @@ export async function executeMultiWithProgress(
pageSize?: number;
maxResultBytes?: number;
resultKeyColumns?: string[];
tableDataPreview?: boolean;
resultSessionId?: string;
clientSessionId?: string;
timeoutSecs?: number;
+2
View File
@@ -1148,6 +1148,7 @@ export async function executeMulti(
pageSize?: number;
maxResultBytes?: number;
resultKeyColumns?: string[];
tableDataPreview?: boolean;
resultSessionId?: string;
clientSessionId?: string;
timeoutSecs?: number;
@@ -1194,6 +1195,7 @@ export async function executeMultiWithProgress(
pageSize?: number;
maxResultBytes?: number;
resultKeyColumns?: string[];
tableDataPreview?: boolean;
resultSessionId?: string;
clientSessionId?: string;
timeoutSecs?: number;
@@ -1,6 +1,27 @@
import type { QueryResult } from "@/types/database";
import type { ColumnInfo, DatabaseType, QueryResult } from "@/types/database";
export const TABLE_DATA_RESULT_MAX_BYTES = 32 * 1024 * 1024;
export const TABLE_DATA_CELL_PREVIEW_MIN_SIZE = 1;
export const TABLE_DATA_CELL_PREVIEW_SIZE = 8 * 1024;
export const TABLE_DATA_PREVIEW_CONTENT_MAX_BYTES = 24 * 1024 * 1024;
export const TABLE_DATA_TEXT_SERIALIZED_BYTES_PER_CHARACTER = 6;
const TABLE_DATA_LARGE_VALUE_MARKER_PREFIX = "__DBX_LARGE_VALUE_BYTES_";
export function canUseTableDataLargeValuePreview(databaseType: DatabaseType | undefined, columns: readonly ColumnInfo[], primaryKeys: readonly string[]): boolean {
return (databaseType === "mysql" || databaseType === "postgres") && columns.length > 0 && primaryKeys.length > 0 && !columns.some((column) => column.name.toLocaleUpperCase().startsWith(TABLE_DATA_LARGE_VALUE_MARKER_PREFIX));
}
export function tableDataLargeValuePreviewOptions(databaseType: DatabaseType | undefined, columns: readonly ColumnInfo[], primaryKeys: readonly string[], pageSize?: number): { columnTypes: string[]; largeValuePreviewSize: number } | Record<string, never> {
if (!canUseTableDataLargeValuePreview(databaseType, columns, primaryKeys)) return {};
const keyColumns = new Set(primaryKeys.map((column) => column.toLocaleLowerCase()));
const previewColumnCount = columns.filter((column) => !keyColumns.has(column.name.toLocaleLowerCase())).length;
if (previewColumnCount === 0) return {};
const budgetedSize = Math.floor(TABLE_DATA_PREVIEW_CONTENT_MAX_BYTES / Math.max(1, pageSize ?? 1) / previewColumnCount / TABLE_DATA_TEXT_SERIALIZED_BYTES_PER_CHARACTER);
return {
columnTypes: columns.map((column) => column.data_type),
largeValuePreviewSize: Math.max(TABLE_DATA_CELL_PREVIEW_MIN_SIZE, Math.min(TABLE_DATA_CELL_PREVIEW_SIZE, budgetedSize)),
};
}
export function largeValueCellKey(rowIndex: number, columnIndex: number): string {
return `${rowIndex}:${columnIndex}`;
@@ -13,6 +13,8 @@ export interface BuildTableSelectSqlOptions {
tableType?: string;
primaryKeys?: string[];
columns?: string[];
columnTypes?: string[];
largeValuePreviewSize?: number;
fallbackOrderColumns?: string[];
orderBy?: string;
limit?: number;