fix(grid): bound large table previews by byte budget
This commit is contained in:
19 files changed
+823
-23
No files matched your search
@@ -1,8 +1,63 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { appendLargeValueCells, largeValueCellMap, remapLargeValueCells } from "@/lib/dataGrid/dataGridLargeValues";
|
||||
import { appendLargeValueCells, canUseTableDataLargeValuePreview, largeValueCellMap, remapLargeValueCells, TABLE_DATA_CELL_PREVIEW_SIZE, TABLE_DATA_PREVIEW_CONTENT_MAX_BYTES, tableDataLargeValuePreviewOptions } from "@/lib/dataGrid/dataGridLargeValues";
|
||||
import { buildDataGridCellDetail } from "@/lib/dataGrid/dataGridDetail";
|
||||
import type { ColumnInfo } from "@/types/database";
|
||||
|
||||
function column(name: string, dataType: string, isPrimaryKey = false): ColumnInfo {
|
||||
return {
|
||||
name,
|
||||
data_type: dataType,
|
||||
is_nullable: false,
|
||||
column_default: null,
|
||||
is_primary_key: isPrimaryKey,
|
||||
extra: null,
|
||||
};
|
||||
}
|
||||
|
||||
describe("data grid large-value metadata", () => {
|
||||
it("enables bounded table previews only for supported databases with stable keys", () => {
|
||||
const columns = [column("id", "bigint", true), column("payload", "longtext")];
|
||||
|
||||
expect(tableDataLargeValuePreviewOptions("mysql", columns, ["id"], 100)).toEqual({
|
||||
columnTypes: ["bigint", "longtext"],
|
||||
largeValuePreviewSize: TABLE_DATA_CELL_PREVIEW_SIZE,
|
||||
});
|
||||
expect(tableDataLargeValuePreviewOptions("postgres", columns, ["id"], 100)).toEqual({
|
||||
columnTypes: ["bigint", "longtext"],
|
||||
largeValuePreviewSize: TABLE_DATA_CELL_PREVIEW_SIZE,
|
||||
});
|
||||
expect(tableDataLargeValuePreviewOptions("sqlite", columns, ["id"], 100)).toEqual({});
|
||||
expect(tableDataLargeValuePreviewOptions("mysql", columns, [], 100)).toEqual({});
|
||||
expect(tableDataLargeValuePreviewOptions("mysql", [columns[0]!], ["id"], 100)).toEqual({});
|
||||
});
|
||||
|
||||
it("disables marker parsing when a real column uses the reserved preview prefix", () => {
|
||||
const columns = [column("id", "bigint", true), column("__dbx_large_value_bytes_t_0", "text")];
|
||||
|
||||
expect(canUseTableDataLargeValuePreview("postgres", columns, ["id"])).toBe(false);
|
||||
expect(tableDataLargeValuePreviewOptions("postgres", columns, ["id"], 100)).toEqual({});
|
||||
});
|
||||
|
||||
it("shrinks each cell preview by serialized byte budget without changing page size", () => {
|
||||
const columns = [column("id", "bigint", true), ...Array.from({ length: 256 }, (_, index) => column(`payload_${index}`, "text"))];
|
||||
const options = tableDataLargeValuePreviewOptions("postgres", columns, ["ID"], 1_000);
|
||||
|
||||
expect(options).toEqual({
|
||||
columnTypes: columns.map((item) => item.data_type),
|
||||
largeValuePreviewSize: 16,
|
||||
});
|
||||
|
||||
if (!("largeValuePreviewSize" in options)) throw new Error("expected preview options");
|
||||
const previewSize = options.largeValuePreviewSize;
|
||||
const previewCellCount = 1_000 * 256;
|
||||
const serializedSamples = ["中".repeat(previewSize), "😀".repeat(previewSize), "\u0000".repeat(previewSize)];
|
||||
for (const sample of serializedSamples) {
|
||||
const serializedContent = JSON.stringify(sample).slice(1, -1);
|
||||
const serializedBytes = new TextEncoder().encode(serializedContent).byteLength * previewCellCount;
|
||||
expect(serializedBytes).toBeLessThanOrEqual(TABLE_DATA_PREVIEW_CONTENT_MAX_BYTES);
|
||||
}
|
||||
});
|
||||
|
||||
it("offsets appended segment rows and discards metadata beyond the append cap", () => {
|
||||
expect(
|
||||
appendLargeValueCells(
|
||||
|
||||
@@ -929,6 +929,7 @@ export async function executeMulti(
|
||||
pageSize?: number;
|
||||
maxResultBytes?: number;
|
||||
resultKeyColumns?: string[];
|
||||
tableDataPreview?: boolean;
|
||||
resultSessionId?: string;
|
||||
clientSessionId?: string;
|
||||
timeoutSecs?: number;
|
||||
@@ -971,6 +972,7 @@ export async function executeMultiWithProgress(
|
||||
pageSize?: number;
|
||||
maxResultBytes?: number;
|
||||
resultKeyColumns?: string[];
|
||||
tableDataPreview?: boolean;
|
||||
resultSessionId?: string;
|
||||
clientSessionId?: string;
|
||||
timeoutSecs?: number;
|
||||
|
||||
@@ -1148,6 +1148,7 @@ export async function executeMulti(
|
||||
pageSize?: number;
|
||||
maxResultBytes?: number;
|
||||
resultKeyColumns?: string[];
|
||||
tableDataPreview?: boolean;
|
||||
resultSessionId?: string;
|
||||
clientSessionId?: string;
|
||||
timeoutSecs?: number;
|
||||
@@ -1194,6 +1195,7 @@ export async function executeMultiWithProgress(
|
||||
pageSize?: number;
|
||||
maxResultBytes?: number;
|
||||
resultKeyColumns?: string[];
|
||||
tableDataPreview?: boolean;
|
||||
resultSessionId?: string;
|
||||
clientSessionId?: string;
|
||||
timeoutSecs?: number;
|
||||
|
||||
@@ -1,6 +1,27 @@
|
||||
import type { QueryResult } from "@/types/database";
|
||||
import type { ColumnInfo, DatabaseType, QueryResult } from "@/types/database";
|
||||
|
||||
export const TABLE_DATA_RESULT_MAX_BYTES = 32 * 1024 * 1024;
|
||||
export const TABLE_DATA_CELL_PREVIEW_MIN_SIZE = 1;
|
||||
export const TABLE_DATA_CELL_PREVIEW_SIZE = 8 * 1024;
|
||||
export const TABLE_DATA_PREVIEW_CONTENT_MAX_BYTES = 24 * 1024 * 1024;
|
||||
export const TABLE_DATA_TEXT_SERIALIZED_BYTES_PER_CHARACTER = 6;
|
||||
const TABLE_DATA_LARGE_VALUE_MARKER_PREFIX = "__DBX_LARGE_VALUE_BYTES_";
|
||||
|
||||
export function canUseTableDataLargeValuePreview(databaseType: DatabaseType | undefined, columns: readonly ColumnInfo[], primaryKeys: readonly string[]): boolean {
|
||||
return (databaseType === "mysql" || databaseType === "postgres") && columns.length > 0 && primaryKeys.length > 0 && !columns.some((column) => column.name.toLocaleUpperCase().startsWith(TABLE_DATA_LARGE_VALUE_MARKER_PREFIX));
|
||||
}
|
||||
|
||||
export function tableDataLargeValuePreviewOptions(databaseType: DatabaseType | undefined, columns: readonly ColumnInfo[], primaryKeys: readonly string[], pageSize?: number): { columnTypes: string[]; largeValuePreviewSize: number } | Record<string, never> {
|
||||
if (!canUseTableDataLargeValuePreview(databaseType, columns, primaryKeys)) return {};
|
||||
const keyColumns = new Set(primaryKeys.map((column) => column.toLocaleLowerCase()));
|
||||
const previewColumnCount = columns.filter((column) => !keyColumns.has(column.name.toLocaleLowerCase())).length;
|
||||
if (previewColumnCount === 0) return {};
|
||||
const budgetedSize = Math.floor(TABLE_DATA_PREVIEW_CONTENT_MAX_BYTES / Math.max(1, pageSize ?? 1) / previewColumnCount / TABLE_DATA_TEXT_SERIALIZED_BYTES_PER_CHARACTER);
|
||||
return {
|
||||
columnTypes: columns.map((column) => column.data_type),
|
||||
largeValuePreviewSize: Math.max(TABLE_DATA_CELL_PREVIEW_MIN_SIZE, Math.min(TABLE_DATA_CELL_PREVIEW_SIZE, budgetedSize)),
|
||||
};
|
||||
}
|
||||
|
||||
export function largeValueCellKey(rowIndex: number, columnIndex: number): string {
|
||||
return `${rowIndex}:${columnIndex}`;
|
||||
|
||||
@@ -13,6 +13,8 @@ export interface BuildTableSelectSqlOptions {
|
||||
tableType?: string;
|
||||
primaryKeys?: string[];
|
||||
columns?: string[];
|
||||
columnTypes?: string[];
|
||||
largeValuePreviewSize?: number;
|
||||
fallbackOrderColumns?: string[];
|
||||
orderBy?: string;
|
||||
limit?: number;
|
||||
|
||||
Reference in new issue
Block a user