perf(mysql): limit large table result memory

This commit is contained in:
zipg authored and GitHub committed 2026-08-12 14:06:33 +08:00
1 parent 1259c34db8
commit c27a4dd83d
32 files changed
+1441 -177

No files matched your search

@@ -0,0 +1,59 @@
import { describe, expect, it } from "vitest";
import { appendLargeValueCells, largeValueCellMap, remapLargeValueCells } from "@/lib/dataGrid/dataGridLargeValues";
import { buildDataGridCellDetail } from "@/lib/dataGrid/dataGridDetail";
describe("data grid large-value metadata", () => {
it("offsets appended segment rows and discards metadata beyond the append cap", () => {
expect(
appendLargeValueCells(
[{ row_index: 0, column_index: 1, original_bytes: 10_000 }],
[
{ row_index: 0, column_index: 1, original_bytes: 20_000 },
{ row_index: 1, column_index: 2, original_bytes: 30_000 },
{ row_index: 2, column_index: 1, original_bytes: 40_000 },
],
2,
2,
),
).toEqual([
{ row_index: 0, column_index: 1, original_bytes: 10_000 },
{ row_index: 2, column_index: 1, original_bytes: 20_000 },
{ row_index: 3, column_index: 2, original_bytes: 30_000 },
]);
});
it("remaps metadata to locally sorted row positions", () => {
expect(
remapLargeValueCells(
[
{ row_index: 0, column_index: 1, original_bytes: 10_000 },
{ row_index: 2, column_index: 1, original_bytes: 30_000 },
],
[2, 0, 1],
),
).toEqual([
{ row_index: 1, column_index: 1, original_bytes: 10_000 },
{ row_index: 0, column_index: 1, original_bytes: 30_000 },
]);
});
it("indexes metadata by source row and column", () => {
const result = { large_value_cells: [{ row_index: 4, column_index: 2, original_bytes: 65_536 }] };
expect(largeValueCellMap(result).get("4:2")).toEqual(result.large_value_cells[0]);
});
it("marks a backend-bounded value as truncated even when the preview is shorter than the UI limit", () => {
const detail = buildDataGridCellDetail({
rowIndex: 0,
rowId: 1,
row: ["preview..."],
columns: ["payload"],
columnIndex: 0,
displayValue: String,
isEditable: true,
isValuePreviewTruncated: true,
});
expect(detail?.isValuePreviewTruncated).toBe(true);
});
});
@@ -163,6 +163,25 @@ describe("tab result cache statement execution metadata", () => {
expect(result?.spatial_values).toEqual([[4326], [3857], [null]]);
});
it("preserves large-value preview metadata", () => {
const encoded = encodeTabResultSnapshot({
result: {
columns: ["id", "payload"],
rows: [[1, "preview..."]],
large_value_cells: [{ row_index: 0, column_index: 1, original_bytes: 1_000_000 }],
affected_rows: 0,
execution_time_ms: 1,
},
resultLocalSortOriginalRows: [[1, "preview..."]],
resultLocalSortOriginalLargeValueCells: [{ row_index: 0, column_index: 1, original_bytes: 1_000_000 }],
cachedAt: 1,
});
const restored = decodeTabResultSnapshot(encoded);
expect(restored?.result?.large_value_cells).toEqual([{ row_index: 0, column_index: 1, original_bytes: 1_000_000 }]);
expect(restored?.resultLocalSortOriginalLargeValueCells).toEqual([{ row_index: 0, column_index: 1, original_bytes: 1_000_000 }]);
});
it("restores multi-result, pagination, local-sort, and editable metadata fixtures", () => {
const restored = decodeTabResultSnapshot(encodeTabResultSnapshot(queryResultLifecycleSnapshot()));
+4
View File
@@ -927,6 +927,8 @@ export async function executeMulti(
catalog?: string;
fetchSize?: number;
pageSize?: number;
maxResultBytes?: number;
resultKeyColumns?: string[];
resultSessionId?: string;
clientSessionId?: string;
timeoutSecs?: number;
@@ -967,6 +969,8 @@ export async function executeMultiWithProgress(
catalog?: string;
fetchSize?: number;
pageSize?: number;
maxResultBytes?: number;
resultKeyColumns?: string[];
resultSessionId?: string;
clientSessionId?: string;
timeoutSecs?: number;
+4
View File
@@ -1145,6 +1145,8 @@ export async function executeMulti(
catalog?: string;
fetchSize?: number;
pageSize?: number;
maxResultBytes?: number;
resultKeyColumns?: string[];
resultSessionId?: string;
clientSessionId?: string;
timeoutSecs?: number;
@@ -1189,6 +1191,8 @@ export async function executeMultiWithProgress(
catalog?: string;
fetchSize?: number;
pageSize?: number;
maxResultBytes?: number;
resultKeyColumns?: string[];
resultSessionId?: string;
clientSessionId?: string;
timeoutSecs?: number;
@@ -49,6 +49,7 @@ export interface BuildDataGridCellDetailOptions {
displayValue: (value: CellValue, columnIndex: number) => string;
isEditable: boolean;
includeBinaryImagePreview?: boolean;
isValuePreviewTruncated?: boolean;
}
export interface BuildDataGridRowDetailOptions {
@@ -62,6 +63,7 @@ export interface BuildDataGridRowDetailOptions {
commentByColumn?: ReadonlyMap<string, string>;
displayValue: (value: CellValue, columnIndex: number) => string;
isEditableColumn?: (columnIndex: number) => boolean;
isValuePreviewTruncated?: (columnIndex: number) => boolean;
}
export interface BuildDataGridColumnDetailRow {
@@ -69,6 +71,7 @@ export interface BuildDataGridColumnDetailRow {
rowId: number;
row: readonly CellValue[];
isEditable?: boolean;
isValuePreviewTruncated?: boolean;
}
export interface BuildDataGridColumnDetailOptions {
@@ -105,7 +108,7 @@ export function buildDataGridCellDetail(options: BuildDataGridCellDetailOptions)
rawValuePreview,
displayValue,
displayValuePreview,
isValuePreviewTruncated: rawValuePreview.length < rawValue.length || displayValuePreview.length < displayValue.length,
isValuePreviewTruncated: options.isValuePreviewTruncated === true || rawValuePreview.length < rawValue.length || displayValuePreview.length < displayValue.length,
imagePreviewUrl: cellImagePreviewUrl(value, type, { binary: options.includeBinaryImagePreview !== false }),
length: value === null ? 0 : String(value).length,
formattedJson,
@@ -131,6 +134,7 @@ export function buildDataGridColumnDetail(options: BuildDataGridColumnDetailOpti
displayValue: options.displayValue,
isEditable: row.isEditable ?? false,
includeBinaryImagePreview: false,
isValuePreviewTruncated: row.isValuePreviewTruncated,
}),
)
.filter((field): field is DataGridCellDetail => field !== null);
@@ -159,6 +163,7 @@ export function buildDataGridRowDetail(options: BuildDataGridRowDetailOptions):
displayValue: options.displayValue,
isEditable: options.isEditableColumn?.(columnIndex) ?? false,
includeBinaryImagePreview: false,
isValuePreviewTruncated: options.isValuePreviewTruncated?.(columnIndex),
}),
)
.filter((field): field is DataGridCellDetail => field !== null);
@@ -0,0 +1,29 @@
import type { QueryResult } from "@/types/database";
export const TABLE_DATA_RESULT_MAX_BYTES = 32 * 1024 * 1024;
export function largeValueCellKey(rowIndex: number, columnIndex: number): string {
return `${rowIndex}:${columnIndex}`;
}
export function largeValueCellMap(result: Pick<QueryResult, "large_value_cells">): Map<string, NonNullable<QueryResult["large_value_cells"]>[number]> {
return new Map((result.large_value_cells ?? []).map((cell) => [largeValueCellKey(cell.row_index, cell.column_index), cell]));
}
export function appendLargeValueCells(previous: QueryResult["large_value_cells"], segment: QueryResult["large_value_cells"], rowOffset: number, appendedRowCount: number): QueryResult["large_value_cells"] {
const appended = (segment ?? []).filter((cell) => cell.row_index < appendedRowCount).map((cell) => ({ ...cell, row_index: cell.row_index + rowOffset }));
const combined = [...(previous ?? []), ...appended];
return combined.length > 0 ? combined : undefined;
}
export function remapLargeValueCells(cells: QueryResult["large_value_cells"], rowIndexes: number[]): QueryResult["large_value_cells"] {
if (!cells?.length) return undefined;
const targetBySource = new Map(rowIndexes.map((sourceIndex, targetIndex) => [sourceIndex, targetIndex]));
const remapped = cells
.map((cell) => {
const rowIndex = targetBySource.get(cell.row_index);
return rowIndex === undefined ? undefined : { ...cell, row_index: rowIndex };
})
.filter((cell): cell is NonNullable<typeof cell> => !!cell);
return remapped.length > 0 ? remapped : undefined;
}
@@ -27,6 +27,7 @@ export interface TabResultSnapshot {
* still return to the original result order.
*/
resultLocalSortOriginalRows?: QueryResult["rows"];
resultLocalSortOriginalLargeValueCells?: QueryResult["large_value_cells"];
resultLocalSortOriginalMongoDocuments?: QueryResult["mongo_documents"];
resultLocalSortOriginalMongoCopyDocuments?: QueryResult["mongo_copy_documents"];
resultRuns?: QueryTab["resultRuns"];
@@ -48,6 +49,7 @@ interface ColumnarQueryResult {
columns: string[];
spatial_columns?: QueryResult["spatial_columns"];
spatial_values?: QueryResult["spatial_values"];
large_value_cells?: QueryResult["large_value_cells"];
execution_error?: true;
statement_index?: number;
column_types?: string[];
@@ -328,6 +330,7 @@ function stripSessionIds(result: QueryResult | undefined): QueryResult | undefin
column_types: result.column_types ? [...result.column_types] : undefined,
spatial_columns: result.spatial_columns?.map((entry) => ({ column_index: entry.column_index, srid: entry.srid })),
spatial_values: result.spatial_values?.map((row) => [...row]),
large_value_cells: result.large_value_cells?.map((cell) => ({ ...cell })),
rows: result.rows.map((row) => [...row]),
mongo_documents: result.mongo_documents ? clonePlain(result.mongo_documents) : undefined,
mongo_copy_documents: result.mongo_copy_documents ? clonePlain(result.mongo_copy_documents) : undefined,
@@ -353,6 +356,7 @@ function stripResultRunSessionIds(resultRuns: QueryTab["resultRuns"]): QueryTab[
result: stripSessionIds(run.result),
results: stripResultSessionIds(run.results),
resultLocalSortOriginalRows: run.resultLocalSortOriginalRows?.map((row) => [...row]),
resultLocalSortOriginalLargeValueCells: run.resultLocalSortOriginalLargeValueCells?.map((cell) => ({ ...cell })),
resultLocalSortOriginalMongoDocuments: run.resultLocalSortOriginalMongoDocuments ? clonePlain(run.resultLocalSortOriginalMongoDocuments) : undefined,
resultLocalSortOriginalMongoCopyDocuments: run.resultLocalSortOriginalMongoCopyDocuments ? clonePlain(run.resultLocalSortOriginalMongoCopyDocuments) : undefined,
resultSessionId: undefined,
@@ -369,6 +373,7 @@ function toColumnarResult(result: QueryResult | undefined): ColumnarQueryResult
column_types: result.column_types ? [...result.column_types] : undefined,
spatial_columns: result.spatial_columns?.map((entry) => ({ column_index: entry.column_index, srid: entry.srid })),
spatial_values: result.spatial_values?.map((row) => [...row]),
large_value_cells: result.large_value_cells?.map((cell) => ({ ...cell })),
columnValues,
rowCount: result.rows.length,
mongo_documents: result.mongo_documents ? clonePlain(result.mongo_documents) : undefined,
@@ -394,6 +399,7 @@ function fromColumnarResult(result: ColumnarQueryResult | undefined): QueryResul
column_types: result.column_types ? [...result.column_types] : undefined,
spatial_columns: result.spatial_columns?.map((entry) => ({ column_index: entry.column_index, srid: entry.srid })),
spatial_values: result.spatial_values?.map((row) => [...row]),
large_value_cells: result.large_value_cells?.map((cell) => ({ ...cell })),
rows,
mongo_documents: result.mongo_documents ? clonePlain(result.mongo_documents) : undefined,
mongo_copy_documents: result.mongo_copy_documents ? clonePlain(result.mongo_copy_documents) : undefined,