perf(mysql): limit large table result memory
This commit is contained in:
32 files changed
+1441
-177
No files matched your search
@@ -0,0 +1,59 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { appendLargeValueCells, largeValueCellMap, remapLargeValueCells } from "@/lib/dataGrid/dataGridLargeValues";
|
||||
import { buildDataGridCellDetail } from "@/lib/dataGrid/dataGridDetail";
|
||||
|
||||
describe("data grid large-value metadata", () => {
|
||||
it("offsets appended segment rows and discards metadata beyond the append cap", () => {
|
||||
expect(
|
||||
appendLargeValueCells(
|
||||
[{ row_index: 0, column_index: 1, original_bytes: 10_000 }],
|
||||
[
|
||||
{ row_index: 0, column_index: 1, original_bytes: 20_000 },
|
||||
{ row_index: 1, column_index: 2, original_bytes: 30_000 },
|
||||
{ row_index: 2, column_index: 1, original_bytes: 40_000 },
|
||||
],
|
||||
2,
|
||||
2,
|
||||
),
|
||||
).toEqual([
|
||||
{ row_index: 0, column_index: 1, original_bytes: 10_000 },
|
||||
{ row_index: 2, column_index: 1, original_bytes: 20_000 },
|
||||
{ row_index: 3, column_index: 2, original_bytes: 30_000 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("remaps metadata to locally sorted row positions", () => {
|
||||
expect(
|
||||
remapLargeValueCells(
|
||||
[
|
||||
{ row_index: 0, column_index: 1, original_bytes: 10_000 },
|
||||
{ row_index: 2, column_index: 1, original_bytes: 30_000 },
|
||||
],
|
||||
[2, 0, 1],
|
||||
),
|
||||
).toEqual([
|
||||
{ row_index: 1, column_index: 1, original_bytes: 10_000 },
|
||||
{ row_index: 0, column_index: 1, original_bytes: 30_000 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("indexes metadata by source row and column", () => {
|
||||
const result = { large_value_cells: [{ row_index: 4, column_index: 2, original_bytes: 65_536 }] };
|
||||
expect(largeValueCellMap(result).get("4:2")).toEqual(result.large_value_cells[0]);
|
||||
});
|
||||
|
||||
it("marks a backend-bounded value as truncated even when the preview is shorter than the UI limit", () => {
|
||||
const detail = buildDataGridCellDetail({
|
||||
rowIndex: 0,
|
||||
rowId: 1,
|
||||
row: ["preview..."],
|
||||
columns: ["payload"],
|
||||
columnIndex: 0,
|
||||
displayValue: String,
|
||||
isEditable: true,
|
||||
isValuePreviewTruncated: true,
|
||||
});
|
||||
|
||||
expect(detail?.isValuePreviewTruncated).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -163,6 +163,25 @@ describe("tab result cache statement execution metadata", () => {
|
||||
expect(result?.spatial_values).toEqual([[4326], [3857], [null]]);
|
||||
});
|
||||
|
||||
it("preserves large-value preview metadata", () => {
|
||||
const encoded = encodeTabResultSnapshot({
|
||||
result: {
|
||||
columns: ["id", "payload"],
|
||||
rows: [[1, "preview..."]],
|
||||
large_value_cells: [{ row_index: 0, column_index: 1, original_bytes: 1_000_000 }],
|
||||
affected_rows: 0,
|
||||
execution_time_ms: 1,
|
||||
},
|
||||
resultLocalSortOriginalRows: [[1, "preview..."]],
|
||||
resultLocalSortOriginalLargeValueCells: [{ row_index: 0, column_index: 1, original_bytes: 1_000_000 }],
|
||||
cachedAt: 1,
|
||||
});
|
||||
|
||||
const restored = decodeTabResultSnapshot(encoded);
|
||||
expect(restored?.result?.large_value_cells).toEqual([{ row_index: 0, column_index: 1, original_bytes: 1_000_000 }]);
|
||||
expect(restored?.resultLocalSortOriginalLargeValueCells).toEqual([{ row_index: 0, column_index: 1, original_bytes: 1_000_000 }]);
|
||||
});
|
||||
|
||||
it("restores multi-result, pagination, local-sort, and editable metadata fixtures", () => {
|
||||
const restored = decodeTabResultSnapshot(encodeTabResultSnapshot(queryResultLifecycleSnapshot()));
|
||||
|
||||
|
||||
@@ -927,6 +927,8 @@ export async function executeMulti(
|
||||
catalog?: string;
|
||||
fetchSize?: number;
|
||||
pageSize?: number;
|
||||
maxResultBytes?: number;
|
||||
resultKeyColumns?: string[];
|
||||
resultSessionId?: string;
|
||||
clientSessionId?: string;
|
||||
timeoutSecs?: number;
|
||||
@@ -967,6 +969,8 @@ export async function executeMultiWithProgress(
|
||||
catalog?: string;
|
||||
fetchSize?: number;
|
||||
pageSize?: number;
|
||||
maxResultBytes?: number;
|
||||
resultKeyColumns?: string[];
|
||||
resultSessionId?: string;
|
||||
clientSessionId?: string;
|
||||
timeoutSecs?: number;
|
||||
|
||||
@@ -1145,6 +1145,8 @@ export async function executeMulti(
|
||||
catalog?: string;
|
||||
fetchSize?: number;
|
||||
pageSize?: number;
|
||||
maxResultBytes?: number;
|
||||
resultKeyColumns?: string[];
|
||||
resultSessionId?: string;
|
||||
clientSessionId?: string;
|
||||
timeoutSecs?: number;
|
||||
@@ -1189,6 +1191,8 @@ export async function executeMultiWithProgress(
|
||||
catalog?: string;
|
||||
fetchSize?: number;
|
||||
pageSize?: number;
|
||||
maxResultBytes?: number;
|
||||
resultKeyColumns?: string[];
|
||||
resultSessionId?: string;
|
||||
clientSessionId?: string;
|
||||
timeoutSecs?: number;
|
||||
|
||||
@@ -49,6 +49,7 @@ export interface BuildDataGridCellDetailOptions {
|
||||
displayValue: (value: CellValue, columnIndex: number) => string;
|
||||
isEditable: boolean;
|
||||
includeBinaryImagePreview?: boolean;
|
||||
isValuePreviewTruncated?: boolean;
|
||||
}
|
||||
|
||||
export interface BuildDataGridRowDetailOptions {
|
||||
@@ -62,6 +63,7 @@ export interface BuildDataGridRowDetailOptions {
|
||||
commentByColumn?: ReadonlyMap<string, string>;
|
||||
displayValue: (value: CellValue, columnIndex: number) => string;
|
||||
isEditableColumn?: (columnIndex: number) => boolean;
|
||||
isValuePreviewTruncated?: (columnIndex: number) => boolean;
|
||||
}
|
||||
|
||||
export interface BuildDataGridColumnDetailRow {
|
||||
@@ -69,6 +71,7 @@ export interface BuildDataGridColumnDetailRow {
|
||||
rowId: number;
|
||||
row: readonly CellValue[];
|
||||
isEditable?: boolean;
|
||||
isValuePreviewTruncated?: boolean;
|
||||
}
|
||||
|
||||
export interface BuildDataGridColumnDetailOptions {
|
||||
@@ -105,7 +108,7 @@ export function buildDataGridCellDetail(options: BuildDataGridCellDetailOptions)
|
||||
rawValuePreview,
|
||||
displayValue,
|
||||
displayValuePreview,
|
||||
isValuePreviewTruncated: rawValuePreview.length < rawValue.length || displayValuePreview.length < displayValue.length,
|
||||
isValuePreviewTruncated: options.isValuePreviewTruncated === true || rawValuePreview.length < rawValue.length || displayValuePreview.length < displayValue.length,
|
||||
imagePreviewUrl: cellImagePreviewUrl(value, type, { binary: options.includeBinaryImagePreview !== false }),
|
||||
length: value === null ? 0 : String(value).length,
|
||||
formattedJson,
|
||||
@@ -131,6 +134,7 @@ export function buildDataGridColumnDetail(options: BuildDataGridColumnDetailOpti
|
||||
displayValue: options.displayValue,
|
||||
isEditable: row.isEditable ?? false,
|
||||
includeBinaryImagePreview: false,
|
||||
isValuePreviewTruncated: row.isValuePreviewTruncated,
|
||||
}),
|
||||
)
|
||||
.filter((field): field is DataGridCellDetail => field !== null);
|
||||
@@ -159,6 +163,7 @@ export function buildDataGridRowDetail(options: BuildDataGridRowDetailOptions):
|
||||
displayValue: options.displayValue,
|
||||
isEditable: options.isEditableColumn?.(columnIndex) ?? false,
|
||||
includeBinaryImagePreview: false,
|
||||
isValuePreviewTruncated: options.isValuePreviewTruncated?.(columnIndex),
|
||||
}),
|
||||
)
|
||||
.filter((field): field is DataGridCellDetail => field !== null);
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
import type { QueryResult } from "@/types/database";
|
||||
|
||||
export const TABLE_DATA_RESULT_MAX_BYTES = 32 * 1024 * 1024;
|
||||
|
||||
export function largeValueCellKey(rowIndex: number, columnIndex: number): string {
|
||||
return `${rowIndex}:${columnIndex}`;
|
||||
}
|
||||
|
||||
export function largeValueCellMap(result: Pick<QueryResult, "large_value_cells">): Map<string, NonNullable<QueryResult["large_value_cells"]>[number]> {
|
||||
return new Map((result.large_value_cells ?? []).map((cell) => [largeValueCellKey(cell.row_index, cell.column_index), cell]));
|
||||
}
|
||||
|
||||
export function appendLargeValueCells(previous: QueryResult["large_value_cells"], segment: QueryResult["large_value_cells"], rowOffset: number, appendedRowCount: number): QueryResult["large_value_cells"] {
|
||||
const appended = (segment ?? []).filter((cell) => cell.row_index < appendedRowCount).map((cell) => ({ ...cell, row_index: cell.row_index + rowOffset }));
|
||||
const combined = [...(previous ?? []), ...appended];
|
||||
return combined.length > 0 ? combined : undefined;
|
||||
}
|
||||
|
||||
export function remapLargeValueCells(cells: QueryResult["large_value_cells"], rowIndexes: number[]): QueryResult["large_value_cells"] {
|
||||
if (!cells?.length) return undefined;
|
||||
const targetBySource = new Map(rowIndexes.map((sourceIndex, targetIndex) => [sourceIndex, targetIndex]));
|
||||
const remapped = cells
|
||||
.map((cell) => {
|
||||
const rowIndex = targetBySource.get(cell.row_index);
|
||||
return rowIndex === undefined ? undefined : { ...cell, row_index: rowIndex };
|
||||
})
|
||||
.filter((cell): cell is NonNullable<typeof cell> => !!cell);
|
||||
return remapped.length > 0 ? remapped : undefined;
|
||||
}
|
||||
@@ -27,6 +27,7 @@ export interface TabResultSnapshot {
|
||||
* still return to the original result order.
|
||||
*/
|
||||
resultLocalSortOriginalRows?: QueryResult["rows"];
|
||||
resultLocalSortOriginalLargeValueCells?: QueryResult["large_value_cells"];
|
||||
resultLocalSortOriginalMongoDocuments?: QueryResult["mongo_documents"];
|
||||
resultLocalSortOriginalMongoCopyDocuments?: QueryResult["mongo_copy_documents"];
|
||||
resultRuns?: QueryTab["resultRuns"];
|
||||
@@ -48,6 +49,7 @@ interface ColumnarQueryResult {
|
||||
columns: string[];
|
||||
spatial_columns?: QueryResult["spatial_columns"];
|
||||
spatial_values?: QueryResult["spatial_values"];
|
||||
large_value_cells?: QueryResult["large_value_cells"];
|
||||
execution_error?: true;
|
||||
statement_index?: number;
|
||||
column_types?: string[];
|
||||
@@ -328,6 +330,7 @@ function stripSessionIds(result: QueryResult | undefined): QueryResult | undefin
|
||||
column_types: result.column_types ? [...result.column_types] : undefined,
|
||||
spatial_columns: result.spatial_columns?.map((entry) => ({ column_index: entry.column_index, srid: entry.srid })),
|
||||
spatial_values: result.spatial_values?.map((row) => [...row]),
|
||||
large_value_cells: result.large_value_cells?.map((cell) => ({ ...cell })),
|
||||
rows: result.rows.map((row) => [...row]),
|
||||
mongo_documents: result.mongo_documents ? clonePlain(result.mongo_documents) : undefined,
|
||||
mongo_copy_documents: result.mongo_copy_documents ? clonePlain(result.mongo_copy_documents) : undefined,
|
||||
@@ -353,6 +356,7 @@ function stripResultRunSessionIds(resultRuns: QueryTab["resultRuns"]): QueryTab[
|
||||
result: stripSessionIds(run.result),
|
||||
results: stripResultSessionIds(run.results),
|
||||
resultLocalSortOriginalRows: run.resultLocalSortOriginalRows?.map((row) => [...row]),
|
||||
resultLocalSortOriginalLargeValueCells: run.resultLocalSortOriginalLargeValueCells?.map((cell) => ({ ...cell })),
|
||||
resultLocalSortOriginalMongoDocuments: run.resultLocalSortOriginalMongoDocuments ? clonePlain(run.resultLocalSortOriginalMongoDocuments) : undefined,
|
||||
resultLocalSortOriginalMongoCopyDocuments: run.resultLocalSortOriginalMongoCopyDocuments ? clonePlain(run.resultLocalSortOriginalMongoCopyDocuments) : undefined,
|
||||
resultSessionId: undefined,
|
||||
@@ -369,6 +373,7 @@ function toColumnarResult(result: QueryResult | undefined): ColumnarQueryResult
|
||||
column_types: result.column_types ? [...result.column_types] : undefined,
|
||||
spatial_columns: result.spatial_columns?.map((entry) => ({ column_index: entry.column_index, srid: entry.srid })),
|
||||
spatial_values: result.spatial_values?.map((row) => [...row]),
|
||||
large_value_cells: result.large_value_cells?.map((cell) => ({ ...cell })),
|
||||
columnValues,
|
||||
rowCount: result.rows.length,
|
||||
mongo_documents: result.mongo_documents ? clonePlain(result.mongo_documents) : undefined,
|
||||
@@ -394,6 +399,7 @@ function fromColumnarResult(result: ColumnarQueryResult | undefined): QueryResul
|
||||
column_types: result.column_types ? [...result.column_types] : undefined,
|
||||
spatial_columns: result.spatial_columns?.map((entry) => ({ column_index: entry.column_index, srid: entry.srid })),
|
||||
spatial_values: result.spatial_values?.map((row) => [...row]),
|
||||
large_value_cells: result.large_value_cells?.map((cell) => ({ ...cell })),
|
||||
rows,
|
||||
mongo_documents: result.mongo_documents ? clonePlain(result.mongo_documents) : undefined,
|
||||
mongo_copy_documents: result.mongo_copy_documents ? clonePlain(result.mongo_copy_documents) : undefined,
|
||||
|
||||
Reference in new issue
Block a user