perf(mysql): limit large table result memory

This commit is contained in:
zipg authored and GitHub committed 2026-08-12 14:06:33 +08:00
1 parent 1259c34db8
commit c27a4dd83d
32 files changed
+1441 -177

No files matched your search

@@ -166,6 +166,58 @@ describe("queryStore table data refresh", () => {
expect(store.tabs.find((tab) => tab.id === secondTabId)?.result).toBeUndefined();
});
it("keeps the configured MySQL page size when results contain large-value previews", async () => {
mocks.getConnectionConfig.mockReturnValue({
id: "mysql-1",
name: "MySQL",
db_type: "mysql",
database: "app",
query_timeout_secs: 30,
});
const { useQueryStore } = await import("@/stores/queryStore");
const store = useQueryStore();
const tabId = store.createTab("mysql-1", "app", "users", "data");
store.setTableMeta(tabId, {
tableName: "users",
tableType: "TABLE",
columns: [
{ name: "id", data_type: "bigint", is_nullable: false, column_default: null, is_primary_key: true, extra: null },
{ name: "payload", data_type: "longtext", is_nullable: true, column_default: null, is_primary_key: false, extra: null },
],
primaryKeys: ["id"],
});
const tab = store.tabs.find((candidate) => candidate.id === tabId)!;
tab.resultPageLimit = 100;
tab.resultPageOffset = 0;
mocks.executeMulti.mockResolvedValueOnce([
{
columns: ["id", "payload"],
rows: Array.from({ length: 100 }, (_, index) => [String(index + 1), "preview..."]),
large_value_cells: Array.from({ length: 100 }, (_, index) => ({ row_index: index, column_index: 1, original_bytes: 1_000_000 })),
affected_rows: 100,
execution_time_ms: 1,
},
]);
await expect(store.refreshDataTab(tabId)).resolves.toBe(true);
expect(tab.result?.rows).toHaveLength(100);
expect(tab.resultPageLimit).toBe(100);
expect(mocks.executeMulti).toHaveBeenCalledWith(
"mysql-1",
"app",
expect.any(String),
undefined,
expect.any(String),
expect.objectContaining({
maxRows: 100,
fetchSize: 100,
maxResultBytes: 32 * 1024 * 1024,
resultKeyColumns: ["id"],
}),
);
});
it("keeps a MySQL table refresh unqualified in the selected database context", async () => {
mocks.getConnectionConfig.mockReturnValue({
id: "mysql-1",
@@ -216,6 +268,7 @@ describe("queryStore table data refresh", () => {
tab.resultSortMode = "database";
tab.resultSortedSql = 'SELECT * FROM public.users ORDER BY "old_name" ASC';
tab.resultLocalSortOriginalRows = [[1, "alpha"]];
tab.resultLocalSortOriginalLargeValueCells = [{ row_index: 0, column_index: 1, original_bytes: 1_000_000 }];
tab.resultLocalSortOriginalMongoDocuments = [{ id: 1, old_name: "alpha" }];
tab.resultLocalSortOriginalMongoCopyDocuments = [{ id: 1, old_name: "alpha" }];
tab.resultPageLimit = 25;
@@ -238,6 +291,7 @@ describe("queryStore table data refresh", () => {
expect(tab.resultSortMode).toBeUndefined();
expect(tab.resultSortedSql).toBeUndefined();
expect(tab.resultLocalSortOriginalRows).toBeUndefined();
expect(tab.resultLocalSortOriginalLargeValueCells).toBeUndefined();
expect(tab.resultLocalSortOriginalMongoDocuments).toBeUndefined();
expect(tab.resultLocalSortOriginalMongoCopyDocuments).toBeUndefined();
expect(tab.orderByInput).toBeUndefined();
+25 -1
View File
@@ -46,6 +46,7 @@ import { connectionObjectTreeNodeSchema, connectionQueryExecutionSchema, connect
import { frontendQueryTimeoutDelayMs, frontendQueryTimeoutSecsForSql, queryTimeoutSecsForConnection } from "@/lib/sql/queryTimeout";
import { queryResultNameFromPreamble, queryResultSourceLabel } from "@/lib/sql/queryResultSource";
import { beginDataGridNativeSelectionBlock, finishDataGridNativeSelectionBlock } from "@/lib/dataGrid/dataGridNativeSelection";
import { appendLargeValueCells, remapLargeValueCells, TABLE_DATA_RESULT_MAX_BYTES } from "@/lib/dataGrid/dataGridLargeValues";
import { simpleDataGridOrderByReferencesMissingColumn, sortDataGridRowIndexes, type DataGridSortDirection } from "@/lib/dataGrid/dataGridSort";
import { normalizeResultPageSize } from "@/lib/dataGrid/paginationPageSize";
import { agentProtocolQueryResultMaxRows, capQueryResultTotal, effectiveQueryResultMaxRows, limitQueryPagination, queryResultLimitReached } from "@/lib/dataGrid/queryResultRowLimit";
@@ -150,6 +151,7 @@ function droppedTableObjectSchemaCandidates(target: DroppedTableObjectTarget): S
function markQueryResultRowsRaw(result: QueryResult): QueryResult {
markRaw(result.rows);
if (result.large_value_cells) markRaw(result.large_value_cells);
if (result.mongo_documents) markRaw(result.mongo_documents);
if (result.mongo_copy_documents) markRaw(result.mongo_copy_documents);
return result;
@@ -201,6 +203,7 @@ export function appendQueryResultSegment(previous: QueryResult, segment: QueryRe
rows: [...previous.rows, ...segment.rows.slice(0, appendedRowCount)],
spatial_columns: spatial_columns.length > 0 ? spatial_columns : undefined,
spatial_values: appendParallelValues(previous.spatial_values, segment.spatial_values),
large_value_cells: appendLargeValueCells(previous.large_value_cells, segment.large_value_cells, previous.rows.length, appendedRowCount),
mongo_documents: appendParallelValues(previous.mongo_documents, segment.mongo_documents),
mongo_copy_documents: appendParallelValues(previous.mongo_copy_documents, segment.mongo_copy_documents),
execution_time_ms: (previous.execution_time_ms ?? 0) + (segment.execution_time_ms ?? 0),
@@ -213,6 +216,7 @@ function markQueryResultRunsRowsRaw(resultRuns: NonNullable<QueryTab["resultRuns
if (run.result) markQueryResultRowsRaw(run.result);
if (run.results) markQueryResultsRowsRaw(run.results);
if (run.resultLocalSortOriginalRows) markRaw(run.resultLocalSortOriginalRows);
if (run.resultLocalSortOriginalLargeValueCells) markRaw(run.resultLocalSortOriginalLargeValueCells);
if (run.resultLocalSortOriginalMongoDocuments) markRaw(run.resultLocalSortOriginalMongoDocuments);
if (run.resultLocalSortOriginalMongoCopyDocuments) markRaw(run.resultLocalSortOriginalMongoCopyDocuments);
}
@@ -927,6 +931,7 @@ export const useQueryStore = defineStore("query", () => {
tab.batchSqlExecution = undefined;
tab.resultEditorFingerprint = undefined;
tab.resultLocalSortOriginalRows = undefined;
tab.resultLocalSortOriginalLargeValueCells = undefined;
tab.resultLocalSortOriginalMongoDocuments = undefined;
tab.resultLocalSortOriginalMongoCopyDocuments = undefined;
tab.resultSortMode = undefined;
@@ -954,6 +959,7 @@ export const useQueryStore = defineStore("query", () => {
tab.resultSortDirection = undefined;
tab.resultSortMode = undefined;
tab.resultLocalSortOriginalRows = undefined;
tab.resultLocalSortOriginalLargeValueCells = undefined;
tab.resultLocalSortOriginalMongoDocuments = undefined;
tab.resultLocalSortOriginalMongoCopyDocuments = undefined;
tab.orderByInput = undefined;
@@ -978,6 +984,7 @@ export const useQueryStore = defineStore("query", () => {
run.result = undefined;
run.results = undefined;
run.resultLocalSortOriginalRows = undefined;
run.resultLocalSortOriginalLargeValueCells = undefined;
run.resultLocalSortOriginalMongoDocuments = undefined;
run.resultLocalSortOriginalMongoCopyDocuments = undefined;
run.resultSessionId = undefined;
@@ -1007,6 +1014,7 @@ export const useQueryStore = defineStore("query", () => {
tab.resultSortDirection = run.resultSortDirection;
tab.resultSortMode = run.resultSortMode;
tab.resultLocalSortOriginalRows = run.resultLocalSortOriginalRows;
tab.resultLocalSortOriginalLargeValueCells = run.resultLocalSortOriginalLargeValueCells;
tab.resultLocalSortOriginalMongoDocuments = run.resultLocalSortOriginalMongoDocuments;
tab.resultLocalSortOriginalMongoCopyDocuments = run.resultLocalSortOriginalMongoCopyDocuments;
tab.orderByInput = run.orderByInput;
@@ -1243,6 +1251,7 @@ export const useQueryStore = defineStore("query", () => {
resultSortDirection: tab.resultSortDirection,
resultSortMode: tab.resultSortMode,
resultLocalSortOriginalRows: tab.resultLocalSortOriginalRows,
resultLocalSortOriginalLargeValueCells: tab.resultLocalSortOriginalLargeValueCells,
resultLocalSortOriginalMongoDocuments: tab.resultLocalSortOriginalMongoDocuments,
resultLocalSortOriginalMongoCopyDocuments: tab.resultLocalSortOriginalMongoCopyDocuments,
orderByInput: tab.orderByInput,
@@ -1339,6 +1348,7 @@ export const useQueryStore = defineStore("query", () => {
resultSortDirection: tab.resultSortDirection,
resultSortMode: tab.resultSortMode,
resultLocalSortOriginalRows: tab.resultLocalSortOriginalRows,
resultLocalSortOriginalLargeValueCells: tab.resultLocalSortOriginalLargeValueCells,
resultLocalSortOriginalMongoDocuments: tab.resultLocalSortOriginalMongoDocuments,
resultLocalSortOriginalMongoCopyDocuments: tab.resultLocalSortOriginalMongoCopyDocuments,
orderByInput: tab.orderByInput,
@@ -1392,6 +1402,7 @@ export const useQueryStore = defineStore("query", () => {
if (!tab.resultLocalSortOriginalRows) {
tab.resultLocalSortOriginalRows = tab.result.rows.slice();
tab.resultLocalSortOriginalLargeValueCells = tab.result.large_value_cells?.map((cell) => ({ ...cell }));
tab.resultLocalSortOriginalMongoDocuments = tab.result.mongo_documents?.slice();
tab.resultLocalSortOriginalMongoCopyDocuments = tab.result.mongo_copy_documents?.slice();
}
@@ -1404,7 +1415,8 @@ export const useQueryStore = defineStore("query", () => {
const mongo_documents = originalMongoDocuments ? rowIndexes.map((index) => originalMongoDocuments[index]) : undefined;
const originalMongoCopyDocuments = tab.resultLocalSortOriginalMongoCopyDocuments;
const mongo_copy_documents = originalMongoCopyDocuments ? rowIndexes.map((index) => originalMongoCopyDocuments[index]) : undefined;
assignDisplayedResult(tab, { ...tab.result, rows, mongo_documents, mongo_copy_documents });
const large_value_cells = remapLargeValueCells(tab.resultLocalSortOriginalLargeValueCells, rowIndexes);
assignDisplayedResult(tab, { ...tab.result, rows, large_value_cells, mongo_documents, mongo_copy_documents });
tab.resultSortColumn = direction ? column : undefined;
tab.resultSortColumnIndex = direction ? columnIndex : undefined;
@@ -1413,6 +1425,7 @@ export const useQueryStore = defineStore("query", () => {
tab.resultSortedSql = undefined;
if (!direction) {
tab.resultLocalSortOriginalRows = undefined;
tab.resultLocalSortOriginalLargeValueCells = undefined;
tab.resultLocalSortOriginalMongoDocuments = undefined;
tab.resultLocalSortOriginalMongoCopyDocuments = undefined;
}
@@ -2544,6 +2557,7 @@ export const useQueryStore = defineStore("query", () => {
resultSortDirection: undefined,
resultSortMode: undefined,
resultLocalSortOriginalRows: undefined,
resultLocalSortOriginalLargeValueCells: undefined,
resultLocalSortOriginalMongoDocuments: undefined,
resultLocalSortOriginalMongoCopyDocuments: undefined,
orderByInput: undefined,
@@ -3124,6 +3138,7 @@ export const useQueryStore = defineStore("query", () => {
tab.resultSortMode = undefined;
tab.resultSortedSql = undefined;
tab.resultLocalSortOriginalRows = undefined;
tab.resultLocalSortOriginalLargeValueCells = undefined;
tab.resultLocalSortOriginalMongoDocuments = undefined;
tab.resultLocalSortOriginalMongoCopyDocuments = undefined;
}
@@ -3808,6 +3823,7 @@ export const useQueryStore = defineStore("query", () => {
const previousDisplayedSql = tab.resultBaseSql ?? tab.lastExecutedSql ?? tab.sql;
tab.lastExecutedSql = sql;
tab.resultLocalSortOriginalRows = undefined;
tab.resultLocalSortOriginalLargeValueCells = undefined;
tab.resultLocalSortOriginalMongoDocuments = undefined;
tab.resultLocalSortOriginalMongoCopyDocuments = undefined;
if (captureResultRun && tab.result && !tab.activeResultRunId) {
@@ -4465,6 +4481,12 @@ export const useQueryStore = defineStore("query", () => {
: { maxRows: pageLimit, fetchSize: pageLimit }
: { maxRows: agentProtocolQueryResultMaxRows(queryResultMaxRows) }),
...(executionClientSessionId ? { clientSessionId: executionClientSessionId } : {}),
...(tab.mode === "data" && effectiveDbType === "mysql"
? {
maxResultBytes: TABLE_DATA_RESULT_MAX_BYTES,
resultKeyColumns: tableMetaForDataTab(tab)?.primaryKeys ?? [],
}
: {}),
timeoutSecs: queryTimeoutSecs,
catalog: executionCatalog,
continueOnError: settingsStore.editorSettings.continueOnErrorOnBatch,
@@ -5177,6 +5199,7 @@ export const useQueryStore = defineStore("query", () => {
tab.activeResultIndex = index;
tab.result = tab.results[index];
tab.resultLocalSortOriginalRows = undefined;
tab.resultLocalSortOriginalLargeValueCells = undefined;
tab.resultLocalSortOriginalMongoDocuments = undefined;
tab.resultLocalSortOriginalMongoCopyDocuments = undefined;
tab.resultSortColumn = undefined;
@@ -5300,6 +5323,7 @@ export const useQueryStore = defineStore("query", () => {
tab.resultEditorFingerprint = snapshot.resultEditorFingerprint;
tab.result = snapshot.result ? markQueryResultRowsRaw(snapshot.result) : results?.[activeIndex] ? markQueryResultRowsRaw(results[activeIndex]) : undefined;
tab.resultLocalSortOriginalRows = snapshot.resultLocalSortOriginalRows ? markRaw(snapshot.resultLocalSortOriginalRows) : undefined;
tab.resultLocalSortOriginalLargeValueCells = snapshot.resultLocalSortOriginalLargeValueCells?.map((cell) => ({ ...cell }));
tab.resultLocalSortOriginalMongoDocuments = snapshot.resultLocalSortOriginalMongoDocuments ? markRaw(snapshot.resultLocalSortOriginalMongoDocuments) : undefined;
tab.resultLocalSortOriginalMongoCopyDocuments = snapshot.resultLocalSortOriginalMongoCopyDocuments ? markRaw(snapshot.resultLocalSortOriginalMongoCopyDocuments) : undefined;
// 快照编解码会重建负载,落盘前的各 run 估算值不再对应恢复后的对象,