perf(mysql): limit large table result memory
This commit is contained in:
32 files changed
+1441
-177
No files matched your search
@@ -166,6 +166,58 @@ describe("queryStore table data refresh", () => {
|
||||
expect(store.tabs.find((tab) => tab.id === secondTabId)?.result).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps the configured MySQL page size when results contain large-value previews", async () => {
|
||||
mocks.getConnectionConfig.mockReturnValue({
|
||||
id: "mysql-1",
|
||||
name: "MySQL",
|
||||
db_type: "mysql",
|
||||
database: "app",
|
||||
query_timeout_secs: 30,
|
||||
});
|
||||
const { useQueryStore } = await import("@/stores/queryStore");
|
||||
const store = useQueryStore();
|
||||
const tabId = store.createTab("mysql-1", "app", "users", "data");
|
||||
store.setTableMeta(tabId, {
|
||||
tableName: "users",
|
||||
tableType: "TABLE",
|
||||
columns: [
|
||||
{ name: "id", data_type: "bigint", is_nullable: false, column_default: null, is_primary_key: true, extra: null },
|
||||
{ name: "payload", data_type: "longtext", is_nullable: true, column_default: null, is_primary_key: false, extra: null },
|
||||
],
|
||||
primaryKeys: ["id"],
|
||||
});
|
||||
const tab = store.tabs.find((candidate) => candidate.id === tabId)!;
|
||||
tab.resultPageLimit = 100;
|
||||
tab.resultPageOffset = 0;
|
||||
mocks.executeMulti.mockResolvedValueOnce([
|
||||
{
|
||||
columns: ["id", "payload"],
|
||||
rows: Array.from({ length: 100 }, (_, index) => [String(index + 1), "preview..."]),
|
||||
large_value_cells: Array.from({ length: 100 }, (_, index) => ({ row_index: index, column_index: 1, original_bytes: 1_000_000 })),
|
||||
affected_rows: 100,
|
||||
execution_time_ms: 1,
|
||||
},
|
||||
]);
|
||||
|
||||
await expect(store.refreshDataTab(tabId)).resolves.toBe(true);
|
||||
|
||||
expect(tab.result?.rows).toHaveLength(100);
|
||||
expect(tab.resultPageLimit).toBe(100);
|
||||
expect(mocks.executeMulti).toHaveBeenCalledWith(
|
||||
"mysql-1",
|
||||
"app",
|
||||
expect.any(String),
|
||||
undefined,
|
||||
expect.any(String),
|
||||
expect.objectContaining({
|
||||
maxRows: 100,
|
||||
fetchSize: 100,
|
||||
maxResultBytes: 32 * 1024 * 1024,
|
||||
resultKeyColumns: ["id"],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps a MySQL table refresh unqualified in the selected database context", async () => {
|
||||
mocks.getConnectionConfig.mockReturnValue({
|
||||
id: "mysql-1",
|
||||
@@ -216,6 +268,7 @@ describe("queryStore table data refresh", () => {
|
||||
tab.resultSortMode = "database";
|
||||
tab.resultSortedSql = 'SELECT * FROM public.users ORDER BY "old_name" ASC';
|
||||
tab.resultLocalSortOriginalRows = [[1, "alpha"]];
|
||||
tab.resultLocalSortOriginalLargeValueCells = [{ row_index: 0, column_index: 1, original_bytes: 1_000_000 }];
|
||||
tab.resultLocalSortOriginalMongoDocuments = [{ id: 1, old_name: "alpha" }];
|
||||
tab.resultLocalSortOriginalMongoCopyDocuments = [{ id: 1, old_name: "alpha" }];
|
||||
tab.resultPageLimit = 25;
|
||||
@@ -238,6 +291,7 @@ describe("queryStore table data refresh", () => {
|
||||
expect(tab.resultSortMode).toBeUndefined();
|
||||
expect(tab.resultSortedSql).toBeUndefined();
|
||||
expect(tab.resultLocalSortOriginalRows).toBeUndefined();
|
||||
expect(tab.resultLocalSortOriginalLargeValueCells).toBeUndefined();
|
||||
expect(tab.resultLocalSortOriginalMongoDocuments).toBeUndefined();
|
||||
expect(tab.resultLocalSortOriginalMongoCopyDocuments).toBeUndefined();
|
||||
expect(tab.orderByInput).toBeUndefined();
|
||||
|
||||
@@ -46,6 +46,7 @@ import { connectionObjectTreeNodeSchema, connectionQueryExecutionSchema, connect
|
||||
import { frontendQueryTimeoutDelayMs, frontendQueryTimeoutSecsForSql, queryTimeoutSecsForConnection } from "@/lib/sql/queryTimeout";
|
||||
import { queryResultNameFromPreamble, queryResultSourceLabel } from "@/lib/sql/queryResultSource";
|
||||
import { beginDataGridNativeSelectionBlock, finishDataGridNativeSelectionBlock } from "@/lib/dataGrid/dataGridNativeSelection";
|
||||
import { appendLargeValueCells, remapLargeValueCells, TABLE_DATA_RESULT_MAX_BYTES } from "@/lib/dataGrid/dataGridLargeValues";
|
||||
import { simpleDataGridOrderByReferencesMissingColumn, sortDataGridRowIndexes, type DataGridSortDirection } from "@/lib/dataGrid/dataGridSort";
|
||||
import { normalizeResultPageSize } from "@/lib/dataGrid/paginationPageSize";
|
||||
import { agentProtocolQueryResultMaxRows, capQueryResultTotal, effectiveQueryResultMaxRows, limitQueryPagination, queryResultLimitReached } from "@/lib/dataGrid/queryResultRowLimit";
|
||||
@@ -150,6 +151,7 @@ function droppedTableObjectSchemaCandidates(target: DroppedTableObjectTarget): S
|
||||
|
||||
function markQueryResultRowsRaw(result: QueryResult): QueryResult {
|
||||
markRaw(result.rows);
|
||||
if (result.large_value_cells) markRaw(result.large_value_cells);
|
||||
if (result.mongo_documents) markRaw(result.mongo_documents);
|
||||
if (result.mongo_copy_documents) markRaw(result.mongo_copy_documents);
|
||||
return result;
|
||||
@@ -201,6 +203,7 @@ export function appendQueryResultSegment(previous: QueryResult, segment: QueryRe
|
||||
rows: [...previous.rows, ...segment.rows.slice(0, appendedRowCount)],
|
||||
spatial_columns: spatial_columns.length > 0 ? spatial_columns : undefined,
|
||||
spatial_values: appendParallelValues(previous.spatial_values, segment.spatial_values),
|
||||
large_value_cells: appendLargeValueCells(previous.large_value_cells, segment.large_value_cells, previous.rows.length, appendedRowCount),
|
||||
mongo_documents: appendParallelValues(previous.mongo_documents, segment.mongo_documents),
|
||||
mongo_copy_documents: appendParallelValues(previous.mongo_copy_documents, segment.mongo_copy_documents),
|
||||
execution_time_ms: (previous.execution_time_ms ?? 0) + (segment.execution_time_ms ?? 0),
|
||||
@@ -213,6 +216,7 @@ function markQueryResultRunsRowsRaw(resultRuns: NonNullable<QueryTab["resultRuns
|
||||
if (run.result) markQueryResultRowsRaw(run.result);
|
||||
if (run.results) markQueryResultsRowsRaw(run.results);
|
||||
if (run.resultLocalSortOriginalRows) markRaw(run.resultLocalSortOriginalRows);
|
||||
if (run.resultLocalSortOriginalLargeValueCells) markRaw(run.resultLocalSortOriginalLargeValueCells);
|
||||
if (run.resultLocalSortOriginalMongoDocuments) markRaw(run.resultLocalSortOriginalMongoDocuments);
|
||||
if (run.resultLocalSortOriginalMongoCopyDocuments) markRaw(run.resultLocalSortOriginalMongoCopyDocuments);
|
||||
}
|
||||
@@ -927,6 +931,7 @@ export const useQueryStore = defineStore("query", () => {
|
||||
tab.batchSqlExecution = undefined;
|
||||
tab.resultEditorFingerprint = undefined;
|
||||
tab.resultLocalSortOriginalRows = undefined;
|
||||
tab.resultLocalSortOriginalLargeValueCells = undefined;
|
||||
tab.resultLocalSortOriginalMongoDocuments = undefined;
|
||||
tab.resultLocalSortOriginalMongoCopyDocuments = undefined;
|
||||
tab.resultSortMode = undefined;
|
||||
@@ -954,6 +959,7 @@ export const useQueryStore = defineStore("query", () => {
|
||||
tab.resultSortDirection = undefined;
|
||||
tab.resultSortMode = undefined;
|
||||
tab.resultLocalSortOriginalRows = undefined;
|
||||
tab.resultLocalSortOriginalLargeValueCells = undefined;
|
||||
tab.resultLocalSortOriginalMongoDocuments = undefined;
|
||||
tab.resultLocalSortOriginalMongoCopyDocuments = undefined;
|
||||
tab.orderByInput = undefined;
|
||||
@@ -978,6 +984,7 @@ export const useQueryStore = defineStore("query", () => {
|
||||
run.result = undefined;
|
||||
run.results = undefined;
|
||||
run.resultLocalSortOriginalRows = undefined;
|
||||
run.resultLocalSortOriginalLargeValueCells = undefined;
|
||||
run.resultLocalSortOriginalMongoDocuments = undefined;
|
||||
run.resultLocalSortOriginalMongoCopyDocuments = undefined;
|
||||
run.resultSessionId = undefined;
|
||||
@@ -1007,6 +1014,7 @@ export const useQueryStore = defineStore("query", () => {
|
||||
tab.resultSortDirection = run.resultSortDirection;
|
||||
tab.resultSortMode = run.resultSortMode;
|
||||
tab.resultLocalSortOriginalRows = run.resultLocalSortOriginalRows;
|
||||
tab.resultLocalSortOriginalLargeValueCells = run.resultLocalSortOriginalLargeValueCells;
|
||||
tab.resultLocalSortOriginalMongoDocuments = run.resultLocalSortOriginalMongoDocuments;
|
||||
tab.resultLocalSortOriginalMongoCopyDocuments = run.resultLocalSortOriginalMongoCopyDocuments;
|
||||
tab.orderByInput = run.orderByInput;
|
||||
@@ -1243,6 +1251,7 @@ export const useQueryStore = defineStore("query", () => {
|
||||
resultSortDirection: tab.resultSortDirection,
|
||||
resultSortMode: tab.resultSortMode,
|
||||
resultLocalSortOriginalRows: tab.resultLocalSortOriginalRows,
|
||||
resultLocalSortOriginalLargeValueCells: tab.resultLocalSortOriginalLargeValueCells,
|
||||
resultLocalSortOriginalMongoDocuments: tab.resultLocalSortOriginalMongoDocuments,
|
||||
resultLocalSortOriginalMongoCopyDocuments: tab.resultLocalSortOriginalMongoCopyDocuments,
|
||||
orderByInput: tab.orderByInput,
|
||||
@@ -1339,6 +1348,7 @@ export const useQueryStore = defineStore("query", () => {
|
||||
resultSortDirection: tab.resultSortDirection,
|
||||
resultSortMode: tab.resultSortMode,
|
||||
resultLocalSortOriginalRows: tab.resultLocalSortOriginalRows,
|
||||
resultLocalSortOriginalLargeValueCells: tab.resultLocalSortOriginalLargeValueCells,
|
||||
resultLocalSortOriginalMongoDocuments: tab.resultLocalSortOriginalMongoDocuments,
|
||||
resultLocalSortOriginalMongoCopyDocuments: tab.resultLocalSortOriginalMongoCopyDocuments,
|
||||
orderByInput: tab.orderByInput,
|
||||
@@ -1392,6 +1402,7 @@ export const useQueryStore = defineStore("query", () => {
|
||||
|
||||
if (!tab.resultLocalSortOriginalRows) {
|
||||
tab.resultLocalSortOriginalRows = tab.result.rows.slice();
|
||||
tab.resultLocalSortOriginalLargeValueCells = tab.result.large_value_cells?.map((cell) => ({ ...cell }));
|
||||
tab.resultLocalSortOriginalMongoDocuments = tab.result.mongo_documents?.slice();
|
||||
tab.resultLocalSortOriginalMongoCopyDocuments = tab.result.mongo_copy_documents?.slice();
|
||||
}
|
||||
@@ -1404,7 +1415,8 @@ export const useQueryStore = defineStore("query", () => {
|
||||
const mongo_documents = originalMongoDocuments ? rowIndexes.map((index) => originalMongoDocuments[index]) : undefined;
|
||||
const originalMongoCopyDocuments = tab.resultLocalSortOriginalMongoCopyDocuments;
|
||||
const mongo_copy_documents = originalMongoCopyDocuments ? rowIndexes.map((index) => originalMongoCopyDocuments[index]) : undefined;
|
||||
assignDisplayedResult(tab, { ...tab.result, rows, mongo_documents, mongo_copy_documents });
|
||||
const large_value_cells = remapLargeValueCells(tab.resultLocalSortOriginalLargeValueCells, rowIndexes);
|
||||
assignDisplayedResult(tab, { ...tab.result, rows, large_value_cells, mongo_documents, mongo_copy_documents });
|
||||
|
||||
tab.resultSortColumn = direction ? column : undefined;
|
||||
tab.resultSortColumnIndex = direction ? columnIndex : undefined;
|
||||
@@ -1413,6 +1425,7 @@ export const useQueryStore = defineStore("query", () => {
|
||||
tab.resultSortedSql = undefined;
|
||||
if (!direction) {
|
||||
tab.resultLocalSortOriginalRows = undefined;
|
||||
tab.resultLocalSortOriginalLargeValueCells = undefined;
|
||||
tab.resultLocalSortOriginalMongoDocuments = undefined;
|
||||
tab.resultLocalSortOriginalMongoCopyDocuments = undefined;
|
||||
}
|
||||
@@ -2544,6 +2557,7 @@ export const useQueryStore = defineStore("query", () => {
|
||||
resultSortDirection: undefined,
|
||||
resultSortMode: undefined,
|
||||
resultLocalSortOriginalRows: undefined,
|
||||
resultLocalSortOriginalLargeValueCells: undefined,
|
||||
resultLocalSortOriginalMongoDocuments: undefined,
|
||||
resultLocalSortOriginalMongoCopyDocuments: undefined,
|
||||
orderByInput: undefined,
|
||||
@@ -3124,6 +3138,7 @@ export const useQueryStore = defineStore("query", () => {
|
||||
tab.resultSortMode = undefined;
|
||||
tab.resultSortedSql = undefined;
|
||||
tab.resultLocalSortOriginalRows = undefined;
|
||||
tab.resultLocalSortOriginalLargeValueCells = undefined;
|
||||
tab.resultLocalSortOriginalMongoDocuments = undefined;
|
||||
tab.resultLocalSortOriginalMongoCopyDocuments = undefined;
|
||||
}
|
||||
@@ -3808,6 +3823,7 @@ export const useQueryStore = defineStore("query", () => {
|
||||
const previousDisplayedSql = tab.resultBaseSql ?? tab.lastExecutedSql ?? tab.sql;
|
||||
tab.lastExecutedSql = sql;
|
||||
tab.resultLocalSortOriginalRows = undefined;
|
||||
tab.resultLocalSortOriginalLargeValueCells = undefined;
|
||||
tab.resultLocalSortOriginalMongoDocuments = undefined;
|
||||
tab.resultLocalSortOriginalMongoCopyDocuments = undefined;
|
||||
if (captureResultRun && tab.result && !tab.activeResultRunId) {
|
||||
@@ -4465,6 +4481,12 @@ export const useQueryStore = defineStore("query", () => {
|
||||
: { maxRows: pageLimit, fetchSize: pageLimit }
|
||||
: { maxRows: agentProtocolQueryResultMaxRows(queryResultMaxRows) }),
|
||||
...(executionClientSessionId ? { clientSessionId: executionClientSessionId } : {}),
|
||||
...(tab.mode === "data" && effectiveDbType === "mysql"
|
||||
? {
|
||||
maxResultBytes: TABLE_DATA_RESULT_MAX_BYTES,
|
||||
resultKeyColumns: tableMetaForDataTab(tab)?.primaryKeys ?? [],
|
||||
}
|
||||
: {}),
|
||||
timeoutSecs: queryTimeoutSecs,
|
||||
catalog: executionCatalog,
|
||||
continueOnError: settingsStore.editorSettings.continueOnErrorOnBatch,
|
||||
@@ -5177,6 +5199,7 @@ export const useQueryStore = defineStore("query", () => {
|
||||
tab.activeResultIndex = index;
|
||||
tab.result = tab.results[index];
|
||||
tab.resultLocalSortOriginalRows = undefined;
|
||||
tab.resultLocalSortOriginalLargeValueCells = undefined;
|
||||
tab.resultLocalSortOriginalMongoDocuments = undefined;
|
||||
tab.resultLocalSortOriginalMongoCopyDocuments = undefined;
|
||||
tab.resultSortColumn = undefined;
|
||||
@@ -5300,6 +5323,7 @@ export const useQueryStore = defineStore("query", () => {
|
||||
tab.resultEditorFingerprint = snapshot.resultEditorFingerprint;
|
||||
tab.result = snapshot.result ? markQueryResultRowsRaw(snapshot.result) : results?.[activeIndex] ? markQueryResultRowsRaw(results[activeIndex]) : undefined;
|
||||
tab.resultLocalSortOriginalRows = snapshot.resultLocalSortOriginalRows ? markRaw(snapshot.resultLocalSortOriginalRows) : undefined;
|
||||
tab.resultLocalSortOriginalLargeValueCells = snapshot.resultLocalSortOriginalLargeValueCells?.map((cell) => ({ ...cell }));
|
||||
tab.resultLocalSortOriginalMongoDocuments = snapshot.resultLocalSortOriginalMongoDocuments ? markRaw(snapshot.resultLocalSortOriginalMongoDocuments) : undefined;
|
||||
tab.resultLocalSortOriginalMongoCopyDocuments = snapshot.resultLocalSortOriginalMongoCopyDocuments ? markRaw(snapshot.resultLocalSortOriginalMongoCopyDocuments) : undefined;
|
||||
// 快照编解码会重建负载,落盘前的各 run 估算值不再对应恢复后的对象,
|
||||
|
||||
Reference in new issue
Block a user