feat(grid): allow load all to exceed the per-request row cap

Closes #10752
This commit is contained in:
lxk955 authored and GitHub committed 2026-10-05 23:58:49 +08:00
1 parent f8b7ffe516
commit d58a79d6c8
8 files changed
+167 -12

No files matched your search

+17 -7
View File
@@ -222,6 +222,7 @@ import { dataGridHeaderContentWidth, scrollbarGutterWidth } from "@/lib/dataGrid
import {
canFetchNextDataGridSegment,
canGoNextDataGridPage,
dataGridLoadAllInitialTarget,
dataGridLoadAllNextSegment,
dataGridLoadAllSegment,
dataGridTotalRowCountLabelKey,
@@ -3817,14 +3818,17 @@ function loadAllRowsAndGoToLast() {
// against those cursors is untested, so ES/Easysearch grids keep the
// reveal-only shortcut instead of loading everything.
if (isResultsContext.value && (resolvedDatabaseType.value === "elasticsearch" || resolvedDatabaseType.value === "easysearch")) return;
const segment = dataGridLoadAllSegment(props.result.rows.length, infiniteScrollMaxRows.value, !infiniteScrollAllLoaded && canFetchNextInfiniteScrollSegment.value);
const effectiveTotal = paginationTotalRowCount.value ?? displayedTotalRowCount.value;
const canFetchMore = canFetchNextInfiniteScrollSegment.value && (!totalRowCountIsExact.value || effectiveTotal === undefined || props.result.rows.length < effectiveTotal);
const targetMaxRows = dataGridLoadAllInitialTarget(props.result.rows.length, infiniteScrollMaxRows.value, totalRowCountIsExact.value ? effectiveTotal : undefined);
const segment = dataGridLoadAllSegment(props.result.rows.length, targetMaxRows, canFetchMore);
if (!segment) {
loadAllRowsActive.value = true;
infiniteScrollAllLoaded = true;
selectAndRevealLastLoadedRow();
return;
}
const knownTotal = displayedTotalRowCount.value;
const knownTotal = effectiveTotal;
const remaining = typeof knownTotal === "number" && Number.isFinite(knownTotal) && knownTotal >= props.result.rows.length ? knownTotal - props.result.rows.length : segment.limit;
if (remaining > LOAD_ALL_ROWS_CONFIRM_ROW_THRESHOLD) {
pendingLoadAllRows.value = { remaining };
@@ -3837,6 +3841,7 @@ function loadAllRowsAndGoToLast() {
function startLoadAllRows(segment: { offset: number; limit: number }) {
loadAllRowsActive.value = true;
loadAllRowsLoopActive = true;
infiniteScrollAllLoaded = false;
infiniteScrollLoadAllPending = true;
infiniteScrollLoading.value = true;
isInfiniteScrollPaginating.value = true;
@@ -3852,12 +3857,14 @@ function startLoadAllRows(segment: { offset: number; limit: number }) {
// The per-request result-row cap bounds each chunk, never the run (#10752).
function finishOrContinueLoadAllRun(requestedOffset: number | undefined, requestedLimit: number | undefined): boolean {
if (!loadAllRowsLoopActive) return false;
const nextSegment = canFetchNextInfiniteScrollSegment.value
const effectiveTotal = paginationTotalRowCount.value ?? displayedTotalRowCount.value;
const canFetchMore = canFetchNextInfiniteScrollSegment.value && (!totalRowCountIsExact.value || effectiveTotal === undefined || props.result.rows.length < effectiveTotal);
const nextSegment = canFetchMore
? dataGridLoadAllNextSegment({
loadedRowCount: props.result.rows.length,
requestedOffset: requestedOffset ?? props.result.rows.length,
requestedLimit: requestedLimit ?? pageSize.value,
totalRowCount: totalRowCountIsExact.value ? displayedTotalRowCount.value : undefined,
totalRowCount: totalRowCountIsExact.value ? effectiveTotal : undefined,
})
: null;
if (!nextSegment) {
@@ -3875,7 +3882,10 @@ function confirmLoadAllRows() {
if (!pending) return;
pendingLoadAllRows.value = undefined;
loadAllRowsConfirmOpen.value = false;
const segment = dataGridLoadAllSegment(props.result.rows.length, infiniteScrollMaxRows.value, !infiniteScrollAllLoaded && canFetchNextInfiniteScrollSegment.value);
const effectiveTotal = paginationTotalRowCount.value ?? displayedTotalRowCount.value;
const canFetchMore = canFetchNextInfiniteScrollSegment.value && (!totalRowCountIsExact.value || effectiveTotal === undefined || props.result.rows.length < effectiveTotal);
const targetMaxRows = dataGridLoadAllInitialTarget(props.result.rows.length, infiniteScrollMaxRows.value, totalRowCountIsExact.value ? effectiveTotal : undefined);
const segment = dataGridLoadAllSegment(props.result.rows.length, targetMaxRows, canFetchMore);
if (segment) startLoadAllRows(segment);
}
function checkInfiniteScroll(scroller: HTMLElement) {
@@ -10740,10 +10750,10 @@ watch(
// While a "load all" run is active the per-request row cap must not be
// read as the end of data — only the requested-vs-appended count and an
// exact known total end the run (#10752).
...(loadAllRowsLoopActive && appendRequestedLimit ? { loadAll: { requestedLimit: appendRequestedLimit, totalRowCount: totalRowCountIsExact.value ? displayedTotalRowCount.value : undefined } } : {}),
...(loadAllRowsLoopActive && appendRequestedLimit ? { loadAll: { requestedLimit: appendRequestedLimit, totalRowCount: totalRowCountIsExact.value ? (paginationTotalRowCount.value ?? displayedTotalRowCount.value) : undefined } } : {}),
});
if (appendCompletion) {
if (infiniteScrollEnabled.value) {
if (infiniteScrollEnabled.value || loadAllRowsLoopActive) {
currentPage.value = appendCompletion.loadedPage;
lastInfiniteScrollPage = Math.max(0, appendCompletion.loadedPage - 1);
infiniteScrollAllLoaded = appendCompletion.allLoaded;
@@ -2,6 +2,7 @@ import { computed, reactive } from "vue";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { useDataGridActions } from "@/composables/useDataGridActions";
import { clearTableMetadataCache } from "@/lib/metadata/tableMetadataCache";
import { MAX_QUERY_RESULT_MAX_ROWS } from "@/lib/dataGrid/queryResultRowLimit";
import { restoredDataTabReloadFilters } from "@/lib/table/tableDataRefresh";
import type { IndexInfo, QueryTab } from "@/types/database";
@@ -618,6 +619,37 @@ describe("useDataGridActions", () => {
expect(mocks.executeTabSql).toHaveBeenCalledWith("tab-1", expect.any(String), expect.objectContaining({ pagination: { offset: 100, limit: 99_900, sessionId: undefined, clientSessionId: undefined } }));
});
it("allows load-all to append rows past the continuous query result cap", async () => {
mocks.queryResultMaxRows = 100_000;
const tab = tableDataTab({
id: "tab-large",
resultTotalRowCount: 260_000,
resultPageLimit: 100,
resultPageOffset: 0,
result: {
columns: ["id"],
rows: Array.from({ length: 100_000 }, (_, index) => [index + 1]),
affected_rows: 0,
execution_time_ms: 1,
},
});
mocks.buildTableSelectSql.mockResolvedValueOnce('SELECT * FROM "public"."users" LIMIT 100000 OFFSET 100000;');
mocks.tabs.push(tab);
const actions = useDataGridActions(computed(() => tab));
await actions.onPaginate(tab.id, 100_000, 100_000, "", undefined, true);
expect(mocks.executeTabSql).toHaveBeenCalledWith(
"tab-large",
expect.any(String),
expect.objectContaining({
appendResult: {
maxRows: MAX_QUERY_RESULT_MAX_ROWS,
},
}),
);
});
it("continues the matching Cassandra table cursor and client session", async () => {
mocks.infiniteScroll = false;
mocks.getConfig.mockReturnValue({ id: "cassandra-1", db_type: "cassandra" });
@@ -21,7 +21,7 @@ import { applyMongoFindSort } from "@/lib/mongo/mongoShellCommand";
import { uuid } from "@/lib/common/utils";
import { simpleDataGridOrderByReferencesMissingColumn, type DataGridSortMode } from "@/lib/dataGrid/dataGridSort";
import type { DataGridReloadIntent } from "@/lib/dataGrid/dataGridToolbar";
import { continuousQueryResultMaxRows } from "@/lib/dataGrid/queryResultRowLimit";
import { continuousQueryResultMaxRows, MAX_QUERY_RESULT_MAX_ROWS } from "@/lib/dataGrid/queryResultRowLimit";
import { queryResultBaseSql, queryResultExecutionSql } from "@/lib/tabs/tabPresentation";
import { sqlExecutionTargetCapabilities } from "@/lib/database/sqlExecutionTargetCapabilities";
import { usesAgentCursorForTableData } from "@/lib/database/databaseDriverManifest";
@@ -503,7 +503,7 @@ export function useDataGridActions(activeTab: ComputedRef<QueryTab | undefined>)
const appendOptions = appendResult
? {
appendResult: {
maxRows: continuousQueryResultMaxRows(settingsStore.editorSettings.queryResultMaxRowsEnabled, settingsStore.editorSettings.queryResultMaxRows),
maxRows: appendRequested ? MAX_QUERY_RESULT_MAX_ROWS : continuousQueryResultMaxRows(settingsStore.editorSettings.queryResultMaxRowsEnabled, settingsStore.editorSettings.queryResultMaxRows),
},
}
: {};
@@ -1,6 +1,6 @@
import { describe, expect, it } from "vitest";
import { dataGridBottomScrollTop, dataGridInfiniteScrollAppendCompletion, didDataGridInfiniteScrollContextChange, isDataGridAtScrollBottom, restoredDataGridScrollLeft } from "@/lib/dataGrid/dataGridInfiniteScroll";
import { dataGridLoadAllNextSegment } from "@/lib/dataGrid/dataGridPagination";
import { dataGridLoadAllInitialTarget, dataGridLoadAllNextSegment } from "@/lib/dataGrid/dataGridPagination";
describe("data grid bottom anchoring", () => {
it("keeps DOM rows anchored when scrollbar padding increases the scroll height", () => {
@@ -145,3 +145,22 @@ describe("dataGridLoadAllNextSegment", () => {
expect(dataGridLoadAllNextSegment({ loadedRowCount: 200_000, requestedOffset: 200_000, requestedLimit: 100_000 })).toBeNull();
});
});
describe("dataGridLoadAllInitialTarget", () => {
it("targets the cap or total when below the chunk limit", () => {
expect(dataGridLoadAllInitialTarget(100, 100_000, 260_000)).toBe(100_000);
expect(dataGridLoadAllInitialTarget(100, 100_000, 50_000)).toBe(50_000);
expect(dataGridLoadAllInitialTarget(100, 100_000)).toBe(100_000);
});
it("targets the next chunk increment when already at or past the chunk limit", () => {
expect(dataGridLoadAllInitialTarget(100_000, 100_000, 260_000)).toBe(200_000);
expect(dataGridLoadAllInitialTarget(200_000, 100_000, 260_000)).toBe(260_000);
expect(dataGridLoadAllInitialTarget(260_000, 100_000, 260_000)).toBe(260_000);
});
it("advances by chunk increment when total is unknown", () => {
expect(dataGridLoadAllInitialTarget(100_000, 100_000)).toBe(200_000);
expect(dataGridLoadAllInitialTarget(200_000, 100_000)).toBe(300_000);
});
});
@@ -163,6 +163,21 @@ export function dataGridLoadAllSegment(loadedRowCount: number, maxRows: number,
return { offset, limit: boundedMaxRows - offset };
}
/**
* Target cumulative row count for the initial chunk of an explicit "load all"
* run. Each chunk is bounded by `chunkLimit` (the per-request result-row cap,
* e.g. 100,000), but an existing dataset that has already reached or exceeded
* `chunkLimit` must still be able to start loading subsequent chunks up to
* `totalRowCount` (if known) or by another chunk increment (#10752).
*/
export function dataGridLoadAllInitialTarget(loadedRowCount: number, chunkLimit: number, totalRowCount?: number): number {
const loaded = Number.isFinite(loadedRowCount) ? Math.max(0, Math.trunc(loadedRowCount)) : 0;
const chunk = Number.isFinite(chunkLimit) ? Math.max(1, Math.trunc(chunkLimit)) : 1;
const chunkCap = loaded < chunk ? chunk : loaded + chunk;
const total = typeof totalRowCount === "number" && Number.isFinite(totalRowCount) && totalRowCount >= 0 ? Math.trunc(totalRowCount) : undefined;
return total !== undefined ? Math.min(chunkCap, total) : chunkCap;
}
/**
* Next chunk of an explicit "load all" run, issued after the previous chunk
* completed. The per-request result-row cap bounds each request, not the run:
+1 -1
View File
@@ -7724,7 +7724,7 @@ export const useQueryStore = defineStore("query", () => {
return producedResult;
}
const queryResultMaxRows = tab.mode === "query" ? effectiveQueryResultMaxRows(settingsStore.editorSettings.queryResultMaxRowsEnabled, settingsStore.editorSettings.queryResultMaxRows) : undefined;
const queryResultMaxRows = tab.mode === "query" ? (options?.appendResult ? options.appendResult.maxRows : effectiveQueryResultMaxRows(settingsStore.editorSettings.queryResultMaxRowsEnabled, settingsStore.editorSettings.queryResultMaxRows)) : undefined;
if (tab.mode === "query") {
const prepared = await prepareEditableQueryExecution(tab, sqlToExecute, conn, effectiveDbType, executionDatabase, traceId, elapsed);